Makussu/bunkerweb
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

edit visibility of Job members and integration of a generic checker for nginx

Browse Source
This commit is contained in:
bunkerity 2021-07-22 23:07:35 +02:00
parent 9a207dfdc5
commit 0dc2a5ec25
No known key found for this signature in database
GPG Key ID: 3D80806F12602A7C
8 changed files with 98 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
confs/site/main-lua.conf
View File

@ -69,6 +69,7 @@ local iputils = require "resty.iputils"
local behavior = require "behavior" local behavior = require "behavior"
local logger = require "logger" local logger = require "logger"
local redis = require "resty.redis" local redis = require "resty.redis"
local checker = require "checker"
-- user variables -- user variables
local antibot_uri = "{{ ANTIBOT_URI }}" local antibot_uri = "{{ ANTIBOT_URI }}"
@ -157,8 +158,8 @@ end
-- check if IP is in proxies list -- check if IP is in proxies list
if use_proxies then if use_proxies then
local value, flags = ngx.shared.proxies_data:get(iputils.ip2bin(ngx.var.remote_addr)) local checker = checker:new("proxies", ngx.shared.proxies_data, redis_client, "simple")
if value ~= nil then if checker:check(iputils.ip2bin(ngx.var.remote_addr)) then
logger.log(ngx.WARN, "PROXIES", "IP " .. ngx.var.remote_addr .. " is in proxies list") logger.log(ngx.WARN, "PROXIES", "IP " .. ngx.var.remote_addr .. " is in proxies list")
ngx.exit(ngx.HTTP_FORBIDDEN) ngx.exit(ngx.HTTP_FORBIDDEN)
end end
@ -166,8 +167,8 @@ end
-- check if IP is in abusers list -- check if IP is in abusers list
if use_abusers then if use_abusers then
local value, flags = ngx.shared.abusers_data:get(iputils.ip2bin(ngx.var.remote_addr)) local checker = checker:new("abusers", ngx.shared.abusers_data, redis_client, "simple")
if value ~= nil then if checker:check(iputils.ip2bin(ngx.var.remote_addr)) then
logger.log(ngx.WARN, "ABUSERS", "IP " .. ngx.var.remote_addr .. " is in abusers list") logger.log(ngx.WARN, "ABUSERS", "IP " .. ngx.var.remote_addr .. " is in abusers list")
ngx.exit(ngx.HTTP_FORBIDDEN) ngx.exit(ngx.HTTP_FORBIDDEN)
end end
@ -175,8 +176,8 @@ end
-- check if IP is in TOR exit nodes list -- check if IP is in TOR exit nodes list
if use_tor_exit_nodes then if use_tor_exit_nodes then
local value, flags = ngx.shared.tor_exit_nodes_data:get(iputils.ip2bin(ngx.var.remote_addr)) local checker = checker:new("exit-nodes", ngx.shared.tor_exit_nodes_data, redis_client, "simple")
if value ~= nil then if checker:check(iputils.ip2bin(ngx.var.remote_addr)) then
logger.log(ngx.WARN, "TOR", "IP " .. ngx.var.remote_addr .. " is in TOR exit nodes list") logger.log(ngx.WARN, "TOR", "IP " .. ngx.var.remote_addr .. " is in TOR exit nodes list")
ngx.exit(ngx.HTTP_FORBIDDEN) ngx.exit(ngx.HTTP_FORBIDDEN)
end end
@ -193,23 +194,9 @@ if use_user_agents and ngx.var.http_user_agent ~= nil then
end end
end end
if not whitelisted then if not whitelisted then
local value, flags = ngx.shared.user_agents_cache:get(ngx.var.http_user_agent) local checker = checker:new("user-agents", ngx.shared.user_agents_data, redis_client, "match")
if value == nil then if checker:check(ngx.var.http_user_agent) then
local patterns = ngx.shared.user_agents_data:get_keys(0) logger.log(ngx.WARN, "USER-AGENTS", "User-Agent " .. ngx.var.http_user_agent .. " is blacklisted")
for i, pattern in ipairs(patterns) do
if string.match(ngx.var.http_user_agent, pattern) then
value = "ko"
ngx.shared.user_agents_cache:set(ngx.var.http_user_agent, "ko", 86400)
break
end
end
if value == nil then
value = "ok"
ngx.shared.user_agents_cache:set(ngx.var.http_user_agent, "ok", 86400)
end
end
if value == "ko" then
logger.log(ngx.WARN, "USER-AGENT", "User-Agent " .. ngx.var.http_user_agent .. " is blacklisted")
ngx.exit(ngx.HTTP_FORBIDDEN) ngx.exit(ngx.HTTP_FORBIDDEN)
end end
end end
@ -217,23 +204,9 @@ end
-- check if referrer is allowed -- check if referrer is allowed
if use_referrer and ngx.var.http_referer ~= nil then if use_referrer and ngx.var.http_referer ~= nil then
local value, flags = ngx.shared.referrers_cache:get(ngx.var.http_referer) local checker = checker:new("referrers", ngx.shared.referrers_data, redis_client, "match")
if value == nil then if checker:check(ngx.var.http_referer) then
local patterns = ngx.shared.referrers_data:get_keys(0) logger.log(ngx.WARN, "REFERRERS", "Referrer " .. ngx.var.http_referer .. " is blacklisted")
for i, pattern in ipairs(patterns) do
if string.match(ngx.var.http_referer, pattern) then
value = "ko"
ngx.shared.referrers_cache:set(ngx.var.http_referer, "ko", 86400)
break
end
end
if value == nil then
value = "ok"
ngx.shared.referrers_cache:set(ngx.var.http_referer, "ok", 86400)
end
end
if value == "ko" then
logger.log(ngx.WARN, "REFERRER", "Referrer " .. ngx.var.http_referer .. " is blacklisted")
ngx.exit(ngx.HTTP_FORBIDDEN) ngx.exit(ngx.HTTP_FORBIDDEN)
end end
end end

4
jobs/Abusers.py
View File

@ -12,8 +12,8 @@ class Abusers(Job) :
regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$"
super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache) super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache)
def _Job__edit(self, chunk) : def _edit(self, chunk) :
if self.__redis != None : if self._redis != None :
network = chunk.decode("utf-8") network = chunk.decode("utf-8")
if re.match(network, r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\/?[0-9]+$") : if re.match(network, r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\/?[0-9]+$") :
ips = [] ips = []

4
jobs/ExitNodes.py
View File

@ -12,8 +12,8 @@ class ExitNodes(Job) :
regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$"
super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache) super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache)
def _Job__edit(self, chunk) : def _edit(self, chunk) :
if self.__redis != None : if self._redis != None :
network = chunk.decode("utf-8") network = chunk.decode("utf-8")
if re.match(network, r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\/?[0-9]+$") : if re.match(network, r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\/?[0-9]+$") :
ips = [] ips = []

122
jobs/Job.py
View File

@ -8,37 +8,37 @@ class JobRet(enum.Enum) :
class Job(abc.ABC) : class Job(abc.ABC) :
def __init__(self, name, data, filename=None, redis_host=None, type="line", regex=r"^.+$", copy_cache=False) : def __init__(self, name, data, filename=None, redis_host=None, type="line", regex=r"^.+$", copy_cache=False) :
self.__name = name self._name = name
self.__data = data self._data = data
self.__filename = filename self._filename = filename
self.__redis = None self._redis = None
if redis_host != None : if redis_host != None :
self.__redis = redis.Redis(host=redis_host, port=6379, db=0) self._redis = redis.Redis(host=redis_host, port=6379, db=0)
try : try :
self.__redis.echo("test") self._redis.echo("test")
except : except :
self.__log("can't connect to redis host " + redis_host) self._log("can't connect to redis host " + redis_host)
self.__type = type self._type = type
self.__regex = regex self._regex = regex
self.__copy_cache = copy_cache self._copy_cache = copy_cache
def __log(self, data) : def _log(self, data) :
when = datetime.datetime.today().strftime("[%Y-%m-%d %H:%M:%S]") when = datetime.datetime.today().strftime("[%Y-%m-%d %H:%M:%S]")
what = self.__name + " - " + data + "\n" what = self._name + " - " + data + "\n"
with open("/var/log/nginx/jobs.log", "a") as f : with open("/var/log/nginx/jobs.log", "a") as f :
f.write(when + " " + what) f.write(when + " " + what)
def run(self) : def run(self) :
ret = JobRet.KO ret = JobRet.KO
try : try :
if self.__type == "line" or self.__type == "file" : if self._type == "line" or self._type == "file" :
if self.__copy_cache : if self._copy_cache :
ret = self.__from_cache() ret = self.__from_cache()
if ret != JobRet.KO : if ret != JobRet.KO :
return ret return ret
ret = self.__external() ret = self.__external()
self.__to_cache() self.__to_cache()
elif self.__type == "exec" : elif self._type == "exec" :
return self.__exec() return self.__exec()
except Exception as e : except Exception as e :
self.__log("exception while running job : " + traceback.format_exc()) self.__log("exception while running job : " + traceback.format_exc())
@ -46,43 +46,45 @@ class Job(abc.ABC) :
return ret return ret
def __external(self) : def __external(self) :
if self.__redis == None : if self._redis == None :
if os.path.isfile("/tmp/" + self.__filename) : if os.path.isfile("/tmp/" + self._filename) :
os.remove("/tmp/" + self.__filename) os.remove("/tmp/" + self._filename)
file = open("/tmp/" + self.__filename, "ab") file = open("/tmp/" + self._filename, "ab")
elif self.__redis != None : elif self._redis != None :
pipe = self.__redis.pipeline() pipe = self._redis.pipeline()
count = 0 count = 0
for url in self.__data : for url in self._data :
data = self.__download_data(url) data = self.__download_data(url)
for chunk in data : for chunk in data :
if self.__type == "line" : if self._type == "line" :
if not re.match(self.__regex, chunk.decode("utf-8")) : if not re.match(self._regex, chunk.decode("utf-8")) :
continue continue
chunks = self.__edit(chunk) chunks = self._edit(chunk)
if self.__redis == None : if self._redis == None :
if self.__type == "line" : if self._type == "line" :
chunk += b"\n"
file.write(chunk)
else :
if self.__type == "line" :
for chunk in chunks : for chunk in chunks :
pipe.set(self.__name + "_" + chunk, "1") file.write(chunk + b"\n")
else : else :
pipe.set(self.__name + "_" + chunk, "1") file.write(chunk)
else :
if self._type == "line" :
for chunk in chunks :
pipe.set(self._name + "_" + chunk, "1")
else :
pipe.set(self._name + "_" + chunk, "1")
count += 1 count += 1
if self.__redis == None : if self._redis == None :
file.close() file.close()
if count > 0 : if count > 0 :
shutil.copyfile("/tmp/" + self.__filename, "/etc/nginx/" + self.__filename) shutil.copyfile("/tmp/" + self._filename, "/etc/nginx/" + self._filename)
os.remove("/tmp/" + self.__filename) os.remove("/tmp/" + self._filename)
return JobRet.OK_RELOAD return JobRet.OK_RELOAD
elif self.__redis != None and count > 0 : elif self._redis != None and count > 0 :
self.__redis.delete(self.__redis.keys(self.__name + "_*")) self._redis.delete(self._redis.keys(self._name + "_*"))
pipe.execute() pipe.execute()
return JobRet.OK_RELOAD return JobRet.OK_RELOAD
@ -92,57 +94,57 @@ class Job(abc.ABC) :
r = requests.get(url, stream=True) r = requests.get(url, stream=True)
if not r or r.status_code != 200 : if not r or r.status_code != 200 :
raise Exception("can't download data at " + url) raise Exception("can't download data at " + url)
if self.__type == "line" : if self._type == "line" :
return r.iter_lines() return r.iter_lines()
return r.iter_content(chunk_size=8192) return r.iter_content(chunk_size=8192)
def __exec(self) : def __exec(self) :
proc = subprocess.run(self.__data, capture_output=True) proc = subprocess.run(self._data, capture_output=True)
stdout = proc.stdout.decode("ascii") stdout = proc.stdout.decode("ascii")
stderr = proc.stderr.decode("err") stderr = proc.stderr.decode("err")
if len(stdout) > 1 : if len(stdout) > 1 :
self.__log("stdout = " + stdout) self._log("stdout = " + stdout)
if len(stderr) > 1 : if len(stderr) > 1 :
self.__log("stderr = " + stderr) self._log("stderr = " + stderr)
if proc.returncode != 0 : if proc.returncode != 0 :
return JobRet.KO return JobRet.KO
# TODO : check if reload is needed ? # TODO : check if reload is needed ?
return JobRet.OK_RELOAD return JobRet.OK_RELOAD
def __edit(self, chunk) : def _edit(self, chunk) :
return [chunk] return [chunk]
def __from_cache(self) : def __from_cache(self) :
if not os.path.isfile("/opt/bunkerized-nginx/cache/" + self.__filename) : if not os.path.isfile("/opt/bunkerized-nginx/cache/" + self._filename) :
return JobRet.KO return JobRet.KO
if self.__redis == None or self.__type == "file" : if self._redis == None or self._type == "file" :
if not os.path.isfile("/etc/nginx/" + self.__filename) or not filecmp.cmp("/opt/bunkerized-nginx/cache/" + self.__filename, "/etc/nginx/" + self.__filename, shallow=False) : if not os.path.isfile("/etc/nginx/" + self._filename) or not filecmp.cmp("/opt/bunkerized-nginx/cache/" + self._filename, "/etc/nginx/" + self._filename, shallow=False) :
shutil.copyfile("/opt/bunkerized-nginx/cache/" + self.__filename, "/etc/nginx/" + self.__filename) shutil.copyfile("/opt/bunkerized-nginx/cache/" + self._filename, "/etc/nginx/" + self._filename)
return JobRet.OK_RELOAD return JobRet.OK_RELOAD
return JobRet.OK_NO_RELOAD return JobRet.OK_NO_RELOAD
if self.__redis != None and self.__type == "line" : if self._redis != None and self._type == "line" :
self.__redis.delete(self.__redis.keys(self.__name + "_*")) self._redis.delete(self._redis.keys(self._name + "_*"))
with open("/opt/bunkerized-nginx/cache/" + self.__filename) as f : with open("/opt/bunkerized-nginx/cache/" + self._filename) as f :
pipe = self.__redis.pipeline() pipe = self._redis.pipeline()
while True : while True :
line = f.readline() line = f.readline()
if not line : if not line :
break break
line = line.strip() line = line.strip()
pipe.set(self.__name + "_" + line, "1") pipe.set(self._name + "_" + line, "1")
pipe.execute() pipe.execute()
return JobRet.OK_NO_RELOAD return JobRet.OK_NO_RELOAD
return JobRet.KO return JobRet.KO
def __to_cache(self) : def __to_cache(self) :
if self.__redis == None or self.__type == "file" : if self._redis == None or self._type == "file" :
shutil.copyfile("/etc/nginx/" + self.__filename, "/opt/bunkerized-nginx/cache/" + self.__filename) shutil.copyfile("/etc/nginx/" + self._filename, "/opt/bunkerized-nginx/cache/" + self._filename)
elif self.__redis != None and self.__type == "line" : elif self._redis != None and self._type == "line" :
if os.path.isfile("/opt/bunkerized-nginx/cache/" + self.__filename) : if os.path.isfile("/opt/bunkerized-nginx/cache/" + self._filename) :
os.remove("/opt/bunkerized-nginx/cache/" + self.__filename) os.remove("/opt/bunkerized-nginx/cache/" + self._filename)
with open("/opt/bunkerized-nginx/cache/" + self.__filename, "a") as f : with open("/opt/bunkerized-nginx/cache/" + self._filename, "a") as f :
for key in self.__redis.keys(self.__name + "_*") : for key in self._redis.keys(self._name + "_*") :
f.write(self.__redis.get(key) + "\n") f.write(self._redis.get(key) + "\n")

4
jobs/Proxies.py
View File

@ -12,8 +12,8 @@ class Proxies(Job) :
regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$"
super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache) super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache)
def _Job__edit(self, chunk) : def _edit(self, chunk) :
if self.__redis != None : if self._redis != None :
network = chunk.decode("utf-8") network = chunk.decode("utf-8")
if re.match(network, r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\/?[0-9]+$") : if re.match(network, r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\/?[0-9]+$") :
ips = [] ips = []

4
jobs/Referrers.py
View File

@ -10,5 +10,5 @@ class Referrers(Job) :
regex = r"^.+$" regex = r"^.+$"
super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache) super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache)
def _Job__edit(self, chunk) : def _edit(self, chunk) :
return chunk.replace(b".", b"%.").replace(b"-", b"%-") return [chunk.replace(b".", b"%.").replace(b"-", b"%-")]

4
jobs/UserAgents.py
View File

@ -10,5 +10,5 @@ class UserAgents(Job) :
regex = r"^.+$" regex = r"^.+$"
super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache) super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache)
def _Job__edit(self, chunk) : def _edit(self, chunk) :
return chunk.replace(b"\\ ", b" ").replace(b"\\.", b"%.").replace(b"\\\\", b"\\").replace(b"-", b"%-") return [chunk.replace(b"\\ ", b" ").replace(b"\\.", b"%.").replace(b"\\\\", b"\\").replace(b"-", b"%-")]

23
lua/checker.lua
View File

@ -1,41 +1,42 @@
local M = {} local M = {}
local redis = require "resty.redis" local redis = require "resty.redis"
local mt = { __index = M }
function M.new(self, name, data_dict, redis_client, type) function M.new(self, name, data_dict, redis_client, type)
return selfmetatable({ return setmetatable({
__name = name, __name = name,
__data_dict = data_dict, __data_dict = data_dict,
__redis_client = redis_client, __redis_client = redis_client,
__type = type __type = type
}) }, mt)
end end
function M.check(self, data) : function M.check(self, data)
-- without redis -- without redis
if self.__data_dict ~= nil and redis_client == nil then if self.__data_dict ~= nil and self.__redis_client == nil then
if self.__type == "simple" then if self.__type == "simple" then
local value, flags = self.__data_dict:get(data) local value, flags = self.__data_dict:get(data)
return ~= nil return value ~= nil
else if self.__type == "match" then elseif self.__type == "match" then
local patterns = self.__data_dict:get_keys(0) local patterns = self.__data_dict:get_keys(0)
for i, pattern in ipairs(patterns) do for i, pattern in ipairs(patterns) do
if string.match(data, pattern) then if string.match(data, pattern) then
return true return true
end end
end end
return false
end end
-- with redis -- with redis
else if data_dict == nil and redis_client ~= nil then elseif self.__data_dict == nil and self.__redis_client ~= nil then
if self.__type == "simple" then if self.__type == "simple" then
local res, err = self.__redis_client:get(self.__name .. "_" .. data) local res, err = self.__redis_client:get(self.__name .. "_" .. data)
return res and res ~= ngx.null return res and res ~= ngx.null
else if self.__type == "match" then elseif self.__type == "match" then
local patterns = self.__redis_client:keys(self.__name .. "_*") local patterns = self.__redis_client:keys(self.__name .. "_*")
if patterns then if patterns then
for i, pattern in ipairs(patterns) do for i, pattern in ipairs(patterns) do
if string.match(data, pattern) do if string.match(data, pattern) then
return true return true
end end
end end
@ -45,4 +46,6 @@ function M.check(self, data) :
return false return false
end
return M return M