diff --git a/confs/site/main-lua.conf b/confs/site/main-lua.conf index 87efd8e..d0c3299 100644 --- a/confs/site/main-lua.conf +++ b/confs/site/main-lua.conf @@ -69,6 +69,7 @@ local iputils = require "resty.iputils" local behavior = require "behavior" local logger = require "logger" local redis = require "resty.redis" +local checker = require "checker" -- user variables local antibot_uri = "{{ ANTIBOT_URI }}" @@ -157,8 +158,8 @@ end -- check if IP is in proxies list if use_proxies then - local value, flags = ngx.shared.proxies_data:get(iputils.ip2bin(ngx.var.remote_addr)) - if value ~= nil then + local checker = checker:new("proxies", ngx.shared.proxies_data, redis_client, "simple") + if checker:check(iputils.ip2bin(ngx.var.remote_addr)) then logger.log(ngx.WARN, "PROXIES", "IP " .. ngx.var.remote_addr .. " is in proxies list") ngx.exit(ngx.HTTP_FORBIDDEN) end @@ -166,8 +167,8 @@ end -- check if IP is in abusers list if use_abusers then - local value, flags = ngx.shared.abusers_data:get(iputils.ip2bin(ngx.var.remote_addr)) - if value ~= nil then + local checker = checker:new("abusers", ngx.shared.abusers_data, redis_client, "simple") + if checker:check(iputils.ip2bin(ngx.var.remote_addr)) then logger.log(ngx.WARN, "ABUSERS", "IP " .. ngx.var.remote_addr .. " is in abusers list") ngx.exit(ngx.HTTP_FORBIDDEN) end @@ -175,8 +176,8 @@ end -- check if IP is in TOR exit nodes list if use_tor_exit_nodes then - local value, flags = ngx.shared.tor_exit_nodes_data:get(iputils.ip2bin(ngx.var.remote_addr)) - if value ~= nil then + local checker = checker:new("exit-nodes", ngx.shared.tor_exit_nodes_data, redis_client, "simple") + if checker:check(iputils.ip2bin(ngx.var.remote_addr)) then logger.log(ngx.WARN, "TOR", "IP " .. ngx.var.remote_addr .. " is in TOR exit nodes list") ngx.exit(ngx.HTTP_FORBIDDEN) end @@ -193,23 +194,9 @@ if use_user_agents and ngx.var.http_user_agent ~= nil then end end if not whitelisted then - local value, flags = ngx.shared.user_agents_cache:get(ngx.var.http_user_agent) - if value == nil then - local patterns = ngx.shared.user_agents_data:get_keys(0) - for i, pattern in ipairs(patterns) do - if string.match(ngx.var.http_user_agent, pattern) then - value = "ko" - ngx.shared.user_agents_cache:set(ngx.var.http_user_agent, "ko", 86400) - break - end - end - if value == nil then - value = "ok" - ngx.shared.user_agents_cache:set(ngx.var.http_user_agent, "ok", 86400) - end - end - if value == "ko" then - logger.log(ngx.WARN, "USER-AGENT", "User-Agent " .. ngx.var.http_user_agent .. " is blacklisted") + local checker = checker:new("user-agents", ngx.shared.user_agents_data, redis_client, "match") + if checker:check(ngx.var.http_user_agent) then + logger.log(ngx.WARN, "USER-AGENTS", "User-Agent " .. ngx.var.http_user_agent .. " is blacklisted") ngx.exit(ngx.HTTP_FORBIDDEN) end end @@ -217,23 +204,9 @@ end -- check if referrer is allowed if use_referrer and ngx.var.http_referer ~= nil then - local value, flags = ngx.shared.referrers_cache:get(ngx.var.http_referer) - if value == nil then - local patterns = ngx.shared.referrers_data:get_keys(0) - for i, pattern in ipairs(patterns) do - if string.match(ngx.var.http_referer, pattern) then - value = "ko" - ngx.shared.referrers_cache:set(ngx.var.http_referer, "ko", 86400) - break - end - end - if value == nil then - value = "ok" - ngx.shared.referrers_cache:set(ngx.var.http_referer, "ok", 86400) - end - end - if value == "ko" then - logger.log(ngx.WARN, "REFERRER", "Referrer " .. ngx.var.http_referer .. " is blacklisted") + local checker = checker:new("referrers", ngx.shared.referrers_data, redis_client, "match") + if checker:check(ngx.var.http_referer) then + logger.log(ngx.WARN, "REFERRERS", "Referrer " .. ngx.var.http_referer .. " is blacklisted") ngx.exit(ngx.HTTP_FORBIDDEN) end end diff --git a/jobs/Abusers.py b/jobs/Abusers.py index f14151f..2b368b7 100644 --- a/jobs/Abusers.py +++ b/jobs/Abusers.py @@ -12,8 +12,8 @@ class Abusers(Job) : regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache) - def _Job__edit(self, chunk) : - if self.__redis != None : + def _edit(self, chunk) : + if self._redis != None : network = chunk.decode("utf-8") if re.match(network, r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\/?[0-9]+$") : ips = [] diff --git a/jobs/ExitNodes.py b/jobs/ExitNodes.py index 7a2e39f..2a4abeb 100644 --- a/jobs/ExitNodes.py +++ b/jobs/ExitNodes.py @@ -12,8 +12,8 @@ class ExitNodes(Job) : regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache) - def _Job__edit(self, chunk) : - if self.__redis != None : + def _edit(self, chunk) : + if self._redis != None : network = chunk.decode("utf-8") if re.match(network, r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\/?[0-9]+$") : ips = [] diff --git a/jobs/Job.py b/jobs/Job.py index ee5e4ed..9432627 100644 --- a/jobs/Job.py +++ b/jobs/Job.py @@ -8,37 +8,37 @@ class JobRet(enum.Enum) : class Job(abc.ABC) : def __init__(self, name, data, filename=None, redis_host=None, type="line", regex=r"^.+$", copy_cache=False) : - self.__name = name - self.__data = data - self.__filename = filename - self.__redis = None + self._name = name + self._data = data + self._filename = filename + self._redis = None if redis_host != None : - self.__redis = redis.Redis(host=redis_host, port=6379, db=0) + self._redis = redis.Redis(host=redis_host, port=6379, db=0) try : - self.__redis.echo("test") + self._redis.echo("test") except : - self.__log("can't connect to redis host " + redis_host) - self.__type = type - self.__regex = regex - self.__copy_cache = copy_cache + self._log("can't connect to redis host " + redis_host) + self._type = type + self._regex = regex + self._copy_cache = copy_cache - def __log(self, data) : + def _log(self, data) : when = datetime.datetime.today().strftime("[%Y-%m-%d %H:%M:%S]") - what = self.__name + " - " + data + "\n" + what = self._name + " - " + data + "\n" with open("/var/log/nginx/jobs.log", "a") as f : f.write(when + " " + what) def run(self) : ret = JobRet.KO try : - if self.__type == "line" or self.__type == "file" : - if self.__copy_cache : + if self._type == "line" or self._type == "file" : + if self._copy_cache : ret = self.__from_cache() if ret != JobRet.KO : return ret ret = self.__external() self.__to_cache() - elif self.__type == "exec" : + elif self._type == "exec" : return self.__exec() except Exception as e : self.__log("exception while running job : " + traceback.format_exc()) @@ -46,43 +46,45 @@ class Job(abc.ABC) : return ret def __external(self) : - if self.__redis == None : - if os.path.isfile("/tmp/" + self.__filename) : - os.remove("/tmp/" + self.__filename) - file = open("/tmp/" + self.__filename, "ab") + if self._redis == None : + if os.path.isfile("/tmp/" + self._filename) : + os.remove("/tmp/" + self._filename) + file = open("/tmp/" + self._filename, "ab") - elif self.__redis != None : - pipe = self.__redis.pipeline() + elif self._redis != None : + pipe = self._redis.pipeline() count = 0 - for url in self.__data : + for url in self._data : data = self.__download_data(url) for chunk in data : - if self.__type == "line" : - if not re.match(self.__regex, chunk.decode("utf-8")) : + if self._type == "line" : + if not re.match(self._regex, chunk.decode("utf-8")) : continue - chunks = self.__edit(chunk) - if self.__redis == None : - if self.__type == "line" : - chunk += b"\n" - file.write(chunk) - else : - if self.__type == "line" : + chunks = self._edit(chunk) + if self._redis == None : + if self._type == "line" : for chunk in chunks : - pipe.set(self.__name + "_" + chunk, "1") + file.write(chunk + b"\n") else : - pipe.set(self.__name + "_" + chunk, "1") + file.write(chunk) + else : + if self._type == "line" : + for chunk in chunks : + pipe.set(self._name + "_" + chunk, "1") + else : + pipe.set(self._name + "_" + chunk, "1") count += 1 - if self.__redis == None : + if self._redis == None : file.close() if count > 0 : - shutil.copyfile("/tmp/" + self.__filename, "/etc/nginx/" + self.__filename) - os.remove("/tmp/" + self.__filename) + shutil.copyfile("/tmp/" + self._filename, "/etc/nginx/" + self._filename) + os.remove("/tmp/" + self._filename) return JobRet.OK_RELOAD - elif self.__redis != None and count > 0 : - self.__redis.delete(self.__redis.keys(self.__name + "_*")) + elif self._redis != None and count > 0 : + self._redis.delete(self._redis.keys(self._name + "_*")) pipe.execute() return JobRet.OK_RELOAD @@ -92,57 +94,57 @@ class Job(abc.ABC) : r = requests.get(url, stream=True) if not r or r.status_code != 200 : raise Exception("can't download data at " + url) - if self.__type == "line" : + if self._type == "line" : return r.iter_lines() return r.iter_content(chunk_size=8192) def __exec(self) : - proc = subprocess.run(self.__data, capture_output=True) + proc = subprocess.run(self._data, capture_output=True) stdout = proc.stdout.decode("ascii") stderr = proc.stderr.decode("err") if len(stdout) > 1 : - self.__log("stdout = " + stdout) + self._log("stdout = " + stdout) if len(stderr) > 1 : - self.__log("stderr = " + stderr) + self._log("stderr = " + stderr) if proc.returncode != 0 : return JobRet.KO # TODO : check if reload is needed ? return JobRet.OK_RELOAD - def __edit(self, chunk) : + def _edit(self, chunk) : return [chunk] def __from_cache(self) : - if not os.path.isfile("/opt/bunkerized-nginx/cache/" + self.__filename) : + if not os.path.isfile("/opt/bunkerized-nginx/cache/" + self._filename) : return JobRet.KO - if self.__redis == None or self.__type == "file" : - if not os.path.isfile("/etc/nginx/" + self.__filename) or not filecmp.cmp("/opt/bunkerized-nginx/cache/" + self.__filename, "/etc/nginx/" + self.__filename, shallow=False) : - shutil.copyfile("/opt/bunkerized-nginx/cache/" + self.__filename, "/etc/nginx/" + self.__filename) + if self._redis == None or self._type == "file" : + if not os.path.isfile("/etc/nginx/" + self._filename) or not filecmp.cmp("/opt/bunkerized-nginx/cache/" + self._filename, "/etc/nginx/" + self._filename, shallow=False) : + shutil.copyfile("/opt/bunkerized-nginx/cache/" + self._filename, "/etc/nginx/" + self._filename) return JobRet.OK_RELOAD return JobRet.OK_NO_RELOAD - if self.__redis != None and self.__type == "line" : - self.__redis.delete(self.__redis.keys(self.__name + "_*")) - with open("/opt/bunkerized-nginx/cache/" + self.__filename) as f : - pipe = self.__redis.pipeline() + if self._redis != None and self._type == "line" : + self._redis.delete(self._redis.keys(self._name + "_*")) + with open("/opt/bunkerized-nginx/cache/" + self._filename) as f : + pipe = self._redis.pipeline() while True : line = f.readline() if not line : break line = line.strip() - pipe.set(self.__name + "_" + line, "1") + pipe.set(self._name + "_" + line, "1") pipe.execute() return JobRet.OK_NO_RELOAD return JobRet.KO def __to_cache(self) : - if self.__redis == None or self.__type == "file" : - shutil.copyfile("/etc/nginx/" + self.__filename, "/opt/bunkerized-nginx/cache/" + self.__filename) - elif self.__redis != None and self.__type == "line" : - if os.path.isfile("/opt/bunkerized-nginx/cache/" + self.__filename) : - os.remove("/opt/bunkerized-nginx/cache/" + self.__filename) - with open("/opt/bunkerized-nginx/cache/" + self.__filename, "a") as f : - for key in self.__redis.keys(self.__name + "_*") : - f.write(self.__redis.get(key) + "\n") + if self._redis == None or self._type == "file" : + shutil.copyfile("/etc/nginx/" + self._filename, "/opt/bunkerized-nginx/cache/" + self._filename) + elif self._redis != None and self._type == "line" : + if os.path.isfile("/opt/bunkerized-nginx/cache/" + self._filename) : + os.remove("/opt/bunkerized-nginx/cache/" + self._filename) + with open("/opt/bunkerized-nginx/cache/" + self._filename, "a") as f : + for key in self._redis.keys(self._name + "_*") : + f.write(self._redis.get(key) + "\n") diff --git a/jobs/Proxies.py b/jobs/Proxies.py index 4863627..2d7c626 100644 --- a/jobs/Proxies.py +++ b/jobs/Proxies.py @@ -12,8 +12,8 @@ class Proxies(Job) : regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache) - def _Job__edit(self, chunk) : - if self.__redis != None : + def _edit(self, chunk) : + if self._redis != None : network = chunk.decode("utf-8") if re.match(network, r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\/?[0-9]+$") : ips = [] diff --git a/jobs/Referrers.py b/jobs/Referrers.py index 1ad744c..7b3e74b 100644 --- a/jobs/Referrers.py +++ b/jobs/Referrers.py @@ -10,5 +10,5 @@ class Referrers(Job) : regex = r"^.+$" super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache) - def _Job__edit(self, chunk) : - return chunk.replace(b".", b"%.").replace(b"-", b"%-") + def _edit(self, chunk) : + return [chunk.replace(b".", b"%.").replace(b"-", b"%-")] diff --git a/jobs/UserAgents.py b/jobs/UserAgents.py index 10f3959..577947e 100644 --- a/jobs/UserAgents.py +++ b/jobs/UserAgents.py @@ -10,5 +10,5 @@ class UserAgents(Job) : regex = r"^.+$" super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache) - def _Job__edit(self, chunk) : - return chunk.replace(b"\\ ", b" ").replace(b"\\.", b"%.").replace(b"\\\\", b"\\").replace(b"-", b"%-") + def _edit(self, chunk) : + return [chunk.replace(b"\\ ", b" ").replace(b"\\.", b"%.").replace(b"\\\\", b"\\").replace(b"-", b"%-")] diff --git a/lua/checker.lua b/lua/checker.lua index 8a2f7cf..ffca301 100644 --- a/lua/checker.lua +++ b/lua/checker.lua @@ -1,41 +1,42 @@ local M = {} local redis = require "resty.redis" +local mt = { __index = M } + function M.new(self, name, data_dict, redis_client, type) - return selfmetatable({ + return setmetatable({ __name = name, __data_dict = data_dict, __redis_client = redis_client, __type = type - }) + }, mt) end -function M.check(self, data) : +function M.check(self, data) -- without redis - if self.__data_dict ~= nil and redis_client == nil then + if self.__data_dict ~= nil and self.__redis_client == nil then if self.__type == "simple" then local value, flags = self.__data_dict:get(data) - return ~= nil - else if self.__type == "match" then + return value ~= nil + elseif self.__type == "match" then local patterns = self.__data_dict:get_keys(0) for i, pattern in ipairs(patterns) do if string.match(data, pattern) then return true end end - return false end -- with redis - else if data_dict == nil and redis_client ~= nil then + elseif self.__data_dict == nil and self.__redis_client ~= nil then if self.__type == "simple" then local res, err = self.__redis_client:get(self.__name .. "_" .. data) return res and res ~= ngx.null - else if self.__type == "match" then + elseif self.__type == "match" then local patterns = self.__redis_client:keys(self.__name .. "_*") if patterns then for i, pattern in ipairs(patterns) do - if string.match(data, pattern) do + if string.match(data, pattern) then return true end end @@ -45,4 +46,6 @@ function M.check(self, data) : return false +end + return M