diff --git a/gen/main.py b/gen/main.py index 17b241c..0fe156a 100755 --- a/gen/main.py +++ b/gen/main.py @@ -1,6 +1,6 @@ #!/usr/bin/python3 -import argparse, os, sys, shutil +import argparse, os, sys, shutil, glob import utils from Configurator import Configurator diff --git a/jobs/Abusers.py b/jobs/Abusers.py index a048940..f14151f 100644 --- a/jobs/Abusers.py +++ b/jobs/Abusers.py @@ -1,5 +1,7 @@ from Job import Job +import re, ipaddress + class Abusers(Job) : def __init__(self, redis_host=None, copy_cache=False) : @@ -9,3 +11,13 @@ class Abusers(Job) : type = "line" regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache) + + def _Job__edit(self, chunk) : + if self.__redis != None : + network = chunk.decode("utf-8") + if re.match(network, r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\/?[0-9]+$") : + ips = [] + for ip in ipaddress.IPv4Network(network) : + ips.append(str(ip).encode("utf-8")) + return [chunk] + diff --git a/jobs/ExitNodes.py b/jobs/ExitNodes.py index d7d64ed..7a2e39f 100644 --- a/jobs/ExitNodes.py +++ b/jobs/ExitNodes.py @@ -1,5 +1,7 @@ from Job import Job +import re, ipaddress + class ExitNodes(Job) : def __init__(self, redis_host=None, copy_cache=False) : @@ -9,3 +11,12 @@ class ExitNodes(Job) : type = "line" regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache) + + def _Job__edit(self, chunk) : + if self.__redis != None : + network = chunk.decode("utf-8") + if re.match(network, r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\/?[0-9]+$") : + ips = [] + for ip in ipaddress.IPv4Network(network) : + ips.append(str(ip).encode("utf-8")) + return [chunk] diff --git a/jobs/Job.py b/jobs/Job.py index 9cd6854..ee5e4ed 100644 --- a/jobs/Job.py +++ b/jobs/Job.py @@ -61,13 +61,17 @@ class Job(abc.ABC) : if self.__type == "line" : if not re.match(self.__regex, chunk.decode("utf-8")) : continue - chunk = self.__edit(chunk) + chunks = self.__edit(chunk) if self.__redis == None : if self.__type == "line" : chunk += b"\n" file.write(chunk) else : - pipe.set(self.__name + "_" + chunk, "1") + if self.__type == "line" : + for chunk in chunks : + pipe.set(self.__name + "_" + chunk, "1") + else : + pipe.set(self.__name + "_" + chunk, "1") count += 1 if self.__redis == None : @@ -106,7 +110,7 @@ class Job(abc.ABC) : return JobRet.OK_RELOAD def __edit(self, chunk) : - return chunk + return [chunk] def __from_cache(self) : if not os.path.isfile("/opt/bunkerized-nginx/cache/" + self.__filename) : diff --git a/jobs/Proxies.py b/jobs/Proxies.py index e30793a..4863627 100644 --- a/jobs/Proxies.py +++ b/jobs/Proxies.py @@ -1,5 +1,7 @@ from Job import Job +import re, ipaddress + class Proxies(Job) : def __init__(self, redis_host=None, copy_cache=False) : @@ -9,3 +11,12 @@ class Proxies(Job) : type = "line" regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache) + + def _Job__edit(self, chunk) : + if self.__redis != None : + network = chunk.decode("utf-8") + if re.match(network, r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\/?[0-9]+$") : + ips = [] + for ip in ipaddress.IPv4Network(network) : + ips.append(str(ip).encode("utf-8")) + return [chunk] diff --git a/lua/checker.lua b/lua/checker.lua new file mode 100644 index 0000000..8a2f7cf --- /dev/null +++ b/lua/checker.lua @@ -0,0 +1,48 @@ +local M = {} +local redis = require "resty.redis" + +function M.new(self, name, data_dict, redis_client, type) + return selfmetatable({ + __name = name, + __data_dict = data_dict, + __redis_client = redis_client, + __type = type + }) +end + +function M.check(self, data) : + -- without redis + if self.__data_dict ~= nil and redis_client == nil then + if self.__type == "simple" then + local value, flags = self.__data_dict:get(data) + return ~= nil + else if self.__type == "match" then + local patterns = self.__data_dict:get_keys(0) + for i, pattern in ipairs(patterns) do + if string.match(data, pattern) then + return true + end + end + return false + end + + -- with redis + else if data_dict == nil and redis_client ~= nil then + if self.__type == "simple" then + local res, err = self.__redis_client:get(self.__name .. "_" .. data) + return res and res ~= ngx.null + else if self.__type == "match" then + local patterns = self.__redis_client:keys(self.__name .. "_*") + if patterns then + for i, pattern in ipairs(patterns) do + if string.match(data, pattern) do + return true + end + end + end + end + end + + return false + +return M