diff --git a/jobs/Abusers.py b/jobs/Abusers.py index 32bafe1..a048940 100644 --- a/jobs/Abusers.py +++ b/jobs/Abusers.py @@ -2,10 +2,10 @@ from Job import Job class Abusers(Job) : - def __init__(self, redis_host=None) : + def __init__(self, redis_host=None, copy_cache=False) : name = "abusers" data = ["https://iplists.firehol.org/files/firehol_abusers_30d.netset"] filename = "abusers.list" type = "line" regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" - super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex) + super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache) diff --git a/jobs/CertbotNew.py b/jobs/CertbotNew.py index abba9b2..7a5e445 100644 --- a/jobs/CertbotNew.py +++ b/jobs/CertbotNew.py @@ -2,8 +2,8 @@ from Job import Job class CertbotRenew(Job) : - def __init__(self, redis_host=None, domain="", email="") : + def __init__(self, redis_host=None, copy_cache=False, domain="", email="") : name = "certbot-new" data = ["certbot", "certonly", "--webroot", "-w", "/opt/bunkerized-nginx/acme-challenge", "-n", "-d", domain, "--email", email, "--agree-tos"] type = "exec" - super().__init__(name, data, filename, redis_host=redis_host, type=type) + super().__init__(name, data, filename, redis_host=redis_host, type=type, copy_cache=copy_cache) diff --git a/jobs/CertbotRenew.py b/jobs/CertbotRenew.py index 36a48f9..520eb27 100644 --- a/jobs/CertbotRenew.py +++ b/jobs/CertbotRenew.py @@ -2,8 +2,8 @@ from Job import Job class CertbotRenew(Job) : - def __init__(self, redis_host=None) : + def __init__(self, redis_host=None, copy_cache=False) : name = "certbot-renew" data = ["certbot", "renew", "--deploy-hook", "/opt/bunkerized-nginx/jobs/reload.py"] type = "exec" - super().__init__(name, data, filename, redis_host=redis_host, type=type) + super().__init__(name, data, filename, redis_host=redis_host, type=type, copy_cache=copy_cache) diff --git a/jobs/ExitNodes.py b/jobs/ExitNodes.py index 40712a4..576da6e 100644 --- a/jobs/ExitNodes.py +++ b/jobs/ExitNodes.py @@ -2,10 +2,10 @@ from Job import Job class ExitNodes(Job) : - def __init__(self, redis_host=None) : + def __init__(self, redis_host=None, copy_cache=copy_cache) : name = "exit-nodes" data = ["https://iplists.firehol.org/files/tor_exits.ipset"] filename = "tor-exit-nodes.list" type = "line" regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" - super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex) + super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache) diff --git a/jobs/GeoIP.py b/jobs/GeoIP.py index 2a13957..c163f79 100644 --- a/jobs/GeoIP.py +++ b/jobs/GeoIP.py @@ -4,12 +4,12 @@ import datetime, gzip class GeoIP(Job) : - def __init__(self, redis_host=None) : + def __init__(self, redis_host=None, copy_cache=False) : name = "geoip" data = ["https://download.db-ip.com/free/dbip-country-lite-" + datetime.datetime.today().strftime("%Y-%m") + ".mmdb.gz"] filename = "geoip.mmdb.gz" type = "file" - super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex) + super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache) def run(self) : super().run() diff --git a/jobs/Job.py b/jobs/Job.py index b54880a..c664c9b 100644 --- a/jobs/Job.py +++ b/jobs/Job.py @@ -2,7 +2,7 @@ import abc, requests, redis, os, datetime, traceback class Job(abc.ABC) : - def __init__(self, name, data, filename, redis_host=None, type="line", regex=r"^.+$") : + def __init__(self, name, data, filename, redis_host=None, type="line", regex=r"^.+$", copy_cache=False) : self.__name = name self.__data = data self.__filename = filename @@ -15,6 +15,7 @@ class Job(abc.ABC) : self.__log("can't connect to redis host " + redis_host) self.__type = type self.__regex = regex + self.__copy_cache = copy_cache def __log(self, data) : when = datetime.datetime.today().strftime("[%Y-%m-%d %H:%M:%S]") @@ -25,7 +26,10 @@ class Job(abc.ABC) : def run(self) : try : if self.__type == "line" or self.__type == "file" : + if self.__copy_cache and self.__from_cache() : + return True self.__external() + self.__to_cache() elif self.__type == "exec" : self.__exec() except Exception as e : @@ -84,3 +88,31 @@ class Job(abc.ABC) : self.__log("stderr = " + stderr) if proc.returncode != 0 : raise Exception("error code " + str(proc.returncode)) + + def __from_cache(self) : + if not os.path.isfile("/opt/bunkerized-nginx/cache/" + self.__filename) : + return False + if self.__redis == None or self.__type == "file" : + shutil.copyfile("/opt/bunkerized-nginx/cache/" + self.__filename, "/etc/nginx/" + self.__filename) + elif self.__redis != None and self.__type == "line" : + self.__redis.del(self.__redis.keys(self.__name + "_*")) + with open("/opt/bunkerized-nginx/cache/" + self.__filename) as f : + pipe = self.__redis.pipeline() + while True : + line = f.readline() + if not line : + break + line = line.strip() + pipe.set(self.__name + "_" + line, "1") + pipe.execute() + return True + + def __to_cache(self) : + if self.__redis == None or self.__type == "file" : + shutil.copyfile("/etc/nginx/" + self.__filename, "/opt/bunkerized-nginx/cache/" + self.__filename) + elif self.__redis != None and self.__type == "line" : + if os.path.isfile("/opt/bunkerized-nginx/cache/" + self.__filename) : + os.remove("/opt/bunkerized-nginx/cache/" + self.__filename) + with open("/opt/bunkerized-nginx/cache/" + self.__filename, "a") as f : + for key in self.__redis.keys(self.__name + "_*") : + f.write(self.__redis.get(key) + "\n") diff --git a/jobs/Proxies.py b/jobs/Proxies.py index 0b04f95..3295e94 100644 --- a/jobs/Proxies.py +++ b/jobs/Proxies.py @@ -2,10 +2,10 @@ from Job import Job class Proxies(Job) : - def __init__(self, redis_host=None) : + def __init__(self, redis_host=None, copy_cache=False) : name = "proxies" data = ["https://iplists.firehol.org/files/firehol_proxies.netset"] filename = "proxies.list" type = "line" regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" - super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex) + super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex; copy_cache=copy_cache) diff --git a/jobs/Referrers.py b/jobs/Referrers.py index 1744809..e873df1 100644 --- a/jobs/Referrers.py +++ b/jobs/Referrers.py @@ -2,10 +2,10 @@ from Job import Job class Referrers(Job) : - def __init__(self, redis_host=None) : + def __init__(self, redis_host=None, copy_cache=False) : name = "referrers" data = ["https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list"] filename = "referrers.list" type = "line" regex = r"^.+$" - super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex) + super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache) diff --git a/jobs/UserAgents.py b/jobs/UserAgents.py index 5d170d3..99f6162 100644 --- a/jobs/UserAgents.py +++ b/jobs/UserAgents.py @@ -2,10 +2,10 @@ from Job import Job class UserAgents(Job) : - def __init__(self, redis_host=None) : + def __init__(self, redis_host=None, copy_cache=False) : name = "user-agents" data = ["https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list", "https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Crawlers.txt"] filename = "user-agents.list" type = "line" regex = r"^.+$" - super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex) + super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache) diff --git a/jobs/requirements.txt b/jobs/requirements.txt new file mode 100644 index 0000000..28bd01f --- /dev/null +++ b/jobs/requirements.txt @@ -0,0 +1,2 @@ +requests +redis