From fccf14627f6b928bf31daac9a65d8e19bc9a3930 Mon Sep 17 00:00:00 2001 From: bunkerity Date: Tue, 20 Jul 2021 11:41:31 +0200 Subject: [PATCH] jobs - python stubs --- jobs/.Job.py.swp | Bin 1024 -> 0 bytes jobs/Abusers.py | 11 +++++++++++ jobs/CertbotNew.py | 9 +++++++++ jobs/CertbotRenew.py | 9 +++++++++ jobs/ExitNodes.py | 6 +++--- jobs/GeoIP.py | 27 +++++++++++++++++++++++++++ jobs/Job.py | 15 ++++++++++++--- jobs/Proxies.py | 11 +++++++++++ jobs/Referrers.py | 11 +++++++++++ jobs/UserAgents.py | 11 +++++++++++ 10 files changed, 104 insertions(+), 6 deletions(-) delete mode 100644 jobs/.Job.py.swp create mode 100644 jobs/Abusers.py create mode 100644 jobs/CertbotNew.py create mode 100644 jobs/CertbotRenew.py create mode 100644 jobs/GeoIP.py create mode 100644 jobs/Proxies.py create mode 100644 jobs/Referrers.py create mode 100644 jobs/UserAgents.py diff --git a/jobs/.Job.py.swp b/jobs/.Job.py.swp deleted file mode 100644 index 497c7c30df39c8f247902708730763cc7c409ccf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1024 zcmYc?$V<%2S1{HyVn6{47#J8zi&KkG1aWX&T!Xzs`~!5|gIt|m{CsezVqnnEFDTJ3 z&o9c>Pb$sJPA$r;N=?ztOV7-!(9g#HoE$Vl)IsLx8dn2u0Tp08vyJ AQ2+n{ diff --git a/jobs/Abusers.py b/jobs/Abusers.py new file mode 100644 index 0000000..32bafe1 --- /dev/null +++ b/jobs/Abusers.py @@ -0,0 +1,11 @@ +from Job import Job + +class Abusers(Job) : + + def __init__(self, redis_host=None) : + name = "abusers" + data = ["https://iplists.firehol.org/files/firehol_abusers_30d.netset"] + filename = "abusers.list" + type = "line" + regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" + super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex) diff --git a/jobs/CertbotNew.py b/jobs/CertbotNew.py new file mode 100644 index 0000000..abba9b2 --- /dev/null +++ b/jobs/CertbotNew.py @@ -0,0 +1,9 @@ +from Job import Job + +class CertbotRenew(Job) : + + def __init__(self, redis_host=None, domain="", email="") : + name = "certbot-new" + data = ["certbot", "certonly", "--webroot", "-w", "/opt/bunkerized-nginx/acme-challenge", "-n", "-d", domain, "--email", email, "--agree-tos"] + type = "exec" + super().__init__(name, data, filename, redis_host=redis_host, type=type) diff --git a/jobs/CertbotRenew.py b/jobs/CertbotRenew.py new file mode 100644 index 0000000..36a48f9 --- /dev/null +++ b/jobs/CertbotRenew.py @@ -0,0 +1,9 @@ +from Job import Job + +class CertbotRenew(Job) : + + def __init__(self, redis_host=None) : + name = "certbot-renew" + data = ["certbot", "renew", "--deploy-hook", "/opt/bunkerized-nginx/jobs/reload.py"] + type = "exec" + super().__init__(name, data, filename, redis_host=redis_host, type=type) diff --git a/jobs/ExitNodes.py b/jobs/ExitNodes.py index 6e6091f..40712a4 100644 --- a/jobs/ExitNodes.py +++ b/jobs/ExitNodes.py @@ -3,9 +3,9 @@ from Job import Job class ExitNodes(Job) : def __init__(self, redis_host=None) : - name = "ExitNodes" - urls = ["https://iplists.firehol.org/files/tor_exits.ipset"] + name = "exit-nodes" + data = ["https://iplists.firehol.org/files/tor_exits.ipset"] filename = "tor-exit-nodes.list" type = "line" regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" - super().__init__(name, urls, filename, redis_host=redis_host, type=type, regex=regex) + super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex) diff --git a/jobs/GeoIP.py b/jobs/GeoIP.py new file mode 100644 index 0000000..2a13957 --- /dev/null +++ b/jobs/GeoIP.py @@ -0,0 +1,27 @@ +from Job import Job + +import datetime, gzip + +class GeoIP(Job) : + + def __init__(self, redis_host=None) : + name = "geoip" + data = ["https://download.db-ip.com/free/dbip-country-lite-" + datetime.datetime.today().strftime("%Y-%m") + ".mmdb.gz"] + filename = "geoip.mmdb.gz" + type = "file" + super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex) + + def run(self) : + super().run() + count = 0 + with gzip.open("/etc/nginx/geoip.mmdb.gz", "rb") as f : + with open("/tmp/geoip.mmdb", "w") as f2 + while True : + chunk = f.read(8192) + if not chunk : + break + f2.write(chunk) + count += 1 + shutil.copyfile("/tmp/geoip.mmdb", "/etc/nginx/geoip.mmdb") + os.remove("/tmp/geoip.mmdb") + os.remove("/etc/nginx/geoip.mmdb.gz") diff --git a/jobs/Job.py b/jobs/Job.py index 33b169e..7097acf 100644 --- a/jobs/Job.py +++ b/jobs/Job.py @@ -2,9 +2,9 @@ import abc, requests, redis, os class Job(abc.ABC) : - def __init__(self, name, urls, filename, redis_host=None, type="line", regex=r"^.*$") : + def __init__(self, name, data, filename, redis_host=None, type="line", regex=r"^.+$") : self.__name = name - self.__urls = urls + self.__data = data self.__filename = filename self.__redis = None if redis_host != None : @@ -13,6 +13,12 @@ class Job(abc.ABC) : self.__regex = regex def run(self) : + if self.__type == "line" or self.__type == "file" : + self.__external() + elif self.__type == "exec" : + self.__exec() + + def __external(self) : if self.__redis == None : if os.path.isfile("/tmp/" + self.__filename) : os.remove("/tmp/" + self.__filename) @@ -22,7 +28,7 @@ class Job(abc.ABC) : pipe = self.__redis.pipeline() count = 0 - for url in self.__urls : + for url in self.__data : data = self.__download_data(url) for chunk in data : if self.__type == "line" and not re.match(self.__regex, chunk) : @@ -52,3 +58,6 @@ class Job(abc.ABC) : if self.__type == "line" : return r.iter_lines() return r.iter_content(chunk_size=8192) + + def __exec(self) : + proc = subprocess.run(self.__data, capture_output=True) diff --git a/jobs/Proxies.py b/jobs/Proxies.py new file mode 100644 index 0000000..0b04f95 --- /dev/null +++ b/jobs/Proxies.py @@ -0,0 +1,11 @@ +from Job import Job + +class Proxies(Job) : + + def __init__(self, redis_host=None) : + name = "proxies" + data = ["https://iplists.firehol.org/files/firehol_proxies.netset"] + filename = "proxies.list" + type = "line" + regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" + super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex) diff --git a/jobs/Referrers.py b/jobs/Referrers.py new file mode 100644 index 0000000..1744809 --- /dev/null +++ b/jobs/Referrers.py @@ -0,0 +1,11 @@ +from Job import Job + +class Referrers(Job) : + + def __init__(self, redis_host=None) : + name = "referrers" + data = ["https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list"] + filename = "referrers.list" + type = "line" + regex = r"^.+$" + super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex) diff --git a/jobs/UserAgents.py b/jobs/UserAgents.py new file mode 100644 index 0000000..5d170d3 --- /dev/null +++ b/jobs/UserAgents.py @@ -0,0 +1,11 @@ +from Job import Job + +class UserAgents(Job) : + + def __init__(self, redis_host=None) : + name = "user-agents" + data = ["https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list", "https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Crawlers.txt"] + filename = "user-agents.list" + type = "line" + regex = r"^.+$" + super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex)