From 5f845680ff38031e1e527dc947e7eff84740a9c9 Mon Sep 17 00:00:00 2001 From: bunkerity Date: Wed, 21 Jul 2021 14:42:55 +0200 Subject: [PATCH] jobs - edit referrers and user-agents data and init work on autoconf integration --- autoconf/Config.py | 14 +++++--------- autoconf/Dockerfile | 13 ++++++------- autoconf/dependencies.sh | 5 ----- autoconf/prepare.sh | 2 ++ autoconf/reload.py | 19 ------------------- autoconf/requirements.txt | 3 +++ jobs/Job.py | 11 ++++++++--- jobs/Referrers.py | 3 +++ jobs/UserAgents.py | 3 +++ jobs/main.py | 8 ++++++-- jobs/reload.py | 18 +++++++++++------- 11 files changed, 47 insertions(+), 52 deletions(-) delete mode 100644 autoconf/dependencies.sh delete mode 100644 autoconf/reload.py create mode 100644 autoconf/requirements.txt diff --git a/autoconf/Config.py b/autoconf/Config.py index d0bce69..b0812f5 100644 --- a/autoconf/Config.py +++ b/autoconf/Config.py @@ -9,9 +9,9 @@ class Config : self.__swarm = swarm self.__api = api - def __jobs(self, type) : - utils.log("[*] Starting jobs (type = " + type + ") ...") - proc = subprocess.run(["/bin/su", "-c", "/opt/bunkerized-nginx/entrypoint/" + type + "-jobs.sh", "nginx"], capture_output=True) + def __jobs(self) : + utils.log("[*] Starting jobs") + proc = subprocess.run(["/bin/su", "-c", "/opt/bunkerized-nginx/entrypoint/jobs.sh", "nginx"], capture_output=True) stdout = proc.stdout.decode("ascii") stderr = proc.stderr.decode("ascii") if len(stdout) > 1 : @@ -71,7 +71,7 @@ class Config : # We're done if proc.returncode == 0 : if self.__swarm : - return self.__jobs("pre") + return self.__jobs() return True utils.log("[!] Error while generating site config for " + env["SERVER_NAME"] + " : return code = " + str(proc.returncode)) @@ -80,11 +80,7 @@ class Config : return False def reload(self, instances) : - if self.__api_call(instances, "/reload") : - if self.__swarm : - return self.__jobs("post") - return True - return False + return self.__api_call(instances, "/reload") def __ping(self, instances) : return self.__api_call(instances, "/ping") diff --git a/autoconf/Dockerfile b/autoconf/Dockerfile index 6ff498f..13fdddf 100644 --- a/autoconf/Dockerfile +++ b/autoconf/Dockerfile @@ -1,20 +1,19 @@ FROM alpine -COPY autoconf/dependencies.sh /tmp -RUN chmod +x /tmp/dependencies.sh && \ - /tmp/dependencies.sh && \ - rm -f /tmp/dependencies.sh && \ - mkdir /opt/bunkerized-nginx - COPY gen/ /opt/bunkerized-nginx/gen COPY entrypoint/ /opt/bunkerized-nginx/entrypoint COPY confs/global/ /opt/bunkerized-nginx/confs/global COPY confs/site/ /opt/bunkerized-nginx/confs/site -COPY scripts/ /opt/bunkerized-nginx/scripts +COPY jobs/ /opt/bunkerized-nginx/jobs COPY settings.json /opt/bunkerized-nginx/ COPY misc/cron /etc/crontabs/nginx COPY autoconf/* /opt/bunkerized-nginx/entrypoint/ +RUN apk add --no-cache py3-pip bash certbot curl openssl && \ + pip3 install -r /opt/bunkerized-nginx/gen/requirements.txt && \ + pip3 install -r /opt/bunkerized-nginx/entrypoint/requirements.txt && \ + pip3 install -r /opt/bunkerized-nginx/jobs/requirements.txt + COPY autoconf/prepare.sh /tmp RUN chmod +x /tmp/prepare.sh && \ /tmp/prepare.sh && \ diff --git a/autoconf/dependencies.sh b/autoconf/dependencies.sh deleted file mode 100644 index 565c2ea..0000000 --- a/autoconf/dependencies.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -# install dependencies -apk add py3-pip bash certbot curl openssl -pip3 install docker requests jinja2 diff --git a/autoconf/prepare.sh b/autoconf/prepare.sh index 3a6cdf9..4288b67 100644 --- a/autoconf/prepare.sh +++ b/autoconf/prepare.sh @@ -14,6 +14,8 @@ find /opt/bunkerized-nginx -type f -exec chmod 0740 {} \; find /opt/bunkerized-nginx -type d -exec chmod 0750 {} \; chmod ugo+x /opt/bunkerized-nginx/entrypoint/* /opt/bunkerized-nginx/scripts/* chmod ugo+x /opt/bunkerized-nginx/gen/main.py +chmod ugo+x /opt/bunkerized-nginx/jobs/main.py +chmod ugo+x /opt/bunkerized-nginx/jobs/reload.py chmod 770 /opt/bunkerized-nginx chmod 440 /opt/bunkerized-nginx/settings.json diff --git a/autoconf/reload.py b/autoconf/reload.py deleted file mode 100644 index 6fe1815..0000000 --- a/autoconf/reload.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/python3 - -import sys, socket, os - -if not os.path.exists("/tmp/autoconf.sock") : - sys.exit(1) - -try : - client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - client.connect("/tmp/autoconf.sock") - client.send("reload".encode("utf-8")) - data = client.recv(512) - client.close() - if not data or data.decode("utf-8") != "ok" : - sys.exit(3) -except Exception as e : - sys.exit(2) - -sys.exit(0) diff --git a/autoconf/requirements.txt b/autoconf/requirements.txt new file mode 100644 index 0000000..ddb40d3 --- /dev/null +++ b/autoconf/requirements.txt @@ -0,0 +1,3 @@ +docker +requests +jinja2 diff --git a/jobs/Job.py b/jobs/Job.py index f9afc29..eb3d618 100644 --- a/jobs/Job.py +++ b/jobs/Job.py @@ -50,15 +50,17 @@ class Job(abc.ABC) : for url in self.__data : data = self.__download_data(url) for chunk in data : - if self.__type == "line" and not re.match(self.__regex, chunk.decode("utf-8")) : - continue - count += 1 + if self.__type == "line" : + if not re.match(self.__regex, chunk.decode("utf-8")) : + continue + chunk = self.__edit(chunk) if self.__redis == None : if self.__type == "line" : chunk += b"\n" file.write(chunk) else : pipe.set(self.__name + "_" + chunk, "1") + count += 1 if self.__redis == None : file.close() @@ -89,6 +91,9 @@ class Job(abc.ABC) : if proc.returncode != 0 : raise Exception("error code " + str(proc.returncode)) + def __edit(self, chunk) : + return chunk + def __from_cache(self) : if not os.path.isfile("/opt/bunkerized-nginx/cache/" + self.__filename) : return False diff --git a/jobs/Referrers.py b/jobs/Referrers.py index e873df1..d1dd403 100644 --- a/jobs/Referrers.py +++ b/jobs/Referrers.py @@ -9,3 +9,6 @@ class Referrers(Job) : type = "line" regex = r"^.+$" super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache) + + def __edit(self, chunk) : + return chunk.replace(b".", b"%.").replace(b"-", b"%-") diff --git a/jobs/UserAgents.py b/jobs/UserAgents.py index 99f6162..162a0b3 100644 --- a/jobs/UserAgents.py +++ b/jobs/UserAgents.py @@ -9,3 +9,6 @@ class UserAgents(Job) : type = "line" regex = r"^.+$" super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache) + + def __edit(self, chunk) : + return chunk.replace(b"\\ ", b" ").replace(b"\\.", b"%.").replace(b"\\\\", b"\\").replace(b"-", b"%-") diff --git a/jobs/main.py b/jobs/main.py index b3ba08d..053c984 100644 --- a/jobs/main.py +++ b/jobs/main.py @@ -59,10 +59,14 @@ if __name__ == "__main__" : # TODO : only reload if needed do_reload = True if do_reload : - if not reload() : + ret = reload() + if ret == 0 : + print("[*] Reload operation successfully executed") + elif ret == 1 : print("[!] Error while doing reload operation") sys.exit(1) - print("[*] Reload operation successfully executed") + elif ret == 2 : + print("[*] Skipped reload operation because nginx is not running") # Done sys.exit(0) diff --git a/jobs/reload.py b/jobs/reload.py index 7661c90..c0b82ae 100644 --- a/jobs/reload.py +++ b/jobs/reload.py @@ -11,8 +11,8 @@ def reload() : print(proc.stdout.decode("ascii")) if len(proc.stderr.decode("ascii")) > 1 : print(proc.stderr.decode("ascii")) - return False - return True + return 0 + return 1 # Autoconf case (Docker, Swarm and Ingress) if os.path.exists("/tmp/autoconf.sock") and stat.S_ISSOCK(os.stat("/tmp/autoconf.sock")) : @@ -23,17 +23,21 @@ def reload() : client.close() if not data or data.decode("utf-8") != "ok" : print("[!] Can't reload nginx (data not ok)") - return False - return True + return 0 + return 1 - return False + return 2 if __name__ == "__main__" : try : print("[*] Starting reload operation ...") - if not reload() : + ret = reload() + if ret == 0 : sys.exit(1) - print("[*] Reload operation successfully executed") + elif ret == 1 : + print("[*] Reload operation successfully executed") + elif ret == 2 : + print("[*] Skipped reload operation because nginx is not running") sys.exit(0) except : print("[!] Can't reload nginx (exception)")