jobs - edit referrers and user-agents data and init work on autoconf integration

autoconf/Config.py

@@ -9,9 +9,9 @@ class Config :
 		self.__swarm = swarm
 		self.__api = api
 
-	def __jobs(self, type) :
+	def __jobs(self) :
-		utils.log("[*] Starting jobs (type = " + type + ") ...")
+		utils.log("[*] Starting jobs")
-		proc = subprocess.run(["/bin/su", "-c", "/opt/bunkerized-nginx/entrypoint/" + type + "-jobs.sh", "nginx"], capture_output=True)
+		proc = subprocess.run(["/bin/su", "-c", "/opt/bunkerized-nginx/entrypoint/jobs.sh", "nginx"], capture_output=True)
 		stdout = proc.stdout.decode("ascii")
 		stderr = proc.stderr.decode("ascii")
 		if len(stdout) > 1 :
@@ -71,7 +71,7 @@ class Config :
 			# We're done
 			if proc.returncode == 0 :
 				if self.__swarm :
-					return self.__jobs("pre")
+					return self.__jobs()
 				return True
 			utils.log("[!] Error while generating site config for " + env["SERVER_NAME"] + " : return code = " + str(proc.returncode))
 
@@ -80,11 +80,7 @@ class Config :
 		return False
 
 	def reload(self, instances) :
-		if self.__api_call(instances, "/reload") :
+		return self.__api_call(instances, "/reload")
-			if self.__swarm :
-				return self.__jobs("post")
-			return True
-		return False
 
 	def __ping(self, instances) :
 		return self.__api_call(instances, "/ping")

autoconf/Dockerfile

@@ -1,20 +1,19 @@
 FROM alpine
 
-COPY autoconf/dependencies.sh /tmp
-RUN chmod +x /tmp/dependencies.sh && \
-    /tmp/dependencies.sh && \
-    rm -f /tmp/dependencies.sh && \
-    mkdir /opt/bunkerized-nginx
-
 COPY gen/ /opt/bunkerized-nginx/gen
 COPY entrypoint/ /opt/bunkerized-nginx/entrypoint
 COPY confs/global/ /opt/bunkerized-nginx/confs/global
 COPY confs/site/ /opt/bunkerized-nginx/confs/site
-COPY scripts/ /opt/bunkerized-nginx/scripts
+COPY jobs/ /opt/bunkerized-nginx/jobs
 COPY settings.json /opt/bunkerized-nginx/
 COPY misc/cron /etc/crontabs/nginx
 COPY autoconf/* /opt/bunkerized-nginx/entrypoint/
 
+RUN apk add --no-cache py3-pip bash certbot curl openssl && \
+    pip3 install -r /opt/bunkerized-nginx/gen/requirements.txt && \
+    pip3 install -r /opt/bunkerized-nginx/entrypoint/requirements.txt && \
+    pip3 install -r /opt/bunkerized-nginx/jobs/requirements.txt
+
 COPY autoconf/prepare.sh /tmp
 RUN chmod +x /tmp/prepare.sh && \
     /tmp/prepare.sh && \

autoconf/dependencies.sh

@@ -1,5 +0,0 @@
-#!/bin/sh
-
-# install dependencies
-apk add py3-pip bash certbot curl openssl
-pip3 install docker requests jinja2

autoconf/prepare.sh

@@ -14,6 +14,8 @@ find /opt/bunkerized-nginx -type f -exec chmod 0740 {} \;
 find /opt/bunkerized-nginx -type d -exec chmod 0750 {} \;
 chmod ugo+x /opt/bunkerized-nginx/entrypoint/* /opt/bunkerized-nginx/scripts/*
 chmod ugo+x /opt/bunkerized-nginx/gen/main.py
+chmod ugo+x /opt/bunkerized-nginx/jobs/main.py
+chmod ugo+x /opt/bunkerized-nginx/jobs/reload.py
 chmod 770 /opt/bunkerized-nginx
 chmod 440 /opt/bunkerized-nginx/settings.json
 

autoconf/reload.py

@@ -1,19 +0,0 @@
-#!/usr/bin/python3
-
-import sys, socket, os
-
-if not os.path.exists("/tmp/autoconf.sock") :
-	sys.exit(1)
-
-try :
-	client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-	client.connect("/tmp/autoconf.sock")
-	client.send("reload".encode("utf-8"))
-	data = client.recv(512)
-	client.close()
-	if not data or data.decode("utf-8") != "ok" :
-		sys.exit(3)
-except Exception as e :
-	sys.exit(2)
-
-sys.exit(0)

autoconf/requirements.txt

@@ -0,0 +1,3 @@
+docker
+requests
+jinja2

jobs/Job.py

@@ -50,15 +50,17 @@ class Job(abc.ABC) :
 		for url in self.__data :
 			data = self.__download_data(url)
 			for chunk in data :
-				if self.__type == "line" and not re.match(self.__regex, chunk.decode("utf-8")) :
+				if self.__type == "line" :
-					continue
+					if not re.match(self.__regex, chunk.decode("utf-8")) :
-				count += 1
+						continue
+					chunk = self.__edit(chunk)
 				if self.__redis == None :
 					if self.__type == "line" :
 						chunk += b"\n"
 					file.write(chunk)
 				else :
 					pipe.set(self.__name + "_" + chunk, "1")
+				count += 1
 
 		if self.__redis == None :
 			file.close()
@@ -89,6 +91,9 @@ class Job(abc.ABC) :
 		if proc.returncode != 0 :
 			raise Exception("error code " + str(proc.returncode))
 
+	def __edit(self, chunk) :
+		return chunk
+
 	def __from_cache(self) :
 		if not os.path.isfile("/opt/bunkerized-nginx/cache/" + self.__filename) :
 			return False

jobs/Referrers.py

@@ -9,3 +9,6 @@ class Referrers(Job) :
 		type = "line"
 		regex = r"^.+$"
 		super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache)
+
+	def __edit(self, chunk) :
+		return chunk.replace(b".", b"%.").replace(b"-", b"%-")

jobs/UserAgents.py

@@ -9,3 +9,6 @@ class UserAgents(Job) :
 		type = "line"
 		regex = r"^.+$"
 		super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache)
+
+	def __edit(self, chunk) :
+		return chunk.replace(b"\\ ", b" ").replace(b"\\.", b"%.").replace(b"\\\\", b"\\").replace(b"-", b"%-")

jobs/main.py

@@ -59,10 +59,14 @@ if __name__ == "__main__" :
 	# TODO : only reload if needed
 	do_reload = True
 	if do_reload :
-		if not reload() :
+		ret = reload()
+		if ret == 0 :
+			print("[*] Reload operation successfully executed")
+		elif ret == 1 :
 			print("[!] Error while doing reload operation")
 			sys.exit(1)
-		print("[*] Reload operation successfully executed")
+		elif ret == 2 :
+			print("[*] Skipped reload operation because nginx is not running")
 
 	# Done
 	sys.exit(0)

jobs/reload.py

@@ -11,8 +11,8 @@ def reload() :
 				print(proc.stdout.decode("ascii"))
 			if len(proc.stderr.decode("ascii")) > 1 :
 				print(proc.stderr.decode("ascii"))
-			return False
+			return 0
-		return True
+		return 1
 
 	# Autoconf case (Docker, Swarm and Ingress)
 	if os.path.exists("/tmp/autoconf.sock") and stat.S_ISSOCK(os.stat("/tmp/autoconf.sock")) :
@@ -23,17 +23,21 @@ def reload() :
 		client.close()
 		if not data or data.decode("utf-8") != "ok" :
 			print("[!] Can't reload nginx (data not ok)")
-			return False
+			return 0
-		return True
+		return 1
 
-	return False
+	return 2
 
 if __name__ == "__main__" :
 	try :
 		print("[*] Starting reload operation ...")
-		if not reload() :
+		ret = reload()
+		if ret == 0 :
 			sys.exit(1)
-		print("[*] Reload operation successfully executed")
+		elif ret == 1 :
+			print("[*] Reload operation successfully executed")
+		elif ret == 2 :
+			print("[*] Skipped reload operation because nginx is not running")
 		sys.exit(0)
 	except :
 		print("[!] Can't reload nginx (exception)")