jobs - cleaning the mess when using autoconf without swarm mode

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## v1.2.7 - 2021/06/14
+
+- Add custom robots.txt and sitemap to RTD
+- Fix missing GeoIP DB bug when using BLACKLIST/WHITELIST_COUNTRY
+- Add underscore "_" to allowed chars for CUSTOM_HTTPS_CERT/KEY
+- Fix bug when using automatic self-signed certificate
+- Build and push images from GitHub actions instead of Docker Hub autobuild
+- Display the reason when generator is ignoring a variable
+- Various bug fixes related to certbot and jobs
+- Split jobs into pre and post jobs
+
 ## v1.2.6 - 2021/06/06
 
 - Move from "ghetto-style" shell scripts to generic jinja2 templating

autoconf/Config.py

@@ -9,12 +9,12 @@ class Config :
 		self.__swarm = swarm
 		self.__api = api
 
-	def __jobs(self) :
+	def __jobs(self, type) :
-		utils.log("[*] Starting jobs ...")
+		utils.log("[*] Starting jobs (type = " + type + ") ...")
-		proc = subprocess.run(["/bin/su", "-c", "/opt/entrypoint/jobs.sh", "nginx"], capture_output=True)
+		proc = subprocess.run(["/bin/su", "-c", "/opt/entrypoint/" + type + "-jobs.sh", "nginx"], capture_output=True)
 		stdout = proc.stdout.decode("ascii")
 		stderr = proc.stderr.decode("ascii")
-		if stdout != "" :
+		if len(stdout) > 1 :
 			utils.log("[*] Jobs stdout :")
 			utils.log(stdout)
 		if stderr != "" :
@@ -61,7 +61,7 @@ class Config :
 			# Print stdout/stderr
 			stdout = proc.stdout.decode("ascii")
 			stderr = proc.stderr.decode("ascii")
-			if stdout != "" :
+			if len(stdout) > 1 :
 				utils.log("[*] Generator output :")
 				utils.log(stdout)
 			if stderr != "" :
@@ -71,7 +71,7 @@ class Config :
 			# We're done
 			if proc.returncode == 0 :
 				if self.__swarm :
-					return self.__jobs()
+					return self.__jobs("pre")
 				return True
 			utils.log("[!] Error while generating site config for " + env["SERVER_NAME"] + " : return code = " + str(proc.returncode))
 
@@ -80,7 +80,11 @@ class Config :
 		return False
 
 	def reload(self, instances) :
-		return self.__api_call(instances, "/reload")
+		if self.__api_call(instances, "/reload") :
+			if self.__swarm :
+				return self.__jobs("post")
+			return True
+		return False
 
 	def __ping(self, instances) :
 		return self.__api_call(instances, "/ping")

entrypoint/entrypoint.sh

@@ -16,13 +16,14 @@ trap "trap_exit" TERM INT QUIT
 function trap_reload() {
 	echo "[*] Catched reload operation"
 	if [ "$SWARM_MODE" != "yes" ] ; then
-		/opt/entrypoint/jobs.sh
+		/opt/entrypoint/pre-jobs.sh
 	fi
 	if [ -f /tmp/nginx.pid ] ; then
 		echo "[*] Reloading nginx ..."
 		nginx -s reload
 		if [ $? -eq 0 ] ; then
 			echo "[*] Reload successfull"
+			/opt/entrypoint/post-jobs.sh
 		else
 			echo "[!] Reload failed"
 		fi
@@ -58,8 +59,8 @@ if [ ! -f "/etc/nginx/global.env" ] ; then
 		# call the generator
 		/opt/gen/main.py --settings /opt/settings.json --templates /opt/confs --output /etc/nginx --variables /tmp/variables.env
 
-		# external jobs
+		# pre-jobs
-		/opt/entrypoint/jobs.sh
+		/opt/entrypoint/pre-jobs.sh
 	fi
 else
 	echo "[*] Skipping configuration process"
@@ -97,6 +98,9 @@ if [ "$1" == "test" ] ; then
 	exit 1
 fi
 
+# post jobs
+/opt/entrypoint/post-jobs.sh
+
 # wait for nginx
 wait "$pid"
 while [ -f "/tmp/nginx.pid" ] ; do

entrypoint/post-jobs.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# load some functions
+. /opt/entrypoint/utils.sh
+
+# User-Agents
+if [ "$(has_value BLOCK_USER_AGENT yes)" != "" ] ; then
+	if [ -f "/cache/user-agents.list" ] && [ "$(wc -l /cache/user-agents.list | cut -d ' ' -f 1)" -gt 1 ] ; then
+		echo "[*] Copying cached user-agents.list ..."
+		cp /cache/user-agents.list /etc/nginx/user-agents.list
+	elif [ "$(ps aux | grep "user-agents\.sh")" = "" ] ; then
+		echo "[*] Downloading bad user-agent list (in background) ..."
+		/opt/scripts/user-agents.sh > /dev/null 2>&1 &
+	fi
+fi
+
+# Referrers
+if [ "$(has_value BLOCK_REFERRER yes)" != "" ] ; then
+	if [ -f "/cache/referrers.list" ] && [ "$(wc -l /cache/referrers.list | cut -d ' ' -f 1)" -gt 1 ] ; then
+		echo "[*] Copying cached referrers.list ..."
+		cp /cache/referrers.list /etc/nginx/referrers.list
+	elif [ "$(ps aux | grep "referrers\.sh")" = "" ] ; then
+		echo "[*] Downloading bad referrer list (in background) ..."
+		/opt/scripts/referrers.sh > /dev/null 2>&1 &
+	fi
+fi
+
+# exit nodes
+if [ "$(has_value BLOCK_TOR_EXIT_NODE yes)" != "" ] ; then
+	if [ -f "/cache/tor-exit-nodes.list" ] && [ "$(wc -l /cache/tor-exit-nodes.list | cut -d ' ' -f 1)" -gt 1 ] ; then
+		echo "[*] Copying cached tor-exit-nodes.list ..."
+		cp /cache/tor-exit-nodes.list /etc/nginx/tor-exit-nodes.list
+	elif [ "$(ps aux | grep "exit-nodes\.sh")" = "" ] ; then
+		echo "[*] Downloading tor exit nodes list (in background) ..."
+		/opt/scripts/exit-nodes.sh > /dev/null 2>&1 &
+	fi
+fi
+
+# proxies
+if [ "$(has_value BLOCK_PROXIES yes)" != "" ] ; then
+	if [ -f "/cache/proxies.list" ] && [ "$(wc -l /cache/proxies.list | cut -d ' ' -f 1)" -gt 1 ] ; then
+		echo "[*] Copying cached proxies.list ..."
+		cp /cache/proxies.list /etc/nginx/proxies.list
+	elif [ "$(ps aux | grep "proxies\.sh")" = "" ] ; then
+		echo "[*] Downloading proxies list (in background) ..."
+		/opt/scripts/proxies.sh > /dev/null 2>&1 &
+	fi
+fi
+
+# abusers
+if [ "$(has_value BLOCK_ABUSERS yes)" != "" ] ; then
+	if [ -f "/cache/abusers.list" ] && [ "$(wc -l /cache/abusers.list | cut -d ' ' -f 1)" -gt 1 ] ; then
+		echo "[*] Copying cached abusers.list ..."
+		cp /cache/abusers.list /etc/nginx/abusers.list
+	elif [ "$(ps aux | grep "abusers\.sh")" = "" ] ; then
+		echo "[*] Downloading abusers list (in background) ..."
+		/opt/scripts/abusers.sh > /dev/null 2>&1 &
+	fi
+fi

entrypoint/pre-jobs.sh

@@ -78,58 +78,3 @@ if [ "$(has_value BLACKLIST_COUNTRY ".\+")" != "" ] || [ "$(has_value WHITELIST_
 		/opt/scripts/geoip.sh > /dev/null 2>&1
 	fi
 fi
-
-# User-Agents
-if [ "$(has_value BLOCK_USER_AGENT yes)" != "" ] ; then
-	if [ -f "/cache/user-agents.list" ] && [ "$(wc -l /cache/user-agents.list | cut -d ' ' -f 1)" -gt 1 ] ; then
-		echo "[*] Copying cached user-agents.list ..."
-		cp /cache/user-agents.list /etc/nginx/user-agents.list
-	elif [ "$(ps aux | grep "user-agents\.sh")" = "" ] ; then
-		echo "[*] Downloading bad user-agent list (in background) ..."
-		/opt/scripts/user-agents.sh > /dev/null 2>&1 &
-	fi
-fi
-
-# Referrers
-if [ "$(has_value BLOCK_REFERRER yes)" != "" ] ; then
-	if [ -f "/cache/referrers.list" ] && [ "$(wc -l /cache/referrers.list | cut -d ' ' -f 1)" -gt 1 ] ; then
-		echo "[*] Copying cached referrers.list ..."
-		cp /cache/referrers.list /etc/nginx/referrers.list
-	elif [ "$(ps aux | grep "referrers\.sh")" = "" ] ; then
-		echo "[*] Downloading bad referrer list (in background) ..."
-		/opt/scripts/referrers.sh > /dev/null 2>&1 &
-	fi
-fi
-
-# exit nodes
-if [ "$(has_value BLOCK_TOR_EXIT_NODE yes)" != "" ] ; then
-	if [ -f "/cache/tor-exit-nodes.list" ] && [ "$(wc -l /cache/tor-exit-nodes.list | cut -d ' ' -f 1)" -gt 1 ] ; then
-		echo "[*] Copying cached tor-exit-nodes.list ..."
-		cp /cache/tor-exit-nodes.list /etc/nginx/tor-exit-nodes.list
-	elif [ "$(ps aux | grep "exit-nodes\.sh")" = "" ] ; then
-		echo "[*] Downloading tor exit nodes list (in background) ..."
-		/opt/scripts/exit-nodes.sh > /dev/null 2>&1 &
-	fi
-fi
-
-# proxies
-if [ "$(has_value BLOCK_PROXIES yes)" != "" ] ; then
-	if [ -f "/cache/proxies.list" ] && [ "$(wc -l /cache/proxies.list | cut -d ' ' -f 1)" -gt 1 ] ; then
-		echo "[*] Copying cached proxies.list ..."
-		cp /cache/proxies.list /etc/nginx/proxies.list
-	elif [ "$(ps aux | grep "proxies\.sh")" = "" ] ; then
-		echo "[*] Downloading proxies list (in background) ..."
-		/opt/scripts/proxies.sh > /dev/null 2>&1 &
-	fi
-fi
-
-# abusers
-if [ "$(has_value BLOCK_ABUSERS yes)" != "" ] ; then
-	if [ -f "/cache/abusers.list" ] && [ "$(wc -l /cache/abusers.list | cut -d ' ' -f 1)" -gt 1 ] ; then
-		echo "[*] Copying cached abusers.list ..."
-		cp /cache/abusers.list /etc/nginx/abusers.list
-	elif [ "$(ps aux | grep "abusers\.sh")" = "" ] ; then
-		echo "[*] Downloading abusers list (in background) ..."
-		/opt/scripts/abusers.sh > /dev/null 2>&1 &
-	fi
-fi

gen/main.py

@@ -52,12 +52,13 @@ if __name__ == "__main__" :
 	#print(config)
 
 	# Remove old config
-	for filename in os.listdir(args.output):
+	# TODO : remove unnecessary files after rendering
-		file_path = os.path.join(args.output, filename)
+#	for filename in os.listdir(args.output):
-		if os.path.isfile(file_path) or os.path.islink(file_path):
+#		file_path = os.path.join(args.output, filename)
-			os.unlink(file_path)
+#		if os.path.isfile(file_path) or os.path.islink(file_path):
-		elif os.path.isdir(file_path):
+#			os.unlink(file_path)
-			shutil.rmtree(file_path)
+#		elif os.path.isdir(file_path):
+#			shutil.rmtree(file_path)
 
 	# Generate the files from templates and config
 	templator = Templator(config, args.templates, args.output, args.target)