diff --git a/CHANGELOG.md b/CHANGELOG.md index e6561fb..5c23750 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## v1.2.7 - 2021/06/14 + +- Add custom robots.txt and sitemap to RTD +- Fix missing GeoIP DB bug when using BLACKLIST/WHITELIST_COUNTRY +- Add underscore "_" to allowed chars for CUSTOM_HTTPS_CERT/KEY +- Fix bug when using automatic self-signed certificate +- Build and push images from GitHub actions instead of Docker Hub autobuild +- Display the reason when generator is ignoring a variable +- Various bug fixes related to certbot and jobs +- Split jobs into pre and post jobs + ## v1.2.6 - 2021/06/06 - Move from "ghetto-style" shell scripts to generic jinja2 templating diff --git a/autoconf/Config.py b/autoconf/Config.py index ab5fbd6..c961356 100644 --- a/autoconf/Config.py +++ b/autoconf/Config.py @@ -9,12 +9,12 @@ class Config : self.__swarm = swarm self.__api = api - def __jobs(self) : - utils.log("[*] Starting jobs ...") - proc = subprocess.run(["/bin/su", "-c", "/opt/entrypoint/jobs.sh", "nginx"], capture_output=True) + def __jobs(self, type) : + utils.log("[*] Starting jobs (type = " + type + ") ...") + proc = subprocess.run(["/bin/su", "-c", "/opt/entrypoint/" + type + "-jobs.sh", "nginx"], capture_output=True) stdout = proc.stdout.decode("ascii") stderr = proc.stderr.decode("ascii") - if stdout != "" : + if len(stdout) > 1 : utils.log("[*] Jobs stdout :") utils.log(stdout) if stderr != "" : @@ -61,7 +61,7 @@ class Config : # Print stdout/stderr stdout = proc.stdout.decode("ascii") stderr = proc.stderr.decode("ascii") - if stdout != "" : + if len(stdout) > 1 : utils.log("[*] Generator output :") utils.log(stdout) if stderr != "" : @@ -71,7 +71,7 @@ class Config : # We're done if proc.returncode == 0 : if self.__swarm : - return self.__jobs() + return self.__jobs("pre") return True utils.log("[!] Error while generating site config for " + env["SERVER_NAME"] + " : return code = " + str(proc.returncode)) @@ -80,7 +80,11 @@ class Config : return False def reload(self, instances) : - return self.__api_call(instances, "/reload") + if self.__api_call(instances, "/reload") : + if self.__swarm : + return self.__jobs("post") + return True + return False def __ping(self, instances) : return self.__api_call(instances, "/ping") diff --git a/entrypoint/entrypoint.sh b/entrypoint/entrypoint.sh index 1d7b64a..ed7e7bb 100644 --- a/entrypoint/entrypoint.sh +++ b/entrypoint/entrypoint.sh @@ -16,13 +16,14 @@ trap "trap_exit" TERM INT QUIT function trap_reload() { echo "[*] Catched reload operation" if [ "$SWARM_MODE" != "yes" ] ; then - /opt/entrypoint/jobs.sh + /opt/entrypoint/pre-jobs.sh fi if [ -f /tmp/nginx.pid ] ; then echo "[*] Reloading nginx ..." nginx -s reload if [ $? -eq 0 ] ; then echo "[*] Reload successfull" + /opt/entrypoint/post-jobs.sh else echo "[!] Reload failed" fi @@ -58,8 +59,8 @@ if [ ! -f "/etc/nginx/global.env" ] ; then # call the generator /opt/gen/main.py --settings /opt/settings.json --templates /opt/confs --output /etc/nginx --variables /tmp/variables.env - # external jobs - /opt/entrypoint/jobs.sh + # pre-jobs + /opt/entrypoint/pre-jobs.sh fi else echo "[*] Skipping configuration process" @@ -97,6 +98,9 @@ if [ "$1" == "test" ] ; then exit 1 fi +# post jobs +/opt/entrypoint/post-jobs.sh + # wait for nginx wait "$pid" while [ -f "/tmp/nginx.pid" ] ; do diff --git a/entrypoint/post-jobs.sh b/entrypoint/post-jobs.sh new file mode 100644 index 0000000..91f78b9 --- /dev/null +++ b/entrypoint/post-jobs.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# load some functions +. /opt/entrypoint/utils.sh + +# User-Agents +if [ "$(has_value BLOCK_USER_AGENT yes)" != "" ] ; then + if [ -f "/cache/user-agents.list" ] && [ "$(wc -l /cache/user-agents.list | cut -d ' ' -f 1)" -gt 1 ] ; then + echo "[*] Copying cached user-agents.list ..." + cp /cache/user-agents.list /etc/nginx/user-agents.list + elif [ "$(ps aux | grep "user-agents\.sh")" = "" ] ; then + echo "[*] Downloading bad user-agent list (in background) ..." + /opt/scripts/user-agents.sh > /dev/null 2>&1 & + fi +fi + +# Referrers +if [ "$(has_value BLOCK_REFERRER yes)" != "" ] ; then + if [ -f "/cache/referrers.list" ] && [ "$(wc -l /cache/referrers.list | cut -d ' ' -f 1)" -gt 1 ] ; then + echo "[*] Copying cached referrers.list ..." + cp /cache/referrers.list /etc/nginx/referrers.list + elif [ "$(ps aux | grep "referrers\.sh")" = "" ] ; then + echo "[*] Downloading bad referrer list (in background) ..." + /opt/scripts/referrers.sh > /dev/null 2>&1 & + fi +fi + +# exit nodes +if [ "$(has_value BLOCK_TOR_EXIT_NODE yes)" != "" ] ; then + if [ -f "/cache/tor-exit-nodes.list" ] && [ "$(wc -l /cache/tor-exit-nodes.list | cut -d ' ' -f 1)" -gt 1 ] ; then + echo "[*] Copying cached tor-exit-nodes.list ..." + cp /cache/tor-exit-nodes.list /etc/nginx/tor-exit-nodes.list + elif [ "$(ps aux | grep "exit-nodes\.sh")" = "" ] ; then + echo "[*] Downloading tor exit nodes list (in background) ..." + /opt/scripts/exit-nodes.sh > /dev/null 2>&1 & + fi +fi + +# proxies +if [ "$(has_value BLOCK_PROXIES yes)" != "" ] ; then + if [ -f "/cache/proxies.list" ] && [ "$(wc -l /cache/proxies.list | cut -d ' ' -f 1)" -gt 1 ] ; then + echo "[*] Copying cached proxies.list ..." + cp /cache/proxies.list /etc/nginx/proxies.list + elif [ "$(ps aux | grep "proxies\.sh")" = "" ] ; then + echo "[*] Downloading proxies list (in background) ..." + /opt/scripts/proxies.sh > /dev/null 2>&1 & + fi +fi + +# abusers +if [ "$(has_value BLOCK_ABUSERS yes)" != "" ] ; then + if [ -f "/cache/abusers.list" ] && [ "$(wc -l /cache/abusers.list | cut -d ' ' -f 1)" -gt 1 ] ; then + echo "[*] Copying cached abusers.list ..." + cp /cache/abusers.list /etc/nginx/abusers.list + elif [ "$(ps aux | grep "abusers\.sh")" = "" ] ; then + echo "[*] Downloading abusers list (in background) ..." + /opt/scripts/abusers.sh > /dev/null 2>&1 & + fi +fi diff --git a/entrypoint/jobs.sh b/entrypoint/pre-jobs.sh similarity index 61% rename from entrypoint/jobs.sh rename to entrypoint/pre-jobs.sh index e787a01..ccda706 100644 --- a/entrypoint/jobs.sh +++ b/entrypoint/pre-jobs.sh @@ -78,58 +78,3 @@ if [ "$(has_value BLACKLIST_COUNTRY ".\+")" != "" ] || [ "$(has_value WHITELIST_ /opt/scripts/geoip.sh > /dev/null 2>&1 fi fi - -# User-Agents -if [ "$(has_value BLOCK_USER_AGENT yes)" != "" ] ; then - if [ -f "/cache/user-agents.list" ] && [ "$(wc -l /cache/user-agents.list | cut -d ' ' -f 1)" -gt 1 ] ; then - echo "[*] Copying cached user-agents.list ..." - cp /cache/user-agents.list /etc/nginx/user-agents.list - elif [ "$(ps aux | grep "user-agents\.sh")" = "" ] ; then - echo "[*] Downloading bad user-agent list (in background) ..." - /opt/scripts/user-agents.sh > /dev/null 2>&1 & - fi -fi - -# Referrers -if [ "$(has_value BLOCK_REFERRER yes)" != "" ] ; then - if [ -f "/cache/referrers.list" ] && [ "$(wc -l /cache/referrers.list | cut -d ' ' -f 1)" -gt 1 ] ; then - echo "[*] Copying cached referrers.list ..." - cp /cache/referrers.list /etc/nginx/referrers.list - elif [ "$(ps aux | grep "referrers\.sh")" = "" ] ; then - echo "[*] Downloading bad referrer list (in background) ..." - /opt/scripts/referrers.sh > /dev/null 2>&1 & - fi -fi - -# exit nodes -if [ "$(has_value BLOCK_TOR_EXIT_NODE yes)" != "" ] ; then - if [ -f "/cache/tor-exit-nodes.list" ] && [ "$(wc -l /cache/tor-exit-nodes.list | cut -d ' ' -f 1)" -gt 1 ] ; then - echo "[*] Copying cached tor-exit-nodes.list ..." - cp /cache/tor-exit-nodes.list /etc/nginx/tor-exit-nodes.list - elif [ "$(ps aux | grep "exit-nodes\.sh")" = "" ] ; then - echo "[*] Downloading tor exit nodes list (in background) ..." - /opt/scripts/exit-nodes.sh > /dev/null 2>&1 & - fi -fi - -# proxies -if [ "$(has_value BLOCK_PROXIES yes)" != "" ] ; then - if [ -f "/cache/proxies.list" ] && [ "$(wc -l /cache/proxies.list | cut -d ' ' -f 1)" -gt 1 ] ; then - echo "[*] Copying cached proxies.list ..." - cp /cache/proxies.list /etc/nginx/proxies.list - elif [ "$(ps aux | grep "proxies\.sh")" = "" ] ; then - echo "[*] Downloading proxies list (in background) ..." - /opt/scripts/proxies.sh > /dev/null 2>&1 & - fi -fi - -# abusers -if [ "$(has_value BLOCK_ABUSERS yes)" != "" ] ; then - if [ -f "/cache/abusers.list" ] && [ "$(wc -l /cache/abusers.list | cut -d ' ' -f 1)" -gt 1 ] ; then - echo "[*] Copying cached abusers.list ..." - cp /cache/abusers.list /etc/nginx/abusers.list - elif [ "$(ps aux | grep "abusers\.sh")" = "" ] ; then - echo "[*] Downloading abusers list (in background) ..." - /opt/scripts/abusers.sh > /dev/null 2>&1 & - fi -fi diff --git a/gen/main.py b/gen/main.py index f341336..6729740 100755 --- a/gen/main.py +++ b/gen/main.py @@ -52,12 +52,13 @@ if __name__ == "__main__" : #print(config) # Remove old config - for filename in os.listdir(args.output): - file_path = os.path.join(args.output, filename) - if os.path.isfile(file_path) or os.path.islink(file_path): - os.unlink(file_path) - elif os.path.isdir(file_path): - shutil.rmtree(file_path) + # TODO : remove unnecessary files after rendering +# for filename in os.listdir(args.output): +# file_path = os.path.join(args.output, filename) +# if os.path.isfile(file_path) or os.path.islink(file_path): +# os.unlink(file_path) +# elif os.path.isdir(file_path): +# shutil.rmtree(file_path) # Generate the files from templates and config templator = Templator(config, args.templates, args.output, args.target)