jobs - SelfSignedCert, runner and reloader

entrypoint/entrypoint.sh

@@ -61,8 +61,8 @@ if [ ! -f "/etc/nginx/global.env" ] ; then
 		# call the generator
 		/opt/bunkerized-nginx/gen/main.py --settings /opt/bunkerized-nginx/settings.json --templates /opt/bunkerized-nginx/confs --output /etc/nginx --variables /tmp/variables.env
 
-		# pre-jobs
+		# jobs
-		/opt/bunkerized-nginx/entrypoint/pre-jobs.sh
+		/opt/bunkerized-nginx/entrypoint/jobs.sh
 	fi
 else
 	echo "[*] Skipping configuration process"
@@ -100,9 +100,6 @@ if [ "$1" == "test" ] ; then
 	exit 1
 fi
 
-# post jobs
-/opt/bunkerized-nginx/entrypoint/post-jobs.sh
-
 # wait for nginx
 wait "$pid"
 while [ -f "/tmp/nginx.pid" ] ; do

entrypoint/jobs.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# load some functions
+. /opt/bunkerized-nginx/entrypoint/utils.sh
+
+# self signed certs for sites
+files=$(has_value GENERATE_SELF_SIGNED_SSL yes)
+if [ "$files" != "" ] ; then
+	for file in $files ; do
+		site=$(echo $file | cut -f 4 -d '/')
+		dest="/etc/nginx/"
+		if [ "$site" != "site.env" ] ; then
+			dest="${dest}/${site}/"
+		fi
+		SELF_SIGNED_SSL_EXPIRY="$(sed -nE 's/^SELF_SIGNED_SSL_EXPIRY=(.*)$/\1/p' $file)"
+		SELF_SIGNED_SSL_COUNTRY="$(sed -nE 's/^SELF_SIGNED_SSL_COUNTRY=(.*)$/\1/p' $file)"
+		SELF_SIGNED_SSL_STATE="$(sed -nE 's/^SELF_SIGNED_SSL_STATE=(.*)$/\1/p' $file)"
+		SELF_SIGNED_SSL_CITY="$(sed -nE 's/^SELF_SIGNED_SSL_CITY=(.*)$/\1/p' $file)"
+		SELF_SIGNED_SSL_ORG="$(sed -nE 's/^SELF_SIGNED_SSL_ORG=(.*)$/\1/p' $file)"
+		SELF_SIGNED_SSL_OU="$(sed -nE 's/^SELF_SIGNED_SSL_OU=(.*)$/\1/p' $file)"
+		SELF_SIGNED_SSL_CN="$(sed -nE 's/^SELF_SIGNED_SSL_CN=(.*)$/\1/p' $file)"
+		/opt/bunkerized-nginx/jobs/main.py --name self-signed-cert --dst_cert "${dest}self-cert.pem" --dst_key "${dest}self-key.pem" --days "$SELF_SIGNED_SSL_EXPIRY" --subj "/C=$SELF_SIGNED_SSL_COUNTRY/ST=$SELF_SIGNED_SSL_STATE/L=$SELF_SIGNED_SSL_CITY/O=$SELF_SIGNED_SSL_ORG/OU=$SELF_SIGNED_SSL_OU/CN=$SELF_SIGNED_SSL_CN"
+		if [ $? -eq 0 ] ; then
+			echo "[*] Generated self-signed certificate ${dest}self-cert.pem with key ${dest}self-key.pem"
+		else
+			echo "[!] Error while generating self-signed certificate"
+		fi
+	done
+fi
+
+# self signed cert for default server
+if [ "$(has_value AUTO_LETS_ENCRYPT yes)" != "" ] || [ "$(has_value GENERATE_SELF_SIGNED_SSL yes)" != "" ] || [ "$(has_value USE_CUSTOM_HTTPS yes)" != "" ] ; then
+	SELF_SIGNED_SSL_EXPIRY="999"
+	SELF_SIGNED_SSL_COUNTRY="US"
+	SELF_SIGNED_SSL_STATE="Utah"
+	SELF_SIGNED_SSL_CITY="Lehi"
+	SELF_SIGNED_SSL_ORG="Your Company, Inc."
+	SELF_SIGNED_SSL_OU="IT"
+	SELF_SIGNED_SSL_CN="www.yourdomain.com"
+	/opt/bunkerized-nginx/jobs/main.py --name self-signed-cert --dst_cert "/etc/nginx/default-cert.pem" --dst_key "/etc/nginx/default-key.pem" --days "$SELF_SIGNED_SSL_EXPIRY" --subj "/C=$SELF_SIGNED_SSL_COUNTRY/ST=$SELF_SIGNED_SSL_STATE/L=$SELF_SIGNED_SSL_CITY/O=$SELF_SIGNED_SSL_ORG/OU=$SELF_SIGNED_SSL_OU/CN=$SELF_SIGNED_SSL_CN"
+	if [ $? -eq 0 ] ; then
+		echo "[*] Generated self-signed certificate for default server"
+	else
+		echo "[!] Error while generating self-signed certificate for default server"
+	fi
+fi
+
+# certbot
+files=$(has_value AUTO_LETS_ENCRYPT yes)
+if [ "$files" != "" ] ; then
+	for file in $files ; do
+		if [ "$(echo "$file" | grep 'site.env$')" = "" ] ; then
+			continue
+		fi
+		SERVER_NAME="$(sed -nE 's/^SERVER_NAME=(.*)$/\1/p' $file)"
+		FIRST_SERVER="$(echo $SERVER_NAME | cut -d ' ' -f 1)"
+		EMAIL_LETS_ENCRYPT="$(sed -nE 's/^EMAIL_LETS_ENCRYPT=(.*)$/\1/p' $file)"
+		if [ "$EMAIL_LETS_ENCRYPT" = "" ] ; then
+			EMAIL_LETS_ENCRYPT="contact@${FIRST_SERVER}"
+		fi
+		certbot_output=$(/opt/bunkerized-nginx/scripts/certbot-new.sh "$(echo -n $SERVER_NAME | sed 's/ /,/g')" "$EMAIL_LETS_ENCRYPT" 2>&1)
+		if [ $? -eq 0 ] ; then
+			echo "[*] Certbot new successfully executed for domain(s) $(echo -n $SERVER_NAME | sed 's/ /,/g')"
+		else
+			echo "[*] Error while executing certbot new : $certbot_output"
+		fi
+	done
+fi
+
+
+# GeoIP
+if [ "$(has_value BLACKLIST_COUNTRY ".\+")" != "" ] || [ "$(has_value WHITELIST_COUNTRY ".\+")" != "" ] ; then
+	if [ -f "/opt/bunkerized-nginx/cache/geoip.mmdb" ] ; then
+		echo "[*] Copying cached geoip.mmdb ..."
+		cp /opt/bunkerized-nginx/cache/geoip.mmdb /etc/nginx/geoip.mmdb
+	elif [ "$(ps aux | grep "geoip\.sh")" = "" ] ; then
+		echo "[*] Downloading GeoIP database ..."
+		/opt/bunkerized-nginx/scripts/geoip.sh > /dev/null 2>&1
+	fi
+fi

helpers/install.sh

@@ -695,6 +695,10 @@ do_and_check_cmd cp -r /tmp/bunkerized-nginx/confs /opt/bunkerized-nginx
 echo "[*] Copy scripts"
 do_and_check_cmd cp -r /tmp/bunkerized-nginx/scripts /opt/bunkerized-nginx
 
+# Copy scripts
+echo "[*] Copy jobs"
+do_and_check_cmd cp -r /tmp/bunkerized-nginx/jobs /opt/bunkerized-nginx
+
 # Copy LUA
 echo "[*] Copy LUA"
 do_and_check_cmd cp -r /tmp/bunkerized-nginx/lua /opt/bunkerized-nginx
@@ -796,6 +800,7 @@ do_and_check_cmd chmod 770 /opt/bunkerized-nginx/acme-challenge
 do_and_check_cmd chmod 750 /opt/bunkerized-nginx/scripts/*
 do_and_check_cmd chmod 750 /opt/bunkerized-nginx/entrypoint/*
 do_and_check_cmd chmod 750 /opt/bunkerized-nginx/gen/main.py
+do_and_check_cmd chmod 750 /opt/bunkerized-nginx/jobs/main.py
 # Set permissions for /usr/local/bin/bunkerized-nginx
 do_and_check_cmd chown root:root /usr/local/bin/bunkerized-nginx
 do_and_check_cmd chmod 750 /usr/local/bin/bunkerized-nginx
@@ -865,29 +870,37 @@ do_and_check_cmd cp /tmp/bunkerized-nginx/misc/cron "$CRON_PATH"
 do_and_check_cmd chown root:nginx "$CRON_PATH"
 do_and_check_cmd chmod 740 "$CRON_PATH"
 
+# Don't install external things on Docker image
+if [ "$OS" = "alpine" ] ; then
+	cd "$old_dir"
+	cleanup
+	echo "[*] bunkerized-nginx successfully installed !"
+	exit 0
+fi
+
 # Download abusers list
 echo "[*] Download abusers list"
-do_and_check_cmd /opt/bunkerized-nginx/scripts/abusers.sh
+do_and_check_cmd /opt/bunkerized-nginx/jobs/main.py --name abusers
 
 # Download TOR exit nodes list
 echo "[*] Download TOR exit nodes list"
-do_and_check_cmd /opt/bunkerized-nginx/scripts/exit-nodes.sh
+do_and_check_cmd /opt/bunkerized-nginx/jobs/main.py --name exit-nodes
 
 # Download proxies list
 echo "[*] Download proxies list"
-do_and_check_cmd /opt/bunkerized-nginx/scripts/proxies.sh
+do_and_check_cmd /opt/bunkerized-nginx/jobs/main.py --name proxies
 
 # Download referrers list
 echo "[*] Download referrers list"
-do_and_check_cmd /opt/bunkerized-nginx/scripts/referrers.sh
+do_and_check_cmd /opt/bunkerized-nginx/jobs/main.py --name referrers
 
 # Download user agents list
 echo "[*] Download user agents list"
-do_and_check_cmd /opt/bunkerized-nginx/scripts/user-agents.sh
+do_and_check_cmd /opt/bunkerized-nginx/jobs/main.py --name user-agents
 
 # Download geoip database
 echo "[*] Download geoip DB"
-do_and_check_cmd /opt/bunkerized-nginx/scripts/geoip.sh
+do_and_check_cmd /opt/bunkerized-nginx/jobs/main.py --name geoip
 
 # We're done
 cd "$old_dir"

jobs/CertbotNew.py

@@ -6,4 +6,4 @@ class CertbotRenew(Job) :
 		name = "certbot-new"
 		data = ["certbot", "certonly", "--webroot", "-w", "/opt/bunkerized-nginx/acme-challenge", "-n", "-d", domain, "--email", email, "--agree-tos"]
 		type = "exec"
-		super().__init__(name, data, filename, redis_host=redis_host, type=type, copy_cache=copy_cache)
+		super().__init__(name, data, filename=None, redis_host=redis_host, type=type, copy_cache=copy_cache)

jobs/CertbotRenew.py

@@ -6,4 +6,4 @@ class CertbotRenew(Job) :
 		name = "certbot-renew"
 		data = ["certbot", "renew", "--deploy-hook", "/opt/bunkerized-nginx/jobs/reload.py"]
 		type = "exec"
-		super().__init__(name, data, filename, redis_host=redis_host, type=type, copy_cache=copy_cache)
+		super().__init__(name, data, filename=None, redis_host=redis_host, type=type, copy_cache=copy_cache)

jobs/ExitNodes.py

@@ -2,7 +2,7 @@ from Job import Job
 
 class ExitNodes(Job) :
 
-	def __init__(self, redis_host=None, copy_cache=copy_cache) :
+	def __init__(self, redis_host=None, copy_cache=False) :
 		name = "exit-nodes"
 		data = ["https://iplists.firehol.org/files/tor_exits.ipset"]
 		filename = "tor-exit-nodes.list"

jobs/Job.py

@@ -2,7 +2,7 @@ import abc, requests, redis, os, datetime, traceback
 
 class Job(abc.ABC) :
 
-	def __init__(self, name, data, filename, redis_host=None, type="line", regex=r"^.+$", copy_cache=False) :
+	def __init__(self, name, data, filename=None, redis_host=None, type="line", regex=r"^.+$", copy_cache=False) :
 		self.__name = name
 		self.__data = data
 		self.__filename = filename

jobs/SelfSignedCert.py

@@ -0,0 +1,9 @@
+from Job import Job
+
+class SelfSignedCert(Job) :
+
+	def __init__(self, redis_host=None, copy_cache=False, dst_cert="/etc/nginx/default-cert.pem", dst_key="/etc/nginx/default-key.pem", expiry="999", subj="CN=www.example.com") :
+		name = "self-signed-cert"
+		data = ["openssl", "req", "-nodes", "-x509", "-newkey", "rsa:4096", "-keyout", dst_key, "-out", dst_cert, "-days", expiry, "-subj", subj]
+		type = "exec"
+		super().__init__(name, data, filename=None, redis_host=redis_host, type=type, copy_cache=copy_cache)

jobs/main.py

@@ -0,0 +1,53 @@
+import argparse, sys
+
+sys.path.append("/opt/bunkerized-nginx/jobs")
+
+import Abusers, CertbotNew, CertbotRenew, ExitNodes, GeoIP, Proxies, Referrers, SelfSignedCert, UserAgents
+
+JOBS = {
+	"abusers": Abusers.Abusers,
+	"certbot-new": CertbotNew.CertbotNew,
+	"certbot-renew": CertbotRenew.CertbotRenew,
+	"exit-nodes": ExitNodes.ExitNodes,
+	"geoip": GeoIP.GeoIP,
+	"proxies": Proxies.Proxies,
+	"referrers": Referrers.Referrers,
+	"self-signed-cert": SelfSignedCert.SelfSignedCert,
+	"user-agents": UserAgents.UserAgents
+}
+
+if __name__ == "__main__" :
+
+	# Parse arguments
+	parser = argparse.ArgumentParser(description="job runner for bunkerized-nginx")
+	parser.add_argument("--name", default="", type=str, help="job to run (e.g : abusers or certbot-new or certbot-renew ...)")
+	parser.add_argument("--redis", default=None, type=str, help="hostname of the redis server if any")
+	parser.add_argument("--cache", action="store_true", help="copy data from cache if available")
+	parser.add_argument("--domain", default="", type=str, help="domain(s) for certbot-new job (e.g. : www.example.com or app1.example.com,app2.example.com)")
+	parser.add_argument("--email", default="", type=str, help="email for certbot-new job (e.g. : contact@example.com)")
+	parser.add_argument("--dst_cert", default="", type=str, help="certificate path for self-signed-cert job (e.g. : /etc/nginx/default-cert.pem)")
+	parser.add_argument("--dst_key", default="", type=str, help="key path for self-signed-cert job (e.g. : /etc/nginx/default-key.pem)")
+	parser.add_argument("--expiry", default="", type=str, help="number of validity days for self-signed-cert job (e.g. : 365)")
+	parser.add_argument("--subj", default="", type=str, help="certificate subject for self-signed-cert job (e.g. : OU=X/CN=Y...)")
+	args = parser.parse_args()
+
+	# Check job name
+	if not args.name in JOBS :
+		print("[!] unknown job " + args.job)
+		sys.exit(1)
+
+	# Run job
+	ret = 0
+	if job == "certbot-new" :
+		instance = JOBS[job](redis_host=args.redis, copy_cache=args.cache, domain=args.domain, email=args.email)
+	elif job == "self-signed-cert" :
+		instance = JOBS[job](redis_host=args.redis, copy_cache=args.cache, dst_cert=args.dst_cert, dst_key=args.dst_key, expiry=args.expiry, subj=args.subj)
+	else :
+		instance = JOBS[job](redis_host=args.redis, copy_cache=args.cache)
+	if not instance.run() :
+		print("[!] error while running job " + job)
+		sys.exit(1)
+	print("[*] job " + job + " successfully executed")
+	sys.exit(0)
+
+	# TODO : reload

jobs/reload.py

@@ -0,0 +1,27 @@
+import docker, subprocess, os, stat, sys
+
+if __name__ == "__main__" :
+
+	# Linux or single Docker use case
+	if os.path.isfile("/usr/sbin/nginx") :
+		proc = subprocess.run(["/usr/sbin/nginx", "-s", "reload"], capture_output=True)
+		if proc.returncode != 0 :
+			print("[!] can't reload nginx (status code = " + str(proc.returncode) + ")"
+			if len(proc.stdout.decode("ascii")) > 1 :
+				print(proc.stdout.decode("ascii"))
+			if len(proc.stderr.decode("ascii")) > 1 :
+				print(proc.stderr.decode("ascii"))
+			sys.exit(1)
+
+	# Autoconf case (Docker, Swarm and Ingress)
+	mode = os.stat("/tmp/autoconf.sock")
+	elif stat.S_ISSOCK(mode) :
+		client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+		client.connect("/tmp/autoconf.sock")
+		client.send("reload".encode("utf-8"))
+		data = client.recv(512)
+		client.close()
+        if not data or data.decode("utf-8") != "ok" :
+                sys.exit(2)
+
+	sys.exit(0)