From 366e39f591b1ef5b5d9e845d22e0f174c12b803b Mon Sep 17 00:00:00 2001 From: bunkerity Date: Tue, 20 Jul 2021 22:52:01 +0200 Subject: [PATCH] jobs - SelfSignedCert, runner and reloader --- entrypoint/entrypoint.sh | 7 +--- entrypoint/jobs.sh | 80 ++++++++++++++++++++++++++++++++++++++++ helpers/install.sh | 25 ++++++++++--- jobs/CertbotNew.py | 2 +- jobs/CertbotRenew.py | 2 +- jobs/ExitNodes.py | 2 +- jobs/Job.py | 2 +- jobs/SelfSignedCert.py | 9 +++++ jobs/main.py | 53 ++++++++++++++++++++++++++ jobs/reload.py | 27 ++++++++++++++ 10 files changed, 194 insertions(+), 15 deletions(-) create mode 100644 entrypoint/jobs.sh create mode 100644 jobs/SelfSignedCert.py create mode 100644 jobs/main.py create mode 100644 jobs/reload.py diff --git a/entrypoint/entrypoint.sh b/entrypoint/entrypoint.sh index 7f1948d..ad0ea75 100644 --- a/entrypoint/entrypoint.sh +++ b/entrypoint/entrypoint.sh @@ -61,8 +61,8 @@ if [ ! -f "/etc/nginx/global.env" ] ; then # call the generator /opt/bunkerized-nginx/gen/main.py --settings /opt/bunkerized-nginx/settings.json --templates /opt/bunkerized-nginx/confs --output /etc/nginx --variables /tmp/variables.env - # pre-jobs - /opt/bunkerized-nginx/entrypoint/pre-jobs.sh + # jobs + /opt/bunkerized-nginx/entrypoint/jobs.sh fi else echo "[*] Skipping configuration process" @@ -100,9 +100,6 @@ if [ "$1" == "test" ] ; then exit 1 fi -# post jobs -/opt/bunkerized-nginx/entrypoint/post-jobs.sh - # wait for nginx wait "$pid" while [ -f "/tmp/nginx.pid" ] ; do diff --git a/entrypoint/jobs.sh b/entrypoint/jobs.sh new file mode 100644 index 0000000..f255bf6 --- /dev/null +++ b/entrypoint/jobs.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +# load some functions +. /opt/bunkerized-nginx/entrypoint/utils.sh + +# self signed certs for sites +files=$(has_value GENERATE_SELF_SIGNED_SSL yes) +if [ "$files" != "" ] ; then + for file in $files ; do + site=$(echo $file | cut -f 4 -d '/') + dest="/etc/nginx/" + if [ "$site" != "site.env" ] ; then + dest="${dest}/${site}/" + fi + SELF_SIGNED_SSL_EXPIRY="$(sed -nE 's/^SELF_SIGNED_SSL_EXPIRY=(.*)$/\1/p' $file)" + SELF_SIGNED_SSL_COUNTRY="$(sed -nE 's/^SELF_SIGNED_SSL_COUNTRY=(.*)$/\1/p' $file)" + SELF_SIGNED_SSL_STATE="$(sed -nE 's/^SELF_SIGNED_SSL_STATE=(.*)$/\1/p' $file)" + SELF_SIGNED_SSL_CITY="$(sed -nE 's/^SELF_SIGNED_SSL_CITY=(.*)$/\1/p' $file)" + SELF_SIGNED_SSL_ORG="$(sed -nE 's/^SELF_SIGNED_SSL_ORG=(.*)$/\1/p' $file)" + SELF_SIGNED_SSL_OU="$(sed -nE 's/^SELF_SIGNED_SSL_OU=(.*)$/\1/p' $file)" + SELF_SIGNED_SSL_CN="$(sed -nE 's/^SELF_SIGNED_SSL_CN=(.*)$/\1/p' $file)" + /opt/bunkerized-nginx/jobs/main.py --name self-signed-cert --dst_cert "${dest}self-cert.pem" --dst_key "${dest}self-key.pem" --days "$SELF_SIGNED_SSL_EXPIRY" --subj "/C=$SELF_SIGNED_SSL_COUNTRY/ST=$SELF_SIGNED_SSL_STATE/L=$SELF_SIGNED_SSL_CITY/O=$SELF_SIGNED_SSL_ORG/OU=$SELF_SIGNED_SSL_OU/CN=$SELF_SIGNED_SSL_CN" + if [ $? -eq 0 ] ; then + echo "[*] Generated self-signed certificate ${dest}self-cert.pem with key ${dest}self-key.pem" + else + echo "[!] Error while generating self-signed certificate" + fi + done +fi + +# self signed cert for default server +if [ "$(has_value AUTO_LETS_ENCRYPT yes)" != "" ] || [ "$(has_value GENERATE_SELF_SIGNED_SSL yes)" != "" ] || [ "$(has_value USE_CUSTOM_HTTPS yes)" != "" ] ; then + SELF_SIGNED_SSL_EXPIRY="999" + SELF_SIGNED_SSL_COUNTRY="US" + SELF_SIGNED_SSL_STATE="Utah" + SELF_SIGNED_SSL_CITY="Lehi" + SELF_SIGNED_SSL_ORG="Your Company, Inc." + SELF_SIGNED_SSL_OU="IT" + SELF_SIGNED_SSL_CN="www.yourdomain.com" + /opt/bunkerized-nginx/jobs/main.py --name self-signed-cert --dst_cert "/etc/nginx/default-cert.pem" --dst_key "/etc/nginx/default-key.pem" --days "$SELF_SIGNED_SSL_EXPIRY" --subj "/C=$SELF_SIGNED_SSL_COUNTRY/ST=$SELF_SIGNED_SSL_STATE/L=$SELF_SIGNED_SSL_CITY/O=$SELF_SIGNED_SSL_ORG/OU=$SELF_SIGNED_SSL_OU/CN=$SELF_SIGNED_SSL_CN" + if [ $? -eq 0 ] ; then + echo "[*] Generated self-signed certificate for default server" + else + echo "[!] Error while generating self-signed certificate for default server" + fi +fi + +# certbot +files=$(has_value AUTO_LETS_ENCRYPT yes) +if [ "$files" != "" ] ; then + for file in $files ; do + if [ "$(echo "$file" | grep 'site.env$')" = "" ] ; then + continue + fi + SERVER_NAME="$(sed -nE 's/^SERVER_NAME=(.*)$/\1/p' $file)" + FIRST_SERVER="$(echo $SERVER_NAME | cut -d ' ' -f 1)" + EMAIL_LETS_ENCRYPT="$(sed -nE 's/^EMAIL_LETS_ENCRYPT=(.*)$/\1/p' $file)" + if [ "$EMAIL_LETS_ENCRYPT" = "" ] ; then + EMAIL_LETS_ENCRYPT="contact@${FIRST_SERVER}" + fi + certbot_output=$(/opt/bunkerized-nginx/scripts/certbot-new.sh "$(echo -n $SERVER_NAME | sed 's/ /,/g')" "$EMAIL_LETS_ENCRYPT" 2>&1) + if [ $? -eq 0 ] ; then + echo "[*] Certbot new successfully executed for domain(s) $(echo -n $SERVER_NAME | sed 's/ /,/g')" + else + echo "[*] Error while executing certbot new : $certbot_output" + fi + done +fi + + +# GeoIP +if [ "$(has_value BLACKLIST_COUNTRY ".\+")" != "" ] || [ "$(has_value WHITELIST_COUNTRY ".\+")" != "" ] ; then + if [ -f "/opt/bunkerized-nginx/cache/geoip.mmdb" ] ; then + echo "[*] Copying cached geoip.mmdb ..." + cp /opt/bunkerized-nginx/cache/geoip.mmdb /etc/nginx/geoip.mmdb + elif [ "$(ps aux | grep "geoip\.sh")" = "" ] ; then + echo "[*] Downloading GeoIP database ..." + /opt/bunkerized-nginx/scripts/geoip.sh > /dev/null 2>&1 + fi +fi diff --git a/helpers/install.sh b/helpers/install.sh index 11bdd9f..00dd34e 100755 --- a/helpers/install.sh +++ b/helpers/install.sh @@ -695,6 +695,10 @@ do_and_check_cmd cp -r /tmp/bunkerized-nginx/confs /opt/bunkerized-nginx echo "[*] Copy scripts" do_and_check_cmd cp -r /tmp/bunkerized-nginx/scripts /opt/bunkerized-nginx +# Copy scripts +echo "[*] Copy jobs" +do_and_check_cmd cp -r /tmp/bunkerized-nginx/jobs /opt/bunkerized-nginx + # Copy LUA echo "[*] Copy LUA" do_and_check_cmd cp -r /tmp/bunkerized-nginx/lua /opt/bunkerized-nginx @@ -796,6 +800,7 @@ do_and_check_cmd chmod 770 /opt/bunkerized-nginx/acme-challenge do_and_check_cmd chmod 750 /opt/bunkerized-nginx/scripts/* do_and_check_cmd chmod 750 /opt/bunkerized-nginx/entrypoint/* do_and_check_cmd chmod 750 /opt/bunkerized-nginx/gen/main.py +do_and_check_cmd chmod 750 /opt/bunkerized-nginx/jobs/main.py # Set permissions for /usr/local/bin/bunkerized-nginx do_and_check_cmd chown root:root /usr/local/bin/bunkerized-nginx do_and_check_cmd chmod 750 /usr/local/bin/bunkerized-nginx @@ -865,29 +870,37 @@ do_and_check_cmd cp /tmp/bunkerized-nginx/misc/cron "$CRON_PATH" do_and_check_cmd chown root:nginx "$CRON_PATH" do_and_check_cmd chmod 740 "$CRON_PATH" +# Don't install external things on Docker image +if [ "$OS" = "alpine" ] ; then + cd "$old_dir" + cleanup + echo "[*] bunkerized-nginx successfully installed !" + exit 0 +fi + # Download abusers list echo "[*] Download abusers list" -do_and_check_cmd /opt/bunkerized-nginx/scripts/abusers.sh +do_and_check_cmd /opt/bunkerized-nginx/jobs/main.py --name abusers # Download TOR exit nodes list echo "[*] Download TOR exit nodes list" -do_and_check_cmd /opt/bunkerized-nginx/scripts/exit-nodes.sh +do_and_check_cmd /opt/bunkerized-nginx/jobs/main.py --name exit-nodes # Download proxies list echo "[*] Download proxies list" -do_and_check_cmd /opt/bunkerized-nginx/scripts/proxies.sh +do_and_check_cmd /opt/bunkerized-nginx/jobs/main.py --name proxies # Download referrers list echo "[*] Download referrers list" -do_and_check_cmd /opt/bunkerized-nginx/scripts/referrers.sh +do_and_check_cmd /opt/bunkerized-nginx/jobs/main.py --name referrers # Download user agents list echo "[*] Download user agents list" -do_and_check_cmd /opt/bunkerized-nginx/scripts/user-agents.sh +do_and_check_cmd /opt/bunkerized-nginx/jobs/main.py --name user-agents # Download geoip database echo "[*] Download geoip DB" -do_and_check_cmd /opt/bunkerized-nginx/scripts/geoip.sh +do_and_check_cmd /opt/bunkerized-nginx/jobs/main.py --name geoip # We're done cd "$old_dir" diff --git a/jobs/CertbotNew.py b/jobs/CertbotNew.py index 7a5e445..55e22ad 100644 --- a/jobs/CertbotNew.py +++ b/jobs/CertbotNew.py @@ -6,4 +6,4 @@ class CertbotRenew(Job) : name = "certbot-new" data = ["certbot", "certonly", "--webroot", "-w", "/opt/bunkerized-nginx/acme-challenge", "-n", "-d", domain, "--email", email, "--agree-tos"] type = "exec" - super().__init__(name, data, filename, redis_host=redis_host, type=type, copy_cache=copy_cache) + super().__init__(name, data, filename=None, redis_host=redis_host, type=type, copy_cache=copy_cache) diff --git a/jobs/CertbotRenew.py b/jobs/CertbotRenew.py index 520eb27..6a7eb14 100644 --- a/jobs/CertbotRenew.py +++ b/jobs/CertbotRenew.py @@ -6,4 +6,4 @@ class CertbotRenew(Job) : name = "certbot-renew" data = ["certbot", "renew", "--deploy-hook", "/opt/bunkerized-nginx/jobs/reload.py"] type = "exec" - super().__init__(name, data, filename, redis_host=redis_host, type=type, copy_cache=copy_cache) + super().__init__(name, data, filename=None, redis_host=redis_host, type=type, copy_cache=copy_cache) diff --git a/jobs/ExitNodes.py b/jobs/ExitNodes.py index 576da6e..d7d64ed 100644 --- a/jobs/ExitNodes.py +++ b/jobs/ExitNodes.py @@ -2,7 +2,7 @@ from Job import Job class ExitNodes(Job) : - def __init__(self, redis_host=None, copy_cache=copy_cache) : + def __init__(self, redis_host=None, copy_cache=False) : name = "exit-nodes" data = ["https://iplists.firehol.org/files/tor_exits.ipset"] filename = "tor-exit-nodes.list" diff --git a/jobs/Job.py b/jobs/Job.py index c664c9b..f60c330 100644 --- a/jobs/Job.py +++ b/jobs/Job.py @@ -2,7 +2,7 @@ import abc, requests, redis, os, datetime, traceback class Job(abc.ABC) : - def __init__(self, name, data, filename, redis_host=None, type="line", regex=r"^.+$", copy_cache=False) : + def __init__(self, name, data, filename=None, redis_host=None, type="line", regex=r"^.+$", copy_cache=False) : self.__name = name self.__data = data self.__filename = filename diff --git a/jobs/SelfSignedCert.py b/jobs/SelfSignedCert.py new file mode 100644 index 0000000..35c8102 --- /dev/null +++ b/jobs/SelfSignedCert.py @@ -0,0 +1,9 @@ +from Job import Job + +class SelfSignedCert(Job) : + + def __init__(self, redis_host=None, copy_cache=False, dst_cert="/etc/nginx/default-cert.pem", dst_key="/etc/nginx/default-key.pem", expiry="999", subj="CN=www.example.com") : + name = "self-signed-cert" + data = ["openssl", "req", "-nodes", "-x509", "-newkey", "rsa:4096", "-keyout", dst_key, "-out", dst_cert, "-days", expiry, "-subj", subj] + type = "exec" + super().__init__(name, data, filename=None, redis_host=redis_host, type=type, copy_cache=copy_cache) diff --git a/jobs/main.py b/jobs/main.py new file mode 100644 index 0000000..00f33c6 --- /dev/null +++ b/jobs/main.py @@ -0,0 +1,53 @@ +import argparse, sys + +sys.path.append("/opt/bunkerized-nginx/jobs") + +import Abusers, CertbotNew, CertbotRenew, ExitNodes, GeoIP, Proxies, Referrers, SelfSignedCert, UserAgents + +JOBS = { + "abusers": Abusers.Abusers, + "certbot-new": CertbotNew.CertbotNew, + "certbot-renew": CertbotRenew.CertbotRenew, + "exit-nodes": ExitNodes.ExitNodes, + "geoip": GeoIP.GeoIP, + "proxies": Proxies.Proxies, + "referrers": Referrers.Referrers, + "self-signed-cert": SelfSignedCert.SelfSignedCert, + "user-agents": UserAgents.UserAgents +} + +if __name__ == "__main__" : + + # Parse arguments + parser = argparse.ArgumentParser(description="job runner for bunkerized-nginx") + parser.add_argument("--name", default="", type=str, help="job to run (e.g : abusers or certbot-new or certbot-renew ...)") + parser.add_argument("--redis", default=None, type=str, help="hostname of the redis server if any") + parser.add_argument("--cache", action="store_true", help="copy data from cache if available") + parser.add_argument("--domain", default="", type=str, help="domain(s) for certbot-new job (e.g. : www.example.com or app1.example.com,app2.example.com)") + parser.add_argument("--email", default="", type=str, help="email for certbot-new job (e.g. : contact@example.com)") + parser.add_argument("--dst_cert", default="", type=str, help="certificate path for self-signed-cert job (e.g. : /etc/nginx/default-cert.pem)") + parser.add_argument("--dst_key", default="", type=str, help="key path for self-signed-cert job (e.g. : /etc/nginx/default-key.pem)") + parser.add_argument("--expiry", default="", type=str, help="number of validity days for self-signed-cert job (e.g. : 365)") + parser.add_argument("--subj", default="", type=str, help="certificate subject for self-signed-cert job (e.g. : OU=X/CN=Y...)") + args = parser.parse_args() + + # Check job name + if not args.name in JOBS : + print("[!] unknown job " + args.job) + sys.exit(1) + + # Run job + ret = 0 + if job == "certbot-new" : + instance = JOBS[job](redis_host=args.redis, copy_cache=args.cache, domain=args.domain, email=args.email) + elif job == "self-signed-cert" : + instance = JOBS[job](redis_host=args.redis, copy_cache=args.cache, dst_cert=args.dst_cert, dst_key=args.dst_key, expiry=args.expiry, subj=args.subj) + else : + instance = JOBS[job](redis_host=args.redis, copy_cache=args.cache) + if not instance.run() : + print("[!] error while running job " + job) + sys.exit(1) + print("[*] job " + job + " successfully executed") + sys.exit(0) + + # TODO : reload diff --git a/jobs/reload.py b/jobs/reload.py new file mode 100644 index 0000000..3ba4fca --- /dev/null +++ b/jobs/reload.py @@ -0,0 +1,27 @@ +import docker, subprocess, os, stat, sys + +if __name__ == "__main__" : + + # Linux or single Docker use case + if os.path.isfile("/usr/sbin/nginx") : + proc = subprocess.run(["/usr/sbin/nginx", "-s", "reload"], capture_output=True) + if proc.returncode != 0 : + print("[!] can't reload nginx (status code = " + str(proc.returncode) + ")" + if len(proc.stdout.decode("ascii")) > 1 : + print(proc.stdout.decode("ascii")) + if len(proc.stderr.decode("ascii")) > 1 : + print(proc.stderr.decode("ascii")) + sys.exit(1) + + # Autoconf case (Docker, Swarm and Ingress) + mode = os.stat("/tmp/autoconf.sock") + elif stat.S_ISSOCK(mode) : + client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + client.connect("/tmp/autoconf.sock") + client.send("reload".encode("utf-8")) + data = client.recv(512) + client.close() + if not data or data.decode("utf-8") != "ok" : + sys.exit(2) + + sys.exit(0)