From c40fb331759ee0ac0b7e6e3cee96f9c1f90976cd Mon Sep 17 00:00:00 2001 From: bunkerity Date: Wed, 17 Mar 2021 12:16:56 +0100 Subject: [PATCH] road to swarm - automatic reload after jobs --- autoconf/AutoConf.py | 2 +- autoconf/Config.py | 2 +- autoconf/Dockerfile | 8 +++++++- autoconf/Dockerfile-amd64 | 8 +++++++- autoconf/Dockerfile-arm32v7 | 7 ++++++- autoconf/Dockerfile-arm64v8 | 7 ++++++- autoconf/Dockerfile-i386 | 8 +++++++- autoconf/ReloadServer.py | 29 +++++++++++++++++------------ scripts/abusers.sh | 2 +- scripts/certbot-renew-hook.sh | 2 +- scripts/exit-nodes.sh | 2 +- scripts/geoip.sh | 2 +- scripts/proxies.sh | 2 +- scripts/referrers.sh | 2 +- scripts/user-agents.sh | 2 +- 15 files changed, 59 insertions(+), 26 deletions(-) diff --git a/autoconf/AutoConf.py b/autoconf/AutoConf.py index 02d05fa..a69db1f 100644 --- a/autoconf/AutoConf.py +++ b/autoconf/AutoConf.py @@ -12,7 +12,7 @@ class AutoConf : self.__config = Config(self.__swarm, api) def reload(self) : - return self.__config.reload(self.instances) + return self.__config.reload(self.__instances) def pre_process(self, objs) : for instance in objs : diff --git a/autoconf/Config.py b/autoconf/Config.py index ba8fc06..592ea96 100644 --- a/autoconf/Config.py +++ b/autoconf/Config.py @@ -23,7 +23,7 @@ class Config : if self.globalconf(instances) : i = 0 started = False - while i < 5 : + while i < 10 : if self.__ping(instances) : started = True break diff --git a/autoconf/Dockerfile b/autoconf/Dockerfile index dd90486..6731fd5 100644 --- a/autoconf/Dockerfile +++ b/autoconf/Dockerfile @@ -17,7 +17,13 @@ RUN apk add py3-pip apache2-utils bash certbot curl logrotate openssl && \ chmod 770 /var/log/letsencrypt && \ mkdir /var/lib/letsencrypt && \ chown root:nginx /var/lib/letsencrypt && \ - chmod 770 /var/lib/letsencrypt + chmod 770 /var/lib/letsencrypt && \ + mkdir /cache && \ + chown root:nginx /cache && \ + chmod 770 /cache && \ + touch /var/log/jobs.log && \ + chown root:nginx /var/log/jobs.log && \ + chmod 770 /var/log/jobs.log COPY scripts/* /opt/scripts/ COPY confs/site/ /opt/confs/site diff --git a/autoconf/Dockerfile-amd64 b/autoconf/Dockerfile-amd64 index c729597..b515047 100644 --- a/autoconf/Dockerfile-amd64 +++ b/autoconf/Dockerfile-amd64 @@ -1,5 +1,9 @@ +FROM nginx:stable-alpine AS builder + FROM amd64/alpine +COPY --from=builder /etc/nginx/ /opt/confs/nginx + RUN apk add py3-pip apache2-utils bash certbot curl logrotate openssl && \ pip3 install docker requests && \ mkdir /opt/entrypoint && \ @@ -14,10 +18,12 @@ RUN apk add py3-pip apache2-utils bash certbot curl logrotate openssl && \ chown root:nginx /var/lib/letsencrypt && \ chmod 770 /var/lib/letsencrypt +COPY scripts/* /opt/scripts/ +COPY confs/global/ /opt/confs/global COPY confs/site/ /opt/confs/site COPY entrypoint/* /opt/entrypoint/ COPY autoconf/* /opt/entrypoint/ -RUN chmod +x /opt/entrypoint/*.py /opt/entrypoint/*.sh +RUN chmod +x /opt/entrypoint/*.py /opt/entrypoint/*.sh /opt/scripts/*.sh VOLUME /etc/nginx diff --git a/autoconf/Dockerfile-arm32v7 b/autoconf/Dockerfile-arm32v7 index 1d69c0c..b6ab404 100644 --- a/autoconf/Dockerfile-arm32v7 +++ b/autoconf/Dockerfile-arm32v7 @@ -3,9 +3,12 @@ FROM alpine AS builder ENV QEMU_URL https://github.com/balena-io/qemu/releases/download/v4.0.0%2Bbalena2/qemu-4.0.0.balena2-arm.tar.gz RUN apk add curl && curl -L ${QEMU_URL} | tar zxvf - -C . --strip-components 1 +FROM nginx:stable-alpine AS builder2 + FROM arm32v7/alpine COPY --from=builder qemu-arm-static /usr/bin +COPY --from=builder2 /etc/nginx/ /opt/confs/nginx RUN apk add py3-pip apache2-utils bash certbot curl logrotate openssl && \ pip3 install docker requests && \ @@ -21,10 +24,12 @@ RUN apk add py3-pip apache2-utils bash certbot curl logrotate openssl && \ chown root:nginx /var/lib/letsencrypt && \ chmod 770 /var/lib/letsencrypt +COPY scripts/* /opt/scripts/ +COPY confs/global/ /opt/confs/global COPY confs/site/ /opt/confs/site COPY entrypoint/* /opt/entrypoint/ COPY autoconf/* /opt/entrypoint/ -RUN chmod +x /opt/entrypoint/*.py /opt/entrypoint/*.sh +RUN chmod +x /opt/entrypoint/*.py /opt/entrypoint/*.sh /opt/scripts/*.sh VOLUME /etc/nginx diff --git a/autoconf/Dockerfile-arm64v8 b/autoconf/Dockerfile-arm64v8 index aa1a704..db5f77b 100644 --- a/autoconf/Dockerfile-arm64v8 +++ b/autoconf/Dockerfile-arm64v8 @@ -3,9 +3,12 @@ FROM alpine AS builder ENV QEMU_URL https://github.com/balena-io/qemu/releases/download/v4.0.0%2Bbalena2/qemu-4.0.0.balena2-aarch64.tar.gz RUN apk add curl && curl -L ${QEMU_URL} | tar zxvf - -C . --strip-components 1 +FROM nginx:stable-alpine AS builder2 + FROM arm64v8/alpine COPY --from=builder qemu-aarch64-static /usr/bin +COPY --from=builder2 /etc/nginx/ /opt/confs/nginx RUN apk add py3-pip apache2-utils bash certbot curl logrotate openssl && \ pip3 install docker requests && \ @@ -21,10 +24,12 @@ RUN apk add py3-pip apache2-utils bash certbot curl logrotate openssl && \ chown root:nginx /var/lib/letsencrypt && \ chmod 770 /var/lib/letsencrypt +COPY scripts/* /opt/scripts/ +COPY confs/global/ /opt/confs/global COPY confs/site/ /opt/confs/site COPY entrypoint/* /opt/entrypoint/ COPY autoconf/* /opt/entrypoint/ -RUN chmod +x /opt/entrypoint/*.py /opt/entrypoint/*.sh +RUN chmod +x /opt/entrypoint/*.py /opt/entrypoint/*.sh /opt/scripts/*.sh VOLUME /etc/nginx diff --git a/autoconf/Dockerfile-i386 b/autoconf/Dockerfile-i386 index 7a4df44..b0eaff6 100644 --- a/autoconf/Dockerfile-i386 +++ b/autoconf/Dockerfile-i386 @@ -1,5 +1,9 @@ +FROM nginx:stable-alpine AS builder + FROM i386/alpine +COPY --from=builder /etc/nginx/ /opt/confs/nginx + RUN apk add py3-pip apache2-utils bash certbot curl logrotate openssl && \ pip3 install docker requests && \ mkdir /opt/entrypoint && \ @@ -14,10 +18,12 @@ RUN apk add py3-pip apache2-utils bash certbot curl logrotate openssl && \ chown root:nginx /var/lib/letsencrypt && \ chmod 770 /var/lib/letsencrypt +COPY scripts/* /opt/scripts/ +COPY confs/global/ /opt/confs/global COPY confs/site/ /opt/confs/site COPY entrypoint/* /opt/entrypoint/ COPY autoconf/* /opt/entrypoint/ -RUN chmod +x /opt/entrypoint/*.py /opt/entrypoint/*.sh +RUN chmod +x /opt/entrypoint/*.py /opt/entrypoint/*.sh /opt/scripts/*.sh VOLUME /etc/nginx diff --git a/autoconf/ReloadServer.py b/autoconf/ReloadServer.py index 344791a..6af8262 100644 --- a/autoconf/ReloadServer.py +++ b/autoconf/ReloadServer.py @@ -1,20 +1,25 @@ -import socketserver, threading +import socketserver, threading, utils, os, stat -class ReloadServerHandler(socketserver.BaseRequestHandler): +class ReloadServerHandler(socketserver.StreamRequestHandler): def handle(self) : - data = self.request.recv(512) - if not data : - return - with self.server.lock : - ret = self.server.autoconf.reload() - if ret : - self.request.sendall("ok") - else : - self.request.sendall("ko") + try : + data = self.request.recv(512) + if not data : + return + with self.server.lock : + ret = self.server.autoconf.reload() + if ret : + self.request.sendall("ok".encode("utf-8")) + else : + self.request.sendall("ko".encode("utf-8")) + except Exception as e : + utils.log("Exception " + str(e)) def run_reload_server(autoconf, lock) : - server = socketserver.UnixStreamServer("/tmp/autoconf.pid", ReloadServerHandler) + server = socketserver.UnixStreamServer("/tmp/autoconf.sock", ReloadServerHandler) + os.chown("/tmp/autoconf.sock", 0, 101) + os.chmod("/tmp/autoconf.sock", 0o770) server.autoconf = autoconf server.lock = lock thread = threading.Thread(target=server.serve_forever) diff --git a/scripts/abusers.sh b/scripts/abusers.sh index 772d781..96f8c21 100755 --- a/scripts/abusers.sh +++ b/scripts/abusers.sh @@ -10,7 +10,7 @@ cp /etc/nginx/block-abusers.conf /cache if [ -f /tmp/nginx.pid ] ; then RELOAD="/usr/sbin/nginx -s reload > /dev/null 2>&1" # if we are in autoconf -elif [ -f /tmp/autoconf.sock ] ; then +elif [ -S /tmp/autoconf.sock ] ; then RELOAD="/opt/entrypoint/reload.py" fi diff --git a/scripts/certbot-renew-hook.sh b/scripts/certbot-renew-hook.sh index 8800e76..34f46e0 100644 --- a/scripts/certbot-renew-hook.sh +++ b/scripts/certbot-renew-hook.sh @@ -9,7 +9,7 @@ job_log "[CERTBOT] certificates have been renewed" if [ -f /tmp/nginx.pid ] ; then RELOAD="/usr/sbin/nginx -s reload > /dev/null 2>&1" # if we are in autoconf -elif [ -f /tmp/autoconf.sock ] ; then +elif [ -S /tmp/autoconf.sock ] ; then RELOAD="echo reload > /tmp/autoconf.sock" fi diff --git a/scripts/exit-nodes.sh b/scripts/exit-nodes.sh index b093686..d319b87 100644 --- a/scripts/exit-nodes.sh +++ b/scripts/exit-nodes.sh @@ -10,7 +10,7 @@ cp /etc/nginx/block-tor-exit-node.conf /cache if [ -f /tmp/nginx.pid ] ; then RELOAD="/usr/sbin/nginx -s reload > /dev/null 2>&1" # if we are in autoconf -elif [ -f /tmp/autoconf.sock ] ; then +elif [ -S /tmp/autoconf.sock ] ; then RELOAD="/opt/entrypoint/reload.py" fi diff --git a/scripts/geoip.sh b/scripts/geoip.sh index 8105f3a..2eec70f 100644 --- a/scripts/geoip.sh +++ b/scripts/geoip.sh @@ -7,7 +7,7 @@ if [ -f /tmp/nginx.pid ] ; then RELOAD="/usr/sbin/nginx -s reload > /dev/null 2>&1" # if we are in autoconf -elif [ -f /tmp/autoconf.sock ] ; then +elif [ -S /tmp/autoconf.sock ] ; then RELOAD="/opt/entrypoint/reload.py" fi diff --git a/scripts/proxies.sh b/scripts/proxies.sh index 3194fcf..f6d1659 100755 --- a/scripts/proxies.sh +++ b/scripts/proxies.sh @@ -10,7 +10,7 @@ cp /etc/nginx/block-proxies.conf /cache if [ -f /tmp/nginx.pid ] ; then RELOAD="/usr/sbin/nginx -s reload > /dev/null 2>&1" # if we are in autoconf -elif [ -f /tmp/autoconf.sock ] ; then +elif [ -S /tmp/autoconf.sock ] ; then RELOAD="/opt/entrypoint/reload.py" fi diff --git a/scripts/referrers.sh b/scripts/referrers.sh index a9aaf98..65be9a3 100755 --- a/scripts/referrers.sh +++ b/scripts/referrers.sh @@ -10,7 +10,7 @@ cp /etc/nginx/map-referrer.conf /cache if [ -f /tmp/nginx.pid ] ; then RELOAD="/usr/sbin/nginx -s reload > /dev/null 2>&1" # if we are in autoconf -elif [ -f /tmp/autoconf.sock ] ; then +elif [ -S /tmp/autoconf.sock ] ; then RELOAD="/opt/entrypoint/reload.py" fi diff --git a/scripts/user-agents.sh b/scripts/user-agents.sh index 67853b0..1e1ee87 100755 --- a/scripts/user-agents.sh +++ b/scripts/user-agents.sh @@ -10,7 +10,7 @@ cp /etc/nginx/map-user-agent.conf /cache if [ -f /tmp/nginx.pid ] ; then RELOAD="/usr/sbin/nginx -s reload > /dev/null 2>&1" # if we are in autoconf -elif [ -f /tmp/autoconf.sock ] ; then +elif [ -S /tmp/autoconf.sock ] ; then RELOAD="/opt/entrypoint/reload.py" fi