autoconf - various bug fixes with DockerController

This commit is contained in:
bunkerity 2021-07-29 15:43:51 +02:00
parent 7180378d0c
commit 1a32e7c02c
No known key found for this signature in database
GPG Key ID: 3D80806F12602A7C
16 changed files with 134 additions and 82 deletions

View File

@ -12,8 +12,10 @@ RUN chmod +x /tmp/docker.sh && \
/tmp/docker.sh && \ /tmp/docker.sh && \
rm -f /tmp/docker.sh rm -f /tmp/docker.sh
# Fix CVE-2021-22901, CVE-2021-22898, CVE-2021-22897, CVE-2021-33560 and CVE-2021-36159 # Fix CVE-2021-22901, CVE-2021-22898, CVE-2021-22897 and CVE-2021-33560
RUN apk add "curl>=7.77.0-r0" "libgcrypt>=1.8.8-r0" "apk-tools>=2.12.6-r0" RUN apk add "curl>=7.77.0-r0" "libgcrypt>=1.8.8-r0"
# Fix CVE-2021-36159
#RUN apk add "apk-tools>=2.12.6-r0"
VOLUME /www /http-confs /server-confs /modsec-confs /modsec-crs-confs /cache /pre-server-confs /acme-challenge /plugins VOLUME /www /http-confs /server-confs /modsec-confs /modsec-crs-confs /cache /pre-server-confs /acme-challenge /plugins

View File

@ -1,19 +1,18 @@
#!/usr/bin/python3 #!/usr/bin/python3
import utils
import subprocess, shutil, os, traceback, requests, time import subprocess, shutil, os, traceback, requests, time
from Controller import ControllerType import Controller
from logger import log from logger import log
class Config : class Config :
def __init__(type, api_uri) : def __init__(self, type, api_uri) :
self.__type = type self.__type = type
self.__api_uri = api_uri self.__api_uri = api_uri
def gen(env) : def gen(self, env) :
try : try :
# Write environment variables to a file # Write environment variables to a file
with open("/tmp/variables.env", "w") as f : with open("/tmp/variables.env", "w") as f :
@ -27,32 +26,32 @@ class Config :
stdout = proc.stdout.decode("ascii") stdout = proc.stdout.decode("ascii")
stderr = proc.stderr.decode("ascii") stderr = proc.stderr.decode("ascii")
if len(stdout) > 1 : if len(stdout) > 1 :
log("CONFIG", "INFO", "generator output : " + stdout) log("config", "INFO", "generator output : " + stdout)
if stderr != "" : if stderr != "" :
log("CONFIG", "ERROR", "generator error : " + stderr) log("config", "ERROR", "generator error : " + stderr)
# We're done # We're done
if proc.returncode == 0 : if proc.returncode == 0 :
if self.__type == ControllerType.SWARM or self.__type == ControllerType.KUBERNETES : if self.__type == Controller.Type.SWARM or self.__type == Controller.Type.KUBERNETES :
return self.__jobs() return self.__jobs()
return True return True
log("CONFIG", "ERROR", "error while generating config (return code = " + str(proc.returncode) + ")") log("config", "ERROR", "error while generating config (return code = " + str(proc.returncode) + ")")
except Exception as e : except Exception as e :
log("CONFIG", "ERROR", "exception while generating site config : " + traceback.format_exc()) log("config", "ERROR", "exception while generating site config : " + traceback.format_exc())
return False return False
def reload(self, instances) : def reload(self, instances) :
ret = True ret = True
if self.__type == ControllerType.DOCKER : if self.__type == Controller.Type.DOCKER :
for instance in instances : for instance in instances :
try : try :
instance.kill("SIGHUP") instance.kill("SIGHUP")
except : except :
ret = False ret = False
elif self.__type == ControllerType.SWARM : elif self.__type == Controller.Type.SWARM :
ret = self.__api_call(instances, "/reload") ret = self.__api_call(instances, "/reload")
elif self.__type == ControllerType.KUBERNETES : elif self.__type == Controller.Type.KUBERNETES :
ret = self.__api_call(instances, "/reload") ret = self.__api_call(instances, "/reload")
return ret return ret
@ -61,9 +60,9 @@ class Config :
def wait(self, instances) : def wait(self, instances) :
ret = True ret = True
if self.__type == ControllerType.DOCKER : if self.__type == Controller.Type.DOCKER :
ret = self.__wait_docker() ret = self.__wait_docker()
elif self.__type == ControllerType.SWARM or self.__type == ControllerType.KUBERNETES : elif self.__type == Controller.Type.SWARM or self.__type == Controller.Type.KUBERNETES :
ret = self.__wait_api() ret = self.__wait_api()
return ret return ret
@ -96,20 +95,20 @@ class Config :
started = True started = True
break break
i = i + 1 i = i + 1
log("CONFIG", "INFO" "waiting " + str(i) + " seconds before retrying to contact bunkerized-nginx instances") log("config", "INFO" "waiting " + str(i) + " seconds before retrying to contact bunkerized-nginx instances")
if started : if started :
log("CONFIG", "INFO", "bunkerized-nginx instances started") log("config", "INFO", "bunkerized-nginx instances started")
return True return True
else : else :
log("CONFIG", "ERROR", "bunkerized-nginx instances are not started") log("config", "ERROR", "bunkerized-nginx instances are not started")
except Exception as e : except Exception as e :
log("CONFIG", "ERROR", "exception while waiting for bunkerized-nginx instances : " + traceback.format_exc()) log("config", "ERROR", "exception while waiting for bunkerized-nginx instances : " + traceback.format_exc())
return False return False
def __api_call(self, instances, path) : def __api_call(self, instances, path) :
ret = True ret = True
urls = [] urls = []
if self.__type == ControllerType.SWARM : if self.__type == Controller.Type.SWARM :
for instance in instances : for instance in instances :
name = instance.name name = instance.name
for task in instance.tasks() : for task in instance.tasks() :
@ -117,8 +116,8 @@ class Config :
taskID = task["ID"] taskID = task["ID"]
url = "http://" + name + "." + nodeID + "." + taskID + ":8080" + self.__api_uri + path url = "http://" + name + "." + nodeID + "." + taskID + ":8080" + self.__api_uri + path
urls.append(url) urls.append(url)
elif self.__type == ControllerType.KUBERNETES : elif self.__type == Controller.Type.KUBERNETES :
log("CONFIG", "ERROR", "TODO get urls for k8s") log("config", "ERROR", "TODO get urls for k8s")
for url in urls : for url in urls :
try : try :
@ -126,8 +125,8 @@ class Config :
except : except :
pass pass
if req and req.status_code == 200 and req.text == "ok" : if req and req.status_code == 200 and req.text == "ok" :
log("CONFIG", "INFO", "successfully sent API order to " + url) log("config", "INFO", "successfully sent API order to " + url)
else : else :
log("CONFIG", "INFO", "failed API order to " + url) log("config", "INFO", "failed API order to " + url)
ret = False ret = False
return ret return ret

View File

@ -1,7 +1,9 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from enum import Enum
from Config import Config from Config import Config
class ControllerType(Enum) : class Type(Enum) :
DOCKER = 1 DOCKER = 1
SWARM = 2 SWARM = 2
KUBERNETES = 3 KUBERNETES = 3
@ -28,7 +30,7 @@ class Controller(ABC) :
return self.__config.gen(env) return self.__config.gen(env)
@abstractmethod @abstractmethod
def process_events(self) : def process_events(self, current_env) :
pass pass
@abstractmethod @abstractmethod

View File

@ -1,11 +1,12 @@
import docker import docker
from Controller import Controller, ControllerType import Controller
import utils
class DockerController(Controller) : from logger import log
class DockerController(Controller.Controller) :
def __init__(self) : def __init__(self) :
super().__init__(ControllerType.DOCKER) super().__init__(Controller.Type.DOCKER)
# TODO : honor env vars like DOCKER_HOST # TODO : honor env vars like DOCKER_HOST
self.__client = docker.DockerClient(base_url='unix:///var/run/docker.sock') self.__client = docker.DockerClient(base_url='unix:///var/run/docker.sock')
@ -21,22 +22,38 @@ class DockerController(Controller) :
for variable in instance.attrs["Config"]["Env"] : for variable in instance.attrs["Config"]["Env"] :
env[variable.split("=")[0]] = variable.replace(variable.split("=")[0] + "=", "", 1) env[variable.split("=")[0]] = variable.replace(variable.split("=")[0] + "=", "", 1)
first_servers = [] first_servers = []
if "SERVER_NAME" in env : if "SERVER_NAME" in env and env["SERVER_NAME"] != "" :
first_servers = env["SERVER_NAME"].split(" ") first_servers = env["SERVER_NAME"].split(" ")
for container in self.__get_containers() : for container in self.__get_containers() :
first_server = container.labels["bunkerized-nginx.SERVER_NAME"].split(" ")[0] first_server = container.labels["bunkerized-nginx.SERVER_NAME"].split(" ")[0]
first_servers.append(first_server) first_servers.append(first_server)
for variable, value in instance.labels.items() : for variable, value in container.labels.items() :
if variable.startswith("bunkerized-nginx.") : if variable.startswith("bunkerized-nginx.") and variable != "bunkerized-nginx.AUTOCONF" :
env[first_server + "_" + variable.replace("bunkerized-nginx.", "", 1)] = value env[first_server + "_" + variable.replace("bunkerized-nginx.", "", 1)] = value
env["SERVER_NAME"] = " ".join(first_servers) if len(first_servers) == 0 :
env["SERVER_NAME"] = ""
else :
env["SERVER_NAME"] = " ".join(first_servers)
return self._fix_env(env) return self._fix_env(env)
def process_events(self, current_env) : def process_events(self, current_env) :
old_env = current_env old_env = current_env
for event in client.events(decode=True, filter={"type": "container", "label": ["bunkerized-nginx.AUTOCONF", "bunkerized-nginx.SERVER_NAME"]}) : # TODO : check why filter isn't working as expected
#for event in self.__client.events(decode=True, filters={"type": "container", "label": ["bunkerized-nginx.AUTOCONF", "bunkerized-nginx.SERVER_NAME"]}) :
for event in self.__client.events(decode=True, filters={"type": "container"}) :
new_env = self.get_env() new_env = self.get_env()
if new_env != old_env : if new_env != old_env :
log("controller", "INFO", "generating new configuration")
if self.gen_conf(new_env) : if self.gen_conf(new_env) :
old_env.copy(new_env) old_env = new_env.copy()
log("CONTROLLER", "INFO", "successfully generated new configuration") log("controller", "INFO", "successfully generated new configuration")
if self.reload() :
log("controller", "INFO", "successful reload")
else :
log("controller", "ERROR", "failed reload")
else :
log("controller", "ERROR", "can't generate new configuration")
def reload(self) :
return self._reload(self.__get_instances())

View File

@ -1,14 +1,14 @@
from kubernetes import client, config, watch from kubernetes import client, config, watch
from threading import Thread from threading import Thread
from Controller import Controller, ControllerType import Controller
from logger import log from logger import log
class IngressController : class IngressController(Controller.Controller) :
def __init__(self, api_uri) : def __init__(self, api_uri) :
super().__init__(ControllerType.KUBERNETES, api_uri=api_uri, lock=Lock()) super().__init__(Controller.Type.KUBERNETES, api_uri=api_uri, lock=Lock())
config.load_incluster_config() config.load_incluster_config()
self.__api = client.CoreV1Api() self.__api = client.CoreV1Api()
self.__extensions_api = client.ExtensionsV1beta1Api() self.__extensions_api = client.ExtensionsV1beta1Api()
@ -79,7 +79,7 @@ class IngressController :
new_env = self.get_env() new_env = self.get_env()
if new_env != self.__old_env() : if new_env != self.__old_env() :
if self.gen_conf(new_env, lock=False) : if self.gen_conf(new_env, lock=False) :
self.__old_env.copy(new_env) self.__old_env = new_env.copy()
log("CONTROLLER", "INFO", "successfully generated new configuration") log("CONTROLLER", "INFO", "successfully generated new configuration")
self.lock.release() self.lock.release()
@ -90,6 +90,9 @@ class IngressController :
new_env = self.get_env() new_env = self.get_env()
if new_env != self.__old_env() : if new_env != self.__old_env() :
if self.gen_conf(new_env, lock=False) : if self.gen_conf(new_env, lock=False) :
self.__old_env.copy(new_env) self.__old_env = new_env.copy()
log("CONTROLLER", "INFO", "successfully generated new configuration") log("CONTROLLER", "INFO", "successfully generated new configuration")
self.lock.release() self.lock.release()
def reload(self) :
return self._reload(self.__get_ingresses())

View File

@ -21,7 +21,7 @@ class ReloadServerHandler(socketserver.StreamRequestHandler):
locked = False locked = False
self.request.sendall(b"ok") self.request.sendall(b"ok")
elif data == b"reload" : elif data == b"reload" :
ret = self.server.controller.reload() : ret = self.server.controller.reload()
if ret : if ret :
self.request.sendall(b"ok") self.request.sendall(b"ok")
else : else :

View File

@ -1,11 +1,13 @@
import docker import docker
from Controller import Controller, ControllerType
import utils
class SwarmController(Controller) : from logger import log
import Controller
class SwarmController(Controller.Controller) :
def __init__(self, api_uri) : def __init__(self, api_uri) :
super().__init__(ControllerType.SWARM, api_uri=api_uri, lock=Lock()) super().__init__(Controller.Type.SWARM, api_uri=api_uri, lock=Lock())
# TODO : honor env vars like DOCKER_HOST # TODO : honor env vars like DOCKER_HOST
self.__client = docker.DockerClient(base_url='unix:///var/run/docker.sock') self.__client = docker.DockerClient(base_url='unix:///var/run/docker.sock')
@ -21,24 +23,39 @@ class SwarmController(Controller) :
for variable in instance.attrs["Spec"]["TaskTemplate"]["ContainerSpec"]["Env"] : for variable in instance.attrs["Spec"]["TaskTemplate"]["ContainerSpec"]["Env"] :
env[variable.split("=")[0]] = variable.replace(variable.split("=")[0] + "=", "", 1) env[variable.split("=")[0]] = variable.replace(variable.split("=")[0] + "=", "", 1)
first_servers = [] first_servers = []
if "SERVER_NAME" in env : if "SERVER_NAME" in env and env["SERVER_NAME"] != "" :
first_servers = env["SERVER_NAME"].split(" ") first_servers = env["SERVER_NAME"].split(" ")
for service in self.__get_services() : for service in self.__get_services() :
first_server = service.attrs["Spec"]["Labels"]["bunkerized-nginx.SERVER_NAME"].split(" ")[0] first_server = service.attrs["Spec"]["Labels"]["bunkerized-nginx.SERVER_NAME"].split(" ")[0]
first_servers.append(first_server) first_servers.append(first_server)
for variable, value in service.attrs["Spec"]["Labels"].items() : for variable, value in service.attrs["Spec"]["Labels"].items() :
if variable.startswith("bunkerized-nginx.") : if variable.startswith("bunkerized-nginx.") and variable != "bunkerized-nginx.AUTOCONF" :
env[first_server + "_" + variable.replace("bunkerized-nginx.", "", 1)] = value env[first_server + "_" + variable.replace("bunkerized-nginx.", "", 1)] = value
env["SERVER_NAME"] = " ".join(first_servers) if len(first_servers) == 0 :
env["SERVER_NAME"] = ""
else :
env["SERVER_NAME"] = " ".join(first_servers)
return self._fix_env(env) return self._fix_env(env)
def process_events(self, current_env) : def process_events(self, current_env) :
old_env = current_env old_env = current_env
for event in client.events(decode=True, filter={"type": "service", "label": ["bunkerized-nginx.AUTOCONF", "bunkerized-nginx.SERVER_NAME"]}) : # TODO : check why filter isn't working as expected
#for event in self.__client.events(decode=True, filters={"type": "service", "label": ["bunkerized-nginx.AUTOCONF", "bunkerized-nginx.SERVER_NAME"]}) :
for event in self.__client.events(decode=True, filters={"type": "service"}) :
new_env = self.get_env() new_env = self.get_env()
if new_env != old_env : if new_env != old_env :
self.lock.acquire() self.lock.acquire()
if self.gen_conf(new_env, lock=False) : log("controller", "INFO", "generating new configuration")
old_env.copy(new_env) if self.gen_conf(new_env) :
log("CONTROLLER", "INFO", "successfully generated new configuration") old_env = new_env.copy()
log("controller", "INFO", "successfully generated new configuration")
if self.reload() :
log("controller", "INFO", "successful reload")
else :
log("controller", "ERROR", "failed reload")
else :
log("controller", "ERROR", "can't generate new configuration")
self.lock.release() self.lock.release()
def reload(self) :
return self._reload(self.__get_instances())

View File

@ -6,7 +6,7 @@ import docker, os, stat, sys, select, threading
from DockerController import DockerController from DockerController import DockerController
from SwarmController import SwarmController from SwarmController import SwarmController
from KubernetesController import KubernetesController from IngressController import IngressController
from logger import log from logger import log
@ -21,7 +21,7 @@ if swarm :
controller = SwarmController(api_uri) controller = SwarmController(api_uri)
elif kubernetes : elif kubernetes :
log("autoconf", "INFO", "kubernetes mode detected") log("autoconf", "INFO", "kubernetes mode detected")
controller = KubernetesController(api_uri) controller = IngressController(api_uri)
else : else :
log("autoconf", "INFO", "docker mode detected") log("autoconf", "INFO", "docker mode detected")
controller = DockerController() controller = DockerController()
@ -32,8 +32,8 @@ if swarm or kubernetes :
# Apply the first config for existing services # Apply the first config for existing services
current_env = controller.get_env() current_env = controller.get_env()
if env != {} : if current_env != {} :
controller.gen_conf(current_env) controller.gen_conf(current_env)
# Process events # Process events
controller.process_events() controller.process_events(current_env)

View File

@ -1,33 +1,35 @@
#!/bin/bash #!/bin/bash
echo "[*] Starting bunkerized-nginx ..." . /opt/bunkerize-nginx/entrypoint/utils.sh
log "entrypoint" "INFO" "starting bunkerized-nginx ..."
# trap SIGTERM and SIGINT # trap SIGTERM and SIGINT
function trap_exit() { function trap_exit() {
echo "[*] Catched stop operation" log "stop" "INFO" "catched stop operation"
echo "[*] Stopping crond ..." log "stop" "INFO" "stopping crond ..."
pkill -TERM crond pkill -TERM crond
echo "[*] Stopping nginx ..." log "stop" "INFO" "stopping nginx ..."
/usr/sbin/nginx -s stop /usr/sbin/nginx -s stop
} }
trap "trap_exit" TERM INT QUIT trap "trap_exit" TERM INT QUIT
# trap SIGHUP # trap SIGHUP
function trap_reload() { function trap_reload() {
echo "[*] Catched reload operation" log "reload" "INFO" "catched reload operation"
if [ "$SWARM_MODE" != "yes" ] ; then if [ "$SWARM_MODE" != "yes" ] ; then
/opt/bunkerized-nginx/entrypoint/jobs.sh /opt/bunkerized-nginx/entrypoint/jobs.sh
fi fi
if [ -f /tmp/nginx.pid ] ; then if [ -f /tmp/nginx.pid ] ; then
echo "[*] Reloading nginx ..." log "reload" "INFO" "reloading nginx ..."
nginx -s reload nginx -s reload
if [ $? -eq 0 ] ; then if [ $? -eq 0 ] ; then
echo "[*] Reload successfull" log "reload" "INFO" "reloading successful"
else else
echo "[!] Reload failed" log "reload" "ERROR" "reloading failed"
fi fi
else else
echo "[!] Ignored reload operation because nginx is not running" log "reload" "INFO" "ignored reload operation because nginx is not running"
fi fi
} }
trap "trap_reload" HUP trap "trap_reload" HUP
@ -35,7 +37,7 @@ trap "trap_reload" HUP
# do the configuration magic if needed # do the configuration magic if needed
if [ ! -f "/etc/nginx/global.env" ] ; then if [ ! -f "/etc/nginx/global.env" ] ; then
echo "[*] Configuring bunkerized-nginx ..." log "entrypoint" "INFO" "configuring bunkerized-nginx ..."
# check permissions # check permissions
if [ "$SWARM_MODE" != "yes" ] ; then if [ "$SWARM_MODE" != "yes" ] ; then
@ -50,27 +52,34 @@ if [ ! -f "/etc/nginx/global.env" ] ; then
# start temp nginx to solve Let's Encrypt challenges if needed # start temp nginx to solve Let's Encrypt challenges if needed
/opt/bunkerized-nginx/entrypoint/nginx-temp.sh /opt/bunkerized-nginx/entrypoint/nginx-temp.sh
# only do config if we are not in swarm mode # only do config if we are not in swarm/kubernetes mode
if [ "$SWARM_MODE" != "yes" ] ; then if [ "$SWARM_MODE" != "yes" ] && [ "$KUBERNETES_MODE" != "yes" ] ; then
# export the variables # export the variables
env | grep -E -v "^(HOSTNAME|PWD|PKG_RELEASE|NJS_VERSION|SHLVL|PATH|_|NGINX_VERSION|HOME)=" > "/tmp/variables.env" env | grep -E -v "^(HOSTNAME|PWD|PKG_RELEASE|NJS_VERSION|SHLVL|PATH|_|NGINX_VERSION|HOME)=" > "/tmp/variables.env"
# call the generator # call the generator
/opt/bunkerized-nginx/gen/main.py --settings /opt/bunkerized-nginx/settings.json --templates /opt/bunkerized-nginx/confs --output /etc/nginx --variables /tmp/variables.env gen_ret="$(/opt/bunkerized-nginx/gen/main.py --settings /opt/bunkerized-nginx/settings.json --templates /opt/bunkerized-nginx/confs --output /etc/nginx --variables /tmp/variables.env 2>&1)"
if [ "$?" -ne 0 ] ; then
log "entrypoint" "ERROR" "generator failed : $gen_ret"
exit 1
fi
if [ "$gen_ret" != "" ] ; then
log "entrypoint" "INFO" "generator output : $gen_ret"
fi
# call jobs # call jobs
/opt/bunkerized-nginx/entrypoint/jobs.sh /opt/bunkerized-nginx/entrypoint/jobs.sh
fi fi
else else
echo "[*] Skipping configuration process" log "entrypoint" "INFO" "skipping configuration process"
fi fi
# start crond # start crond
crond crond
# wait until config has been generated if we are in swarm mode # wait until config has been generated if we are in swarm mode
if [ "$SWARM_MODE" = "yes" ] ; then if [ "$SWARM_MODE" = "yes" ] || [ "$KUBERNETES_MODE" = "yes" ] ; then
echo "[*] Waiting until config has been generated ..." log "entrypoint" "INFO" "waiting until config has been generated ..."
while [ ! -f "/etc/nginx/autoconf" ] ; do while [ ! -f "/etc/nginx/autoconf" ] ; do
sleep 1 sleep 1
done done
@ -82,7 +91,7 @@ if [ -f "/tmp/nginx-temp.pid" ] ; then
fi fi
# run nginx # run nginx
echo "[*] Running nginx ..." log "entrypoint" "INFO" "running nginx ..."
nginx -g 'daemon off;' & nginx -g 'daemon off;' &
pid="$!" pid="$!"
@ -104,5 +113,5 @@ while [ -f "/tmp/nginx.pid" ] ; do
done done
# sigterm trapped # sigterm trapped
echo "[*] bunkerized-nginx stopped" log "entrypoint" "INFO" "bunkerized-nginx stopped"
exit 0 exit 0

View File

@ -32,9 +32,12 @@ function has_value() {
done done
} }
# log to jobs.log # log to stdout
function job_log() { function log() {
when="$(date '+[%Y-%m-%d %H:%M:%S]')" when="$(date '+[%Y-%m-%d %H:%M:%S]')"
what="$1" category="$1"
echo "$when $what" >> /var/log/nginx/jobs.log severity="$2"
message="$3"
echo "$when $category - $severity - $message"
} }

View File

@ -25,7 +25,7 @@ class Configurator :
if check : if check :
self.__variables[var] = value self.__variables[var] = value
else : else :
print("Ignoring " + var + "=" + value + " (" + reason + ")") print("ignoring " + var + "=" + value + " (" + reason + ")", file=sys.stderr)
def get_config(self) : def get_config(self) :
config = {} config = {}