From f950abdc24aa336a793d9213933db89ba647b9a8 Mon Sep 17 00:00:00 2001 From: bunkerity Date: Thu, 27 May 2021 12:34:00 +0200 Subject: [PATCH] templating - started integration into autoconf --- autoconf/AutoConf.py | 71 +++++++++++++++++----- autoconf/Config.py | 69 ++++----------------- autoconf/prepare.sh | 2 +- confs/global/multisite-default-server.conf | 3 +- confs/global/nginx.conf | 31 +++++++--- settings.json | 2 +- 6 files changed, 95 insertions(+), 83 deletions(-) diff --git a/autoconf/AutoConf.py b/autoconf/AutoConf.py index 4e1fad0..491aa92 100644 --- a/autoconf/AutoConf.py +++ b/autoconf/AutoConf.py @@ -1,3 +1,5 @@ +# TODO : hard tests, jobs, check state when generating env, ... + from Config import Config import utils import os @@ -7,7 +9,7 @@ class AutoConf : self.__swarm = swarm self.__servers = {} self.__instances = {} - self.__sites = {} + self.__env = {} self.__config = Config(self.__swarm, api) def get_server(self, id) : @@ -18,6 +20,21 @@ class AutoConf : def reload(self) : return self.__config.reload(self.__instances) + def __gen_env(self) : + self.__env.clear() + # TODO : check actual state (e.g. : running ?) + for instance in self.__instances : + (id, name, labels) = self.__get_infos(self.__instances[instance]) + for label in labels : + if label.startswith("bunkerized-nginx.") : + self.__env[label.replace("bunkerized-nginx.", "", 1)] = labels[label] + for server in self.__servers : + (id, name, labels) = self.__get_infos(self.__servers[server]) + first_server = labels["bunkerized-nginx.SERVER_NAME"].split(" ")[0] + for label in labels : + if label.startswith("bunkerized-nginx.") : + self.__env[first_server + "_" + label.replace("bunkerized-nginx.", "", 1)] = labels[label] + def pre_process(self, objs) : for instance in objs : (id, name, labels) = self.__get_infos(instance) @@ -60,22 +77,32 @@ class AutoConf : return (id, name, labels) def __process_instance(self, instance, event, id, name, labels) : + if event == "create" : self.__instances[id] = instance + self.__gen_env() if self.__swarm and len(self.__instances) == 1 : - if self.__config.initconf(self.__instances) : + if self.__config.generate(self.__env) : utils.log("[*] Initial config succeeded") + with open("/etc/nginx/autoconf", "w") as f : + f.write("ok") else : utils.log("[!] Initial config failed") utils.log("[*] bunkerized-nginx instance created : " + name + " / " + id) + elif event == "start" : self.__instances[id].reload() + self.__gen_env() utils.log("[*] bunkerized-nginx instance started : " + name + " / " + id) + elif event == "die" : self.__instances[id].reload() + self.__gen_env() utils.log("[*] bunkerized-nginx instance stopped : " + name + " / " + id) + elif event == "destroy" or event == "remove" : del self.__instances[id] + self.__gen_env() if self.__swarm and len(self.__instances) == 0 : with open("/etc/crontabs/nginx", "w") as f : f.write("") @@ -84,47 +111,59 @@ class AutoConf : utils.log("[*] bunkerized-nginx instance removed : " + name + " / " + id) def __process_server(self, instance, event, id, name, labels) : + vars = { k.replace("bunkerized-nginx.", "", 1) : v for k, v in labels.items() if k.startswith("bunkerized-nginx.")} + if event == "create" : utils.log("[*] Generating config for " + vars["SERVER_NAME"] + " ...") - if self.__config.generate(self.__instances, vars) : + self.__servers[id] = instance + self.__gen_env() + if self.__config.generate(self.__env) : utils.log("[*] Generated config for " + vars["SERVER_NAME"]) - self.__servers[id] = instance if self.__swarm : utils.log("[*] Activating config for " + vars["SERVER_NAME"] + " ...") - if self.__config.activate(self.__instances, vars) : + if self.__config.reload(self.__instances) : utils.log("[*] Activated config for " + vars["SERVER_NAME"]) else : - utils.log("[!] Can't activate config for " + vars["SERVER_NAME"]) + utils.log("[!] Can't activate config for " + vars["SERVER_NAME"]) else : utils.log("[!] Can't generate config for " + vars["SERVER_NAME"]) + del self.__servers[id] + self.__gen_env() + self.__config.generate(self.__env) + elif event == "start" : if id in self.__servers : self.__servers[id].reload() utils.log("[*] Activating config for " + vars["SERVER_NAME"] + " ...") - if self.__config.activate(self.__instances, vars) : + self.__gen_env() + if self.__config.reload(self.__instances) : utils.log("[*] Activated config for " + vars["SERVER_NAME"]) else : utils.log("[!] Can't activate config for " + vars["SERVER_NAME"]) + elif event == "die" : if id in self.__servers : self.__servers[id].reload() utils.log("[*] Deactivating config for " + vars["SERVER_NAME"]) - if self.__config.deactivate(self.__instances, vars) : + self.__gen_env() + if self.__config.reload() : utils.log("[*] Deactivated config for " + vars["SERVER_NAME"]) else : utils.log("[!] Can't deactivate config for " + vars["SERVER_NAME"]) + elif event == "destroy" or event == "remove" : if id in self.__servers : - if self.__swarm : - utils.log("[*] Deactivating config for " + vars["SERVER_NAME"]) - if self.__config.deactivate(self.__instances, vars) : - utils.log("[*] Deactivated config for " + vars["SERVER_NAME"]) - else : - utils.log("[!] Can't deactivate config for " + vars["SERVER_NAME"]) - del self.__servers[id] utils.log("[*] Removing config for " + vars["SERVER_NAME"]) - if self.__config.remove(vars) : + del self.__servers[id] + self.__gen_env() + if self.__config.generate(self.__env) : utils.log("[*] Removed config for " + vars["SERVER_NAME"]) else : utils.log("[!] Can't remove config for " + vars["SERVER_NAME"]) + utils.log("[*] Deactivating config for " + vars["SERVER_NAME"]) + if self.__config.reload(self.__instances) : + utils.log("[*] Deactivated config for " + vars["SERVER_NAME"]) + else : + utils.log("[!] Can't deactivate config for " + vars["SERVER_NAME"]) + diff --git a/autoconf/Config.py b/autoconf/Config.py index 1158128..09ce6c9 100644 --- a/autoconf/Config.py +++ b/autoconf/Config.py @@ -9,7 +9,6 @@ class Config : self.__swarm = swarm self.__api = api - def initconf(self, instances) : try : for instance_id, instance in instances.items() : @@ -59,48 +58,24 @@ class Config : utils.log("[!] Error while initializing config : " + str(e)) return False - def globalconf(self, instances) : + def generate(self, env) : try : - for instance_id, instance in instances.items() : - env = instance.attrs["Spec"]["TaskTemplate"]["ContainerSpec"]["Env"] - break - vars = {} - for var_value in env : - var = var_value.split("=")[0] - value = var_value.replace(var + "=", "", 1) - vars[var] = value - proc = subprocess.run(["/bin/su", "-s", "/opt/entrypoint/global-config.sh", "nginx"], env=vars, capture_output=True) - if proc.returncode == 0 : - return True - else : - utils.log("[*] Error while generating global config : return code = " + str(proc.returncode)) - except Exception as e : - utils.log("[!] Exception while generating global config : " + str(e)) - return False + # Write environment variables to fs + with open("/tmp/variables.env", "w") as f : + for k, v in env.items() : + f.write(k + "=" + v + "\n") - def generate(self, instances, vars) : - try : - # Get env vars from bunkerized-nginx instances - vars_instances = {} - for instance_id, instance in instances.items() : - if self.__swarm : - env = instance.attrs["Spec"]["TaskTemplate"]["ContainerSpec"]["Env"] - else : - env = instance.attrs["Config"]["Env"] - for var_value in env : - var = var_value.split("=")[0] - value = var_value.replace(var + "=", "", 1) - vars_instances[var] = value - vars_defaults = vars.copy() - vars_defaults.update(vars_instances) - vars_defaults.update(vars) - # Call site-config.sh to generate the config - proc = subprocess.run(["/bin/su", "-s", "/bin/sh", "-c", "/opt/entrypoint/site-config.sh" + " \"" + vars["SERVER_NAME"] + "\"", "nginx"], env=vars_defaults, capture_output=True) - if proc.returncode == 0 and vars_defaults["MULTISITE"] == "yes" and self.__swarm : - proc = subprocess.run(["/bin/su", "-s", "/opt/entrypoint/multisite-config.sh", "nginx"], env=vars_defaults, capture_output=True) + # Call the generator + proc = subprocess.run(["/bin/su", "-c", "/opt/gen/main.py --settings /opt/settings.json --templates /opt/confs --output /etc/nginx --variables /tmp/variables.env", "nginx"], capture_output=True) + + # Print stdout/stderr just in case + # TODO + + # We're done if proc.returncode == 0 : return True utils.log("[!] Error while generating site config for " + vars["SERVER_NAME"] + " : return code = " + str(proc.returncode)) + except Exception as e : utils.log("[!] Exception while generating site config : " + str(e)) return False @@ -149,24 +124,6 @@ class Config : return False - def remove(self, vars) : - try : - # Get first server name - first_server_name = vars["SERVER_NAME"].split(" ")[0] - - # Check if file exists - if not os.path.isfile("/etc/nginx/" + first_server_name + "/server.conf") : - utils.log("[!] /etc/nginx/" + first_server_name + "/server.conf doesn't exist") - return False - - # Remove the folder - shutil.rmtree("/etc/nginx/" + first_server_name) - return True - except Exception as e : - utils.log("[!] Error while deactivating config : " + str(e)) - - return False - def reload(self, instances) : return self.__api_call(instances, "/reload") diff --git a/autoconf/prepare.sh b/autoconf/prepare.sh index 622dced..e8f37c2 100644 --- a/autoconf/prepare.sh +++ b/autoconf/prepare.sh @@ -2,7 +2,7 @@ # create nginx user addgroup -g 101 nginx -adduser -h /var/cache/nginx -g nginx -s /sbin/nologin -G nginx -D -H -u 101 nginx +adduser -h /var/cache/nginx -g nginx -s /bin/sh -G nginx -D -H -u 101 nginx # prepare /opt chown -R root:nginx /opt diff --git a/confs/global/multisite-default-server.conf b/confs/global/multisite-default-server.conf index 74d4029..f4c3659 100644 --- a/confs/global/multisite-default-server.conf +++ b/confs/global/multisite-default-server.conf @@ -1,6 +1,5 @@ -toto server { - {% if LISTEN_HTTP == "yes" %}listen 0.0.0.0:{{ HTTP_PORT }} default_server{% endif %} + {% if LISTEN_HTTP == "yes" %}listen 0.0.0.0:{{ HTTP_PORT }} default_server{% endif %}; server_name _; {% if has_value("AUTO_LETS_ENCRYPT", "yes") %}include /etc/nginx/multisite-default-server-https.conf;{% endif %} {% if DISABLE_DEFAULT_SERVER == "yes" %}include /etc/nginx/multisite-disable-default-server.conf;{% endif %} diff --git a/confs/global/nginx.conf b/confs/global/nginx.conf index aa24be7..0db66c9 100644 --- a/confs/global/nginx.conf +++ b/confs/global/nginx.conf @@ -116,14 +116,31 @@ http { {% if MULTISITE == "yes" %}include /etc/nginx/multisite-default-server.conf;{% endif %} # server config(s) - {% if SWARM_MODE == "no" %} - {% if MULTISITE == "yes" %} - {% for server_name in SERVER_NAME.split(" ") %} - include /etc/nginx/{{ server_name }}/server.conf; - {% endfor %} - {% else %} + {% if MULTISITE == "yes" and SERVER_NAME != "" %} + {% set map_servers = {} %} + {% for server_name in SERVER_NAME.split(" ") %} + {% if server_name + "_SERVER_NAME" in all %} + {% set x = map_servers.update({server_name : [all[server_name] + "_SERVER_NAME"].split(" ")}) %} + {% endif %} + {% endfor %} + {% for server_name in SERVER_NAME.split(" ") %} + {% if not server_name in map_servers %} + {% set found = {"res": false} %} + {% for first_server, servers in map_servers.items() %} + {% if server_name in servers %} + {% set x = found.update({"res" : true}) %} + {% endif %} + {% endfor %} + {% if not found["res"] %} + {% set x = map_servers.update({server_name : [server_name]}) %} + {% endif %} + {% endif %} + {% endfor %} + {% for first_server in map_servers %} + include /etc/nginx/{{ first_server }}/server.conf; + {% endfor %} + {% elif MULTISITE == "no" %} include /etc/nginx/server.conf; - {% endif %} {% endif %} # API diff --git a/settings.json b/settings.json index ed8d843..6ae7477 100644 --- a/settings.json +++ b/settings.json @@ -843,7 +843,7 @@ "env": "SERVER_NAME", "id": "server-name", "label": "Server name", - "regex": "^([a-z\\-0-9]+\\.? ?)+$", + "regex": "^([a-z\\-0-9]+\\.? ?)*$", "type": "text" }, {