From 2db967ad1dea7665e9750963218cabe6972bc126 Mon Sep 17 00:00:00 2001 From: bunkerity Date: Mon, 31 May 2021 17:49:37 +0200 Subject: [PATCH] templating - road to web ui --- gen/Templator.py | 26 +++-- gen/test.txt | 2 +- ui/Config.py | 91 +++++++++++++++++ ui/Docker.py | 17 ++-- ui/Dockerfile | 8 +- ui/Dockerfile-amd64 | 8 +- ui/Dockerfile-arm32v7 | 8 +- ui/Dockerfile-arm64v8 | 8 +- ui/Dockerfile-i386 | 8 +- ui/entrypoint.py | 110 +++++++++------------ ui/templates/services-edit.html | 4 +- ui/templates/services-new.html | 6 +- ui/utils.py | 7 -- ui/wrappers.py | 168 -------------------------------- 14 files changed, 186 insertions(+), 285 deletions(-) create mode 100644 ui/Config.py delete mode 100644 ui/wrappers.py diff --git a/gen/Templator.py b/gen/Templator.py index e58a56c..82f6712 100644 --- a/gen/Templator.py +++ b/gen/Templator.py @@ -3,37 +3,45 @@ import jinja2, glob, os, pathlib, copy, crypt, random, string class Templator : def __init__(self, config, input_path, output_path, target_path) : - self.__config = config + self.__config_global = copy.deepcopy(config) + if config["MULTISITE"] == "yes" and config["SERVER_NAME"] != "" : + self.__config_sites = {} + for server_name in config["SERVER_NAME"].split(" ") : + self.__config_sites[server_name] = {} + for k, v in config.items() : + if k.startswith(server_name + "_") : + self.__config_sites[server_name][k.replace(server_name + "_", "", 1)] = v + del self.__config_global[k] self.__input_path = input_path self.__output_path = output_path self.__target_path = target_path if not self.__target_path.endswith("/") : self.__target_path += "/" - self.__template_env = jinja2.Environment(loader=jinja2.FileSystemLoader(searchpath=self.__input_path), trim_blocks=True, lstrip_blocks=True) + self.__template_env = jinja2.Environment(loader=jinja2.FileSystemLoader(searchpath=self.__input_path), lstrip_blocks=True) def render_global(self) : return self.__render("global") def render_site(self, server_name=None, first_server=None) : if server_name is None : - server_name = self.__config["SERVER_NAME"] + server_name = self.__config_global["SERVER_NAME"] if first_server is None : - first_server = self.__config["SERVER_NAME"].split(" ")[0] + first_server = self.__config_global["SERVER_NAME"].split(" ")[0] return self.__render("site", server_name, first_server) def __prepare_config(self, type, server_name=None, first_server=None) : - real_config = copy.deepcopy(self.__config) + real_config = copy.deepcopy(self.__config_global) + if type == "site" and self.__config_global["MULTISITE"] == "yes" : + site_config = copy.deepcopy(self.__config_sites[first_server]) + real_config.update(site_config) if not server_name is None : real_config["SERVER_NAME"] = server_name if not first_server is None : real_config["FIRST_SERVER"] = first_server real_config["NGINX_PREFIX"] = self.__target_path - if real_config["MULTISITE"] == "yes" and type == "site" : + if self.__config_global["MULTISITE"] == "yes" and type == "site" : real_config["NGINX_PREFIX"] += first_server + "/" real_config["ROOT_FOLDER"] += "/" + first_server - for variable, value in self.__config.items() : - if variable.startswith(first_server + "_") : - real_config[variable.replace(first_server + "_", "", 1)] = value if real_config["ROOT_SITE_SUBFOLDER"] != "" : real_config["ROOT_FOLDER"] += "/" + real_config["ROOT_SITE_SUBFOLDER"] return real_config diff --git a/gen/test.txt b/gen/test.txt index 94b0266..0a00e6e 100644 --- a/gen/test.txt +++ b/gen/test.txt @@ -1 +1 @@ -./main.py --settings /opt/work/bunkerized-nginx/settings.json --templates /opt/work/bunkerized-nginx/confs2 --output /tmp/debug --variables /tmp/variables.env +./main.py --settings /opt/work/bunkerized-nginx/settings.json --templates /opt/work/bunkerized-nginx/confs --output /tmp/debug --variables /tmp/variables.env diff --git a/ui/Config.py b/ui/Config.py new file mode 100644 index 0000000..dcdb0ab --- /dev/null +++ b/ui/Config.py @@ -0,0 +1,91 @@ +import json, uuid, glob + +class Config : + + def __init__(self) : + with open("/opt/settings.json", "r") as f : + self.__settings = json.loads(f.read()) + + def __env_to_dict(self, filename) : + with open(filename, "r") as f : + env = f.read() + data = {} + for line in env.split("\n") : + var = line.split("=")[0] + val = line.replace(var + "=", "", 1) + data[var] = val + return data + + def __dict_to_env(self, filename, variables) : + env = "" + for k, v in variables.items() : + env += k + "=" + v + "\n" + with open(filename, "w") as f : + f.write(env) + + def __gen_conf(self, global_conf, services_conf) : + conf = copy.deepcopy(global_conf) + for service in services_conf : + first_server = service["SERVER_NAME"].split(" ")[0] + for k, v in service.items() : + conf[first_server + "_" + k] = v + env_file = "/tmp/" + str(uuid.uuid4()) + ".env" + self.__dict_to_env(env_file, conf) + proc = subprocess.run(["/bin/su", "-c", "/opt/gen/main.py --settings /opt/settings.json --templates /opt/confs --output /etc/nginx --variables " + env_file, "nginx"], capture_output=True) + stderr = proc.stderr.decode("ascii") + if stderr != "" or proc.returncode != 0 : + raise Exception("Error from generator (return code = " + str(proc.returncode) + ") : " + stderr) + + def get_settings(self) : + return self.__settings + + def get_services(self) : + services = [] + for filename in glob.iglob("/etc/nginx/**/site.env") : + env = self.__env_to_dict(filename) + services.append(env) + return services + + def check_variables(self, variables) : + for k, v in variables.items() : + check = False + for category in self.__settings : + for param in self.__settings[category]["params"] : + if type != "multiple" : + real_params = [param] + else : + real_params = param + for real_param in real_params : + if k == real_param["env"] and real_param["context"] == "multisite" and re.search(real_param["regex"], v) : + check = True + if not check : + raise Exception("Variable " + k + " is not valid.") + + def new_service(self, variables) : + global_env = self.__env_to_dict("/etc/nginx/global.env") + services = self.get_services() + for service in services : + if service["SERVER_NAME"] == variables["SERVER_NAME"] or service["SERVER_NAME"] in variables["SERVER_NAME"].split(" ") : + raise Exception("Service " + service["SERVER_NAME"] + " already exists.") + services.append(variables) + self.__gen_conf(global_env, services) + + def edit_service(self, old_server_name, variables) : + self.delete_service(old_server_name) + self.new_service(variables) + + + def delete_service(self, server_name) : + global_env = self.__env_to_dict("/etc/nginx/global.env") + services = self.get_services() + new_services = [] + found = False + for service in services : + if service["SERVER_NAME"].split(" ")[0] == server_name : + found = True + else : + new_services.append(service) + if not found : + raise Exception("Can't delete missing " + server_name + " configuration.") + self.__gen_conf(global_env, new_services) + \ No newline at end of file diff --git a/ui/Docker.py b/ui/Docker.py index bdfcd9e..ad5cfa3 100644 --- a/ui/Docker.py +++ b/ui/Docker.py @@ -17,16 +17,21 @@ class Docker : return self.__client.containers.get(id) def reload_instance(self, id) : - return self.get_instance(id).kill(signal="SIGHUP") + self.get_instance(id).kill(signal="SIGHUP") + return "Instance " + id + " has been reloaded." def start_instance(self, id) : - return self.get_instance(id).start() + self.get_instance(id).start() + return "Instance " + id + " has been started." def stop_instance(self, id) : - return self.get_instance(id).stop() + self.get_instance(id).stop() + return "Instance " + id + " has been stopped." def restart_instance(self, id) : - return self.get_instance(id).restart() + self.get_instance(id).restart() + return "Instance " + id + " has been restarted." - def remove_instance(self, id) : - return self.get_instance(id).remove(v=True, force=True) + def delete_instance(self, id) : + self.get_instance(id).remove(v=True, force=True) + return "Instance " + id + " has been deleted." diff --git a/ui/Dockerfile b/ui/Dockerfile index 47386b7..c473f26 100644 --- a/ui/Dockerfile +++ b/ui/Dockerfile @@ -10,14 +10,12 @@ RUN chmod +x /tmp/dependencies.sh && \ rm -f /tmp/dependencies.sh COPY gen/ /opt/gen -#COPY entrypoint/ /opt/entrypoint -#COPY confs/global/ /opt/confs/global COPY confs/site/ /opt/confs/site -COPY ui/* /opt/entrypoint/ +COPY ui/ /opt/entrypoint COPY settings.json /opt COPY ui/prepare.sh /tmp - chmod +x /tmp/prepare && \ +RUN chmod +x /tmp/prepare.sh && \ /tmp/prepare.sh && \ rm -f /tmp/prepare.sh @@ -25,4 +23,4 @@ EXPOSE 5000 WORKDIR /opt/entrypoint ENV FLASK_APP entrypoint.py -ENTRYPOINT ["/usr/bin/python3", "-m", "flask", "run", "--host=0.0.0.0"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/python3", "-m", "flask", "run", "--host=0.0.0.0"] diff --git a/ui/Dockerfile-amd64 b/ui/Dockerfile-amd64 index 1063fef..22b1060 100644 --- a/ui/Dockerfile-amd64 +++ b/ui/Dockerfile-amd64 @@ -10,14 +10,12 @@ RUN chmod +x /tmp/dependencies.sh && \ rm -f /tmp/dependencies.sh COPY gen/ /opt/gen -#COPY entrypoint/ /opt/entrypoint -#COPY confs/global/ /opt/confs/global COPY confs/site/ /opt/confs/site -COPY ui/* /opt/entrypoint/ +COPY ui/ /opt/entrypoint COPY settings.json /opt COPY ui/prepare.sh /tmp - chmod +x /tmp/prepare && \ +RUN chmod +x /tmp/prepare && \ /tmp/prepare.sh && \ rm -f /tmp/prepare.sh @@ -25,4 +23,4 @@ EXPOSE 5000 WORKDIR /opt/entrypoint ENV FLASK_APP entrypoint.py -ENTRYPOINT ["/usr/bin/python3", "-m", "flask", "run", "--host=0.0.0.0"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/python3", "-m", "flask", "run", "--host=0.0.0.0"] diff --git a/ui/Dockerfile-arm32v7 b/ui/Dockerfile-arm32v7 index 7858cb8..1b7119b 100644 --- a/ui/Dockerfile-arm32v7 +++ b/ui/Dockerfile-arm32v7 @@ -16,14 +16,12 @@ RUN chmod +x /tmp/dependencies.sh && \ rm -f /tmp/dependencies.sh COPY gen/ /opt/gen -#COPY entrypoint/ /opt/entrypoint -#COPY confs/global/ /opt/confs/global COPY confs/site/ /opt/confs/site -COPY ui/* /opt/entrypoint/ +COPY ui/ /opt/entrypoint COPY settings.json /opt COPY ui/prepare.sh /tmp - chmod +x /tmp/prepare && \ +RUN chmod +x /tmp/prepare && \ /tmp/prepare.sh && \ rm -f /tmp/prepare.sh @@ -31,4 +29,4 @@ EXPOSE 5000 WORKDIR /opt/entrypoint ENV FLASK_APP entrypoint.py -ENTRYPOINT ["/usr/bin/python3", "-m", "flask", "run", "--host=0.0.0.0"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/python3", "-m", "flask", "run", "--host=0.0.0.0"] diff --git a/ui/Dockerfile-arm64v8 b/ui/Dockerfile-arm64v8 index 022ca24..24cf1b8 100644 --- a/ui/Dockerfile-arm64v8 +++ b/ui/Dockerfile-arm64v8 @@ -16,14 +16,12 @@ RUN chmod +x /tmp/dependencies.sh && \ rm -f /tmp/dependencies.sh COPY gen/ /opt/gen -#COPY entrypoint/ /opt/entrypoint -#COPY confs/global/ /opt/confs/global COPY confs/site/ /opt/confs/site -COPY ui/* /opt/entrypoint/ +COPY ui/ /opt/entrypoint COPY settings.json /opt COPY ui/prepare.sh /tmp - chmod +x /tmp/prepare && \ +RUN chmod +x /tmp/prepare && \ /tmp/prepare.sh && \ rm -f /tmp/prepare.sh @@ -31,4 +29,4 @@ EXPOSE 5000 WORKDIR /opt/entrypoint ENV FLASK_APP entrypoint.py -ENTRYPOINT ["/usr/bin/python3", "-m", "flask", "run", "--host=0.0.0.0"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/python3", "-m", "flask", "run", "--host=0.0.0.0"] diff --git a/ui/Dockerfile-i386 b/ui/Dockerfile-i386 index e254a13..b8768be 100644 --- a/ui/Dockerfile-i386 +++ b/ui/Dockerfile-i386 @@ -10,14 +10,12 @@ RUN chmod +x /tmp/dependencies.sh && \ rm -f /tmp/dependencies.sh COPY gen/ /opt/gen -#COPY entrypoint/ /opt/entrypoint -#COPY confs/global/ /opt/confs/global COPY confs/site/ /opt/confs/site -COPY ui/* /opt/entrypoint/ +COPY ui/ /opt/entrypoint COPY settings.json /opt COPY ui/prepare.sh /tmp - chmod +x /tmp/prepare && \ +RUN chmod +x /tmp/prepare && \ /tmp/prepare.sh && \ rm -f /tmp/prepare.sh @@ -25,4 +23,4 @@ EXPOSE 5000 WORKDIR /opt/entrypoint ENV FLASK_APP entrypoint.py -ENTRYPOINT ["/usr/bin/python3", "-m", "flask", "run", "--host=0.0.0.0"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/python3", "-m", "flask", "run", "--host=0.0.0.0"] diff --git a/ui/entrypoint.py b/ui/entrypoint.py index 09e8997..6687c26 100644 --- a/ui/entrypoint.py +++ b/ui/entrypoint.py @@ -3,7 +3,8 @@ from flask import Flask, render_template, current_app, request from Docker import Docker -import Docker, wrappers, utils +from Config import Config +import utils import os, json, re app = Flask(__name__, static_url_path="/", static_folder="static", template_folder="templates") @@ -11,9 +12,8 @@ ABSOLUTE_URI = "" if "ABSOLUTE_URI" in os.environ : ABSOLUTE_URI = os.environ["ABSOLUTE_URI"] app.config["ABSOLUTE_URI"] = ABSOLUTE_URI -with open("/opt/settings.json", "r") as f : - app.config["CONFIG"] = json.loads(f.read()) app.config["DOCKER"] = Docker() +app.config["CONFIG"] = Config() app.jinja_env.globals.update(env_to_summary_class=utils.env_to_summary_class) app.jinja_env.globals.update(form_service_gen=utils.form_service_gen) app.jinja_env.globals.update(form_service_gen_multiple=utils.form_service_gen_multiple) @@ -24,7 +24,7 @@ app.jinja_env.globals.update(form_service_gen_multiple_values=utils.form_service def home() : try : instances_number = len(app.config["DOCKER"].get_instances()) - services_number = 0 # TODO + services_number = len(app.config["CONFIG"].get_services()) return render_template("home.html", title="Home", instances_number=instances_number, services_number=services_number) except Exception as e : return render_template("error.html", title="Error", error=e) @@ -38,88 +38,70 @@ def instances() : # Check operation if not "operation" in request.form or not request.form["operation"] in ["reload", "start", "stop", "restart", "delete"] : - return render_template("error.html", title="Error", error="Missing operation parameter on /instances.") + raise Exception("Missing operation parameter on /instances.") # Check that all fields are present if not "INSTANCE_ID" in request.form : - return render_template("error.html", title="Error", error="Missing INSTANCE_ID parameter.") + raise Exception("Missing INSTANCE_ID parameter.") # Do the operation if request.form["operation"] == "reload" : - app.config["DOCKER"].reload(request_form["INSTANCE_ID"]) + operation = app.config["DOCKER"].reload(request_form["INSTANCE_ID"]) elif request.form["operation"] == "start" : - app.config["DOCKER"].start(request_form["INSTANCE_ID"]) + operation = app.config["DOCKER"].start(request_form["INSTANCE_ID"]) elif request.form["operation"] == "stop" : - app.config["DOCKER"].stop(request_form["INSTANCE_ID"]) + operation = app.config["DOCKER"].stop(request_form["INSTANCE_ID"]) elif request.form["operation"] == "restart" : - app.config["DOCKER"].restart(request_form["INSTANCE_ID"]) + operation = app.config["DOCKER"].restart(request_form["INSTANCE_ID"]) elif request.form["operation"] == "delete" : - app.config["DOCKER"].remove(request_form["INSTANCE_ID"]) + operation = app.config["DOCKER"].delete(request_form["INSTANCE_ID"]) # Display instances instances = app.config["DOCKER"].get_instances() - return render_template("instances.html", title="Instances", instances=instances, operation="todo") + return render_template("instances.html", title="Instances", instances=instances, operation=operation) except Exception as e : - return render_template("error.html", title="Error", error=e) + return render_template("error.html", title="Error", error=str(e)) @app.route('/services', methods=["GET", "POST"]) def services(): + try : + # Manage services + operation = "" + if request.method == "POST" : - # Get the client - check, client = wrappers.get_client() - if not check : - return render_template("error.html", title="Error", error=client) + # Check operation + if not "operation" in request.form or not request.form["operation"] in ["new", "edit", "delete"] : + raise Exception("Missing operation parameter on /services.") - # Manage services - operation = "" - if request.method == "POST" : + # Check variables + variables = copy.deepcopy(request.form) + if not "OLD_SERVER_NAME" in request.form and request.form["operation"] == "edit" : + raise Exception("Missing OLD_SERVER_NAME parameter.") + if request.form["operation"] in ["new", "edit"] : + del variables["operation"] + if request.form["operation"] == "edit" : + del variables["OLD_SERVER_NAME"] + app.config["CONFIG"].check_variables(variables) - # Check operation - if not "operation" in request.form or not request.form["operation"] in ["new", "edit", "delete"] : - return render_template("error.html", title="Error", error="Missing operation parameter on /services.") + # Delete + elif request.form["operation"] == "delete" : + if not "SERVER_NAME" in request.form : + raise Exception("Missing SERVER_NAME parameter.") + app.config["CONFIG"].check_variables({"SERVER_NAME" : request.form["SERVER_NAME"]}) - # Check that all fields are present and they match the corresponding regex - env = {} - env["MULTISITE"] = "yes" - if request.form["operation"] in ["new", "edit"] : - for category in current_app.config["CONFIG"] : - for param in current_app.config["CONFIG"][category]["params"] : - if param["type"] == "multiple" : - for param_user in request.form : - if param_user.startswith(param["params"][0]["env"]) : - suffix = param_user.replace(param["params"][0]["env"], "") - for param_multiple in param["params"] : - if not param_multiple["env"] + suffix in request.form : - return render_template("error.html", title="Error", error="Missing " + param["env"] + " parameter.") - if not re.search(param_multiple["regex"], request.form[param_multiple["env"] + suffix]) : - return render_template("error.html", title="Error", error="Parameter " + param["env"] + " doesn't match regex.") - env[param_multiple["env"] + suffix] = request.form[param_multiple["env"] + suffix] - else : - if not param["env"] in request.form : - return render_template("error.html", title="Error", error="Missing " + param["env"] + " parameter.") - if not re.search(param["regex"], request.form[param["env"]]) : - return render_template("error.html", title="Error", error="Parameter " + param["env"] + " doesn't match regex.") - env[param["env"]] = request.form[param["env"]] - if request.form["operation"] == "edit" : - if not "OLD_SERVER_NAME" in request.form : - return render_template("error.html", title="Error", error="Missing OLD_SERVER_NAME parameter.") - if not re.search("^([a-z\-0-9]+\.?)+$", request.form["OLD_SERVER_NAME"]) : - return render_template("error.html", title="Error", error="Parameter OLD_SERVER_NAME doesn't match regex.") - elif request.form["operation"] == "delete" : - if not "SERVER_NAME" in request.form : - return render_template("error.html", title="Error", error="Missing SERVER_NAME parameter.") - if not re.search("^([a-z\-0-9]+\.?)+$", request.form["SERVER_NAME"]) : - return render_template("error.html", title="Error", error="Parameter SERVER_NAME doesn't match regex.") + # Do the operation + if request.form["operation"] == "new" : + operation = app.config["CONFIG"].new_service(variables) + elif request.form["operation"] == "edit" : + operation = app.config["CONFIG"].edit_service(request.form["OLD_SERVER_NAME"], variables) + elif request.form["operation"] == "delete" : + operation = app.config["CONFIG"].delete_service(request.form["SERVER_NAME"]) - # Do the operation - check, operation = wrappers.operation_service(client, request.form, env) - if not check : - render_template("error.html", title="Error", error=operation) + # Display services + services = app.config["CONFIG"].get_services() + return render_template("services.html", title="Services", services=services, operation=operation) - # Display services - check, services = wrappers.get_services() - if not check : - return render_template("error.html", title="Error", error=services) - return render_template("services.html", title="Services", services=services, operation=operation) + except Exception as e : + return render_template("error.html", title="Error", error=str(e)) \ No newline at end of file diff --git a/ui/templates/services-edit.html b/ui/templates/services-edit.html index c4d790b..e81bd4b 100644 --- a/ui/templates/services-edit.html +++ b/ui/templates/services-edit.html @@ -8,7 +8,7 @@