From 6e66571fb9c3cb0cedbeafefbda145d63aa0fc73 Mon Sep 17 00:00:00 2001 From: florian Date: Wed, 28 Jul 2021 23:25:05 +0200 Subject: [PATCH] various cleaning --- autoconf/src/AutoConf.py | 195 ------------------------------ autoconf/src/DockerController.py | 2 +- autoconf/src/IngressController.py | 6 +- autoconf/src/ReloadServer.py | 6 +- autoconf/src/SwarmController.py | 4 +- autoconf/src/app.py | 11 +- autoconf/src/logger.py | 6 + autoconf/src/utils.py | 24 ---- jobs/main.py | 5 +- ui/hooks/post_push | 12 -- ui/hooks/pre_build | 5 - ui/utils.py | 4 +- 12 files changed, 26 insertions(+), 254 deletions(-) delete mode 100644 autoconf/src/AutoConf.py create mode 100644 autoconf/src/logger.py delete mode 100644 autoconf/src/utils.py delete mode 100644 ui/hooks/post_push delete mode 100644 ui/hooks/pre_build diff --git a/autoconf/src/AutoConf.py b/autoconf/src/AutoConf.py deleted file mode 100644 index 5651e6b..0000000 --- a/autoconf/src/AutoConf.py +++ /dev/null @@ -1,195 +0,0 @@ -from Config import Config - -import utils - -import os, time - -class AutoConf : - - def __init__(self, swarm, api) : - self.__swarm = swarm - self.__servers = {} - self.__instances = {} - self.__env = {} - self.__config = Config(self.__swarm, api) - - def get_server(self, id) : - if id in self.__servers : - return self.__servers[id] - return False - - def reload(self) : - return self.__config.reload(self.__instances) - - def __gen_env(self) : - self.__env.clear() - # TODO : check actual state (e.g. : running, stopped ?) - for id, instance in self.__instances.items() : - env = [] - if self.__swarm : - env = instance.attrs["Spec"]["TaskTemplate"]["ContainerSpec"]["Env"] - else : - env = instance.attrs["Config"]["Env"] - for entry in env : - self.__env[entry.split("=")[0]] = entry.replace(entry.split("=")[0] + "=", "", 1) - blacklist = ["NGINX_VERSION", "NJS_VERSION", "PATH", "PKG_RELEASE"] - for entry in blacklist : - if entry in self.__env : - del self.__env[entry] - if not "SERVER_NAME" in self.__env or self.__env["SERVER_NAME"] == "" : - self.__env["SERVER_NAME"] = [] - else : - self.__env["SERVER_NAME"] = self.__env["SERVER_NAME"].split(" ") - for server in self.__servers : - (id, name, labels) = self.__get_infos(self.__servers[server]) - first_server = labels["bunkerized-nginx.SERVER_NAME"].split(" ")[0] - for label in labels : - if label.startswith("bunkerized-nginx.") : - self.__env[first_server + "_" + label.replace("bunkerized-nginx.", "", 1)] = labels[label] - for server_name in labels["bunkerized-nginx.SERVER_NAME"].split(" ") : - if not server_name in self.__env["SERVER_NAME"] : - self.__env["SERVER_NAME"].append(server_name) - self.__env["SERVER_NAME"] = " ".join(self.__env["SERVER_NAME"]) - - def pre_process(self, objs) : - for instance in objs : - (id, name, labels) = self.__get_infos(instance) - if "bunkerized-nginx.AUTOCONF" in labels : - if self.__swarm : - self.__process_instance(instance, "create", id, name, labels) - else : - if instance.status in ("restarting", "running", "created", "exited") : - self.__process_instance(instance, "create", id, name, labels) - if instance.status == "running" : - self.__process_instance(instance, "start", id, name, labels) - - for server in objs : - (id, name, labels) = self.__get_infos(server) - if "bunkerized-nginx.SERVER_NAME" in labels : - if self.__swarm : - self.__process_server(server, "create", id, name, labels) - else : - if server.status in ("restarting", "running", "created", "exited") : - self.__process_server(server, "create", id, name, labels) - if server.status == "running" : - self.__process_server(server, "start", id, name, labels) - - def process(self, obj, event) : - (id, name, labels) = self.__get_infos(obj) - if "bunkerized-nginx.AUTOCONF" in labels : - self.__process_instance(obj, event, id, name, labels) - elif "bunkerized-nginx.SERVER_NAME" in labels : - self.__process_server(obj, event, id, name, labels) - - def __get_infos(self, obj) : - if self.__swarm : - id = obj.id - name = obj.name - labels = obj.attrs["Spec"]["Labels"] - else : - id = obj.id - name = obj.name - labels = obj.labels - return (id, name, labels) - - def __process_instance(self, instance, event, id, name, labels) : - - if event == "create" : - self.__instances[id] = instance - self.__gen_env() - utils.log("[*] bunkerized-nginx instance created : " + name + " / " + id) - if self.__swarm and len(self.__instances) == 1 : - if self.__config.generate(self.__env) : - utils.log("[*] Initial config succeeded") - if not self.__config.swarm_wait(self.__instances) : - utils.log("[!] Removing bunkerized-nginx instances from list (API not available)") - del self.__instances[id] - else : - utils.log("[!] Initial config failed") - elif not self.__swarm and len(self.__instances) == 1 : - utils.log("[*] Wait until bunkerized-nginx is healthy (timeout = 120s) ...") - i = 0 - healthy = False - while i < 120 : - self.__instances[id].reload() - if self.__instances[id].attrs["State"]["Health"]["Status"] == "healthy" : - healthy = True - break - time.sleep(1) - i = i + 1 - if not healthy : - utils.log("[!] Removing bunkerized-nginx instances from list (unhealthy)") - del self.__instances[id] - - elif event == "start" : - self.__instances[id].reload() - self.__gen_env() - utils.log("[*] bunkerized-nginx instance started : " + name + " / " + id) - - elif event == "die" : - self.__instances[id].reload() - self.__gen_env() - utils.log("[*] bunkerized-nginx instance stopped : " + name + " / " + id) - - elif event == "destroy" or event == "remove" : - del self.__instances[id] - self.__gen_env() - utils.log("[*] bunkerized-nginx instance removed : " + name + " / " + id) - - def __process_server(self, instance, event, id, name, labels) : - - vars = { k.replace("bunkerized-nginx.", "", 1) : v for k, v in labels.items() if k.startswith("bunkerized-nginx.")} - - if event == "create" : - utils.log("[*] Generating config for " + vars["SERVER_NAME"] + " ...") - self.__servers[id] = instance - self.__gen_env() - if self.__config.generate(self.__env) : - utils.log("[*] Generated config for " + vars["SERVER_NAME"]) - if self.__swarm : - utils.log("[*] Activating config for " + vars["SERVER_NAME"] + " ...") - if self.__config.reload(self.__instances) : - utils.log("[*] Activated config for " + vars["SERVER_NAME"]) - else : - utils.log("[!] Can't activate config for " + vars["SERVER_NAME"]) - else : - utils.log("[!] Can't generate config for " + vars["SERVER_NAME"]) - del self.__servers[id] - self.__gen_env() - self.__config.generate(self.__env) - - elif event == "start" : - if id in self.__servers : - self.__servers[id].reload() - utils.log("[*] Activating config for " + vars["SERVER_NAME"] + " ...") - self.__gen_env() - if self.__config.reload(self.__instances) : - utils.log("[*] Activated config for " + vars["SERVER_NAME"]) - else : - utils.log("[!] Can't activate config for " + vars["SERVER_NAME"]) - - elif event == "die" : - if id in self.__servers : - self.__servers[id].reload() - utils.log("[*] Deactivating config for " + vars["SERVER_NAME"]) - self.__gen_env() - if self.__config.reload(self.__instances) : - utils.log("[*] Deactivated config for " + vars["SERVER_NAME"]) - else : - utils.log("[!] Can't deactivate config for " + vars["SERVER_NAME"]) - - elif event == "destroy" or event == "remove" : - if id in self.__servers : - utils.log("[*] Removing config for " + vars["SERVER_NAME"]) - del self.__servers[id] - self.__gen_env() - if self.__config.generate(self.__env) : - utils.log("[*] Removed config for " + vars["SERVER_NAME"]) - else : - utils.log("[!] Can't remove config for " + vars["SERVER_NAME"]) - utils.log("[*] Deactivating config for " + vars["SERVER_NAME"]) - if self.__config.reload(self.__instances) : - utils.log("[*] Deactivated config for " + vars["SERVER_NAME"]) - else : - utils.log("[!] Can't deactivate config for " + vars["SERVER_NAME"]) - diff --git a/autoconf/src/DockerController.py b/autoconf/src/DockerController.py index fc2bf3f..aebdc02 100644 --- a/autoconf/src/DockerController.py +++ b/autoconf/src/DockerController.py @@ -39,4 +39,4 @@ class DockerController(Controller) : if new_env != old_env : if self.gen_conf(new_env) : old_env.copy(new_env) - utils.log("[*] Successfully generated new configuration") + log("CONTROLLER", "INFO", "successfully generated new configuration") diff --git a/autoconf/src/IngressController.py b/autoconf/src/IngressController.py index 27d4f2a..58afd46 100644 --- a/autoconf/src/IngressController.py +++ b/autoconf/src/IngressController.py @@ -3,6 +3,8 @@ from threading import Thread from Controller import Controller, ControllerType +from logger import log + class IngressController : def __init__(self, api_uri) : @@ -78,7 +80,7 @@ class IngressController : if new_env != self.__old_env() : if self.gen_conf(new_env, lock=False) : self.__old_env.copy(new_env) - utils.log("[*] Successfully generated new configuration") + log("CONTROLLER", "INFO", "successfully generated new configuration") self.lock.release() def __watch_service(self) : @@ -89,5 +91,5 @@ class IngressController : if new_env != self.__old_env() : if self.gen_conf(new_env, lock=False) : self.__old_env.copy(new_env) - utils.log("[*] Successfully generated new configuration") + log("CONTROLLER", "INFO", "successfully generated new configuration") self.lock.release() diff --git a/autoconf/src/ReloadServer.py b/autoconf/src/ReloadServer.py index 1003f71..e33fbfc 100644 --- a/autoconf/src/ReloadServer.py +++ b/autoconf/src/ReloadServer.py @@ -1,4 +1,6 @@ -import socketserver, threading, utils, os, stat +import socketserver, threading, os, stat + +from logger import log class ReloadServerHandler(socketserver.StreamRequestHandler): @@ -25,7 +27,7 @@ class ReloadServerHandler(socketserver.StreamRequestHandler): else : self.request.sendall(b"ko") except Exception as e : - utils.log("Exception ReloadServer : " + str(e)) + log("RELOADSERVER", "ERROR", "exception : " + str(e)) if locked : self.server.controller.lock.release() diff --git a/autoconf/src/SwarmController.py b/autoconf/src/SwarmController.py index 1a1e557..50271ac 100644 --- a/autoconf/src/SwarmController.py +++ b/autoconf/src/SwarmController.py @@ -38,7 +38,7 @@ class SwarmController(Controller) : new_env = self.get_env() if new_env != old_env : self.lock.acquire() - if self.gen_conf(new_env) : + if self.gen_conf(new_env, lock=False) : old_env.copy(new_env) - utils.log("[*] Successfully generated new configuration") + log("CONTROLLER", "INFO", "successfully generated new configuration") self.lock.release() diff --git a/autoconf/src/app.py b/autoconf/src/app.py index fcab991..3faf990 100644 --- a/autoconf/src/app.py +++ b/autoconf/src/app.py @@ -1,14 +1,15 @@ #!/usr/bin/python3 -from AutoConf import AutoConf from ReloadServer import run_reload_server -import utils + import docker, os, stat, sys, select, threading from DockerController import DockerController from SwarmController import SwarmController from KubernetesController import KubernetesController +from logger import log + # Get variables swarm = os.getenv("SWARM_MODE", "no") == "yes" kubernetes = os.getenv("KUBERNETES_MODE", "no") == "yes" @@ -16,13 +17,13 @@ api_uri = os.getenv("API_URI", "") # Instantiate the controller if swarm : - utils.log("[*] Swarm mode detected") + log("autoconf", "INFO", "swarm mode detected") controller = SwarmController(api_uri) elif kubernetes : - utils.log("[*] Kubernetes mode detected") + log("autoconf", "INFO", "kubernetes mode detected") controller = KubernetesController(api_uri) else : - utils.log("[*] Docker mode detected") + log("autoconf", "INFO", "docker mode detected") controller = DockerController() # Run the reload server in background if needed diff --git a/autoconf/src/logger.py b/autoconf/src/logger.py new file mode 100644 index 0000000..47deb8b --- /dev/null +++ b/autoconf/src/logger.py @@ -0,0 +1,6 @@ +import datetime + +def log(title, severity, msg) : + when = datetime.datetime.today().strftime("[%Y-%m-%d %H:%M:%S]") + what = title + " - " + severity + " - " + msg + print(when + " " + what, flush=True) diff --git a/autoconf/src/utils.py b/autoconf/src/utils.py deleted file mode 100644 index 10b1071..0000000 --- a/autoconf/src/utils.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/python3 - -import datetime - -def log(event) : - print("[" + str(datetime.datetime.now().replace(microsecond=0)) + "] " + event, flush=True) - -def replace_in_file(file, old_str, new_str) : - with open(file) as f : - data = f.read() - data = data[::-1].replace(old_str[::-1], new_str[::-1], 1)[::-1] - with open(file, "w") as f : - f.write(data) - -def install_cron(service, vars, crons) : - for var in vars : - if var in crons : - with open("/etc/crontabs/root", "a+") as f : - f.write(vars[var] + " /opt/cron/" + crons[var] + ".py " + service["Actor"]["ID"]) - -def uninstall_cron(service, vars, crons) : - for var in vars : - if var in crons : - replace_in_file("/etc/crontabs/root", vars[var] + " /opt/cron/" + crons[var] + ".py " + service["Actor"]["ID"] + "\n", "") diff --git a/jobs/main.py b/jobs/main.py index 36e27db..2d5ae4d 100644 --- a/jobs/main.py +++ b/jobs/main.py @@ -60,10 +60,7 @@ if __name__ == "__main__" : ret = instance.run() if ret == JobRet.KO : log("job", "ERROR", "error while running job " + job) - if reload_socket != None : - reload_socket.sendall(b"unlock") - reload_socket.recv(512) - reload_socket.close() + management.unlock() sys.exit(1) log("job", "INFO", "job " + job + " successfully executed") diff --git a/ui/hooks/post_push b/ui/hooks/post_push deleted file mode 100644 index 23d13c5..0000000 --- a/ui/hooks/post_push +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -curl -Lo manifest-tool https://github.com/estesp/manifest-tool/releases/download/v1.0.3/manifest-tool-linux-amd64 -chmod +x manifest-tool - -VERSION=$(cat VERSION | tr -d '\n') -if [ "$SOURCE_BRANCH" = "dev" ] ; then - ./manifest-tool push from-args --ignore-missing --platforms linux/amd64,linux/386,linux/arm/v7,linux/arm64/v8 --template bunkerity/bunkerized-nginx-ui:dev-ARCHVARIANT --target bunkerity/bunkerized-nginx-ui:dev -elif [ "$SOURCE_BRANCH" = "master" ] ; then - ./manifest-tool push from-args --ignore-missing --platforms linux/amd64,linux/386,linux/arm/v7,linux/arm64/v8 --template bunkerity/bunkerized-nginx-ui:ARCHVARIANT --target bunkerity/bunkerized-nginx-ui:${VERSION} - ./manifest-tool push from-args --ignore-missing --platforms linux/amd64,linux/386,linux/arm/v7,linux/arm64/v8 --template bunkerity/bunkerized-nginx-ui:ARCHVARIANT --target bunkerity/bunkerized-nginx-ui:latest -fi diff --git a/ui/hooks/pre_build b/ui/hooks/pre_build deleted file mode 100644 index acf05f1..0000000 --- a/ui/hooks/pre_build +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -# Register qemu-*-static for all supported processors except the -# current one, but also remove all registered binfmt_misc before -docker run --rm --privileged multiarch/qemu-user-static:register --reset diff --git a/ui/utils.py b/ui/utils.py index 8c778c0..15752d7 100644 --- a/ui/utils.py +++ b/ui/utils.py @@ -4,8 +4,8 @@ import datetime, re, json, os def get_variables() : vars = {} - vars["DOCKER_HOST"] = "unix:///var/run/docker.sock" - vars["API_URI"] = "" + vars["DOCKER_HOST"] = "unix:///var/run/docker.sock" + vars["API_URI"] = "" vars["ABSOLUTE_URI"] = "" vars["FLASK_SECRET"] = os.urandom(32) vars["ADMIN_USERNAME"] = "admin"