From c9a6b6c27df42ee929bf2a236c1a90f9c2feb518 Mon Sep 17 00:00:00 2001 From: bunkerity Date: Fri, 13 Aug 2021 10:02:14 +0200 Subject: [PATCH] autoconf - fixed infinite lock --- autoconf/src/Config.py | 24 +------- autoconf/src/Controller.py | 14 ++++- autoconf/src/DockerController.py | 47 +++++++++------ autoconf/src/IngressController.py | 98 ++++++++++++++++++------------- autoconf/src/ReloadServer.py | 2 - autoconf/src/SwarmController.py | 49 ++++++++++------ docs/integrations.md | 2 +- 7 files changed, 127 insertions(+), 109 deletions(-) diff --git a/autoconf/src/Config.py b/autoconf/src/Config.py index 14c72ef..823ab3b 100644 --- a/autoconf/src/Config.py +++ b/autoconf/src/Config.py @@ -8,10 +8,9 @@ from logger import log class Config : - def __init__(self, type, api_uri, lock=None, http_port="8080") : + def __init__(self, type, api_uri, http_port="8080") : self.__type = type self.__api_uri = api_uri - self.__lock = lock self.__http_port = http_port def __jobs(self) : @@ -29,14 +28,7 @@ class Config : return True def gen(self, env) : - locked = False try : - # Lock - if self.__lock : - log("config", "ERROR", "lock") - self.__lock.acquire() - locked = True - # Write environment variables to a file with open("/tmp/variables.env", "w") as f : for k, v in env.items() : @@ -45,12 +37,6 @@ class Config : # Call the generator proc = subprocess.run(["/bin/su", "-c", "/opt/bunkerized-nginx/gen/main.py --settings /opt/bunkerized-nginx/settings.json --templates /opt/bunkerized-nginx/confs --output /etc/nginx --variables /tmp/variables.env", "nginx"], capture_output=True) - # Unlock - if self.__lock : - log("config", "ERROR", "release") - self.__lock.release() - locked = False - # Print stdout/stderr stdout = proc.stdout.decode("ascii") stderr = proc.stderr.decode("ascii") @@ -68,8 +54,6 @@ class Config : except Exception as e : log("config", "ERROR", "exception while generating site config : " + traceback.format_exc()) - if locked : - self.__lock.release() return False def reload(self, instances) : @@ -137,9 +121,6 @@ class Config : return False def __api_call(self, instances, path) : - if self.__lock : - log("config", "ERROR", "lock") - self.__lock.acquire() ret = True nb = 0 urls = [] @@ -174,7 +155,4 @@ class Config : else : log("config", "INFO", "failed API order to " + url) ret = False - if self.__lock : - log("config", "ERROR", "release") - self.__lock.release() return ret and nb > 0 diff --git a/autoconf/src/Controller.py b/autoconf/src/Controller.py index ed33b43..7980a08 100644 --- a/autoconf/src/Controller.py +++ b/autoconf/src/Controller.py @@ -11,7 +11,7 @@ class Type(Enum) : class Controller(ABC) : def __init__(self, type, api_uri=None, lock=None, http_port="8080") : - self._config = Config(type, api_uri, lock=lock, http_port=http_port) + self._config = Config(type, api_uri, http_port=http_port) self.lock = lock @abstractmethod @@ -27,7 +27,11 @@ class Controller(ABC) : return fixed_env def gen_conf(self, env) : - return self._config.gen(env) + try : + ret = self._config.gen(env) + except : + ret = False + return ret @abstractmethod def wait(self) : @@ -42,4 +46,8 @@ class Controller(ABC) : pass def _reload(self, instances) : - return self._config.reload(instances) + try : + ret = self._config.reload(instances) + except : + ret = False + return ret diff --git a/autoconf/src/DockerController.py b/autoconf/src/DockerController.py index 1cb54ac..e200d38 100644 --- a/autoconf/src/DockerController.py +++ b/autoconf/src/DockerController.py @@ -42,30 +42,39 @@ class DockerController(Controller.Controller) : for event in self.__client.events(decode=True, filters={"type": "container"}) : new_env = self.get_env() if new_env != old_env : - log("controller", "INFO", "generating new configuration") - if self.gen_conf(new_env) : - old_env = new_env.copy() - log("controller", "INFO", "successfully generated new configuration") - if self.reload() : - log("controller", "INFO", "successful reload") + try : + log("controller", "INFO", "generating new configuration") + if self.gen_conf(new_env) : + old_env = new_env.copy() + log("controller", "INFO", "successfully generated new configuration") + if self.reload() : + log("controller", "INFO", "successful reload") + else : + log("controller", "ERROR", "failed reload") else : - log("controller", "ERROR", "failed reload") - else : - log("controller", "ERROR", "can't generate new configuration") + log("controller", "ERROR", "can't generate new configuration") + except : + log("controller", "ERROR", "exception while receiving event") def reload(self) : return self._reload(self.__get_instances()) def wait(self) : - # Wait for a container - instances = self.__get_instances() - while len(instances) == 0 : - time.sleep(1) + try : + # Wait for a container instances = self.__get_instances() - # Generate first config - env = self.get_env() - if not self.gen_conf(env) : - return False, env - # Wait for nginx - return self._config.wait(instances), env + while len(instances) == 0 : + time.sleep(1) + instances = self.__get_instances() + # Generate first config + env = self.get_env() + if not self.gen_conf(env) : + self.lock.release() + return False, env + # Wait for nginx + self.lock.release() + return self._config.wait(instances), env + except : + pass + return False, {} diff --git a/autoconf/src/IngressController.py b/autoconf/src/IngressController.py index a8e02a5..6907d3d 100644 --- a/autoconf/src/IngressController.py +++ b/autoconf/src/IngressController.py @@ -14,7 +14,6 @@ class IngressController(Controller.Controller) : self.__api = client.CoreV1Api() self.__extensions_api = client.ExtensionsV1beta1Api() self.__old_env = {} - self.__internal_lock = Lock() def __get_pods(self) : return self.__api.list_pod_for_all_namespaces(watch=False, label_selector="bunkerized-nginx").items @@ -110,68 +109,83 @@ class IngressController(Controller.Controller) : def __watch_pod(self) : w = watch.Watch() for event in w.stream(self.__api.list_pod_for_all_namespaces, label_selector="bunkerized-nginx") : - self.__internal_lock.acquire() + self.lock.acquire() new_env = self.get_env() if new_env != self.__old_env : - if self.gen_conf(new_env) : - self.__old_env = new_env.copy() - log("CONTROLLER", "INFO", "successfully generated new configuration") - if self.reload() : - log("controller", "INFO", "successful reload") - else : - log("controller", "ERROR", "failed reload") - self.__internal_lock.release() + try : + if self.gen_conf(new_env) : + self.__old_env = new_env.copy() + log("CONTROLLER", "INFO", "successfully generated new configuration") + if self.reload() : + log("controller", "INFO", "successful reload") + else : + log("controller", "ERROR", "failed reload") + except : + log("controller", "ERROR", "exception while receiving event") + self.lock.release() def __watch_ingress(self) : w = watch.Watch() for event in w.stream(self.__extensions_api.list_ingress_for_all_namespaces, label_selector="bunkerized-nginx") : - self.__internal_lock.acquire() + self.lock.acquire() new_env = self.get_env() if new_env != self.__old_env : - if self.gen_conf(new_env) : - self.__old_env = new_env.copy() - log("CONTROLLER", "INFO", "successfully generated new configuration") - if self.reload() : - log("controller", "INFO", "successful reload") - else : - log("controller", "ERROR", "failed reload") - self.__internal_lock.release() + try : + if self.gen_conf(new_env) : + self.__old_env = new_env.copy() + log("CONTROLLER", "INFO", "successfully generated new configuration") + if self.reload() : + log("controller", "INFO", "successful reload") + else : + log("controller", "ERROR", "failed reload") + except : + log("controller", "ERROR", "exception while receiving event") + self.lock.release() def __watch_service(self) : w = watch.Watch() for event in w.stream(self.__api.list_service_for_all_namespaces, label_selector="bunkerized-nginx") : - self.__internal_lock.acquire() + self.lock.acquire() new_env = self.get_env() if new_env != self.__old_env : - if self.gen_conf(new_env) : - self.__old_env = new_env.copy() - log("CONTROLLER", "INFO", "successfully generated new configuration") - if self.reload() : - log("controller", "INFO", "successful reload") - else : - log("controller", "ERROR", "failed reload") - self.__internal_lock.release() + try : + if self.gen_conf(new_env) : + self.__old_env = new_env.copy() + log("CONTROLLER", "INFO", "successfully generated new configuration") + if self.reload() : + log("controller", "INFO", "successful reload") + else : + log("controller", "ERROR", "failed reload") + except : + log("controller", "ERROR", "exception while receiving event") + self.lock.release() def reload(self) : return self._reload(self.__get_services(autoconf=True)) def wait(self) : - # Wait for at least one bunkerized-nginx pod - pods = self.__get_pods() - while len(pods) == 0 : - time.sleep(1) + self.lock.acquire() + try : + # Wait for at least one bunkerized-nginx pod pods = self.__get_pods() + while len(pods) == 0 : + time.sleep(1) + pods = self.__get_pods() - # Wait for at least one bunkerized-nginx service - services = self.__get_services(autoconf=True) - while len(services) == 0 : - time.sleep(1) + # Wait for at least one bunkerized-nginx service services = self.__get_services(autoconf=True) + while len(services) == 0 : + time.sleep(1) + services = self.__get_services(autoconf=True) - # Generate first config - env = self.get_env() - if not self.gen_conf(env) : - return False, env + # Generate first config + env = self.get_env() + if not self.gen_conf(env) : + return False, env - # Wait for bunkerized-nginx - return self._config.wait(services), env + # Wait for bunkerized-nginx + return self._config.wait(services), env + except : + pass + self.lock.release() + return False, {} diff --git a/autoconf/src/ReloadServer.py b/autoconf/src/ReloadServer.py index 3eb5a9f..2415518 100644 --- a/autoconf/src/ReloadServer.py +++ b/autoconf/src/ReloadServer.py @@ -13,12 +13,10 @@ class ReloadServerHandler(socketserver.StreamRequestHandler): if not data or not data in [b"lock", b"reload", b"unlock"] : break if data == b"lock" : - log("RELOADSERVER", "ERROR", "lock") self.server.controller.lock.acquire() locked = True self.request.sendall(b"ok") elif data == b"unlock" : - log("RELOADSERVER", "ERROR", "unlock") self.server.controller.lock.release() locked = False self.request.sendall(b"ok") diff --git a/autoconf/src/SwarmController.py b/autoconf/src/SwarmController.py index 61a2923..171e0c2 100644 --- a/autoconf/src/SwarmController.py +++ b/autoconf/src/SwarmController.py @@ -44,29 +44,40 @@ class SwarmController(Controller.Controller) : for event in self.__client.events(decode=True, filters={"type": "service"}) : new_env = self.get_env() if new_env != old_env : - log("controller", "INFO", "generating new configuration") - if self.gen_conf(new_env) : - old_env = new_env.copy() - log("controller", "INFO", "successfully generated new configuration") - if self.reload() : - log("controller", "INFO", "successful reload") + self.lock.acquire() + try : + log("controller", "INFO", "generating new configuration") + if self.gen_conf(new_env) : + old_env = new_env.copy() + log("controller", "INFO", "successfully generated new configuration") + if self.reload() : + log("controller", "INFO", "successful reload") + else : + log("controller", "ERROR", "failed reload") else : - log("controller", "ERROR", "failed reload") - else : - log("controller", "ERROR", "can't generate new configuration") + log("controller", "ERROR", "can't generate new configuration") + except : + log("controller", "ERROR", "exception while receiving event") + self.lock.release() def reload(self) : return self._reload(self.__get_instances()) def wait(self) : - # Wait for a service - instances = self.__get_instances() - while len(instances) == 0 : - time.sleep(1) + self.lock.acquire() + try : + # Wait for a service instances = self.__get_instances() - # Generate first config - env = self.get_env() - if not self.gen_conf(env) : - return False, env - # Wait for nginx - return self._config.wait(instances), env + while len(instances) == 0 : + time.sleep(1) + instances = self.__get_instances() + # Generate first config + env = self.get_env() + if not self.gen_conf(env) : + return False, env + # Wait for nginx + return self._config.wait(instances), env + except : + pass + self.lock.release() + return False, {} diff --git a/docs/integrations.md b/docs/integrations.md index 4424f8c..9cb95de 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -319,7 +319,7 @@ $ docker service create \ Or do the same with docker-compose if you wish : ```yaml -version: '3' +version: '3.8' services: