swarm/k8s - less storage, more API

.github/workflows/build-bunkerized-nginx-autoconf.yml

@@ -20,7 +20,6 @@ jobs:
       - name: Run Trivy security scanner
         uses: aquasecurity/trivy-action@master
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}
           image-ref: 'bunkerized-nginx-autoconf'
           format: 'table'
           exit-code: '1'

.github/workflows/build-bunkerized-nginx-ui.yml

@@ -20,7 +20,6 @@ jobs:
       - name: Run Trivy security scanner
         uses: aquasecurity/trivy-action@master
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}
           image-ref: 'bunkerized-nginx-ui'
           format: 'table'
           exit-code: '1'

.github/workflows/build-bunkerized-nginx.yml

@@ -20,7 +20,6 @@ jobs:
       - name: Run Trivy security scanner
         uses: aquasecurity/trivy-action@master
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}
           image-ref: 'bunkerized-nginx'
           format: 'table'
           exit-code: '1'

autoconf/Dockerfile

@@ -11,7 +11,7 @@ COPY autoconf/entrypoint.sh /opt/bunkerized-nginx/entrypoint/
 COPY autoconf/requirements.txt /opt/bunkerized-nginx/entrypoint/
 COPY autoconf/src/* /opt/bunkerized-nginx/entrypoint/
 
-RUN apk add --no-cache py3-pip bash certbot curl openssl && \
+RUN apk add --no-cache py3-pip bash certbot curl openssl socat && \
     pip3 install -r /opt/bunkerized-nginx/gen/requirements.txt && \
     pip3 install -r /opt/bunkerized-nginx/entrypoint/requirements.txt && \
     pip3 install -r /opt/bunkerized-nginx/jobs/requirements.txt
@@ -24,4 +24,6 @@ RUN chmod +x /tmp/prepare.sh && \
 # Fix CVE-2021-36159
 RUN apk add "apk-tools>=2.12.6-r0"
 
+#VOLUME /http-confs /server-confs /modsec-confs /modsec-crs-confs /cache /etc/letsencrypt /acme-challenge
+
 ENTRYPOINT ["/opt/bunkerized-nginx/entrypoint/entrypoint.sh"]

autoconf/prepare.sh

@@ -29,6 +29,11 @@ mkdir /var/log/letsencrypt
 chown nginx:nginx /var/log/letsencrypt
 chmod 770 /var/log/letsencrypt
 
+# prepare /etc/nginx
+mkdir /etc/nginx
+chown root:nginx /etc/nginx
+chmod 770 /etc/nginx
+
 # prepare /etc/letsencrypt
 mkdir /etc/letsencrypt
 chown root:nginx /etc/letsencrypt
@@ -51,6 +56,18 @@ mkdir /acme-challenge
 chown root:nginx /acme-challenge
 chmod 770 /acme-challenge
 
+# prepare /http-confs
+ln -s /http-confs /opt/bunkerized-nginx/http-confs
+mkdir /http-confs
+chown root:nginx /http-confs
+chmod 770 /http-confs
+
+# prepare /server-confs
+ln -s /server-confs /opt/bunkerized-nginx/server-confs
+mkdir /server-confs
+chown root:nginx /server-confs
+chmod 770 /server-confs
+
 # prepare /modsec-confs
 ln -s /modsec-confs /opt/bunkerized-nginx/modsec-confs
 mkdir /modsec-confs

autoconf/src/Config.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python3
 
-import subprocess, shutil, os, traceback, requests, time, dns.resolver
+import subprocess, shutil, os, traceback, requests, time, dns.resolver, io, tarfile
 
 import Controller
 
@@ -79,9 +79,6 @@ class Config :
 
 	def send(self, instances) :
 		ret = True
-		if self.__type == Controller.Type.DOCKER :
-			return ret
-		elif self.__type == Controller.Type.SWARM or self.__type == Controller.Type.KUBERNERTES :
 		fail = False
 		for name, path in CONFIGS.items() :
 			file = self.__tarball(path)
@@ -93,10 +90,14 @@ class Config :
 			ret = False
 		return ret
 
-	def __tarball(path) :
+	def stop_temp(self, instances) :
+		return self.__api_call(instances, "/stop-temp")
+
+	def __tarball(self, path) :
 		file = io.BytesIO()
 		with tarfile.open(mode="w:gz", fileobj=file) as tar :
 			tar.add(path, arcname=".")
+		file.seek(0, 0)
 		return file
 
 	def __ping(self, instances) :
@@ -178,7 +179,8 @@ class Config :
 				if file == None :
 					req = requests.post(url)
 				else :
-					req = requests.post(url, {'file': file})
+					file.seek(0, 0)
+					req = requests.post(url, files={'file': file})
 			except :
 				pass
 			if req and req.status_code == 200 and req.text == "ok" :

autoconf/src/Controller.py

@@ -1,3 +1,4 @@
+import traceback
 from abc import ABC, abstractmethod
 from enum import Enum
 
@@ -55,6 +56,13 @@ class Controller(ABC) :
 	def _send(self, instances) :
 		try :
 			ret = self._config.send(instances)
-		except :
+		except Exception as e :
+			ret = False
+		return ret
+
+	def _stop_temp(self, instances) :
+		try :
+			ret = self._config.stop_temp(instances)
+		except Exception as e :
 			ret = False
 		return ret

autoconf/src/IngressController.py

@@ -138,6 +138,9 @@ class IngressController(Controller.Controller) :
 	def send(self) :
 		return self._send(self.__get_services(autoconf=True))
 
+	def stop_temp(self) :
+		return self._stop_temp(self.__get_services(autoconf=True))
+
 	def wait(self) :
 		self.lock.acquire()
 		try :
@@ -146,20 +149,28 @@ class IngressController(Controller.Controller) :
 			while len(pods) == 0 :
 				time.sleep(1)
 				pods = self.__get_pods()
-
 			# Wait for at least one bunkerized-nginx service
 			services = self.__get_services(autoconf=True)
 			while len(services) == 0 :
 				time.sleep(1)
 				services = self.__get_services(autoconf=True)
-
 			# Generate first config
 			env = self.get_env()
 			if not self.gen_conf(env) :
 				self.lock.release()
 				return False, env
-
+			# Send the config
+			if not self.send() :
+				self.lock.release()
+				return False, env
+			# Stop the temporary server
+			if not self.stop_temp() :
+				self.lock.release()
+				return False, env
 			# Wait for bunkerized-nginx
+			if not self._config.wait(instances) :
+				self.lock.release()
+				return False, env
 			self.lock.release()
 			return self._config.wait(services), env
 		except :

autoconf/src/ReloadServer.py

@@ -20,6 +20,12 @@ class ReloadServerHandler(socketserver.StreamRequestHandler):
 					self.server.controller.lock.release()
 					locked = False
 					self.request.sendall(b"ok")
+				elif data == b"acme" :
+					ret = self.server.controller.send()
+					if ret :
+						self.request.sendall(b"ok")
+					else :
+						self.request.sendall(b"ko")
 				elif data == b"reload" :
 					ret = self.server.controller.reload()
 					if ret :

autoconf/src/SwarmController.py

@@ -64,6 +64,9 @@ class SwarmController(Controller.Controller) :
 	def send(self) :
 		return self._send(self.__get_instances())
 
+	def stop_temp(self) :
+		return self._stop_temp(self.__get_instances())
+
 	def wait(self) :
 		self.lock.acquire()
 		try :
@@ -72,14 +75,29 @@ class SwarmController(Controller.Controller) :
 			while len(instances) == 0 :
 				time.sleep(1)
 				instances = self.__get_instances()
+			# Wait for temporary bunkerized-nginx
+			if not self._config.wait(instances) :
+				self.lock.release()
+				return False, env
 			# Generate first config
 			env = self.get_env()
 			if not self.gen_conf(env) :
 				self.lock.release()
 				return False, env
-			# Wait for nginx
+			# Send the config
+			if not self.send() :
 				self.lock.release()
-			return self._config.wait(instances), env
+				return False, env
+			# Stop the temporary server
+			if not self.stop_temp() :
+				self.lock.release()
+				return False, env
+			# Wait for bunkerized-nginx
+			if not self._config.wait(instances) :
+				self.lock.release()
+				return False, env
+			self.lock.release()
+			return True, env
 		except :
 			pass
 		self.lock.release()

confs/global/api-temp.conf

@@ -1,6 +1,4 @@
-location ~ ^%API_URI%/ping {
+client_max_body_size 1G;
-	return 444;
-}
 
 location ~ %API_URI% {
 
@@ -15,10 +13,10 @@ rewrite_by_lua_block {
 		ngx.header.content_type = 'text/plain'
 		if api.do_api_call(api_uri) then
 			logger.log(ngx.NOTICE, "API", "API call " .. ngx.var.request_uri .. " successfull from " .. ngx.var.remote_addr)
-			ngx.say("ok")
+			ngx.print("ok")
 		else
 			logger.log(ngx.WARN, "API", "API call " .. ngx.var.request_uri .. " failed from " .. ngx.var.remote_addr)
-			ngx.say("ko")
+			ngx.print("ko")
 		end
 
 		ngx.exit(ngx.HTTP_OK)
@@ -29,3 +27,4 @@ rewrite_by_lua_block {
 }
 
 }
+

confs/global/api.conf

@@ -1,4 +1,5 @@
 # todo : if api_uri == "random"
+client_max_body_size 1G;
 rewrite_by_lua_block {
 	
 	local api		= require "api"

confs/global/nginx-temp.conf

@@ -1,6 +1,6 @@
 load_module /usr/lib/nginx/modules/ngx_http_lua_module.so;
 
-daemon on;
+#daemon on;
 
 pid /tmp/nginx-temp.pid;
 

entrypoint/entrypoint.sh

@@ -49,7 +49,7 @@ if [ ! -f "/etc/nginx/global.env" ] ; then
 		exit 1
 	fi
 
-	# start temp nginx to solve Let's Encrypt challenges if needed
+	# start temp nginx to solve Let's Encrypt challenges if needed and serve API
 	/opt/bunkerized-nginx/entrypoint/nginx-temp.sh
 
 	# only do config if we are not in swarm/kubernetes mode
@@ -75,15 +75,16 @@ else
 fi
 
 # start crond
-crond
+if [ "$SWARM_MODE" != "yes" ] && [ "$KUBERNETES_MODE" != "yes" ] ; then
-
+	crond
-# wait until config has been generated if we are in swarm mode
-if [ "$SWARM_MODE" = "yes" ] || [ "$KUBERNETES_MODE" = "yes" ] ; then
-	log "entrypoint" "INFO" "waiting until config has been generated ..."
-	while [ ! -f "/etc/nginx/autoconf" ] ; do
-		sleep 1
-	done
 fi
+# wait until config has been generated if we are in swarm mode
+#if [ "$SWARM_MODE" = "yes" ] || [ "$KUBERNETES_MODE" = "yes" ] ; then
+#	log "entrypoint" "INFO" "waiting until config has been generated ..."
+#	while [ ! -f "/etc/nginx/autoconf" ] ; do
+#		sleep 1
+#	done
+#fi
 
 # stop temp config if needed
 if [ -f "/tmp/nginx-temp.pid" ] ; then

entrypoint/nginx-temp.sh

@@ -7,7 +7,7 @@
 if [ "$(has_value AUTO_LETS_ENCRYPT yes)" != "" ] || [ "$SWARM_MODE" = "yes" ] || [ "$AUTO_LETS_ENCRYPT" = "yes" ] || [ "$KUBERNETES_MODE" = "yes" ] ; then
 	cp /opt/bunkerized-nginx/confs/global/nginx-temp.conf /tmp/nginx-temp.conf
 	cp /opt/bunkerized-nginx/confs/global/api-temp.conf /tmp/api.conf
-	if [ "$SWARM_MODE" = "yes" ] ; then
+	if [ "$SWARM_MODE" = "yes" ] || [ "$KUBERNETES_MODE" = "yes" ] ; then
 		replace_in_file "/tmp/nginx-temp.conf" "%USE_API%" "include /tmp/api.conf;"
 		replace_in_file "/tmp/api.conf" "%API_URI%" "$API_URI"
 		API_WHITELIST_IP="${API_WHITELIST_IP-192.168.0.0/16 172.16.0.0/12 10.0.0.0/8}"
@@ -18,10 +18,15 @@ if [ "$(has_value AUTO_LETS_ENCRYPT yes)" != "" ] || [ "$SWARM_MODE" = "yes" ] |
 	fi
 	HTTP_PORT="${HTTP_PORT-8080}"
 	replace_in_file "/tmp/nginx-temp.conf" "%HTTP_PORT%" "$HTTP_PORT"
-	nginx -c /tmp/nginx-temp.conf
+	if [ "$SWARM_MODE" = "yes" ] || [ "$KUBERNETES_MODE" = "yes" ] ; then
-	if [ "$?" -eq 0 ] ; then
+		log "nginx-temp" "INFO" "start temporary nginx server and wait for autoconf events..."
-		echo "[*] Successfully started temp nginx"
+		nginx -c /tmp/nginx-temp.conf -g 'daemon off;'
 	else
-		echo "[!] Can't start temp nginx"
+		nginx -c /tmp/nginx-temp.conf -g 'daemon on;'
+		if [ "$?" -eq 0 ] ; then
+			log "nginx-temp" "INFO" "successfully started temp nginx"
+		else
+			log "nginx-temp" "ERROR" "can't start temp nginx"
+		fi
 	fi
 fi

helpers/install.sh

@@ -610,6 +610,12 @@ git_secure_clone https://github.com/openresty/lua-resty-redis.git 91585affcd9a8d
 echo "[*] Install lua-resty-redis"
 CHANGE_DIR="/tmp/bunkerized-nginx/lua-resty-redis" do_and_check_cmd make PREFIX=/opt/bunkerized-nginx/deps LUA_LIB_DIR=/opt/bunkerized-nginx/deps/lib/lua install
 
+# Download and install lua-resty-upload
+echo "[*] Clone openresty/lua-resty-upload"
+git_secure_clone https://github.com/openresty/lua-resty-upload.git 7baca92c7e741979ae5857989bbf6cc0402d6126
+echo "[*] Install lua-resty-upload"
+CHANGE_DIR="/tmp/bunkerized-nginx/lua-resty-upload" do_and_check_cmd make PREFIX=/opt/bunkerized-nginx/deps LUA_LIB_DIR=/opt/bunkerized-nginx/deps/lib/lua install
+
 # Download nginx and decompress sources
 echo "[*] Download nginx-${NGINX_VERSION}.tar.gz"
 do_and_check_cmd wget -O "/tmp/bunkerized-nginx/nginx-${NGINX_VERSION}.tar.gz" "https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz"

jobs/certbot-auth.sh

@@ -1,3 +1,9 @@
 #!/bin/bash
 
+. /opt/bunkerized-nginx/entrypoint/utils.sh
+
 echo $CERTBOT_VALIDATION > /opt/bunkerized-nginx/acme-challenge/.well-known/acme-challenge/$CERTBOT_TOKEN
+
+if [ -S "/tmp/autoconf.sock" ] ; then
+	echo -e "lock\nacme\nunlock" | socat UNIX-CONNECT:/tmp/autoconf.sock -
+fi

lua/api.lua

@@ -2,6 +2,7 @@ local M		= {}
 local api_list	= {}
 local iputils	= require "resty.iputils"
 local upload	= require "resty.upload"
+local logger	= require "logger"
 
 api_list["^/ping$"] = function ()
 	return true
@@ -27,6 +28,10 @@ api_list["^/stop$"] = function ()
 	return os.execute("/usr/sbin/nginx -s quit") == 0
 end
 
+api_list["^/stop%-temp$"] = function ()
+	return os.execute("/usr/sbin/nginx -c /tmp/nginx-temp.conf -s stop") == 0
+end
+
 api_list["^/conf$"] = function ()
 	if not M.save_file("/tmp/conf.tar.gz") then
 		return false
@@ -69,7 +74,7 @@ api_list["^/modsec$"] = function ()
 	return M.extract_file("/tmp/modsec.tar.gz", "/modsec-confs/")
 end
 
-api_list["^/modsec-crs$"] = function ()
+api_list["^/modsec%-crs$"] = function ()
 	if not M.save_file("/tmp/modsec-crs.tar.gz") then
 		return false
 	end
@@ -79,6 +84,7 @@ end
 function M.save_file (name)
 	local form, err = upload:new(4096)
 	if not form then
+		logger.log(ngx.ERR, "API", err)
 		return false
 	end
 	form:set_timeout(1000)
@@ -87,6 +93,7 @@ function M.save_file (name)
 		local typ, res, err = form:read()
 		if not typ then
 			file:close()
+			logger.log(ngx.ERR, "API", "not typ")
 			return false
 		end
 		if typ == "eof" then
@@ -102,7 +109,7 @@ function M.save_file (name)
 end
 
 function M.extract_file(archive, destination)
-	return os.execute("tar xzf " .. archive .. " -C " .. destination)
+	return os.execute("tar xzf " .. archive .. " -C " .. destination) == 0
 end
 
 function M.is_api_call (api_uri, api_whitelist_ip)