api - client side (untested)

autoconf/Dockerfile

@@ -10,6 +10,7 @@ COPY misc/cron-autoconf /etc/crontabs/root
 COPY autoconf/entrypoint.sh /opt/bunkerized-nginx/entrypoint/
 COPY autoconf/requirements.txt /opt/bunkerized-nginx/entrypoint/
 COPY autoconf/src/* /opt/bunkerized-nginx/entrypoint/
+COPY VERSION /opt/bunkerized-nginx
 
 RUN apk add --no-cache py3-pip bash certbot curl openssl socat && \
     pip3 install -r /opt/bunkerized-nginx/gen/requirements.txt && \
@@ -21,9 +22,6 @@ RUN chmod +x /tmp/prepare.sh && \
     /tmp/prepare.sh && \
     rm -f /tmp/prepare.sh
 
-# Fix CVE-2021-36159
-RUN apk add "apk-tools>=2.12.6-r0"
-
 #VOLUME /http-confs /server-confs /modsec-confs /modsec-crs-confs /cache /etc/letsencrypt /acme-challenge
 
 ENTRYPOINT ["/opt/bunkerized-nginx/entrypoint/entrypoint.sh"]

confs/global/init-lua.conf

@@ -87,19 +87,13 @@ if use_remote_api then
 	f:close()
 
 	-- Save and ask a machine ID if needed
-	local f = io.open("/opt/bunkerized-nginx/cache/machine.id", "rw")
+	local f = io.open("/etc/nginx/machine.id", "rw")
 	if f == nil then
-		local res, id = remoteapi.register()
+		id = nil
-		if not res then
+		logger.log(ngx.ERR, "REMOTE API", "USE_REMOTE_API is set to yes but machine ID is not generated - communication with {{ REMOTE_API_SERVER }} won't work")
-			logger.log(ngx.ERR, "REMOTE API", "Can't register to the remote API")
 	else
-			logger.log(ngx.ERR, "REMOTE API", "Successfully registered to the remote API")
-			f:write(data)
-			ngx.shared.remote_api:set("id", data, 0)
-		end
-	else
-		logger.log(ngx.ERR, "REMOTE API", "*NOT AN ERROR* Using existing machine ID from cache")	
 		id = f:read("*all")
+		logger.log(ngx.ERR, "REMOTE API", "*NOT AN ERROR* Machine ID = " .. id)	
 	end
 	f:close()
 

confs/site/log-lua.conf

@@ -11,7 +11,10 @@ local bad_behavior_count_time   = {{ BAD_BEHAVIOR_COUNT_TIME }}
 local bad_behavior_ban_time     = {{ BAD_BEHAVIOR_BAN_TIME }}
 
 if use_bad_behavior then
-	behavior.count(bad_behavior_status_codes, bad_behavior_threshold, bad_behavior_count_time, bad_behavior_ban_time)
+	local new_bad_behavior_ban = false
+	if not behavior.is_banned() then
+		new_bad_behavior_ban = behavior.count(bad_behavior_status_codes, bad_behavior_threshold, bad_behavior_count_time, bad_behavior_ban_time)
+	end
 end
 
 -- remote API
@@ -20,8 +23,11 @@ local remoteapi			= require "remoteapi"
 
 if use_remote_api then
 	if ngx.status == ngx.HTTP_FORBIDDEN then
-		-- TODO check if IP is global + good reason
+		local reason = "other"
-		local res, data = remoteapi.ip(ngx.var.remote_addr, "other")
+		if use_bad_behavior and new_bad_behavior_ban then
+			reason = "behavior"
+		end
+		local res, data = remoteapi.ip(ngx.var.remote_addr, reason)
 		if res then
 			logger.log(ngx.NOTICE, "REMOTE API", "Successfully reported ip " .. ngx.var.remote_addr)			
 		else

jobs/Job.py

@@ -116,7 +116,10 @@ class Job(abc.ABC) :
 		if self._redis == None :
 			if os.path.isfile("/tmp/" + self._filename) :
 				os.remove("/tmp/" + self._filename)
-			file = open("/tmp/" + self._filename, "ab")
+			mode = "a"
+			if self._type == "file" :
+				mode = "ab"
+			file = open("/tmp/" + self._filename, mode)
 
 		elif self._redis != None :
 			pipe = self._redis.pipeline()
@@ -126,19 +129,20 @@ class Job(abc.ABC) :
 			data = self.__download_data(url)
 			for chunk in data :
 				if self._type == ["line", "json"] :
-					if not re.match(self._regex, chunk.decode("utf-8")) :
+					if not re.match(self._regex, chunk) :
 						continue
-					chunks = self._edit(chunk)
 				if self._redis == None :
 					if self._type in ["line", "json"] :
-						for chunk in chunks :
+						chunks = self._edit(chunk)
-							file.write(chunk + b"\n")
+						for more_chunk in chunks :
+							file.write(more_chunk + "\n")
 					else :
 						file.write(chunk)
 				else :
 					if self._type in ["line", "json"] :
-						for chunk in chunks :
+						chunks = self._edit(chunk)
-							pipe.set(self._name + "_" + chunk, "1", ex=self._redis_ex)
+						for more_chunk in chunks :
+							pipe.set(self._name + "_" + more_chunk, "1", ex=self._redis_ex)
 					else :
 						pipe.set(self._name + "_" + chunk, "1", ex=self._redis_ex)
 				count += 1
@@ -161,7 +165,7 @@ class Job(abc.ABC) :
 		if not r or r.status_code != 200 :
 			raise Exception("can't download data at " + url)
 		if self._type == "line" :
-			return r.iter_lines()
+			return r.iter_lines(decode_unicode=True)
 		if self._type == "json" :
 			try :
 				return self._json(r.json())

jobs/main.py

@@ -4,7 +4,7 @@ import argparse, sys, re
 
 sys.path.append("/opt/bunkerized-nginx/jobs")
 
-import Abusers, CertbotNew, CertbotRenew, ExitNodes, GeoIP, Proxies, Referrers, SelfSignedCert, UserAgents
+import Abusers, CertbotNew, CertbotRenew, ExitNodes, GeoIP, Proxies, Referrers, SelfSignedCert, UserAgents, RemoteApiDatabase, RemoteApiRegister
 from Job import JobRet, JobManagement, ReloadRet
 
 from logger import log

lua/behavior.lua

@@ -16,17 +16,18 @@ function M.count (status_codes, threshold, count_time, ban_time)
 			local ok, err = ngx.shared.behavior_count:set(ngx.var.remote_addr, count, count_time)
 			if not ok then
 				logger.log(ngx.ERR, "BEHAVIOR", "not enough memory allocated to behavior_ip_count")
-				return
+				return false
 			end
 			if count >= threshold then
 				logger.log(ngx.WARN, "BEHAVIOR", "threshold reached for " .. ngx.var.remote_addr .. " (" .. count .. " / " .. threshold .. ") : IP is banned for " .. ban_time .. " seconds")
 				local ok, err = ngx.shared.behavior_ban:safe_set(ngx.var.remote_addr, true, ban_time)
 				if not ok then
 					logger.log(ngx.ERR, "BEHAVIOR", "not enough memory allocated to behavior_ip_ban")
-					return
+					return false
 				end
+				return true
 			end
-			break
+			return false
 		end
 	end
 end

ui/Dockerfile

@@ -9,15 +9,13 @@ COPY confs/site/ /opt/bunkerized-nginx/confs/site
 COPY confs/global/ /opt/bunkerized-nginx/confs/global
 COPY ui/ /opt/bunkerized-nginx/ui
 COPY settings.json /opt/bunkerized-nginx
+COPY VERSION /opt/bunkerized-nginx
 
 COPY ui/prepare.sh /tmp
 RUN chmod +x /tmp/prepare.sh && \
     /tmp/prepare.sh && \
     rm -f /tmp/prepare.sh
 
-# Fix CVE-2021-36159
-RUN apk add "apk-tools>=2.12.6-r0"
-
 EXPOSE 5000
 
 WORKDIR /opt/bunkerized-nginx/ui