From fdc02be0519949b645fecdbc0a4a870ba1197711 Mon Sep 17 00:00:00 2001
From: bunkerity <contact@bunkerity.com>
Date: Thu, 7 Oct 2021 12:00:20 +0200
Subject: [PATCH] remote API - basic send of bad IPs

---
 confs/global/init-lua.conf | 25 +++++++++----------------
 confs/site/log-lua.conf    | 24 ++++++++++++++++++------
 entrypoint/jobs.sh         |  2 +-
 jobs/Job.py                |  3 ++-
 jobs/RemoteApiDatabase.py  |  3 ++-
 jobs/RemoteApiRegister.py  |  2 +-
 lua/remoteapi.lua          |  1 +
 misc/cron                  |  2 +-
 misc/cron-autoconf         |  2 +-
 misc/cron-linux            |  2 +-
 10 files changed, 37 insertions(+), 29 deletions(-)

diff --git a/confs/global/init-lua.conf b/confs/global/init-lua.conf
index 0c22556..8e82131 100644
--- a/confs/global/init-lua.conf
+++ b/confs/global/init-lua.conf
@@ -83,30 +83,23 @@ if use_remote_api then
 
 	-- Save version
 	local f = io.open("/opt/bunkerized-nginx/VERSION", "r")
-	ngx.shared.remote_api:set("version", f:read("*all"), 0)
+	ngx.shared.remote_api:set("version", f:read("*all"):gsub("[\r\n]", ""), 0)
 	f:close()
 
-	-- Save and ask a machine ID if needed
-	local f = io.open("/etc/nginx/machine.id", "rw")
+	-- Save machine ID
+	local f = io.open("/etc/nginx/machine.id", "r")
 	if f == nil then
-		id = nil
+		id = "empty"
 		logger.log(ngx.ERR, "REMOTE API", "USE_REMOTE_API is set to yes but machine ID is not generated - communication with {{ REMOTE_API_SERVER }} won't work")
 	else
-		id = f:read("*all")
+		id = f:read("*all"):gsub("[\r\n]", "")
 		logger.log(ngx.ERR, "REMOTE API", "*NOT AN ERROR* Machine ID = " .. id)	
+		f:close()
 	end
-	f:close()
-
-	-- Test the machine ID
-	if id ~= nil then
-		local res, pong = remoteapi.ping()
-		if not res or pong ~= "pong" then
-			logger.log(ngx.ERR, "REMOTE API", "Ping failed, the remote server may be down or your machine ID is invalid")
-		else
-			logger.log(ngx.ERR, "REMOTE API", "*NOT AN ERROR* Ping successful")
-		end
-	end
+	ngx.shared.remote_api:set("id", id, 0)
 
+	-- TODO : ping (blocking socket)
+	-- TODO : load database
 end
 
 }
diff --git a/confs/site/log-lua.conf b/confs/site/log-lua.conf
index c33bb3d..8b7f5c2 100644
--- a/confs/site/log-lua.conf
+++ b/confs/site/log-lua.conf
@@ -21,17 +21,29 @@ end
 local use_remote_api            = {% if USE_REMOTE_API == "yes" %}true{% else %}false{% endif +%}
 local remoteapi			= require "remoteapi"
 
-if use_remote_api then
+if use_remote_api and ngx.shared.remote_api:get("id") ~= "empty" then
 	if ngx.status == ngx.HTTP_FORBIDDEN then
 		local reason = "other"
 		if use_bad_behavior and new_bad_behavior_ban then
 			reason = "behavior"
 		end
-		local res, data = remoteapi.ip(ngx.var.remote_addr, reason)
-		if res then
-			logger.log(ngx.NOTICE, "REMOTE API", "Successfully reported ip " .. ngx.var.remote_addr)			
-		else
-			logger.log(ngx.ERR, "REMOTE API", "Error while reporting ip " .. ngx.var.remote_addr .. " : " .. data)
+		local report_ip = function (premature, ip, reason)
+			if premature then
+				return
+			end
+			local remoteapi = require "remoteapi"
+			local logger = require "logger"
+			local res, data = remoteapi.ip(ip, reason)
+			-- TODO : find a way to log ?
+--			if res then
+--				logger.log(ngx.ERR, "REMOTE API", "Successfully reported ip " .. ngx.var.remote_addr)			
+--			else
+--				logger.log(ngx.ERR, "REMOTE API", "Error while reporting ip " .. ngx.var.remote_addr .. " : " .. data)
+--			end
+		end
+		local ok, err = ngx.timer.at(0, report_ip, ngx.var.remote_addr, reason)
+		if not ok then
+			logger.log(ngx.ERR, "REMOTE API", "Error while creating report timer " .. err)
 		end
 	end
 end
diff --git a/entrypoint/jobs.sh b/entrypoint/jobs.sh
index dd5b15f..c2a48a4 100644
--- a/entrypoint/jobs.sh
+++ b/entrypoint/jobs.sh
@@ -92,6 +92,6 @@ fi
 if [ "$(has_value USE_REMOTE_API yes)" != "" ] ; then
 	/opt/bunkerized-nginx/jobs/main.py --name remote-api-register --server "$(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2)" --version "$(cat /opt/bunkerized-nginx/VERSION)"
 	if [ $? -eq 0 ] ; then
-		/opt/bunkerized-nginx/jobs/main.py --name remote-api-database --server "$(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2)" --version "$(cat /opt/bunkerized-nginx/VERSION)" --id "$(cat /opt/bunkerized-nginx/machine.id)"
+		/opt/bunkerized-nginx/jobs/main.py --name remote-api-database --server "$(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2)" --version "$(cat /opt/bunkerized-nginx/VERSION)" --id "$(cat /opt/bunkerized-nginx/cache/machine.id)"
 	fi
 fi
diff --git a/jobs/Job.py b/jobs/Job.py
index 4c8a8a6..37c9265 100644
--- a/jobs/Job.py
+++ b/jobs/Job.py
@@ -128,10 +128,11 @@ class Job(abc.ABC) :
 		for url in self._data :
 			data = self.__download_data(url)
 			for chunk in data :
-				if isinstance(chunk, bytes) :
+				if isinstance(chunk, bytes) and self._type in ["line", "json"] :
 					chunk = chunk.decode("utf-8")
 				if self._type in ["line", "json"] :
 					if not re.match(self._regex, chunk) :
+						log(self._name, "WARN", chunk + " doesn't match regex " + self._regex)
 						continue
 				if self._redis == None :
 					if self._type in ["line", "json"] :
diff --git a/jobs/RemoteApiDatabase.py b/jobs/RemoteApiDatabase.py
index 4381422..00fa4c8 100644
--- a/jobs/RemoteApiDatabase.py
+++ b/jobs/RemoteApiDatabase.py
@@ -7,9 +7,10 @@ class RemoteApiDatabase(Job) :
 		data = [server + "/db"]
 		filename = "remote-api.db"
 		type = "json"
+		redis_ex = 3600
 		regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$"
 		json_data = {"version": version, "id": id}
-		super().__init__(name, data, filename, type=type, redis_host=redis_host, redis_ex=redis_ex, regex=regex, copy_cache=copy_cache, json_data=json_data, method=method)
+		super().__init__(name, data, filename, type=type, redis_host=redis_host, redis_ex=redis_ex, regex=regex, copy_cache=copy_cache, json_data=json_data)
 
 	def _json(self, data) :
 		return data["data"]
diff --git a/jobs/RemoteApiRegister.py b/jobs/RemoteApiRegister.py
index 822273c..c06fc07 100644
--- a/jobs/RemoteApiRegister.py
+++ b/jobs/RemoteApiRegister.py
@@ -13,4 +13,4 @@ class RemoteApiRegister(Job) :
 		super().__init__(name, data, filename, type=type, regex=regex, copy_cache=True, json_data=json_data, method=method)
 
 	def _json(self, data) :
-		return data["data"]
+		return [data["data"]]
diff --git a/lua/remoteapi.lua b/lua/remoteapi.lua
index 164571c..d9256f9 100644
--- a/lua/remoteapi.lua
+++ b/lua/remoteapi.lua
@@ -58,6 +58,7 @@ function M.ping()
 end
 
 function M.ip(ip, reason)
+	-- TODO : check if IP is global
 	local request = {
 		["ip"] = ip,
 		["reason"] = reason
diff --git a/misc/cron b/misc/cron
index c63ebcd..9ff7e25 100644
--- a/misc/cron
+++ b/misc/cron
@@ -3,6 +3,6 @@
 45 0 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_REFERRER yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name referrers >> /var/log/nginx/jobs.log 2>&1
 0 1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_ABUSERS yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name abusers >> /var/log/nginx/jobs.log 2>&1
 0 2 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_PROXIES yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name proxies >> /var/log/nginx/jobs.log 2>&1
-30 */1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value USE_REMOTE_API yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name remote-api-database --server "$(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2)" --version "$(cat /opt/bunkerized-nginx/VERSION)" --id "$(cat /opt/bunkerized-nginx/machine.id)" >> /var/log/nginx/jobs.log 2>&1
+30 */1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value USE_REMOTE_API yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name remote-api-database --server "$(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2)" --version "$(cat /opt/bunkerized-nginx/VERSION)" --id "$(cat /opt/bunkerized-nginx/cache/machine.id)" >> /var/log/nginx/jobs.log 2>&1
 0 */1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_TOR_EXIT_NODE yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name exit-nodes >> /var/log/nginx/jobs.log 2>&1
 0 3 2 * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ [ "$(has_value BLACKLIST_COUNTRY ".\+")" != "" ] || [ "$(has_value WHITELIST_COUNTRY ".\+")" != "" ] ] && /opt/bunkerized-nginx/jobs/main.py --reload --name geoip >> /var/log/nginx/jobs.log 2>&1
diff --git a/misc/cron-autoconf b/misc/cron-autoconf
index f5296b7..4a4a61e 100644
--- a/misc/cron-autoconf
+++ b/misc/cron-autoconf
@@ -3,6 +3,6 @@
 45 0 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_REFERRER yes)" != "" ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --lock --name referrers" nginx >> /var/log/nginx/jobs.log 2>&1
 0 1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_ABUSERS yes)" != "" ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --lock --name abusers" nginx >> /var/log/nginx/jobs.log 2>&1
 0 2 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_PROXIES yes)" != "" ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --lock --name proxies" nginx >> /var/log/nginx/jobs.log 2>&1
-30 */1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value USE_REMOTE_API yes)" != "" ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --name remote-api-database --server $(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2) --version $(cat /opt/bunkerized-nginx/VERSION) --id $(cat /opt/bunkerized-nginx/machine.id)" nginx >> /var/log/nginx/jobs.log 2>&1
+30 */1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value USE_REMOTE_API yes)" != "" ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --name remote-api-database --server $(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2) --version $(cat /opt/bunkerized-nginx/VERSION) --id $(cat /opt/bunkerized-nginx/cache/machine.id)" nginx >> /var/log/nginx/jobs.log 2>&1
 0 */1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_TOR_EXIT_NODE yes)" != "" ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --lock --name exit-nodes" nginx >> /var/log/nginx/jobs.log 2>&1
 0 3 2 * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ [ "$(has_value BLACKLIST_COUNTRY ".\+")" != "" ] || [ "$(has_value WHITELIST_COUNTRY ".\+")" != "" ] ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --lock --name geoip" nginx >> /var/log/nginx/jobs.log 2>&1
diff --git a/misc/cron-linux b/misc/cron-linux
index 85f86cc..b1faf2e 100644
--- a/misc/cron-linux
+++ b/misc/cron-linux
@@ -3,6 +3,6 @@
 45 0 * * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_REFERRER yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name referrers >> /var/log/nginx/jobs.log 2>&1
 0 1 * * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_ABUSERS yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name abusers >> /var/log/nginx/jobs.log 2>&1
 0 2 * * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_PROXIES yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name proxies >> /var/log/nginx/jobs.log 2>&1
-30 */1 * * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value USE_REMOTE_API yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name remote-api-database --server "$(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2)" --version "$(cat /opt/bunkerized-nginx/VERSION)" --id "$(cat /opt/bunkerized-nginx/machine.id)" >> /var/log/nginx/jobs.log 2>&1
+30 */1 * * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value USE_REMOTE_API yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name remote-api-database --server "$(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2)" --version "$(cat /opt/bunkerized-nginx/VERSION)" --id "$(cat /opt/bunkerized-nginx/cache/machine.id)" >> /var/log/nginx/jobs.log 2>&1
 0 */1 * * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_TOR_EXIT_NODE yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name exit-nodes >> /var/log/nginx/jobs.log 2>&1
 0 3 2 * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ [ "$(has_value BLACKLIST_COUNTRY ".\+")" != "" ] || [ "$(has_value WHITELIST_COUNTRY ".\+")" != "" ] ] && /opt/bunkerized-nginx/jobs/main.py --reload --name geoip >> /var/log/nginx/jobs.log 2>&1