Makussu/bunkerweb
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

remote API - basic send of bad IPs

Browse Source
This commit is contained in:
bunkerity 2021-10-07 12:00:20 +02:00
parent fb799765a4
commit fdc02be051
No known key found for this signature in database
GPG Key ID: 3D80806F12602A7C
10 changed files with 37 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
confs/global/init-lua.conf
View File

@ -83,30 +83,23 @@ if use_remote_api then
-- Save version -- Save version
local f = io.open("/opt/bunkerized-nginx/VERSION", "r") local f = io.open("/opt/bunkerized-nginx/VERSION", "r")
ngx.shared.remote_api:set("version", f:read("*all"), 0) ngx.shared.remote_api:set("version", f:read("*all"):gsub("[\r\n]", ""), 0)
f:close() f:close()
-- Save and ask a machine ID if needed -- Save machine ID
local f = io.open("/etc/nginx/machine.id", "rw") local f = io.open("/etc/nginx/machine.id", "r")
if f == nil then if f == nil then
id = nil id = "empty"
logger.log(ngx.ERR, "REMOTE API", "USE_REMOTE_API is set to yes but machine ID is not generated - communication with {{ REMOTE_API_SERVER }} won't work") logger.log(ngx.ERR, "REMOTE API", "USE_REMOTE_API is set to yes but machine ID is not generated - communication with {{ REMOTE_API_SERVER }} won't work")
else else
id = f:read("*all") id = f:read("*all"):gsub("[\r\n]", "")
logger.log(ngx.ERR, "REMOTE API", "*NOT AN ERROR* Machine ID = " .. id) logger.log(ngx.ERR, "REMOTE API", "*NOT AN ERROR* Machine ID = " .. id)
f:close()
end end
f:close() ngx.shared.remote_api:set("id", id, 0)
-- Test the machine ID
if id ~= nil then
local res, pong = remoteapi.ping()
if not res or pong ~= "pong" then
logger.log(ngx.ERR, "REMOTE API", "Ping failed, the remote server may be down or your machine ID is invalid")
else
logger.log(ngx.ERR, "REMOTE API", "*NOT AN ERROR* Ping successful")
end
end
-- TODO : ping (blocking socket)
-- TODO : load database
end end
} }

24
confs/site/log-lua.conf
View File

@ -21,17 +21,29 @@ end
local use_remote_api = {% if USE_REMOTE_API == "yes" %}true{% else %}false{% endif +%} local use_remote_api = {% if USE_REMOTE_API == "yes" %}true{% else %}false{% endif +%}
local remoteapi = require "remoteapi" local remoteapi = require "remoteapi"
if use_remote_api then if use_remote_api and ngx.shared.remote_api:get("id") ~= "empty" then
if ngx.status == ngx.HTTP_FORBIDDEN then if ngx.status == ngx.HTTP_FORBIDDEN then
local reason = "other" local reason = "other"
if use_bad_behavior and new_bad_behavior_ban then if use_bad_behavior and new_bad_behavior_ban then
reason = "behavior" reason = "behavior"
end end
local res, data = remoteapi.ip(ngx.var.remote_addr, reason) local report_ip = function (premature, ip, reason)
if res then if premature then
logger.log(ngx.NOTICE, "REMOTE API", "Successfully reported ip " .. ngx.var.remote_addr) return
else end
logger.log(ngx.ERR, "REMOTE API", "Error while reporting ip " .. ngx.var.remote_addr .. " : " .. data) local remoteapi = require "remoteapi"
local logger = require "logger"
local res, data = remoteapi.ip(ip, reason)
-- TODO : find a way to log ?
-- if res then
-- logger.log(ngx.ERR, "REMOTE API", "Successfully reported ip " .. ngx.var.remote_addr)
-- else
-- logger.log(ngx.ERR, "REMOTE API", "Error while reporting ip " .. ngx.var.remote_addr .. " : " .. data)
-- end
end
local ok, err = ngx.timer.at(0, report_ip, ngx.var.remote_addr, reason)
if not ok then
logger.log(ngx.ERR, "REMOTE API", "Error while creating report timer " .. err)
end end
end end
end end

2
entrypoint/jobs.sh
View File

@ -92,6 +92,6 @@ fi
if [ "$(has_value USE_REMOTE_API yes)" != "" ] ; then if [ "$(has_value USE_REMOTE_API yes)" != "" ] ; then
/opt/bunkerized-nginx/jobs/main.py --name remote-api-register --server "$(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2)" --version "$(cat /opt/bunkerized-nginx/VERSION)" /opt/bunkerized-nginx/jobs/main.py --name remote-api-register --server "$(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2)" --version "$(cat /opt/bunkerized-nginx/VERSION)"
if [ $? -eq 0 ] ; then if [ $? -eq 0 ] ; then
/opt/bunkerized-nginx/jobs/main.py --name remote-api-database --server "$(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2)" --version "$(cat /opt/bunkerized-nginx/VERSION)" --id "$(cat /opt/bunkerized-nginx/machine.id)" /opt/bunkerized-nginx/jobs/main.py --name remote-api-database --server "$(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2)" --version "$(cat /opt/bunkerized-nginx/VERSION)" --id "$(cat /opt/bunkerized-nginx/cache/machine.id)"
fi fi
fi fi

3
jobs/Job.py
View File

@ -128,10 +128,11 @@ class Job(abc.ABC) :
for url in self._data : for url in self._data :
data = self.__download_data(url) data = self.__download_data(url)
for chunk in data : for chunk in data :
if isinstance(chunk, bytes) : if isinstance(chunk, bytes) and self._type in ["line", "json"] :
chunk = chunk.decode("utf-8") chunk = chunk.decode("utf-8")
if self._type in ["line", "json"] : if self._type in ["line", "json"] :
if not re.match(self._regex, chunk) : if not re.match(self._regex, chunk) :
log(self._name, "WARN", chunk + " doesn't match regex " + self._regex)
continue continue
if self._redis == None : if self._redis == None :
if self._type in ["line", "json"] : if self._type in ["line", "json"] :

3
jobs/RemoteApiDatabase.py
View File

@ -7,9 +7,10 @@ class RemoteApiDatabase(Job) :
data = [server + "/db"] data = [server + "/db"]
filename = "remote-api.db" filename = "remote-api.db"
type = "json" type = "json"
redis_ex = 3600
regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$" regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$"
json_data = {"version": version, "id": id} json_data = {"version": version, "id": id}
super().__init__(name, data, filename, type=type, redis_host=redis_host, redis_ex=redis_ex, regex=regex, copy_cache=copy_cache, json_data=json_data, method=method) super().__init__(name, data, filename, type=type, redis_host=redis_host, redis_ex=redis_ex, regex=regex, copy_cache=copy_cache, json_data=json_data)
def _json(self, data) : def _json(self, data) :
return data["data"] return data["data"]

2
jobs/RemoteApiRegister.py
View File

@ -13,4 +13,4 @@ class RemoteApiRegister(Job) :
super().__init__(name, data, filename, type=type, regex=regex, copy_cache=True, json_data=json_data, method=method) super().__init__(name, data, filename, type=type, regex=regex, copy_cache=True, json_data=json_data, method=method)
def _json(self, data) : def _json(self, data) :
return data["data"] return [data["data"]]

1
lua/remoteapi.lua
View File

@ -58,6 +58,7 @@ function M.ping()
end end
function M.ip(ip, reason) function M.ip(ip, reason)
-- TODO : check if IP is global
local request = { local request = {
["ip"] = ip, ["ip"] = ip,
["reason"] = reason ["reason"] = reason

2
misc/cron
View File

@ -3,6 +3,6 @@
45 0 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_REFERRER yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name referrers >> /var/log/nginx/jobs.log 2>&1 45 0 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_REFERRER yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name referrers >> /var/log/nginx/jobs.log 2>&1
0 1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_ABUSERS yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name abusers >> /var/log/nginx/jobs.log 2>&1 0 1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_ABUSERS yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name abusers >> /var/log/nginx/jobs.log 2>&1
0 2 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_PROXIES yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name proxies >> /var/log/nginx/jobs.log 2>&1 0 2 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_PROXIES yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name proxies >> /var/log/nginx/jobs.log 2>&1
30 */1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value USE_REMOTE_API yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name remote-api-database --server "$(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2)" --version "$(cat /opt/bunkerized-nginx/VERSION)" --id "$(cat /opt/bunkerized-nginx/machine.id)" >> /var/log/nginx/jobs.log 2>&1 30 */1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value USE_REMOTE_API yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name remote-api-database --server "$(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2)" --version "$(cat /opt/bunkerized-nginx/VERSION)" --id "$(cat /opt/bunkerized-nginx/cache/machine.id)" >> /var/log/nginx/jobs.log 2>&1
0 */1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_TOR_EXIT_NODE yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name exit-nodes >> /var/log/nginx/jobs.log 2>&1 0 */1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_TOR_EXIT_NODE yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name exit-nodes >> /var/log/nginx/jobs.log 2>&1
0 3 2 * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ [ "$(has_value BLACKLIST_COUNTRY ".\+")" != "" ] || [ "$(has_value WHITELIST_COUNTRY ".\+")" != "" ] ] && /opt/bunkerized-nginx/jobs/main.py --reload --name geoip >> /var/log/nginx/jobs.log 2>&1 0 3 2 * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ [ "$(has_value BLACKLIST_COUNTRY ".\+")" != "" ] || [ "$(has_value WHITELIST_COUNTRY ".\+")" != "" ] ] && /opt/bunkerized-nginx/jobs/main.py --reload --name geoip >> /var/log/nginx/jobs.log 2>&1

2
misc/cron-autoconf
View File

@ -3,6 +3,6 @@
45 0 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_REFERRER yes)" != "" ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --lock --name referrers" nginx >> /var/log/nginx/jobs.log 2>&1 45 0 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_REFERRER yes)" != "" ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --lock --name referrers" nginx >> /var/log/nginx/jobs.log 2>&1
0 1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_ABUSERS yes)" != "" ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --lock --name abusers" nginx >> /var/log/nginx/jobs.log 2>&1 0 1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_ABUSERS yes)" != "" ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --lock --name abusers" nginx >> /var/log/nginx/jobs.log 2>&1
0 2 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_PROXIES yes)" != "" ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --lock --name proxies" nginx >> /var/log/nginx/jobs.log 2>&1 0 2 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_PROXIES yes)" != "" ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --lock --name proxies" nginx >> /var/log/nginx/jobs.log 2>&1
30 */1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value USE_REMOTE_API yes)" != "" ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --name remote-api-database --server $(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2) --version $(cat /opt/bunkerized-nginx/VERSION) --id $(cat /opt/bunkerized-nginx/machine.id)" nginx >> /var/log/nginx/jobs.log 2>&1 30 */1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value USE_REMOTE_API yes)" != "" ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --name remote-api-database --server $(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2) --version $(cat /opt/bunkerized-nginx/VERSION) --id $(cat /opt/bunkerized-nginx/cache/machine.id)" nginx >> /var/log/nginx/jobs.log 2>&1
0 */1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_TOR_EXIT_NODE yes)" != "" ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --lock --name exit-nodes" nginx >> /var/log/nginx/jobs.log 2>&1 0 */1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_TOR_EXIT_NODE yes)" != "" ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --lock --name exit-nodes" nginx >> /var/log/nginx/jobs.log 2>&1
0 3 2 * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ [ "$(has_value BLACKLIST_COUNTRY ".\+")" != "" ] || [ "$(has_value WHITELIST_COUNTRY ".\+")" != "" ] ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --lock --name geoip" nginx >> /var/log/nginx/jobs.log 2>&1 0 3 2 * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ [ "$(has_value BLACKLIST_COUNTRY ".\+")" != "" ] || [ "$(has_value WHITELIST_COUNTRY ".\+")" != "" ] ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --lock --name geoip" nginx >> /var/log/nginx/jobs.log 2>&1

2
misc/cron-linux
View File

@ -3,6 +3,6 @@
45 0 * * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_REFERRER yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name referrers >> /var/log/nginx/jobs.log 2>&1 45 0 * * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_REFERRER yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name referrers >> /var/log/nginx/jobs.log 2>&1
0 1 * * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_ABUSERS yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name abusers >> /var/log/nginx/jobs.log 2>&1 0 1 * * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_ABUSERS yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name abusers >> /var/log/nginx/jobs.log 2>&1
0 2 * * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_PROXIES yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name proxies >> /var/log/nginx/jobs.log 2>&1 0 2 * * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_PROXIES yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name proxies >> /var/log/nginx/jobs.log 2>&1
30 */1 * * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value USE_REMOTE_API yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name remote-api-database --server "$(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2)" --version "$(cat /opt/bunkerized-nginx/VERSION)" --id "$(cat /opt/bunkerized-nginx/machine.id)" >> /var/log/nginx/jobs.log 2>&1 30 */1 * * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value USE_REMOTE_API yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name remote-api-database --server "$(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2)" --version "$(cat /opt/bunkerized-nginx/VERSION)" --id "$(cat /opt/bunkerized-nginx/cache/machine.id)" >> /var/log/nginx/jobs.log 2>&1
0 */1 * * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_TOR_EXIT_NODE yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name exit-nodes >> /var/log/nginx/jobs.log 2>&1 0 */1 * * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_TOR_EXIT_NODE yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name exit-nodes >> /var/log/nginx/jobs.log 2>&1
0 3 2 * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ [ "$(has_value BLACKLIST_COUNTRY ".\+")" != "" ] || [ "$(has_value WHITELIST_COUNTRY ".\+")" != "" ] ] && /opt/bunkerized-nginx/jobs/main.py --reload --name geoip >> /var/log/nginx/jobs.log 2>&1 0 3 2 * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ [ "$(has_value BLACKLIST_COUNTRY ".\+")" != "" ] || [ "$(has_value WHITELIST_COUNTRY ".\+")" != "" ] ] && /opt/bunkerized-nginx/jobs/main.py --reload --name geoip >> /var/log/nginx/jobs.log 2>&1