From d53f02b5b3fb3b371e67770077b9b1160c5824df Mon Sep 17 00:00:00 2001 From: bunkerity Date: Wed, 6 Oct 2021 15:41:55 +0200 Subject: [PATCH] api - client side (untested) --- autoconf/Dockerfile | 4 +--- confs/global/init-lua.conf | 14 ++++---------- confs/site/log-lua.conf | 12 +++++++++--- jobs/Job.py | 20 ++++++++++++-------- jobs/main.py | 2 +- lua/behavior.lua | 7 ++++--- ui/Dockerfile | 4 +--- 7 files changed, 32 insertions(+), 31 deletions(-) diff --git a/autoconf/Dockerfile b/autoconf/Dockerfile index 085cfdf..f802de8 100644 --- a/autoconf/Dockerfile +++ b/autoconf/Dockerfile @@ -10,6 +10,7 @@ COPY misc/cron-autoconf /etc/crontabs/root COPY autoconf/entrypoint.sh /opt/bunkerized-nginx/entrypoint/ COPY autoconf/requirements.txt /opt/bunkerized-nginx/entrypoint/ COPY autoconf/src/* /opt/bunkerized-nginx/entrypoint/ +COPY VERSION /opt/bunkerized-nginx RUN apk add --no-cache py3-pip bash certbot curl openssl socat && \ pip3 install -r /opt/bunkerized-nginx/gen/requirements.txt && \ @@ -21,9 +22,6 @@ RUN chmod +x /tmp/prepare.sh && \ /tmp/prepare.sh && \ rm -f /tmp/prepare.sh -# Fix CVE-2021-36159 -RUN apk add "apk-tools>=2.12.6-r0" - #VOLUME /http-confs /server-confs /modsec-confs /modsec-crs-confs /cache /etc/letsencrypt /acme-challenge ENTRYPOINT ["/opt/bunkerized-nginx/entrypoint/entrypoint.sh"] diff --git a/confs/global/init-lua.conf b/confs/global/init-lua.conf index e6f035e..0c22556 100644 --- a/confs/global/init-lua.conf +++ b/confs/global/init-lua.conf @@ -87,19 +87,13 @@ if use_remote_api then f:close() -- Save and ask a machine ID if needed - local f = io.open("/opt/bunkerized-nginx/cache/machine.id", "rw") + local f = io.open("/etc/nginx/machine.id", "rw") if f == nil then - local res, id = remoteapi.register() - if not res then - logger.log(ngx.ERR, "REMOTE API", "Can't register to the remote API") - else - logger.log(ngx.ERR, "REMOTE API", "Successfully registered to the remote API") - f:write(data) - ngx.shared.remote_api:set("id", data, 0) - end + id = nil + logger.log(ngx.ERR, "REMOTE API", "USE_REMOTE_API is set to yes but machine ID is not generated - communication with {{ REMOTE_API_SERVER }} won't work") else - logger.log(ngx.ERR, "REMOTE API", "*NOT AN ERROR* Using existing machine ID from cache") id = f:read("*all") + logger.log(ngx.ERR, "REMOTE API", "*NOT AN ERROR* Machine ID = " .. id) end f:close() diff --git a/confs/site/log-lua.conf b/confs/site/log-lua.conf index 470d655..c33bb3d 100644 --- a/confs/site/log-lua.conf +++ b/confs/site/log-lua.conf @@ -11,7 +11,10 @@ local bad_behavior_count_time = {{ BAD_BEHAVIOR_COUNT_TIME }} local bad_behavior_ban_time = {{ BAD_BEHAVIOR_BAN_TIME }} if use_bad_behavior then - behavior.count(bad_behavior_status_codes, bad_behavior_threshold, bad_behavior_count_time, bad_behavior_ban_time) + local new_bad_behavior_ban = false + if not behavior.is_banned() then + new_bad_behavior_ban = behavior.count(bad_behavior_status_codes, bad_behavior_threshold, bad_behavior_count_time, bad_behavior_ban_time) + end end -- remote API @@ -20,8 +23,11 @@ local remoteapi = require "remoteapi" if use_remote_api then if ngx.status == ngx.HTTP_FORBIDDEN then - -- TODO check if IP is global + good reason - local res, data = remoteapi.ip(ngx.var.remote_addr, "other") + local reason = "other" + if use_bad_behavior and new_bad_behavior_ban then + reason = "behavior" + end + local res, data = remoteapi.ip(ngx.var.remote_addr, reason) if res then logger.log(ngx.NOTICE, "REMOTE API", "Successfully reported ip " .. ngx.var.remote_addr) else diff --git a/jobs/Job.py b/jobs/Job.py index f24e399..c2fcd71 100644 --- a/jobs/Job.py +++ b/jobs/Job.py @@ -116,7 +116,10 @@ class Job(abc.ABC) : if self._redis == None : if os.path.isfile("/tmp/" + self._filename) : os.remove("/tmp/" + self._filename) - file = open("/tmp/" + self._filename, "ab") + mode = "a" + if self._type == "file" : + mode = "ab" + file = open("/tmp/" + self._filename, mode) elif self._redis != None : pipe = self._redis.pipeline() @@ -126,19 +129,20 @@ class Job(abc.ABC) : data = self.__download_data(url) for chunk in data : if self._type == ["line", "json"] : - if not re.match(self._regex, chunk.decode("utf-8")) : + if not re.match(self._regex, chunk) : continue - chunks = self._edit(chunk) if self._redis == None : if self._type in ["line", "json"] : - for chunk in chunks : - file.write(chunk + b"\n") + chunks = self._edit(chunk) + for more_chunk in chunks : + file.write(more_chunk + "\n") else : file.write(chunk) else : if self._type in ["line", "json"] : - for chunk in chunks : - pipe.set(self._name + "_" + chunk, "1", ex=self._redis_ex) + chunks = self._edit(chunk) + for more_chunk in chunks : + pipe.set(self._name + "_" + more_chunk, "1", ex=self._redis_ex) else : pipe.set(self._name + "_" + chunk, "1", ex=self._redis_ex) count += 1 @@ -161,7 +165,7 @@ class Job(abc.ABC) : if not r or r.status_code != 200 : raise Exception("can't download data at " + url) if self._type == "line" : - return r.iter_lines() + return r.iter_lines(decode_unicode=True) if self._type == "json" : try : return self._json(r.json()) diff --git a/jobs/main.py b/jobs/main.py index e096bb4..07a81ea 100644 --- a/jobs/main.py +++ b/jobs/main.py @@ -4,7 +4,7 @@ import argparse, sys, re sys.path.append("/opt/bunkerized-nginx/jobs") -import Abusers, CertbotNew, CertbotRenew, ExitNodes, GeoIP, Proxies, Referrers, SelfSignedCert, UserAgents +import Abusers, CertbotNew, CertbotRenew, ExitNodes, GeoIP, Proxies, Referrers, SelfSignedCert, UserAgents, RemoteApiDatabase, RemoteApiRegister from Job import JobRet, JobManagement, ReloadRet from logger import log diff --git a/lua/behavior.lua b/lua/behavior.lua index 33828e4..c517a89 100644 --- a/lua/behavior.lua +++ b/lua/behavior.lua @@ -16,17 +16,18 @@ function M.count (status_codes, threshold, count_time, ban_time) local ok, err = ngx.shared.behavior_count:set(ngx.var.remote_addr, count, count_time) if not ok then logger.log(ngx.ERR, "BEHAVIOR", "not enough memory allocated to behavior_ip_count") - return + return false end if count >= threshold then logger.log(ngx.WARN, "BEHAVIOR", "threshold reached for " .. ngx.var.remote_addr .. " (" .. count .. " / " .. threshold .. ") : IP is banned for " .. ban_time .. " seconds") local ok, err = ngx.shared.behavior_ban:safe_set(ngx.var.remote_addr, true, ban_time) if not ok then logger.log(ngx.ERR, "BEHAVIOR", "not enough memory allocated to behavior_ip_ban") - return + return false end + return true end - break + return false end end end diff --git a/ui/Dockerfile b/ui/Dockerfile index 1393edd..61f35a2 100644 --- a/ui/Dockerfile +++ b/ui/Dockerfile @@ -9,15 +9,13 @@ COPY confs/site/ /opt/bunkerized-nginx/confs/site COPY confs/global/ /opt/bunkerized-nginx/confs/global COPY ui/ /opt/bunkerized-nginx/ui COPY settings.json /opt/bunkerized-nginx +COPY VERSION /opt/bunkerized-nginx COPY ui/prepare.sh /tmp RUN chmod +x /tmp/prepare.sh && \ /tmp/prepare.sh && \ rm -f /tmp/prepare.sh -# Fix CVE-2021-36159 -RUN apk add "apk-tools>=2.12.6-r0" - EXPOSE 5000 WORKDIR /opt/bunkerized-nginx/ui