diff --git a/CHANGELOG.md b/CHANGELOG.md index 2986056..601a61d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## v1.3.2 - - Use API instead of a shared folder for Swarm and Kubernetes integrations +- Beta integration of distributed bad IPs database through a remote API - Various bug fixes related to antibot feature - Fix Moodle example diff --git a/confs/global/init-lua.conf b/confs/global/init-lua.conf index 8e82131..4a7962d 100644 --- a/confs/global/init-lua.conf +++ b/confs/global/init-lua.conf @@ -4,6 +4,7 @@ local dataloader = require "dataloader" local logger = require "logger" local cjson = require "cjson" local remoteapi = require "remoteapi" +local iputils = require "resty.iputils" local use_redis = {% if USE_REDIS == "yes" %}true{% else %}false{% endif +%} @@ -15,6 +16,30 @@ local use_referrers = {% if has_value("BLOCK_REFERRER", "yes") %}true{% else %} local use_remote_api = {% if has_value("USE_REMOTE_API", "yes") %}true{% else %}false{% endif +%} + +-- Load reserved IPs +local reserved_ips = { + "0.0.0.0/8", + "10.0.0.0/8", + "100.64.0.0/10", + "127.0.0.0/8", + "169.254.0.0/16", + "172.16.0.0/12", + "192.0.0.0/24", + "192.0.2.0/24", + "192.88.99.0/24", + "192.168.0.0/16", + "198.18.0.0/15", + "198.51.100.0/24", + "203.0.113.0/24", + "224.0.0.0/4", + "233.252.0.0/24", + "240.0.0.0/4", + "255.255.255.255/32" +} +ngx.shared.reserved_ips:safe_set("cidrs", iputils.parse_cidrs(reserved_ips), 0) + +-- Load blacklists if not use_redis then if use_proxies then dataloader.load_ip("/etc/nginx/proxies.list", ngx.shared.proxies_data) @@ -87,19 +112,33 @@ if use_remote_api then f:close() -- Save machine ID + local id = "empty" local f = io.open("/etc/nginx/machine.id", "r") if f == nil then - id = "empty" logger.log(ngx.ERR, "REMOTE API", "USE_REMOTE_API is set to yes but machine ID is not generated - communication with {{ REMOTE_API_SERVER }} won't work") else id = f:read("*all"):gsub("[\r\n]", "") - logger.log(ngx.ERR, "REMOTE API", "*NOT AN ERROR* Machine ID = " .. id) + logger.log(ngx.ERR, "REMOTE API", "*NOT AN ERROR* Using existing machine ID (" .. id .. ")") f:close() end ngx.shared.remote_api:set("id", id, 0) - -- TODO : ping (blocking socket) - -- TODO : load database + -- Ping the remote API + local ping = "ko" + if id ~= "empty" then + if remoteapi.ping2() then + ping = "ok" + logger.log(ngx.ERR, "REMOTE API", "*NOT AN ERROR* Successfully requested the remote API") + else + logger.log(ngx.ERR, "REMOTE API", "Can't contact the remote API, feature will be disabled") + end + end + ngx.shared.remote_api:set("ping", ping, 0) + + -- Load the database + if ping ~= "ko" then + dataloader.load_ip("/etc/nginx/remote-api.db", ngx.shared.remote_api_db) + end end } diff --git a/confs/global/nginx.conf b/confs/global/nginx.conf index 84dfbcb..96fe66b 100644 --- a/confs/global/nginx.conf +++ b/confs/global/nginx.conf @@ -91,7 +91,9 @@ http { {% if has_value("USE_BAD_BEHAVIOR", "yes") %}lua_shared_dict behavior_ban 10m;{% endif +%} {% if has_value("USE_BAD_BEHAVIOR", "yes") %}lua_shared_dict behavior_count 10m;{% endif +%} lua_shared_dict plugins_data 10m; + lua_shared_dict reserved_ips 1m; {% if has_value("USE_REMOTE_API", "yes") %}lua_shared_dict remote_api 1m;{% endif +%} + {% if has_value("USE_REMOTE_API", "yes") %}lua_shared_dict remote_api_db 10m;{% endif +%} # shared memory zone for limit_req {% if has_value("USE_LIMIT_REQ", "yes") %}limit_req_zone $binary_remote_addr$uri zone=limit:{{ LIMIT_REQ_CACHE }} rate={{ LIMIT_REQ_RATE }};{% endif +%} diff --git a/confs/site/log-lua.conf b/confs/site/log-lua.conf index 8b7f5c2..3020285 100644 --- a/confs/site/log-lua.conf +++ b/confs/site/log-lua.conf @@ -20,8 +20,9 @@ end -- remote API local use_remote_api = {% if USE_REMOTE_API == "yes" %}true{% else %}false{% endif +%} local remoteapi = require "remoteapi" +local iputils = require "resty.iputils" -if use_remote_api and ngx.shared.remote_api:get("id") ~= "empty" then +if use_remote_api and not iputils.ip_in_cidrs(ngx.var.remote_addr, ngx.shared.reserved_ips:get("data")) and ngx.shared.remote_api:get("id") ~= "empty" and ngx.shared.remote_api:get("ping") ~= "ko" then if ngx.status == ngx.HTTP_FORBIDDEN then local reason = "other" if use_bad_behavior and new_bad_behavior_ban then @@ -34,16 +35,13 @@ if use_remote_api and ngx.shared.remote_api:get("id") ~= "empty" then local remoteapi = require "remoteapi" local logger = require "logger" local res, data = remoteapi.ip(ip, reason) - -- TODO : find a way to log ? --- if res then --- logger.log(ngx.ERR, "REMOTE API", "Successfully reported ip " .. ngx.var.remote_addr) --- else --- logger.log(ngx.ERR, "REMOTE API", "Error while reporting ip " .. ngx.var.remote_addr .. " : " .. data) --- end + -- TODO : find a way to log end local ok, err = ngx.timer.at(0, report_ip, ngx.var.remote_addr, reason) if not ok then logger.log(ngx.ERR, "REMOTE API", "Error while creating report timer " .. err) + else + logger.log(ngx.NOTICE, "REMOTE API", "Reporting " .. ngx.var.remote_addr .. "(reason: " .. reason .. ") to the remote API") end end end diff --git a/confs/site/main-lua.conf b/confs/site/main-lua.conf index d0c3299..a145415 100644 --- a/confs/site/main-lua.conf +++ b/confs/site/main-lua.conf @@ -57,6 +57,9 @@ local dnsbl_list = {% raw %}{{% endraw %}{% if DNSBL_LIST != "" %}{% set elemen -- bad behavior local use_bad_behavior = {% if USE_BAD_BEHAVIOR == "yes" %}true{% else %}false{% endif +%} +-- remote API +local use_remote_api = {% if USE_REMOTE_API == "yes" %}true{% else %}false{% endif +%} + -- include LUA code local whitelist = require "whitelist" local blacklist = require "blacklist" @@ -224,6 +227,15 @@ if use_dnsbl and not dnsbl.cached() then end end +-- check if IP is in distributed DB +if use_remote_api then + local checker = checker:new("remote-api-db", ngx.shared.remote_api_db, redis_client, "simple") + if checker:check(iputils.ip2bin(ngx.var.remote_addr)) then + logger.log(ngx.WARN, "REMOTE API", "IP " .. ngx.var.remote_addr .. " is in the distributed DB") + ngx.exit(ngx.HTTP_FORBIDDEN) + end +end + -- cookie check if use_antibot_cookie and ngx.var.uri ~= "/favicon.ico" then if not cookie.is_set("uri") then diff --git a/jobs/Job.py b/jobs/Job.py index 37c9265..795ac75 100644 --- a/jobs/Job.py +++ b/jobs/Job.py @@ -132,7 +132,7 @@ class Job(abc.ABC) : chunk = chunk.decode("utf-8") if self._type in ["line", "json"] : if not re.match(self._regex, chunk) : - log(self._name, "WARN", chunk + " doesn't match regex " + self._regex) + #log(self._name, "WARN", chunk + " doesn't match regex " + self._regex) continue if self._redis == None : if self._type in ["line", "json"] : diff --git a/lua/remoteapi.lua b/lua/remoteapi.lua index d9256f9..c564086 100644 --- a/lua/remoteapi.lua +++ b/lua/remoteapi.lua @@ -39,6 +39,28 @@ function M.gen_data(use_id, data) return all_data end +function M.ping2() + local https = require "ssl.https" + local ltn12 = require "ltn12" + local request_body = cjson.encode(M.gen_data(true, {})) + local response_body = {} + local res, code, headers, status = https.request { + url = ngx.shared.remote_api:get("server") .. "/ping", + method = "GET", + headers = { + ["Content-Type"] = "application/json", + ["User-Agent"] = "bunkerized-nginx/" .. ngx.shared.remote_api:get("version"), + ["Content-Length"] = request_body:len() + }, + source = ltn12.source.string(request_body), + sink = ltn12.sink.table(response_body) + } + if res and status == 200 and response_body["data"] == "pong" then + return true + end + return false +end + function M.register() local request = {} local res, status, data = M.send("POST", "/register", M.gen_data(false, request)) @@ -58,7 +80,6 @@ function M.ping() end function M.ip(ip, reason) - -- TODO : check if IP is global local request = { ["ip"] = ip, ["reason"] = reason