bunkerweb 1.4.0

core/bunkernet/bunkernet.lua

@@ -0,0 +1,222 @@
+local _M = {}
+_M.__index = _M
+
+local utils		= require "utils"
+local datastore	= require "datastore"
+local logger	= require "logger"
+local cjson		= require "cjson"
+local http		= require "resty.http"
+
+function _M.new()
+	local self = setmetatable({}, _M)
+	local server, err = datastore:get("variable_BUNKERNET_SERVER")
+	if not server then
+		return nil, "can't get BUNKERNET_SERVER from datastore : " .. err
+	end
+	self.server = server
+	local id, err = datastore:get("plugin_bunkernet_id")
+	if not id then
+		self.id = nil
+	else
+		self.id = id
+	end
+	return self, nil
+end
+
+function _M:init()
+	local init_needed, err = utils.has_variable("USE_BUNKERNET", "yes")
+	if init_needed == nil then
+		return false, err
+	end
+	if not init_needed then
+		return true, "no service uses BunkerNet, skipping init"
+	end
+	-- Check if instance ID is present
+	local f, err = io.open("/opt/bunkerweb/cache/bunkernet/instance.id", "r")
+	if not f then
+		return false, "can't read instance id : " .. err
+	end
+	-- Retrieve instance ID
+	id = f:read("*all"):gsub("[\r\n]", "")
+	f:close()
+	self.id = id
+	-- TODO : regex check just in case
+	-- Send a ping with the ID
+	--local ok, err, status, response = self:ping()
+	-- BunkerNet server is down or instance can't access it
+	--if not ok then
+		--return false, "can't send request to BunkerNet service : " .. err
+	-- Local instance ID is unknown to the server, let's delete it
+	--elseif status == 401 then
+		--local ok, message = os.remove("/opt/bunkerweb/cache/bunkernet/instance.id")
+		--if not ok then
+			--return false, "can't remove instance ID " .. message
+		--end
+		--return false, "instance ID is not valid"
+	--elseif status == 429 then
+		--return false, "sent too many requests to the BunkerNet service"
+	--elseif status ~= 200 then
+		--return false, "unknown error from BunkerNet service (HTTP status = " .. tostring(status) .. ")"
+	--end
+	-- Store ID in datastore
+	local ok, err = datastore:set("plugin_bunkernet_id", id)
+	if not ok then
+		return false, "can't save instance ID to the datastore : " .. err
+	end
+	-- Load databases
+	local ret = true
+	local i = 0
+	local db = {
+		ip = {}
+	}
+	f, err = io.open("/opt/bunkerweb/cache/bunkernet/ip.list", "r")
+	if not f then
+		ret = false
+	else
+		for line in f:lines() do
+			if utils.is_ipv4(line) and utils.ip_is_global(line) then
+				table.insert(db.ip, line)
+				i = i + 1
+			end
+		end
+	end
+	if not ret then
+		return false, "error while reading database : " .. err
+	end
+	f:close()
+	local ok, err = datastore:set("plugin_bunkernet_db", cjson.encode(db))
+	if not ok then
+		return false, "can't store BunkerNet database into datastore : " .. err
+	end
+	return true, "successfully connected to the BunkerNet service " .. self.server .. " with machine ID " .. id .. " and " .. tostring(i) .. " bad IPs in database"
+end
+
+function _M:request(method, url, data)
+	local httpc, err = http.new()
+	if not httpc then
+		return false, "can't instantiate http object : " .. err, nil, nil
+	end
+	local all_data = {
+		id = self.id,
+		integration = utils.get_integration(),
+		version = utils.get_version()
+	}
+	for k, v in pairs(data) do
+		all_data[k] = v
+	end
+	local res, err = httpc:request_uri(self.server .. url, {
+		method = method,
+		body = cjson.encode(all_data),
+		headers = {
+			["Content-Type"] = "application/json",
+			["User-Agent"] = "BunkerWeb/" .. utils.get_version()
+		}
+	})
+	httpc:close()
+	if not res then
+		return false, "error while sending request : " .. err, nil, nil
+	end
+	if res.status ~= 200 then
+		return false, "status code != 200", res.status, nil
+	end
+	local ok, ret = pcall(cjson.decode, res.body)
+	if not ok then
+		return false, "error while decoding json : " .. ret, nil, nil
+	end
+	return true, "success", res.status, ret
+end
+
+function _M:ping()
+	return self:request("GET", "/ping", {})
+end
+
+function _M:report(ip, reason, method, url, headers)
+	local data = {
+		ip = ip,
+		reason = reason,
+		method = method,
+		url = url,
+		headers = headers
+	}
+	return self:request("POST", "/report", data)
+end
+
+function _M:log()
+	-- Check if BunkerNet is activated
+	local use_bunkernet = utils.get_variable("USE_BUNKERNET")
+	if use_bunkernet ~= "yes" then
+		return true, "bunkernet not activated"
+	end
+	-- Check if BunkerNet ID is generated
+	if not self.id then
+		return true, "bunkernet ID is not generated"
+	end
+	-- Check if IP has been blocked
+	local reason = utils.get_reason()
+	if not reason then
+		return true, "ip is not blocked"
+	end
+	if reason == "bunkernet" then
+		return true, "skipping report because the reason is bunkernet"
+	end
+	-- Check if IP is global
+	local is_global, err = utils.ip_is_global(ngx.var.remote_addr)
+	if is_global == nil then
+		return false, "error while checking if IP is global " .. err
+	end
+	if not is_global then
+		return true, "IP is not global"
+	end
+	-- Only report if it hasn't been reported for the same reason recently
+	--local reported = datastore:get("plugin_bunkernet_cache_" .. ngx.var.remote_addr .. reason)
+	--if reported then
+		--return true, "ip already reported recently"
+	--end
+	local function report_callback(premature, obj, ip, reason, method, url, headers)
+		local ok, err, status, data = obj:report(ip, reason, method, url, headers)
+		if status == 429 then
+			logger.log(ngx.WARN, "BUNKERNET", "BunkerNet API is rate limiting us")
+		elseif not ok then
+			logger.log(ngx.ERR, "BUNKERNET", "Can't report IP : " .. err)
+		else
+			logger.log(ngx.NOTICE, "BUNKERNET", "Successfully reported IP " .. ip .. " (reason : " .. reason .. ")")
+			--local ok, err = datastore:set("plugin_bunkernet_cache_" .. ip .. reason, true, 3600)
+			--if not ok then
+				--logger.log(ngx.ERR, "BUNKERNET", "Can't store cached report : " .. err)
+			--end
+		end
+	end
+	local hdr, err = ngx.timer.at(0, report_callback, self, ngx.var.remote_addr, reason, ngx.var.request_method, ngx.var.request_uri, ngx.req.get_headers())
+	if not hdr then
+		return false, "can't create report timer : " .. err
+	end
+	return true, "created report timer"
+end
+
+function _M:access()
+	local use_bunkernet = utils.get_variable("USE_BUNKERNET")
+	if use_bunkernet ~= "yes" then
+		return true, "bunkernet not activated", false, nil
+	end
+	-- Check if BunkerNet ID is generated
+	if not self.id then
+		return true, "bunkernet ID is not generated"
+	end
+	local data, err = datastore:get("plugin_bunkernet_db")
+	if not data then
+		return false, "can't get bunkernet db : " .. err, false, nil
+	end
+	local db = cjson.decode(data)
+	for index, value in ipairs(db.ip) do
+		if value == ngx.var.remote_addr then
+			return true, "ip is in database", true, ngx.exit(ngx.HTTP_FORBIDDEN)
+		end
+	end
+	return true, "ip is not in database", false, nil
+end
+
+function _M:api()
+	return false, nil, nil
+end
+
+return _M

core/bunkernet/confs/default-server-http/bunkernet.conf

@@ -0,0 +1,51 @@
+log_by_lua_block {
+	local bunkernet					= require "bunkernet.bunkernet"
+	local utils 					= require "utils"
+	local datastore					= require "datastore"
+	local logger					= require "logger"
+	local disable_default_server	= utils.get_variable("DISABLE_DEFAULT_SERVER", false)
+	local use_bunkernet				= utils.has_variable("USE_BUNKERNET", "yes")
+	
+	if disable_default_server == "yes" and use_bunkernet then
+		-- Instantiate bunkernet
+		local bnet, err = bunkernet.new()
+		if not bnet then
+			ngx.log(ngx.ERR, "BUNKERNET", "can't instantiate bunkernet " .. err)
+			return
+		end
+		-- Check if BunkerNet ID is generated
+		if not bnet.id then
+			return
+		end
+		-- Check if IP has been blocked
+		if ngx.status ~= ngx.HTTP_CLOSE then
+			return
+		end
+		-- Only report if it hasn't been reported for the same reason recently
+		local reported = datastore:get("plugin_bunkernet_cache_" .. ngx.var.remote_addr .. "default")
+		if reported then
+			return
+		end
+		-- report callback called in a light thread
+		local function report_callback(premature, obj, ip, reason, method, url, headers)
+			local ok, err, status, data = obj:report(ip, reason, method, url, headers)
+			if not ok then
+				logger.log(ngx.ERR, "BUNKERNET", "Can't report IP : " .. err)
+			elseif status ~= 200 then
+				logger.log(ngx.ERR, "BUNKERNET", "Error from remote server : " .. tostring(status))
+			else
+				logger.log(ngx.NOTICE, "BUNKERNET", "Successfully reported IP " .. ip .. " (reason : " .. reason .. ")")
+				local ok, err = datastore:set("plugin_bunkernet_cache_" .. ip .. reason, true, 3600)
+				if not ok then
+					logger.log(ngx.ERR, "BUNKERNET", "Can't store cached report : " .. err)
+				end
+			end
+		end
+		-- Set a timer at the end of log()
+		local hdr, err = ngx.timer.at(0, report_callback, bnet, ngx.var.remote_addr, "default", ngx.var.request_method, ngx.var.request_uri, ngx.req.get_headers())
+		if not hdr then
+			logger.log(ngx.ERR, "BUNKERNET", "can't create report timer : " .. err)
+		end
+		return
+	end
+}

core/bunkernet/jobs/bunkernet-data.py

@@ -0,0 +1,85 @@
+#!/usr/bin/python3
+
+import sys, os, traceback
+
+sys.path.append("/opt/bunkerweb/deps/python")
+sys.path.append("/opt/bunkerweb/utils")
+sys.path.append("/opt/bunkerweb/core/bunkernet/jobs")
+
+import logger, jobs
+from bunkernet import data
+
+status = 0
+
+try :
+
+    # Check if at least a server has BunkerNet activated
+    bunkernet_activated = False
+    # Multisite case
+    if os.getenv("MULTISITE") == "yes" :
+        for first_server in os.getenv("SERVER_NAME").split(" ") :
+            if os.getenv(first_server + "_USE_BUNKERNET", os.getenv("USE_BUNKERNET")) == "yes" :
+                bunkernet_activated = True
+                break
+    # Singlesite case
+    elif os.getenv("USE_BUNKERNET") == "yes" :
+        bunkernet_activated = True
+    if not bunkernet_activated :
+        logger.log("BUNKERNET", "ℹ️", "BunkerNet is not activated, skipping download...")
+        os._exit(0)
+    
+    # Create directory if it doesn't exist
+    os.makedirs("/opt/bunkerweb/cache/bunkernet", exist_ok=True)
+    
+    # Check if ID is present
+    if not os.path.isfile("/opt/bunkerweb/cache/bunkernet/instance.id") :
+        logger.log("BUNKERNET", "❌", "Not downloading BunkerNet data because instance is not registered")
+        os._exit(2)
+
+    # Don't go further if the cache is fresh
+    if jobs.is_cached_file("/opt/bunkerweb/cache/bunkernet/ip.list", "day") :
+        logger.log("BUNKERNET", "ℹ️", "BunkerNet list is already in cache, skipping download...")
+        os._exit(0)
+
+    # Download data
+    logger.log("BUNKERNET", "ℹ️", "Downloading BunkerNet data ...")
+    ok, status, data = data()
+    if not ok :
+        logger.log("BUNKERNET", "❌", "Error while sending data request to BunkerNet API : " + data)
+        os._exit(2)
+    elif status == 429 :
+        logger.log("BUNKERNET", "⚠️", "BunkerNet API is rate limiting us, trying again later...")
+        os._exit(0)
+    elif data["result"] != "ok" :
+        logger.log("BUNKERNET", "❌", "Received error from BunkerNet API while sending db request : " + data["data"] + ", removing instance ID")
+        os._exit(2)
+    logger.log("BUNKERNET", "ℹ️", "Successfully downloaded data from BunkerNet API")
+
+    # Writing data to file
+    logger.log("BUNKERNET", "ℹ️", "Saving BunkerNet data ...")
+    with open("/opt/bunkerweb/tmp/bunkernet-ip.list", "w") as f :
+        for ip in data["data"] :
+            f.write(ip + "\n")
+    
+    # Check if file has changed
+    file_hash = jobs.file_hash("/opt/bunkerweb/tmp/bunkernet-ip.list")
+    cache_hash = jobs.cache_hash("/opt/bunkerweb/cache/bunkernet/ip.list")
+    if file_hash == cache_hash :
+        logger.log("BUNKERNET", "ℹ️", "New file is identical to cache file, reload is not needed")
+        os._exit(0)
+
+    # Put file in cache
+    cached, err = jobs.cache_file("/opt/bunkerweb/tmp/bunkernet-ip.list", "/opt/bunkerweb/cache/bunkernet/ip.list", file_hash)
+    if not cached :
+        logger.log("BUNKERNET", "❌", "Error while caching BunkerNet data : " + err)
+        os._exit(2)
+    logger.log("BUNKERNET", "ℹ️", "Successfully saved BunkerNet data")
+
+    status = 1
+
+except :
+    status = 2
+    logger.log("BUNKERNET", "❌", "Exception while running bunkernet-data.py :")
+    print(traceback.format_exc())
+
+sys.exit(status)

core/bunkernet/jobs/bunkernet-register.py

@@ -0,0 +1,81 @@
+#!/usr/bin/python3
+
+import sys, os, traceback
+
+sys.path.append("/opt/bunkerweb/deps/python")
+sys.path.append("/opt/bunkerweb/utils")
+sys.path.append("/opt/bunkerweb/core/bunkernet/jobs")
+
+import logger
+from bunkernet import register, ping, get_id
+
+status = 0
+
+try :
+
+    # Check if at least a server has BunkerNet activated
+    bunkernet_activated = False
+    # Multisite case
+    if os.getenv("MULTISITE") == "yes" :
+        for first_server in os.getenv("SERVER_NAME").split(" ") :
+            if os.getenv(first_server + "_USE_BUNKERNET", os.getenv("USE_BUNKERNET")) == "yes" :
+                bunkernet_activated = True
+                break
+    # Singlesite case
+    elif os.getenv("USE_BUNKERNET") == "yes" :
+        bunkernet_activated = True
+    if not bunkernet_activated :
+        logger.log("BUNKERNET", "ℹ️", "BunkerNet is not activated, skipping registration...")
+        os._exit(0)
+    
+    # Create directory if it doesn't exist
+    os.makedirs("/opt/bunkerweb/cache/bunkernet", exist_ok=True)
+    
+    # Ask an ID if needed
+    if not os.path.isfile("/opt/bunkerweb/cache/bunkernet/instance.id") :
+        logger.log("BUNKERNET", "ℹ️", "Registering instance on BunkerNet API ...")
+        ok, status, data = register()
+        if not ok :
+            logger.log("BUNKERNET", "❌", "Error while sending register request to BunkerNet API : " + data)
+            os._exit(1)
+        elif status == 429 :
+            logger.log("BUNKERNET", "⚠️", "BunkerNet API is rate limiting us, trying again later...")
+            os._exit(0)
+        elif status != 200 :
+            logger.log("BUNKERNET", "❌", "Error " + str(status) + " from BunkerNet API : " + data["data"])
+            os._exit(1)
+        elif data["result"] != "ok" :
+            logger.log("BUNKERNET", "❌", "Received error from BunkerNet API while sending register request : " + data["data"])
+            os._exit(1)
+        with open("/opt/bunkerweb/cache/bunkernet/instance.id", "w") as f :
+            f.write(data["data"])
+        logger.log("BUNKERNET", "ℹ️", "Successfully registered on BunkerNet API with instance id " + get_id())
+    else :
+        logger.log("BUNKERNET", "ℹ️", "Already registered on BunkerNet API with instance id " + get_id())
+
+    # Ping
+    logger.log("BUNKERNET", "ℹ️", "Checking connectivity with BunkerNet API ...")
+    ok, status, data = ping()
+    if not ok :
+        logger.log("BUNKERNET", "❌", "Error while sending ping request to BunkerNet API : " + data)
+        os._exit(2)
+    elif status == 429 :
+        logger.log("BUNKERNET", "⚠️", "BunkerNet API is rate limiting us, trying again later...")
+        os._exit(0)
+    elif status == 401 :
+        logger.log("BUNKERNET", "⚠️", "Instance ID is not registered, removing it and retrying a register later...")
+        os.remove("/opt/bunkerweb/cache/bunkernet/instance.id")
+        os._exit(1)
+    elif data["result"] != "ok" :
+        logger.log("BUNKERNET", "❌", "Received error from BunkerNet API while sending ping request : " + data["data"] + ", removing instance ID")
+        os._exit(1)
+    logger.log("BUNKERNET", "ℹ️", "Successfully checked connectivity with BunkerNet API")
+
+    status = 1
+
+except :
+    status = 2
+    logger.log("BUNKERNET", "❌", "Exception while running bunkernet-register.py :")
+    print(traceback.format_exc())
+
+sys.exit(status)

core/bunkernet/jobs/bunkernet.py

@@ -0,0 +1,55 @@
+import requests, traceback
+from os import getenv
+
+def request(method, url, _id=None) :
+    data = {
+        "integration": get_integration(),
+        "version": get_version()
+    }
+    headers = {
+        "User-Agent": "BunkerWeb/" + get_version()
+    }
+    if _id is not None :
+        data["id"] = _id
+    try :
+        resp = requests.request(method, getenv("BUNKERNET_SERVER") + url, json=data, headers=headers, timeout=5)
+        status = resp.status_code
+        if status == 429 :
+            return True, 429, "rate limited"
+        raw_data = resp.json()
+        assert "result" in raw_data
+        assert "data" in raw_data
+    except Exception as e :
+        return False, None, traceback.format_exc()
+    return True, status, raw_data
+    
+def register() :
+    return request("POST", "/register")
+
+def ping() :
+    return request("GET", "/ping", _id=get_id())
+
+def data() :
+    return request("GET", "/db", _id=get_id())
+
+def get_id() :
+	with open("/opt/bunkerweb/cache/bunkernet/instance.id", "r") as f :
+		return f.read().strip()
+
+def get_version() :
+    with open("/opt/bunkerweb/VERSION", "r") as f :
+        return f.read().strip()
+
+def get_integration() :
+    try :
+        if getenv("KUBERNETES_MODE") == "yes" :
+            return "kubernetes"
+        if getenv("SWARM_MODE") == "yes" :
+            return "swarm"
+        with open("/etc/os-release", "r") as f :
+            if f.read().contains("Alpine") :
+                return "docker"
+        return "linux"
+    except :
+        return "unknown"
+

core/bunkernet/plugin.json

@@ -0,0 +1,41 @@
+{
+	"id": "bunkernet",
+	"order": 2,
+	"name": "BunkerNet",
+	"description": "Share threat data with other BunkerWeb instances via BunkerNet.",
+	"version": "0.1",
+	"settings": {
+		"USE_BUNKERNET": {
+			"context": "multisite",
+			"default": "yes",
+			"help": "Activate BunkerNet feature.",
+			"id": "use-bunkernet",
+			"label": "Activate BunkerNet",
+			"regex": "^(yes|no)$",
+			"type": "check"
+		},
+		"BUNKERNET_SERVER": {
+			"context": "global",
+			"default": "https://api.bunkerweb.io",
+			"help": "Address of the BunkerNet API.",
+			"id": "bunkernet-server",
+			"label": "BunkerNet server",
+			"regex": "^.*$",
+			"type": "text"
+		}
+	},
+	"jobs": [
+		{
+			"name": "bunkernet-register",
+			"file": "bunkernet-register.py",
+			"every": "hour",
+			"reload": true
+		},
+		{
+			"name": "bunkernet-data",
+			"file": "bunkernet-data.py",
+			"every": "hour",
+			"reload": true
+		}
+	]
+}