Makussu/bunkerweb
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

bunkerweb 1.4.0

Browse Source
This commit is contained in:
bunkerity
2022-06-03 17:24:14 +02:00
parent 3a078326c5
commit a9f886804a
5245 changed files with 1432051 additions and 27894 deletions
Download Patch File Download Diff File Expand all files Collapse all files

152
core/whitelist/jobs/whitelist-download.py Executable file
View File

@@ -0,0 +1,152 @@
#!/usr/bin/python3
import sys, os, traceback
sys.path.append("/opt/bunkerweb/deps/python")
sys.path.append("/opt/bunkerweb/utils")
import logger, jobs, requests, ipaddress
def check_line(kind, line) :
if kind == "IP" :
if "/" in line :
try :
ipaddress.ip_network(line)
return True, line
except :
pass
else :
try :
ipaddress.ip_address(line)
return True, line
except :
pass
return False, ""
elif kind == "RDNS" :
if re.match(r"^(\.?[A-Za-z0-9\-]+)*\.[A-Za-z]{2,}$", line) :
return True, line.lower()
return False, ""
elif kind == "ASN" :
real_line = line.replace("AS", "")
if re.match(r"^\d+$", real_line) :
return True, real_line
elif kind == "USER_AGENT" :
return True, line.replace("\\ ", " ").replace("\\.", "%.").replace("\\\\", "\\").replace("-", "%-")
elif kind == "URI" :
if re.match(r"^/", line) :
return True, line
return False, ""
status = 0
try :
# Check if at least a server has Whitelist activated
whitelist_activated = False
# Multisite case
if os.getenv("MULTISITE") == "yes" :
for first_server in os.getenv("SERVER_NAME").split(" ") :
if os.getenv(first_server + "_USE_WHITELIST", os.getenv("USE_WHITELIST")) == "yes" :
whitelist_activated = True
break
# Singlesite case
elif os.getenv("USE_WHITELIST") == "yes" :
whitelist_activated = True
if not whitelist_activated :
logger.log("WHITELIST", "ℹ️", "Whitelist is not activated, skipping downloads...")
os._exit(0)
# Create directories if they don't exist
os.makedirs("/opt/bunkerweb/cache/whitelist", exist_ok=True)
os.makedirs("/opt/bunkerweb/tmp/whitelist", exist_ok=True)
# Our urls data
urls = {
"IP": [],
"RDNS": [],
"ASN" : [],
"USER_AGENT": [],
"URI": []
}
# Don't go further if the cache is fresh
kinds_fresh = {
"IP": True,
"RDNS": True,
"ASN" : True,
"USER_AGENT": True,
"URI": True
}
all_fresh = True
for kind in kinds_fresh :
if not jobs.is_cached_file("/opt/bunkerweb/cache/whitelist/" + kind + ".list", "hour") :
kinds_fresh[kind] = False
all_fresh = False
logger.log("WHITELIST", "ℹ️", "Whitelist for " + kind + " is not cached, processing downloads...")
else :
logger.log("WHITELIST", "ℹ️", "Whitelist for " + kind + " is already in cache, skipping downloads...")
if all_fresh :
os._exit(0)
# Get URLs
urls = {
"IP": [],
"RDNS": [],
"ASN" : [],
"USER_AGENT": [],
"URI": []
}
for kind in urls :
for url in os.getenv("WHITELIST_" + kind + "_URLS", "").split(" ") :
if url != "" and url not in urls[kind] :
urls[kind].append(url)
# Loop on kinds
for kind, urls_list in urls.items() :
if kinds_fresh[kind] :
continue
# Write combined data of the kind to a single temp file
for url in urls_list :
try :
logger.log("WHITELIST", "ℹ️", "Downloading whitelist data from " + url + " ...")
resp = requests.get(url, stream=True)
if resp.status_code != 200 :
continue
i = 0
with open("/opt/bunkerweb/tmp/whitelist/" + kind + ".list", "w") as f :
for line in resp.iter_lines(decode_unicode=True) :
line = line.strip()
if kind != "USER_AGENT" :
line = line.strip().split(" ")[0]
if line == "" or line.startswith("#") or line.startswith(";") :
continue
ok, data = check_line(kind, line)
if ok :
f.write(data + "\n")
i += 1
logger.log("WHITELIST", "ℹ️", "Downloaded " + str(i) + " bad " + kind)
# Check if file has changed
file_hash = jobs.file_hash("/opt/bunkerweb/tmp/whitelist/" + kind + ".list")
cache_hash = jobs.cache_hash("/opt/bunkerweb/cache/whitelist/" + kind + ".list")
if file_hash == cache_hash :
logger.log("WHITELIST", "ℹ️", "New file " + kind + ".list is identical to cache file, reload is not needed")
else :
logger.log("WHITELIST", "ℹ️", "New file " + kind + ".list is different than cache file, reload is needed")
# Put file in cache
cached, err = jobs.cache_file("/opt/bunkerweb/tmp/whitelist/" + kind + ".list", "/opt/bunkerweb/cache/whitelist/" + kind + ".list", file_hash)
if not cached :
logger.log("WHITELIST", "❌", "Error while caching whitelist : " + err)
status = 2
if status != 2 :
status = 1
except :
status = 2
logger.log("WHITELIST", "❌", "Exception while getting whitelist from " + url + " :")
print(traceback.format_exc())
except :
status = 2
logger.log("WHITELIST", "❌", "Exception while running whitelist-download.py :")
print(traceback.format_exc())
sys.exit(status)

125
core/whitelist/plugin.json Normal file
View File

@@ -0,0 +1,125 @@
{
"id": "whitelist",
"order": 2,
"name": "Whitelist",
"description": "Allow access based on internal and external IP/network/rDNS/ASN whitelists.",
"version": "0.1",
"settings": {
"USE_WHITELIST": {
"context": "multisite",
"default": "yes",
"help": "Activate whitelist feature.",
"id": "use-whitelist",
"label": "Activate whitelisting",
"regex": "^(yes|no)$",
"type": "check"
},
"WHITELIST_IP_URLS": {
"context": "global",
"default": "",
"help": "List of URLs, separated with spaces, containing good IP/network to whitelist.",
"id": "whitelist-ip-urls",
"label": "Whitelist IP/network URLs",
"regex": "^.*$",
"type": "text"
},
"WHITELIST_IP": {
"context": "multisite",
"default": "20.191.45.212 40.88.21.235 40.76.173.151 40.76.163.7 20.185.79.47 52.142.26.175 20.185.79.15 52.142.24.149 40.76.162.208 40.76.163.23 40.76.162.191 40.76.162.247 54.208.102.37 107.21.1.8",
"help": "List of IP/network, separated with spaces, to whitelist.",
"id": "whitelist-ip",
"label": "Whitelist IP/network",
"regex": "^.*$",
"type": "text"
},
"WHITELIST_RDNS": {
"context": "multisite",
"default": ".google.com .googlebot.com .yandex.ru .yandex.net .yandex.com .search.msn.com .baidu.com .baidu.jp .crawl.yahoo.net .fwd.linkedin.com .twitter.com .twttr.com .discord.com",
"help": "List of reverse DNS suffixes, separated with spaces, to whitelist.",
"id": "whitelist-rdns",
"label": "Whitelist reverse DNS",
"regex": "^.*$",
"type": "text"
},
"WHITELIST_RDNS_URLS": {
"context": "global",
"default": "",
"help": "List of URLs, separated with spaces, containing reverse DNS suffixes to whitelist.",
"id": "whitelist-rdns-urls",
"label": "Whitelist reverse DNS URLs",
"regex": "^.*$",
"type": "text"
},
"WHITELIST_RDNS_GLOBAL": {
"context": "multisite",
"default": "yes",
"help": "Only perform RDNS whitelist checks on global IP addresses.",
"id": "whitelist-rdns-global",
"label": "Whitelist reverse DNS global IPs",
"regex": "^.*$",
"type": "text"
},
"WHITELIST_ASN": {
"context": "multisite",
"default": "32934",
"help": "List of ASN numbers, separated with spaces, to whitelist.",
"id": "whitelist-asn",
"label": "Whitelist ASN",
"regex": "^.*$",
"type": "text"
},
"WHITELIST_ASN_URLS": {
"context": "global",
"default": "",
"help": "List of URLs, separated with spaces, containing ASN to whitelist.",
"id": "whitelist-rdns-urls",
"label": "Whitelist ASN URLs",
"regex": "^.*$",
"type": "text"
},
"WHITELIST_USER_AGENT": {
"context": "multisite",
"default": "",
"help": "List of User-Agent, separated with spaces, to whitelist.",
"id": "whitelist-user-agent",
"label": "Whitelist User-Agent",
"regex": "^.*$",
"type": "text"
},
"WHITELIST_USER_AGENT_URLS": {
"context": "global",
"default": "",
"help": "List of URLs, separated with spaces, containing good User-Agent to whitelist.",
"id": "whitelist-user-agent-urls",
"label": "Whitelist User-Agent URLs",
"regex": "^.*$",
"type": "text"
},
"WHITELIST_URI": {
"context": "multisite",
"default": "",
"help": "List of URI, separated with spaces, to whitelist.",
"id": "whitelist-uri",
"label": "Whitelist URI",
"regex": "^.*$",
"type": "text"
},
"WHITELIST_URI_URLS": {
"context": "global",
"default": "",
"help": "List of URLs, separated with spaces, containing bad URI to whitelist.",
"id": "whitelist-uri-urls",
"label": "Whitelist URI URLs",
"regex": "^.*$",
"type": "text"
}
},
"jobs": [
{
"name": "whitelist-download",
"file": "whitelist-download.py",
"every": "hour",
"reload": true
}
]
}

245
core/whitelist/whitelist.lua Normal file
View File

@@ -0,0 +1,245 @@
local _M = {}
_M.__index = _M
local utils = require "utils"
local datastore = require "datastore"
local logger = require "logger"
local cjson = require "cjson"
local ipmatcher = require "resty.ipmatcher"
function _M.new()
local self = setmetatable({}, _M)
return self, nil
end
function _M:init()
-- Check if init is needed
local init_needed, err = utils.has_variable("USE_WHITELIST", "yes")
if init_needed == nil then
return false, err
end
if not init_needed then
return true, "no service uses Whitelist, skipping init"
end
-- Read whitelists
local whitelists = {
["IP"] = {},
["RDNS"] = {},
["ASN"] = {},
["USER_AGENT"] = {},
["URI"] = {}
}
local i = 0
for kind, _ in pairs(whitelists) do
local f, err = io.open("/opt/bunkerweb/cache/whitelist/" .. kind .. ".list", "r")
if f then
for line in f:lines() do
table.insert(whitelists[kind], line)
i = i + 1
end
f:close()
end
end
-- Load them into datastore
local ok, err = datastore:set("plugin_whitelist_list", cjson.encode(whitelists))
if not ok then
return false, "can't store Whitelist list into datastore : " .. err
end
return true, "successfully loaded " .. tostring(i) .. " whitelisted IP/network/rDNS/ASN/User-Agent/URI"
end
function _M:access()
-- Check if access is needed
local access_needed, err = utils.get_variable("USE_WHITELIST")
if access_needed == nil then
return false, err
end
if access_needed ~= "yes" then
return true, "Whitelist not activated"
end
-- Check the cache
local cached_ip, err = self:is_in_cache("ip" .. ngx.var.remote_addr)
if cached_ip and cached_ip ~= "ok" then
return true, "IP is in whitelist cache (info = " .. cached_ip .. ")", true, ngx.OK
end
local cached_uri, err = self:is_in_cache("uri" .. ngx.var.uri)
if cached_uri and cached_uri ~= "ok" then
return true, "URI is in whitelist cache (info = " .. cached_uri .. ")", true, ngx.OK
end
local cached_ua = true
if ngx.var.http_user_agent then
cached_ua, err = self:is_in_cache("ua" .. ngx.var.http_user_agent)
if cached_ua and cached_ua ~= "ok" then
return true, "User-Agent is in whitelist cache (info = " .. cached_ua .. ")", true, ngx.OK
end
end
if cached_ip and cached_uri and cached_ua then
return true, "full request is in whitelist cache (not whitelisted)", nil, nil
end
-- Get list
local data, err = datastore:get("plugin_whitelist_list")
if not data then
return false, "can't get Whitelist list : " .. err, false, nil
end
local ok, whitelists = pcall(cjson.decode, data)
if not ok then
return false, "error while decoding whitelists : " .. whitelists, false, nil
end
-- Return value
local ret, ret_err = true, "success"
-- Check if IP is in IP/net whitelist
local ip_net, err = utils.get_variable("WHITELIST_IP")
if ip_net and ip_net ~= "" then
for element in ip_net:gmatch("%S+") do
table.insert(whitelists["IP"], element)
end
end
if not cached_ip then
local ipm, err = ipmatcher.new(whitelists["IP"])
if not ipm then
ret = false
ret_err = "can't instantiate ipmatcher " .. err
else
if ipm:match(ngx.var.remote_addr) then
self:add_to_cache("ip" .. ngx.var.remote_addr, "ip/net")
return ret, "client IP " .. ngx.var.remote_addr .. " is in whitelist", true, ngx.OK
end
end
end
-- Check if rDNS is in whitelist
local rdns_global, err = utils.get_variable("WHITELIST_RDNS_GLOBAL")
local check = true
if not rdns_global then
logger.log(ngx.ERR, "WHITELIST", "Error while getting WHITELIST_RDNS_GLOBAL variable : " .. err)
elseif rdns_global == "yes" then
check, err = utils.ip_is_global(ngx.var.remote_addr)
if check == nil then
logger.log(ngx.ERR, "WHITELIST", "Error while getting checking if IP is global : " .. err)
end
end
if not cached_ip and check then
local rdns, err = utils.get_rdns(ngx.var.remote_addr)
if not rdns then
ret = false
ret_err = "error while trying to get reverse dns : " .. err
else
local rdns_list, err = utils.get_variable("WHITELIST_RDNS")
if rdns_list and rdns_list ~= "" then
for element in rdns_list:gmatch("%S+") do
table.insert(whitelists["RDNS"], element)
end
end
for i, suffix in ipairs(whitelists["RDNS"]) do
if rdns:sub(-#suffix) == suffix then
self:add_to_cache("ip" .. ngx.var.remote_addr, "rDNS " .. suffix)
return ret, "client IP " .. ngx.var.remote_addr .. " is in whitelist (info = rDNS " .. suffix .. ")", true, ngx.OK
end
end
end
end
-- Check if ASN is in whitelist
if not cached_ip then
if utils.ip_is_global(ngx.var.remote_addr) then
local asn, err = utils.get_asn(ngx.var.remote_addr)
if not asn then
ret = false
ret_err = "error while trying to get asn number : " .. err
else
local asn_list, err = utils.get_variable("WHITELIST_ASN")
if asn_list and asn_list ~= "" then
for element in asn_list:gmatch("%S+") do
table.insert(whitelists["ASN"], element)
end
end
for i, asn_bl in ipairs(whitelists["ASN"]) do
if tostring(asn) == asn_bl then
self:add_to_cache("ip" .. ngx.var.remote_addr, "ASN " .. tostring(asn))
return ret, "client IP " .. ngx.var.remote_addr .. " is in whitelist (kind = ASN " .. tostring(asn) .. ")", true, ngx.OK
end
end
end
end
end
-- IP is not whitelisted
local ok, err = self:add_to_cache("ip" .. ngx.var.remote_addr, "ok")
if not ok then
ret = false
ret_err = err
end
-- Check if User-Agent is in whitelist
if not cached_ua and ngx.var.http_user_agent then
local ua_list, err = utils.get_variable("WHITELIST_USER_AGENT")
if ua_list and ua_list ~= "" then
for element in ua_list:gmatch("%S+") do
table.insert(whitelists["USER_AGENT"], element)
end
end
for i, ua_bl in ipairs(whitelists["USER_AGENT"]) do
if ngx.var.http_user_agent:match(ua_bl) then
self:add_to_cache("ua" .. ngx.var.http_user_agent, "UA " .. ua_bl)
return ret, "client User-Agent " .. ngx.var.http_user_agent .. " is in whitelist (matched " .. ua_bl .. ")", true, ngx.OK
end
end
-- UA is not whitelisted
local ok, err = self:add_to_cache("ua" .. ngx.var.http_user_agent, "ok")
if not ok then
ret = false
ret_err = err
end
end
-- Check if URI is in whitelist
if not cached_uri then
local uri_list, err = utils.get_variable("WHITELIST_URI")
if uri_list and uri_list ~= "" then
for element in uri_list:gmatch("%S+") do
table.insert(whitelists["URI"], element)
end
end
for i, uri_bl in ipairs(whitelists["URI"]) do
if ngx.var.uri:match(uri_bl) then
self:add_to_cache("uri" .. ngx.var.uri, "URI " .. uri_bl)
return ret, "client URI " .. ngx.var.uri .. " is in whitelist (matched " .. uri_bl .. ")", true, ngx.OK
end
end
end
-- URI is not whitelisted
local ok, err = self:add_to_cache("uri" .. ngx.var.uri, "ok")
if not ok then
ret = false
ret_err = err
end
return ret, "IP is not in list (error = " .. ret_err .. ")", false, nil
end
function _M:is_in_cache(ele)
local kind, err = datastore:get("plugin_whitelist_cache_" .. ngx.var.server_name .. ele)
if not kind then
if err ~= "not found" then
logger.log(ngx.ERR, "WHITELIST", "Error while accessing cache : " .. err)
end
return false, err
end
return kind, "success"
end
function _M:add_to_cache(ele, kind)
local ok, err = datastore:set("plugin_whitelist_cache_" .. ngx.var.server_name .. ele, kind, 3600)
if not ok then
logger.log(ngx.ERR, "WHITELIST", "Error while adding element to cache : " .. err)
return false, err
end
return true, "success"
end
return _M

35
core/whitelist/whitelist.txt Normal file
View File

@@ -0,0 +1,35 @@
RDNS
.google.com
.googlebot.com
.yandex.ru
.yandex.net
.yandex.com
.search.msn.com
.baidu.com
.baidu.jp
.crawl.yahoo.net
.fwd.linkedin.com
.twitter.com
.twttr.com
.discord.com
ASN
AS32934 facebook
IP
duckduckgo
20.191.45.212
40.88.21.235
40.76.173.151
40.76.163.7
20.185.79.47
52.142.26.175
20.185.79.15
52.142.24.149
40.76.162.208
40.76.163.23
40.76.162.191
40.76.162.247
54.208.102.37
107.21.1.8