From fd61df205fca6568b91006ff3c152edb1bb7d7aa Mon Sep 17 00:00:00 2001 From: bunkerity Date: Mon, 10 May 2021 17:51:07 +0200 Subject: [PATCH] performance - move external blacklists checks from nginx to LUA --- .../{block-abusers.conf => abusers.list} | 0 confs/global/init-lua.conf | 21 +++++++++++++ confs/global/nginx.conf | 12 +++++++ .../{block-proxies.conf => proxies.list} | 0 ...tor-exit-node.conf => tor-exit-nodes.list} | 0 confs/site/main-lua.conf | 31 +++++++++++++++++++ confs/site/server.conf | 3 -- entrypoint/entrypoint.sh | 2 +- entrypoint/global-config.sh | 15 +++++++++ entrypoint/site-config.sh | 12 +++---- lua/dataloader.lua | 31 +++++++++++++++++++ scripts/abusers.sh | 17 +++++----- scripts/exit-nodes.sh | 18 +++++------ scripts/proxies.sh | 17 +++++----- 14 files changed, 141 insertions(+), 38 deletions(-) rename confs/global/{block-abusers.conf => abusers.list} (100%) create mode 100644 confs/global/init-lua.conf rename confs/global/{block-proxies.conf => proxies.list} (100%) rename confs/global/{block-tor-exit-node.conf => tor-exit-nodes.list} (100%) create mode 100644 lua/dataloader.lua diff --git a/confs/global/block-abusers.conf b/confs/global/abusers.list similarity index 100% rename from confs/global/block-abusers.conf rename to confs/global/abusers.list diff --git a/confs/global/init-lua.conf b/confs/global/init-lua.conf new file mode 100644 index 0000000..b110a1c --- /dev/null +++ b/confs/global/init-lua.conf @@ -0,0 +1,21 @@ +init_by_lua_block { + +local dataloader = require "dataloader" + +local use_proxies = %USE_PROXIES% +local use_abusers = %USE_ABUSERS% +local use_tor_exit_nodes = %USE_TOR_EXIT_NODES% + +if use_proxies then + dataloader.load_ip("/etc/nginx/proxies.list", ngx.shared.proxies_data) +end + +if use_abusers then + dataloader.load_ip("/etc/nginx/abusers.list", ngx.shared.abusers_data) +end + +if use_tor_exit_nodes then + dataloader.load_ip("/etc/nginx/tor-exit-nodes.list", ngx.shared.tor_exit_nodes_data) +end + +} diff --git a/confs/global/nginx.conf b/confs/global/nginx.conf index b1127e6..32eecca 100644 --- a/confs/global/nginx.conf +++ b/confs/global/nginx.conf @@ -94,6 +94,15 @@ http { # shared memory zone for limit_conn %LIMIT_CONN_ZONE% + # shared memory zone for proxies + %BLOCK_PROXIES% + + # shared memory zone for abusers + %BLOCK_ABUSERS% + + # shared memory zone for TOR exit nodes + %BLOCK_TOR_EXIT_NODES% + # whitelist or blacklist country %USE_COUNTRY% @@ -109,6 +118,9 @@ http { # custom http confs include /http-confs/*.conf; + # LUA init block + include /etc/nginx/init-lua.conf; + # default server when MULTISITE=yes %MULTISITE_DEFAULT_SERVER% diff --git a/confs/global/block-proxies.conf b/confs/global/proxies.list similarity index 100% rename from confs/global/block-proxies.conf rename to confs/global/proxies.list diff --git a/confs/global/block-tor-exit-node.conf b/confs/global/tor-exit-nodes.list similarity index 100% rename from confs/global/block-tor-exit-node.conf rename to confs/global/tor-exit-nodes.list diff --git a/confs/site/main-lua.conf b/confs/site/main-lua.conf index 78c27e6..01ed144 100644 --- a/confs/site/main-lua.conf +++ b/confs/site/main-lua.conf @@ -7,6 +7,9 @@ local use_lets_encrypt = %USE_LETS_ENCRYPT% local use_whitelist_ip = %USE_WHITELIST_IP% local use_whitelist_reverse = %USE_WHITELIST_REVERSE% local use_user_agent = %USE_USER_AGENT% +local use_proxies = %USE_PROXIES% +local use_abusers = %USE_ABUSERS% +local use_tor_exit_nodes = %USE_TOR_EXIT_NODES% local use_referrer = %USE_REFERRER% local use_country = %USE_COUNTRY% local use_blacklist_ip = %USE_BLACKLIST_IP% @@ -26,6 +29,7 @@ local cookie = require "cookie" local javascript = require "javascript" local captcha = require "captcha" local recaptcha = require "recaptcha" +local iputils = require "resty.iputils" -- user variables local antibot_uri = "%ANTIBOT_URI%" @@ -94,6 +98,33 @@ if use_blacklist_reverse and not blacklist.reverse_cached() then end end +-- check if IP is in proxies list +if use_proxies then + local value, flags = ngx.shared.proxies_data:get(iputils.ip2bin(ngx.var.remote_addr)) + if value ~= nil then + ngx.log(ngx.NOTICE, "[BLOCK] IP " .. ngx.var.remote_addr .. " is in proxies list") + ngx.exit(ngx.HTTP_FORBIDDEN) + end +end + +-- check if IP is in abusers list +if use_abusers then + local value, flags = ngx.shared.abusers_data:get(iputils.ip2bin(ngx.var.remote_addr)) + if value ~= nil then + ngx.log(ngx.NOTICE, "[BLOCK] IP " .. ngx.var.remote_addr .. " is in abusers list") + ngx.exit(ngx.HTTP_FORBIDDEN) + end +end + +-- check if IP is in TOR exit nodes list +if use_tor_exit_nodes then + local value, flags = ngx.shared.tor_exit_nodes_data:get(iputils.ip2bin(ngx.var.remote_addr)) + if value ~= nil then + ngx.log(ngx.NOTICE, "[BLOCK] IP " .. ngx.var.remote_addr .. " is in TOR exit nodes list") + ngx.exit(ngx.HTTP_FORBIDDEN) + end +end + -- check if user-agent is allowed if use_user_agent and ngx.var.bad_user_agent == "yes" then local block = true diff --git a/confs/site/server.conf b/confs/site/server.conf index ef90871..3ed54b2 100644 --- a/confs/site/server.conf +++ b/confs/site/server.conf @@ -27,9 +27,6 @@ server { %REFERRER_POLICY% %FEATURE_POLICY% %PERMISSIONS_POLICY% - %BLOCK_TOR_EXIT_NODE% - %BLOCK_PROXIES% - %BLOCK_ABUSERS% %COOKIE_FLAGS% %ERRORS% %USE_FAIL2BAN% diff --git a/entrypoint/entrypoint.sh b/entrypoint/entrypoint.sh index c00bad0..6c77409 100644 --- a/entrypoint/entrypoint.sh +++ b/entrypoint/entrypoint.sh @@ -132,7 +132,7 @@ else fi # list of log files to display -LOGS="/var/log/access.log /var/log/error.log /var/log/jobs.log /var/log/nginx/modsec_audit.log" +LOGS="/var/log/access.log /var/log/error.log /var/log/jobs.log /var/log/nginx/error.log /var/log/nginx/modsec_audit.log" # start fail2ban if [ "$USE_FAIL2BAN" = "yes" ] ; then diff --git a/entrypoint/global-config.sh b/entrypoint/global-config.sh index 64bdc6d..49ac949 100644 --- a/entrypoint/global-config.sh +++ b/entrypoint/global-config.sh @@ -130,16 +130,31 @@ fi # block TOR exit nodes if [ "$(has_value BLOCK_TOR_EXIT_NODE yes)" != "" ] ; then echo "$BLOCK_TOR_EXIT_NODE_CRON /opt/scripts/exit-nodes.sh" >> /etc/crontabs/nginx + replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_TOR_EXIT_NODES%" "lua_shared_dict tor_exit_nodes_data 1m;" + replace_in_file "/etc/nginx/init-lua.conf" "%USE_TOR_EXIT_NODES%" "true" +else + replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_TOR_EXIT_NODES%" "" + replace_in_file "/etc/nginx/init-lua.conf" "%USE_TOR_EXIT_NODES%" "false" fi # block proxies if [ "$(has_value BLOCK_PROXIES yes)" != "" ] ; then echo "$BLOCK_PROXIES_CRON /opt/scripts/proxies.sh" >> /etc/crontabs/nginx + replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_PROXIES%" "lua_shared_dict proxies_data 250m;" + replace_in_file "/etc/nginx/init-lua.conf" "%USE_PROXIES%" "true" +else + replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_PROXIES%" "" + replace_in_file "/etc/nginx/init-lua.conf" "%USE_PROXIES%" "false" fi # block abusers if [ "$(has_value BLOCK_ABUSERS yes)" != "" ] ; then echo "$BLOCK_ABUSERS_CRON /opt/scripts/abusers.sh" >> /etc/crontabs/nginx + replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_ABUSERS%" "lua_shared_dict abusers_data 50m;" + replace_in_file "/etc/nginx/init-lua.conf" "%USE_ABUSERS%" "true" +else + replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_ABUSERS%" "" + replace_in_file "/etc/nginx/init-lua.conf" "%USE_ABUSERS%" "false" fi # DNS resolvers diff --git a/entrypoint/site-config.sh b/entrypoint/site-config.sh index ab417ef..64c5578 100644 --- a/entrypoint/site-config.sh +++ b/entrypoint/site-config.sh @@ -307,23 +307,23 @@ fi # block TOR exit nodes if [ "$BLOCK_TOR_EXIT_NODE" = "yes" ] ; then - replace_in_file "${NGINX_PREFIX}server.conf" "%BLOCK_TOR_EXIT_NODE%" "include /etc/nginx/block-tor-exit-node.conf;" + replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_TOR_EXIT_NODES%" "true" else - replace_in_file "${NGINX_PREFIX}server.conf" "%BLOCK_TOR_EXIT_NODE%" "" + replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_TOR_EXIT_NODES%" "false" fi # block proxies if [ "$BLOCK_PROXIES" = "yes" ] ; then - replace_in_file "${NGINX_PREFIX}server.conf" "%BLOCK_PROXIES%" "include /etc/nginx/block-proxies.conf;" + replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_PROXIES%" "true" else - replace_in_file "${NGINX_PREFIX}server.conf" "%BLOCK_PROXIES%" "" + replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_PROXIES%" "false" fi # block abusers if [ "$BLOCK_ABUSERS" = "yes" ] ; then - replace_in_file "${NGINX_PREFIX}server.conf" "%BLOCK_ABUSERS%" "include /etc/nginx/block-abusers.conf;" + replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_ABUSERS%" "true" else - replace_in_file "${NGINX_PREFIX}server.conf" "%BLOCK_ABUSERS%" "" + replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_ABUSERS%" "false" fi # HTTPS config diff --git a/lua/dataloader.lua b/lua/dataloader.lua new file mode 100644 index 0000000..8d3fc9d --- /dev/null +++ b/lua/dataloader.lua @@ -0,0 +1,31 @@ +local M = {} +local iputils = require "resty.iputils" + +function M.load_ip (path, dict) + local file = io.open(path, "r") + if not file then + ngx.log(ngx.ERR, "[INIT] can't open " .. path) + else + io.input(file) + local i = 0 + for line in io.lines() do + if string.match(line, "/") then + local lower, upper = iputils.parse_cidr(line) + local bin_ip = lower + while bin_ip <= upper do + dict:set(bin_ip, true, 0) + bin_ip = bin_ip + 1 + i = i + 1 + end + else + local bin_ip, bin_octets = iputils.ip2bin(line) + dict:set(bin_ip, true, 0) + i = i + 1 + end + end + ngx.log(ngx.ERR, "[INIT] loaded " .. tostring(i) .. " IPs from " .. path) + io.close(file) + end +end + +return M diff --git a/scripts/abusers.sh b/scripts/abusers.sh index 246464f..3b430e1 100755 --- a/scripts/abusers.sh +++ b/scripts/abusers.sh @@ -4,12 +4,11 @@ . /opt/entrypoint/utils.sh # copy old conf to cache -cp /etc/nginx/block-abusers.conf /cache +cp /etc/nginx/abusers.list /cache # generate the new conf curl -s "https://iplists.firehol.org/files/firehol_abusers_30d.netset" | \ - grep -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" | \ - sed 's/^/deny /;s/$/;/' > /tmp/block-abusers.conf + grep -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" > /tmp/abusers.list # if we are running nginx if [ -f /tmp/nginx.pid ] ; then @@ -20,28 +19,28 @@ elif [ -S /tmp/autoconf.sock ] ; then fi # check if we have at least 1 line -lines="$(wc -l /tmp/block-abusers.conf | cut -d ' ' -f 1)" +lines="$(wc -l /tmp/abusers.list | cut -d ' ' -f 1)" if [ "$lines" -gt 1 ] ; then job_log "[BLACKLIST] abusers list updated ($lines entries)" # reload nginx with the new config - mv /tmp/block-abusers.conf /etc/nginx/block-abusers.conf + mv /tmp/abusers.list /etc/nginx/abusers.list if [ "$RELOAD" != "" ] ; then $RELOAD > /dev/null 2>&1 # new config is ok : save it in the cache if [ "$?" -eq 0 ] ; then - cp /etc/nginx/block-abusers.conf /cache + cp /etc/nginx/abusers.list /cache job_log "[NGINX] successfull nginx reload after abusers list update" else job_log "[NGINX] failed nginx reload after abusers list update fallback to old list" - cp /cache/block-abusers.conf /etc/nginx + cp /cache/abusers.list /etc/nginx $RELOAD > /dev/null 2>&1 fi else - cp /etc/nginx/block-abusers.conf /cache + cp /etc/nginx/abusers.list /cache fi else job_log "[BLACKLIST] can't update abusers list" fi -rm -f /tmp/block-abusers.conf 2> /dev/null +rm -f /tmp/abusers.list 2> /dev/null diff --git a/scripts/exit-nodes.sh b/scripts/exit-nodes.sh index b32a5b6..fc51e56 100644 --- a/scripts/exit-nodes.sh +++ b/scripts/exit-nodes.sh @@ -4,12 +4,11 @@ . /opt/entrypoint/utils.sh # copy old conf to cache -cp /etc/nginx/block-tor-exit-node.conf /cache +cp /etc/nginx/tor-exit-nodes.list /cache # generate the new conf curl -s "https://iplists.firehol.org/files/tor_exits.ipset" | \ - grep -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" | \ - sed 's/^/deny /;s/$/;/' > /tmp/block-tor-exit-node.conf + grep -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" > /tmp/tor-exit-nodes.list # if we are running nginx if [ -f /tmp/nginx.pid ] ; then @@ -20,28 +19,27 @@ elif [ -S /tmp/autoconf.sock ] ; then fi # check if we have at least 1 line -lines="$(wc -l /tmp/block-tor-exit-node.conf | cut -d ' ' -f 1)" +lines="$(wc -l /tmp/tor-exit-nodes.list | cut -d ' ' -f 1)" if [ "$lines" -gt 1 ] ; then job_log "[BLACKLIST] TOR exit node list updated ($lines entries)" # reload nginx with the new config - mv /tmp/block-tor-exit-node.conf /etc/nginx/block-tor-exit-node.conf + mv /tmp/tor-exit-nodes.list /etc/nginx/tor-exit-nodes.list if [ "$RELOAD" != "" ] ; then $RELOAD > /dev/null 2>&1 # new config is ok : save it in the cache if [ "$?" -eq 0 ] ; then - cp /etc/nginx/block-tor-exit-node.conf /cache + cp /etc/nginx/tor-exit-nodes.list /cache job_log "[NGINX] successfull nginx reload after TOR exit node list update" else job_log "[NGINX] failed nginx reload after TOR exit node list update fallback to old list" - cp /cache/block-tor-exit-node.conf /etc/nginx + cp /cache/tor-exit-nodes.list /etc/nginx $RELOAD > /dev/null 2>&1 fi else - cp /etc/nginx/block-tor-exit-node.conf /cache + cp /etc/nginx/tor-exit-nodes.list /cache fi else job_log "[BLACKLIST] can't update TOR exit node list" fi -rm -f /tmp/block-tor-exit-node.conf 2> /dev/null - +rm -f /tmp/tor-exit-nodes.list 2> /dev/null diff --git a/scripts/proxies.sh b/scripts/proxies.sh index ab730c3..7e2084d 100755 --- a/scripts/proxies.sh +++ b/scripts/proxies.sh @@ -4,12 +4,11 @@ . /opt/entrypoint/utils.sh # copy old conf to cache -cp /etc/nginx/block-proxies.conf /cache +cp /etc/nginx/proxies.list /cache # generate the new conf curl -s "https://iplists.firehol.org/files/firehol_proxies.netset" | \ - grep -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" | \ - sed 's/^/deny /;s/$/;/' > /tmp/block-proxies.conf + grep -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" > /tmp/proxies.list # if we are running nginx if [ -f /tmp/nginx.pid ] ; then @@ -20,28 +19,28 @@ elif [ -S /tmp/autoconf.sock ] ; then fi # check if we have at least 1 line -lines="$(wc -l /tmp/block-proxies.conf | cut -d ' ' -f 1)" +lines="$(wc -l /tmp/proxies.list | cut -d ' ' -f 1)" if [ "$lines" -gt 1 ] ; then job_log "[BLACKLIST] proxies list updated ($lines entries)" # reload nginx with the new config - mv /tmp/block-proxies.conf /etc/nginx/block-proxies.conf + mv /tmp/proxies.list /etc/nginx/proxies.list if [ "$RELOAD" != "" ] ; then $RELOAD > /dev/null 2>&1 # new config is ok : save it in the cache if [ "$?" -eq 0 ] ; then - cp /etc/nginx/block-proxies.conf /cache + cp /etc/nginx/proxies.list /cache job_log "[NGINX] successfull nginx reload after proxies list update" else job_log "[NGINX] failed nginx reload after proxies list update fallback to old list" - cp /cache/block-proxies.conf /etc/nginx + cp /cache/proxies.list /etc/nginx $RELOAD > /dev/null 2>&1 fi else - cp /etc/nginx/block-proxies.conf /cache + cp /etc/nginx/proxies.list /cache fi else job_log "[BLACKLIST] can't update proxies list" fi -rm -f /tmp/block-proxies.conf 2> /dev/null +rm -f /tmp/proxies.list 2> /dev/null