performance - move bad user-agents and referrers checks from nginx to LUA with caching

confs/global/init-lua.conf

@@ -1,10 +1,12 @@
 init_by_lua_block {
 
-local dataloader	= require "dataloader"
+local dataloader		= require "dataloader"
 
 local use_proxies		= %USE_PROXIES%
 local use_abusers		= %USE_ABUSERS%
 local use_tor_exit_nodes	= %USE_TOR_EXIT_NODES%
+local use_user_agents		= %USE_USER_AGENTS%
+local use_referrers		= %USE_REFERRERS%
 
 if use_proxies then
 	dataloader.load_ip("/etc/nginx/proxies.list", ngx.shared.proxies_data)
@@ -18,4 +20,12 @@ if use_tor_exit_nodes then
 	dataloader.load_ip("/etc/nginx/tor-exit-nodes.list", ngx.shared.tor_exit_nodes_data)
 end
 
+if use_user_agents then
+	dataloader.load_raw("/etc/nginx/user-agents.list", ngx.shared.user_agents_data)
+end
+
+if use_referrers then
+	dataloader.load_raw("/etc/nginx/referrers.list", ngx.shared.referrers_data)
+end
+
 }

confs/global/map-referrer.conf

@@ -1 +0,0 @@
-map $http_referer $bad_referrer { hostnames; default no; }

confs/global/map-user-agent.conf

@@ -1 +0,0 @@
-map $http_user_agent $bad_user_agent { default no; }

confs/global/nginx.conf

@@ -84,6 +84,11 @@ http {
 	%BLACKLIST_IP_CACHE%
 	%BLACKLIST_REVERSE_CACHE%
 	%DNSBL_CACHE%
+	%BLOCK_PROXIES%
+	%BLOCK_ABUSERS%
+	%BLOCK_TOR_EXIT_NODES%
+	%BLOCK_USER_AGENTS%
+	%BLOCK_REFERRERS%
 
 	# crowdsec init
 	%USE_CROWDSEC%
@@ -94,24 +99,9 @@ http {
 	# shared memory zone for limit_conn
 	%LIMIT_CONN_ZONE%
 
-	# shared memory zone for proxies
-	%BLOCK_PROXIES%
-
-	# shared memory zone for abusers
-	%BLOCK_ABUSERS%
-
-	# shared memory zone for TOR exit nodes
-	%BLOCK_TOR_EXIT_NODES%
-
 	# whitelist or blacklist country
 	%USE_COUNTRY%
 
-	# list of blocked user agents
-	%BLOCK_USER_AGENT%
-
-	# list of blocked referrers
-	%BLOCK_REFERRER%
-
 	# zone for proxy_cache
 	%PROXY_CACHE_PATH%