performance - move bad user-agents and referrers checks from nginx to LUA with caching

confs/global/init-lua.conf

@@ -1,10 +1,12 @@
 init_by_lua_block {
 
-local dataloader	= require "dataloader"
+local dataloader		= require "dataloader"
 
 local use_proxies		= %USE_PROXIES%
 local use_abusers		= %USE_ABUSERS%
 local use_tor_exit_nodes	= %USE_TOR_EXIT_NODES%
+local use_user_agents		= %USE_USER_AGENTS%
+local use_referrers		= %USE_REFERRERS%
 
 if use_proxies then
 	dataloader.load_ip("/etc/nginx/proxies.list", ngx.shared.proxies_data)
@@ -18,4 +20,12 @@ if use_tor_exit_nodes then
 	dataloader.load_ip("/etc/nginx/tor-exit-nodes.list", ngx.shared.tor_exit_nodes_data)
 end
 
+if use_user_agents then
+	dataloader.load_raw("/etc/nginx/user-agents.list", ngx.shared.user_agents_data)
+end
+
+if use_referrers then
+	dataloader.load_raw("/etc/nginx/referrers.list", ngx.shared.referrers_data)
+end
+
 }

confs/global/map-referrer.conf

@@ -1 +0,0 @@
-map $http_referer $bad_referrer { hostnames; default no; }

confs/global/map-user-agent.conf

@@ -1 +0,0 @@
-map $http_user_agent $bad_user_agent { default no; }

confs/global/nginx.conf

@@ -84,6 +84,11 @@ http {
 	%BLACKLIST_IP_CACHE%
 	%BLACKLIST_REVERSE_CACHE%
 	%DNSBL_CACHE%
+	%BLOCK_PROXIES%
+	%BLOCK_ABUSERS%
+	%BLOCK_TOR_EXIT_NODES%
+	%BLOCK_USER_AGENTS%
+	%BLOCK_REFERRERS%
 
 	# crowdsec init
 	%USE_CROWDSEC%
@@ -94,24 +99,9 @@ http {
 	# shared memory zone for limit_conn
 	%LIMIT_CONN_ZONE%
 
-	# shared memory zone for proxies
-	%BLOCK_PROXIES%
-
-	# shared memory zone for abusers
-	%BLOCK_ABUSERS%
-
-	# shared memory zone for TOR exit nodes
-	%BLOCK_TOR_EXIT_NODES%
-
 	# whitelist or blacklist country
 	%USE_COUNTRY%
 
-	# list of blocked user agents
-	%BLOCK_USER_AGENT%
-
-	# list of blocked referrers
-	%BLOCK_REFERRER%
-
 	# zone for proxy_cache
 	%PROXY_CACHE_PATH%
 

confs/site/main-lua.conf

@@ -6,11 +6,11 @@ access_by_lua_block {
 local use_lets_encrypt		= %USE_LETS_ENCRYPT%
 local use_whitelist_ip		= %USE_WHITELIST_IP%
 local use_whitelist_reverse	= %USE_WHITELIST_REVERSE%
-local use_user_agent		= %USE_USER_AGENT%
+local use_user_agents		= %USE_USER_AGENTS%
 local use_proxies		= %USE_PROXIES%
 local use_abusers		= %USE_ABUSERS%
 local use_tor_exit_nodes	= %USE_TOR_EXIT_NODES%
-local use_referrer		= %USE_REFERRER%
+local use_referrers		= %USE_REFERRERS%
 local use_country		= %USE_COUNTRY%
 local use_blacklist_ip		= %USE_BLACKLIST_IP%
 local use_blacklist_reverse	= %USE_BLACKLIST_REVERSE%
@@ -126,25 +126,59 @@ if use_tor_exit_nodes then
 end
 
 -- check if user-agent is allowed
-if use_user_agent and ngx.var.bad_user_agent == "yes" then
-	local block = true
+if use_user_agents then
+	local whitelisted = false
 	for k, v in pairs(whitelist_user_agent) do
 		if string.match(ngx.var.http_user_agent, v) then
 			ngx.log(ngx.NOTICE, "[ALLOW] User-Agent " .. ngx.var.http_user_agent .. " is whitelisted")
-			block = false
+			whitelisted = true
 			break
 		end
 	end
-	if block then
-		ngx.log(ngx.NOTICE, "[BLOCK] User-Agent " .. ngx.var.http_user_agent .. " is blacklisted")
-		ngx.exit(ngx.HTTP_FORBIDDEN)
+	if not whitelisted then
+		local value, flags = ngx.shared.user_agents_cache:get(ngx.var.http_user_agent)
+		if value == nil then
+			local patterns = ngx.shared.user_agents_data:get_keys(0)
+			for i, pattern in ipairs(patterns) do
+				if string.match(ngx.var.http_user_agent, pattern) then
+					value = "ko"
+					ngx.shared.user_agents_cache:set(ngx.var.http_user_agent, "ko", 86400)
+					break
+				end
+			end
+			if value == nil then
+				value = "ok"
+				ngx.shared.user_agents_cache:set(ngx.var.http_user_agent, "ok", 86400)
+			end
+		end
+		if value == "ko" then
+			ngx.log(ngx.NOTICE, "[BLOCK] User-Agent " .. ngx.var.http_user_agent .. " is blacklisted")
+			ngx.exit(ngx.HTTP_FORBIDDEN)
+		end
 	end
 end
 
 -- check if referrer is allowed
-if use_referrer and ngx.var.bad_referrer == "yes" then
-	ngx.log(ngx.NOTICE, "[BLOCK] Referrer " .. ngx.var.http_referer .. " is blacklisted")
-	ngx.exit(ngx.HTTP_FORBIDDEN)
+if use_referrer then
+	local value, flags = ngx.shared.referrers_cache:get(ngx.var.http_referer)
+	if value == nil then
+		local patterns = ngx.shared.referrers_data:get_keys(0)
+		for i, pattern in ipairs(patterns) do
+			if string.match(ngx.var.http_referer, pattern) then
+				value = "ko"
+				ngx.shared.referrers_cache:set(ngx.var.http_referer, "ko", 86400)
+				break
+			end
+		end
+		if value == nil then
+			value = "ok"
+			ngx.shared.referrers_cache:set(ngx.var.http_referer, "ok", 86400)
+		end
+	end
+	if value == "ko" then
+		ngx.log(ngx.NOTICE, "[BLOCK] Referrer " .. ngx.var.http_referer .. " is blacklisted")
+		ngx.exit(ngx.HTTP_FORBIDDEN)
+	end
 end
 
 -- check if country is allowed