Makussu/bunkerweb
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

performance - move bad user-agents and referrers checks from nginx to LUA with caching

Browse Source
This commit is contained in:
bunkerity
2021-05-11 15:30:16 +02:00
parent 42c3fb8740
commit b1d03cd11c
13 changed files with 149 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
scripts/referrers.sh
View File

@@ -4,14 +4,16 @@
. /opt/entrypoint/utils.sh
# save old conf
cp /etc/nginx/map-referrer.conf /cache
cp /etc/nginx/referrers.list /cache
# generate new conf
BLACKLIST="$(curl -s https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list)"
BLACKLIST="$(curl -s https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list | sed 's:\.:%\.:g;s:\-:%\-:g')"
if [ "$?" -ne 0 ] ; then
job_log "[BLACKLIST] can't update referrers list"
echo -n "" > /tmp/referrers.list
else
echo -e "$BLACKLIST" > /tmp/referrers.list
fi
echo -e "map \$http_referer \$bad_referrer { hostnames; default no; $(echo -e "$BLACKLIST" | sed 's/^/"~/;s/$/" yes;/') }" > /tmp/map-referrer.conf
# if we are running nginx
if [ -f /tmp/nginx.pid ] ; then
@@ -22,25 +24,25 @@ elif [ -S /tmp/autoconf.sock ] ; then
fi
# check number of lines
lines="$(wc -l /tmp/map-referrer.conf | cut -d ' ' -f 1)"
lines="$(wc -l /tmp/referrers.list | cut -d ' ' -f 1)"
if [ "$lines" -gt 1 ] ; then
mv /tmp/map-referrer.conf /etc/nginx/map-referrer.conf
mv /tmp/referrers.list /etc/nginx/referrers.list
job_log "[BLACKLIST] referrers list updated ($lines entries)"
if [ "$RELOAD" != "" ] ; then
$RELOAD > /dev/null 2>&1
if [ "$?" -eq 0 ] ; then
cp /etc/nginx/map-referrer.conf /cache
cp /etc/nginx/referrers.list /cache
job_log "[NGINX] successfull nginx reload after referrers list update"
else
cp /cache/map-referrer.conf /etc/nginx
cp /cache/referrers.list /etc/nginx
job_log "[NGINX] failed nginx reload after referrers list update fallback to old list"
$RELOAD > /dev/null 2>&1
fi
else
cp /etc/nginx/map-referrer.conf /cache
cp /etc/nginx/referrers.list /cache
fi
else
job_log "[BLACKLIST] can't update referrers list"
fi
rm -f /tmp/map-referrer.conf 2> /dev/null
rm -f /tmp/referrers.list 2> /dev/null

21
scripts/user-agents.sh
View File

@@ -4,15 +4,16 @@
. /opt/entrypoint/utils.sh
# save old conf
cp /etc/nginx/map-user-agent.conf /cache
cp /etc/nginx/user-agents.list /cache
# generate new conf
IFS= BLACKLIST="$(curl -s https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list;
curl -s https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Crawlers.txt | sort -u)"
BLACKLIST="$( (curl -s https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list ; curl -s https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Crawlers.txt) | sort -u | sed 's:\\ : :g;s:\\\.:%\.:g;s:\\\\:\\:g;s:\\/:/:g;s:\-:%\-:g')"
if [ "$?" -ne 0 ] ; then
job_log "[BLACKLIST] can't update user-agent list"
echo -n "" > /tmp/user-agents.list
else
echo -e "$BLACKLIST" > /tmp/user-agents.list
fi
echo -e "map \$http_user_agent \$bad_user_agent { default no; $(echo $BLACKLIST | sed 's: :\\ :g;s:^:~*:;s:$: yes;:') }" > /tmp/map-user-agent.conf
# if we are running nginx
if [ -f /tmp/nginx.pid ] ; then
@@ -23,25 +24,25 @@ elif [ -S /tmp/autoconf.sock ] ; then
fi
# check number of lines
lines="$(wc -l /tmp/map-user-agent.conf | cut -d ' ' -f 1)"
lines="$(wc -l /tmp/user-agents.list | cut -d ' ' -f 1)"
if [ "$lines" -gt 1 ] ; then
mv /tmp/map-user-agent.conf /etc/nginx/map-user-agent.conf
mv /tmp/user-agents.list /etc/nginx/user-agents.list
job_log "[BLACKLIST] user-agent list updated ($lines entries)"
if [ "$RELOAD" != "" ] ; then
$RELOAD > /dev/null 2>&1
if [ "$?" -eq 0 ] ; then
cp /etc/nginx/map-user-agent.conf /cache
cp /etc/nginx/user-agents.list /cache
job_log "[NGINX] successfull nginx reload after user-agent list update"
else
cp /cache/map-user-agent.conf /etc/nginx
cp /cache/user-agents.list /etc/nginx
job_log "[NGINX] failed nginx reload after user-agent list update fallback to old list"
$RELOAD > /dev/null 2>&1
fi
else
cp /etc/nginx/map-user-agent.conf /cache
cp /etc/nginx/user-agents.list /cache
fi
else
job_log "[BLACKLIST] can't update user-agent list"
fi
rm -f /tmp/map-user-agent.conf 2> /dev/null
rm -f /tmp/user-agents.list 2> /dev/null