Makussu/bunkerweb
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

JOBS - be more verbose about jobs failure/success

Browse Source
This commit is contained in:
bunkerity 2020-12-16 11:43:41 +01:00
parent 373988670a
commit 119e963612
No known key found for this signature in database
GPG Key ID: 654FFF51CEF7CC47
17 changed files with 166 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@ -25,7 +25,7 @@ RUN apk --no-cache add certbot libstdc++ libmaxminddb geoip pcre yajl fail2ban c
rm -f /var/log/nginx/* && \ rm -f /var/log/nginx/* && \
chown root:nginx /var/log/nginx && \ chown root:nginx /var/log/nginx && \
chmod 750 /var/log/nginx && \ chmod 750 /var/log/nginx && \
touch /var/log/nginx/error.log /var/log/nginx/modsec_audit.log && \ touch /var/log/nginx/error.log /var/log/nginx/modsec_audit.log /var/log/jobs.log && \
chown nginx:nginx /var/log/nginx/*.log && \ chown nginx:nginx /var/log/nginx/*.log && \
mkdir /acme-challenge && \ mkdir /acme-challenge && \
chown root:nginx /acme-challenge && \ chown root:nginx /acme-challenge && \

2
Dockerfile-amd64
View File

@ -25,7 +25,7 @@ RUN apk --no-cache add certbot libstdc++ libmaxminddb geoip pcre yajl fail2ban c
rm -f /var/log/nginx/* && \ rm -f /var/log/nginx/* && \
chown root:nginx /var/log/nginx && \ chown root:nginx /var/log/nginx && \
chmod 750 /var/log/nginx && \ chmod 750 /var/log/nginx && \
touch /var/log/nginx/error.log /var/log/nginx/modsec_audit.log && \ touch /var/log/nginx/error.log /var/log/nginx/modsec_audit.log /var/log/jobs.log && \
chown nginx:nginx /var/log/nginx/*.log && \ chown nginx:nginx /var/log/nginx/*.log && \
mkdir /acme-challenge && \ mkdir /acme-challenge && \
chown root:nginx /acme-challenge && \ chown root:nginx /acme-challenge && \

2
Dockerfile-arm32v7
View File

@ -32,7 +32,7 @@ RUN apk --no-cache add certbot libstdc++ libmaxminddb geoip pcre yajl fail2ban c
rm -f /var/log/nginx/* && \ rm -f /var/log/nginx/* && \
chown root:nginx /var/log/nginx && \ chown root:nginx /var/log/nginx && \
chmod 750 /var/log/nginx && \ chmod 750 /var/log/nginx && \
touch /var/log/nginx/error.log /var/log/nginx/modsec_audit.log && \ touch /var/log/nginx/error.log /var/log/nginx/modsec_audit.log /var/log/jobs.log && \
chown nginx:nginx /var/log/nginx/*.log && \ chown nginx:nginx /var/log/nginx/*.log && \
mkdir /acme-challenge && \ mkdir /acme-challenge && \
chown root:nginx /acme-challenge && \ chown root:nginx /acme-challenge && \

2
Dockerfile-arm64v8
View File

@ -32,7 +32,7 @@ RUN apk --no-cache add certbot libstdc++ libmaxminddb geoip pcre yajl fail2ban c
rm -f /var/log/nginx/* && \ rm -f /var/log/nginx/* && \
chown root:nginx /var/log/nginx && \ chown root:nginx /var/log/nginx && \
chmod 750 /var/log/nginx && \ chmod 750 /var/log/nginx && \
touch /var/log/nginx/error.log /var/log/nginx/modsec_audit.log && \ touch /var/log/nginx/error.log /var/log/nginx/modsec_audit.log /var/log/jobs.log && \
chown nginx:nginx /var/log/nginx/*.log && \ chown nginx:nginx /var/log/nginx/*.log && \
mkdir /acme-challenge && \ mkdir /acme-challenge && \
chown root:nginx /acme-challenge && \ chown root:nginx /acme-challenge && \

2
Dockerfile-i386
View File

@ -25,7 +25,7 @@ RUN apk --no-cache add certbot libstdc++ libmaxminddb geoip pcre yajl fail2ban c
rm -f /var/log/nginx/* && \ rm -f /var/log/nginx/* && \
chown root:nginx /var/log/nginx && \ chown root:nginx /var/log/nginx && \
chmod 750 /var/log/nginx && \ chmod 750 /var/log/nginx && \
touch /var/log/nginx/error.log /var/log/nginx/modsec_audit.log && \ touch /var/log/nginx/error.log /var/log/nginx/modsec_audit.log /var/log/jobs.log && \
chown nginx:nginx /var/log/nginx/*.log && \ chown nginx:nginx /var/log/nginx/*.log && \
mkdir /acme-challenge && \ mkdir /acme-challenge && \
chown root:nginx /acme-challenge && \ chown root:nginx /acme-challenge && \

2
entrypoint/entrypoint.sh
View File

@ -85,7 +85,7 @@ echo "[*] Running nginx ..."
su -s "/usr/sbin/nginx" nginx su -s "/usr/sbin/nginx" nginx
# list of log files to display # list of log files to display
LOGS="/var/log/access.log /var/log/error.log" LOGS="/var/log/access.log /var/log/error.log /var/log/jobs.log"
# start fail2ban # start fail2ban
if [ "$USE_FAIL2BAN" = "yes" ] ; then if [ "$USE_FAIL2BAN" = "yes" ] ; then

24
scripts/abusers.sh
View File

@ -1,14 +1,32 @@
#!/bin/sh #!/bin/sh
echo "" > /etc/nginx/block-abusers.conf . /opt/scripts/utils.sh
if [ ! -f /etc/nginx/block-abusers.conf ] ; then
echo "" > /etc/nginx/block-abusers.conf
fi
echo "" > /cache/block-abusers.conf
curl -s "https://iplists.firehol.org/files/firehol_abusers_30d.netset" | grep -v "^\#.*" | curl -s "https://iplists.firehol.org/files/firehol_abusers_30d.netset" | grep -v "^\#.*" |
while read entry ; do while read entry ; do
check=$(echo $entry | grep -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$") check=$(echo $entry | grep -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$")
if [ "$check" != "" ] ; then if [ "$check" != "" ] ; then
echo "deny ${entry};" >> /etc/nginx/block-abusers.conf echo "deny ${entry};" >> /cache/block-abusers.conf
fi fi
done done
cp /etc/nginx/block-abusers.conf /cache
cp /cache/block-abusers.conf /etc/nginx/block-abusers.conf
lines="$(wc -l /etc/nginx/block-abusers.conf | cut -d ' ' - f1)"
if [ "$lines" -gt 1 ] ; then
job_log "[BLACKLIST] abusers list updated ($lines entries)"
else
job_log "[BLACKLIST] can't update abusers list"
fi
if [ -f /tmp/nginx.pid ] ; then if [ -f /tmp/nginx.pid ] ; then
/usr/sbin/nginx -s reload > /dev/null 2>&1 /usr/sbin/nginx -s reload > /dev/null 2>&1
if [ "$?" -eq 0 ] ; then
job_log "[NGINX] successfull nginx reload after abusers list update"
else
job_log "[NGINX] failed nginx reload after abusers list update"
fi
fi fi

4
scripts/certbot-new.sh
View File

@ -2,8 +2,12 @@
# generate certificate # generate certificate
certbot certonly --webroot -w /acme-challenge -n -d "$1" --email "$2" --agree-tos certbot certonly --webroot -w /acme-challenge -n -d "$1" --email "$2" --agree-tos
if [ "$?" -ne 0 ] ; then
exit 1
fi
# fix rights # fix rights
chown -R root:nginx /etc/letsencrypt chown -R root:nginx /etc/letsencrypt
chmod -R 740 /etc/letsencrypt chmod -R 740 /etc/letsencrypt
find /etc/letsencrypt -type d -exec chmod 750 {} \; find /etc/letsencrypt -type d -exec chmod 750 {} \;
exit 0

20
scripts/certbot-renew-hook.sh Normal file
View File

@ -0,0 +1,20 @@
#!/bin/sh
. /opt/scripts/utils.sh
job_log "[CERTBOT] certificates have been renewed"
# fix rights
chown -R root:nginx /etc/letsencrypt
chmod -R 740 /etc/letsencrypt
find /etc/letsencrypt -type d -exec chmod 750 {} \;
# reload nginx
if [ -f /tmp/nginx.pid ] ; then
/usr/sbin/nginx -s reload > /dev/null 2>&1
if [ "$?" -eq 0 ] ; then
job_log "[NGINX] successfull nginx reload after certbot renew"
else
job_log "[NGINX] failed nginx reload after certbot renew"
fi
fi

10
scripts/certbot-renew.sh
View File

@ -1,7 +1,15 @@
#!/bin/sh #!/bin/sh
. /opt/scripts/utils.sh
# ask new certificates if needed # ask new certificates if needed
certbot renew certbot renew --deploy-hook /opt/scripts/certbot-renew-hook.sh
if [ "$?" -eq 0 ] ; then
job_log "[CERTBOT] renew operation done"
else
job_log "[CERTBOT] renew operation failed"
fi
# fix rights # fix rights
chown -R root:nginx /etc/letsencrypt chown -R root:nginx /etc/letsencrypt

24
scripts/exit-nodes.sh
View File

@ -1,14 +1,32 @@
#!/bin/sh #!/bin/sh
echo "" > /etc/nginx/block-tor-exit-node.conf . /opt/scripts/utils.sh
if [ ! -f /etc/nginx/block-tor-exit-node.conf ] ; then
echo "" > /etc/nginx/block-tor-exit-node.conf
fi
echo "" > /cache/block-tor-exit-node.conf
curl -s "https://iplists.firehol.org/files/tor_exits.ipset" | grep -v "^\#.*" | curl -s "https://iplists.firehol.org/files/tor_exits.ipset" | grep -v "^\#.*" |
while read entry ; do while read entry ; do
check=$(echo $entry | grep -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$") check=$(echo $entry | grep -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$")
if [ "$check" != "" ] ; then if [ "$check" != "" ] ; then
echo "deny ${entry};" >> /etc/nginx/block-tor-exit-node.conf echo "deny ${entry};" >> /cache/block-tor-exit-node.conf
fi fi
done done
cp /etc/nginx/block-tor-exit-node.conf /cache
cp /cache/block-tor-exit-node.conf /etc/nginx/block-tor-exit-node.conf
lines="$(wc -l /etc/nginx/block-tor-exit-node.conf | cut -d ' ' - f1)"
if [ "$lines" -gt 1 ] ; then
job_log "[BLACKLIST] TOR exit node list updated ($lines entries)"
else
job_log "[BLACKLIST] can't update TOR exit node list"
fi
if [ -f /tmp/nginx.pid ] ; then if [ -f /tmp/nginx.pid ] ; then
/usr/sbin/nginx -s reload > /dev/null 2>&1 /usr/sbin/nginx -s reload > /dev/null 2>&1
if [ "$?" -eq 0 ] ; then
job_log "[NGINX] successfull nginx reload after TOR exit node list update"
else
job_log "[NGINX] failed nginx reload after TOR exit node list update"
fi
fi fi

21
scripts/geoip.sh
View File

@ -1,12 +1,25 @@
#!/bin/sh #!/bin/sh
. /opt/scripts/utils.sh
# MMDB from https://db-ip.com/db/download/ip-to-country-lite # MMDB from https://db-ip.com/db/download/ip-to-country-lite
URL="https://download.db-ip.com/free/dbip-country-lite-$(date +%Y-%m).mmdb.gz" URL="https://download.db-ip.com/free/dbip-country-lite-$(date +%Y-%m).mmdb.gz"
wget -O /etc/nginx/geoip.mmdb.gz "$URL" > /dev/null 2>&1 wget -O /cache/geoip.mmdb.gz "$URL" > /dev/null 2>&1
if [ -f /etc/nginx/geoip.mmdb.gz ] ; then if [ -f /cache/geoip.mmdb.gz ] ; then
gunzip -f /etc/nginx/geoip.mmdb.gz gunzip -f /cache/geoip.mmdb.gz > /dev/null 2>&1
cp /etc/nginx/geoip.mmdb /cache if [ "$?" -ne 0 ] ; then
job_log "[GEOIP] can't extract DB from $URL"
exit 1
fi
cp /cache/geoip.mmdb /etc/nginx/geoip.mmdb
if [ -f /tmp/nginx.pid ] ; then if [ -f /tmp/nginx.pid ] ; then
/usr/sbin/nginx -s reload > /dev/null 2>&1 /usr/sbin/nginx -s reload > /dev/null 2>&1
if [ "$?" -eq 0 ] ; then
job_log "[NGINX] successfull nginx reload after GeoIP DB update"
else
job_log "[NGINX] failed nginx reload after GeoIP DB update"
fi
fi fi
else
job_log "[GEOIP] can't download DB from $URL"
fi fi

7
scripts/logrotate.sh
View File

@ -7,5 +7,10 @@ pkill -HUP rsyslogd
fail2ban-client flushlogs fail2ban-client flushlogs
if [ -f /tmp/nginx.pid ] ; then if [ -f /tmp/nginx.pid ] ; then
/usr/sbin/nginx -s reload > /dev/null 2>&1 /usr/sbin/nginx -s reload > /dev/null 2>&1
if [ "$?" -eq 0 ] ; then
job_log "[NGINX] successfull nginx reload after logrotate"
else
job_log "[NGINX] failed nginx reload after logrotate"
fi
fi fi

24
scripts/proxies.sh
View File

@ -1,14 +1,32 @@
#!/bin/sh #!/bin/sh
echo "" > /etc/nginx/block-proxies.conf . /opt/scripts/utils.sh
if [ ! -f /etc/nginx/block-proxies.conf ] ; then
echo "" > /etc/nginx/block-proxies.conf
fi
echo "" > /cache/block-proxies.conf
curl -s "https://iplists.firehol.org/files/firehol_proxies.netset" | grep -v "^\#.*" | curl -s "https://iplists.firehol.org/files/firehol_proxies.netset" | grep -v "^\#.*" |
while read entry ; do while read entry ; do
check=$(echo $entry | grep -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$") check=$(echo $entry | grep -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$")
if [ "$check" != "" ] ; then if [ "$check" != "" ] ; then
echo "deny ${entry};" >> /etc/nginx/block-proxies.conf echo "deny ${entry};" >> /cache/block-proxies.conf
fi fi
done done
cp /etc/nginx/block-proxies.conf /cache
cp /cache/block-proxies.conf /etc/nginx/block-proxies.conf
lines="$(wc -l /etc/nginx/block-proxies.conf | cut -d ' ' - f1)"
if [ "$lines" -gt 1 ] ; then
job_log "[BLACKLIST] proxies list updated ($lines entries)"
else
job_log "[BLACKLIST] can't update proxies list"
fi
if [ -f /tmp/nginx.pid ] ; then if [ -f /tmp/nginx.pid ] ; then
/usr/sbin/nginx -s reload > /dev/null 2>&1 /usr/sbin/nginx -s reload > /dev/null 2>&1
if [ "$?" -eq 0 ] ; then
job_log "[NGINX] successfull nginx reload after proxies list update"
else
job_log "[NGINX] failed nginx reload after proxies list update"
fi
fi fi

19
scripts/referrers.sh
View File

@ -1,15 +1,30 @@
#!/bin/sh #!/bin/sh
. /opt/scripts/utils.sh
echo "map \$http_referer \$bad_referrer { hostnames; default no; }" > /etc/nginx/map-referrer.conf
echo "map \$http_referer \$bad_referrer { hostnames; default no; }" > /cache/map-referrer.conf
BLACKLIST="$(curl -s https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list)" BLACKLIST="$(curl -s https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list)"
if [ "$?" -ne 0 ] ; then
job_log "[BLACKLIST] can't update referrers list"
fi
DATA="" DATA=""
IFS=$'\n' IFS=$'\n'
for ref in $BLACKLIST ; do for ref in $BLACKLIST ; do
DATA="${DATA}\"~${ref}\" yes;\n" DATA="${DATA}\"~${ref}\" yes;\n"
done done
echo -e "map \$http_referer \$bad_referrer { hostnames; default no; $DATA }" > /etc/nginx/map-referrer.conf echo -e "map \$http_referer \$bad_referrer { hostnames; default no; $DATA }" > /cache/map-referrer.conf
cp /etc/nginx/map-referrer.conf /cache cp /cache/map-referrer.conf /etc/nginx/map-referrer.conf
lines="$(wc -l /etc/nginx/map-referrer.conf | cut -d ' ' -f 1)"
job_log "[BLACKLIST] referrers list updated ($lines entries)"
if [ -f /tmp/nginx.pid ] ; then if [ -f /tmp/nginx.pid ] ; then
/usr/sbin/nginx -s reload > /dev/null 2>&1 /usr/sbin/nginx -s reload > /dev/null 2>&1
if [ "$?" -eq 0 ] ; then
job_log "[NGINX] successfull nginx reload after referrers list update"
else
job_log "[NGINX] failed nginx reload after referrers list update"
fi
fi fi

18
scripts/user-agents.sh
View File

@ -1,7 +1,14 @@
#!/bin/sh #!/bin/sh
echo "map \$http_user_agent \$bad_user_agent { default no; }" > /etc/nginx/map-user-agent.conf
echo "map \$http_user_agent \$bad_user_agent { default no; }" > /cache/map-user-agent.conf
BLACKLIST="$(curl -s https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) BLACKLIST="$(curl -s https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
$(curl -s https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Crawlers.txt)" $(curl -s https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Crawlers.txt)"
if [ "$?" -ne 0 ] ; then
job_log "[BLACKLIST] can't update user-agent list"
fi
DATA="" DATA=""
IFS=$'\n' IFS=$'\n'
for ua in $BLACKLIST ; do for ua in $BLACKLIST ; do
@ -9,9 +16,16 @@ for ua in $BLACKLIST ; do
done done
DATA_ESCAPED=$(echo "$DATA" | sed 's: :\\\\ :g' | sed 's:\\\\ yes;: yes;:g' | sed 's:\\\\\\ :\\\\ :g') DATA_ESCAPED=$(echo "$DATA" | sed 's: :\\\\ :g' | sed 's:\\\\ yes;: yes;:g' | sed 's:\\\\\\ :\\\\ :g')
echo -e "map \$http_user_agent \$bad_user_agent { default no; $DATA_ESCAPED }" > /etc/nginx/map-user-agent.conf echo -e "map \$http_user_agent \$bad_user_agent { default no; $DATA_ESCAPED }" > /cache/map-user-agent.conf
cp /etc/nginx/map-user-agent.conf /cache cp /cache/map-user-agent.conf /etc/nginx/map-user-agent.conf
lines="$(wc -l /etc/nginx/map-user-agent.conf | cut -d ' ' -f 1)"
job_log "[BLACKLIST] user-agent list updated ($lines entries)"
if [ -f /tmp/nginx.pid ] ; then if [ -f /tmp/nginx.pid ] ; then
/usr/sbin/nginx -s reload > /dev/null 2>&1 /usr/sbin/nginx -s reload > /dev/null 2>&1
if [ "$?" -eq 0 ] ; then
job_log "[NGINX] successfull nginx reload after user-agent list update"
else
job_log "[NGINX] failed nginx reload after user-agent list update"
fi
fi fi

8
scripts/utils.sh Normal file
View File

@ -0,0 +1,8 @@
#!/bin/sh
function job_log() {
when="$(date +[%d/%m/%Y %H:%M:%S])
what="$1"
echo "$when $what" >> /var/log/jobs.log
}