From afc66788556015889936164c9298640a1d191fe7 Mon Sep 17 00:00:00 2001 From: bunkerity Date: Wed, 17 Mar 2021 17:55:56 +0100 Subject: [PATCH] road to v1.2.3 - fixing bugs --- README.md | 8 ++++---- autoconf/Config.py | 1 + autoconf/Dockerfile | 5 ++++- autoconf/entrypoint.sh | 4 +++- autoconf/misc/logrotate.conf | 23 +++++++++++++++++++++++ confs/site/main-lua.conf | 22 +++++++++++----------- entrypoint/defaults.sh | 2 +- entrypoint/site-config.sh | 23 ++++++++++++++--------- entrypoint/utils.sh | 14 ++++++++++---- logs/logrotate.conf | 2 +- prepare.sh | 14 ++++++++++++++ scripts/abusers.sh | 2 +- scripts/certbot-renew-hook.sh | 2 +- scripts/certbot-renew.sh | 2 +- scripts/exit-nodes.sh | 2 +- scripts/geoip.sh | 2 +- scripts/logrotate.sh | 2 +- scripts/proxies.sh | 2 +- scripts/referrers.sh | 2 +- scripts/user-agents.sh | 2 +- scripts/utils.sh | 8 -------- ui/config.json | 6 +++--- 22 files changed, 98 insertions(+), 52 deletions(-) create mode 100644 autoconf/misc/logrotate.conf delete mode 100644 scripts/utils.sh diff --git a/README.md b/README.md index 99f6108..79457cd 100644 --- a/README.md +++ b/README.md @@ -1083,11 +1083,11 @@ Default value : *.googlebot.com .google.com .search.msn.com .crawl.yahoot.net .c Context : *global* The list of reverse DNS suffixes to whitelist when `USE_WHITELIST_REVERSE` is set to *yes*. The default list contains suffixes of major search engines. -`WHITELIST_USERAGENT_LIST` -Values : *"useragent1", "^[user]agent2"* -Default value : ** +`WHITELIST_USER_AGENT` +Values : *\* +Default value : Context : *global*, *multisite* -Whitelist user agent from be blocked by `BLOCK_USER_AGENT` +Whitelist user agent from being blocked by `BLOCK_USER_AGENT`. ### Custom blacklisting diff --git a/autoconf/Config.py b/autoconf/Config.py index 592ea96..42d1298 100644 --- a/autoconf/Config.py +++ b/autoconf/Config.py @@ -28,6 +28,7 @@ class Config : started = True break i = i + 1 + utils.log("[!] Waiting " + str(i) + " seconds before retrying to contact nginx instances") time.sleep(i) if started : proc = subprocess.run(["/bin/su", "-s", "/opt/entrypoint/jobs.sh", "nginx"], env=vars, capture_output=True) diff --git a/autoconf/Dockerfile b/autoconf/Dockerfile index 6731fd5..2703020 100644 --- a/autoconf/Dockerfile +++ b/autoconf/Dockerfile @@ -23,8 +23,11 @@ RUN apk add py3-pip apache2-utils bash certbot curl logrotate openssl && \ chmod 770 /cache && \ touch /var/log/jobs.log && \ chown root:nginx /var/log/jobs.log && \ - chmod 770 /var/log/jobs.log + chmod 770 /var/log/jobs.log && \ + chown -R root:nginx /opt/confs/nginx && \ + chmod -R 770 /opt/confs/nginx +COPY autoconf/misc/logrotate.conf /etc/logrotate.conf COPY scripts/* /opt/scripts/ COPY confs/site/ /opt/confs/site COPY confs/global/ /opt/confs/global diff --git a/autoconf/entrypoint.sh b/autoconf/entrypoint.sh index 5fb1ec1..0f5a4ce 100644 --- a/autoconf/entrypoint.sh +++ b/autoconf/entrypoint.sh @@ -2,7 +2,9 @@ echo "[*] Starting autoconf ..." -cp -r /opt/confs/nginx/* /etc/nginx +if [ "$SWARM_MODE" = "yes" ] ; then + cp -r /opt/confs/nginx/* /etc/nginx +fi # trap SIGTERM and SIGINT function trap_exit() { diff --git a/autoconf/misc/logrotate.conf b/autoconf/misc/logrotate.conf new file mode 100644 index 0000000..bb90b1f --- /dev/null +++ b/autoconf/misc/logrotate.conf @@ -0,0 +1,23 @@ +/var/log/*.log /var/log/letsencrypt/*.log { + # compress old files using gzip + compress + + # rotate everyday + daily + + # remove old logs after X days + maxage 7 + rotate 7 + + # no errors if a file is missing + missingok + + # disable mailing + nomail + + # mininum size of a logfile before rotating + minsize 10M + + # make a copy and truncate the files + copytruncate +} diff --git a/confs/site/main-lua.conf b/confs/site/main-lua.conf index 6b1af89..c2e9c2c 100644 --- a/confs/site/main-lua.conf +++ b/confs/site/main-lua.conf @@ -30,7 +30,7 @@ local recaptcha = require "recaptcha" -- user variables local antibot_uri = "%ANTIBOT_URI%" -local whitelist_useragent_list = {%WHITELIST_USERAGENT_LIST%} +local whitelist_user_agent = {%WHITELIST_USER_AGENT%} -- check if it's let's encrypt bot if use_lets_encrypt and string.match(ngx.var.request_uri, "^/.well-known/acme-challenge/") then @@ -90,16 +90,16 @@ end if use_user_agent and ngx.var.bad_user_agent == "yes" then local headers = ngx.req.get_headers() local ua = headers["User-Agent"] - if not whitelist_useragent_list ~= "" then - local k_ua_white, v_ua_white = next(whitelist_useragent_list, nil) - while v_ua_white do - local rst_whitelist = string.match(ua, v_ua_white) - if rst_whitelist ~= nil and rst_whitelist ~= "" then - ngx.log(ngx.WARN, "[ALLOW] User-Agent " .. ngx.var.http_user_agent .. " is whitelisted") - ngx.exit(ngx.OK) - end - k_ua_white, v_ua_white = next(whitelist_useragent_list, k_ua_white) - end + if not whitelist_user_agent ~= "" then + local k_ua_white, v_ua_white = next(whitelist_user_agent, nil) + while v_ua_white do + local rst_whitelist = string.match(ua, v_ua_white) + if rst_whitelist ~= nil and rst_whitelist ~= "" then + ngx.log(ngx.WARN, "[ALLOW] User-Agent " .. ngx.var.http_user_agent .. " is whitelisted") + ngx.exit(ngx.OK) + end + k_ua_white, v_ua_white = next(whitelist_user_agent, k_ua_white) + end end ngx.log(ngx.WARN, "[BLOCK] User-Agent " .. ngx.var.http_user_agent .. " is blacklisted") ngx.exit(ngx.HTTP_FORBIDDEN) diff --git a/entrypoint/defaults.sh b/entrypoint/defaults.sh index 460c282..119f4b0 100644 --- a/entrypoint/defaults.sh +++ b/entrypoint/defaults.sh @@ -45,7 +45,7 @@ DISABLE_DEFAULT_SERVER="${DISABLE_DEFAULT_SERVER-no}" SERVER_NAME="${SERVER_NAME-www.bunkerity.com}" ALLOWED_METHODS="${ALLOWED_METHODS-GET|POST|HEAD}" BLOCK_USER_AGENT="${BLOCK_USER_AGENT-yes}" -WHITELIST_USERAGENT_LIST="${WHITELIST_USERAGENT_LIST-}" +WHITELIST_USER_AGENT="${WHITELIST_USER_AGENT-}" BLOCK_USER_AGENT_CRON="${BLOCK_USER_AGENT_CRON-5 0 * * *}" BLOCK_REFERRER="${BLOCK_REFERRER-yes}" BLOCK_REFERRER_CRON="${BLOCK_REFERRER_CRON-10 0 * * *}" diff --git a/entrypoint/site-config.sh b/entrypoint/site-config.sh index 559e45c..028b3e5 100644 --- a/entrypoint/site-config.sh +++ b/entrypoint/site-config.sh @@ -24,12 +24,12 @@ if [ "$MULTISITE" = "yes" ] ; then sed -i "/^${server}_.*=.*/d" "${NGINX_PREFIX}nginx.env" fi done - for var in $(compgen -e) ; do + for var in $(cut -d '=' -f 1 "${NGINX_PREFIX}nginx.env") ; do name=$(echo "$var") check=$(echo "$name" | grep "^$1_") if [ "$check" != "" ] ; then repl_name=$(echo "$name" | sed "s~${1}_~~") - repl_value=$(echo "${!var}") + repl_value=$(env | grep -E "^${name}=" | sed "s~^${name}=~~") read -r "$repl_name" <<< $repl_value sed -i "/^${repl_name}=.*/d" "${NGINX_PREFIX}nginx.env" sed -i "/^${name}=.*/d" "${NGINX_PREFIX}nginx.env" @@ -76,10 +76,13 @@ if [ "$USE_REVERSE_PROXY" = "yes" ] ; then replace_in_file "${NGINX_PREFIX}reverse-proxy-${i}.conf" "%REVERSE_PROXY_URL%" "$value" replace_in_file "${NGINX_PREFIX}reverse-proxy-${i}.conf" "%REVERSE_PROXY_HOST%" "$host_value" if [ "$custom_headers_value" != "" ] ; then - IFS=';' ;for header_value in $(echo "$custom_headers_value") ; do - replace_in_file "${NGINX_PREFIX}reverse-proxy-${i}.conf" "%REVERSE_PROXY_CUSTOM_HEADERS%" "more_set_headers $header_value;\n%REVERSE_PROXY_CUSTOM_HEADERS%" - done - replace_in_file "${NGINX_PREFIX}reverse-proxy-${i}.conf" "%REVERSE_PROXY_CUSTOM_HEADERS%" "" + IFS_=$IFS + IFS=';' + for header_value in $(echo "$custom_headers_value") ; do + replace_in_file "${NGINX_PREFIX}reverse-proxy-${i}.conf" "%REVERSE_PROXY_CUSTOM_HEADERS%" "more_set_headers $header_value;\n%REVERSE_PROXY_CUSTOM_HEADERS%" + done + IFS=$IFS_ + replace_in_file "${NGINX_PREFIX}reverse-proxy-${i}.conf" "%REVERSE_PROXY_CUSTOM_HEADERS%" "" fi replace_in_file "${NGINX_PREFIX}reverse-proxy-${i}.conf" "%REVERSE_PROXY_HEADERS%" "include ${NGINX_PREFIX}reverse-proxy-headers.conf;" if [ "$ws_value" = "yes" ] ; then @@ -284,13 +287,15 @@ fi # block bad UA if [ "$BLOCK_USER_AGENT" = "yes" ] ; then replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_USER_AGENT%" "true" - if [ "$WHITELIST_USERAGENT_LIST" != "" ] ; then - replace_in_file "${NGINX_PREFIX}main-lua.conf" "%WHITELIST_USERAGENT_LIST%" "$WHITELIST_USERAGENT_LIST" + if [ "$WHITELIST_USER_AGENT" != "" ] ; then + list=$(spaces_to_lua "$WHITELIST_USER_AGENT") + replace_in_file "${NGINX_PREFIX}main-lua.conf" "%WHITELIST_USER_AGENT%" "$list" else - replace_in_file "${NGINX_PREFIX}main-lua.conf" "%WHITELIST_USERAGENT_LIST%" "" + replace_in_file "${NGINX_PREFIX}main-lua.conf" "%WHITELIST_USER_AGENT%" "" fi else replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_USER_AGENT%" "false" + replace_in_file "${NGINX_PREFIX}main-lua.conf" "%WHITELIST_USER_AGENT%" "" fi # block bad referrer diff --git a/entrypoint/utils.sh b/entrypoint/utils.sh index 3cb3f36..be83ccc 100644 --- a/entrypoint/utils.sh +++ b/entrypoint/utils.sh @@ -28,13 +28,19 @@ function has_value() { echo "ok" return 0 fi - for var in $(compgen -e) ; do + for var in $(env | grep -E "^.*_${1}=") ; do domain=$(echo "$var" | cut -d '_' -f 1) - name=$(echo "$var" | cut -d '=' -f 1 | sed "s~${domain}_~~") - value=$(echo "${!var}") - if [ "$name" == "$1" ] && [ "$value" == "$2" ] ; then + value=$(echo "$var" | sed "s~^${domain}_${1}=~~") + if [ "$value" == "$2" ] ; then echo "ok" return 0 fi done } + +# log to jobs.log +function job_log() { + when="$(date '+[%Y-%m-%d %H:%M:%S]')" + what="$1" + echo "$when $what" >> /var/log/jobs.log +} diff --git a/logs/logrotate.conf b/logs/logrotate.conf index 32df410..4880add 100644 --- a/logs/logrotate.conf +++ b/logs/logrotate.conf @@ -1,4 +1,4 @@ -/var/log/*.log /var/log/clamav/*.log /var/log/nginx/*.log { +/var/log/*.log /var/log/clamav/*.log /var/log/nginx/*.log /var/log/letsencrypt/*.log { # compress old files using gzip compress diff --git a/prepare.sh b/prepare.sh index 0f82ea9..2bbd32b 100644 --- a/prepare.sh +++ b/prepare.sh @@ -3,6 +3,9 @@ # install dependencies apk --no-cache add certbot libstdc++ libmaxminddb geoip pcre yajl fail2ban clamav apache2-utils rsyslog openssl lua libgd go jq mariadb-connector-c bash brotli +# temp fix ? +chmod 644 /usr/lib/python3.8/site-packages/fail2ban-*/* + # custom entrypoint mkdir /opt/entrypoint.d @@ -34,6 +37,9 @@ chmod 770 /var/log/*.log mkdir /var/log/letsencrypt chown nginx:nginx /var/log/letsencrypt chmod 770 /var/log/letsencrypt +touch /var/log/clamav.log +chown root:nginx /var/log/clamav.log +chmod 770 /var/log/clamav.log # prepare /acme-challenge mkdir /acme-challenge @@ -78,3 +84,11 @@ chown root:nginx /etc/rsyslog.conf touch /etc/crontabs/nginx chown root:nginx /etc/crontabs/nginx chmod 660 /etc/crontabs/nginx + +# prepare /var/log/clamav +chown root:nginx /var/log/clamav +chmod 770 /var/log/clamav + +# prepare /var/lib/clamav +chown root:nginx /var/lib/clamav +chmod 770 /var/lib/clamav diff --git a/scripts/abusers.sh b/scripts/abusers.sh index 96f8c21..c768434 100755 --- a/scripts/abusers.sh +++ b/scripts/abusers.sh @@ -1,7 +1,7 @@ #!/bin/sh # load some functions -. /opt/scripts/utils.sh +. /opt/entrypoint/utils.sh # copy old conf to cache cp /etc/nginx/block-abusers.conf /cache diff --git a/scripts/certbot-renew-hook.sh b/scripts/certbot-renew-hook.sh index 34f46e0..0dc9b19 100644 --- a/scripts/certbot-renew-hook.sh +++ b/scripts/certbot-renew-hook.sh @@ -1,7 +1,7 @@ #!/bin/sh # load some functions -. /opt/scripts/utils.sh +. /opt/entrypoint/utils.sh job_log "[CERTBOT] certificates have been renewed" diff --git a/scripts/certbot-renew.sh b/scripts/certbot-renew.sh index 996b900..c567c23 100644 --- a/scripts/certbot-renew.sh +++ b/scripts/certbot-renew.sh @@ -1,7 +1,7 @@ #!/bin/sh # load some functions -. /opt/scripts/utils.sh +. /opt/entrypoint/utils.sh # ask new certificates if needed certbot renew --deploy-hook /opt/scripts/certbot-renew-hook.sh diff --git a/scripts/exit-nodes.sh b/scripts/exit-nodes.sh index d319b87..8ecfdb1 100644 --- a/scripts/exit-nodes.sh +++ b/scripts/exit-nodes.sh @@ -1,7 +1,7 @@ #!/bin/sh # load some functions -. /opt/scripts/utils.sh +. /opt/entrypoint/utils.sh # copy old conf to cache cp /etc/nginx/block-tor-exit-node.conf /cache diff --git a/scripts/geoip.sh b/scripts/geoip.sh index 2eec70f..d28bf61 100644 --- a/scripts/geoip.sh +++ b/scripts/geoip.sh @@ -1,7 +1,7 @@ #!/bin/sh # load some functions -. /opt/scripts/utils.sh +. /opt/entrypoint/utils.sh # if we are running nginx if [ -f /tmp/nginx.pid ] ; then diff --git a/scripts/logrotate.sh b/scripts/logrotate.sh index 7f88c9b..6b467b3 100644 --- a/scripts/logrotate.sh +++ b/scripts/logrotate.sh @@ -1,7 +1,7 @@ #!/bin/sh # load some functions -. /opt/scripts/utils.sh +. /opt/entrypoint/utils.sh logrotate -f /etc/logrotate.conf > /dev/null 2>&1 diff --git a/scripts/proxies.sh b/scripts/proxies.sh index f6d1659..3108e81 100755 --- a/scripts/proxies.sh +++ b/scripts/proxies.sh @@ -1,7 +1,7 @@ #!/bin/sh # load some functions -. /opt/scripts/utils.sh +. /opt/entrypoint/utils.sh # copy old conf to cache cp /etc/nginx/block-proxies.conf /cache diff --git a/scripts/referrers.sh b/scripts/referrers.sh index 65be9a3..6f4932e 100755 --- a/scripts/referrers.sh +++ b/scripts/referrers.sh @@ -1,7 +1,7 @@ #!/bin/sh # load some functions -. /opt/scripts/utils.sh +. /opt/entrypoint/utils.sh # save old conf cp /etc/nginx/map-referrer.conf /cache diff --git a/scripts/user-agents.sh b/scripts/user-agents.sh index 1e1ee87..9207053 100755 --- a/scripts/user-agents.sh +++ b/scripts/user-agents.sh @@ -1,7 +1,7 @@ #!/bin/sh # load some functions -. /opt/scripts/utils.sh +. /opt/entrypoint/utils.sh # save old conf cp /etc/nginx/map-user-agent.conf /cache diff --git a/scripts/utils.sh b/scripts/utils.sh deleted file mode 100644 index 722ebe6..0000000 --- a/scripts/utils.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -function job_log() { - when="$(date '+[%d/%m/%Y %H:%M:%S]')" - what="$1" - echo "$when $what" >> /var/log/jobs.log -} - diff --git a/ui/config.json b/ui/config.json index c7f3a20..965bb23 100644 --- a/ui/config.json +++ b/ui/config.json @@ -687,10 +687,10 @@ }, { "type":"text", - "label":"Whitelist user agent list", - "env":"WHITELIST_USERAGENT_LIST", + "label":"Whitelist user agent", + "env":"WHITELIST_USER_AGENT", "regex":".*", - "id":"whitelist-user-agent-list", + "id":"whitelist-user-agent", "default":"" } ]