jobs - various bugs fixed and old files removed

jobs/CertbotNew.py

@@ -1,6 +1,6 @@
 from Job import Job
 
-class CertbotRenew(Job) :
+class CertbotNew(Job) :
 
 	def __init__(self, redis_host=None, copy_cache=False, domain="", email="") :
 		name = "certbot-new"

jobs/GeoIP.py

@@ -15,7 +15,7 @@ class GeoIP(Job) :
 		super().run()
 		count = 0
 		with gzip.open("/etc/nginx/geoip.mmdb.gz", "rb") as f :
-			with open("/tmp/geoip.mmdb", "w") as f2
+			with open("/tmp/geoip.mmdb", "w") as f2 :
 				while True :
 					chunk = f.read(8192)
 					if not chunk :

jobs/Job.py

@@ -1,4 +1,4 @@
-import abc, requests, redis, os, datetime, traceback
+import abc, requests, redis, os, datetime, traceback, re, shutil
 
 class Job(abc.ABC) :
 
@@ -41,7 +41,7 @@ class Job(abc.ABC) :
 		if self.__redis == None :
 			if os.path.isfile("/tmp/" + self.__filename) :
 				os.remove("/tmp/" + self.__filename)
-			file = open("/tmp/" + self.__filename, "a")
+			file = open("/tmp/" + self.__filename, "ab")
 
 		elif self.__redis != None :
 			pipe = self.__redis.pipeline()
@@ -50,7 +50,7 @@ class Job(abc.ABC) :
 		for url in self.__data :
 			data = self.__download_data(url)
 			for chunk in data :
-				if self.__type == "line" and not re.match(self.__regex, chunk) :
+				if self.__type == "line" and not re.match(self.__regex, chunk.decode("utf-8")) :
 					continue
 				count += 1
 				if self.__redis == None :
@@ -67,7 +67,7 @@ class Job(abc.ABC) :
 			os.remove("/tmp/" + self.__filename)
 
 		elif self.__redis != None and count > 0 :
-			self.__redis.del(self.__redis.keys(self.__name + "_*"))
+			self.__redis.delete(self.__redis.keys(self.__name + "_*"))
 			pipe.execute()
 
 	def __download_data(self, url) :
@@ -95,7 +95,7 @@ class Job(abc.ABC) :
 		if self.__redis == None or self.__type == "file" :
 			shutil.copyfile("/opt/bunkerized-nginx/cache/" + self.__filename, "/etc/nginx/" + self.__filename)
 		elif self.__redis != None and self.__type == "line" :
-			self.__redis.del(self.__redis.keys(self.__name + "_*"))
+			self.__redis.delete(self.__redis.keys(self.__name + "_*"))
 			with open("/opt/bunkerized-nginx/cache/" + self.__filename) as f :
 				pipe = self.__redis.pipeline()
 				while True :

jobs/Proxies.py

@@ -8,4 +8,4 @@ class Proxies(Job) :
 		filename = "proxies.list"
 		type = "line"
 		regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$"
-		super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex; copy_cache=copy_cache)
+		super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache)

jobs/main.py

@@ -1,9 +1,13 @@
+#!/usr/bin/python3
+
 import argparse, sys
 
 sys.path.append("/opt/bunkerized-nginx/jobs")
 
 import Abusers, CertbotNew, CertbotRenew, ExitNodes, GeoIP, Proxies, Referrers, SelfSignedCert, UserAgents
 
+from reload import reload
+
 JOBS = {
 	"abusers": Abusers.Abusers,
 	"certbot-new": CertbotNew.CertbotNew,
@@ -33,10 +37,12 @@ if __name__ == "__main__" :
 
 	# Check job name
 	if not args.name in JOBS :
-		print("[!] unknown job " + args.job)
+		print("[!] unknown job " + args.name)
 		sys.exit(1)
+	job = args.name
 
 	# Run job
+	print("[*] Executing job " + job)
 	ret = 0
 	if job == "certbot-new" :
 		instance = JOBS[job](redis_host=args.redis, copy_cache=args.cache, domain=args.domain, email=args.email)
@@ -45,9 +51,18 @@ if __name__ == "__main__" :
 	else :
 		instance = JOBS[job](redis_host=args.redis, copy_cache=args.cache)
 	if not instance.run() :
-		print("[!] error while running job " + job)
+		print("[!] Error while running job " + job)
 		sys.exit(1)
-	print("[*] job " + job + " successfully executed")
-	sys.exit(0)
+	print("[*] Job " + job + " successfully executed")
 
-	# TODO : reload
+	# Reload
+	# TODO : only reload if needed
+	do_reload = True
+	if do_reload :
+		if not reload() :
+			print("[!] Error while doing reload operation")
+			sys.exit(1)
+		print("[*] Reload operation successfully executed")
+
+	# Done
+	sys.exit(0)

jobs/reload.py

@@ -1,27 +1,41 @@
-import docker, subprocess, os, stat, sys
+import docker, subprocess, os, stat, sys, traceback
 
-if __name__ == "__main__" :
+def reload() :
 
 	# Linux or single Docker use case
-	if os.path.isfile("/usr/sbin/nginx") :
+	if os.path.isfile("/usr/sbin/nginx") and os.path.isfile("/tmp/nginx.pid") :
 		proc = subprocess.run(["/usr/sbin/nginx", "-s", "reload"], capture_output=True)
 		if proc.returncode != 0 :
-			print("[!] can't reload nginx (status code = " + str(proc.returncode) + ")"
+			print("[!] Can't reload nginx (status code = " + str(proc.returncode) + ")")
 			if len(proc.stdout.decode("ascii")) > 1 :
 				print(proc.stdout.decode("ascii"))
 			if len(proc.stderr.decode("ascii")) > 1 :
 				print(proc.stderr.decode("ascii"))
-			sys.exit(1)
+			return False
+		return True
 
 	# Autoconf case (Docker, Swarm and Ingress)
-	mode = os.stat("/tmp/autoconf.sock")
-	elif stat.S_ISSOCK(mode) :
+	if os.path.exists("/tmp/autoconf.sock") and stat.S_ISSOCK(os.stat("/tmp/autoconf.sock")) :
 		client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
 		client.connect("/tmp/autoconf.sock")
 		client.send("reload".encode("utf-8"))
 		data = client.recv(512)
 		client.close()
-        if not data or data.decode("utf-8") != "ok" :
-                sys.exit(2)
+		if not data or data.decode("utf-8") != "ok" :
+			print("[!] Can't reload nginx (data not ok)")
+			return False
+		return True
 
-	sys.exit(0)
+	return False
+
+if __name__ == "__main__" :
+	try :
+		print("[*] Starting reload operation ...")
+		if not reload() :
+			sys.exit(1)
+		print("[*] Reload operation successfully executed")
+		sys.exit(0)
+	except :
+		print("[!] Can't reload nginx (exception)")
+		print(traceback.format_exc())
+		sys.exit(2)

jobs/requirements.txt

@@ -1,2 +1,3 @@
 requests
 redis
+docker