jobs - edit referrers and user-agents data and init work on autoconf integration

jobs/Job.py

@@ -50,15 +50,17 @@ class Job(abc.ABC) :
 		for url in self.__data :
 			data = self.__download_data(url)
 			for chunk in data :
-				if self.__type == "line" and not re.match(self.__regex, chunk.decode("utf-8")) :
-					continue
-				count += 1
+				if self.__type == "line" :
+					if not re.match(self.__regex, chunk.decode("utf-8")) :
+						continue
+					chunk = self.__edit(chunk)
 				if self.__redis == None :
 					if self.__type == "line" :
 						chunk += b"\n"
 					file.write(chunk)
 				else :
 					pipe.set(self.__name + "_" + chunk, "1")
+				count += 1
 
 		if self.__redis == None :
 			file.close()
@@ -89,6 +91,9 @@ class Job(abc.ABC) :
 		if proc.returncode != 0 :
 			raise Exception("error code " + str(proc.returncode))
 
+	def __edit(self, chunk) :
+		return chunk
+
 	def __from_cache(self) :
 		if not os.path.isfile("/opt/bunkerized-nginx/cache/" + self.__filename) :
 			return False

jobs/Referrers.py

@@ -9,3 +9,6 @@ class Referrers(Job) :
 		type = "line"
 		regex = r"^.+$"
 		super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache)
+
+	def __edit(self, chunk) :
+		return chunk.replace(b".", b"%.").replace(b"-", b"%-")

jobs/UserAgents.py

@@ -9,3 +9,6 @@ class UserAgents(Job) :
 		type = "line"
 		regex = r"^.+$"
 		super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache)
+
+	def __edit(self, chunk) :
+		return chunk.replace(b"\\ ", b" ").replace(b"\\.", b"%.").replace(b"\\\\", b"\\").replace(b"-", b"%-")

jobs/main.py

@@ -59,10 +59,14 @@ if __name__ == "__main__" :
 	# TODO : only reload if needed
 	do_reload = True
 	if do_reload :
-		if not reload() :
+		ret = reload()
+		if ret == 0 :
+			print("[*] Reload operation successfully executed")
+		elif ret == 1 :
 			print("[!] Error while doing reload operation")
 			sys.exit(1)
-		print("[*] Reload operation successfully executed")
+		elif ret == 2 :
+			print("[*] Skipped reload operation because nginx is not running")
 
 	# Done
 	sys.exit(0)

jobs/reload.py

@@ -11,8 +11,8 @@ def reload() :
 				print(proc.stdout.decode("ascii"))
 			if len(proc.stderr.decode("ascii")) > 1 :
 				print(proc.stderr.decode("ascii"))
-			return False
-		return True
+			return 0
+		return 1
 
 	# Autoconf case (Docker, Swarm and Ingress)
 	if os.path.exists("/tmp/autoconf.sock") and stat.S_ISSOCK(os.stat("/tmp/autoconf.sock")) :
@@ -23,17 +23,21 @@ def reload() :
 		client.close()
 		if not data or data.decode("utf-8") != "ok" :
 			print("[!] Can't reload nginx (data not ok)")
-			return False
-		return True
+			return 0
+		return 1
 
-	return False
+	return 2
 
 if __name__ == "__main__" :
 	try :
 		print("[*] Starting reload operation ...")
-		if not reload() :
+		ret = reload()
+		if ret == 0 :
 			sys.exit(1)
-		print("[*] Reload operation successfully executed")
+		elif ret == 1 :
+			print("[*] Reload operation successfully executed")
+		elif ret == 2 :
+			print("[*] Skipped reload operation because nginx is not running")
 		sys.exit(0)
 	except :
 		print("[!] Can't reload nginx (exception)")