diff --git a/jobs/.Job.py.swp b/jobs/.Job.py.swp new file mode 100644 index 0000000..497c7c3 Binary files /dev/null and b/jobs/.Job.py.swp differ diff --git a/jobs/ExitNodes.py b/jobs/ExitNodes.py new file mode 100644 index 0000000..6e6091f --- /dev/null +++ b/jobs/ExitNodes.py @@ -0,0 +1,11 @@ +from Job import Job + +class ExitNodes(Job) : + + def __init__(self, redis_host=None) : + name = "ExitNodes" + urls = ["https://iplists.firehol.org/files/tor_exits.ipset"] + filename = "tor-exit-nodes.list" + type = "line" + regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" + super().__init__(name, urls, filename, redis_host=redis_host, type=type, regex=regex) diff --git a/jobs/Job.py b/jobs/Job.py new file mode 100644 index 0000000..33b169e --- /dev/null +++ b/jobs/Job.py @@ -0,0 +1,54 @@ +import abc, requests, redis, os + +class Job(abc.ABC) : + + def __init__(self, name, urls, filename, redis_host=None, type="line", regex=r"^.*$") : + self.__name = name + self.__urls = urls + self.__filename = filename + self.__redis = None + if redis_host != None : + self.__redis = redis.Redis(host=redis_host, port=6379, db=0) + self.__type = type + self.__regex = regex + + def run(self) : + if self.__redis == None : + if os.path.isfile("/tmp/" + self.__filename) : + os.remove("/tmp/" + self.__filename) + file = open("/tmp/" + self.__filename, "a") + + elif self.__redis != None : + pipe = self.__redis.pipeline() + + count = 0 + for url in self.__urls : + data = self.__download_data(url) + for chunk in data : + if self.__type == "line" and not re.match(self.__regex, chunk) : + continue + count += 1 + if self.__redis == None : + if self.__type == "line" : + chunk += b"\n" + file.write(chunk) + else : + pipe.set(self.__name + "_" + chunk, "1") + + if self.__redis == None : + file.close() + if count > 0 : + shutil.copyfile("/tmp/" + self.__filename, "/etc/nginx/" + self.__filename) + os.remove("/tmp/" + self.__filename) + + elif self.__redis != None and count > 0 : + self.__redis.del(self.__redis.keys(self.__name + "_*")) + pipe.execute() + + def __download_data(self, url) : + r = requests.get(url, stream=True) + if not r or r.status_code != 200 : + return False + if self.__type == "line" : + return r.iter_lines() + return r.iter_content(chunk_size=8192)