From fb799765a40ad21b1e2f1f5bcc3839d9670d6361 Mon Sep 17 00:00:00 2001 From: bunkerity Date: Wed, 6 Oct 2021 21:09:27 +0200 Subject: [PATCH] jobs - fix str/bytes hell --- jobs/Job.py | 8 +++++--- jobs/Referrers.py | 2 +- jobs/UserAgents.py | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/jobs/Job.py b/jobs/Job.py index c2fcd71..4c8a8a6 100644 --- a/jobs/Job.py +++ b/jobs/Job.py @@ -128,7 +128,9 @@ class Job(abc.ABC) : for url in self._data : data = self.__download_data(url) for chunk in data : - if self._type == ["line", "json"] : + if isinstance(chunk, bytes) : + chunk = chunk.decode("utf-8") + if self._type in ["line", "json"] : if not re.match(self._regex, chunk) : continue if self._redis == None : @@ -207,7 +209,7 @@ class Job(abc.ABC) : return JobRet.OK_RELOAD return JobRet.OK_NO_RELOAD - if self._redis != None and self._type == "line" : + if self._redis != None and self._type in ["line", "json"] : with open("/opt/bunkerized-nginx/cache/" + self._filename) as f : pipe = self._redis.pipeline() while True : @@ -224,7 +226,7 @@ class Job(abc.ABC) : def __to_cache(self) : if self._redis == None or self._type == "file" : shutil.copyfile("/etc/nginx/" + self._filename, "/opt/bunkerized-nginx/cache/" + self._filename) - elif self._redis != None and self._type == "line" : + elif self._redis != None and self._type in ["line", "json"] : if os.path.isfile("/opt/bunkerized-nginx/cache/" + self._filename) : os.remove("/opt/bunkerized-nginx/cache/" + self._filename) with open("/opt/bunkerized-nginx/cache/" + self._filename, "a") as f : diff --git a/jobs/Referrers.py b/jobs/Referrers.py index 7c7a5e9..dd8a8f1 100644 --- a/jobs/Referrers.py +++ b/jobs/Referrers.py @@ -12,4 +12,4 @@ class Referrers(Job) : super().__init__(name, data, filename, redis_host=redis_host, redis_ex=redis_ex, type=type, regex=regex, copy_cache=copy_cache) def _edit(self, chunk) : - return [chunk.replace(b".", b"%.").replace(b"-", b"%-")] + return [chunk.replace(".", "%.").replace("-", "%-")] diff --git a/jobs/UserAgents.py b/jobs/UserAgents.py index 8101cde..b0da32e 100644 --- a/jobs/UserAgents.py +++ b/jobs/UserAgents.py @@ -12,4 +12,4 @@ class UserAgents(Job) : super().__init__(name, data, filename, redis_host=redis_host, redis_ex=redis_ex, type=type, regex=regex, copy_cache=copy_cache) def _edit(self, chunk) : - return [chunk.replace(b"\\ ", b" ").replace(b"\\.", b"%.").replace(b"\\\\", b"\\").replace(b"-", b"%-")] + return [chunk.replace("\\ ", " ").replace("\\.", "%.").replace("\\\\", "\\").replace("-", "%-")]