Makussu/bunkerweb
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

jobs - fix str/bytes hell

Browse Source
This commit is contained in:
bunkerity 2021-10-06 21:09:27 +02:00
parent d53f02b5b3
commit fb799765a4
No known key found for this signature in database
GPG Key ID: 3D80806F12602A7C
3 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
jobs/Job.py
View File

@ -128,7 +128,9 @@ class Job(abc.ABC) :
for url in self._data : for url in self._data :
data = self.__download_data(url) data = self.__download_data(url)
for chunk in data : for chunk in data :
if self._type == ["line", "json"] : if isinstance(chunk, bytes) :
chunk = chunk.decode("utf-8")
if self._type in ["line", "json"] :
if not re.match(self._regex, chunk) : if not re.match(self._regex, chunk) :
continue continue
if self._redis == None : if self._redis == None :
@ -207,7 +209,7 @@ class Job(abc.ABC) :
return JobRet.OK_RELOAD return JobRet.OK_RELOAD
return JobRet.OK_NO_RELOAD return JobRet.OK_NO_RELOAD
if self._redis != None and self._type == "line" : if self._redis != None and self._type in ["line", "json"] :
with open("/opt/bunkerized-nginx/cache/" + self._filename) as f : with open("/opt/bunkerized-nginx/cache/" + self._filename) as f :
pipe = self._redis.pipeline() pipe = self._redis.pipeline()
while True : while True :
@ -224,7 +226,7 @@ class Job(abc.ABC) :
def __to_cache(self) : def __to_cache(self) :
if self._redis == None or self._type == "file" : if self._redis == None or self._type == "file" :
shutil.copyfile("/etc/nginx/" + self._filename, "/opt/bunkerized-nginx/cache/" + self._filename) shutil.copyfile("/etc/nginx/" + self._filename, "/opt/bunkerized-nginx/cache/" + self._filename)
elif self._redis != None and self._type == "line" : elif self._redis != None and self._type in ["line", "json"] :
if os.path.isfile("/opt/bunkerized-nginx/cache/" + self._filename) : if os.path.isfile("/opt/bunkerized-nginx/cache/" + self._filename) :
os.remove("/opt/bunkerized-nginx/cache/" + self._filename) os.remove("/opt/bunkerized-nginx/cache/" + self._filename)
with open("/opt/bunkerized-nginx/cache/" + self._filename, "a") as f : with open("/opt/bunkerized-nginx/cache/" + self._filename, "a") as f :

2
jobs/Referrers.py
View File

@ -12,4 +12,4 @@ class Referrers(Job) :
super().__init__(name, data, filename, redis_host=redis_host, redis_ex=redis_ex, type=type, regex=regex, copy_cache=copy_cache) super().__init__(name, data, filename, redis_host=redis_host, redis_ex=redis_ex, type=type, regex=regex, copy_cache=copy_cache)
def _edit(self, chunk) : def _edit(self, chunk) :
return [chunk.replace(b".", b"%.").replace(b"-", b"%-")] return [chunk.replace(".", "%.").replace("-", "%-")]

2
jobs/UserAgents.py
View File

@ -12,4 +12,4 @@ class UserAgents(Job) :
super().__init__(name, data, filename, redis_host=redis_host, redis_ex=redis_ex, type=type, regex=regex, copy_cache=copy_cache) super().__init__(name, data, filename, redis_host=redis_host, redis_ex=redis_ex, type=type, regex=regex, copy_cache=copy_cache)
def _edit(self, chunk) : def _edit(self, chunk) :
return [chunk.replace(b"\\ ", b" ").replace(b"\\.", b"%.").replace(b"\\\\", b"\\").replace(b"-", b"%-")] return [chunk.replace("\\ ", " ").replace("\\.", "%.").replace("\\\\", "\\").replace("-", "%-")]