paulbsd/pybl
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Comparison on flavors

Browse Source 
Code cleaning ...
This commit is contained in:
Paul 2016-02-20 12:40:51 +01:00
parent 080505a032
commit 43c6164e10
2 changed files with 110 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

109
py-squid-blacklists-cdb.py
View File

@ -5,69 +5,78 @@ import os
import re
import urllib
from urlparse import urlparse
try:
import config
import config
except ImportError:
print("Please create config.py using config.py.sample")
exit()
print("Please create config.py using config.py.sample")
exit()
try:
import cdb
import cdb
except ImportError:
print("Please install python-cdb from pypi or via package manager")
exit()
print("Please install python-cdb from pypi or via package manager")
exit()
def make_list(files):
blacklists = []
for l in files:
splitlist = l.split("/")
list_type = splitlist[len(splitlist)-2]
blacklists.append([list_type,l])
return blacklists
blacklists = []
for l in files:
splitlist = l.split("/")
list_type = splitlist[len(splitlist) - 2]
blacklists.append([list_type, l])
return blacklists
def make_db(blacklist_files,config):
lib = []
for bl in blacklist_files:
bl_cdb_file = ("%s/%s.cdb" % (config.blacklists_dir,bl[0]))
bl_cdb_file_tmp = ("%s/%s.tmp" % (config.blacklists_dir,bl[0]))
if(bl[0] in config.blacklists):
if not os.path.isfile(bl_cdb_file):
cdb_file = cdb.cdbmake(bl_cdb_file,bl_cdb_file_tmp)
cache = dict()
f = open(bl[1], "r")
for line in f:
cdb_file.add(line.strip("\n"),"True")
cdb_file.finish()
lib.append(bl_cdb_file)
return lib
def compare(outline,blacklist_cache):
result = False
for blacklist in blacklist_cache:
cdb_file = cdb.init(blacklist)
tmpline = outline
while not result and tmpline != "":
try:
result = cdb_file[tmpline]
pass
except KeyError:
pass
tmpline = tmpline.partition('.')[2]
return result
def make_db(blacklist_files, config):
lib = []
for bl in blacklist_files:
bl_cdb_file = ("%s/%s.cdb" % (config.blacklists_dir, bl[0]))
bl_cdb_file_tmp = ("%s/%s.tmp" % (config.blacklists_dir, bl[0]))
if (bl[0] in config.blacklists):
if not os.path.isfile(bl_cdb_file):
cdb_file = cdb.cdbmake(bl_cdb_file, bl_cdb_file_tmp)
cache = dict()
f = open(bl[1], "r")
for line in f:
cdb_file.add(line.strip("\n"), "True")
cdb_file.finish()
lib.append(bl_cdb_file)
return lib
def compare(outline, blacklist_cache):
result = False
for blacklist in blacklist_cache:
cdb_file = cdb.init(blacklist)
tmpline = outline
while not result and tmpline != "":
try:
result = cdb_file[tmpline]
pass
except KeyError:
pass
tmpline = tmpline.partition('.')[2]
return result
def squid_response(response):
sys.stdout.write("%s\n" % response)
sys.stdout.flush()
sys.stdout.write("%s\n" % response)
sys.stdout.flush()
domain_files = [os.path.join(dp, f) for dp, dn, fn in os.walk(os.path.expanduser(config.blacklists_dir)) for f in fn if re.match(r"domains*", f)]
blacklist_files = make_list(domain_files)
blacklist_cache = make_db(blacklist_files,config)
blacklist_cache = make_db(blacklist_files, config)
while True:
line = sys.stdin.readline().strip()
outline = urlparse(line).netloc
if line:
if compare(outline,blacklist_cache):
squid_response("OK")
else:
squid_response("ERR")
try:
line = sys.stdin.readline().strip()
outline = urlparse(line).netloc
if line:
if compare(outline, blacklist_cache):
squid_response("OK")
else:
squid_response("ERR")
except KeyboardInterrupt:
break

93
py-squid-blacklists.py
View File

@ -5,59 +5,68 @@ import os
import re
import urllib
from urlparse import urlparse
try:
import config
import config
except ImportError:
print("Please create config.py using config.py.sample")
exit()
print("Please create config.py using config.py.sample")
exit()
def make_list(files):
blacklists = []
for l in files:
splitlist = l.split("/")
list_type = splitlist[len(splitlist)-2]
blacklists.append([list_type,l])
return blacklists
blacklists = []
for l in files:
splitlist = l.split("/")
list_type = splitlist[len(splitlist) - 2]
blacklists.append([list_type, l])
return blacklists
def make_db(blacklist_files,blacklists):
lib = dict()
for bl in blacklist_files:
if(bl[0] in blacklists):
cache = dict()
f = open(bl[1], "r")
for line in f:
cache[line.strip("\n")] = True
lib[bl[0]] = cache
del cache
return lib
def compare(outline,blacklist_cache):
result = False
for blacklist in blacklist_cache:
tmpline = outline
while not result and tmpline != "":
try:
result = blacklist_cache[blacklist][tmpline]
pass
except KeyError:
pass
tmpline = tmpline.partition('.')[2]
return result
def make_db(blacklist_files, config):
lib = dict()
for bl in blacklist_files:
if (bl[0] in config.blacklists):
cache = dict()
f = open(bl[1], "r")
for line in f:
cache[line.strip("\n")] = True
lib[bl[0]] = cache
del cache
return lib
def compare(outline, blacklist_cache):
result = False
for blacklist in blacklist_cache:
tmpline = outline
while not result and tmpline != "":
try:
result = blacklist_cache[blacklist][tmpline]
pass
except KeyError:
pass
tmpline = tmpline.partition('.')[2]
return result
def squid_response(response):
sys.stdout.write("%s\n" % response)
sys.stdout.flush()
sys.stdout.write("%s\n" % response)
sys.stdout.flush()
domain_files = [os.path.join(dp, f) for dp, dn, fn in os.walk(os.path.expanduser(config.blacklists_dir)) for f in fn if re.match(r"domains*", f)]
blacklist_files = make_list(domain_files)
blacklist_cache = make_db(blacklist_files,config.blacklists)
blacklist_cache = make_db(blacklist_files, config)
while True:
line = sys.stdin.readline().strip()
outline = urlparse(line).netloc
if line:
if compare(outline,blacklist_cache):
squid_response("OK")
else:
squid_response("ERR")
try:
line = sys.stdin.readline().strip()
outline = urlparse(line).netloc
if line:
if compare(outline, blacklist_cache):
squid_response("OK")
else:
squid_response("ERR")
except KeyboardInterrupt:
break