From 43c6164e10b205414e731ca47a242709ac7a8698 Mon Sep 17 00:00:00 2001 From: Paul Lecuq Date: Sat, 20 Feb 2016 12:40:51 +0100 Subject: [PATCH] Comparison on flavors Code cleaning ... --- py-squid-blacklists-cdb.py | 109 ++++++++++++++++++++----------------- py-squid-blacklists.py | 93 +++++++++++++++++-------------- 2 files changed, 110 insertions(+), 92 deletions(-) diff --git a/py-squid-blacklists-cdb.py b/py-squid-blacklists-cdb.py index fc37c15..05c7350 100755 --- a/py-squid-blacklists-cdb.py +++ b/py-squid-blacklists-cdb.py @@ -5,69 +5,78 @@ import os import re import urllib from urlparse import urlparse + try: - import config + import config except ImportError: - print("Please create config.py using config.py.sample") - exit() + print("Please create config.py using config.py.sample") + exit() try: - import cdb + import cdb except ImportError: - print("Please install python-cdb from pypi or via package manager") - exit() + print("Please install python-cdb from pypi or via package manager") + exit() + def make_list(files): - blacklists = [] - for l in files: - splitlist = l.split("/") - list_type = splitlist[len(splitlist)-2] - blacklists.append([list_type,l]) - return blacklists + blacklists = [] + for l in files: + splitlist = l.split("/") + list_type = splitlist[len(splitlist) - 2] + blacklists.append([list_type, l]) + return blacklists -def make_db(blacklist_files,config): - lib = [] - for bl in blacklist_files: - bl_cdb_file = ("%s/%s.cdb" % (config.blacklists_dir,bl[0])) - bl_cdb_file_tmp = ("%s/%s.tmp" % (config.blacklists_dir,bl[0])) - if(bl[0] in config.blacklists): - if not os.path.isfile(bl_cdb_file): - cdb_file = cdb.cdbmake(bl_cdb_file,bl_cdb_file_tmp) - cache = dict() - f = open(bl[1], "r") - for line in f: - cdb_file.add(line.strip("\n"),"True") - cdb_file.finish() - lib.append(bl_cdb_file) - return lib -def compare(outline,blacklist_cache): - result = False - for blacklist in blacklist_cache: - cdb_file = cdb.init(blacklist) - tmpline = outline - while not result and tmpline != "": - try: - result = cdb_file[tmpline] - pass - except KeyError: - pass - tmpline = tmpline.partition('.')[2] - return result +def make_db(blacklist_files, config): + lib = [] + for bl in blacklist_files: + bl_cdb_file = ("%s/%s.cdb" % (config.blacklists_dir, bl[0])) + bl_cdb_file_tmp = ("%s/%s.tmp" % (config.blacklists_dir, bl[0])) + if (bl[0] in config.blacklists): + if not os.path.isfile(bl_cdb_file): + cdb_file = cdb.cdbmake(bl_cdb_file, bl_cdb_file_tmp) + cache = dict() + f = open(bl[1], "r") + for line in f: + cdb_file.add(line.strip("\n"), "True") + cdb_file.finish() + lib.append(bl_cdb_file) + return lib + + +def compare(outline, blacklist_cache): + result = False + for blacklist in blacklist_cache: + cdb_file = cdb.init(blacklist) + tmpline = outline + while not result and tmpline != "": + try: + result = cdb_file[tmpline] + pass + except KeyError: + pass + tmpline = tmpline.partition('.')[2] + return result + def squid_response(response): - sys.stdout.write("%s\n" % response) - sys.stdout.flush() + sys.stdout.write("%s\n" % response) + sys.stdout.flush() + domain_files = [os.path.join(dp, f) for dp, dn, fn in os.walk(os.path.expanduser(config.blacklists_dir)) for f in fn if re.match(r"domains*", f)] blacklist_files = make_list(domain_files) -blacklist_cache = make_db(blacklist_files,config) +blacklist_cache = make_db(blacklist_files, config) while True: - line = sys.stdin.readline().strip() - outline = urlparse(line).netloc - if line: - if compare(outline,blacklist_cache): - squid_response("OK") - else: - squid_response("ERR") + try: + line = sys.stdin.readline().strip() + outline = urlparse(line).netloc + if line: + if compare(outline, blacklist_cache): + squid_response("OK") + else: + squid_response("ERR") + except KeyboardInterrupt: + break \ No newline at end of file diff --git a/py-squid-blacklists.py b/py-squid-blacklists.py index a21695a..e2826fb 100755 --- a/py-squid-blacklists.py +++ b/py-squid-blacklists.py @@ -5,59 +5,68 @@ import os import re import urllib from urlparse import urlparse + try: - import config + import config except ImportError: - print("Please create config.py using config.py.sample") - exit() + print("Please create config.py using config.py.sample") + exit() + def make_list(files): - blacklists = [] - for l in files: - splitlist = l.split("/") - list_type = splitlist[len(splitlist)-2] - blacklists.append([list_type,l]) - return blacklists + blacklists = [] + for l in files: + splitlist = l.split("/") + list_type = splitlist[len(splitlist) - 2] + blacklists.append([list_type, l]) + return blacklists -def make_db(blacklist_files,blacklists): - lib = dict() - for bl in blacklist_files: - if(bl[0] in blacklists): - cache = dict() - f = open(bl[1], "r") - for line in f: - cache[line.strip("\n")] = True - lib[bl[0]] = cache - del cache - return lib -def compare(outline,blacklist_cache): - result = False - for blacklist in blacklist_cache: - tmpline = outline - while not result and tmpline != "": - try: - result = blacklist_cache[blacklist][tmpline] - pass - except KeyError: - pass - tmpline = tmpline.partition('.')[2] - return result +def make_db(blacklist_files, config): + lib = dict() + for bl in blacklist_files: + if (bl[0] in config.blacklists): + cache = dict() + f = open(bl[1], "r") + for line in f: + cache[line.strip("\n")] = True + lib[bl[0]] = cache + del cache + return lib + + +def compare(outline, blacklist_cache): + result = False + for blacklist in blacklist_cache: + tmpline = outline + while not result and tmpline != "": + try: + result = blacklist_cache[blacklist][tmpline] + pass + except KeyError: + pass + tmpline = tmpline.partition('.')[2] + return result + def squid_response(response): - sys.stdout.write("%s\n" % response) - sys.stdout.flush() + sys.stdout.write("%s\n" % response) + sys.stdout.flush() + domain_files = [os.path.join(dp, f) for dp, dn, fn in os.walk(os.path.expanduser(config.blacklists_dir)) for f in fn if re.match(r"domains*", f)] blacklist_files = make_list(domain_files) -blacklist_cache = make_db(blacklist_files,config.blacklists) +blacklist_cache = make_db(blacklist_files, config) while True: - line = sys.stdin.readline().strip() - outline = urlparse(line).netloc - if line: - if compare(outline,blacklist_cache): - squid_response("OK") - else: - squid_response("ERR") + try: + line = sys.stdin.readline().strip() + outline = urlparse(line).netloc + if line: + if compare(outline, blacklist_cache): + squid_response("OK") + else: + squid_response("ERR") + except KeyboardInterrupt: + break