py-squid-blacklists: various updates

.gitignore

@@ -60,3 +60,5 @@ target/
 
 #Ipython Notebook
 .ipynb_checkpoints
+
+config.py

README.md

@@ -1,2 +1,8 @@
 # py-squid-blacklists
 Squid helper handling squidguard blacklists written in python
+
+* Only supports domains blacklists actually (ie : google.com, www.google.com, api.google.com, etc.)
+* All blacklists are loaded in RAM
+* Usable as an external acl plugin of squid
+* Written because of poor developpement on squidguard and bad support of blacklists files using squid3
+* Tested on Debian 8 / python 2.7.9

config.py.sample

@@ -1,2 +1,3 @@
+blacklists_fetch = "http://dsi.ut-capitole.fr/blacklists/download/blacklists.tar.gz"
 blacklists_dir = "/usr/local/py-squid-blacklists/blacklists/"
 blacklists = ["adult","malware"]

py-squid-blacklists.py

@@ -1,12 +1,17 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 import sys
 import os
 import re
 import logging
 import time
+import urllib
 from urlparse import urlparse
+try:
 	from config import *
+except ImportError:
+	print("Please create config.py using config.py.sample")
+	exit()
 
 domain_files = [os.path.join(dp, f) for dp, dn, fn in os.walk(os.path.expanduser(blacklists_dir)) for f in fn if re.match(r"domains*", f)]
 
@@ -29,27 +34,23 @@ def make_db(blacklist_files):
 		lib[blacklist[0]] = cache
 	return lib
 
-def compare(outline,blacklist_cache):
+def compare(outline,blacklist_cache,blacklists):
 	result = False
+	for blacklist in blacklists:
 		while not result and outline != "":
 			try:
-			result=blacklist_cache['adult'][outline]
+				result = blacklist_cache[blacklist][outline]
 			except KeyError:
 				pass
 			outline = outline.partition('.')[2]
 	return result
 
-def grant():
+def squid_response(response):
-	sys.stdout.write( 'OK\n' )
+	sys.stdout.write("%s\n" % response)
-	sys.stdout.flush()
-
-def deny():
-	sys.stdout.write( 'ERR\n' )
 	sys.stdout.flush()
 
 
 blacklist_cache=[]
-
 blacklist_files = make_list(domain_files)
 blacklist_cache = make_db(blacklist_files)
 
@@ -57,7 +58,7 @@ while True:
 	line = sys.stdin.readline().strip()
 	outline = urlparse(line).netloc
 	if line:
-		if compare(outline,blacklist_cache):
+		if compare(outline,blacklist_cache,blacklists):
-			grant()
+			squid_response("OK")
 		else:
-			deny()
+			squid_response("ERR")