From 88b5ce30039ff7d249ea7b31f98ea926cc739cfe Mon Sep 17 00:00:00 2001
From: Paul Lecuq <paul@paulbsd.com>
Date: Wed, 3 Feb 2016 21:31:33 +0100
Subject: [PATCH] py-squid-blacklists: various updates

---
 .gitignore             |  2 ++
 README.md              |  6 ++++++
 config.py.sample       |  1 +
 py-squid-blacklists.py | 47 +++++++++++++++++++++---------------------
 4 files changed, 33 insertions(+), 23 deletions(-)

diff --git a/.gitignore b/.gitignore
index 1dbc687..8d996cd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -60,3 +60,5 @@ target/
 
 #Ipython Notebook
 .ipynb_checkpoints
+
+config.py
diff --git a/README.md b/README.md
index 155e70c..d40ef67 100644
--- a/README.md
+++ b/README.md
@@ -1,2 +1,8 @@
 # py-squid-blacklists
 Squid helper handling squidguard blacklists written in python
+
+* Only supports domains blacklists actually (ie : google.com, www.google.com, api.google.com, etc.)
+* All blacklists are loaded in RAM
+* Usable as an external acl plugin of squid
+* Written because of poor developpement on squidguard and bad support of blacklists files using squid3
+* Tested on Debian 8 / python 2.7.9
diff --git a/config.py.sample b/config.py.sample
index 4e82a35..6bb9d6d 100644
--- a/config.py.sample
+++ b/config.py.sample
@@ -1,2 +1,3 @@
+blacklists_fetch = "http://dsi.ut-capitole.fr/blacklists/download/blacklists.tar.gz"
 blacklists_dir = "/usr/local/py-squid-blacklists/blacklists/"
 blacklists = ["adult","malware"]
diff --git a/py-squid-blacklists.py b/py-squid-blacklists.py
index b595d85..b966032 100755
--- a/py-squid-blacklists.py
+++ b/py-squid-blacklists.py
@@ -1,17 +1,22 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 import sys
 import os
 import re
 import logging
 import time
+import urllib
 from urlparse import urlparse
-from config import *
+try:
+	from config import *
+except ImportError:
+	print("Please create config.py using config.py.sample")
+	exit()
 
 domain_files = [os.path.join(dp, f) for dp, dn, fn in os.walk(os.path.expanduser(blacklists_dir)) for f in fn if re.match(r"domains*", f)]
 
 def make_list(files):
-	blacklists=[]
+	blacklists = []
 	for f in files:
 		splitlist = f.split("/")
 		list_type = splitlist[len(splitlist)-2]
@@ -21,35 +26,31 @@ def make_list(files):
 def make_db(blacklist_files):
 	lib = dict()
 	for blacklist in blacklist_files:
-		cache= dict()
-		values=[]
+		cache = dict()
+		values = []
 		f = open(blacklist[1], "r")
 		for line in f:
 			cache[line.strip("\n")] = True
-		lib[blacklist[0]]=cache
+		lib[blacklist[0]] = cache
 	return lib
 
-def compare(outline,blacklist_cache):
+def compare(outline,blacklist_cache,blacklists):
 	result = False
-	while not result and outline != "":
-		try:
-			result=blacklist_cache['adult'][outline]
-		except KeyError:
-			pass
-		outline = outline.partition('.')[2]
+	for blacklist in blacklists:
+		while not result and outline != "":
+			try:
+				result = blacklist_cache[blacklist][outline]
+			except KeyError:
+				pass
+			outline = outline.partition('.')[2]
 	return result
 
-def grant():
-	sys.stdout.write( 'OK\n' )
-	sys.stdout.flush()
-
-def deny():
-	sys.stdout.write( 'ERR\n' )
+def squid_response(response):
+	sys.stdout.write("%s\n" % response)
 	sys.stdout.flush()
 
 
 blacklist_cache=[]
-
 blacklist_files = make_list(domain_files)
 blacklist_cache = make_db(blacklist_files)
 
@@ -57,7 +58,7 @@ while True:
 	line = sys.stdin.readline().strip()
 	outline = urlparse(line).netloc
 	if line:
-		if compare(outline,blacklist_cache):
-			grant()
+		if compare(outline,blacklist_cache,blacklists):
+			squid_response("OK")
 		else:
-			deny()
+			squid_response("ERR")