py-squid-blacklists: various updates

This commit is contained in:
Paul 2016-02-03 21:31:33 +01:00
parent 8fe5aac738
commit 88b5ce3003
4 changed files with 33 additions and 23 deletions

2
.gitignore vendored
View File

@ -60,3 +60,5 @@ target/
#Ipython Notebook
.ipynb_checkpoints
config.py

View File

@ -1,2 +1,8 @@
# py-squid-blacklists
Squid helper handling squidguard blacklists written in python
* Only supports domains blacklists actually (ie : google.com, www.google.com, api.google.com, etc.)
* All blacklists are loaded in RAM
* Usable as an external acl plugin of squid
* Written because of poor developpement on squidguard and bad support of blacklists files using squid3
* Tested on Debian 8 / python 2.7.9

View File

@ -1,2 +1,3 @@
blacklists_fetch = "http://dsi.ut-capitole.fr/blacklists/download/blacklists.tar.gz"
blacklists_dir = "/usr/local/py-squid-blacklists/blacklists/"
blacklists = ["adult","malware"]

View File

@ -1,17 +1,22 @@
#!/usr/bin/python
#!/usr/bin/env python
import sys
import os
import re
import logging
import time
import urllib
from urlparse import urlparse
from config import *
try:
from config import *
except ImportError:
print("Please create config.py using config.py.sample")
exit()
domain_files = [os.path.join(dp, f) for dp, dn, fn in os.walk(os.path.expanduser(blacklists_dir)) for f in fn if re.match(r"domains*", f)]
def make_list(files):
blacklists=[]
blacklists = []
for f in files:
splitlist = f.split("/")
list_type = splitlist[len(splitlist)-2]
@ -21,35 +26,31 @@ def make_list(files):
def make_db(blacklist_files):
lib = dict()
for blacklist in blacklist_files:
cache= dict()
values=[]
cache = dict()
values = []
f = open(blacklist[1], "r")
for line in f:
cache[line.strip("\n")] = True
lib[blacklist[0]]=cache
lib[blacklist[0]] = cache
return lib
def compare(outline,blacklist_cache):
def compare(outline,blacklist_cache,blacklists):
result = False
for blacklist in blacklists:
while not result and outline != "":
try:
result=blacklist_cache['adult'][outline]
result = blacklist_cache[blacklist][outline]
except KeyError:
pass
outline = outline.partition('.')[2]
return result
def grant():
sys.stdout.write( 'OK\n' )
sys.stdout.flush()
def deny():
sys.stdout.write( 'ERR\n' )
def squid_response(response):
sys.stdout.write("%s\n" % response)
sys.stdout.flush()
blacklist_cache=[]
blacklist_files = make_list(domain_files)
blacklist_cache = make_db(blacklist_files)
@ -57,7 +58,7 @@ while True:
line = sys.stdin.readline().strip()
outline = urlparse(line).netloc
if line:
if compare(outline,blacklist_cache):
grant()
if compare(outline,blacklist_cache,blacklists):
squid_response("OK")
else:
deny()
squid_response("ERR")