updated ipbl

go.mod

@@ -21,7 +21,6 @@ require (
 	github.com/mattn/go-isatty v0.0.16 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/onsi/ginkgo v1.16.5 // indirect
 	github.com/onsi/gomega v1.22.1 // indirect
 	github.com/syndtr/goleveldb v1.0.0 // indirect
@@ -31,7 +30,5 @@ require (
 	golang.org/x/net v0.0.0-20221014081412-f15817d10f9b // indirect
 	golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43 // indirect
 	golang.org/x/text v0.3.8 // indirect
-	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 	xorm.io/builder v0.3.12 // indirect
 )

src/models/host.go

@@ -0,0 +1,54 @@
+package models
+
+import (
+	"xorm.io/xorm"
+)
+
+func (host *Host) GetOrCreate(session *xorm.Session) (apihost *APIHost, err error) {
+	has, err := session.Get(host)
+	if err != nil {
+		return
+	}
+	if !has {
+		session.Insert(host)
+	} else {
+		session.ID(host.ID).Update(host)
+	}
+	host.Get(session)
+	apihost = host.APIFormat()
+	return
+}
+
+func (host *Host) Get(session *xorm.Session) (apihost *APIHost, err error) {
+	has, err := session.Get(host)
+	if !has || err != nil {
+		return
+	}
+	apihost = host.APIFormat()
+	return
+}
+
+func (host *Host) APIFormat() *APIHost {
+	if host == nil {
+		return &APIHost{}
+	}
+	return &APIHost{
+		Host: host.Host,
+	}
+}
+
+func (host *Host) APIParse(apihost APIHost) (err error) {
+	*host = Host{
+		Host: apihost.Host}
+	return
+}
+
+type Host struct {
+	ID   int    `xorm:"pk autoincr"`
+	Host string `xorm:"text unique host" json:"host"`
+}
+
+type APIHost struct {
+	ID   int    `json:"-"`
+	Host string `json:"host"`
+}

src/models/ip.go

@@ -37,19 +37,28 @@ func GetIPsLast(ctx *context.Context, config *config.Config, interval string) (a
 }
 
 func (ip *IP) GetOrCreate(session *xorm.Session) (apiip *APIIP, err error) {
-	var tmpip = IP{IP: ip.IP}
+	var tmpip = &IP{IP: ip.IP}
-	has, err := session.Get(&tmpip)
+	has, err := session.Get(tmpip)
+	if err != nil {
+		log.Println(err)
+	}
+
 	ip.City.GetOrCreate(session)
 	ip.Country.GetOrCreate(session)
 	ip.AutonomousSystem.GetOrCreate(session)
+	ip.Host.GetOrCreate(session)
+	ip.Src.GetOrCreate(session)
+	session.Commit()
 
 	if !has {
 		session.Insert(ip)
 	} else {
 		ip.ID = tmpip.ID
 		session.ID(ip.ID).AllCols().Update(ip)
-		session.ID(ip.ID).Cols("city", "country", "autonomous_system").Update(ip)
+		session.ID(ip.ID).Cols("city_id", "country_id", "as_id").Update(ip)
 	}
+	session.Commit()
+
 	ip.Get(session)
 	apiip = ip.APIFormat()
 	return
@@ -117,13 +126,14 @@ func InsertIPBulk(session *xorm.Session, ips *[]IP) (numinsert int64, numupdate
 }
 
 func ScanIP(cfg *config.Config) (err error) {
-	cols := []string{"rdns", "autonomous_system", "city", "country", "updated"}
-	session := cfg.Db.NewSession()
-	defer session.Close()
-
 	for {
+		session := cfg.Db.NewSession()
 		orphans := []IP{}
-		if session.Asc("updated").Limit(ScanLimit).Find(&orphans); len(orphans) > 0 {
+		err = session.Where("as_id is null or country_id is null or src_id is null or host_id is null").Asc("updated").Limit(ScanLimit).Find(&orphans)
+		session.Close()
+
+		if err == nil && len(orphans) > 0 {
+			session := cfg.Db.NewSession()
 			err = session.Begin()
 			if err != nil {
 				log.Println(err)
@@ -135,41 +145,33 @@ func ScanIP(cfg *config.Config) (err error) {
 					continue
 				}
 				as := &AutonomousSystem{ASID: query.APIAS.ASID, ASName: query.APIAS.ASName}
-				as.GetOrCreate(session)
 				orphan.AutonomousSystem = as
 
 				if query.APICity != "" {
-					var city = &City{CityName: query.APICity}
+					var city = City{CityName: query.APICity}
-					city.GetOrCreate(session)
+					orphan.City = &city
-					orphan.City = city
-
 				}
 
 				if query.APICountry != "" {
-					var country = &Country{CountryName: query.APICountry}
+					var country = Country{CountryName: query.APICountry}
-					country.GetOrCreate(session)
+					orphan.Country = &country
-					orphan.Country = country
 				}
 
-				orphan.Rdns.String = query.Rdns
+				orphan.Rdns = sql.NullString{String: query.Rdns, Valid: true}
-				orphan.Rdns.Valid = true
 				log.Printf("%s -> \"%s\"\n", orphan.IP, query.Rdns)
 
-				_, err = session.ID(orphan.ID).Cols(cols...).Update(&orphan)
+				orphan.GetOrCreate(session)
 
-				if err != nil {
-					session.Rollback()
-					log.Println(err)
-				}
-			}
 				err = session.Commit()
+				fmt.Println(err)
 				if err != nil {
 					log.Println(err)
 				}
+				session.Close()
+			}
 		} else {
 			time.Sleep(10 * time.Second)
 		}
-
 	}
 }
 
@@ -251,6 +253,8 @@ type IP struct {
 	AutonomousSystem *AutonomousSystem `xorm:"as_id int index default NULL"`
 	City             *City             `xorm:"city_id int index default NULL"`
 	Country          *Country          `xorm:"country_id int index default NULL"`
+	Src              *Src              `xorm:"src_id int index default NULL"`
+	Host             *Host             `xorm:"host_id int index default NULL"`
 	Whois            string            `xorm:"text index default null"`
 	Created          time.Time         `xorm:"created notnull"`
 	Updated          time.Time         `xorm:"updated notnull"`

src/models/models.go

@@ -28,6 +28,8 @@ func init() {
 		new(Country),
 		new(Event),
 		new(IP),
+		new(Src),
+		new(Host),
 	)
 	for _, name := range []string{"SSL", "UID"} {
 		names.LintGonicMapper[name] = true

src/models/src.go

@@ -0,0 +1,54 @@
+package models
+
+import (
+	"xorm.io/xorm"
+)
+
+func (src *Src) GetOrCreate(session *xorm.Session) (apisrc *APISrc, err error) {
+	has, err := session.Get(src)
+	if err != nil {
+		return
+	}
+	if !has {
+		session.Insert(src)
+	} else {
+		session.ID(src.ID).Update(src)
+	}
+	src.Get(session)
+	apisrc = src.APIFormat()
+	return
+}
+
+func (src *Src) Get(session *xorm.Session) (apisrc *APISrc, err error) {
+	has, err := session.Get(src)
+	if !has || err != nil {
+		return
+	}
+	apisrc = src.APIFormat()
+	return
+}
+
+func (src *Src) APIFormat() *APISrc {
+	if src == nil {
+		return &APISrc{}
+	}
+	return &APISrc{
+		Src: src.Src,
+	}
+}
+
+func (src *Src) APIParse(apisrc APISrc) (err error) {
+	*src = Src{
+		Src: apisrc.Src}
+	return
+}
+
+type Src struct {
+	ID  int    `xorm:"pk autoincr"`
+	Src string `xorm:"text unique(srcindex) src" json:"src"`
+}
+
+type APISrc struct {
+	ID  int    `json:"-"`
+	Src string `json:"src"`
+}

vendor/modules.txt

@@ -43,8 +43,6 @@ github.com/modern-go/concurrent
 # github.com/modern-go/reflect2 v1.0.2
 ## explicit; go 1.12
 github.com/modern-go/reflect2
-# github.com/nxadm/tail v1.4.8
-## explicit; go 1.13
 # github.com/onsi/ginkgo v1.16.5
 ## explicit; go 1.16
 # github.com/onsi/gomega v1.22.1
@@ -92,10 +90,6 @@ golang.org/x/text/unicode/norm
 # gopkg.in/ini.v1 v1.67.0
 ## explicit
 gopkg.in/ini.v1
-# gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7
-## explicit
-# gopkg.in/yaml.v3 v3.0.1
-## explicit
 # gopkg.in/zeromq/goczmq.v4 v4.1.0
 ## explicit
 gopkg.in/zeromq/goczmq.v4