From 467ce67fa6a3a3c174abf00d333e526e9e0236c5 Mon Sep 17 00:00:00 2001 From: Paul Lecuq Date: Sun, 23 Oct 2022 18:16:43 +0200 Subject: [PATCH] updated ipbl --- go.mod | 3 --- src/models/host.go | 54 ++++++++++++++++++++++++++++++++++++++++++++ src/models/ip.go | 54 ++++++++++++++++++++++++-------------------- src/models/models.go | 2 ++ src/models/src.go | 54 ++++++++++++++++++++++++++++++++++++++++++++ vendor/modules.txt | 6 ----- 6 files changed, 139 insertions(+), 34 deletions(-) create mode 100644 src/models/host.go create mode 100644 src/models/src.go diff --git a/go.mod b/go.mod index fbfea58..9b49b67 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,6 @@ require ( github.com/mattn/go-isatty v0.0.16 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/nxadm/tail v1.4.8 // indirect github.com/onsi/ginkgo v1.16.5 // indirect github.com/onsi/gomega v1.22.1 // indirect github.com/syndtr/goleveldb v1.0.0 // indirect @@ -31,7 +30,5 @@ require ( golang.org/x/net v0.0.0-20221014081412-f15817d10f9b // indirect golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43 // indirect golang.org/x/text v0.3.8 // indirect - gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect xorm.io/builder v0.3.12 // indirect ) diff --git a/src/models/host.go b/src/models/host.go new file mode 100644 index 0000000..9c2c5f1 --- /dev/null +++ b/src/models/host.go @@ -0,0 +1,54 @@ +package models + +import ( + "xorm.io/xorm" +) + +func (host *Host) GetOrCreate(session *xorm.Session) (apihost *APIHost, err error) { + has, err := session.Get(host) + if err != nil { + return + } + if !has { + session.Insert(host) + } else { + session.ID(host.ID).Update(host) + } + host.Get(session) + apihost = host.APIFormat() + return +} + +func (host *Host) Get(session *xorm.Session) (apihost *APIHost, err error) { + has, err := session.Get(host) + if !has || err != nil { + return + } + apihost = host.APIFormat() + return +} + +func (host *Host) APIFormat() *APIHost { + if host == nil { + return &APIHost{} + } + return &APIHost{ + Host: host.Host, + } +} + +func (host *Host) APIParse(apihost APIHost) (err error) { + *host = Host{ + Host: apihost.Host} + return +} + +type Host struct { + ID int `xorm:"pk autoincr"` + Host string `xorm:"text unique host" json:"host"` +} + +type APIHost struct { + ID int `json:"-"` + Host string `json:"host"` +} diff --git a/src/models/ip.go b/src/models/ip.go index da235e9..d7f964e 100644 --- a/src/models/ip.go +++ b/src/models/ip.go @@ -37,19 +37,28 @@ func GetIPsLast(ctx *context.Context, config *config.Config, interval string) (a } func (ip *IP) GetOrCreate(session *xorm.Session) (apiip *APIIP, err error) { - var tmpip = IP{IP: ip.IP} - has, err := session.Get(&tmpip) + var tmpip = &IP{IP: ip.IP} + has, err := session.Get(tmpip) + if err != nil { + log.Println(err) + } + ip.City.GetOrCreate(session) ip.Country.GetOrCreate(session) ip.AutonomousSystem.GetOrCreate(session) + ip.Host.GetOrCreate(session) + ip.Src.GetOrCreate(session) + session.Commit() if !has { session.Insert(ip) } else { ip.ID = tmpip.ID session.ID(ip.ID).AllCols().Update(ip) - session.ID(ip.ID).Cols("city", "country", "autonomous_system").Update(ip) + session.ID(ip.ID).Cols("city_id", "country_id", "as_id").Update(ip) } + session.Commit() + ip.Get(session) apiip = ip.APIFormat() return @@ -117,13 +126,14 @@ func InsertIPBulk(session *xorm.Session, ips *[]IP) (numinsert int64, numupdate } func ScanIP(cfg *config.Config) (err error) { - cols := []string{"rdns", "autonomous_system", "city", "country", "updated"} - session := cfg.Db.NewSession() - defer session.Close() - for { + session := cfg.Db.NewSession() orphans := []IP{} - if session.Asc("updated").Limit(ScanLimit).Find(&orphans); len(orphans) > 0 { + err = session.Where("as_id is null or country_id is null or src_id is null or host_id is null").Asc("updated").Limit(ScanLimit).Find(&orphans) + session.Close() + + if err == nil && len(orphans) > 0 { + session := cfg.Db.NewSession() err = session.Begin() if err != nil { log.Println(err) @@ -135,41 +145,33 @@ func ScanIP(cfg *config.Config) (err error) { continue } as := &AutonomousSystem{ASID: query.APIAS.ASID, ASName: query.APIAS.ASName} - as.GetOrCreate(session) orphan.AutonomousSystem = as if query.APICity != "" { - var city = &City{CityName: query.APICity} - city.GetOrCreate(session) - orphan.City = city - + var city = City{CityName: query.APICity} + orphan.City = &city } if query.APICountry != "" { - var country = &Country{CountryName: query.APICountry} - country.GetOrCreate(session) - orphan.Country = country + var country = Country{CountryName: query.APICountry} + orphan.Country = &country } - orphan.Rdns.String = query.Rdns - orphan.Rdns.Valid = true + orphan.Rdns = sql.NullString{String: query.Rdns, Valid: true} log.Printf("%s -> \"%s\"\n", orphan.IP, query.Rdns) - _, err = session.ID(orphan.ID).Cols(cols...).Update(&orphan) + orphan.GetOrCreate(session) + err = session.Commit() + fmt.Println(err) if err != nil { - session.Rollback() log.Println(err) } - } - err = session.Commit() - if err != nil { - log.Println(err) + session.Close() } } else { time.Sleep(10 * time.Second) } - } } @@ -251,6 +253,8 @@ type IP struct { AutonomousSystem *AutonomousSystem `xorm:"as_id int index default NULL"` City *City `xorm:"city_id int index default NULL"` Country *Country `xorm:"country_id int index default NULL"` + Src *Src `xorm:"src_id int index default NULL"` + Host *Host `xorm:"host_id int index default NULL"` Whois string `xorm:"text index default null"` Created time.Time `xorm:"created notnull"` Updated time.Time `xorm:"updated notnull"` diff --git a/src/models/models.go b/src/models/models.go index 90e1d79..2b2b752 100644 --- a/src/models/models.go +++ b/src/models/models.go @@ -28,6 +28,8 @@ func init() { new(Country), new(Event), new(IP), + new(Src), + new(Host), ) for _, name := range []string{"SSL", "UID"} { names.LintGonicMapper[name] = true diff --git a/src/models/src.go b/src/models/src.go new file mode 100644 index 0000000..178fa3e --- /dev/null +++ b/src/models/src.go @@ -0,0 +1,54 @@ +package models + +import ( + "xorm.io/xorm" +) + +func (src *Src) GetOrCreate(session *xorm.Session) (apisrc *APISrc, err error) { + has, err := session.Get(src) + if err != nil { + return + } + if !has { + session.Insert(src) + } else { + session.ID(src.ID).Update(src) + } + src.Get(session) + apisrc = src.APIFormat() + return +} + +func (src *Src) Get(session *xorm.Session) (apisrc *APISrc, err error) { + has, err := session.Get(src) + if !has || err != nil { + return + } + apisrc = src.APIFormat() + return +} + +func (src *Src) APIFormat() *APISrc { + if src == nil { + return &APISrc{} + } + return &APISrc{ + Src: src.Src, + } +} + +func (src *Src) APIParse(apisrc APISrc) (err error) { + *src = Src{ + Src: apisrc.Src} + return +} + +type Src struct { + ID int `xorm:"pk autoincr"` + Src string `xorm:"text unique(srcindex) src" json:"src"` +} + +type APISrc struct { + ID int `json:"-"` + Src string `json:"src"` +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 20a1d53..12c31d4 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -43,8 +43,6 @@ github.com/modern-go/concurrent # github.com/modern-go/reflect2 v1.0.2 ## explicit; go 1.12 github.com/modern-go/reflect2 -# github.com/nxadm/tail v1.4.8 -## explicit; go 1.13 # github.com/onsi/ginkgo v1.16.5 ## explicit; go 1.16 # github.com/onsi/gomega v1.22.1 @@ -92,10 +90,6 @@ golang.org/x/text/unicode/norm # gopkg.in/ini.v1 v1.67.0 ## explicit gopkg.in/ini.v1 -# gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 -## explicit -# gopkg.in/yaml.v3 v3.0.1 -## explicit # gopkg.in/zeromq/goczmq.v4 v4.1.0 ## explicit gopkg.in/zeromq/goczmq.v4