From 5b95e817de20c301c9dceb8941590f14099d91b8 Mon Sep 17 00:00:00 2001 From: Paul Lecuq Date: Tue, 17 Jan 2023 18:36:44 +0100 Subject: [PATCH] updated ipbl * removed unused columns in tables * added multithreaded support for ip scans * added health endpoint --- src/models/as.go | 14 ++++-- src/models/city.go | 8 ++- src/models/country.go | 12 +++-- src/models/event.go | 21 ++++---- src/models/host.go | 8 ++- src/models/hostinfo.go | 3 +- src/models/ip.go | 108 +++++++++++++++++++++++++++-------------- src/models/src.go | 8 ++- src/routers/funcs.go | 4 ++ 9 files changed, 124 insertions(+), 62 deletions(-) diff --git a/src/models/as.go b/src/models/as.go index 041345f..c32c9e0 100644 --- a/src/models/as.go +++ b/src/models/as.go @@ -1,6 +1,10 @@ package models -import "xorm.io/xorm" +import ( + "time" + + "xorm.io/xorm" +) func (as *AutonomousSystem) GetOrCreate(session *xorm.Session) (apias *APIAutonomousSystem, err error) { has, err := session.Get(as) @@ -44,9 +48,11 @@ func (as *AutonomousSystem) APIParse(apias APIAutonomousSystem) (err error) { } type AutonomousSystem struct { - ID int `xorm:"pk autoincr"` - ASID int `xorm:"integer unique(asindex) as_id"` - ASName string `xorm:"text unique(asindex) as_name"` + ID int `xorm:"pk autoincr"` + ASID int `xorm:"integer unique(asindex) as_id"` + ASName string `xorm:"text unique(asindex) as_name"` + Created time.Time `xorm:"created notnull"` + Updated time.Time `xorm:"updated notnull"` } type APIAutonomousSystem struct { diff --git a/src/models/city.go b/src/models/city.go index 526e41e..2b5dec7 100644 --- a/src/models/city.go +++ b/src/models/city.go @@ -1,6 +1,8 @@ package models import ( + "time" + "xorm.io/xorm" ) @@ -44,8 +46,10 @@ func (city *City) APIParse(apicity APICity) (err error) { } type City struct { - ID int `xorm:"pk autoincr"` - CityName string `xorm:"text unique(cityindex) city_name" json:"city_name"` + ID int `xorm:"pk autoincr"` + CityName string `xorm:"text unique(cityindex) city_name" json:"city_name"` + Created time.Time `xorm:"created notnull"` + Updated time.Time `xorm:"updated notnull"` } type APICity struct { diff --git a/src/models/country.go b/src/models/country.go index 4333f7f..f4b23b1 100644 --- a/src/models/country.go +++ b/src/models/country.go @@ -1,6 +1,10 @@ package models -import "xorm.io/xorm" +import ( + "time" + + "xorm.io/xorm" +) func (country *Country) GetOrCreate(session *xorm.Session) (apicountry *APICountry, err error) { has, err := session.Get(country) @@ -42,8 +46,10 @@ func (country *Country) APIParse(apicountry APICountry) (err error) { } type Country struct { - ID int `xorm:"pk autoincr"` - CountryName string `xorm:"text unique(countryindex) country_name" json:"country_name"` + ID int `xorm:"pk autoincr"` + CountryName string `xorm:"text unique(countryindex) country_name" json:"country_name"` + Created time.Time `xorm:"created notnull"` + Updated time.Time `xorm:"updated notnull"` } type APICountry struct { diff --git a/src/models/event.go b/src/models/event.go index 3bfc74b..6335381 100644 --- a/src/models/event.go +++ b/src/models/event.go @@ -1,7 +1,6 @@ package models import ( - "database/sql" "time" "git.paulbsd.com/paulbsd/ipbl/src/config" @@ -33,22 +32,22 @@ func (event *Event) APIFormat() *APIEvent { func (event *Event) APIParse(session *xorm.Session, apievent APIEvent) (err error) { *event = Event{ - IP: &IP{IP: apievent.IPData.IP}, - Src: apievent.IPData.Src, - Hostname: sql.NullString{ - String: apievent.IPData.Hostname, - Valid: true}, + IP: &IP{IP: apievent.IPData.IP}, + Host: &Host{Host: apievent.IPData.Hostname}, + Src: &Src{Src: apievent.IPData.Src}, } event.IP.GetOrCreate(session) + event.Host.GetOrCreate(session) + event.Src.GetOrCreate(session) return } type Event struct { - ID int `xorm:"pk autoincr"` - Src string `xorm:"text notnull"` - Hostname sql.NullString `xorm:"text default NULL"` - IP *IP `xorm:"int ip_id"` - Created time.Time `xorm:"created notnull"` + ID int `xorm:"pk autoincr"` + Src *Src `xorm:"int src_id"` + Host *Host `xorm:"int host_id"` + IP *IP `xorm:"int ip_id"` + Created time.Time `xorm:"created notnull"` } type APIEvent struct { diff --git a/src/models/host.go b/src/models/host.go index 9c2c5f1..4366b90 100644 --- a/src/models/host.go +++ b/src/models/host.go @@ -1,6 +1,8 @@ package models import ( + "time" + "xorm.io/xorm" ) @@ -44,8 +46,10 @@ func (host *Host) APIParse(apihost APIHost) (err error) { } type Host struct { - ID int `xorm:"pk autoincr"` - Host string `xorm:"text unique host" json:"host"` + ID int `xorm:"pk autoincr"` + Host string `xorm:"text unique host" json:"host"` + Created time.Time `xorm:"created notnull"` + Updated time.Time `xorm:"updated notnull"` } type APIHost struct { diff --git a/src/models/hostinfo.go b/src/models/hostinfo.go index 2f5bcc9..2d1ac99 100644 --- a/src/models/hostinfo.go +++ b/src/models/hostinfo.go @@ -35,8 +35,7 @@ func ProcessHostInfo(cfg *config.Config, hostinfos []HostInfo) (err error) { func HostInfoToIP(session *xorm.Session, hi *HostInfo) (ip IP) { ip = IP{ - IP: hi.IP, - Whois: hi.Whois, + IP: hi.IP, Rdns: sql.NullString{ String: hi.Rdns, Valid: true}, diff --git a/src/models/ip.go b/src/models/ip.go index 8ae0f4a..bb875eb 100644 --- a/src/models/ip.go +++ b/src/models/ip.go @@ -8,13 +8,14 @@ import ( "io" "log" "net/http" + "sync" "time" "git.paulbsd.com/paulbsd/ipbl/src/config" "xorm.io/xorm" ) -const SCANLIMIT = 10 +const SCANLIMIT = 100 const IPINFO_WS = "https://ip.paulbsd.com" @@ -42,8 +43,6 @@ func (ip *IP) GetOrCreate(session *xorm.Session) (apiip *APIIP, err error) { ip.City.GetOrCreate(session) ip.Country.GetOrCreate(session) ip.AutonomousSystem.GetOrCreate(session) - ip.Host.GetOrCreate(session) - ip.Src.GetOrCreate(session) session.Commit() var tmpip *IP @@ -132,43 +131,81 @@ func InsertIPBulk(session *xorm.Session, ips *[]IP) (numinsert int64, numupdate } func ScanIP(cfg *config.Config) (err error) { - queryclient := http.Client{} + var numthreads = 8 for { session := cfg.Db.NewSession() orphans := []IP{} - err = session.Where("(as_id is null or country_id is null or city_id is null or rdns = '') and updated < now()-'1d'::interval").Asc("updated").Limit(SCANLIMIT).Find(&orphans) + err = session.Where(`((as_id IS NULL + OR country_id IS NULL + OR city_id IS NULL + OR rdns = '' + OR rdns IS NULL) AND updated < now()-'1d'::interval) OR + (as_id IS NULL + AND country_id IS NULL + AND city_id IS NULL + AND rdns IS NULL) + `).Desc("updated").Limit(SCANLIMIT).Find(&orphans) session.Close() if err == nil && len(orphans) > 0 { + orphanchan := make(chan IP) + done := make(chan bool) + var wg sync.WaitGroup + + for thr := range make([]int, numthreads) { + go ScanOrphan(&wg, orphanchan, done, thr, cfg) + } + + for _, orphan := range orphans { + orphanchan <- orphan + } + + close(orphanchan) + <-done + wg.Wait() + + } else { + time.Sleep(30 * time.Second) + } + } +} + +func ScanOrphan(wg *sync.WaitGroup, orphans chan IP, done chan bool, thr int, cfg *config.Config) (err error) { + wg.Add(1) + + for { + orphan, more := <-orphans + if more { session := cfg.Db.NewSession() if err != nil { log.Println(err) } - for _, orphan := range orphans { - query, err := QueryInfo(&queryclient, orphan.IP) - if err != nil { - log.Println(err) - continue - } - as := &AutonomousSystem{ASID: query.APIAS.ASID, ASName: query.APIAS.ASName} - orphan.AutonomousSystem = as - - if query.APICity != "" { - var city = City{CityName: query.APICity} - orphan.City = &city - } - - if query.APICountry != "" { - var country = Country{CountryName: query.APICountry} - orphan.Country = &country - } - - orphan.Rdns = sql.NullString{String: query.Rdns, Valid: true} - log.Printf("%s -> \"%s\"\n", orphan.IP, query.Rdns) - - orphan.GetOrCreate(session) + queryclient := http.Client{} + var query QueryIP + query, err := QueryInfo(&queryclient, orphan.IP) + if err != nil { + log.Println(err) + continue } + var as = AutonomousSystem{ASID: query.APIAS.Number, ASName: query.APIAS.Org} + orphan.AutonomousSystem = &as + + if query.APICity != "" { + var city = City{CityName: query.APICity} + orphan.City = &city + } + + if query.APICountry != "" { + var country = Country{CountryName: query.APICountry} + orphan.Country = &country + } + + orphan.Rdns = sql.NullString{String: query.Rdns, Valid: true} + log.Printf("%s -> \"%s\"\n", orphan.IP, query.Rdns) + + orphan.GetOrCreate(session) + err = session.Commit() if err != nil { log.Println(err) @@ -178,9 +215,11 @@ func ScanIP(cfg *config.Config) (err error) { log.Println(err) } } else { - time.Sleep(2 * time.Minute) + //fmt.Printf("All orphan migrated on thread num %d\n", thr) + wg.Done() + done <- true + return nil } - time.Sleep(1 * time.Second) } } @@ -261,9 +300,6 @@ type IP struct { AutonomousSystem *AutonomousSystem `xorm:"as_id int index default null"` City *City `xorm:"city_id int index default null"` Country *Country `xorm:"country_id int index default null"` - Src *Src `xorm:"src_id int index default null"` - Host *Host `xorm:"host_id int index default null"` - Whois string `xorm:"text index default null"` Created time.Time `xorm:"created notnull"` Updated time.Time `xorm:"updated notnull"` } @@ -280,12 +316,12 @@ type APIIP struct { type QueryIP struct { IP string `json:"ip"` Rdns string `json:"hostname"` - APIAS QueryAutonomousSystem `json:"asn"` + APIAS QueryAutonomousSystem `json:"as"` APICity string `json:"city"` APICountry string `json:"country"` } type QueryAutonomousSystem struct { - ASID int `json:"id"` - ASName string `json:"name"` + Number int `json:"number"` + Org string `json:"org"` } diff --git a/src/models/src.go b/src/models/src.go index 178fa3e..7c3653c 100644 --- a/src/models/src.go +++ b/src/models/src.go @@ -1,6 +1,8 @@ package models import ( + "time" + "xorm.io/xorm" ) @@ -44,8 +46,10 @@ func (src *Src) APIParse(apisrc APISrc) (err error) { } type Src struct { - ID int `xorm:"pk autoincr"` - Src string `xorm:"text unique(srcindex) src" json:"src"` + ID int `xorm:"pk autoincr"` + Src string `xorm:"text unique(srcindex) src" json:"src"` + Created time.Time `xorm:"created notnull"` + Updated time.Time `xorm:"updated notnull"` } type APISrc struct { diff --git a/src/routers/funcs.go b/src/routers/funcs.go index 4c22c44..5133b21 100644 --- a/src/routers/funcs.go +++ b/src/routers/funcs.go @@ -22,6 +22,10 @@ func RegisterRoutes(e *echo.Echo, ctx *context.Context, cfg *config.Config) { `) }) + e.GET("/health", func(c echo.Context) error { + return c.HTML(http.StatusOK, `OK`) + }) + e.GET("/ip/:ip", func(c echo.Context) (err error) { session := cfg.Db.NewSession() defer session.Close()