paulbsd/ipbl
1
0
Fork 0
Code Issues Pull Requests Projects Releases 29 Wiki Activity

Merge branch 'develop-xorm'
Some checks failed
continuous-integration/drone/push Build is failing
Details
continuous-integration/drone/tag Build is failing
Details

Browse Source
This commit is contained in:
Paul 2023-01-21 09:59:32 +01:00
commit 90b7f68004
20 changed files with 853 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
go.mod
View File

@ -13,14 +13,15 @@ require (
require (
github.com/goccy/go-json v0.10.0 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/labstack/gommon v0.4.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/onsi/ginkgo v1.16.4 // indirect
github.com/onsi/gomega v1.14.0 // indirect
github.com/onsi/ginkgo v1.16.5 // indirect
github.com/onsi/gomega v1.22.1 // indirect
github.com/syndtr/goleveldb v1.0.0 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasttemplate v1.2.2 // indirect

19
go.sum
View File

@ -98,8 +98,6 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
@ -111,7 +109,8 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@ -284,13 +283,13 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.14.0 h1:ep6kpPVwmr/nTbklSx2nrLNSIO62DoYAhnPNIMhK8gI=
github.com/onsi/gomega v1.14.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
github.com/onsi/gomega v1.22.1 h1:pY8O4lBfsHKZHM/6nrxkhVPUznOlIu3quZcKP/M20KI=
github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=
github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
@ -442,7 +441,6 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@ -483,7 +481,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@ -553,8 +550,6 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@ -574,8 +569,6 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

18
src/database/main.go
View File

@ -10,17 +10,13 @@ import (
"git.paulbsd.com/paulbsd/ipbl/src/models"
_ "github.com/lib/pq"
"xorm.io/xorm"
"xorm.io/xorm/dialects"
"xorm.io/xorm/names"
)
func Initialize(ctx *context.Context, cfg *config.Config) (err error) {
var databaseEngine = "postgres"
var tables = []interface{}{models.IP{},
models.Cfg{},
models.CfgSet{},
models.CfgTrustlist{},
models.CfgZMQ{},
models.HostInfo{}}
//cacher := caches.NewLRUCacher(caches.NewMemoryStore(), 10)
cfg.Db, err = xorm.NewEngine(databaseEngine,
fmt.Sprintf("%s://%s:%s@%s/%s",
@ -33,18 +29,18 @@ func Initialize(ctx *context.Context, cfg *config.Config) (err error) {
log.Fatalln(err)
}
cfg.Db.SetMapper(names.GonicMapper{})
cfg.Db.SetQuotePolicy(dialects.QuotePolicyReserved)
cfg.Db.ShowSQL(cfg.Switchs.Debug)
//cfg.Db.SetDefaultCacher(cacher)
if cfg.Switchs.Drop {
log.Println("Dropping tables")
cfg.Db.DropTables(tables)
models.DropTables(ctx, cfg)
os.Exit(0)
}
log.Println("Syncing tables")
for _, table := range tables {
cfg.Db.CreateTables(table)
err = cfg.Db.Sync2(table)
}
err = models.NewEngine(ctx, cfg)
return
}

62
src/models/as.go Normal file
View File

@ -0,0 +1,62 @@
package models
import (
"time"
"xorm.io/xorm"
)
func (as *AutonomousSystem) GetOrCreate(session *xorm.Session) (apias *APIAutonomousSystem, err error) {
has, err := session.Get(as)
if err != nil {
return
}
if !has {
session.Insert(as)
} else {
session.ID(as.ID).Update(as)
}
as.Get(session)
apias = as.APIFormat()
return
}
func (as *AutonomousSystem) Get(session *xorm.Session) (apias *APIAutonomousSystem, err error) {
has, err := session.Get(as)
if !has || err != nil {
return
}
apias = as.APIFormat()
return
}
func (as *AutonomousSystem) APIFormat() *APIAutonomousSystem {
if as == nil {
return &APIAutonomousSystem{}
}
return &APIAutonomousSystem{
ASID: as.ASID,
ASName: as.ASName,
}
}
func (as *AutonomousSystem) APIParse(apias APIAutonomousSystem) (err error) {
*as = AutonomousSystem{
ASID: apias.ASID,
ASName: apias.ASName}
return
}
type AutonomousSystem struct {
ID int `xorm:"pk autoincr"`
ASID int `xorm:"integer unique(asindex) as_id"`
ASName string `xorm:"text unique(asindex) as_name"`
Created time.Time `xorm:"created notnull"`
Updated time.Time `xorm:"updated notnull"`
}
type APIAutonomousSystem struct {
ID int `json:"-"`
ASID int `json:"as_id"`
ASName string `json:"as_name"`
}

30
src/models/cfg.go
View File

@ -118,17 +118,10 @@ func DiscoverURLS(cfg config.Config, routes []*echo.Route) (Discovery, error) {
return disc, nil
}
type CfgTrustlist struct {
ID int `xorm:"pk autoincr" json:"-"`
IP string `xorm:"text notnull" json:"ip"`
}
type CfgZMQ struct {
ID int `xorm:"pk autoincr" json:"-"`
Type string `xorm:"text notnull" json:"type"`
Hostname string `xorm:"text notnull" json:"hostname"`
Port int `json:"port"`
Subscription string `json:"subscription"`
type Cfg struct {
ID int `xorm:"pk autoincr" json:"-"`
Key string `xorm:"text notnull unique" json:"key"`
Value string `xorm:"text default" json:"value"`
}
type CfgSet struct {
@ -142,10 +135,17 @@ type CfgSet struct {
Enabled bool `xorm:"notnull default true" json:"-"`
}
type Cfg struct {
ID int `xorm:"pk autoincr" json:"-"`
Key string `xorm:"text notnull unique" json:"key"`
Value string `xorm:"text default" json:"value"`
type CfgTrustlist struct {
ID int `xorm:"pk autoincr" json:"-"`
IP string `xorm:"text notnull" json:"ip"`
}
type CfgZMQ struct {
ID int `xorm:"pk autoincr" json:"-"`
Type string `xorm:"text notnull" json:"type"`
Hostname string `xorm:"text notnull" json:"hostname"`
Port int `json:"port"`
Subscription string `json:"subscription"`
}
type Discovery struct {

58
src/models/city.go Normal file
View File

@ -0,0 +1,58 @@
package models
import (
"time"
"xorm.io/xorm"
)
func (city *City) GetOrCreate(session *xorm.Session) (apicity *APICity, err error) {
has, err := session.Get(city)
if err != nil {
return
}
if !has {
session.Insert(city)
} else {
session.ID(city.ID).Update(city)
}
city.Get(session)
apicity = city.APIFormat()
return
}
func (city *City) Get(session *xorm.Session) (apicity *APICity, err error) {
has, err := session.Get(city)
if !has || err != nil {
return
}
apicity = city.APIFormat()
return
}
func (city *City) APIFormat() *APICity {
if city == nil {
return &APICity{}
}
return &APICity{
CityName: city.CityName,
}
}
func (city *City) APIParse(apicity APICity) (err error) {
*city = City{
CityName: apicity.CityName}
return
}
type City struct {
ID int `xorm:"pk autoincr"`
CityName string `xorm:"text unique(cityindex) city_name" json:"city_name"`
Created time.Time `xorm:"created notnull"`
Updated time.Time `xorm:"updated notnull"`
}
type APICity struct {
ID int `json:"-"`
CityName string `json:"city_name"`
}

58
src/models/country.go Normal file
View File

@ -0,0 +1,58 @@
package models
import (
"time"
"xorm.io/xorm"
)
func (country *Country) GetOrCreate(session *xorm.Session) (apicountry *APICountry, err error) {
has, err := session.Get(country)
if err != nil {
return
}
if !has {
session.Insert(country)
} else {
session.ID(country.ID).Update(country)
}
apicountry = country.APIFormat()
country.Get(session)
return
}
func (country *Country) Get(session *xorm.Session) (apicountry *APICountry, err error) {
has, err := session.Get(country)
if !has || err != nil {
return
}
apicountry = country.APIFormat()
return
}
func (country *Country) APIFormat() *APICountry {
if country == nil {
return &APICountry{}
}
return &APICountry{
CountryName: country.CountryName,
}
}
func (country *Country) APIParse(apicountry APICountry) (err error) {
*country = Country{
CountryName: apicountry.CountryName}
return
}
type Country struct {
ID int `xorm:"pk autoincr"`
CountryName string `xorm:"text unique(countryindex) country_name" json:"country_name"`
Created time.Time `xorm:"created notnull"`
Updated time.Time `xorm:"updated notnull"`
}
type APICountry struct {
ID int `json:"-"`
CountryName string `json:"country_name"`
}

64
src/models/event.go Normal file
View File

@ -0,0 +1,64 @@
package models
import (
"time"
"git.paulbsd.com/paulbsd/ipbl/src/config"
"xorm.io/xorm"
)
func (event *Event) Insert(cfg *config.Config) (err error) {
session := cfg.Db.NewSession()
defer session.Close()
_, err = session.Insert(event)
err = session.Commit()
return
}
/*
func (event *Event) APIFormat() *APIEvent {
if event == nil {
return nil
}
return &APIEvent{
MSgType: "none",
IPData: {
IP: event.IP.IP,
Src: event.Src,
Hostname: event.Hostname.String,
},
}
}*/
func (event *Event) APIParse(session *xorm.Session, apievent APIEvent) (err error) {
*event = Event{
IP: &IP{IP: apievent.IPData.IP},
Host: &Host{Host: apievent.IPData.Hostname},
Src: &Src{Src: apievent.IPData.Src},
}
event.IP.GetOrCreate(session)
event.Host.GetOrCreate(session)
event.Src.GetOrCreate(session)
return
}
type Event struct {
ID int `xorm:"pk autoincr"`
Src *Src `xorm:"int src_id"`
Host *Host `xorm:"int host_id"`
IP *IP `xorm:"int ip_id"`
Created time.Time `xorm:"created notnull"`
}
type APIEvent struct {
MsgType string `json:"msgtype"`
Mode string `json:"mode"`
Hostname string `json:"hostname"`
IPData struct {
IP string `json:"ip"`
Src string `json:"src"`
Hostname string `json:"hostname"`
Date string `json:"date"`
Created string `json:"created"`
} `json:"ipdata"`
}

58
src/models/host.go Normal file
View File

@ -0,0 +1,58 @@
package models
import (
"time"
"xorm.io/xorm"
)
func (host *Host) GetOrCreate(session *xorm.Session) (apihost *APIHost, err error) {
has, err := session.Get(host)
if err != nil {
return
}
if !has {
session.Insert(host)
} else {
session.ID(host.ID).Update(host)
}
host.Get(session)
apihost = host.APIFormat()
return
}
func (host *Host) Get(session *xorm.Session) (apihost *APIHost, err error) {
has, err := session.Get(host)
if !has || err != nil {
return
}
apihost = host.APIFormat()
return
}
func (host *Host) APIFormat() *APIHost {
if host == nil {
return &APIHost{}
}
return &APIHost{
Host: host.Host,
}
}
func (host *Host) APIParse(apihost APIHost) (err error) {
*host = Host{
Host: apihost.Host}
return
}
type Host struct {
ID int `xorm:"pk autoincr"`
Host string `xorm:"text unique host" json:"host"`
Created time.Time `xorm:"created notnull"`
Updated time.Time `xorm:"updated notnull"`
}
type APIHost struct {
ID int `json:"-"`
Host string `json:"host"`
}

69
src/models/hostinfo.go
View File

@ -1,28 +1,55 @@
package models
import (
"time"
"context"
"database/sql"
"git.paulbsd.com/paulbsd/ipbl/src/config"
"xorm.io/xorm"
)
func GetIPsWithoutHostInfo(ctx *context.Context, config *config.Config) (apiips []string, err error) {
res, err := config.Db.Query(`
SELECT ip
FROM ip
WHERE ip NOT IN (
SELECT ip FROM host_info)
ORDER BY RANDOM()
LIMIT 10;`)
for _, r := range res {
apiips = append(apiips, string(r["ip"]))
}
return
}
func ProcessHostInfo(cfg *config.Config, hostinfos []HostInfo) (err error) {
session := cfg.Db.NewSession()
defer session.Close()
for _, hi := range hostinfos {
var i IP = HostInfoToIP(session, &hi)
i.GetOrCreate(session)
}
session.Commit()
return
}
func HostInfoToIP(session *xorm.Session, hi *HostInfo) (ip IP) {
ip = IP{
IP: hi.IP,
Rdns: sql.NullString{
String: hi.Rdns,
Valid: true},
City: &City{CityName: hi.City},
Country: &Country{CountryName: hi.Country},
AutonomousSystem: &AutonomousSystem{ID: 0},
}
return
}
type HostInfo struct {
ID int `xorm:"pk autoincr"`
IP string `xorm:"text unique" json:"ip"`
RawWhois string `xorm:"text" json:"raw_whois"`
ReverseDNS string `xorm:"text" json:"reverse_dns"`
City string `xorm:"text" json:"city"`
Country string `xorm:"text" json:"country"`
Created time.Time `xorm:"created notnull"`
Updated time.Time `xorm:"updated notnull"`
}
type ScanResult struct {
Protocol string `json:"proto"`
PortID int `json:"port_id"`
State string `json:"state"`
ServiceName string `json:"service"`
}
type ASN struct {
ID int `json:"id"`
Name string `json:"name"`
IP string `json:"ip"`
Whois string `json:"whois"`
Rdns string `json:"rdns"`
City string `json:"city"`
Country string `json:"country"`
}

294
src/models/ip.go
View File

@ -3,15 +3,21 @@ package models
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"io"
"log"
"net"
"net/http"
"sync"
"time"
"git.paulbsd.com/paulbsd/ipbl/src/config"
"xorm.io/xorm"
)
const ScanLimit = 500
const SCANLIMIT = 100
const IPINFO_WS = "https://ip.paulbsd.com"
var lastday = time.Now().Add(-(time.Hour * 24))
@ -33,48 +39,54 @@ func GetIPsLast(ctx *context.Context, config *config.Config, interval string) (a
return
}
func GetIPsWithoutHostInfo(ctx *context.Context, config *config.Config) (apiips []string, err error) {
var ips []IP
err = config.Db.In("ip").GroupBy("ip").Find(&ips)
for _, ml := range ips {
apiips = append(apiips, ml.IP)
func (ip *IP) GetOrCreate(session *xorm.Session) (apiip *APIIP, err error) {
ip.City.GetOrCreate(session)
ip.Country.GetOrCreate(session)
ip.AutonomousSystem.GetOrCreate(session)
session.Commit()
var tmpip *IP
if ip.ID != 0 {
tmpip = &IP{ID: ip.ID, IP: ip.IP}
} else {
tmpip = &IP{IP: ip.IP}
}
has, err := session.Get(tmpip)
if err != nil {
log.Println(err)
}
if !has {
session.Insert(ip)
} else {
ip.ID = tmpip.ID
session.ID(ip.ID).AllCols().Update(ip)
session.ID(ip.ID).Cols("city_id", "country_id", "as_id").Update(ip)
}
session.Commit()
ip.Get(session)
apiip = ip.APIFormat()
return
}
func GetIP(ctx *context.Context, cfg *config.Config, ipquery interface{}) (apiip *APIIP, err error) {
session := cfg.Db.NewSession()
defer session.Close()
var ip IP
has, err := session.Where("ip = ?", ipquery).Get(&ip)
if !has {
func (ip *IP) Get(session *xorm.Session) (apiip *APIIP, err error) {
has, err := session.Get(ip)
if !has || err != nil {
err = fmt.Errorf("not found")
return nil, err
}
if err != nil {
return
}
apiip = ip.APIFormat()
session.Commit()
return
}
func (ip *IP) UpdateRDNS() (result string, err error) {
res, err := net.LookupAddr(ip.IP)
if err != nil {
result = ""
return
func (ip *IP) InsertOrUpdate(session *xorm.Session) (numinsert int64, numupdate int64, err error) {
//var ips = []IP{}
//err = session.Where("ip = ?", ip.IP).And("src = ?", ip.Src).And("hostname = ?", ip.Hostname).Find(&ips)
has, err := session.Get(ip)
if has {
session.ID(ip.ID).Update(&IP{})
}
result = res[0]
return
}
func (ip *IP) InsertOrUpdate(cfg *config.Config) (numinsert int64, numupdate int64, err error) {
session := cfg.Db.NewSession()
defer session.Close()
var ips = []IP{}
err = session.Where("ip = ?", ip.IP).And("src = ?", ip.Src).And("hostname = ?", ip.Hostname).Find(&ips)
if len(ips) > 0 {
/*if len(ips) > 0 {
numupdate, err = session.Where("id = ?", ips[0].ID).Cols("updated").Update(&IP{})
if err != nil {
log.Println(err)
@ -84,16 +96,16 @@ func (ip *IP) InsertOrUpdate(cfg *config.Config) (numinsert int64, numupdate int
if err != nil {
log.Println(err)
}
}
}*/
session.Commit()
return
}
func InsertIPBulk(cfg *config.Config, ips *[]IP) (numinsert int64, numupdate int64, err error) {
func InsertIPBulk(session *xorm.Session, ips *[]IP) (numinsert int64, numupdate int64, err error) {
for _, ip := range *ips {
numinsert, numupdate, err = ip.InsertOrUpdate(cfg)
numinsert, numupdate, err = ip.InsertOrUpdate(session)
}
Cleanup(cfg)
Cleanup(session)
return
/*var iplist []string
hostname := (*ips)[0].Hostname.String
@ -119,43 +131,129 @@ func InsertIPBulk(cfg *config.Config, ips *[]IP) (numinsert int64, numupdate int
}
func ScanIP(cfg *config.Config) (err error) {
var numthreads = 8
for {
orphans := []IP{}
session := cfg.Db.NewSession()
err := session.Begin()
if err != nil {
log.Println(err)
}
if session.Where("rdns IS NULL").Asc("ip").Limit(ScanLimit).Find(&orphans); len(orphans) > 0 {
for _, orphan := range orphans {
reverse, _ := orphan.UpdateRDNS()
log.Printf("%s -> \"%s\"\n", orphan.IP, reverse)
orphan.Rdns.String = reverse
orphan.Rdns.Valid = true
_, err = session.ID(orphan.ID).Cols("rdns").Update(&orphan)
orphans := []IP{}
err = session.Where(`((as_id IS NULL
OR country_id IS NULL
OR city_id IS NULL
OR rdns = ''
OR rdns IS NULL) AND updated < now()-'1d'::interval) OR
(as_id IS NULL
AND country_id IS NULL
AND city_id IS NULL
AND rdns IS NULL)
`).Desc("updated").Limit(SCANLIMIT).Find(&orphans)
session.Close()
if err != nil {
session.Rollback()
log.Println(err)
}
if err == nil && len(orphans) > 0 {
orphanchan := make(chan IP)
done := make(chan bool)
var wg sync.WaitGroup
for thr := range make([]int, numthreads) {
go ScanOrphan(&wg, orphanchan, done, thr, cfg)
}
for _, orphan := range orphans {
orphanchan <- orphan
}
close(orphanchan)
<-done
wg.Wait()
} else {
time.Sleep(30 * time.Second)
}
}
}
func ScanOrphan(wg *sync.WaitGroup, orphans chan IP, done chan bool, thr int, cfg *config.Config) (err error) {
wg.Add(1)
for {
orphan, more := <-orphans
if more {
session := cfg.Db.NewSession()
if err != nil {
log.Println(err)
}
queryclient := http.Client{}
var query QueryIP
query, err := QueryInfo(&queryclient, orphan.IP)
if err != nil {
log.Println(err)
continue
}
var as = AutonomousSystem{ASID: query.APIAS.Number, ASName: query.APIAS.Org}
orphan.AutonomousSystem = &as
if query.APICity != "" {
var city = City{CityName: query.APICity}
orphan.City = &city
}
if query.APICountry != "" {
var country = Country{CountryName: query.APICountry}
orphan.Country = &country
}
orphan.Rdns = sql.NullString{String: query.Rdns, Valid: true}
log.Printf("%s -> \"%s\"\n", orphan.IP, query.Rdns)
orphan.GetOrCreate(session)
err = session.Commit()
if err != nil {
log.Println(err)
}
err = session.Close()
if err != nil {
log.Println(err)
}
} else {
time.Sleep(10 * time.Second)
//fmt.Printf("All orphan migrated on thread num %d\n", thr)
wg.Done()
done <- true
return nil
}
session.Close()
}
}
func Cleanup(cfg *config.Config) (err error) {
session := cfg.Db.NewSession()
defer session.Close()
results, _ := session.Query("select * from ip where ip in (select ip from ip group by ip having count(ip) > 1) and hostname is null order by updated desc;")
func QueryInfo(client *http.Client, ip string) (query QueryIP, err error) {
var url = fmt.Sprintf("%s/%s", IPINFO_WS, ip)
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Accept", "*/*")
req.Header.Add("User-Agent", "ipbl")
res, _ := client.Do(req)
data, _ := io.ReadAll(res.Body)
err = json.Unmarshal(data, &query)
return
}
func Cleanup(session *xorm.Session) (err error) {
results, _ := session.Query(`
SELECT *
FROM ip
WHERE ip IN (
SELECT ip
FROM ip
GROUP BY ip
HAVING COUNT(ip) > 1)
AND hostname IS NULL
ORDER BY updated DESC;`)
if len(results) > 0 {
_, err := session.Query("delete from ip where ip in (select ip from ip group by ip having count(ip) > 1) and hostname is null;")
_, err := session.Query(`
DELETE FROM ip
WHERE ip IN (
SELECT ip
FROM ip
GROUP by ip
HAVING COUNT(ip) > 1)
AND hostname IS NULL;`)
if err != nil {
log.Println("error deleting orphans")
}
@ -166,33 +264,29 @@ func Cleanup(cfg *config.Config) (err error) {
func (ip *IP) APIFormat() *APIIP {
if ip == nil {
return nil
return &APIIP{}
}
return &APIIP{
IP: ip.IP,
Rdns: ip.Rdns.String,
Src: ip.Src,
Hostname: ip.Hostname.String,
City: ip.City.String,
Country: ip.Country.String,
Date: ip.Updated.Local().String(),
IP: ip.IP,
Rdns: ip.Rdns.String,
APIAS: *ip.AutonomousSystem.APIFormat(),
APICity: ip.City.APIFormat().CityName,
APICountry: ip.Country.APIFormat().CountryName,
}
}
func (ip *APIIP) APIConvert() *IP {
if ip == nil {
return nil
}
return &IP{
IP: ip.IP,
func (ip *IP) APIParse(apiip APIIP) (err error) {
*ip = IP{
IP: apiip.IP,
Rdns: sql.NullString{
String: ip.Rdns,
Valid: false},
Src: ip.Src,
Hostname: sql.NullString{
String: ip.Hostname,
String: apiip.Rdns,
Valid: true},
AutonomousSystem: &AutonomousSystem{
ASID: apiip.APIAS.ASID,
ASName: apiip.APIAS.ASName,
},
}
return
}
func (ip *APIIP) BeforeInsert() (err error) {
@ -200,24 +294,34 @@ func (ip *APIIP) BeforeInsert() (err error) {
}
type IP struct {
ID int `xorm:"pk autoincr"`
IP string `xorm:"text notnull unique(ipsrc)" json:"ip"`
Rdns sql.NullString `xorm:"text default"`
Src string `xorm:"text notnull unique(ipsrc)" json:"src"`
Hostname sql.NullString `xorm:"text default '' unique(ipsrc)" json:"hostname"`
City sql.NullString `xorm:"text default '' unique(ipsrc)" json:"city"`
Country sql.NullString `xorm:"text default '' unique(ipsrc)" json:"country"`
Created time.Time `xorm:"created notnull"`
Updated time.Time `xorm:"updated notnull"`
ID int `xorm:"pk autoincr"`
IP string `xorm:"text notnull"`
Rdns sql.NullString `xorm:"text default ''"`
AutonomousSystem *AutonomousSystem `xorm:"as_id int index default null"`
City *City `xorm:"city_id int index default null"`
Country *Country `xorm:"country_id int index default null"`
Created time.Time `xorm:"created notnull"`
Updated time.Time `xorm:"updated notnull"`
}
type APIIP struct {
IP string `json:"ip"`
Rdns string `json:"rdns"`
Src string `json:"src"`
Hostname string `json:"hostname"`
City string `json:"city"`
Country string `json:"country"`
Date string `json:"date"`
Mode string `json:"mode"`
IP string `json:"ip"`
Rdns string `json:"rdns"`
APIAS APIAutonomousSystem `json:"as"`
APICity string `json:"city"`
APICountry string `json:"country"`
APIWhois string `json:"whois"`
}
type QueryIP struct {
IP string `json:"ip"`
Rdns string `json:"hostname"`
APIAS QueryAutonomousSystem `json:"as"`
APICity string `json:"city"`
APICountry string `json:"country"`
}
type QueryAutonomousSystem struct {
Number int `json:"number"`
Org string `json:"org"`
}

67
src/models/models.go Normal file
View File

@ -0,0 +1,67 @@
package models
import (
"context"
"fmt"
"git.paulbsd.com/paulbsd/ipbl/src/config"
"xorm.io/xorm"
"xorm.io/xorm/names"
)
var (
x *xorm.Engine
tables []interface{}
// HasEngine specifies if we have a xorm.Engine
HasEngine bool
)
func init() {
tables = append(tables,
new(AutonomousSystem),
new(Cfg),
new(CfgSet),
new(CfgTrustlist),
new(CfgZMQ),
new(City),
new(Country),
new(Event),
new(IP),
new(Src),
new(Host),
)
for _, name := range []string{"SSL", "UID"} {
names.LintGonicMapper[name] = true
}
}
// NewEngine initializes a new xorm.Engine
func NewEngine(ctx *context.Context, config *config.Config) (err error) {
var x = config.Db
if err = x.Ping(); err != nil {
return err
}
if err = x.Sync2(tables...); err != nil {
return fmt.Errorf("sync database struct error: %v", err)
}
return nil
}
// DropTables initializes a new xorm.Engine
func DropTables(ctx *context.Context, config *config.Config) (err error) {
var x = config.Db
if err = x.Ping(); err != nil {
return err
}
if err = x.Sync2(tables...); err != nil {
return fmt.Errorf("sync database struct error: %v", err)
}
return nil
}

65
src/models/scanresult.go Normal file
View File

@ -0,0 +1,65 @@
package models
import (
"time"
"git.paulbsd.com/paulbsd/ipbl/src/config"
)
func (sr *ScanResult) GetOrCreate(cfg *config.Config) (apisr *APIScanResult, err error) {
has, err := cfg.Db.Get(sr)
if !has {
cfg.Db.Insert(sr)
} else {
cfg.Db.ID(sr.ID).Update(sr)
}
sr.Get(cfg)
apisr = sr.APIFormat()
return
}
func (sr *ScanResult) Get(cfg *config.Config) (apisr *APIScanResult, err error) {
has, err := cfg.Db.Get(sr)
if !has || err != nil {
return
}
apisr = sr.APIFormat()
return
}
func (sr *ScanResult) APIFormat() *APIScanResult {
if sr == nil {
return &APIScanResult{}
}
return &APIScanResult{
Protocol: sr.Protocol,
PortID: sr.PortID,
State: sr.State,
ServiceName: sr.ServiceName,
}
}
func (sr *ScanResult) APIParse(apisr APIScanResult) (err error) {
*sr = ScanResult{
Protocol: apisr.Protocol,
PortID: apisr.PortID}
return
}
type ScanResult struct {
ID int `xorm:"pk autoincr"`
Protocol string `xorm:"text default ''"`
PortID int `xorm:"default ''"`
State string `xorm:"text default ''"`
ServiceName string `xorm:"text default ''"`
Created time.Time `xorm:"created notnull"`
Updated time.Time `xorm:"updated notnull"`
}
type APIScanResult struct {
ID int `json:"id"`
Protocol string `json:"proto"`
PortID int `json:"port_id"`
State string `json:"state"`
ServiceName string `json:"service"`
}

58
src/models/src.go Normal file
View File

@ -0,0 +1,58 @@
package models
import (
"time"
"xorm.io/xorm"
)
func (src *Src) GetOrCreate(session *xorm.Session) (apisrc *APISrc, err error) {
has, err := session.Get(src)
if err != nil {
return
}
if !has {
session.Insert(src)
} else {
session.ID(src.ID).Update(src)
}
src.Get(session)
apisrc = src.APIFormat()
return
}
func (src *Src) Get(session *xorm.Session) (apisrc *APISrc, err error) {
has, err := session.Get(src)
if !has || err != nil {
return
}
apisrc = src.APIFormat()
return
}
func (src *Src) APIFormat() *APISrc {
if src == nil {
return &APISrc{}
}
return &APISrc{
Src: src.Src,
}
}
func (src *Src) APIParse(apisrc APISrc) (err error) {
*src = Src{
Src: apisrc.Src}
return
}
type Src struct {
ID int `xorm:"pk autoincr"`
Src string `xorm:"text unique(srcindex) src" json:"src"`
Created time.Time `xorm:"created notnull"`
Updated time.Time `xorm:"updated notnull"`
}
type APISrc struct {
ID int `json:"-"`
Src string `json:"src"`
}

48
src/routers/funcs.go
View File

@ -22,23 +22,31 @@ func RegisterRoutes(e *echo.Echo, ctx *context.Context, cfg *config.Config) {
</html>`)
})
e.GET("/health", func(c echo.Context) error {
return c.HTML(http.StatusOK, `OK`)
})
e.GET("/ip/:ip", func(c echo.Context) (err error) {
ret, err := models.GetIP(ctx, cfg, c.Param("ip"))
session := cfg.Db.NewSession()
defer session.Close()
var ip = models.IP{IP: c.Param("ip")}
ret, err := ip.Get(session)
return Result(c, err, ret)
})
e.POST("/ip", func(c echo.Context) (err error) {
session := cfg.Db.NewSession()
var apiip = new(models.APIIP)
var ip = new(models.IP)
var msg = "No IP inserted"
err = c.Bind(ip)
err = c.Bind(apiip)
if err != nil {
return Result(c, fmt.Errorf("error when parsing body"), "")
}
numinsert, numupdate, err := ip.InsertOrUpdate(cfg)
ip.APIParse(*apiip)
_, err = ip.GetOrCreate(session)
if err != nil {
return Result(c, err, "")
}
msg = fmt.Sprintf("zmq: Inserted %d IP, Updated %d IP", numinsert, numupdate)
log.Println(msg)
return Result(c, err, msg)
})
e.GET("/ips", func(c echo.Context) (err error) {
@ -62,6 +70,8 @@ func RegisterRoutes(e *echo.Echo, ctx *context.Context, cfg *config.Config) {
return Result(c, err, ret)
})
e.POST("/ips", func(c echo.Context) (err error) {
session := cfg.Db.NewSession()
defer session.Close()
var apiips = []models.APIIP{}
var ips = []models.IP{}
var msg string
@ -70,23 +80,39 @@ func RegisterRoutes(e *echo.Echo, ctx *context.Context, cfg *config.Config) {
return Result(c, err, apiips)
}
for _, apiip := range apiips {
ips = append(ips, *apiip.APIConvert())
var ip = new(models.IP)
ip.APIParse(apiip)
ips = append(ips, *ip)
}
numinsert, numupdate, _ := models.InsertIPBulk(cfg, &ips)
numinsert, numupdate, _ := models.InsertIPBulk(session, &ips)
msg = fmt.Sprintf("zmq: Inserted %d IP, Updated %d IP", numinsert, numupdate)
log.Println(msg)
return Result(c, err, msg)
})
e.POST("/event", func(c echo.Context) (err error) {
session := cfg.Db.NewSession()
defer session.Close()
var apievent = new(models.APIEvent)
var event = new(models.Event)
err = c.Bind(apievent)
if err != nil {
return Result(c, fmt.Errorf("error when parsing body"), "")
}
event.APIParse(session, *apievent)
err = event.Insert(cfg)
if err != nil {
return Result(c, err, "")
}
return Result(c, err, "OK")
})
e.POST("/hostinfo", func(c echo.Context) (err error) {
var hostinfos = []models.HostInfo{}
err = c.Bind(&hostinfos)
if err != nil {
return Result(c, err, hostinfos)
}
num, err := cfg.Db.Insert(&hostinfos)
msg := fmt.Sprintf("Inserted %v host info (%d)", hostinfos, num)
log.Println(msg)
return Result(c, err, msg)
models.ProcessHostInfo(cfg, hostinfos)
return Result(c, err, "")
})
e.GET("/config", func(c echo.Context) (err error) {
trustlists, err := models.GetGlobalConfig(*cfg)

3
src/routers/main.go
View File

@ -24,6 +24,9 @@ func RunServer(ctx *context.Context, cfg *config.Config) (err error) {
func Result(c echo.Context, inputerr error, data interface{}) (err error) {
if inputerr != nil {
if inputerr.Error() == "not found" {
return c.String(http.StatusNotFound, "Content not found")
}
if inputerr.Error() == "Not Found" {
return c.String(http.StatusNotFound, "Content not found")
}

37
src/zmqrouter/main.go
View File

@ -4,6 +4,7 @@ import (
"encoding/json"
"fmt"
"log"
"time"
"git.paulbsd.com/paulbsd/ipbl/src/config"
"git.paulbsd.com/paulbsd/ipbl/src/models"
@ -30,37 +31,57 @@ func Init(cfg *config.Config) (err error) {
func Handle(cfg *config.Config, reqsock *goczmq.Sock, pubsock *goczmq.Sock, channel string) (err error) {
log.Println("Start handling zmq sockets")
var lastip string
for {
var msg = "err"
var req, err = reqsock.RecvMessage()
if err != nil {
log.Println("unable to receive message from req socket")
log.Println("unable to receive message from req socket", err)
continue
}
var topub [][]byte
for _, val := range req {
var apiip = models.APIIP{}
var ip = models.IP{}
err = json.Unmarshal(val, &apiip)
var apievent = models.APIEvent{}
var event = models.Event{}
err = json.Unmarshal(val, &apievent)
if err != nil {
log.Println("unable to parse ip address", err)
time.Sleep(time.Second)
continue
}
if apievent.IPData.IP != "" && apievent.IPData.IP == lastip {
continue
}
if apiip.Mode != "init" {
ip = *apiip.APIConvert()
numinsert, numupdate, err := ip.InsertOrUpdate(cfg)
if apievent.MsgType == "add" {
session := cfg.Db.NewSession()
event.APIParse(session, apievent)
session.Close()
err := event.Insert(cfg)
if err != nil {
log.Println(err)
}
log.Printf("zmq: Inserted %d IP, Updated %d IP\n", numinsert, numupdate)
log.Printf("zmq: Inserted event")
}
if apievent.MsgType == "file" {
apievent.MsgType = "zmq"
}
val, err = json.Marshal(apievent)
if err != nil {
fmt.Println(err)
}
tmpval := fmt.Sprintf("%s %s", channel, string(val))
ipjson := []byte(tmpval)
topub = append(topub, ipjson)
lastip = apievent.IPData.IP
}
err = pubsock.SendMessage(topub)

6
src/zmqrouter/pubsub.go
View File

@ -1,10 +1,14 @@
package zmqrouter
import (
"fmt"
"gopkg.in/zeromq/goczmq.v4"
)
const pubsubport int = 9999
func InitPub() (sock *goczmq.Sock, err error) {
sock, err = goczmq.NewPub("tcp://*:9999")
sock, err = goczmq.NewPub(fmt.Sprintf("tcp://*:%d", pubsubport))
return
}

6
src/zmqrouter/reqrep.go
View File

@ -1,10 +1,14 @@
package zmqrouter
import (
"fmt"
"gopkg.in/zeromq/goczmq.v4"
)
const repreqport int = 9998
func InitRep() (sock *goczmq.Sock, err error) {
sock, err = goczmq.NewRep("tcp://*:9998")
sock, err = goczmq.NewRep(fmt.Sprintf("tcp://*:%d", repreqport))
return
}

10
vendor/modules.txt vendored
View File

@ -12,6 +12,8 @@ github.com/goccy/go-json/internal/runtime
# github.com/golang/snappy v0.0.4
## explicit
github.com/golang/snappy
# github.com/google/go-cmp v0.5.9
## explicit; go 1.13
# github.com/json-iterator/go v1.1.12
## explicit; go 1.12
github.com/json-iterator/go
@ -39,10 +41,10 @@ github.com/modern-go/concurrent
# github.com/modern-go/reflect2 v1.0.2
## explicit; go 1.12
github.com/modern-go/reflect2
# github.com/onsi/ginkgo v1.16.4
## explicit; go 1.15
# github.com/onsi/gomega v1.14.0
## explicit; go 1.14
# github.com/onsi/ginkgo v1.16.5
## explicit; go 1.16
# github.com/onsi/gomega v1.22.1
## explicit; go 1.18
# github.com/syndtr/goleveldb v1.0.0
## explicit
github.com/syndtr/goleveldb/leveldb