Merge branch 'develop-xorm'

go.mod

@@ -13,14 +13,15 @@ require (
 require (
 	github.com/goccy/go-json v0.10.0 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/labstack/gommon v0.4.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/onsi/ginkgo v1.16.4 // indirect
+	github.com/onsi/ginkgo v1.16.5 // indirect
-	github.com/onsi/gomega v1.14.0 // indirect
+	github.com/onsi/gomega v1.22.1 // indirect
 	github.com/syndtr/goleveldb v1.0.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.2 // indirect

go.sum

@@ -98,8 +98,6 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
@@ -111,7 +109,8 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -284,13 +283,13 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
-github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
+github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
-github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
+github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.14.0 h1:ep6kpPVwmr/nTbklSx2nrLNSIO62DoYAhnPNIMhK8gI=
+github.com/onsi/gomega v1.22.1 h1:pY8O4lBfsHKZHM/6nrxkhVPUznOlIu3quZcKP/M20KI=
-github.com/onsi/gomega v1.14.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
+github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=
 github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
@@ -442,7 +441,6 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
 golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -483,7 +481,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -553,8 +550,6 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ
 google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -574,8 +569,6 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

src/database/main.go

@@ -10,17 +10,13 @@ import (
 	"git.paulbsd.com/paulbsd/ipbl/src/models"
 	_ "github.com/lib/pq"
 	"xorm.io/xorm"
+	"xorm.io/xorm/dialects"
 	"xorm.io/xorm/names"
 )
 
 func Initialize(ctx *context.Context, cfg *config.Config) (err error) {
 	var databaseEngine = "postgres"
-	var tables = []interface{}{models.IP{},
+	//cacher := caches.NewLRUCacher(caches.NewMemoryStore(), 10)
-		models.Cfg{},
-		models.CfgSet{},
-		models.CfgTrustlist{},
-		models.CfgZMQ{},
-		models.HostInfo{}}
 
 	cfg.Db, err = xorm.NewEngine(databaseEngine,
 		fmt.Sprintf("%s://%s:%s@%s/%s",
@@ -33,18 +29,18 @@ func Initialize(ctx *context.Context, cfg *config.Config) (err error) {
 		log.Fatalln(err)
 	}
 	cfg.Db.SetMapper(names.GonicMapper{})
+	cfg.Db.SetQuotePolicy(dialects.QuotePolicyReserved)
 	cfg.Db.ShowSQL(cfg.Switchs.Debug)
 
+	//cfg.Db.SetDefaultCacher(cacher)
+
 	if cfg.Switchs.Drop {
 		log.Println("Dropping tables")
-		cfg.Db.DropTables(tables)
+		models.DropTables(ctx, cfg)
 		os.Exit(0)
 	}
 
 	log.Println("Syncing tables")
-	for _, table := range tables {
+	err = models.NewEngine(ctx, cfg)
-		cfg.Db.CreateTables(table)
-		err = cfg.Db.Sync2(table)
-	}
 	return
 }

src/models/as.go

@@ -0,0 +1,62 @@
+package models
+
+import (
+	"time"
+
+	"xorm.io/xorm"
+)
+
+func (as *AutonomousSystem) GetOrCreate(session *xorm.Session) (apias *APIAutonomousSystem, err error) {
+	has, err := session.Get(as)
+	if err != nil {
+		return
+	}
+	if !has {
+		session.Insert(as)
+	} else {
+		session.ID(as.ID).Update(as)
+	}
+	as.Get(session)
+	apias = as.APIFormat()
+	return
+}
+
+func (as *AutonomousSystem) Get(session *xorm.Session) (apias *APIAutonomousSystem, err error) {
+	has, err := session.Get(as)
+	if !has || err != nil {
+		return
+	}
+	apias = as.APIFormat()
+	return
+}
+
+func (as *AutonomousSystem) APIFormat() *APIAutonomousSystem {
+	if as == nil {
+		return &APIAutonomousSystem{}
+	}
+	return &APIAutonomousSystem{
+		ASID:   as.ASID,
+		ASName: as.ASName,
+	}
+}
+
+func (as *AutonomousSystem) APIParse(apias APIAutonomousSystem) (err error) {
+	*as = AutonomousSystem{
+		ASID:   apias.ASID,
+		ASName: apias.ASName}
+	return
+}
+
+type AutonomousSystem struct {
+	ID      int       `xorm:"pk autoincr"`
+	ASID    int       `xorm:"integer unique(asindex) as_id"`
+	ASName  string    `xorm:"text unique(asindex) as_name"`
+	Created time.Time `xorm:"created notnull"`
+	Updated time.Time `xorm:"updated notnull"`
+}
+
+type APIAutonomousSystem struct {
+	ID     int    `json:"-"`
+	ASID   int    `json:"as_id"`
+	ASName string `json:"as_name"`
+}

src/models/cfg.go

@@ -118,17 +118,10 @@ func DiscoverURLS(cfg config.Config, routes []*echo.Route) (Discovery, error) {
 	return disc, nil
 }
 
-type CfgTrustlist struct {
+type Cfg struct {
-	ID int    `xorm:"pk autoincr" json:"-"`
+	ID    int    `xorm:"pk autoincr" json:"-"`
-	IP string `xorm:"text notnull" json:"ip"`
+	Key   string `xorm:"text notnull unique" json:"key"`
-}
+	Value string `xorm:"text default" json:"value"`
-
-type CfgZMQ struct {
-	ID           int    `xorm:"pk autoincr" json:"-"`
-	Type         string `xorm:"text notnull" json:"type"`
-	Hostname     string `xorm:"text notnull" json:"hostname"`
-	Port         int    `json:"port"`
-	Subscription string `json:"subscription"`
 }
 
 type CfgSet struct {
@@ -142,10 +135,17 @@ type CfgSet struct {
 	Enabled   bool   `xorm:"notnull default true" json:"-"`
 }
 
-type Cfg struct {
+type CfgTrustlist struct {
-	ID    int    `xorm:"pk autoincr" json:"-"`
+	ID int    `xorm:"pk autoincr" json:"-"`
-	Key   string `xorm:"text notnull unique" json:"key"`
+	IP string `xorm:"text notnull" json:"ip"`
-	Value string `xorm:"text default" json:"value"`
+}
+
+type CfgZMQ struct {
+	ID           int    `xorm:"pk autoincr" json:"-"`
+	Type         string `xorm:"text notnull" json:"type"`
+	Hostname     string `xorm:"text notnull" json:"hostname"`
+	Port         int    `json:"port"`
+	Subscription string `json:"subscription"`
 }
 
 type Discovery struct {

src/models/city.go

@@ -0,0 +1,58 @@
+package models
+
+import (
+	"time"
+
+	"xorm.io/xorm"
+)
+
+func (city *City) GetOrCreate(session *xorm.Session) (apicity *APICity, err error) {
+	has, err := session.Get(city)
+	if err != nil {
+		return
+	}
+	if !has {
+		session.Insert(city)
+	} else {
+		session.ID(city.ID).Update(city)
+	}
+	city.Get(session)
+	apicity = city.APIFormat()
+	return
+}
+
+func (city *City) Get(session *xorm.Session) (apicity *APICity, err error) {
+	has, err := session.Get(city)
+	if !has || err != nil {
+		return
+	}
+	apicity = city.APIFormat()
+	return
+}
+
+func (city *City) APIFormat() *APICity {
+	if city == nil {
+		return &APICity{}
+	}
+	return &APICity{
+		CityName: city.CityName,
+	}
+}
+
+func (city *City) APIParse(apicity APICity) (err error) {
+	*city = City{
+		CityName: apicity.CityName}
+	return
+}
+
+type City struct {
+	ID       int       `xorm:"pk autoincr"`
+	CityName string    `xorm:"text unique(cityindex) city_name" json:"city_name"`
+	Created  time.Time `xorm:"created notnull"`
+	Updated  time.Time `xorm:"updated notnull"`
+}
+
+type APICity struct {
+	ID       int    `json:"-"`
+	CityName string `json:"city_name"`
+}

src/models/country.go

@@ -0,0 +1,58 @@
+package models
+
+import (
+	"time"
+
+	"xorm.io/xorm"
+)
+
+func (country *Country) GetOrCreate(session *xorm.Session) (apicountry *APICountry, err error) {
+	has, err := session.Get(country)
+	if err != nil {
+		return
+	}
+	if !has {
+		session.Insert(country)
+	} else {
+		session.ID(country.ID).Update(country)
+	}
+	apicountry = country.APIFormat()
+	country.Get(session)
+	return
+}
+
+func (country *Country) Get(session *xorm.Session) (apicountry *APICountry, err error) {
+	has, err := session.Get(country)
+	if !has || err != nil {
+		return
+	}
+	apicountry = country.APIFormat()
+	return
+}
+
+func (country *Country) APIFormat() *APICountry {
+	if country == nil {
+		return &APICountry{}
+	}
+	return &APICountry{
+		CountryName: country.CountryName,
+	}
+}
+
+func (country *Country) APIParse(apicountry APICountry) (err error) {
+	*country = Country{
+		CountryName: apicountry.CountryName}
+	return
+}
+
+type Country struct {
+	ID          int       `xorm:"pk autoincr"`
+	CountryName string    `xorm:"text unique(countryindex) country_name" json:"country_name"`
+	Created     time.Time `xorm:"created notnull"`
+	Updated     time.Time `xorm:"updated notnull"`
+}
+
+type APICountry struct {
+	ID          int    `json:"-"`
+	CountryName string `json:"country_name"`
+}

src/models/event.go

@@ -0,0 +1,64 @@
+package models
+
+import (
+	"time"
+
+	"git.paulbsd.com/paulbsd/ipbl/src/config"
+	"xorm.io/xorm"
+)
+
+func (event *Event) Insert(cfg *config.Config) (err error) {
+	session := cfg.Db.NewSession()
+	defer session.Close()
+	_, err = session.Insert(event)
+	err = session.Commit()
+	return
+}
+
+/*
+func (event *Event) APIFormat() *APIEvent {
+	if event == nil {
+		return nil
+	}
+	return &APIEvent{
+		MSgType: "none",
+		IPData: {
+			IP:       event.IP.IP,
+			Src:      event.Src,
+			Hostname: event.Hostname.String,
+		},
+	}
+}*/
+
+func (event *Event) APIParse(session *xorm.Session, apievent APIEvent) (err error) {
+	*event = Event{
+		IP:   &IP{IP: apievent.IPData.IP},
+		Host: &Host{Host: apievent.IPData.Hostname},
+		Src:  &Src{Src: apievent.IPData.Src},
+	}
+	event.IP.GetOrCreate(session)
+	event.Host.GetOrCreate(session)
+	event.Src.GetOrCreate(session)
+	return
+}
+
+type Event struct {
+	ID      int       `xorm:"pk autoincr"`
+	Src     *Src      `xorm:"int src_id"`
+	Host    *Host     `xorm:"int host_id"`
+	IP      *IP       `xorm:"int ip_id"`
+	Created time.Time `xorm:"created notnull"`
+}
+
+type APIEvent struct {
+	MsgType  string `json:"msgtype"`
+	Mode     string `json:"mode"`
+	Hostname string `json:"hostname"`
+	IPData   struct {
+		IP       string `json:"ip"`
+		Src      string `json:"src"`
+		Hostname string `json:"hostname"`
+		Date     string `json:"date"`
+		Created  string `json:"created"`
+	} `json:"ipdata"`
+}

src/models/host.go

@@ -0,0 +1,58 @@
+package models
+
+import (
+	"time"
+
+	"xorm.io/xorm"
+)
+
+func (host *Host) GetOrCreate(session *xorm.Session) (apihost *APIHost, err error) {
+	has, err := session.Get(host)
+	if err != nil {
+		return
+	}
+	if !has {
+		session.Insert(host)
+	} else {
+		session.ID(host.ID).Update(host)
+	}
+	host.Get(session)
+	apihost = host.APIFormat()
+	return
+}
+
+func (host *Host) Get(session *xorm.Session) (apihost *APIHost, err error) {
+	has, err := session.Get(host)
+	if !has || err != nil {
+		return
+	}
+	apihost = host.APIFormat()
+	return
+}
+
+func (host *Host) APIFormat() *APIHost {
+	if host == nil {
+		return &APIHost{}
+	}
+	return &APIHost{
+		Host: host.Host,
+	}
+}
+
+func (host *Host) APIParse(apihost APIHost) (err error) {
+	*host = Host{
+		Host: apihost.Host}
+	return
+}
+
+type Host struct {
+	ID      int       `xorm:"pk autoincr"`
+	Host    string    `xorm:"text unique host" json:"host"`
+	Created time.Time `xorm:"created notnull"`
+	Updated time.Time `xorm:"updated notnull"`
+}
+
+type APIHost struct {
+	ID   int    `json:"-"`
+	Host string `json:"host"`
+}

src/models/hostinfo.go

@@ -1,28 +1,55 @@
 package models
 
 import (
-	"time"
+	"context"
+	"database/sql"
+
+	"git.paulbsd.com/paulbsd/ipbl/src/config"
+	"xorm.io/xorm"
 )
 
+func GetIPsWithoutHostInfo(ctx *context.Context, config *config.Config) (apiips []string, err error) {
+	res, err := config.Db.Query(`
+		SELECT ip
+		FROM ip
+		WHERE ip NOT IN (
+			SELECT ip FROM host_info)
+		ORDER BY RANDOM()
+		LIMIT 10;`)
+	for _, r := range res {
+		apiips = append(apiips, string(r["ip"]))
+	}
+	return
+}
+
+func ProcessHostInfo(cfg *config.Config, hostinfos []HostInfo) (err error) {
+	session := cfg.Db.NewSession()
+	defer session.Close()
+	for _, hi := range hostinfos {
+		var i IP = HostInfoToIP(session, &hi)
+		i.GetOrCreate(session)
+	}
+	session.Commit()
+	return
+}
+
+func HostInfoToIP(session *xorm.Session, hi *HostInfo) (ip IP) {
+	ip = IP{
+		IP: hi.IP,
+		Rdns: sql.NullString{
+			String: hi.Rdns,
+			Valid:  true},
+		City:             &City{CityName: hi.City},
+		Country:          &Country{CountryName: hi.Country},
+		AutonomousSystem: &AutonomousSystem{ID: 0},
+	}
+	return
+}
+
 type HostInfo struct {
-	ID         int       `xorm:"pk autoincr"`
+	IP      string `json:"ip"`
-	IP         string    `xorm:"text unique" json:"ip"`
+	Whois   string `json:"whois"`
-	RawWhois   string    `xorm:"text" json:"raw_whois"`
+	Rdns    string `json:"rdns"`
-	ReverseDNS string    `xorm:"text" json:"reverse_dns"`
+	City    string `json:"city"`
-	City       string    `xorm:"text" json:"city"`
+	Country string `json:"country"`
-	Country    string    `xorm:"text" json:"country"`
-	Created    time.Time `xorm:"created notnull"`
-	Updated    time.Time `xorm:"updated notnull"`
-}
-
-type ScanResult struct {
-	Protocol    string `json:"proto"`
-	PortID      int    `json:"port_id"`
-	State       string `json:"state"`
-	ServiceName string `json:"service"`
-}
-
-type ASN struct {
-	ID   int    `json:"id"`
-	Name string `json:"name"`
 }

src/models/ip.go

@@ -3,15 +3,21 @@ package models
 import (
 	"context"
 	"database/sql"
+	"encoding/json"
 	"fmt"
+	"io"
 	"log"
-	"net"
+	"net/http"
+	"sync"
 	"time"
 
 	"git.paulbsd.com/paulbsd/ipbl/src/config"
+	"xorm.io/xorm"
 )
 
-const ScanLimit = 500
+const SCANLIMIT = 100
+
+const IPINFO_WS = "https://ip.paulbsd.com"
 
 var lastday = time.Now().Add(-(time.Hour * 24))
 
@@ -33,48 +39,54 @@ func GetIPsLast(ctx *context.Context, config *config.Config, interval string) (a
 	return
 }
 
-func GetIPsWithoutHostInfo(ctx *context.Context, config *config.Config) (apiips []string, err error) {
+func (ip *IP) GetOrCreate(session *xorm.Session) (apiip *APIIP, err error) {
-	var ips []IP
+	ip.City.GetOrCreate(session)
-	err = config.Db.In("ip").GroupBy("ip").Find(&ips)
+	ip.Country.GetOrCreate(session)
-	for _, ml := range ips {
+	ip.AutonomousSystem.GetOrCreate(session)
-		apiips = append(apiips, ml.IP)
+	session.Commit()
+
+	var tmpip *IP
+	if ip.ID != 0 {
+		tmpip = &IP{ID: ip.ID, IP: ip.IP}
+	} else {
+		tmpip = &IP{IP: ip.IP}
 	}
+	has, err := session.Get(tmpip)
+	if err != nil {
+		log.Println(err)
+	}
+	if !has {
+		session.Insert(ip)
+	} else {
+		ip.ID = tmpip.ID
+		session.ID(ip.ID).AllCols().Update(ip)
+		session.ID(ip.ID).Cols("city_id", "country_id", "as_id").Update(ip)
+	}
+	session.Commit()
+
+	ip.Get(session)
+	apiip = ip.APIFormat()
 	return
 }
 
-func GetIP(ctx *context.Context, cfg *config.Config, ipquery interface{}) (apiip *APIIP, err error) {
+func (ip *IP) Get(session *xorm.Session) (apiip *APIIP, err error) {
-	session := cfg.Db.NewSession()
+	has, err := session.Get(ip)
-	defer session.Close()
+	if !has || err != nil {
-	var ip IP
-	has, err := session.Where("ip = ?", ipquery).Get(&ip)
-	if !has {
 		err = fmt.Errorf("not found")
 		return nil, err
 	}
-	if err != nil {
-		return
-	}
 	apiip = ip.APIFormat()
-	session.Commit()
 	return
 }
 
-func (ip *IP) UpdateRDNS() (result string, err error) {
+func (ip *IP) InsertOrUpdate(session *xorm.Session) (numinsert int64, numupdate int64, err error) {
-	res, err := net.LookupAddr(ip.IP)
+	//var ips = []IP{}
-	if err != nil {
+	//err = session.Where("ip = ?", ip.IP).And("src = ?", ip.Src).And("hostname = ?", ip.Hostname).Find(&ips)
-		result = ""
+	has, err := session.Get(ip)
-		return
+	if has {
+		session.ID(ip.ID).Update(&IP{})
 	}
-	result = res[0]
+	/*if len(ips) > 0 {
-	return
-}
-
-func (ip *IP) InsertOrUpdate(cfg *config.Config) (numinsert int64, numupdate int64, err error) {
-	session := cfg.Db.NewSession()
-	defer session.Close()
-	var ips = []IP{}
-	err = session.Where("ip = ?", ip.IP).And("src = ?", ip.Src).And("hostname = ?", ip.Hostname).Find(&ips)
-	if len(ips) > 0 {
 		numupdate, err = session.Where("id = ?", ips[0].ID).Cols("updated").Update(&IP{})
 		if err != nil {
 			log.Println(err)
@@ -84,16 +96,16 @@ func (ip *IP) InsertOrUpdate(cfg *config.Config) (numinsert int64, numupdate int
 		if err != nil {
 			log.Println(err)
 		}
-	}
+	}*/
 	session.Commit()
 	return
 }
 
-func InsertIPBulk(cfg *config.Config, ips *[]IP) (numinsert int64, numupdate int64, err error) {
+func InsertIPBulk(session *xorm.Session, ips *[]IP) (numinsert int64, numupdate int64, err error) {
 	for _, ip := range *ips {
-		numinsert, numupdate, err = ip.InsertOrUpdate(cfg)
+		numinsert, numupdate, err = ip.InsertOrUpdate(session)
 	}
-	Cleanup(cfg)
+	Cleanup(session)
 	return
 	/*var iplist []string
 	hostname := (*ips)[0].Hostname.String
@@ -119,43 +131,129 @@ func InsertIPBulk(cfg *config.Config, ips *[]IP) (numinsert int64, numupdate int
 }
 
 func ScanIP(cfg *config.Config) (err error) {
+	var numthreads = 8
 	for {
-		orphans := []IP{}
 		session := cfg.Db.NewSession()
-		err := session.Begin()
+		orphans := []IP{}
-		if err != nil {
+		err = session.Where(`((as_id IS NULL
-			log.Println(err)
+							  OR country_id IS NULL
-		}
+							  OR city_id IS NULL
-		if session.Where("rdns IS NULL").Asc("ip").Limit(ScanLimit).Find(&orphans); len(orphans) > 0 {
+							  OR rdns = ''
-			for _, orphan := range orphans {
+							  OR rdns IS NULL) AND updated < now()-'1d'::interval) OR 
-				reverse, _ := orphan.UpdateRDNS()
+							  (as_id IS NULL
-				log.Printf("%s -> \"%s\"\n", orphan.IP, reverse)
+								AND country_id IS NULL
-				orphan.Rdns.String = reverse
+								AND city_id IS NULL
-				orphan.Rdns.Valid = true
+								AND rdns IS NULL)
-				_, err = session.ID(orphan.ID).Cols("rdns").Update(&orphan)
+							`).Desc("updated").Limit(SCANLIMIT).Find(&orphans)
+		session.Close()
 
-				if err != nil {
+		if err == nil && len(orphans) > 0 {
-					session.Rollback()
+			orphanchan := make(chan IP)
-					log.Println(err)
+			done := make(chan bool)
-				}
+			var wg sync.WaitGroup
+
+			for thr := range make([]int, numthreads) {
+				go ScanOrphan(&wg, orphanchan, done, thr, cfg)
 			}
+
+			for _, orphan := range orphans {
+				orphanchan <- orphan
+			}
+
+			close(orphanchan)
+			<-done
+			wg.Wait()
+
+		} else {
+			time.Sleep(30 * time.Second)
+		}
+	}
+}
+
+func ScanOrphan(wg *sync.WaitGroup, orphans chan IP, done chan bool, thr int, cfg *config.Config) (err error) {
+	wg.Add(1)
+
+	for {
+		orphan, more := <-orphans
+		if more {
+			session := cfg.Db.NewSession()
+			if err != nil {
+				log.Println(err)
+			}
+			queryclient := http.Client{}
+			var query QueryIP
+			query, err := QueryInfo(&queryclient, orphan.IP)
+			if err != nil {
+				log.Println(err)
+				continue
+			}
+
+			var as = AutonomousSystem{ASID: query.APIAS.Number, ASName: query.APIAS.Org}
+			orphan.AutonomousSystem = &as
+
+			if query.APICity != "" {
+				var city = City{CityName: query.APICity}
+				orphan.City = &city
+			}
+
+			if query.APICountry != "" {
+				var country = Country{CountryName: query.APICountry}
+				orphan.Country = &country
+			}
+
+			orphan.Rdns = sql.NullString{String: query.Rdns, Valid: true}
+			log.Printf("%s -> \"%s\"\n", orphan.IP, query.Rdns)
+
+			orphan.GetOrCreate(session)
+
 			err = session.Commit()
 			if err != nil {
 				log.Println(err)
 			}
+			err = session.Close()
+			if err != nil {
+				log.Println(err)
+			}
 		} else {
-			time.Sleep(10 * time.Second)
+			//fmt.Printf("All orphan migrated on thread num %d\n", thr)
+			wg.Done()
+			done <- true
+			return nil
 		}
-		session.Close()
 	}
 }
 
-func Cleanup(cfg *config.Config) (err error) {
+func QueryInfo(client *http.Client, ip string) (query QueryIP, err error) {
-	session := cfg.Db.NewSession()
+	var url = fmt.Sprintf("%s/%s", IPINFO_WS, ip)
-	defer session.Close()
+	req, _ := http.NewRequest("GET", url, nil)
-	results, _ := session.Query("select * from ip where ip in (select ip from ip group by ip having count(ip) > 1) and hostname is null order by updated desc;")
+	req.Header.Add("Accept", "*/*")
+	req.Header.Add("User-Agent", "ipbl")
+	res, _ := client.Do(req)
+	data, _ := io.ReadAll(res.Body)
+	err = json.Unmarshal(data, &query)
+	return
+}
+
+func Cleanup(session *xorm.Session) (err error) {
+	results, _ := session.Query(`
+		SELECT *
+		FROM ip
+		WHERE ip IN (
+			SELECT ip
+			FROM ip
+			GROUP BY ip
+			HAVING COUNT(ip) > 1)
+		AND hostname IS NULL
+		ORDER BY updated DESC;`)
 	if len(results) > 0 {
-		_, err := session.Query("delete from ip where ip in (select ip from ip group by ip having count(ip) > 1) and hostname is null;")
+		_, err := session.Query(`
+			DELETE FROM ip
+			WHERE ip IN (
+				SELECT ip
+				FROM ip
+				GROUP by ip
+				HAVING COUNT(ip) > 1)
+			AND hostname IS NULL;`)
 		if err != nil {
 			log.Println("error deleting orphans")
 		}
@@ -166,33 +264,29 @@ func Cleanup(cfg *config.Config) (err error) {
 
 func (ip *IP) APIFormat() *APIIP {
 	if ip == nil {
-		return nil
+		return &APIIP{}
 	}
 	return &APIIP{
-		IP:       ip.IP,
+		IP:         ip.IP,
-		Rdns:     ip.Rdns.String,
+		Rdns:       ip.Rdns.String,
-		Src:      ip.Src,
+		APIAS:      *ip.AutonomousSystem.APIFormat(),
-		Hostname: ip.Hostname.String,
+		APICity:    ip.City.APIFormat().CityName,
-		City:     ip.City.String,
+		APICountry: ip.Country.APIFormat().CountryName,
-		Country:  ip.Country.String,
-		Date:     ip.Updated.Local().String(),
 	}
 }
 
-func (ip *APIIP) APIConvert() *IP {
+func (ip *IP) APIParse(apiip APIIP) (err error) {
-	if ip == nil {
+	*ip = IP{
-		return nil
+		IP: apiip.IP,
-	}
-	return &IP{
-		IP: ip.IP,
 		Rdns: sql.NullString{
-			String: ip.Rdns,
+			String: apiip.Rdns,
-			Valid:  false},
-		Src: ip.Src,
-		Hostname: sql.NullString{
-			String: ip.Hostname,
 			Valid:  true},
+		AutonomousSystem: &AutonomousSystem{
+			ASID:   apiip.APIAS.ASID,
+			ASName: apiip.APIAS.ASName,
+		},
 	}
+	return
 }
 
 func (ip *APIIP) BeforeInsert() (err error) {
@@ -200,24 +294,34 @@ func (ip *APIIP) BeforeInsert() (err error) {
 }
 
 type IP struct {
-	ID       int            `xorm:"pk autoincr"`
+	ID               int               `xorm:"pk autoincr"`
-	IP       string         `xorm:"text notnull unique(ipsrc)" json:"ip"`
+	IP               string            `xorm:"text notnull"`
-	Rdns     sql.NullString `xorm:"text default"`
+	Rdns             sql.NullString    `xorm:"text default ''"`
-	Src      string         `xorm:"text notnull unique(ipsrc)" json:"src"`
+	AutonomousSystem *AutonomousSystem `xorm:"as_id int index default null"`
-	Hostname sql.NullString `xorm:"text default '' unique(ipsrc)" json:"hostname"`
+	City             *City             `xorm:"city_id int index default null"`
-	City     sql.NullString `xorm:"text default '' unique(ipsrc)" json:"city"`
+	Country          *Country          `xorm:"country_id int index default null"`
-	Country  sql.NullString `xorm:"text default '' unique(ipsrc)" json:"country"`
+	Created          time.Time         `xorm:"created notnull"`
-	Created  time.Time      `xorm:"created notnull"`
+	Updated          time.Time         `xorm:"updated notnull"`
-	Updated  time.Time      `xorm:"updated notnull"`
 }
 
 type APIIP struct {
-	IP       string `json:"ip"`
+	IP         string              `json:"ip"`
-	Rdns     string `json:"rdns"`
+	Rdns       string              `json:"rdns"`
-	Src      string `json:"src"`
+	APIAS      APIAutonomousSystem `json:"as"`
-	Hostname string `json:"hostname"`
+	APICity    string              `json:"city"`
-	City     string `json:"city"`
+	APICountry string              `json:"country"`
-	Country  string `json:"country"`
+	APIWhois   string              `json:"whois"`
-	Date     string `json:"date"`
+}
-	Mode     string `json:"mode"`
+
+type QueryIP struct {
+	IP         string                `json:"ip"`
+	Rdns       string                `json:"hostname"`
+	APIAS      QueryAutonomousSystem `json:"as"`
+	APICity    string                `json:"city"`
+	APICountry string                `json:"country"`
+}
+
+type QueryAutonomousSystem struct {
+	Number int    `json:"number"`
+	Org    string `json:"org"`
 }

src/models/models.go

@@ -0,0 +1,67 @@
+package models
+
+import (
+	"context"
+	"fmt"
+
+	"git.paulbsd.com/paulbsd/ipbl/src/config"
+	"xorm.io/xorm"
+	"xorm.io/xorm/names"
+)
+
+var (
+	x      *xorm.Engine
+	tables []interface{}
+
+	// HasEngine specifies if we have a xorm.Engine
+	HasEngine bool
+)
+
+func init() {
+	tables = append(tables,
+		new(AutonomousSystem),
+		new(Cfg),
+		new(CfgSet),
+		new(CfgTrustlist),
+		new(CfgZMQ),
+		new(City),
+		new(Country),
+		new(Event),
+		new(IP),
+		new(Src),
+		new(Host),
+	)
+	for _, name := range []string{"SSL", "UID"} {
+		names.LintGonicMapper[name] = true
+	}
+}
+
+// NewEngine initializes a new xorm.Engine
+func NewEngine(ctx *context.Context, config *config.Config) (err error) {
+	var x = config.Db
+
+	if err = x.Ping(); err != nil {
+		return err
+	}
+
+	if err = x.Sync2(tables...); err != nil {
+		return fmt.Errorf("sync database struct error: %v", err)
+	}
+
+	return nil
+}
+
+// DropTables initializes a new xorm.Engine
+func DropTables(ctx *context.Context, config *config.Config) (err error) {
+	var x = config.Db
+
+	if err = x.Ping(); err != nil {
+		return err
+	}
+
+	if err = x.Sync2(tables...); err != nil {
+		return fmt.Errorf("sync database struct error: %v", err)
+	}
+
+	return nil
+}

src/models/scanresult.go

@@ -0,0 +1,65 @@
+package models
+
+import (
+	"time"
+
+	"git.paulbsd.com/paulbsd/ipbl/src/config"
+)
+
+func (sr *ScanResult) GetOrCreate(cfg *config.Config) (apisr *APIScanResult, err error) {
+	has, err := cfg.Db.Get(sr)
+	if !has {
+		cfg.Db.Insert(sr)
+	} else {
+		cfg.Db.ID(sr.ID).Update(sr)
+	}
+	sr.Get(cfg)
+	apisr = sr.APIFormat()
+	return
+}
+
+func (sr *ScanResult) Get(cfg *config.Config) (apisr *APIScanResult, err error) {
+	has, err := cfg.Db.Get(sr)
+	if !has || err != nil {
+		return
+	}
+	apisr = sr.APIFormat()
+	return
+}
+
+func (sr *ScanResult) APIFormat() *APIScanResult {
+	if sr == nil {
+		return &APIScanResult{}
+	}
+	return &APIScanResult{
+		Protocol:    sr.Protocol,
+		PortID:      sr.PortID,
+		State:       sr.State,
+		ServiceName: sr.ServiceName,
+	}
+}
+
+func (sr *ScanResult) APIParse(apisr APIScanResult) (err error) {
+	*sr = ScanResult{
+		Protocol: apisr.Protocol,
+		PortID:   apisr.PortID}
+	return
+}
+
+type ScanResult struct {
+	ID          int       `xorm:"pk autoincr"`
+	Protocol    string    `xorm:"text default ''"`
+	PortID      int       `xorm:"default ''"`
+	State       string    `xorm:"text default ''"`
+	ServiceName string    `xorm:"text default ''"`
+	Created     time.Time `xorm:"created notnull"`
+	Updated     time.Time `xorm:"updated notnull"`
+}
+
+type APIScanResult struct {
+	ID          int    `json:"id"`
+	Protocol    string `json:"proto"`
+	PortID      int    `json:"port_id"`
+	State       string `json:"state"`
+	ServiceName string `json:"service"`
+}

src/models/src.go

@@ -0,0 +1,58 @@
+package models
+
+import (
+	"time"
+
+	"xorm.io/xorm"
+)
+
+func (src *Src) GetOrCreate(session *xorm.Session) (apisrc *APISrc, err error) {
+	has, err := session.Get(src)
+	if err != nil {
+		return
+	}
+	if !has {
+		session.Insert(src)
+	} else {
+		session.ID(src.ID).Update(src)
+	}
+	src.Get(session)
+	apisrc = src.APIFormat()
+	return
+}
+
+func (src *Src) Get(session *xorm.Session) (apisrc *APISrc, err error) {
+	has, err := session.Get(src)
+	if !has || err != nil {
+		return
+	}
+	apisrc = src.APIFormat()
+	return
+}
+
+func (src *Src) APIFormat() *APISrc {
+	if src == nil {
+		return &APISrc{}
+	}
+	return &APISrc{
+		Src: src.Src,
+	}
+}
+
+func (src *Src) APIParse(apisrc APISrc) (err error) {
+	*src = Src{
+		Src: apisrc.Src}
+	return
+}
+
+type Src struct {
+	ID      int       `xorm:"pk autoincr"`
+	Src     string    `xorm:"text unique(srcindex) src" json:"src"`
+	Created time.Time `xorm:"created notnull"`
+	Updated time.Time `xorm:"updated notnull"`
+}
+
+type APISrc struct {
+	ID  int    `json:"-"`
+	Src string `json:"src"`
+}

src/routers/funcs.go

@@ -22,23 +22,31 @@ func RegisterRoutes(e *echo.Echo, ctx *context.Context, cfg *config.Config) {
 		</html>`)
 	})
 
+	e.GET("/health", func(c echo.Context) error {
+		return c.HTML(http.StatusOK, `OK`)
+	})
+
 	e.GET("/ip/:ip", func(c echo.Context) (err error) {
-		ret, err := models.GetIP(ctx, cfg, c.Param("ip"))
+		session := cfg.Db.NewSession()
+		defer session.Close()
+		var ip = models.IP{IP: c.Param("ip")}
+		ret, err := ip.Get(session)
 		return Result(c, err, ret)
 	})
 	e.POST("/ip", func(c echo.Context) (err error) {
+		session := cfg.Db.NewSession()
+		var apiip = new(models.APIIP)
 		var ip = new(models.IP)
 		var msg = "No IP inserted"
-		err = c.Bind(ip)
+		err = c.Bind(apiip)
 		if err != nil {
 			return Result(c, fmt.Errorf("error when parsing body"), "")
 		}
-		numinsert, numupdate, err := ip.InsertOrUpdate(cfg)
+		ip.APIParse(*apiip)
+		_, err = ip.GetOrCreate(session)
 		if err != nil {
 			return Result(c, err, "")
 		}
-		msg = fmt.Sprintf("zmq: Inserted %d IP, Updated %d IP", numinsert, numupdate)
-		log.Println(msg)
 		return Result(c, err, msg)
 	})
 	e.GET("/ips", func(c echo.Context) (err error) {
@@ -62,6 +70,8 @@ func RegisterRoutes(e *echo.Echo, ctx *context.Context, cfg *config.Config) {
 		return Result(c, err, ret)
 	})
 	e.POST("/ips", func(c echo.Context) (err error) {
+		session := cfg.Db.NewSession()
+		defer session.Close()
 		var apiips = []models.APIIP{}
 		var ips = []models.IP{}
 		var msg string
@@ -70,23 +80,39 @@ func RegisterRoutes(e *echo.Echo, ctx *context.Context, cfg *config.Config) {
 			return Result(c, err, apiips)
 		}
 		for _, apiip := range apiips {
-			ips = append(ips, *apiip.APIConvert())
+			var ip = new(models.IP)
+			ip.APIParse(apiip)
+			ips = append(ips, *ip)
 		}
-		numinsert, numupdate, _ := models.InsertIPBulk(cfg, &ips)
+		numinsert, numupdate, _ := models.InsertIPBulk(session, &ips)
 		msg = fmt.Sprintf("zmq: Inserted %d IP, Updated %d IP", numinsert, numupdate)
 		log.Println(msg)
 		return Result(c, err, msg)
 	})
+	e.POST("/event", func(c echo.Context) (err error) {
+		session := cfg.Db.NewSession()
+		defer session.Close()
+		var apievent = new(models.APIEvent)
+		var event = new(models.Event)
+		err = c.Bind(apievent)
+		if err != nil {
+			return Result(c, fmt.Errorf("error when parsing body"), "")
+		}
+		event.APIParse(session, *apievent)
+		err = event.Insert(cfg)
+		if err != nil {
+			return Result(c, err, "")
+		}
+		return Result(c, err, "OK")
+	})
 	e.POST("/hostinfo", func(c echo.Context) (err error) {
 		var hostinfos = []models.HostInfo{}
 		err = c.Bind(&hostinfos)
 		if err != nil {
 			return Result(c, err, hostinfos)
 		}
-		num, err := cfg.Db.Insert(&hostinfos)
+		models.ProcessHostInfo(cfg, hostinfos)
-		msg := fmt.Sprintf("Inserted %v host info (%d)", hostinfos, num)
+		return Result(c, err, "")
-		log.Println(msg)
-		return Result(c, err, msg)
 	})
 	e.GET("/config", func(c echo.Context) (err error) {
 		trustlists, err := models.GetGlobalConfig(*cfg)

src/routers/main.go

@@ -24,6 +24,9 @@ func RunServer(ctx *context.Context, cfg *config.Config) (err error) {
 
 func Result(c echo.Context, inputerr error, data interface{}) (err error) {
 	if inputerr != nil {
+		if inputerr.Error() == "not found" {
+			return c.String(http.StatusNotFound, "Content not found")
+		}
 		if inputerr.Error() == "Not Found" {
 			return c.String(http.StatusNotFound, "Content not found")
 		}

src/zmqrouter/main.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"log"
+	"time"
 
 	"git.paulbsd.com/paulbsd/ipbl/src/config"
 	"git.paulbsd.com/paulbsd/ipbl/src/models"
@@ -30,37 +31,57 @@ func Init(cfg *config.Config) (err error) {
 func Handle(cfg *config.Config, reqsock *goczmq.Sock, pubsock *goczmq.Sock, channel string) (err error) {
 	log.Println("Start handling zmq sockets")
 
+	var lastip string
 	for {
 		var msg = "err"
 		var req, err = reqsock.RecvMessage()
 
 		if err != nil {
-			log.Println("unable to receive message from req socket")
+			log.Println("unable to receive message from req socket", err)
 			continue
 		}
 
 		var topub [][]byte
 		for _, val := range req {
-			var apiip = models.APIIP{}
-			var ip = models.IP{}
 
-			err = json.Unmarshal(val, &apiip)
+			var apievent = models.APIEvent{}
+			var event = models.Event{}
+
+			err = json.Unmarshal(val, &apievent)
 			if err != nil {
 				log.Println("unable to parse ip address", err)
+				time.Sleep(time.Second)
+				continue
+			}
+			if apievent.IPData.IP != "" && apievent.IPData.IP == lastip {
 				continue
 			}
-			if apiip.Mode != "init" {
-				ip = *apiip.APIConvert()
 
-				numinsert, numupdate, err := ip.InsertOrUpdate(cfg)
+			if apievent.MsgType == "add" {
+				session := cfg.Db.NewSession()
+				event.APIParse(session, apievent)
+				session.Close()
+
+				err := event.Insert(cfg)
 				if err != nil {
 					log.Println(err)
 				}
-				log.Printf("zmq: Inserted %d IP, Updated %d IP\n", numinsert, numupdate)
+				log.Printf("zmq: Inserted event")
 			}
+
+			if apievent.MsgType == "file" {
+				apievent.MsgType = "zmq"
+			}
+
+			val, err = json.Marshal(apievent)
+			if err != nil {
+				fmt.Println(err)
+			}
+
 			tmpval := fmt.Sprintf("%s %s", channel, string(val))
 			ipjson := []byte(tmpval)
 			topub = append(topub, ipjson)
+			lastip = apievent.IPData.IP
 		}
 
 		err = pubsock.SendMessage(topub)

src/zmqrouter/pubsub.go

@@ -1,10 +1,14 @@
 package zmqrouter
 
 import (
+	"fmt"
+
 	"gopkg.in/zeromq/goczmq.v4"
 )
 
+const pubsubport int = 9999
+
 func InitPub() (sock *goczmq.Sock, err error) {
-	sock, err = goczmq.NewPub("tcp://*:9999")
+	sock, err = goczmq.NewPub(fmt.Sprintf("tcp://*:%d", pubsubport))
 	return
 }

src/zmqrouter/reqrep.go

@@ -1,10 +1,14 @@
 package zmqrouter
 
 import (
+	"fmt"
+
 	"gopkg.in/zeromq/goczmq.v4"
 )
 
+const repreqport int = 9998
+
 func InitRep() (sock *goczmq.Sock, err error) {
-	sock, err = goczmq.NewRep("tcp://*:9998")
+	sock, err = goczmq.NewRep(fmt.Sprintf("tcp://*:%d", repreqport))
 	return
 }

vendor/modules.txt

@@ -12,6 +12,8 @@ github.com/goccy/go-json/internal/runtime
 # github.com/golang/snappy v0.0.4
 ## explicit
 github.com/golang/snappy
+# github.com/google/go-cmp v0.5.9
+## explicit; go 1.13
 # github.com/json-iterator/go v1.1.12
 ## explicit; go 1.12
 github.com/json-iterator/go
@@ -39,10 +41,10 @@ github.com/modern-go/concurrent
 # github.com/modern-go/reflect2 v1.0.2
 ## explicit; go 1.12
 github.com/modern-go/reflect2
-# github.com/onsi/ginkgo v1.16.4
+# github.com/onsi/ginkgo v1.16.5
-## explicit; go 1.15
+## explicit; go 1.16
-# github.com/onsi/gomega v1.14.0
+# github.com/onsi/gomega v1.22.1
-## explicit; go 1.14
+## explicit; go 1.18
 # github.com/syndtr/goleveldb v1.0.0
 ## explicit
 github.com/syndtr/goleveldb/leveldb