diff --git a/src/api/ip.go b/src/api/ip.go deleted file mode 100644 index 04d440e..0000000 --- a/src/api/ip.go +++ /dev/null @@ -1,8 +0,0 @@ -package api - -type IP struct { - IP string `json:"ip"` - Rdns string `json:"rdns"` - Src string `json:"src"` - Hostname string `json:"hostname"` -} diff --git a/src/database/main.go b/src/database/main.go index 9a25af6..edba9ee 100644 --- a/src/database/main.go +++ b/src/database/main.go @@ -15,7 +15,12 @@ import ( func Initialize(ctx *context.Context, cfg *config.Config) (err error) { var databaseEngine = "postgres" - var tables = []interface{}{models.IP{}, models.Cfg{}, models.Src{}} + var tables = []interface{}{models.IP{}, + models.Cfg{}, + models.Src{}, + models.CfgSet{}, + models.CfgTrustlist{}, + models.CfgZMQ{}} cfg.Db, err = xorm.NewEngine(databaseEngine, fmt.Sprintf("%s://%s:%s@%s/%s", diff --git a/src/models/cfg.go b/src/models/cfg.go index 2a2b405..b597693 100644 --- a/src/models/cfg.go +++ b/src/models/cfg.go @@ -14,14 +14,17 @@ import ( var ipv4_cidr_regex = `^(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.|)){4}\/([1-3])?([0-9])?$)` func GetTrustlists(cfg config.Config) (res []string, err error) { - var w = Cfg{Key: "trustlist"} - if exists, _ := cfg.Db.Get(&w); exists { - res = strings.Split(w.Value, ",") + var w []CfgTrustlist + err = cfg.Db.Find(&w) + if len(w) > 0 { + for _, a := range w { + res = append(res, a.IP) + } } return } -func (wl Trustlist) InsertOrUpdate(cfg config.Config) (err error) { +func (wl CfgTrustlist) InsertOrUpdate(cfg config.Config) (err error) { var w = Cfg{Key: "trustlist"} exists, _ := cfg.Db.Get(&w) if exists { @@ -38,37 +41,30 @@ func (wl Trustlist) InsertOrUpdate(cfg config.Config) (err error) { return fmt.Errorf("no trustlist updated") } -func (wl Trustlist) Delete(cfg config.Config, ip string) (err error) { - var w = Cfg{Key: "trustlist"} +func (wl CfgTrustlist) Delete(cfg config.Config, ip string) (affected int64, err error) { + var w = CfgTrustlist{IP: ip} exists, _ := cfg.Db.Get(&w) - var updated []string if exists { - existing, _ := GetTrustlists(cfg) - for _, sip := range existing { - if sip != ip { - updated = append(updated, sip) - } - } - w.Value = strings.Join(updated, ",") - cfg.Db.ID(w.ID).Update(&w) - } - return fmt.Errorf("no trustlist updated") -} - -func (wl Trustlist) Verify() bool { - reg := regexp.MustCompile(ipv4_cidr_regex) - return reg.MatchString(wl.IP) -} - -func GetFolders(cfg config.Config) (res []Folder, err error) { - var w = Cfg{Key: "folders"} - if exists, _ := cfg.Db.Get(&w); exists { - err = json.Unmarshal([]byte(w.Value), &res) + affected, err = cfg.Db.ID(w.ID).Update(&w) + return } return } -func InsertOrUpdateFolders(cfg config.Config, folders []Folder) (res string, err error) { +func (wl CfgTrustlist) Verify() bool { + reg := regexp.MustCompile(ipv4_cidr_regex) + return reg.MatchString(wl.IP) +} + +func GetSets(cfg config.Config) (res []CfgSet, err error) { + var w = []CfgSet{} + if err := cfg.Db.Find(&w); err == nil { + return w, err + } + return +} + +func InsertOrUpdateSets(cfg config.Config, folders []CfgSet) (res string, err error) { var w = Cfg{Key: "folders"} if exists, _ := cfg.Db.Get(&w); exists { resbytes, err := json.Marshal(folders) @@ -84,10 +80,10 @@ func InsertOrUpdateFolders(cfg config.Config, folders []Folder) (res string, err return } -func GetZMQ(cfg config.Config, key string) (res ZMQ, err error) { - var w = Cfg{Key: fmt.Sprintf("zmq%s", key)} - if exists, _ := cfg.Db.Get(&w); exists { - err = json.Unmarshal([]byte(w.Value), &res) +func GetZMQ(cfg config.Config) (res []CfgZMQ, err error) { + var w = []CfgZMQ{} + if err = cfg.Db.Find(&w); err == nil { + return w, err } return } @@ -107,24 +103,25 @@ func DiscoverURLS(cfg config.Config, routes []*echo.Route) (Discovery, error) { return disc, nil } -type Trustlist struct { - IP string `json:"ip"` +type CfgTrustlist struct { + ID int `xorm:"pk autoincr" json:"-"` + IP string `xorm:"text notnull" json:"ip"` +} +type CfgZMQ struct { + ID int `xorm:"pk autoincr" json:"-"` + Type string `xorm:"text notnull" json:"type"` + Hostname string `xorm:"text notnull" json:"hostname"` + Port int `json:"port"` + Subscription string `json:"subscription"` } -type Folder struct { - Path string `json:"path"` - Sets []Set `json:"sets"` -} - -type ZMQ struct { - Hostname string `json:"hostname"` - Port int `json:"port"` -} - -type Set struct { - Type string `json:"type"` - Filename string `json:"filename"` - Regex string `json:"regex"` +type CfgSet struct { + ID int `xorm:"pk autoincr" json:"-"` + Path string `xorm:"text notnull default" json:"path"` + Type string `xorm:"text notnull" json:"type"` + Filename string `xorm:"text notnull" json:"filename"` + Regex string `xorm:"text notnull" json:"regex"` + Enabled bool `xorm:"notnull default true" json:"-"` } type Cfg struct { diff --git a/src/models/ip.go b/src/models/ip.go index fbbb8ad..88186a9 100644 --- a/src/models/ip.go +++ b/src/models/ip.go @@ -8,31 +8,30 @@ import ( "net" "time" - "git.paulbsd.com/paulbsd/ipbl/src/api" "git.paulbsd.com/paulbsd/ipbl/src/config" ) //var lastday = time.Now().Add(-(time.Hour * 24)) -func GetIPs(ctx *context.Context, config *config.Config, limit int) (apimailboxes []*api.IP, err error) { +func GetIPs(ctx *context.Context, config *config.Config, limit int) (apiips []*APIIP, err error) { var ips []IP err = config.Db.Limit(limit).Desc("created").Find(&ips) for _, ml := range ips { - apimailboxes = append(apimailboxes, ml.APIFormat()) + apiips = append(apiips, ml.APIFormat()) } return } -func GetIPsLast(ctx *context.Context, config *config.Config, interval string) (apimailboxes []*api.IP, err error) { +func GetIPsLast(ctx *context.Context, config *config.Config, interval string) (apiips []*APIIP, err error) { var ips []IP err = config.Db.Where("updated >= (now()-?::interval)", interval).GroupBy("ip").Find(&ips) for _, ml := range ips { - apimailboxes = append(apimailboxes, ml.APIFormat()) + apiips = append(apiips, ml.APIFormat()) } return } -func GetIP(ctx *context.Context, config *config.Config, ipquery interface{}) (apiip *api.IP, err error) { +func GetIP(ctx *context.Context, config *config.Config, ipquery interface{}) (apiip *APIIP, err error) { var ip IP has, err := config.Db.Where("ip = ?", ipquery).Get(&ip) if !has { @@ -63,22 +62,25 @@ func (i *IP) InsertIP(cfg *config.Config) (num int64, err error) { func InsertIPBulk(cfg *config.Config, ips *[]IP) (numinserts int64, numupdates int64, numfail int64, err error) { var iplist []string + hostname := (*ips)[0].Hostname for _, ip := range *ips { iplist = append(iplist, ip.IP) } var searchips []IP - cfg.Db.In("ip", iplist).Find(&searchips) + cfg.Db.In("ip", iplist).Where("hostname = ?", hostname).Find(&searchips) var toupdateips []string for _, ip := range searchips { toupdateips = append(toupdateips, ip.IP) } - numupdates, _ = cfg.Db.In("ip", toupdateips).Cols("updated").Update(&IP{}) + numupdates, _ = cfg.Db.In("ip", toupdateips).Where("hostname = ?", hostname).Cols("updated").Update(&IP{}) var toinsertip, _ = differ(*ips, searchips) numinserts, err = cfg.Db.Insert(toinsertip) + Cleanup(cfg) + return } @@ -102,15 +104,37 @@ func ScanIP(cfg *config.Config) (err error) { } } -func (ip *IP) APIFormat() *api.IP { +func Cleanup(cfg *config.Config) (err error) { + results, _ := cfg.Db.Query("select * from ip where ip in (select ip from ip group by ip having count(ip) > 1) and hostname is null order by updated desc;") + if len(results) > 0 { + _, err := cfg.Db.Query("delete from ip where ip in (select ip from ip group by ip having count(ip) > 1) and hostname is null;") + if err != nil { + log.Println("error deleting orphans") + } + } + return +} + +func (ip *IP) APIFormat() *APIIP { if ip == nil { return nil } - return &api.IP{ + return &APIIP{ IP: ip.IP, Rdns: ip.Rdns.String, Src: ip.Src, - Hostname: ip.Hostname, + Hostname: ip.Hostname.String, + } +} +func (ip *APIIP) APIConvert() *IP { + if ip == nil { + return nil + } + return &IP{ + IP: ip.IP, + Rdns: sql.NullString{String: ip.Rdns, Valid: true}, + Src: ip.Src, + Hostname: sql.NullString{String: ip.Hostname, Valid: true}, } } @@ -119,7 +143,14 @@ type IP struct { IP string `xorm:"text notnull unique(ipsrc)" json:"ip"` Rdns sql.NullString `xorm:"text default" json:"rdns"` Src string `xorm:"text notnull unique(ipsrc)" json:"src"` - Hostname string `xorm:"text notnull unique(ipsrc)" json:"hostname"` + Hostname sql.NullString `xorm:"text default '' unique(ipsrc)" json:"hostname"` Created time.Time `xorm:"created notnull" json:"-"` Updated time.Time `xorm:"updated notnull" json:"-"` } + +type APIIP struct { + IP string `json:"ip"` + Rdns string `json:"rdns"` + Src string `json:"src"` + Hostname string `json:"hostname"` +} diff --git a/src/routers/funcs.go b/src/routers/funcs.go index f041222..974bf90 100644 --- a/src/routers/funcs.go +++ b/src/routers/funcs.go @@ -60,11 +60,15 @@ func RegisterRoutes(e *echo.Echo, ctx *context.Context, cfg *config.Config) { return Result(c, err, ret) }) e.POST("/ips", func(c echo.Context) (err error) { + var apiips = []models.APIIP{} var ips = []models.IP{} var msg string - err = c.Bind(&ips) + err = c.Bind(&apiips) if err != nil { - return Result(c, err, ips) + return Result(c, err, apiips) + } + for _, v := range apiips { + ips = append(ips, *v.APIConvert()) } numinsert, numupdate, _, _ := models.InsertIPBulk(cfg, &ips) if numinsert > 0 { @@ -86,7 +90,7 @@ func RegisterRoutes(e *echo.Echo, ctx *context.Context, cfg *config.Config) { return Result(c, err, trustlists) }) e.POST("/config/trustlist", func(c echo.Context) (err error) { - var cidr models.Trustlist + var cidr models.CfgTrustlist err = c.Bind(&cidr) if err == nil && cidr.Verify() { err = cidr.InsertOrUpdate(*cfg) @@ -96,45 +100,25 @@ func RegisterRoutes(e *echo.Echo, ctx *context.Context, cfg *config.Config) { }) e.DELETE("/config/trustlist/:ip", func(c echo.Context) (err error) { var ip = c.Param("ip") - var cidr models.Trustlist - err = cidr.Delete(*cfg, ip) - if err != nil { - return c.JSON(http.StatusOK, "Deleted old CIDR") - } + var cidr models.CfgTrustlist + _, err = cidr.Delete(*cfg, ip) return }) - e.GET("/config/folders", func(c echo.Context) (err error) { - folders, err := models.GetFolders(*cfg) - return Result(c, err, folders) + e.GET("/config/sets", func(c echo.Context) (err error) { + sets, err := models.GetSets(*cfg) + return Result(c, err, sets) }) e.POST("/config/folders", func(c echo.Context) (err error) { - var folders []models.Folder - err = c.Bind(&folders) + var sets []models.CfgSet + err = c.Bind(&sets) if err != nil { return Result(c, err, "Unable to parse JSON") } - _, err = models.InsertOrUpdateFolders(*cfg, folders) - return Result(c, err, folders) + _, err = models.InsertOrUpdateSets(*cfg, sets) + return Result(c, err, sets) }) - e.GET("/config/folders", func(c echo.Context) (err error) { - folders, err := models.GetFolders(*cfg) - return Result(c, err, folders) - }) - e.POST("/config/folders", func(c echo.Context) (err error) { - var folders []models.Folder - err = c.Bind(&folders) - if err != nil { - return Result(c, err, "Unable to parse JSON") - } - _, err = models.InsertOrUpdateFolders(*cfg, folders) - return Result(c, err, folders) - }) - e.GET("/config/zmqps", func(c echo.Context) (err error) { - folders, err := models.GetZMQ(*cfg, "ps") - return Result(c, err, folders) - }) - e.GET("/config/zmqrr", func(c echo.Context) (err error) { - folders, err := models.GetZMQ(*cfg, "rr") + e.GET("/config/zmq", func(c echo.Context) (err error) { + folders, err := models.GetZMQ(*cfg) return Result(c, err, folders) }) e.GET("/discovery", func(c echo.Context) (err error) {