paulbsd/ipbl
1
0
Fork 0
Code Issues Pull Requests Projects Releases 30 Wiki Activity

changed tables schemas
All checks were successful
continuous-integration/drone/push Build is passing
Details
continuous-integration/drone/tag Build is passing
Details

Browse Source
This commit is contained in:
Paul 2022-03-19 14:48:50 +01:00
parent 881969f81e
commit 83995c26a5
5 changed files with 113 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/api/ip.go
View File

@ -1,8 +0,0 @@
package api
type IP struct {
IP string `json:"ip"`
Rdns string `json:"rdns"`
Src string `json:"src"`
Hostname string `json:"hostname"`
}

7
src/database/main.go
View File

@ -15,7 +15,12 @@ import (
func Initialize(ctx *context.Context, cfg *config.Config) (err error) {
var databaseEngine = "postgres"
var tables = []interface{}{models.IP{}, models.Cfg{}, models.Src{}}
var tables = []interface{}{models.IP{},
models.Cfg{},
models.Src{},
models.CfgSet{},
models.CfgTrustlist{},
models.CfgZMQ{}}
cfg.Db, err = xorm.NewEngine(databaseEngine,
fmt.Sprintf("%s://%s:%s@%s/%s",

95
src/models/cfg.go
View File

@ -14,14 +14,17 @@ import (
var ipv4_cidr_regex = `^(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.|)){4}\/([1-3])?([0-9])?$)`
func GetTrustlists(cfg config.Config) (res []string, err error) {
var w = Cfg{Key: "trustlist"}
if exists, _ := cfg.Db.Get(&w); exists {
res = strings.Split(w.Value, ",")
var w []CfgTrustlist
err = cfg.Db.Find(&w)
if len(w) > 0 {
for _, a := range w {
res = append(res, a.IP)
}
}
return
}
func (wl Trustlist) InsertOrUpdate(cfg config.Config) (err error) {
func (wl CfgTrustlist) InsertOrUpdate(cfg config.Config) (err error) {
var w = Cfg{Key: "trustlist"}
exists, _ := cfg.Db.Get(&w)
if exists {
@ -38,37 +41,30 @@ func (wl Trustlist) InsertOrUpdate(cfg config.Config) (err error) {
return fmt.Errorf("no trustlist updated")
}
func (wl Trustlist) Delete(cfg config.Config, ip string) (err error) {
var w = Cfg{Key: "trustlist"}
func (wl CfgTrustlist) Delete(cfg config.Config, ip string) (affected int64, err error) {
var w = CfgTrustlist{IP: ip}
exists, _ := cfg.Db.Get(&w)
var updated []string
if exists {
existing, _ := GetTrustlists(cfg)
for _, sip := range existing {
if sip != ip {
updated = append(updated, sip)
}
}
w.Value = strings.Join(updated, ",")
cfg.Db.ID(w.ID).Update(&w)
}
return fmt.Errorf("no trustlist updated")
}
func (wl Trustlist) Verify() bool {
reg := regexp.MustCompile(ipv4_cidr_regex)
return reg.MatchString(wl.IP)
}
func GetFolders(cfg config.Config) (res []Folder, err error) {
var w = Cfg{Key: "folders"}
if exists, _ := cfg.Db.Get(&w); exists {
err = json.Unmarshal([]byte(w.Value), &res)
affected, err = cfg.Db.ID(w.ID).Update(&w)
return
}
return
}
func InsertOrUpdateFolders(cfg config.Config, folders []Folder) (res string, err error) {
func (wl CfgTrustlist) Verify() bool {
reg := regexp.MustCompile(ipv4_cidr_regex)
return reg.MatchString(wl.IP)
}
func GetSets(cfg config.Config) (res []CfgSet, err error) {
var w = []CfgSet{}
if err := cfg.Db.Find(&w); err == nil {
return w, err
}
return
}
func InsertOrUpdateSets(cfg config.Config, folders []CfgSet) (res string, err error) {
var w = Cfg{Key: "folders"}
if exists, _ := cfg.Db.Get(&w); exists {
resbytes, err := json.Marshal(folders)
@ -84,10 +80,10 @@ func InsertOrUpdateFolders(cfg config.Config, folders []Folder) (res string, err
return
}
func GetZMQ(cfg config.Config, key string) (res ZMQ, err error) {
var w = Cfg{Key: fmt.Sprintf("zmq%s", key)}
if exists, _ := cfg.Db.Get(&w); exists {
err = json.Unmarshal([]byte(w.Value), &res)
func GetZMQ(cfg config.Config) (res []CfgZMQ, err error) {
var w = []CfgZMQ{}
if err = cfg.Db.Find(&w); err == nil {
return w, err
}
return
}
@ -107,24 +103,25 @@ func DiscoverURLS(cfg config.Config, routes []*echo.Route) (Discovery, error) {
return disc, nil
}
type Trustlist struct {
IP string `json:"ip"`
type CfgTrustlist struct {
ID int `xorm:"pk autoincr" json:"-"`
IP string `xorm:"text notnull" json:"ip"`
}
type CfgZMQ struct {
ID int `xorm:"pk autoincr" json:"-"`
Type string `xorm:"text notnull" json:"type"`
Hostname string `xorm:"text notnull" json:"hostname"`
Port int `json:"port"`
Subscription string `json:"subscription"`
}
type Folder struct {
Path string `json:"path"`
Sets []Set `json:"sets"`
}
type ZMQ struct {
Hostname string `json:"hostname"`
Port int `json:"port"`
}
type Set struct {
Type string `json:"type"`
Filename string `json:"filename"`
Regex string `json:"regex"`
type CfgSet struct {
ID int `xorm:"pk autoincr" json:"-"`
Path string `xorm:"text notnull default" json:"path"`
Type string `xorm:"text notnull" json:"type"`
Filename string `xorm:"text notnull" json:"filename"`
Regex string `xorm:"text notnull" json:"regex"`
Enabled bool `xorm:"notnull default true" json:"-"`
}
type Cfg struct {

55
src/models/ip.go
View File

@ -8,31 +8,30 @@ import (
"net"
"time"
"git.paulbsd.com/paulbsd/ipbl/src/api"
"git.paulbsd.com/paulbsd/ipbl/src/config"
)
//var lastday = time.Now().Add(-(time.Hour * 24))
func GetIPs(ctx *context.Context, config *config.Config, limit int) (apimailboxes []*api.IP, err error) {
func GetIPs(ctx *context.Context, config *config.Config, limit int) (apiips []*APIIP, err error) {
var ips []IP
err = config.Db.Limit(limit).Desc("created").Find(&ips)
for _, ml := range ips {
apimailboxes = append(apimailboxes, ml.APIFormat())
apiips = append(apiips, ml.APIFormat())
}
return
}
func GetIPsLast(ctx *context.Context, config *config.Config, interval string) (apimailboxes []*api.IP, err error) {
func GetIPsLast(ctx *context.Context, config *config.Config, interval string) (apiips []*APIIP, err error) {
var ips []IP
err = config.Db.Where("updated >= (now()-?::interval)", interval).GroupBy("ip").Find(&ips)
for _, ml := range ips {
apimailboxes = append(apimailboxes, ml.APIFormat())
apiips = append(apiips, ml.APIFormat())
}
return
}
func GetIP(ctx *context.Context, config *config.Config, ipquery interface{}) (apiip *api.IP, err error) {
func GetIP(ctx *context.Context, config *config.Config, ipquery interface{}) (apiip *APIIP, err error) {
var ip IP
has, err := config.Db.Where("ip = ?", ipquery).Get(&ip)
if !has {
@ -63,22 +62,25 @@ func (i *IP) InsertIP(cfg *config.Config) (num int64, err error) {
func InsertIPBulk(cfg *config.Config, ips *[]IP) (numinserts int64, numupdates int64, numfail int64, err error) {
var iplist []string
hostname := (*ips)[0].Hostname
for _, ip := range *ips {
iplist = append(iplist, ip.IP)
}
var searchips []IP
cfg.Db.In("ip", iplist).Find(&searchips)
cfg.Db.In("ip", iplist).Where("hostname = ?", hostname).Find(&searchips)
var toupdateips []string
for _, ip := range searchips {
toupdateips = append(toupdateips, ip.IP)
}
numupdates, _ = cfg.Db.In("ip", toupdateips).Cols("updated").Update(&IP{})
numupdates, _ = cfg.Db.In("ip", toupdateips).Where("hostname = ?", hostname).Cols("updated").Update(&IP{})
var toinsertip, _ = differ(*ips, searchips)
numinserts, err = cfg.Db.Insert(toinsertip)
Cleanup(cfg)
return
}
@ -102,15 +104,37 @@ func ScanIP(cfg *config.Config) (err error) {
}
}
func (ip *IP) APIFormat() *api.IP {
func Cleanup(cfg *config.Config) (err error) {
results, _ := cfg.Db.Query("select * from ip where ip in (select ip from ip group by ip having count(ip) > 1) and hostname is null order by updated desc;")
if len(results) > 0 {
_, err := cfg.Db.Query("delete from ip where ip in (select ip from ip group by ip having count(ip) > 1) and hostname is null;")
if err != nil {
log.Println("error deleting orphans")
}
}
return
}
func (ip *IP) APIFormat() *APIIP {
if ip == nil {
return nil
}
return &api.IP{
return &APIIP{
IP: ip.IP,
Rdns: ip.Rdns.String,
Src: ip.Src,
Hostname: ip.Hostname,
Hostname: ip.Hostname.String,
}
}
func (ip *APIIP) APIConvert() *IP {
if ip == nil {
return nil
}
return &IP{
IP: ip.IP,
Rdns: sql.NullString{String: ip.Rdns, Valid: true},
Src: ip.Src,
Hostname: sql.NullString{String: ip.Hostname, Valid: true},
}
}
@ -119,7 +143,14 @@ type IP struct {
IP string `xorm:"text notnull unique(ipsrc)" json:"ip"`
Rdns sql.NullString `xorm:"text default" json:"rdns"`
Src string `xorm:"text notnull unique(ipsrc)" json:"src"`
Hostname string `xorm:"text notnull unique(ipsrc)" json:"hostname"`
Hostname sql.NullString `xorm:"text default '' unique(ipsrc)" json:"hostname"`
Created time.Time `xorm:"created notnull" json:"-"`
Updated time.Time `xorm:"updated notnull" json:"-"`
}
type APIIP struct {
IP string `json:"ip"`
Rdns string `json:"rdns"`
Src string `json:"src"`
Hostname string `json:"hostname"`
}

52
src/routers/funcs.go
View File

@ -60,11 +60,15 @@ func RegisterRoutes(e *echo.Echo, ctx *context.Context, cfg *config.Config) {
return Result(c, err, ret)
})
e.POST("/ips", func(c echo.Context) (err error) {
var apiips = []models.APIIP{}
var ips = []models.IP{}
var msg string
err = c.Bind(&ips)
err = c.Bind(&apiips)
if err != nil {
return Result(c, err, ips)
return Result(c, err, apiips)
}
for _, v := range apiips {
ips = append(ips, *v.APIConvert())
}
numinsert, numupdate, _, _ := models.InsertIPBulk(cfg, &ips)
if numinsert > 0 {
@ -86,7 +90,7 @@ func RegisterRoutes(e *echo.Echo, ctx *context.Context, cfg *config.Config) {
return Result(c, err, trustlists)
})
e.POST("/config/trustlist", func(c echo.Context) (err error) {
var cidr models.Trustlist
var cidr models.CfgTrustlist
err = c.Bind(&cidr)
if err == nil && cidr.Verify() {
err = cidr.InsertOrUpdate(*cfg)
@ -96,45 +100,25 @@ func RegisterRoutes(e *echo.Echo, ctx *context.Context, cfg *config.Config) {
})
e.DELETE("/config/trustlist/:ip", func(c echo.Context) (err error) {
var ip = c.Param("ip")
var cidr models.Trustlist
err = cidr.Delete(*cfg, ip)
if err != nil {
return c.JSON(http.StatusOK, "Deleted old CIDR")
}
var cidr models.CfgTrustlist
_, err = cidr.Delete(*cfg, ip)
return
})
e.GET("/config/folders", func(c echo.Context) (err error) {
folders, err := models.GetFolders(*cfg)
return Result(c, err, folders)
e.GET("/config/sets", func(c echo.Context) (err error) {
sets, err := models.GetSets(*cfg)
return Result(c, err, sets)
})
e.POST("/config/folders", func(c echo.Context) (err error) {
var folders []models.Folder
err = c.Bind(&folders)
var sets []models.CfgSet
err = c.Bind(&sets)
if err != nil {
return Result(c, err, "Unable to parse JSON")
}
_, err = models.InsertOrUpdateFolders(*cfg, folders)
return Result(c, err, folders)
_, err = models.InsertOrUpdateSets(*cfg, sets)
return Result(c, err, sets)
})
e.GET("/config/folders", func(c echo.Context) (err error) {
folders, err := models.GetFolders(*cfg)
return Result(c, err, folders)
})
e.POST("/config/folders", func(c echo.Context) (err error) {
var folders []models.Folder
err = c.Bind(&folders)
if err != nil {
return Result(c, err, "Unable to parse JSON")
}
_, err = models.InsertOrUpdateFolders(*cfg, folders)
return Result(c, err, folders)
})
e.GET("/config/zmqps", func(c echo.Context) (err error) {
folders, err := models.GetZMQ(*cfg, "ps")
return Result(c, err, folders)
})
e.GET("/config/zmqrr", func(c echo.Context) (err error) {
folders, err := models.GetZMQ(*cfg, "rr")
e.GET("/config/zmq", func(c echo.Context) (err error) {
folders, err := models.GetZMQ(*cfg)
return Result(c, err, folders)
})
e.GET("/discovery", func(c echo.Context) (err error) {