From 410d4dd8100b35393824a6e384ac0e0053982c1c Mon Sep 17 00:00:00 2001 From: Paul Lecuq Date: Fri, 3 Nov 2023 10:04:48 +0100 Subject: [PATCH] feat: implement new config handling * new config endpoint with v=2 param * fix messages in websockets handlers --- src/database/main.go | 28 ++++++++++++++++ src/models/cfg.go | 77 +++++++++++++++++++++++++++++++++++++++----- src/models/models.go | 1 + src/routers/funcs.go | 8 +++-- src/ws/pubsub.go | 2 +- src/ws/reqrep.go | 2 +- 6 files changed, 106 insertions(+), 12 deletions(-) diff --git a/src/database/main.go b/src/database/main.go index 19e1cdb..c52245d 100644 --- a/src/database/main.go +++ b/src/database/main.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "os" + "strings" "git.paulbsd.com/paulbsd/ipbl/src/config" "git.paulbsd.com/paulbsd/ipbl/src/models" @@ -42,5 +43,32 @@ func Initialize(ctx *context.Context, cfg *config.Config) (err error) { log.Println("Syncing tables") err = models.NewEngine(ctx, cfg) + + migrate(cfg) + + return +} + +func migrate(cfg *config.Config) (err error) { + var a []models.CfgSet + num, err := cfg.Db.FindAndCount(&a) + fmt.Printf("%d sets found\n", num) + if err != nil { + fmt.Println(err) + } + for _, i := range a { + var r = i.Regex + r = strings.Trim(r, "()") + rs := strings.Split(r, "|") + for _, v := range rs { + if v != "" { + n := models.CfgExpr{ + Expr: v, + Set: &i, + } + cfg.Db.Insert(&n) + } + } + } return } diff --git a/src/models/cfg.go b/src/models/cfg.go index 24370fa..daae8c5 100644 --- a/src/models/cfg.go +++ b/src/models/cfg.go @@ -25,25 +25,25 @@ func GetTrustlists(cfg config.Config) (res []string, err error) { return } -func (wl CfgTrustlist) InsertOrUpdate(cfg config.Config) (err error) { +func (tl CfgTrustlist) InsertOrUpdate(cfg config.Config) (err error) { var w = Cfg{Key: "trustlist"} exists, _ := cfg.Db.Get(&w) if exists { existing, _ := GetTrustlists(cfg) for _, j := range existing { - if j == wl.IP { + if j == tl.IP { return fmt.Errorf("ip %s already in config", j) } } - existing = append(existing, wl.IP) + existing = append(existing, tl.IP) w.Value = strings.Join(existing, ",") cfg.Db.ID(w.ID).Update(&w) } return fmt.Errorf("no trustlist updated") } -func (wl CfgTrustlist) Delete(cfg config.Config, ip string) (affected int64, err error) { +func (tl CfgTrustlist) Delete(cfg config.Config, ip string) (affected int64, err error) { var w = CfgTrustlist{IP: ip} exists, _ := cfg.Db.Get(&w) @@ -54,15 +54,28 @@ func (wl CfgTrustlist) Delete(cfg config.Config, ip string) (affected int64, err return } -func (wl CfgTrustlist) Verify() bool { +func (tl CfgTrustlist) Verify() bool { reg := regexp.MustCompile(ipv4_cidr_regex) - return reg.MatchString(wl.IP) + return reg.MatchString(tl.IP) +} + +func (cfgset *CfgSet) BuildRegex(cfg config.Config) { + var rr []CfgExpr + err := cfg.Db.Where("cfgset_id = $1 AND enabled", cfgset.ID).OrderBy("expr").Find(&rr) + if err != nil { + fmt.Println(err) + } + regs := []string{} + for _, v := range rr { + regs = append(regs, v.Expr) + } + cfgset.Regex = fmt.Sprintf("(%s)", strings.Join(regs, "|")) } func GetSets(cfg config.Config) (res []CfgSet, err error) { var w = []CfgSet{} - if err := cfg.Db.Where("enabled is true").Find(&w); err == nil { + if err := cfg.Db.Where("enabled").Find(&w); err == nil { return w, err } return @@ -77,6 +90,40 @@ func GetGlobalConfig(cfg config.Config) (res []Cfg, err error) { return } +func GetAllConfigv2(cfg config.Config) (res CfgAllv2, err error) { + var c = make(map[string]string) + var tmpcfg = []Cfg{} + if err := cfg.Db.Find(&tmpcfg); err == nil { + for _, v := range tmpcfg { + c[v.Key] = v.Value + } + res.Cfg = c + } + + var sets = []CfgSet{} + if err := cfg.Db.Find(&sets); err == nil { + for i, _ := range sets { + sets[i].BuildRegex(cfg) + } + res.Sets = sets + } + + var tl = []CfgTrustlist{} + if err := cfg.Db.Find(&tl); err == nil { + for _, v := range tl { + res.Trustlists = append(res.Trustlists, v.IP) + + } + } + + var ws = []CfgWS{} + if err := cfg.Db.Find(&ws); err == nil { + res.WS = ws + } + + return +} + func InsertOrUpdateSets(cfg config.Config, folders []CfgSet) (res string, err error) { var w = Cfg{Key: "folders"} @@ -118,6 +165,13 @@ func DiscoverURLS(cfg config.Config, routes []*echo.Route) (Discovery, error) { return disc, nil } +type CfgAllv2 struct { + Cfg map[string]string `json:"cfg"` + Sets []CfgSet `json:"sets"` + Trustlists []string `json:"trustlists"` + WS []CfgWS `json:"ws"` +} + type Cfg struct { ID int `xorm:"pk autoincr" json:"-"` Key string `xorm:"text notnull unique" json:"key"` @@ -129,12 +183,19 @@ type CfgSet struct { Path string `xorm:"text notnull default" json:"path"` Src string `xorm:"text notnull" json:"src"` Filename string `xorm:"text notnull" json:"filename"` - Regex string `xorm:"text notnull" json:"regex"` + Regex string `xorm:"-" json:"regex"` Blocktime int64 `xorm:"notnull default 60" json:"blocktime"` TryFail int64 `xorm:"notnull default 5" json:"tryfail"` Enabled bool `xorm:"notnull default true" json:"-"` } +type CfgExpr struct { + ID int `xorm:"pk autoincr" json:"-"` + Expr string `xorm:"text notnull unique(exprindex) index default ''"` + Enabled bool `xorm:"notnull default true" json:"-"` + Set *CfgSet `xorm:"cfgset_id int unique(exprindex) default null"` +} + type CfgTrustlist struct { ID int `xorm:"pk autoincr" json:"-"` IP string `xorm:"text notnull" json:"ip"` diff --git a/src/models/models.go b/src/models/models.go index 4a7f6e5..354951a 100644 --- a/src/models/models.go +++ b/src/models/models.go @@ -22,6 +22,7 @@ func init() { new(AutonomousSystem), new(Cfg), new(CfgSet), + new(CfgExpr), new(CfgTrustlist), new(CfgWS), new(City), diff --git a/src/routers/funcs.go b/src/routers/funcs.go index 56c2f55..538b394 100644 --- a/src/routers/funcs.go +++ b/src/routers/funcs.go @@ -117,8 +117,12 @@ func RegisterRoutes(e *echo.Echo, ctx *context.Context, cfg *config.Config) { return Result(c, err, "") }) e.GET("/config", func(c echo.Context) (err error) { - trustlists, err := models.GetGlobalConfig(*cfg) - return Result(c, err, trustlists) + if c.QueryParam("v") == "2" { + globalconfig, err := models.GetAllConfigv2(*cfg) + return Result(c, err, globalconfig) + } + globalconfig, err := models.GetGlobalConfig(*cfg) + return Result(c, err, globalconfig) }) e.GET("/config/trustlist", func(c echo.Context) (err error) { trustlists, err := models.GetTrustlists(*cfg) diff --git a/src/ws/pubsub.go b/src/ws/pubsub.go index 19d22b4..603458a 100644 --- a/src/ws/pubsub.go +++ b/src/ws/pubsub.go @@ -32,7 +32,7 @@ func HandleWSPS(c echo.Context, cfg *config.Config) (err error) { err = websocket.Message.Receive(ws, "OK") if err != nil { - log.Println(err) + log.Printf("disconnect: %s (from pubsub channel)\n", welcome.Hostname) } for { diff --git a/src/ws/reqrep.go b/src/ws/reqrep.go index 309722d..e300a3c 100644 --- a/src/ws/reqrep.go +++ b/src/ws/reqrep.go @@ -36,7 +36,7 @@ func HandleWSRR(c echo.Context, cfg *config.Config) error { var msg []byte err := websocket.Message.Receive(ws, &msg) if err != nil { - log.Println(err) + log.Printf("disconnect: %s (from reqrep channel)", welcome.Hostname) return }