From e47ed6adfe89739c8a667ae7517ae4ad95f7db96 Mon Sep 17 00:00:00 2001
From: Paul Lecuq <paul@paulbsd.com>
Date: Sun, 6 Feb 2022 00:24:10 +0100
Subject: [PATCH] optimized inserts and updates on ip table

---
 src/models/ip.go | 67 +++++++++++++++++++++++++++++++++---------------
 1 file changed, 47 insertions(+), 20 deletions(-)

diff --git a/src/models/ip.go b/src/models/ip.go
index af1d57d..f0b37e3 100644
--- a/src/models/ip.go
+++ b/src/models/ip.go
@@ -6,12 +6,15 @@ import (
 	"fmt"
 	"log"
 	"net"
+	"reflect"
 	"time"
 
 	"git.paulbsd.com/paulbsd/ipbl/src/api"
 	"git.paulbsd.com/paulbsd/ipbl/src/config"
 )
 
+var lastday = time.Now().Add(-(time.Hour * 24))
+
 // GetIPs ...
 func GetIPs(ctx *context.Context, config *config.Config, limit int) (apimailboxes []*api.IP, err error) {
 	var ips []IP
@@ -66,28 +69,24 @@ func (i *IP) InsertIP(cfg *config.Config) (num int64, err error) {
 
 // InsertIPBulk ...
 func InsertIPBulk(cfg *config.Config, ips *[]IP) (numinserts int64, numupdates int64, numfail int64, err error) {
-	lastday := time.Now().Add(-(time.Hour * 24))
+
+	var iplist []string
 	for _, ip := range *ips {
-		s_ip := IP{IP: ip.IP}
-		res, _ := cfg.Db.Get(&s_ip)
-		if res {
-			if !s_ip.Updated.After(lastday) {
-				num, err := cfg.Db.ID(s_ip.ID).Update(&s_ip)
-				if err != nil {
-					numfail++
-					continue
-				}
-				numupdates += num
-			}
-		} else {
-			num, err := cfg.Db.Insert(ip)
-			if err != nil {
-				numfail++
-				continue
-			}
-			numinserts += num
-		}
+		iplist = append(iplist, ip.IP)
 	}
+
+	var searchips []IP
+	cfg.Db.In("ip", iplist).Find(&searchips)
+
+	var toupdateips []string
+	for _, ip := range searchips {
+		toupdateips = append(toupdateips, ip.IP)
+	}
+	cfg.Db.In("ip", toupdateips).Cols("updated").Update(&IP{})
+
+	var toinsertip, _ = differ(*ips, searchips)
+	numinserts, err = cfg.Db.Insert(toinsertip)
+
 	return
 }
 
@@ -131,6 +130,29 @@ func (ip *IP) APIFormat() *api.IP {
 	}
 }
 
+func differ(sl1 []IP, sl2 []IP) (toinsert []IP, err error) {
+	var m = make(map[string]IPDiffer)
+	longslice := append(sl1, sl2...)
+
+	for _, v2 := range longslice {
+		if _, v := m[v2.IP]; !v {
+			m[v2.IP] = IPDiffer{IP: v2, Num: 1}
+		} else {
+			if this, ok := m[v2.IP]; ok {
+				this.Num += 1
+				m[v2.IP] = this
+			}
+		}
+	}
+
+	for _, j := range reflect.ValueOf(m).MapKeys() {
+		if m[j.String()].Num == 1 {
+			toinsert = append(toinsert, m[j.String()].IP)
+		}
+	}
+	return
+}
+
 // IP describe IP objects
 type IP struct {
 	ID      int            `xorm:"pk autoincr" json:"-"`
@@ -140,3 +162,8 @@ type IP struct {
 	Created time.Time      `xorm:"created notnull" json:"-"`
 	Updated time.Time      `xorm:"updated notnull" json:"-"`
 }
+
+type IPDiffer struct {
+	IP  IP
+	Num int
+}