added incremental update using max(date) on existing tables

cmd/coronafana/coronafana.go

@@ -15,6 +15,7 @@ func main() {
 	var cr coronafana.Coronafana
 
 	cfg.GetConfig()
+
 	cr, err := coronafana.GetData(cfg)
 	if err != nil {
 		log.Fatalln(err)
@@ -30,6 +31,11 @@ func main() {
 		log.Fatalln(err)
 	}
 
+	err = cr.GetMaxDates(cfg, *db)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
 	// Processes data for 'Global'
 	_, err = db.Exec(cfg.DbSchemaGlobal)
 	if err != nil {
@@ -46,8 +52,10 @@ func main() {
 	if err != nil {
 		log.Fatalln(err)
 	}
+
 	err = cr.InsertPaysData(cfg, *db)
 	if err != nil {
 		log.Fatalln(err)
 	}
+
 }

src/config/main.go

@@ -13,7 +13,7 @@ func (config *Config) GetConfig() (err error) {
 
 	flag.Usage = utils.Usage
 
-	flag.StringVar(&configfile, "configfile", "coronafana.ini", "config file to use with coronafana section")
+	flag.StringVar(&configfile, "configfile", "coronafana.ini", "Config file to use with coronafana section")
 	flag.Parse()
 
 	cfg, err := ini.Load(configfile)
@@ -21,38 +21,81 @@ func (config *Config) GetConfig() (err error) {
 		return
 	}
 
-	coronafanasection := cfg.Section("coronafana")
+	*config = Config{
+		DbHostname: "localhost",
+		DbName:     "database",
+		DbUsername: "username",
+		DbPassword: "password",
+		DbTable:    "corona",
+		CoronaURL:  "https://coronavirus.politologue.com/data/coronavirus/coronacsv.aspx?format=json"}
 
-	config.DbHostname = coronafanasection.Key("db_hostname").MustString("localhost")
+	err = cfg.Section("coronafana").MapTo(config)
-	config.DbName = coronafanasection.Key("db_name").MustString("database")
+	if err != nil {
-	config.DbUsername = coronafanasection.Key("db_username").MustString("username")
+		return
-	config.DbPassword = coronafanasection.Key("db_password").MustString("password")
+	}
-	config.DbTable = coronafanasection.Key("db_table").MustString("corona")
-	config.Corona.URL = coronafanasection.Key("corona_url").MustString("")
 
-	config.DbSchemaGlobal = "CREATE TABLE IF NOT EXISTS coronaglobaldata 	(`id` int(8) NOT NULL AUTO_INCREMENT, `date` datetime NOT NULL, `infection` int(8) DEFAULT NULL, `deces` int(8) DEFAULT NULL, `guerisons` varchar(50) DEFAULT NULL, `tauxdeces` float(10) DEFAULT NULL, `tauxguerison` float(10) DEFAULT NULL, `tauxinfection` float(10) DEFAULT NULL, PRIMARY KEY (`id`), UNIQUE KEY `date` (`date`) USING BTREE) ENGINE=InnoDB DEFAULT CHARSET=utf8;"
+	config.DbSchemaGlobal = `
+		CREATE TABLE IF NOT EXISTS coronaglobaldata
+		(
+			id int(8) NOT NULL AUTO_INCREMENT,
+			date datetime NOT NULL,
+			infection int(8) DEFAULT NULL,
+			deces int(8) DEFAULT NULL,
+			guerisons varchar(50) DEFAULT NULL,
+			tauxdeces float(10) DEFAULT NULL,
+			tauxguerison float(10) DEFAULT NULL,
+			tauxinfection float(10) DEFAULT NULL,
+			PRIMARY KEY (id),
+			UNIQUE KEY date (date) USING BTREE
+		) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+		`
 
-	config.DbInsertGlobal = "INSERT INTO coronaglobaldata (date, infection, deces, guerisons, tauxdeces, tauxguerison, tauxinfection) VALUES (:date, :infection, :deces, :guerisons, :tauxdeces, :tauxguerison, :tauxinfection) ON DUPLICATE KEY UPDATE date=:date,infection=:infection, deces=:deces, guerisons=:guerisons, tauxdeces=:tauxdeces, tauxguerison=:tauxguerison, tauxinfection=:tauxinfection;"
+	config.DbInsertGlobal = `
+		INSERT INTO coronaglobaldata (date, infection, deces, guerisons, tauxdeces, tauxguerison, tauxinfection)
+		VALUES (:date, :infection, :deces, :guerisons, :tauxdeces, :tauxguerison, :tauxinfection)
+		ON DUPLICATE KEY UPDATE date=:date,infection=:infection, deces=:deces, guerisons=:guerisons, tauxdeces=:tauxdeces, tauxguerison=:tauxguerison, tauxinfection=:tauxinfection;
+	`
 
-	config.DbSchemaPays = "CREATE TABLE IF NOT EXISTS coronapaysdata (`id` int(8) NOT NULL AUTO_INCREMENT, `date` datetime NOT NULL, `pays` varchar(50) NOT NULL, `infection` int(8) DEFAULT NULL, `deces` int(8) DEFAULT NULL, `guerisons` varchar(50) DEFAULT NULL, `tauxdeces` float(10) DEFAULT NULL, `tauxguerison` float(10) DEFAULT NULL, `tauxinfection` float(10) DEFAULT NULL, PRIMARY KEY (`id`), UNIQUE KEY `date` (`date`,`pays`) USING BTREE) ENGINE=InnoDB DEFAULT CHARSET=utf8;"
+	config.DbMaxDateGlobal = `SELECT max(date) FROM coronaglobaldata;`
 
-	config.DbInsertPays = "INSERT INTO coronapaysdata (date, pays, infection, deces, guerisons, tauxdeces, tauxguerison, tauxinfection) VALUES (:date, :pays, :infection, :deces, :guerisons, :tauxdeces, :tauxguerison, :tauxinfection) ON DUPLICATE KEY UPDATE date=:date, pays=:pays,infection=:infection, deces=:deces, guerisons=:guerisons, tauxdeces=:tauxdeces, tauxguerison=:tauxguerison, tauxinfection=:tauxinfection;"
+	config.DbSchemaPays = `
+		CREATE TABLE IF NOT EXISTS coronapaysdata
+		(
+			id int(8) NOT NULL AUTO_INCREMENT,
+			date datetime NOT NULL,
+			pays varchar(50) NOT NULL,
+			infection int(8) DEFAULT NULL,
+			deces int(8) DEFAULT NULL,
+			guerisons varchar(50) DEFAULT NULL,
+			tauxdeces float(10) DEFAULT NULL,
+			tauxguerison float(10) DEFAULT NULL,
+			tauxinfection float(10) DEFAULT NULL,
+			PRIMARY KEY (id),
+			UNIQUE KEY date (date,pays) USING BTREE
+		) ENGINE=InnoDB DEFAULT CHARSET=utf8;`
+
+	config.DbInsertPays = `
+		INSERT INTO coronapaysdata (date, pays, infection, deces, guerisons, tauxdeces, tauxguerison, tauxinfection)
+		VALUES (:date, :pays, :infection, :deces, :guerisons, :tauxdeces, :tauxguerison, :tauxinfection)
+		ON DUPLICATE KEY UPDATE date=:date, pays=:pays,infection=:infection, deces=:deces, guerisons=:guerisons, tauxdeces=:tauxdeces, tauxguerison=:tauxguerison, tauxinfection=:tauxinfection;`
+
+	config.DbMaxDatePays = `SELECT max(date) FROM coronapaysdata;`
 
 	return
 }
 
 // Config is the global config of g2g
 type Config struct {
-	DbHostname     string
+	DbHostname      string `ini:"db_hostname"`
-	DbName         string
+	DbName          string `ini:"db_name"`
-	DbUsername     string
+	DbUsername      string `ini:"db_username"`
-	DbPassword     string
+	DbPassword      string `ini:"db_password"`
-	DbTable        string
+	DbTable         string `ini:"db_table"`
-	DbSchemaGlobal string
+	DbSchemaGlobal  string
-	DbInsertGlobal string
+	DbInsertGlobal  string
-	DbSchemaPays   string
+	DbMaxDateGlobal string
-	DbInsertPays   string
+	DbSchemaPays    string
-	Corona         struct {
+	DbInsertPays    string
-		URL string
+	DbMaxDatePays   string
-	}
+	CoronaURL       string
 }

src/coronafana/main.go

@@ -2,11 +2,15 @@ package coronafana
 
 import (
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"strings"
+	"time"
 
 	"git.paulbsd.com/paulbsd/coronafana/src/config"
+	"github.com/jinzhu/gorm"
 	"github.com/jmoiron/sqlx"
 )
 
@@ -14,7 +18,9 @@ import (
 func GetData(cfg config.Config) (cr Coronafana, err error) {
 	var client http.Client
 
-	resp, err := client.Get(cfg.Corona.URL)
+	log.Println("Getting JSON file ...")
+
+	resp, err := client.Get(cfg.CoronaURL)
 	if err != nil {
 		return
 	}
@@ -30,62 +36,122 @@ func GetData(cfg config.Config) (cr Coronafana, err error) {
 		return
 	}
 
+	log.Println("JSON file fetched.")
+
 	return
 }
 
 // InsertGlobalData insert data to MySQL / MariaDB
 func (cr Coronafana) InsertGlobalData(cfg config.Config, db sqlx.DB) (err error) {
 	var tx *sqlx.Tx
+	var i int
+
+	log.Println("Start inserting global data ...")
 	tx = db.MustBegin()
 	txStmtglobal, err := tx.PrepareNamed(cfg.DbInsertGlobal)
-	for _, dt := range cr.GlobalData {
-		_, err = txStmtglobal.Exec(&dt)
-		if err != nil {
-			return
-		}
 
+	for _, dt := range cr.GlobalData {
+		if t, _ := GetParsedDate(dt.Date); t.After(cr.MaxDateGlobal) {
+			_, err = txStmtglobal.Exec(&dt)
+			if err != nil {
+				return
+			}
+			i++
+		}
 	}
+
 	err = tx.Commit()
 	if err != nil {
 		return
 	}
 
+	if i > 0 {
+		log.Println(fmt.Sprintf("%d global entries to inserted", i))
+	} else {
+		log.Println("No entry inserted")
+	}
+
 	return
 }
 
 // InsertPaysData insert data to MySQL / MariaDB
 func (cr Coronafana) InsertPaysData(cfg config.Config, db sqlx.DB) (err error) {
 	var tx *sqlx.Tx
+	var i int
+
+	log.Println("Start inserting country data ...")
 	tx = db.MustBegin()
 	cr.ReplacePaysQuotes()
+
 	txStmtpays, err := tx.PrepareNamed(cfg.DbInsertPays)
+
 	for _, dt := range cr.PaysData {
-		_, err = txStmtpays.Exec(&dt)
+		if t, _ := GetParsedDate(dt.Date); t.After(cr.MaxDatePays) {
-		if err != nil {
+			_, err = txStmtpays.Exec(&dt)
-			return
+			if err != nil {
+				return
+			}
+			i++
 		}
 	}
+
 	err = tx.Commit()
 	if err != nil {
 		return
 	}
+
+	if i > 0 {
+		log.Println(fmt.Sprintf("%d global entries to inserted", i))
+	} else {
+		log.Println("No entry inserted")
+	}
 	return
 }
 
 // ReplacePaysQuotes escapes quotes in Pays names
-func (cr Coronafana) ReplacePaysQuotes() (err error) {
+func (cr *Coronafana) ReplacePaysQuotes() (err error) {
-	for _, pad := range cr.PaysData {
+	for _, pdata := range cr.PaysData {
-		pad.Pays = strings.Replace(pad.Pays, `'`, `\'`, -1)
+		pdata.Pays = strings.Replace(pdata.Pays, `'`, `\'`, -1)
 	}
-	return err
+	return
+}
+
+// GetMaxDates get max dates on table
+func (cr *Coronafana) GetMaxDates(cfg config.Config, db sqlx.DB) (err error) {
+	var maxDateGlobal, maxDatePays string
+
+	err = db.Get(&maxDateGlobal, cfg.DbMaxDateGlobal)
+	if err != nil {
+		return
+	}
+
+	err = db.Get(&maxDatePays, cfg.DbMaxDatePays)
+
+	if err != nil {
+		return
+	}
+
+	cr.MaxDateGlobal, err = time.Parse("2006-01-02 15:04:05", maxDateGlobal)
+	cr.MaxDatePays, err = time.Parse("2006-01-02 15:04:05", maxDatePays)
+
+	return
+}
+
+// GetParsedDate returns parsed date as time.Time
+func GetParsedDate(date string) (res time.Time, err error) {
+	res, err = time.Parse("2006-01-02T15:04:05", date)
+	return
 }
 
 // Coronafana is the main struct
 type Coronafana struct {
-	Source      string `json:"Source"`
+	gorm.Model
-	Information string `json:"Information"`
+	Source        string `json:"Source"`
-	Utilisation string `json:"Utilisation"`
+	Information   string `json:"Information"`
-	GlobalData  []struct {
+	Utilisation   string `json:"Utilisation"`
+	MaxDateGlobal time.Time
+	MaxDatePays   time.Time
+	GlobalData    []struct {
 		Date          string  `json:"Date" gorm:"primary_key" db:"date"`
 		Infection     int     `json:"Infection" db:"infection"`
 		Deces         int     `json:"Deces" db:"deces"`