added incremental update using max(date) on existing tables

This commit is contained in:
Paul 2020-03-30 00:07:32 +02:00
parent 6936f7cab7
commit 4c58ca76d2
3 changed files with 158 additions and 41 deletions

View File

@ -15,6 +15,7 @@ func main() {
var cr coronafana.Coronafana var cr coronafana.Coronafana
cfg.GetConfig() cfg.GetConfig()
cr, err := coronafana.GetData(cfg) cr, err := coronafana.GetData(cfg)
if err != nil { if err != nil {
log.Fatalln(err) log.Fatalln(err)
@ -30,6 +31,11 @@ func main() {
log.Fatalln(err) log.Fatalln(err)
} }
err = cr.GetMaxDates(cfg, *db)
if err != nil {
log.Fatalln(err)
}
// Processes data for 'Global' // Processes data for 'Global'
_, err = db.Exec(cfg.DbSchemaGlobal) _, err = db.Exec(cfg.DbSchemaGlobal)
if err != nil { if err != nil {
@ -46,8 +52,10 @@ func main() {
if err != nil { if err != nil {
log.Fatalln(err) log.Fatalln(err)
} }
err = cr.InsertPaysData(cfg, *db) err = cr.InsertPaysData(cfg, *db)
if err != nil { if err != nil {
log.Fatalln(err) log.Fatalln(err)
} }
} }

View File

@ -13,7 +13,7 @@ func (config *Config) GetConfig() (err error) {
flag.Usage = utils.Usage flag.Usage = utils.Usage
flag.StringVar(&configfile, "configfile", "coronafana.ini", "config file to use with coronafana section") flag.StringVar(&configfile, "configfile", "coronafana.ini", "Config file to use with coronafana section")
flag.Parse() flag.Parse()
cfg, err := ini.Load(configfile) cfg, err := ini.Load(configfile)
@ -21,38 +21,81 @@ func (config *Config) GetConfig() (err error) {
return return
} }
coronafanasection := cfg.Section("coronafana") *config = Config{
DbHostname: "localhost",
DbName: "database",
DbUsername: "username",
DbPassword: "password",
DbTable: "corona",
CoronaURL: "https://coronavirus.politologue.com/data/coronavirus/coronacsv.aspx?format=json"}
config.DbHostname = coronafanasection.Key("db_hostname").MustString("localhost") err = cfg.Section("coronafana").MapTo(config)
config.DbName = coronafanasection.Key("db_name").MustString("database") if err != nil {
config.DbUsername = coronafanasection.Key("db_username").MustString("username") return
config.DbPassword = coronafanasection.Key("db_password").MustString("password") }
config.DbTable = coronafanasection.Key("db_table").MustString("corona")
config.Corona.URL = coronafanasection.Key("corona_url").MustString("")
config.DbSchemaGlobal = "CREATE TABLE IF NOT EXISTS coronaglobaldata (`id` int(8) NOT NULL AUTO_INCREMENT, `date` datetime NOT NULL, `infection` int(8) DEFAULT NULL, `deces` int(8) DEFAULT NULL, `guerisons` varchar(50) DEFAULT NULL, `tauxdeces` float(10) DEFAULT NULL, `tauxguerison` float(10) DEFAULT NULL, `tauxinfection` float(10) DEFAULT NULL, PRIMARY KEY (`id`), UNIQUE KEY `date` (`date`) USING BTREE) ENGINE=InnoDB DEFAULT CHARSET=utf8;" config.DbSchemaGlobal = `
CREATE TABLE IF NOT EXISTS coronaglobaldata
(
id int(8) NOT NULL AUTO_INCREMENT,
date datetime NOT NULL,
infection int(8) DEFAULT NULL,
deces int(8) DEFAULT NULL,
guerisons varchar(50) DEFAULT NULL,
tauxdeces float(10) DEFAULT NULL,
tauxguerison float(10) DEFAULT NULL,
tauxinfection float(10) DEFAULT NULL,
PRIMARY KEY (id),
UNIQUE KEY date (date) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
`
config.DbInsertGlobal = "INSERT INTO coronaglobaldata (date, infection, deces, guerisons, tauxdeces, tauxguerison, tauxinfection) VALUES (:date, :infection, :deces, :guerisons, :tauxdeces, :tauxguerison, :tauxinfection) ON DUPLICATE KEY UPDATE date=:date,infection=:infection, deces=:deces, guerisons=:guerisons, tauxdeces=:tauxdeces, tauxguerison=:tauxguerison, tauxinfection=:tauxinfection;" config.DbInsertGlobal = `
INSERT INTO coronaglobaldata (date, infection, deces, guerisons, tauxdeces, tauxguerison, tauxinfection)
VALUES (:date, :infection, :deces, :guerisons, :tauxdeces, :tauxguerison, :tauxinfection)
ON DUPLICATE KEY UPDATE date=:date,infection=:infection, deces=:deces, guerisons=:guerisons, tauxdeces=:tauxdeces, tauxguerison=:tauxguerison, tauxinfection=:tauxinfection;
`
config.DbSchemaPays = "CREATE TABLE IF NOT EXISTS coronapaysdata (`id` int(8) NOT NULL AUTO_INCREMENT, `date` datetime NOT NULL, `pays` varchar(50) NOT NULL, `infection` int(8) DEFAULT NULL, `deces` int(8) DEFAULT NULL, `guerisons` varchar(50) DEFAULT NULL, `tauxdeces` float(10) DEFAULT NULL, `tauxguerison` float(10) DEFAULT NULL, `tauxinfection` float(10) DEFAULT NULL, PRIMARY KEY (`id`), UNIQUE KEY `date` (`date`,`pays`) USING BTREE) ENGINE=InnoDB DEFAULT CHARSET=utf8;" config.DbMaxDateGlobal = `SELECT max(date) FROM coronaglobaldata;`
config.DbInsertPays = "INSERT INTO coronapaysdata (date, pays, infection, deces, guerisons, tauxdeces, tauxguerison, tauxinfection) VALUES (:date, :pays, :infection, :deces, :guerisons, :tauxdeces, :tauxguerison, :tauxinfection) ON DUPLICATE KEY UPDATE date=:date, pays=:pays,infection=:infection, deces=:deces, guerisons=:guerisons, tauxdeces=:tauxdeces, tauxguerison=:tauxguerison, tauxinfection=:tauxinfection;" config.DbSchemaPays = `
CREATE TABLE IF NOT EXISTS coronapaysdata
(
id int(8) NOT NULL AUTO_INCREMENT,
date datetime NOT NULL,
pays varchar(50) NOT NULL,
infection int(8) DEFAULT NULL,
deces int(8) DEFAULT NULL,
guerisons varchar(50) DEFAULT NULL,
tauxdeces float(10) DEFAULT NULL,
tauxguerison float(10) DEFAULT NULL,
tauxinfection float(10) DEFAULT NULL,
PRIMARY KEY (id),
UNIQUE KEY date (date,pays) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`
config.DbInsertPays = `
INSERT INTO coronapaysdata (date, pays, infection, deces, guerisons, tauxdeces, tauxguerison, tauxinfection)
VALUES (:date, :pays, :infection, :deces, :guerisons, :tauxdeces, :tauxguerison, :tauxinfection)
ON DUPLICATE KEY UPDATE date=:date, pays=:pays,infection=:infection, deces=:deces, guerisons=:guerisons, tauxdeces=:tauxdeces, tauxguerison=:tauxguerison, tauxinfection=:tauxinfection;`
config.DbMaxDatePays = `SELECT max(date) FROM coronapaysdata;`
return return
} }
// Config is the global config of g2g // Config is the global config of g2g
type Config struct { type Config struct {
DbHostname string DbHostname string `ini:"db_hostname"`
DbName string DbName string `ini:"db_name"`
DbUsername string DbUsername string `ini:"db_username"`
DbPassword string DbPassword string `ini:"db_password"`
DbTable string DbTable string `ini:"db_table"`
DbSchemaGlobal string DbSchemaGlobal string
DbInsertGlobal string DbInsertGlobal string
DbMaxDateGlobal string
DbSchemaPays string DbSchemaPays string
DbInsertPays string DbInsertPays string
Corona struct { DbMaxDatePays string
URL string CoronaURL string
}
} }

View File

@ -2,11 +2,15 @@ package coronafana
import ( import (
"encoding/json" "encoding/json"
"fmt"
"io/ioutil" "io/ioutil"
"log"
"net/http" "net/http"
"strings" "strings"
"time"
"git.paulbsd.com/paulbsd/coronafana/src/config" "git.paulbsd.com/paulbsd/coronafana/src/config"
"github.com/jinzhu/gorm"
"github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx"
) )
@ -14,7 +18,9 @@ import (
func GetData(cfg config.Config) (cr Coronafana, err error) { func GetData(cfg config.Config) (cr Coronafana, err error) {
var client http.Client var client http.Client
resp, err := client.Get(cfg.Corona.URL) log.Println("Getting JSON file ...")
resp, err := client.Get(cfg.CoronaURL)
if err != nil { if err != nil {
return return
} }
@ -30,61 +36,121 @@ func GetData(cfg config.Config) (cr Coronafana, err error) {
return return
} }
log.Println("JSON file fetched.")
return return
} }
// InsertGlobalData insert data to MySQL / MariaDB // InsertGlobalData insert data to MySQL / MariaDB
func (cr Coronafana) InsertGlobalData(cfg config.Config, db sqlx.DB) (err error) { func (cr Coronafana) InsertGlobalData(cfg config.Config, db sqlx.DB) (err error) {
var tx *sqlx.Tx var tx *sqlx.Tx
var i int
log.Println("Start inserting global data ...")
tx = db.MustBegin() tx = db.MustBegin()
txStmtglobal, err := tx.PrepareNamed(cfg.DbInsertGlobal) txStmtglobal, err := tx.PrepareNamed(cfg.DbInsertGlobal)
for _, dt := range cr.GlobalData { for _, dt := range cr.GlobalData {
if t, _ := GetParsedDate(dt.Date); t.After(cr.MaxDateGlobal) {
_, err = txStmtglobal.Exec(&dt) _, err = txStmtglobal.Exec(&dt)
if err != nil { if err != nil {
return return
} }
i++
} }
}
err = tx.Commit() err = tx.Commit()
if err != nil { if err != nil {
return return
} }
if i > 0 {
log.Println(fmt.Sprintf("%d global entries to inserted", i))
} else {
log.Println("No entry inserted")
}
return return
} }
// InsertPaysData insert data to MySQL / MariaDB // InsertPaysData insert data to MySQL / MariaDB
func (cr Coronafana) InsertPaysData(cfg config.Config, db sqlx.DB) (err error) { func (cr Coronafana) InsertPaysData(cfg config.Config, db sqlx.DB) (err error) {
var tx *sqlx.Tx var tx *sqlx.Tx
var i int
log.Println("Start inserting country data ...")
tx = db.MustBegin() tx = db.MustBegin()
cr.ReplacePaysQuotes() cr.ReplacePaysQuotes()
txStmtpays, err := tx.PrepareNamed(cfg.DbInsertPays) txStmtpays, err := tx.PrepareNamed(cfg.DbInsertPays)
for _, dt := range cr.PaysData { for _, dt := range cr.PaysData {
if t, _ := GetParsedDate(dt.Date); t.After(cr.MaxDatePays) {
_, err = txStmtpays.Exec(&dt) _, err = txStmtpays.Exec(&dt)
if err != nil { if err != nil {
return return
} }
i++
} }
}
err = tx.Commit() err = tx.Commit()
if err != nil { if err != nil {
return return
} }
if i > 0 {
log.Println(fmt.Sprintf("%d global entries to inserted", i))
} else {
log.Println("No entry inserted")
}
return return
} }
// ReplacePaysQuotes escapes quotes in Pays names // ReplacePaysQuotes escapes quotes in Pays names
func (cr Coronafana) ReplacePaysQuotes() (err error) { func (cr *Coronafana) ReplacePaysQuotes() (err error) {
for _, pad := range cr.PaysData { for _, pdata := range cr.PaysData {
pad.Pays = strings.Replace(pad.Pays, `'`, `\'`, -1) pdata.Pays = strings.Replace(pdata.Pays, `'`, `\'`, -1)
} }
return err return
}
// GetMaxDates get max dates on table
func (cr *Coronafana) GetMaxDates(cfg config.Config, db sqlx.DB) (err error) {
var maxDateGlobal, maxDatePays string
err = db.Get(&maxDateGlobal, cfg.DbMaxDateGlobal)
if err != nil {
return
}
err = db.Get(&maxDatePays, cfg.DbMaxDatePays)
if err != nil {
return
}
cr.MaxDateGlobal, err = time.Parse("2006-01-02 15:04:05", maxDateGlobal)
cr.MaxDatePays, err = time.Parse("2006-01-02 15:04:05", maxDatePays)
return
}
// GetParsedDate returns parsed date as time.Time
func GetParsedDate(date string) (res time.Time, err error) {
res, err = time.Parse("2006-01-02T15:04:05", date)
return
} }
// Coronafana is the main struct // Coronafana is the main struct
type Coronafana struct { type Coronafana struct {
gorm.Model
Source string `json:"Source"` Source string `json:"Source"`
Information string `json:"Information"` Information string `json:"Information"`
Utilisation string `json:"Utilisation"` Utilisation string `json:"Utilisation"`
MaxDateGlobal time.Time
MaxDatePays time.Time
GlobalData []struct { GlobalData []struct {
Date string `json:"Date" gorm:"primary_key" db:"date"` Date string `json:"Date" gorm:"primary_key" db:"date"`
Infection int `json:"Infection" db:"infection"` Infection int `json:"Infection" db:"infection"`