paulbsd/coronafana
1
0
Fork 0
Code Issues Pull Requests Releases 6 Wiki Activity

added incremental update using max(date) on existing tables

Browse Source
This commit is contained in:
Paul 2020-03-30 00:07:32 +02:00
parent 6936f7cab7
commit 4c58ca76d2
3 changed files with 158 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
cmd/coronafana/coronafana.go
View File

@ -15,6 +15,7 @@ func main() {
var cr coronafana.Coronafana
cfg.GetConfig()
cr, err := coronafana.GetData(cfg)
if err != nil {
log.Fatalln(err)
@ -30,6 +31,11 @@ func main() {
log.Fatalln(err)
}
err = cr.GetMaxDates(cfg, *db)
if err != nil {
log.Fatalln(err)
}
// Processes data for 'Global'
_, err = db.Exec(cfg.DbSchemaGlobal)
if err != nil {
@ -46,8 +52,10 @@ func main() {
if err != nil {
log.Fatalln(err)
}
err = cr.InsertPaysData(cfg, *db)
if err != nil {
log.Fatalln(err)
}
}

83
src/config/main.go
View File

@ -13,7 +13,7 @@ func (config *Config) GetConfig() (err error) {
flag.Usage = utils.Usage
flag.StringVar(&configfile, "configfile", "coronafana.ini", "config file to use with coronafana section")
flag.StringVar(&configfile, "configfile", "coronafana.ini", "Config file to use with coronafana section")
flag.Parse()
cfg, err := ini.Load(configfile)
@ -21,38 +21,81 @@ func (config *Config) GetConfig() (err error) {
return
}
coronafanasection := cfg.Section("coronafana")
*config = Config{
DbHostname: "localhost",
DbName: "database",
DbUsername: "username",
DbPassword: "password",
DbTable: "corona",
CoronaURL: "https://coronavirus.politologue.com/data/coronavirus/coronacsv.aspx?format=json"}
config.DbHostname = coronafanasection.Key("db_hostname").MustString("localhost")
config.DbName = coronafanasection.Key("db_name").MustString("database")
config.DbUsername = coronafanasection.Key("db_username").MustString("username")
config.DbPassword = coronafanasection.Key("db_password").MustString("password")
config.DbTable = coronafanasection.Key("db_table").MustString("corona")
config.Corona.URL = coronafanasection.Key("corona_url").MustString("")
err = cfg.Section("coronafana").MapTo(config)
if err != nil {
return
}
config.DbSchemaGlobal = "CREATE TABLE IF NOT EXISTS coronaglobaldata (`id` int(8) NOT NULL AUTO_INCREMENT, `date` datetime NOT NULL, `infection` int(8) DEFAULT NULL, `deces` int(8) DEFAULT NULL, `guerisons` varchar(50) DEFAULT NULL, `tauxdeces` float(10) DEFAULT NULL, `tauxguerison` float(10) DEFAULT NULL, `tauxinfection` float(10) DEFAULT NULL, PRIMARY KEY (`id`), UNIQUE KEY `date` (`date`) USING BTREE) ENGINE=InnoDB DEFAULT CHARSET=utf8;"
config.DbSchemaGlobal = `
CREATE TABLE IF NOT EXISTS coronaglobaldata
(
id int(8) NOT NULL AUTO_INCREMENT,
date datetime NOT NULL,
infection int(8) DEFAULT NULL,
deces int(8) DEFAULT NULL,
guerisons varchar(50) DEFAULT NULL,
tauxdeces float(10) DEFAULT NULL,
tauxguerison float(10) DEFAULT NULL,
tauxinfection float(10) DEFAULT NULL,
PRIMARY KEY (id),
UNIQUE KEY date (date) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
`
config.DbInsertGlobal = "INSERT INTO coronaglobaldata (date, infection, deces, guerisons, tauxdeces, tauxguerison, tauxinfection) VALUES (:date, :infection, :deces, :guerisons, :tauxdeces, :tauxguerison, :tauxinfection) ON DUPLICATE KEY UPDATE date=:date,infection=:infection, deces=:deces, guerisons=:guerisons, tauxdeces=:tauxdeces, tauxguerison=:tauxguerison, tauxinfection=:tauxinfection;"
config.DbInsertGlobal = `
INSERT INTO coronaglobaldata (date, infection, deces, guerisons, tauxdeces, tauxguerison, tauxinfection)
VALUES (:date, :infection, :deces, :guerisons, :tauxdeces, :tauxguerison, :tauxinfection)
ON DUPLICATE KEY UPDATE date=:date,infection=:infection, deces=:deces, guerisons=:guerisons, tauxdeces=:tauxdeces, tauxguerison=:tauxguerison, tauxinfection=:tauxinfection;
`
config.DbSchemaPays = "CREATE TABLE IF NOT EXISTS coronapaysdata (`id` int(8) NOT NULL AUTO_INCREMENT, `date` datetime NOT NULL, `pays` varchar(50) NOT NULL, `infection` int(8) DEFAULT NULL, `deces` int(8) DEFAULT NULL, `guerisons` varchar(50) DEFAULT NULL, `tauxdeces` float(10) DEFAULT NULL, `tauxguerison` float(10) DEFAULT NULL, `tauxinfection` float(10) DEFAULT NULL, PRIMARY KEY (`id`), UNIQUE KEY `date` (`date`,`pays`) USING BTREE) ENGINE=InnoDB DEFAULT CHARSET=utf8;"
config.DbMaxDateGlobal = `SELECT max(date) FROM coronaglobaldata;`
config.DbInsertPays = "INSERT INTO coronapaysdata (date, pays, infection, deces, guerisons, tauxdeces, tauxguerison, tauxinfection) VALUES (:date, :pays, :infection, :deces, :guerisons, :tauxdeces, :tauxguerison, :tauxinfection) ON DUPLICATE KEY UPDATE date=:date, pays=:pays,infection=:infection, deces=:deces, guerisons=:guerisons, tauxdeces=:tauxdeces, tauxguerison=:tauxguerison, tauxinfection=:tauxinfection;"
config.DbSchemaPays = `
CREATE TABLE IF NOT EXISTS coronapaysdata
(
id int(8) NOT NULL AUTO_INCREMENT,
date datetime NOT NULL,
pays varchar(50) NOT NULL,
infection int(8) DEFAULT NULL,
deces int(8) DEFAULT NULL,
guerisons varchar(50) DEFAULT NULL,
tauxdeces float(10) DEFAULT NULL,
tauxguerison float(10) DEFAULT NULL,
tauxinfection float(10) DEFAULT NULL,
PRIMARY KEY (id),
UNIQUE KEY date (date,pays) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`
config.DbInsertPays = `
INSERT INTO coronapaysdata (date, pays, infection, deces, guerisons, tauxdeces, tauxguerison, tauxinfection)
VALUES (:date, :pays, :infection, :deces, :guerisons, :tauxdeces, :tauxguerison, :tauxinfection)
ON DUPLICATE KEY UPDATE date=:date, pays=:pays,infection=:infection, deces=:deces, guerisons=:guerisons, tauxdeces=:tauxdeces, tauxguerison=:tauxguerison, tauxinfection=:tauxinfection;`
config.DbMaxDatePays = `SELECT max(date) FROM coronapaysdata;`
return
}
// Config is the global config of g2g
type Config struct {
DbHostname string
DbName string
DbUsername string
DbPassword string
DbTable string
DbHostname string `ini:"db_hostname"`
DbName string `ini:"db_name"`
DbUsername string `ini:"db_username"`
DbPassword string `ini:"db_password"`
DbTable string `ini:"db_table"`
DbSchemaGlobal string
DbInsertGlobal string
DbMaxDateGlobal string
DbSchemaPays string
DbInsertPays string
Corona struct {
URL string
}
DbMaxDatePays string
CoronaURL string
}

78
src/coronafana/main.go
View File

@ -2,11 +2,15 @@ package coronafana
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"strings"
"time"
"git.paulbsd.com/paulbsd/coronafana/src/config"
"github.com/jinzhu/gorm"
"github.com/jmoiron/sqlx"
)
@ -14,7 +18,9 @@ import (
func GetData(cfg config.Config) (cr Coronafana, err error) {
var client http.Client
resp, err := client.Get(cfg.Corona.URL)
log.Println("Getting JSON file ...")
resp, err := client.Get(cfg.CoronaURL)
if err != nil {
return
}
@ -30,61 +36,121 @@ func GetData(cfg config.Config) (cr Coronafana, err error) {
return
}
log.Println("JSON file fetched.")
return
}
// InsertGlobalData insert data to MySQL / MariaDB
func (cr Coronafana) InsertGlobalData(cfg config.Config, db sqlx.DB) (err error) {
var tx *sqlx.Tx
var i int
log.Println("Start inserting global data ...")
tx = db.MustBegin()
txStmtglobal, err := tx.PrepareNamed(cfg.DbInsertGlobal)
for _, dt := range cr.GlobalData {
if t, _ := GetParsedDate(dt.Date); t.After(cr.MaxDateGlobal) {
_, err = txStmtglobal.Exec(&dt)
if err != nil {
return
}
i++
}
}
err = tx.Commit()
if err != nil {
return
}
if i > 0 {
log.Println(fmt.Sprintf("%d global entries to inserted", i))
} else {
log.Println("No entry inserted")
}
return
}
// InsertPaysData insert data to MySQL / MariaDB
func (cr Coronafana) InsertPaysData(cfg config.Config, db sqlx.DB) (err error) {
var tx *sqlx.Tx
var i int
log.Println("Start inserting country data ...")
tx = db.MustBegin()
cr.ReplacePaysQuotes()
txStmtpays, err := tx.PrepareNamed(cfg.DbInsertPays)
for _, dt := range cr.PaysData {
if t, _ := GetParsedDate(dt.Date); t.After(cr.MaxDatePays) {
_, err = txStmtpays.Exec(&dt)
if err != nil {
return
}
i++
}
}
err = tx.Commit()
if err != nil {
return
}
if i > 0 {
log.Println(fmt.Sprintf("%d global entries to inserted", i))
} else {
log.Println("No entry inserted")
}
return
}
// ReplacePaysQuotes escapes quotes in Pays names
func (cr Coronafana) ReplacePaysQuotes() (err error) {
for _, pad := range cr.PaysData {
pad.Pays = strings.Replace(pad.Pays, `'`, `\'`, -1)
func (cr *Coronafana) ReplacePaysQuotes() (err error) {
for _, pdata := range cr.PaysData {
pdata.Pays = strings.Replace(pdata.Pays, `'`, `\'`, -1)
}
return err
return
}
// GetMaxDates get max dates on table
func (cr *Coronafana) GetMaxDates(cfg config.Config, db sqlx.DB) (err error) {
var maxDateGlobal, maxDatePays string
err = db.Get(&maxDateGlobal, cfg.DbMaxDateGlobal)
if err != nil {
return
}
err = db.Get(&maxDatePays, cfg.DbMaxDatePays)
if err != nil {
return
}
cr.MaxDateGlobal, err = time.Parse("2006-01-02 15:04:05", maxDateGlobal)
cr.MaxDatePays, err = time.Parse("2006-01-02 15:04:05", maxDatePays)
return
}
// GetParsedDate returns parsed date as time.Time
func GetParsedDate(date string) (res time.Time, err error) {
res, err = time.Parse("2006-01-02T15:04:05", date)
return
}
// Coronafana is the main struct
type Coronafana struct {
gorm.Model
Source string `json:"Source"`
Information string `json:"Information"`
Utilisation string `json:"Utilisation"`
MaxDateGlobal time.Time
MaxDatePays time.Time
GlobalData []struct {
Date string `json:"Date" gorm:"primary_key" db:"date"`
Infection int `json:"Infection" db:"infection"`