diff --git a/cmd/coronafana/coronafana.go b/cmd/coronafana/coronafana.go index 5a1cb11..c0fa236 100644 --- a/cmd/coronafana/coronafana.go +++ b/cmd/coronafana/coronafana.go @@ -15,6 +15,7 @@ func main() { var cr coronafana.Coronafana cfg.GetConfig() + cr, err := coronafana.GetData(cfg) if err != nil { log.Fatalln(err) @@ -30,6 +31,11 @@ func main() { log.Fatalln(err) } + err = cr.GetMaxDates(cfg, *db) + if err != nil { + log.Fatalln(err) + } + // Processes data for 'Global' _, err = db.Exec(cfg.DbSchemaGlobal) if err != nil { @@ -46,8 +52,10 @@ func main() { if err != nil { log.Fatalln(err) } + err = cr.InsertPaysData(cfg, *db) if err != nil { log.Fatalln(err) } + } diff --git a/src/config/main.go b/src/config/main.go index 2a22c09..f81444b 100644 --- a/src/config/main.go +++ b/src/config/main.go @@ -13,7 +13,7 @@ func (config *Config) GetConfig() (err error) { flag.Usage = utils.Usage - flag.StringVar(&configfile, "configfile", "coronafana.ini", "config file to use with coronafana section") + flag.StringVar(&configfile, "configfile", "coronafana.ini", "Config file to use with coronafana section") flag.Parse() cfg, err := ini.Load(configfile) @@ -21,38 +21,81 @@ func (config *Config) GetConfig() (err error) { return } - coronafanasection := cfg.Section("coronafana") + *config = Config{ + DbHostname: "localhost", + DbName: "database", + DbUsername: "username", + DbPassword: "password", + DbTable: "corona", + CoronaURL: "https://coronavirus.politologue.com/data/coronavirus/coronacsv.aspx?format=json"} - config.DbHostname = coronafanasection.Key("db_hostname").MustString("localhost") - config.DbName = coronafanasection.Key("db_name").MustString("database") - config.DbUsername = coronafanasection.Key("db_username").MustString("username") - config.DbPassword = coronafanasection.Key("db_password").MustString("password") - config.DbTable = coronafanasection.Key("db_table").MustString("corona") - config.Corona.URL = coronafanasection.Key("corona_url").MustString("") + err = cfg.Section("coronafana").MapTo(config) + if err != nil { + return + } - config.DbSchemaGlobal = "CREATE TABLE IF NOT EXISTS coronaglobaldata (`id` int(8) NOT NULL AUTO_INCREMENT, `date` datetime NOT NULL, `infection` int(8) DEFAULT NULL, `deces` int(8) DEFAULT NULL, `guerisons` varchar(50) DEFAULT NULL, `tauxdeces` float(10) DEFAULT NULL, `tauxguerison` float(10) DEFAULT NULL, `tauxinfection` float(10) DEFAULT NULL, PRIMARY KEY (`id`), UNIQUE KEY `date` (`date`) USING BTREE) ENGINE=InnoDB DEFAULT CHARSET=utf8;" + config.DbSchemaGlobal = ` + CREATE TABLE IF NOT EXISTS coronaglobaldata + ( + id int(8) NOT NULL AUTO_INCREMENT, + date datetime NOT NULL, + infection int(8) DEFAULT NULL, + deces int(8) DEFAULT NULL, + guerisons varchar(50) DEFAULT NULL, + tauxdeces float(10) DEFAULT NULL, + tauxguerison float(10) DEFAULT NULL, + tauxinfection float(10) DEFAULT NULL, + PRIMARY KEY (id), + UNIQUE KEY date (date) USING BTREE + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + ` - config.DbInsertGlobal = "INSERT INTO coronaglobaldata (date, infection, deces, guerisons, tauxdeces, tauxguerison, tauxinfection) VALUES (:date, :infection, :deces, :guerisons, :tauxdeces, :tauxguerison, :tauxinfection) ON DUPLICATE KEY UPDATE date=:date,infection=:infection, deces=:deces, guerisons=:guerisons, tauxdeces=:tauxdeces, tauxguerison=:tauxguerison, tauxinfection=:tauxinfection;" + config.DbInsertGlobal = ` + INSERT INTO coronaglobaldata (date, infection, deces, guerisons, tauxdeces, tauxguerison, tauxinfection) + VALUES (:date, :infection, :deces, :guerisons, :tauxdeces, :tauxguerison, :tauxinfection) + ON DUPLICATE KEY UPDATE date=:date,infection=:infection, deces=:deces, guerisons=:guerisons, tauxdeces=:tauxdeces, tauxguerison=:tauxguerison, tauxinfection=:tauxinfection; + ` - config.DbSchemaPays = "CREATE TABLE IF NOT EXISTS coronapaysdata (`id` int(8) NOT NULL AUTO_INCREMENT, `date` datetime NOT NULL, `pays` varchar(50) NOT NULL, `infection` int(8) DEFAULT NULL, `deces` int(8) DEFAULT NULL, `guerisons` varchar(50) DEFAULT NULL, `tauxdeces` float(10) DEFAULT NULL, `tauxguerison` float(10) DEFAULT NULL, `tauxinfection` float(10) DEFAULT NULL, PRIMARY KEY (`id`), UNIQUE KEY `date` (`date`,`pays`) USING BTREE) ENGINE=InnoDB DEFAULT CHARSET=utf8;" + config.DbMaxDateGlobal = `SELECT max(date) FROM coronaglobaldata;` - config.DbInsertPays = "INSERT INTO coronapaysdata (date, pays, infection, deces, guerisons, tauxdeces, tauxguerison, tauxinfection) VALUES (:date, :pays, :infection, :deces, :guerisons, :tauxdeces, :tauxguerison, :tauxinfection) ON DUPLICATE KEY UPDATE date=:date, pays=:pays,infection=:infection, deces=:deces, guerisons=:guerisons, tauxdeces=:tauxdeces, tauxguerison=:tauxguerison, tauxinfection=:tauxinfection;" + config.DbSchemaPays = ` + CREATE TABLE IF NOT EXISTS coronapaysdata + ( + id int(8) NOT NULL AUTO_INCREMENT, + date datetime NOT NULL, + pays varchar(50) NOT NULL, + infection int(8) DEFAULT NULL, + deces int(8) DEFAULT NULL, + guerisons varchar(50) DEFAULT NULL, + tauxdeces float(10) DEFAULT NULL, + tauxguerison float(10) DEFAULT NULL, + tauxinfection float(10) DEFAULT NULL, + PRIMARY KEY (id), + UNIQUE KEY date (date,pays) USING BTREE + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;` + + config.DbInsertPays = ` + INSERT INTO coronapaysdata (date, pays, infection, deces, guerisons, tauxdeces, tauxguerison, tauxinfection) + VALUES (:date, :pays, :infection, :deces, :guerisons, :tauxdeces, :tauxguerison, :tauxinfection) + ON DUPLICATE KEY UPDATE date=:date, pays=:pays,infection=:infection, deces=:deces, guerisons=:guerisons, tauxdeces=:tauxdeces, tauxguerison=:tauxguerison, tauxinfection=:tauxinfection;` + + config.DbMaxDatePays = `SELECT max(date) FROM coronapaysdata;` return } // Config is the global config of g2g type Config struct { - DbHostname string - DbName string - DbUsername string - DbPassword string - DbTable string - DbSchemaGlobal string - DbInsertGlobal string - DbSchemaPays string - DbInsertPays string - Corona struct { - URL string - } + DbHostname string `ini:"db_hostname"` + DbName string `ini:"db_name"` + DbUsername string `ini:"db_username"` + DbPassword string `ini:"db_password"` + DbTable string `ini:"db_table"` + DbSchemaGlobal string + DbInsertGlobal string + DbMaxDateGlobal string + DbSchemaPays string + DbInsertPays string + DbMaxDatePays string + CoronaURL string } diff --git a/src/coronafana/main.go b/src/coronafana/main.go index 64a7997..4f8cefc 100644 --- a/src/coronafana/main.go +++ b/src/coronafana/main.go @@ -2,11 +2,15 @@ package coronafana import ( "encoding/json" + "fmt" "io/ioutil" + "log" "net/http" "strings" + "time" "git.paulbsd.com/paulbsd/coronafana/src/config" + "github.com/jinzhu/gorm" "github.com/jmoiron/sqlx" ) @@ -14,7 +18,9 @@ import ( func GetData(cfg config.Config) (cr Coronafana, err error) { var client http.Client - resp, err := client.Get(cfg.Corona.URL) + log.Println("Getting JSON file ...") + + resp, err := client.Get(cfg.CoronaURL) if err != nil { return } @@ -30,62 +36,122 @@ func GetData(cfg config.Config) (cr Coronafana, err error) { return } + log.Println("JSON file fetched.") + return } // InsertGlobalData insert data to MySQL / MariaDB func (cr Coronafana) InsertGlobalData(cfg config.Config, db sqlx.DB) (err error) { var tx *sqlx.Tx + var i int + + log.Println("Start inserting global data ...") tx = db.MustBegin() txStmtglobal, err := tx.PrepareNamed(cfg.DbInsertGlobal) - for _, dt := range cr.GlobalData { - _, err = txStmtglobal.Exec(&dt) - if err != nil { - return - } + for _, dt := range cr.GlobalData { + if t, _ := GetParsedDate(dt.Date); t.After(cr.MaxDateGlobal) { + _, err = txStmtglobal.Exec(&dt) + if err != nil { + return + } + i++ + } } + err = tx.Commit() if err != nil { return } + if i > 0 { + log.Println(fmt.Sprintf("%d global entries to inserted", i)) + } else { + log.Println("No entry inserted") + } + return } // InsertPaysData insert data to MySQL / MariaDB func (cr Coronafana) InsertPaysData(cfg config.Config, db sqlx.DB) (err error) { var tx *sqlx.Tx + var i int + + log.Println("Start inserting country data ...") tx = db.MustBegin() cr.ReplacePaysQuotes() + txStmtpays, err := tx.PrepareNamed(cfg.DbInsertPays) + for _, dt := range cr.PaysData { - _, err = txStmtpays.Exec(&dt) - if err != nil { - return + if t, _ := GetParsedDate(dt.Date); t.After(cr.MaxDatePays) { + _, err = txStmtpays.Exec(&dt) + if err != nil { + return + } + i++ } } + err = tx.Commit() if err != nil { return } + + if i > 0 { + log.Println(fmt.Sprintf("%d global entries to inserted", i)) + } else { + log.Println("No entry inserted") + } return } // ReplacePaysQuotes escapes quotes in Pays names -func (cr Coronafana) ReplacePaysQuotes() (err error) { - for _, pad := range cr.PaysData { - pad.Pays = strings.Replace(pad.Pays, `'`, `\'`, -1) +func (cr *Coronafana) ReplacePaysQuotes() (err error) { + for _, pdata := range cr.PaysData { + pdata.Pays = strings.Replace(pdata.Pays, `'`, `\'`, -1) } - return err + return +} + +// GetMaxDates get max dates on table +func (cr *Coronafana) GetMaxDates(cfg config.Config, db sqlx.DB) (err error) { + var maxDateGlobal, maxDatePays string + + err = db.Get(&maxDateGlobal, cfg.DbMaxDateGlobal) + if err != nil { + return + } + + err = db.Get(&maxDatePays, cfg.DbMaxDatePays) + + if err != nil { + return + } + + cr.MaxDateGlobal, err = time.Parse("2006-01-02 15:04:05", maxDateGlobal) + cr.MaxDatePays, err = time.Parse("2006-01-02 15:04:05", maxDatePays) + + return +} + +// GetParsedDate returns parsed date as time.Time +func GetParsedDate(date string) (res time.Time, err error) { + res, err = time.Parse("2006-01-02T15:04:05", date) + return } // Coronafana is the main struct type Coronafana struct { - Source string `json:"Source"` - Information string `json:"Information"` - Utilisation string `json:"Utilisation"` - GlobalData []struct { + gorm.Model + Source string `json:"Source"` + Information string `json:"Information"` + Utilisation string `json:"Utilisation"` + MaxDateGlobal time.Time + MaxDatePays time.Time + GlobalData []struct { Date string `json:"Date" gorm:"primary_key" db:"date"` Infection int `json:"Infection" db:"infection"` Deces int `json:"Deces" db:"deces"`