refactored fuelprices project
This commit is contained in:
parent
4aaa9cad3c
commit
812d458657
18
Makefile
Normal file
18
Makefile
Normal file
@ -0,0 +1,18 @@
|
||||
# fuelprices Makefile
|
||||
|
||||
GOCMD=go
|
||||
GOBUILDCMD=${GOCMD} build
|
||||
GOOPTIONS=-mod=vendor -ldflags="-s -w"
|
||||
|
||||
RMCMD=rm
|
||||
BINNAME=fuelprices
|
||||
|
||||
SRCFILES=cmd/fuelprices/*.go
|
||||
|
||||
all: build
|
||||
|
||||
build:
|
||||
${GOBUILDCMD} ${GOOPTIONS} ${SRCFILES}
|
||||
|
||||
clean:
|
||||
${RMCMD} -f ${BINNAME}
|
||||
13
README.md
13
README.md
@ -1,8 +1,7 @@
|
||||
# fuelprices
|
||||
## Summary
|
||||
weather is a small program that fetch weather informations, and store them to influxdb
|
||||
|
||||
## Usage
|
||||
## Summary
|
||||
|
||||
fuelprices is a small tool designed to fetch fuel prices from France open data and send them to influxdb
|
||||
|
||||
## Howto
|
||||
@ -10,10 +9,10 @@ fuelprices is a small tool designed to fetch fuel prices from France open data a
|
||||
### Build
|
||||
|
||||
```shell
|
||||
go build -mod=vendor
|
||||
make
|
||||
```
|
||||
|
||||
### Sample config in fuelprices.ini
|
||||
## Sample config in fuelprices.ini
|
||||
|
||||
```ini
|
||||
[fuelprices]
|
||||
@ -40,14 +39,14 @@ database=database
|
||||
./fuelprices -configfile fuelprices.ini
|
||||
```
|
||||
|
||||
### Todo
|
||||
## Todo
|
||||
|
||||
- Add tests
|
||||
|
||||
## License
|
||||
|
||||
```text
|
||||
Copyright (c) 2019 PaulBSD
|
||||
Copyright (c) 2020 PaulBSD
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
|
||||
@ -1,38 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
)
|
||||
|
||||
func main() {
|
||||
var fpc FuelPricesConfig
|
||||
var zipfile ZipFile
|
||||
var xmlfile XMLFile
|
||||
var prices []Price
|
||||
var err error
|
||||
|
||||
err = fpc.GetConfig()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
err = DownloadFile(&fpc, &zipfile)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
err = ExtractZip(&fpc, &zipfile, &xmlfile)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
err = GetPrices(&fpc, &prices, &xmlfile)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
err = SendToInflux(&fpc, &prices)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
191
functions.go
191
functions.go
@ -1,191 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/antchfx/xmlquery"
|
||||
_ "github.com/influxdata/influxdb1-client"
|
||||
client "github.com/influxdata/influxdb1-client/v2"
|
||||
"gopkg.in/ini.v1"
|
||||
)
|
||||
|
||||
// GetConfig fetch config from ini file
|
||||
func (fpc *FuelPricesConfig) GetConfig() error {
|
||||
flag.Usage = Usage
|
||||
|
||||
flag.StringVar(&fpc.ConfigPath, "configfile", "common.ini", "config file to use with fuelprices section")
|
||||
flag.Parse()
|
||||
|
||||
config, err := ini.Load(fpc.ConfigPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fuelpricesSection := config.Section("fuelprices")
|
||||
fpc.RemoteURL = fuelpricesSection.Key("remote_url").MustString("https://donnees.roulez-eco.fr/opendata/instantane")
|
||||
fpc.RemoteFilename = fuelpricesSection.Key("remote_filename").MustString("PrixCarburants_instantane.xml")
|
||||
fpc.XPathBase = fuelpricesSection.Key("xpath_base").MustString(".//pdv[@id='%s']/prix[@nom='%s']")
|
||||
fpc.Table = fuelpricesSection.Key("table").MustString("fuel_price")
|
||||
fpc.Pos = fuelpricesSection.Key("pos").Strings(",")
|
||||
if len(fpc.Pos) < 1 {
|
||||
err := errors.New("No pos defined")
|
||||
return err
|
||||
}
|
||||
fpc.Types = fuelpricesSection.Key("types").Strings(",")
|
||||
if len(fpc.Types) < 1 {
|
||||
err := errors.New("No fuel types defined")
|
||||
return err
|
||||
}
|
||||
|
||||
influxdbSection := config.Section("influxdb")
|
||||
fpc.InfluxURL = influxdbSection.Key("url").MustString("http://localhost:8086")
|
||||
fpc.InfluxUser = influxdbSection.Key("username").MustString("username")
|
||||
fpc.InfluxPass = influxdbSection.Key("password").MustString("password")
|
||||
fpc.InfluxDB = influxdbSection.Key("database").MustString("me")
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DownloadFile fetch file from webserver
|
||||
func DownloadFile(fpc *FuelPricesConfig, zipfile *ZipFile) error {
|
||||
pollTo := 30 * time.Millisecond
|
||||
|
||||
client := &http.Client{Timeout: pollTo * time.Second, Transport: &http.Transport{
|
||||
IdleConnTimeout: pollTo,
|
||||
DisableCompression: false,
|
||||
}}
|
||||
|
||||
resp, err := client.Get(fpc.RemoteURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
zipfile.Content, err = ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
time.Sleep(pollTo)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ExtractZip get the XML file to be processed
|
||||
func ExtractZip(fpc *FuelPricesConfig, zipfile *ZipFile, xmlfile *XMLFile) error {
|
||||
unzipped, err := zip.NewReader(bytes.NewReader(zipfile.Content), int64(len(zipfile.Content)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, file := range unzipped.File {
|
||||
if file.Name == fpc.RemoteFilename {
|
||||
rc, err := file.Open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
xmlfile.Content, err = ioutil.ReadAll(rc)
|
||||
rc.Close()
|
||||
} else {
|
||||
log.Fatal("File not found")
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// GetPrices parses the XML file and get values of prices
|
||||
func GetPrices(fpc *FuelPricesConfig, prices *[]Price, xmlfile *XMLFile) error {
|
||||
var xml *xmlquery.Node
|
||||
var valueattr = "valeur"
|
||||
|
||||
file := bytes.NewReader(xmlfile.Content)
|
||||
xml, err := xmlquery.Parse(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, station := range fpc.Pos {
|
||||
for _, fuel := range fpc.Types {
|
||||
|
||||
query := fmt.Sprintf(fpc.XPathBase, station, fuel)
|
||||
list := xmlquery.FindOne(xml, query)
|
||||
|
||||
if list != nil {
|
||||
for _, i := range list.Attr {
|
||||
if i.Name.Local == valueattr {
|
||||
if s, err := strconv.ParseFloat(i.Value, 64); err == nil {
|
||||
*prices = append(*prices, Price{ID: station, Fuel: fuel, Amount: s})
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log.Println(fmt.Sprintf("Fuel type not found for point of sale, skipping. Query : %s", query))
|
||||
}
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// SendToInflux sends time series data to influxdb
|
||||
func SendToInflux(fpc *FuelPricesConfig, prices *[]Price) error {
|
||||
httpClient, err := client.NewHTTPClient(client.HTTPConfig{
|
||||
Addr: fpc.InfluxURL,
|
||||
Username: fpc.InfluxUser,
|
||||
Password: fpc.InfluxPass,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer httpClient.Close()
|
||||
|
||||
bp, err := client.NewBatchPoints(client.BatchPointsConfig{
|
||||
Database: fpc.InfluxDB,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, p := range *prices {
|
||||
|
||||
tags := map[string]string{"pdv": p.ID, "fuel": p.Fuel}
|
||||
fields := map[string]interface{}{"value": p.Amount}
|
||||
|
||||
point, _ := client.NewPoint(
|
||||
fpc.Table,
|
||||
tags,
|
||||
fields,
|
||||
time.Now(),
|
||||
)
|
||||
|
||||
log.Println(point)
|
||||
|
||||
bp.AddPoint(point)
|
||||
err = httpClient.Write(bp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Usage displays possible arguments
|
||||
func Usage() {
|
||||
flag.PrintDefaults()
|
||||
os.Exit(1)
|
||||
}
|
||||
4
go.mod
4
go.mod
@ -1,6 +1,6 @@
|
||||
module fuelprices
|
||||
module git.paulbsd.com/paulbsd/fuelprices
|
||||
|
||||
go 1.12
|
||||
go 1.13
|
||||
|
||||
require (
|
||||
github.com/antchfx/xmlquery v1.0.0
|
||||
|
||||
65
src/config/main.go
Normal file
65
src/config/main.go
Normal file
@ -0,0 +1,65 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"flag"
|
||||
|
||||
"git.paulbsd.com/paulbsd/fuelprices/src/utils"
|
||||
"gopkg.in/ini.v1"
|
||||
)
|
||||
|
||||
// GetConfig fetch config from ini file
|
||||
func (c *Config) GetConfig() (err error) {
|
||||
flag.Usage = utils.Usage
|
||||
|
||||
flag.StringVar(&c.ConfigPath, "configfile", "common.ini", "config file to use with fuelprices section")
|
||||
flag.Parse()
|
||||
|
||||
config, err := ini.Load(c.ConfigPath)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
fuelpricesSection := config.Section("fuelprices")
|
||||
c.RemoteURL = fuelpricesSection.Key("remote_url").MustString("https://donnees.roulez-eco.fr/opendata/instantane")
|
||||
c.RemoteFilename = fuelpricesSection.Key("remote_filename").MustString("PrixCarburants_instantane.xml")
|
||||
c.XPathBase = fuelpricesSection.Key("xpath_base").MustString(".//pdv[@id='%s']/prix[@nom='%s']")
|
||||
c.Table = fuelpricesSection.Key("table").MustString("fuel_price")
|
||||
c.Pos = fuelpricesSection.Key("pos").Strings(",")
|
||||
if len(c.Pos) < 1 {
|
||||
err = errors.New("No pos defined")
|
||||
return
|
||||
}
|
||||
c.Types = fuelpricesSection.Key("types").Strings(",")
|
||||
if len(c.Types) < 1 {
|
||||
err = errors.New("No fuel types defined")
|
||||
return
|
||||
}
|
||||
|
||||
influxdbSection := config.Section("influxdb")
|
||||
c.InfluxURL = influxdbSection.Key("url").MustString("http://localhost:8086")
|
||||
c.InfluxUser = influxdbSection.Key("username").MustString("username")
|
||||
c.InfluxPass = influxdbSection.Key("password").MustString("password")
|
||||
c.InfluxDB = influxdbSection.Key("database").MustString("me")
|
||||
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Config is the main configuration
|
||||
type Config struct {
|
||||
ConfigPath string
|
||||
RemoteURL string
|
||||
RemoteFilename string
|
||||
XPathBase string
|
||||
Pos []string
|
||||
Types []string
|
||||
Table string
|
||||
InfluxURL string
|
||||
InfluxUser string
|
||||
InfluxPass string
|
||||
InfluxDB string
|
||||
}
|
||||
98
src/price/main.go
Normal file
98
src/price/main.go
Normal file
@ -0,0 +1,98 @@
|
||||
package price
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"log"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.paulbsd.com/paulbsd/fuelprices/src/config"
|
||||
"git.paulbsd.com/paulbsd/fuelprices/src/xmlfile"
|
||||
"github.com/antchfx/xmlquery"
|
||||
client "github.com/influxdata/influxdb1-client/v2"
|
||||
)
|
||||
|
||||
// GetPrices parses the XML file and get values of prices
|
||||
func GetPrices(c *config.Config, prices *[]Price, xmlfile *xmlfile.XMLFile) (err error) {
|
||||
var xml *xmlquery.Node
|
||||
var valueattr = "valeur"
|
||||
|
||||
file := bytes.NewReader(xmlfile.Content)
|
||||
xml, err = xmlquery.Parse(file)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
for _, station := range c.Pos {
|
||||
for _, fuel := range c.Types {
|
||||
|
||||
query := fmt.Sprintf(c.XPathBase, station, fuel)
|
||||
list := xmlquery.FindOne(xml, query)
|
||||
|
||||
if list != nil {
|
||||
for _, i := range list.Attr {
|
||||
if i.Name.Local == valueattr {
|
||||
if s, err := strconv.ParseFloat(i.Value, 64); err == nil {
|
||||
*prices = append(*prices, Price{ID: station, Fuel: fuel, Amount: s})
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log.Println(fmt.Sprintf("Fuel type not found for point of sale, skipping. Query : %s", query))
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// SendToInflux sends time series data to influxdb
|
||||
func SendToInflux(c *config.Config, prices *[]Price) (err error) {
|
||||
httpClient, err := client.NewHTTPClient(client.HTTPConfig{
|
||||
Addr: c.InfluxURL,
|
||||
Username: c.InfluxUser,
|
||||
Password: c.InfluxPass,
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
defer httpClient.Close()
|
||||
|
||||
bp, err := client.NewBatchPoints(client.BatchPointsConfig{
|
||||
Database: c.InfluxDB,
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
for _, p := range *prices {
|
||||
|
||||
tags := map[string]string{"pdv": p.ID, "fuel": p.Fuel}
|
||||
fields := map[string]interface{}{"value": p.Amount}
|
||||
|
||||
point, _ := client.NewPoint(
|
||||
c.Table,
|
||||
tags,
|
||||
fields,
|
||||
time.Now(),
|
||||
)
|
||||
|
||||
log.Println(point)
|
||||
|
||||
bp.AddPoint(point)
|
||||
err = httpClient.Write(bp)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Price contains price of points of sale
|
||||
type Price struct {
|
||||
ID string
|
||||
Fuel string
|
||||
Amount float64
|
||||
}
|
||||
12
src/utils/main.go
Normal file
12
src/utils/main.go
Normal file
@ -0,0 +1,12 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"os"
|
||||
)
|
||||
|
||||
// Usage displays possible arguments
|
||||
func Usage() {
|
||||
flag.PrintDefaults()
|
||||
os.Exit(1)
|
||||
}
|
||||
6
src/xmlfile/main.go
Normal file
6
src/xmlfile/main.go
Normal file
@ -0,0 +1,6 @@
|
||||
package xmlfile
|
||||
|
||||
// XMLFile contains prices
|
||||
type XMLFile struct {
|
||||
Content []byte
|
||||
}
|
||||
67
src/zipfile/main.go
Normal file
67
src/zipfile/main.go
Normal file
@ -0,0 +1,67 @@
|
||||
package zipfile
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"git.paulbsd.com/paulbsd/fuelprices/src/config"
|
||||
"git.paulbsd.com/paulbsd/fuelprices/src/xmlfile"
|
||||
)
|
||||
|
||||
// DownloadFile fetch file from webserver
|
||||
func (zipfile *ZipFile) DownloadFile(c *config.Config) (err error) {
|
||||
pollTo := 30 * time.Millisecond
|
||||
|
||||
client := &http.Client{Timeout: pollTo * time.Second, Transport: &http.Transport{
|
||||
IdleConnTimeout: pollTo,
|
||||
DisableCompression: false,
|
||||
}}
|
||||
|
||||
resp, err := client.Get(c.RemoteURL)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
zipfile.Content, err = ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
time.Sleep(pollTo)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ExtractZip get the XML file to be processed
|
||||
func (zipfile *ZipFile) ExtractZip(c *config.Config, xmlfile *xmlfile.XMLFile) (err error) {
|
||||
unzipped, err := zip.NewReader(bytes.NewReader(zipfile.Content), int64(len(zipfile.Content)))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
for _, file := range unzipped.File {
|
||||
if file.Name == c.RemoteFilename {
|
||||
rc, err := file.Open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
xmlfile.Content, err = ioutil.ReadAll(rc)
|
||||
rc.Close()
|
||||
} else {
|
||||
log.Fatal("File not found")
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// ZipFile source zipped file
|
||||
type ZipFile struct {
|
||||
Filename string
|
||||
Filepath string
|
||||
Content []byte
|
||||
}
|
||||
35
types.go
35
types.go
@ -1,35 +0,0 @@
|
||||
package main
|
||||
|
||||
// ZipFile source zipped file
|
||||
type ZipFile struct {
|
||||
Filename string
|
||||
Filepath string
|
||||
Content []byte
|
||||
}
|
||||
|
||||
// XMLFile contains prices
|
||||
type XMLFile struct {
|
||||
Content []byte
|
||||
}
|
||||
|
||||
// FuelPricesConfig is the main configuration
|
||||
type FuelPricesConfig struct {
|
||||
ConfigPath string
|
||||
RemoteURL string
|
||||
RemoteFilename string
|
||||
XPathBase string
|
||||
Pos []string
|
||||
Types []string
|
||||
Table string
|
||||
InfluxURL string
|
||||
InfluxUser string
|
||||
InfluxPass string
|
||||
InfluxDB string
|
||||
}
|
||||
|
||||
// Price contains price of points of sale
|
||||
type Price struct {
|
||||
ID string
|
||||
Fuel string
|
||||
Amount float64
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user