paulbsd/fuelprices
1
0
Fork 0
Code Issues Pull Requests Releases 4 Wiki Activity

refactored fuelprices project

Browse Source
This commit is contained in:
Paul 2020-02-04 07:46:44 +01:00
parent 4aaa9cad3c
commit 812d458657
11 changed files with 275 additions and 274 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
Makefile Normal file
View File

@ -0,0 +1,18 @@
# fuelprices Makefile
GOCMD=go
GOBUILDCMD=${GOCMD} build
GOOPTIONS=-mod=vendor -ldflags="-s -w"
RMCMD=rm
BINNAME=fuelprices
SRCFILES=cmd/fuelprices/*.go
all: build
build:
${GOBUILDCMD} ${GOOPTIONS} ${SRCFILES}
clean:
${RMCMD} -f ${BINNAME}

15
README.md
View File

@ -1,8 +1,7 @@
# fuelprices
## Summary
weather is a small program that fetch weather informations, and store them to influxdb
## Usage
## Summary
fuelprices is a small tool designed to fetch fuel prices from France open data and send them to influxdb
## Howto
@ -10,10 +9,10 @@ fuelprices is a small tool designed to fetch fuel prices from France open data a
### Build
```shell
go build -mod=vendor
make
```
### Sample config in fuelprices.ini
## Sample config in fuelprices.ini
```ini
[fuelprices]
@ -40,14 +39,14 @@ database=database
./fuelprices -configfile fuelprices.ini
```
### Todo
## Todo
- Add tests
## License
```text
Copyright (c) 2019 PaulBSD
Copyright (c) 2020 PaulBSD
All rights reserved.
Redistribution and use in source and binary forms, with or without
@ -60,4 +59,4 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
The views and conclusions contained in the software and documentation are those of the authors and should not be interpreted as representing official policies, either expressed or implied, of the fuelprices project.
```
```

38
fuelprices.go
View File

@ -1,38 +0,0 @@
package main
import (
"log"
)
func main() {
var fpc FuelPricesConfig
var zipfile ZipFile
var xmlfile XMLFile
var prices []Price
var err error
err = fpc.GetConfig()
if err != nil {
log.Fatal(err)
}
err = DownloadFile(&fpc, &zipfile)
if err != nil {
log.Fatal(err)
}
err = ExtractZip(&fpc, &zipfile, &xmlfile)
if err != nil {
log.Fatal(err)
}
err = GetPrices(&fpc, &prices, &xmlfile)
if err != nil {
log.Fatal(err)
}
err = SendToInflux(&fpc, &prices)
if err != nil {
log.Fatal(err)
}
}

191
functions.go
View File

@ -1,191 +0,0 @@
package main
import (
"archive/zip"
"bytes"
"errors"
"flag"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"strconv"
"time"
"github.com/antchfx/xmlquery"
_ "github.com/influxdata/influxdb1-client"
client "github.com/influxdata/influxdb1-client/v2"
"gopkg.in/ini.v1"
)
// GetConfig fetch config from ini file
func (fpc *FuelPricesConfig) GetConfig() error {
flag.Usage = Usage
flag.StringVar(&fpc.ConfigPath, "configfile", "common.ini", "config file to use with fuelprices section")
flag.Parse()
config, err := ini.Load(fpc.ConfigPath)
if err != nil {
return err
}
fuelpricesSection := config.Section("fuelprices")
fpc.RemoteURL = fuelpricesSection.Key("remote_url").MustString("https://donnees.roulez-eco.fr/opendata/instantane")
fpc.RemoteFilename = fuelpricesSection.Key("remote_filename").MustString("PrixCarburants_instantane.xml")
fpc.XPathBase = fuelpricesSection.Key("xpath_base").MustString(".//pdv[@id='%s']/prix[@nom='%s']")
fpc.Table = fuelpricesSection.Key("table").MustString("fuel_price")
fpc.Pos = fuelpricesSection.Key("pos").Strings(",")
if len(fpc.Pos) < 1 {
err := errors.New("No pos defined")
return err
}
fpc.Types = fuelpricesSection.Key("types").Strings(",")
if len(fpc.Types) < 1 {
err := errors.New("No fuel types defined")
return err
}
influxdbSection := config.Section("influxdb")
fpc.InfluxURL = influxdbSection.Key("url").MustString("http://localhost:8086")
fpc.InfluxUser = influxdbSection.Key("username").MustString("username")
fpc.InfluxPass = influxdbSection.Key("password").MustString("password")
fpc.InfluxDB = influxdbSection.Key("database").MustString("me")
if err != nil {
return err
}
return nil
}
// DownloadFile fetch file from webserver
func DownloadFile(fpc *FuelPricesConfig, zipfile *ZipFile) error {
pollTo := 30 * time.Millisecond
client := &http.Client{Timeout: pollTo * time.Second, Transport: &http.Transport{
IdleConnTimeout: pollTo,
DisableCompression: false,
}}
resp, err := client.Get(fpc.RemoteURL)
if err != nil {
return err
}
defer resp.Body.Close()
zipfile.Content, err = ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
time.Sleep(pollTo)
return nil
}
// ExtractZip get the XML file to be processed
func ExtractZip(fpc *FuelPricesConfig, zipfile *ZipFile, xmlfile *XMLFile) error {
unzipped, err := zip.NewReader(bytes.NewReader(zipfile.Content), int64(len(zipfile.Content)))
if err != nil {
return err
}
for _, file := range unzipped.File {
if file.Name == fpc.RemoteFilename {
rc, err := file.Open()
if err != nil {
return err
}
xmlfile.Content, err = ioutil.ReadAll(rc)
rc.Close()
} else {
log.Fatal("File not found")
}
}
return err
}
// GetPrices parses the XML file and get values of prices
func GetPrices(fpc *FuelPricesConfig, prices *[]Price, xmlfile *XMLFile) error {
var xml *xmlquery.Node
var valueattr = "valeur"
file := bytes.NewReader(xmlfile.Content)
xml, err := xmlquery.Parse(file)
if err != nil {
return err
}
for _, station := range fpc.Pos {
for _, fuel := range fpc.Types {
query := fmt.Sprintf(fpc.XPathBase, station, fuel)
list := xmlquery.FindOne(xml, query)
if list != nil {
for _, i := range list.Attr {
if i.Name.Local == valueattr {
if s, err := strconv.ParseFloat(i.Value, 64); err == nil {
*prices = append(*prices, Price{ID: station, Fuel: fuel, Amount: s})
}
}
}
} else {
log.Println(fmt.Sprintf("Fuel type not found for point of sale, skipping. Query : %s", query))
}
}
}
return err
}
// SendToInflux sends time series data to influxdb
func SendToInflux(fpc *FuelPricesConfig, prices *[]Price) error {
httpClient, err := client.NewHTTPClient(client.HTTPConfig{
Addr: fpc.InfluxURL,
Username: fpc.InfluxUser,
Password: fpc.InfluxPass,
})
if err != nil {
return err
}
defer httpClient.Close()
bp, err := client.NewBatchPoints(client.BatchPointsConfig{
Database: fpc.InfluxDB,
})
if err != nil {
return err
}
for _, p := range *prices {
tags := map[string]string{"pdv": p.ID, "fuel": p.Fuel}
fields := map[string]interface{}{"value": p.Amount}
point, _ := client.NewPoint(
fpc.Table,
tags,
fields,
time.Now(),
)
log.Println(point)
bp.AddPoint(point)
err = httpClient.Write(bp)
if err != nil {
return err
}
}
return nil
}
// Usage displays possible arguments
func Usage() {
flag.PrintDefaults()
os.Exit(1)
}

4
go.mod
View File

@ -1,6 +1,6 @@
module fuelprices
module git.paulbsd.com/paulbsd/fuelprices
go 1.12
go 1.13
require (
github.com/antchfx/xmlquery v1.0.0

65
src/config/main.go Normal file
View File

@ -0,0 +1,65 @@
package config
import (
"errors"
"flag"
"git.paulbsd.com/paulbsd/fuelprices/src/utils"
"gopkg.in/ini.v1"
)
// GetConfig fetch config from ini file
func (c *Config) GetConfig() (err error) {
flag.Usage = utils.Usage
flag.StringVar(&c.ConfigPath, "configfile", "common.ini", "config file to use with fuelprices section")
flag.Parse()
config, err := ini.Load(c.ConfigPath)
if err != nil {
return
}
fuelpricesSection := config.Section("fuelprices")
c.RemoteURL = fuelpricesSection.Key("remote_url").MustString("https://donnees.roulez-eco.fr/opendata/instantane")
c.RemoteFilename = fuelpricesSection.Key("remote_filename").MustString("PrixCarburants_instantane.xml")
c.XPathBase = fuelpricesSection.Key("xpath_base").MustString(".//pdv[@id='%s']/prix[@nom='%s']")
c.Table = fuelpricesSection.Key("table").MustString("fuel_price")
c.Pos = fuelpricesSection.Key("pos").Strings(",")
if len(c.Pos) < 1 {
err = errors.New("No pos defined")
return
}
c.Types = fuelpricesSection.Key("types").Strings(",")
if len(c.Types) < 1 {
err = errors.New("No fuel types defined")
return
}
influxdbSection := config.Section("influxdb")
c.InfluxURL = influxdbSection.Key("url").MustString("http://localhost:8086")
c.InfluxUser = influxdbSection.Key("username").MustString("username")
c.InfluxPass = influxdbSection.Key("password").MustString("password")
c.InfluxDB = influxdbSection.Key("database").MustString("me")
if err != nil {
return
}
return nil
}
// Config is the main configuration
type Config struct {
ConfigPath string
RemoteURL string
RemoteFilename string
XPathBase string
Pos []string
Types []string
Table string
InfluxURL string
InfluxUser string
InfluxPass string
InfluxDB string
}

98
src/price/main.go Normal file
View File

@ -0,0 +1,98 @@
package price
import (
"bytes"
"fmt"
"log"
"strconv"
"time"
"git.paulbsd.com/paulbsd/fuelprices/src/config"
"git.paulbsd.com/paulbsd/fuelprices/src/xmlfile"
"github.com/antchfx/xmlquery"
client "github.com/influxdata/influxdb1-client/v2"
)
// GetPrices parses the XML file and get values of prices
func GetPrices(c *config.Config, prices *[]Price, xmlfile *xmlfile.XMLFile) (err error) {
var xml *xmlquery.Node
var valueattr = "valeur"
file := bytes.NewReader(xmlfile.Content)
xml, err = xmlquery.Parse(file)
if err != nil {
return
}
for _, station := range c.Pos {
for _, fuel := range c.Types {
query := fmt.Sprintf(c.XPathBase, station, fuel)
list := xmlquery.FindOne(xml, query)
if list != nil {
for _, i := range list.Attr {
if i.Name.Local == valueattr {
if s, err := strconv.ParseFloat(i.Value, 64); err == nil {
*prices = append(*prices, Price{ID: station, Fuel: fuel, Amount: s})
}
}
}
} else {
log.Println(fmt.Sprintf("Fuel type not found for point of sale, skipping. Query : %s", query))
}
}
}
return
}
// SendToInflux sends time series data to influxdb
func SendToInflux(c *config.Config, prices *[]Price) (err error) {
httpClient, err := client.NewHTTPClient(client.HTTPConfig{
Addr: c.InfluxURL,
Username: c.InfluxUser,
Password: c.InfluxPass,
})
if err != nil {
return
}
defer httpClient.Close()
bp, err := client.NewBatchPoints(client.BatchPointsConfig{
Database: c.InfluxDB,
})
if err != nil {
return
}
for _, p := range *prices {
tags := map[string]string{"pdv": p.ID, "fuel": p.Fuel}
fields := map[string]interface{}{"value": p.Amount}
point, _ := client.NewPoint(
c.Table,
tags,
fields,
time.Now(),
)
log.Println(point)
bp.AddPoint(point)
err = httpClient.Write(bp)
if err != nil {
return
}
}
return nil
}
// Price contains price of points of sale
type Price struct {
ID string
Fuel string
Amount float64
}

12
src/utils/main.go Normal file
View File

@ -0,0 +1,12 @@
package utils
import (
"flag"
"os"
)
// Usage displays possible arguments
func Usage() {
flag.PrintDefaults()
os.Exit(1)
}

6
src/xmlfile/main.go Normal file
View File

@ -0,0 +1,6 @@
package xmlfile
// XMLFile contains prices
type XMLFile struct {
Content []byte
}

67
src/zipfile/main.go Normal file
View File

@ -0,0 +1,67 @@
package zipfile
import (
"archive/zip"
"bytes"
"io/ioutil"
"log"
"net/http"
"time"
"git.paulbsd.com/paulbsd/fuelprices/src/config"
"git.paulbsd.com/paulbsd/fuelprices/src/xmlfile"
)
// DownloadFile fetch file from webserver
func (zipfile *ZipFile) DownloadFile(c *config.Config) (err error) {
pollTo := 30 * time.Millisecond
client := &http.Client{Timeout: pollTo * time.Second, Transport: &http.Transport{
IdleConnTimeout: pollTo,
DisableCompression: false,
}}
resp, err := client.Get(c.RemoteURL)
if err != nil {
return
}
defer resp.Body.Close()
zipfile.Content, err = ioutil.ReadAll(resp.Body)
if err != nil {
return
}
time.Sleep(pollTo)
return nil
}
// ExtractZip get the XML file to be processed
func (zipfile *ZipFile) ExtractZip(c *config.Config, xmlfile *xmlfile.XMLFile) (err error) {
unzipped, err := zip.NewReader(bytes.NewReader(zipfile.Content), int64(len(zipfile.Content)))
if err != nil {
return
}
for _, file := range unzipped.File {
if file.Name == c.RemoteFilename {
rc, err := file.Open()
if err != nil {
return err
}
xmlfile.Content, err = ioutil.ReadAll(rc)
rc.Close()
} else {
log.Fatal("File not found")
}
}
return
}
// ZipFile source zipped file
type ZipFile struct {
Filename string
Filepath string
Content []byte
}

35
types.go
View File

@ -1,35 +0,0 @@
package main
// ZipFile source zipped file
type ZipFile struct {
Filename string
Filepath string
Content []byte
}
// XMLFile contains prices
type XMLFile struct {
Content []byte
}
// FuelPricesConfig is the main configuration
type FuelPricesConfig struct {
ConfigPath string
RemoteURL string
RemoteFilename string
XPathBase string
Pos []string
Types []string
Table string
InfluxURL string
InfluxUser string
InfluxPass string
InfluxDB string
}
// Price contains price of points of sale
type Price struct {
ID string
Fuel string
Amount float64
}