125 lines
3.1 KiB
Go
125 lines
3.1 KiB
Go
package main
|
|
|
|
import (
|
|
"flag"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"net/http"
|
|
"os"
|
|
"regexp"
|
|
"strings"
|
|
|
|
"github.com/antchfx/htmlquery"
|
|
"github.com/jmoiron/sqlx"
|
|
"golang.org/x/text/encoding/charmap"
|
|
"gopkg.in/ini.v1"
|
|
)
|
|
|
|
// GetConfig fetch configuration
|
|
func (config *Config) GetConfig() error {
|
|
var configfile string
|
|
|
|
flag.Usage = Usage
|
|
|
|
flag.StringVar(&configfile, "configfile", "qrz.ini", "config file to use with qrz section")
|
|
flag.Parse()
|
|
|
|
cfg, err := ini.Load(configfile)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
qrzsection := cfg.Section("qrz")
|
|
|
|
config.DbHostname = qrzsection.Key("db_hostname").MustString("localhost")
|
|
config.DbName = qrzsection.Key("db_name").MustString("database")
|
|
config.DbUsername = qrzsection.Key("db_username").MustString("username")
|
|
config.DbPassword = qrzsection.Key("db_password").MustString("password")
|
|
config.DbSchema = qrzsection.Key("db_schema").MustString("")
|
|
config.DbTable = qrzsection.Key("db_table").MustString("qrz")
|
|
config.FrsConfig.URL = qrzsection.Key("frs_url").MustString("")
|
|
config.FrsConfig.XPath = qrzsection.Key("frs_xpath").MustString("//tr")
|
|
config.FrsConfig.Regex = qrzsection.Key("frs_regex").MustString(".*")
|
|
|
|
return nil
|
|
}
|
|
|
|
// GetBody fetch html page
|
|
func GetBody(config Config) (string, error) {
|
|
var client http.Client
|
|
|
|
resp, err := client.Get(config.FrsConfig.URL)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
body, err := ioutil.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
dec := charmap.ISO8859_1.NewDecoder()
|
|
output, _ := dec.Bytes(body)
|
|
bodyString := string(output)
|
|
|
|
return bodyString, nil
|
|
}
|
|
|
|
// GetFrsEntries get FRS entries from html body
|
|
func GetFrsEntries(config Config, body string) ([]FrsEntry, error) {
|
|
var err error
|
|
|
|
var frsentries []FrsEntry
|
|
re := regexp.MustCompile(config.FrsConfig.Regex)
|
|
|
|
htmlpage, err := htmlquery.Parse(strings.NewReader(body))
|
|
if err != nil {
|
|
return frsentries, err
|
|
}
|
|
|
|
for _, n := range htmlquery.Find(htmlpage, config.FrsConfig.XPath) {
|
|
var frs FrsEntry
|
|
td := htmlquery.Find(n, "//td")
|
|
if re.MatchString(htmlquery.InnerText(td[0])) {
|
|
frs.QRZ = strings.Replace(htmlquery.InnerText(td[0]), "'", "\\'", -1)
|
|
frs.Name = strings.Replace(htmlquery.InnerText(td[1]), "'", "\\'", -1)
|
|
frs.City = strings.Replace(htmlquery.InnerText(td[2]), "'", "\\'", -1)
|
|
frs.Dept = strings.Replace(htmlquery.InnerText(td[3]), "'", "\\'", -1)
|
|
frs.Country = strings.Replace(htmlquery.InnerText(td[4]), "'", "\\'", -1)
|
|
|
|
frsentries = append(frsentries, frs)
|
|
}
|
|
}
|
|
return frsentries, nil
|
|
}
|
|
|
|
// InsertFrsEntryToDB inserts frs entries to database
|
|
func InsertFrsEntryToDB(config Config, db sqlx.DB, frsPeople []FrsEntry) error {
|
|
var err error
|
|
tx := db.MustBegin()
|
|
var qrzNum int
|
|
|
|
fmt.Println(fmt.Sprintf("Starting insert %d entries", len(frsPeople)))
|
|
|
|
for _, j := range frsPeople {
|
|
query := fmt.Sprintf(config.DbInsertStatement, config.DbTable, j.QRZ, j.Name, j.City, j.Dept, j.Country)
|
|
tx.MustExec(query)
|
|
qrzNum++
|
|
}
|
|
|
|
err = tx.Commit()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fmt.Println(fmt.Sprintf("Commited %d entries", qrzNum))
|
|
|
|
return nil
|
|
}
|
|
|
|
// Usage displays possible arguments
|
|
func Usage() {
|
|
flag.PrintDefaults()
|
|
os.Exit(1)
|
|
}
|