package main import ( "flag" "fmt" "io/ioutil" "net/http" "os" "regexp" "strings" "github.com/antchfx/htmlquery" "github.com/jmoiron/sqlx" "golang.org/x/text/encoding/charmap" "gopkg.in/ini.v1" ) // GetConfig fetch configuration func (config *Config) GetConfig() error { var configfile string flag.Usage = Usage flag.StringVar(&configfile, "configfile", "qrz.ini", "config file to use with qrz section") flag.Parse() cfg, err := ini.Load(configfile) if err != nil { return err } qrzsection := cfg.Section("qrz") config.DbHostname = qrzsection.Key("db_hostname").MustString("localhost") config.DbName = qrzsection.Key("db_name").MustString("database") config.DbUsername = qrzsection.Key("db_username").MustString("username") config.DbPassword = qrzsection.Key("db_password").MustString("password") config.DbSchema = qrzsection.Key("db_schema").MustString("") config.DbTable = qrzsection.Key("db_table").MustString("qrz") config.FrsConfig.URL = qrzsection.Key("frs_url").MustString("") config.FrsConfig.XPath = qrzsection.Key("frs_xpath").MustString("//tr") config.FrsConfig.Regex = qrzsection.Key("frs_regex").MustString(".*") return nil } // GetBody fetch html page func GetBody(config Config) (string, error) { var client http.Client resp, err := client.Get(config.FrsConfig.URL) if err != nil { return "", err } body, err := ioutil.ReadAll(resp.Body) if err != nil { return "", err } defer resp.Body.Close() dec := charmap.ISO8859_1.NewDecoder() output, _ := dec.Bytes(body) bodyString := string(output) return bodyString, nil } // GetFrsEntries get FRS entries from html body func GetFrsEntries(config Config, body string) ([]FrsEntry, error) { var err error var frsentries []FrsEntry re := regexp.MustCompile(config.FrsConfig.Regex) htmlpage, err := htmlquery.Parse(strings.NewReader(body)) if err != nil { return frsentries, err } for _, n := range htmlquery.Find(htmlpage, config.FrsConfig.XPath) { var frs FrsEntry td := htmlquery.Find(n, "//td") if re.MatchString(htmlquery.InnerText(td[0])) { frs.QRZ = strings.Replace(htmlquery.InnerText(td[0]), "'", "\\'", -1) frs.Name = strings.Replace(htmlquery.InnerText(td[1]), "'", "\\'", -1) frs.City = strings.Replace(htmlquery.InnerText(td[2]), "'", "\\'", -1) frs.Dept = strings.Replace(htmlquery.InnerText(td[3]), "'", "\\'", -1) frs.Country = strings.Replace(htmlquery.InnerText(td[4]), "'", "\\'", -1) frsentries = append(frsentries, frs) } } return frsentries, nil } // InsertFrsEntryToDB inserts frs entries to database func InsertFrsEntryToDB(config Config, db sqlx.DB, frsPeople []FrsEntry) error { var err error tx := db.MustBegin() var qrzNum int fmt.Println(fmt.Sprintf("Starting insert %d entries", len(frsPeople))) for _, j := range frsPeople { query := fmt.Sprintf(config.DbInsertStatement, config.DbTable, j.QRZ, j.Name, j.City, j.Dept, j.Country) tx.MustExec(query) qrzNum++ } err = tx.Commit() if err != nil { return err } fmt.Println(fmt.Sprintf("Commited %d entries", qrzNum)) return nil } // Usage displays possible arguments func Usage() { flag.PrintDefaults() os.Exit(1) }