paulbsd/qrz
1
0
Fork 0
Code Issues Pull Requests Releases 14 Wiki Activity

handled automatic group generation

Browse Source
This commit is contained in:
Paul 2021-01-30 12:50:15 +01:00
parent 8fac3c0a57
commit 21dcea6c7f
2 changed files with 39 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
src/config/main.go
View File

@ -40,48 +40,9 @@ func (config *Config) GetConfig() error {
config.Cron = qrzsection.Key("cron").MustString("@every 1h")
config.URLBase = `http://groupe-frs.hamstation.eu/index_qrz_liste_%s.php`
//config.URLBase = "http://groupe-frs.hamstation.eu"
config.URLBaseForGroups = "http://groupe-frs.hamstation.eu/bdd/menu_listing_division.php"
config.QrzGroups = []string{
"1",
"2",
"3",
"9",
"103",
"104",
"107",
"119",
"13",
"14",
"146",
"147",
"15",
"156",
"16",
"161",
"163",
"18",
"188",
"214",
"233",
"25",
"26",
"29",
"30",
"31",
"32",
"34",
"43",
"44",
"49",
"54",
"64",
"66",
"76",
"79",
"84",
"97",
"98"}
config.QrzGroups = []string{}
return nil
}
@ -94,6 +55,7 @@ type Config struct {
DbUsername string
DbPassword string
URLBase string
URLBaseForGroups string
QrzGroups []string
Cron string
Port int

39
src/qrz/main.go
View File

@ -29,7 +29,9 @@ func Run(config config.Config) (err error) {
log.Fatalln(err)
}
for _, group := range config.QrzGroups {
groups, err := getGroups(config.URLBaseForGroups)
for _, group := range groups {
url := fmt.Sprintf(config.URLBase, group)
log.Println(fmt.Sprintf("Processing the %s group with URL %s", group, url))
@ -66,25 +68,32 @@ func Run(config config.Config) (err error) {
return
}
func getGroups(urlbase string) (urls []string, err error) {
var client http.Client
func getGroups(urlbase string) (groups []string, err error) {
re1 := regexp.MustCompile(`.*document.write\('(.*)'\).*`)
re2 := regexp.MustCompile(`.*_(\d{1,3})\.php$`)
xpath := `//center/form/select/*`
resp, err := client.Get(urlbase)
clt := &http.Client{}
resp, err := clt.Get(urlbase)
a, err := ioutil.ReadAll(resp.Body)
retstr := string(a)
b := re1.FindStringSubmatch(retstr)
body := b[1]
htmlpage, err := htmlquery.Parse(strings.NewReader(body))
if err != nil {
return nil, err
return
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
query := htmlquery.Find(htmlpage, xpath)
for _, opt := range query {
rematch := re2.FindStringSubmatch(opt.Attr[0].Val)
if len(rematch) > 1 {
groups = append(groups, rematch[1])
}
}
defer resp.Body.Close()
fmt.Println(string(body))
htmlpage, err := htmlquery.Parse(strings.NewReader(string(body)))
q := htmlquery.Find(htmlpage, `//*[@id="menu"]/center[2]/form/select`)
fmt.Println(q)
return
}