handled automatic group generation

src/config/main.go

@@ -40,48 +40,9 @@ func (config *Config) GetConfig() error {
 	config.Cron = qrzsection.Key("cron").MustString("@every 1h")
 
 	config.URLBase = `http://groupe-frs.hamstation.eu/index_qrz_liste_%s.php`
-	//config.URLBase = "http://groupe-frs.hamstation.eu"
+	config.URLBaseForGroups = "http://groupe-frs.hamstation.eu/bdd/menu_listing_division.php"
 
-	config.QrzGroups = []string{
+	config.QrzGroups = []string{}
-		"1",
-		"2",
-		"3",
-		"9",
-		"103",
-		"104",
-		"107",
-		"119",
-		"13",
-		"14",
-		"146",
-		"147",
-		"15",
-		"156",
-		"16",
-		"161",
-		"163",
-		"18",
-		"188",
-		"214",
-		"233",
-		"25",
-		"26",
-		"29",
-		"30",
-		"31",
-		"32",
-		"34",
-		"43",
-		"44",
-		"49",
-		"54",
-		"64",
-		"66",
-		"76",
-		"79",
-		"84",
-		"97",
-		"98"}
 
 	return nil
 }
@@ -94,6 +55,7 @@ type Config struct {
 	DbUsername       string
 	DbPassword       string
 	URLBase          string
+	URLBaseForGroups string
 	QrzGroups        []string
 	Cron             string
 	Port             int

src/qrz/main.go

@@ -29,7 +29,9 @@ func Run(config config.Config) (err error) {
 		log.Fatalln(err)
 	}
 
-	for _, group := range config.QrzGroups {
+	groups, err := getGroups(config.URLBaseForGroups)
+
+	for _, group := range groups {
 		url := fmt.Sprintf(config.URLBase, group)
 		log.Println(fmt.Sprintf("Processing the %s group with URL %s", group, url))
 
@@ -66,25 +68,32 @@ func Run(config config.Config) (err error) {
 	return
 }
 
-func getGroups(urlbase string) (urls []string, err error) {
+func getGroups(urlbase string) (groups []string, err error) {
-	var client http.Client
+	re1 := regexp.MustCompile(`.*document.write\('(.*)'\).*`)
+	re2 := regexp.MustCompile(`.*_(\d{1,3})\.php$`)
+	xpath := `//center/form/select/*`
 
-	resp, err := client.Get(urlbase)
+	clt := &http.Client{}
+	resp, err := clt.Get(urlbase)
+	a, err := ioutil.ReadAll(resp.Body)
+	retstr := string(a)
+
+	b := re1.FindStringSubmatch(retstr)
+	body := b[1]
+
+	htmlpage, err := htmlquery.Parse(strings.NewReader(body))
 	if err != nil {
-		return nil, err
+		return
 	}
 
-	body, err := ioutil.ReadAll(resp.Body)
+	query := htmlquery.Find(htmlpage, xpath)
-	if err != nil {
+
-		return nil, err
+	for _, opt := range query {
+		rematch := re2.FindStringSubmatch(opt.Attr[0].Val)
+		if len(rematch) > 1 {
+			groups = append(groups, rematch[1])
+		}
 	}
-	defer resp.Body.Close()
-	fmt.Println(string(body))
-
-	htmlpage, err := htmlquery.Parse(strings.NewReader(string(body)))
-
-	q := htmlquery.Find(htmlpage, `//*[@id="menu"]/center[2]/form/select`)
-	fmt.Println(q)
 
 	return
 }