handled automatic group generation

src/config/main.go

@@ -40,64 +40,26 @@ func (config *Config) GetConfig() error {
 	config.Cron = qrzsection.Key("cron").MustString("@every 1h")
 
 	config.URLBase = `http://groupe-frs.hamstation.eu/index_qrz_liste_%s.php`
-	//config.URLBase = "http://groupe-frs.hamstation.eu"
+	config.URLBaseForGroups = "http://groupe-frs.hamstation.eu/bdd/menu_listing_division.php"
 
-	config.QrzGroups = []string{
+	config.QrzGroups = []string{}
-		"1",
-		"2",
-		"3",
-		"9",
-		"103",
-		"104",
-		"107",
-		"119",
-		"13",
-		"14",
-		"146",
-		"147",
-		"15",
-		"156",
-		"16",
-		"161",
-		"163",
-		"18",
-		"188",
-		"214",
-		"233",
-		"25",
-		"26",
-		"29",
-		"30",
-		"31",
-		"32",
-		"34",
-		"43",
-		"44",
-		"49",
-		"54",
-		"64",
-		"66",
-		"76",
-		"79",
-		"84",
-		"97",
-		"98"}
 
 	return nil
 }
 
 // Config is the global config of qrz
 type Config struct {
-	Db         *xorm.Engine
+	Db               *xorm.Engine
-	DbHostname string
+	DbHostname       string
-	DbName     string
+	DbName           string
-	DbUsername string
+	DbUsername       string
-	DbPassword string
+	DbPassword       string
-	URLBase    string
+	URLBase          string
-	QrzGroups  []string
+	URLBaseForGroups string
-	Cron       string
+	QrzGroups        []string
-	Port       int
+	Cron             string
-	NoFeed     bool
+	Port             int
-	Debug      bool
+	NoFeed           bool
-	Version    string
+	Debug            bool
+	Version          string
 }

src/qrz/main.go

@@ -29,7 +29,9 @@ func Run(config config.Config) (err error) {
 		log.Fatalln(err)
 	}
 
-	for _, group := range config.QrzGroups {
+	groups, err := getGroups(config.URLBaseForGroups)
+
+	for _, group := range groups {
 		url := fmt.Sprintf(config.URLBase, group)
 		log.Println(fmt.Sprintf("Processing the %s group with URL %s", group, url))
 
@@ -66,25 +68,32 @@ func Run(config config.Config) (err error) {
 	return
 }
 
-func getGroups(urlbase string) (urls []string, err error) {
+func getGroups(urlbase string) (groups []string, err error) {
-	var client http.Client
+	re1 := regexp.MustCompile(`.*document.write\('(.*)'\).*`)
+	re2 := regexp.MustCompile(`.*_(\d{1,3})\.php$`)
+	xpath := `//center/form/select/*`
 
-	resp, err := client.Get(urlbase)
+	clt := &http.Client{}
+	resp, err := clt.Get(urlbase)
+	a, err := ioutil.ReadAll(resp.Body)
+	retstr := string(a)
+
+	b := re1.FindStringSubmatch(retstr)
+	body := b[1]
+
+	htmlpage, err := htmlquery.Parse(strings.NewReader(body))
 	if err != nil {
-		return nil, err
+		return
 	}
 
-	body, err := ioutil.ReadAll(resp.Body)
+	query := htmlquery.Find(htmlpage, xpath)
-	if err != nil {
+
-		return nil, err
+	for _, opt := range query {
+		rematch := re2.FindStringSubmatch(opt.Attr[0].Val)
+		if len(rematch) > 1 {
+			groups = append(groups, rematch[1])
+		}
 	}
-	defer resp.Body.Close()
-	fmt.Println(string(body))
-
-	htmlpage, err := htmlquery.Parse(strings.NewReader(string(body)))
-
-	q := htmlquery.Find(htmlpage, `//*[@id="menu"]/center[2]/form/select`)
-	fmt.Println(q)
 
 	return
 }