added parallelization

functions.go

@@ -10,6 +10,7 @@ import (
 	"log"
 	"net/http"
 	"os"
+	"sync"
 
 	"gopkg.in/ini.v1"
 )
@@ -40,9 +41,17 @@ func GetConfig(config *Config) error {
 	if err != nil {
 		return err
 	}
+	globalconfig.Threads, err = globalsection.Key("threads").Int()
+	if err != nil {
+		return err
+	}
 
 	githubconfig.StarsPages = githubsection.Key("stars_pages").String()
-	githubconfig.MaxPagesNumber, err = githubsection.Key("max_pages_number").Int()
+	githubconfig.PageNum, err = githubsection.Key("page_num").Int()
+	if err != nil {
+		return err
+	}
+	githubconfig.MaxPerPage, err = githubsection.Key("max_per_page").Int()
 	if err != nil {
 		return err
 	}
@@ -51,7 +60,6 @@ func GetConfig(config *Config) error {
 	githubconfig.AuthPassword = githubsection.Key("auth_password").String()
 	githubconfig.ContentType = githubsection.Key("content_type").String()
 
-	gogsconfig.UID = -1
 	gogsconfig.Username = gogssection.Key("username").String()
 	gogsconfig.DestUsername = gogssection.Key("dest_username").String()
 	gogsconfig.RepoURLTmpl = gogssection.Key("repo_url_tmpl").String()
@@ -76,9 +84,11 @@ func GetReposFromGitHub(config *Config) ([]GitHubRepo, error) {
 	var repolist []GitHubRepo
 	var repo []GitHubRepo
 
-	for num := 1; num <= config.githubconfig.MaxPagesNumber; num++ {
+	for num := 1; num <= config.githubconfig.PageNum; num++ {
-		url := fmt.Sprintf(config.githubconfig.StarsPages, config.githubconfig.AuthUsername, num)
+		url := fmt.Sprintf(config.githubconfig.StarsPages, config.githubconfig.AuthUsername, config.githubconfig.MaxPerPage, num)
+		fmt.Println(url)
 
+		fmt.Println("Getting GitHub starred repos")
 		resp, err := InvokeGitHub(config, url)
 		if err != nil {
 			return nil, err
@@ -93,11 +103,11 @@ func GetReposFromGitHub(config *Config) ([]GitHubRepo, error) {
 		if err != nil {
 			return nil, err
 		}
-
 		for _, elem := range repo {
 			repolist = append(repolist, elem)
 		}
 	}
+
 	fmt.Println(fmt.Sprintf("%d repositories from Github fetched", len(repolist)))
 	return repolist, nil
 }
@@ -152,7 +162,7 @@ func CheckGogsExistingRepo(config *Config, repo GitHubRepo) (bool, error) {
 	} else if resp.StatusCode == 404 {
 		isexists = false
 	} else {
-		err = errors.New(fmt.Sprintf("Can't determine error, cancelling, error %d in gogs webservice", resp.StatusCode))
+		err = fmt.Errorf("Can't determine error, cancelling, error %d in gogs webservice", resp.StatusCode)
 		return false, err
 	}
 
@@ -188,7 +198,7 @@ func GetGogsUserUID(config *Config) error {
 
 	err = json.Unmarshal(*respbody, &gogsorg)
 	if err != nil {
-		err = errors.New("Failed to parse user ID from gogs webservice, check auth if possible")
+		err = errors.New("Failed to parse user ID from gogs webservice, check auth")
 		return err
 	}
 
@@ -197,47 +207,71 @@ func GetGogsUserUID(config *Config) error {
 	return nil
 }
 
+// RunMigration run threads for migration
+func RunMigration(config *Config, repolist []GitHubRepo) {
+	jobs := make(chan GitHubRepo)
+	done := make(chan bool)
+
+	var wg sync.WaitGroup
+	for thr := range make([]int, config.globalconfig.Threads) {
+		go MigrateReposToGogs(config, &wg, jobs, done, thr)
+	}
+
+	for _, repo := range repolist {
+		jobs <- repo
+	}
+	close(jobs)
+	<-done
+	wg.Wait()
+}
+
 // MigrateReposToGogs migrates input repositories to gogs
-func MigrateReposToGogs(config *Config, repolist []GitHubRepo) error {
+func MigrateReposToGogs(config *Config, wg *sync.WaitGroup, jobs chan GitHubRepo, done chan bool, thr int) error {
-	fmt.Println("Migrate repos to Gogs...")
+	wg.Add(1)
+	for {
+		elem, more := <-jobs
+		if more {
+			client := &http.Client{}
+			var resp *http.Response
 
-	client := &http.Client{}
+			existingrepo, err := CheckGogsExistingRepo(config, elem)
-
-	for _, elem := range repolist {
-		existingrepo, err := CheckGogsExistingRepo(config, elem)
-		if err != nil {
-			return err
-		}
-		if !existingrepo {
-			jsondata := fmt.Sprintf(`{"uid" : %d, "repo_name" : "%s" , "mirror" : %v, "clone_addr" : "%s"}`, config.gogsconfig.UID, elem.Name, true, elem.CloneURL)
-
-			req, err := http.NewRequest("POST", config.gogsconfig.MigrateURL, bytes.NewBufferString(jsondata))
 			if err != nil {
 				return err
 			}
+			if !existingrepo {
+				jsondata := fmt.Sprintf(`{"uid" : %d, "repo_name" : "%s" , "mirror" : %v, "clone_addr" : "%s"}`, config.gogsconfig.UID, elem.Name, true, elem.CloneURL)
 
-			req.Header.Add("Content-Type", config.gogsconfig.ContentType)
+				req, err := http.NewRequest("POST", config.gogsconfig.MigrateURL, bytes.NewBufferString(jsondata))
-			req.Header.Set("User-Agent", config.globalconfig.UserAgent)
+				if err != nil {
-			req.Header.Set("Authorization", config.gogsconfig.AuthToken)
+					return err
+				}
 
-			client.Timeout = config.globalconfig.RequestTimeout
+				req.Header.Add("Content-Type", config.gogsconfig.ContentType)
+				req.Header.Set("User-Agent", config.globalconfig.UserAgent)
+				req.Header.Set("Authorization", config.gogsconfig.AuthToken)
 
-			fmt.Println(fmt.Sprintf("Migrating repo %s to gogs rest api", elem.Name))
+				client.Timeout = config.globalconfig.RequestTimeout
-			resp, err := client.Do(req)
+
-			if err != nil {
+				fmt.Println(fmt.Sprintf("Migrating repo %s to gogs rest api", elem.Name))
-				return err
+				resp, err = client.Do(req)
+				if err != nil {
+					return err
+				}
+				fmt.Println(fmt.Sprintf("Repo %s migrated with status code %d on gogs webservice", elem.Name, resp.StatusCode))
+				if resp.StatusCode != 201 {
+					err = fmt.Errorf("Error when migrating repo %s to gogs with status code %d on gogs webservice", elem.Name, resp.StatusCode)
+					log.Println(err)
+				}
+			} else {
+				fmt.Println(fmt.Sprintf("Not migrating, %s exists in gogs", elem.Name))
 			}
-			fmt.Println(fmt.Sprintf("Repo %s migrated with status code %d", elem.Name, resp.StatusCode))
-			if resp.StatusCode != 201 {
-				err = errors.New(fmt.Sprintf("Error when migrating repo %s to gogs, status code %d", elem.Name, resp.StatusCode))
-				log.Println(err)
-			}
-
 		} else {
-			fmt.Println(fmt.Sprintf("Not migrating, %s exists in gogs", elem.Name))
+			fmt.Println(fmt.Sprintf("All repo migrated on thread num %d", thr))
+			wg.Done()
+			done <- true
+			return nil
 		}
 	}
-	return nil
 }
 
 // Usage displays possible arguments

github-to-gogs.go

@@ -23,8 +23,5 @@ func main() {
 		log.Fatal(err)
 	}
 
-	err = MigrateReposToGogs(&config, repolist)
+	RunMigration(&config, repolist)
-	if err != nil {
-		log.Fatal(err)
-	}
 }

types.go

@@ -33,15 +33,17 @@ type Config struct {
 type GlobalConfig struct {
 	RequestTimeout time.Duration
 	UserAgent      string
+	Threads        int
 }
 
 // GitHubConfig is the GitHub config
 type GitHubConfig struct {
-	StarsPages     string
+	StarsPages   string
-	MaxPagesNumber int
+	MaxPerPage   int
-	AuthUsername   string
+	PageNum      int
-	AuthPassword   string
+	AuthUsername string
-	ContentType    string
+	AuthPassword string
+	ContentType  string
 }
 
 // GogsConfig is the Gogs config