added parallelization

functions.go

@@ -10,6 +10,7 @@ import (
 	"log"
 	"net/http"
 	"os"
+	"sync"
 
 	"gopkg.in/ini.v1"
 )
@@ -40,9 +41,17 @@ func GetConfig(config *Config) error {
 	if err != nil {
 		return err
 	}
+	globalconfig.Threads, err = globalsection.Key("threads").Int()
+	if err != nil {
+		return err
+	}
 
 	githubconfig.StarsPages = githubsection.Key("stars_pages").String()
-	githubconfig.MaxPagesNumber, err = githubsection.Key("max_pages_number").Int()
+	githubconfig.PageNum, err = githubsection.Key("page_num").Int()
+	if err != nil {
+		return err
+	}
+	githubconfig.MaxPerPage, err = githubsection.Key("max_per_page").Int()
 	if err != nil {
 		return err
 	}
@@ -51,7 +60,6 @@ func GetConfig(config *Config) error {
 	githubconfig.AuthPassword = githubsection.Key("auth_password").String()
 	githubconfig.ContentType = githubsection.Key("content_type").String()
 
-	gogsconfig.UID = -1
 	gogsconfig.Username = gogssection.Key("username").String()
 	gogsconfig.DestUsername = gogssection.Key("dest_username").String()
 	gogsconfig.RepoURLTmpl = gogssection.Key("repo_url_tmpl").String()
@@ -76,9 +84,11 @@ func GetReposFromGitHub(config *Config) ([]GitHubRepo, error) {
 	var repolist []GitHubRepo
 	var repo []GitHubRepo
 
-	for num := 1; num <= config.githubconfig.MaxPagesNumber; num++ {
+	for num := 1; num <= config.githubconfig.PageNum; num++ {
-		url := fmt.Sprintf(config.githubconfig.StarsPages, config.githubconfig.AuthUsername, num)
+		url := fmt.Sprintf(config.githubconfig.StarsPages, config.githubconfig.AuthUsername, config.githubconfig.MaxPerPage, num)
+		fmt.Println(url)
 
+		fmt.Println("Getting GitHub starred repos")
 		resp, err := InvokeGitHub(config, url)
 		if err != nil {
 			return nil, err
@@ -93,11 +103,11 @@ func GetReposFromGitHub(config *Config) ([]GitHubRepo, error) {
 		if err != nil {
 			return nil, err
 		}
-
 		for _, elem := range repo {
 			repolist = append(repolist, elem)
 		}
 	}
+
 	fmt.Println(fmt.Sprintf("%d repositories from Github fetched", len(repolist)))
 	return repolist, nil
 }
@@ -152,7 +162,7 @@ func CheckGogsExistingRepo(config *Config, repo GitHubRepo) (bool, error) {
 	} else if resp.StatusCode == 404 {
 		isexists = false
 	} else {
-		err = errors.New(fmt.Sprintf("Can't determine error, cancelling, error %d in gogs webservice", resp.StatusCode))
+		err = fmt.Errorf("Can't determine error, cancelling, error %d in gogs webservice", resp.StatusCode)
 		return false, err
 	}
 
@@ -188,7 +198,7 @@ func GetGogsUserUID(config *Config) error {
 
 	err = json.Unmarshal(*respbody, &gogsorg)
 	if err != nil {
-		err = errors.New("Failed to parse user ID from gogs webservice, check auth if possible")
+		err = errors.New("Failed to parse user ID from gogs webservice, check auth")
 		return err
 	}
 
@@ -197,13 +207,33 @@ func GetGogsUserUID(config *Config) error {
 	return nil
 }
 
+// RunMigration run threads for migration
+func RunMigration(config *Config, repolist []GitHubRepo) {
+	jobs := make(chan GitHubRepo)
+	done := make(chan bool)
+
+	var wg sync.WaitGroup
+	for thr := range make([]int, config.globalconfig.Threads) {
+		go MigrateReposToGogs(config, &wg, jobs, done, thr)
+	}
+
+	for _, repo := range repolist {
+		jobs <- repo
+	}
+	close(jobs)
+	<-done
+	wg.Wait()
+}
+
 // MigrateReposToGogs migrates input repositories to gogs
-func MigrateReposToGogs(config *Config, repolist []GitHubRepo) error {
+func MigrateReposToGogs(config *Config, wg *sync.WaitGroup, jobs chan GitHubRepo, done chan bool, thr int) error {
-	fmt.Println("Migrate repos to Gogs...")
+	wg.Add(1)
-
+	for {
+		elem, more := <-jobs
+		if more {
 			client := &http.Client{}
+			var resp *http.Response
 
-	for _, elem := range repolist {
 			existingrepo, err := CheckGogsExistingRepo(config, elem)
 			if err != nil {
 				return err
@@ -223,22 +253,26 @@ func MigrateReposToGogs(config *Config, repolist []GitHubRepo) error {
 				client.Timeout = config.globalconfig.RequestTimeout
 
 				fmt.Println(fmt.Sprintf("Migrating repo %s to gogs rest api", elem.Name))
-			resp, err := client.Do(req)
+				resp, err = client.Do(req)
 				if err != nil {
 					return err
 				}
-			fmt.Println(fmt.Sprintf("Repo %s migrated with status code %d", elem.Name, resp.StatusCode))
+				fmt.Println(fmt.Sprintf("Repo %s migrated with status code %d on gogs webservice", elem.Name, resp.StatusCode))
 				if resp.StatusCode != 201 {
-				err = errors.New(fmt.Sprintf("Error when migrating repo %s to gogs, status code %d", elem.Name, resp.StatusCode))
+					err = fmt.Errorf("Error when migrating repo %s to gogs with status code %d on gogs webservice", elem.Name, resp.StatusCode)
 					log.Println(err)
 				}
-
 			} else {
 				fmt.Println(fmt.Sprintf("Not migrating, %s exists in gogs", elem.Name))
 			}
-	}
+		} else {
+			fmt.Println(fmt.Sprintf("All repo migrated on thread num %d", thr))
+			wg.Done()
+			done <- true
 			return nil
 		}
+	}
+}
 
 // Usage displays possible arguments
 func Usage() {

github-to-gogs.go

@@ -23,8 +23,5 @@ func main() {
 		log.Fatal(err)
 	}
 
-	err = MigrateReposToGogs(&config, repolist)
+	RunMigration(&config, repolist)
-	if err != nil {
-		log.Fatal(err)
-	}
 }

types.go

@@ -33,12 +33,14 @@ type Config struct {
 type GlobalConfig struct {
 	RequestTimeout time.Duration
 	UserAgent      string
+	Threads        int
 }
 
 // GitHubConfig is the GitHub config
 type GitHubConfig struct {
 	StarsPages   string
-	MaxPagesNumber int
+	MaxPerPage   int
+	PageNum      int
 	AuthUsername string
 	AuthPassword string
 	ContentType  string