From 14fe05f4d5e9305c02a1b45c213cdd7a464744de Mon Sep 17 00:00:00 2001 From: Paul Lecuq Date: Wed, 10 Jul 2019 15:07:41 +0200 Subject: [PATCH] added parallelization --- functions.go | 106 ++++++++++++++++++++++++++++++---------------- github-to-gogs.go | 5 +-- types.go | 12 +++--- 3 files changed, 78 insertions(+), 45 deletions(-) diff --git a/functions.go b/functions.go index 79c81af..d59d145 100644 --- a/functions.go +++ b/functions.go @@ -10,6 +10,7 @@ import ( "log" "net/http" "os" + "sync" "gopkg.in/ini.v1" ) @@ -40,9 +41,17 @@ func GetConfig(config *Config) error { if err != nil { return err } + globalconfig.Threads, err = globalsection.Key("threads").Int() + if err != nil { + return err + } githubconfig.StarsPages = githubsection.Key("stars_pages").String() - githubconfig.MaxPagesNumber, err = githubsection.Key("max_pages_number").Int() + githubconfig.PageNum, err = githubsection.Key("page_num").Int() + if err != nil { + return err + } + githubconfig.MaxPerPage, err = githubsection.Key("max_per_page").Int() if err != nil { return err } @@ -51,7 +60,6 @@ func GetConfig(config *Config) error { githubconfig.AuthPassword = githubsection.Key("auth_password").String() githubconfig.ContentType = githubsection.Key("content_type").String() - gogsconfig.UID = -1 gogsconfig.Username = gogssection.Key("username").String() gogsconfig.DestUsername = gogssection.Key("dest_username").String() gogsconfig.RepoURLTmpl = gogssection.Key("repo_url_tmpl").String() @@ -76,9 +84,11 @@ func GetReposFromGitHub(config *Config) ([]GitHubRepo, error) { var repolist []GitHubRepo var repo []GitHubRepo - for num := 1; num <= config.githubconfig.MaxPagesNumber; num++ { - url := fmt.Sprintf(config.githubconfig.StarsPages, config.githubconfig.AuthUsername, num) + for num := 1; num <= config.githubconfig.PageNum; num++ { + url := fmt.Sprintf(config.githubconfig.StarsPages, config.githubconfig.AuthUsername, config.githubconfig.MaxPerPage, num) + fmt.Println(url) + fmt.Println("Getting GitHub starred repos") resp, err := InvokeGitHub(config, url) if err != nil { return nil, err @@ -93,11 +103,11 @@ func GetReposFromGitHub(config *Config) ([]GitHubRepo, error) { if err != nil { return nil, err } - for _, elem := range repo { repolist = append(repolist, elem) } } + fmt.Println(fmt.Sprintf("%d repositories from Github fetched", len(repolist))) return repolist, nil } @@ -152,7 +162,7 @@ func CheckGogsExistingRepo(config *Config, repo GitHubRepo) (bool, error) { } else if resp.StatusCode == 404 { isexists = false } else { - err = errors.New(fmt.Sprintf("Can't determine error, cancelling, error %d in gogs webservice", resp.StatusCode)) + err = fmt.Errorf("Can't determine error, cancelling, error %d in gogs webservice", resp.StatusCode) return false, err } @@ -188,7 +198,7 @@ func GetGogsUserUID(config *Config) error { err = json.Unmarshal(*respbody, &gogsorg) if err != nil { - err = errors.New("Failed to parse user ID from gogs webservice, check auth if possible") + err = errors.New("Failed to parse user ID from gogs webservice, check auth") return err } @@ -197,47 +207,71 @@ func GetGogsUserUID(config *Config) error { return nil } +// RunMigration run threads for migration +func RunMigration(config *Config, repolist []GitHubRepo) { + jobs := make(chan GitHubRepo) + done := make(chan bool) + + var wg sync.WaitGroup + for thr := range make([]int, config.globalconfig.Threads) { + go MigrateReposToGogs(config, &wg, jobs, done, thr) + } + + for _, repo := range repolist { + jobs <- repo + } + close(jobs) + <-done + wg.Wait() +} + // MigrateReposToGogs migrates input repositories to gogs -func MigrateReposToGogs(config *Config, repolist []GitHubRepo) error { - fmt.Println("Migrate repos to Gogs...") +func MigrateReposToGogs(config *Config, wg *sync.WaitGroup, jobs chan GitHubRepo, done chan bool, thr int) error { + wg.Add(1) + for { + elem, more := <-jobs + if more { + client := &http.Client{} + var resp *http.Response - client := &http.Client{} - - for _, elem := range repolist { - existingrepo, err := CheckGogsExistingRepo(config, elem) - if err != nil { - return err - } - if !existingrepo { - jsondata := fmt.Sprintf(`{"uid" : %d, "repo_name" : "%s" , "mirror" : %v, "clone_addr" : "%s"}`, config.gogsconfig.UID, elem.Name, true, elem.CloneURL) - - req, err := http.NewRequest("POST", config.gogsconfig.MigrateURL, bytes.NewBufferString(jsondata)) + existingrepo, err := CheckGogsExistingRepo(config, elem) if err != nil { return err } + if !existingrepo { + jsondata := fmt.Sprintf(`{"uid" : %d, "repo_name" : "%s" , "mirror" : %v, "clone_addr" : "%s"}`, config.gogsconfig.UID, elem.Name, true, elem.CloneURL) - req.Header.Add("Content-Type", config.gogsconfig.ContentType) - req.Header.Set("User-Agent", config.globalconfig.UserAgent) - req.Header.Set("Authorization", config.gogsconfig.AuthToken) + req, err := http.NewRequest("POST", config.gogsconfig.MigrateURL, bytes.NewBufferString(jsondata)) + if err != nil { + return err + } - client.Timeout = config.globalconfig.RequestTimeout + req.Header.Add("Content-Type", config.gogsconfig.ContentType) + req.Header.Set("User-Agent", config.globalconfig.UserAgent) + req.Header.Set("Authorization", config.gogsconfig.AuthToken) - fmt.Println(fmt.Sprintf("Migrating repo %s to gogs rest api", elem.Name)) - resp, err := client.Do(req) - if err != nil { - return err + client.Timeout = config.globalconfig.RequestTimeout + + fmt.Println(fmt.Sprintf("Migrating repo %s to gogs rest api", elem.Name)) + resp, err = client.Do(req) + if err != nil { + return err + } + fmt.Println(fmt.Sprintf("Repo %s migrated with status code %d on gogs webservice", elem.Name, resp.StatusCode)) + if resp.StatusCode != 201 { + err = fmt.Errorf("Error when migrating repo %s to gogs with status code %d on gogs webservice", elem.Name, resp.StatusCode) + log.Println(err) + } + } else { + fmt.Println(fmt.Sprintf("Not migrating, %s exists in gogs", elem.Name)) } - fmt.Println(fmt.Sprintf("Repo %s migrated with status code %d", elem.Name, resp.StatusCode)) - if resp.StatusCode != 201 { - err = errors.New(fmt.Sprintf("Error when migrating repo %s to gogs, status code %d", elem.Name, resp.StatusCode)) - log.Println(err) - } - } else { - fmt.Println(fmt.Sprintf("Not migrating, %s exists in gogs", elem.Name)) + fmt.Println(fmt.Sprintf("All repo migrated on thread num %d", thr)) + wg.Done() + done <- true + return nil } } - return nil } // Usage displays possible arguments diff --git a/github-to-gogs.go b/github-to-gogs.go index a950026..4920c92 100644 --- a/github-to-gogs.go +++ b/github-to-gogs.go @@ -23,8 +23,5 @@ func main() { log.Fatal(err) } - err = MigrateReposToGogs(&config, repolist) - if err != nil { - log.Fatal(err) - } + RunMigration(&config, repolist) } diff --git a/types.go b/types.go index 15d0adc..85150fd 100644 --- a/types.go +++ b/types.go @@ -33,15 +33,17 @@ type Config struct { type GlobalConfig struct { RequestTimeout time.Duration UserAgent string + Threads int } // GitHubConfig is the GitHub config type GitHubConfig struct { - StarsPages string - MaxPagesNumber int - AuthUsername string - AuthPassword string - ContentType string + StarsPages string + MaxPerPage int + PageNum int + AuthUsername string + AuthPassword string + ContentType string } // GogsConfig is the Gogs config