paulbsd/g2g
1
0
Fork 0
Code Issues Pull Requests Releases 4 Wiki Activity

added parallelization

Browse Source
This commit is contained in:
Paul 2019-07-10 15:07:41 +02:00
parent 2cd78236ab
commit 14fe05f4d5
3 changed files with 78 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
functions.go
View File

@ -10,6 +10,7 @@ import (
"log"
"net/http"
"os"
"sync"
"gopkg.in/ini.v1"
)
@ -40,9 +41,17 @@ func GetConfig(config *Config) error {
if err != nil {
return err
}
globalconfig.Threads, err = globalsection.Key("threads").Int()
if err != nil {
return err
}
githubconfig.StarsPages = githubsection.Key("stars_pages").String()
githubconfig.MaxPagesNumber, err = githubsection.Key("max_pages_number").Int()
githubconfig.PageNum, err = githubsection.Key("page_num").Int()
if err != nil {
return err
}
githubconfig.MaxPerPage, err = githubsection.Key("max_per_page").Int()
if err != nil {
return err
}
@ -51,7 +60,6 @@ func GetConfig(config *Config) error {
githubconfig.AuthPassword = githubsection.Key("auth_password").String()
githubconfig.ContentType = githubsection.Key("content_type").String()
gogsconfig.UID = -1
gogsconfig.Username = gogssection.Key("username").String()
gogsconfig.DestUsername = gogssection.Key("dest_username").String()
gogsconfig.RepoURLTmpl = gogssection.Key("repo_url_tmpl").String()
@ -76,9 +84,11 @@ func GetReposFromGitHub(config *Config) ([]GitHubRepo, error) {
var repolist []GitHubRepo
var repo []GitHubRepo
for num := 1; num <= config.githubconfig.MaxPagesNumber; num++ {
url := fmt.Sprintf(config.githubconfig.StarsPages, config.githubconfig.AuthUsername, num)
for num := 1; num <= config.githubconfig.PageNum; num++ {
url := fmt.Sprintf(config.githubconfig.StarsPages, config.githubconfig.AuthUsername, config.githubconfig.MaxPerPage, num)
fmt.Println(url)
fmt.Println("Getting GitHub starred repos")
resp, err := InvokeGitHub(config, url)
if err != nil {
return nil, err
@ -93,11 +103,11 @@ func GetReposFromGitHub(config *Config) ([]GitHubRepo, error) {
if err != nil {
return nil, err
}
for _, elem := range repo {
repolist = append(repolist, elem)
}
}
fmt.Println(fmt.Sprintf("%d repositories from Github fetched", len(repolist)))
return repolist, nil
}
@ -152,7 +162,7 @@ func CheckGogsExistingRepo(config *Config, repo GitHubRepo) (bool, error) {
} else if resp.StatusCode == 404 {
isexists = false
} else {
err = errors.New(fmt.Sprintf("Can't determine error, cancelling, error %d in gogs webservice", resp.StatusCode))
err = fmt.Errorf("Can't determine error, cancelling, error %d in gogs webservice", resp.StatusCode)
return false, err
}
@ -188,7 +198,7 @@ func GetGogsUserUID(config *Config) error {
err = json.Unmarshal(*respbody, &gogsorg)
if err != nil {
err = errors.New("Failed to parse user ID from gogs webservice, check auth if possible")
err = errors.New("Failed to parse user ID from gogs webservice, check auth")
return err
}
@ -197,47 +207,71 @@ func GetGogsUserUID(config *Config) error {
return nil
}
// RunMigration run threads for migration
func RunMigration(config *Config, repolist []GitHubRepo) {
jobs := make(chan GitHubRepo)
done := make(chan bool)
var wg sync.WaitGroup
for thr := range make([]int, config.globalconfig.Threads) {
go MigrateReposToGogs(config, &wg, jobs, done, thr)
}
for _, repo := range repolist {
jobs <- repo
}
close(jobs)
<-done
wg.Wait()
}
// MigrateReposToGogs migrates input repositories to gogs
func MigrateReposToGogs(config *Config, repolist []GitHubRepo) error {
fmt.Println("Migrate repos to Gogs...")
func MigrateReposToGogs(config *Config, wg *sync.WaitGroup, jobs chan GitHubRepo, done chan bool, thr int) error {
wg.Add(1)
for {
elem, more := <-jobs
if more {
client := &http.Client{}
var resp *http.Response
client := &http.Client{}
for _, elem := range repolist {
existingrepo, err := CheckGogsExistingRepo(config, elem)
if err != nil {
return err
}
if !existingrepo {
jsondata := fmt.Sprintf(`{"uid" : %d, "repo_name" : "%s" , "mirror" : %v, "clone_addr" : "%s"}`, config.gogsconfig.UID, elem.Name, true, elem.CloneURL)
req, err := http.NewRequest("POST", config.gogsconfig.MigrateURL, bytes.NewBufferString(jsondata))
existingrepo, err := CheckGogsExistingRepo(config, elem)
if err != nil {
return err
}
if !existingrepo {
jsondata := fmt.Sprintf(`{"uid" : %d, "repo_name" : "%s" , "mirror" : %v, "clone_addr" : "%s"}`, config.gogsconfig.UID, elem.Name, true, elem.CloneURL)
req.Header.Add("Content-Type", config.gogsconfig.ContentType)
req.Header.Set("User-Agent", config.globalconfig.UserAgent)
req.Header.Set("Authorization", config.gogsconfig.AuthToken)
req, err := http.NewRequest("POST", config.gogsconfig.MigrateURL, bytes.NewBufferString(jsondata))
if err != nil {
return err
}
client.Timeout = config.globalconfig.RequestTimeout
req.Header.Add("Content-Type", config.gogsconfig.ContentType)
req.Header.Set("User-Agent", config.globalconfig.UserAgent)
req.Header.Set("Authorization", config.gogsconfig.AuthToken)
fmt.Println(fmt.Sprintf("Migrating repo %s to gogs rest api", elem.Name))
resp, err := client.Do(req)
if err != nil {
return err
client.Timeout = config.globalconfig.RequestTimeout
fmt.Println(fmt.Sprintf("Migrating repo %s to gogs rest api", elem.Name))
resp, err = client.Do(req)
if err != nil {
return err
}
fmt.Println(fmt.Sprintf("Repo %s migrated with status code %d on gogs webservice", elem.Name, resp.StatusCode))
if resp.StatusCode != 201 {
err = fmt.Errorf("Error when migrating repo %s to gogs with status code %d on gogs webservice", elem.Name, resp.StatusCode)
log.Println(err)
}
} else {
fmt.Println(fmt.Sprintf("Not migrating, %s exists in gogs", elem.Name))
}
fmt.Println(fmt.Sprintf("Repo %s migrated with status code %d", elem.Name, resp.StatusCode))
if resp.StatusCode != 201 {
err = errors.New(fmt.Sprintf("Error when migrating repo %s to gogs, status code %d", elem.Name, resp.StatusCode))
log.Println(err)
}
} else {
fmt.Println(fmt.Sprintf("Not migrating, %s exists in gogs", elem.Name))
fmt.Println(fmt.Sprintf("All repo migrated on thread num %d", thr))
wg.Done()
done <- true
return nil
}
}
return nil
}
// Usage displays possible arguments

5
github-to-gogs.go
View File

@ -23,8 +23,5 @@ func main() {
log.Fatal(err)
}
err = MigrateReposToGogs(&config, repolist)
if err != nil {
log.Fatal(err)
}
RunMigration(&config, repolist)
}

12
types.go
View File

@ -33,15 +33,17 @@ type Config struct {
type GlobalConfig struct {
RequestTimeout time.Duration
UserAgent string
Threads int
}
// GitHubConfig is the GitHub config
type GitHubConfig struct {
StarsPages string
MaxPagesNumber int
AuthUsername string
AuthPassword string
ContentType string
StarsPages string
MaxPerPage int
PageNum int
AuthUsername string
AuthPassword string
ContentType string
}
// GogsConfig is the Gogs config