From 948d2a5eda9b067243d011f327280d39939c2a7b Mon Sep 17 00:00:00 2001 From: Paul Lecuq Date: Sat, 24 Aug 2019 21:53:02 +0200 Subject: [PATCH] renamed github-to-gogs to g2g, change configuration structure --- README.md | 38 +++++----- functions.go | 137 ++++++++++++++---------------------- github-to-gogs.go => g2g.go | 2 +- g2g.ini.sample | 18 +++++ github-to-gogs.ini.sample | 20 ------ go.mod | 2 +- types.go | 59 ++++++---------- 7 files changed, 113 insertions(+), 163 deletions(-) rename github-to-gogs.go => g2g.go (90%) create mode 100644 g2g.ini.sample delete mode 100644 github-to-gogs.ini.sample diff --git a/README.md b/README.md index 71f48fd..51ec013 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -# github-to-gogs +# g2g ## Summary -github-to-gogs is a small program that migrate stars made on github projects to self owned gogs/gitea instance +g2g is a small program that migrate stars made on github projects to self owned gogs/gitea instance ## Howto @@ -12,37 +12,33 @@ github-to-gogs is a small program that migrate stars made on github projects to go build ``` -### Sample config in github-to-gogs.ini +### Sample config in g2g.ini ```ini -[global] +[g2g] request_timeout=1200s user_agent="Golang" threads=4 -[github] -stars_pages="https://api.github.com/users/%s/starred?page=%d" -max_per_page=500 -page_num=3 -auth_username="user" -auth_password="pass" -content_type="application/x-www-form-urlencoded" +github_stars_pages="https://api.github.com/users/%s/starred?page=%d" +github_max_per_page=500 +github_page_num=3 +github_auth_username="user" +github_auth_password="pass" -[gogs] -username="user" -dest_username="user_or_org" -repo_url_tmpl="https://gogs.example.com/api/v1/repos/%s/%s" -orgs_url_tmpl="https://git.paulbsd.com/api/v1/orgs/%s" -migrate_url="https://gogs.example.com/api/v1/repos/migrate" -auth_token="token xxxx" -content_type="application/json" -mirror=true +gitea_username="user" +gitea_dest_username="user_or_org" +gitea_repo_url_tmpl="https://gogs.example.com/api/v1/repos/%s/%s" +gitea_orgs_url_tmpl="https://git.paulbsd.com/api/v1/orgs/%s" +gitea_migrate_url="https://gogs.example.com/api/v1/repos/migrate" +gitea_auth_token="token xxxx" +gitea_mirror=true ``` ### Run ```bash -./github-to-gogs -configfile github-to-gogs.ini +./g2g -configfile g2g.ini ``` ## License diff --git a/functions.go b/functions.go index 2721472..a43914d 100644 --- a/functions.go +++ b/functions.go @@ -19,13 +19,9 @@ import ( func (config *Config) GetConfig() error { var configfile string - var globalconfig GlobalConfig - var githubconfig GitHubConfig - var gogsconfig GogsConfig - flag.Usage = Usage - flag.StringVar(&configfile, "configfile", "github-to-gogs.ini", "config file to use with github-to-gogs section") + flag.StringVar(&configfile, "configfile", "g2g.ini", "config file to use with g2g section") flag.Parse() cfg, err := ini.Load(configfile) @@ -33,50 +29,25 @@ func (config *Config) GetConfig() error { return err } - globalsection := cfg.Section("global") - githubsection := cfg.Section("github") - gogssection := cfg.Section("gogs") + g2gsection := cfg.Section("g2g") - globalconfig.UserAgent = globalsection.Key("user_agent").String() - globalconfig.RequestTimeout, err = globalsection.Key("request_timeout").Duration() - if err != nil { - return err - } - globalconfig.Threads, err = globalsection.Key("threads").Int() - if err != nil { - return err - } + config.UserAgent = g2gsection.Key("user_agent").MustString("Golang") + config.RequestTimeout = g2gsection.Key("request_timeout").MustDuration(1200) + config.Threads = g2gsection.Key("threads").MustInt(2) + config.GitHubPageNum = g2gsection.Key("github_page_num").MustInt(3) + config.GitHubMaxPerPage = g2gsection.Key("github_max_per_page").MustInt(100) + config.GitHubAuthUsername = g2gsection.Key("github_auth_username").MustString("username") + config.GitHubAuthPassword = g2gsection.Key("github_auth_password").MustString("password") - githubconfig.PageNum, err = githubsection.Key("page_num").Int() - if err != nil { - return err - } - githubconfig.MaxPerPage, err = githubsection.Key("max_per_page").Int() - if err != nil { - return err - } + config.GiteaUsername = g2gsection.Key("gitea_username").MustString("username") + config.GiteaDestUsername = g2gsection.Key("gitea_dest_username").MustString("dest_username") + config.GiteaRepoURLTmpl = g2gsection.Key("gitea_repo_url_tmpl").MustString("repo_url") + config.GiteaOrgsURLTmpl = g2gsection.Key("gitea_orgs_url_tmpl").MustString("orgs_url") + config.GiteaMigrateURL = g2gsection.Key("gitea_migrate_url").MustString("migrate_url") + config.GiteaAuthToken = g2gsection.Key("gitea_auth_token").MustString("token") + config.GiteaMirror = g2gsection.Key("gitea_mirror").MustBool(true) - githubconfig.AuthUsername = githubsection.Key("auth_username").String() - githubconfig.AuthPassword = githubsection.Key("auth_password").String() - githubconfig.ContentType = githubsection.Key("content_type").String() - - gogsconfig.Username = gogssection.Key("username").String() - gogsconfig.DestUsername = gogssection.Key("dest_username").String() - gogsconfig.RepoURLTmpl = gogssection.Key("repo_url_tmpl").String() - gogsconfig.OrgsURLTmpl = gogssection.Key("orgs_url_tmpl").String() - gogsconfig.MigrateURL = gogssection.Key("migrate_url").String() - gogsconfig.AuthToken = gogssection.Key("auth_token").String() - gogsconfig.ContentType = gogssection.Key("content_type").String() - gogsconfig.Mirror, err = gogssection.Key("mirror").Bool() - if err != nil { - return err - } - - *config = Config{globalconfig: globalconfig, - githubconfig: githubconfig, - gogsconfig: gogsconfig} - - return nil + return err } // GetReposFromGitHub get starred repositories from github @@ -85,8 +56,8 @@ func GetReposFromGitHub(config *Config) ([]GitHubRepo, error) { var repofulllist []GitHubRepo fmt.Println("Getting GitHub starred repos") - for num := 1; num <= config.githubconfig.PageNum; num++ { - url := fmt.Sprintf("https://api.github.com/users/%s/starred?per_page=%d&page=%d", config.githubconfig.AuthUsername, config.githubconfig.MaxPerPage, num) + for num := 1; num <= config.GitHubPageNum; num++ { + url := fmt.Sprintf("https://api.github.com/users/%s/starred?per_page=%d&page=%d", config.GitHubAuthUsername, config.GitHubMaxPerPage, num) fmt.Println(url) resp, err := InvokeGitHub(config, url) @@ -120,9 +91,9 @@ func InvokeGitHub(config *Config, url string) (*http.Response, error) { return nil, err } - req.Header.Add("Content-Type", config.githubconfig.ContentType) - req.Header.Set("User-Agent", config.globalconfig.UserAgent) - req.SetBasicAuth(config.githubconfig.AuthUsername, config.githubconfig.AuthPassword) + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("User-Agent", config.UserAgent) + req.SetBasicAuth(config.GitHubAuthUsername, config.GitHubAuthPassword) client := &http.Client{} resp, err := client.Do(req) @@ -133,15 +104,15 @@ func InvokeGitHub(config *Config, url string) (*http.Response, error) { return resp, nil } -// CheckGogsExistingRepo checks if there's an existing repository in gogs -func CheckGogsExistingRepo(config *Config, repo GitHubRepo) (bool, error) { +// CheckGiteaExistingRepo checks if there's an existing repository in gitea +func CheckGiteaExistingRepo(config *Config, repo GitHubRepo) (bool, error) { var isexists bool - gogsrepourl := fmt.Sprintf(config.gogsconfig.RepoURLTmpl, config.gogsconfig.DestUsername, repo.Name) + gitearepourl := fmt.Sprintf(config.GiteaRepoURLTmpl, config.GiteaDestUsername, repo.Name) - req, err := http.NewRequest("GET", gogsrepourl, nil) - req.Header.Set("Authorization", config.gogsconfig.AuthToken) - req.Header.Set("User-Agent", config.globalconfig.UserAgent) + req, err := http.NewRequest("GET", gitearepourl, nil) + req.Header.Set("Authorization", config.GiteaAuthToken) + req.Header.Set("User-Agent", config.UserAgent) client := &http.Client{} resp, err := client.Do(req) @@ -154,26 +125,26 @@ func CheckGogsExistingRepo(config *Config, repo GitHubRepo) (bool, error) { } else if resp.StatusCode == 404 { isexists = false } else { - err = fmt.Errorf("Can't determine error, cancelling, error %d in gogs webservice", resp.StatusCode) + err = fmt.Errorf("Can't determine error, cancelling, error %d in gitea webservice", resp.StatusCode) return false, err } return isexists, nil } -// GetGogsUserUID get the logued user identifier -func (config *Config) GetGogsUserUID() error { - var gogsorg GogsOrg +// GetGiteaUserUID get the logued user identifier +func (config *Config) GetGiteaUserUID() error { + var giteaorg GiteaOrg - gogsrepourl := fmt.Sprintf(config.gogsconfig.OrgsURLTmpl, config.gogsconfig.DestUsername) + gitearepourl := fmt.Sprintf(config.GiteaOrgsURLTmpl, config.GiteaDestUsername) - req, err := http.NewRequest("GET", gogsrepourl, nil) + req, err := http.NewRequest("GET", gitearepourl, nil) if err != nil { return err } - req.Header.Set("Authorization", config.gogsconfig.AuthToken) - req.Header.Set("User-Agent", config.globalconfig.UserAgent) + req.Header.Set("Authorization", config.GiteaAuthToken) + req.Header.Set("User-Agent", config.UserAgent) client := &http.Client{} resp, err := client.Do(req) @@ -182,19 +153,19 @@ func (config *Config) GetGogsUserUID() error { } if resp.StatusCode != 200 { - err = errors.New("Error invoking gogs webservice") + err = errors.New("Error invoking gitea webservice") return err } respbody, err := ioutil.ReadAll(resp.Body) - err = json.Unmarshal(respbody, &gogsorg) + err = json.Unmarshal(respbody, &giteaorg) if err != nil { - err = errors.New("Failed to parse user ID from gogs webservice, check auth") + err = errors.New("Failed to parse user ID from gitea webservice, check auth") return err } - config.gogsconfig.UID = gogsorg.ID + config.GiteaUID = giteaorg.ID return nil } @@ -205,8 +176,8 @@ func RunMigration(config *Config, repolist []GitHubRepo) { done := make(chan bool) var wg sync.WaitGroup - for thr := range make([]int, config.globalconfig.Threads) { - go MigrateReposToGogs(config, &wg, repochan, done, thr) + for thr := range make([]int, config.Threads) { + go MigrateReposToGitea(config, &wg, repochan, done, thr) } for _, repo := range repolist { @@ -217,33 +188,33 @@ func RunMigration(config *Config, repolist []GitHubRepo) { wg.Wait() } -// MigrateReposToGogs migrates input repositories to gogs -func MigrateReposToGogs(config *Config, wg *sync.WaitGroup, jobs chan GitHubRepo, done chan bool, thr int) error { +// MigrateReposToGitea migrates input repositories to gitea +func MigrateReposToGitea(config *Config, wg *sync.WaitGroup, jobs chan GitHubRepo, done chan bool, thr int) error { wg.Add(1) for { elem, more := <-jobs if more { client := &http.Client{} - existingrepo, err := CheckGogsExistingRepo(config, elem) + existingrepo, err := CheckGiteaExistingRepo(config, elem) if err != nil { return err } if !existingrepo { - jsondata := fmt.Sprintf(`{"uid" : %d, "repo_name" : "%s" , "mirror" : %v, "clone_addr" : "%s"}`, config.gogsconfig.UID, elem.Name, true, elem.CloneURL) + jsondata := fmt.Sprintf(`{"uid" : %d, "repo_name" : "%s" , "mirror" : %v, "clone_addr" : "%s"}`, config.GiteaUID, elem.Name, true, elem.CloneURL) - req, err := http.NewRequest("POST", config.gogsconfig.MigrateURL, bytes.NewBufferString(jsondata)) + req, err := http.NewRequest("POST", config.GiteaMigrateURL, bytes.NewBufferString(jsondata)) if err != nil { return err } - req.Header.Add("Content-Type", config.gogsconfig.ContentType) - req.Header.Set("User-Agent", config.globalconfig.UserAgent) - req.Header.Set("Authorization", config.gogsconfig.AuthToken) + req.Header.Add("Content-Type", "application/json") + req.Header.Set("User-Agent", config.UserAgent) + req.Header.Set("Authorization", config.GiteaAuthToken) - client.Timeout = config.globalconfig.RequestTimeout + client.Timeout = config.RequestTimeout - fmt.Println(fmt.Sprintf("Migrating repo %s to gogs rest api", elem.Name)) + fmt.Println(fmt.Sprintf("Migrating repo %s to gitea rest api", elem.Name)) resp, err := client.Do(req) if err != nil { @@ -251,11 +222,11 @@ func MigrateReposToGogs(config *Config, wg *sync.WaitGroup, jobs chan GitHubRepo } if resp.StatusCode != 201 { - err = fmt.Errorf("Error when migrating repo %s to gogs with status code %d on gogs webservice", elem.Name, resp.StatusCode) + err = fmt.Errorf("Error when migrating repo %s to gitea with status code %d on gitea webservice", elem.Name, resp.StatusCode) log.Println(err) } } else { - fmt.Println(fmt.Sprintf("Not migrating, %s exists in gogs", elem.Name)) + fmt.Println(fmt.Sprintf("Not migrating, %s exists in gitea", elem.Name)) } } else { fmt.Println(fmt.Sprintf("All repo migrated on thread num %d", thr)) diff --git a/github-to-gogs.go b/g2g.go similarity index 90% rename from github-to-gogs.go rename to g2g.go index bc62bbc..c3ec131 100644 --- a/github-to-gogs.go +++ b/g2g.go @@ -13,7 +13,7 @@ func main() { log.Fatal(err) } - err = config.GetGogsUserUID() + err = config.GetGiteaUserUID() if err != nil { log.Fatal(err) } diff --git a/g2g.ini.sample b/g2g.ini.sample new file mode 100644 index 0000000..717a3aa --- /dev/null +++ b/g2g.ini.sample @@ -0,0 +1,18 @@ +[g2g] +request_timeout=1200s +user_agent="Golang" +threads=4 + +github_stars_pages="https://api.github.com/users/%s/starred?page=%d" +github_max_per_page=500 +github_page_num=3 +github_auth_username="user" +github_auth_password="pass" + +gitea_username="user" +gitea_dest_username="user_or_org" +gitea_repo_url_tmpl="https://gogs.example.com/api/v1/repos/%s/%s" +gitea_orgs_url_tmpl="https://git.paulbsd.com/api/v1/orgs/%s" +gitea_migrate_url="https://gogs.example.com/api/v1/repos/migrate" +gitea_auth_token="token xxxx" +gitea_mirror=true \ No newline at end of file diff --git a/github-to-gogs.ini.sample b/github-to-gogs.ini.sample deleted file mode 100644 index 5b6728d..0000000 --- a/github-to-gogs.ini.sample +++ /dev/null @@ -1,20 +0,0 @@ -[global] -request_timeout=1200s -user_agent="Golang" - -[github] -stars_pages="https://api.github.com/users/%s/starred?page=%d" -max_pages_number=8 -auth_username="user" -auth_password="pass" -content_type="application/x-www-form-urlencoded" - -[gogs] -username="user" -dest_username="user_or_org" -repo_url_tmpl="https://gogs.example.com/api/v1/repos/%s/%s" -orgs_url_tmpl="https://git.paulbsd.com/api/v1/orgs/%s" -migrate_url="https://gogs.example.com/api/v1/repos/migrate" -auth_token="token xxxx" -content_type="application/json" -mirror=true diff --git a/go.mod b/go.mod index 413a33a..a0db408 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github-to-gogs +module g2g go 1.12 diff --git a/types.go b/types.go index a215849..b25d168 100644 --- a/types.go +++ b/types.go @@ -8,52 +8,37 @@ type GitHubRepo struct { CloneURL string `json:"clone_url"` } -// GogsOrg gogsrepo struct -type GogsOrg struct { +// GiteaOrg gitearepo struct +type GiteaOrg struct { ID int `json:"id"` Username string `json:"username"` } -// GogsMigrateRepo defines the repo to migrate -type GogsMigrateRepo struct { +// GiteaMigrateRepo defines the repo to migrate +type GiteaMigrateRepo struct { Name string CloneURL string UID int Mirror bool } -// Config merge all config elements +// Config is the global config of g2g type Config struct { - globalconfig GlobalConfig - gogsconfig GogsConfig - githubconfig GitHubConfig -} - -// GlobalConfig is the global config of github-to-gogs -type GlobalConfig struct { - RequestTimeout time.Duration - UserAgent string - Threads int -} - -// GitHubConfig is the GitHub config -type GitHubConfig struct { - MaxPerPage int - PageNum int - AuthUsername string - AuthPassword string - ContentType string -} - -// GogsConfig is the Gogs config -type GogsConfig struct { - UID int - Username string - DestUsername string - RepoURLTmpl string - OrgsURLTmpl string - MigrateURL string - AuthToken string - ContentType string - Mirror bool + RequestTimeout time.Duration + UserAgent string + Threads int + GitHubMaxPerPage int + GitHubPageNum int + GitHubAuthUsername string + GitHubAuthPassword string + GitHubContentType string + GiteaUID int + GiteaUsername string + GiteaDestUsername string + GiteaRepoURLTmpl string + GiteaOrgsURLTmpl string + GiteaMigrateURL string + GiteaAuthToken string + GiteaContentType string + GiteaMirror bool }