diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..0ec831d --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,17 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Launch", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${fileDirname}", + "env": {}, + "args": [] + } + ] +} \ No newline at end of file diff --git a/config.go b/config.go new file mode 100644 index 0000000..598e4ed --- /dev/null +++ b/config.go @@ -0,0 +1,48 @@ +package main + +import ( + "flag" + "time" + + "gopkg.in/ini.v1" +) + +// GetConfig fetch configuration +func (config *Config) GetConfig() error { + var configfile string + + flag.Usage = Usage + + flag.StringVar(&configfile, "configfile", "g2g.ini", "config file to use with g2g section") + flag.Parse() + + cfg, err := ini.Load(configfile) + if err != nil { + return err + } + + g2gsection := cfg.Section("g2g") + g2gsection.MapTo(config) + + return err +} + +// Config is the global config of g2g +type Config struct { + G2gRequestTimeout time.Duration `ini:"g2g_request_timeout"` + G2gThreads int `ini:"g2g_threads"` + GitHubMaxPerPage int `ini:"github_max_per_page"` + GitHubPageNum int `ini:"github_page_num"` + GitHubAuthUsername string `ini:"github_auth_username"` + GitHubAuthPassword string `ini:"github_auth_password"` + GitHubContentType string + GiteaUID int + GiteaUsername string `ini:"gitea_username"` + GiteaDestUsername string `ini:"gitea_dest_username"` + GiteaRepoURLTmpl string `ini:"gitea_repo_url_tmpl"` + GiteaOrgsURLTmpl string `ini:"gitea_orgs_url_tmpl"` + GiteaMigrateURL string `ini:"gitea_migrate_url"` + GiteaAuthToken string `ini:"gitea_auth_token"` + GiteaContentType string + GiteaMirror bool `ini:"gitea_mirror"` +} diff --git a/config_test.go b/config_test.go new file mode 100644 index 0000000..a6b460b --- /dev/null +++ b/config_test.go @@ -0,0 +1,29 @@ +package main + +import ( + "fmt" + "testing" +) + +func TestGetConfig(t *testing.T) { + got := ` +[g2g] +request_timeout=1200s +threads=4 + +github_stars_pages="https://api.github.com/users/%s/starred?page=%d" +github_max_per_page=500 +github_page_num=3 +github_auth_username="user" +github_auth_password="pass" + +gitea_username="user" +gitea_dest_username="user_or_org" +gitea_repo_url_tmpl="https://gogs.example.com/api/v1/repos/%s/%s" +gitea_orgs_url_tmpl="https://git.paulbsd.com/api/v1/orgs/%s" +gitea_migrate_url="https://gogs.example.com/api/v1/repos/migrate" +gitea_auth_token="token xxxx" +gitea_mirror=true +` + fmt.Println(got) +} diff --git a/functions.go b/functions.go index a43914d..8f67e64 100644 --- a/functions.go +++ b/functions.go @@ -11,45 +11,8 @@ import ( "net/http" "os" "sync" - - "gopkg.in/ini.v1" ) -// GetConfig fetch configuration -func (config *Config) GetConfig() error { - var configfile string - - flag.Usage = Usage - - flag.StringVar(&configfile, "configfile", "g2g.ini", "config file to use with g2g section") - flag.Parse() - - cfg, err := ini.Load(configfile) - if err != nil { - return err - } - - g2gsection := cfg.Section("g2g") - - config.UserAgent = g2gsection.Key("user_agent").MustString("Golang") - config.RequestTimeout = g2gsection.Key("request_timeout").MustDuration(1200) - config.Threads = g2gsection.Key("threads").MustInt(2) - config.GitHubPageNum = g2gsection.Key("github_page_num").MustInt(3) - config.GitHubMaxPerPage = g2gsection.Key("github_max_per_page").MustInt(100) - config.GitHubAuthUsername = g2gsection.Key("github_auth_username").MustString("username") - config.GitHubAuthPassword = g2gsection.Key("github_auth_password").MustString("password") - - config.GiteaUsername = g2gsection.Key("gitea_username").MustString("username") - config.GiteaDestUsername = g2gsection.Key("gitea_dest_username").MustString("dest_username") - config.GiteaRepoURLTmpl = g2gsection.Key("gitea_repo_url_tmpl").MustString("repo_url") - config.GiteaOrgsURLTmpl = g2gsection.Key("gitea_orgs_url_tmpl").MustString("orgs_url") - config.GiteaMigrateURL = g2gsection.Key("gitea_migrate_url").MustString("migrate_url") - config.GiteaAuthToken = g2gsection.Key("gitea_auth_token").MustString("token") - config.GiteaMirror = g2gsection.Key("gitea_mirror").MustBool(true) - - return err -} - // GetReposFromGitHub get starred repositories from github func GetReposFromGitHub(config *Config) ([]GitHubRepo, error) { var repopartiallist []GitHubRepo @@ -60,7 +23,7 @@ func GetReposFromGitHub(config *Config) ([]GitHubRepo, error) { url := fmt.Sprintf("https://api.github.com/users/%s/starred?per_page=%d&page=%d", config.GitHubAuthUsername, config.GitHubMaxPerPage, num) fmt.Println(url) - resp, err := InvokeGitHub(config, url) + resp, err := GetGitHubRepos(config, url) if err != nil { return nil, err } @@ -84,15 +47,14 @@ func GetReposFromGitHub(config *Config) ([]GitHubRepo, error) { return repofulllist, nil } -// InvokeGitHub ... -func InvokeGitHub(config *Config, url string) (*http.Response, error) { +// GetGitHubRepos fetchs starred repositories on GitHub +func GetGitHubRepos(config *Config, url string) (*http.Response, error) { req, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err } req.Header.Add("Content-Type", "application/x-www-form-urlencoded") - req.Header.Set("User-Agent", config.UserAgent) req.SetBasicAuth(config.GitHubAuthUsername, config.GitHubAuthPassword) client := &http.Client{} @@ -101,18 +63,21 @@ func InvokeGitHub(config *Config, url string) (*http.Response, error) { return nil, err } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("Error on status code %s", resp.Status) + } + return resp, nil } // CheckGiteaExistingRepo checks if there's an existing repository in gitea func CheckGiteaExistingRepo(config *Config, repo GitHubRepo) (bool, error) { - var isexists bool + var isExists bool gitearepourl := fmt.Sprintf(config.GiteaRepoURLTmpl, config.GiteaDestUsername, repo.Name) req, err := http.NewRequest("GET", gitearepourl, nil) req.Header.Set("Authorization", config.GiteaAuthToken) - req.Header.Set("User-Agent", config.UserAgent) client := &http.Client{} resp, err := client.Do(req) @@ -120,16 +85,15 @@ func CheckGiteaExistingRepo(config *Config, repo GitHubRepo) (bool, error) { return false, err } - if resp.StatusCode == 200 { - isexists = true - } else if resp.StatusCode == 404 { - isexists = false + if resp.StatusCode == http.StatusOK { + isExists = true + } else if resp.StatusCode == http.StatusNotFound { + isExists = false } else { - err = fmt.Errorf("Can't determine error, cancelling, error %d in gitea webservice", resp.StatusCode) - return false, err + return false, fmt.Errorf("Can't determine error, cancelling, error %s in gitea webservice", resp.Status) } - return isexists, nil + return isExists, nil } // GetGiteaUserUID get the logued user identifier @@ -144,7 +108,6 @@ func (config *Config) GetGiteaUserUID() error { } req.Header.Set("Authorization", config.GiteaAuthToken) - req.Header.Set("User-Agent", config.UserAgent) client := &http.Client{} resp, err := client.Do(req) @@ -176,7 +139,7 @@ func RunMigration(config *Config, repolist []GitHubRepo) { done := make(chan bool) var wg sync.WaitGroup - for thr := range make([]int, config.Threads) { + for thr := range make([]int, config.G2gThreads) { go MigrateReposToGitea(config, &wg, repochan, done, thr) } @@ -209,10 +172,9 @@ func MigrateReposToGitea(config *Config, wg *sync.WaitGroup, jobs chan GitHubRep } req.Header.Add("Content-Type", "application/json") - req.Header.Set("User-Agent", config.UserAgent) req.Header.Set("Authorization", config.GiteaAuthToken) - client.Timeout = config.RequestTimeout + client.Timeout = config.G2gRequestTimeout fmt.Println(fmt.Sprintf("Migrating repo %s to gitea rest api", elem.Name)) @@ -242,3 +204,23 @@ func Usage() { flag.PrintDefaults() os.Exit(1) } + +// GitHubRepo githubrepo struct +type GitHubRepo struct { + Name string `json:"name"` + CloneURL string `json:"clone_url"` +} + +// GiteaOrg gitearepo struct +type GiteaOrg struct { + ID int `json:"id"` + Username string `json:"username"` +} + +// GiteaMigrateRepo defines the repo to migrate +type GiteaMigrateRepo struct { + Name string + CloneURL string + UID int + Mirror bool +} diff --git a/g2g.go b/g2g.go index c3ec131..0132723 100644 --- a/g2g.go +++ b/g2g.go @@ -7,6 +7,7 @@ import ( func main() { var config Config var err error + var repolist []GitHubRepo err = config.GetConfig() if err != nil { @@ -18,7 +19,7 @@ func main() { log.Fatal(err) } - repolist, err := GetReposFromGitHub(&config) + repolist, err = GetReposFromGitHub(&config) if err != nil { log.Fatal(err) } diff --git a/g2g.ini.sample b/g2g.ini.sample index 717a3aa..01e7fd3 100644 --- a/g2g.ini.sample +++ b/g2g.ini.sample @@ -1,6 +1,5 @@ [g2g] request_timeout=1200s -user_agent="Golang" threads=4 github_stars_pages="https://api.github.com/users/%s/starred?page=%d" diff --git a/types.go b/types.go deleted file mode 100644 index b25d168..0000000 --- a/types.go +++ /dev/null @@ -1,44 +0,0 @@ -package main - -import "time" - -// GitHubRepo githubrepo struct -type GitHubRepo struct { - Name string `json:"name"` - CloneURL string `json:"clone_url"` -} - -// GiteaOrg gitearepo struct -type GiteaOrg struct { - ID int `json:"id"` - Username string `json:"username"` -} - -// GiteaMigrateRepo defines the repo to migrate -type GiteaMigrateRepo struct { - Name string - CloneURL string - UID int - Mirror bool -} - -// Config is the global config of g2g -type Config struct { - RequestTimeout time.Duration - UserAgent string - Threads int - GitHubMaxPerPage int - GitHubPageNum int - GitHubAuthUsername string - GitHubAuthPassword string - GitHubContentType string - GiteaUID int - GiteaUsername string - GiteaDestUsername string - GiteaRepoURLTmpl string - GiteaOrgsURLTmpl string - GiteaMigrateURL string - GiteaAuthToken string - GiteaContentType string - GiteaMirror bool -}