package main import ( "bytes" "encoding/json" "errors" "flag" "fmt" "io/ioutil" "log" "net/http" "os" "sync" "gopkg.in/ini.v1" ) // GetConfig fetch configuration func (config *Config) GetConfig() error { var configfile string flag.Usage = Usage flag.StringVar(&configfile, "configfile", "g2g.ini", "config file to use with g2g section") flag.Parse() cfg, err := ini.Load(configfile) if err != nil { return err } g2gsection := cfg.Section("g2g") config.UserAgent = g2gsection.Key("user_agent").MustString("Golang") config.RequestTimeout = g2gsection.Key("request_timeout").MustDuration(1200) config.Threads = g2gsection.Key("threads").MustInt(2) config.GitHubPageNum = g2gsection.Key("github_page_num").MustInt(3) config.GitHubMaxPerPage = g2gsection.Key("github_max_per_page").MustInt(100) config.GitHubAuthUsername = g2gsection.Key("github_auth_username").MustString("username") config.GitHubAuthPassword = g2gsection.Key("github_auth_password").MustString("password") config.GiteaUsername = g2gsection.Key("gitea_username").MustString("username") config.GiteaDestUsername = g2gsection.Key("gitea_dest_username").MustString("dest_username") config.GiteaRepoURLTmpl = g2gsection.Key("gitea_repo_url_tmpl").MustString("repo_url") config.GiteaOrgsURLTmpl = g2gsection.Key("gitea_orgs_url_tmpl").MustString("orgs_url") config.GiteaMigrateURL = g2gsection.Key("gitea_migrate_url").MustString("migrate_url") config.GiteaAuthToken = g2gsection.Key("gitea_auth_token").MustString("token") config.GiteaMirror = g2gsection.Key("gitea_mirror").MustBool(true) return err } // GetReposFromGitHub get starred repositories from github func GetReposFromGitHub(config *Config) ([]GitHubRepo, error) { var repopartiallist []GitHubRepo var repofulllist []GitHubRepo fmt.Println("Getting GitHub starred repos") for num := 1; num <= config.GitHubPageNum; num++ { url := fmt.Sprintf("https://api.github.com/users/%s/starred?per_page=%d&page=%d", config.GitHubAuthUsername, config.GitHubMaxPerPage, num) fmt.Println(url) resp, err := InvokeGitHub(config, url) if err != nil { return nil, err } respbody, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, err } err = json.Unmarshal(respbody, &repopartiallist) if err != nil { return nil, err } for _, elem := range repopartiallist { repofulllist = append(repofulllist, elem) } } fmt.Println(fmt.Sprintf("%d repositories from Github fetched", len(repofulllist))) return repofulllist, nil } // InvokeGitHub ... func InvokeGitHub(config *Config, url string) (*http.Response, error) { req, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err } req.Header.Add("Content-Type", "application/x-www-form-urlencoded") req.Header.Set("User-Agent", config.UserAgent) req.SetBasicAuth(config.GitHubAuthUsername, config.GitHubAuthPassword) client := &http.Client{} resp, err := client.Do(req) if err != nil { return nil, err } return resp, nil } // CheckGiteaExistingRepo checks if there's an existing repository in gitea func CheckGiteaExistingRepo(config *Config, repo GitHubRepo) (bool, error) { var isexists bool gitearepourl := fmt.Sprintf(config.GiteaRepoURLTmpl, config.GiteaDestUsername, repo.Name) req, err := http.NewRequest("GET", gitearepourl, nil) req.Header.Set("Authorization", config.GiteaAuthToken) req.Header.Set("User-Agent", config.UserAgent) client := &http.Client{} resp, err := client.Do(req) if err != nil { return false, err } if resp.StatusCode == 200 { isexists = true } else if resp.StatusCode == 404 { isexists = false } else { err = fmt.Errorf("Can't determine error, cancelling, error %d in gitea webservice", resp.StatusCode) return false, err } return isexists, nil } // GetGiteaUserUID get the logued user identifier func (config *Config) GetGiteaUserUID() error { var giteaorg GiteaOrg gitearepourl := fmt.Sprintf(config.GiteaOrgsURLTmpl, config.GiteaDestUsername) req, err := http.NewRequest("GET", gitearepourl, nil) if err != nil { return err } req.Header.Set("Authorization", config.GiteaAuthToken) req.Header.Set("User-Agent", config.UserAgent) client := &http.Client{} resp, err := client.Do(req) if err != nil { return err } if resp.StatusCode != 200 { err = errors.New("Error invoking gitea webservice") return err } respbody, err := ioutil.ReadAll(resp.Body) err = json.Unmarshal(respbody, &giteaorg) if err != nil { err = errors.New("Failed to parse user ID from gitea webservice, check auth") return err } config.GiteaUID = giteaorg.ID return nil } // RunMigration run threads for migration func RunMigration(config *Config, repolist []GitHubRepo) { repochan := make(chan GitHubRepo) done := make(chan bool) var wg sync.WaitGroup for thr := range make([]int, config.Threads) { go MigrateReposToGitea(config, &wg, repochan, done, thr) } for _, repo := range repolist { repochan <- repo } close(repochan) <-done wg.Wait() } // MigrateReposToGitea migrates input repositories to gitea func MigrateReposToGitea(config *Config, wg *sync.WaitGroup, jobs chan GitHubRepo, done chan bool, thr int) error { wg.Add(1) for { elem, more := <-jobs if more { client := &http.Client{} existingrepo, err := CheckGiteaExistingRepo(config, elem) if err != nil { return err } if !existingrepo { jsondata := fmt.Sprintf(`{"uid" : %d, "repo_name" : "%s" , "mirror" : %v, "clone_addr" : "%s"}`, config.GiteaUID, elem.Name, true, elem.CloneURL) req, err := http.NewRequest("POST", config.GiteaMigrateURL, bytes.NewBufferString(jsondata)) if err != nil { return err } req.Header.Add("Content-Type", "application/json") req.Header.Set("User-Agent", config.UserAgent) req.Header.Set("Authorization", config.GiteaAuthToken) client.Timeout = config.RequestTimeout fmt.Println(fmt.Sprintf("Migrating repo %s to gitea rest api", elem.Name)) resp, err := client.Do(req) if err != nil { return err } if resp.StatusCode != 201 { err = fmt.Errorf("Error when migrating repo %s to gitea with status code %d on gitea webservice", elem.Name, resp.StatusCode) log.Println(err) } } else { fmt.Println(fmt.Sprintf("Not migrating, %s exists in gitea", elem.Name)) } } else { fmt.Println(fmt.Sprintf("All repo migrated on thread num %d", thr)) wg.Done() done <- true return nil } } } // Usage displays possible arguments func Usage() { flag.PrintDefaults() os.Exit(1) }