package main import ( "bytes" "encoding/json" "errors" "flag" "fmt" "io/ioutil" "log" "net/http" "os" "sync" "gopkg.in/ini.v1" ) // GetConfig fetch configuration func (config *Config) GetConfig() error { var configfile string var globalconfig GlobalConfig var githubconfig GitHubConfig var gogsconfig GogsConfig flag.Usage = Usage flag.StringVar(&configfile, "configfile", "github-to-gogs.ini", "config file to use with github-to-gogs section") flag.Parse() cfg, err := ini.Load(configfile) if err != nil { return err } globalsection := cfg.Section("global") githubsection := cfg.Section("github") gogssection := cfg.Section("gogs") globalconfig.UserAgent = globalsection.Key("user_agent").String() globalconfig.RequestTimeout, err = globalsection.Key("request_timeout").Duration() if err != nil { return err } globalconfig.Threads, err = globalsection.Key("threads").Int() if err != nil { return err } githubconfig.PageNum, err = githubsection.Key("page_num").Int() if err != nil { return err } githubconfig.MaxPerPage, err = githubsection.Key("max_per_page").Int() if err != nil { return err } githubconfig.AuthUsername = githubsection.Key("auth_username").String() githubconfig.AuthPassword = githubsection.Key("auth_password").String() githubconfig.ContentType = githubsection.Key("content_type").String() gogsconfig.Username = gogssection.Key("username").String() gogsconfig.DestUsername = gogssection.Key("dest_username").String() gogsconfig.RepoURLTmpl = gogssection.Key("repo_url_tmpl").String() gogsconfig.OrgsURLTmpl = gogssection.Key("orgs_url_tmpl").String() gogsconfig.MigrateURL = gogssection.Key("migrate_url").String() gogsconfig.AuthToken = gogssection.Key("auth_token").String() gogsconfig.ContentType = gogssection.Key("content_type").String() gogsconfig.Mirror, err = gogssection.Key("mirror").Bool() if err != nil { return err } *config = Config{globalconfig: globalconfig, githubconfig: githubconfig, gogsconfig: gogsconfig} return nil } // GetReposFromGitHub get starred repositories from github func GetReposFromGitHub(config *Config) ([]GitHubRepo, error) { var repopartiallist []GitHubRepo var repofulllist []GitHubRepo fmt.Println("Getting GitHub starred repos") for num := 1; num <= config.githubconfig.PageNum; num++ { url := fmt.Sprintf("https://api.github.com/users/%s/starred?per_page=%d&page=%d", config.githubconfig.AuthUsername, config.githubconfig.MaxPerPage, num) fmt.Println(url) resp, err := InvokeGitHub(config, url) if err != nil { return nil, err } respbody, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, err } err = json.Unmarshal(respbody, &repopartiallist) if err != nil { return nil, err } for _, elem := range repopartiallist { repofulllist = append(repofulllist, elem) } } fmt.Println(fmt.Sprintf("%d repositories from Github fetched", len(repofulllist))) return repofulllist, nil } // InvokeGitHub ... func InvokeGitHub(config *Config, url string) (*http.Response, error) { req, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err } req.Header.Add("Content-Type", config.githubconfig.ContentType) req.Header.Set("User-Agent", config.globalconfig.UserAgent) req.SetBasicAuth(config.githubconfig.AuthUsername, config.githubconfig.AuthPassword) client := &http.Client{} resp, err := client.Do(req) if err != nil { return nil, err } return resp, nil } // CheckGogsExistingRepo checks if there's an existing repository in gogs func CheckGogsExistingRepo(config *Config, repo GitHubRepo) (bool, error) { var isexists bool gogsrepourl := fmt.Sprintf(config.gogsconfig.RepoURLTmpl, config.gogsconfig.DestUsername, repo.Name) req, err := http.NewRequest("GET", gogsrepourl, nil) req.Header.Set("Authorization", config.gogsconfig.AuthToken) req.Header.Set("User-Agent", config.globalconfig.UserAgent) client := &http.Client{} resp, err := client.Do(req) if err != nil { return false, err } if resp.StatusCode == 200 { isexists = true } else if resp.StatusCode == 404 { isexists = false } else { err = fmt.Errorf("Can't determine error, cancelling, error %d in gogs webservice", resp.StatusCode) return false, err } return isexists, nil } // GetGogsUserUID get the logued user identifier func (config *Config) GetGogsUserUID() error { var gogsorg GogsOrg gogsrepourl := fmt.Sprintf(config.gogsconfig.OrgsURLTmpl, config.gogsconfig.DestUsername) req, err := http.NewRequest("GET", gogsrepourl, nil) if err != nil { return err } req.Header.Set("Authorization", config.gogsconfig.AuthToken) req.Header.Set("User-Agent", config.globalconfig.UserAgent) client := &http.Client{} resp, err := client.Do(req) if err != nil { return err } if resp.StatusCode != 200 { err = errors.New("Error invoking gogs webservice") return err } respbody, err := ioutil.ReadAll(resp.Body) err = json.Unmarshal(respbody, &gogsorg) if err != nil { err = errors.New("Failed to parse user ID from gogs webservice, check auth") return err } config.gogsconfig.UID = gogsorg.ID return nil } // RunMigration run threads for migration func RunMigration(config *Config, repolist []GitHubRepo) { repochan := make(chan GitHubRepo) done := make(chan bool) var wg sync.WaitGroup for thr := range make([]int, config.globalconfig.Threads) { go MigrateReposToGogs(config, &wg, repochan, done, thr) } for _, repo := range repolist { repochan <- repo } close(repochan) <-done wg.Wait() } // MigrateReposToGogs migrates input repositories to gogs func MigrateReposToGogs(config *Config, wg *sync.WaitGroup, jobs chan GitHubRepo, done chan bool, thr int) error { wg.Add(1) for { elem, more := <-jobs if more { client := &http.Client{} existingrepo, err := CheckGogsExistingRepo(config, elem) if err != nil { return err } if !existingrepo { jsondata := fmt.Sprintf(`{"uid" : %d, "repo_name" : "%s" , "mirror" : %v, "clone_addr" : "%s"}`, config.gogsconfig.UID, elem.Name, true, elem.CloneURL) req, err := http.NewRequest("POST", config.gogsconfig.MigrateURL, bytes.NewBufferString(jsondata)) if err != nil { return err } req.Header.Add("Content-Type", config.gogsconfig.ContentType) req.Header.Set("User-Agent", config.globalconfig.UserAgent) req.Header.Set("Authorization", config.gogsconfig.AuthToken) client.Timeout = config.globalconfig.RequestTimeout fmt.Println(fmt.Sprintf("Migrating repo %s to gogs rest api", elem.Name)) resp, err := client.Do(req) if err != nil { return err } if resp.StatusCode != 201 { err = fmt.Errorf("Error when migrating repo %s to gogs with status code %d on gogs webservice", elem.Name, resp.StatusCode) log.Println(err) } } else { fmt.Println(fmt.Sprintf("Not migrating, %s exists in gogs", elem.Name)) } } else { fmt.Println(fmt.Sprintf("All repo migrated on thread num %d", thr)) wg.Done() done <- true return nil } } } // Usage displays possible arguments func Usage() { flag.PrintDefaults() os.Exit(1) }