renamed github-to-gogs to g2g, change configuration structure
This commit is contained in:
parent
a24c3933b6
commit
948d2a5eda
38
README.md
38
README.md
@ -1,8 +1,8 @@
|
||||
# github-to-gogs
|
||||
# g2g
|
||||
|
||||
## Summary
|
||||
|
||||
github-to-gogs is a small program that migrate stars made on github projects to self owned gogs/gitea instance
|
||||
g2g is a small program that migrate stars made on github projects to self owned gogs/gitea instance
|
||||
|
||||
## Howto
|
||||
|
||||
@ -12,37 +12,33 @@ github-to-gogs is a small program that migrate stars made on github projects to
|
||||
go build
|
||||
```
|
||||
|
||||
### Sample config in github-to-gogs.ini
|
||||
### Sample config in g2g.ini
|
||||
|
||||
```ini
|
||||
[global]
|
||||
[g2g]
|
||||
request_timeout=1200s
|
||||
user_agent="Golang"
|
||||
threads=4
|
||||
|
||||
[github]
|
||||
stars_pages="https://api.github.com/users/%s/starred?page=%d"
|
||||
max_per_page=500
|
||||
page_num=3
|
||||
auth_username="user"
|
||||
auth_password="pass"
|
||||
content_type="application/x-www-form-urlencoded"
|
||||
github_stars_pages="https://api.github.com/users/%s/starred?page=%d"
|
||||
github_max_per_page=500
|
||||
github_page_num=3
|
||||
github_auth_username="user"
|
||||
github_auth_password="pass"
|
||||
|
||||
[gogs]
|
||||
username="user"
|
||||
dest_username="user_or_org"
|
||||
repo_url_tmpl="https://gogs.example.com/api/v1/repos/%s/%s"
|
||||
orgs_url_tmpl="https://git.paulbsd.com/api/v1/orgs/%s"
|
||||
migrate_url="https://gogs.example.com/api/v1/repos/migrate"
|
||||
auth_token="token xxxx"
|
||||
content_type="application/json"
|
||||
mirror=true
|
||||
gitea_username="user"
|
||||
gitea_dest_username="user_or_org"
|
||||
gitea_repo_url_tmpl="https://gogs.example.com/api/v1/repos/%s/%s"
|
||||
gitea_orgs_url_tmpl="https://git.paulbsd.com/api/v1/orgs/%s"
|
||||
gitea_migrate_url="https://gogs.example.com/api/v1/repos/migrate"
|
||||
gitea_auth_token="token xxxx"
|
||||
gitea_mirror=true
|
||||
```
|
||||
|
||||
### Run
|
||||
|
||||
```bash
|
||||
./github-to-gogs -configfile github-to-gogs.ini
|
||||
./g2g -configfile g2g.ini
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
139
functions.go
139
functions.go
@ -19,13 +19,9 @@ import (
|
||||
func (config *Config) GetConfig() error {
|
||||
var configfile string
|
||||
|
||||
var globalconfig GlobalConfig
|
||||
var githubconfig GitHubConfig
|
||||
var gogsconfig GogsConfig
|
||||
|
||||
flag.Usage = Usage
|
||||
|
||||
flag.StringVar(&configfile, "configfile", "github-to-gogs.ini", "config file to use with github-to-gogs section")
|
||||
flag.StringVar(&configfile, "configfile", "g2g.ini", "config file to use with g2g section")
|
||||
flag.Parse()
|
||||
|
||||
cfg, err := ini.Load(configfile)
|
||||
@ -33,51 +29,26 @@ func (config *Config) GetConfig() error {
|
||||
return err
|
||||
}
|
||||
|
||||
globalsection := cfg.Section("global")
|
||||
githubsection := cfg.Section("github")
|
||||
gogssection := cfg.Section("gogs")
|
||||
g2gsection := cfg.Section("g2g")
|
||||
|
||||
config.UserAgent = g2gsection.Key("user_agent").MustString("Golang")
|
||||
config.RequestTimeout = g2gsection.Key("request_timeout").MustDuration(1200)
|
||||
config.Threads = g2gsection.Key("threads").MustInt(2)
|
||||
config.GitHubPageNum = g2gsection.Key("github_page_num").MustInt(3)
|
||||
config.GitHubMaxPerPage = g2gsection.Key("github_max_per_page").MustInt(100)
|
||||
config.GitHubAuthUsername = g2gsection.Key("github_auth_username").MustString("username")
|
||||
config.GitHubAuthPassword = g2gsection.Key("github_auth_password").MustString("password")
|
||||
|
||||
config.GiteaUsername = g2gsection.Key("gitea_username").MustString("username")
|
||||
config.GiteaDestUsername = g2gsection.Key("gitea_dest_username").MustString("dest_username")
|
||||
config.GiteaRepoURLTmpl = g2gsection.Key("gitea_repo_url_tmpl").MustString("repo_url")
|
||||
config.GiteaOrgsURLTmpl = g2gsection.Key("gitea_orgs_url_tmpl").MustString("orgs_url")
|
||||
config.GiteaMigrateURL = g2gsection.Key("gitea_migrate_url").MustString("migrate_url")
|
||||
config.GiteaAuthToken = g2gsection.Key("gitea_auth_token").MustString("token")
|
||||
config.GiteaMirror = g2gsection.Key("gitea_mirror").MustBool(true)
|
||||
|
||||
globalconfig.UserAgent = globalsection.Key("user_agent").String()
|
||||
globalconfig.RequestTimeout, err = globalsection.Key("request_timeout").Duration()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
globalconfig.Threads, err = globalsection.Key("threads").Int()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
githubconfig.PageNum, err = githubsection.Key("page_num").Int()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
githubconfig.MaxPerPage, err = githubsection.Key("max_per_page").Int()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
githubconfig.AuthUsername = githubsection.Key("auth_username").String()
|
||||
githubconfig.AuthPassword = githubsection.Key("auth_password").String()
|
||||
githubconfig.ContentType = githubsection.Key("content_type").String()
|
||||
|
||||
gogsconfig.Username = gogssection.Key("username").String()
|
||||
gogsconfig.DestUsername = gogssection.Key("dest_username").String()
|
||||
gogsconfig.RepoURLTmpl = gogssection.Key("repo_url_tmpl").String()
|
||||
gogsconfig.OrgsURLTmpl = gogssection.Key("orgs_url_tmpl").String()
|
||||
gogsconfig.MigrateURL = gogssection.Key("migrate_url").String()
|
||||
gogsconfig.AuthToken = gogssection.Key("auth_token").String()
|
||||
gogsconfig.ContentType = gogssection.Key("content_type").String()
|
||||
gogsconfig.Mirror, err = gogssection.Key("mirror").Bool()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
*config = Config{globalconfig: globalconfig,
|
||||
githubconfig: githubconfig,
|
||||
gogsconfig: gogsconfig}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetReposFromGitHub get starred repositories from github
|
||||
func GetReposFromGitHub(config *Config) ([]GitHubRepo, error) {
|
||||
@ -85,8 +56,8 @@ func GetReposFromGitHub(config *Config) ([]GitHubRepo, error) {
|
||||
var repofulllist []GitHubRepo
|
||||
|
||||
fmt.Println("Getting GitHub starred repos")
|
||||
for num := 1; num <= config.githubconfig.PageNum; num++ {
|
||||
url := fmt.Sprintf("https://api.github.com/users/%s/starred?per_page=%d&page=%d", config.githubconfig.AuthUsername, config.githubconfig.MaxPerPage, num)
|
||||
for num := 1; num <= config.GitHubPageNum; num++ {
|
||||
url := fmt.Sprintf("https://api.github.com/users/%s/starred?per_page=%d&page=%d", config.GitHubAuthUsername, config.GitHubMaxPerPage, num)
|
||||
fmt.Println(url)
|
||||
|
||||
resp, err := InvokeGitHub(config, url)
|
||||
@ -120,9 +91,9 @@ func InvokeGitHub(config *Config, url string) (*http.Response, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Add("Content-Type", config.githubconfig.ContentType)
|
||||
req.Header.Set("User-Agent", config.globalconfig.UserAgent)
|
||||
req.SetBasicAuth(config.githubconfig.AuthUsername, config.githubconfig.AuthPassword)
|
||||
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
|
||||
req.Header.Set("User-Agent", config.UserAgent)
|
||||
req.SetBasicAuth(config.GitHubAuthUsername, config.GitHubAuthPassword)
|
||||
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(req)
|
||||
@ -133,15 +104,15 @@ func InvokeGitHub(config *Config, url string) (*http.Response, error) {
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// CheckGogsExistingRepo checks if there's an existing repository in gogs
|
||||
func CheckGogsExistingRepo(config *Config, repo GitHubRepo) (bool, error) {
|
||||
// CheckGiteaExistingRepo checks if there's an existing repository in gitea
|
||||
func CheckGiteaExistingRepo(config *Config, repo GitHubRepo) (bool, error) {
|
||||
var isexists bool
|
||||
|
||||
gogsrepourl := fmt.Sprintf(config.gogsconfig.RepoURLTmpl, config.gogsconfig.DestUsername, repo.Name)
|
||||
gitearepourl := fmt.Sprintf(config.GiteaRepoURLTmpl, config.GiteaDestUsername, repo.Name)
|
||||
|
||||
req, err := http.NewRequest("GET", gogsrepourl, nil)
|
||||
req.Header.Set("Authorization", config.gogsconfig.AuthToken)
|
||||
req.Header.Set("User-Agent", config.globalconfig.UserAgent)
|
||||
req, err := http.NewRequest("GET", gitearepourl, nil)
|
||||
req.Header.Set("Authorization", config.GiteaAuthToken)
|
||||
req.Header.Set("User-Agent", config.UserAgent)
|
||||
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(req)
|
||||
@ -154,26 +125,26 @@ func CheckGogsExistingRepo(config *Config, repo GitHubRepo) (bool, error) {
|
||||
} else if resp.StatusCode == 404 {
|
||||
isexists = false
|
||||
} else {
|
||||
err = fmt.Errorf("Can't determine error, cancelling, error %d in gogs webservice", resp.StatusCode)
|
||||
err = fmt.Errorf("Can't determine error, cancelling, error %d in gitea webservice", resp.StatusCode)
|
||||
return false, err
|
||||
}
|
||||
|
||||
return isexists, nil
|
||||
}
|
||||
|
||||
// GetGogsUserUID get the logued user identifier
|
||||
func (config *Config) GetGogsUserUID() error {
|
||||
var gogsorg GogsOrg
|
||||
// GetGiteaUserUID get the logued user identifier
|
||||
func (config *Config) GetGiteaUserUID() error {
|
||||
var giteaorg GiteaOrg
|
||||
|
||||
gogsrepourl := fmt.Sprintf(config.gogsconfig.OrgsURLTmpl, config.gogsconfig.DestUsername)
|
||||
gitearepourl := fmt.Sprintf(config.GiteaOrgsURLTmpl, config.GiteaDestUsername)
|
||||
|
||||
req, err := http.NewRequest("GET", gogsrepourl, nil)
|
||||
req, err := http.NewRequest("GET", gitearepourl, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req.Header.Set("Authorization", config.gogsconfig.AuthToken)
|
||||
req.Header.Set("User-Agent", config.globalconfig.UserAgent)
|
||||
req.Header.Set("Authorization", config.GiteaAuthToken)
|
||||
req.Header.Set("User-Agent", config.UserAgent)
|
||||
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(req)
|
||||
@ -182,19 +153,19 @@ func (config *Config) GetGogsUserUID() error {
|
||||
}
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
err = errors.New("Error invoking gogs webservice")
|
||||
err = errors.New("Error invoking gitea webservice")
|
||||
return err
|
||||
}
|
||||
|
||||
respbody, err := ioutil.ReadAll(resp.Body)
|
||||
|
||||
err = json.Unmarshal(respbody, &gogsorg)
|
||||
err = json.Unmarshal(respbody, &giteaorg)
|
||||
if err != nil {
|
||||
err = errors.New("Failed to parse user ID from gogs webservice, check auth")
|
||||
err = errors.New("Failed to parse user ID from gitea webservice, check auth")
|
||||
return err
|
||||
}
|
||||
|
||||
config.gogsconfig.UID = gogsorg.ID
|
||||
config.GiteaUID = giteaorg.ID
|
||||
|
||||
return nil
|
||||
}
|
||||
@ -205,8 +176,8 @@ func RunMigration(config *Config, repolist []GitHubRepo) {
|
||||
done := make(chan bool)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for thr := range make([]int, config.globalconfig.Threads) {
|
||||
go MigrateReposToGogs(config, &wg, repochan, done, thr)
|
||||
for thr := range make([]int, config.Threads) {
|
||||
go MigrateReposToGitea(config, &wg, repochan, done, thr)
|
||||
}
|
||||
|
||||
for _, repo := range repolist {
|
||||
@ -217,33 +188,33 @@ func RunMigration(config *Config, repolist []GitHubRepo) {
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// MigrateReposToGogs migrates input repositories to gogs
|
||||
func MigrateReposToGogs(config *Config, wg *sync.WaitGroup, jobs chan GitHubRepo, done chan bool, thr int) error {
|
||||
// MigrateReposToGitea migrates input repositories to gitea
|
||||
func MigrateReposToGitea(config *Config, wg *sync.WaitGroup, jobs chan GitHubRepo, done chan bool, thr int) error {
|
||||
wg.Add(1)
|
||||
for {
|
||||
elem, more := <-jobs
|
||||
if more {
|
||||
client := &http.Client{}
|
||||
|
||||
existingrepo, err := CheckGogsExistingRepo(config, elem)
|
||||
existingrepo, err := CheckGiteaExistingRepo(config, elem)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !existingrepo {
|
||||
jsondata := fmt.Sprintf(`{"uid" : %d, "repo_name" : "%s" , "mirror" : %v, "clone_addr" : "%s"}`, config.gogsconfig.UID, elem.Name, true, elem.CloneURL)
|
||||
jsondata := fmt.Sprintf(`{"uid" : %d, "repo_name" : "%s" , "mirror" : %v, "clone_addr" : "%s"}`, config.GiteaUID, elem.Name, true, elem.CloneURL)
|
||||
|
||||
req, err := http.NewRequest("POST", config.gogsconfig.MigrateURL, bytes.NewBufferString(jsondata))
|
||||
req, err := http.NewRequest("POST", config.GiteaMigrateURL, bytes.NewBufferString(jsondata))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req.Header.Add("Content-Type", config.gogsconfig.ContentType)
|
||||
req.Header.Set("User-Agent", config.globalconfig.UserAgent)
|
||||
req.Header.Set("Authorization", config.gogsconfig.AuthToken)
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Set("User-Agent", config.UserAgent)
|
||||
req.Header.Set("Authorization", config.GiteaAuthToken)
|
||||
|
||||
client.Timeout = config.globalconfig.RequestTimeout
|
||||
client.Timeout = config.RequestTimeout
|
||||
|
||||
fmt.Println(fmt.Sprintf("Migrating repo %s to gogs rest api", elem.Name))
|
||||
fmt.Println(fmt.Sprintf("Migrating repo %s to gitea rest api", elem.Name))
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
@ -251,11 +222,11 @@ func MigrateReposToGogs(config *Config, wg *sync.WaitGroup, jobs chan GitHubRepo
|
||||
}
|
||||
|
||||
if resp.StatusCode != 201 {
|
||||
err = fmt.Errorf("Error when migrating repo %s to gogs with status code %d on gogs webservice", elem.Name, resp.StatusCode)
|
||||
err = fmt.Errorf("Error when migrating repo %s to gitea with status code %d on gitea webservice", elem.Name, resp.StatusCode)
|
||||
log.Println(err)
|
||||
}
|
||||
} else {
|
||||
fmt.Println(fmt.Sprintf("Not migrating, %s exists in gogs", elem.Name))
|
||||
fmt.Println(fmt.Sprintf("Not migrating, %s exists in gitea", elem.Name))
|
||||
}
|
||||
} else {
|
||||
fmt.Println(fmt.Sprintf("All repo migrated on thread num %d", thr))
|
||||
|
||||
@ -13,7 +13,7 @@ func main() {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
err = config.GetGogsUserUID()
|
||||
err = config.GetGiteaUserUID()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
18
g2g.ini.sample
Normal file
18
g2g.ini.sample
Normal file
@ -0,0 +1,18 @@
|
||||
[g2g]
|
||||
request_timeout=1200s
|
||||
user_agent="Golang"
|
||||
threads=4
|
||||
|
||||
github_stars_pages="https://api.github.com/users/%s/starred?page=%d"
|
||||
github_max_per_page=500
|
||||
github_page_num=3
|
||||
github_auth_username="user"
|
||||
github_auth_password="pass"
|
||||
|
||||
gitea_username="user"
|
||||
gitea_dest_username="user_or_org"
|
||||
gitea_repo_url_tmpl="https://gogs.example.com/api/v1/repos/%s/%s"
|
||||
gitea_orgs_url_tmpl="https://git.paulbsd.com/api/v1/orgs/%s"
|
||||
gitea_migrate_url="https://gogs.example.com/api/v1/repos/migrate"
|
||||
gitea_auth_token="token xxxx"
|
||||
gitea_mirror=true
|
||||
@ -1,20 +0,0 @@
|
||||
[global]
|
||||
request_timeout=1200s
|
||||
user_agent="Golang"
|
||||
|
||||
[github]
|
||||
stars_pages="https://api.github.com/users/%s/starred?page=%d"
|
||||
max_pages_number=8
|
||||
auth_username="user"
|
||||
auth_password="pass"
|
||||
content_type="application/x-www-form-urlencoded"
|
||||
|
||||
[gogs]
|
||||
username="user"
|
||||
dest_username="user_or_org"
|
||||
repo_url_tmpl="https://gogs.example.com/api/v1/repos/%s/%s"
|
||||
orgs_url_tmpl="https://git.paulbsd.com/api/v1/orgs/%s"
|
||||
migrate_url="https://gogs.example.com/api/v1/repos/migrate"
|
||||
auth_token="token xxxx"
|
||||
content_type="application/json"
|
||||
mirror=true
|
||||
53
types.go
53
types.go
@ -8,52 +8,37 @@ type GitHubRepo struct {
|
||||
CloneURL string `json:"clone_url"`
|
||||
}
|
||||
|
||||
// GogsOrg gogsrepo struct
|
||||
type GogsOrg struct {
|
||||
// GiteaOrg gitearepo struct
|
||||
type GiteaOrg struct {
|
||||
ID int `json:"id"`
|
||||
Username string `json:"username"`
|
||||
}
|
||||
|
||||
// GogsMigrateRepo defines the repo to migrate
|
||||
type GogsMigrateRepo struct {
|
||||
// GiteaMigrateRepo defines the repo to migrate
|
||||
type GiteaMigrateRepo struct {
|
||||
Name string
|
||||
CloneURL string
|
||||
UID int
|
||||
Mirror bool
|
||||
}
|
||||
|
||||
// Config merge all config elements
|
||||
// Config is the global config of g2g
|
||||
type Config struct {
|
||||
globalconfig GlobalConfig
|
||||
gogsconfig GogsConfig
|
||||
githubconfig GitHubConfig
|
||||
}
|
||||
|
||||
// GlobalConfig is the global config of github-to-gogs
|
||||
type GlobalConfig struct {
|
||||
RequestTimeout time.Duration
|
||||
UserAgent string
|
||||
Threads int
|
||||
}
|
||||
|
||||
// GitHubConfig is the GitHub config
|
||||
type GitHubConfig struct {
|
||||
MaxPerPage int
|
||||
PageNum int
|
||||
AuthUsername string
|
||||
AuthPassword string
|
||||
ContentType string
|
||||
}
|
||||
|
||||
// GogsConfig is the Gogs config
|
||||
type GogsConfig struct {
|
||||
UID int
|
||||
Username string
|
||||
DestUsername string
|
||||
RepoURLTmpl string
|
||||
OrgsURLTmpl string
|
||||
MigrateURL string
|
||||
AuthToken string
|
||||
ContentType string
|
||||
Mirror bool
|
||||
GitHubMaxPerPage int
|
||||
GitHubPageNum int
|
||||
GitHubAuthUsername string
|
||||
GitHubAuthPassword string
|
||||
GitHubContentType string
|
||||
GiteaUID int
|
||||
GiteaUsername string
|
||||
GiteaDestUsername string
|
||||
GiteaRepoURLTmpl string
|
||||
GiteaOrgsURLTmpl string
|
||||
GiteaMigrateURL string
|
||||
GiteaAuthToken string
|
||||
GiteaContentType string
|
||||
GiteaMirror bool
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user