paulbsd/g2g
1
0
Fork 0
Code Issues Pull Requests Releases 4 Wiki Activity

renamed github-to-gogs to g2g, change configuration structure

Browse Source
This commit is contained in:
Paul 2019-08-24 21:53:02 +02:00
parent a24c3933b6
commit 948d2a5eda
7 changed files with 113 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
README.md
View File

@ -1,8 +1,8 @@
# github-to-gogs # g2g
## Summary ## Summary
github-to-gogs is a small program that migrate stars made on github projects to self owned gogs/gitea instance g2g is a small program that migrate stars made on github projects to self owned gogs/gitea instance
## Howto ## Howto
@ -12,37 +12,33 @@ github-to-gogs is a small program that migrate stars made on github projects to
go build go build
``` ```
### Sample config in github-to-gogs.ini ### Sample config in g2g.ini
```ini ```ini
[global] [g2g]
request_timeout=1200s request_timeout=1200s
user_agent="Golang" user_agent="Golang"
threads=4 threads=4
[github] github_stars_pages="https://api.github.com/users/%s/starred?page=%d"
stars_pages="https://api.github.com/users/%s/starred?page=%d" github_max_per_page=500
max_per_page=500 github_page_num=3
page_num=3 github_auth_username="user"
auth_username="user" github_auth_password="pass"
auth_password="pass"
content_type="application/x-www-form-urlencoded"
[gogs] gitea_username="user"
username="user" gitea_dest_username="user_or_org"
dest_username="user_or_org" gitea_repo_url_tmpl="https://gogs.example.com/api/v1/repos/%s/%s"
repo_url_tmpl="https://gogs.example.com/api/v1/repos/%s/%s" gitea_orgs_url_tmpl="https://git.paulbsd.com/api/v1/orgs/%s"
orgs_url_tmpl="https://git.paulbsd.com/api/v1/orgs/%s" gitea_migrate_url="https://gogs.example.com/api/v1/repos/migrate"
migrate_url="https://gogs.example.com/api/v1/repos/migrate" gitea_auth_token="token xxxx"
auth_token="token xxxx" gitea_mirror=true
content_type="application/json"
mirror=true
``` ```
### Run ### Run
```bash ```bash
./github-to-gogs -configfile github-to-gogs.ini ./g2g -configfile g2g.ini
``` ```
## License ## License

139
functions.go
View File

@ -19,13 +19,9 @@ import (
func (config *Config) GetConfig() error { func (config *Config) GetConfig() error {
var configfile string var configfile string
var globalconfig GlobalConfig
var githubconfig GitHubConfig
var gogsconfig GogsConfig
flag.Usage = Usage flag.Usage = Usage
flag.StringVar(&configfile, "configfile", "github-to-gogs.ini", "config file to use with github-to-gogs section") flag.StringVar(&configfile, "configfile", "g2g.ini", "config file to use with g2g section")
flag.Parse() flag.Parse()
cfg, err := ini.Load(configfile) cfg, err := ini.Load(configfile)
@ -33,50 +29,25 @@ func (config *Config) GetConfig() error {
return err return err
} }
globalsection := cfg.Section("global") g2gsection := cfg.Section("g2g")
githubsection := cfg.Section("github")
gogssection := cfg.Section("gogs") config.UserAgent = g2gsection.Key("user_agent").MustString("Golang")
config.RequestTimeout = g2gsection.Key("request_timeout").MustDuration(1200)
config.Threads = g2gsection.Key("threads").MustInt(2)
config.GitHubPageNum = g2gsection.Key("github_page_num").MustInt(3)
config.GitHubMaxPerPage = g2gsection.Key("github_max_per_page").MustInt(100)
config.GitHubAuthUsername = g2gsection.Key("github_auth_username").MustString("username")
config.GitHubAuthPassword = g2gsection.Key("github_auth_password").MustString("password")
config.GiteaUsername = g2gsection.Key("gitea_username").MustString("username")
config.GiteaDestUsername = g2gsection.Key("gitea_dest_username").MustString("dest_username")
config.GiteaRepoURLTmpl = g2gsection.Key("gitea_repo_url_tmpl").MustString("repo_url")
config.GiteaOrgsURLTmpl = g2gsection.Key("gitea_orgs_url_tmpl").MustString("orgs_url")
config.GiteaMigrateURL = g2gsection.Key("gitea_migrate_url").MustString("migrate_url")
config.GiteaAuthToken = g2gsection.Key("gitea_auth_token").MustString("token")
config.GiteaMirror = g2gsection.Key("gitea_mirror").MustBool(true)
globalconfig.UserAgent = globalsection.Key("user_agent").String()
globalconfig.RequestTimeout, err = globalsection.Key("request_timeout").Duration()
if err != nil {
return err return err
}
globalconfig.Threads, err = globalsection.Key("threads").Int()
if err != nil {
return err
}
githubconfig.PageNum, err = githubsection.Key("page_num").Int()
if err != nil {
return err
}
githubconfig.MaxPerPage, err = githubsection.Key("max_per_page").Int()
if err != nil {
return err
}
githubconfig.AuthUsername = githubsection.Key("auth_username").String()
githubconfig.AuthPassword = githubsection.Key("auth_password").String()
githubconfig.ContentType = githubsection.Key("content_type").String()
gogsconfig.Username = gogssection.Key("username").String()
gogsconfig.DestUsername = gogssection.Key("dest_username").String()
gogsconfig.RepoURLTmpl = gogssection.Key("repo_url_tmpl").String()
gogsconfig.OrgsURLTmpl = gogssection.Key("orgs_url_tmpl").String()
gogsconfig.MigrateURL = gogssection.Key("migrate_url").String()
gogsconfig.AuthToken = gogssection.Key("auth_token").String()
gogsconfig.ContentType = gogssection.Key("content_type").String()
gogsconfig.Mirror, err = gogssection.Key("mirror").Bool()
if err != nil {
return err
}
*config = Config{globalconfig: globalconfig,
githubconfig: githubconfig,
gogsconfig: gogsconfig}
return nil
} }
// GetReposFromGitHub get starred repositories from github // GetReposFromGitHub get starred repositories from github
@ -85,8 +56,8 @@ func GetReposFromGitHub(config *Config) ([]GitHubRepo, error) {
var repofulllist []GitHubRepo var repofulllist []GitHubRepo
fmt.Println("Getting GitHub starred repos") fmt.Println("Getting GitHub starred repos")
for num := 1; num <= config.githubconfig.PageNum; num++ { for num := 1; num <= config.GitHubPageNum; num++ {
url := fmt.Sprintf("https://api.github.com/users/%s/starred?per_page=%d&page=%d", config.githubconfig.AuthUsername, config.githubconfig.MaxPerPage, num) url := fmt.Sprintf("https://api.github.com/users/%s/starred?per_page=%d&page=%d", config.GitHubAuthUsername, config.GitHubMaxPerPage, num)
fmt.Println(url) fmt.Println(url)
resp, err := InvokeGitHub(config, url) resp, err := InvokeGitHub(config, url)
@ -120,9 +91,9 @@ func InvokeGitHub(config *Config, url string) (*http.Response, error) {
return nil, err return nil, err
} }
req.Header.Add("Content-Type", config.githubconfig.ContentType) req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("User-Agent", config.globalconfig.UserAgent) req.Header.Set("User-Agent", config.UserAgent)
req.SetBasicAuth(config.githubconfig.AuthUsername, config.githubconfig.AuthPassword) req.SetBasicAuth(config.GitHubAuthUsername, config.GitHubAuthPassword)
client := &http.Client{} client := &http.Client{}
resp, err := client.Do(req) resp, err := client.Do(req)
@ -133,15 +104,15 @@ func InvokeGitHub(config *Config, url string) (*http.Response, error) {
return resp, nil return resp, nil
} }
// CheckGogsExistingRepo checks if there's an existing repository in gogs // CheckGiteaExistingRepo checks if there's an existing repository in gitea
func CheckGogsExistingRepo(config *Config, repo GitHubRepo) (bool, error) { func CheckGiteaExistingRepo(config *Config, repo GitHubRepo) (bool, error) {
var isexists bool var isexists bool
gogsrepourl := fmt.Sprintf(config.gogsconfig.RepoURLTmpl, config.gogsconfig.DestUsername, repo.Name) gitearepourl := fmt.Sprintf(config.GiteaRepoURLTmpl, config.GiteaDestUsername, repo.Name)
req, err := http.NewRequest("GET", gogsrepourl, nil) req, err := http.NewRequest("GET", gitearepourl, nil)
req.Header.Set("Authorization", config.gogsconfig.AuthToken) req.Header.Set("Authorization", config.GiteaAuthToken)
req.Header.Set("User-Agent", config.globalconfig.UserAgent) req.Header.Set("User-Agent", config.UserAgent)
client := &http.Client{} client := &http.Client{}
resp, err := client.Do(req) resp, err := client.Do(req)
@ -154,26 +125,26 @@ func CheckGogsExistingRepo(config *Config, repo GitHubRepo) (bool, error) {
} else if resp.StatusCode == 404 { } else if resp.StatusCode == 404 {
isexists = false isexists = false
} else { } else {
err = fmt.Errorf("Can't determine error, cancelling, error %d in gogs webservice", resp.StatusCode) err = fmt.Errorf("Can't determine error, cancelling, error %d in gitea webservice", resp.StatusCode)
return false, err return false, err
} }
return isexists, nil return isexists, nil
} }
// GetGogsUserUID get the logued user identifier // GetGiteaUserUID get the logued user identifier
func (config *Config) GetGogsUserUID() error { func (config *Config) GetGiteaUserUID() error {
var gogsorg GogsOrg var giteaorg GiteaOrg
gogsrepourl := fmt.Sprintf(config.gogsconfig.OrgsURLTmpl, config.gogsconfig.DestUsername) gitearepourl := fmt.Sprintf(config.GiteaOrgsURLTmpl, config.GiteaDestUsername)
req, err := http.NewRequest("GET", gogsrepourl, nil) req, err := http.NewRequest("GET", gitearepourl, nil)
if err != nil { if err != nil {
return err return err
} }
req.Header.Set("Authorization", config.gogsconfig.AuthToken) req.Header.Set("Authorization", config.GiteaAuthToken)
req.Header.Set("User-Agent", config.globalconfig.UserAgent) req.Header.Set("User-Agent", config.UserAgent)
client := &http.Client{} client := &http.Client{}
resp, err := client.Do(req) resp, err := client.Do(req)
@ -182,19 +153,19 @@ func (config *Config) GetGogsUserUID() error {
} }
if resp.StatusCode != 200 { if resp.StatusCode != 200 {
err = errors.New("Error invoking gogs webservice") err = errors.New("Error invoking gitea webservice")
return err return err
} }
respbody, err := ioutil.ReadAll(resp.Body) respbody, err := ioutil.ReadAll(resp.Body)
err = json.Unmarshal(respbody, &gogsorg) err = json.Unmarshal(respbody, &giteaorg)
if err != nil { if err != nil {
err = errors.New("Failed to parse user ID from gogs webservice, check auth") err = errors.New("Failed to parse user ID from gitea webservice, check auth")
return err return err
} }
config.gogsconfig.UID = gogsorg.ID config.GiteaUID = giteaorg.ID
return nil return nil
} }
@ -205,8 +176,8 @@ func RunMigration(config *Config, repolist []GitHubRepo) {
done := make(chan bool) done := make(chan bool)
var wg sync.WaitGroup var wg sync.WaitGroup
for thr := range make([]int, config.globalconfig.Threads) { for thr := range make([]int, config.Threads) {
go MigrateReposToGogs(config, &wg, repochan, done, thr) go MigrateReposToGitea(config, &wg, repochan, done, thr)
} }
for _, repo := range repolist { for _, repo := range repolist {
@ -217,33 +188,33 @@ func RunMigration(config *Config, repolist []GitHubRepo) {
wg.Wait() wg.Wait()
} }
// MigrateReposToGogs migrates input repositories to gogs // MigrateReposToGitea migrates input repositories to gitea
func MigrateReposToGogs(config *Config, wg *sync.WaitGroup, jobs chan GitHubRepo, done chan bool, thr int) error { func MigrateReposToGitea(config *Config, wg *sync.WaitGroup, jobs chan GitHubRepo, done chan bool, thr int) error {
wg.Add(1) wg.Add(1)
for { for {
elem, more := <-jobs elem, more := <-jobs
if more { if more {
client := &http.Client{} client := &http.Client{}
existingrepo, err := CheckGogsExistingRepo(config, elem) existingrepo, err := CheckGiteaExistingRepo(config, elem)
if err != nil { if err != nil {
return err return err
} }
if !existingrepo { if !existingrepo {
jsondata := fmt.Sprintf(`{"uid" : %d, "repo_name" : "%s" , "mirror" : %v, "clone_addr" : "%s"}`, config.gogsconfig.UID, elem.Name, true, elem.CloneURL) jsondata := fmt.Sprintf(`{"uid" : %d, "repo_name" : "%s" , "mirror" : %v, "clone_addr" : "%s"}`, config.GiteaUID, elem.Name, true, elem.CloneURL)
req, err := http.NewRequest("POST", config.gogsconfig.MigrateURL, bytes.NewBufferString(jsondata)) req, err := http.NewRequest("POST", config.GiteaMigrateURL, bytes.NewBufferString(jsondata))
if err != nil { if err != nil {
return err return err
} }
req.Header.Add("Content-Type", config.gogsconfig.ContentType) req.Header.Add("Content-Type", "application/json")
req.Header.Set("User-Agent", config.globalconfig.UserAgent) req.Header.Set("User-Agent", config.UserAgent)
req.Header.Set("Authorization", config.gogsconfig.AuthToken) req.Header.Set("Authorization", config.GiteaAuthToken)
client.Timeout = config.globalconfig.RequestTimeout client.Timeout = config.RequestTimeout
fmt.Println(fmt.Sprintf("Migrating repo %s to gogs rest api", elem.Name)) fmt.Println(fmt.Sprintf("Migrating repo %s to gitea rest api", elem.Name))
resp, err := client.Do(req) resp, err := client.Do(req)
if err != nil { if err != nil {
@ -251,11 +222,11 @@ func MigrateReposToGogs(config *Config, wg *sync.WaitGroup, jobs chan GitHubRepo
} }
if resp.StatusCode != 201 { if resp.StatusCode != 201 {
err = fmt.Errorf("Error when migrating repo %s to gogs with status code %d on gogs webservice", elem.Name, resp.StatusCode) err = fmt.Errorf("Error when migrating repo %s to gitea with status code %d on gitea webservice", elem.Name, resp.StatusCode)
log.Println(err) log.Println(err)
} }
} else { } else {
fmt.Println(fmt.Sprintf("Not migrating, %s exists in gogs", elem.Name)) fmt.Println(fmt.Sprintf("Not migrating, %s exists in gitea", elem.Name))
} }
} else { } else {
fmt.Println(fmt.Sprintf("All repo migrated on thread num %d", thr)) fmt.Println(fmt.Sprintf("All repo migrated on thread num %d", thr))

2
github-to-gogs.go → g2g.go
View File

@ -13,7 +13,7 @@ func main() {
log.Fatal(err) log.Fatal(err)
} }
err = config.GetGogsUserUID() err = config.GetGiteaUserUID()
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }

18
g2g.ini.sample Normal file
View File

@ -0,0 +1,18 @@
[g2g]
request_timeout=1200s
user_agent="Golang"
threads=4
github_stars_pages="https://api.github.com/users/%s/starred?page=%d"
github_max_per_page=500
github_page_num=3
github_auth_username="user"
github_auth_password="pass"
gitea_username="user"
gitea_dest_username="user_or_org"
gitea_repo_url_tmpl="https://gogs.example.com/api/v1/repos/%s/%s"
gitea_orgs_url_tmpl="https://git.paulbsd.com/api/v1/orgs/%s"
gitea_migrate_url="https://gogs.example.com/api/v1/repos/migrate"
gitea_auth_token="token xxxx"
gitea_mirror=true

20
github-to-gogs.ini.sample
View File

@ -1,20 +0,0 @@
[global]
request_timeout=1200s
user_agent="Golang"
[github]
stars_pages="https://api.github.com/users/%s/starred?page=%d"
max_pages_number=8
auth_username="user"
auth_password="pass"
content_type="application/x-www-form-urlencoded"
[gogs]
username="user"
dest_username="user_or_org"
repo_url_tmpl="https://gogs.example.com/api/v1/repos/%s/%s"
orgs_url_tmpl="https://git.paulbsd.com/api/v1/orgs/%s"
migrate_url="https://gogs.example.com/api/v1/repos/migrate"
auth_token="token xxxx"
content_type="application/json"
mirror=true

2
go.mod
View File

@ -1,4 +1,4 @@
module github-to-gogs module g2g
go 1.12 go 1.12

53
types.go
View File

@ -8,52 +8,37 @@ type GitHubRepo struct {
CloneURL string `json:"clone_url"` CloneURL string `json:"clone_url"`
} }
// GogsOrg gogsrepo struct // GiteaOrg gitearepo struct
type GogsOrg struct { type GiteaOrg struct {
ID int `json:"id"` ID int `json:"id"`
Username string `json:"username"` Username string `json:"username"`
} }
// GogsMigrateRepo defines the repo to migrate // GiteaMigrateRepo defines the repo to migrate
type GogsMigrateRepo struct { type GiteaMigrateRepo struct {
Name string Name string
CloneURL string CloneURL string
UID int UID int
Mirror bool Mirror bool
} }
// Config merge all config elements // Config is the global config of g2g
type Config struct { type Config struct {
globalconfig GlobalConfig
gogsconfig GogsConfig
githubconfig GitHubConfig
}
// GlobalConfig is the global config of github-to-gogs
type GlobalConfig struct {
RequestTimeout time.Duration RequestTimeout time.Duration
UserAgent string UserAgent string
Threads int Threads int
} GitHubMaxPerPage int
GitHubPageNum int
// GitHubConfig is the GitHub config GitHubAuthUsername string
type GitHubConfig struct { GitHubAuthPassword string
MaxPerPage int GitHubContentType string
PageNum int GiteaUID int
AuthUsername string GiteaUsername string
AuthPassword string GiteaDestUsername string
ContentType string GiteaRepoURLTmpl string
} GiteaOrgsURLTmpl string
GiteaMigrateURL string
// GogsConfig is the Gogs config GiteaAuthToken string
type GogsConfig struct { GiteaContentType string
UID int GiteaMirror bool
Username string
DestUsername string
RepoURLTmpl string
OrgsURLTmpl string
MigrateURL string
AuthToken string
ContentType string
Mirror bool
} }