paulbsd/g2g
1
0
Fork 0
Code Issues Pull Requests Releases 4 Wiki Activity

renamed github-to-gogs to g2g, change configuration structure

Browse Source
This commit is contained in:
Paul 2019-08-24 21:53:02 +02:00
parent a24c3933b6
commit 948d2a5eda
7 changed files with 113 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
README.md
View File

@ -1,8 +1,8 @@
# github-to-gogs
# g2g
## Summary
github-to-gogs is a small program that migrate stars made on github projects to self owned gogs/gitea instance
g2g is a small program that migrate stars made on github projects to self owned gogs/gitea instance
## Howto
@ -12,37 +12,33 @@ github-to-gogs is a small program that migrate stars made on github projects to
go build
```
### Sample config in github-to-gogs.ini
### Sample config in g2g.ini
```ini
[global]
[g2g]
request_timeout=1200s
user_agent="Golang"
threads=4
[github]
stars_pages="https://api.github.com/users/%s/starred?page=%d"
max_per_page=500
page_num=3
auth_username="user"
auth_password="pass"
content_type="application/x-www-form-urlencoded"
github_stars_pages="https://api.github.com/users/%s/starred?page=%d"
github_max_per_page=500
github_page_num=3
github_auth_username="user"
github_auth_password="pass"
[gogs]
username="user"
dest_username="user_or_org"
repo_url_tmpl="https://gogs.example.com/api/v1/repos/%s/%s"
orgs_url_tmpl="https://git.paulbsd.com/api/v1/orgs/%s"
migrate_url="https://gogs.example.com/api/v1/repos/migrate"
auth_token="token xxxx"
content_type="application/json"
mirror=true
gitea_username="user"
gitea_dest_username="user_or_org"
gitea_repo_url_tmpl="https://gogs.example.com/api/v1/repos/%s/%s"
gitea_orgs_url_tmpl="https://git.paulbsd.com/api/v1/orgs/%s"
gitea_migrate_url="https://gogs.example.com/api/v1/repos/migrate"
gitea_auth_token="token xxxx"
gitea_mirror=true
```
### Run
```bash
./github-to-gogs -configfile github-to-gogs.ini
./g2g -configfile g2g.ini
```
## License

137
functions.go
View File

@ -19,13 +19,9 @@ import (
func (config *Config) GetConfig() error {
var configfile string
var globalconfig GlobalConfig
var githubconfig GitHubConfig
var gogsconfig GogsConfig
flag.Usage = Usage
flag.StringVar(&configfile, "configfile", "github-to-gogs.ini", "config file to use with github-to-gogs section")
flag.StringVar(&configfile, "configfile", "g2g.ini", "config file to use with g2g section")
flag.Parse()
cfg, err := ini.Load(configfile)
@ -33,50 +29,25 @@ func (config *Config) GetConfig() error {
return err
}
globalsection := cfg.Section("global")
githubsection := cfg.Section("github")
gogssection := cfg.Section("gogs")
g2gsection := cfg.Section("g2g")
globalconfig.UserAgent = globalsection.Key("user_agent").String()
globalconfig.RequestTimeout, err = globalsection.Key("request_timeout").Duration()
if err != nil {
return err
}
globalconfig.Threads, err = globalsection.Key("threads").Int()
if err != nil {
return err
}
config.UserAgent = g2gsection.Key("user_agent").MustString("Golang")
config.RequestTimeout = g2gsection.Key("request_timeout").MustDuration(1200)
config.Threads = g2gsection.Key("threads").MustInt(2)
config.GitHubPageNum = g2gsection.Key("github_page_num").MustInt(3)
config.GitHubMaxPerPage = g2gsection.Key("github_max_per_page").MustInt(100)
config.GitHubAuthUsername = g2gsection.Key("github_auth_username").MustString("username")
config.GitHubAuthPassword = g2gsection.Key("github_auth_password").MustString("password")
githubconfig.PageNum, err = githubsection.Key("page_num").Int()
if err != nil {
return err
}
githubconfig.MaxPerPage, err = githubsection.Key("max_per_page").Int()
if err != nil {
return err
}
config.GiteaUsername = g2gsection.Key("gitea_username").MustString("username")
config.GiteaDestUsername = g2gsection.Key("gitea_dest_username").MustString("dest_username")
config.GiteaRepoURLTmpl = g2gsection.Key("gitea_repo_url_tmpl").MustString("repo_url")
config.GiteaOrgsURLTmpl = g2gsection.Key("gitea_orgs_url_tmpl").MustString("orgs_url")
config.GiteaMigrateURL = g2gsection.Key("gitea_migrate_url").MustString("migrate_url")
config.GiteaAuthToken = g2gsection.Key("gitea_auth_token").MustString("token")
config.GiteaMirror = g2gsection.Key("gitea_mirror").MustBool(true)
githubconfig.AuthUsername = githubsection.Key("auth_username").String()
githubconfig.AuthPassword = githubsection.Key("auth_password").String()
githubconfig.ContentType = githubsection.Key("content_type").String()
gogsconfig.Username = gogssection.Key("username").String()
gogsconfig.DestUsername = gogssection.Key("dest_username").String()
gogsconfig.RepoURLTmpl = gogssection.Key("repo_url_tmpl").String()
gogsconfig.OrgsURLTmpl = gogssection.Key("orgs_url_tmpl").String()
gogsconfig.MigrateURL = gogssection.Key("migrate_url").String()
gogsconfig.AuthToken = gogssection.Key("auth_token").String()
gogsconfig.ContentType = gogssection.Key("content_type").String()
gogsconfig.Mirror, err = gogssection.Key("mirror").Bool()
if err != nil {
return err
}
*config = Config{globalconfig: globalconfig,
githubconfig: githubconfig,
gogsconfig: gogsconfig}
return nil
return err
}
// GetReposFromGitHub get starred repositories from github
@ -85,8 +56,8 @@ func GetReposFromGitHub(config *Config) ([]GitHubRepo, error) {
var repofulllist []GitHubRepo
fmt.Println("Getting GitHub starred repos")
for num := 1; num <= config.githubconfig.PageNum; num++ {
url := fmt.Sprintf("https://api.github.com/users/%s/starred?per_page=%d&page=%d", config.githubconfig.AuthUsername, config.githubconfig.MaxPerPage, num)
for num := 1; num <= config.GitHubPageNum; num++ {
url := fmt.Sprintf("https://api.github.com/users/%s/starred?per_page=%d&page=%d", config.GitHubAuthUsername, config.GitHubMaxPerPage, num)
fmt.Println(url)
resp, err := InvokeGitHub(config, url)
@ -120,9 +91,9 @@ func InvokeGitHub(config *Config, url string) (*http.Response, error) {
return nil, err
}
req.Header.Add("Content-Type", config.githubconfig.ContentType)
req.Header.Set("User-Agent", config.globalconfig.UserAgent)
req.SetBasicAuth(config.githubconfig.AuthUsername, config.githubconfig.AuthPassword)
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("User-Agent", config.UserAgent)
req.SetBasicAuth(config.GitHubAuthUsername, config.GitHubAuthPassword)
client := &http.Client{}
resp, err := client.Do(req)
@ -133,15 +104,15 @@ func InvokeGitHub(config *Config, url string) (*http.Response, error) {
return resp, nil
}
// CheckGogsExistingRepo checks if there's an existing repository in gogs
func CheckGogsExistingRepo(config *Config, repo GitHubRepo) (bool, error) {
// CheckGiteaExistingRepo checks if there's an existing repository in gitea
func CheckGiteaExistingRepo(config *Config, repo GitHubRepo) (bool, error) {
var isexists bool
gogsrepourl := fmt.Sprintf(config.gogsconfig.RepoURLTmpl, config.gogsconfig.DestUsername, repo.Name)
gitearepourl := fmt.Sprintf(config.GiteaRepoURLTmpl, config.GiteaDestUsername, repo.Name)
req, err := http.NewRequest("GET", gogsrepourl, nil)
req.Header.Set("Authorization", config.gogsconfig.AuthToken)
req.Header.Set("User-Agent", config.globalconfig.UserAgent)
req, err := http.NewRequest("GET", gitearepourl, nil)
req.Header.Set("Authorization", config.GiteaAuthToken)
req.Header.Set("User-Agent", config.UserAgent)
client := &http.Client{}
resp, err := client.Do(req)
@ -154,26 +125,26 @@ func CheckGogsExistingRepo(config *Config, repo GitHubRepo) (bool, error) {
} else if resp.StatusCode == 404 {
isexists = false
} else {
err = fmt.Errorf("Can't determine error, cancelling, error %d in gogs webservice", resp.StatusCode)
err = fmt.Errorf("Can't determine error, cancelling, error %d in gitea webservice", resp.StatusCode)
return false, err
}
return isexists, nil
}
// GetGogsUserUID get the logued user identifier
func (config *Config) GetGogsUserUID() error {
var gogsorg GogsOrg
// GetGiteaUserUID get the logued user identifier
func (config *Config) GetGiteaUserUID() error {
var giteaorg GiteaOrg
gogsrepourl := fmt.Sprintf(config.gogsconfig.OrgsURLTmpl, config.gogsconfig.DestUsername)
gitearepourl := fmt.Sprintf(config.GiteaOrgsURLTmpl, config.GiteaDestUsername)
req, err := http.NewRequest("GET", gogsrepourl, nil)
req, err := http.NewRequest("GET", gitearepourl, nil)
if err != nil {
return err
}
req.Header.Set("Authorization", config.gogsconfig.AuthToken)
req.Header.Set("User-Agent", config.globalconfig.UserAgent)
req.Header.Set("Authorization", config.GiteaAuthToken)
req.Header.Set("User-Agent", config.UserAgent)
client := &http.Client{}
resp, err := client.Do(req)
@ -182,19 +153,19 @@ func (config *Config) GetGogsUserUID() error {
}
if resp.StatusCode != 200 {
err = errors.New("Error invoking gogs webservice")
err = errors.New("Error invoking gitea webservice")
return err
}
respbody, err := ioutil.ReadAll(resp.Body)
err = json.Unmarshal(respbody, &gogsorg)
err = json.Unmarshal(respbody, &giteaorg)
if err != nil {
err = errors.New("Failed to parse user ID from gogs webservice, check auth")
err = errors.New("Failed to parse user ID from gitea webservice, check auth")
return err
}
config.gogsconfig.UID = gogsorg.ID
config.GiteaUID = giteaorg.ID
return nil
}
@ -205,8 +176,8 @@ func RunMigration(config *Config, repolist []GitHubRepo) {
done := make(chan bool)
var wg sync.WaitGroup
for thr := range make([]int, config.globalconfig.Threads) {
go MigrateReposToGogs(config, &wg, repochan, done, thr)
for thr := range make([]int, config.Threads) {
go MigrateReposToGitea(config, &wg, repochan, done, thr)
}
for _, repo := range repolist {
@ -217,33 +188,33 @@ func RunMigration(config *Config, repolist []GitHubRepo) {
wg.Wait()
}
// MigrateReposToGogs migrates input repositories to gogs
func MigrateReposToGogs(config *Config, wg *sync.WaitGroup, jobs chan GitHubRepo, done chan bool, thr int) error {
// MigrateReposToGitea migrates input repositories to gitea
func MigrateReposToGitea(config *Config, wg *sync.WaitGroup, jobs chan GitHubRepo, done chan bool, thr int) error {
wg.Add(1)
for {
elem, more := <-jobs
if more {
client := &http.Client{}
existingrepo, err := CheckGogsExistingRepo(config, elem)
existingrepo, err := CheckGiteaExistingRepo(config, elem)
if err != nil {
return err
}
if !existingrepo {
jsondata := fmt.Sprintf(`{"uid" : %d, "repo_name" : "%s" , "mirror" : %v, "clone_addr" : "%s"}`, config.gogsconfig.UID, elem.Name, true, elem.CloneURL)
jsondata := fmt.Sprintf(`{"uid" : %d, "repo_name" : "%s" , "mirror" : %v, "clone_addr" : "%s"}`, config.GiteaUID, elem.Name, true, elem.CloneURL)
req, err := http.NewRequest("POST", config.gogsconfig.MigrateURL, bytes.NewBufferString(jsondata))
req, err := http.NewRequest("POST", config.GiteaMigrateURL, bytes.NewBufferString(jsondata))
if err != nil {
return err
}
req.Header.Add("Content-Type", config.gogsconfig.ContentType)
req.Header.Set("User-Agent", config.globalconfig.UserAgent)
req.Header.Set("Authorization", config.gogsconfig.AuthToken)
req.Header.Add("Content-Type", "application/json")
req.Header.Set("User-Agent", config.UserAgent)
req.Header.Set("Authorization", config.GiteaAuthToken)
client.Timeout = config.globalconfig.RequestTimeout
client.Timeout = config.RequestTimeout
fmt.Println(fmt.Sprintf("Migrating repo %s to gogs rest api", elem.Name))
fmt.Println(fmt.Sprintf("Migrating repo %s to gitea rest api", elem.Name))
resp, err := client.Do(req)
if err != nil {
@ -251,11 +222,11 @@ func MigrateReposToGogs(config *Config, wg *sync.WaitGroup, jobs chan GitHubRepo
}
if resp.StatusCode != 201 {
err = fmt.Errorf("Error when migrating repo %s to gogs with status code %d on gogs webservice", elem.Name, resp.StatusCode)
err = fmt.Errorf("Error when migrating repo %s to gitea with status code %d on gitea webservice", elem.Name, resp.StatusCode)
log.Println(err)
}
} else {
fmt.Println(fmt.Sprintf("Not migrating, %s exists in gogs", elem.Name))
fmt.Println(fmt.Sprintf("Not migrating, %s exists in gitea", elem.Name))
}
} else {
fmt.Println(fmt.Sprintf("All repo migrated on thread num %d", thr))

2
github-to-gogs.go → g2g.go
View File

@ -13,7 +13,7 @@ func main() {
log.Fatal(err)
}
err = config.GetGogsUserUID()
err = config.GetGiteaUserUID()
if err != nil {
log.Fatal(err)
}

18
g2g.ini.sample Normal file
View File

@ -0,0 +1,18 @@
[g2g]
request_timeout=1200s
user_agent="Golang"
threads=4
github_stars_pages="https://api.github.com/users/%s/starred?page=%d"
github_max_per_page=500
github_page_num=3
github_auth_username="user"
github_auth_password="pass"
gitea_username="user"
gitea_dest_username="user_or_org"
gitea_repo_url_tmpl="https://gogs.example.com/api/v1/repos/%s/%s"
gitea_orgs_url_tmpl="https://git.paulbsd.com/api/v1/orgs/%s"
gitea_migrate_url="https://gogs.example.com/api/v1/repos/migrate"
gitea_auth_token="token xxxx"
gitea_mirror=true

20
github-to-gogs.ini.sample
View File

@ -1,20 +0,0 @@
[global]
request_timeout=1200s
user_agent="Golang"
[github]
stars_pages="https://api.github.com/users/%s/starred?page=%d"
max_pages_number=8
auth_username="user"
auth_password="pass"
content_type="application/x-www-form-urlencoded"
[gogs]
username="user"
dest_username="user_or_org"
repo_url_tmpl="https://gogs.example.com/api/v1/repos/%s/%s"
orgs_url_tmpl="https://git.paulbsd.com/api/v1/orgs/%s"
migrate_url="https://gogs.example.com/api/v1/repos/migrate"
auth_token="token xxxx"
content_type="application/json"
mirror=true

2
go.mod
View File

@ -1,4 +1,4 @@
module github-to-gogs
module g2g
go 1.12

59
types.go
View File

@ -8,52 +8,37 @@ type GitHubRepo struct {
CloneURL string `json:"clone_url"`
}
// GogsOrg gogsrepo struct
type GogsOrg struct {
// GiteaOrg gitearepo struct
type GiteaOrg struct {
ID int `json:"id"`
Username string `json:"username"`
}
// GogsMigrateRepo defines the repo to migrate
type GogsMigrateRepo struct {
// GiteaMigrateRepo defines the repo to migrate
type GiteaMigrateRepo struct {
Name string
CloneURL string
UID int
Mirror bool
}
// Config merge all config elements
// Config is the global config of g2g
type Config struct {
globalconfig GlobalConfig
gogsconfig GogsConfig
githubconfig GitHubConfig
}
// GlobalConfig is the global config of github-to-gogs
type GlobalConfig struct {
RequestTimeout time.Duration
UserAgent string
Threads int
}
// GitHubConfig is the GitHub config
type GitHubConfig struct {
MaxPerPage int
PageNum int
AuthUsername string
AuthPassword string
ContentType string
}
// GogsConfig is the Gogs config
type GogsConfig struct {
UID int
Username string
DestUsername string
RepoURLTmpl string
OrgsURLTmpl string
MigrateURL string
AuthToken string
ContentType string
Mirror bool
RequestTimeout time.Duration
UserAgent string
Threads int
GitHubMaxPerPage int
GitHubPageNum int
GitHubAuthUsername string
GitHubAuthPassword string
GitHubContentType string
GiteaUID int
GiteaUsername string
GiteaDestUsername string
GiteaRepoURLTmpl string
GiteaOrgsURLTmpl string
GiteaMigrateURL string
GiteaAuthToken string
GiteaContentType string
GiteaMirror bool
}