paulbsd/g2g
1
0
Fork 0
Code Issues Pull Requests Releases 4 Wiki Activity

various changes

Browse Source
This commit is contained in:
Paul 2019-10-12 10:08:05 +02:00
parent c73a600096
commit 53f605dffc
7 changed files with 132 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
.vscode/launch.json vendored Normal file
View File

@ -0,0 +1,17 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Launch",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${fileDirname}",
"env": {},
"args": []
}
]
}

48
config.go Normal file
View File

@ -0,0 +1,48 @@
package main
import (
"flag"
"time"
"gopkg.in/ini.v1"
)
// GetConfig fetch configuration
func (config *Config) GetConfig() error {
var configfile string
flag.Usage = Usage
flag.StringVar(&configfile, "configfile", "g2g.ini", "config file to use with g2g section")
flag.Parse()
cfg, err := ini.Load(configfile)
if err != nil {
return err
}
g2gsection := cfg.Section("g2g")
g2gsection.MapTo(config)
return err
}
// Config is the global config of g2g
type Config struct {
G2gRequestTimeout time.Duration `ini:"g2g_request_timeout"`
G2gThreads int `ini:"g2g_threads"`
GitHubMaxPerPage int `ini:"github_max_per_page"`
GitHubPageNum int `ini:"github_page_num"`
GitHubAuthUsername string `ini:"github_auth_username"`
GitHubAuthPassword string `ini:"github_auth_password"`
GitHubContentType string
GiteaUID int
GiteaUsername string `ini:"gitea_username"`
GiteaDestUsername string `ini:"gitea_dest_username"`
GiteaRepoURLTmpl string `ini:"gitea_repo_url_tmpl"`
GiteaOrgsURLTmpl string `ini:"gitea_orgs_url_tmpl"`
GiteaMigrateURL string `ini:"gitea_migrate_url"`
GiteaAuthToken string `ini:"gitea_auth_token"`
GiteaContentType string
GiteaMirror bool `ini:"gitea_mirror"`
}

29
config_test.go Normal file
View File

@ -0,0 +1,29 @@
package main
import (
"fmt"
"testing"
)
func TestGetConfig(t *testing.T) {
got := `
[g2g]
request_timeout=1200s
threads=4
github_stars_pages="https://api.github.com/users/%s/starred?page=%d"
github_max_per_page=500
github_page_num=3
github_auth_username="user"
github_auth_password="pass"
gitea_username="user"
gitea_dest_username="user_or_org"
gitea_repo_url_tmpl="https://gogs.example.com/api/v1/repos/%s/%s"
gitea_orgs_url_tmpl="https://git.paulbsd.com/api/v1/orgs/%s"
gitea_migrate_url="https://gogs.example.com/api/v1/repos/migrate"
gitea_auth_token="token xxxx"
gitea_mirror=true
`
fmt.Println(got)
}

90
functions.go
View File

@ -11,45 +11,8 @@ import (
"net/http"
"os"
"sync"
"gopkg.in/ini.v1"
)
// GetConfig fetch configuration
func (config *Config) GetConfig() error {
var configfile string
flag.Usage = Usage
flag.StringVar(&configfile, "configfile", "g2g.ini", "config file to use with g2g section")
flag.Parse()
cfg, err := ini.Load(configfile)
if err != nil {
return err
}
g2gsection := cfg.Section("g2g")
config.UserAgent = g2gsection.Key("user_agent").MustString("Golang")
config.RequestTimeout = g2gsection.Key("request_timeout").MustDuration(1200)
config.Threads = g2gsection.Key("threads").MustInt(2)
config.GitHubPageNum = g2gsection.Key("github_page_num").MustInt(3)
config.GitHubMaxPerPage = g2gsection.Key("github_max_per_page").MustInt(100)
config.GitHubAuthUsername = g2gsection.Key("github_auth_username").MustString("username")
config.GitHubAuthPassword = g2gsection.Key("github_auth_password").MustString("password")
config.GiteaUsername = g2gsection.Key("gitea_username").MustString("username")
config.GiteaDestUsername = g2gsection.Key("gitea_dest_username").MustString("dest_username")
config.GiteaRepoURLTmpl = g2gsection.Key("gitea_repo_url_tmpl").MustString("repo_url")
config.GiteaOrgsURLTmpl = g2gsection.Key("gitea_orgs_url_tmpl").MustString("orgs_url")
config.GiteaMigrateURL = g2gsection.Key("gitea_migrate_url").MustString("migrate_url")
config.GiteaAuthToken = g2gsection.Key("gitea_auth_token").MustString("token")
config.GiteaMirror = g2gsection.Key("gitea_mirror").MustBool(true)
return err
}
// GetReposFromGitHub get starred repositories from github
func GetReposFromGitHub(config *Config) ([]GitHubRepo, error) {
var repopartiallist []GitHubRepo
@ -60,7 +23,7 @@ func GetReposFromGitHub(config *Config) ([]GitHubRepo, error) {
url := fmt.Sprintf("https://api.github.com/users/%s/starred?per_page=%d&page=%d", config.GitHubAuthUsername, config.GitHubMaxPerPage, num)
fmt.Println(url)
resp, err := InvokeGitHub(config, url)
resp, err := GetGitHubRepos(config, url)
if err != nil {
return nil, err
}
@ -84,15 +47,14 @@ func GetReposFromGitHub(config *Config) ([]GitHubRepo, error) {
return repofulllist, nil
}
// InvokeGitHub ...
func InvokeGitHub(config *Config, url string) (*http.Response, error) {
// GetGitHubRepos fetchs starred repositories on GitHub
func GetGitHubRepos(config *Config, url string) (*http.Response, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("User-Agent", config.UserAgent)
req.SetBasicAuth(config.GitHubAuthUsername, config.GitHubAuthPassword)
client := &http.Client{}
@ -101,18 +63,21 @@ func InvokeGitHub(config *Config, url string) (*http.Response, error) {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("Error on status code %s", resp.Status)
}
return resp, nil
}
// CheckGiteaExistingRepo checks if there's an existing repository in gitea
func CheckGiteaExistingRepo(config *Config, repo GitHubRepo) (bool, error) {
var isexists bool
var isExists bool
gitearepourl := fmt.Sprintf(config.GiteaRepoURLTmpl, config.GiteaDestUsername, repo.Name)
req, err := http.NewRequest("GET", gitearepourl, nil)
req.Header.Set("Authorization", config.GiteaAuthToken)
req.Header.Set("User-Agent", config.UserAgent)
client := &http.Client{}
resp, err := client.Do(req)
@ -120,16 +85,15 @@ func CheckGiteaExistingRepo(config *Config, repo GitHubRepo) (bool, error) {
return false, err
}
if resp.StatusCode == 200 {
isexists = true
} else if resp.StatusCode == 404 {
isexists = false
if resp.StatusCode == http.StatusOK {
isExists = true
} else if resp.StatusCode == http.StatusNotFound {
isExists = false
} else {
err = fmt.Errorf("Can't determine error, cancelling, error %d in gitea webservice", resp.StatusCode)
return false, err
return false, fmt.Errorf("Can't determine error, cancelling, error %s in gitea webservice", resp.Status)
}
return isexists, nil
return isExists, nil
}
// GetGiteaUserUID get the logued user identifier
@ -144,7 +108,6 @@ func (config *Config) GetGiteaUserUID() error {
}
req.Header.Set("Authorization", config.GiteaAuthToken)
req.Header.Set("User-Agent", config.UserAgent)
client := &http.Client{}
resp, err := client.Do(req)
@ -176,7 +139,7 @@ func RunMigration(config *Config, repolist []GitHubRepo) {
done := make(chan bool)
var wg sync.WaitGroup
for thr := range make([]int, config.Threads) {
for thr := range make([]int, config.G2gThreads) {
go MigrateReposToGitea(config, &wg, repochan, done, thr)
}
@ -209,10 +172,9 @@ func MigrateReposToGitea(config *Config, wg *sync.WaitGroup, jobs chan GitHubRep
}
req.Header.Add("Content-Type", "application/json")
req.Header.Set("User-Agent", config.UserAgent)
req.Header.Set("Authorization", config.GiteaAuthToken)
client.Timeout = config.RequestTimeout
client.Timeout = config.G2gRequestTimeout
fmt.Println(fmt.Sprintf("Migrating repo %s to gitea rest api", elem.Name))
@ -242,3 +204,23 @@ func Usage() {
flag.PrintDefaults()
os.Exit(1)
}
// GitHubRepo githubrepo struct
type GitHubRepo struct {
Name string `json:"name"`
CloneURL string `json:"clone_url"`
}
// GiteaOrg gitearepo struct
type GiteaOrg struct {
ID int `json:"id"`
Username string `json:"username"`
}
// GiteaMigrateRepo defines the repo to migrate
type GiteaMigrateRepo struct {
Name string
CloneURL string
UID int
Mirror bool
}

3
g2g.go
View File

@ -7,6 +7,7 @@ import (
func main() {
var config Config
var err error
var repolist []GitHubRepo
err = config.GetConfig()
if err != nil {
@ -18,7 +19,7 @@ func main() {
log.Fatal(err)
}
repolist, err := GetReposFromGitHub(&config)
repolist, err = GetReposFromGitHub(&config)
if err != nil {
log.Fatal(err)
}

1
g2g.ini.sample
View File

@ -1,6 +1,5 @@
[g2g]
request_timeout=1200s
user_agent="Golang"
threads=4
github_stars_pages="https://api.github.com/users/%s/starred?page=%d"

44
types.go
View File

@ -1,44 +0,0 @@
package main
import "time"
// GitHubRepo githubrepo struct
type GitHubRepo struct {
Name string `json:"name"`
CloneURL string `json:"clone_url"`
}
// GiteaOrg gitearepo struct
type GiteaOrg struct {
ID int `json:"id"`
Username string `json:"username"`
}
// GiteaMigrateRepo defines the repo to migrate
type GiteaMigrateRepo struct {
Name string
CloneURL string
UID int
Mirror bool
}
// Config is the global config of g2g
type Config struct {
RequestTimeout time.Duration
UserAgent string
Threads int
GitHubMaxPerPage int
GitHubPageNum int
GitHubAuthUsername string
GitHubAuthPassword string
GitHubContentType string
GiteaUID int
GiteaUsername string
GiteaDestUsername string
GiteaRepoURLTmpl string
GiteaOrgsURLTmpl string
GiteaMigrateURL string
GiteaAuthToken string
GiteaContentType string
GiteaMirror bool
}