From 1ea8773b34531a74a019cb17dcd882aca45c3717 Mon Sep 17 00:00:00 2001 From: Paul Lecuq Date: Fri, 4 Dec 2020 20:23:52 +0100 Subject: [PATCH] updates on g2g --- .drone.yml | 117 +++++++++- README.md | 1 + ci-build.sh | 18 +- go.mod | 8 +- go.sum | 13 +- src/g2g/main.go | 22 +- vendor/gopkg.in/ini.v1/.travis.yml | 19 -- vendor/gopkg.in/ini.v1/Makefile | 2 +- vendor/gopkg.in/ini.v1/README.md | 11 +- vendor/gopkg.in/ini.v1/codecov.yml | 9 + vendor/gopkg.in/ini.v1/data_source.go | 2 + vendor/gopkg.in/ini.v1/file.go | 173 +++++++++++--- vendor/gopkg.in/ini.v1/ini.go | 31 ++- vendor/gopkg.in/ini.v1/key.go | 127 ++++++---- vendor/gopkg.in/ini.v1/parser.go | 84 +++++-- vendor/gopkg.in/ini.v1/section.go | 14 +- vendor/gopkg.in/ini.v1/struct.go | 320 ++++++++++++++++++++------ vendor/modules.txt | 9 +- 18 files changed, 740 insertions(+), 240 deletions(-) delete mode 100644 vendor/gopkg.in/ini.v1/.travis.yml create mode 100644 vendor/gopkg.in/ini.v1/codecov.yml diff --git a/.drone.yml b/.drone.yml index 98100c0..f76427a 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,3 +1,24 @@ +--- +kind: pipeline +type: docker +name: cleanup-before + +steps: + - name: clean + image: alpine + commands: + - rm -rf /build/* + volumes: + - name: build + path: /build + when: + event: tag + +volumes: + - name: build + host: + path: /tmp/g2g/build + --- kind: pipeline type: docker @@ -11,18 +32,17 @@ steps: environment: GOOS: linux GOARCH: amd64 - - name: release - image: plugins/gitea-release - settings: - base_url: https://git.paulbsd.com - api_key: - from_secret: gitea_token - files: "*.tar.gz" - checksum: - - sha256 - - sha512 - when: - event: tag + volumes: + - name: build + path: /build + +volumes: + - name: build + host: + path: /tmp/g2g/build + +depends_on: + - cleanup-before --- kind: pipeline @@ -37,6 +57,33 @@ steps: environment: GOOS: linux GOARCH: arm64 + volumes: + - name: build + path: /build + +volumes: + - name: build + host: + path: /tmp/g2g/build + +depends_on: + - cleanup-before + +--- +kind: pipeline +type: docker +name: gitea-release + +steps: + - name: move + image: alpine + commands: + - mv build/* ./ + volumes: + - name: build + path: /drone/src/build + when: + event: tag - name: release image: plugins/gitea-release settings: @@ -47,5 +94,51 @@ steps: checksum: - sha256 - sha512 + title: VERSION + volumes: + - name: build + path: /drone/src/build when: event: tag + - name: ls + image: alpine + commands: + - find . + volumes: + - name: build + path: /drone/src/build + when: + event: tag + +volumes: + - name: build + host: + path: /tmp/g2g/build + +depends_on: + - default-linux-amd64 + - default-linux-arm64 + +--- +kind: pipeline +type: docker +name: cleanup-after + +steps: + - name: clean + image: alpine + commands: + - rm -rf /build/* + volumes: + - name: build + path: /build + when: + event: tag + +volumes: + - name: build + host: + path: /tmp/g2g/build + +depends_on: + - gitea-release diff --git a/README.md b/README.md index 50fb5e3..6cf8f72 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ ## Summary g2g is a small program that migrate stars made on github projects to self owned gogs/gitea instance +It supports multithread processing ## Howto diff --git a/ci-build.sh b/ci-build.sh index d317c65..ea8d04d 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -8,24 +8,38 @@ GOOPTIONS="-mod=vendor" SRCFILES=cmd/g2g/*.go build() { + echo "Begin of build" if [[ ! -z $DRONE_TAG ]] then + echo "Drone tag set, let's do a release" VERSION=$DRONE_TAG + echo "${PROJECTNAME} ${VERSION}" > /build/VERSION elif [[ ! -z $DRONE_TAG ]] then + echo "Drone not set, let's only do a build" VERSION=$DRONE_COMMIT fi if [[ ! -z $VERSION && ! -z $GOOS && ! -z $GOARCH ]] then + echo "Let's set a release name" RELEASENAME=${PROJECTNAME}-${VERSION}-${GOOS}-${GOARCH} fi + + echo "Building project" go build -o ${PROJECTNAME} ${GOOPTIONS} ${SRCFILES} + if [[ ! -z $DRONE_TAG ]] then - tar -czvf ${RELEASENAME}.tar.gz ${PROJECTNAME} + echo "Let's make archives" + mkdir -p /build + tar -czvf /build/${RELEASENAME}.tar.gz ${PROJECTNAME} fi + + echo "Removing binary file" rm ${PROJECTNAME} + + echo "End of build" } clean() { @@ -40,5 +54,7 @@ case $1 in clean ;; *) + echo "No options choosen" + exit 1 ;; esac diff --git a/go.mod b/go.mod index 88c57ac..02681d4 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,10 @@ module git.paulbsd.com/paulbsd/g2g -go 1.13 +go 1.15 require ( - github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect - gopkg.in/ini.v1 v1.48.0 + github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect + github.com/smartystreets/assertions v1.2.0 // indirect + github.com/smartystreets/goconvey v1.6.4 // indirect + gopkg.in/ini.v1 v1.62.0 ) diff --git a/go.sum b/go.sum index 69be99d..ed97569 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,20 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0= +github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= -github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 h1:TFlARGu6Czu1z7q93HTxcP1P+/ZFC/IKythI5RzrnRg= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -gopkg.in/ini.v1 v1.48.0 h1:URjZc+8ugRY5mL5uUeQH/a63JcHwdX9xZaWvmNWD7z8= -gopkg.in/ini.v1 v1.48.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/src/g2g/main.go b/src/g2g/main.go index ef31b8f..ea261f5 100644 --- a/src/g2g/main.go +++ b/src/g2g/main.go @@ -154,6 +154,7 @@ func RunMigration(config *config.Config, repolist []GitHubRepo) { // MigrateReposToGitea migrates input repositories to gitea func MigrateReposToGitea(config *config.Config, wg *sync.WaitGroup, jobs chan GitHubRepo, done chan bool, thr int) error { wg.Add(1) + for { elem, more := <-jobs if more { @@ -164,9 +165,14 @@ func MigrateReposToGitea(config *config.Config, wg *sync.WaitGroup, jobs chan Gi return err } if !existingrepo { - jsondata := fmt.Sprintf(`{"uid" : %d, "repo_name" : "%s" , "mirror" : %v, "clone_addr" : "%s"}`, config.GiteaUID, elem.Name, true, elem.CloneURL) + repo := GiteaRepo{ + UID: config.GiteaUID, + RepoName: elem.Name, + Mirror: true, + CloneAddr: elem.CloneURL} + jsondata, err := json.Marshal(repo) - req, err := http.NewRequest("POST", config.GiteaMigrateURL, bytes.NewBufferString(jsondata)) + req, err := http.NewRequest("POST", config.GiteaMigrateURL, bytes.NewBufferString(string(jsondata))) if err != nil { return err } @@ -211,10 +217,10 @@ type GiteaOrg struct { Username string `json:"username"` } -// GiteaMigrateRepo defines the repo to migrate -type GiteaMigrateRepo struct { - Name string - CloneURL string - UID int - Mirror bool +// GiteaRepo defines the repo to migrate +type GiteaRepo struct { + UID int `json:"uid"` + RepoName string `json:"repo_name"` + Mirror bool `json:"mirror"` + CloneAddr string `json:"clone_addr"` } diff --git a/vendor/gopkg.in/ini.v1/.travis.yml b/vendor/gopkg.in/ini.v1/.travis.yml deleted file mode 100644 index 08682ef..0000000 --- a/vendor/gopkg.in/ini.v1/.travis.yml +++ /dev/null @@ -1,19 +0,0 @@ -sudo: false -language: go -go: - - 1.6.x - - 1.7.x - - 1.8.x - - 1.9.x - - 1.10.x - - 1.11.x - - 1.12.x - -install: skip -script: - - go get golang.org/x/tools/cmd/cover - - go get github.com/smartystreets/goconvey - - mkdir -p $HOME/gopath/src/gopkg.in - - ln -s $HOME/gopath/src/github.com/go-ini/ini $HOME/gopath/src/gopkg.in/ini.v1 - - cd $HOME/gopath/src/gopkg.in/ini.v1 - - go test -v -cover -race diff --git a/vendor/gopkg.in/ini.v1/Makefile b/vendor/gopkg.in/ini.v1/Makefile index af27ff0..f3b0dae 100644 --- a/vendor/gopkg.in/ini.v1/Makefile +++ b/vendor/gopkg.in/ini.v1/Makefile @@ -6,7 +6,7 @@ test: go test -v -cover -race bench: - go test -v -cover -race -test.bench=. -test.benchmem + go test -v -cover -test.bench=. -test.benchmem vet: go vet diff --git a/vendor/gopkg.in/ini.v1/README.md b/vendor/gopkg.in/ini.v1/README.md index 44e1fcd..5d65658 100644 --- a/vendor/gopkg.in/ini.v1/README.md +++ b/vendor/gopkg.in/ini.v1/README.md @@ -1,5 +1,9 @@ -INI [![Build Status](https://travis-ci.org/go-ini/ini.svg?branch=master)](https://travis-ci.org/go-ini/ini) [![Sourcegraph](https://img.shields.io/badge/view%20on-Sourcegraph-brightgreen.svg)](https://sourcegraph.com/github.com/go-ini/ini) -=== +# INI + +[![GitHub Workflow Status](https://img.shields.io/github/workflow/status/go-ini/ini/Go?logo=github&style=for-the-badge)](https://github.com/go-ini/ini/actions?query=workflow%3AGo) +[![codecov](https://img.shields.io/codecov/c/github/go-ini/ini/master?logo=codecov&style=for-the-badge)](https://codecov.io/gh/go-ini/ini) +[![GoDoc](https://img.shields.io/badge/GoDoc-Reference-blue?style=for-the-badge&logo=go)](https://pkg.go.dev/github.com/go-ini/ini?tab=doc) +[![Sourcegraph](https://img.shields.io/badge/view%20on-Sourcegraph-brightgreen.svg?style=for-the-badge&logo=sourcegraph)](https://sourcegraph.com/github.com/go-ini/ini) ![](https://avatars0.githubusercontent.com/u/10216035?v=3&s=200) @@ -7,7 +11,7 @@ Package ini provides INI file read and write functionality in Go. ## Features -- Load from multiple data sources(`[]byte`, file and `io.ReadCloser`) with overwrites. +- Load from multiple data sources(file, `[]byte`, `io.Reader` and `io.ReadCloser`) with overwrites. - Read with recursion values. - Read with parent-child sections. - Read with auto-increment key names. @@ -32,6 +36,7 @@ Please add `-u` flag to update in the future. - [Getting Started](https://ini.unknwon.io/docs/intro/getting_started) - [API Documentation](https://gowalker.org/gopkg.in/ini.v1) +- 中国大陆镜像：https://ini.unknwon.cn ## License diff --git a/vendor/gopkg.in/ini.v1/codecov.yml b/vendor/gopkg.in/ini.v1/codecov.yml new file mode 100644 index 0000000..fc947f2 --- /dev/null +++ b/vendor/gopkg.in/ini.v1/codecov.yml @@ -0,0 +1,9 @@ +coverage: + range: "60...95" + status: + project: + default: + threshold: 1% + +comment: + layout: 'diff, files' diff --git a/vendor/gopkg.in/ini.v1/data_source.go b/vendor/gopkg.in/ini.v1/data_source.go index dc0277e..c3a541f 100644 --- a/vendor/gopkg.in/ini.v1/data_source.go +++ b/vendor/gopkg.in/ini.v1/data_source.go @@ -68,6 +68,8 @@ func parseDataSource(source interface{}) (dataSource, error) { return &sourceData{s}, nil case io.ReadCloser: return &sourceReadCloser{s}, nil + case io.Reader: + return &sourceReadCloser{ioutil.NopCloser(s)}, nil default: return nil, fmt.Errorf("error parsing data source: unknown type %q", s) } diff --git a/vendor/gopkg.in/ini.v1/file.go b/vendor/gopkg.in/ini.v1/file.go index 017b77c..b96d172 100644 --- a/vendor/gopkg.in/ini.v1/file.go +++ b/vendor/gopkg.in/ini.v1/file.go @@ -25,7 +25,7 @@ import ( "sync" ) -// File represents a combination of a or more INI file(s) in memory. +// File represents a combination of one or more INI files in memory. type File struct { options LoadOptions dataSources []dataSource @@ -36,8 +36,12 @@ type File struct { // To keep data in order. sectionList []string + // To keep track of the index of a section with same name. + // This meta list is only used with non-unique section names are allowed. + sectionIndexes []int + // Actual data is stored here. - sections map[string]*Section + sections map[string][]*Section NameMapper ValueMapper @@ -48,27 +52,40 @@ func newFile(dataSources []dataSource, opts LoadOptions) *File { if len(opts.KeyValueDelimiters) == 0 { opts.KeyValueDelimiters = "=:" } + if len(opts.KeyValueDelimiterOnWrite) == 0 { + opts.KeyValueDelimiterOnWrite = "=" + } + if len(opts.ChildSectionDelimiter) == 0 { + opts.ChildSectionDelimiter = "." + } + return &File{ BlockMode: true, dataSources: dataSources, - sections: make(map[string]*Section), - sectionList: make([]string, 0, 10), + sections: make(map[string][]*Section), options: opts, } } // Empty returns an empty file object. -func Empty() *File { - // Ignore error here, we sure our data is good. - f, _ := Load([]byte("")) +func Empty(opts ...LoadOptions) *File { + var opt LoadOptions + if len(opts) > 0 { + opt = opts[0] + } + + // Ignore error here, we are sure our data is good. + f, _ := LoadSources(opt, []byte("")) return f } // NewSection creates a new section. func (f *File) NewSection(name string) (*Section, error) { if len(name) == 0 { - return nil, errors.New("error creating new section: empty section name") - } else if f.options.Insensitive && name != DefaultSection { + return nil, errors.New("empty section name") + } + + if (f.options.Insensitive || f.options.InsensitiveSections) && name != DefaultSection { name = strings.ToLower(name) } @@ -77,13 +94,20 @@ func (f *File) NewSection(name string) (*Section, error) { defer f.lock.Unlock() } - if inSlice(name, f.sectionList) { - return f.sections[name], nil + if !f.options.AllowNonUniqueSections && inSlice(name, f.sectionList) { + return f.sections[name][0], nil } f.sectionList = append(f.sectionList, name) - f.sections[name] = newSection(f, name) - return f.sections[name], nil + + // NOTE: Append to indexes must happen before appending to sections, + // otherwise index will have off-by-one problem. + f.sectionIndexes = append(f.sectionIndexes, len(f.sections[name])) + + sec := newSection(f, name) + f.sections[name] = append(f.sections[name], sec) + + return sec, nil } // NewRawSection creates a new section with an unparseable body. @@ -110,10 +134,20 @@ func (f *File) NewSections(names ...string) (err error) { // GetSection returns section by given name. func (f *File) GetSection(name string) (*Section, error) { + secs, err := f.SectionsByName(name) + if err != nil { + return nil, err + } + + return secs[0], err +} + +// SectionsByName returns all sections with given name. +func (f *File) SectionsByName(name string) ([]*Section, error) { if len(name) == 0 { name = DefaultSection } - if f.options.Insensitive { + if f.options.Insensitive || f.options.InsensitiveSections { name = strings.ToLower(name) } @@ -122,11 +156,12 @@ func (f *File) GetSection(name string) (*Section, error) { defer f.lock.RUnlock() } - sec := f.sections[name] - if sec == nil { - return nil, fmt.Errorf("section '%s' does not exist", name) + secs := f.sections[name] + if len(secs) == 0 { + return nil, fmt.Errorf("section %q does not exist", name) } - return sec, nil + + return secs, nil } // Section assumes named section exists and returns a zero-value when not. @@ -141,6 +176,19 @@ func (f *File) Section(name string) *Section { return sec } +// SectionWithIndex assumes named section exists and returns a new section when not. +func (f *File) SectionWithIndex(name string, index int) *Section { + secs, err := f.SectionsByName(name) + if err != nil || len(secs) <= index { + // NOTE: It's OK here because the only possible error is empty section name, + // but if it's empty, this piece of code won't be executed. + newSec, _ := f.NewSection(name) + return newSec + } + + return secs[index] +} + // Sections returns a list of Section stored in the current instance. func (f *File) Sections() []*Section { if f.BlockMode { @@ -150,7 +198,7 @@ func (f *File) Sections() []*Section { sections := make([]*Section, len(f.sectionList)) for i, name := range f.sectionList { - sections[i] = f.sections[name] + sections[i] = f.sections[name][f.sectionIndexes[i]] } return sections } @@ -167,24 +215,70 @@ func (f *File) SectionStrings() []string { return list } -// DeleteSection deletes a section. +// DeleteSection deletes a section or all sections with given name. func (f *File) DeleteSection(name string) { - if f.BlockMode { - f.lock.Lock() - defer f.lock.Unlock() + secs, err := f.SectionsByName(name) + if err != nil { + return + } + + for i := 0; i < len(secs); i++ { + // For non-unique sections, it is always needed to remove the first one so + // in the next iteration, the subsequent section continue having index 0. + // Ignoring the error as index 0 never returns an error. + _ = f.DeleteSectionWithIndex(name, 0) + } +} + +// DeleteSectionWithIndex deletes a section with given name and index. +func (f *File) DeleteSectionWithIndex(name string, index int) error { + if !f.options.AllowNonUniqueSections && index != 0 { + return fmt.Errorf("delete section with non-zero index is only allowed when non-unique sections is enabled") } if len(name) == 0 { name = DefaultSection } - - for i, s := range f.sectionList { - if s == name { - f.sectionList = append(f.sectionList[:i], f.sectionList[i+1:]...) - delete(f.sections, name) - return - } + if f.options.Insensitive || f.options.InsensitiveSections { + name = strings.ToLower(name) } + + if f.BlockMode { + f.lock.Lock() + defer f.lock.Unlock() + } + + // Count occurrences of the sections + occurrences := 0 + + sectionListCopy := make([]string, len(f.sectionList)) + copy(sectionListCopy, f.sectionList) + + for i, s := range sectionListCopy { + if s != name { + continue + } + + if occurrences == index { + if len(f.sections[name]) <= 1 { + delete(f.sections, name) // The last one in the map + } else { + f.sections[name] = append(f.sections[name][:index], f.sections[name][index+1:]...) + } + + // Fix section lists + f.sectionList = append(f.sectionList[:i], f.sectionList[i+1:]...) + f.sectionIndexes = append(f.sectionIndexes[:i], f.sectionIndexes[i+1:]...) + + } else if occurrences > index { + // Fix the indices of all following sections with this name. + f.sectionIndexes[i-1]-- + } + + occurrences++ + } + + return nil } func (f *File) reload(s dataSource) error { @@ -203,11 +297,14 @@ func (f *File) Reload() (err error) { if err = f.reload(s); err != nil { // In loose mode, we create an empty default section for nonexistent files. if os.IsNotExist(err) && f.options.Loose { - f.parse(bytes.NewBuffer(nil)) + _ = f.parse(bytes.NewBuffer(nil)) continue } return err } + if f.options.ShortCircuit { + return nil + } } return nil } @@ -230,16 +327,16 @@ func (f *File) Append(source interface{}, others ...interface{}) error { } func (f *File) writeToBuffer(indent string) (*bytes.Buffer, error) { - equalSign := DefaultFormatLeft + "=" + DefaultFormatRight + equalSign := DefaultFormatLeft + f.options.KeyValueDelimiterOnWrite + DefaultFormatRight if PrettyFormat || PrettyEqual { - equalSign = " = " + equalSign = fmt.Sprintf(" %s ", f.options.KeyValueDelimiterOnWrite) } // Use buffer to make sure target is safe until finish encoding. buf := bytes.NewBuffer(nil) for i, sname := range f.sectionList { - sec := f.Section(sname) + sec := f.SectionWithIndex(sname, f.sectionIndexes[i]) if len(sec.Comment) > 0 { // Support multiline comments lines := strings.Split(sec.Comment, LineBreak) @@ -256,7 +353,7 @@ func (f *File) writeToBuffer(indent string) (*bytes.Buffer, error) { } } - if i > 0 || DefaultHeader { + if i > 0 || DefaultHeader || (i == 0 && strings.ToUpper(sec.name) != DefaultSection) { if _, err := buf.WriteString("[" + sname + "]" + LineBreak); err != nil { return nil, err } @@ -282,7 +379,7 @@ func (f *File) writeToBuffer(indent string) (*bytes.Buffer, error) { } // Count and generate alignment length and buffer spaces using the - // longest key. Keys may be modifed if they contain certain characters so + // longest key. Keys may be modified if they contain certain characters so // we need to take that into account in our calculation. alignLength := 0 if PrettyFormat { @@ -360,6 +457,8 @@ func (f *File) writeToBuffer(indent string) (*bytes.Buffer, error) { val = `"""` + val + `"""` } else if !f.options.IgnoreInlineComment && strings.ContainsAny(val, "#;") { val = "`" + val + "`" + } else if len(strings.TrimSpace(val)) != len(val) { + val = `"` + val + `"` } if _, err := buf.WriteString(equalSign + val + LineBreak); err != nil { return nil, err @@ -403,7 +502,7 @@ func (f *File) WriteTo(w io.Writer) (int64, error) { // SaveToIndent writes content to file system with given value indention. func (f *File) SaveToIndent(filename, indent string) error { // Note: Because we are truncating with os.Create, - // so it's safer to save to a temporary file location and rename afte done. + // so it's safer to save to a temporary file location and rename after done. buf, err := f.writeToBuffer(indent) if err != nil { return err diff --git a/vendor/gopkg.in/ini.v1/ini.go b/vendor/gopkg.in/ini.v1/ini.go index 9de6171..23f0742 100644 --- a/vendor/gopkg.in/ini.v1/ini.go +++ b/vendor/gopkg.in/ini.v1/ini.go @@ -18,8 +18,10 @@ package ini import ( + "os" "regexp" "runtime" + "strings" ) const ( @@ -29,14 +31,8 @@ const ( // Maximum allowed depth when recursively substituing variable names. depthValues = 99 - version = "1.48.0" ) -// Version returns current package version literal. -func Version() string { - return version -} - var ( // LineBreak is the delimiter to determine or compose a new line. // This variable will be changed to "\r\n" automatically on Windows at package init time. @@ -61,8 +57,10 @@ var ( DefaultFormatRight = "" ) +var inTest = len(os.Args) > 0 && strings.HasSuffix(strings.TrimSuffix(os.Args[0], ".exe"), ".test") + func init() { - if runtime.GOOS == "windows" { + if runtime.GOOS == "windows" && !inTest { LineBreak = "\r\n" } } @@ -73,12 +71,18 @@ type LoadOptions struct { Loose bool // Insensitive indicates whether the parser forces all section and key names to lowercase. Insensitive bool + // InsensitiveSections indicates whether the parser forces all section to lowercase. + InsensitiveSections bool + // InsensitiveKeys indicates whether the parser forces all key names to lowercase. + InsensitiveKeys bool // IgnoreContinuation indicates whether to ignore continuation lines while parsing. IgnoreContinuation bool // IgnoreInlineComment indicates whether to ignore comments at the end of value and treat it as part of value. IgnoreInlineComment bool // SkipUnrecognizableLines indicates whether to skip unrecognizable lines that do not conform to key/value pairs. SkipUnrecognizableLines bool + // ShortCircuit indicates whether to ignore other configuration sources after loaded the first available configuration source. + ShortCircuit bool // AllowBooleanKeys indicates whether to allow boolean type keys or treat as value is missing. // This type of keys are mostly used in my.cnf. AllowBooleanKeys bool @@ -109,10 +113,23 @@ type LoadOptions struct { UnparseableSections []string // KeyValueDelimiters is the sequence of delimiters that are used to separate key and value. By default, it is "=:". KeyValueDelimiters string + // KeyValueDelimiterOnWrite is the delimiter that are used to separate key and value output. By default, it is "=". + KeyValueDelimiterOnWrite string + // ChildSectionDelimiter is the delimiter that is used to separate child sections. By default, it is ".". + ChildSectionDelimiter string // PreserveSurroundedQuote indicates whether to preserve surrounded quote (single and double quotes). PreserveSurroundedQuote bool + // DebugFunc is called to collect debug information (currently only useful to debug parsing Python-style multiline values). + DebugFunc DebugFunc + // ReaderBufferSize is the buffer size of the reader in bytes. + ReaderBufferSize int + // AllowNonUniqueSections indicates whether to allow sections with the same name multiple times. + AllowNonUniqueSections bool } +// DebugFunc is the type of function called to log parse events. +type DebugFunc func(message string) + // LoadSources allows caller to apply customized options for loading from data source(s). func LoadSources(opts LoadOptions, source interface{}, others ...interface{}) (_ *File, err error) { sources := make([]dataSource, len(others)+1) diff --git a/vendor/gopkg.in/ini.v1/key.go b/vendor/gopkg.in/ini.v1/key.go index 62f9146..8baafd9 100644 --- a/vendor/gopkg.in/ini.v1/key.go +++ b/vendor/gopkg.in/ini.v1/key.go @@ -147,10 +147,15 @@ func (k *Key) transformValue(val string) string { noption := vr[2 : len(vr)-2] // Search in the same section. + // If not found or found the key itself, then search again in default section. nk, err := k.s.GetKey(noption) if err != nil || k == nk { - // Search again in default section. nk, _ = k.s.f.Section("").GetKey(noption) + if nk == nil { + // Stop when no results found in the default section, + // and returns the value as-is. + break + } } // Substitute by new value and take off leading '%(' and trailing ')s'. @@ -681,99 +686,127 @@ func (k *Key) StrictTimes(delim string) ([]time.Time, error) { // parseBools transforms strings to bools. func (k *Key) parseBools(strs []string, addInvalid, returnOnInvalid bool) ([]bool, error) { vals := make([]bool, 0, len(strs)) - for _, str := range strs { + parser := func(str string) (interface{}, error) { val, err := parseBool(str) - if err != nil && returnOnInvalid { - return nil, err - } - if err == nil || addInvalid { - vals = append(vals, val) + return val, err + } + rawVals, err := k.doParse(strs, addInvalid, returnOnInvalid, parser) + if err == nil { + for _, val := range rawVals { + vals = append(vals, val.(bool)) } } - return vals, nil + return vals, err } // parseFloat64s transforms strings to float64s. func (k *Key) parseFloat64s(strs []string, addInvalid, returnOnInvalid bool) ([]float64, error) { vals := make([]float64, 0, len(strs)) - for _, str := range strs { + parser := func(str string) (interface{}, error) { val, err := strconv.ParseFloat(str, 64) - if err != nil && returnOnInvalid { - return nil, err - } - if err == nil || addInvalid { - vals = append(vals, val) + return val, err + } + rawVals, err := k.doParse(strs, addInvalid, returnOnInvalid, parser) + if err == nil { + for _, val := range rawVals { + vals = append(vals, val.(float64)) } } - return vals, nil + return vals, err } // parseInts transforms strings to ints. func (k *Key) parseInts(strs []string, addInvalid, returnOnInvalid bool) ([]int, error) { vals := make([]int, 0, len(strs)) - for _, str := range strs { - valInt64, err := strconv.ParseInt(str, 0, 64) - val := int(valInt64) - if err != nil && returnOnInvalid { - return nil, err - } - if err == nil || addInvalid { - vals = append(vals, val) + parser := func(str string) (interface{}, error) { + val, err := strconv.ParseInt(str, 0, 64) + return val, err + } + rawVals, err := k.doParse(strs, addInvalid, returnOnInvalid, parser) + if err == nil { + for _, val := range rawVals { + vals = append(vals, int(val.(int64))) } } - return vals, nil + return vals, err } // parseInt64s transforms strings to int64s. func (k *Key) parseInt64s(strs []string, addInvalid, returnOnInvalid bool) ([]int64, error) { vals := make([]int64, 0, len(strs)) - for _, str := range strs { + parser := func(str string) (interface{}, error) { val, err := strconv.ParseInt(str, 0, 64) - if err != nil && returnOnInvalid { - return nil, err - } - if err == nil || addInvalid { - vals = append(vals, val) + return val, err + } + + rawVals, err := k.doParse(strs, addInvalid, returnOnInvalid, parser) + if err == nil { + for _, val := range rawVals { + vals = append(vals, val.(int64)) } } - return vals, nil + return vals, err } // parseUints transforms strings to uints. func (k *Key) parseUints(strs []string, addInvalid, returnOnInvalid bool) ([]uint, error) { vals := make([]uint, 0, len(strs)) - for _, str := range strs { - val, err := strconv.ParseUint(str, 0, 0) - if err != nil && returnOnInvalid { - return nil, err - } - if err == nil || addInvalid { - vals = append(vals, uint(val)) + parser := func(str string) (interface{}, error) { + val, err := strconv.ParseUint(str, 0, 64) + return val, err + } + + rawVals, err := k.doParse(strs, addInvalid, returnOnInvalid, parser) + if err == nil { + for _, val := range rawVals { + vals = append(vals, uint(val.(uint64))) } } - return vals, nil + return vals, err } // parseUint64s transforms strings to uint64s. func (k *Key) parseUint64s(strs []string, addInvalid, returnOnInvalid bool) ([]uint64, error) { vals := make([]uint64, 0, len(strs)) - for _, str := range strs { + parser := func(str string) (interface{}, error) { val, err := strconv.ParseUint(str, 0, 64) - if err != nil && returnOnInvalid { - return nil, err - } - if err == nil || addInvalid { - vals = append(vals, val) + return val, err + } + rawVals, err := k.doParse(strs, addInvalid, returnOnInvalid, parser) + if err == nil { + for _, val := range rawVals { + vals = append(vals, val.(uint64)) } } - return vals, nil + return vals, err } + +type Parser func(str string) (interface{}, error) + + // parseTimesFormat transforms strings to times in given format. func (k *Key) parseTimesFormat(format string, strs []string, addInvalid, returnOnInvalid bool) ([]time.Time, error) { vals := make([]time.Time, 0, len(strs)) - for _, str := range strs { + parser := func(str string) (interface{}, error) { val, err := time.Parse(format, str) + return val, err + } + rawVals, err := k.doParse(strs, addInvalid, returnOnInvalid, parser) + if err == nil { + for _, val := range rawVals { + vals = append(vals, val.(time.Time)) + } + } + return vals, err +} + + +// doParse transforms strings to different types +func (k *Key) doParse(strs []string, addInvalid, returnOnInvalid bool, parser Parser) ([]interface{}, error) { + vals := make([]interface{}, 0, len(strs)) + for _, str := range strs { + val, err := parser(str) if err != nil && returnOnInvalid { return nil, err } diff --git a/vendor/gopkg.in/ini.v1/parser.go b/vendor/gopkg.in/ini.v1/parser.go index 7c22a25..6514716 100644 --- a/vendor/gopkg.in/ini.v1/parser.go +++ b/vendor/gopkg.in/ini.v1/parser.go @@ -25,7 +25,9 @@ import ( "unicode" ) -var pythonMultiline = regexp.MustCompile("^(\\s+)([^\n]+)") +const minReaderBufferSize = 4096 + +var pythonMultiline = regexp.MustCompile(`^([\t\f ]+)(.*)`) type parserOptions struct { IgnoreContinuation bool @@ -35,6 +37,8 @@ type parserOptions struct { UnescapeValueDoubleQuotes bool UnescapeValueCommentSymbols bool PreserveSurroundedQuote bool + DebugFunc DebugFunc + ReaderBufferSize int } type parser struct { @@ -46,9 +50,20 @@ type parser struct { comment *bytes.Buffer } +func (p *parser) debug(format string, args ...interface{}) { + if p.options.DebugFunc != nil { + p.options.DebugFunc(fmt.Sprintf(format, args...)) + } +} + func newParser(r io.Reader, opts parserOptions) *parser { + size := opts.ReaderBufferSize + if size < minReaderBufferSize { + size = minReaderBufferSize + } + return &parser{ - buf: bufio.NewReader(r), + buf: bufio.NewReaderSize(r, size), options: opts, count: 1, comment: &bytes.Buffer{}, @@ -69,7 +84,10 @@ func (p *parser) BOM() error { case mask[0] == 254 && mask[1] == 255: fallthrough case mask[0] == 255 && mask[1] == 254: - p.buf.Read(mask) + _, err = p.buf.Read(mask) + if err != nil { + return err + } case mask[0] == 239 && mask[1] == 187: mask, err := p.buf.Peek(3) if err != nil && err != io.EOF { @@ -78,7 +96,10 @@ func (p *parser) BOM() error { return nil } if mask[2] == 191 { - p.buf.Read(mask) + _, err = p.buf.Read(mask) + if err != nil { + return err + } } } return nil @@ -120,7 +141,7 @@ func readKeyName(delimiters string, in []byte) (string, int, error) { } // Get out key name - endIdx := -1 + var endIdx int if len(keyQuote) > 0 { startIdx := len(keyQuote) // FIXME: fail case -> """"""name"""=value @@ -166,7 +187,7 @@ func (p *parser) readMultilines(line, val, valQuote string) (string, error) { } val += next if p.isEOF { - return "", fmt.Errorf("missing closing key quote from '%s' to '%s'", line, next) + return "", fmt.Errorf("missing closing key quote from %q to %q", line, next) } } return val, nil @@ -285,33 +306,55 @@ func (p *parser) readPythonMultilines(line string, bufferSize int) (string, erro parserBufferPeekResult, _ := p.buf.Peek(bufferSize) peekBuffer := bytes.NewBuffer(parserBufferPeekResult) + indentSize := 0 for { peekData, peekErr := peekBuffer.ReadBytes('\n') if peekErr != nil { if peekErr == io.EOF { + p.debug("readPythonMultilines: io.EOF, peekData: %q, line: %q", string(peekData), line) return line, nil } + + p.debug("readPythonMultilines: failed to peek with error: %v", peekErr) return "", peekErr } + p.debug("readPythonMultilines: parsing %q", string(peekData)) + peekMatches := pythonMultiline.FindStringSubmatch(string(peekData)) + p.debug("readPythonMultilines: matched %d parts", len(peekMatches)) + for n, v := range peekMatches { + p.debug(" %d: %q", n, v) + } + + // Return if not a Python multiline value. if len(peekMatches) != 3 { + p.debug("readPythonMultilines: end of value, got: %q", line) return line, nil } - // NOTE: Return if not a python-ini multi-line value. - currentIdentSize := len(peekMatches[1]) - if currentIdentSize <= 0 { + // Determine indent size and line prefix. + currentIndentSize := len(peekMatches[1]) + if indentSize < 1 { + indentSize = currentIndentSize + p.debug("readPythonMultilines: indent size is %d", indentSize) + } + + // Make sure each line is indented at least as far as first line. + if currentIndentSize < indentSize { + p.debug("readPythonMultilines: end of value, current indent: %d, expected indent: %d, line: %q", currentIndentSize, indentSize, line) return line, nil } - // NOTE: Just advance the parser reader (buffer) in-sync with the peek buffer. - _, err := p.readUntil('\n') + // Advance the parser reader (buffer) in-sync with the peek buffer. + _, err := p.buf.Discard(len(peekData)) if err != nil { + p.debug("readPythonMultilines: failed to skip to the end, returning error") return "", err } - line += fmt.Sprintf("\n%s", peekMatches[2]) + // Handle indented empty line. + line += "\n" + peekMatches[1][indentSize:] + peekMatches[2] } } @@ -325,6 +368,8 @@ func (f *File) parse(reader io.Reader) (err error) { UnescapeValueDoubleQuotes: f.options.UnescapeValueDoubleQuotes, UnescapeValueCommentSymbols: f.options.UnescapeValueCommentSymbols, PreserveSurroundedQuote: f.options.PreserveSurroundedQuote, + DebugFunc: f.options.DebugFunc, + ReaderBufferSize: f.options.ReaderBufferSize, }) if err = p.BOM(); err != nil { return fmt.Errorf("BOM: %v", err) @@ -332,7 +377,7 @@ func (f *File) parse(reader io.Reader) (err error) { // Ignore error because default section name is never empty string. name := DefaultSection - if f.options.Insensitive { + if f.options.Insensitive || f.options.InsensitiveSections { name = strings.ToLower(DefaultSection) } section, _ := f.NewSection(name) @@ -348,8 +393,8 @@ func (f *File) parse(reader io.Reader) (err error) { // the size of the parser buffer is found. // TODO(unknwon): When Golang 1.10 is the lowest version supported, replace with `parserBufferSize := p.buf.Size()`. parserBufferSize := 0 - // NOTE: Peek 1kb at a time. - currentPeekSize := 1024 + // NOTE: Peek 4kb at a time. + currentPeekSize := minReaderBufferSize if f.options.AllowPythonMultilineValues { for { @@ -374,7 +419,10 @@ func (f *File) parse(reader io.Reader) (err error) { if f.options.AllowNestedValues && isLastValueEmpty && len(line) > 0 { if line[0] == ' ' || line[0] == '\t' { - lastRegularKey.addNestedValue(string(bytes.TrimSpace(line))) + err = lastRegularKey.addNestedValue(string(bytes.TrimSpace(line))) + if err != nil { + return err + } continue } } @@ -414,14 +462,14 @@ func (f *File) parse(reader io.Reader) (err error) { section.Comment = strings.TrimSpace(p.comment.String()) - // Reset aotu-counter and comments + // Reset auto-counter and comments p.comment.Reset() p.count = 1 inUnparseableSection = false for i := range f.options.UnparseableSections { if f.options.UnparseableSections[i] == name || - (f.options.Insensitive && strings.ToLower(f.options.UnparseableSections[i]) == strings.ToLower(name)) { + ((f.options.Insensitive || f.options.InsensitiveSections) && strings.EqualFold(f.options.UnparseableSections[i], name)) { inUnparseableSection = true continue } diff --git a/vendor/gopkg.in/ini.v1/section.go b/vendor/gopkg.in/ini.v1/section.go index 0bd3e13..afaa97c 100644 --- a/vendor/gopkg.in/ini.v1/section.go +++ b/vendor/gopkg.in/ini.v1/section.go @@ -66,7 +66,7 @@ func (s *Section) SetBody(body string) { func (s *Section) NewKey(name, val string) (*Key, error) { if len(name) == 0 { return nil, errors.New("error creating new key: empty key name") - } else if s.f.options.Insensitive { + } else if s.f.options.Insensitive || s.f.options.InsensitiveKeys { name = strings.ToLower(name) } @@ -109,7 +109,7 @@ func (s *Section) GetKey(name string) (*Key, error) { if s.f.BlockMode { s.f.lock.RLock() } - if s.f.options.Insensitive { + if s.f.options.Insensitive || s.f.options.InsensitiveKeys { name = strings.ToLower(name) } key := s.keys[name] @@ -121,7 +121,7 @@ func (s *Section) GetKey(name string) (*Key, error) { // Check if it is a child-section. sname := s.name for { - if i := strings.LastIndex(sname, "."); i > -1 { + if i := strings.LastIndex(sname, s.f.options.ChildSectionDelimiter); i > -1 { sname = sname[:i] sec, err := s.f.GetSection(sname) if err != nil { @@ -131,7 +131,7 @@ func (s *Section) GetKey(name string) (*Key, error) { } break } - return nil, fmt.Errorf("error when getting key of section '%s': key '%s' not exists", s.name, name) + return nil, fmt.Errorf("error when getting key of section %q: key %q not exists", s.name, name) } return key, nil } @@ -188,7 +188,7 @@ func (s *Section) ParentKeys() []*Key { var parentKeys []*Key sname := s.name for { - if i := strings.LastIndex(sname, "."); i > -1 { + if i := strings.LastIndex(sname, s.f.options.ChildSectionDelimiter); i > -1 { sname = sname[:i] sec, err := s.f.GetSection(sname) if err != nil { @@ -245,11 +245,11 @@ func (s *Section) DeleteKey(name string) { // For example, "[parent.child1]" and "[parent.child12]" are child sections // of section "[parent]". func (s *Section) ChildSections() []*Section { - prefix := s.name + "." + prefix := s.name + s.f.options.ChildSectionDelimiter children := make([]*Section, 0, 3) for _, name := range s.f.sectionList { if strings.HasPrefix(name, prefix) { - children = append(children, s.f.sections[name]) + children = append(children, s.f.sections[name]...) } } return children diff --git a/vendor/gopkg.in/ini.v1/struct.go b/vendor/gopkg.in/ini.v1/struct.go index 2764534..a486b2f 100644 --- a/vendor/gopkg.in/ini.v1/struct.go +++ b/vendor/gopkg.in/ini.v1/struct.go @@ -155,23 +155,45 @@ func wrapStrictError(err error, isStrict bool) error { // but it does not return error for failing parsing, // because we want to use default value that is already assigned to struct. func setWithProperType(t reflect.Type, key *Key, field reflect.Value, delim string, allowShadow, isStrict bool) error { - switch t.Kind() { + vt := t + isPtr := t.Kind() == reflect.Ptr + if isPtr { + vt = t.Elem() + } + switch vt.Kind() { case reflect.String: - if len(key.String()) == 0 { - return nil + stringVal := key.String() + if isPtr { + field.Set(reflect.ValueOf(&stringVal)) + } else if len(stringVal) > 0 { + field.SetString(key.String()) } - field.SetString(key.String()) case reflect.Bool: boolVal, err := key.Bool() if err != nil { return wrapStrictError(err, isStrict) } - field.SetBool(boolVal) + if isPtr { + field.Set(reflect.ValueOf(&boolVal)) + } else { + field.SetBool(boolVal) + } case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - durationVal, err := key.Duration() - // Skip zero value - if err == nil && int64(durationVal) > 0 { - field.Set(reflect.ValueOf(durationVal)) + // ParseDuration will not return err for `0`, so check the type name + if vt.Name() == "Duration" { + durationVal, err := key.Duration() + if err != nil { + if intVal, err := key.Int64(); err == nil { + field.SetInt(intVal) + return nil + } + return wrapStrictError(err, isStrict) + } + if isPtr { + field.Set(reflect.ValueOf(&durationVal)) + } else if int64(durationVal) > 0 { + field.Set(reflect.ValueOf(durationVal)) + } return nil } @@ -179,13 +201,23 @@ func setWithProperType(t reflect.Type, key *Key, field reflect.Value, delim stri if err != nil { return wrapStrictError(err, isStrict) } - field.SetInt(intVal) + if isPtr { + pv := reflect.New(t.Elem()) + pv.Elem().SetInt(intVal) + field.Set(pv) + } else { + field.SetInt(intVal) + } // byte is an alias for uint8, so supporting uint8 breaks support for byte case reflect.Uint, reflect.Uint16, reflect.Uint32, reflect.Uint64: durationVal, err := key.Duration() // Skip zero value if err == nil && uint64(durationVal) > 0 { - field.Set(reflect.ValueOf(durationVal)) + if isPtr { + field.Set(reflect.ValueOf(&durationVal)) + } else { + field.Set(reflect.ValueOf(durationVal)) + } return nil } @@ -193,52 +225,59 @@ func setWithProperType(t reflect.Type, key *Key, field reflect.Value, delim stri if err != nil { return wrapStrictError(err, isStrict) } - field.SetUint(uintVal) + if isPtr { + pv := reflect.New(t.Elem()) + pv.Elem().SetUint(uintVal) + field.Set(pv) + } else { + field.SetUint(uintVal) + } case reflect.Float32, reflect.Float64: floatVal, err := key.Float64() if err != nil { return wrapStrictError(err, isStrict) } - field.SetFloat(floatVal) + if isPtr { + pv := reflect.New(t.Elem()) + pv.Elem().SetFloat(floatVal) + field.Set(pv) + } else { + field.SetFloat(floatVal) + } case reflectTime: timeVal, err := key.Time() if err != nil { return wrapStrictError(err, isStrict) } - field.Set(reflect.ValueOf(timeVal)) + if isPtr { + field.Set(reflect.ValueOf(&timeVal)) + } else { + field.Set(reflect.ValueOf(timeVal)) + } case reflect.Slice: return setSliceWithProperType(key, field, delim, allowShadow, isStrict) - case reflect.Ptr: - switch t.Elem().Kind() { - case reflect.Bool: - boolVal, err := key.Bool() - if err != nil { - return wrapStrictError(err, isStrict) - } - field.Set(reflect.ValueOf(&boolVal)) - default: - return fmt.Errorf("unsupported type '%s'", t) - } default: - return fmt.Errorf("unsupported type '%s'", t) + return fmt.Errorf("unsupported type %q", t) } return nil } -func parseTagOptions(tag string) (rawName string, omitEmpty bool, allowShadow bool) { - opts := strings.SplitN(tag, ",", 3) +func parseTagOptions(tag string) (rawName string, omitEmpty bool, allowShadow bool, allowNonUnique bool, extends bool) { + opts := strings.SplitN(tag, ",", 5) rawName = opts[0] - if len(opts) > 1 { - omitEmpty = opts[1] == "omitempty" + for _, opt := range opts[1:] { + omitEmpty = omitEmpty || (opt == "omitempty") + allowShadow = allowShadow || (opt == "allowshadow") + allowNonUnique = allowNonUnique || (opt == "nonunique") + extends = extends || (opt == "extends") } - if len(opts) > 2 { - allowShadow = opts[2] == "allowshadow" - } - return rawName, omitEmpty, allowShadow + return rawName, omitEmpty, allowShadow, allowNonUnique, extends } -func (s *Section) mapTo(val reflect.Value, isStrict bool) error { +// mapToField maps the given value to the matching field of the given section. +// The sectionIndex is the index (if non unique sections are enabled) to which the value should be added. +func (s *Section) mapToField(val reflect.Value, isStrict bool, sectionIndex int, sectionName string) error { if val.Kind() == reflect.Ptr { val = val.Elem() } @@ -253,7 +292,7 @@ func (s *Section) mapTo(val reflect.Value, isStrict bool) error { continue } - rawName, _, allowShadow := parseTagOptions(tag) + rawName, _, allowShadow, allowNonUnique, extends := parseTagOptions(tag) fieldName := s.parseFieldName(tpField.Name, rawName) if len(fieldName) == 0 || !field.CanSet() { continue @@ -261,63 +300,116 @@ func (s *Section) mapTo(val reflect.Value, isStrict bool) error { isStruct := tpField.Type.Kind() == reflect.Struct isStructPtr := tpField.Type.Kind() == reflect.Ptr && tpField.Type.Elem().Kind() == reflect.Struct - isAnonymous := tpField.Type.Kind() == reflect.Ptr && tpField.Anonymous - if isAnonymous { + isAnonymousPtr := tpField.Type.Kind() == reflect.Ptr && tpField.Anonymous + if isAnonymousPtr { field.Set(reflect.New(tpField.Type.Elem())) } - if isAnonymous || isStruct || isStructPtr { - if sec, err := s.f.GetSection(fieldName); err == nil { - // Only set the field to non-nil struct value if we have - // a section for it. Otherwise, we end up with a non-nil - // struct ptr even though there is no data. + if extends && (isAnonymousPtr || (isStruct && tpField.Anonymous)) { + if isStructPtr && field.IsNil() { + field.Set(reflect.New(tpField.Type.Elem())) + } + fieldSection := s + if rawName != "" { + sectionName = s.name + s.f.options.ChildSectionDelimiter + rawName + if secs, err := s.f.SectionsByName(sectionName); err == nil && sectionIndex < len(secs) { + fieldSection = secs[sectionIndex] + } + } + if err := fieldSection.mapToField(field, isStrict, sectionIndex, sectionName); err != nil { + return fmt.Errorf("map to field %q: %v", fieldName, err) + } + } else if isAnonymousPtr || isStruct || isStructPtr { + if secs, err := s.f.SectionsByName(fieldName); err == nil { + if len(secs) <= sectionIndex { + return fmt.Errorf("there are not enough sections (%d <= %d) for the field %q", len(secs), sectionIndex, fieldName) + } + // Only set the field to non-nil struct value if we have a section for it. + // Otherwise, we end up with a non-nil struct ptr even though there is no data. if isStructPtr && field.IsNil() { field.Set(reflect.New(tpField.Type.Elem())) } - if err = sec.mapTo(field, isStrict); err != nil { - return fmt.Errorf("error mapping field(%s): %v", fieldName, err) + if err = secs[sectionIndex].mapToField(field, isStrict, sectionIndex, fieldName); err != nil { + return fmt.Errorf("map to field %q: %v", fieldName, err) } continue } } + // Map non-unique sections + if allowNonUnique && tpField.Type.Kind() == reflect.Slice { + newField, err := s.mapToSlice(fieldName, field, isStrict) + if err != nil { + return fmt.Errorf("map to slice %q: %v", fieldName, err) + } + + field.Set(newField) + continue + } + if key, err := s.GetKey(fieldName); err == nil { delim := parseDelim(tpField.Tag.Get("delim")) if err = setWithProperType(tpField.Type, key, field, delim, allowShadow, isStrict); err != nil { - return fmt.Errorf("error mapping field(%s): %v", fieldName, err) + return fmt.Errorf("set field %q: %v", fieldName, err) } } } return nil } -// MapTo maps section to given struct. -func (s *Section) MapTo(v interface{}) error { +// mapToSlice maps all sections with the same name and returns the new value. +// The type of the Value must be a slice. +func (s *Section) mapToSlice(secName string, val reflect.Value, isStrict bool) (reflect.Value, error) { + secs, err := s.f.SectionsByName(secName) + if err != nil { + return reflect.Value{}, err + } + + typ := val.Type().Elem() + for i, sec := range secs { + elem := reflect.New(typ) + if err = sec.mapToField(elem, isStrict, i, sec.name); err != nil { + return reflect.Value{}, fmt.Errorf("map to field from section %q: %v", secName, err) + } + + val = reflect.Append(val, elem.Elem()) + } + return val, nil +} + +// mapTo maps a section to object v. +func (s *Section) mapTo(v interface{}, isStrict bool) error { typ := reflect.TypeOf(v) val := reflect.ValueOf(v) if typ.Kind() == reflect.Ptr { typ = typ.Elem() val = val.Elem() } else { - return errors.New("cannot map to non-pointer struct") + return errors.New("not a pointer to a struct") } - return s.mapTo(val, false) + if typ.Kind() == reflect.Slice { + newField, err := s.mapToSlice(s.name, val, isStrict) + if err != nil { + return err + } + + val.Set(newField) + return nil + } + + return s.mapToField(val, isStrict, 0, s.name) +} + +// MapTo maps section to given struct. +func (s *Section) MapTo(v interface{}) error { + return s.mapTo(v, false) } // StrictMapTo maps section to given struct in strict mode, // which returns all possible error including value parsing error. func (s *Section) StrictMapTo(v interface{}) error { - typ := reflect.TypeOf(v) - val := reflect.ValueOf(v) - if typ.Kind() == reflect.Ptr { - typ = typ.Elem() - val = val.Elem() - } else { - return errors.New("cannot map to non-pointer struct") - } - - return s.mapTo(val, true) + return s.mapTo(v, true) } // MapTo maps file to given struct. @@ -395,10 +487,10 @@ func reflectSliceWithProperType(key *Key, field reflect.Value, delim string, all if i == 0 { keyWithShadows = newKey(key.s, key.name, val) } else { - keyWithShadows.AddShadow(val) + _ = keyWithShadows.AddShadow(val) } } - key = keyWithShadows + *key = *keyWithShadows return nil } @@ -448,7 +540,7 @@ func reflectWithProperType(t reflect.Type, key *Key, field reflect.Value, delim return reflectWithProperType(t.Elem(), key, field.Elem(), delim, allowShadow) } default: - return fmt.Errorf("unsupported type '%s'", t) + return fmt.Errorf("unsupported type %q", t) } return nil } @@ -476,6 +568,11 @@ func isEmptyValue(v reflect.Value) bool { return false } +// StructReflector is the interface implemented by struct types that can extract themselves into INI objects. +type StructReflector interface { + ReflectINIStruct(*File) error +} + func (s *Section) reflectFrom(val reflect.Value) error { if val.Kind() == reflect.Ptr { val = val.Elem() @@ -483,6 +580,10 @@ func (s *Section) reflectFrom(val reflect.Value) error { typ := val.Type() for i := 0; i < typ.NumField(); i++ { + if !val.Field(i).CanInterface() { + continue + } + field := val.Field(i) tpField := typ.Field(i) @@ -491,17 +592,28 @@ func (s *Section) reflectFrom(val reflect.Value) error { continue } - rawName, omitEmpty, allowShadow := parseTagOptions(tag) + rawName, omitEmpty, allowShadow, allowNonUnique, extends := parseTagOptions(tag) if omitEmpty && isEmptyValue(field) { continue } + if r, ok := field.Interface().(StructReflector); ok { + return r.ReflectINIStruct(s.f) + } + fieldName := s.parseFieldName(tpField.Name, rawName) if len(fieldName) == 0 || !field.CanSet() { continue } - if (tpField.Type.Kind() == reflect.Ptr && tpField.Anonymous) || + if extends && tpField.Anonymous && (tpField.Type.Kind() == reflect.Ptr || tpField.Type.Kind() == reflect.Struct) { + if err := s.reflectFrom(field); err != nil { + return fmt.Errorf("reflect from field %q: %v", fieldName, err) + } + continue + } + + if (tpField.Type.Kind() == reflect.Ptr && tpField.Type.Elem().Kind() == reflect.Struct) || (tpField.Type.Kind() == reflect.Struct && tpField.Type.Name() != "Time") { // Note: The only error here is section doesn't exist. sec, err := s.f.GetSection(fieldName) @@ -516,12 +628,41 @@ func (s *Section) reflectFrom(val reflect.Value) error { } if err = sec.reflectFrom(field); err != nil { - return fmt.Errorf("error reflecting field (%s): %v", fieldName, err) + return fmt.Errorf("reflect from field %q: %v", fieldName, err) } continue } - // Note: Same reason as secion. + if allowNonUnique && tpField.Type.Kind() == reflect.Slice { + slice := field.Slice(0, field.Len()) + if field.Len() == 0 { + return nil + } + sliceOf := field.Type().Elem().Kind() + + for i := 0; i < field.Len(); i++ { + if sliceOf != reflect.Struct && sliceOf != reflect.Ptr { + return fmt.Errorf("field %q is not a slice of pointer or struct", fieldName) + } + + sec, err := s.f.NewSection(fieldName) + if err != nil { + return err + } + + // Add comment from comment tag + if len(sec.Comment) == 0 { + sec.Comment = tpField.Tag.Get("comment") + } + + if err := sec.reflectFrom(slice.Index(i)); err != nil { + return fmt.Errorf("reflect from field %q: %v", fieldName, err) + } + } + continue + } + + // Note: Same reason as section. key, err := s.GetKey(fieldName) if err != nil { key, _ = s.NewKey(fieldName, "") @@ -532,23 +673,58 @@ func (s *Section) reflectFrom(val reflect.Value) error { key.Comment = tpField.Tag.Get("comment") } - if err = reflectWithProperType(tpField.Type, key, field, parseDelim(tpField.Tag.Get("delim")), allowShadow); err != nil { - return fmt.Errorf("error reflecting field (%s): %v", fieldName, err) + delim := parseDelim(tpField.Tag.Get("delim")) + if err = reflectWithProperType(tpField.Type, key, field, delim, allowShadow); err != nil { + return fmt.Errorf("reflect field %q: %v", fieldName, err) } } return nil } -// ReflectFrom reflects secion from given struct. +// ReflectFrom reflects section from given struct. It overwrites existing ones. func (s *Section) ReflectFrom(v interface{}) error { typ := reflect.TypeOf(v) val := reflect.ValueOf(v) + + if s.name != DefaultSection && s.f.options.AllowNonUniqueSections && + (typ.Kind() == reflect.Slice || typ.Kind() == reflect.Ptr) { + // Clear sections to make sure none exists before adding the new ones + s.f.DeleteSection(s.name) + + if typ.Kind() == reflect.Ptr { + sec, err := s.f.NewSection(s.name) + if err != nil { + return err + } + return sec.reflectFrom(val.Elem()) + } + + slice := val.Slice(0, val.Len()) + sliceOf := val.Type().Elem().Kind() + if sliceOf != reflect.Ptr { + return fmt.Errorf("not a slice of pointers") + } + + for i := 0; i < slice.Len(); i++ { + sec, err := s.f.NewSection(s.name) + if err != nil { + return err + } + + err = sec.reflectFrom(slice.Index(i)) + if err != nil { + return fmt.Errorf("reflect from %dth field: %v", i, err) + } + } + + return nil + } + if typ.Kind() == reflect.Ptr { - typ = typ.Elem() val = val.Elem() } else { - return errors.New("cannot reflect from non-pointer struct") + return errors.New("not a pointer to a struct") } return s.reflectFrom(val) diff --git a/vendor/modules.txt b/vendor/modules.txt index 879230e..42c3521 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,2 +1,9 @@ -# gopkg.in/ini.v1 v1.48.0 +# github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 +## explicit +# github.com/smartystreets/assertions v1.2.0 +## explicit +# github.com/smartystreets/goconvey v1.6.4 +## explicit +# gopkg.in/ini.v1 v1.62.0 +## explicit gopkg.in/ini.v1