Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Validate migration files #18203

Merged
merged 6 commits into from
Jan 26, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ _testmain.go
coverage.all
cpu.out

/modules/migration/bindata.go
/modules/migration/bindata.go.hash
/modules/options/bindata.go
/modules/options/bindata.go.hash
/modules/public/bindata.go
Expand Down
5 changes: 5 additions & 0 deletions cmd/restore_repo.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,10 @@ var CmdRestoreRepository = cli.Command{
Usage: `Which items will be restored, one or more units should be separated as comma.
wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.`,
},
cli.BoolFlag{
Name: "validation",
Usage: "Sanity check the content of the files before trying to load them",
},
},
}

Expand All @@ -58,6 +62,7 @@ func runRestoreRepository(c *cli.Context) error {
c.String("owner_name"),
c.String("repo_name"),
c.StringSlice("units"),
c.Bool("validation"),
)
if statusCode == http.StatusOK {
return nil
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ require (
github.com/quasoft/websspi v1.0.0
github.com/rs/xid v1.3.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 // indirect
github.com/sergi/go-diff v1.2.0
github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect
github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1039,6 +1039,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 h1:TToq11gyfNlrMFZiYujSekIsPd9AmsA2Bj/iv+s4JHE=
github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
Expand Down
2 changes: 1 addition & 1 deletion integrations/dump_restore_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ func TestDumpRestore(t *testing.T) {
//

newreponame := "restoredrepo"
err = migrations.RestoreRepository(ctx, d, repo.OwnerName, newreponame, []string{"labels", "milestones", "issues", "comments"})
err = migrations.RestoreRepository(ctx, d, repo.OwnerName, newreponame, []string{"labels", "milestones", "issues", "comments"}, false)
assert.NoError(t, err)

newrepo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{Name: newreponame}).(*repo_model.Repository)
Expand Down
112 changes: 112 additions & 0 deletions modules/migration/file_format.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package migration

import (
"fmt"
"os"
"strings"

"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log"

"github.com/santhosh-tekuri/jsonschema/v5"
"gopkg.in/yaml.v2"
)

// Load project data from file, with optional validation
func Load(filename string, data interface{}, validation bool) error {
isJSON := strings.HasSuffix(filename, ".json")

bs, err := os.ReadFile(filename)
if err != nil {
return err
}

if validation {
err := validate(bs, data, isJSON)
if err != nil {
return err
}
}
return unmarshal(bs, data, isJSON)
}

func unmarshal(bs []byte, data interface{}, isJSON bool) error {
if isJSON {
return json.Unmarshal(bs, data)
}
return yaml.Unmarshal(bs, data)
}

func getSchema(filename string) (*jsonschema.Schema, error) {
c := jsonschema.NewCompiler()
c.LoadURL = openSchema
return c.Compile(filename)
}

func validate(bs []byte, datatype interface{}, isJSON bool) error {
var v interface{}
err := unmarshal(bs, &v, isJSON)
if err != nil {
return err
}
if !isJSON {
v, err = toStringKeys(v)
if err != nil {
return err
}
}

var schemaFilename string
switch datatype := datatype.(type) {
case *[]*Issue:
schemaFilename = "issue.json"
case *[]*Milestone:
schemaFilename = "milestone.json"
default:
return fmt.Errorf("file_format:validate: %T has not a validation implemented", datatype)
}

sch, err := getSchema(schemaFilename)
if err != nil {
return err
}
err = sch.Validate(v)
if err != nil {
log.Error("migration validation with %s failed for\n%s", schemaFilename, string(bs))
}
return err
}

func toStringKeys(val interface{}) (interface{}, error) {
var err error
switch val := val.(type) {
case map[interface{}]interface{}:
m := make(map[string]interface{})
for k, v := range val {
k, ok := k.(string)
if !ok {
return nil, fmt.Errorf("found non-string key %T %s", k, k)
}
m[k], err = toStringKeys(v)
if err != nil {
return nil, err
}
}
return m, nil
case []interface{}:
l := make([]interface{}, len(val))
for i, v := range val {
l[i], err = toStringKeys(v)
if err != nil {
return nil, err
}
}
return l, nil
default:
return val, nil
}
}
39 changes: 39 additions & 0 deletions modules/migration/file_format_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package migration

import (
"strings"
"testing"

"github.com/santhosh-tekuri/jsonschema/v5"
"github.com/stretchr/testify/assert"
)

func TestMigrationJSON_IssueOK(t *testing.T) {
issues := make([]*Issue, 0, 10)
err := Load("file_format_testdata/issue_a.json", &issues, true)
assert.NoError(t, err)
err = Load("file_format_testdata/issue_a.yml", &issues, true)
assert.NoError(t, err)
}

func TestMigrationJSON_IssueFail(t *testing.T) {
issues := make([]*Issue, 0, 10)
err := Load("file_format_testdata/issue_b.json", &issues, true)
if _, ok := err.(*jsonschema.ValidationError); ok {
errors := strings.Split(err.(*jsonschema.ValidationError).GoString(), "\n")
assert.Contains(t, errors[1], "missing properties")
assert.Contains(t, errors[1], "poster_id")
} else {
t.Fatalf("got: type %T with value %s, want: *jsonschema.ValidationError", err, err)
}
}

func TestMigrationJSON_MilestoneOK(t *testing.T) {
milestones := make([]*Milestone, 0, 10)
err := Load("file_format_testdata/milestones.json", &milestones, true)
assert.NoError(t, err)
}
14 changes: 14 additions & 0 deletions modules/migration/file_format_testdata/issue_a.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
[
{
"number": 1,
"poster_id": 1,
"poster_name": "name_a",
"title": "title_a",
"content": "content_a",
"state": "closed",
"is_locked": false,
"created": "1985-04-12T23:20:50.52Z",
"updated": "1986-04-12T23:20:50.52Z",
"closed": "1987-04-12T23:20:50.52Z"
}
]
10 changes: 10 additions & 0 deletions modules/migration/file_format_testdata/issue_a.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
- number: 1
poster_id: 1
poster_name: name_a
title: title_a
content: content_a
state: closed
is_locked: false
created: 2021-05-27T15:24:13+02:00
updated: 2021-11-11T10:52:45+01:00
closed: 2021-11-11T10:52:45+01:00
5 changes: 5 additions & 0 deletions modules/migration/file_format_testdata/issue_b.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
[
{
"number": 1
}
]
20 changes: 20 additions & 0 deletions modules/migration/file_format_testdata/milestones.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
[
{
"title": "title_a",
"description": "description_a",
"deadline": "1988-04-12T23:20:50.52Z",
"created": "1985-04-12T23:20:50.52Z",
"updated": "1986-04-12T23:20:50.52Z",
"closed": "1987-04-12T23:20:50.52Z",
"state": "closed"
},
{
"title": "title_b",
"description": "description_b",
"deadline": "1998-04-12T23:20:50.52Z",
"created": "1995-04-12T23:20:50.52Z",
"updated": "1996-04-12T23:20:50.52Z",
"closed": null,
"state": "open"
}
]
32 changes: 16 additions & 16 deletions modules/migration/issue.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,21 +28,21 @@ func (c BasicIssueContext) ForeignID() int64 {

// Issue is a standard issue information
type Issue struct {
Number int64
PosterID int64 `yaml:"poster_id"`
PosterName string `yaml:"poster_name"`
PosterEmail string `yaml:"poster_email"`
Title string
Content string
Ref string
Milestone string
State string // closed, open
IsLocked bool `yaml:"is_locked"`
Created time.Time
Updated time.Time
Closed *time.Time
Labels []*Label
Reactions []*Reaction
Assignees []string
Number int64 `json:"number"`
PosterID int64 `yaml:"poster_id" json:"poster_id"`
PosterName string `yaml:"poster_name" json:"poster_name"`
PosterEmail string `yaml:"poster_email" json:"poster_email"`
Title string `json:"title"`
Content string `json:"content"`
Ref string `json:"ref"`
Milestone string `json:"milestone"`
State string `json:"state"` // closed, open
IsLocked bool `yaml:"is_locked" json:"is_locked"`
Created time.Time `json:"created"`
Updated time.Time `json:"updated"`
Closed *time.Time `json:"closed"`
Labels []*Label `json:"labels"`
Reactions []*Reaction `json:"reactions"`
Assignees []string `json:"assignees"`
Context IssueContext `yaml:"-"`
}
6 changes: 3 additions & 3 deletions modules/migration/label.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ package migration

// Label defines a standard label information
type Label struct {
Name string
Color string
Description string
Name string `json:"name"`
Color string `json:"color"`
Description string `json:"description"`
}
14 changes: 7 additions & 7 deletions modules/migration/milestone.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@ import "time"

// Milestone defines a standard milestone
type Milestone struct {
Title string
Description string
Deadline *time.Time
Created time.Time
Updated *time.Time
Closed *time.Time
State string // open, closed
Title string `json:"title"`
Description string `json:"description"`
Deadline *time.Time `json:"deadline"`
Created time.Time `json:"created"`
Updated *time.Time `json:"updated"`
Closed *time.Time `json:"closed"`
State string `json:"state"` // open, closed
}
6 changes: 3 additions & 3 deletions modules/migration/reaction.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ package migration

// Reaction represents a reaction to an issue/pr/comment.
type Reaction struct {
UserID int64 `yaml:"user_id"`
UserName string `yaml:"user_name"`
Content string
UserID int64 `yaml:"user_id" json:"user_id"`
UserName string `yaml:"user_name" json:"user_name"`
Content string `json:"content"`
}
Loading