From dca09f8b4adb40dc0bf23763f43e7006c5ec98cc Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Thu, 28 Dec 2023 19:30:39 +0800 Subject: [PATCH 1/2] fix --- models/git/branch.go | 24 +++++++++++++++++++++-- models/migrations/migrations.go | 2 ++ models/migrations/v1_22/v283_test.go | 16 +++++++-------- models/migrations/v1_22/v284.go | 29 ++++++++++++++++++++++++++++ models/migrations/v1_22/v284_test.go | 25 ++++++++++++++++++++++++ tests/integration/api_branch_test.go | 12 ++++++++++++ 6 files changed, 98 insertions(+), 10 deletions(-) create mode 100644 models/migrations/v1_22/v284.go create mode 100644 models/migrations/v1_22/v284_test.go diff --git a/models/git/branch.go b/models/git/branch.go index ffd1d7ed164a..72141662f243 100644 --- a/models/git/branch.go +++ b/models/git/branch.go @@ -13,10 +13,12 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" "xorm.io/builder" + "xorm.io/xorm/schemas" ) // ErrBranchNotExist represents an error that branch with such name does not exist. @@ -103,7 +105,7 @@ func (err ErrBranchesEqual) Unwrap() error { type Branch struct { ID int64 RepoID int64 `xorm:"UNIQUE(s)"` - Name string `xorm:"UNIQUE(s) NOT NULL"` // git's ref-name is case-sensitive internally, however, in some databases (mssql, mysql, by default), it's case-insensitive at the moment + Name string `xorm:"UNIQUE(s) NOT NULL"` CommitID string CommitMessage string `xorm:"TEXT"` // it only stores the message summary (the first line) PusherID int64 @@ -117,6 +119,20 @@ type Branch struct { UpdatedUnix timeutil.TimeStamp `xorm:"updated"` } +// TableCollations is to make the "name" column case-sensitve for MySQL/MSSQL +func (b *Branch) TableCollations() []*schemas.Collation { + if setting.Database.Type.IsMySQL() { + return []*schemas.Collation{ + {Column: "name", Name: "utf8mb4_bin"}, + } + } else if setting.Database.Type.IsMSSQL() { + return []*schemas.Collation{ + {Column: "name", Name: "Latin1_General_CS_AS"}, + } + } + return nil +} + func (b *Branch) LoadDeletedBy(ctx context.Context) (err error) { if b.DeletedBy == nil { b.DeletedBy, err = user_model.GetUserByID(ctx, b.DeletedByID) @@ -380,7 +396,11 @@ func RenameBranch(ctx context.Context, repo *repo_model.Repository, from, to str // except the indicate branch func FindRecentlyPushedNewBranches(ctx context.Context, repoID, userID int64, excludeBranchName string) (BranchList, error) { branches := make(BranchList, 0, 2) - subQuery := builder.Select("head_branch").From("pull_request"). + sqlHeadBranch := "head_branch" + if setting.Database.Type.IsMSSQL() { + sqlHeadBranch = "CAST(head_branch AS nvarchar(255)) COLLATE Latin1_General_CS_AS" // FIXME: a dirty hack for MSSQL collation + } + subQuery := builder.Select(sqlHeadBranch).From("pull_request"). InnerJoin("issue", "issue.id = pull_request.issue_id"). Where(builder.Eq{ "pull_request.head_repo_id": repoID, diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 578cbca035d8..45c912d1861d 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -552,6 +552,8 @@ var migrations = []Migration{ NewMigration("Add Index to pull_auto_merge.doer_id", v1_22.AddIndexToPullAutoMergeDoerID), // v283 -> v284 NewMigration("Add combined Index to issue_user.uid and issue_id", v1_22.AddCombinedIndexToIssueUser), + // v284 -> v285 + NewMigration("Alter branch name collation to be case-sensitive", v1_22.AlterBranchNameCollation), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v1_22/v283_test.go b/models/migrations/v1_22/v283_test.go index 864f47f840c0..bb09cc2e0605 100644 --- a/models/migrations/v1_22/v283_test.go +++ b/models/migrations/v1_22/v283_test.go @@ -7,22 +7,22 @@ import ( "testing" "code.gitea.io/gitea/models/migrations/base" + + "github.com/stretchr/testify/assert" ) func Test_AddCombinedIndexToIssueUser(t *testing.T) { type IssueUser struct { - UID int64 `xorm:"INDEX unique(uid_to_issue)"` // User ID. - IssueID int64 `xorm:"INDEX unique(uid_to_issue)"` + ID int64 `xorm:"pk autoincr"` + UID int64 `xorm:"INDEX"` // User ID. + IssueID int64 `xorm:"INDEX"` + IsRead bool + IsMentioned bool } // Prepare and load the testing database x, deferable := base.PrepareTestEnv(t, 0, new(IssueUser)) defer deferable() - if x == nil || t.Failed() { - return - } - if err := AddCombinedIndexToIssueUser(x); err != nil { - t.Fatal(err) - } + assert.NoError(t, AddCombinedIndexToIssueUser(x)) } diff --git a/models/migrations/v1_22/v284.go b/models/migrations/v1_22/v284.go new file mode 100644 index 000000000000..5a8cba4cc63f --- /dev/null +++ b/models/migrations/v1_22/v284.go @@ -0,0 +1,29 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package v1_22 //nolint + +import ( + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + + "xorm.io/xorm" +) + +func AlterBranchNameCollation(x *xorm.Engine) error { + if setting.Database.Type.IsMySQL() { + _, err := x.Exec("ALTER TABLE branch MODIFY COLUMN `name` VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL") + return err + } else if setting.Database.Type.IsMSSQL() { + if _, err := x.Exec("DROP INDEX UQE_branch_s ON branch"); err != nil { + log.Error("Failed to drop index UQE_branch_s on branch: %v", err) // ignore this error, in case the index has been dropped in previous migration + } + if _, err := x.Exec("ALTER TABLE branch ALTER COLUMN [name] nvarchar(255) COLLATE Latin1_General_CS_AS NOT NULL"); err != nil { + return err + } + if _, err := x.Exec("CREATE UNIQUE NONCLUSTERED INDEX UQE_branch_s ON branch (repo_id ASC, [name] ASC)"); err != nil { + return err + } + } + return nil +} diff --git a/models/migrations/v1_22/v284_test.go b/models/migrations/v1_22/v284_test.go new file mode 100644 index 000000000000..12980dce08ab --- /dev/null +++ b/models/migrations/v1_22/v284_test.go @@ -0,0 +1,25 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package v1_22 //nolint + +import ( + "testing" + + "code.gitea.io/gitea/models/migrations/base" + + "github.com/stretchr/testify/assert" +) + +func TestAlterBranchNameCollation(t *testing.T) { + type Branch struct { + ID int64 + RepoID int64 `xorm:"UNIQUE(s)"` + Name string `xorm:"UNIQUE(s) NOT NULL"` + } + + x, deferable := base.PrepareTestEnv(t, 0, new(Branch)) + defer deferable() + + assert.NoError(t, AlterBranchNameCollation(x)) +} diff --git a/tests/integration/api_branch_test.go b/tests/integration/api_branch_test.go index 28e690b35689..78f16fa7a414 100644 --- a/tests/integration/api_branch_test.go +++ b/tests/integration/api_branch_test.go @@ -142,6 +142,18 @@ func testAPICreateBranches(t *testing.T, giteaURL *url.URL) { NewBranch: "branch_2", ExpectedHTTPStatus: http.StatusCreated, }, + // Trying to create a case-sensitive branch name + { + OldBranch: "new_branch_from_master_1", + NewBranch: "Branch_2", + ExpectedHTTPStatus: http.StatusCreated, + }, + // Trying to create a branch with UTF8 + { + OldBranch: "master", + NewBranch: "test-👀", + ExpectedHTTPStatus: http.StatusCreated, + }, // Trying to create from a branch which does not exist { OldBranch: "does_not_exist", From 7e7326a70b756e697e6cc4311016ac140c22f998 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Fri, 29 Dec 2023 18:33:38 +0800 Subject: [PATCH 2/2] if a table is already using utf8mb4, do not convert it again. --- docs/content/help/faq.en-us.md | 2 +- docs/content/help/faq.zh-cn.md | 4 +--- models/db/convert.go | 5 +++++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/content/help/faq.en-us.md b/docs/content/help/faq.en-us.md index e6350936ef81..0404bc4eaec2 100644 --- a/docs/content/help/faq.en-us.md +++ b/docs/content/help/faq.en-us.md @@ -387,7 +387,7 @@ the `utf8` charset, and connections which use the `utf8` charset will not use th Please run `gitea doctor convert`, or run `ALTER DATABASE database_name CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;` for the database_name and run `ALTER TABLE table_name CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;` -for each table in the database. +for tables which are not using `utf8mb4` in the database. ## Why are Emoji displaying only as placeholders or in monochrome diff --git a/docs/content/help/faq.zh-cn.md b/docs/content/help/faq.zh-cn.md index 909ca7e5e213..83a4943f1164 100644 --- a/docs/content/help/faq.zh-cn.md +++ b/docs/content/help/faq.zh-cn.md @@ -390,9 +390,7 @@ SET GLOBAL innodb_large_prefix=1; utf8 字符集的表和连接将不会使用它。 请运行 `gitea doctor convert` 或对数据库运行 `ALTER DATABASE database_name CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;` -并对每个表运行 `ALTER TABLE table_name CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;`。 - -您还需要将`app.ini`文件中的数据库字符集设置为`CHARSET=utf8mb4`。 +并对不是 `utf8mb4` 的表运行 `ALTER TABLE table_name CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;`。 ## 为什么 Emoji 只显示占位符或单色图像 diff --git a/models/db/convert.go b/models/db/convert.go index 112c8575ca2c..6c4716314b05 100644 --- a/models/db/convert.go +++ b/models/db/convert.go @@ -6,6 +6,7 @@ package db import ( "fmt" "strconv" + "strings" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -34,6 +35,10 @@ func ConvertUtf8ToUtf8mb4() error { return err } + if strings.HasPrefix(table.Collation, "utf8mb4") { + fmt.Printf("skip table %q because it is already using utf8mb4\n", table.Name) + continue + } if _, err := x.Exec(fmt.Sprintf("ALTER TABLE `%s` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;", table.Name)); err != nil { return err }