Add --case.insensitive flag to gitp4transfer and equivalent -insensit…

…ive to run_conversion.sh
rcowham · Apr 24, 2023 · a1cc8d5 · a1cc8d5
1 parent cb28685
commit a1cc8d5
Show file tree

Hide file tree

Showing 4 changed files with 177 additions and 42 deletions.
diff --git a/journal/journal.go b/journal/journal.go
@@ -5,6 +5,7 @@ import (
 	"io"
 	"os"
 	"strings"
+	"time"
 )
 
 // Example of journal records written for 2004.1
@@ -211,30 +212,39 @@ func ReplaceWildcards(filename string) string {
 }
 
 type Journal struct {
-	filename string
-	w        io.Writer
+	filename        string
+	w               io.Writer
+	caseInsensitive bool
 }
 
 var p4client = "git-client"
 var p4user = "git-user"
 var statusSubmitted = "1"
 
-func (j *Journal) CreateJournal() {
+func (j *Journal) CreateJournal(caseInsensitive bool) {
 
 	f, err := os.Create(j.filename)
 	if err != nil {
 		panic(err)
 	}
 	j.w = f
+	j.caseInsensitive = caseInsensitive
 }
 
 func (j *Journal) SetWriter(w io.Writer) {
 	j.w = w
 }
 
-func (j *Journal) WriteHeader(depot string) {
+func (j *Journal) WriteHeader(depot string, caseInsensitive bool) {
 
-	hdr := `@pv@ 0 @db.depot@ @%s@ 0 @subdir@ @%s/...@ 
+	hdr := ""
+	if caseInsensitive {
+		// If case insensitive then write the appropriate header
+		// Otherwise we leave this unset, and the user can choose by restoring jnl/ckp with p4d and appropriate flag/default value.
+		// Set server version = 18 - see https://www.perforce.com/perforce/doc.current/schema/index.html#releases
+		hdr = fmt.Sprintf("@nx@ 0 %d @18@ 2 0 0 0 0 @@ @@ @@ @@ @@\n", time.Now().Unix())
+	}
+	hdr = hdr + `@pv@ 0 @db.depot@ @%s@ 0 @subdir@ @%s/...@ 
 @pv@ 3 @db.domain@ @%s@ 100 @@ @@ @@ @@ @git-user@ 0 0 0 1 @Created by git-user@ 
 @pv@ 3 @db.user@ @git-user@ @git-user@@git-client@ @@ 0 0 @git-user@ @@ 0 @@ 0 
 @pv@ 0 @db.view@ @git-client@ 0 0 @//git-client/...@ @//import/...@ 
@@ -310,6 +320,9 @@ func (j *Journal) WriteRev(depotFile string, depotRev int, action FileAction, fi
 
 	depotFile = ReplaceWildcards(depotFile)
 	lbrFile = ReplaceWildcards(lbrFile)
+	if j.caseInsensitive {
+		lbrFile = strings.ToLower(lbrFile)
+	}
 
 	// @pv@ 3 @db.rev@ @//import/trunk/src/file.txt@ 1 1 0 1 1363872228 1363872228 00000000000000000000000000000000 @//import/trunk/src/file.txt@ @1.1@ 1
 	_, err := fmt.Fprintf(j.w,

diff --git a/main.go b/main.go
@@ -75,14 +75,15 @@ func Humanize(b int) string {
 }
 
 type GitParserOptions struct {
-	config        *config.Config
-	gitImportFile string
-	archiveRoot   string
-	dryRun        bool
-	dummyArchives bool
-	graphFile     string
-	maxCommits    int
-	debugCommit   int // For debug breakpoint
+	config          *config.Config
+	gitImportFile   string
+	archiveRoot     string
+	dryRun          bool
+	dummyArchives   bool
+	caseInsensitive bool // If true then create case insensitive checkpoint for Linux and lowercase archive files
+	graphFile       string
+	maxCommits      int
+	debugCommit     int // For debug breakpoint
 }
 
 type GitAction int
@@ -596,13 +597,16 @@ func (b *GitBlob) SaveBlob(pool *pond.WorkerPool, archiveRoot string, dummyFlag
 	return nil
 }
 
-func (gf *GitFile) CreateArchiveFile(pool *pond.WorkerPool, depotRoot string, matcher *BlobFileMatcher, changeNo int) {
+func (gf *GitFile) CreateArchiveFile(pool *pond.WorkerPool, caseInsensitive bool, depotRoot string, matcher *BlobFileMatcher, changeNo int) {
 	if gf.action == delete || (gf.action == rename && !gf.isDirtyRename && !gf.isPseudoRename && !gf.isDoubleRename) ||
 		gf.blob == nil || !gf.blob.hasData {
 		return
 	}
 	// Fix wildcards
 	depotFile := journal.ReplaceWildcards(gf.p4.depotFile[2:])
+	if caseInsensitive {
+		depotFile = strings.ToLower(depotFile)
+	}
 	rootDir := path.Join(depotRoot, fmt.Sprintf("%s,d", depotFile))
 	if gf.blob.blobFileName == "" {
 		gf.logger.Debugf(fmt.Sprintf("NoBlobFound: %s", depotFile))
@@ -1631,6 +1635,10 @@ func main() {
 			"default.branch",
 			"Name of default git branch (overrides config).",
 		).Default(config.DefaultBranch).Short('b').String()
+		caseInsensitive = kingpin.Flag(
+			"case.insensitive",
+			"Create checkpoint case-insensitive mode (for Linux) and lowercase archive files. If not set, then OS default applies.",
+		).Bool()
 		dummyArchives = kingpin.Flag(
 			"dummy",
 			"Create dummy (small) archive files - for quick analysis of large repos.",
@@ -1705,14 +1713,15 @@ func main() {
 	logger.Infof("Starting %s, gitimport: %v", startTime, *gitimport)
 
 	opts := &GitParserOptions{
-		config:        cfg,
-		gitImportFile: *gitimport,
-		archiveRoot:   *archive,
-		dryRun:        *dryrun,
-		dummyArchives: *dummyArchives,
-		maxCommits:    *maxCommits,
-		graphFile:     *outputGraph,
-		debugCommit:   *debugCommit,
+		config:          cfg,
+		gitImportFile:   *gitimport,
+		archiveRoot:     *archive,
+		dryRun:          *dryrun,
+		dummyArchives:   *dummyArchives,
+		caseInsensitive: *caseInsensitive,
+		maxCommits:      *maxCommits,
+		graphFile:       *outputGraph,
+		debugCommit:     *debugCommit,
 	}
 	logger.Infof("Options: %+v", opts)
 	g, err := NewGitP4Transfer(logger, opts)
@@ -1748,13 +1757,13 @@ func main() {
 	defer f.Close()
 
 	j.SetWriter(f)
-	j.WriteHeader(opts.config.ImportDepot)
+	j.WriteHeader(opts.config.ImportDepot, opts.caseInsensitive)
 
 	for c := range commitChan {
 		j.WriteChange(c.commit.Mark, c.user, c.commit.Msg, int(c.commit.Author.Time.Unix()))
 		for _, f := range c.files {
 			if !*dryrun {
-				f.CreateArchiveFile(pool, opts.archiveRoot, g.blobFileMatcher, c.commit.Mark)
+				f.CreateArchiveFile(pool, opts.caseInsensitive, opts.archiveRoot, g.blobFileMatcher, c.commit.Mark)
 			} else if f.blob != nil && f.blob.hasData && !f.blob.dataRemoved {
 				f.blob.blob.Data = "" // Allow contents to be GC'ed
 				f.blob.dataRemoved = true

diff --git a/main_test.go b/main_test.go
@@ -8,6 +8,7 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
@@ -243,12 +244,12 @@ func runTransferWithDump(t *testing.T, logger *logrus.Logger, output string, opt
 
 	j := journal.Journal{}
 	j.SetWriter(buf)
-	j.WriteHeader(opts.config.ImportDepot)
+	j.WriteHeader(opts.config.ImportDepot, opts.caseInsensitive)
 
 	for _, c := range commits {
 		j.WriteChange(c.commit.Mark, user, c.commit.Msg, int(c.commit.Author.Time.Unix()))
 		for _, f := range c.files {
-			f.CreateArchiveFile(nil, p4t.serverRoot, g.blobFileMatcher, c.commit.Mark)
+			f.CreateArchiveFile(nil, opts.caseInsensitive, p4t.serverRoot, g.blobFileMatcher, c.commit.Mark)
 			f.WriteJournal(&j, &c)
 		}
 	}
@@ -341,7 +342,7 @@ func TestAdd(t *testing.T) {
 	buf := new(bytes.Buffer)
 	j := journal.Journal{}
 	j.SetWriter(buf)
-	j.WriteHeader(opts.config.ImportDepot)
+	j.WriteHeader(opts.config.ImportDepot, opts.caseInsensitive)
 	c = commits[0]
 	j.WriteChange(c.commit.Mark, defaultP4user, c.commit.Msg, int(c.commit.Author.Time.Unix()))
 	f = c.files[0]
@@ -365,7 +366,7 @@ func TestAdd(t *testing.T) {
 
 	jnl := filepath.Join(p4t.serverRoot, "jnl.0")
 	writeToFile(jnl, expectedJournal)
-	f.CreateArchiveFile(nil, p4t.serverRoot, g.blobFileMatcher, c.commit.Mark)
+	f.CreateArchiveFile(nil, opts.caseInsensitive, p4t.serverRoot, g.blobFileMatcher, c.commit.Mark)
 	runCmd("p4d -r . -jr jnl.0")
 	runCmd("p4d -r . -J journal -xu")
 	runCmd("p4 storage -r")
@@ -498,6 +499,109 @@ func TestMaxCommits(t *testing.T) {
 	assert.Regexp(t, fmt.Sprintf(`Change 2 on .* by %s@git\-client`, user), result)
 }
 
+func TestCaseInsensitive(t *testing.T) { // Check case sensitive flag set
+	logger := createLogger()
+	logger.Debugf("======== Test: %s", t.Name())
+
+	d := createGitRepo(t)
+	os.Chdir(d)
+	logger.Debugf("Git repo: %s", d)
+	src := "SRC.txt"
+	srcContents1 := "contents\n"
+	writeToFile(src, srcContents1)
+	runCmd("git add .")
+	runCmd("git commit -m initial")
+
+	r := runTransferOpts(t, logger, &GitParserOptions{
+		caseInsensitive: true,
+		config:          &config.Config{ImportDepot: "IMPORT", DefaultBranch: "main"},
+	})
+	logger.Debugf("Server root: %s", r)
+
+	result, err := runCmd("p4 files //...")
+	assert.Equal(t, nil, err)
+	assert.Equal(t, "//IMPORT/main/SRC.txt#1 - add change 2 (text+C)\n", result)
+
+	result, err = runCmd("p4 print -q //IMPORT/main/SRC.txt#1")
+	assert.Equal(t, nil, err)
+	assert.Equal(t, srcContents1, result)
+
+	result, err = runCmd("p4 verify -qu //...")
+	assert.Equal(t, "<nil>", fmt.Sprint(err))
+	assert.Equal(t, "", result)
+
+	result, err = runCmd("p4 fstat -Ob //IMPORT/main/SRC.txt")
+	assert.Equal(t, nil, err)
+	assert.Regexp(t, `headType text\+C`, result)
+	assert.Regexp(t, `lbrType text\+C`, result)
+	assert.Regexp(t, `lbrPath .*/import/main/src.txt,d/1.2.gz`, result)
+
+	files, err := ioutil.ReadDir(r)
+	if err != nil {
+		log.Fatal(err)
+	}
+	foundUpper := false
+	foundLower := false
+	for _, f := range files {
+		if f.Name() == "IMPORT" {
+			foundUpper = true
+		} else if f.Name() == "import" {
+			foundLower = true
+		}
+	}
+	assert.False(t, foundUpper)
+	assert.True(t, foundLower)
+}
+
+func TestCaseSensitive(t *testing.T) { // Test when case sensitive is specified
+	logger := createLogger()
+	logger.Debugf("======== Test: %s", t.Name())
+
+	d := createGitRepo(t)
+	os.Chdir(d)
+	logger.Debugf("Git repo: %s", d)
+	src := "SRC.txt"
+	srcContents1 := "contents\n"
+	writeToFile(src, srcContents1)
+	runCmd("git add .")
+	runCmd("git commit -m initial")
+
+	r := runTransferOpts(t, logger, &GitParserOptions{
+		caseInsensitive: false,
+		config:          &config.Config{ImportDepot: "IMPORT", DefaultBranch: "main"},
+	})
+	logger.Debugf("Server root: %s", r)
+
+	result, err := runCmd("p4 files //...")
+	assert.Equal(t, nil, err)
+	assert.Equal(t, "//IMPORT/main/SRC.txt#1 - add change 2 (text+C)\n", result)
+
+	result, err = runCmd("p4 print -q //IMPORT/main/SRC.txt#1")
+	assert.Equal(t, nil, err)
+	assert.Equal(t, srcContents1, result)
+
+	result, err = runCmd("p4 verify -qu //...")
+	assert.Equal(t, "<nil>", fmt.Sprint(err))
+	assert.Equal(t, "", result)
+
+	files, err := ioutil.ReadDir(r)
+	if err != nil {
+		log.Fatal(err)
+	}
+	foundUpper := false
+	foundLower := false
+	for _, f := range files {
+		if f.Name() == "IMPORT" {
+			foundUpper = true
+		} else if f.Name() == "import" {
+			foundLower = true
+		}
+	}
+	assert.True(t, foundUpper)
+	assert.False(t, foundLower)
+
+}
+
 func TestAddSameFile(t *testing.T) {
 	// Ensure single archive in git
 	logger := createLogger()

diff --git a/run_conversion.sh b/run_conversion.sh
@@ -25,19 +25,20 @@ function usage
 
    echo "USAGE for run_conversion.sh:
  
-run_conversion.sh <git_fast_export> [-p <P4Root>] [-d] [-dummy] [-depot <import depot>] [-graph <graphFile.dot>] [-m <max commits>] [-t <parallel threads>]
- 
+run_conversion.sh <git_fast_export> [-p <P4Root>] [-d] [-dummy] [-insensitive] [-depot <import depot>] [-graph <graphFile.dot>] [-m <max commits>] [-t <parallel threads>]
+
    or
 
 run_conversion.sh -h
 
-    -d          Debug
-    -depot      Depot to use for this import (default is 'import')
-    -dummy      Create dummy archives as placeholders (no real content) - much faster
-    -graph      Create Graphviz output showing commit structure
-    -m          Max no of commits to process
-    -t          No of parallel threads to use (default is No of CPUs)
-    <P4Root>    Directory to use as resulting P4Root - will default to a tmp dir
+    -d           Debug
+    -depot       Depot to use for this import (default is 'import')
+    -dummy       Create dummy archives as placeholders (no real content) - much faster
+    -graph       Create Graphviz output showing commit structure
+    -insensitive Specify case insensitive checkpoint (and lowercase archive files) - for Linux servers
+    -m           Max no of commits to process
+    -t           No of parallel threads to use (default is No of CPUs)
+    <P4Root>     Directory to use as resulting P4Root - will default to a tmp dir
     <git_fast_export> The (input) git fast-export format file (required)
 
 Examples:
@@ -53,6 +54,7 @@ Examples:
 declare -i shiftArgs=0
 declare -i Debug=0
 declare -i Dummy=0
+declare -i CaseInsensitive=0
 declare -i MaxCommits=0
 declare -i ParallelThreads=0
 declare ConfigFile=""
@@ -71,6 +73,7 @@ while [[ $# -gt 0 ]]; do
         (-d) Debug=1;;
         (-depot) ImportDepot=1;;
         (-dummy) Dummy=1;;
+        (-insensitive) CaseInsensitive=1;;
         (-graph) GraphFile=$2; shiftArgs=1;;
         (-m) MaxCommits=$2; shiftArgs=1;;
         (-t) ParallelThreads=$2; shiftArgs=1;;
@@ -108,6 +111,12 @@ DummyFlag=""
 if [[ $Dummy -ne 0 ]]; then
     DummyFlag="--dummy"
 fi
+CaseInsensitiveFlag=""
+P4DCaseFlag=""
+if [[ $CaseInsensitive -ne 0 ]]; then
+    CaseInsensitiveFlag="--case.insensitive"
+    P4DCaseFlag="-C1"
+fi
 GraphArgs=""
 if [[ ! -z $GraphFile ]]; then
     GraphArgs="--graphfile=$GraphFile"
@@ -117,8 +126,8 @@ if [[ ! -z $ConfigFile ]]; then
     ConfigArgs="--config=$ConfigFile"
 fi
 
-echo ./gitp4transfer --archive.root="$P4Root" $DebugFlag $DummyFlag $MaxCommitArgs $ParallelThreadArgs $GraphArgs --import.depot="$ImportDepot" --journal="$P4Root/jnl.0" "$GitFile"
-./gitp4transfer --archive.root="$P4Root" $ConfigArgs $DebugFlag $DummyFlag $MaxCommitArgs $ParallelThreadArgs $GraphArgs --import.depot="$ImportDepot" --journal="$P4Root/jnl.0" "$GitFile"
+echo ./gitp4transfer --archive.root="$P4Root" $DebugFlag $DummyFlag $CaseInsensitiveFlag $MaxCommitArgs $ParallelThreadArgs $GraphArgs --import.depot="$ImportDepot" --journal="$P4Root/jnl.0" "$GitFile"
+./gitp4transfer --archive.root="$P4Root" $ConfigArgs $DebugFlag $DummyFlag $CaseInsensitiveFlag $MaxCommitArgs $ParallelThreadArgs $GraphArgs --import.depot="$ImportDepot" --journal="$P4Root/jnl.0" "$GitFile"
 
 if [[ $? -ne 0 ]]; then
     echo "Server is in directory:"
@@ -129,9 +138,9 @@ fi
 pushd "$P4Root"
 curr_dir=$(pwd)
 
-declare P4PORT="rsh:p4d -r \"$curr_dir\" -L log -vserver=3 -i"
-p4d -r . -jr jnl.0
-p4d -r . -J journal -xu
+declare P4PORT="rsh:p4d $P4DCaseFlag -r \"$curr_dir\" -L log -vserver=3 -i"
+p4d $P4DCaseFlag -r . -jr jnl.0
+p4d $P4DCaseFlag -r . -J journal -xu
 p4 -p "$P4PORT" storage -r
 p4 -p "$P4PORT" storage -w
 p4 -p "$P4PORT" configure set monitor=1