diff --git a/journal/journal.go b/journal/journal.go index 0ae277f..d887a09 100644 --- a/journal/journal.go +++ b/journal/journal.go @@ -5,6 +5,7 @@ import ( "io" "os" "strings" + "time" ) // Example of journal records written for 2004.1 @@ -211,30 +212,39 @@ func ReplaceWildcards(filename string) string { } type Journal struct { - filename string - w io.Writer + filename string + w io.Writer + caseInsensitive bool } var p4client = "git-client" var p4user = "git-user" var statusSubmitted = "1" -func (j *Journal) CreateJournal() { +func (j *Journal) CreateJournal(caseInsensitive bool) { f, err := os.Create(j.filename) if err != nil { panic(err) } j.w = f + j.caseInsensitive = caseInsensitive } func (j *Journal) SetWriter(w io.Writer) { j.w = w } -func (j *Journal) WriteHeader(depot string) { +func (j *Journal) WriteHeader(depot string, caseInsensitive bool) { - hdr := `@pv@ 0 @db.depot@ @%s@ 0 @subdir@ @%s/...@ + hdr := "" + if caseInsensitive { + // If case insensitive then write the appropriate header + // Otherwise we leave this unset, and the user can choose by restoring jnl/ckp with p4d and appropriate flag/default value. + // Set server version = 18 - see https://www.perforce.com/perforce/doc.current/schema/index.html#releases + hdr = fmt.Sprintf("@nx@ 0 %d @18@ 2 0 0 0 0 @@ @@ @@ @@ @@\n", time.Now().Unix()) + } + hdr = hdr + `@pv@ 0 @db.depot@ @%s@ 0 @subdir@ @%s/...@ @pv@ 3 @db.domain@ @%s@ 100 @@ @@ @@ @@ @git-user@ 0 0 0 1 @Created by git-user@ @pv@ 3 @db.user@ @git-user@ @git-user@@git-client@ @@ 0 0 @git-user@ @@ 0 @@ 0 @pv@ 0 @db.view@ @git-client@ 0 0 @//git-client/...@ @//import/...@ @@ -310,6 +320,9 @@ func (j *Journal) WriteRev(depotFile string, depotRev int, action FileAction, fi depotFile = ReplaceWildcards(depotFile) lbrFile = ReplaceWildcards(lbrFile) + if j.caseInsensitive { + lbrFile = strings.ToLower(lbrFile) + } // @pv@ 3 @db.rev@ @//import/trunk/src/file.txt@ 1 1 0 1 1363872228 1363872228 00000000000000000000000000000000 @//import/trunk/src/file.txt@ @1.1@ 1 _, err := fmt.Fprintf(j.w, diff --git a/main.go b/main.go index 2eb393b..d87211e 100644 --- a/main.go +++ b/main.go @@ -75,14 +75,15 @@ func Humanize(b int) string { } type GitParserOptions struct { - config *config.Config - gitImportFile string - archiveRoot string - dryRun bool - dummyArchives bool - graphFile string - maxCommits int - debugCommit int // For debug breakpoint + config *config.Config + gitImportFile string + archiveRoot string + dryRun bool + dummyArchives bool + caseInsensitive bool // If true then create case insensitive checkpoint for Linux and lowercase archive files + graphFile string + maxCommits int + debugCommit int // For debug breakpoint } type GitAction int @@ -596,13 +597,16 @@ func (b *GitBlob) SaveBlob(pool *pond.WorkerPool, archiveRoot string, dummyFlag return nil } -func (gf *GitFile) CreateArchiveFile(pool *pond.WorkerPool, depotRoot string, matcher *BlobFileMatcher, changeNo int) { +func (gf *GitFile) CreateArchiveFile(pool *pond.WorkerPool, caseInsensitive bool, depotRoot string, matcher *BlobFileMatcher, changeNo int) { if gf.action == delete || (gf.action == rename && !gf.isDirtyRename && !gf.isPseudoRename && !gf.isDoubleRename) || gf.blob == nil || !gf.blob.hasData { return } // Fix wildcards depotFile := journal.ReplaceWildcards(gf.p4.depotFile[2:]) + if caseInsensitive { + depotFile = strings.ToLower(depotFile) + } rootDir := path.Join(depotRoot, fmt.Sprintf("%s,d", depotFile)) if gf.blob.blobFileName == "" { gf.logger.Debugf(fmt.Sprintf("NoBlobFound: %s", depotFile)) @@ -1631,6 +1635,10 @@ func main() { "default.branch", "Name of default git branch (overrides config).", ).Default(config.DefaultBranch).Short('b').String() + caseInsensitive = kingpin.Flag( + "case.insensitive", + "Create checkpoint case-insensitive mode (for Linux) and lowercase archive files. If not set, then OS default applies.", + ).Bool() dummyArchives = kingpin.Flag( "dummy", "Create dummy (small) archive files - for quick analysis of large repos.", @@ -1705,14 +1713,15 @@ func main() { logger.Infof("Starting %s, gitimport: %v", startTime, *gitimport) opts := &GitParserOptions{ - config: cfg, - gitImportFile: *gitimport, - archiveRoot: *archive, - dryRun: *dryrun, - dummyArchives: *dummyArchives, - maxCommits: *maxCommits, - graphFile: *outputGraph, - debugCommit: *debugCommit, + config: cfg, + gitImportFile: *gitimport, + archiveRoot: *archive, + dryRun: *dryrun, + dummyArchives: *dummyArchives, + caseInsensitive: *caseInsensitive, + maxCommits: *maxCommits, + graphFile: *outputGraph, + debugCommit: *debugCommit, } logger.Infof("Options: %+v", opts) g, err := NewGitP4Transfer(logger, opts) @@ -1748,13 +1757,13 @@ func main() { defer f.Close() j.SetWriter(f) - j.WriteHeader(opts.config.ImportDepot) + j.WriteHeader(opts.config.ImportDepot, opts.caseInsensitive) for c := range commitChan { j.WriteChange(c.commit.Mark, c.user, c.commit.Msg, int(c.commit.Author.Time.Unix())) for _, f := range c.files { if !*dryrun { - f.CreateArchiveFile(pool, opts.archiveRoot, g.blobFileMatcher, c.commit.Mark) + f.CreateArchiveFile(pool, opts.caseInsensitive, opts.archiveRoot, g.blobFileMatcher, c.commit.Mark) } else if f.blob != nil && f.blob.hasData && !f.blob.dataRemoved { f.blob.blob.Data = "" // Allow contents to be GC'ed f.blob.dataRemoved = true diff --git a/main_test.go b/main_test.go index 9b5c366..1ef8cd6 100644 --- a/main_test.go +++ b/main_test.go @@ -8,6 +8,7 @@ import ( "flag" "fmt" "io" + "io/ioutil" "log" "os" "os/exec" @@ -243,12 +244,12 @@ func runTransferWithDump(t *testing.T, logger *logrus.Logger, output string, opt j := journal.Journal{} j.SetWriter(buf) - j.WriteHeader(opts.config.ImportDepot) + j.WriteHeader(opts.config.ImportDepot, opts.caseInsensitive) for _, c := range commits { j.WriteChange(c.commit.Mark, user, c.commit.Msg, int(c.commit.Author.Time.Unix())) for _, f := range c.files { - f.CreateArchiveFile(nil, p4t.serverRoot, g.blobFileMatcher, c.commit.Mark) + f.CreateArchiveFile(nil, opts.caseInsensitive, p4t.serverRoot, g.blobFileMatcher, c.commit.Mark) f.WriteJournal(&j, &c) } } @@ -341,7 +342,7 @@ func TestAdd(t *testing.T) { buf := new(bytes.Buffer) j := journal.Journal{} j.SetWriter(buf) - j.WriteHeader(opts.config.ImportDepot) + j.WriteHeader(opts.config.ImportDepot, opts.caseInsensitive) c = commits[0] j.WriteChange(c.commit.Mark, defaultP4user, c.commit.Msg, int(c.commit.Author.Time.Unix())) f = c.files[0] @@ -365,7 +366,7 @@ func TestAdd(t *testing.T) { jnl := filepath.Join(p4t.serverRoot, "jnl.0") writeToFile(jnl, expectedJournal) - f.CreateArchiveFile(nil, p4t.serverRoot, g.blobFileMatcher, c.commit.Mark) + f.CreateArchiveFile(nil, opts.caseInsensitive, p4t.serverRoot, g.blobFileMatcher, c.commit.Mark) runCmd("p4d -r . -jr jnl.0") runCmd("p4d -r . -J journal -xu") runCmd("p4 storage -r") @@ -498,6 +499,109 @@ func TestMaxCommits(t *testing.T) { assert.Regexp(t, fmt.Sprintf(`Change 2 on .* by %s@git\-client`, user), result) } +func TestCaseInsensitive(t *testing.T) { // Check case sensitive flag set + logger := createLogger() + logger.Debugf("======== Test: %s", t.Name()) + + d := createGitRepo(t) + os.Chdir(d) + logger.Debugf("Git repo: %s", d) + src := "SRC.txt" + srcContents1 := "contents\n" + writeToFile(src, srcContents1) + runCmd("git add .") + runCmd("git commit -m initial") + + r := runTransferOpts(t, logger, &GitParserOptions{ + caseInsensitive: true, + config: &config.Config{ImportDepot: "IMPORT", DefaultBranch: "main"}, + }) + logger.Debugf("Server root: %s", r) + + result, err := runCmd("p4 files //...") + assert.Equal(t, nil, err) + assert.Equal(t, "//IMPORT/main/SRC.txt#1 - add change 2 (text+C)\n", result) + + result, err = runCmd("p4 print -q //IMPORT/main/SRC.txt#1") + assert.Equal(t, nil, err) + assert.Equal(t, srcContents1, result) + + result, err = runCmd("p4 verify -qu //...") + assert.Equal(t, "", fmt.Sprint(err)) + assert.Equal(t, "", result) + + result, err = runCmd("p4 fstat -Ob //IMPORT/main/SRC.txt") + assert.Equal(t, nil, err) + assert.Regexp(t, `headType text\+C`, result) + assert.Regexp(t, `lbrType text\+C`, result) + assert.Regexp(t, `lbrPath .*/import/main/src.txt,d/1.2.gz`, result) + + files, err := ioutil.ReadDir(r) + if err != nil { + log.Fatal(err) + } + foundUpper := false + foundLower := false + for _, f := range files { + if f.Name() == "IMPORT" { + foundUpper = true + } else if f.Name() == "import" { + foundLower = true + } + } + assert.False(t, foundUpper) + assert.True(t, foundLower) +} + +func TestCaseSensitive(t *testing.T) { // Test when case sensitive is specified + logger := createLogger() + logger.Debugf("======== Test: %s", t.Name()) + + d := createGitRepo(t) + os.Chdir(d) + logger.Debugf("Git repo: %s", d) + src := "SRC.txt" + srcContents1 := "contents\n" + writeToFile(src, srcContents1) + runCmd("git add .") + runCmd("git commit -m initial") + + r := runTransferOpts(t, logger, &GitParserOptions{ + caseInsensitive: false, + config: &config.Config{ImportDepot: "IMPORT", DefaultBranch: "main"}, + }) + logger.Debugf("Server root: %s", r) + + result, err := runCmd("p4 files //...") + assert.Equal(t, nil, err) + assert.Equal(t, "//IMPORT/main/SRC.txt#1 - add change 2 (text+C)\n", result) + + result, err = runCmd("p4 print -q //IMPORT/main/SRC.txt#1") + assert.Equal(t, nil, err) + assert.Equal(t, srcContents1, result) + + result, err = runCmd("p4 verify -qu //...") + assert.Equal(t, "", fmt.Sprint(err)) + assert.Equal(t, "", result) + + files, err := ioutil.ReadDir(r) + if err != nil { + log.Fatal(err) + } + foundUpper := false + foundLower := false + for _, f := range files { + if f.Name() == "IMPORT" { + foundUpper = true + } else if f.Name() == "import" { + foundLower = true + } + } + assert.True(t, foundUpper) + assert.False(t, foundLower) + +} + func TestAddSameFile(t *testing.T) { // Ensure single archive in git logger := createLogger() diff --git a/run_conversion.sh b/run_conversion.sh index ff2cc0b..060f295 100755 --- a/run_conversion.sh +++ b/run_conversion.sh @@ -25,19 +25,20 @@ function usage echo "USAGE for run_conversion.sh: -run_conversion.sh [-p ] [-d] [-dummy] [-depot ] [-graph ] [-m ] [-t ] - +run_conversion.sh [-p ] [-d] [-dummy] [-insensitive] [-depot ] [-graph ] [-m ] [-t ] + or run_conversion.sh -h - -d Debug - -depot Depot to use for this import (default is 'import') - -dummy Create dummy archives as placeholders (no real content) - much faster - -graph Create Graphviz output showing commit structure - -m Max no of commits to process - -t No of parallel threads to use (default is No of CPUs) - Directory to use as resulting P4Root - will default to a tmp dir + -d Debug + -depot Depot to use for this import (default is 'import') + -dummy Create dummy archives as placeholders (no real content) - much faster + -graph Create Graphviz output showing commit structure + -insensitive Specify case insensitive checkpoint (and lowercase archive files) - for Linux servers + -m Max no of commits to process + -t No of parallel threads to use (default is No of CPUs) + Directory to use as resulting P4Root - will default to a tmp dir The (input) git fast-export format file (required) Examples: @@ -53,6 +54,7 @@ Examples: declare -i shiftArgs=0 declare -i Debug=0 declare -i Dummy=0 +declare -i CaseInsensitive=0 declare -i MaxCommits=0 declare -i ParallelThreads=0 declare ConfigFile="" @@ -71,6 +73,7 @@ while [[ $# -gt 0 ]]; do (-d) Debug=1;; (-depot) ImportDepot=1;; (-dummy) Dummy=1;; + (-insensitive) CaseInsensitive=1;; (-graph) GraphFile=$2; shiftArgs=1;; (-m) MaxCommits=$2; shiftArgs=1;; (-t) ParallelThreads=$2; shiftArgs=1;; @@ -108,6 +111,12 @@ DummyFlag="" if [[ $Dummy -ne 0 ]]; then DummyFlag="--dummy" fi +CaseInsensitiveFlag="" +P4DCaseFlag="" +if [[ $CaseInsensitive -ne 0 ]]; then + CaseInsensitiveFlag="--case.insensitive" + P4DCaseFlag="-C1" +fi GraphArgs="" if [[ ! -z $GraphFile ]]; then GraphArgs="--graphfile=$GraphFile" @@ -117,8 +126,8 @@ if [[ ! -z $ConfigFile ]]; then ConfigArgs="--config=$ConfigFile" fi -echo ./gitp4transfer --archive.root="$P4Root" $DebugFlag $DummyFlag $MaxCommitArgs $ParallelThreadArgs $GraphArgs --import.depot="$ImportDepot" --journal="$P4Root/jnl.0" "$GitFile" -./gitp4transfer --archive.root="$P4Root" $ConfigArgs $DebugFlag $DummyFlag $MaxCommitArgs $ParallelThreadArgs $GraphArgs --import.depot="$ImportDepot" --journal="$P4Root/jnl.0" "$GitFile" +echo ./gitp4transfer --archive.root="$P4Root" $DebugFlag $DummyFlag $CaseInsensitiveFlag $MaxCommitArgs $ParallelThreadArgs $GraphArgs --import.depot="$ImportDepot" --journal="$P4Root/jnl.0" "$GitFile" +./gitp4transfer --archive.root="$P4Root" $ConfigArgs $DebugFlag $DummyFlag $CaseInsensitiveFlag $MaxCommitArgs $ParallelThreadArgs $GraphArgs --import.depot="$ImportDepot" --journal="$P4Root/jnl.0" "$GitFile" if [[ $? -ne 0 ]]; then echo "Server is in directory:" @@ -129,9 +138,9 @@ fi pushd "$P4Root" curr_dir=$(pwd) -declare P4PORT="rsh:p4d -r \"$curr_dir\" -L log -vserver=3 -i" -p4d -r . -jr jnl.0 -p4d -r . -J journal -xu +declare P4PORT="rsh:p4d $P4DCaseFlag -r \"$curr_dir\" -L log -vserver=3 -i" +p4d $P4DCaseFlag -r . -jr jnl.0 +p4d $P4DCaseFlag -r . -J journal -xu p4 -p "$P4PORT" storage -r p4 -p "$P4PORT" storage -w p4 -p "$P4PORT" configure set monitor=1