Feature/list commits with children

catalog/cataloger_diff.go

@@ -160,7 +160,7 @@ func (c *cataloger) diffFromSon(tx db.Tx, sonID, fatherID int64) (Differences, e
 		return nil, fmt.Errorf("son lineage failed: %w", err)
 	}
 
-	sonLineageValues := getLineageAsValues(sonLineage, sonID)
+	sonLineageValues := getLineageAsValues(sonLineage, sonID, MaxCommitID)
 	mainDiffFromSon := sqDiffFromSonV(fatherID, sonID, effectiveCommits.FatherEffectiveCommit, effectiveCommits.SonEffectiveCommit, fatherLineage, sonLineageValues)
 	diffFromSonSQL, args, err := mainDiffFromSon.
 		Prefix("CREATE TEMP TABLE " + diffResultsTableName + " ON COMMIT DROP AS").

catalog/cataloger_list_commits.go

@@ -30,18 +30,26 @@ func (c *cataloger) ListCommits(ctx context.Context, repository, branch string,
 		fromCommitID = ref.CommitID
 	}
 	res, err := c.db.Transact(func(tx db.Tx) (interface{}, error) {
+
 		branchID, err := c.getBranchIDCache(tx, repository, branch)
 		if err != nil {
 			return nil, err
 		}
-		lineage, err := getLineage(tx, branchID, CommittedID)
+		lineage, err := getLineage(tx, branchID, fromCommitID)
 		if err != nil {
 			return nil, fmt.Errorf("get lineage: %w", err)
 		}
-		lineageAsValuesTable := getLineageAsValues(lineage, branchID)
-		query := `SELECT b_name.name as branch_name,c.commit_id,c.previous_commit_id,c.committer,c.message,c.creation_date,c.metadata,
+		lineageAsValuesTable := getLineageAsValues(lineage, branchID, fromCommitID)
+		cte := `WITH RECURSIVE lineage_graph AS (
+    select branch_id,commit_id from ` + lineageAsValuesTable + `
+	union all
+	select * from (Select distinct on (c.branch_id,c.merge_source_branch) merge_source_branch,merge_source_commit from catalog_commits c 
+	join lineage_graph l on l.branch_id = c.branch_id and c.merge_type='from_son'  and c.merge_source_commit < l.commit_id
+	order by c.branch_id,c.merge_source_branch,c.commit_id desc )t)
+`
+		query := cte + `SELECT b_name.name as branch_name,c.commit_id,c.previous_commit_id,c.committer,c.message,c.creation_date,c.metadata,
 				COALESCE(bb.name,'') as merge_source_branch_name,COALESCE(c.merge_source_commit,0) as merge_source_commit
-			FROM catalog_commits c JOIN (SELECT * FROM ` + lineageAsValuesTable + `) l  ON  c.branch_id = l.branch_id and c.commit_id <= l.commit_id
+			FROM catalog_commits c JOIN lineage_graph l  ON  c.branch_id = l.branch_id and c.commit_id <= l.commit_id
 				JOIN catalog_branches b_name ON c.branch_id = b_name.id
 				LEFT JOIN catalog_branches bb ON bb.id = c.merge_source_branch
 			WHERE c.commit_id < $1

catalog/cataloger_list_commits_test.go

@@ -45,9 +45,9 @@ func TestCataloger_ListCommits(t *testing.T) {
 				limit:         -1,
 			},
 			want: []*CommitLog{
-				{Reference: commits[2].Reference, Committer: "tester", Message: "commit3", Metadata: Metadata{}, Parents: []string{"~KJ8Wd1Rs96a"}},
-				{Reference: commits[1].Reference, Committer: "tester", Message: "commit2", Metadata: Metadata{}, Parents: []string{"~KJ8Wd1Rs96Z"}},
-				{Reference: commits[0].Reference, Committer: "tester", Message: "commit1", Metadata: Metadata{}, Parents: []string{"~KJ8Wd1Rs96Y"}},
+				{Reference: commits[2].Reference, Committer: "tester", Message: "commit3 on branch master", Metadata: Metadata{}, Parents: []string{"~KJ8Wd1Rs96a"}},
+				{Reference: commits[1].Reference, Committer: "tester", Message: "commit2 on branch master", Metadata: Metadata{}, Parents: []string{"~KJ8Wd1Rs96Z"}},
+				{Reference: commits[0].Reference, Committer: "tester", Message: "commit1 on branch master", Metadata: Metadata{}, Parents: []string{"~KJ8Wd1Rs96Y"}},
 				{Reference: initialCommitReference, Committer: CatalogerCommitter, Message: createRepositoryCommitMessage, Metadata: Metadata{}},
 			},
 			wantMore: false,
@@ -62,8 +62,8 @@ func TestCataloger_ListCommits(t *testing.T) {
 				limit:         2,
 			},
 			want: []*CommitLog{
-				{Reference: commits[2].Reference, Committer: "tester", Message: "commit3", Metadata: Metadata{}, Parents: []string{"~KJ8Wd1Rs96a"}},
-				{Reference: commits[1].Reference, Committer: "tester", Message: "commit2", Metadata: Metadata{}, Parents: []string{"~KJ8Wd1Rs96Z"}},
+				{Reference: commits[2].Reference, Committer: "tester", Message: "commit3 on branch master", Metadata: Metadata{}, Parents: []string{"~KJ8Wd1Rs96a"}},
+				{Reference: commits[1].Reference, Committer: "tester", Message: "commit2 on branch master", Metadata: Metadata{}, Parents: []string{"~KJ8Wd1Rs96Z"}},
 			},
 			wantMore: true,
 			wantErr:  false,
@@ -91,7 +91,7 @@ func TestCataloger_ListCommits(t *testing.T) {
 				limit:         1,
 			},
 			want: []*CommitLog{
-				{Reference: commits[1].Reference, Committer: "tester", Message: "commit2", Metadata: Metadata{}, Parents: []string{"~KJ8Wd1Rs96Z"}},
+				{Reference: commits[1].Reference, Committer: "tester", Message: "commit2 on branch master", Metadata: Metadata{}, Parents: []string{"~KJ8Wd1Rs96Z"}},
 			},
 			wantMore: true,
 			wantErr:  false,
@@ -209,7 +209,7 @@ func setupListCommitsByBranchData(t *testing.T, ctx context.Context, c Cataloger
 		}, CreateEntryParams{}); err != nil {
 			t.Fatal("Write entry for list repository commits failed", err)
 		}
-		message := "commit" + strconv.Itoa(i+1)
+		message := "commit" + strconv.Itoa(i+1) + " on branch " + branch
 		commitLog, err := c.Commit(ctx, repository, branch, message, "tester", nil)
 		if err != nil {
 			t.Fatalf("Commit for list repository commits failed '%s': %s", message, err)
@@ -275,3 +275,140 @@ func TestCataloger_ListCommits_Lineage(t *testing.T) {
 		t.Error("br_1 did not inherit commits correctly", diff)
 	}
 }
+
+func TestCataloger_ListCommits_Lineage_from_son(t *testing.T) {
+	ctx := context.Background()
+	c := testCataloger(t)
+
+	repository := testCatalogerRepo(t, ctx, c, "repository", "master")
+	_ = setupListCommitsByBranchData(t, ctx, c, repository, "master")
+
+	testCatalogerBranch(t, ctx, c, repository, "br_1_1", "master")
+	testCatalogerBranch(t, ctx, c, repository, "br_1_2", "br_1_1")
+	masterCommits, _, err := c.ListCommits(ctx, repository, "master", "", 100)
+	testutil.MustDo(t, "list master commits", err)
+
+	br1Commits, _, err := c.ListCommits(ctx, repository, "br_1_1", "", 100)
+	testutil.MustDo(t, "list br_1_1 commits", err)
+
+	// get all commits without the first one
+	if diff := deep.Equal(masterCommits, br1Commits[1:]); diff != nil {
+		t.Error("br_1_1 did not inherit commits correctly", diff)
+	}
+
+	b2Commits, _, err := c.ListCommits(ctx, repository, "br_1_2", "", 100)
+	testutil.MustDo(t, "list br_1_2 commits", err)
+
+	if diff := deep.Equal(br1Commits, b2Commits[1:]); diff != nil {
+		t.Error("br_1_2 did not inherit commits correctly", diff)
+	}
+
+	if err := c.CreateEntry(ctx, repository, "master", Entry{
+		Path:            "master-file",
+		Checksum:        "ssss",
+		PhysicalAddress: "xxxxxxx",
+		Size:            10000,
+	}, CreateEntryParams{}); err != nil {
+		t.Fatal("Write entry for list repository commits failed", err)
+	}
+	_, err = c.Commit(ctx, repository, "master", "commit master-file on master", "tester", nil)
+	if err != nil {
+		t.Fatalf("Commit for list repository commits failed '%s': %s", "master commit failed", err)
+	}
+	_, err = c.Merge(ctx, repository, "master", "br_1_1", "tester", "merge master to br_1_1", nil)
+	testutil.MustDo(t, "merge master  into br_1_1", err)
+
+	got, _, err := c.ListCommits(ctx, repository, "br_1_2", "", 100)
+	testutil.MustDo(t, "list br_1_2 commits", err)
+	if diff := deep.Equal(got, b2Commits); diff != nil {
+		t.Error("br_1_2 changed although not merged", diff)
+	}
+	masterCommits, _, err = c.ListCommits(ctx, repository, "master", "", 100)
+	testutil.MustDo(t, "list master commits", err)
+
+	br_1_1_base_list, _, err := c.ListCommits(ctx, repository, "br_1_1", "", 100)
+	testutil.MustDo(t, "list br_1_1 commits", err)
+	if diff := deep.Equal(masterCommits[0], br_1_1_base_list[1]); diff != nil {
+		t.Error("br_1_1 did not inherit commits correctly", diff)
+	}
+
+	testCatalogerBranch(t, ctx, c, repository, "br_2_1", "master")
+	testCatalogerBranch(t, ctx, c, repository, "br_2_2", "br_2_1")
+	if err := c.CreateEntry(ctx, repository, "br_2_2", Entry{
+		Path:            "master-file",
+		Checksum:        "zzzzz",
+		PhysicalAddress: "yyyyy",
+		Size:            20000,
+	}, CreateEntryParams{}); err != nil {
+		t.Fatal("Write entry to br_2_2 failed", err)
+	}
+	_, err = c.Commit(ctx, repository, "br_2_2", "commit master-file to br_2_2", "tester", nil)
+	if err != nil {
+		t.Fatalf("Commit for list repository commits failed '%s': %s", "br_2_2  commit failed", err)
+	}
+	br_2_2_list, _, err := c.ListCommits(ctx, repository, "br_2_2", "", 100)
+	testutil.MustDo(t, "list br_2_2  commits", err)
+	_ = br_2_2_list
+	_, err = c.Merge(ctx, repository, "br_2_2", "br_2_1", "tester", "merge br_2_2 to br_2_1", nil)
+	testutil.MustDo(t, "merge br_2_2  into br_2_1", err)
+	br_2_1_list, _, err := c.ListCommits(ctx, repository, "br_2_1", "", 100)
+	testutil.MustDo(t, "list br_2_1  commits", err)
+	_ = br_2_1_list
+	masterList, _, err := c.ListCommits(ctx, repository, "master", "", 100)
+	testutil.MustDo(t, "list master commits", err)
+	if diff := deep.Equal(masterCommits, masterList); diff != nil {
+		t.Error("master commits changed befor merge", diff)
+	}
+	merge_2, err := c.Merge(ctx, repository, "br_2_1", "master", "tester", "merge br_2_1 to master", nil)
+	testutil.MustDo(t, "merge br_2_1  into master", err)
+	if merge_2.Differences[0].Type != DifferenceTypeChanged || merge_2.Differences[0].Path != "master-file" {
+		t.Error("merge br_2_1 into master with unexpected results", merge_2.Differences[0])
+	}
+
+	masterList, _, err = c.ListCommits(ctx, repository, "master", "", 100)
+	testutil.MustDo(t, "list master commits", err)
+	if diff := deep.Equal(br_2_1_list, masterList[1:]); diff != nil {
+		t.Error("master commits list mismatch with br_2_1_list", diff)
+	}
+
+	br_1_1_list, _, err := c.ListCommits(ctx, repository, "br_1_1", "", 100)
+	testutil.MustDo(t, "list br_1_1 commits", err)
+	if diff := deep.Equal(br_1_1_base_list, br_1_1_list); diff != nil {
+		t.Error("br_1_1 commits changed before merge", diff)
+	}
+	_, err = c.Merge(ctx, repository, "master", "br_1_1", "tester", "merge master to br_1_1", nil)
+	testutil.MustDo(t, "merge master  into br_1_1", err)
+	br_1_1_list, _, err = c.ListCommits(ctx, repository, "br_1_1", "", 100)
+	testutil.MustDo(t, "list br_1_1 commits", err)
+	if diff := deep.Equal(masterList[:5], br_1_1_list[1:6]); diff != nil {
+		t.Error("master 5 first different from br_1_1 [1:6]", diff)
+	}
+	if diff := deep.Equal(masterList[6:], br_1_1_list[9:]); diff != nil {
+		t.Error("master 5 first different from br_1_1 [1:6]", diff)
+	}
+	// test that a change to br_2_2 does not prpagate to master
+	if err := c.CreateEntry(ctx, repository, "br_2_2", Entry{
+		Path:            "no-propagate-file",
+		Checksum:        "aaaaaaaa",
+		PhysicalAddress: "yybbbbbbyyy",
+		Size:            20000,
+	}, CreateEntryParams{}); err != nil {
+		t.Fatal("Write no-propagate-file to br_2_2 failed", err)
+	}
+	_, err = c.Commit(ctx, repository, "br_2_2", "commit master-file to br_2_2", "tester", nil)
+	if err != nil {
+		t.Fatalf("no-propagate-Commit for list repository commits failed '%s': %s", "br_2_2  commit failed", err)
+	}
+	_, err = c.Merge(ctx, repository, "br_2_2", "br_2_1", "tester", "merge br_2_2 to br_2_1", nil)
+	testutil.MustDo(t, "second merge br_2_2  into br_2_1", err)
+	new_br_2_1_list, _, err := c.ListCommits(ctx, repository, "br_2_1", "", 100)
+	testutil.MustDo(t, "second list br_2_1 commits", err)
+	if diff := deep.Equal(br_2_1_list, new_br_2_1_list); diff == nil {
+		t.Error("br_2_1 commits did not changed after merge", diff)
+	}
+	new_master_list, _, err := c.ListCommits(ctx, repository, "master", "", 100)
+	testutil.MustDo(t, "third list master commits", err)
+	if diff := deep.Equal(new_master_list, masterList); diff != nil {
+		t.Error("master commits  changed without merge", diff)
+	}
+}

catalog/db.go

@@ -142,9 +142,9 @@ func getLineage(tx db.Tx, branchID int64, commitID CommitID) ([]lineageCommit, e
 	return requestedLineage, nil
 }
 
-func getLineageAsValues(lineage []lineageCommit, branchID int64) string {
+func getLineageAsValues(lineage []lineageCommit, branchID int64, commitID CommitID) string {
 	valArray := make([]string, 1, len(lineage)+1)
-	valArray[0] = fmt.Sprintf("(0,%d,%d)", branchID, MaxCommitID)
+	valArray[0] = fmt.Sprintf("(0,%d::bigint,%d::bigint)", branchID, commitID)
 	for precedence, lineageBranch := range lineage {
 		valArray = append(valArray, fmt.Sprintf("(%d,%d,%d)", precedence+1, lineageBranch.BranchID, lineageBranch.CommitID))
 	}

catalog/views.go

@@ -147,7 +147,7 @@ func sqDiffFromSonV(fatherID, sonID int64, fatherEffectiveCommit, sonEffectiveCo
 }
 
 func sqDiffFromFatherV(fatherID, sonID int64, lastSonMergeWithFather CommitID, fatherUncommittedLineage, sonUncommittedLineage []lineageCommit) sq.SelectBuilder {
-	sonLineageValues := getLineageAsValues(sonUncommittedLineage, sonID)
+	sonLineageValues := getLineageAsValues(sonUncommittedLineage, sonID, MaxCommitID)
 	sonLineage := sqEntriesLineage(sonID, UncommittedID, sonUncommittedLineage)
 	sqSon := sq.Select("*").
 		FromSelect(sonLineage, "s").

ddl/000001_initial.up.sql

@@ -107,32 +107,40 @@ CREATE SEQUENCE catalog_commit_id_seq
     NO MAXVALUE
     CACHE 10;
 
-CREATE TABLE catalog_commits (
-    branch_id           bigint                                 NOT NULL,
-    commit_id           bigint                                 NOT NULL,
-    previous_commit_id  bigint                                 NOT NULL,
-    committer           character varying,
-    message             character varying,
-    creation_date       timestamp with time zone DEFAULT now() NOT NULL,
-    metadata            jsonb,
-    merge_source_branch bigint,
-    merge_source_commit bigint,
-    merge_type          catalog_merge_type               DEFAULT 'none'::catalog_merge_type,
-    lineage_commits     bigint[]                 DEFAULT array []::bigint[]
-);
-
-CREATE TABLE catalog_entries (
-    branch_id bigint NOT NULL,
-    path character varying COLLATE "C" NOT NULL,
+CREATE TABLE catalog_commits(
+                                branch_id           bigint                                 NOT NULL,
+                                commit_id           bigint                                 NOT NULL,
+                                previous_commit_id  bigint                                 NOT NULL,
+                                committer           character varying,
+                                message             character varying,
+                                creation_date       timestamp with time zone DEFAULT now() NOT NULL,
+                                metadata            jsonb,
+                                merge_source_branch bigint,
+                                merge_source_commit bigint,
+                                merge_type          catalog_merge_type       DEFAULT 'none'::catalog_merge_type,
+                                lineage_commits     bigint[]                 DEFAULT array []::bigint[]
+);
+
+CREATE INDEX commits_lineage_idx
+    ON catalog_commits USING btree
+        (merge_type ASC NULLS LAST, branch_id ASC NULLS LAST, commit_id DESC NULLS LAST)
+    INCLUDE (lineage_commits, merge_source_branch, merge_source_commit)
+    TABLESPACE pg_default
+    WHERE merge_type in ('from_father'::merge_type, 'from_son'::merge_type);
+
+CREATE TABLE catalog_entries
+(
+    branch_id        bigint                                                   NOT NULL,
+    path             character varying COLLATE "C"                            NOT NULL,
     physical_address character varying,
-    creation_date timestamp with time zone DEFAULT now() NOT NULL,
-    size bigint NOT NULL,
-    checksum character varying(64) NOT NULL,
-    metadata jsonb,
-    min_commit bigint DEFAULT 0 NOT NULL,
-    max_commit bigint DEFAULT catalog_max_commit_id() NOT NULL,
+    creation_date    timestamp with time zone DEFAULT now()                   NOT NULL,
+    size             bigint                                                   NOT NULL,
+    checksum         character varying(64)                                    NOT NULL,
+    metadata         jsonb,
+    min_commit       bigint                   DEFAULT 0                       NOT NULL,
+    max_commit       bigint                   DEFAULT catalog_max_commit_id() NOT NULL,
     -- If set, entry has expired.  Requests to retrieve may return "410 Gone".
-    is_expired BOOLEAN DEFAULT false NOT NULL
+    is_expired       BOOLEAN                  DEFAULT false                   NOT NULL
 );
 ALTER TABLE ONLY catalog_entries ALTER COLUMN path SET STATISTICS 10000;