Add updated_at field to stash_id's

Stored as a time (not OptionalTime), and so on migration set all
existing StashIDs to have time zero, then on scrape update.

graphql/schema/types/stash-box.graphql

@@ -13,11 +13,13 @@ input StashBoxInput {
 type StashID {
   endpoint: String!
   stash_id: String!
+  updated_at: Time!
 }
 
 input StashIDInput {
   endpoint: String!
   stash_id: String!
+  updated_at: Time!
 }
 
 input StashBoxFingerprintSubmissionInput {

internal/identify/identify.go

@@ -8,6 +8,7 @@ import (
 	"errors"
 	"fmt"
 	"strconv"
+	"time"
 
 	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/models"
@@ -244,7 +245,8 @@ func (t *SceneIdentifier) getSceneUpdater(ctx context.Context, s *models.Scene,
 		}
 	}
 
-	stashIDs, err := rel.stashIDs(ctx)
+	currentTime := time.Now()
+	stashIDs, err := rel.stashIDs(ctx, currentTime)
 	if err != nil {
 		return nil, err
 	}

internal/identify/scene.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/models"
@@ -182,7 +183,7 @@ func (g sceneRelationships) tags(ctx context.Context) ([]int, error) {
 	return tagIDs, nil
 }
 
-func (g sceneRelationships) stashIDs(ctx context.Context) ([]models.StashID, error) {
+func (g sceneRelationships) stashIDs(ctx context.Context, updateTime time.Time) ([]models.StashID, error) {
 	remoteSiteID := g.result.result.RemoteSiteID
 	fieldStrategy := g.fieldOptions["stash_ids"]
 	target := g.scene
@@ -210,22 +211,19 @@ func (g sceneRelationships) stashIDs(ctx context.Context) ([]models.StashID, err
 
 	for i, stashID := range stashIDs {
 		if endpoint == stashID.Endpoint {
-			// if stashID is the same, then don't set
-			if stashID.StashID == *remoteSiteID {
-				return nil, nil
-			}
-
 			// replace the stash id and return
 			stashID.StashID = *remoteSiteID
+			stashID.UpdatedAt = updateTime
 			stashIDs[i] = stashID
 			return stashIDs, nil
 		}
 	}
 
 	// not found, create new entry
 	stashIDs = append(stashIDs, models.StashID{
-		StashID:  *remoteSiteID,
-		Endpoint: endpoint,
+		StashID:   *remoteSiteID,
+		Endpoint:  endpoint,
+		UpdatedAt: updateTime,
 	})
 
 	if sliceutil.SliceSame(originalStashIDs, stashIDs) {

internal/identify/scene_test.go

@@ -5,6 +5,7 @@ import (
 	"reflect"
 	"strconv"
 	"testing"
+	"time"
 
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/models/mocks"
@@ -548,8 +549,9 @@ func Test_sceneRelationships_stashIDs(t *testing.T) {
 		ID: sceneWithStashID,
 		StashIDs: models.NewRelatedStashIDs([]models.StashID{
 			{
-				StashID:  remoteSiteID,
-				Endpoint: existingEndpoint,
+				StashID:   remoteSiteID,
+				Endpoint:  existingEndpoint,
+				UpdatedAt: time.Time{},
 			},
 		}),
 	}
@@ -605,7 +607,13 @@ func Test_sceneRelationships_stashIDs(t *testing.T) {
 			defaultOptions,
 			existingEndpoint,
 			&remoteSiteID,
-			nil,
+			[]models.StashID{
+				{
+					StashID:   remoteSiteID,
+					Endpoint:  existingEndpoint,
+					UpdatedAt: time.Time{},
+				},
+			},
 			false,
 		},
 		{
@@ -616,8 +624,9 @@ func Test_sceneRelationships_stashIDs(t *testing.T) {
 			&newRemoteSiteID,
 			[]models.StashID{
 				{
-					StashID:  newRemoteSiteID,
-					Endpoint: existingEndpoint,
+					StashID:   newRemoteSiteID,
+					Endpoint:  existingEndpoint,
+					UpdatedAt: time.Time{},
 				},
 			},
 			false,
@@ -630,12 +639,14 @@ func Test_sceneRelationships_stashIDs(t *testing.T) {
 			&newRemoteSiteID,
 			[]models.StashID{
 				{
-					StashID:  remoteSiteID,
-					Endpoint: existingEndpoint,
+					StashID:   remoteSiteID,
+					Endpoint:  existingEndpoint,
+					UpdatedAt: time.Time{},
 				},
 				{
-					StashID:  newRemoteSiteID,
-					Endpoint: newEndpoint,
+					StashID:   newRemoteSiteID,
+					Endpoint:  newEndpoint,
+					UpdatedAt: time.Time{},
 				},
 			},
 			false,
@@ -650,8 +661,9 @@ func Test_sceneRelationships_stashIDs(t *testing.T) {
 			&newRemoteSiteID,
 			[]models.StashID{
 				{
-					StashID:  newRemoteSiteID,
-					Endpoint: newEndpoint,
+					StashID:   newRemoteSiteID,
+					Endpoint:  newEndpoint,
+					UpdatedAt: time.Time{},
 				},
 			},
 			false,
@@ -681,7 +693,8 @@ func Test_sceneRelationships_stashIDs(t *testing.T) {
 				},
 			}
 
-			got, err := tr.stashIDs(testCtx)
+			got, err := tr.stashIDs(testCtx, time.Time{})
+
 			if (err != nil) != tt.wantErr {
 				t.Errorf("sceneRelationships.stashIDs() error = %v, wantErr %v", err, tt.wantErr)
 				return

pkg/models/model_scraped_item.go

@@ -3,6 +3,7 @@ package models
 import (
 	"context"
 	"strconv"
+	"time"
 
 	"github.com/stashapp/stash/pkg/sliceutil/stringslice"
 	"github.com/stashapp/stash/pkg/utils"
@@ -29,8 +30,9 @@ func (s *ScrapedStudio) ToStudio(endpoint string, excluded map[string]bool) *Stu
 	if s.RemoteSiteID != nil && endpoint != "" {
 		ret.StashIDs = NewRelatedStashIDs([]StashID{
 			{
-				Endpoint: endpoint,
-				StashID:  *s.RemoteSiteID,
+				Endpoint:  endpoint,
+				StashID:   *s.RemoteSiteID,
+				UpdatedAt: time.Now(),
 			},
 		})
 	}
@@ -65,6 +67,7 @@ func (s *ScrapedStudio) GetImage(ctx context.Context, excluded map[string]bool)
 func (s *ScrapedStudio) ToPartial(id string, endpoint string, excluded map[string]bool, existingStashIDs []StashID) StudioPartial {
 	ret := NewStudioPartial()
 	ret.ID, _ = strconv.Atoi(id)
+	currentTime := time.Now()
 
 	if s.Name != "" && !excluded["name"] {
 		ret.Name = NewOptionalString(s.Name)
@@ -90,8 +93,9 @@ func (s *ScrapedStudio) ToPartial(id string, endpoint string, excluded map[strin
 			Mode:     RelationshipUpdateModeSet,
 		}
 		ret.StashIDs.Set(StashID{
-			Endpoint: endpoint,
-			StashID:  *s.RemoteSiteID,
+			Endpoint:  endpoint,
+			StashID:   *s.RemoteSiteID,
+			UpdatedAt: currentTime,
 		})
 	}
 
@@ -137,6 +141,7 @@ func (ScrapedPerformer) IsScrapedContent() {}
 
 func (p *ScrapedPerformer) ToPerformer(endpoint string, excluded map[string]bool) *Performer {
 	ret := NewPerformer()
+	currentTime := time.Now()
 	ret.Name = *p.Name
 
 	if p.Aliases != nil && !excluded["aliases"] {
@@ -244,8 +249,9 @@ func (p *ScrapedPerformer) ToPerformer(endpoint string, excluded map[string]bool
 	if p.RemoteSiteID != nil && endpoint != "" {
 		ret.StashIDs = NewRelatedStashIDs([]StashID{
 			{
-				Endpoint: endpoint,
-				StashID:  *p.RemoteSiteID,
+				Endpoint:  endpoint,
+				StashID:   *p.RemoteSiteID,
+				UpdatedAt: currentTime,
 			},
 		})
 	}
@@ -375,8 +381,9 @@ func (p *ScrapedPerformer) ToPartial(endpoint string, excluded map[string]bool,
 			Mode:     RelationshipUpdateModeSet,
 		}
 		ret.StashIDs.Set(StashID{
-			Endpoint: endpoint,
-			StashID:  *p.RemoteSiteID,
+			Endpoint:  endpoint,
+			StashID:   *p.RemoteSiteID,
+			UpdatedAt: time.Now(),
 		})
 	}
 

pkg/models/model_scraped_item_test.go

@@ -87,6 +87,11 @@ func Test_scrapedToStudioInput(t *testing.T) {
 
 			got.CreatedAt = time.Time{}
 			got.UpdatedAt = time.Time{}
+			if got.StashIDs.Loaded() && len(got.StashIDs.List()) > 0 {
+				for stid := range got.StashIDs.List() {
+					got.StashIDs.List()[stid].UpdatedAt = time.Time{}
+				}
+			}
 			assert.Equal(t, tt.want, got)
 		})
 	}
@@ -243,6 +248,12 @@ func Test_scrapedToPerformerInput(t *testing.T) {
 
 			got.CreatedAt = time.Time{}
 			got.UpdatedAt = time.Time{}
+
+			if got.StashIDs.Loaded() && len(got.StashIDs.List()) > 0 {
+				for stid := range got.StashIDs.List() {
+					got.StashIDs.List()[stid].UpdatedAt = time.Time{}
+				}
+			}
 			assert.Equal(t, tt.want, got)
 		})
 	}
@@ -263,7 +274,7 @@ func TestScrapedStudio_ToPartial(t *testing.T) {
 		images            = []string{image}
 
 		existingEndpoint = "existingEndpoint"
-		existingStashID  = StashID{"existingStashID", existingEndpoint}
+		existingStashID  = StashID{"existingStashID", existingEndpoint, time.Time{}}
 		existingStashIDs = []StashID{existingStashID}
 	)
 
@@ -362,6 +373,11 @@ func TestScrapedStudio_ToPartial(t *testing.T) {
 
 			// unset updatedAt - we don't need to compare it
 			got.UpdatedAt = OptionalTime{}
+			if got.StashIDs != nil && len(got.StashIDs.StashIDs) > 0 {
+				for stid := range got.StashIDs.StashIDs {
+					got.StashIDs.StashIDs[stid].UpdatedAt = time.Time{}
+				}
+			}
 
 			assert.Equal(t, tt.want, got)
 		})

pkg/models/stash_ids.go

@@ -1,8 +1,11 @@
 package models
 
+import "time"
+
 type StashID struct {
-	StashID  string `db:"stash_id" json:"stash_id"`
-	Endpoint string `db:"endpoint" json:"endpoint"`
+	StashID   string    `db:"stash_id" json:"stash_id"`
+	Endpoint  string    `db:"endpoint" json:"endpoint"`
+	UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
 }
 
 type UpdateStashIDs struct {

pkg/sqlite/database.go

@@ -34,7 +34,7 @@ const (
 	cacheSizeEnv = "STASH_SQLITE_CACHE_SIZE"
 )
 
-var appSchemaVersion uint = 68
+var appSchemaVersion uint = 69
 
 //go:embed migrations/*.sql
 var migrationsBox embed.FS

pkg/sqlite/migrations/69_stash_id_updated_at.up.sql

@@ -0,0 +1,3 @@
+ALTER TABLE `performer_stash_ids` ADD COLUMN `updated_at` datetime not null default '1970-01-01T00:00:00Z';
+ALTER TABLE `scene_stash_ids` ADD COLUMN `updated_at` datetime not null default '1970-01-01T00:00:00Z';
+ALTER TABLE `studio_stash_ids` ADD COLUMN `updated_at` datetime not null default '1970-01-01T00:00:00Z';

pkg/sqlite/repository.go

@@ -14,11 +14,6 @@ import (
 
 const idColumn = "id"
 
-type objectList interface {
-	Append(o interface{})
-	New() interface{}
-}
-
 type repository struct {
 	tableName string
 	idColumn  string
@@ -124,17 +119,6 @@ func (r *repository) queryFunc(ctx context.Context, query string, args []interfa
 	return nil
 }
 
-func (r *repository) query(ctx context.Context, query string, args []interface{}, out objectList) error {
-	return r.queryFunc(ctx, query, args, false, func(rows *sqlx.Rows) error {
-		object := out.New()
-		if err := rows.StructScan(object); err != nil {
-			return err
-		}
-		out.Append(object)
-		return nil
-	})
-}
-
 func (r *repository) queryStruct(ctx context.Context, query string, args []interface{}, out interface{}) error {
 	if err := r.queryFunc(ctx, query, args, true, func(rows *sqlx.Rows) error {
 		if err := rows.StructScan(out); err != nil {
@@ -421,18 +405,25 @@ type stashIDRepository struct {
 type stashIDs []models.StashID
 
 func (s *stashIDs) Append(o interface{}) {
-	*s = append(*s, *o.(*models.StashID))
+	*s = append(*s, o.(models.StashID))
 }
 
 func (s *stashIDs) New() interface{} {
 	return &models.StashID{}
 }
 
 func (r *stashIDRepository) get(ctx context.Context, id int) ([]models.StashID, error) {
-	query := fmt.Sprintf("SELECT stash_id, endpoint from %s WHERE %s = ?", r.tableName, r.idColumn)
+	query := fmt.Sprintf("SELECT stash_id, endpoint, updated_at from %s WHERE %s = ?", r.tableName, r.idColumn)
 	var ret stashIDs
-	err := r.query(ctx, query, []interface{}{id}, &ret)
-	return []models.StashID(ret), err
+	err := r.queryFunc(ctx, query, []interface{}{id}, false, func(rows *sqlx.Rows) error {
+		var v stashIDRow
+		if err := rows.StructScan(&v); err != nil {
+			return err
+		}
+		ret.Append(v.resolve())
+		return nil
+	})
+	return ret, err
 }
 
 type filesRepository struct {