restructure responsibilities around indexing and generative content

- remove autotag/summarise from index consumer
- fix some issues with auto tag
- refactor node mutation logic

app/resources/mq/message_types.go

@@ -17,6 +17,10 @@ import (
 )
 
 type IndexNode struct {
+	ID library.NodeID
+}
+
+type AutoFillNode struct {
 	ID               library.NodeID
 	SummariseContent bool
 	AutoTag          bool

app/services/generative/generative.go

@@ -15,7 +15,7 @@ type Tagger interface {
 }
 
 type Summariser interface {
-	Summarise(ctx context.Context, object datagraph.Item) (string, error)
+	Summarise(ctx context.Context, content datagraph.Content) (string, error)
 }
 
 var (

app/services/generative/summary.go

@@ -12,7 +12,7 @@ import (
 )
 
 var SummarisePrompt = template.Must(template.New("").Parse(`
-Write a short few paragraphs that are somewhat engaging but remaining relatively neutral in tone in the style of a wikipedia introduction about "{{ .Name }}". Focus on providing unique insights and interesting details while keeping the tone conversational and approachable. Imagine this will be read by someone browsing a directory or knowledgebase.
+Write a short few paragraphs that are somewhat engaging but remaining relatively neutral in tone in the style of a wikipedia introduction based on the specified content. Focus on providing unique insights and interesting details while keeping the tone conversational and approachable. Imagine this will be read by someone browsing a directory or knowledgebase.
 
 Be aware that the input to this may include broken HTML and other artifacts from the web and due to the nature of web scraping, there may be parts that do not make sense.
 
@@ -29,11 +29,10 @@ Content:
 {{ .Content }}
 `))
 
-func (g *generator) Summarise(ctx context.Context, object datagraph.Item) (string, error) {
+func (g *generator) Summarise(ctx context.Context, content datagraph.Content) (string, error) {
 	template := strings.Builder{}
 	err := SummarisePrompt.Execute(&template, map[string]any{
-		"Name":    object.GetName(),
-		"Content": object.GetContent().Plaintext(),
+		"Content": content.Plaintext(),
 	})
 	if err != nil {
 		return "", fault.Wrap(err, fctx.With(ctx))

app/services/library/node_fill/filler.go

@@ -25,20 +25,23 @@ import (
 var errFillRuleNotAvailale = fault.New("fill rule not available")
 
 type Filler struct {
-	nodeWriter *node_writer.Writer
-	indexQueue pubsub.Topic[mq.IndexNode]
-	assetQueue pubsub.Topic[mq.DownloadAsset]
+	nodeWriter    *node_writer.Writer
+	indexQueue    pubsub.Topic[mq.IndexNode]
+	assetQueue    pubsub.Topic[mq.DownloadAsset]
+	autoFillQueue pubsub.Topic[mq.AutoFillNode]
 }
 
 func New(
 	nodeWriter *node_writer.Writer,
 	indexQueue pubsub.Topic[mq.IndexNode],
 	assetQueue pubsub.Topic[mq.DownloadAsset],
+	autoFillQueue pubsub.Topic[mq.AutoFillNode],
 ) *Filler {
 	return &Filler{
-		nodeWriter: nodeWriter,
-		indexQueue: indexQueue,
-		assetQueue: assetQueue,
+		nodeWriter:    nodeWriter,
+		indexQueue:    indexQueue,
+		assetQueue:    assetQueue,
+		autoFillQueue: autoFillQueue,
 	}
 }
 
@@ -71,7 +74,7 @@ func (f *Filler) FillContentFromLink(ctx context.Context, link *link_ref.LinkRef
 			return fault.Wrap(err, fctx.With(ctx))
 		}
 
-		f.assetQueue.Publish(ctx, dt.Map(wc.Content.Media(), func(u string) mq.DownloadAsset {
+		err = f.assetQueue.Publish(ctx, dt.Map(wc.Content.Media(), func(u string) mq.DownloadAsset {
 			return mq.DownloadAsset{
 				URL: u,
 				ContentFillRule: opt.New(asset.ContentFillCommand{
@@ -80,17 +83,37 @@ func (f *Filler) FillContentFromLink(ctx context.Context, link *link_ref.LinkRef
 				}),
 			}
 		})...)
+		if err != nil {
+			return fault.Wrap(err, fctx.With(ctx))
+		}
+
+		err = f.autoFillQueue.Publish(ctx, mq.AutoFillNode{
+			ID:               library.NodeID(n.Mark.ID()),
+			SummariseContent: true,
+			AutoTag:          true,
+		})
+		if err != nil {
+			return fault.Wrap(err, fctx.With(ctx))
+		}
 
 		if vis == visibility.VisibilityPublished {
 			if err := f.indexQueue.Publish(ctx, mq.IndexNode{
-				ID:               library.NodeID(n.Mark.ID()),
-				SummariseContent: true,
-				AutoTag:          true,
+				ID: library.NodeID(n.Mark.ID()),
 			}); err != nil {
 				return fault.Wrap(err, fctx.With(ctx))
 			}
 		}
 
+	case asset.ContentFillRuleReplace:
+		err = f.autoFillQueue.Publish(ctx, mq.AutoFillNode{
+			ID:               library.NodeID(library.NodeID(cfr.TargetNodeID)),
+			SummariseContent: true,
+			AutoTag:          true,
+		})
+		if err != nil {
+			return fault.Wrap(err, fctx.With(ctx))
+		}
+
 	default:
 		return fault.Wrap(errFillRuleNotAvailale, fctx.With(ctx))
 	}

app/services/library/node_mutate/create.go

@@ -3,22 +3,17 @@ package node_mutate
 import (
 	"context"
 
-	"github.com/Southclaws/dt"
 	"github.com/Southclaws/fault"
 	"github.com/Southclaws/fault/fctx"
 	"github.com/Southclaws/fault/fmsg"
-	"github.com/rs/xid"
+	"github.com/Southclaws/opt"
 
 	"github.com/Southclaws/storyden/app/resources/account"
-	"github.com/Southclaws/storyden/app/resources/asset"
 	"github.com/Southclaws/storyden/app/resources/library"
-	"github.com/Southclaws/storyden/app/resources/library/node_writer"
 	"github.com/Southclaws/storyden/app/resources/mark"
 	"github.com/Southclaws/storyden/app/resources/mq"
 	"github.com/Southclaws/storyden/app/resources/rbac"
-	"github.com/Southclaws/storyden/app/resources/tag/tag_ref"
 	"github.com/Southclaws/storyden/app/resources/visibility"
-	"github.com/Southclaws/storyden/app/services/link/fetcher"
 )
 
 func (s *Manager) Create(ctx context.Context,
@@ -42,42 +37,14 @@ func (s *Manager) Create(ctx context.Context,
 		}
 	}
 
-	opts, err := s.applyOpts(ctx, p)
+	pre, err := s.preMutation(ctx, p, opt.NewEmpty[library.Node]())
 	if err != nil {
 		return nil, fault.Wrap(err, fctx.With(ctx))
 	}
-
-	if v, ok := p.AssetSources.Get(); ok {
-		for _, source := range v {
-			a, err := s.fetcher.CopyAsset(ctx, source)
-			if err != nil {
-				return nil, fault.Wrap(err, fctx.With(ctx))
-			}
-
-			opts = append(opts, node_writer.WithAssets([]asset.AssetID{a.ID}))
-		}
-	}
+	opts := pre.opts
 
 	nodeSlug := p.Slug.Or(mark.NewSlugFromName(name))
 
-	if u, ok := p.URL.Get(); ok {
-		ln, err := s.fetcher.Fetch(ctx, u, fetcher.Options{})
-		if err == nil {
-			opts = append(opts, node_writer.WithLink(xid.ID(ln.ID)))
-		}
-	}
-
-	if tags, ok := p.Tags.Get(); ok {
-		newTags, err := s.tagWriter.Add(ctx, tags...)
-		if err != nil {
-			return nil, fault.Wrap(err, fctx.With(ctx))
-		}
-
-		tagIDs := dt.Map(newTags, func(t *tag_ref.Tag) tag_ref.ID { return t.ID })
-
-		opts = append(opts, node_writer.WithTagsAdd(tagIDs...))
-	}
-
 	n, err := s.nodeWriter.Create(ctx, owner, name, nodeSlug, opts...)
 	if err != nil {
 		return nil, fault.Wrap(err, fctx.With(ctx))

app/services/library/node_mutate/mutate.go

@@ -1,13 +1,9 @@
 package node_mutate
 
 import (
-	"context"
 	"net/url"
 
-	"github.com/Southclaws/fault"
-	"github.com/Southclaws/fault/fctx"
 	"github.com/Southclaws/opt"
-	"github.com/rs/xid"
 	"go.uber.org/zap"
 
 	"github.com/Southclaws/storyden/app/resources/account/account_querier"
@@ -19,9 +15,11 @@ import (
 	"github.com/Southclaws/storyden/app/resources/library/node_writer"
 	"github.com/Southclaws/storyden/app/resources/mark"
 	"github.com/Southclaws/storyden/app/resources/mq"
+	"github.com/Southclaws/storyden/app/resources/tag"
 	"github.com/Southclaws/storyden/app/resources/tag/tag_ref"
 	"github.com/Southclaws/storyden/app/resources/tag/tag_writer"
 	"github.com/Southclaws/storyden/app/resources/visibility"
+	"github.com/Southclaws/storyden/app/services/generative"
 	"github.com/Southclaws/storyden/app/services/link/fetcher"
 	"github.com/Southclaws/storyden/app/services/tag/autotagger"
 	"github.com/Southclaws/storyden/internal/deletable"
@@ -41,26 +39,11 @@ type Partial struct {
 	AssetsAdd        opt.Optional[[]asset.AssetID]
 	AssetsRemove     opt.Optional[[]asset.AssetID]
 	AssetSources     opt.Optional[[]string]
+	TagFill          opt.Optional[tag.TagFillCommand]
 	ContentFill      opt.Optional[asset.ContentFillCommand]
 	ContentSummarise opt.Optional[bool]
 }
 
-func (p Partial) Opts() (opts []node_writer.Option) {
-	p.Name.Call(func(value string) { opts = append(opts, node_writer.WithName(value)) })
-	p.Slug.Call(func(value mark.Slug) { opts = append(opts, node_writer.WithSlug(value.String())) })
-	p.PrimaryImage.Call(func(value xid.ID) {
-		opts = append(opts, node_writer.WithPrimaryImage(value))
-	}, func() {
-		opts = append(opts, node_writer.WithPrimaryImageRemoved())
-	})
-	p.Content.Call(func(value datagraph.Content) { opts = append(opts, node_writer.WithContent(value)) })
-	p.Metadata.Call(func(value map[string]any) { opts = append(opts, node_writer.WithMetadata(value)) })
-	p.AssetsAdd.Call(func(value []asset.AssetID) { opts = append(opts, node_writer.WithAssets(value)) })
-	p.AssetsRemove.Call(func(value []asset.AssetID) { opts = append(opts, node_writer.WithAssetsRemoved(value)) })
-	p.Visibility.Call(func(value visibility.Visibility) { opts = append(opts, node_writer.WithVisibility(value)) })
-	return
-}
-
 type Manager struct {
 	logger            *zap.Logger
 	accountQuery      *account_querier.Querier
@@ -70,6 +53,7 @@ type Manager struct {
 	tagger            *autotagger.Tagger
 	nc                node_children.Repository
 	fetcher           *fetcher.Fetcher
+	summariser        generative.Summariser
 	indexQueue        pubsub.Topic[mq.IndexNode]
 	deleteQueue       pubsub.Topic[mq.DeleteNode]
 	assetAnalyseQueue pubsub.Topic[mq.AnalyseAsset]
@@ -84,6 +68,7 @@ func New(
 	tagger *autotagger.Tagger,
 	nc node_children.Repository,
 	fetcher *fetcher.Fetcher,
+	summariser generative.Summariser,
 	indexQueue pubsub.Topic[mq.IndexNode],
 	deleteQueue pubsub.Topic[mq.DeleteNode],
 	assetAnalyseQueue pubsub.Topic[mq.AnalyseAsset],
@@ -97,23 +82,9 @@ func New(
 		tagger:            tagger,
 		nc:                nc,
 		fetcher:           fetcher,
+		summariser:        summariser,
 		indexQueue:        indexQueue,
 		deleteQueue:       deleteQueue,
 		assetAnalyseQueue: assetAnalyseQueue,
 	}
 }
-
-func (s *Manager) applyOpts(ctx context.Context, p Partial) ([]node_writer.Option, error) {
-	opts := p.Opts()
-
-	if parentSlug, ok := p.Parent.Get(); ok {
-		parent, err := s.nodeQuerier.Get(ctx, parentSlug)
-		if err != nil {
-			return nil, fault.Wrap(err, fctx.With(ctx))
-		}
-
-		opts = append(opts, node_writer.WithParent(library.NodeID(parent.Mark.ID())))
-	}
-
-	return opts, nil
-}