From 5d8b329f3d1e06679ce57360c2138fb3b4ed2cdd Mon Sep 17 00:00:00 2001 From: barney Date: Sat, 14 Dec 2024 17:19:58 -0500 Subject: [PATCH 1/3] restructure responsibilities around indexing and generative content - remove autotag/summarise from index consumer - fix some issues with auto tag - refactor node mutation logic --- app/resources/mq/message_types.go | 4 + app/services/generative/generative.go | 2 +- app/services/generative/summary.go | 7 +- app/services/library/node_fill/filler.go | 43 +++- app/services/library/node_mutate/create.go | 39 +-- app/services/library/node_mutate/mutate.go | 41 +-- app/services/library/node_mutate/options.go | 233 ++++++++++++++++++ app/services/library/node_mutate/update.go | 121 +-------- app/services/library/node_semdex/autofill.go | 35 +++ app/services/library/node_semdex/indexer.go | 88 +------ .../library/node_semdex/node_semdex.go | 57 ++++- app/transports/http/bindings/nodes.go | 31 +-- .../LibraryPageScreen/useLibraryPageScreen.ts | 10 +- 13 files changed, 392 insertions(+), 319 deletions(-) create mode 100644 app/services/library/node_mutate/options.go create mode 100644 app/services/library/node_semdex/autofill.go diff --git a/app/resources/mq/message_types.go b/app/resources/mq/message_types.go index 27fd1e583..a8517a6e5 100644 --- a/app/resources/mq/message_types.go +++ b/app/resources/mq/message_types.go @@ -17,6 +17,10 @@ import ( ) type IndexNode struct { + ID library.NodeID +} + +type AutoFillNode struct { ID library.NodeID SummariseContent bool AutoTag bool diff --git a/app/services/generative/generative.go b/app/services/generative/generative.go index 2102daa4b..cccc7b7fd 100644 --- a/app/services/generative/generative.go +++ b/app/services/generative/generative.go @@ -15,7 +15,7 @@ type Tagger interface { } type Summariser interface { - Summarise(ctx context.Context, object datagraph.Item) (string, error) + Summarise(ctx context.Context, content datagraph.Content) (string, error) } var ( diff --git a/app/services/generative/summary.go b/app/services/generative/summary.go index a460b7303..936c189a1 100644 --- a/app/services/generative/summary.go +++ b/app/services/generative/summary.go @@ -12,7 +12,7 @@ import ( ) var SummarisePrompt = template.Must(template.New("").Parse(` -Write a short few paragraphs that are somewhat engaging but remaining relatively neutral in tone in the style of a wikipedia introduction about "{{ .Name }}". Focus on providing unique insights and interesting details while keeping the tone conversational and approachable. Imagine this will be read by someone browsing a directory or knowledgebase. +Write a short few paragraphs that are somewhat engaging but remaining relatively neutral in tone in the style of a wikipedia introduction based on the specified content. Focus on providing unique insights and interesting details while keeping the tone conversational and approachable. Imagine this will be read by someone browsing a directory or knowledgebase. Be aware that the input to this may include broken HTML and other artifacts from the web and due to the nature of web scraping, there may be parts that do not make sense. @@ -29,11 +29,10 @@ Content: {{ .Content }} `)) -func (g *generator) Summarise(ctx context.Context, object datagraph.Item) (string, error) { +func (g *generator) Summarise(ctx context.Context, content datagraph.Content) (string, error) { template := strings.Builder{} err := SummarisePrompt.Execute(&template, map[string]any{ - "Name": object.GetName(), - "Content": object.GetContent().Plaintext(), + "Content": content.Plaintext(), }) if err != nil { return "", fault.Wrap(err, fctx.With(ctx)) diff --git a/app/services/library/node_fill/filler.go b/app/services/library/node_fill/filler.go index 423a20272..d76df28d1 100644 --- a/app/services/library/node_fill/filler.go +++ b/app/services/library/node_fill/filler.go @@ -25,20 +25,23 @@ import ( var errFillRuleNotAvailale = fault.New("fill rule not available") type Filler struct { - nodeWriter *node_writer.Writer - indexQueue pubsub.Topic[mq.IndexNode] - assetQueue pubsub.Topic[mq.DownloadAsset] + nodeWriter *node_writer.Writer + indexQueue pubsub.Topic[mq.IndexNode] + assetQueue pubsub.Topic[mq.DownloadAsset] + autoFillQueue pubsub.Topic[mq.AutoFillNode] } func New( nodeWriter *node_writer.Writer, indexQueue pubsub.Topic[mq.IndexNode], assetQueue pubsub.Topic[mq.DownloadAsset], + autoFillQueue pubsub.Topic[mq.AutoFillNode], ) *Filler { return &Filler{ - nodeWriter: nodeWriter, - indexQueue: indexQueue, - assetQueue: assetQueue, + nodeWriter: nodeWriter, + indexQueue: indexQueue, + assetQueue: assetQueue, + autoFillQueue: autoFillQueue, } } @@ -71,7 +74,7 @@ func (f *Filler) FillContentFromLink(ctx context.Context, link *link_ref.LinkRef return fault.Wrap(err, fctx.With(ctx)) } - f.assetQueue.Publish(ctx, dt.Map(wc.Content.Media(), func(u string) mq.DownloadAsset { + err = f.assetQueue.Publish(ctx, dt.Map(wc.Content.Media(), func(u string) mq.DownloadAsset { return mq.DownloadAsset{ URL: u, ContentFillRule: opt.New(asset.ContentFillCommand{ @@ -80,17 +83,37 @@ func (f *Filler) FillContentFromLink(ctx context.Context, link *link_ref.LinkRef }), } })...) + if err != nil { + return fault.Wrap(err, fctx.With(ctx)) + } + + err = f.autoFillQueue.Publish(ctx, mq.AutoFillNode{ + ID: library.NodeID(n.Mark.ID()), + SummariseContent: true, + AutoTag: true, + }) + if err != nil { + return fault.Wrap(err, fctx.With(ctx)) + } if vis == visibility.VisibilityPublished { if err := f.indexQueue.Publish(ctx, mq.IndexNode{ - ID: library.NodeID(n.Mark.ID()), - SummariseContent: true, - AutoTag: true, + ID: library.NodeID(n.Mark.ID()), }); err != nil { return fault.Wrap(err, fctx.With(ctx)) } } + case asset.ContentFillRuleReplace: + err = f.autoFillQueue.Publish(ctx, mq.AutoFillNode{ + ID: library.NodeID(library.NodeID(cfr.TargetNodeID)), + SummariseContent: true, + AutoTag: true, + }) + if err != nil { + return fault.Wrap(err, fctx.With(ctx)) + } + default: return fault.Wrap(errFillRuleNotAvailale, fctx.With(ctx)) } diff --git a/app/services/library/node_mutate/create.go b/app/services/library/node_mutate/create.go index e14c55932..200e1e250 100644 --- a/app/services/library/node_mutate/create.go +++ b/app/services/library/node_mutate/create.go @@ -3,22 +3,17 @@ package node_mutate import ( "context" - "github.com/Southclaws/dt" "github.com/Southclaws/fault" "github.com/Southclaws/fault/fctx" "github.com/Southclaws/fault/fmsg" - "github.com/rs/xid" + "github.com/Southclaws/opt" "github.com/Southclaws/storyden/app/resources/account" - "github.com/Southclaws/storyden/app/resources/asset" "github.com/Southclaws/storyden/app/resources/library" - "github.com/Southclaws/storyden/app/resources/library/node_writer" "github.com/Southclaws/storyden/app/resources/mark" "github.com/Southclaws/storyden/app/resources/mq" "github.com/Southclaws/storyden/app/resources/rbac" - "github.com/Southclaws/storyden/app/resources/tag/tag_ref" "github.com/Southclaws/storyden/app/resources/visibility" - "github.com/Southclaws/storyden/app/services/link/fetcher" ) func (s *Manager) Create(ctx context.Context, @@ -42,42 +37,14 @@ func (s *Manager) Create(ctx context.Context, } } - opts, err := s.applyOpts(ctx, p) + pre, err := s.preMutation(ctx, p, opt.NewEmpty[library.Node]()) if err != nil { return nil, fault.Wrap(err, fctx.With(ctx)) } - - if v, ok := p.AssetSources.Get(); ok { - for _, source := range v { - a, err := s.fetcher.CopyAsset(ctx, source) - if err != nil { - return nil, fault.Wrap(err, fctx.With(ctx)) - } - - opts = append(opts, node_writer.WithAssets([]asset.AssetID{a.ID})) - } - } + opts := pre.opts nodeSlug := p.Slug.Or(mark.NewSlugFromName(name)) - if u, ok := p.URL.Get(); ok { - ln, err := s.fetcher.Fetch(ctx, u, fetcher.Options{}) - if err == nil { - opts = append(opts, node_writer.WithLink(xid.ID(ln.ID))) - } - } - - if tags, ok := p.Tags.Get(); ok { - newTags, err := s.tagWriter.Add(ctx, tags...) - if err != nil { - return nil, fault.Wrap(err, fctx.With(ctx)) - } - - tagIDs := dt.Map(newTags, func(t *tag_ref.Tag) tag_ref.ID { return t.ID }) - - opts = append(opts, node_writer.WithTagsAdd(tagIDs...)) - } - n, err := s.nodeWriter.Create(ctx, owner, name, nodeSlug, opts...) if err != nil { return nil, fault.Wrap(err, fctx.With(ctx)) diff --git a/app/services/library/node_mutate/mutate.go b/app/services/library/node_mutate/mutate.go index cd2690b8d..3a992d9b9 100644 --- a/app/services/library/node_mutate/mutate.go +++ b/app/services/library/node_mutate/mutate.go @@ -1,13 +1,9 @@ package node_mutate import ( - "context" "net/url" - "github.com/Southclaws/fault" - "github.com/Southclaws/fault/fctx" "github.com/Southclaws/opt" - "github.com/rs/xid" "go.uber.org/zap" "github.com/Southclaws/storyden/app/resources/account/account_querier" @@ -19,9 +15,11 @@ import ( "github.com/Southclaws/storyden/app/resources/library/node_writer" "github.com/Southclaws/storyden/app/resources/mark" "github.com/Southclaws/storyden/app/resources/mq" + "github.com/Southclaws/storyden/app/resources/tag" "github.com/Southclaws/storyden/app/resources/tag/tag_ref" "github.com/Southclaws/storyden/app/resources/tag/tag_writer" "github.com/Southclaws/storyden/app/resources/visibility" + "github.com/Southclaws/storyden/app/services/generative" "github.com/Southclaws/storyden/app/services/link/fetcher" "github.com/Southclaws/storyden/app/services/tag/autotagger" "github.com/Southclaws/storyden/internal/deletable" @@ -41,26 +39,11 @@ type Partial struct { AssetsAdd opt.Optional[[]asset.AssetID] AssetsRemove opt.Optional[[]asset.AssetID] AssetSources opt.Optional[[]string] + TagFill opt.Optional[tag.TagFillCommand] ContentFill opt.Optional[asset.ContentFillCommand] ContentSummarise opt.Optional[bool] } -func (p Partial) Opts() (opts []node_writer.Option) { - p.Name.Call(func(value string) { opts = append(opts, node_writer.WithName(value)) }) - p.Slug.Call(func(value mark.Slug) { opts = append(opts, node_writer.WithSlug(value.String())) }) - p.PrimaryImage.Call(func(value xid.ID) { - opts = append(opts, node_writer.WithPrimaryImage(value)) - }, func() { - opts = append(opts, node_writer.WithPrimaryImageRemoved()) - }) - p.Content.Call(func(value datagraph.Content) { opts = append(opts, node_writer.WithContent(value)) }) - p.Metadata.Call(func(value map[string]any) { opts = append(opts, node_writer.WithMetadata(value)) }) - p.AssetsAdd.Call(func(value []asset.AssetID) { opts = append(opts, node_writer.WithAssets(value)) }) - p.AssetsRemove.Call(func(value []asset.AssetID) { opts = append(opts, node_writer.WithAssetsRemoved(value)) }) - p.Visibility.Call(func(value visibility.Visibility) { opts = append(opts, node_writer.WithVisibility(value)) }) - return -} - type Manager struct { logger *zap.Logger accountQuery *account_querier.Querier @@ -70,6 +53,7 @@ type Manager struct { tagger *autotagger.Tagger nc node_children.Repository fetcher *fetcher.Fetcher + summariser generative.Summariser indexQueue pubsub.Topic[mq.IndexNode] deleteQueue pubsub.Topic[mq.DeleteNode] assetAnalyseQueue pubsub.Topic[mq.AnalyseAsset] @@ -84,6 +68,7 @@ func New( tagger *autotagger.Tagger, nc node_children.Repository, fetcher *fetcher.Fetcher, + summariser generative.Summariser, indexQueue pubsub.Topic[mq.IndexNode], deleteQueue pubsub.Topic[mq.DeleteNode], assetAnalyseQueue pubsub.Topic[mq.AnalyseAsset], @@ -97,23 +82,9 @@ func New( tagger: tagger, nc: nc, fetcher: fetcher, + summariser: summariser, indexQueue: indexQueue, deleteQueue: deleteQueue, assetAnalyseQueue: assetAnalyseQueue, } } - -func (s *Manager) applyOpts(ctx context.Context, p Partial) ([]node_writer.Option, error) { - opts := p.Opts() - - if parentSlug, ok := p.Parent.Get(); ok { - parent, err := s.nodeQuerier.Get(ctx, parentSlug) - if err != nil { - return nil, fault.Wrap(err, fctx.With(ctx)) - } - - opts = append(opts, node_writer.WithParent(library.NodeID(parent.Mark.ID()))) - } - - return opts, nil -} diff --git a/app/services/library/node_mutate/options.go b/app/services/library/node_mutate/options.go new file mode 100644 index 000000000..c6f70c590 --- /dev/null +++ b/app/services/library/node_mutate/options.go @@ -0,0 +1,233 @@ +package node_mutate + +import ( + "context" + + "github.com/Southclaws/dt" + "github.com/Southclaws/fault" + "github.com/Southclaws/fault/fctx" + "github.com/Southclaws/opt" + "github.com/rs/xid" + "github.com/samber/lo" + + "github.com/Southclaws/storyden/app/resources/asset" + "github.com/Southclaws/storyden/app/resources/datagraph" + "github.com/Southclaws/storyden/app/resources/library" + "github.com/Southclaws/storyden/app/resources/library/node_writer" + "github.com/Southclaws/storyden/app/resources/mark" + "github.com/Southclaws/storyden/app/resources/mq" + "github.com/Southclaws/storyden/app/resources/tag" + "github.com/Southclaws/storyden/app/resources/tag/tag_ref" + "github.com/Southclaws/storyden/app/resources/visibility" + "github.com/Southclaws/storyden/app/services/link/fetcher" +) + +type preMutationResult struct { + opts []node_writer.Option + + // Ideally, this API should only return node writer options, but because of + // a weird public API design choice I made, the PATCH /nodes endpoint also + // returns tag suggestions which can be opted out of being applied directly. + // This may change in future but it would require breaking public API change + // and it works pretty well at the moment as an API design, so not critical. + tags opt.Optional[tag_ref.Names] +} + +// preMutation constructs node_writer options for a create or partial update. +func (s *Manager) preMutation(ctx context.Context, p Partial, current opt.Optional[library.Node]) (*preMutationResult, error) { + opts := []node_writer.Option{} + + // Apply all primitive options. These are just basic partial updates. + p.Name.Call(func(value string) { opts = append(opts, node_writer.WithName(value)) }) + p.Slug.Call(func(value mark.Slug) { opts = append(opts, node_writer.WithSlug(value.String())) }) + p.PrimaryImage.Call(func(value xid.ID) { + opts = append(opts, node_writer.WithPrimaryImage(value)) + }, func() { + opts = append(opts, node_writer.WithPrimaryImageRemoved()) + }) + p.Content.Call(func(value datagraph.Content) { opts = append(opts, node_writer.WithContent(value)) }) + p.Metadata.Call(func(value map[string]any) { opts = append(opts, node_writer.WithMetadata(value)) }) + p.AssetsAdd.Call(func(value []asset.AssetID) { opts = append(opts, node_writer.WithAssets(value)) }) + p.AssetsRemove.Call(func(value []asset.AssetID) { opts = append(opts, node_writer.WithAssetsRemoved(value)) }) + p.Visibility.Call(func(value visibility.Visibility) { opts = append(opts, node_writer.WithVisibility(value)) }) + + // If the mutation includes a parent node, we need to query it because the + // WithParent API only accepts a node ID, not a node mark (slug or ID). + if parentSlug, ok := p.Parent.Get(); ok { + parent, err := s.nodeQuerier.Get(ctx, parentSlug) + if err != nil { + return nil, fault.Wrap(err, fctx.With(ctx)) + } + + opts = append(opts, node_writer.WithParent(library.NodeID(parent.Mark.ID()))) + } + + // If the mutation includes asset sources (so, URLs to assets to be added) + // download them and append them to the node's asset list. + if v, ok := p.AssetSources.Get(); ok { + o, err := s.buildAssetSourcesOpts(ctx, v) + if err != nil { + return nil, fault.Wrap(err, fctx.With(ctx)) + } + opts = append(opts, o...) + } + + // If assets have been added to the node and there's a content fill rule, + // queue the assets for extraction for the node's content etc. + assetsAdd, assetsAddSet := p.AssetsAdd.Get() + if assetsAddSet && p.ContentFill.Ok() { + if err := s.assetAnalyseQueue.Publish(ctx, dt.Map(assetsAdd, func(a asset.AssetID) mq.AnalyseAsset { + return mq.AnalyseAsset{ + AssetID: a, + ContentFillRule: p.ContentFill, + } + })...); err != nil { + return nil, fault.Wrap(err, fctx.With(ctx)) + } + } + + // If there's a URL being applied, fetch its contents. + if u, ok := p.URL.Get(); ok { + ln, err := s.fetcher.Fetch(ctx, u, fetcher.Options{}) + if err == nil { + opts = append(opts, node_writer.WithLink(xid.ID(ln.ID))) + } + } + + // The content to use during pre-mutation tasks such as tag suggestion, auto + // title generation and content summarisation. If it's a new node, this will + // be the content submitted for the new node, if it's an update, either pick + // the new content if specified in the partial, or the current node content. + content := p.Content.Or(current.OrZero().Content.OrZero()) + + var tags opt.Optional[tag_ref.Names] + if tfr, ok := p.TagFill.Get(); ok { + suggested, err := s.buildTagSuggestionOpts(ctx, content, tfr.FillRule) + if err != nil { + return nil, fault.Wrap(err, fctx.With(ctx)) + } + + if tfr.FillRule == tag.TagFillRuleReplace { + tags = opt.New(suggested) + } else { + tags = p.Tags + } + } else { + tags = p.Tags + } + + if t, ok := tags.Get(); ok { + n, ok := current.Get() + if ok { + tagOpts, err := s.createDeleteTagsForExistingNode(ctx, &n, t) + if err != nil { + return nil, fault.Wrap(err, fctx.With(ctx)) + } + opts = append(opts, tagOpts...) + } else { + tagOpts, err := s.createDeleteTagsForNewNode(ctx, t) + if err != nil { + return nil, fault.Wrap(err, fctx.With(ctx)) + } + opts = append(opts, tagOpts...) + } + } + + if p.ContentSummarise.OrZero() { + opt, err := s.buildSummaryOpts(ctx, content) + if err != nil { + return nil, fault.Wrap(err, fctx.With(ctx)) + } + + opts = append(opts, opt) + } + + return &preMutationResult{ + opts: opts, + tags: tags, + }, nil +} + +func (s *Manager) buildAssetSourcesOpts(ctx context.Context, sources []string) ([]node_writer.Option, error) { + opts := []node_writer.Option{} + + for _, source := range sources { + a, err := s.fetcher.CopyAsset(ctx, source) + if err != nil { + return nil, fault.Wrap(err, fctx.With(ctx)) + } + + opts = append(opts, node_writer.WithAssets([]asset.AssetID{a.ID})) + } + + return opts, nil +} + +func (s *Manager) createDeleteTagsForNewNode(ctx context.Context, tags tag_ref.Names) ([]node_writer.Option, error) { + opts := []node_writer.Option{} + + newTags, err := s.tagWriter.Add(ctx, tags...) + if err != nil { + return nil, fault.Wrap(err, fctx.With(ctx)) + } + + addIDs := dt.Map(newTags, func(t *tag_ref.Tag) tag_ref.ID { return t.ID }) + + opts = append(opts, node_writer.WithTagsAdd(addIDs...)) + + return opts, nil +} + +func (s *Manager) createDeleteTagsForExistingNode(ctx context.Context, n *library.Node, tags tag_ref.Names) ([]node_writer.Option, error) { + opts := []node_writer.Option{} + + currentTagNames := n.Tags.Names() + + toCreate, toRemove := lo.Difference(tags, currentTagNames) + + newTags, err := s.tagWriter.Add(ctx, toCreate...) + if err != nil { + return nil, fault.Wrap(err, fctx.With(ctx)) + } + + addIDs := dt.Map(newTags, func(t *tag_ref.Tag) tag_ref.ID { return t.ID }) + removeIDs := dt.Reduce(n.Tags, func(acc []tag_ref.ID, prev *tag_ref.Tag) []tag_ref.ID { + if lo.Contains(toRemove, prev.Name) { + acc = append(acc, prev.ID) + } + return acc + }, []tag_ref.ID{}) + + opts = append(opts, node_writer.WithTagsAdd(addIDs...)) + opts = append(opts, node_writer.WithTagsRemove(removeIDs...)) + + return opts, nil +} + +func (s *Manager) buildTagSuggestionOpts(ctx context.Context, content datagraph.Content, tfr tag.TagFillRule) (tag_ref.Names, error) { + // Only bother if there's any actual content to work with! + if content.IsEmpty() { + return nil, nil + } + + gathered, err := s.tagger.Gather(ctx, tfr, content) + if err != nil { + return nil, fault.Wrap(err, fctx.With(ctx)) + } + + return gathered, nil +} + +func (s *Manager) buildSummaryOpts(ctx context.Context, content datagraph.Content) (node_writer.Option, error) { + summary, err := s.summariser.Summarise(ctx, content) + if err != nil { + return nil, fault.Wrap(err, fctx.With(ctx)) + } + + newContent, err := datagraph.NewRichText(summary) + if err != nil { + return nil, fault.Wrap(err, fctx.With(ctx)) + } + + return node_writer.WithContent(newContent), nil +} diff --git a/app/services/library/node_mutate/update.go b/app/services/library/node_mutate/update.go index d2c4812e7..674085491 100644 --- a/app/services/library/node_mutate/update.go +++ b/app/services/library/node_mutate/update.go @@ -3,49 +3,25 @@ package node_mutate import ( "context" - "github.com/Southclaws/dt" "github.com/Southclaws/fault" "github.com/Southclaws/fault/fctx" "github.com/Southclaws/opt" - "github.com/rs/xid" - "github.com/samber/lo" "go.uber.org/zap" - "github.com/Southclaws/storyden/app/resources/asset" "github.com/Southclaws/storyden/app/resources/library" - "github.com/Southclaws/storyden/app/resources/library/node_writer" "github.com/Southclaws/storyden/app/resources/mq" - "github.com/Southclaws/storyden/app/resources/tag" "github.com/Southclaws/storyden/app/resources/tag/tag_ref" "github.com/Southclaws/storyden/app/resources/visibility" "github.com/Southclaws/storyden/app/services/authentication/session" "github.com/Southclaws/storyden/app/services/library/node_auth" - "github.com/Southclaws/storyden/app/services/link/fetcher" ) -type Option func(*updateOptions) - -type updateOptions struct { - tagFillRule opt.Optional[tag.TagFillRule] -} - -func WithTagFillRule(fr tag.TagFillRule) Option { - return func(uo *updateOptions) { - uo.tagFillRule = opt.New(fr) - } -} - type Updated struct { library.Node TagSuggestions opt.Optional[tag_ref.Names] } -func (s *Manager) Update(ctx context.Context, qk library.QueryKey, p Partial, options ...Option) (*Updated, error) { - updateOpts := updateOptions{} - for _, fn := range options { - fn(&updateOpts) - } - +func (s *Manager) Update(ctx context.Context, qk library.QueryKey, p Partial) (*Updated, error) { accountID, err := session.GetAccountID(ctx) if err != nil { return nil, fault.Wrap(err, fctx.With(ctx)) @@ -65,96 +41,12 @@ func (s *Manager) Update(ctx context.Context, qk library.QueryKey, p Partial, op return nil, fault.Wrap(err, fctx.With(ctx)) } - opts, err := s.applyOpts(ctx, p) + pre, err := s.preMutation(ctx, p, opt.NewPtr(n)) if err != nil { return nil, fault.Wrap(err, fctx.With(ctx)) } - // TODO: Queue this for background processing - if v, ok := p.AssetSources.Get(); ok { - for _, source := range v { - a, err := s.fetcher.CopyAsset(ctx, source) - if err != nil { - return nil, fault.Wrap(err, fctx.With(ctx)) - } - - opts = append(opts, node_writer.WithAssets([]asset.AssetID{a.ID})) - } - } - - assetsAdd, assetsAddSet := p.AssetsAdd.Get() - if assetsAddSet && p.ContentFill.Ok() { - - messages := dt.Map(assetsAdd, func(a asset.AssetID) mq.AnalyseAsset { - return mq.AnalyseAsset{ - AssetID: a, - ContentFillRule: p.ContentFill, - } - }) - - if err := s.assetAnalyseQueue.Publish(ctx, messages...); err != nil { - return nil, fault.Wrap(err, fctx.With(ctx)) - } - } - - if u, ok := p.URL.Get(); ok { - ln, err := s.fetcher.Fetch(ctx, u, fetcher.Options{}) - if err == nil { - opts = append(opts, node_writer.WithLink(xid.ID(ln.ID))) - } - } - - suggestedTags := opt.NewEmpty[tag_ref.Names]() - if tfr, ok := updateOpts.tagFillRule.Get(); ok { - // If the update query contains new content, use that, otherwise, fall - // back to the current content in the node. - content := p.Content.Or(n.Content.OrZero()) - - // Only bother if there's any actual content to work with! - if !content.IsEmpty() { - gathered, err := s.tagger.Gather(ctx, tfr, content) - if err != nil { - return nil, fault.Wrap(err, fctx.With(ctx)) - } - - switch tfr { - case tag.TagFillRuleQuery: - suggestedTags = opt.New(gathered) - - case tag.TagFillRuleReplace: - if t, ok := p.Tags.Get(); ok { - p.Tags = opt.New(append(t, gathered...)) - } else { - p.Tags = opt.New(gathered) - } - default: - } - } - } - - if tags, ok := p.Tags.Get(); ok { - currentTagNames := n.Tags.Names() - - toCreate, toRemove := lo.Difference(tags, currentTagNames) - - newTags, err := s.tagWriter.Add(ctx, toCreate...) - if err != nil { - return nil, fault.Wrap(err, fctx.With(ctx)) - } - - addIDs := dt.Map(newTags, func(t *tag_ref.Tag) tag_ref.ID { return t.ID }) - removeIDs := dt.Reduce(n.Tags, func(acc []tag_ref.ID, prev *tag_ref.Tag) []tag_ref.ID { - if lo.Contains(toRemove, prev.Name) { - acc = append(acc, prev.ID) - } - return acc - }, []tag_ref.ID{}) - - opts = append(opts, node_writer.WithTagsAdd(addIDs...)) - opts = append(opts, node_writer.WithTagsRemove(removeIDs...)) - } - - n, err = s.nodeWriter.Update(ctx, qk, opts...) + n, err = s.nodeWriter.Update(ctx, qk, pre.opts...) if err != nil { return nil, fault.Wrap(err, fctx.With(ctx)) } @@ -163,13 +55,14 @@ func (s *Manager) Update(ctx context.Context, qk library.QueryKey, p Partial, op if err := s.indexQueue.Publish(ctx, mq.IndexNode{ ID: library.NodeID(n.Mark.ID()), }); err != nil { - return nil, fault.Wrap(err, fctx.With(ctx)) + s.logger.Error("failed to publish index post message", zap.Error(err)) } } else { if err := s.deleteQueue.Publish(ctx, mq.DeleteNode{ ID: library.NodeID(n.GetID()), }); err != nil { - s.logger.Error("failed to publish index post message", zap.Error(err)) + // failing to publish the deletion message is worthy of an error. + return nil, fault.Wrap(err, fctx.With(ctx)) } } @@ -177,7 +70,7 @@ func (s *Manager) Update(ctx context.Context, qk library.QueryKey, p Partial, op u := Updated{ Node: *n, - TagSuggestions: suggestedTags, + TagSuggestions: pre.tags, } return &u, nil diff --git a/app/services/library/node_semdex/autofill.go b/app/services/library/node_semdex/autofill.go new file mode 100644 index 000000000..1f8796b70 --- /dev/null +++ b/app/services/library/node_semdex/autofill.go @@ -0,0 +1,35 @@ +package node_semdex + +import ( + "context" + + "github.com/Southclaws/fault" + "github.com/Southclaws/fault/fctx" + "github.com/Southclaws/opt" + "github.com/rs/xid" + + "github.com/Southclaws/storyden/app/resources/library" + "github.com/Southclaws/storyden/app/resources/tag" + "github.com/Southclaws/storyden/app/services/library/node_mutate" +) + +func (i *semdexer) autofill(ctx context.Context, id library.NodeID, summarise bool, autotag bool) error { + qk := library.NewID(xid.ID(id)) + + p := node_mutate.Partial{} + + if summarise { + p.ContentSummarise = opt.New(true) + } + + if autotag { + p.TagFill = opt.New(tag.TagFillCommand{FillRule: tag.TagFillRuleReplace}) + } + + _, err := i.nodeUpdater.Update(ctx, qk, p) + if err != nil { + return fault.Wrap(err, fctx.With(ctx)) + } + + return nil +} diff --git a/app/services/library/node_semdex/indexer.go b/app/services/library/node_semdex/indexer.go index e96552ce7..c78a3360a 100644 --- a/app/services/library/node_semdex/indexer.go +++ b/app/services/library/node_semdex/indexer.go @@ -3,21 +3,15 @@ package node_semdex import ( "context" - "github.com/Southclaws/dt" "github.com/Southclaws/fault" "github.com/Southclaws/fault/fctx" "github.com/rs/xid" - "github.com/samber/lo" - "go.uber.org/zap" - "github.com/Southclaws/storyden/app/resources/datagraph" "github.com/Southclaws/storyden/app/resources/library" "github.com/Southclaws/storyden/app/resources/library/node_writer" - "github.com/Southclaws/storyden/app/resources/tag" - "github.com/Southclaws/storyden/app/resources/tag/tag_ref" ) -func (i *semdexer) index(ctx context.Context, id library.NodeID, summarise bool, autotag bool) error { +func (i *semdexer) index(ctx context.Context, id library.NodeID) error { qk := library.NewID(xid.ID(id)) node, err := i.nodeQuerier.Get(ctx, qk) @@ -30,33 +24,7 @@ func (i *semdexer) index(ctx context.Context, id library.NodeID, summarise bool, return fault.Wrap(err, fctx.With(ctx)) } - opts := []node_writer.Option{ - node_writer.WithIndexed(), - } - - content := node.GetContent() - - if summarise { - summarisedContent, err := i.getSummary(ctx, node) - if err != nil { - i.logger.Warn("failed to summarise node", zap.Error(err), zap.String("node_id", node.GetID().String())) - } else { - opts = append(opts, node_writer.WithContent(*summarisedContent)) - } - - content = *summarisedContent - } - - if autotag { - tagOpts, err := i.generateTags(ctx, node, content) - if err != nil { - i.logger.Warn("failed to autotag node", zap.Error(err), zap.String("node_id", node.GetID().String())) - } - - opts = append(opts, tagOpts...) - } - - _, err = i.nodeWriter.Update(ctx, qk, opts...) + _, err = i.nodeWriter.Update(ctx, qk, node_writer.WithIndexed()) if err != nil { return fault.Wrap(err, fctx.With(ctx)) } @@ -64,20 +32,6 @@ func (i *semdexer) index(ctx context.Context, id library.NodeID, summarise bool, return nil } -func (i *semdexer) getSummary(ctx context.Context, p datagraph.Item) (*datagraph.Content, error) { - summary, err := i.summariser.Summarise(ctx, p) - if err != nil { - return nil, fault.Wrap(err, fctx.With(ctx)) - } - - content, err := datagraph.NewRichText(summary) - if err != nil { - return nil, fault.Wrap(err, fctx.With(ctx)) - } - - return &content, nil -} - func (i *semdexer) deindex(ctx context.Context, id library.NodeID) error { qk := library.NewID(xid.ID(id)) @@ -93,41 +47,3 @@ func (i *semdexer) deindex(ctx context.Context, id library.NodeID) error { return nil } - -func (i *semdexer) generateTags(ctx context.Context, n *library.Node, content datagraph.Content) ([]node_writer.Option, error) { - gathered, err := i.tagger.Gather(ctx, tag.TagFillRuleReplace, content) - if err != nil { - return nil, fault.Wrap(err, fctx.With(ctx)) - } - - tagOpts, err := i.applyTags(ctx, n, gathered) - if err != nil { - return nil, fault.Wrap(err, fctx.With(ctx)) - } - - return tagOpts, nil -} - -func (i *semdexer) applyTags(ctx context.Context, n *library.Node, tags []tag_ref.Name) (opts []node_writer.Option, err error) { - currentTagNames := n.Tags.Names() - - toCreate, toRemove := lo.Difference(tags, currentTagNames) - - newTags, err := i.tagWriter.Add(ctx, toCreate...) - if err != nil { - return nil, fault.Wrap(err, fctx.With(ctx)) - } - - addIDs := dt.Map(newTags, func(t *tag_ref.Tag) tag_ref.ID { return t.ID }) - removeIDs := dt.Reduce(n.Tags, func(acc []tag_ref.ID, prev *tag_ref.Tag) []tag_ref.ID { - if lo.Contains(toRemove, prev.Name) { - acc = append(acc, prev.ID) - } - return acc - }, []tag_ref.ID{}) - - opts = append(opts, node_writer.WithTagsAdd(addIDs...)) - opts = append(opts, node_writer.WithTagsRemove(removeIDs...)) - - return opts, nil -} diff --git a/app/services/library/node_semdex/node_semdex.go b/app/services/library/node_semdex/node_semdex.go index 34a314616..b83469f6c 100644 --- a/app/services/library/node_semdex/node_semdex.go +++ b/app/services/library/node_semdex/node_semdex.go @@ -11,7 +11,8 @@ import ( "github.com/Southclaws/storyden/app/resources/library/node_writer" "github.com/Southclaws/storyden/app/resources/mq" "github.com/Southclaws/storyden/app/resources/tag/tag_writer" - "github.com/Southclaws/storyden/app/services/generative" + "github.com/Southclaws/storyden/app/services/authentication/session" + "github.com/Southclaws/storyden/app/services/library/node_mutate" "github.com/Southclaws/storyden/app/services/semdex" "github.com/Southclaws/storyden/app/services/tag/autotagger" "github.com/Southclaws/storyden/internal/config" @@ -25,6 +26,7 @@ func Build() fx.Option { fx.Provide( queue.New[mq.IndexNode], queue.New[mq.DeleteNode], + queue.New[mq.AutoFillNode], ), fx.Invoke(newSemdexer), ) @@ -40,17 +42,22 @@ var ( ) type semdexer struct { - logger *zap.Logger - db *ent.Client - nodeQuerier *node_querier.Querier - nodeWriter *node_writer.Writer + logger *zap.Logger + db *ent.Client + + nodeQuerier *node_querier.Querier + nodeWriter *node_writer.Writer + nodeUpdater *node_mutate.Manager + indexQueue pubsub.Topic[mq.IndexNode] deleteQueue pubsub.Topic[mq.DeleteNode] + autoFillQueue pubsub.Topic[mq.AutoFillNode] + semdexMutator semdex.Mutator semdexQuerier semdex.Querier - summariser generative.Summariser - tagger *autotagger.Tagger - tagWriter *tag_writer.Writer + + tagger *autotagger.Tagger + tagWriter *tag_writer.Writer } func newSemdexer( @@ -62,11 +69,13 @@ func newSemdexer( db *ent.Client, nodeQuerier *node_querier.Querier, nodeWriter *node_writer.Writer, + nodeUpdater *node_mutate.Manager, indexQueue pubsub.Topic[mq.IndexNode], deleteQueue pubsub.Topic[mq.DeleteNode], + autoFillQueue pubsub.Topic[mq.AutoFillNode], semdexMutator semdex.Mutator, semdexQuerier semdex.Querier, - summariser generative.Summariser, + tagger *autotagger.Tagger, tagWriter *tag_writer.Writer, ) { @@ -79,13 +88,14 @@ func newSemdexer( db: db, nodeQuerier: nodeQuerier, nodeWriter: nodeWriter, + nodeUpdater: nodeUpdater, indexQueue: indexQueue, deleteQueue: deleteQueue, semdexMutator: semdexMutator, semdexQuerier: semdexQuerier, - summariser: summariser, - tagger: tagger, - tagWriter: tagWriter, + + tagger: tagger, + tagWriter: tagWriter, } lc.Append(fx.StartHook(func(hctx context.Context) error { @@ -107,7 +117,7 @@ func newSemdexer( go func() { for msg := range sub { - if err := re.index(ctx, msg.Payload.ID, msg.Payload.SummariseContent, msg.Payload.AutoTag); err != nil { + if err := re.index(ctx, msg.Payload.ID); err != nil { l.Error("failed to index node", zap.Error(err)) } @@ -136,4 +146,25 @@ func newSemdexer( return nil })) + + lc.Append(fx.StartHook(func(_ context.Context) error { + sub, err := autoFillQueue.Subscribe(ctx) + if err != nil { + return err + } + + go func() { + for msg := range sub { + ctx = session.GetSessionFromMessage(ctx, msg) + + if err := re.autofill(ctx, msg.Payload.ID, msg.Payload.SummariseContent, msg.Payload.AutoTag); err != nil { + l.Error("failed to autofill node", zap.Error(err)) + } + + msg.Ack() + } + }() + + return nil + })) } diff --git a/app/transports/http/bindings/nodes.go b/app/transports/http/bindings/nodes.go index 770a82ae1..db4f2c360 100644 --- a/app/transports/http/bindings/nodes.go +++ b/app/transports/http/bindings/nodes.go @@ -18,6 +18,7 @@ import ( "github.com/Southclaws/storyden/app/resources/library" "github.com/Southclaws/storyden/app/resources/library/node_traversal" "github.com/Southclaws/storyden/app/resources/mark" + "github.com/Southclaws/storyden/app/resources/tag" "github.com/Southclaws/storyden/app/resources/tag/tag_ref" "github.com/Southclaws/storyden/app/resources/visibility" "github.com/Southclaws/storyden/app/services/authentication/session" @@ -197,7 +198,7 @@ func (c *Nodes) NodeGet(ctx context.Context, request openapi.NodeGetRequestObjec } func (c *Nodes) NodeUpdate(ctx context.Context, request openapi.NodeUpdateRequestObject) (openapi.NodeUpdateResponseObject, error) { - richContent, err := opt.MapErr(opt.NewPtr(request.Body.Content), datagraph.NewRichText) + content, err := opt.MapErr(opt.NewPtr(request.Body.Content), datagraph.NewRichText) if err != nil { return nil, fault.Wrap(err, fctx.With(ctx), ftag.With(ftag.InvalidArgument)) } @@ -213,40 +214,40 @@ func (c *Nodes) NodeUpdate(ctx context.Context, request openapi.NodeUpdateReques return nil, fault.Wrap(err, fctx.With(ctx), ftag.With(ftag.InvalidArgument)) } - tags := opt.Map(opt.NewPtr(request.Body.Tags), func(tags []string) tag_ref.Names { - return dt.Map(tags, deserialiseTagName) - }) - slug, err := deserialiseInputSlug(request.Body.Slug) if err != nil { return nil, fault.Wrap(err, fctx.With(ctx)) } - primaryImage := deletable.NewMap(request.Body.PrimaryImageAssetId, deserialiseAssetID) - - opts := []node_mutate.Option{} - tagFillRuleParam, err := opt.MapErr(opt.NewPtr(request.Params.TagFillRule), deserialiseTagFillRule) if err != nil { return nil, fault.Wrap(err, fctx.With(ctx)) } - if tfr, ok := tagFillRuleParam.Get(); ok { - opts = append(opts, node_mutate.WithTagFillRule(tfr)) - } + tags := opt.Map(opt.NewPtr(request.Body.Tags), func(tags []string) tag_ref.Names { + return dt.Map(tags, deserialiseTagName) + }) - node, err := c.nodeMutator.Update(ctx, deserialiseNodeMark(request.NodeSlug), node_mutate.Partial{ + primaryImage := deletable.NewMap(request.Body.PrimaryImageAssetId, deserialiseAssetID) + + partial := node_mutate.Partial{ Name: opt.NewPtr(request.Body.Name), Slug: slug, AssetsAdd: opt.NewPtrMap(request.Body.AssetIds, deserialiseAssetIDs), AssetSources: opt.NewPtrMap(request.Body.AssetSources, deserialiseAssetSources), URL: url, - Content: richContent, + Content: content, PrimaryImage: primaryImage, Parent: opt.NewPtrMap(request.Body.Parent, deserialiseNodeMark), Tags: tags, Metadata: opt.NewPtr((*map[string]any)(request.Body.Meta)), - }, opts...) + } + + if tfr, ok := tagFillRuleParam.Get(); ok { + partial.TagFill = opt.New(tag.TagFillCommand{FillRule: tfr}) + } + + node, err := c.nodeMutator.Update(ctx, deserialiseNodeMark(request.NodeSlug), partial) if err != nil { return nil, fault.Wrap(err, fctx.With(ctx)) } diff --git a/web/src/screens/library/LibraryPageScreen/useLibraryPageScreen.ts b/web/src/screens/library/LibraryPageScreen/useLibraryPageScreen.ts index 4a0eb0c6a..1d2373884 100644 --- a/web/src/screens/library/LibraryPageScreen/useLibraryPageScreen.ts +++ b/web/src/screens/library/LibraryPageScreen/useLibraryPageScreen.ts @@ -19,6 +19,7 @@ import { useSession } from "src/auth"; import { handle } from "@/api/client"; import { assetUpload } from "@/api/openapi-client/assets"; +import { linkCreate } from "@/api/openapi-client/links"; import { useLibraryMutation } from "@/lib/library/library"; import { CoverImage, @@ -223,13 +224,12 @@ export function useLibraryPageScreen({ node }: Props) { form.setValue("content", link.description); - await updateNode( - node.slug, + await linkCreate( + { url: link.url }, { - content: link.description, - primary_image_asset_id: link.primary_image?.id, + content_fill_rule: "replace", + node_content_fill_target: node.id, }, - coverConfig, ); }, { From 98549671433b91da15d570fbc0048e38a52f42fd Mon Sep 17 00:00:00 2001 From: barney Date: Sun, 15 Dec 2024 15:08:17 -0500 Subject: [PATCH 2/3] add a mock embedding provider --- internal/infrastructure/ai/ai.go | 8 ++++ internal/infrastructure/ai/mock.go | 56 ++++++++++++++++++++++++++++ internal/infrastructure/ai/openai.go | 13 +++++-- 3 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 internal/infrastructure/ai/mock.go diff --git a/internal/infrastructure/ai/ai.go b/internal/infrastructure/ai/ai.go index 05f8635f3..7b041ae6a 100644 --- a/internal/infrastructure/ai/ai.go +++ b/internal/infrastructure/ai/ai.go @@ -12,6 +12,7 @@ type Result struct { type Prompter interface { Prompt(ctx context.Context, input string) (*Result, error) + EmbeddingFunc() func(ctx context.Context, text string) ([]float32, error) } func New(cfg config.Config) (Prompter, error) { @@ -19,6 +20,9 @@ func New(cfg config.Config) (Prompter, error) { case "openai": return newOpenAI(cfg) + case "mock": + return newMock() + default: return &Disabled{}, nil } @@ -29,3 +33,7 @@ type Disabled struct{} func (d *Disabled) Prompt(ctx context.Context, input string) (*Result, error) { return nil, nil } + +func (d *Disabled) EmbeddingFunc() func(ctx context.Context, text string) ([]float32, error) { + return nil +} diff --git a/internal/infrastructure/ai/mock.go b/internal/infrastructure/ai/mock.go new file mode 100644 index 000000000..f9659398d --- /dev/null +++ b/internal/infrastructure/ai/mock.go @@ -0,0 +1,56 @@ +package ai + +import ( + "context" + "math" +) + +type Mock struct{} + +func newMock() (*Mock, error) { + return &Mock{}, nil +} + +func (o *Mock) Prompt(ctx context.Context, input string) (*Result, error) { + var output string + if len(input) < 120 { + output = input + } else { + output = input[:116] + "..." + } + return &Result{ + Answer: "An answer for " + output, + }, nil +} + +const mockEmbeddingSize = 3072 + +func (o *Mock) EmbeddingFunc() func(ctx context.Context, text string) ([]float32, error) { + return func(ctx context.Context, text string) ([]float32, error) { + embedding := make([]float32, mockEmbeddingSize) + + for _, v := range text { + for i := range mockEmbeddingSize { + c := ((float32(v % 256)) / 256) * float32(((i+1)*3071)%65535) + embedding[i] = c + } + } + + return normalizeVector(embedding), nil + } +} + +func normalizeVector(vec []float32) []float32 { + var n float32 + for _, v := range vec { + n += v * v + } + n = float32(math.Sqrt(float64(n))) + + r := make([]float32, len(vec)) + for i, v := range vec { + r[i] = v / n + } + + return r +} diff --git a/internal/infrastructure/ai/openai.go b/internal/infrastructure/ai/openai.go index 42109fd44..7ee83daac 100644 --- a/internal/infrastructure/ai/openai.go +++ b/internal/infrastructure/ai/openai.go @@ -3,21 +3,24 @@ package ai import ( "context" + "github.com/Southclaws/fault" + "github.com/Southclaws/fault/fctx" "github.com/openai/openai-go" "github.com/openai/openai-go/option" + "github.com/philippgille/chromem-go" - "github.com/Southclaws/fault" - "github.com/Southclaws/fault/fctx" "github.com/Southclaws/storyden/internal/config" ) type OpenAI struct { client *openai.Client + ef func(ctx context.Context, text string) ([]float32, error) } func newOpenAI(cfg config.Config) (*OpenAI, error) { client := openai.NewClient(option.WithAPIKey(cfg.OpenAIKey)) - return &OpenAI{client: client}, nil + ef := chromem.NewEmbeddingFuncOpenAI(cfg.OpenAIKey, chromem.EmbeddingModelOpenAI3Large) + return &OpenAI{client: client, ef: ef}, nil } func (o *OpenAI) Prompt(ctx context.Context, input string) (*Result, error) { @@ -39,3 +42,7 @@ func (o *OpenAI) Prompt(ctx context.Context, input string) (*Result, error) { Answer: res.Choices[0].Message.Content, }, nil } + +func (o *OpenAI) EmbeddingFunc() func(ctx context.Context, text string) ([]float32, error) { + return o.ef +} From 9e38766681378edf4411148dda92cc96362ea8ea Mon Sep 17 00:00:00 2001 From: barney Date: Sun, 15 Dec 2024 15:08:40 -0500 Subject: [PATCH 3/3] use custom embedding provider in chromem --- app/services/semdex/semdexer/chromem_semdexer/chromem.go | 9 +++++---- app/services/semdex/semdexer/semdexer.go | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/app/services/semdex/semdexer/chromem_semdexer/chromem.go b/app/services/semdex/semdexer/chromem_semdexer/chromem.go index c1efe74ae..291487032 100644 --- a/app/services/semdex/semdexer/chromem_semdexer/chromem.go +++ b/app/services/semdex/semdexer/chromem_semdexer/chromem.go @@ -17,6 +17,7 @@ import ( "github.com/Southclaws/storyden/app/services/search/searcher" "github.com/Southclaws/storyden/app/services/semdex" "github.com/Southclaws/storyden/internal/config" + "github.com/Southclaws/storyden/internal/infrastructure/ai" ) type chromemRefIndex struct { @@ -25,17 +26,17 @@ type chromemRefIndex struct { hydrator *hydrate.Hydrator } -func New(cfg config.Config, rh *hydrate.Hydrator) (semdex.Semdexer, error) { +func New(cfg config.Config, rh *hydrate.Hydrator, aip ai.Prompter) (semdex.Semdexer, error) { db, err := chromem.NewPersistentDB(cfg.SemdexLocalPath, false) if err != nil { return nil, err } - if cfg.OpenAIKey == "" { - return nil, fault.New("OpenAI API key is required for embedded semdexer") + if _, ok := aip.(*ai.Disabled); ok { + return nil, fault.New("a language model provider must be enabled for the embedded semdexer to be enabled") } - ef := chromem.NewEmbeddingFuncOpenAI(cfg.OpenAIKey, chromem.EmbeddingModelOpenAI3Large) + ef := aip.EmbeddingFunc() collection, err := db.GetOrCreateCollection("semdex", nil, ef) if err != nil { diff --git a/app/services/semdex/semdexer/semdexer.go b/app/services/semdex/semdexer/semdexer.go index 16547dd5d..675912f48 100644 --- a/app/services/semdex/semdexer/semdexer.go +++ b/app/services/semdex/semdexer/semdexer.go @@ -10,6 +10,7 @@ import ( "github.com/Southclaws/storyden/app/services/semdex/semdexer/chromem_semdexer" "github.com/Southclaws/storyden/app/services/semdex/semdexer/weaviate_semdexer" "github.com/Southclaws/storyden/internal/config" + "github.com/Southclaws/storyden/internal/infrastructure/ai" weaviate_infra "github.com/Southclaws/storyden/internal/infrastructure/weaviate" ) @@ -19,6 +20,7 @@ func newSemdexer( weaviateClassName weaviate_infra.WeaviateClassName, hydrator *hydrate.Hydrator, + prompter ai.Prompter, ) (semdex.Semdexer, error) { if cfg.SemdexProvider != "" && cfg.LanguageModelProvider == "" { return nil, fault.New("semdex requires a language model provider to be enabled") @@ -26,11 +28,9 @@ func newSemdexer( switch cfg.SemdexProvider { case "chromem": - - return chromem_semdexer.New(cfg, hydrator) + return chromem_semdexer.New(cfg, hydrator, prompter) case "weaviate": - return weaviate_semdexer.New(wc, weaviateClassName, hydrator), nil default: