From c6ade8990231ef46dafe7e634e9aeea958dd36d5 Mon Sep 17 00:00:00 2001 From: Rick Hennigan Date: Tue, 1 Oct 2024 10:40:19 -0400 Subject: [PATCH 01/10] Bugfix: Ensure history menu is queued to avoid dynamic timeouts --- Source/Chatbook/ChatModes/UI.wl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Source/Chatbook/ChatModes/UI.wl b/Source/Chatbook/ChatModes/UI.wl index c2c74433..817bfaf7 100644 --- a/Source/Chatbook/ChatModes/UI.wl +++ b/Source/Chatbook/ChatModes/UI.wl @@ -858,7 +858,11 @@ createHistoryMenu[ nbo_NotebookObject ] := Enclose[ appName = ConfirmBy[ CurrentChatSettings[ nbo, "AppName" ], StringQ, "AppName" ]; chats = ConfirmMatch[ ListSavedChats @ appName, { ___Association }, "Chats" ]; If[ chats === { }, Throw @ ActionMenu[ "History", { "Nothing here yet" :> Null } ] ]; - ActionMenu[ "History", makeHistoryMenuItem[ nbo ] /@ Take[ chats, UpTo @ $maxHistoryItems ] ] + ActionMenu[ + "History", + makeHistoryMenuItem[ nbo ] /@ Take[ chats, UpTo @ $maxHistoryItems ], + Method -> "Queued" + ] ], throwInternalFailure ]; From ada1b0a424a71c1bb17f6cfe3bfb3d1da9dbadb7 Mon Sep 17 00:00:00 2001 From: Rick Hennigan Date: Tue, 1 Oct 2024 10:40:58 -0400 Subject: [PATCH 02/10] Specify model for code assistance separately from global preferences --- Source/Chatbook/ChatModes/ShowCodeAssistance.wl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Source/Chatbook/ChatModes/ShowCodeAssistance.wl b/Source/Chatbook/ChatModes/ShowCodeAssistance.wl index e2819936..a1f3fb1f 100644 --- a/Source/Chatbook/ChatModes/ShowCodeAssistance.wl +++ b/Source/Chatbook/ChatModes/ShowCodeAssistance.wl @@ -14,6 +14,8 @@ $workspaceChatWidth = 325; $codeAssistanceBaseSettings = <| "AppName" -> "CodeAssistance", + "Authentication" -> Automatic, (* TODO *) + "Model" -> <| "Service" -> "OpenAI", "Name" -> "gpt-4o" |>, "PromptGenerators" -> { "RelatedDocumentation" }, "ServiceCaller" -> "CodeAssistance", "ToolOptions" -> <| "WolframLanguageEvaluator" -> <| "AppendURIPrompt" -> True, "Method" -> "Session" |> |>, From d9453dfc952e8b4ac6ae6375b1268f02d0d470e1 Mon Sep 17 00:00:00 2001 From: Rick Hennigan Date: Tue, 1 Oct 2024 10:42:26 -0400 Subject: [PATCH 03/10] Downsize user image and use locally defined fallback images --- Source/Chatbook/ChatModes/UI.wl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Source/Chatbook/ChatModes/UI.wl b/Source/Chatbook/ChatModes/UI.wl index 817bfaf7..01dabfd9 100644 --- a/Source/Chatbook/ChatModes/UI.wl +++ b/Source/Chatbook/ChatModes/UI.wl @@ -30,7 +30,7 @@ $inputFieldFrameOptions = Sequence[ FrameStyle -> Directive[ AbsoluteThickness[ 2 ], RGBColor[ "#a3c9f2" ] ] ]; -$userImageParams = <| "size" -> 50, "default" -> "identicon", "rating" -> "G" |>; +$userImageParams = <| "size" -> 40, "default" -> "404", "rating" -> "G" |>; $defaultUserImage = Graphics[ { @@ -41,7 +41,7 @@ $defaultUserImage = Graphics[ Disk[ { 0, 1 }, 1 ], Disk[ { 0, -1.8 }, { 1.65, 2 } ] }, - ImageSize -> 25, + ImageSize -> 20, PlotRange -> { { -2.4, 2.4 }, { -2.0, 2.8 } } ]; @@ -51,7 +51,7 @@ $inputFieldBox = None; $inlineChatScrollPosition = 0.0; $lastScrollPosition = 0.0; -$maxHistoryItems = 50; +$maxHistoryItems = 25; (* ::**************************************************************************************************************:: *) (* ::Section::Closed:: *) @@ -717,12 +717,12 @@ userImage[ ] := userImage[ $CloudUserID ]; userImage[ user_String ] := Enclose[ Module[ { hash, url, image }, - hash = Hash[ ToLowerCase @ StringTrim @ user, "MD5", "HexString" ]; - url = ConfirmBy[ URLBuild[ { "https://www.gravatar.com/avatar/", hash }, $userImageParams ], StringQ, "URL" ]; - image = ConfirmBy[ Import @ url, ImageQ, "Image" ]; - userImage[ user ] = Show[ image, ImageSize -> 25 ] + hash = Hash[ ToLowerCase @ StringTrim @ user, "MD5", "HexString" ]; + url = ConfirmBy[ URLBuild[ { "https://www.gravatar.com/avatar/", hash }, $userImageParams ], StringQ, "URL" ]; + image = ConfirmBy[ Quiet @ Import @ url, ImageQ, "Image" ]; + userImage[ user ] = Show[ image, ImageSize -> 20 ] ], - $defaultUserImage & + (userImage[ user ] = $defaultUserImage) & ]; userImage[ other_ ] := From b80e7e93e19a79fb82a1616529de23e17a15a1f9 Mon Sep 17 00:00:00 2001 From: Rick Hennigan Date: Tue, 1 Oct 2024 10:51:45 -0400 Subject: [PATCH 04/10] Ensure resource versions are always explicitly defined --- Source/Chatbook/Common.wl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Source/Chatbook/Common.wl b/Source/Chatbook/Common.wl index b8b810da..cbd8e833 100644 --- a/Source/Chatbook/Common.wl +++ b/Source/Chatbook/Common.wl @@ -127,7 +127,9 @@ $resourceVersions = <| "ClickToCopy" -> "1.0.0", "GPTTokenizer" -> "1.1.0", "MessageFailure" -> "1.0.0", - "ReplaceContext" -> "1.0.0" + "RelativeTimeString" -> "1.0.0", + "ReplaceContext" -> "1.0.0", + "SelectByCurrentValue" -> "1.0.1" |>; (* ::**************************************************************************************************************:: *) @@ -541,7 +543,7 @@ importResourceFunction::failure = "[ERROR] Failed to import resource function `1 importResourceFunction // Attributes = { HoldFirst }; importResourceFunction[ symbol_Symbol, name_String ] := - importResourceFunction[ symbol, name, Lookup[ $resourceVersions, name, "Latest" ] ]; + importResourceFunction[ symbol, name, Lookup[ $resourceVersions, name ] ]; importResourceFunction[ symbol_Symbol, name_String, version_String ] /; $mxFlag := Enclose[ Block[ { PrintTemporary }, From a19f046b142bb1cf68e976cafd58ae5863bd4f4d Mon Sep 17 00:00:00 2001 From: Rick Hennigan Date: Tue, 1 Oct 2024 10:54:55 -0400 Subject: [PATCH 05/10] Define default app name in Common.wl and resolve in settings --- Source/Chatbook/Common.wl | 1 + Source/Chatbook/CommonSymbols.wl | 1 + Source/Chatbook/Settings.wl | 9 +++++---- Source/Chatbook/Storage.wl | 1 - 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Source/Chatbook/Common.wl b/Source/Chatbook/Common.wl index cbd8e833..aa0cbcef 100644 --- a/Source/Chatbook/Common.wl +++ b/Source/Chatbook/Common.wl @@ -93,6 +93,7 @@ Needs[ "Wolfram`Chatbook`" ]; $cloudNotebooks := TrueQ @ CloudSystem`$CloudNotebooks; $chatIndicatorSymbol = "\|01f4ac"; +$defaultAppName = "Default"; $chatDelimiterStyles = { "ChatBlockDivider", "ChatDelimiter", "ExcludedChatDelimiter" }; $chatIgnoredStyles = { "ChatExcluded" }; diff --git a/Source/Chatbook/CommonSymbols.wl b/Source/Chatbook/CommonSymbols.wl index aa2eb98f..45e81705 100644 --- a/Source/Chatbook/CommonSymbols.wl +++ b/Source/Chatbook/CommonSymbols.wl @@ -31,6 +31,7 @@ BeginPackage[ "Wolfram`Chatbook`Common`" ]; `$currentChatSettings; `$currentSettingsCache; `$customToolFormatter; +`$defaultAppName; `$defaultChatSettings; `$defaultChatTools; `$defaultMaxCellStringLength; diff --git a/Source/Chatbook/Settings.wl b/Source/Chatbook/Settings.wl index 45958241..9b08ffc7 100644 --- a/Source/Chatbook/Settings.wl +++ b/Source/Chatbook/Settings.wl @@ -18,7 +18,7 @@ $cloudInheritanceFix := $cloudNotebooks; (* cSpell: ignore AIAPI *) $defaultChatSettings = <| - "AppName" -> None, + "AppName" -> Automatic, "Assistance" -> Automatic, "Authentication" -> Automatic, "AutoFormat" -> True, @@ -308,6 +308,7 @@ resolveAutoSetting[ settings_, key_ -> value_ ] := <| settings, key -> resolveAu resolveAutoSetting // endDefinition; resolveAutoSetting0 // beginDefinition; +resolveAutoSetting0[ as_, "AppName" ] := $defaultAppName; resolveAutoSetting0[ as_, "Assistance" ] := False; resolveAutoSetting0[ as_, "AutoSaveConversations" ] := autoSaveConversationsQ @ as; resolveAutoSetting0[ as_, "BypassResponseChecking" ] := bypassResponseCheckingQ @ as; @@ -317,15 +318,15 @@ resolveAutoSetting0[ as_, "EnableLLMServices" ] := $useLLMServices; resolveAutoSetting0[ as_, "ForceSynchronous" ] := forceSynchronousQ @ as; resolveAutoSetting0[ as_, "HandlerFunctionsKeys" ] := chatHandlerFunctionsKeys @ as; resolveAutoSetting0[ as_, "IncludeHistory" ] := Automatic; -resolveAutoSetting0[ as_, "PromptGenerators" ] := { }; -resolveAutoSetting0[ as_, "PromptGeneratorMessageRole" ] := "System"; -resolveAutoSetting0[ as_, "PromptGeneratorMessagePosition" ] := 2; resolveAutoSetting0[ as_, "MaxCellStringLength" ] := chooseMaxCellStringLength @ as; resolveAutoSetting0[ as_, "MaxContextTokens" ] := autoMaxContextTokens @ as; resolveAutoSetting0[ as_, "MaxOutputCellStringLength" ] := chooseMaxOutputCellStringLength @ as; resolveAutoSetting0[ as_, "MaxTokens" ] := autoMaxTokens @ as; resolveAutoSetting0[ as_, "Multimodal" ] := multimodalQ @ as; resolveAutoSetting0[ as_, "NotebookWriteMethod" ] := "PreemptiveLink"; +resolveAutoSetting0[ as_, "PromptGeneratorMessagePosition" ] := 2; +resolveAutoSetting0[ as_, "PromptGeneratorMessageRole" ] := "System"; +resolveAutoSetting0[ as_, "PromptGenerators" ] := { }; resolveAutoSetting0[ as_, "ShowMinimized" ] := Automatic; resolveAutoSetting0[ as_, "StreamingOutputMethod" ] := "PartialDynamic"; resolveAutoSetting0[ as_, "TokenBudgetMultiplier" ] := 1; diff --git a/Source/Chatbook/Storage.wl b/Source/Chatbook/Storage.wl index 7980e934..1d5ba37e 100644 --- a/Source/Chatbook/Storage.wl +++ b/Source/Chatbook/Storage.wl @@ -17,7 +17,6 @@ Needs[ "Wolfram`Chatbook`Common`" ]; $maxTitleGenerationMessages = 10; $savedChatDataVersion = 1; $rootStorageName = "SavedChats"; -$defaultAppName = "Default"; $defaultConversationTitle = "Untitled Chat"; $maxFilesDisplay = 50; $timestampPrefixLength = 7; (* good for about 1000 years *) From 90a453e0f07ee57b340c93ce3786606abbd2e6fe Mon Sep 17 00:00:00 2001 From: Rick Hennigan Date: Tue, 1 Oct 2024 10:57:37 -0400 Subject: [PATCH 06/10] Added "CacheEmbeddings" option --- Source/Chatbook/CommonSymbols.wl | 2 + .../PromptGenerators/VectorDatabases.wl | 52 ++++++++++++------- 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/Source/Chatbook/CommonSymbols.wl b/Source/Chatbook/CommonSymbols.wl index 45e81705..2374bc8a 100644 --- a/Source/Chatbook/CommonSymbols.wl +++ b/Source/Chatbook/CommonSymbols.wl @@ -153,6 +153,8 @@ BeginPackage[ "Wolfram`Chatbook`Common`" ]; `getAvailableServiceNames; `getBoxObjectFromBoxID; `getChatGroupSettings; +`getEmbedding; +`getEmbeddings; `getHandlerFunctions; `getInlineChatPrompt; `getModelList; diff --git a/Source/Chatbook/PromptGenerators/VectorDatabases.wl b/Source/Chatbook/PromptGenerators/VectorDatabases.wl index 0f46c7e9..251f38e9 100644 --- a/Source/Chatbook/PromptGenerators/VectorDatabases.wl +++ b/Source/Chatbook/PromptGenerators/VectorDatabases.wl @@ -15,9 +15,10 @@ HoldComplete[ (* ::**************************************************************************************************************:: *) (* ::Section::Closed:: *) (*Configuration*) -$vectorDBNames = { "DocumentationURIs", "WolframAlphaQueries" }; -$dbVersion = "1.1.0"; -$allowDownload = True; +$vectorDBNames = { "DocumentationURIs", "WolframAlphaQueries" }; +$dbVersion = "1.1.0"; +$allowDownload = True; +$cacheEmbeddings = True; $embeddingDimension = 384; $maxNeighbors = 50; @@ -693,14 +694,15 @@ cacheVectorDBResult // endDefinition; (* ::Subsection::Closed:: *) (*getEmbedding*) getEmbedding // beginDefinition; +getEmbedding // Options = { "CacheEmbeddings" -> $cacheEmbeddings }; -getEmbedding[ string_String ] := +getEmbedding[ string_String, opts: OptionsPattern[ ] ] := With[ { embedding = $embeddingCache[ string ] }, embedding /; NumericArrayQ @ embedding ]; -getEmbedding[ string_String ] := Enclose[ - First @ ConfirmMatch[ getEmbeddings @ { string }, { _NumericArray }, "Embedding" ], +getEmbedding[ string_String, opts: OptionsPattern[ ] ] := Enclose[ + First @ ConfirmMatch[ getEmbeddings[ { string }, opts ], { _NumericArray }, "Embedding" ], throwInternalFailure ]; @@ -710,19 +712,31 @@ getEmbedding // endDefinition; (* ::Subsection::Closed:: *) (*getEmbeddings*) getEmbeddings // beginDefinition; +getEmbeddings // Options = { "CacheEmbeddings" -> $cacheEmbeddings }; -getEmbeddings[ { } ] := { }; +getEmbeddings[ { }, opts: OptionsPattern[ ] ] := { }; -getEmbeddings[ strings: { __String } ] := Enclose[ +getEmbeddings[ strings: { __String }, opts: OptionsPattern[ ] ] := + If[ TrueQ @ OptionValue[ "CacheEmbeddings" ], + getEmbeddings0 @ strings, + Block[ { $cacheEmbeddings = False }, getAndCacheEmbeddings @ strings ] + ] // LogChatTiming[ "GetEmbeddings" ]; + +getEmbeddings // endDefinition; + + +getEmbeddings0 // beginDefinition; + +getEmbeddings0[ strings: { __String } ] := Enclose[ Module[ { notCached }, notCached = Select[ strings, ! KeyExistsQ[ $embeddingCache, # ] & ]; ConfirmMatch[ getAndCacheEmbeddings @ notCached, { ___NumericArray }, "CacheEmbeddings" ]; ConfirmMatch[ Lookup[ $embeddingCache, strings ], { __NumericArray }, "Result" ] - ] // LogChatTiming[ "GetEmbeddings" ], + ], throwInternalFailure ]; -getEmbeddings // endDefinition; +getEmbeddings0 // endDefinition; (* ::**************************************************************************************************************:: *) (* ::Subsubsection::Closed:: *) @@ -742,10 +756,7 @@ getAndCacheEmbeddings[ strings: { __String } ] /; $embeddingModel === "SentenceB ConfirmAssert[ Length @ strings === Length @ vectors, "LengthCheck" ]; - MapThread[ - ($embeddingCache[ #1 ] = toTinyVector @ #2) &, - { strings, vectors } - ] + MapThread[ cacheEmbedding, { strings, vectors } ] ], throwInternalFailure ]; @@ -771,16 +782,21 @@ getAndCacheEmbeddings[ strings: { __String } ] := Enclose[ ConfirmAssert[ Length @ strings === Length @ vectors, "LengthCheck" ]; - MapThread[ - ($embeddingCache[ #1 ] = toTinyVector @ #2) &, - { strings, vectors } - ] + MapThread[ cacheEmbedding, { strings, vectors } ] ], throwInternalFailure ]; getAndCacheEmbeddings // endDefinition; +(* ::**************************************************************************************************************:: *) +(* ::Subsubsection::Closed:: *) +(*cacheEmbedding*) +cacheEmbedding // beginDefinition; +cacheEmbedding[ key_String, vector_ ] /; ! $cacheEmbeddings := toTinyVector @ vector; +cacheEmbedding[ key_String, vector_ ] := $embeddingCache[ key ] = toTinyVector @ vector; +cacheEmbedding // endDefinition; + (* ::**************************************************************************************************************:: *) (* ::Subsubsection::Closed:: *) (*sentenceBERTEmbedding*) From fad56b543fa4c0c79f190dee9bb46d67974e1946 Mon Sep 17 00:00:00 2001 From: Rick Hennigan Date: Tue, 1 Oct 2024 11:00:06 -0400 Subject: [PATCH 07/10] Saving chats now also includes embeddings for search --- Source/Chatbook/Storage.wl | 313 +++++++++++++++++++++++++++++++------ 1 file changed, 263 insertions(+), 50 deletions(-) diff --git a/Source/Chatbook/Storage.wl b/Source/Chatbook/Storage.wl index 1d5ba37e..8455e714 100644 --- a/Source/Chatbook/Storage.wl +++ b/Source/Chatbook/Storage.wl @@ -14,22 +14,36 @@ Needs[ "Wolfram`Chatbook`Common`" ]; (* ::**************************************************************************************************************:: *) (* ::Section::Closed:: *) (*Configuration*) -$maxTitleGenerationMessages = 10; -$savedChatDataVersion = 1; +$maxTitleGenerationMessages = 10; (* 5 input/output pairs *) +$savedChatDataVersion = 2; $rootStorageName = "SavedChats"; $defaultConversationTitle = "Untitled Chat"; -$maxFilesDisplay = 50; +$maxChatItems = Infinity; $timestampPrefixLength = 7; (* good for about 1000 years *) $$timestampPrefix = Repeated[ LetterCharacter|DigitCharacter, { $timestampPrefixLength } ]; -$$chatMetadata = KeyValuePattern @ { +$metaKeys = { "AppName", "ConversationTitle", "ConversationUUID", "Date", "Version" }; + +(* TODO: these patterns might need to move to Common.wl *) +$$conversationData = KeyValuePattern @ { + "AppName" -> _String, + "ConversationTitle" -> _String, + "ConversationUUID" -> _String, + "Date" -> _Real, + "Version" -> $savedChatDataVersion +}; + +$$conversationFullData = KeyValuePattern @ { "AppName" -> _String, "ConversationTitle" -> _String, "ConversationUUID" -> _String, "Date" -> _Real, - "Version" -> _Integer + "Messages" -> _List, + "Version" -> $savedChatDataVersion }; +$$legacyData = KeyValuePattern[ "Version" -> _? (LessThan @ $savedChatDataVersion ) ]; + $$appSpec = $$string | All | _NotebookObject; $generatedTitleCache = <| |>; @@ -38,8 +52,14 @@ $generatedTitleCache = <| |>; (* ::Section::Closed:: *) (*ListSavedChats*) ListSavedChats // beginDefinition; -ListSavedChats[ ] := catchMine @ ListSavedChats @ All; -ListSavedChats[ appSpec: $$appSpec ] := catchMine @ LogChatTiming @ listSavedChats @ appSpec; +ListSavedChats // Options = { MaxItems -> $maxChatItems }; + +ListSavedChats[ opts: OptionsPattern[ ] ] := + catchMine @ ListSavedChats[ All, opts ]; + +ListSavedChats[ appSpec: $$appSpec, opts: OptionsPattern[ ] ] := + catchMine @ LogChatTiming @ listSavedChats[ appSpec, OptionValue[ MaxItems ] ]; + ListSavedChats // endExportedDefinition; (* ::**************************************************************************************************************:: *) @@ -47,8 +67,8 @@ ListSavedChats // endExportedDefinition; (*listSavedChats*) listSavedChats // beginDefinition; -listSavedChats[ appSpec: $$appSpec ] := Enclose[ - Catch @ Module[ { appName, dirName, root, depth, files, sorted }, +listSavedChats[ appSpec: $$appSpec, maxItems_? Positive ] := Enclose[ + Catch @ Module[ { appName, dirName, root, depth, files, sorted, take }, appName = ConfirmMatch[ determineAppName @ appSpec, $$string | All, "Name" ]; dirName = If[ StringQ @ appName, appName, Nothing ]; @@ -61,14 +81,14 @@ listSavedChats[ appSpec: $$appSpec ] := Enclose[ depth = If[ StringQ @ appName, 2, 3 ]; - files = FileNames[ "metadata.wxf", root, { depth } ]; If[ files === { }, Throw @ { } ]; (* show most recent first *) sorted = If[ StringQ @ appName, Reverse @ files, ReverseSortBy[ files, FileNameTake[ #, { -2 } ] & ] ]; + take = ConfirmMatch[ Take[ sorted, UpTo @ Floor @ maxItems ], { ___String }, "Take" ]; - ConfirmMatch[ readChatMetaFile /@ Take[ sorted, UpTo @ $maxFilesDisplay ], { ___Association }, "Metadata" ] + ConfirmMatch[ readChatMetaFile /@ take, { ___Association }, "Metadata" ] ], throwInternalFailure ]; @@ -98,17 +118,18 @@ notebookAppName // endDefinition; (*readChatMetaFile*) readChatMetaFile // beginDefinition; readChatMetaFile[ file_String ] := readChatMetaFile[ file, Quiet @ Developer`ReadWXFFile @ file ]; -readChatMetaFile[ file_String, as: $$chatMetadata ] := <| as, "Path" -> File @ DirectoryName @ file |>; +readChatMetaFile[ file_String, as_Association ] := checkChatDataVersion @ as; readChatMetaFile[ file_String, _? FailureQ ] := Nothing; (* corrupt WXF file (should we auto-remove it?) *) readChatMetaFile // endDefinition; (* ::**************************************************************************************************************:: *) (* ::Section::Closed:: *) (*LoadChat*) +(* TODO: LoadChat[NotebookObject[...], spec]*) LoadChat // beginDefinition; -LoadChat[ as_Association ] := catchMine @ LoadChat[ as[ "AppName" ], as[ "ConversationUUID" ] ]; -LoadChat[ uuid_String ] := catchMine @ LogChatTiming @ loadChat[ $defaultAppName, uuid ]; -LoadChat[ appName_String, uuid_String ] := catchMine @ LogChatTiming @ loadChat[ appName, uuid ]; +LoadChat[ as: KeyValuePattern[ "ConversationUUID" -> _String ] ] := catchMine @ LogChatTiming @ loadChat @ as; +LoadChat[ uuid_String ] := catchMine @ LoadChat @ <| "ConversationUUID" -> uuid |>; +LoadChat[ app_String, uuid_String ] := catchMine @ LoadChat @ <| "AppName" -> app, "ConversationUUID" -> uuid |>; LoadChat // endExportedDefinition; (* ::**************************************************************************************************************:: *) @@ -116,21 +137,10 @@ LoadChat // endExportedDefinition; (*loadChat*) loadChat // beginDefinition; -loadChat[ appName_String, uuid_String ] := Enclose[ - Catch @ Module[ { root, dirs, dir, file, data }, - - root = ConfirmBy[ - ChatbookFilesDirectory[ { $rootStorageName, appName }, "EnsureDirectory" -> False ], - StringQ, - "Root" - ]; - - dirs = ConfirmMatch[ Sort @ FileNames[ $$timestampPrefix ~~ "_" ~~ uuid, root ], { ___String }, "Directories" ]; - If[ dirs === { }, Throw @ Missing[ "NotFound" ] ]; - dir = ConfirmBy[ First[ dirs, $Failed ], StringQ, "Directory" ]; - file = ConfirmBy[ FileNameJoin @ { dir, "data.wxf" }, StringQ, "File" ]; - If[ ! FileExistsQ @ file, Throw @ Missing[ "NotFound" ] ]; - data = ConfirmBy[ Developer`ReadWXFFile @ file, AssociationQ, "Data" ]; +loadChat[ as_Association ] := Enclose[ + Catch @ Module[ { data }, + data = ConfirmMatch[ getChatConversationData @ as, $$conversationFullData|_Missing, "Data" ]; + If[ MissingQ @ data, Throw @ data ]; ConfirmBy[ restoreAttachments @ data, AssociationQ, "RestoreAttachments" ]; data ], @@ -139,6 +149,60 @@ loadChat[ appName_String, uuid_String ] := Enclose[ loadChat // endDefinition; +(* ::**************************************************************************************************************:: *) +(* ::Subsection::Closed:: *) +(*getChatConversationData*) +getChatConversationData // beginDefinition; + +getChatConversationData[ KeyValuePattern @ { "AppName" -> appName_String, "ConversationUUID" -> uuid_String } ] := + getChatConversationData[ appName, uuid ]; + +getChatConversationData[ KeyValuePattern[ "ConversationUUID" -> uuid_String ] ] := + getChatConversationData @ uuid; + +getChatConversationData[ uuid_String ] := Enclose[ + Catch @ Module[ { root, dir }, + root = ConfirmBy[ storageDirectory[ ], StringQ, "Root" ]; + dir = First[ conversationFileNames[ uuid, root, { 2 } ], Throw @ Missing[ "NotFound" ] ]; + ConfirmMatch[ getChatConversationData0 @ dir, $$conversationFullData|_Missing, "Data" ] + ], + throwInternalFailure +]; + +getChatConversationData[ appName_String, uuid_String ] := Enclose[ + Catch @ Module[ { root, dir }, + root = ConfirmBy[ storageDirectory @ appName, StringQ, "Root" ]; + dir = First[ conversationFileNames[ uuid, root ], Throw @ Missing[ "NotFound" ] ]; + ConfirmMatch[ getChatConversationData0 @ dir, $$conversationFullData|_Missing, "Data" ] + ], + throwInternalFailure +]; + +getChatConversationData // endDefinition; + + +getChatConversationData0 // beginDefinition; + +getChatConversationData0[ dir_String ] := Enclose[ + Catch @ Module[ { fail, file, data }, + fail = Function[ Quiet @ DeleteDirectory[ dir, DeleteContents -> True ]; Throw @ Missing[ "NotFound" ] ]; + + file = FileNameJoin @ { dir, "data.wxf" }; + If[ ! FileExistsQ @ file, fail[ ] ]; + + data = Quiet @ Developer`ReadWXFFile @ file; + If[ ! AssociationQ @ data, fail[ ] ]; + + checkChatDataVersion @ data + ], + throwInternalFailure +]; + +getChatConversationData0[ missing_Missing ] := + missing; + +getChatConversationData0 // endDefinition; + (* ::**************************************************************************************************************:: *) (* ::Subsubsection::Closed:: *) (*restoreAttachments*) @@ -178,14 +242,8 @@ deleteChat // beginDefinition; deleteChat[ appName_String, uuid_String ] := Enclose[ Catch @ Module[ { root, dirs, dir }, - - root = ConfirmBy[ - ChatbookFilesDirectory[ { $rootStorageName, appName }, "EnsureDirectory" -> False ], - StringQ, - "Root" - ]; - - dirs = ConfirmMatch[ Sort @ FileNames[ $$timestampPrefix ~~ "_" ~~ uuid, root ], { ___String }, "Directories" ]; + root = ConfirmBy[ storageDirectory @ appName, StringQ, "Root" ]; + dirs = ConfirmMatch[ conversationFileNames[ uuid, root ], { ___String }, "Directories" ]; If[ dirs === { }, Throw @ Missing[ "NotFound" ] ]; dir = ConfirmBy[ First[ dirs, $Failed ], StringQ, "Directory" ]; ConfirmMatch[ DeleteDirectory[ dir, DeleteContents -> True ], Null, "DeleteDirectory" ]; @@ -218,11 +276,13 @@ SaveChat // endExportedDefinition; saveChat // beginDefinition; saveChat[ messages0_, settings0_, autoTitle_ ] := Enclose[ - Module[ { settings, messages, appName, metadata, directory, attachments, smallSettings, as }, + Module[ { settings, messages, appName, metadata, vectors, directory, attachments, smallSettings, as }, + settings = If[ TrueQ @ autoTitle, <| settings0, "AutoGenerateTitle" -> True |>, settings0 ]; messages = ConfirmMatch[ prepareMessagesForSaving[ messages0, settings ], $$chatMessages, "Messages" ]; appName = ConfirmBy[ Lookup[ settings, "AppName", $defaultAppName ], StringQ, "AppName" ]; - metadata = ConfirmMatch[ getChatMetadata[ appName, messages, settings ], $$chatMetadata, "Metadata" ]; + metadata = ConfirmMatch[ getChatMetadata[ appName, messages, settings ], $$conversationData, "Metadata" ]; + vectors = ConfirmMatch[ createMessageVectors[ metadata, messages, settings ], { ___NumericArray }, "Vectors" ]; directory = ConfirmBy[ targetDirectory[ appName, metadata ], DirectoryQ, "Directory" ]; attachments = ConfirmBy[ GetAttachments[ messages, All ], AssociationQ, "Attachments" ]; smallSettings = ConfirmBy[ toSmallSettings @ settings, AssociationQ, "Settings" ]; @@ -242,7 +302,8 @@ saveChat[ messages0_, settings0_, autoTitle_ ] := Enclose[ metadata, "Attachments" -> attachments, "Messages" -> messages, - "Settings" -> smallSettings + "Settings" -> smallSettings, + "Vectors" -> vectors |>, directory, PerformanceGoal -> "Size" @@ -272,6 +333,37 @@ saveChat[ messages0_, settings0_, autoTitle_ ] := Enclose[ saveChat // endDefinition; +(* ::**************************************************************************************************************:: *) +(* ::Subsubsection::Closed:: *) +(*createMessageVectors*) +createMessageVectors // beginDefinition; + +createMessageVectors[ metadata_, messages: $$chatMessages, settings_ ] := Enclose[ + Module[ { partitioned, strings, rVectors, iVectors, title, titleVector }, + ConfirmAssert[ Length @ messages >= 2, "LengthCheck" ]; + partitioned = ConfirmBy[ Partition[ messages, UpTo[ 2 ] ], ListQ, "Pairs" ]; + strings = ConfirmMatch[ messagesToString /@ partitioned, { __String }, "Strings" ]; + rVectors = ConfirmMatch[ getEmbeddings @ strings, { __NumericArray }, "Embeddings" ]; + iVectors = ConfirmMatch[ toInt8Vector /@ rVectors, { __NumericArray }, "Int8Vectors" ]; + title = metadata[ "ConversationTitle" ]; + If[ StringQ @ title, + titleVector = ConfirmMatch[ toInt8Vector @ getEmbedding @ title, _NumericArray, "TitleVector" ]; + Prepend[ iVectors, titleVector ], + iVectors + ] + ], + throwInternalFailure +]; + +createMessageVectors // endDefinition; + +(* ::**************************************************************************************************************:: *) +(* ::Subsubsubsection::Closed:: *) +(*toInt8Vector*) +toInt8Vector // beginDefinition; +toInt8Vector[ arr_NumericArray ] := NumericArray[ arr, "Integer8", "ClipAndRound" ]; +toInt8Vector // endDefinition; + (* ::**************************************************************************************************************:: *) (* ::Subsubsection::Closed:: *) (*getAppName*) @@ -313,13 +405,8 @@ cleanupStaleChats // beginDefinition; cleanupStaleChats[ app_String ] := Enclose[ Module[ { root, dirs, grouped, delete }, - root = ConfirmBy[ - ChatbookFilesDirectory[ { $rootStorageName, app }, "EnsureDirectory" -> False ], - StringQ, - "Root" - ]; - - dirs = ConfirmMatch[ Sort @ FileNames[ $$timestampPrefix ~~ "_" ~~ __, root ], { ___String }, "Directories" ]; + root = ConfirmBy[ storageDirectory @ app, StringQ, "Root" ]; + dirs = ConfirmMatch[ conversationFileNames[ All, root ], { ___String }, "Directories" ]; grouped = GatherBy[ dirs, StringDrop[ FileNameTake @ #, $timestampPrefixLength ] & ]; delete = ConfirmMatch[ Flatten[ Most /@ grouped ], { ___String }, "Delete" ]; @@ -374,7 +461,18 @@ dropSystemMessage // endDefinition; (*saveChatFile*) saveChatFile // beginDefinition; -saveChatFile[ type_String, data_, directory_, opts: OptionsPattern[ ] ] := Enclose[ +saveChatFile[ "metadata", data_Association, directory_, opts: OptionsPattern[ ] ] := + saveChatFile0[ "metadata", KeyTake[ data, $metaKeys ], directory, opts ]; + +saveChatFile[ type_String, data_Association, directory_, opts: OptionsPattern[ ] ] := + saveChatFile0[ type, data, directory, opts ]; + +saveChatFile // endDefinition; + + +saveChatFile0 // beginDefinition; + +saveChatFile0[ type_String, data_, directory_, opts: OptionsPattern[ ] ] := Enclose[ Module[ { file }, file = ConfirmBy[ FileNameJoin @ { directory, type <> ".wxf" }, StringQ, "File" ]; ConfirmBy[ Developer`WriteWXFFile[ file, data, opts ], FileExistsQ, "Export" ] @@ -382,7 +480,7 @@ saveChatFile[ type_String, data_, directory_, opts: OptionsPattern[ ] ] := Enclo throwInternalFailure ]; -saveChatFile // endDefinition; +saveChatFile0 // endDefinition; (* ::**************************************************************************************************************:: *) (* ::Subsubsection::Closed:: *) @@ -477,6 +575,121 @@ generateTitleCached0[ hash_Integer, messages_ ] := Enclose[ generateTitleCached0 // endDefinition; +(* ::**************************************************************************************************************:: *) +(* ::Section::Closed:: *) +(*Upgrade Data*) + +(* ::**************************************************************************************************************:: *) +(* ::Subsection::Closed:: *) +(*checkChatDataVersion*) +checkChatDataVersion // beginDefinition; +checkChatDataVersion[ as: $$conversationData ] := as; +checkChatDataVersion[ as: $$legacyData ] := upgradeChatData @ as; +checkChatDataVersion // endDefinition; + +(* ::**************************************************************************************************************:: *) +(* ::Subsection::Closed:: *) +(*upgradeChatData*) +upgradeChatData // beginDefinition; + +upgradeChatData[ as: KeyValuePattern[ "Version" -> oldVersion_Integer ] ] := Enclose[ + Module[ { upgraded, newVersion }, + ConfirmAssert[ oldVersion < $savedChatDataVersion, "OldVersionCheck" ]; + upgraded = ConfirmBy[ upgradeChatData0[ oldVersion, as ], AssociationQ, "Upgraded" ]; + newVersion = ConfirmMatch[ upgraded[ "Version" ], _Integer, "NewVersion" ]; + ConfirmAssert[ oldVersion < newVersion <= $savedChatDataVersion, "NewVersionCheck" ]; + If[ newVersion === $savedChatDataVersion, + upgraded, + upgradeChatData @ upgraded + ] + ], + throwInternalFailure +]; + +upgradeChatData // endDefinition; + + +upgradeChatData0 // beginDefinition; +upgradeChatData0[ 1, as_Association ] := upgradeChatData1 @ as; +upgradeChatData0 // endDefinition; + +(* ::**************************************************************************************************************:: *) +(* ::Subsubsection::Closed:: *) +(*Update from version 1*) + +(* Adds vectors to the saved data: *) +upgradeChatData1 // beginDefinition; + +upgradeChatData1[ metadata_Association ] := Enclose[ + Module[ { appName, directory, file, data, messages, settings, vectors, newData, newMeta }, + + appName = ConfirmBy[ metadata[ "AppName" ], StringQ, "AppName" ]; + directory = ConfirmBy[ targetDirectory[ appName, metadata ], DirectoryQ, "Directory" ]; + file = ConfirmBy[ FileNameJoin @ { directory, "data.wxf" }, FileExistsQ, "File" ]; + data = ConfirmBy[ Developer`ReadWXFFile @ file, AssociationQ, "Data" ]; + messages = ConfirmMatch[ data[ "Messages" ], $$chatMessages, "Messages" ]; + settings = ConfirmBy[ data[ "Settings" ], AssociationQ, "Settings" ]; + vectors = ConfirmMatch[ createMessageVectors[ metadata, messages, settings ], { ___NumericArray }, "Vectors" ]; + newData = <| data, "Vectors" -> vectors, "Version" -> 2 |>; + newMeta = ConfirmBy[ <| metadata, "Version" -> 2 |>, AssociationQ, "Metadata" ]; + + ConfirmBy[ + saveChatFile[ "metadata", newMeta, directory ], + FileExistsQ, + "SaveMetadata" + ]; + + ConfirmBy[ + saveChatFile[ "data", newData, directory, PerformanceGoal -> "Size" ], + FileExistsQ, + "SaveMessages" + ]; + + If[ KeyExistsQ[ metadata, "Messages" ], + newData, + newMeta + ] + ], + throwInternalFailure +]; + +upgradeChatData1 // endDefinition; + +(* ::**************************************************************************************************************:: *) +(* ::Section::Closed:: *) +(*File Utilities*) + +(* ::**************************************************************************************************************:: *) +(* ::Subsection::Closed:: *) +(*storageDirectory*) +storageDirectory // beginDefinition; +storageDirectory[ ] := ChatbookFilesDirectory[ $rootStorageName, "EnsureDirectory" -> False ]; +storageDirectory[ name_String ] := ChatbookFilesDirectory[ { $rootStorageName, name }, "EnsureDirectory" -> False ]; +storageDirectory[ All ] := storageDirectory[ ]; +storageDirectory // endDefinition; + +(* ::**************************************************************************************************************:: *) +(* ::Subsection::Closed:: *) +(*conversationFileNames*) +conversationFileNames // beginDefinition; + +conversationFileNames[ All, args__ ] := + conversationFileNames[ __, args ]; + +conversationFileNames[ pattern_, args__ ] := Enclose[ + Sort @ ConfirmMatch[ FileNames[ conversationFilePattern @ pattern, args ], { ___String }, "FileNames" ], + throwInternalFailure +]; + +conversationFileNames // endDefinition; + +(* ::**************************************************************************************************************:: *) +(* ::Subsection::Closed:: *) +(*conversationFilePattern*) +conversationFilePattern // beginDefinition; +conversationFilePattern[ pattern_ ] := $$timestampPrefix ~~ "_" ~~ pattern; +conversationFilePattern // endDefinition; + (* ::**************************************************************************************************************:: *) (* ::Section::Closed:: *) (*Package Footer*) From b0840794ac4e8c8ceb855a9c52f683de18af9341 Mon Sep 17 00:00:00 2001 From: Rick Hennigan Date: Tue, 1 Oct 2024 20:53:40 -0400 Subject: [PATCH 08/10] Added search functionality --- Source/Chatbook/CommonSymbols.wl | 2 + Source/Chatbook/Main.wl | 7 + Source/Chatbook/Search.wl | 264 +++++++++++++++++++++++++++++++ Source/Chatbook/Storage.wl | 6 + 4 files changed, 279 insertions(+) create mode 100644 Source/Chatbook/Search.wl diff --git a/Source/Chatbook/CommonSymbols.wl b/Source/Chatbook/CommonSymbols.wl index 2374bc8a..f86f1bc6 100644 --- a/Source/Chatbook/CommonSymbols.wl +++ b/Source/Chatbook/CommonSymbols.wl @@ -152,7 +152,9 @@ BeginPackage[ "Wolfram`Chatbook`Common`" ]; `functionTemplateBoxes; `getAvailableServiceNames; `getBoxObjectFromBoxID; +`getChatConversationData; `getChatGroupSettings; +`getChatMetadata; `getEmbedding; `getEmbeddings; `getHandlerFunctions; diff --git a/Source/Chatbook/Main.wl b/Source/Chatbook/Main.wl index 6c42676d..b3dcace7 100644 --- a/Source/Chatbook/Main.wl +++ b/Source/Chatbook/Main.wl @@ -32,6 +32,7 @@ BeginPackage[ "Wolfram`Chatbook`" ]; `$ToolFunctions; `$WorkspaceChat; `AbsoluteCurrentChatSettings; +`AddChatToSearchIndex; `AppendURIInstructions; `BasePrompt; `CachedBoxes; @@ -65,10 +66,12 @@ BeginPackage[ "Wolfram`Chatbook`" ]; `LoadChat; `LogChatTiming; `MakeExpressionURI; +`RebuildChatSearchIndex; `RelatedDocumentation; `RelatedWolframAlphaQueries; `SandboxLinguisticAssistantData; `SaveChat; +`SearchChats; `SetModel; `SetToolOptions; `ShowCodeAssistance; @@ -143,6 +146,7 @@ $ChatbookContexts = { "Wolfram`Chatbook`Prompting`", "Wolfram`Chatbook`ResourceInstaller`", "Wolfram`Chatbook`Sandbox`", + "Wolfram`Chatbook`Search`", "Wolfram`Chatbook`SendChat`", "Wolfram`Chatbook`Serialization`", "Wolfram`Chatbook`Services`", @@ -184,6 +188,7 @@ $ChatbookProtectedNames = "Wolfram`Chatbook`" <> # & /@ { "$ToolFunctions", "$WorkspaceChat", "AbsoluteCurrentChatSettings", + "AddChatToSearchIndex", "AppendURIInstructions", "BasePrompt", "CachedBoxes", @@ -216,10 +221,12 @@ $ChatbookProtectedNames = "Wolfram`Chatbook`" <> # & /@ { "LoadChat", "LogChatTiming", "MakeExpressionURI", + "RebuildChatSearchIndex", "RelatedDocumentation", "RelatedWolframAlphaQueries", "SandboxLinguisticAssistantData", "SaveChat", + "SearchChats", "SetModel", "SetToolOptions", "ShowCodeAssistance", diff --git a/Source/Chatbook/Search.wl b/Source/Chatbook/Search.wl new file mode 100644 index 00000000..14e122c0 --- /dev/null +++ b/Source/Chatbook/Search.wl @@ -0,0 +1,264 @@ +(* ::Section::Closed:: *) +(*Package Header*) +BeginPackage[ "Wolfram`Chatbook`Search`" ]; +Begin[ "`Private`" ]; + +Needs[ "Wolfram`Chatbook`" ]; +Needs[ "Wolfram`Chatbook`Common`" ]; + +(* ::**************************************************************************************************************:: *) +(* ::Section::Closed:: *) +(*Configuration*) +$rootStorageName = "Search"; +$chatSearchIndex = None; +$searchIndexVersion = 1; + +(* ::**************************************************************************************************************:: *) +(* ::Section::Closed:: *) +(*SearchChats*) +SearchChats // beginDefinition; +SearchChats // Options = { MaxItems -> 10 }; + +SearchChats[ app: _String|All, query_String, opts: OptionsPattern[ ] ] := + catchMine @ LogChatTiming @ searchChats[ app, query, OptionValue[ MaxItems ] ]; + +SearchChats[ query_String, opts: OptionsPattern[ ] ] := + catchMine @ SearchChats[ All, query, opts ]; + +SearchChats // endDefinition; + +(* ::**************************************************************************************************************:: *) +(* ::Subsection::Closed:: *) +(*searchChats*) +searchChats // beginDefinition; + +searchChats[ appName_String, query_String, max_? Positive ] := Enclose[ + Catch @ Module[ { index, flat, values, vectors, embedding, idx, results }, + + index = Values @ ConfirmBy[ loadChatSearchIndex @ appName, AssociationQ, "Load" ]; + + flat = ConfirmMatch[ + Flatten[ Thread @ { KeyDrop[ #, "Vectors" ], #Vectors } & /@ index, 1 ], + { { _Association, _NumericArray }... } + ]; + + If[ flat === { }, Throw @ { } ]; + + { values, vectors } = ConfirmMatch[ Transpose @ flat, { _, _ }, "Transpose" ]; + + embedding = ConfirmBy[ getEmbedding[ query, "CacheEmbeddings" -> False ], NumericArrayQ, "Embedding" ]; + + idx = ConfirmMatch[ + Nearest[ Normal @ vectors -> "Index", Normal @ embedding, Floor[ 2*max+1 ] ], + { ___Integer }, + "Nearest" + ]; + + results = ConfirmMatch[ values[[ idx ]], { ___Association }, "Results" ]; + + Take[ DeleteDuplicatesBy[ results, Lookup[ "ConversationUUID" ] ], UpTo @ Floor @ max ] + ], + throwInternalFailure +]; + +searchChats // endDefinition; + +(* ::**************************************************************************************************************:: *) +(* ::Section::Closed:: *) +(*AddChatToSearchIndex*) +AddChatToSearchIndex // beginDefinition; + +AddChatToSearchIndex[ as: KeyValuePattern[ "ConversationUUID" -> _String ] ] := + catchMine @ LogChatTiming @ addChatToSearchIndex @ as; + +AddChatToSearchIndex[ uuid_String ] := + catchMine @ LogChatTiming @ addChatToSearchIndex @ uuid; + +AddChatToSearchIndex[ app_String, uuid_String ] := + catchMine @ LogChatTiming @ addChatToSearchIndex @ <| "AppName" -> app, "ConversationUUID" -> uuid |>; + +AddChatToSearchIndex // endExportedDefinition; + +(* ::**************************************************************************************************************:: *) +(* ::Subsection::Closed:: *) +(*addChatToSearchIndex*) +addChatToSearchIndex // beginDefinition; + +addChatToSearchIndex[ spec_ ] := Enclose[ + Catch @ Module[ { data, appName, uuid, vectors, metadata }, + data = ConfirmMatch[ getChatConversationData @ spec, _Association|_Missing, "Data" ]; + If[ MissingQ @ data, Throw @ Missing[ "NotSaved" ] ]; (* TODO: auto-save here? *) + appName = ConfirmBy[ data[ "AppName" ], StringQ, "AppName" ]; + uuid = ConfirmBy[ data[ "ConversationUUID" ], StringQ, "ConversationUUID" ]; + vectors = ConfirmMatch[ data[ "Vectors" ], { ___NumericArray }, "Vectors" ]; + If[ vectors === { }, Throw @ Missing[ "NoVectors" ] ]; + + ConfirmBy[ loadChatSearchIndex @ appName, AssociationQ, "Load" ]; + ConfirmAssert[ AssociationQ @ $chatSearchIndex[ appName ], "CheckIndex" ]; + + metadata = ConfirmBy[ getChatMetadata @ data, AssociationQ, "Metadata" ]; + $chatSearchIndex[ appName, uuid ] = <| metadata, "Vectors" -> vectors |>; + ConfirmBy[ saveChatIndex @ appName, FileExistsQ, "Save" ]; + + Success[ "AddedChatToSearchIndex", <| "AppName" -> appName, "ConversationUUID" -> uuid |> ] + ], + throwInternalFailure +]; + +addChatToSearchIndex // endDefinition; + +(* ::**************************************************************************************************************:: *) +(* ::Section::Closed:: *) +(*RebuildChatSearchIndex*) +RebuildChatSearchIndex // beginDefinition; +RebuildChatSearchIndex[ appName_String ] := catchMine @ LogChatTiming @ rebuildChatSearchIndex @ appName; +RebuildChatSearchIndex[ All ] := catchMine @ LogChatTiming @ rebuildChatSearchIndex @ All; +RebuildChatSearchIndex[ ] := catchMine @ LogChatTiming @ rebuildChatSearchIndex @ All; +RebuildChatSearchIndex // endExportedDefinition; + +(* ::**************************************************************************************************************:: *) +(* ::Subsection::Closed:: *) +(*rebuildChatSearchIndex*) +rebuildChatSearchIndex // beginDefinition; + +rebuildChatSearchIndex[ appName_String ] := Enclose[ + Module[ { root, chats }, + + root = ConfirmBy[ + ChatbookFilesDirectory[ { $rootStorageName, appName }, "EnsureDirectory" -> False ], + StringQ, + "Root" + ]; + + If[ DirectoryQ @ root, ConfirmMatch[ DeleteDirectory[ root, DeleteContents -> True ], Null, "Delete" ] ]; + + If[ ! AssociationQ @ $chatSearchIndex, $chatSearchIndex = <| |> ]; + chats = ConfirmMatch[ ListSavedChats @ appName, { ___Association }, "Chats" ]; + ConfirmMatch[ addChatToSearchIndex /@ chats, { ___Success }, "AddChatToSearchIndex" ]; + ConfirmBy[ saveChatIndex @ appName, FileExistsQ, "Save" ]; + ConfirmBy[ $chatSearchIndex[ appName ], AssociationQ, "Result" ] + ], + throwInternalFailure +]; + +rebuildChatSearchIndex[ All ] := Enclose[ + Module[ { root, chats }, + + root = ConfirmBy[ + ChatbookFilesDirectory[ $rootStorageName, "EnsureDirectory" -> False ], + StringQ, + "Root" + ]; + + If[ DirectoryQ @ root, ConfirmMatch[ DeleteDirectory[ root, DeleteContents -> True ], Null, "Delete" ] ]; + + $chatSearchIndex = <| |>; + chats = ConfirmMatch[ ListSavedChats[ ], { ___Association }, "Chats" ]; + ConfirmMatch[ addChatToSearchIndex /@ chats, { ___Success }, "AddChatToSearchIndex" ]; + ConfirmMatch[ saveChatIndex[ ], { ___? FileExistsQ }, "Save" ]; + $chatSearchIndex + ], + throwInternalFailure +]; + +rebuildChatSearchIndex // endDefinition; + +(* ::**************************************************************************************************************:: *) +(* ::Section::Closed:: *) +(*Loading/Saving*) + +(* ::**************************************************************************************************************:: *) +(* ::Subsection::Closed:: *) +(*loadChatSearchIndex*) +loadChatSearchIndex // beginDefinition; + +loadChatSearchIndex[ appName_String ] := Enclose[ + Catch @ Module[ { root, file, data, index }, + + If[ ! AssociationQ @ $chatSearchIndex, $chatSearchIndex = <| |> ]; + + If[ AssociationQ @ $chatSearchIndex[ appName ], + Throw @ $chatSearchIndex[ appName ], + $chatSearchIndex[ appName ] = <| |> + ]; + + root = ConfirmBy[ + ChatbookFilesDirectory[ { $rootStorageName, appName }, "EnsureDirectory" -> False ], + StringQ, + "Root" + ]; + + file = FileNameJoin @ { root, "index.wxf" }; + If[ ! FileExistsQ @ file, Throw @ rebuildChatSearchIndex @ appName ]; + + data = Quiet @ Developer`ReadWXFFile @ file; + If[ ! AssociationQ @ data, Throw @ rebuildChatSearchIndex @ appName ]; + + index = ConfirmBy[ data[ "Index" ], AssociationQ, "Index" ]; + $chatSearchIndex[ appName ] = index + ], + throwInternalFailure +]; + +loadChatSearchIndex[ All ] := Enclose[ + Module[ { root, files, names }, + If[ ! AssociationQ @ $chatSearchIndex, $chatSearchIndex = <| |> ]; + + root = ConfirmBy[ + ChatbookFilesDirectory[ $rootStorageName, "EnsureDirectory" -> False ], + StringQ, + "Root" + ]; + + If[ ! DirectoryQ @ root, Throw @ $chatSearchIndex ]; + + files = ConfirmMatch[ FileNames[ "index.wxf", root, { 2 } ], { ___String }, "Files" ]; + If[ files === { }, Throw @ $chatSearchIndex ]; + + names = ConfirmMatch[ FileBaseName @* DirectoryName /@ files, { __String }, "Names" ]; + ConfirmMatch[ loadChatSearchIndex /@ names, { ___Association }, "Load" ]; + + $chatSearchIndex + ], + throwInternalFailure +]; + +loadChatSearchIndex // endDefinition; + +(* ::**************************************************************************************************************:: *) +(* ::Subsection::Closed:: *) +(*saveChatIndex*) +saveChatIndex // beginDefinition; + +saveChatIndex[ appName_String ] := Enclose[ + Module[ { index, root, file, data }, + index = ConfirmBy[ $chatSearchIndex[ appName ], AssociationQ, "Data" ]; + + root = ConfirmBy[ + ChatbookFilesDirectory[ { $rootStorageName, appName }, "EnsureDirectory" -> True ], + StringQ, + "Root" + ]; + + file = FileNameJoin @ { root, "index.wxf" }; + data = <| "Index" -> index, "Version" -> $searchIndexVersion |>; + + ConfirmBy[ Developer`WriteWXFFile[ file, data, PerformanceGoal -> "Size" ], FileExistsQ, "Result" ] + ], + throwInternalFailure +]; + +saveChatIndex[ ] := + saveChatIndex /@ Keys @ $chatSearchIndex; + +saveChatIndex // endDefinition; + +(* ::**************************************************************************************************************:: *) +(* ::Section::Closed:: *) +(*Package Footer*) +addToMXInitialization[ + Null +]; + +End[ ]; +EndPackage[ ]; diff --git a/Source/Chatbook/Storage.wl b/Source/Chatbook/Storage.wl index 8455e714..439b1507 100644 --- a/Source/Chatbook/Storage.wl +++ b/Source/Chatbook/Storage.wl @@ -154,6 +154,9 @@ loadChat // endDefinition; (*getChatConversationData*) getChatConversationData // beginDefinition; +getChatConversationData[ data: $$conversationFullData ] := + data; + getChatConversationData[ KeyValuePattern @ { "AppName" -> appName_String, "ConversationUUID" -> uuid_String } ] := getChatConversationData[ appName, uuid ]; @@ -487,6 +490,9 @@ saveChatFile0 // endDefinition; (*getChatMetadata*) getChatMetadata // beginDefinition; +getChatMetadata[ data: $$conversationData ] := + KeyTake[ data, $metaKeys ]; + getChatMetadata[ appName_, messages_, settings_Association ] := Enclose[ Module[ { uuid, title, date, version }, From 74408daae4b13475331cc31540122801f4ec7b28 Mon Sep 17 00:00:00 2001 From: Rick Hennigan Date: Tue, 1 Oct 2024 20:53:58 -0400 Subject: [PATCH 09/10] Index chat when saving --- Source/Chatbook/Storage.wl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Source/Chatbook/Storage.wl b/Source/Chatbook/Storage.wl index 439b1507..0d407d93 100644 --- a/Source/Chatbook/Storage.wl +++ b/Source/Chatbook/Storage.wl @@ -327,6 +327,8 @@ saveChat[ messages0_, settings0_, autoTitle_ ] := Enclose[ ConfirmMatch[ cleanupStaleChats @ appName, { ___String }, "Cleanup" ]; + ConfirmMatch[ AddChatToSearchIndex @ as, _Success, "AddToSearchIndex" ]; + updateDynamics[ "SavedChats" ]; Success[ "Saved", as ] From df6c7aacf15aec5a6d6133cd0a95f289ef8bd6ee Mon Sep 17 00:00:00 2001 From: Rick Hennigan Date: Tue, 1 Oct 2024 20:54:22 -0400 Subject: [PATCH 10/10] Specify max tokens for a few more models --- Source/Chatbook/Settings.wl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Source/Chatbook/Settings.wl b/Source/Chatbook/Settings.wl index 9b08ffc7..ae799e74 100644 --- a/Source/Chatbook/Settings.wl +++ b/Source/Chatbook/Settings.wl @@ -481,6 +481,7 @@ autoMaxContextTokens // endDefinition; autoMaxContextTokens0 // beginDefinition; autoMaxContextTokens0[ name_String ] := autoMaxContextTokens0 @ StringSplit[ name, "-"|Whitespace ]; +autoMaxContextTokens0[ { ___, "o1" , ___ } ] := 2^17; autoMaxContextTokens0[ { ___, "gpt"|"chatgpt", "4o" , ___ } ] := 2^17; autoMaxContextTokens0[ { ___, "gpt", "4", "vision" , ___ } ] := 2^17; autoMaxContextTokens0[ { ___, "gpt", "4", "turbo" , ___ } ] := 2^17; @@ -493,6 +494,7 @@ autoMaxContextTokens0[ { ___, "gpt", "3.5" , ___ } ] := 2^12; autoMaxContextTokens0[ { ___, "chat", "bison", "001" , ___ } ] := 20000; autoMaxContextTokens0[ { ___, "gemini", ___, "pro", "vision", ___ } ] := 12288; autoMaxContextTokens0[ { ___, "gemini", ___, "pro" , ___ } ] := 30720; +autoMaxContextTokens0[ { ___, "phi3.5" , ___ } ] := 2^17; autoMaxContextTokens0[ _List ] := 2^12; autoMaxContextTokens0 // endDefinition;