adding current progress

Signed-off-by: Francisco Javier Arceo <farceo@redhat.com>

Author: franciscojavierarceo

docs/reference/alpha-vector-database.md

@@ -108,4 +108,16 @@ def print_online_features(features):
         print(key, " : ", value)
 
 print_online_features(features)
+```
+
+### Configuration
+We offer two Online Store options for Vector Databases. PGVector and SQLite.
+
+#### Installation with SQLite
+If you are using `pyenv` to manage your Python versions, you can install the SQLite extension with the following command:
+```bash
+PYTHON_CONFIGURE_OPTS="--enable-loadable-sqlite-extensions" \
+    LDFLAGS="-L/opt/homebrew/opt/sqlite/lib" \
+    CPPFLAGS="-I/opt/homebrew/opt/sqlite/include" \
+    pyenv install
 ```

sdk/python/feast/infra/online_stores/sqlite.py

@@ -13,12 +13,13 @@
 # limitations under the License.
 import itertools
 import os
+import json
 import sqlite3
+import sqlite_vss
 from datetime import datetime
 from pathlib import Path
 from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple
 
-import sqlite_vss
 from pydantic import StrictStr
 
 from feast import Entity
@@ -110,9 +111,9 @@ def online_write_batch(
                     created_ts = to_naive_utc(created_ts)
 
                 for feature_name, val in values.items():
-                    vector_val = None
                     if config.online_store.vss_enabled:
-                        vector_val = self._get_list_val_str(val)
+                        print('using vector search')
+                        vector_val = json.dumps(val)
                         conn.execute(
                             f"""
                                 UPDATE {_table_id(project, table)}
@@ -121,7 +122,7 @@ def online_write_batch(
                             """,
                             (
                                 # SET
-                                val.SerializeToString(),
+                                str(val),
                                 vector_val,
                                 timestamp,
                                 created_ts,
@@ -138,13 +139,14 @@ def online_write_batch(
                             (
                                 entity_key_bin,
                                 feature_name,
-                                val.SerializeToString(),
+                                str(val),
                                 vector_val,
                                 timestamp,
                                 created_ts,
                             ),
                         )
                     else:
+                        print('not using vector search')
                         conn.execute(
                             f"""
                                 UPDATE {_table_id(project, table)}
@@ -243,8 +245,8 @@ def update(
         project = config.project
 
         for table in tables_to_keep:
-            conn.execute(
-                f"CREATE TABLE IF NOT EXISTS {_table_id(project, table)} (entity_key BLOB, feature_name TEXT, value BLOB, event_ts timestamp, created_ts timestamp,  PRIMARY KEY(entity_key, feature_name))"
+            self.conn.execute(
+                f"CREATE TABLE IF NOT EXISTS {_table_id(project, table)} (entity_key BLOB, feature_name TEXT, value BLOB, vector_value BLOB, event_ts timestamp, created_ts timestamp,  PRIMARY KEY(entity_key, feature_name))"
             )
             conn.execute(
                 f"CREATE INDEX IF NOT EXISTS {_table_id(project, table)}_ek ON {_table_id(project, table)} (entity_key);"
@@ -282,20 +284,6 @@ def teardown(
         except FileNotFoundError:
             pass
 
-    def _get_list_val_str(self, val: ValueProto) -> str:
-        if val.HasField("string_list_val"):
-            return ",".join(val.string_list_val.val)
-        elif val.HasField("bytes_list_val"):
-            return ",".join(map(str, val.bytes_list_val.val))
-        elif val.HasField("int64_list_val"):
-            return ",".join(map(str, val.int64_list_val.val))
-        elif val.HasField("float_list_val"):
-            return ",".join(map(str, val.float_list_val.val))
-        elif val.HasField("double_list_val"):
-            return ",".join(map(str, val.double_list_val.val))
-        else:
-            raise ValueError("Unsupported list value type")
-
     def retrieve_online_documents(
         self,
         config: RepoConfig,
@@ -434,7 +422,7 @@ def from_proto(sqlite_table_proto: SqliteTableProto) -> Any:
 
     def update(self):
         self.conn.execute(
-            f"CREATE TABLE IF NOT EXISTS {self.name} (entity_key BLOB, feature_name TEXT, value BLOB, event_ts timestamp, created_ts timestamp,  PRIMARY KEY(entity_key, feature_name))"
+            f"CREATE TABLE IF NOT EXISTS {self.name} (entity_key BLOB, feature_name TEXT, value BLOB, vector_value BLOB, event_ts timestamp, created_ts timestamp,  PRIMARY KEY(entity_key, feature_name))"
         )
         self.conn.execute(
             f"CREATE INDEX IF NOT EXISTS {self.name}_ek ON {self.name} (entity_key);"

sdk/python/tests/unit/online_store/test_online_retrieval.py

@@ -426,6 +426,7 @@ def test_get_online_documents() -> None:
     with runner.local_repo(
         get_example_repo("example_feature_repo_1.py"), "file"
     ) as store:
+        store.config.online_store.vss_enabled = True
         # Write some data to two tables
         document_embeddings_fv = store.get_feature_view(name="document_embeddings")
 
@@ -434,15 +435,11 @@ def test_get_online_documents() -> None:
         item_key = EntityKeyProto(
             join_keys=["item_id"], entity_values=[ValueProto(int64_val=0)]
         )
-        provider.online_write_batch(
-            config=store.config,
-            table=document_embeddings_fv,
-            data=[
+        data = [
                 (
                     item_key,
                     {
                         "Embeddings": [
-                            np.array(
                                 [
                                     0.17517076,
                                     -0.1259909,
@@ -454,9 +451,7 @@ def test_get_online_documents() -> None:
                                     0.01173803,
                                     -0.0573408,
                                     0.02616226,
-                                ]
-                            ),
-                            np.array(
+                                ],
                                 [
                                     0.18517076,
                                     -0.1259909,
@@ -468,9 +463,7 @@ def test_get_online_documents() -> None:
                                     0.01173803,
                                     -0.0573408,
                                     0.02616226,
-                                ]
-                            ),
-                            np.array(
+                                ],
                                 [
                                     0.19517076,
                                     -0.1259909,
@@ -482,9 +475,7 @@ def test_get_online_documents() -> None:
                                     0.01173803,
                                     -0.0573408,
                                     0.02616226,
-                                ]
-                            ),
-                            np.array(
+                                ],
                                 [
                                     0.20517076,
                                     -0.1259909,
@@ -497,17 +488,20 @@ def test_get_online_documents() -> None:
                                     -0.0573408,
                                     0.02616226,
                                 ]
-                            ),
-                        ]
+                        ],
                     },
                     datetime.utcnow(),
                     datetime.utcnow(),
                 )
-            ],
+            ]
+        provider.online_write_batch(
+            config=store.config,
+            table=document_embeddings_fv,
+            data=data,
             progress=None,
         )
 
-        query = np.array(
+        query_embedding = np.array(
             [
                 0.17517076,
                 -0.1259909,
@@ -523,7 +517,7 @@ def test_get_online_documents() -> None:
         )
         # Retrieve two features using two keys, one valid one non-existing
         result = store.retrieve_online_documents(
-            feature="document_embeddings:Embeddings", query=query, top_k=3
+            feature="document_embeddings:Embeddings", query=query_embedding, top_k=3
         ).to_dict()
 
         assert "Embeddings" in result