Build-To-Use: 3.3.0-20

CBL-5629: Update zlib to 1.3.1 (#2032)
CBL-5627: Update min MacOS version to 12.0 (#2033)
CBL-5539: Add an API to check if a vector index is trained or not (#2035)
CBL-5628: Update mbedtls to 2.28.8 (#2027)
374d485 Support latest vectorsearch (dev branch) and hybrid queries (#1980)
5c3c854 Lazy vector index updating (#1949)
CBL-5522: Port - N1QL Parser has exponential slowdown for redundant parentheses (#1984)
ab19634 Part of CBL 5579 in order to facilitate VS on .NET Android (#1993)
CBL-5507: Fix index-past-end in CookieStore (#1982)
CBL-5591: Binary Decoder to account for the new Logging object path (#1995)
294c3f8 Define _LIBCPP_REMOVE_TRANSITIVE_INCLUDES (#1987)
CBL-5438: DateTime standard format parser (#1977)
CBL-5498: Util changes for ConnectedClient (#1978)
CBL-5450: Remote rev KeepBody flag could be cleared accidentally
f8a8de2 Remove UWP builds from build scripts (#1954)
CBL-5425: Binary Encoder to encode the (Logging) object path (#1986)
CBL-4661: Fix ROUND_EVEN. (#1981)

C/Cpp_include/c4.hh

@@ -30,6 +30,7 @@
 #include "c4Database.hh"       // IWYU pragma: keep
 #include "c4Document.hh"       // IWYU pragma: keep
 #include "c4DocEnumerator.hh"  // IWYU pragma: keep
+#include "c4Index.hh"          // IWYU pragma: keep
 #include "c4Listener.hh"       // IWYU pragma: keep
 #include "c4Observer.hh"       // IWYU pragma: keep
 #include "c4Query.hh"          // IWYU pragma: keep

C/Cpp_include/c4Base.hh

@@ -52,6 +52,7 @@ namespace litecore {
     class ExclusiveTransaction;
     class FilePath;
     class KeyStore;
+    class LazyIndexUpdate;
     class LiveQuerier;
     class Query;
     class QueryEnumerator;

C/Cpp_include/c4Collection.hh

@@ -97,12 +97,16 @@ struct C4Collection
     virtual void createIndex(slice name, slice indexSpec, C4QueryLanguage indexLanguage, C4IndexType indexType,
                              const C4IndexOptions* C4NULLABLE indexOptions = nullptr) = 0;
 
+    virtual Retained<C4Index> getIndex(slice name) = 0;
+
     virtual void deleteIndex(slice name) = 0;
 
     virtual alloc_slice getIndexesInfo(bool fullInfo = true) const = 0;
 
     virtual alloc_slice getIndexRows(slice name) const = 0;
 
+    virtual bool isIndexTrained(slice name) const = 0;
+
     // Observers:
 
     using CollectionObserverCallback = std::function<void(C4CollectionObserver*)>;

C/Cpp_include/c4Index.hh

@@ -0,0 +1,103 @@
+//
+// c4Index.hh
+//
+// Copyright 2024-Present Couchbase, Inc.
+//
+// Use of this software is governed by the Business Source License included
+// in the file licenses/BSL-Couchbase.txt.  As of the Change Date specified
+// in that file, in accordance with the Business Source License, use of this
+// software will be governed by the Apache License, Version 2.0, included in
+// the file licenses/APL2.txt.
+//
+
+#pragma once
+#include "c4Base.hh"
+#include "c4IndexTypes.h"
+#include "fleece/InstanceCounted.hh"
+
+C4_ASSUME_NONNULL_BEGIN
+
+// ************************************************************************
+// This header is part of the LiteCore C++ API.
+// If you use this API, you must _statically_ link LiteCore;
+// the dynamic library only exports the C API.
+// ************************************************************************
+
+
+/** Represents an index. Acts as a factory for C4IndexUpdater objects. */
+struct C4Index
+    : public fleece::RefCounted
+    , public fleece::InstanceCountedIn<C4Index>
+    , C4Base {
+    C4Collection* getCollection() const { return _collection; }
+
+    slice getName() const { return _name; }
+
+#ifdef COUCHBASE_ENTERPRISE
+    /// Finds new or updated documents for which vectors need to be recomputed by the application.
+    /// If there are none, returns NULL.
+    /// @param limit  The maximum number of documents/vectors to return. If this is less than
+    ///               the total number, the rest will be returned on the next call to `beginUpdate`.
+    /// @warning  Do not call `beginUpdate` again until you're done with the returned updater;
+    ///           it's not valid to have more than one update in progress at a time.
+    Retained<struct C4IndexUpdater> beginUpdate(size_t limit);
+#endif
+
+  protected:
+    friend class litecore::CollectionImpl;
+    static Retained<C4Index> getIndex(C4Collection*, slice name);
+
+    Retained<C4Collection> _collection;
+    std::string            _name;
+};
+
+#ifdef COUCHBASE_ENTERPRISE
+
+/** Describes a set of index values that need to be computed by the application,
+    to update a lazy index after its Collection has changed.
+    You should:
+
+    1. Call `valueAt` for each of the `count` items to get the Fleece value, and:
+      1.1. Compute a vector from this value
+      1.2. Call `setVectorAt` with the resulting vector, or with nullptr if none.
+    2. Finally, open a transaction and call `finish` to apply the updates to the index.
+
+    If you need to abandon an update, simply release the updater without calling `finish`. */
+struct C4IndexUpdater final
+    : public fleece::RefCounted
+    , public fleece::InstanceCountedIn<C4IndexUpdater>
+    , C4Base {
+    /// The number of vectors to compute.
+    size_t count() const;
+
+    /// Returns the i'th value to compute a vector from.
+    /// This is the value of the expression in the index spec.
+    FLValue valueAt(size_t i) const;
+
+    /// Sets the vector for the i'th value. A NULL pointer means there is no vector and any
+    /// existing vector should be removed from the index.
+    void setVectorAt(size_t i, const float* C4NULLABLE vector, size_t dimension);
+
+    /// Tells the updater that the i'th vector can't be computed at this time, e.g. because of
+    /// a transient network error. The associated document will be returned again in the next
+    /// call to `C4Index::beginUpdate()`.
+    void skipVectorAt(size_t i);
+
+    /// Updates the index with the computed vectors, removes any index rows for which no vector
+    /// was given, and updates the index's latest sequence.
+    /// @returns  True if the index is now completely up-to-date; false if there are more vectors
+    ///           to index (including ones changed since the call to `C4Index::beginUpdate`.)
+    bool finish();
+
+  private:
+    friend struct C4IndexImpl;
+    C4IndexUpdater(Retained<litecore::LazyIndexUpdate>, C4Collection*);
+    ~C4IndexUpdater();
+
+    Retained<litecore::LazyIndexUpdate> _update;
+    Retained<C4Collection>              _collection;
+};
+
+#endif
+
+C4_ASSUME_NONNULL_END

C/c4.exp

@@ -319,6 +319,7 @@ _c4coll_getDocumentCount
 _c4coll_getLastSequence
 _c4coll_getDoc
 _c4coll_getDocBySequence
+_c4coll_getIndex
 _c4coll_isValid
 _c4coll_putDoc
 _c4coll_createDoc

C/c4CAPI.cc

@@ -21,6 +21,7 @@
 #include "c4DocEnumerator.h"
 #include "c4Document+Fleece.h"
 #include "c4Index.h"
+#include "c4Index.hh"
 #include "c4DocEnumerator.hh"
 #include "c4ExceptionUtils.hh"
 #include "c4Observer.hh"
@@ -30,6 +31,7 @@
 #include "c4Replicator.h"
 #include "c4Private.h"
 #include "NumConversion.hh"
+#include "CollectionImpl.hh"
 #include "StringUtil.hh"
 #include <sstream>
 
@@ -349,6 +351,15 @@ bool c4coll_createIndex(C4Collection* coll, C4String name, C4String indexSpec, C
     return tryCatch(outError, [&] { coll->createIndex(name, indexSpec, queryLanguage, indexType, indexOptions); });
 }
 
+C4Index* C4NULLABLE c4coll_getIndex(C4Collection* coll, C4String name, C4Error* C4NULLABLE outError) noexcept {
+    returnIfCollectionInvalid(coll, outError, nullptr);
+    return tryCatch<C4Index*>(outError, [&] {
+        auto index = coll->getIndex(name);
+        if ( !index ) c4error_return(LiteCoreDomain, kC4ErrorMissingIndex, nullslice, outError);
+        return std::move(index).detach();
+    });
+}
+
 bool c4coll_deleteIndex(C4Collection* coll, C4String name, C4Error* C4NULLABLE outError) noexcept {
     returnIfCollectionInvalid(coll, outError, false);
     return tryCatch(outError, [&] { coll->deleteIndex(name); });
@@ -541,6 +552,11 @@ bool c4db_createIndex2(C4Database* database, C4Slice name, C4Slice indexSpec, C4
     return c4coll_createIndex(coll, name, indexSpec, queryLanguage, indexType, indexOptions, outError);
 }
 
+bool c4coll_isIndexTrained(C4Collection* collection, C4Slice name, C4Error* outError) noexcept {
+    memset(outError, 0, sizeof(C4Error));
+    return tryCatch(outError, [=] { return collection->isIndexTrained(name); });
+}
+
 // semi-deprecated
 bool c4db_deleteIndex(C4Database* database, C4Slice name, C4Error* outError) noexcept {
     auto coll = database->getDefaultCollection();
@@ -1027,6 +1043,35 @@ C4QueryEnumerator* c4queryobs_getEnumerator(C4QueryObserver* obs, bool forget, C
     return asInternal(obs)->getEnumeratorImpl(forget, outError).detach();
 }
 
+#pragma mark - LAZY INDEX API: (EE)
+
+#ifdef COUCHBASE_ENTERPRISE
+
+C4IndexUpdater* C4NULLABLE c4index_beginUpdate(C4Index* index, size_t limit, C4Error* outError) noexcept {
+    return tryCatch<C4IndexUpdater*>(outError, [&] { return index->beginUpdate(limit).detach(); });
+}
+
+size_t c4indexupdater_count(C4IndexUpdater* update) noexcept { return update->count(); }
+
+FLValue c4indexupdater_valueAt(C4IndexUpdater* update, size_t i) noexcept {
+    return tryCatch<FLValue>(nullptr, [&] { return update->valueAt(i); });
+}
+
+bool c4indexupdater_setVectorAt(C4IndexUpdater* update, size_t i, const float vec[], size_t dimension,
+                                C4Error* outError) noexcept {
+    return tryCatch(outError, [&] { update->setVectorAt(i, vec, dimension); });
+}
+
+bool c4indexupdater_skipVectorAt(C4IndexUpdater* update, size_t i) noexcept {
+    return tryCatch(nullptr, [&] { update->skipVectorAt(i); });
+}
+
+bool c4indexupdater_finish(C4IndexUpdater* update, C4Error* outError) noexcept {
+    return tryCatch(outError, [&] { update->finish(); });
+}
+
+#endif
+
 #pragma mark - CERTIFICATE API: (EE)
 
 

C/c4Error.cc

@@ -194,28 +194,20 @@ __cold C4Error C4Error::fromException(const exception& x) noexcept {
     return ErrorTable::instance().makeError((C4ErrorDomain)e.domain, e.code, {e.what(), e.backtrace});
 }
 
-__cold C4Error C4Error::fromCurrentException() noexcept {
-    // This rigamarole recovers the current exception being thrown...
-    auto xp = std::current_exception();
-    if ( xp ) {
-        try {
-            std::rethrow_exception(xp);
-        } catch ( const std::exception& x ) {
-            // Now we have the exception, so we can record it in outError:
-            return C4Error::fromException(x);
-        } catch ( ... ) {}
-    }
-    return ErrorTable::instance().makeError(LiteCoreDomain, kC4ErrorUnexpectedError,
-                                            {"Unknown C++ exception", Backtrace::capture(1)});
+__cold C4Error C4Error::fromCurrentException() noexcept { return fromException(error::convertCurrentException()); }
+
+static string getMessage(const C4Error& c4err) {
+    auto info = ErrorTable::instance().copy(c4err);
+    return info ? info->message : "";
 }
 
-[[noreturn]] __cold void C4Error::raise() const {
-    if ( auto info = ErrorTable::instance().copy(*this); info ) {
-        throw error{error::Domain(domain), code, info->message, info->backtrace};
-    } else {
-        error::_throw(error::Domain(domain), code);
+namespace litecore {
+    error::error(const C4Error& c4err) : error(error::Domain(c4err.domain), c4err.code, getMessage(c4err)) {
+        if ( auto info = ErrorTable::instance().copy(c4err) ) backtrace = info->backtrace;
     }
-}
+}  // namespace litecore
+
+[[noreturn]] __cold void C4Error::raise() const { throw litecore::error(*this); }
 
 [[noreturn]] __cold void C4Error::raise(C4ErrorDomain domain, int code, const char* format, ...) {
     std::string message;

C/c4Index.cc

@@ -0,0 +1,88 @@
+//
+// c4Index.cc
+//
+// Copyright © 2024 Couchbase. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include "c4Index.hh"
+
+#include "CollectionImpl.hh"
+#include "DatabaseImpl.hh"
+#include "LazyIndex.hh"
+
+
+using namespace std;
+using namespace fleece;
+using namespace litecore;
+
+struct C4IndexImpl final : public C4Index {
+    C4IndexImpl(C4Collection* c, slice name) : _spec(asInternal(c)->keyStore().getIndex(name)) {
+        _collection = c;
+        _name       = name;
+    }
+
+#ifdef COUCHBASE_ENTERPRISE
+    Retained<C4IndexUpdater> beginUpdate(size_t limit) {
+        if ( !_lazy ) _lazy = new LazyIndex(asInternal(_collection)->keyStore(), _name);
+        Retained<LazyIndexUpdate> update = _lazy->beginUpdate(limit);
+        if ( update ) return new C4IndexUpdater(std::move(update), _collection);
+        else
+            return nullptr;
+    }
+#endif
+
+    optional<IndexSpec>           _spec;
+    Retained<litecore::LazyIndex> _lazy;
+};
+
+inline C4IndexImpl* asInternal(C4Index* index) { return static_cast<C4IndexImpl*>(index); }
+
+Retained<C4Index> C4Index::getIndex(C4Collection* c, slice name) {
+    Retained<C4IndexImpl> index = new C4IndexImpl(c, name);
+    if ( !index->_spec ) index = nullptr;
+    return index;
+}
+
+
+#ifdef COUCHBASE_ENTERPRISE
+
+Retained<C4IndexUpdater> C4Index::beginUpdate(size_t limit) { return asInternal(this)->beginUpdate(limit); }
+
+C4IndexUpdater::C4IndexUpdater(Retained<litecore::LazyIndexUpdate> u, C4Collection* c)
+    : _update(std::move(u)), _collection(c) {}
+
+C4IndexUpdater::~C4IndexUpdater() = default;
+
+size_t C4IndexUpdater::count() const { return _update->count(); }
+
+FLValue C4IndexUpdater::valueAt(size_t i) const { return _update->valueAt(i); }
+
+void C4IndexUpdater::setVectorAt(size_t i, const float* vector, size_t dimension) {
+    _update->setVectorAt(i, vector, dimension);
+}
+
+void C4IndexUpdater::skipVectorAt(size_t i) { return _update->skipVectorAt(i); }
+
+bool C4IndexUpdater::finish() {
+    auto                    db = _collection->getDatabase();
+    C4Database::Transaction txn(db);
+    bool                    done = _update->finish(asInternal(db)->transaction());
+    txn.commit();
+    _update     = nullptr;
+    _collection = nullptr;
+    return done;
+}
+
+#endif

C/c4_ee.exp

@@ -360,6 +360,7 @@ _c4coll_getDocumentCount
 _c4coll_getLastSequence
 _c4coll_getDoc
 _c4coll_getDocBySequence
+_c4coll_getIndex
 _c4coll_isValid
 _c4coll_putDoc
 _c4coll_createDoc
@@ -479,5 +480,11 @@ _c4keypair_privateKeyData
 _c4keypair_publicKeyData
 _c4keypair_publicKeyDigest
 
+_c4index_beginUpdate
+_c4indexupdater_count
+_c4indexupdater_valueAt
+_c4indexupdater_setVectorAt
+_c4indexupdater_finish
+
 # Apple specific
 _FLEncoder_WriteNSObject