Skip to content

Commit

Permalink
Merge pull request #604 from couchbase/fix/lazy_fts
Browse files Browse the repository at this point in the history 
Don't create SQLite FTS or RTree tables until needed
  • Loading branch information
@pasin
pasin committed Feb 18, 2015
2 parents 578d270 + fecc442 commit f9c373d
Show file tree
Hide file tree
Showing 3 changed files with 161 additions and 36 deletions.
11 changes: 1 addition & 10 deletions Source/CBL_SQLiteStorage.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@
@implementation CBL_SQLiteStorage
{
NSString* _directory;
__weak CBLManager* _manager;
BOOL _readOnly;
NSCache* _docIDs;
}
Expand All @@ -64,7 +63,7 @@ @implementation CBL_SQLiteStorage


+ (void) initialize {
// Test the features of the actual SQLite implementation at runtime. This is necessary because
// Test the version of the actual SQLite implementation at runtime. Necessary because
// the app might be linked with a custom version of SQLite (like SQLCipher) instead of the
// system library, so the actual version/features may differ from what was declared in
// sqlite3.h at compile time.
Expand All @@ -81,11 +80,6 @@ + (void) initialize {
#endif
Assert(sqlite3_libversion_number() >= 3007000,
@"SQLite library is too old (%s); needs to be at least 3.7", sqlite3_libversion());
Assert(sqlite3_compileoption_used("SQLITE_ENABLE_FTS3")
|| sqlite3_compileoption_used("SQLITE_ENABLE_FTS4"),
@"SQLite isn't built with full-text indexing (FTS3 or FTS4)");
Assert(sqlite3_compileoption_used("SQLITE_ENABLE_RTREE"),
@"SQLite isn't built with geo-indexing (R-tree)");
}
}

Expand All @@ -107,7 +101,6 @@ - (BOOL) openInDirectory: (NSString*)directory
{
_directory = [directory copy];
_readOnly = readOnly;
_manager = manager;
NSString* path = [_directory stringByAppendingPathComponent: kDBFilename];
_fmdb = [[CBL_FMDatabase alloc] initWithPath: path];
_fmdb.dispatchQueue = manager.dispatchQueue;
Expand Down Expand Up @@ -325,8 +318,6 @@ CREATE TABLE info (\
key TEXT PRIMARY KEY,\
value TEXT);\
\
CREATE VIRTUAL TABLE fulltext USING fts4(content, tokenize=unicodesn);\
CREATE VIRTUAL TABLE bboxes USING rtree(rowid, x0, x1, y0, y1);\
PRAGMA user_version = 17"; // at the end, update user_version
//OPT: Would be nice to use partial indexes but that requires SQLite 3.8 and makes the
// db file only readable by SQLite 3.8+, i.e. the file would not be portable to iOS 8
Expand Down
89 changes: 64 additions & 25 deletions Source/CBL_SQLiteViewStorage.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ @implementation CBL_SQLiteViewStorage
int _viewID;
CBLViewCollation _collation;
NSString* _mapTableName;
BOOL _hasFullTextTrigger, _hasGeoTrigger;
BOOL _initializedFullTextSchema, _initializedRTreeSchema;
}

@synthesize name=_name, delegate=_delegate;
Expand Down Expand Up @@ -94,7 +94,7 @@ - (NSString*) queryString: (NSString*)sql {
}


- (CBLStatus) runStatements: (NSString*)sql error: (NSError**)outError {
- (BOOL) runStatements: (NSString*)sql error: (NSError**)outError {
CBL_SQLiteStorage* db = _dbStorage;
return [db inTransaction: ^CBLStatus {
if ([_dbStorage runStatements: [self queryString: sql] error: outError])
Expand Down Expand Up @@ -135,6 +135,56 @@ - (void) deleteIndex {
}


- (BOOL) createFullTextSchema {
if (_initializedFullTextSchema)
return YES;
if (!sqlite3_compileoption_used("SQLITE_ENABLE_FTS3")
&& !sqlite3_compileoption_used("SQLITE_ENABLE_FTS4")) {
Warn(@"Can't index full text: SQLite isn't built with FTS3 or FTS4 module");
return NO;
}
NSString* sql = @"\
CREATE VIRTUAL TABLE IF NOT EXISTS fulltext USING fts4(content, tokenize=unicodesn);\
CREATE INDEX IF NOT EXISTS 'maps_#_by_fulltext' ON 'maps_#'(fulltext_id); \
CREATE TRIGGER IF NOT EXISTS 'del_maps_#_fulltext' \
DELETE ON 'maps_#' WHEN old.fulltext_id not null BEGIN \
DELETE FROM fulltext WHERE rowid=old.fulltext_id| END";
//OPT: Would be nice to use partial indexes but that requires SQLite 3.8 and makes
// the db file only readable by SQLite 3.8+, i.e. the file would not be portable to
// iOS 8 which only has SQLite 3.7 :(
// On the above index we could add "WHERE fulltext_id not null".
NSError* error;
if (![self runStatements: sql error: &error]) {
Warn(@"Error initializing fts4 schema: %@", error);
return NO;
}
_initializedFullTextSchema = YES;
return YES;
}


- (BOOL) createRTreeSchema {
if (_initializedRTreeSchema)
return YES;
if (!sqlite3_compileoption_used("SQLITE_ENABLE_RTREE")) {
Warn(@"Can't geo-query: SQLite isn't built with Rtree module");
return NO;
}
NSString* sql = @"\
CREATE VIRTUAL TABLE IF NOT EXISTS bboxes USING rtree(rowid, x0, x1, y0, y1);\
CREATE TRIGGER IF NOT EXISTS 'del_maps_#_bbox' \
DELETE ON 'maps_#' WHEN old.bbox_id not null BEGIN \
DELETE FROM bboxes WHERE rowid=old.bbox_id| END";
NSError* error;
if (![self runStatements: sql error: &error]) {
Warn(@"Error initializing rtree schema: %@", error);
return NO;
}
_initializedRTreeSchema = YES;
return YES;
}


- (void) deleteView {
CBL_SQLiteStorage* db = _dbStorage;
[db inTransaction: ^CBLStatus {
Expand Down Expand Up @@ -446,36 +496,18 @@ - (CBLStatus) _emitKey: (UU id)key
BOOL ok;
NSString* text = specialKey.text;
if (text) {
if (![self createFullTextSchema])
return kCBLStatusNotImplemented;
ok = [fmdb executeUpdate: @"INSERT INTO fulltext (content) VALUES (?)", text];
fullTextID = @(fmdb.lastInsertRowId);
if (!_hasFullTextTrigger) {
// Create triggers only when needed, because they prevent the SQLite DELETE
// 'truncate optimization' <http://sqlite.org/lang_delete.html>
[fmdb executeUpdate: [self queryString:
@"CREATE TRIGGER IF NOT EXISTS 'del_maps_#_fulltext' "
"DELETE ON 'maps_#' WHEN old.fulltext_id not null BEGIN "
"DELETE FROM fulltext WHERE rowid=old.fulltext_id; END"]];
[fmdb executeUpdate: [self queryString: @"CREATE INDEX IF NOT EXISTS 'maps_#_by_fulltext' "
"ON 'maps_#'(fulltext_id)"]];
//OPT: Would be nice to use partial indexes but that requires SQLite 3.8 and makes the
// db file only readable by SQLite 3.8+, i.e. the file would not be portable to iOS 8
// which only has SQLite 3.7 :(
// On the above index we could add "WHERE fulltext_id not null".
_hasFullTextTrigger = YES;
}
} else {
if (![self createRTreeSchema])
return kCBLStatusNotImplemented;
CBLGeoRect rect = specialKey.rect;
ok = [fmdb executeUpdate: @"INSERT INTO bboxes (x0,y0,x1,y1) VALUES (?,?,?,?)",
@(rect.min.x), @(rect.min.y), @(rect.max.x), @(rect.max.y)];
bboxID = @(fmdb.lastInsertRowId);
geoKey = specialKey.geoJSONData;
if (!_hasGeoTrigger) {
[fmdb executeUpdate: [self queryString:
@"CREATE TRIGGER IF NOT EXISTS 'del_maps_#_bbox' "
"DELETE ON 'maps_#' WHEN old.bbox_id not null BEGIN "
"DELETE FROM bboxes WHERE rowid=old.bbox_id; END"]];
_hasGeoTrigger = YES;
}
}
if (!ok)
return dbStorage.lastDbError;
Expand Down Expand Up @@ -528,9 +560,12 @@ - (CBLStatus) _runQueryWithOptions: (const CBLQueryOptions*)options
NSMutableString* sql = [NSMutableString stringWithString: @"SELECT key, value, docid, revs.sequence"];
if (options->includeDocs)
[sql appendString: @", revid, json"];
if (options->bbox)
if (options->bbox) {
if (![self createRTreeSchema])
return kCBLStatusNotImplemented;
[sql appendFormat: @", bboxes.x0, bboxes.y0, bboxes.x1, bboxes.y1, maps_%@.geokey",
self.mapTableName];
}
[sql appendFormat: @" FROM 'maps_%@', revs, docs", self.mapTableName];
if (options->bbox)
[sql appendString: @", bboxes"];
Expand Down Expand Up @@ -765,6 +800,10 @@ - (CBLQueryIteratorBlock) regularQueryWithOptions: (CBLQueryOptions*)options
- (CBLQueryIteratorBlock) fullTextQueryWithOptions: (const CBLQueryOptions*)options
status: (CBLStatus*)outStatus
{
if (![self createFullTextSchema]) {
*outStatus = kCBLStatusNotImplemented;
return nil;
}
NSMutableString* sql = [@"SELECT docs.docid, 'maps_#'.sequence, 'maps_#'.fulltext_id, 'maps_#'.value, "
"offsets(fulltext)" mutableCopy];
if (options->fullTextSnippets)
Expand Down
97 changes: 96 additions & 1 deletion Unit-Tests/View_Tests.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -754,9 +754,104 @@ - (void) test14_LiveQuery_AddingNonIndexedDocsPriorCreatingLiveQuery {
[liveQuery removeObserver:observer forKeyPath:@"rows"];
}


#pragma mark - GEO


static NSDictionary* mkGeoPoint(double x, double y) {
return CBLGeoPointToJSON((CBLGeoPoint){x,y});
}

static NSDictionary* mkGeoRect(double x0, double y0, double x1, double y1) {
return CBLGeoRectToJSON((CBLGeoRect){{x0,y0}, {x1,y1}});
}

- (NSArray*) putGeoDocs {
return @[
[self createDocumentWithProperties: $dict({@"_id", @"22222"}, {@"key", @"two"})],
[self createDocumentWithProperties: $dict({@"_id", @"44444"}, {@"key", @"four"})],
[self createDocumentWithProperties: $dict({@"_id", @"11111"}, {@"key", @"one"})],
[self createDocumentWithProperties: $dict({@"_id", @"33333"}, {@"key", @"three"})],
[self createDocumentWithProperties: $dict({@"_id", @"55555"}, {@"key", @"five"})],
[self createDocumentWithProperties: $dict({@"_id", @"pdx"}, {@"key", @"Portland"},
{@"geoJSON", mkGeoPoint(-122.68, 45.52)})],
[self createDocumentWithProperties: $dict({@"_id", @"aus"}, {@"key", @"Austin"},
{@"geoJSON", mkGeoPoint(-97.75, 30.25)})],
[self createDocumentWithProperties: $dict({@"_id", @"mv"}, {@"key", @"Mountain View"},
{@"geoJSON", mkGeoPoint(-122.08, 37.39)})],
[self createDocumentWithProperties: $dict({@"_id", @"hkg"}, {@"geoJSON", mkGeoPoint(-113.91, 45.52)})],
[self createDocumentWithProperties: $dict({@"_id", @"diy"}, {@"geoJSON", mkGeoPoint(40.12, 37.53)})],
[self createDocumentWithProperties: $dict({@"_id", @"snc"}, {@"geoJSON", mkGeoPoint(-2.205, -80.98)})],

[self createDocumentWithProperties: $dict({@"_id", @"xxx"}, {@"geoJSON",
mkGeoRect(-115,-10, -90, 12)})],
];
}

- (void) test15_GeoQuery {
if (!self.isSQLiteDB)
return; //FIX: Geo support in ForestDB is not complete enough to pass this test

RequireTestCase(CBLGeometry);
RequireTestCase(CBL_View_Index);

CBLView* view = [db viewNamed: @"geovu"];
[view setMapBlock: MAPBLOCK({
if (doc[@"key"])
emit(doc[@"key"], nil);
if (doc[@"geoJSON"])
emit(CBLGeoJSONKey(doc[@"geoJSON"]), nil);
}) version: @"1"];

// Query before any docs are indexed:
CBLQuery* query = [view createQuery];
CBLGeoRect bbox = {{-100, 0}, {180, 90}};
query.boundingBox = bbox;
NSError* error;
NSArray* rows = [[query run: &error] allObjects];
AssertEqual(rows, @[]);

// Create docs:
[self putGeoDocs];

// Bounding-box query:
query = [view createQuery];
query.boundingBox = bbox;
rows = [[query run: &error] allObjects];
NSArray* rowsAsDicts = [rows my_map: ^(CBLQueryRow* row) {return row.asJSONDictionary;}];
NSArray* expectedRows = @[$dict({@"id", @"xxx"},
{@"geometry", mkGeoRect(-115, -10, -90, 12)},
{@"bbox", @[@-115, @-10, @-90, @12]}),
$dict({@"id", @"aus"},
{@"geometry", mkGeoPoint(-97.75, 30.25)},
{@"bbox", @[@-97.75, @30.25, @-97.75, @30.25]}),
$dict({@"id", @"diy"},
{@"geometry", mkGeoPoint(40.12, 37.53)},
{@"bbox", @[@40.12, @37.53, @40.12, @37.53]})];
AssertEqualish(rowsAsDicts, expectedRows);

CBLGeoQueryRow* row = rows[0];
AssertEq(row.boundingBox.min.x, -115);
AssertEq(row.boundingBox.min.y, -10);
AssertEq(row.boundingBox.max.x, -90);
AssertEq(row.boundingBox.max.y, 12);
AssertEqual(row.geometryType, @"Polygon");
AssertEqual(row.geometry, mkGeoRect(-115, -10, -90, 12));

row = rows[1];
AssertEq(row.boundingBox.min.x, -97.75);
AssertEq(row.boundingBox.min.y, 30.25);
AssertEqual(row.geometryType, @"Point");
AssertEqual(row.geometry, mkGeoPoint(-97.75, 30.25));
}



#pragma mark - OTHER

// Make sure that a database's map/reduce functions are shared with the shadow database instance
// running in the background server.
- (void) test15_SharedMapBlocks {
- (void) test16_SharedMapBlocks {
[db setFilterNamed: @"phil" asBlock: ^BOOL(CBLSavedRevision *revision, NSDictionary *params) {
return YES;
}];
Expand Down

0 comments on commit f9c373d

Please sign in to comment.