diff --git a/README.md b/README.md index 10f9f43..214ea53 100644 --- a/README.md +++ b/README.md @@ -234,9 +234,7 @@ make benchmark Number of branches: 1024 branches (can be increased) -Number of commits per branch: 2^32 = 4,294,967,295 commits - -- This value can be increased to 64 bits +Number of commits per branch: 2^64 = 18,446,744,073,709,551,615 commits Concurrent db connections to the same db: XXX readers diff --git a/makefile b/makefile index a3bd53d..76b33f8 100644 --- a/makefile +++ b/makefile @@ -124,7 +124,7 @@ endif clean: rm -f *.o $(LIBRARY) $(LIBNICK1) $(LIBNICK2) $(LIBNICK3) $(LIBNICK4) $(SSHELL) -test: test/test.py +test: test/test.py test/test-64bit-commit-ids.py test/varint.py ifeq ($(OS),Windows_NT) ifeq ($(PY_HOME),) @echo "PY_HOME is not set" @@ -132,22 +132,40 @@ else cd $(PY_HOME)/DLLs && [ ! -f sqlite3-orig.dll ] && mv sqlite3.dll sqlite3-orig.dll || true cp litetree-0.1.dll $(PY_HOME)/DLLs/sqlite3.dll cp $(LMDBPATH)/lmdb.dll $(PY_HOME)/DLLs/lmdb.dll + cd test && python -mpip install lmdb cd test && python test.py -v + cd test && python test-64bit-commit-ids.py -v endif -else ifeq ($(OS),OSX) +else # not Windows +ifneq ($(shell python -c "import lmdb" 2> /dev/null; echo $$?),0) + sudo easy_install cffi + cd test && sudo easy_install lmdb +ifneq ($(shell python -c "import lmdb" 2> /dev/null; echo $$?),0) + git clone --depth=1 https://github.com/dw/py-lmdb + cd py-lmdb && sudo LMDB_FORCE_CPYTHON=1 python setup.py install +ifneq ($(shell python -c "import lmdb" 2> /dev/null; echo $$?),0) + sudo python -c "import cffi" + sudo python -c "import lmdb" +endif +endif +endif +ifeq ($(OS),OSX) ifneq ($(shell python -c "import pysqlite2.dbapi2" 2> /dev/null; echo $$?),0) ifneq ($(shell [ -d $(LIBPATH2) ]; echo $$?),0) @echo "run 'sudo make install' first" endif - git clone https://github.com/ghaering/pysqlite + git clone --depth=1 https://github.com/ghaering/pysqlite cd pysqlite && echo "include_dirs=$(INCPATH)" >> setup.cfg cd pysqlite && echo "library_dirs=$(LIBPATH2)" >> setup.cfg cd pysqlite && python setup.py build cd pysqlite && sudo python setup.py install endif cd test && python test.py -v -else + cd test && python test-64bit-commit-ids.py -v +else # Linux cd test && LD_LIBRARY_PATH=.. python test.py -v + cd test && LD_LIBRARY_PATH=.. python test-64bit-commit-ids.py -v +endif endif benchmark: test/benchmark.py @@ -165,7 +183,7 @@ ifneq ($(shell python -c "import pysqlite2.dbapi2" 2> /dev/null; echo $$?),0) ifneq ($(shell [ -d $(LIBPATH2) ]; echo $$?),0) @echo "run 'sudo make install' first" endif - git clone https://github.com/ghaering/pysqlite + git clone --depth=1 https://github.com/ghaering/pysqlite cd pysqlite && echo "include_dirs=$(INCPATH)" >> setup.cfg cd pysqlite && echo "library_dirs=$(LIBPATH2)" >> setup.cfg cd pysqlite && python setup.py build diff --git a/sqlite3.c b/sqlite3.c index 622cc56..825dace 100644 --- a/sqlite3.c +++ b/sqlite3.c @@ -15365,9 +15365,9 @@ struct branch_info { int visible; /* = not deleted */ char name[128]; /* the branch name */ int source_branch; /* where this branch starts from */ - u32 source_commit; /* the commit on the source branch where this branch starts at. this_branch.first_commit = source_commit + 1 */ - u32 last_commit; /* num_commits = last_commit - source_commit */ - u32 max_commit; /* selected by the user or 0 if none was specified */ + u64 source_commit; /* the commit on the source branch where this branch starts at. this_branch.first_commit = source_commit + 1 */ + u64 last_commit; /* num_commits = last_commit - source_commit */ + u64 max_commit; /* selected by the user or 0 if none was specified */ u32 commit_max_page; /* the maximum page number up to this commit */ u32 max_page; /* the maximum page number on this branch */ u32 txn_max_page; /* The max pgno on the current transaction */ @@ -15377,6 +15377,13 @@ struct branch_info { u8 is_new; /* if this branch was not saved yet */ }; +struct data_chunk { + struct data_chunk *next; + int total; /* total size of the buffer bellow */ + int used; /* used size */ + unsigned char buf[]; +}; + struct lmdb { struct global_lmdb *global; MDB_env *env; @@ -15394,7 +15401,7 @@ struct lmdb { struct branch_info* current_branch; /* pointer to an array item */ char current_branch_name[128]; /* if reloading the array this keeps the selected branch name */ // -- MAYBE NOT NEEDED !! - u32 current_branch_commit; /* same as above */ + u64 current_branch_commit; /* same as above */ }; typedef struct lmdb lmdb; @@ -15408,7 +15415,7 @@ SQLITE_PRIVATE int sqlite3BranchFind(lmdb *lmdb, char *name); SQLITE_PRIVATE int sqlite3BranchOpen(lmdb *lmdb, int branch_id, int open_txn); SQLITE_PRIVATE int sqlite3BranchGetMaxPage(lmdb *lmdb, branch_info *branch); -SQLITE_PRIVATE int sqlite3BranchGetCommitMaxPage(lmdb *lmdb, branch_info *branch, u32 commit_id, Pgno *pMaxPgno); +SQLITE_PRIVATE int sqlite3BranchGetCommitMaxPage(lmdb *lmdb, branch_info *branch, u64 commit_id, Pgno *pMaxPgno); SQLITE_PRIVATE int sqlite3BranchBeginReadTransaction(lmdb *lmdb, int check_for_updates, int *pChanged); SQLITE_PRIVATE void sqlite3BranchEndReadTransaction(lmdb *lmdb); @@ -15424,11 +15431,11 @@ SQLITE_PRIVATE int sqlite3BranchWritePages(lmdb *lmdb, PgHdr *pList, Pgno nTrun SQLITE_PRIVATE char * pragma_get_branch_list(lmdb *lmdb); #if 0 -SQLITE_PRIVATE int pragma_set_current_branch(sqlite3 *db, int iDb, char *zBranch, u32 iCommit); +SQLITE_PRIVATE int pragma_set_current_branch(sqlite3 *db, int iDb, char *zBranch, u64 iCommit); SQLITE_PRIVATE char * pragma_get_current_branch(sqlite3 *db, int iDb, Parse *pParse); SQLITE_PRIVATE int pragma_new_branch(sqlite3 *db, int iDb, char *zBranchName, char *zSourceBranch, char *zSourceCommit); SQLITE_PRIVATE int pragma_rename_branch(sqlite3 *db, int iDb, char *old_name, char *new_name); -SQLITE_PRIVATE int pragma_truncate_branch(sqlite3 *db, int iDb, char *name, u32 to_commit); +SQLITE_PRIVATE int pragma_truncate_branch(sqlite3 *db, int iDb, char *name, u64 to_commit); SQLITE_PRIVATE int pragma_delete_branch(sqlite3 *db, int iDb, char *name); #endif SQLITE_PRIVATE char * pragma_get_branch_info(sqlite3 *db, int iDb, char *name); @@ -30084,6 +30091,17 @@ static int compare2pow63(const char *zNum, int incr){ return c; } +static int compare2pow64(const char *zNum, int incr){ + int c = 0; + int i; + /* 01234567890123456789 */ + const char *pow64 = "18446744073709551615"; + for(i=0; c==0 && i<20; i++){ + c = (zNum[i*incr]-pow64[i])*10; + } + return c; +} + /* ** Convert zNum to a 64-bit signed integer. zNum must be decimal. This ** routine does *not* accept hexadecimal notation. @@ -30221,13 +30239,101 @@ SQLITE_PRIVATE int sqlite3DecOrHexToI64(const char *z, i64 *pOut){ } } +#ifdef UNUSED_FUNCTION SQLITE_PRIVATE u32 sqlite3AtoU32(const char *zNum){ i64 value; int rc = sqlite3Atoi64(zNum, &value, sqlite3Strlen30(zNum), SQLITE_UTF8); - if( rc || value < 0 || value > 0xffffffff ) return 0; + if( rc || value < 0 || value > 0xffffffffLL ) value = 0; + return value; +} +#endif + +SQLITE_PRIVATE int sqlite3AtoU64(const char *zNum, u64 *pNum, int length, u8 enc){ + int incr; + u64 u = 0; + int i; + int c = 0; + int nonNum = 0; /* True if input contains UTF16 with high byte non-zero */ + int rc; /* Baseline return code */ + const char *zStart; + const char *zEnd = zNum + length; + assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); + if( enc==SQLITE_UTF8 ){ + incr = 1; + }else{ + incr = 2; + assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); + for(i=3-enc; i='0' && c<='9'; i+=incr){ + u = u*10 + c - '0'; + } + + *pNum = u; + rc = 0; + + if( (i==0 && zStart==zNum) /* No digits */ + || nonNum /* UTF16 with high-order bytes non-zero */ + ){ + rc = 1; + }else if( &zNum[i]20*incr ? 1 : compare2pow64(zNum, incr); + if( c<=0 ){ + /* zNum is less than or equal to 18446744073709551615 so it fits */ + return rc; + }else{ + /* zNum is greater than 18446744073709551615 so it overflows */ + return 2; + } + } + +} + +SQLITE_PRIVATE u64 atou64(const char *zNum){ + u64 value; + int rc = sqlite3AtoU64(zNum, &value, sqlite3Strlen30(zNum), SQLITE_UTF8); + if( rc ) value = 0; return value; } +SQLITE_PRIVATE int isnumeric(const char *str){ + int found=0; + while( *str ){ + if( *str<'0' || *str>'9' ) return 0; + found = 1; /* at least 1 digit character */ + str++; + } + return found; +} + /* ** If zNum represents an integer that will fit in 32-bits, then set ** *pValue to that integer and return true. Otherwise return false. @@ -30686,6 +30792,171 @@ SQLITE_PRIVATE int sqlite3VarintLen(u64 v){ return i; } +/* +** Decode the varint in the first n bytes z[]. Write the integer value +** into *pResult and return the number of bytes in the varint. +** +** If the decode fails because there are not enough bytes in z[] then +** return 0; +*/ +SQLITE_PRIVATE int sqlite4GetVarint64( + const unsigned char *z, + int n, + u64 *pResult +){ + unsigned int x; + if( n<1 ) return 0; + if( z[0]<=240 ){ + *pResult = z[0]; + return 1; + } + if( z[0]<=248 ){ + if( n<2 ) return 0; + *pResult = (z[0]-241)*256 + z[1] + 240; + return 2; + } + if( n>24); + z[1] = (unsigned char)(y>>16); + z[2] = (unsigned char)(y>>8); + z[3] = (unsigned char)(y); +} + +/* +** Write a varint into z[]. The buffer z[] must be at least 9 characters +** long to accommodate the largest possible varint. Return the number of +** bytes of z[] used. +*/ +SQLITE_PRIVATE int sqlite4PutVarint64(unsigned char *z, u64 x){ + unsigned int w, y; + if( x<=240 ){ + z[0] = (unsigned char)x; + return 1; + } + if( x<=2287 ){ + y = (unsigned int)(x - 240); + z[0] = (unsigned char)(y/256 + 241); + z[1] = (unsigned char)(y%256); + return 2; + } + if( x<=67823 ){ + y = (unsigned int)(x - 2288); + z[0] = 249; + z[1] = (unsigned char)(y/256); + z[2] = (unsigned char)(y%256); + return 3; + } + y = (unsigned int)x; + w = (unsigned int)(x>>32); + if( w==0 ){ + if( y<=16777215 ){ + z[0] = 250; + z[1] = (unsigned char)(y>>16); + z[2] = (unsigned char)(y>>8); + z[3] = (unsigned char)(y); + return 4; + } + z[0] = 251; + varintWrite32(z+1, y); + return 5; + } + if( w<=255 ){ + z[0] = 252; + z[1] = (unsigned char)w; + varintWrite32(z+2, y); + return 6; + } + if( w<=65535 ){ + z[0] = 253; + z[1] = (unsigned char)(w>>8); + z[2] = (unsigned char)w; + varintWrite32(z+3, y); + return 7; + } + if( w<=16777215 ){ + z[0] = 254; + z[1] = (unsigned char)(w>>16); + z[2] = (unsigned char)(w>>8); + z[3] = (unsigned char)w; + varintWrite32(z+4, y); + return 8; + } + z[0] = 255; + varintWrite32(z+1, w); + varintWrite32(z+5, y); + return 9; +} + +#ifdef UNUSED_FUNCTION +/* +** Return the number of bytes required to encode value v as a varint. +*/ +SQLITE_PRIVATE int sqlite4VarintLen(u64 v){ + unsigned char aDummy[9]; + return sqlite4PutVarint64(aDummy, v); +} +#endif + +/* +** Read a varint from buffer z and set *pResult to the value read. +** Return the number of bytes read from the buffer. +*/ +SQLITE_PRIVATE int sqlite4GetVarint32(const unsigned char *z, int n, u32 *pResult){ + u64 value; + int ret = sqlite4GetVarint64(z, n, &value); + if( ret>0 ){ + if( value > UINT32_MAX ){ // 0xffffffffLL + ret = -1; + } else { + *pResult = value; + } + } + return ret; +} + +#ifdef UNUSED_FUNCTION +/* +** Encode v as a varint and write the result to buffer p. Return the +** number of bytes written. +*/ +SQLITE_PRIVATE int sqlite4PutVarint32(unsigned char *p, u32 v){ + return sqlite4PutVarint64(p, v); +} +#endif /* ** Read or write a four-byte big-endian integer value. @@ -49815,6 +50086,23 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ #define BRANCHERROR(X) #endif +#ifdef DEBUGBRANCHES +unsigned char * to_hex(unsigned char *buf, int size) { + static unsigned char *result = NULL; + char *p; + int i; + result = sqlite3_realloc(result, (size * 3) + 1); + if( result ){ + p = (char*)result; + for( i=0; iuseBranches ) return; assert( pWal->writeLock ); aWalData[0] = pWal->hdr.mxFrame; aWalData[1] = pWal->hdr.aFrameCksum[0]; @@ -60656,6 +60945,8 @@ SQLITE_PRIVATE void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){ SQLITE_PRIVATE int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ int rc = SQLITE_OK; + if( pWal->useBranches ) return rc; + assert( pWal->writeLock ); assert( aWalData[3]!=pWal->nCkpt || aWalData[0]<=pWal->hdr.mxFrame ); @@ -61401,6 +61692,7 @@ SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal){ #error "BYTE_ORDER not supported" #endif +#ifdef UNUSED_FUNCTION /* ** Converts the 32bit integer to big endian */ @@ -61426,6 +61718,66 @@ SQLITE_PRIVATE unsigned int tobe32(unsigned int input) { */ #define frombe32 tobe32 +#endif + +#ifdef UNUSED_FUNCTION +/* +** Add a new item to the data_chunk list +*/ +SQLITE_PRIVATE int chunk_list_add(struct data_chunk *list, void *data, int size){ + struct data_chunk *chunk = list; + unsigned char *p; + + if( !list || !data || size <= 0 || size > 255 ) return SQLITE_INTERNAL; + + while( chunk->next ) chunk = chunk->next; + + if( chunk->used + size > chunk->total ){ + chunk->next = (struct data_chunk *) sqlite3_malloc(4096); + if( !chunk->next ) return SQLITE_NOMEM; + chunk = chunk->next; + chunk->next = NULL; + chunk->total = 4096 - offsetof(struct data_chunk, buf); + chunk->used = 0; + } + + p = &chunk->buf[chunk->used]; + *p = size; p++; + memcpy(p, data, size); + chunk->used += (1 + size); + + return SQLITE_OK; +} + +/* +** Release the data_chunk list +*/ +SQLITE_PRIVATE void chunk_list_free(struct data_chunk *list){ + while( list ){ + struct data_chunk *next = list->next; + sqlite3_free(list); + list = next; + } +} +#endif + +/* +** Trims the string, modifying it and returning +** a pointer to its content. +*/ +SQLITE_PRIVATE char * trimstr(char *str){ + /* trim leading spaces */ + while( sqlite3Isspace(*str) ) str++; + if( *str ){ + /* trim trailing spaces */ + char *end = str + strlen(str) - 1; + while( str < end && sqlite3Isspace(*end) ) end--; + /* add the null terminator */ + end[1] = '\0'; + } + return str; +} + /* ** Returns the position of a string inside another, using ** case insensitive comparison. @@ -61695,32 +62047,26 @@ SQLITE_PRIVATE int mdb_str_get_str(MDB_txn *txn, MDB_dbi dbi, char *keyname, cha } /* -** Helper function to read int values from LMDB +** Helper function to read u64 values from LMDB */ -SQLITE_PRIVATE int mdb_str_get_int(MDB_txn *txn, MDB_dbi dbi, char *keyname, int *pvalue){ - MDB_val key, value; +SQLITE_PRIVATE int mdb_get_u64(MDB_txn *txn, MDB_dbi dbi, MDB_val *pkey, u64 *pvalue){ + MDB_val value; int rc; - BRANCHTRACE("mdb_str_get_int key: %s", keyname); - - if( !txn || !keyname || !pvalue ) return MDB_INVALID; + if( !txn || !pkey || !pvalue ) return MDB_INVALID; *pvalue = 0; - /* set the key */ - key.mv_data = keyname; - key.mv_size = strlen(keyname); - /* get the value */ - rc = mdb_get(txn, dbi, &key, &value); + rc = mdb_get(txn, dbi, pkey, &value); if( rc==MDB_SUCCESS ){ - if( value.mv_size == sizeof(int) ){ - *pvalue = *(int*)value.mv_data; - BRANCHTRACE("mdb_str_get_int value: %d", *pvalue); + int ret = sqlite4GetVarint64( (const unsigned char*)value.mv_data, value.mv_size, pvalue); + if( ret>0 ){ + BRANCHTRACE("mdb_get_varint value: %llu", *pvalue); } else { rc = MDB_BAD_VALSIZE; } } else if( rc==MDB_NOTFOUND ){ - BRANCHTRACE("mdb_str_get_int - NOT FOUND"); + BRANCHTRACE("mdb_get_varint - NOT FOUND"); } /* return the result */ @@ -61732,39 +62078,25 @@ SQLITE_PRIVATE int mdb_str_get_int(MDB_txn *txn, MDB_dbi dbi, char *keyname, int ** Helper function to read unsigned int values from LMDB */ SQLITE_PRIVATE int mdb_get_u32(MDB_txn *txn, MDB_dbi dbi, MDB_val *pkey, u32 *pvalue){ - MDB_val value; - int rc; - - BRANCHTRACE("mdb_get_u32"); - - if( !txn || !pkey || !pvalue ) return MDB_INVALID; - *pvalue = 0; - - /* get the value */ - rc = mdb_get(txn, dbi, pkey, &value); + u64 value; + int rc = mdb_get_u64(txn, dbi, pkey, &value); if( rc==MDB_SUCCESS ){ - if( value.mv_size == sizeof(u32) ){ - *pvalue = *(u32*)value.mv_data; - BRANCHTRACE("mdb_get_u32 value: %u", *pvalue); + if( value<=0xFFFFFFFFLL ){ + *pvalue = value; } else { rc = MDB_BAD_VALSIZE; } - } else if( rc==MDB_NOTFOUND ){ - BRANCHTRACE("mdb_get_u32 - NOT FOUND"); } - - /* return the result */ return rc; - } /* ** Helper function to read unsigned int values from LMDB */ -SQLITE_PRIVATE int mdb_str_get_u32(MDB_txn *txn, MDB_dbi dbi, char *keyname, u32 *pvalue){ +SQLITE_PRIVATE int mdb_str_get_u64(MDB_txn *txn, MDB_dbi dbi, char *keyname, u64 *pvalue){ MDB_val key; - BRANCHTRACE("mdb_str_get_u32 key: %s", keyname); + BRANCHTRACE("mdb_str_get_u64 key: %s", keyname); if( !txn || !keyname || !pvalue ) return MDB_INVALID; *pvalue = 0; @@ -61774,23 +62106,24 @@ SQLITE_PRIVATE int mdb_str_get_u32(MDB_txn *txn, MDB_dbi dbi, char *keyname, u32 key.mv_size = strlen(keyname); /* get the value */ - return mdb_get_u32(txn, dbi, &key, pvalue); + return mdb_get_u64(txn, dbi, &key, pvalue); } /* ** Helper function to read unsigned int values from LMDB */ -SQLITE_PRIVATE int mdb_u32_get_u32(MDB_txn *txn, MDB_dbi dbi, u32 ikey, u32 *pvalue){ +SQLITE_PRIVATE int mdb_str_get_u32(MDB_txn *txn, MDB_dbi dbi, char *keyname, u32 *pvalue){ MDB_val key; - BRANCHTRACE("mdb_u32_get_u32 %u", ikey); + BRANCHTRACE("mdb_str_get_u32 key: %s", keyname); - if( !txn ) return MDB_INVALID; + if( !txn || !keyname || !pvalue ) return MDB_INVALID; + *pvalue = 0; /* set the key */ - key.mv_data = &ikey; - key.mv_size = sizeof(u32); + key.mv_data = keyname; + key.mv_size = strlen(keyname); /* get the value */ return mdb_get_u32(txn, dbi, &key, pvalue); @@ -61798,35 +62131,42 @@ SQLITE_PRIVATE int mdb_u32_get_u32(MDB_txn *txn, MDB_dbi dbi, u32 ikey, u32 *pva } /* -** Helper function to write int values to LMDB +** Helper function to read int values from LMDB +** For now it is just calling the unsigned int get function */ -SQLITE_PRIVATE int mdb_str_put_int(MDB_txn *txn, MDB_dbi dbi, char *keyname, int ivalue){ - MDB_val key, value; +SQLITE_PRIVATE int mdb_str_get_int(MDB_txn *txn, MDB_dbi dbi, char *keyname, int *pvalue){ + return mdb_str_get_u32(txn, dbi, keyname, (u32*)pvalue); +} - BRANCHTRACE("mdb_str_put_int %s = %d", keyname, ivalue); +/* +** Helper function to read unsigned int values from LMDB +*/ +SQLITE_PRIVATE int mdb_u64_get_u32(MDB_txn *txn, MDB_dbi dbi, u64 ikey, u32 *pvalue){ + MDB_val key; + unsigned char buf[12]; - if( !txn || !keyname ) return MDB_INVALID; + BRANCHTRACE("mdb_u64_get_u32 %llu", ikey); - /* set the key */ - key.mv_data = keyname; - key.mv_size = strlen(keyname); + if( !txn ) return MDB_INVALID; - /* set the value */ - value.mv_data = &ivalue; - value.mv_size = sizeof(int); + /* set the key */ + //mdb_value_set_u64(&key, buf, ikey); + key.mv_data = buf; + key.mv_size = sqlite4PutVarint64(buf, ikey); - /* store the data */ - return mdb_put(txn, dbi, &key, &value, 0); + /* get the value */ + return mdb_get_u32(txn, dbi, &key, pvalue); } /* ** Helper function to write unsigned int values to LMDB */ -SQLITE_PRIVATE int mdb_str_put_u32(MDB_txn *txn, MDB_dbi dbi, char *keyname, u32 ivalue){ +SQLITE_PRIVATE int mdb_str_put_u64(MDB_txn *txn, MDB_dbi dbi, char *keyname, u64 ivalue){ MDB_val key, value; + unsigned char buf[12]; - BRANCHTRACE("mdb_str_put_u32 %s = %u", keyname, ivalue); + BRANCHTRACE("mdb_str_put_u64 %s = %llu", keyname, ivalue); if( !txn || !keyname ) return MDB_INVALID; @@ -61835,14 +62175,23 @@ SQLITE_PRIVATE int mdb_str_put_u32(MDB_txn *txn, MDB_dbi dbi, char *keyname, u32 key.mv_size = strlen(keyname); /* set the value */ - value.mv_data = &ivalue; - value.mv_size = sizeof(u32); + value.mv_data = buf; + value.mv_size = sqlite4PutVarint64(buf, ivalue); /* store the data */ return mdb_put(txn, dbi, &key, &value, 0); } +#define mdb_str_put_u32 mdb_str_put_u64 + +/* +** Helper function to write int values to LMDB +*/ +SQLITE_PRIVATE int mdb_str_put_int(MDB_txn *txn, MDB_dbi dbi, char *keyname, int ivalue){ + return mdb_str_put_u64(txn, dbi, keyname, ivalue); +} + /* ** Helper function to write null terminated string values to LMDB */ @@ -61869,20 +62218,21 @@ SQLITE_PRIVATE int mdb_str_put_str(MDB_txn *txn, MDB_dbi dbi, char *keyname, cha /* ** Helper function to write int values to LMDB */ -SQLITE_PRIVATE int mdb_u32_put_u32(MDB_txn *txn, MDB_dbi dbi, u32 ikey, u32 ivalue){ +SQLITE_PRIVATE int mdb_u64_put_u32(MDB_txn *txn, MDB_dbi dbi, u64 ikey, u32 ivalue){ MDB_val key, value; + unsigned char buf1[12], buf2[12]; - BRANCHTRACE("mdb_u32_put_u32 %u = %u", ikey, ivalue); + BRANCHTRACE("mdb_u64_put_u32 %llu = %u", ikey, ivalue); if( !txn ) return MDB_INVALID; /* set the key */ - key.mv_data = &ikey; - key.mv_size = sizeof(u32); + key.mv_data = buf1; + key.mv_size = sqlite4PutVarint64(buf1, ikey); /* set the value */ - value.mv_data = &ivalue; - value.mv_size = sizeof(u32); + value.mv_data = buf2; + value.mv_size = sqlite4PutVarint64(buf2, ivalue); /* store the data */ return mdb_put(txn, dbi, &key, &value, 0); @@ -61948,8 +62298,9 @@ SQLITE_PRIVATE int sqlite3BranchInit(Pager *pPager){ } /* open the main database */ - if( (rc=mdb_txn_begin(lmdb->env, NULL, 0, &lmdb->txn)) != MDB_SUCCESS ) goto loc_failed; - if( (rc=mdb_dbi_open(lmdb->txn, NULL, MDB_CREATE, &lmdb->main_db)) != MDB_SUCCESS ) goto loc_failed; + if( (rc=mdb_txn_begin(lmdb->env, NULL, MDB_RDONLY, &lmdb->txn)) != MDB_SUCCESS ) goto loc_failed; + rc = mdb_dbi_open(lmdb->txn, NULL, 0, &lmdb->main_db); + if( rc!=MDB_SUCCESS && rc!=MDB_NOTFOUND ) goto loc_failed; if( (rc=mdb_txn_commit(lmdb->txn)) != MDB_SUCCESS ) goto loc_failed; lmdb->txn = 0; @@ -62005,7 +62356,7 @@ SQLITE_PRIVATE int sqlite3BranchInit(Pager *pPager){ SQLITE_PRIVATE int sqlite3BranchFind(lmdb *lmdb, char *name){ int i; - BRANCHTRACE("sqlite3BranchFind: %s", name); + BRANCHTRACE("sqlite3BranchFind [%s]", name); if( !lmdb || !name ) return -1; @@ -62043,7 +62394,7 @@ UPDATE ... <- move to write txn!!! no problem on single access to the db, but a ** Update the current branch in use ** If the branch with the given name is not found, don't change the current branch */ -SQLITE_PRIVATE int set_current_branch(lmdb *lmdb, char *name, u32 max_commit){ +SQLITE_PRIVATE int set_current_branch(lmdb *lmdb, char *name, u64 max_commit){ branch_info *branch; int branch_id; @@ -62148,6 +62499,7 @@ SQLITE_PRIVATE int put_branch_int(lmdb *lmdb, int branch_id, char *key, int valu return mdb_str_put_int(lmdb->txn, lmdb->main_db, buf, value); } +#ifdef UNUSED_FUNCTION /* ** Helper function to put value to LMDB main db */ @@ -62156,6 +62508,16 @@ SQLITE_PRIVATE int put_branch_u32(lmdb *lmdb, int branch_id, char *key, u32 valu sqlite3_snprintf(sizeof(buf), buf, "b%d.%s", branch_id, key); return mdb_str_put_u32(lmdb->txn, lmdb->main_db, buf, value); } +#endif + +/* +** Helper function to put value to LMDB main db +*/ +SQLITE_PRIVATE int put_branch_u64(lmdb *lmdb, int branch_id, char *key, u64 value){ + char buf[128]; + sqlite3_snprintf(sizeof(buf), buf, "b%d.%s", branch_id, key); + return mdb_str_put_u64(lmdb->txn, lmdb->main_db, buf, value); +} /* ** Helper function to put value to LMDB main db @@ -62175,6 +62537,7 @@ SQLITE_PRIVATE int get_branch_int(lmdb *lmdb, int branch_id, char *key, int *pva return mdb_str_get_int(lmdb->txn, lmdb->main_db, buf, pvalue); } +#ifdef UNUSED_FUNCTION /* ** Helper function to get value from LMDB main db */ @@ -62183,6 +62546,16 @@ SQLITE_PRIVATE int get_branch_u32(lmdb *lmdb, int branch_id, char *key, u32 *pva sqlite3_snprintf(sizeof(buf), buf, "b%d.%s", branch_id, key); return mdb_str_get_u32(lmdb->txn, lmdb->main_db, buf, pvalue); } +#endif + +/* +** Helper function to get value from LMDB main db +*/ +SQLITE_PRIVATE int get_branch_u64(lmdb *lmdb, int branch_id, char *key, u64 *pvalue){ + char buf[128]; + sqlite3_snprintf(sizeof(buf), buf, "b%d.%s", branch_id, key); + return mdb_str_get_u64(lmdb->txn, lmdb->main_db, buf, pvalue); +} /* ** Helper function to get value from LMDB main db @@ -62237,8 +62610,8 @@ SQLITE_PRIVATE int load_branch(lmdb *lmdb, int branch_id){ /* ...and load the values */ if( (rc = get_branch_str(lmdb, branch_id, "name", branch->name )) != MDB_SUCCESS && rc!=MDB_NOTFOUND ) goto loc_failed; if( (rc = get_branch_int(lmdb, branch_id, "source_branch", &branch->source_branch)) != MDB_SUCCESS && rc!=MDB_NOTFOUND ) goto loc_failed; - if( (rc = get_branch_u32(lmdb, branch_id, "source_commit", &branch->source_commit)) != MDB_SUCCESS && rc!=MDB_NOTFOUND ) goto loc_failed; - if( (rc = get_branch_u32(lmdb, branch_id, "last_commit", &branch->last_commit )) != MDB_SUCCESS && rc!=MDB_NOTFOUND ) goto loc_failed; + if( (rc = get_branch_u64(lmdb, branch_id, "source_commit", &branch->source_commit)) != MDB_SUCCESS && rc!=MDB_NOTFOUND ) goto loc_failed; + if( (rc = get_branch_u64(lmdb, branch_id, "last_commit", &branch->last_commit )) != MDB_SUCCESS && rc!=MDB_NOTFOUND ) goto loc_failed; BRANCHTRACE("load_branch OK"); @@ -62278,8 +62651,8 @@ SQLITE_PRIVATE int save_branch(lmdb *lmdb, int branch_id){ if( (rc = put_branch_str(lmdb, branch_id, "name", branch->name )) != MDB_SUCCESS ) goto loc_failed; if( (rc = put_branch_int(lmdb, branch_id, "visible", branch->visible )) != MDB_SUCCESS ) goto loc_failed; if( (rc = put_branch_int(lmdb, branch_id, "source_branch", branch->source_branch)) != MDB_SUCCESS ) goto loc_failed; - if( (rc = put_branch_u32(lmdb, branch_id, "source_commit", branch->source_commit)) != MDB_SUCCESS ) goto loc_failed; - if( (rc = put_branch_u32(lmdb, branch_id, "last_commit", branch->last_commit )) != MDB_SUCCESS ) goto loc_failed; + if( (rc = put_branch_u64(lmdb, branch_id, "source_commit", branch->source_commit)) != MDB_SUCCESS ) goto loc_failed; + if( (rc = put_branch_u64(lmdb, branch_id, "last_commit", branch->last_commit )) != MDB_SUCCESS ) goto loc_failed; /* store the max branch id */ if( (rc = mdb_str_put_int(lmdb->txn, lmdb->main_db, "last_branch_id", lmdb->num_branches)) != MDB_SUCCESS ) goto loc_failed; @@ -62367,7 +62740,7 @@ SQLITE_PRIVATE int sqlite3BranchOpen(lmdb *lmdb, int branch_id, int open_txn){ char zMaxPgDb[64]; int rc=SQLITE_ERROR; - BRANCHTRACE("sqlite3BranchOpen - branch id: %d", branch_id); + BRANCHTRACE("sqlite3BranchOpen [%s] id=%d", lmdb->branches[branch_id].name, branch_id); if( !lmdb || branch_id<=0 || branch_id>lmdb->num_branches ) return SQLITE_ERROR; @@ -62388,8 +62761,8 @@ SQLITE_PRIVATE int sqlite3BranchOpen(lmdb *lmdb, int branch_id, int open_txn){ } /* open the pages subdb and the maxpage subdb */ - if( (rc=mdb_dbi_open(lmdb->txn, zPagesDb, MDB_CREATE , &branch->pages_db)) != MDB_SUCCESS ) goto loc_lmdb_failed; - if( (rc=mdb_dbi_open(lmdb->txn, zMaxPgDb, MDB_CREATE|MDB_INTEGERKEY, &branch->maxpg_db)) != MDB_SUCCESS ) goto loc_lmdb_failed; + if( (rc=mdb_dbi_open(lmdb->txn, zPagesDb, MDB_CREATE, &branch->pages_db)) != MDB_SUCCESS ) goto loc_lmdb_failed; + if( (rc=mdb_dbi_open(lmdb->txn, zMaxPgDb, MDB_CREATE, &branch->maxpg_db)) != MDB_SUCCESS ) goto loc_lmdb_failed; /* open the subdbs from the source branches recursively */ if( branch->source_branch ){ @@ -62760,6 +63133,7 @@ SQLITE_PRIVATE int sqlite3BranchEndWriteTransaction(lmdb *lmdb, int reopen_read_ ** Commit a write transaction. */ SQLITE_PRIVATE int sqlite3BranchCommitTransaction(lmdb *lmdb, int reopen_read_txn){ + int change_counter; int rc; BRANCHTRACE("sqlite3BranchCommitTransaction"); @@ -62770,7 +63144,12 @@ SQLITE_PRIVATE int sqlite3BranchCommitTransaction(lmdb *lmdb, int reopen_read_tx if( !lmdb->inWriteTxn ) return SQLITE_OK; /* update the global change counter */ - rc = mdb_str_put_int(lmdb->txn, lmdb->main_db, "change_counter", lmdb->db_change_counter + 1); + if( lmdb->db_change_counter < 0x7FFFFFFE ){ + change_counter = lmdb->db_change_counter + 1; + } else { + change_counter = 1; /* restart the change counter */ + } + rc = mdb_str_put_int(lmdb->txn, lmdb->main_db, "change_counter", change_counter); if( rc!=MDB_SUCCESS ) return lmdb_error(rc); /* commit the transaction */ @@ -62780,7 +63159,7 @@ SQLITE_PRIVATE int sqlite3BranchCommitTransaction(lmdb *lmdb, int reopen_read_tx if( rc!=MDB_SUCCESS ) return lmdb_error(rc); /* update the change counter in memory */ - lmdb->db_change_counter++; + lmdb->db_change_counter = change_counter; /* close the dbs that must be closed */ sqlite3BranchCommitTempHandles(lmdb); @@ -62833,23 +63212,70 @@ SQLITE_PRIVATE int sqlite3BranchUndoChanges(lmdb *lmdb){ } -#define PAGE_DB_KEY_SIZE 8 /* 32bit pgno + 32bit commit_id */ +/* +** Builds the key used in the pages db for each branch +*/ +SQLITE_PRIVATE void mdb_value_set_u64(MDB_val *key, char *buf, u64 value){ + key->mv_data = buf; + key->mv_size = sqlite4PutVarint64((unsigned char *)buf, value); + BRANCHTRACE("set_varint value: %llu len: %d bytes", value, key->mv_size); +} + +/* +** Read an unsigned int from the key +*/ +SQLITE_PRIVATE int mdb_value_get_u64(MDB_val *key, u64 *pvalue){ + int n; + if( !key || !key->mv_data ) return SQLITE_INTERNAL; + n = sqlite4GetVarint64((unsigned char*)key->mv_data, key->mv_size, pvalue); + return n > 0 ? SQLITE_OK : SQLITE_CORRUPT; +} + +/* +** Read an unsigned int from the key +*/ +SQLITE_PRIVATE int mdb_value_get_u32(MDB_val *key, u32 *pvalue){ + int n; + if( !key || !key->mv_data ) return SQLITE_INTERNAL; + n = sqlite4GetVarint32((unsigned char*)key->mv_data, key->mv_size, pvalue); + return n > 0 ? SQLITE_OK : SQLITE_CORRUPT; +} + +#define PAGE_DB_KEY_MAX_SIZE 32 /* 2 varints = 2 * 9 bytes = 18 bytes */ -#define pgno_from_key(key) frombe32(*(u32*)key.mv_data) -#define commit_from_key(key) frombe32(*(u32*)&((char*)key.mv_data)[4]) +/* +** Retrieves the pgno from the key +*/ +SQLITE_PRIVATE int pgno_from_key(MDB_val *key, Pgno *pvalue){ + return mdb_value_get_u32(key, pvalue); +} + +/* +** Retrieves the commit id from the key +*/ +SQLITE_PRIVATE int commit_from_key(MDB_val *key, u64 *pvalue){ + Pgno pgno; + int n; + if( !key || !key->mv_data ) return SQLITE_INTERNAL; + n = sqlite4GetVarint32( (unsigned char*)key->mv_data, key->mv_size, &pgno); + if( n<=0 ) return SQLITE_CORRUPT; + n = sqlite4GetVarint64( &((unsigned char*)key->mv_data)[n], key->mv_size-n, pvalue); + return n > 0 ? SQLITE_OK : SQLITE_CORRUPT; +} /* ** Builds the key used in the pages db for each branch */ -SQLITE_PRIVATE void branch_set_page_key(MDB_val *key, char *buf, Pgno pgno, u32 commit_id){ +SQLITE_PRIVATE void branch_set_page_key(MDB_val *key, char *buf, Pgno pgno, u64 commit_id){ + int len=0; - *((u32*)buf ) = tobe32(pgno); - *((u32*)&buf[4]) = tobe32(commit_id); + len += sqlite4PutVarint64( (unsigned char *) &buf[len], pgno); + len += sqlite4PutVarint64( (unsigned char *) &buf[len], commit_id); key->mv_data = buf; - key->mv_size = PAGE_DB_KEY_SIZE; + key->mv_size = len; - BRANCHTRACE("key: %02x %02x %02x %02x %02x %02x %02x %02x", buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], buf[7]); + BRANCHTRACE("built page_db varint key pgno=%u commit_id=%llu len: %d bytes", pgno, commit_id, len); } @@ -62866,7 +63292,7 @@ SQLITE_PRIVATE int sqlite3BranchGetMaxPage( MDB_val key, value; MDB_cursor *cursor=0; - BRANCHTRACE("sqlite3BranchGetMaxPage"); + BRANCHTRACE("sqlite3BranchGetMaxPage [%s] id=%d", branch->name, branch->id); if( !lmdb || !branch ) goto loc_exit; @@ -62875,13 +63301,12 @@ SQLITE_PRIVATE int sqlite3BranchGetMaxPage( rc = mdb_cursor_open(lmdb->txn, branch->pages_db, &cursor); if( rc!=MDB_SUCCESS ) goto loc_failed; + /* move to the last record */ switch( (rc=mdb_cursor_get(cursor, &key, &value, MDB_LAST)) ){ case MDB_SUCCESS: - /* check the key size */ - if( key.mv_size != PAGE_DB_KEY_SIZE ) goto loc_exit; - /* read the key */ - BRANCHTRACE("last key: %02x %02x %02x %02x", ((char*)key.mv_data)[0], ((char*)key.mv_data)[1], ((char*)key.mv_data)[2], ((char*)key.mv_data)[3]); - branch->max_page = pgno_from_key(key); + /* get the page number */ + rc = pgno_from_key(&key, &branch->max_page); + if( rc!=SQLITE_OK ) goto loc_exit; break; case MDB_NOTFOUND: /* the db is empty */ @@ -62919,14 +63344,15 @@ SQLITE_PRIVATE int sqlite3BranchGetMaxPage( SQLITE_PRIVATE int sqlite3BranchGetCommitMaxPage( lmdb *lmdb, /* LMDB handle */ branch_info *branch, /* The branch info */ - u32 commit_id, /* The commit id */ + u64 commit_id, /* The commit id */ Pgno *pMaxPgno /* The max pgno will be written here */ ){ int rc=SQLITE_ERROR; MDB_val key, value; MDB_cursor *cursor=0; + char buf[12]; - BRANCHTRACE("sqlite3BranchGetCommitMaxPage on branch [%s] up to commit %u", branch->name, commit_id); + BRANCHTRACE("sqlite3BranchGetCommitMaxPage on branch [%s] up to commit %llu", branch->name, commit_id); if( !lmdb || !branch || !pMaxPgno ) goto loc_exit; @@ -62934,8 +63360,7 @@ SQLITE_PRIVATE int sqlite3BranchGetCommitMaxPage( if( rc!=MDB_SUCCESS ) goto loc_failed; /* set the key */ - key.mv_data = &commit_id; - key.mv_size = sizeof(commit_id); + mdb_value_set_u64(&key, buf, commit_id); /* search for the specific key */ switch( (rc=mdb_cursor_get(cursor, &key, &value, MDB_SET_KEY)) ){ @@ -62955,22 +63380,20 @@ SQLITE_PRIVATE int sqlite3BranchGetCommitMaxPage( loc_exit: + if( rc==SQLITE_OK ){ + BRANCHTRACE("sqlite3BranchGetCommitMaxPage returning max_page=%u", *pMaxPgno); + } else { + BRANCHTRACE("sqlite3BranchGetCommitMaxPage failed"); + } + if( cursor ) mdb_cursor_close(cursor); return rc; loc_found: - /* check the key size */ - if( key.mv_size != sizeof(u32) ){ rc = SQLITE_CORRUPT; goto loc_exit; } - - /* check the value size */ - if( value.mv_size != sizeof(u32) ){ rc = SQLITE_CORRUPT; goto loc_exit; } - /* read the value */ - *pMaxPgno = *(u32*)value.mv_data; - BRANCHTRACE("sqlite3BranchGetCommitMaxPage returning max_page=%u", *pMaxPgno); + rc = mdb_value_get_u32(&value, pMaxPgno); - rc = SQLITE_OK; goto loc_exit; loc_notfound: @@ -62999,16 +63422,16 @@ SQLITE_PRIVATE int sqlite3BranchFindPage( lmdb *lmdb, /* LMDB handle */ Pgno pgno, /* Database page number to read data for */ branch_info *branch, - u32 max_commit, /* Database page number to read data for */ + u64 max_commit, /* Database page number to read data for */ u8 **ppData /* OUT: store the pointer to the data here */ ){ int rc=SQLITE_ERROR; - char buf[PAGE_DB_KEY_SIZE]; + char buf[PAGE_DB_KEY_MAX_SIZE]; MDB_val key, value; MDB_cursor *cursor=0; Pgno this_pgno; - BRANCHTRACE("sqlite3BranchFindPage branch=%s (id=%d) max_commit=%u", branch->name, branch->id, max_commit); + BRANCHTRACE("sqlite3BranchFindPage [%s] id=%d pgno=%u max_commit=%llu", branch->name, branch->id, pgno, max_commit); if( !lmdb || pgno<=0 || !branch || !ppData ) return rc; @@ -63032,8 +63455,10 @@ SQLITE_PRIVATE int sqlite3BranchFindPage( /* check the previous key */ switch( (rc=mdb_cursor_get(cursor, &key, &value, MDB_PREV)) ){ case MDB_SUCCESS: + /* read the pgno */ + rc = pgno_from_key(&key, &this_pgno); + if( rc!=SQLITE_OK ) goto loc_exit; /* has this previous key the same pgno? */ - this_pgno = pgno_from_key(key); if( this_pgno == pgno ) goto loc_found; /* continue to notfound bellow */ case MDB_NOTFOUND: @@ -63052,7 +63477,13 @@ SQLITE_PRIVATE int sqlite3BranchFindPage( loc_found: - BRANCHTRACE("sqlite3BranchFindPage FOUND - from commit %u", commit_from_key(key)); +#ifdef DEBUGBRANCHES + { + u64 commit_id=0; + commit_from_key(&key, &commit_id); + BRANCHTRACE("sqlite3BranchFindPage FOUND - from commit %llu", commit_id); + } +#endif /* check the value size */ if( value.mv_size != lmdb->pageSize-LMDB_PAGE_RESERVED_SPACE ){ @@ -63092,7 +63523,7 @@ SQLITE_PRIVATE int sqlite3BranchGetPage( u8 **ppData /* OUT: store the pointer to the data here */ ){ branch_info *current_branch; - u32 max_commit; + u64 max_commit; int rc=SQLITE_ERROR; BRANCHTRACE("sqlite3BranchGetPage pgno=%d", pgno); @@ -63167,8 +63598,8 @@ SQLITE_PRIVATE int sqlite3BranchWritePages( PgHdr *p; /* Iterator to run through pList with. */ branch_info *branch; MDB_val key, value; - char buf[PAGE_DB_KEY_SIZE]; - u32 current_commit; + char buf[PAGE_DB_KEY_MAX_SIZE]; + u64 current_commit; BRANCHTRACE("sqlite3BranchWritePages nTruncate=%d isCommit=%d", nTruncate, isCommit); @@ -63200,7 +63631,7 @@ SQLITE_PRIVATE int sqlite3BranchWritePages( /* iterate through all the pages to be stored */ for(p=pList; p; p=p->pDirty){ - BRANCHTRACE("commit %u - writting page %u", current_commit, p->pgno); + BRANCHTRACE("commit %llu - writting page %u", current_commit, p->pgno); /* set the key */ branch_set_page_key(&key, buf, p->pgno, current_commit); @@ -63230,12 +63661,12 @@ SQLITE_PRIVATE int sqlite3BranchWritePages( if( branch->txn_max_page > branch->max_page || nTruncate < branch->max_page ){ /* save the number of pages on this commit */ - rc = mdb_u32_put_u32(lmdb->txn, branch->maxpg_db, current_commit, max_page); + rc = mdb_u64_put_u32(lmdb->txn, branch->maxpg_db, current_commit, max_page); if( rc ) return lmdb_error(rc); } /* update the last_commit for this branch */ - rc = put_branch_u32(lmdb, branch->id, "last_commit", current_commit); + rc = put_branch_u64(lmdb, branch->id, "last_commit", current_commit); if( rc ) return lmdb_error(rc); /* commit the transaction */ @@ -63275,14 +63706,14 @@ SQLITE_PRIVATE int sqlite3BranchCheckReloadDb(Pager *pPager, int keep_txn_open){ // PRAGMA branch_result=json // PRAGMA branch_result=rows -SQLITE_PRIVATE int pragma_set_current_branch(sqlite3 *db, int iDb, char *zBranch, u32 iCommit){ +SQLITE_PRIVATE int pragma_set_current_branch(sqlite3 *db, int iDb, char *zBranch, u64 iCommit){ Btree *pBtree = getBtreeFromiDb(db, iDb); Pager *pPager = getPagerFromBtree(pBtree); lmdb *lmdb; branch_info *branch; int rc; - BRANCHTRACE("pragma_set_current_branch to %s at commit %d", zBranch, iCommit); + BRANCHTRACE("pragma_set_current_branch to %s at commit %llu", zBranch, iCommit); if( !pPager || !pPager->lmdb ) return SQLITE_ERROR; lmdb = pPager->lmdb; @@ -63345,7 +63776,7 @@ SQLITE_PRIVATE char * pragma_get_current_branch(sqlite3 *db, int iDb, Parse *pPa if( current_branch->visible ){ if( current_branch->max_commit > 0 ){ /* returns the branch name and the max commit */ - sqlite3_snprintf(sizeof(name), name, "%s.%d", current_branch->name, current_branch->max_commit); + sqlite3_snprintf(sizeof(name), name, "%s.%llu", current_branch->name, current_branch->max_commit); pname = name; } else { /* returns just the branch name */ @@ -63361,7 +63792,7 @@ SQLITE_PRIVATE int pragma_new_branch(sqlite3 *db, int iDb, char *zBranchName, ch Btree *pBtree = getBtreeFromiDb(db, iDb); Pager *pPager = getPagerFromBtree(pBtree); lmdb *lmdb; - u32 iSourceCommit; + u64 iSourceCommit; int source_branch_id, i, rc; branch_info *source_branch; branch_info *new_branch=0; @@ -63374,7 +63805,17 @@ SQLITE_PRIVATE int pragma_new_branch(sqlite3 *db, int iDb, char *zBranchName, ch if( lmdb->inReadTxn || lmdb->inWriteTxn ) return SQLITE_MISUSE; /* check if the name is valid */ + if( !zBranchName ) return SQLITE_MISUSE; + zBranchName = trimstr(zBranchName); + switch( zBranchName[0] ){ + case 0: case '-': + return SQLITE_MISUSE; + } if( strchr(zBranchName,'.') ) return SQLITE_MISUSE; + if( strchr(zBranchName,'(') ) return SQLITE_MISUSE; + if( strchr(zBranchName,')') ) return SQLITE_MISUSE; + if( strchr(zBranchName,'=') ) return SQLITE_MISUSE; + if( isnumeric(zBranchName) ) return SQLITE_MISUSE; /* if the db is being accessed by many db connections */ if( !lmdb->singleConnection ){ @@ -63385,14 +63826,23 @@ SQLITE_PRIVATE int pragma_new_branch(sqlite3 *db, int iDb, char *zBranchName, ch /* check if the new branch name already exists */ if( sqlite3BranchFind(lmdb,zBranchName) > 0 ) return SQLITE_MISUSE; - /* check if the source branch is valid */ - source_branch_id = sqlite3BranchFind(lmdb, zSourceBranch); - if( source_branch_id <= 0 ) return SQLITE_NOTFOUND; - source_branch = &lmdb->branches[source_branch_id]; + if( zSourceBranch ){ + /* check if the source branch is valid */ + source_branch_id = sqlite3BranchFind(lmdb, zSourceBranch); + if( source_branch_id <= 0 ) return SQLITE_NOTFOUND; + source_branch = &lmdb->branches[source_branch_id]; + } else if( lmdb->current_branch ){ + /* start at the current branch */ + source_branch = lmdb->current_branch; + source_branch_id = source_branch->id; + } else { + return SQLITE_MISUSE; + } /* parse the source commit */ if( zSourceCommit ){ - iSourceCommit = sqlite3AtoU32(zSourceCommit); + rc = sqlite3AtoU64(zSourceCommit, &iSourceCommit, strlen(zSourceCommit), SQLITE_UTF8); + if( rc ) return SQLITE_MISUSE; } else { iSourceCommit = source_branch->last_commit; } @@ -63496,8 +63946,8 @@ SQLITE_PRIVATE int pragma_new_branch(sqlite3 *db, int iDb, char *zBranchName, ch ** Retrieves the max commit number that is used by another branch. ** We cannot delete commits equal or previous to this one. */ -SQLITE_PRIVATE u32 get_max_fixed_commit(lmdb *lmdb, int id){ - u32 max_fixed_commit = 0; +SQLITE_PRIVATE u64 get_max_fixed_commit(lmdb *lmdb, int id){ + u64 max_fixed_commit = 0; int i; for( i=1; i <= lmdb->num_branches; i++ ){ /* use base 1 access to the array */ if( lmdb->branches[i].source_branch == id ){ @@ -63511,14 +63961,14 @@ SQLITE_PRIVATE u32 get_max_fixed_commit(lmdb *lmdb, int id){ /* ** Truncate the pages_db and maxpg_db to the given commit number. */ -SQLITE_PRIVATE int branch_truncate_dbs(lmdb *lmdb, branch_info *branch, u32 commit_id){ +SQLITE_PRIVATE int branch_truncate_dbs(lmdb *lmdb, branch_info *branch, u64 max_commit){ int rc=SQLITE_ERROR, rc2; MDB_val key, value; MDB_cursor *cursor=0; - BRANCHTRACE("branch_truncate_dbs to [%s.%u]", branch->name, commit_id); + BRANCHTRACE("branch_truncate_dbs to [%s.%llu]", branch->name, max_commit); - if( !lmdb || !branch || commit_id==0 ) goto loc_exit; + if( !lmdb || !branch || max_commit==0 ) goto loc_exit; /* delete all the pages from posterior commits */ @@ -63533,10 +63983,11 @@ SQLITE_PRIVATE int branch_truncate_dbs(lmdb *lmdb, branch_info *branch, u32 comm } do { - /* check the key size */ - if( key.mv_size != PAGE_DB_KEY_SIZE ){ rc = SQLITE_CORRUPT; goto loc_exit; } + u64 this_commit; + /* get the commit id */ + if( (rc=commit_from_key(&key,&this_commit))!=SQLITE_OK ) goto loc_exit; /* check the key */ - if( commit_from_key(key) > commit_id ){ + if( this_commit > max_commit ){ if( (rc=mdb_cursor_del(cursor,0)) != MDB_SUCCESS ) goto loc_failed; } } while ( (rc2=mdb_cursor_get(cursor, &key, &value, MDB_NEXT)) == MDB_SUCCESS ); @@ -63555,10 +64006,11 @@ SQLITE_PRIVATE int branch_truncate_dbs(lmdb *lmdb, branch_info *branch, u32 comm /* move to the last key */ while( (rc2=mdb_cursor_get(cursor, &key, &value, MDB_LAST)) == MDB_SUCCESS ){ - /* check the key size */ - if( key.mv_size != sizeof(commit_id) ){ rc = SQLITE_CORRUPT; goto loc_exit; } + u64 this_commit; + /* get the commit id */ + if( (rc=mdb_value_get_u64(&key,&this_commit))!=SQLITE_OK ) goto loc_exit; /* check the key */ - if( *(u32*)key.mv_data > commit_id ){ + if( this_commit > max_commit ){ if( (rc=mdb_cursor_del(cursor,0)) != MDB_SUCCESS ) goto loc_failed; } else { break; @@ -63583,7 +64035,7 @@ SQLITE_PRIVATE int branch_truncate_dbs(lmdb *lmdb, branch_info *branch, u32 comm goto loc_exit; // test: check if this fn deleted the correct items (and left the others) - // test: commit ids > max_int32. it must accept up to u32 + // test: commit ids > uint32_max. it must accept up to u64 } @@ -63607,7 +64059,7 @@ SQLITE_PRIVATE int update_branch_source(lmdb *lmdb, int from_id, int to_id){ } SQLITE_PRIVATE int delete_branch(lmdb *lmdb, branch_info *branch){ - u32 max_fixed_commit; + u64 max_fixed_commit; int rc, id; if( !lmdb || !branch ) return SQLITE_INTERNAL; @@ -63711,16 +64163,16 @@ SQLITE_PRIVATE int pragma_delete_branch(sqlite3 *db, int iDb, char *name){ #endif } -SQLITE_PRIVATE int pragma_truncate_branch(sqlite3 *db, int iDb, char *name, u32 to_commit){ +SQLITE_PRIVATE int pragma_truncate_branch(sqlite3 *db, int iDb, char *name, u64 to_commit){ #ifndef SQLITE_OMIT_DELETE_BRANCH Btree *pBtree = getBtreeFromiDb(db, iDb); Pager *pPager = getPagerFromBtree(pBtree); lmdb *lmdb; branch_info *branch; - u32 max_fixed_commit; + u64 max_fixed_commit; int id, rc; - BRANCHTRACE("truncate branch [%s] down to commit [%u]", name, to_commit); + BRANCHTRACE("truncate branch [%s] down to commit [%llu]", name, to_commit); if( !pPager || !pPager->lmdb || !name || to_commit==0 ) return SQLITE_ERROR; lmdb = pPager->lmdb; @@ -63854,13 +64306,13 @@ SQLITE_PRIVATE char * pragma_branch_diff(sqlite3 *db, int iDb, char *point1, cha /* ** Merges forward commits from a single direct child branch into the parent branch */ -SQLITE_PRIVATE int forward_merge_one_branch(lmdb *lmdb, branch_info *parent, branch_info *child, u32 num_commits){ +SQLITE_PRIVATE int forward_merge_one_branch(lmdb *lmdb, branch_info *parent, branch_info *child, u64 num_commits){ MDB_cursor *cursor=0; MDB_val key, value; - u32 start_commit, end_commit, commit; + u64 start_commit, end_commit, commit; int rc, cmd, i; - BRANCHTRACE("forward_merge_one_branch parent=%s child=%s num_commits=%u", parent->name, child->name, num_commits); + BRANCHTRACE("forward_merge_one_branch parent=%s child=%s num_commits=%llu", parent->name, child->name, num_commits); /* if the pages db is not open, open it now */ if( parent->pages_db == 0 ){ @@ -63887,11 +64339,13 @@ SQLITE_PRIVATE int forward_merge_one_branch(lmdb *lmdb, branch_info *parent, bra while( (rc=mdb_cursor_get(cursor,&key,&value,cmd))==MDB_SUCCESS ){ cmd = MDB_NEXT; /* get the commit id */ - commit = commit_from_key(key); + if( (rc=commit_from_key(&key,&commit))!=SQLITE_OK ) goto loc_failed; /* should this commit be moved? */ if( commit >= start_commit && commit <= end_commit ){ - Pgno pgno = pgno_from_key(key); - BRANCHTRACE("moving page %u from commit %u", pgno, commit); + Pgno pgno; + /* get the page number */ + if( (rc=pgno_from_key(&key,&pgno))!=SQLITE_OK ) goto loc_failed; + BRANCHTRACE("moving page %u from commit %llu", pgno, commit); /* check the value size */ if( value.mv_size != lmdb->pageSize-LMDB_PAGE_RESERVED_SPACE ) { rc = SQLITE_CORRUPT; goto loc_failed; } /* copy the page to the parent branch */ @@ -63911,18 +64365,18 @@ SQLITE_PRIVATE int forward_merge_one_branch(lmdb *lmdb, branch_info *parent, bra for( commit = start_commit; commit <= end_commit; commit++ ){ u32 max_page; /* get the number of pages on this commit, if set */ - rc = mdb_u32_get_u32(lmdb->txn, child->maxpg_db, commit, &max_page); + rc = mdb_u64_get_u32(lmdb->txn, child->maxpg_db, commit, &max_page); if( rc==MDB_SUCCESS ){ /* save it on the parent maxpg db */ - rc = mdb_u32_put_u32(lmdb->txn, parent->maxpg_db, commit, max_page); + rc = mdb_u64_put_u32(lmdb->txn, parent->maxpg_db, commit, max_page); if( rc ) goto loc_lmdb_failed; } else if( rc!=MDB_NOTFOUND ) goto loc_lmdb_failed; } /* save the new values to the database */ - rc = put_branch_u32(lmdb, parent->id, "last_commit", parent->last_commit + num_commits); + rc = put_branch_u64(lmdb, parent->id, "last_commit", parent->last_commit + num_commits); if( rc!=MDB_SUCCESS ) goto loc_lmdb_failed; - rc = put_branch_u32(lmdb, child->id, "source_commit", child->source_commit + num_commits); + rc = put_branch_u64(lmdb, child->id, "source_commit", child->source_commit + num_commits); if( rc!=MDB_SUCCESS ) goto loc_lmdb_failed; /* update the values on the branch array */ @@ -63962,7 +64416,7 @@ SQLITE_PRIVATE int forward_merge_one_branch(lmdb *lmdb, branch_info *parent, bra ** This function can be called recursively until it finds the direct child of the parent branch ** Updates the number of remaining commits to merge */ -SQLITE_PRIVATE int forward_merge_sub_branch(lmdb *lmdb, branch_info *parent, branch_info *child, u32 last_commit, u32 *p_remaining_commits){ +SQLITE_PRIVATE int forward_merge_sub_branch(lmdb *lmdb, branch_info *parent, branch_info *child, u64 last_commit, u64 *p_remaining_commits){ int rc; /* if this is not a direct child of the parent branch */ @@ -63974,7 +64428,7 @@ SQLITE_PRIVATE int forward_merge_sub_branch(lmdb *lmdb, branch_info *parent, bra /* if now it is a direct child of the parent branch */ if( child->source_branch == parent->id && *p_remaining_commits > 0 ){ - u32 num_commits; + u64 num_commits; /* the child branch must start at the end of the parent branch */ if( child->source_commit != parent->last_commit ) return SQLITE_MISUSE; /* check how many commits to merge now */ @@ -64008,7 +64462,7 @@ SQLITE_PRIVATE int forward_merge_sub_branch(lmdb *lmdb, branch_info *parent, bra /* ** Deals with the transactions on the forward merge */ -SQLITE_PRIVATE int branch_forward_merge(lmdb *lmdb, branch_info *parent, branch_info *child, u32 num_commits){ +SQLITE_PRIVATE int branch_forward_merge(lmdb *lmdb, branch_info *parent, branch_info *child, u64 num_commits){ int rc; /* start a write transaction */ @@ -64040,7 +64494,7 @@ SQLITE_PRIVATE int pragma_branch_forward_merge(sqlite3 *db, int iDb, char *zpare lmdb *lmdb; branch_info *parent, *child; int id_parent, id_child, rc; - u32 to_commit=0, num_commits=0; + u64 to_commit=0, num_commits=0; char *zto_commit; BRANCHTRACE("forward merge parent=%s child=%s num_commits=%s", zparent, zchild, znum_commits); @@ -64073,11 +64527,11 @@ SQLITE_PRIVATE int pragma_branch_forward_merge(sqlite3 *db, int iDb, char *zpare /* check if both branches are valid */ if( znum_commits ){ - num_commits = sqlite3AtoU32(znum_commits); + num_commits = atou64(znum_commits); if( num_commits==0 ) return SQLITE_MISUSE; } if( zto_commit ){ - to_commit = sqlite3AtoU32(zto_commit); + to_commit = atou64(zto_commit); if( to_commit==0 ) return SQLITE_MISUSE; } @@ -123252,13 +123706,17 @@ SQLITE_PRIVATE void sqlite3Pragma( case PragTyp_BRANCH: { if( zRight ){ char *zName, *zCommit; - u32 iCommit=0; + u64 iCommit=0; + rc = 0; zName = zRight; zCommit = stripchr(zName, '.'); if( zCommit ){ - iCommit = sqlite3AtoU32(zCommit); + rc = sqlite3AtoU64(zCommit, &iCommit, strlen(zCommit), SQLITE_UTF8); + if( rc ) rc = SQLITE_MISUSE; + } + if( rc==SQLITE_OK ){ + rc = pragma_set_current_branch(db, iDb, zName, iCommit); } - rc = pragma_set_current_branch(db, iDb, zName, iCommit); if( rc ){ sqlite3ErrorMsg(pParse, sqlite3ErrStr(rc)); } @@ -123316,13 +123774,17 @@ SQLITE_PRIVATE void sqlite3Pragma( case PragTyp_BRANCH_TRUNCATE: { if( zRight ){ char *zName, *zCommit; - u32 iCommit=0; + u64 iCommit=0; + rc = 0; zName = zRight; zCommit = stripchr(zName, '.'); if( zCommit ){ - iCommit = sqlite3AtoU32(zCommit); + rc = sqlite3AtoU64(zCommit, &iCommit, strlen(zCommit), SQLITE_UTF8); + if( rc ) rc = SQLITE_MISUSE; + } + if( rc==SQLITE_OK ){ + rc = pragma_truncate_branch(db, iDb, zName, iCommit); } - rc = pragma_truncate_branch(db, iDb, zName, iCommit); if( rc ){ sqlite3ErrorMsg(pParse, sqlite3ErrStr(rc)); } @@ -215810,13 +216272,19 @@ SQLITE_PRIVATE void jsonAppendInt(JsonString *p, int value){ sqlite3_snprintf(sizeof(buf), buf, "%d", value); jsonAppendRaw(p, buf, strlen(buf)); } -#endif SQLITE_PRIVATE void jsonAppendU32(JsonString *p, u32 value){ char buf[64]; sqlite3_snprintf(sizeof(buf), buf, "%u", value); jsonAppendRaw(p, buf, strlen(buf)); } +#endif + +SQLITE_PRIVATE void jsonAppendU64(JsonString *p, u64 value){ + char buf[64]; + sqlite3_snprintf(sizeof(buf), buf, "%llu", value); + jsonAppendRaw(p, buf, strlen(buf)); +} /* ** PRAGMA commands ----------------------------------------------------------- @@ -215903,7 +216371,7 @@ SQLITE_PRIVATE char * pragma_get_branch_info(sqlite3 *db, int iDb, char *name){ value = "source_commit"; jsonAppendString(&jx, value, strlen(value)); jsonAppendRaw(&jx, ": ", 2); - jsonAppendU32(&jx, branch->source_commit); + jsonAppendU64(&jx, branch->source_commit); jsonAppendSeparator(&jx); jsonAppendChar(&jx, '\n'); @@ -215912,7 +216380,7 @@ SQLITE_PRIVATE char * pragma_get_branch_info(sqlite3 *db, int iDb, char *name){ value = "total_commits"; jsonAppendString(&jx, value, strlen(value)); jsonAppendRaw(&jx, ": ", 2); - jsonAppendU32(&jx, branch->last_commit); + jsonAppendU64(&jx, branch->last_commit); jsonAppendChar(&jx, '\n'); jsonAppendChar(&jx, '}'); diff --git a/test/convert-to-64bit.py b/test/convert-to-64bit.py new file mode 100644 index 0000000..68052a7 --- /dev/null +++ b/test/convert-to-64bit.py @@ -0,0 +1,131 @@ +# +# Copyright defined in LICENSE.txt +# +import os +import sys +import lmdb +import struct +import varint + + +def delete_file(filepath): + if os.path.exists(filepath): + os.remove(filepath) + + +def convert_db(filename1): + + if not os.path.exists(filename1): + print 'the file does not exist' + quit() + + filename2 = filename1 + '-converted' + delete_file(filename2) + print 'converting', filename, 'to', filename2, '...' + + env1 = lmdb.open(filename1, subdir=False, max_dbs=1024) + env2 = lmdb.open(filename2, subdir=False, max_dbs=1024) + + pages_db1 = [None] # the first element (0) stores None + maxpg_db1 = [None] + + pages_db2 = [None] # the first element (0) stores None + maxpg_db2 = [None] + + with env1.begin(buffers=True) as txn1: + + with env2.begin(buffers=True) as txn2: + + value = txn1.get('last_branch_id') + num_branches = struct.unpack('i', value)[0] + print 'branches:', num_branches + + for branch_id in range(1, num_branches + 1): + dbname = 'b' + str(branch_id) + '-pages' + pages_db1.append(env1.open_db(dbname)) + pages_db2.append(env2.open_db(dbname)) + dbname = 'b' + str(branch_id) + '-maxpage' + maxpg_db1.append(env1.open_db(dbname, integerkey=True)) + maxpg_db2.append(env2.open_db(dbname)) + + + with env1.begin(buffers=True) as txn1: + + with env2.begin(write=True, buffers=True) as txn2: + + txn2.put('last_branch_id', varint.encode(num_branches)) + + value = txn1.get('change_counter') + value = struct.unpack('i', value)[0] + value = varint.encode(value) + txn2.put('change_counter', value) + + for branch_id in range(1, num_branches + 1): + prefix = 'b' + str(branch_id) + + key = prefix + '.name' + name = txn1.get(key) + print 'processing branch:', name + txn2.put(key, name) + + key = prefix + '.visible' + value = txn1.get(key) + value = struct.unpack('i', value)[0] + value = varint.encode(value) + txn2.put(key, value) + + key = prefix + '.source_branch' + value = txn1.get(key) + value = struct.unpack('i', value)[0] + value = varint.encode(value) + txn2.put(key, value) + + key = prefix + '.source_commit' + value = txn1.get(key) + value = struct.unpack('i', value)[0] + value = varint.encode(value) + txn2.put(key, value) + + key = prefix + '.last_commit' + value = txn1.get(key) + value = struct.unpack('i', value)[0] + value = varint.encode(value) + txn2.put(key, value) + + # iterate all the keys from the sub-db + db1 = pages_db1[branch_id] + db2 = pages_db2[branch_id] + for key, value in txn1.cursor(db=db1): + # read the key + pgno = struct.unpack('>i', key[0:4])[0] + commit = struct.unpack('>i', key[4:8] )[0] + print 'page', pgno, 'commit', commit + # write the new key + key2 = varint.encode(pgno) + varint.encode(commit) + txn2.put(key2, value, db=db2) + + # iterate all the keys from the sub-db + db1 = maxpg_db1[branch_id] + db2 = maxpg_db2[branch_id] + for key, value in txn1.cursor(db=db1): + # read the key + commit = struct.unpack('i', key)[0] + print 'commit', commit + # write the new key + key2 = varint.encode(commit) + txn2.put(key2, value, db=db2) + + env1.close() + env2.close() + + print 'done. you can open it with the command: sqlite3 "file:' + filename2 + '?branches=on"' + + + +if len(sys.argv) == 1: + print 'usage: python', sys.argv[0], '' + quit() + +filename = sys.argv[1] + +convert_db(filename) diff --git a/test/test-64bit-commit-ids.py b/test/test-64bit-commit-ids.py new file mode 100644 index 0000000..0120442 --- /dev/null +++ b/test/test-64bit-commit-ids.py @@ -0,0 +1,247 @@ +# +# Copyright defined in LICENSE.txt +# +import unittest +import json +import os +import platform +import lmdb +import varint + +if platform.system() == "Darwin": + import pysqlite2.dbapi2 as sqlite3 +else: + import sqlite3 + +sqlite_version = "3.24.0" + +if sqlite3.sqlite_version != sqlite_version: + print "wrong SQLite version. expected: " + sqlite_version + " found: " + sqlite3.sqlite_version + import sys + sys.exit(1) + +def delete_file(filepath): + if os.path.exists(filepath): + os.remove(filepath) + +v64bit_increment = 0xFFFFFFFE + +class Test64bitCommitIds(unittest.TestCase): + + def test01_create_database(self): + delete_file("test.db") + delete_file("test.db-lock") + + conn = sqlite3.connect('file:test.db?branches=on') + c = conn.cursor() + + c.execute("pragma page_size") + self.assertEqual(c.fetchone()[0], 4096) + + c.execute("pragma journal_mode") + self.assertEqual(c.fetchone()[0], "branches") + + c.execute("pragma branch") + self.assertEqual(c.fetchone()[0], "master") + + c.execute("pragma branches") + self.assertListEqual(c.fetchall(), [("master",)]) + + c.execute("create table t1(name)") + conn.commit() + c.execute("insert into t1 values ('first')") + conn.commit() + c.execute("insert into t1 values ('second')") + conn.commit() + + c.execute("select * from t1") + self.assertListEqual(c.fetchall(), [("first",),("second",)]) + + c.execute("pragma new_branch=test at master.2") + + c.execute("pragma branches") + self.assertListEqual(c.fetchall(), [("master",),("test",)]) + c.execute("pragma branch") + self.assertEqual(c.fetchone()[0], "test") + + c.execute("select * from t1") + self.assertListEqual(c.fetchall(), [("first",)]) + + c.execute("insert into t1 values ('from test branch')") + conn.commit() + c.execute("select * from t1") + self.assertListEqual(c.fetchall(), [("first",),("from test branch",)]) + + c.execute("pragma branch=master") + c.execute("pragma branch") + self.assertEqual(c.fetchone()[0], "master") + + c.execute("select * from t1") + self.assertListEqual(c.fetchall(), [("first",),("second",)]) + + conn.close() + + + def test02_replace_by_64bit_commit_ids(self): + + env = lmdb.open('test.db', subdir=False, max_dbs=1024) + + pages_db = [None] # the first element (0) stores None + maxpg_db = [None] + + with env.begin(buffers=True) as txn: + + value = txn.get('last_branch_id') + num_branches = varint.decode(value)[0] + self.assertEqual(num_branches, 2) + + for branch_id in range(1, num_branches + 1): + pages_db.append(env.open_db('b' + str(branch_id) + '-pages')) + maxpg_db.append(env.open_db('b' + str(branch_id) + '-maxpage')) + self.assertEqual(len(pages_db) - 1, branch_id) + self.assertEqual(len(maxpg_db) - 1, branch_id) + + with env.begin(write=True, buffers=True) as txn: + + value = txn.get('b1.name') + self.assertEqual(bytes(value).decode("utf-8"), "master\x00") + + value = txn.get('b2.name') + self.assertEqual(bytes(value).decode("utf-8"), "test\x00") + + for branch_id in range(1, num_branches + 1): + prefix = 'b' + str(branch_id) + + key = prefix + '.last_commit' + value = txn.get(key) + last_commit = varint.decode(value)[0] + last_commit += v64bit_increment + value = varint.encode(last_commit) + txn.put(key, value) + + key = prefix + '.source_commit' + value = txn.get(key) + source_commit = varint.decode(value)[0] + if source_commit > 0: + source_commit += v64bit_increment + value = varint.encode(source_commit) + txn.put(key, value) + + # iterate all the keys from the sub-db + dbx = pages_db[branch_id] + for key, value in txn.cursor(db=dbx): + res = varint.decode(key) + pgno = res[0] + size1 = res[1] + res = varint.decode(key[size1:len(key)]) + commit = res[0] + size2 = res[1] + if commit < v64bit_increment: + commit += v64bit_increment + key2 = varint.encode(pgno) + varint.encode(commit) + txn.put(key2, value, db=dbx) + txn.delete(key, db=dbx) + + env.close() + + + def test03_64bit_database(self): + conn = sqlite3.connect('file:test.db?branches=on') + c = conn.cursor() + + c.execute("insert into t1 values ('third')") + conn.commit() + + c.execute("insert into t1 values ('fourth')") + c.execute("insert into t1 values ('fifth')") + c.execute("insert into t1 values ('sixth')") + conn.commit() + + c.execute("select * from t1") + self.assertListEqual(c.fetchall(), [("first",),("second",),("third",),("fourth",),("fifth",),("sixth",)]) + + c.execute("pragma branch=test") + c.execute("pragma branch") + self.assertEqual(c.fetchone()[0], "test") + + c.execute("select * from t1") + self.assertListEqual(c.fetchall(), [("first",),("from test branch",)]) + + c.execute("pragma new_branch=sub-test1 at test." + str(v64bit_increment + 2)) + c.execute("pragma new_branch=sub-test2 at test." + str(v64bit_increment + 3)) + c.execute("pragma branches") + self.assertListEqual(c.fetchall(), [("master",),("test",),("sub-test1",),("sub-test2",)]) + c.execute("pragma branch") + self.assertEqual(c.fetchone()[0], "sub-test2") + + c.execute("select * from t1") + self.assertListEqual(c.fetchall(), [("first",),("from test branch",)]) + + c.execute("insert into t1 values ('from sub-test2 branch')") + conn.commit() + + c.execute("pragma branch=sub-test1") + c.execute("pragma branch") + self.assertEqual(c.fetchone()[0], "sub-test1") + + c.execute("select * from t1") + self.assertListEqual(c.fetchall(), [("first",)]) + + c.execute("pragma branch=sub-test2") + c.execute("pragma branch") + self.assertEqual(c.fetchone()[0], "sub-test2") + + c.execute("select * from t1") + self.assertListEqual(c.fetchall(), [("first",),("from test branch",),("from sub-test2 branch",)]) + + c.execute("pragma branch=master." + str(v64bit_increment + 3)) + c.execute("pragma branch") + self.assertEqual(c.fetchone()[0], "master." + str(v64bit_increment + 3)) + + c.execute("select * from t1") + self.assertListEqual(c.fetchall(), [("first",),("second",)]) + + c.execute("pragma branch=master." + str(v64bit_increment + 4)) + c.execute("pragma branch") + self.assertEqual(c.fetchone()[0], "master." + str(v64bit_increment + 4)) + + c.execute("select * from t1") + self.assertListEqual(c.fetchall(), [("first",),("second",),("third",)]) + + conn.close() + + + def test04_branch_info(self): + conn = sqlite3.connect('file:test.db?branches=on') + c = conn.cursor() + + c.execute("pragma journal_mode") + self.assertEqual(c.fetchone()[0], "branches") + + c.execute("pragma branch_info(master)") + obj = json.loads(c.fetchone()[0]) + self.assertEqual(obj["total_commits"], v64bit_increment + 5) + + c.execute("pragma branch_info(test)") + obj = json.loads(c.fetchone()[0]) + self.assertEqual(obj["source_branch"], "master") + self.assertEqual(obj["source_commit"], v64bit_increment + 2) + self.assertEqual(obj["total_commits"], v64bit_increment + 3) + + c.execute("pragma branch_info('sub-test1')") + obj = json.loads(c.fetchone()[0]) + self.assertEqual(obj["source_branch"], "master") + self.assertEqual(obj["source_commit"], v64bit_increment + 2) + self.assertEqual(obj["total_commits"], v64bit_increment + 2) + + c.execute("pragma branch_info('sub-test2')") + obj = json.loads(c.fetchone()[0]) + self.assertEqual(obj["source_branch"], "test") + self.assertEqual(obj["source_commit"], v64bit_increment + 3) + self.assertEqual(obj["total_commits"], v64bit_increment + 4) + + conn.close() + + +if __name__ == '__main__': + unittest.main() diff --git a/test/test.py b/test/test.py index 07e2196..44af5b1 100644 --- a/test/test.py +++ b/test/test.py @@ -383,9 +383,58 @@ def test06_invalid_branch_name(self): c.execute("pragma branches") self.assertListEqual(c.fetchall(), [("master",),("b2",),("b3",),("b4",)]) - # try to create a branch in which its name contains a dot + with self.assertRaises(sqlite3.OperationalError): + c.execute("pragma new_branch= at master.2") + with self.assertRaises(sqlite3.OperationalError): + c.execute("pragma new_branch= at master.2") + with self.assertRaises(sqlite3.OperationalError): + c.execute("pragma new_branch= ") + with self.assertRaises(sqlite3.OperationalError): + c.execute("pragma new_branch=") + with self.assertRaises(sqlite3.OperationalError): c.execute("pragma new_branch=another.branch at master.2") + with self.assertRaises(sqlite3.OperationalError): + c.execute("pragma new_branch=.test. at master.2") + with self.assertRaises(sqlite3.OperationalError): + c.execute("pragma new_branch=.test at master.2") + with self.assertRaises(sqlite3.OperationalError): + c.execute("pragma new_branch=test. at master.2") + + with self.assertRaises(sqlite3.OperationalError): + c.execute("pragma new_branch==test at master.2") + with self.assertRaises(sqlite3.OperationalError): + c.execute("pragma new_branch=test= at master.2") + with self.assertRaises(sqlite3.OperationalError): + c.execute("pragma new_branch=aaa=bbb at master.2") + + with self.assertRaises(sqlite3.OperationalError): + c.execute("pragma new_branch=(test at master.2") + with self.assertRaises(sqlite3.OperationalError): + c.execute("pragma new_branch=test( at master.2") + with self.assertRaises(sqlite3.OperationalError): + c.execute("pragma new_branch=aaa(bbb at master.2") + + with self.assertRaises(sqlite3.OperationalError): + c.execute("pragma new_branch=)test at master.2") + with self.assertRaises(sqlite3.OperationalError): + c.execute("pragma new_branch=test) at master.2") + with self.assertRaises(sqlite3.OperationalError): + c.execute("pragma new_branch=aaa)bbb at master.2") + + # invalid characters at beginning + with self.assertRaises(sqlite3.OperationalError): + c.execute("pragma new_branch=-test at master.2") + with self.assertRaises(sqlite3.OperationalError): + c.execute("pragma new_branch=--test at master.2") + + # numbers + with self.assertRaises(sqlite3.OperationalError): + c.execute("pragma new_branch=3 at master.2") + with self.assertRaises(sqlite3.OperationalError): + c.execute("pragma new_branch=123 at master.2") + with self.assertRaises(sqlite3.OperationalError): + c.execute("pragma new_branch= 123 at master.2") c.execute("pragma branch") self.assertEqual(c.fetchone()[0], "master") @@ -1031,7 +1080,7 @@ def test13_forward_merge(self): self.assertListEqual(c2.fetchall(), [("first",)]) # create a new branch on connection 1 - c1.execute("pragma new_branch=dev at master.2") + c1.execute("pragma new_branch=dev") c1.execute("pragma branch") self.assertEqual(c1.fetchone()[0], "dev") c1.execute("pragma branches") @@ -1774,7 +1823,130 @@ def test14_forward_merge(self): conn2.close() - def test20_closed_connection(self): + def test18_savepoints(self): + delete_file("test4.db") + conn1 = sqlite3.connect('file:test4.db?branches=on') + conn1.isolation_level = None # disables wrapper autocommit + c1 = conn1.cursor() + + # to enforce cache spill + c1.execute("pragma cache_spill=true") + c1.execute("pragma cache_size=2") + + c1.execute("create table t1 (name)") + conn1.commit() + c1.execute("insert into t1 values ('first')") + conn1.commit() + + c1.execute("savepoint s1") + c1.execute("create table t2 (name)") + c1.execute("insert into t1 values ('second')") + + c1.execute("savepoint s2") + c1.execute("create table t3 (name)") + c1.execute("insert into t1 values ('third')") + c1.execute("insert into t2 values ('first')") + c1.execute("insert into t3 values ('first')") + + c1.execute("savepoint s3") + c1.execute("create table t4 (name)") + c1.execute("insert into t1 values ('fourth')") + + c1.execute("savepoint s4") + c1.execute("create table t5 (name)") + c1.execute("insert into t1 values ('5th')") + c1.execute("insert into t2 values ('second')") + c1.execute("insert into t3 values ('second')") + + c1.execute("savepoint s5") + c1.execute("insert into t1 values ('6th')") + c1.execute("insert into t2 values ('third')") + c1.execute("insert into t3 values ('third')") + + c1.execute("select * from t1") + self.assertListEqual(c1.fetchall(), [("first",),("second",),("third",),("fourth",),("5th",),("6th",)]) + c1.execute("select * from t2") + self.assertListEqual(c1.fetchall(), [("first",),("second",),("third",)]) + c1.execute("select * from t3") + self.assertListEqual(c1.fetchall(), [("first",),("second",),("third",)]) + c1.execute("select name from sqlite_master") + self.assertListEqual(c1.fetchall(), [("t1",),("t2",),("t3",),("t4",),("t5",)]) + + c1.execute("rollback to savepoint s5") + + c1.execute("select * from t1") + self.assertListEqual(c1.fetchall(), [("first",),("second",),("third",),("fourth",),("5th",)]) + c1.execute("select * from t2") + self.assertListEqual(c1.fetchall(), [("first",),("second",)]) + c1.execute("select * from t3") + self.assertListEqual(c1.fetchall(), [("first",),("second",)]) + c1.execute("select name from sqlite_master") + self.assertListEqual(c1.fetchall(), [("t1",),("t2",),("t3",),("t4",),("t5",)]) + + c1.execute("release savepoint s5") + + c1.execute("select * from t1") + self.assertListEqual(c1.fetchall(), [("first",),("second",),("third",),("fourth",),("5th",)]) + c1.execute("select * from t2") + self.assertListEqual(c1.fetchall(), [("first",),("second",)]) + c1.execute("select * from t3") + self.assertListEqual(c1.fetchall(), [("first",),("second",)]) + c1.execute("select name from sqlite_master") + self.assertListEqual(c1.fetchall(), [("t1",),("t2",),("t3",),("t4",),("t5",)]) + + c1.execute("rollback to savepoint s4") + + c1.execute("select * from t1") + self.assertListEqual(c1.fetchall(), [("first",),("second",),("third",),("fourth",)]) + c1.execute("select * from t2") + self.assertListEqual(c1.fetchall(), [("first",)]) + c1.execute("select * from t3") + self.assertListEqual(c1.fetchall(), [("first",)]) + c1.execute("select name from sqlite_master") + self.assertListEqual(c1.fetchall(), [("t1",),("t2",),("t3",),("t4",)]) + + c1.execute("rollback to savepoint s3") + + c1.execute("select * from t1") + self.assertListEqual(c1.fetchall(), [("first",),("second",),("third",)]) + c1.execute("select * from t2") + self.assertListEqual(c1.fetchall(), [("first",)]) + c1.execute("select * from t3") + self.assertListEqual(c1.fetchall(), [("first",)]) + c1.execute("select name from sqlite_master") + self.assertListEqual(c1.fetchall(), [("t1",),("t2",),("t3",)]) + + #conn1.commit() + c1.execute("release savepoint s1") + + c1.execute("select * from t1") + self.assertListEqual(c1.fetchall(), [("first",),("second",),("third",)]) + c1.execute("select * from t2") + self.assertListEqual(c1.fetchall(), [("first",)]) + c1.execute("select * from t3") + self.assertListEqual(c1.fetchall(), [("first",)]) + c1.execute("select name from sqlite_master") + self.assertListEqual(c1.fetchall(), [("t1",),("t2",),("t3",)]) + + + conn1.close() + + conn1 = sqlite3.connect('file:test4.db?branches=on') + c1 = conn1.cursor() + + c1.execute("select * from t1") + self.assertListEqual(c1.fetchall(), [("first",),("second",),("third",)]) + c1.execute("select * from t2") + self.assertListEqual(c1.fetchall(), [("first",)]) + c1.execute("select * from t3") + self.assertListEqual(c1.fetchall(), [("first",)]) + c1.execute("select name from sqlite_master") + self.assertListEqual(c1.fetchall(), [("t1",),("t2",),("t3",)]) + + conn1.close() + + + def test19_closed_connection(self): delete_file("test4.db") conn1 = sqlite3.connect('file:test4.db?branches=on') c1 = conn1.cursor() @@ -1797,7 +1969,7 @@ def test20_closed_connection(self): self.assertListEqual(c2.fetchall(), [("first",)]) conn2.close() - # try to write and read on the first connection + # try to write and read on the first connection c1.execute("insert into foo values ('third')") conn1.commit() c1.execute("select * from foo") @@ -1806,6 +1978,51 @@ def test20_closed_connection(self): conn1.close() + def test20_open_while_writing(self): + delete_file("test4.db") + conn1 = sqlite3.connect('file:test4.db?branches=on') + conn1.isolation_level = None # disables wrapper autocommit + c1 = conn1.cursor() + + c1.execute("create table if not exists foo (name)") + conn1.commit() + c1.execute("insert into foo values ('first')") + conn1.commit() + c1.execute("insert into foo values ('second')") + conn1.commit() + + c1.execute("pragma branch_info(master)") + obj = json.loads(c1.fetchone()[0]) + self.assertEqual(obj["total_commits"], 3) + + # start writing on the first connection + c1.execute("begin") + c1.execute("insert into foo values ('third')") + + # open another connection, read the db and close the connection + conn2 = sqlite3.connect('file:test4.db?branches=on') + c2 = conn2.cursor() + c2.execute("select * from foo") + self.assertListEqual(c2.fetchall(), [("first",),("second",)]) + c2.execute("pragma branch=master.2") + c2.execute("select * from foo") + self.assertListEqual(c2.fetchall(), [("first",)]) + conn2.close() + + # continue writing on the first connection + c1.execute("insert into foo values ('fourth')") + conn1.commit() + + c1.execute("pragma branch_info(master)") + obj = json.loads(c1.fetchone()[0]) + self.assertEqual(obj["total_commits"], 4) + + c1.execute("select * from foo") + self.assertListEqual(c1.fetchall(), [("first",),("second",),("third",),("fourth",)]) + + conn1.close() + + def test21_normal_sqlite(self): delete_file("test4.db") conn1 = sqlite3.connect('test4.db') diff --git a/test/varint.py b/test/varint.py new file mode 100644 index 0000000..ac14d80 --- /dev/null +++ b/test/varint.py @@ -0,0 +1,106 @@ +# +# SQLite4 varint +# +# Copyright defined in LICENSE.txt +# + +def encode(num): + + if num < 0: + raise ValueError("The number is negative") + + if num <= 240: + result = chr(num) + + elif num <= 2287: + num -= 240 + result = chr((num >> 8) + 241) + chr(num % 256) + + elif num <= 67823: + num -= 2288 + result = chr(249) + chr(num >> 8) + chr(num % 256) + + else: + + if num > 0xFFFFFFFFFFFFFFFF: + raise ValueError("The number is bigger than an unsigned 64-bit integer") + + # convert the 64-bit number to a buffer in big endian + buf = '' + shift = 56 + while shift >= 0: + buf += chr(num >> shift & 0xFF) + shift -= 8 + + # check how many zeros in the beginning + start = 0 + for i in range(0, 8): + if ord(buf[i]) == 0: + start += 1 + else: + break + + # get the number of used bytes + num_bytes = 8 - start + + # build the result + result = chr(247 + num_bytes) + buf[start:8] + + + return result + + + +def decode(buf): + size = len(buf) + if size < 1: raise ValueError("Invalid varint") + first = ord(buf[0]) + + if first <= 240: + result = first + num_bytes = 1 + + elif first < 249: + if size < 2: raise ValueError("Invalid varint") + second = ord(buf[1]) + result = 240 + ((first - 241) * 256) + second + num_bytes = 2 + + elif first == 249: + if size < 3: raise ValueError("Invalid varint") + second = ord(buf[1]) + third = ord(buf[2]) + result = 2288 + (second * 256) + third + num_bytes = 3 + + else: + num_bytes = first - 247 + if size < num_bytes + 1: raise ValueError("Invalid varint") + result = ord(buf[1]) + for i in range(2, num_bytes + 1): + result = (result << 8) | ord(buf[i]) + num_bytes += 1 + + return (result, num_bytes) + + + +tests = 0 + +def test_encode(num): + print 'testing ', num + buf = encode(num) + num2 = decode(buf)[0] + if num2 != num: + print "FAILED!!!", num, num2 + quit() + global tests + tests += 1 + +if __name__ == '__main__': + num = 11 + while num < 0xFFFFFFFFFFFFFFFF: + test_encode(num) + num *= 3 + print 'OK' + print tests, 'tests'