Index: Makefile.in ================================================================== --- Makefile.in +++ Makefile.in @@ -436,10 +436,11 @@ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/remember.c \ $(TOP)/ext/misc/series.c \ $(TOP)/ext/misc/spellfix.c \ $(TOP)/ext/misc/totype.c \ + $(TOP)/ext/misc/unionvtab.c \ $(TOP)/ext/misc/wholenumber.c # Source code to the library files needed by the test fixture # TESTSRC2 = \ @@ -702,10 +703,15 @@ # Rules to build the LEMON compiler generator # lemon$(BEXE): $(TOP)/tool/lemon.c $(TOP)/tool/lempar.c $(BCC) -o $@ $(TOP)/tool/lemon.c cp $(TOP)/tool/lempar.c . + +# Rules to build the program that generates the source-id +# +mksourceid$(BEXE): $(TOP)/tool/mksourceid.c + $(BCC) -o $@ $(TOP)/tool/mksourceid.c # Rules to build individual *.o files from generated *.c files. This # applies to: # # parse.o @@ -968,11 +974,11 @@ rm -f parse.h ./lemon$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) parse.y mv parse.h parse.h.temp $(TCLSH_CMD) $(TOP)/tool/addopcodes.tcl parse.h.temp >parse.h -sqlite3.h: $(TOP)/src/sqlite.h.in $(TOP)/manifest.uuid $(TOP)/VERSION +sqlite3.h: $(TOP)/src/sqlite.h.in $(TOP)/manifest mksourceid$(BEXE) $(TOP)/VERSION $(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h keywordhash.h: $(TOP)/tool/mkkeywordhash.c $(BCC) -o mkkeywordhash$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)/tool/mkkeywordhash.c ./mkkeywordhash$(BEXE) >keywordhash.h Index: Makefile.msc ================================================================== --- Makefile.msc +++ Makefile.msc @@ -1416,10 +1416,11 @@ $(TOP)\ext\misc\regexp.c \ $(TOP)\ext\misc\remember.c \ $(TOP)\ext\misc\series.c \ $(TOP)\ext\misc\spellfix.c \ $(TOP)\ext\misc\totype.c \ + $(TOP)\ext\misc\unionvtab.c \ $(TOP)\ext\misc\wholenumber.c # Source code to the library files needed by the test fixture # (non-amalgamation) # @@ -1670,10 +1671,16 @@ lemon.exe: $(TOP)\tool\lemon.c lempar.c $(BCC) $(NO_WARN) -Daccess=_access \ -Fe$@ $(TOP)\tool\lemon.c /link $(LDFLAGS) $(NLTLINKOPTS) $(NLTLIBPATHS) +# <> +# Rules to build the source-id generator tool +# +mksourceid.exe: $(TOP)\tool\mksourceid.c + $(BCC) $(NO_WARN) -Fe$@ $(TOP)\tool\mksourceid.c /link $(LDFLAGS) $(NLTLINKOPTS) $(NLTLIBPATHS) + # Rules to build individual *.lo files from generated *.c files. This # applies to: # # parse.lo # opcodes.lo @@ -1948,11 +1955,11 @@ copy $(TOP)\src\parse.y . .\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) parse.y move parse.h parse.h.temp $(TCLSH_CMD) $(TOP)\tool\addopcodes.tcl parse.h.temp > parse.h -$(SQLITE3H): $(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION +$(SQLITE3H): $(TOP)\src\sqlite.h.in $(TOP)\manifest mksourceid.exe $(TOP)\VERSION $(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > $(SQLITE3H) $(MKSQLITE3H_ARGS) sqlite3ext.h: .target_source !IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 type tsrc\sqlite3ext.h | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*\)" "(SQLITE_CALLBACK *)" \ @@ -2260,11 +2267,11 @@ del /Q $(SQLITE3EXE) $(SQLITE3DLL) Replace.exe 2>NUL # <> del /Q sqlite3.c sqlite3.h 2>NUL del /Q opcodes.c opcodes.h 2>NUL del /Q lemon.* lempar.c parse.* 2>NUL - del /Q mkkeywordhash.* keywordhash.h 2>NUL + del /Q mksourceid.* mkkeywordhash.* keywordhash.h 2>NUL del /Q notasharedlib.* 2>NUL -rmdir /Q/S .deps 2>NUL -rmdir /Q/S .libs 2>NUL -rmdir /Q/S tsrc 2>NUL del /Q .target_source 2>NUL Index: VERSION ================================================================== --- VERSION +++ VERSION @@ -1,1 +1,1 @@ -3.20.0 +3.21.0 Index: autoconf/Makefile.msc ================================================================== --- autoconf/Makefile.msc +++ autoconf/Makefile.msc @@ -925,11 +925,11 @@ # Additional compiler options for the shell. These are only effective # when the shell is not being dynamically linked. # !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0 -SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_STMTVTAB !ENDIF # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. Index: configure ================================================================== --- configure +++ configure @@ -1,8 +1,8 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for sqlite 3.20.0. +# Generated by GNU Autoconf 2.69 for sqlite 3.21.0. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. # # @@ -724,12 +724,12 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.20.0' -PACKAGE_STRING='sqlite 3.20.0' +PACKAGE_VERSION='3.21.0' +PACKAGE_STRING='sqlite 3.21.0' PACKAGE_BUGREPORT='' PACKAGE_URL='' # Factoring default headers for most tests. ac_includes_default="\ @@ -907,10 +907,11 @@ enable_memsys3 enable_fts3 enable_fts4 enable_fts5 enable_json1 +enable_update_limit enable_rtree enable_session enable_gcov ' ac_precious_vars='build_alias @@ -1461,11 +1462,11 @@ # if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.20.0 to adapt to many kinds of systems. +\`configure' configures sqlite 3.21.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... To assign environment variables (e.g., CC, CFLAGS...), specify them as VAR=VALUE. See below for descriptions of some of the useful variables. @@ -1526,11 +1527,11 @@ _ACEOF fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.20.0:";; + short | recursive ) echo "Configuration of sqlite 3.21.0:";; esac cat <<\_ACEOF Optional Features: --disable-option-checking ignore unrecognized --enable/--with options @@ -1558,10 +1559,11 @@ --enable-memsys3 Enable MEMSYS3 --enable-fts3 Enable the FTS3 extension --enable-fts4 Enable the FTS4 extension --enable-fts5 Enable the FTS5 extension --enable-json1 Enable the JSON1 extension + --enable-update-limit Enable the UPDATE/DELETE LIMIT clause --enable-rtree Enable the RTREE extension --enable-session Enable the SESSION extension --enable-gcov Enable coverage testing using gcov Optional Packages: @@ -1650,11 +1652,11 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.20.0 +sqlite configure 3.21.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. This configure script is free software; the Free Software Foundation gives unlimited permission to copy, distribute and modify it. @@ -2069,11 +2071,11 @@ } # ac_fn_c_check_header_mongrel cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.20.0, which was +It was created by sqlite $as_me 3.21.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ _ACEOF @@ -3927,17 +3929,17 @@ if ${lt_cv_nm_interface+:} false; then : $as_echo_n "(cached) " >&6 else lt_cv_nm_interface="BSD nm" echo "int some_variable = 0;" > conftest.$ac_ext - (eval echo "\"\$as_me:3932: $ac_compile\"" >&5) + (eval echo "\"\$as_me:3934: $ac_compile\"" >&5) (eval "$ac_compile" 2>conftest.err) cat conftest.err >&5 - (eval echo "\"\$as_me:3935: $NM \\\"conftest.$ac_objext\\\"\"" >&5) + (eval echo "\"\$as_me:3937: $NM \\\"conftest.$ac_objext\\\"\"" >&5) (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out) cat conftest.err >&5 - (eval echo "\"\$as_me:3938: output\"" >&5) + (eval echo "\"\$as_me:3940: output\"" >&5) cat conftest.out >&5 if $GREP 'External.*some_variable' conftest.out > /dev/null; then lt_cv_nm_interface="MS dumpbin" fi rm -f conftest* @@ -5139,11 +5141,11 @@ fi rm -rf conftest* ;; *-*-irix6*) # Find out which ABI we are using. - echo '#line 5144 "configure"' > conftest.$ac_ext + echo '#line 5146 "configure"' > conftest.$ac_ext if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 (eval $ac_compile) 2>&5 ac_status=$? $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then @@ -6664,15 +6666,15 @@ # The option is referenced via a variable to avoid confusing sed. lt_compile=`echo "$ac_compile" | $SED \ -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:6669: $lt_compile\"" >&5) + (eval echo "\"\$as_me:6671: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:6673: \$? = $ac_status" >&5 + echo "$as_me:6675: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' >conftest.exp $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2 @@ -7003,15 +7005,15 @@ # The option is referenced via a variable to avoid confusing sed. lt_compile=`echo "$ac_compile" | $SED \ -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7008: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7010: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:7012: \$? = $ac_status" >&5 + echo "$as_me:7014: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' >conftest.exp $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2 @@ -7108,15 +7110,15 @@ # with a dollar sign (not a hyphen), so the echo should work correctly. lt_compile=`echo "$ac_compile" | $SED \ -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7113: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7115: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:7117: \$? = $ac_status" >&5 + echo "$as_me:7119: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' > out/conftest.exp @@ -7163,15 +7165,15 @@ # with a dollar sign (not a hyphen), so the echo should work correctly. lt_compile=`echo "$ac_compile" | $SED \ -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7168: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7170: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:7172: \$? = $ac_status" >&5 + echo "$as_me:7174: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' > out/conftest.exp @@ -9543,11 +9545,11 @@ lt_cv_dlopen_self=cross else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 9548 "configure" +#line 9550 "configure" #include "confdefs.h" #if HAVE_DLFCN_H #include #endif @@ -9639,11 +9641,11 @@ lt_cv_dlopen_self_static=cross else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 9644 "configure" +#line 9646 "configure" #include "confdefs.h" #if HAVE_DLFCN_H #include #endif @@ -11536,10 +11538,24 @@ fi if test "${enable_json1}" = "yes" ; then OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_JSON1" fi + +######### +# See whether we should enable the LIMIT clause on UPDATE and DELETE +# statements. +# Check whether --enable-update-limit was given. +if test "${enable_update_limit+set}" = set; then : + enableval=$enable_update_limit; enable_udlimit=yes +else + enable_udlimit=no +fi + +if test "${enable_udlimit}" = "yes" ; then + OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT" +fi ######### # See whether we should enable RTREE # Check whether --enable-rtree was given. if test "${enable_rtree+set}" = set; then : @@ -12149,11 +12165,11 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # Save the log message, to keep $0 and so on meaningful, and to # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.20.0, which was +This file was extended by sqlite $as_me 3.21.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES CONFIG_HEADERS = $CONFIG_HEADERS CONFIG_LINKS = $CONFIG_LINKS @@ -12215,11 +12231,11 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -sqlite config.status 3.20.0 +sqlite config.status 3.21.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" Copyright (C) 2012 Free Software Foundation, Inc. This config.status script is free software; the Free Software Foundation Index: configure.ac ================================================================== --- configure.ac +++ configure.ac @@ -641,10 +641,20 @@ [Enable the JSON1 extension]), [enable_json1=yes],[enable_json1=no]) if test "${enable_json1}" = "yes" ; then OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_JSON1" fi + +######### +# See whether we should enable the LIMIT clause on UPDATE and DELETE +# statements. +AC_ARG_ENABLE(update-limit, AC_HELP_STRING([--enable-update-limit], + [Enable the UPDATE/DELETE LIMIT clause]), + [enable_udlimit=yes],[enable_udlimit=no]) +if test "${enable_udlimit}" = "yes" ; then + OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT" +fi ######### # See whether we should enable RTREE AC_ARG_ENABLE(rtree, AC_HELP_STRING([--enable-rtree], [Enable the RTREE extension]), Index: ext/fts3/fts3.c ================================================================== --- ext/fts3/fts3.c +++ ext/fts3/fts3.c @@ -3351,11 +3351,11 @@ assert( iCol>=0 && iCol<=p->nColumn+2 ); switch( iCol-p->nColumn ){ case 0: /* The special 'table-name' column */ - sqlite3_result_pointer(pCtx, pCsr, "fts3cursor"); + sqlite3_result_pointer(pCtx, pCsr, "fts3cursor", 0); break; case 1: /* The docid column */ sqlite3_result_int64(pCtx, pCsr->iPrevId); Index: ext/fts5/fts5_tcl.c ================================================================== --- ext/fts5/fts5_tcl.c +++ ext/fts5/fts5_tcl.c @@ -102,11 +102,11 @@ rc = sqlite3_prepare_v2(db, "SELECT fts5(?1)", -1, &pStmt, 0); if( rc!=SQLITE_OK ){ Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0); return TCL_ERROR; } - sqlite3_bind_pointer(pStmt, 1, (void*)&pApi, "fts5_api_ptr"); + sqlite3_bind_pointer(pStmt, 1, (void*)&pApi, "fts5_api_ptr", 0); sqlite3_step(pStmt); if( sqlite3_finalize(pStmt)!=SQLITE_OK ){ Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0); return TCL_ERROR; Index: ext/fts5/fts5_test_mi.c ================================================================== --- ext/fts5/fts5_test_mi.c +++ ext/fts5/fts5_test_mi.c @@ -73,11 +73,11 @@ int rc; *ppApi = 0; rc = sqlite3_prepare(db, "SELECT fts5(?1)", -1, &pStmt, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_pointer(pStmt, 1, (void*)ppApi, "fts5_api_ptr"); + sqlite3_bind_pointer(pStmt, 1, (void*)ppApi, "fts5_api_ptr", 0); (void)sqlite3_step(pStmt); rc = sqlite3_finalize(pStmt); } return rc; Index: ext/fts5/fts5_vocab.c ================================================================== --- ext/fts5/fts5_vocab.c +++ ext/fts5/fts5_vocab.c @@ -27,10 +27,15 @@ ** ** One row for each term in the database. The value of $doc is set to ** the number of fts5 rows that contain at least one instance of term ** $term. Field $cnt is set to the total number of instances of term ** $term in the database. +** +** instance: +** CREATE TABLE vocab(term, doc, col, offset, PRIMARY KEY()); +** +** One row for each term instance in the database. */ #include "fts5Int.h" @@ -42,11 +47,11 @@ sqlite3_vtab base; char *zFts5Tbl; /* Name of fts5 table */ char *zFts5Db; /* Db containing fts5 table */ sqlite3 *db; /* Database handle */ Fts5Global *pGlobal; /* FTS5 global object for this database */ - int eType; /* FTS5_VOCAB_COL or ROW */ + int eType; /* FTS5_VOCAB_COL, ROW or INSTANCE */ }; struct Fts5VocabCursor { sqlite3_vtab_cursor base; sqlite3_stmt *pStmt; /* Statement holding lock on pIndex */ @@ -62,20 +67,26 @@ Fts5Config *pConfig; /* Fts5 table configuration */ int iCol; i64 *aCnt; i64 *aDoc; - /* Output values used by 'row' and 'col' tables */ + /* Output values used by all tables. */ i64 rowid; /* This table's current rowid value */ Fts5Buffer term; /* Current value of 'term' column */ + + /* Output values Used by 'instance' tables only */ + i64 iInstPos; + int iInstOff; }; -#define FTS5_VOCAB_COL 0 -#define FTS5_VOCAB_ROW 1 +#define FTS5_VOCAB_COL 0 +#define FTS5_VOCAB_ROW 1 +#define FTS5_VOCAB_INSTANCE 2 #define FTS5_VOCAB_COL_SCHEMA "term, col, doc, cnt" #define FTS5_VOCAB_ROW_SCHEMA "term, doc, cnt" +#define FTS5_VOCAB_INST_SCHEMA "term, doc, col, offset" /* ** Bits for the mask used as the idxNum value by xBestIndex/xFilter. */ #define FTS5_VOCAB_TERM_EQ 0x01 @@ -98,10 +109,13 @@ *peType = FTS5_VOCAB_COL; }else if( sqlite3_stricmp(zCopy, "row")==0 ){ *peType = FTS5_VOCAB_ROW; + }else + if( sqlite3_stricmp(zCopy, "instance")==0 ){ + *peType = FTS5_VOCAB_INSTANCE; }else { *pzErr = sqlite3_mprintf("fts5vocab: unknown table type: %Q", zCopy); rc = SQLITE_ERROR; } @@ -159,11 +173,12 @@ sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ char **pzErr /* Write any error message here */ ){ const char *azSchema[] = { "CREATE TABlE vocab(" FTS5_VOCAB_COL_SCHEMA ")", - "CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA ")" + "CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA ")", + "CREATE TABlE vocab(" FTS5_VOCAB_INST_SCHEMA ")" }; Fts5VocabTable *pRet = 0; int rc = SQLITE_OK; /* Return code */ int bDb; @@ -233,10 +248,19 @@ return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr); } /* ** Implementation of the xBestIndex method. +** +** Only constraints of the form: +** +** term <= ? +** term == ? +** term >= ? +** +** are interpreted. Less-than and less-than-or-equal are treated +** identically, as are greater-than and greater-than-or-equal. */ static int fts5VocabBestIndexMethod( sqlite3_vtab *pUnused, sqlite3_index_info *pInfo ){ @@ -376,10 +400,58 @@ sqlite3_finalize(pCsr->pStmt); sqlite3_free(pCsr); return SQLITE_OK; } +static int fts5VocabInstanceNewTerm(Fts5VocabCursor *pCsr){ + int rc = SQLITE_OK; + + if( sqlite3Fts5IterEof(pCsr->pIter) ){ + pCsr->bEof = 1; + }else{ + const char *zTerm; + int nTerm; + zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); + if( pCsr->nLeTerm>=0 ){ + int nCmp = MIN(nTerm, pCsr->nLeTerm); + int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp); + if( bCmp<0 || (bCmp==0 && pCsr->nLeTermbEof = 1; + } + } + + sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm); + } + return rc; +} + +static int fts5VocabInstanceNext(Fts5VocabCursor *pCsr){ + int eDetail = pCsr->pConfig->eDetail; + int rc = SQLITE_OK; + Fts5IndexIter *pIter = pCsr->pIter; + i64 *pp = &pCsr->iInstPos; + int *po = &pCsr->iInstOff; + + while( eDetail==FTS5_DETAIL_NONE + || sqlite3Fts5PoslistNext64(pIter->pData, pIter->nData, po, pp) + ){ + pCsr->iInstPos = 0; + pCsr->iInstOff = 0; + + rc = sqlite3Fts5IterNextScan(pCsr->pIter); + if( rc==SQLITE_OK ){ + rc = fts5VocabInstanceNewTerm(pCsr); + if( eDetail==FTS5_DETAIL_NONE ) break; + } + if( rc ){ + pCsr->bEof = 1; + break; + } + } + + return rc; +} /* ** Advance the cursor to the next row in the table. */ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ @@ -387,18 +459,22 @@ Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab; int rc = SQLITE_OK; int nCol = pCsr->pConfig->nCol; pCsr->rowid++; + + if( pTab->eType==FTS5_VOCAB_INSTANCE ){ + return fts5VocabInstanceNext(pCsr); + } if( pTab->eType==FTS5_VOCAB_COL ){ for(pCsr->iCol++; pCsr->iColiCol++){ if( pCsr->aDoc[pCsr->iCol] ) break; } } - if( pTab->eType==FTS5_VOCAB_ROW || pCsr->iCol>=nCol ){ + if( pTab->eType!=FTS5_VOCAB_COL || pCsr->iCol>=nCol ){ if( sqlite3Fts5IterEof(pCsr->pIter) ){ pCsr->bEof = 1; }else{ const char *zTerm; int nTerm; @@ -418,26 +494,30 @@ memset(pCsr->aDoc, 0, nCol * sizeof(i64)); pCsr->iCol = 0; assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW ); while( rc==SQLITE_OK ){ + int eDetail = pCsr->pConfig->eDetail; const u8 *pPos; int nPos; /* Position list */ i64 iPos = 0; /* 64-bit position read from poslist */ int iOff = 0; /* Current offset within position list */ pPos = pCsr->pIter->pData; nPos = pCsr->pIter->nData; - switch( pCsr->pConfig->eDetail ){ - case FTS5_DETAIL_FULL: - pPos = pCsr->pIter->pData; - nPos = pCsr->pIter->nData; - if( pTab->eType==FTS5_VOCAB_ROW ){ + + switch( pTab->eType ){ + case FTS5_VOCAB_ROW: + if( eDetail==FTS5_DETAIL_FULL ){ while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ pCsr->aCnt[0]++; } - pCsr->aDoc[0]++; - }else{ + } + pCsr->aDoc[0]++; + break; + + case FTS5_VOCAB_COL: + if( eDetail==FTS5_DETAIL_FULL ){ int iCol = -1; while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ int ii = FTS5_POS2COLUMN(iPos); pCsr->aCnt[ii]++; if( iCol!=ii ){ @@ -447,37 +527,34 @@ } pCsr->aDoc[ii]++; iCol = ii; } } - } - break; - - case FTS5_DETAIL_COLUMNS: - if( pTab->eType==FTS5_VOCAB_ROW ){ - pCsr->aDoc[0]++; - }else{ + }else if( eDetail==FTS5_DETAIL_COLUMNS ){ while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff,&iPos) ){ assert_nc( iPos>=0 && iPos=nCol ){ rc = FTS5_CORRUPT; break; } pCsr->aDoc[iPos]++; } + }else{ + assert( eDetail==FTS5_DETAIL_NONE ); + pCsr->aDoc[0]++; } break; - default: - assert( pCsr->pConfig->eDetail==FTS5_DETAIL_NONE ); - pCsr->aDoc[0]++; + default: + assert( pTab->eType==FTS5_VOCAB_INSTANCE ); break; } if( rc==SQLITE_OK ){ rc = sqlite3Fts5IterNextScan(pCsr->pIter); } + if( pTab->eType==FTS5_VOCAB_INSTANCE ) break; if( rc==SQLITE_OK ){ zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ){ break; @@ -503,11 +580,13 @@ int idxNum, /* Strategy index */ const char *zUnused, /* Unused */ int nUnused, /* Number of elements in apVal */ sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ + Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab; Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + int eType = pTab->eType; int rc = SQLITE_OK; int iVal = 0; int f = FTS5INDEX_QUERY_SCAN; const char *zTerm = 0; @@ -543,15 +622,20 @@ memcpy(pCsr->zLeTerm, zCopy, pCsr->nLeTerm+1); } } } - if( rc==SQLITE_OK ){ rc = sqlite3Fts5IndexQuery(pCsr->pIndex, zTerm, nTerm, f, 0, &pCsr->pIter); } - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && eType==FTS5_VOCAB_INSTANCE ){ + rc = fts5VocabInstanceNewTerm(pCsr); + } + if( rc==SQLITE_OK + && !pCsr->bEof + && (eType!=FTS5_VOCAB_INSTANCE || pCsr->pConfig->eDetail!=FTS5_DETAIL_NONE) + ){ rc = fts5VocabNextMethod(pCursor); } return rc; } @@ -589,16 +673,45 @@ }else if( iCol==2 ){ iVal = pCsr->aDoc[pCsr->iCol]; }else{ iVal = pCsr->aCnt[pCsr->iCol]; } - }else{ + }else if( eType==FTS5_VOCAB_ROW ){ assert( iCol==1 || iCol==2 ); if( iCol==1 ){ iVal = pCsr->aDoc[0]; }else{ iVal = pCsr->aCnt[0]; + } + }else{ + int eDetail = pCsr->pConfig->eDetail; + assert( eType==FTS5_VOCAB_INSTANCE ); + switch( iCol ){ + case 1: + sqlite3_result_int64(pCtx, pCsr->pIter->iRowid); + break; + case 2: { + int ii = -1; + if( eDetail==FTS5_DETAIL_FULL ){ + ii = FTS5_POS2COLUMN(pCsr->iInstPos); + }else if( eDetail==FTS5_DETAIL_COLUMNS ){ + ii = pCsr->iInstPos; + } + if( ii>=0 && iipConfig->nCol ){ + const char *z = pCsr->pConfig->azCol[ii]; + sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC); + } + break; + } + default: { + assert( iCol==3 ); + if( eDetail==FTS5_DETAIL_FULL ){ + int ii = FTS5_POS2OFFSET(pCsr->iInstPos); + sqlite3_result_int(pCtx, ii); + } + break; + } } } if( iVal>0 ) sqlite3_result_int64(pCtx, iVal); return SQLITE_OK; Index: ext/fts5/test/fts5ad.test ================================================================== --- ext/fts5/test/fts5ad.test +++ ext/fts5/test/fts5ad.test @@ -229,11 +229,10 @@ set res [prefix_query $prefix] if {$bAsc} { set res [lsort -integer -increasing $res] } set n [llength $res] - if {$T==5} breakpoint do_execsql_test $T.$bAsc.$tn.$n $sql $res } } catchsql COMMIT Index: ext/fts5/test/fts5alter.test ================================================================== --- ext/fts5/test/fts5alter.test +++ ext/fts5/test/fts5alter.test @@ -87,11 +87,10 @@ CREATE VIRTUAL TABLE abc USING fts5(a); INSERT INTO abc(rowid, a) VALUES(1, 'a'); BEGIN; INSERT INTO abc(rowid, a) VALUES(2, 'a'); } -breakpoint do_execsql_test 3.2 { SELECT rowid FROM abc WHERE abc MATCH 'a'; } {1 2} do_execsql_test 3.3 { Index: ext/fts5/test/fts5aux.test ================================================================== --- ext/fts5/test/fts5aux.test +++ ext/fts5/test/fts5aux.test @@ -238,11 +238,10 @@ 2 {"d b" "a d"} {"[d] [b]" "[a] d"} 3 {"a a d"} {"[a] [a] d"} } { execsql { DELETE FROM x1 } foreach row $lRow { execsql { INSERT INTO x1 VALUES($row) } } - breakpoint do_execsql_test 8.$tn { SELECT highlight(x1, 0, '[', ']') FROM x1 WHERE x1 MATCH 'a OR (b AND d)'; } $res } Index: ext/fts5/test/fts5columnsize.test ================================================================== --- ext/fts5/test/fts5columnsize.test +++ ext/fts5/test/fts5columnsize.test @@ -141,11 +141,10 @@ CREATE VIRTUAL TABLE t5 USING fts5(x, columnsize=0); INSERT INTO t5 VALUES('1 2 3 4'); INSERT INTO t5 VALUES('2 4 6 8'); } -breakpoint do_execsql_test 4.1.2 { INSERT INTO t5(t5) VALUES('integrity-check'); } finish_test Index: ext/fts5/test/fts5config.test ================================================================== --- ext/fts5/test/fts5config.test +++ ext/fts5/test/fts5config.test @@ -108,11 +108,10 @@ # do_catchsql_test 5.1 { CREATE VIRTUAL TABLE xx USING fts5(x, tokenize="porter 'ascii"); } {1 {parse error in tokenize directive}} -breakpoint do_catchsql_test 5.2 { CREATE VIRTUAL TABLE xx USING fts5(x, [y[]); } {0 {}} do_catchsql_test 5.3 { ADDED ext/fts5/test/fts5connect.test Index: ext/fts5/test/fts5connect.test ================================================================== --- /dev/null +++ ext/fts5/test/fts5connect.test @@ -0,0 +1,247 @@ +# 2017 August 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + + + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5connect + +ifcapable !fts5 { + finish_test + return +} + +#------------------------------------------------------------------------- +# The tests in this file test the outcome of a schema-reset happening +# within the xConnect() method of an FTS5 table. At one point this +# was causing a problem in SQLite. Each test proceeds as follows: +# +# 1. Connection [db] opens the db and reads from some unrelated, non-FTS5 +# table causing SQLite to load the db schema into memory. +# +# 2. Connection [db2] opens the db and modifies the db schema. +# +# 3. Connection [db] reads or writes an existing fts5 table. That the +# schema has been modified is detected inside the fts5 xConnect() +# callback that is invoked by sqlite3_prepare(). +# +# 4. Verify that the statement in 3 has worked. SQLite should detect +# that the schema has changed and successfully prepare the +# statement against the new schema. +# +# Test plan: +# +# 1.*: Trigger the xConnect()/schema-reset using statements executed +# directly against an FTS5 table. +# +# 2.*: Using various statements executed by various BEFORE triggers. +# +# 3.*: Using various statements executed by various AFTER triggers. +# +# 4.*: Using various statements executed by various INSTEAD OF triggers. +# + + + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft1 USING fts5(a, b); + CREATE TABLE abc(x INTEGER PRIMARY KEY); + CREATE TABLE t1(i INTEGER PRIMARY KEY, a, b); + + INSERT INTO ft1 VALUES('one', 'two'); + INSERT INTO ft1 VALUES('three', 'four'); +} + +foreach {tn sql res} { + 1 "SELECT * FROM ft1" {one two three four} + 2 "REPLACE INTO ft1(rowid, a, b) VALUES(1, 'five', 'six')" {} + 3 "SELECT * FROM ft1" {five six three four} + 4 "INSERT INTO ft1 VALUES('seven', 'eight')" {} + 5 "SELECT * FROM ft1" {five six three four seven eight} + 6 "DELETE FROM ft1 WHERE rowid=2" {} + 7 "UPDATE ft1 SET b='nine' WHERE rowid=1" {} + 8 "SELECT * FROM ft1" {five nine seven eight} +} { + + catch { db close } + catch { db2 close } + sqlite3 db test.db + sqlite3 db2 test.db + + do_test 1.$tn.1 { + db eval { INSERT INTO abc DEFAULT VALUES } + db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable } + } {} + + do_execsql_test 1.$tn.2 $sql $res + + do_execsql_test 1.$tn.3 { + INSERT INTO ft1(ft1) VALUES('integrity-check'); + } +} + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft2 USING fts5(a, b); + CREATE TABLE t2(a, b); + CREATE TABLE log(txt); + + CREATE TRIGGER t2_ai AFTER INSERT ON t2 BEGIN + INSERT INTO ft2(rowid, a, b) VALUES(new.rowid, new.a, new.b); + INSERT INTO log VALUES('insert'); + END; + + CREATE TRIGGER t2_ad AFTER DELETE ON t2 BEGIN + DELETE FROM ft2 WHERE rowid = old.rowid; + INSERT INTO log VALUES('delete'); + END; + + CREATE TRIGGER t2_au AFTER UPDATE ON t2 BEGIN + UPDATE ft2 SET a=new.a, b=new.b WHERE rowid=new.rowid; + INSERT INTO log VALUES('update'); + END; + + INSERT INTO t2 VALUES('one', 'two'); + INSERT INTO t2 VALUES('three', 'four'); +} + +foreach {tn sql res} { + 1 "SELECT * FROM t2" {one two three four} + 2 "REPLACE INTO t2(rowid, a, b) VALUES(1, 'five', 'six')" {} + 3 "SELECT * FROM ft2" {five six three four} + 4 "INSERT INTO t2 VALUES('seven', 'eight')" {} + 5 "SELECT * FROM ft2" {five six three four seven eight} + 6 "DELETE FROM t2 WHERE rowid=2" {} + 7 "UPDATE t2 SET b='nine' WHERE rowid=1" {} + 8 "SELECT * FROM ft2" {five nine seven eight} +} { + + catch { db close } + catch { db2 close } + sqlite3 db test.db + sqlite3 db2 test.db + + do_test 2.$tn.1 { + db eval { INSERT INTO abc DEFAULT VALUES } + db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable } + } {} + + do_execsql_test 2.$tn.2 $sql $res + + do_execsql_test 2.$tn.3 { + INSERT INTO ft2(ft2) VALUES('integrity-check'); + } +} + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE ft3 USING fts5(a, b); + CREATE TABLE t3(a, b); + + CREATE TRIGGER t3_ai BEFORE INSERT ON t3 BEGIN + INSERT INTO ft3(rowid, a, b) VALUES(new.rowid, new.a, new.b); + INSERT INTO log VALUES('insert'); + END; + + CREATE TRIGGER t3_ad BEFORE DELETE ON t3 BEGIN + DELETE FROM ft3 WHERE rowid = old.rowid; + INSERT INTO log VALUES('delete'); + END; + + CREATE TRIGGER t3_au BEFORE UPDATE ON t3 BEGIN + UPDATE ft3 SET a=new.a, b=new.b WHERE rowid=new.rowid; + INSERT INTO log VALUES('update'); + END; + + INSERT INTO t3(rowid, a, b) VALUES(1, 'one', 'two'); + INSERT INTO t3(rowid, a, b) VALUES(2, 'three', 'four'); +} + +foreach {tn sql res} { + 1 "SELECT * FROM t3" {one two three four} + 2 "REPLACE INTO t3(rowid, a, b) VALUES(1, 'five', 'six')" {} + 3 "SELECT * FROM ft3" {five six three four} + 4 "INSERT INTO t3(rowid, a, b) VALUES(3, 'seven', 'eight')" {} + 5 "SELECT * FROM ft3" {five six three four seven eight} + 6 "DELETE FROM t3 WHERE rowid=2" {} + 7 "UPDATE t3 SET b='nine' WHERE rowid=1" {} + 8 "SELECT * FROM ft3" {five nine seven eight} +} { + + catch { db close } + catch { db2 close } + sqlite3 db test.db + sqlite3 db2 test.db + + do_test 3.$tn.1 { + db eval { INSERT INTO abc DEFAULT VALUES } + db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable } + } {} + + do_execsql_test 3.$tn.2 $sql $res + + do_execsql_test 3.$tn.3 { + INSERT INTO ft3(ft3) VALUES('integrity-check'); + } +} + +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE ft4 USING fts5(a, b); + CREATE VIEW v4 AS SELECT rowid, * FROM ft4; + + CREATE TRIGGER t4_ai INSTEAD OF INSERT ON v4 BEGIN + INSERT INTO ft4(rowid, a, b) VALUES(new.rowid, new.a, new.b); + INSERT INTO log VALUES('insert'); + END; + + CREATE TRIGGER t4_ad INSTEAD OF DELETE ON v4 BEGIN + DELETE FROM ft4 WHERE rowid = old.rowid; + INSERT INTO log VALUES('delete'); + END; + + CREATE TRIGGER t4_au INSTEAD OF UPDATE ON v4 BEGIN + UPDATE ft4 SET a=new.a, b=new.b WHERE rowid=new.rowid; + INSERT INTO log VALUES('update'); + END; + + INSERT INTO ft4(rowid, a, b) VALUES(1, 'one', 'two'); + INSERT INTO ft4(rowid, a, b) VALUES(2, 'three', 'four'); +} + +foreach {tn sql res} { + 1 "SELECT * FROM ft4" {one two three four} + 2 "REPLACE INTO v4(rowid, a, b) VALUES(1, 'five', 'six')" {} + 3 "SELECT * FROM ft4" {five six three four} + 4 "INSERT INTO v4(rowid, a, b) VALUES(3, 'seven', 'eight')" {} + 5 "SELECT * FROM ft4" {five six three four seven eight} + 6 "DELETE FROM v4 WHERE rowid=2" {} + 7 "UPDATE v4 SET b='nine' WHERE rowid=1" {} + 8 "SELECT * FROM ft4" {five nine seven eight} +} { + + catch { db close } + catch { db2 close } + sqlite3 db test.db + sqlite3 db2 test.db + + do_test 4.$tn.1 { + db eval { INSERT INTO abc DEFAULT VALUES } + db2 eval { CREATE TABLE newtable(x,y); DROP TABLE newtable } + } {} + + do_execsql_test 4.$tn.2 $sql $res + + do_execsql_test 4.$tn.3 { + INSERT INTO ft3(ft3) VALUES('integrity-check'); + } +} + +finish_test + Index: ext/fts5/test/fts5dlidx.test ================================================================== --- ext/fts5/test/fts5dlidx.test +++ ext/fts5/test/fts5dlidx.test @@ -64,11 +64,10 @@ } execsql { INSERT INTO t1(rowid, x) VALUES($rowid, $doc) } } execsql COMMIT - breakpoint do_test $tn.1 { execsql { INSERT INTO t1(t1) VALUES('integrity-check') } } {} do_fb_test $tn.3.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND x' } $xdoc @@ -122,11 +121,10 @@ } do_execsql_test $tn.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a' } {1} - breakpoint do_execsql_test $tn.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a' ORDER BY rowid DESC } {1} } Index: ext/fts5/test/fts5fault5.test ================================================================== --- ext/fts5/test/fts5fault5.test +++ ext/fts5/test/fts5fault5.test @@ -103,11 +103,10 @@ } } -test { faultsim_test_result {0 {1 10 11 12 13 14 15 16 17 18 19 2}} } -breakpoint do_execsql_test 3.3.0 { SELECT * FROM tv2; } { 0 x 1 {} 1 x 1 {} 10 x 1 {} 11 x 1 {} 12 x 1 {} 13 x 1 {} 14 x 1 {} 15 x 1 {} 16 x 1 {} 17 x 1 {} 18 x 1 {} 19 x 1 {} Index: ext/fts5/test/fts5fault6.test ================================================================== --- ext/fts5/test/fts5fault6.test +++ ext/fts5/test/fts5fault6.test @@ -278,11 +278,10 @@ faultsim_test_result {0 1} } #------------------------------------------------------------------------- catch { db close } -breakpoint do_faultsim_test 6 -faults oom* -prep { sqlite_orig db test.db sqlite3_db_config_lookaside db 0 0 0 } -test { faultsim_test_result {0 {}} {1 {initialization of fts5 failed: }} Index: ext/fts5/test/fts5hash.test ================================================================== --- ext/fts5/test/fts5hash.test +++ ext/fts5/test/fts5hash.test @@ -119,11 +119,10 @@ set small [random_token] if {[sqlite3_fts5_token_hash 1024 $small]==$hash} break } execsql { CREATE VIRTUAL TABLE t2 USING fts5(x, detail=%DETAIL%) } -breakpoint execsql { INSERT INTO t2 VALUES($small || ' ' || $big); } } {} Index: ext/fts5/test/fts5simple.test ================================================================== --- ext/fts5/test/fts5simple.test +++ ext/fts5/test/fts5simple.test @@ -409,11 +409,10 @@ SELECT * FROM x1 WHERE x1 MATCH 'c0 AND (c1 AND (c2 AND (c3 AND (c4 AND (c5 AND (c6 AND (c7 AND (c8 AND (c9 AND (c10 AND (c11 AND (c12 AND (c13 AND (c14 AND (c15 AND (c16 AND (c17 AND (c18 AND (c19 AND (c20 AND (c21 AND (c22 AND (c23 AND (c24 AND (c25 AND (c26 AND (c27 AND (c28 AND (c29 AND (c30 AND (c31 AND (c32 AND (c33 AND (c34 AND (c35 AND (c36 AND (c37 AND (c38 AND (c39 AND (c40 AND (c41 AND (c42 AND (c43 AND (c44 AND (c45 AND (c46 AND (c47 AND (c48 AND (c49 AND (c50 AND (c51 AND (c52 AND (c53 AND (c54 AND (c55 AND (c56 AND (c57 AND (c58 AND (c59 AND (c60 AND (c61 AND (c62 AND (c63 AND (c64 AND (c65 AND (c66 AND (c67 AND (c68 AND (c69 AND (c70 AND (c71 AND (c72 AND (c73 AND (c74 AND (c75 AND (c76 AND (c77 AND (c78 AND (c79 AND (c80 AND (c81 AND (c82 AND (c83 AND (c84 AND (c85 AND (c86 AND (c87 AND (c88 AND (c89 AND (c90 AND (c91 AND (c92 AND (c93 AND (c94 AND (c95 AND (c96 AND (c97 AND (c98 AND (c99 AND (c100 AND (c101 AND (c102 AND (c103 AND (c104 AND (c105 AND (c106 AND (c107 AND (c108 AND (c109 AND (c110 AND (c111 AND (c112 AND (c113 AND (c114 AND (c115 AND (c116 AND (c117 AND (c118 AND (c119 AND (c120 AND (c121 AND (c122 AND (c123 AND (c124 AND (c125 AND (c126 AND (c127 AND (c128 AND (c129 AND (c130 AND (c131 AND (c132 AND (c133 AND (c134 AND (c135 AND (c136 AND (c137 AND (c138 AND (c139 AND (c140 AND (c141 AND (c142 AND (c143 AND (c144 AND (c145 AND (c146 AND (c147 AND (c148 AND (c149 AND (c150 AND (c151 AND (c152 AND (c153 AND (c154 AND (c155 AND (c156 AND (c157 AND (c158 AND (c159 AND (c160 AND (c161 AND (c162 AND (c163 AND (c164 AND (c165 AND (c166 AND (c167 AND (c168 AND (c169 AND (c170 AND (c171 AND (c172 AND (c173 AND (c174 AND (c175 AND (c176 AND (c177 AND (c178 AND (c179 AND (c180 AND (c181 AND (c182 AND (c183 AND (c184 AND (c185 AND (c186 AND (c187 AND (c188 AND (c189 AND (c190 AND (c191 AND (c192 AND (c193 AND (c194 AND (c195 AND (c196 AND (c197 AND (c198 AND (c199 AND c200)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))'; } {1 {fts5: parser stack overflow}} #------------------------------------------------------------------------- reset_db -breakpoint do_execsql_test 20.0 { CREATE VIRTUAL TABLE x1 USING fts5(x); INSERT INTO x1(x1, rank) VALUES('pgsz', 32); INSERT INTO x1(rowid, x) VALUES(11111, 'onetwothree'); } Index: ext/fts5/test/fts5unicode.test ================================================================== --- ext/fts5/test/fts5unicode.test +++ ext/fts5/test/fts5unicode.test @@ -48,11 +48,10 @@ CREATE VIRTUAL TABLE t3 USING fts5(x, tokenize = ascii); INSERT INTO t1 VALUES('\xC0\xC8\xCC'); INSERT INTO t2 VALUES('\xC0\xC8\xCC'); INSERT INTO t3 VALUES('\xC0\xC8\xCC'); " -breakpoint do_execsql_test 2.1 " SELECT 't1' FROM t1 WHERE t1 MATCH '\xE0\xE8\xEC'; SELECT 't2' FROM t2 WHERE t2 MATCH '\xE0\xE8\xEC'; SELECT 't3' FROM t3 WHERE t3 MATCH '\xE0\xE8\xEC'; " {t1 t2} Index: ext/fts5/test/fts5unicode2.test ================================================================== --- ext/fts5/test/fts5unicode2.test +++ ext/fts5/test/fts5unicode2.test @@ -279,11 +279,10 @@ } {0 {}} #------------------------------------------------------------------------- -breakpoint do_unicode_token_test3 5.1 {tokenchars {}} { sqlite3_reset sqlite3_column_int } { sqlite3 sqlite3 reset reset Index: ext/fts5/test/fts5vocab.test ================================================================== --- ext/fts5/test/fts5vocab.test +++ ext/fts5/test/fts5vocab.test @@ -208,11 +208,10 @@ INSERT INTO aux.t1 VALUES('x y z'); INSERT INTO aux.t1 VALUES('m n o'); INSERT INTO aux.t1 VALUES('x n z'); } -breakpoint do_execsql_test 5.1 { CREATE VIRTUAL TABLE temp.vm USING fts5vocab(main, t1, row); CREATE VIRTUAL TABLE temp.vt1 USING fts5vocab(t1, row); CREATE VIRTUAL TABLE temp.vt2 USING fts5vocab(temp, t1, row); CREATE VIRTUAL TABLE temp.va USING fts5vocab(aux, t1, row); ADDED ext/fts5/test/fts5vocab2.test Index: ext/fts5/test/fts5vocab2.test ================================================================== --- /dev/null +++ ext/fts5/test/fts5vocab2.test @@ -0,0 +1,209 @@ +# 2017 August 10 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The tests in this file focus on testing the fts5vocab module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5vocab + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b); + CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, instance); + + INSERT INTO t1 VALUES('one two', 'two three'); + INSERT INTO t1 VALUES('three four', 'four five five five'); +} + +do_execsql_test 1.1 { + SELECT * FROM v1; +} { + five 2 b 1 + five 2 b 2 + five 2 b 3 + four 2 a 1 + four 2 b 0 + one 1 a 0 + three 1 b 1 + three 2 a 0 + two 1 a 1 + two 1 b 0 +} + +do_execsql_test 1.2 { + SELECT * FROM v1 WHERE term='three'; +} { + three 1 b 1 + three 2 a 0 +} + +do_execsql_test 1.3 { + BEGIN; + DELETE FROM t1 WHERE rowid=2; + SELECT * FROM v1; + ROLLBACK; +} { + one 1 a 0 + three 1 b 1 + two 1 a 1 + two 1 b 0 +} + +do_execsql_test 1.4 { + BEGIN; + DELETE FROM t1 WHERE rowid=1; + SELECT * FROM v1; + ROLLBACK; +} { + five 2 b 1 + five 2 b 2 + five 2 b 3 + four 2 a 1 + four 2 b 0 + three 2 a 0 +} + +do_execsql_test 1.5 { + DELETE FROM t1; + SELECT * FROM v1; +} { +} + +#------------------------------------------------------------------------- +# +do_execsql_test 2.0 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS v1; + + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=column); + CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, instance); + + INSERT INTO t1 VALUES('one two', 'two three'); + INSERT INTO t1 VALUES('three four', 'four five five five'); +} + +do_execsql_test 2.1 { + SELECT * FROM v1; +} { + five 2 b {} + four 2 a {} + four 2 b {} + one 1 a {} + three 1 b {} + three 2 a {} + two 1 a {} + two 1 b {} +} + +do_execsql_test 2.2 { + SELECT * FROM v1 WHERE term='three'; +} { + three 1 b {} + three 2 a {} +} + +do_execsql_test 2.3 { + BEGIN; + DELETE FROM t1 WHERE rowid=2; + SELECT * FROM v1; + ROLLBACK; +} { + one 1 a {} + three 1 b {} + two 1 a {} + two 1 b {} +} + +do_execsql_test 2.4 { + BEGIN; + DELETE FROM t1 WHERE rowid=1; + SELECT * FROM v1; + ROLLBACK; +} { + five 2 b {} + four 2 a {} + four 2 b {} + three 2 a {} +} + +do_execsql_test 2.5 { + DELETE FROM t1; + SELECT * FROM v1; +} { +} + +#------------------------------------------------------------------------- +# +do_execsql_test 3.0 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS v1; + + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=none); + CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, instance); + + INSERT INTO t1 VALUES('one two', 'two three'); + INSERT INTO t1 VALUES('three four', 'four five five five'); +} + +do_execsql_test 3.1 { + SELECT * FROM v1; +} { + five 2 {} {} + four 2 {} {} + one 1 {} {} + three 1 {} {} + three 2 {} {} + two 1 {} {} +} + +do_execsql_test 3.2 { + SELECT * FROM v1 WHERE term='three'; +} { + three 1 {} {} + three 2 {} {} +} + +do_execsql_test 3.3 { + BEGIN; + DELETE FROM t1 WHERE rowid=2; + SELECT * FROM v1; + ROLLBACK; +} { + one 1 {} {} + three 1 {} {} + two 1 {} {} +} + +do_execsql_test 3.4 { + BEGIN; + DELETE FROM t1 WHERE rowid=1; + SELECT * FROM v1; + ROLLBACK; +} { + five 2 {} {} + four 2 {} {} + three 2 {} {} +} + +do_execsql_test 3.5 { + DELETE FROM t1; + SELECT * FROM v1; +} { +} + +finish_test + Index: ext/lsm1/lsm_vtab.c ================================================================== --- ext/lsm1/lsm_vtab.c +++ ext/lsm1/lsm_vtab.c @@ -8,12 +8,86 @@ ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* ** -** This file implements a simple virtual table wrapper around the LSM +** This file implements a virtual table for SQLite3 around the LSM ** storage engine from SQLite4. +** +** USAGE +** +** CREATE VIRTUAL TABLE demo USING lsm1(filename,key,keytype,value1,...); +** +** The filename parameter is the name of the LSM database file, which is +** separate and distinct from the SQLite3 database file. +** +** The keytype must be one of: UINT, TEXT, BLOB. All keys must be of that +** one type. "UINT" means unsigned integer. The values may be of any +** SQLite datatype: BLOB, TEXT, INTEGER, FLOAT, or NULL. +** +** The virtual table contains read-only hidden columns: +** +** lsm1_key A BLOB which is the raw LSM key. If the "keytype" +** is BLOB or TEXT then this column is exactly the +** same as the key. For the UINT keytype, this column +** will be a variable-length integer encoding of the key. +** +** lsm1_value A BLOB which is the raw LSM value. All of the value +** columns are packed into this BLOB using the encoding +** described below. +** +** Attempts to write values into the lsm1_key and lsm1_value columns are +** silently ignored. +** +** EXAMPLE +** +** The virtual table declared this way: +** +** CREATE VIRTUAL TABLE demo2 USING lsm1('x.lsm',id,UINT,a,b,c,d); +** +** Results in a new virtual table named "demo2" that acts as if it has +** the following schema: +** +** CREATE TABLE demo2( +** id UINT PRIMARY KEY ON CONFLICT REPLACE, +** a ANY, +** b ANY, +** c ANY, +** d ANY, +** lsm1_key BLOB HIDDEN, +** lsm1_value BLOB HIDDEN +** ) WITHOUT ROWID; +** +** +** +** INTERNALS +** +** The key encoding for BLOB and TEXT is just a copy of the blob or text. +** UTF-8 is used for text. The key encoding for UINT is the variable-length +** integer format at https://sqlite.org/src4/doc/trunk/www/varint.wiki. +** +** The values are encoded as a single blob (since that is what lsm stores as +** its content). There is a "type integer" followed by "content" for each +** value, alternating back and forth. The content might be empty. +** +** TYPE1 CONTENT1 TYPE2 CONTENT2 TYPE3 CONTENT3 .... +** +** Each "type integer" is encoded as a variable-length integer in the +** format of the link above. Let the type integer be T. The actual +** datatype is an integer 0-5 equal to T%6. Values 1 through 5 correspond +** to SQLITE_INTEGER through SQLITE_NULL. The size of the content in bytes +** is T/6. Type value 0 means that the value is an integer whose actual +** values is T/6 and there is no content. The type-value-0 integer format +** only works for integers in the range of 0 through 40. +** +** There is no content for NULL or type-0 integers. For BLOB and TEXT +** values, the content is the blob data or the UTF-8 text data. For +** non-negative integers X, the content is a variable-length integer X*2. +** For negative integers Y, the content is varaible-length integer (1-Y)*2+1. +** For FLOAT values, the content is the IEEE754 floating point value in +** native byte-order. This means that FLOAT values will be corrupted when +** database file is moved between big-endian and little-endian machines. */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 #include "lsm.h" #include @@ -20,18 +94,23 @@ #include /* Forward declaration of subclasses of virtual table objects */ typedef struct lsm1_vtab lsm1_vtab; typedef struct lsm1_cursor lsm1_cursor; +typedef struct lsm1_vblob lsm1_vblob; /* Primitive types */ typedef unsigned char u8; +typedef unsigned int u32; +typedef sqlite3_uint64 u64; /* An open connection to an LSM table */ struct lsm1_vtab { sqlite3_vtab base; /* Base class - must be first */ lsm_db *pDb; /* Open connection to the LSM table */ + u8 keyType; /* SQLITE_BLOB, _TEXT, or _INTEGER */ + u32 nVal; /* Number of value columns */ }; /* lsm1_cursor is a subclass of sqlite3_vtab_cursor which will ** serve as the underlying representation of a cursor that scans @@ -41,11 +120,85 @@ sqlite3_vtab_cursor base; /* Base class - must be first */ lsm_cursor *pLsmCur; /* The LSM cursor */ u8 isDesc; /* 0: scan forward. 1: scan reverse */ u8 atEof; /* True if the scan is complete */ u8 bUnique; /* True if no more than one row of output */ + u8 *zData; /* Content of the current row */ + u32 nData; /* Number of bytes in the current row */ + u8 *aeType; /* Types for all column values */ + u32 *aiOfst; /* Offsets to the various fields */ + u32 *aiLen; /* Length of each field */ + u8 *pKey2; /* Loop termination key, or NULL */ + u32 nKey2; /* Length of the loop termination key */ +}; + +/* An extensible buffer object. +** +** Content can be appended. Space to hold new content is automatically +** allocated. +*/ +struct lsm1_vblob { + u8 *a; /* Space to hold content, from sqlite3_malloc64() */ + u64 n; /* Bytes of space used */ + u64 nAlloc; /* Bytes of space allocated */ + u8 errNoMem; /* True if a memory allocation error has been seen */ }; + +#if defined(__GNUC__) +# define LSM1_NOINLINE __attribute__((noinline)) +#elif defined(_MSC_VER) && _MSC_VER>=1310 +# define LSM1_NOINLINE __declspec(noinline) +#else +# define LSM1_NOINLINE +#endif + + +/* Increase the available space in the vblob object so that it can hold +** at least N more bytes. Return the number of errors. +*/ +static int lsm1VblobEnlarge(lsm1_vblob *p, u32 N){ + if( p->n+N>p->nAlloc ){ + if( p->errNoMem ) return 1; + p->nAlloc += N + (p->nAlloc ? p->nAlloc : N); + p->a = sqlite3_realloc64(p->a, p->nAlloc); + if( p->a==0 ){ + p->n = 0; + p->nAlloc = 0; + p->errNoMem = 1; + return 1; + } + p->nAlloc = sqlite3_msize(p->a); + } + return 0; +} + +/* Append N bytes to a vblob after first enlarging it */ +static LSM1_NOINLINE void lsm1VblobEnlargeAndAppend( + lsm1_vblob *p, + const u8 *pData, + u32 N +){ + if( p->n+N>p->nAlloc && lsm1VblobEnlarge(p, N) ) return; + memcpy(p->a+p->n, pData, N); + p->n += N; +} + +/* Append N bytes to a vblob */ +static void lsm1VblobAppend(lsm1_vblob *p, const u8 *pData, u32 N){ + sqlite3_int64 n = p->n; + if( n+N>p->nAlloc ){ + lsm1VblobEnlargeAndAppend(p, pData, N); + }else{ + p->n += N; + memcpy(p->a+n, pData, N); + } +} + +/* append text to a vblob */ +static void lsm1VblobAppendText(lsm1_vblob *p, const char *z){ + lsm1VblobAppend(p, (u8*)z, (u32)strlen(z)); +} /* Dequote the string */ static void lsm1Dequote(char *z){ int j; char cQuote = z[0]; @@ -74,21 +227,41 @@ char **pzErr ){ lsm1_vtab *pNew; int rc; char *zFilename; + u8 keyType = 0; + int i; + lsm1_vblob sql; + static const char *azTypes[] = { "UINT", "TEXT", "BLOB" }; + static const u8 aeTypes[] = { SQLITE_INTEGER, SQLITE_TEXT, SQLITE_BLOB }; + static const char *azArgName[] = {"filename", "key", "key type", "value1" }; - if( argc!=4 || argv[3]==0 || argv[3][0]==0 ){ - *pzErr = sqlite3_mprintf("filename argument missing"); + for(i=0; ikeyType = keyType; rc = lsm_new(0, &pNew->pDb); if( rc ){ *pzErr = sqlite3_mprintf("lsm_new failed with error code %d", rc); rc = SQLITE_ERROR; goto connect_failed; @@ -101,26 +274,33 @@ *pzErr = sqlite3_mprintf("lsm_open failed with %d", rc); rc = SQLITE_ERROR; goto connect_failed; } -/* Column numbers */ -#define LSM1_COLUMN_KEY 0 -#define LSM1_COLUMN_BLOBKEY 1 -#define LSM1_COLUMN_VALUE 2 -#define LSM1_COLUMN_BLOBVALUE 3 -#define LSM1_COLUMN_COMMAND 4 - - rc = sqlite3_declare_vtab(db, - "CREATE TABLE x(" - " key," /* The primary key. Any non-NULL */ - " blobkey," /* Pure BLOB primary key */ - " value," /* The value associated with key. Any non-NULL */ - " blobvalue," /* Pure BLOB value */ - " command hidden" /* Insert here for control operations */ - ");" - ); + memset(&sql, 0, sizeof(sql)); + lsm1VblobAppendText(&sql, "CREATE TABLE x("); + lsm1VblobAppendText(&sql, argv[4]); + lsm1VblobAppendText(&sql, " "); + lsm1VblobAppendText(&sql, argv[5]); + lsm1VblobAppendText(&sql, " PRIMARY KEY"); + for(i=6; inVal++; + } + lsm1VblobAppendText(&sql, + ", lsm1_command HIDDEN" + ", lsm1_key HIDDEN" + ", lsm1_value HIDDEN) WITHOUT ROWID"); + lsm1VblobAppend(&sql, (u8*)"", 1); + if( sql.errNoMem ){ + rc = SQLITE_NOMEM; + goto connect_failed; + } + rc = sqlite3_declare_vtab(db, (const char*)sql.a); + sqlite3_free(sql.a); + connect_failed: if( rc!=SQLITE_OK ){ if( pNew ){ if( pNew->pDb ) lsm_close(pNew->pDb); sqlite3_free(pNew); @@ -145,13 +325,17 @@ */ static int lsm1Open(sqlite3_vtab *pVtab, sqlite3_vtab_cursor **ppCursor){ lsm1_vtab *p = (lsm1_vtab*)pVtab; lsm1_cursor *pCur; int rc; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) + + p->nVal*(sizeof(pCur->aiOfst)+sizeof(pCur->aiLen)+1) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); + pCur->aiOfst = (u32*)&pCur[1]; + pCur->aiLen = &pCur->aiOfst[p->nVal]; + pCur->aeType = (u8*)&pCur->aiLen[p->nVal]; *ppCursor = &pCur->base; rc = lsm_csr_open(p->pDb, &pCur->pLsmCur); if( rc==LSM_OK ){ rc = SQLITE_OK; }else{ @@ -165,10 +349,11 @@ /* ** Destructor for a lsm1_cursor. */ static int lsm1Close(sqlite3_vtab_cursor *cur){ lsm1_cursor *pCur = (lsm1_cursor*)cur; + sqlite3_free(pCur->pKey2); lsm_csr_close(pCur->pLsmCur); sqlite3_free(pCur); return SQLITE_OK; } @@ -188,10 +373,25 @@ rc = lsm_csr_next(pCur->pLsmCur); } if( rc==LSM_OK && lsm_csr_valid(pCur->pLsmCur)==0 ){ pCur->atEof = 1; } + if( pCur->pKey2 && pCur->atEof==0 ){ + const u8 *pVal; + u32 nVal; + assert( pCur->isDesc==0 ); + rc = lsm_csr_key(pCur->pLsmCur, (const void**)&pVal, (int*)&nVal); + if( rc==LSM_OK ){ + u32 len = pCur->nKey2; + int c; + if( len>nVal ) len = nVal; + c = memcmp(pVal, pCur->pKey2, len); + if( c==0 ) c = nVal - pCur->nKey2; + if( c>0 ) pCur->atEof = 1; + } + } + pCur->zData = 0; } return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; } /* @@ -292,10 +492,18 @@ z[0] = 255; varintWrite32(z+1, w); varintWrite32(z+5, y); return 9; } + +/* Append non-negative integer x as a variable-length integer. +*/ +static void lsm1VblobAppendVarint(lsm1_vblob *p, sqlite3_uint64 x){ + sqlite3_int64 n = p->n; + if( n+9>p->nAlloc && lsm1VblobEnlarge(p, 9) ) return; + p->n += lsm1PutVarint64(p->a+p->n, x); +} /* ** Decode the varint in the first n bytes z[]. Write the integer value ** into *pResult and return the number of bytes in the varint. ** @@ -347,73 +555,78 @@ *pResult = (((sqlite3_uint64)x)<<32) + (0xffffffff & ((z[5]<<24) + (z[6]<<16) + (z[7]<<8) + z[8])); return 9; } -/* -** Generate a key encoding for pValue such that all keys compare in -** lexicographical order. Return an SQLite error code or SQLITE_OK. +/* Encoded a signed integer as a varint. Numbers close to zero uses fewer +** bytes than numbers far away from zero. However, the result is not in +** lexicographical order. ** -** The key encoding is *pnKey bytes in length written into *ppKey. -** Space to hold the key is taken from pSpace if sufficient, or else -** from sqlite3_malloc(). The caller is responsible for freeing malloced -** space. +** Encoding: Non-negative integer X is encoding as an unsigned +** varint X*2. Negative integer Y is encoding as an unsigned +** varint (1-Y)*2 + 1. */ -static int lsm1EncodeKey( - sqlite3_value *pValue, /* Value to be encoded */ - unsigned char **ppKey, /* Write the encoding here */ - int *pnKey, /* Write the size of the encoding here */ - unsigned char *pSpace, /* Use this space if it is large enough */ - int nSpace /* Size of pSpace[] */ +static int lsm1PutSignedVarint64(u8 *z, sqlite3_int64 v){ + sqlite3_uint64 u; + if( v>=0 ){ + u = (sqlite3_uint64)v; + return lsm1PutVarint64(z, u*2); + }else{ + u = (sqlite3_uint64)(-1-v); + return lsm1PutVarint64(z, u*2+1); + } +} + +/* Decoded a signed varint. */ +static int lsm1GetSignedVarint64( + const unsigned char *z, + int n, + sqlite3_int64 *pResult ){ - int eType = sqlite3_value_type(pValue); - *ppKey = 0; - *pnKey = 0; - assert( nSpace>=32 ); - switch( eType ){ - default: { - return SQLITE_ERROR; /* We cannot handle NULL keys */ - } - case SQLITE_BLOB: - case SQLITE_TEXT: { - int nVal = sqlite3_value_bytes(pValue); - const void *pVal; - if( eType==SQLITE_BLOB ){ - eType = LSM1_TYPE_BLOB; - pVal = sqlite3_value_blob(pValue); - }else{ - eType = LSM1_TYPE_TEXT; - pVal = (const void*)sqlite3_value_text(pValue); - if( pVal==0 ) return SQLITE_NOMEM; - } - if( nVal+1>nSpace ){ - pSpace = sqlite3_malloc( nVal+1 ); - if( pSpace==0 ) return SQLITE_NOMEM; - } - pSpace[0] = (unsigned char)eType; - memcpy(&pSpace[1], pVal, nVal); - *ppKey = pSpace; - *pnKey = nVal+1; - break; - } - case SQLITE_INTEGER: { - sqlite3_int64 iVal = sqlite3_value_int64(pValue); - sqlite3_uint64 uVal; - if( iVal<0 ){ - if( iVal==0xffffffffffffffffLL ) return SQLITE_ERROR; - uVal = *(sqlite3_uint64*)&iVal; - eType = LSM1_TYPE_NEGATIVE; - }else{ - uVal = iVal; - eType = LSM1_TYPE_POSITIVE; - } - pSpace[0] = (unsigned char)eType; - *ppKey = pSpace; - *pnKey = 1 + lsm1PutVarint64(&pSpace[1], uVal); - } - } - return SQLITE_OK; + sqlite3_uint64 u = 0; + n = lsm1GetVarint64(z, n, &u); + if( u&1 ){ + *pResult = -1 - (sqlite3_int64)(u>>1); + }else{ + *pResult = (sqlite3_int64)(u>>1); + } + return n; +} + + +/* +** Read the value part of the key-value pair and decode it into columns. +*/ +static int lsm1DecodeValues(lsm1_cursor *pCur){ + lsm1_vtab *pTab = (lsm1_vtab*)(pCur->base.pVtab); + int i, n; + int rc; + u8 eType; + sqlite3_uint64 v; + + if( pCur->zData ) return 1; + rc = lsm_csr_value(pCur->pLsmCur, (const void**)&pCur->zData, + (int*)&pCur->nData); + if( rc ) return 0; + for(i=n=0; inVal; i++){ + v = 0; + n += lsm1GetVarint64(pCur->zData+n, pCur->nData-n, &v); + pCur->aeType[i] = eType = (u8)(v%6); + if( eType==0 ){ + pCur->aiOfst[i] = (u32)(v/6); + pCur->aiLen[i] = 0; + }else{ + pCur->aiOfst[i] = n; + n += (pCur->aiLen[i] = (u32)(v/6)); + } + if( n>pCur->nData ) break; + } + if( inVal ){ + pCur->zData = 0; + return 0; + } + return 1; } /* ** Return values of columns for the row at which the lsm1_cursor ** is currently pointing. @@ -422,115 +635,184 @@ sqlite3_vtab_cursor *cur, /* The cursor */ sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ int i /* Which column to return */ ){ lsm1_cursor *pCur = (lsm1_cursor*)cur; - switch( i ){ - case LSM1_COLUMN_BLOBKEY: { + lsm1_vtab *pTab = (lsm1_vtab*)(cur->pVtab); + if( i==0 ){ + /* The key column */ + const void *pVal; + int nVal; + if( lsm_csr_key(pCur->pLsmCur, &pVal, &nVal)==LSM_OK ){ + if( pTab->keyType==SQLITE_BLOB ){ + sqlite3_result_blob(ctx, pVal, nVal, SQLITE_TRANSIENT); + }else if( pTab->keyType==SQLITE_TEXT ){ + sqlite3_result_text(ctx,(const char*)pVal, nVal, SQLITE_TRANSIENT); + }else{ + const unsigned char *z = (const unsigned char*)pVal; + sqlite3_uint64 v1; + lsm1GetVarint64(z, nVal, &v1); + sqlite3_result_int64(ctx, (sqlite3_int64)v1); + } + } + }else if( i>pTab->nVal ){ + if( i==pTab->nVal+2 ){ /* lsm1_key */ const void *pVal; int nVal; if( lsm_csr_key(pCur->pLsmCur, &pVal, &nVal)==LSM_OK ){ sqlite3_result_blob(ctx, pVal, nVal, SQLITE_TRANSIENT); } - break; - } - case LSM1_COLUMN_KEY: { - const unsigned char *pVal; - int nVal; - if( lsm_csr_key(pCur->pLsmCur, (const void**)&pVal, &nVal)==LSM_OK - && nVal>=1 - ){ - if( pVal[0]==LSM1_TYPE_BLOB ){ - sqlite3_result_blob(ctx, (const void*)&pVal[1],nVal-1, - SQLITE_TRANSIENT); - }else if( pVal[0]==LSM1_TYPE_TEXT ){ - sqlite3_result_text(ctx, (const char*)&pVal[1],nVal-1, - SQLITE_TRANSIENT); - }else if( nVal>=2 && nVal<=10 && - (pVal[0]==LSM1_TYPE_POSITIVE || pVal[0]==LSM1_TYPE_NEGATIVE) - ){ - sqlite3_int64 iVal; - lsm1GetVarint64(pVal+1, nVal-1, (sqlite3_uint64*)&iVal); - sqlite3_result_int64(ctx, iVal); - } - } - break; - } - case LSM1_COLUMN_BLOBVALUE: { + }else if( i==pTab->nVal+3 ){ /* lsm1_value */ const void *pVal; int nVal; - if( lsm_csr_value(pCur->pLsmCur, (const void**)&pVal, &nVal)==LSM_OK ){ + if( lsm_csr_value(pCur->pLsmCur, &pVal, &nVal)==LSM_OK ){ sqlite3_result_blob(ctx, pVal, nVal, SQLITE_TRANSIENT); } - break; - } - case LSM1_COLUMN_VALUE: { - const unsigned char *aVal; - int nVal; - if( lsm_csr_value(pCur->pLsmCur, (const void**)&aVal, &nVal)==LSM_OK - && nVal>=1 - ){ - switch( aVal[0] ){ - case SQLITE_FLOAT: - case SQLITE_INTEGER: { - sqlite3_uint64 x = 0; - int j; - for(j=1; jzData + pCur->aiOfst[i]; + nData = pCur->aiLen[i]; + switch( pCur->aeType[i] ){ + case 0: { /* in-line integer */ + sqlite3_result_int(ctx, pCur->aiOfst[i]); + break; + } + case SQLITE_INTEGER: { + sqlite3_int64 v; + lsm1GetSignedVarint64(zData, nData, &v); + sqlite3_result_int64(ctx, v); + break; + } + case SQLITE_FLOAT: { + double v; + if( nData==sizeof(v) ){ + memcpy(&v, zData, sizeof(v)); + sqlite3_result_double(ctx, v); + } + break; + } + case SQLITE_TEXT: { + sqlite3_result_text(ctx, (const char*)zData, nData, SQLITE_TRANSIENT); + break; + } + case SQLITE_BLOB: { + sqlite3_result_blob(ctx, zData, nData, SQLITE_TRANSIENT); + break; + } + default: { + /* A NULL. Do nothing */ + } } } return SQLITE_OK; } + +/* Parameter "pValue" contains an SQL value that is to be used as +** a key in an LSM table. The type of the key is determined by +** "keyType". Extract the raw bytes used for the key in LSM1. +*/ +static void lsm1KeyFromValue( + int keyType, /* The key type */ + sqlite3_value *pValue, /* The key value */ + u8 *pBuf, /* Storage space for a generated key */ + const u8 **ppKey, /* OUT: the bytes of the key */ + int *pnKey /* OUT: size of the key */ +){ + if( keyType==SQLITE_BLOB ){ + *ppKey = (const u8*)sqlite3_value_blob(pValue); + *pnKey = sqlite3_value_bytes(pValue); + }else if( keyType==SQLITE_TEXT ){ + *ppKey = (const u8*)sqlite3_value_text(pValue); + *pnKey = sqlite3_value_bytes(pValue); + }else{ + sqlite3_int64 v = sqlite3_value_int64(pValue); + if( v<0 ) v = 0; + *pnKey = lsm1PutVarint64(pBuf, v); + *ppKey = pBuf; + } +} /* Move to the first row to return. */ static int lsm1Filter( sqlite3_vtab_cursor *pVtabCursor, int idxNum, const char *idxStr, int argc, sqlite3_value **argv ){ lsm1_cursor *pCur = (lsm1_cursor *)pVtabCursor; + lsm1_vtab *pTab = (lsm1_vtab*)(pCur->base.pVtab); int rc = LSM_OK; + int seekType = -1; + const u8 *pVal = 0; + int nVal; + u8 keyType = pTab->keyType; + u8 aKey1[16]; + pCur->atEof = 1; - if( idxNum==1 ){ - assert( argc==1 ); - pCur->isDesc = 0; - pCur->bUnique = 1; - if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){ - const void *pVal = sqlite3_value_blob(argv[0]); - int nVal = sqlite3_value_bytes(argv[0]); - rc = lsm_csr_seek(pCur->pLsmCur, pVal, nVal, LSM_SEEK_EQ); - } + sqlite3_free(pCur->pKey2); + pCur->pKey2 = 0; + if( idxNum<99 ){ + lsm1KeyFromValue(keyType, argv[0], aKey1, &pVal, &nVal); + } + switch( idxNum ){ + case 0: { /* key==argv[0] */ + assert( argc==1 ); + seekType = LSM_SEEK_EQ; + pCur->isDesc = 0; + pCur->bUnique = 1; + break; + } + case 1: { /* key>=argv[0] AND key<=argv[1] */ + u8 aKey[12]; + seekType = LSM_SEEK_GE; + pCur->isDesc = 0; + pCur->bUnique = 0; + if( keyType==SQLITE_INTEGER ){ + sqlite3_int64 v = sqlite3_value_int64(argv[1]); + if( v<0 ) v = 0; + pCur->nKey2 = lsm1PutVarint64(aKey, (sqlite3_uint64)v); + pCur->pKey2 = sqlite3_malloc( pCur->nKey2 ); + if( pCur->pKey2==0 ) return SQLITE_NOMEM; + memcpy(pCur->pKey2, aKey, pCur->nKey2); + }else{ + pCur->nKey2 = sqlite3_value_bytes(argv[1]); + pCur->pKey2 = sqlite3_malloc( pCur->nKey2 ); + if( pCur->pKey2==0 ) return SQLITE_NOMEM; + if( keyType==SQLITE_BLOB ){ + memcpy(pCur->pKey2, sqlite3_value_blob(argv[1]), pCur->nKey2); + }else{ + memcpy(pCur->pKey2, sqlite3_value_text(argv[1]), pCur->nKey2); + } + } + break; + } + case 2: { /* key>=argv[0] */ + seekType = LSM_SEEK_GE; + pCur->isDesc = 0; + pCur->bUnique = 0; + break; + } + case 3: { /* key<=argv[0] */ + seekType = LSM_SEEK_LE; + pCur->isDesc = 1; + pCur->bUnique = 0; + break; + } + default: { /* full table scan */ + pCur->isDesc = 0; + pCur->bUnique = 0; + break; + } + } + if( pVal ){ + rc = lsm_csr_seek(pCur->pLsmCur, pVal, nVal, seekType); }else{ rc = lsm_csr_first(pCur->pLsmCur); - pCur->isDesc = 0; - pCur->bUnique = 0; } if( rc==LSM_OK && lsm_csr_valid(pCur->pLsmCur)!=0 ){ pCur->atEof = 0; } return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; @@ -538,63 +820,91 @@ /* ** Only comparisons against the key are allowed. The idxNum defines ** which comparisons are available: ** -** 0 Full table scan only -** bit 1 key==?1 single argument for ?1 -** bit 2 key>?1 -** bit 3 key>=?1 -** bit 4 key?1 -** 4 key>=?1 -** 8 key?1 AND key=?1 AND key?1 AND key<=?2 -** 20 key>=?1 AND key<=?2 -** 33..52 Use blobkey in place of key... +** 0 key==?1 +** 1 key>=?1 AND key<=?2 +** 2 key>?1 or key>=?1 +** 3 keyaConstraint; for(i=0; inConstraint && idxNum<16; i++, pConstraint++){ if( pConstraint->usable==0 ) continue; - if( pConstraint->iColumn!=LSM1_COLUMN_KEY ) continue; - if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + if( pConstraint->iColumn!=0 ) continue; switch( pConstraint->op ){ case SQLITE_INDEX_CONSTRAINT_EQ: { - eqIdx = i; - idxNum = 1; + if( idxNum>0 ){ + argIdx = i; + iIdx2 = -1; + idxNum = 0; + omit1 = 1; + } + break; + } + case SQLITE_INDEX_CONSTRAINT_GE: + case SQLITE_INDEX_CONSTRAINT_GT: { + if( idxNum==99 ){ + argIdx = i; + idxNum = 2; + omit1 = pConstraint->op==SQLITE_INDEX_CONSTRAINT_GE; + }else if( idxNum==3 ){ + iIdx2 = idxNum; + omit2 = omit1; + argIdx = i; + idxNum = 1; + omit1 = pConstraint->op==SQLITE_INDEX_CONSTRAINT_GE; + } + break; + } + case SQLITE_INDEX_CONSTRAINT_LE: + case SQLITE_INDEX_CONSTRAINT_LT: { + if( idxNum==99 ){ + argIdx = i; + idxNum = 3; + omit1 = pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE; + }else if( idxNum==2 ){ + iIdx2 = i; + idxNum = 1; + omit1 = pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE; + } break; } } } - if( eqIdx>=0 ){ - pIdxInfo->aConstraintUsage[eqIdx].argvIndex = ++nArg; - pIdxInfo->aConstraintUsage[eqIdx].omit = 1; + if( argIdx>=0 ){ + pIdxInfo->aConstraintUsage[argIdx].argvIndex = ++nArg; + pIdxInfo->aConstraintUsage[argIdx].omit = omit1; + } + if( iIdx2>=0 ){ + pIdxInfo->aConstraintUsage[iIdx2].argvIndex = ++nArg; + pIdxInfo->aConstraintUsage[iIdx2].omit = omit2; } - if( idxNum==1 ){ + if( idxNum==0 ){ pIdxInfo->estimatedCost = (double)1; pIdxInfo->estimatedRows = 1; pIdxInfo->orderByConsumed = 1; + }else if( idxNum==1 ){ + pIdxInfo->estimatedCost = (double)100; + pIdxInfo->estimatedRows = 100; + }else if( idxNum<99 ){ + pIdxInfo->estimatedCost = (double)5000; + pIdxInfo->estimatedRows = 5000; }else{ /* Full table scan */ pIdxInfo->estimatedCost = (double)2147483647; pIdxInfo->estimatedRows = 2147483647; } @@ -613,102 +923,88 @@ int argc, sqlite3_value **argv, sqlite_int64 *pRowid ){ lsm1_vtab *p = (lsm1_vtab*)pVTab; - const void *pKey; - void *pFree = 0; - int nKey; - int eType; - int rc = LSM_OK; - sqlite3_value *pValue; - const unsigned char *pVal; - unsigned char *pData; - int nVal; - unsigned char pSpace[100]; - - if( argc==1 ){ - pVTab->zErrMsg = sqlite3_mprintf("cannot DELETE"); - return SQLITE_ERROR; - } - if( sqlite3_value_type(argv[0])!=SQLITE_NULL ){ - pVTab->zErrMsg = sqlite3_mprintf("cannot UPDATE"); - return SQLITE_ERROR; - } - - /* "INSERT INTO tab(command) VALUES('....')" is used to implement - ** special commands. - */ - if( sqlite3_value_type(argv[2+LSM1_COLUMN_COMMAND])!=SQLITE_NULL ){ - return SQLITE_OK; - } - if( sqlite3_value_type(argv[2+LSM1_COLUMN_BLOBKEY])==SQLITE_BLOB ){ - /* Use the blob key exactly as supplied */ - pKey = sqlite3_value_blob(argv[2+LSM1_COLUMN_BLOBKEY]); - nKey = sqlite3_value_bytes(argv[2+LSM1_COLUMN_BLOBKEY]); - }else{ - /* Use a key encoding that sorts in lexicographical order */ - rc = lsm1EncodeKey(argv[2+LSM1_COLUMN_KEY], - (unsigned char**)&pKey,&nKey, - pSpace,sizeof(pSpace)); - if( rc ) return rc; - if( pKey!=(const void*)pSpace ) pFree = (void*)pKey; - } - if( sqlite3_value_type(argv[2+LSM1_COLUMN_BLOBVALUE])==SQLITE_BLOB ){ - pVal = sqlite3_value_blob(argv[2+LSM1_COLUMN_BLOBVALUE]); - nVal = sqlite3_value_bytes(argv[2+LSM1_COLUMN_BLOBVALUE]); - rc = lsm_insert(p->pDb, pKey, nKey, pVal, nVal); - }else{ - pValue = argv[2+LSM1_COLUMN_VALUE]; - eType = sqlite3_value_type(pValue); - switch( eType ){ - case SQLITE_NULL: { - rc = lsm_delete(p->pDb, pKey, nKey); - break; - } - case SQLITE_BLOB: - case SQLITE_TEXT: { - if( eType==SQLITE_TEXT ){ - pVal = sqlite3_value_text(pValue); - }else{ - pVal = (unsigned char*)sqlite3_value_blob(pValue); - } - nVal = sqlite3_value_bytes(pValue); - pData = sqlite3_malloc( nVal+1 ); - if( pData==0 ){ - rc = SQLITE_NOMEM; - }else{ - pData[0] = (unsigned char)eType; - memcpy(&pData[1], pVal, nVal); - rc = lsm_insert(p->pDb, pKey, nKey, pData, nVal+1); - sqlite3_free(pData); - } - break; - } - case SQLITE_INTEGER: - case SQLITE_FLOAT: { - sqlite3_uint64 x; - unsigned char aVal[9]; - int i; - if( eType==SQLITE_INTEGER ){ - *(sqlite3_int64*)&x = sqlite3_value_int64(pValue); - }else{ - double r = sqlite3_value_double(pValue); - assert( sizeof(r)==sizeof(x) ); - memcpy(&x, &r, sizeof(r)); - } - for(i=8; x>0 && i>=1; i--){ - aVal[i] = x & 0xff; - x >>= 8; - } - aVal[i] = (unsigned char)eType; - rc = lsm_insert(p->pDb, pKey, nKey, &aVal[i], 9-i); - break; - } - } - } - sqlite3_free(pFree); + int nKey, nKey2; + int i; + int rc = LSM_OK; + const u8 *pKey, *pKey2; + unsigned char aKey[16]; + unsigned char pSpace[16]; + lsm1_vblob val; + + if( argc==1 ){ + /* DELETE the record whose key is argv[0] */ + lsm1KeyFromValue(p->keyType, argv[0], aKey, &pKey, &nKey); + lsm_delete(p->pDb, pKey, nKey); + return SQLITE_OK; + } + + if( sqlite3_value_type(argv[0])!=SQLITE_NULL ){ + /* An UPDATE */ + lsm1KeyFromValue(p->keyType, argv[0], aKey, &pKey, &nKey); + lsm1KeyFromValue(p->keyType, argv[1], pSpace, &pKey2, &nKey2); + if( nKey!=nKey2 || memcmp(pKey, pKey2, nKey)!=0 ){ + /* The UPDATE changes the PRIMARY KEY value. DELETE the old key */ + lsm_delete(p->pDb, pKey, nKey); + } + /* Fall through into the INSERT case to complete the UPDATE */ + } + + /* "INSERT INTO tab(lsm1_command) VALUES('....')" is used to implement + ** special commands. + */ + if( sqlite3_value_type(argv[3+p->nVal])!=SQLITE_NULL ){ + return SQLITE_OK; + } + lsm1KeyFromValue(p->keyType, argv[2], aKey, &pKey, &nKey); + memset(&val, 0, sizeof(val)); + for(i=0; inVal; i++){ + sqlite3_value *pArg = argv[3+i]; + u8 eType = sqlite3_value_type(pArg); + switch( eType ){ + case SQLITE_NULL: { + lsm1VblobAppendVarint(&val, SQLITE_NULL); + break; + } + case SQLITE_INTEGER: { + sqlite3_int64 v = sqlite3_value_int64(pArg); + if( v>=0 && v<=240/6 ){ + lsm1VblobAppendVarint(&val, v*6); + }else{ + int n = lsm1PutSignedVarint64(pSpace, v); + lsm1VblobAppendVarint(&val, SQLITE_INTEGER + n*6); + lsm1VblobAppend(&val, pSpace, n); + } + break; + } + case SQLITE_FLOAT: { + double r = sqlite3_value_double(pArg); + lsm1VblobAppendVarint(&val, SQLITE_FLOAT + 8*6); + lsm1VblobAppend(&val, (u8*)&r, sizeof(r)); + break; + } + case SQLITE_BLOB: { + int n = sqlite3_value_bytes(pArg); + lsm1VblobAppendVarint(&val, n*6 + SQLITE_BLOB); + lsm1VblobAppend(&val, sqlite3_value_blob(pArg), n); + break; + } + case SQLITE_TEXT: { + int n = sqlite3_value_bytes(pArg); + lsm1VblobAppendVarint(&val, n*6 + SQLITE_TEXT); + lsm1VblobAppend(&val, sqlite3_value_text(pArg), n); + break; + } + } + } + if( val.errNoMem ){ + return SQLITE_NOMEM; + } + rc = lsm_insert(p->pDb, pKey, nKey, val.a, val.n); + sqlite3_free(val.a); return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; } /* Begin a transaction */ Index: ext/lsm1/test/lsm1_simple.test ================================================================== --- ext/lsm1/test/lsm1_simple.test +++ ext/lsm1/test/lsm1_simple.test @@ -17,40 +17,77 @@ return_if_no_lsm1 load_lsm1_vtab db forcedelete testlsm.db -do_execsql_test 1.0 { - CREATE VIRTUAL TABLE x1 USING lsm1(testlsm.db); - PRAGMA table_info(x1); -} { - 0 key {} 0 {} 0 - 1 blobkey {} 0 {} 0 - 2 value {} 0 {} 0 - 3 blobvalue {} 0 {} 0 -} - -do_execsql_test 1.1 { - INSERT INTO x1(blobkey, blobvalue) VALUES(x'abcd', x'1234'); - SELECT quote(blobkey), quote(blobvalue) FROM x1; -} {X'ABCD' X'1234'} - -do_catchsql_test 1.2 { - UPDATE x1 SET blobvalue = x'7890' WHERE blobkey = x'abcd'; -} {1 {cannot UPDATE}} - -do_catchsql_test 1.3 { - DELETE FROM x1 WHERE blobkey = x'abcd' -} {1 {cannot DELETE}} - -do_test 1.4 { +do_execsql_test 100 { + CREATE VIRTUAL TABLE x1 USING lsm1(testlsm.db,a,UINT,b,c,d); + PRAGMA table_info(x1); +} { + 0 a UINT 1 {} 1 + 1 b {} 0 {} 0 + 2 c {} 0 {} 0 + 3 d {} 0 {} 0 +} + +do_execsql_test 110 { + INSERT INTO x1(a,b,c,d) VALUES(15, 11, 22, 33),(8,'banjo',x'333231',NULL), + (12,NULL,3.25,-559281390); + SELECT a, quote(b), quote(c), quote(d) FROM x1; +} {8 'banjo' X'333231' NULL 12 NULL 3.25 -559281390 15 11 22 33} +do_execsql_test 111 { + SELECT a, quote(lsm1_key), quote(lsm1_value) FROM x1; +} {8 X'08' X'2162616E6A6F1633323105' 12 X'0C' X'05320000000000000A401FFB42ABE9DB' 15 X'0F' X'4284C6'} + +do_execsql_test 120 { + UPDATE x1 SET d = d+1.0 WHERE a=15; + SELECT a, quote(b), quote(c), quote(d) FROM x1; +} {8 'banjo' X'333231' NULL 12 NULL 3.25 -559281390 15 11 22 34.0} + +do_execsql_test 130 { + UPDATE x1 SET a=123456789 WHERE a=12; + SELECT a, quote(b), quote(c), quote(d) FROM x1; +} {8 'banjo' X'333231' NULL 15 11 22 34.0 123456789 NULL 3.25 -559281390} +do_execsql_test 131 { + SELECT quote(lsm1_key), printf('0x%x',a) FROM x1 WHERE a > 100000000; +} {X'FB075BCD15' 0x75bcd15} + +do_execsql_test 140 { + DELETE FROM x1 WHERE a=15; + SELECT a, quote(b), quote(c), quote(d) FROM x1; +} {8 'banjo' X'333231' NULL 123456789 NULL 3.25 -559281390} + +do_test 150 { lsort [glob testlsm.db*] } {testlsm.db testlsm.db-log testlsm.db-shm} db close -do_test 1.5 { +do_test 160 { lsort [glob testlsm.db*] } {testlsm.db} -finish_test +forcedelete testlsm.db +forcedelete test.db +sqlite3 db test.db +load_lsm1_vtab db + + +do_execsql_test 200 { + CREATE VIRTUAL TABLE x1 USING lsm1(testlsm.db,a,TEXT,b,c,d); + PRAGMA table_info(x1); +} { + 0 a TEXT 1 {} 1 + 1 b {} 0 {} 0 + 2 c {} 0 {} 0 + 3 d {} 0 {} 0 +} +do_execsql_test 210 { + INSERT INTO x1(a,b,c,d) VALUES(15, 11, 22, 33),(8,'banjo',x'333231',NULL), + (12,NULL,3.25,-559281390); + SELECT quote(a), quote(b), quote(c), quote(d), '|' FROM x1; +} {'12' NULL 3.25 -559281390 | '15' 11 22 33 | '8' 'banjo' X'333231' NULL |} +do_execsql_test 211 { + SELECT quote(a), quote(lsm1_key), quote(lsm1_value), '|' FROM x1; +} {'12' X'3132' X'05320000000000000A401FFB42ABE9DB' | '15' X'3135' X'4284C6' | '8' X'38' X'2162616E6A6F1633323105' |} +finish_test Index: ext/misc/carray.c ================================================================== --- ext/misc/carray.c +++ ext/misc/carray.c @@ -22,11 +22,11 @@ ** sqlite3_bind_pointer() interface with a pointer type of "carray". ** For example: ** ** static int aX[] = { 53, 9, 17, 2231, 4, 99 }; ** int i = sqlite3_bind_parameter_index(pStmt, "$ptr"); -** sqlite3_bind_value(pStmt, i, aX, "carray"); +** sqlite3_bind_value(pStmt, i, aX, "carray", 0); ** ** There is an optional third parameter to determine the datatype of ** the C-language array. Allowed values of the third parameter are ** 'int32', 'int64', 'double', 'char*'. Example: ** @@ -375,11 +375,11 @@ memcpy(&p, &i64, sizeof(p)); }else{ int i32 = i64 & 0xffffffff; memcpy(&p, &i32, sizeof(p)); } - sqlite3_result_pointer(context, p, "carray"); + sqlite3_result_pointer(context, p, "carray", 0); } #endif /* SQLITE_TEST */ #endif /* SQLITE_OMIT_VIRTUALTABLE */ Index: ext/misc/csv.c ================================================================== --- ext/misc/csv.c +++ ext/misc/csv.c @@ -678,20 +678,20 @@ } memcpy(pCur->azVal[i], z, pCur->rdr.n+1); i++; } }while( pCur->rdr.cTerm==',' ); - while( inCol ){ - sqlite3_free(pCur->azVal[i]); - pCur->azVal[i] = 0; - pCur->aLen[i] = 0; - i++; - } - if( z==0 || pCur->rdr.cTerm==EOF ){ + if( z==0 || (pCur->rdr.cTerm==EOF && inCol) ){ pCur->iRowid = -1; }else{ pCur->iRowid++; + while( inCol ){ + sqlite3_free(pCur->azVal[i]); + pCur->azVal[i] = 0; + pCur->aLen[i] = 0; + i++; + } } return SQLITE_OK; } /* Index: ext/misc/series.c ================================================================== --- ext/misc/series.c +++ ext/misc/series.c @@ -193,12 +193,13 @@ sqlite3_result_int64(ctx, x); return SQLITE_OK; } /* -** Return the rowid for the current row. In this implementation, the -** rowid is the same as the output value. +** Return the rowid for the current row. In this implementation, the +** first row returned is assigned rowid value 1, and each subsequent +** row a value 1 more than that of the previous. */ static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ series_cursor *pCur = (series_cursor*)cur; *pRowid = pCur->iRowid; return SQLITE_OK; ADDED ext/misc/unionvtab.c Index: ext/misc/unionvtab.c ================================================================== --- /dev/null +++ ext/misc/unionvtab.c @@ -0,0 +1,1133 @@ +/* +** 2017 July 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the implementation of the "unionvtab" and "swarmvtab" +** virtual tables. These modules provide read-only access to multiple tables, +** possibly in multiple database files, via a single database object. +** The source tables must have the following characteristics: +** +** * They must all be rowid tables (not VIRTUAL or WITHOUT ROWID +** tables or views). +** +** * Each table must have the same set of columns, declared in +** the same order and with the same declared types. +** +** * The tables must not feature a user-defined column named "_rowid_". +** +** * Each table must contain a distinct range of rowid values. +** +** The difference between the two virtual table modules is that for +** "unionvtab", all source tables must be located in the main database or +** in databases ATTACHed to the main database by the user. For "swarmvtab", +** the tables may be located in any database file on disk. The "swarmvtab" +** implementation takes care of opening and closing database files +** automatically. +** +** UNIONVTAB +** +** A "unionvtab" virtual table is created as follows: +** +** CREATE VIRTUAL TABLE USING unionvtab(); +** +** The implementation evalutes whenever a unionvtab virtual +** table is created or opened. It should return one row for each source +** database table. The four columns required of each row are: +** +** 1. The name of the database containing the table ("main" or "temp" or +** the name of an attached database). Or NULL to indicate that all +** databases should be searched for the table in the usual fashion. +** +** 2. The name of the database table. +** +** 3. The smallest rowid in the range of rowids that may be stored in the +** database table (an integer). +** +** 4. The largest rowid in the range of rowids that may be stored in the +** database table (an integer). +** +** SWARMVTAB +** +** A "swarmvtab" virtual table is created similarly to a unionvtab table: +** +** CREATE VIRTUAL TABLE +** USING swarmvtab(, ); +** +** The difference is that for a swarmvtab table, the first column returned +** by the must return a path or URI that can be used to open +** the database file containing the source table. The option +** is optional. If included, it is the name of an application-defined +** SQL function that is invoked with the URI of the file, if the file +** does not already exist on disk. +*/ + +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** Largest and smallest possible 64-bit signed integers. These macros +** copied from sqliteInt.h. +*/ +#ifndef LARGEST_INT64 +# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) +#endif +#ifndef SMALLEST_INT64 +# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) +#endif + +/* +** The following is also copied from sqliteInt.h. To facilitate coverage +** testing. +*/ +#ifndef ALWAYS +# if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) +# define ALWAYS(X) (1) +# define NEVER(X) (0) +# elif !defined(NDEBUG) +# define ALWAYS(X) ((X)?1:(assert(0),0)) +# define NEVER(X) ((X)?(assert(0),1):0) +# else +# define ALWAYS(X) (X) +# define NEVER(X) (X) +# endif +#endif + +/* +** The swarmvtab module attempts to keep the number of open database files +** at or below this limit. This may not be possible if there are too many +** simultaneous queries. +*/ +#define SWARMVTAB_MAX_OPEN 9 + +typedef struct UnionCsr UnionCsr; +typedef struct UnionTab UnionTab; +typedef struct UnionSrc UnionSrc; + +/* +** Each source table (row returned by the initialization query) is +** represented by an instance of the following structure stored in the +** UnionTab.aSrc[] array. +*/ +struct UnionSrc { + char *zDb; /* Database containing source table */ + char *zTab; /* Source table name */ + sqlite3_int64 iMin; /* Minimum rowid */ + sqlite3_int64 iMax; /* Maximum rowid */ + + /* Fields used by swarmvtab only */ + char *zFile; /* Database file containing table zTab */ + int nUser; /* Current number of users */ + sqlite3 *db; /* Database handle */ + UnionSrc *pNextClosable; /* Next in list of closable sources */ +}; + +/* +** Virtual table type for union vtab. +*/ +struct UnionTab { + sqlite3_vtab base; /* Base class - must be first */ + sqlite3 *db; /* Database handle */ + int bSwarm; /* 1 for "swarmvtab", 0 for "unionvtab" */ + int iPK; /* INTEGER PRIMARY KEY column, or -1 */ + int nSrc; /* Number of elements in the aSrc[] array */ + UnionSrc *aSrc; /* Array of source tables, sorted by rowid */ + + /* Used by swarmvtab only */ + char *zSourceStr; /* Expected unionSourceToStr() value */ + char *zNotFoundCallback; /* UDF to invoke if file not found on open */ + UnionSrc *pClosable; /* First in list of closable sources */ + int nOpen; /* Current number of open sources */ + int nMaxOpen; /* Maximum number of open sources */ +}; + +/* +** Virtual table cursor type for union vtab. +*/ +struct UnionCsr { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3_stmt *pStmt; /* SQL statement to run */ + + /* Used by swarmvtab only */ + sqlite3_int64 iMaxRowid; /* Last rowid to visit */ + int iTab; /* Index of table read by pStmt */ +}; + +/* +** Given UnionTab table pTab and UnionSrc object pSrc, return the database +** handle that should be used to access the table identified by pSrc. This +** is the main db handle for "unionvtab" tables, or the source-specific +** handle for "swarmvtab". +*/ +#define unionGetDb(pTab, pSrc) ((pTab)->bSwarm ? (pSrc)->db : (pTab)->db) + +/* +** If *pRc is other than SQLITE_OK when this function is called, it +** always returns NULL. Otherwise, it attempts to allocate and return +** a pointer to nByte bytes of zeroed memory. If the memory allocation +** is attempted but fails, NULL is returned and *pRc is set to +** SQLITE_NOMEM. +*/ +static void *unionMalloc(int *pRc, int nByte){ + void *pRet; + assert( nByte>0 ); + if( *pRc==SQLITE_OK ){ + pRet = sqlite3_malloc(nByte); + if( pRet ){ + memset(pRet, 0, nByte); + }else{ + *pRc = SQLITE_NOMEM; + } + }else{ + pRet = 0; + } + return pRet; +} + +/* +** If *pRc is other than SQLITE_OK when this function is called, it +** always returns NULL. Otherwise, it attempts to allocate and return +** a copy of the nul-terminated string passed as the second argument. +** If the allocation is attempted but fails, NULL is returned and *pRc is +** set to SQLITE_NOMEM. +*/ +static char *unionStrdup(int *pRc, const char *zIn){ + char *zRet = 0; + if( zIn ){ + int nByte = (int)strlen(zIn) + 1; + zRet = unionMalloc(pRc, nByte); + if( zRet ){ + memcpy(zRet, zIn, nByte); + } + } + return zRet; +} + +/* +** If the first character of the string passed as the only argument to this +** function is one of the 4 that may be used as an open quote character +** in SQL, this function assumes that the input is a well-formed quoted SQL +** string. In this case the string is dequoted in place. +** +** If the first character of the input is not an open quote, then this +** function is a no-op. +*/ +static void unionDequote(char *z){ + if( z ){ + char q = z[0]; + + /* Set stack variable q to the close-quote character */ + if( q=='[' || q=='\'' || q=='"' || q=='`' ){ + int iIn = 1; + int iOut = 0; + if( q=='[' ) q = ']'; + while( ALWAYS(z[iIn]) ){ + if( z[iIn]==q ){ + if( z[iIn+1]!=q ){ + /* Character iIn was the close quote. */ + iIn++; + break; + }else{ + /* Character iIn and iIn+1 form an escaped quote character. Skip + ** the input cursor past both and copy a single quote character + ** to the output buffer. */ + iIn += 2; + z[iOut++] = q; + } + }else{ + z[iOut++] = z[iIn++]; + } + } + z[iOut] = '\0'; + } + } +} + +/* +** This function is a no-op if *pRc is set to other than SQLITE_OK when it +** is called. NULL is returned in this case. +** +** Otherwise, the SQL statement passed as the third argument is prepared +** against the database handle passed as the second. If the statement is +** successfully prepared, a pointer to the new statement handle is +** returned. It is the responsibility of the caller to eventually free the +** statement by calling sqlite3_finalize(). Alternatively, if statement +** compilation fails, NULL is returned, *pRc is set to an SQLite error +** code and *pzErr may be set to an error message buffer allocated by +** sqlite3_malloc(). +*/ +static sqlite3_stmt *unionPrepare( + int *pRc, /* IN/OUT: Error code */ + sqlite3 *db, /* Database handle */ + const char *zSql, /* SQL statement to prepare */ + char **pzErr /* OUT: Error message */ +){ + sqlite3_stmt *pRet = 0; + assert( pzErr ); + if( *pRc==SQLITE_OK ){ + int rc = sqlite3_prepare_v2(db, zSql, -1, &pRet, 0); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("sql error: %s", sqlite3_errmsg(db)); + *pRc = rc; + } + } + return pRet; +} + +/* +** Like unionPrepare(), except prepare the results of vprintf(zFmt, ...) +** instead of a constant SQL string. +*/ +static sqlite3_stmt *unionPreparePrintf( + int *pRc, /* IN/OUT: Error code */ + char **pzErr, /* OUT: Error message */ + sqlite3 *db, /* Database handle */ + const char *zFmt, /* printf() format string */ + ... /* Trailing printf args */ +){ + sqlite3_stmt *pRet = 0; + char *zSql; + va_list ap; + va_start(ap, zFmt); + + zSql = sqlite3_vmprintf(zFmt, ap); + if( *pRc==SQLITE_OK ){ + if( zSql==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + pRet = unionPrepare(pRc, db, zSql, pzErr); + } + } + sqlite3_free(zSql); + + va_end(ap); + return pRet; +} + + +/* +** Call sqlite3_reset() on SQL statement pStmt. If *pRc is set to +** SQLITE_OK when this function is called, then it is set to the +** value returned by sqlite3_reset() before this function exits. +** In this case, *pzErr may be set to point to an error message +** buffer allocated by sqlite3_malloc(). +*/ +#if 0 +static void unionReset(int *pRc, sqlite3_stmt *pStmt, char **pzErr){ + int rc = sqlite3_reset(pStmt); + if( *pRc==SQLITE_OK ){ + *pRc = rc; + if( rc ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(sqlite3_db_handle(pStmt))); + } + } +} +#endif + +/* +** Call sqlite3_finalize() on SQL statement pStmt. If *pRc is set to +** SQLITE_OK when this function is called, then it is set to the +** value returned by sqlite3_finalize() before this function exits. +*/ +static void unionFinalize(int *pRc, sqlite3_stmt *pStmt, char **pzErr){ + sqlite3 *db = sqlite3_db_handle(pStmt); + int rc = sqlite3_finalize(pStmt); + if( *pRc==SQLITE_OK ){ + *pRc = rc; + if( rc ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } + } +} + +/* +** This function is a no-op for unionvtab. For swarmvtab, it attempts to +** close open database files until at most nMax are open. An SQLite error +** code is returned if an error occurs, or SQLITE_OK otherwise. +*/ +static void unionCloseSources(UnionTab *pTab, int nMax){ + while( pTab->pClosable && pTab->nOpen>nMax ){ + UnionSrc **pp; + for(pp=&pTab->pClosable; (*pp)->pNextClosable; pp=&(*pp)->pNextClosable); + assert( (*pp)->db ); + sqlite3_close((*pp)->db); + (*pp)->db = 0; + *pp = 0; + pTab->nOpen--; + } +} + +/* +** xDisconnect method. +*/ +static int unionDisconnect(sqlite3_vtab *pVtab){ + if( pVtab ){ + UnionTab *pTab = (UnionTab*)pVtab; + int i; + for(i=0; inSrc; i++){ + UnionSrc *pSrc = &pTab->aSrc[i]; + sqlite3_free(pSrc->zDb); + sqlite3_free(pSrc->zTab); + sqlite3_free(pSrc->zFile); + sqlite3_close(pSrc->db); + } + sqlite3_free(pTab->zSourceStr); + sqlite3_free(pTab->zNotFoundCallback); + sqlite3_free(pTab->aSrc); + sqlite3_free(pTab); + } + return SQLITE_OK; +} + +/* +** Check that the table identified by pSrc is a rowid table. If not, +** return SQLITE_ERROR and set (*pzErr) to point to an English language +** error message. If the table is a rowid table and no error occurs, +** return SQLITE_OK and leave (*pzErr) unmodified. +*/ +static int unionIsIntkeyTable( + sqlite3 *db, /* Database handle */ + UnionSrc *pSrc, /* Source table to test */ + char **pzErr /* OUT: Error message */ +){ + int bPk = 0; + const char *zType = 0; + int rc; + + sqlite3_table_column_metadata( + db, pSrc->zDb, pSrc->zTab, "_rowid_", &zType, 0, 0, &bPk, 0 + ); + rc = sqlite3_errcode(db); + if( rc==SQLITE_ERROR + || (rc==SQLITE_OK && (!bPk || sqlite3_stricmp("integer", zType))) + ){ + rc = SQLITE_ERROR; + *pzErr = sqlite3_mprintf("no such rowid table: %s%s%s", + (pSrc->zDb ? pSrc->zDb : ""), + (pSrc->zDb ? "." : ""), + pSrc->zTab + ); + } + return rc; +} + +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is +** called. In this case it returns NULL. +** +** Otherwise, this function checks that the source table passed as the +** second argument (a) exists, (b) is not a view and (c) has a column +** named "_rowid_" of type "integer" that is the primary key. +** If this is not the case, *pRc is set to SQLITE_ERROR and NULL is +** returned. +** +** Finally, if the source table passes the checks above, a nul-terminated +** string describing the column names and types belonging to the source +** table is returned. Tables with the same set of column names and types +** cause this function to return identical strings. Is is the responsibility +** of the caller to free the returned string using sqlite3_free() when +** it is no longer required. +*/ +static char *unionSourceToStr( + int *pRc, /* IN/OUT: Error code */ + UnionTab *pTab, /* Virtual table object */ + UnionSrc *pSrc, /* Source table to test */ + char **pzErr /* OUT: Error message */ +){ + char *zRet = 0; + if( *pRc==SQLITE_OK ){ + sqlite3 *db = unionGetDb(pTab, pSrc); + int rc = unionIsIntkeyTable(db, pSrc, pzErr); + sqlite3_stmt *pStmt = unionPrepare(&rc, db, + "SELECT group_concat(quote(name) || '.' || quote(type)) " + "FROM pragma_table_info(?, ?)", pzErr + ); + if( rc==SQLITE_OK ){ + sqlite3_bind_text(pStmt, 1, pSrc->zTab, -1, SQLITE_STATIC); + sqlite3_bind_text(pStmt, 2, pSrc->zDb, -1, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *z = (const char*)sqlite3_column_text(pStmt, 0); + zRet = unionStrdup(&rc, z); + } + unionFinalize(&rc, pStmt, pzErr); + } + *pRc = rc; + } + + return zRet; +} + +/* +** Check that all configured source tables exist and have the same column +** names and datatypes. If this is not the case, or if some other error +** occurs, return an SQLite error code. In this case *pzErr may be set +** to point to an error message buffer allocated by sqlite3_mprintf(). +** Or, if no problems regarding the source tables are detected and no +** other error occurs, SQLITE_OK is returned. +*/ +static int unionSourceCheck(UnionTab *pTab, char **pzErr){ + int rc = SQLITE_OK; + char *z0 = 0; + int i; + + assert( *pzErr==0 ); + z0 = unionSourceToStr(&rc, pTab, &pTab->aSrc[0], pzErr); + for(i=1; inSrc; i++){ + char *z = unionSourceToStr(&rc, pTab, &pTab->aSrc[i], pzErr); + if( rc==SQLITE_OK && sqlite3_stricmp(z, z0) ){ + *pzErr = sqlite3_mprintf("source table schema mismatch"); + rc = SQLITE_ERROR; + } + sqlite3_free(z); + } + sqlite3_free(z0); + + return rc; +} + + +/* +** Try to open the swarmvtab database. If initially unable, invoke the +** not-found callback UDF and then try again. +*/ +static int unionOpenDatabaseInner(UnionTab *pTab, UnionSrc *pSrc, char **pzErr){ + int rc = SQLITE_OK; + static const int openFlags = + SQLITE_OPEN_READONLY | SQLITE_OPEN_URI; + rc = sqlite3_open_v2(pSrc->zFile, &pSrc->db, openFlags, 0); + if( rc==SQLITE_OK ) return rc; + if( pTab->zNotFoundCallback ){ + char *zSql = sqlite3_mprintf("SELECT \"%w\"(%Q);", + pTab->zNotFoundCallback, pSrc->zFile); + sqlite3_close(pSrc->db); + pSrc->db = 0; + if( zSql==0 ){ + *pzErr = sqlite3_mprintf("out of memory"); + return SQLITE_NOMEM; + } + rc = sqlite3_exec(pTab->db, zSql, 0, 0, pzErr); + sqlite3_free(zSql); + if( rc ) return rc; + rc = sqlite3_open_v2(pSrc->zFile, &pSrc->db, openFlags, 0); + } + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(pSrc->db)); + } + return rc; +} + +/* +** This function may only be called for swarmvtab tables. The results of +** calling it on a unionvtab table are undefined. +** +** For a swarmvtab table, this function ensures that source database iSrc +** is open. If the database is opened successfully and the schema is as +** expected, or if it is already open when this function is called, SQLITE_OK +** is returned. +** +** Alternatively If an error occurs while opening the databases, or if the +** database schema is unsuitable, an SQLite error code is returned and (*pzErr) +** may be set to point to an English language error message. In this case it is +** the responsibility of the caller to eventually free the error message buffer +** using sqlite3_free(). +*/ +static int unionOpenDatabase(UnionTab *pTab, int iSrc, char **pzErr){ + int rc = SQLITE_OK; + UnionSrc *pSrc = &pTab->aSrc[iSrc]; + + assert( pTab->bSwarm && iSrcnSrc ); + if( pSrc->db==0 ){ + unionCloseSources(pTab, pTab->nMaxOpen-1); + rc = unionOpenDatabaseInner(pTab, pSrc, pzErr); + if( rc==SQLITE_OK ){ + char *z = unionSourceToStr(&rc, pTab, pSrc, pzErr); + if( rc==SQLITE_OK ){ + if( pTab->zSourceStr==0 ){ + pTab->zSourceStr = z; + }else{ + if( sqlite3_stricmp(z, pTab->zSourceStr) ){ + *pzErr = sqlite3_mprintf("source table schema mismatch"); + rc = SQLITE_ERROR; + } + sqlite3_free(z); + } + } + } + + if( rc==SQLITE_OK ){ + pSrc->pNextClosable = pTab->pClosable; + pTab->pClosable = pSrc; + pTab->nOpen++; + }else{ + sqlite3_close(pSrc->db); + pSrc->db = 0; + } + } + + return rc; +} + + +/* +** This function is a no-op for unionvtab tables. For swarmvtab, increment +** the reference count for source table iTab. If the reference count was +** zero before it was incremented, also remove the source from the closable +** list. +*/ +static void unionIncrRefcount(UnionTab *pTab, int iTab){ + if( pTab->bSwarm ){ + UnionSrc *pSrc = &pTab->aSrc[iTab]; + assert( pSrc->nUser>=0 && pSrc->db ); + if( pSrc->nUser==0 ){ + UnionSrc **pp; + for(pp=&pTab->pClosable; *pp!=pSrc; pp=&(*pp)->pNextClosable); + *pp = pSrc->pNextClosable; + pSrc->pNextClosable = 0; + } + pSrc->nUser++; + } +} + +/* +** Finalize the SQL statement pCsr->pStmt and return the result. +** +** If this is a swarmvtab table (not unionvtab) and pCsr->pStmt was not +** NULL when this function was called, also decrement the reference +** count on the associated source table. If this means the source tables +** refcount is now zero, add it to the closable list. +*/ +static int unionFinalizeCsrStmt(UnionCsr *pCsr){ + int rc = SQLITE_OK; + if( pCsr->pStmt ){ + UnionTab *pTab = (UnionTab*)pCsr->base.pVtab; + UnionSrc *pSrc = &pTab->aSrc[pCsr->iTab]; + rc = sqlite3_finalize(pCsr->pStmt); + pCsr->pStmt = 0; + if( pTab->bSwarm ){ + pSrc->nUser--; + assert( pSrc->nUser>=0 ); + if( pSrc->nUser==0 ){ + pSrc->pNextClosable = pTab->pClosable; + pTab->pClosable = pSrc; + } + unionCloseSources(pTab, pTab->nMaxOpen); + } + } + return rc; +} + +/* +** xConnect/xCreate method. +** +** The argv[] array contains the following: +** +** argv[0] -> module name ("unionvtab" or "swarmvtab") +** argv[1] -> database name +** argv[2] -> table name +** argv[3] -> SQL statement +** argv[4] -> not-found callback UDF name +*/ +static int unionConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + UnionTab *pTab = 0; + int rc = SQLITE_OK; + int bSwarm = (pAux==0 ? 0 : 1); + const char *zVtab = (bSwarm ? "swarmvtab" : "unionvtab"); + + if( sqlite3_stricmp("temp", argv[1]) ){ + /* unionvtab tables may only be created in the temp schema */ + *pzErr = sqlite3_mprintf("%s tables must be created in TEMP schema", zVtab); + rc = SQLITE_ERROR; + }else if( argc!=4 && argc!=5 ){ + *pzErr = sqlite3_mprintf("wrong number of arguments for %s", zVtab); + rc = SQLITE_ERROR; + }else{ + int nAlloc = 0; /* Allocated size of pTab->aSrc[] */ + sqlite3_stmt *pStmt = 0; /* Argument statement */ + char *zArg = unionStrdup(&rc, argv[3]); /* Copy of argument to CVT */ + + /* Prepare the SQL statement. Instead of executing it directly, sort + ** the results by the "minimum rowid" field. This makes it easier to + ** check that there are no rowid range overlaps between source tables + ** and that the UnionTab.aSrc[] array is always sorted by rowid. */ + unionDequote(zArg); + pStmt = unionPreparePrintf(&rc, pzErr, db, + "SELECT * FROM (%z) ORDER BY 3", zArg + ); + + /* Allocate the UnionTab structure */ + pTab = unionMalloc(&rc, sizeof(UnionTab)); + + /* Iterate through the rows returned by the SQL statement specified + ** as an argument to the CREATE VIRTUAL TABLE statement. */ + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zDb = (const char*)sqlite3_column_text(pStmt, 0); + const char *zTab = (const char*)sqlite3_column_text(pStmt, 1); + sqlite3_int64 iMin = sqlite3_column_int64(pStmt, 2); + sqlite3_int64 iMax = sqlite3_column_int64(pStmt, 3); + UnionSrc *pSrc; + + /* Grow the pTab->aSrc[] array if required. */ + if( nAlloc<=pTab->nSrc ){ + int nNew = nAlloc ? nAlloc*2 : 8; + UnionSrc *aNew = (UnionSrc*)sqlite3_realloc( + pTab->aSrc, nNew*sizeof(UnionSrc) + ); + if( aNew==0 ){ + rc = SQLITE_NOMEM; + break; + }else{ + memset(&aNew[pTab->nSrc], 0, (nNew-pTab->nSrc)*sizeof(UnionSrc)); + pTab->aSrc = aNew; + nAlloc = nNew; + } + } + + /* Check for problems with the specified range of rowids */ + if( iMaxnSrc>0 && iMin<=pTab->aSrc[pTab->nSrc-1].iMax) ){ + *pzErr = sqlite3_mprintf("rowid range mismatch error"); + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_OK ){ + pSrc = &pTab->aSrc[pTab->nSrc++]; + pSrc->zTab = unionStrdup(&rc, zTab); + pSrc->iMin = iMin; + pSrc->iMax = iMax; + if( bSwarm ){ + pSrc->zFile = unionStrdup(&rc, zDb); + }else{ + pSrc->zDb = unionStrdup(&rc, zDb); + } + } + } + unionFinalize(&rc, pStmt, pzErr); + pStmt = 0; + + /* Capture the not-found callback UDF name */ + if( rc==SQLITE_OK && argc>=5 ){ + pTab->zNotFoundCallback = unionStrdup(&rc, argv[4]); + unionDequote(pTab->zNotFoundCallback); + } + + /* It is an error if the SELECT statement returned zero rows. If only + ** because there is no way to determine the schema of the virtual + ** table in this case. */ + if( rc==SQLITE_OK && pTab->nSrc==0 ){ + *pzErr = sqlite3_mprintf("no source tables configured"); + rc = SQLITE_ERROR; + } + + /* For unionvtab, verify that all source tables exist and have + ** compatible schemas. For swarmvtab, attach the first database and + ** check that the first table is a rowid table only. */ + if( rc==SQLITE_OK ){ + pTab->db = db; + pTab->bSwarm = bSwarm; + pTab->nMaxOpen = SWARMVTAB_MAX_OPEN; + if( bSwarm ){ + rc = unionOpenDatabase(pTab, 0, pzErr); + }else{ + rc = unionSourceCheck(pTab, pzErr); + } + } + + /* Compose a CREATE TABLE statement and pass it to declare_vtab() */ + if( rc==SQLITE_OK ){ + UnionSrc *pSrc = &pTab->aSrc[0]; + sqlite3 *tdb = unionGetDb(pTab, pSrc); + pStmt = unionPreparePrintf(&rc, pzErr, tdb, "SELECT " + "'CREATE TABLE xyz('" + " || group_concat(quote(name) || ' ' || type, ', ')" + " || ')'," + "max((cid+1) * (type='INTEGER' COLLATE nocase AND pk=1))-1 " + "FROM pragma_table_info(%Q, ?)", + pSrc->zTab, pSrc->zDb + ); + } + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zDecl = (const char*)sqlite3_column_text(pStmt, 0); + rc = sqlite3_declare_vtab(db, zDecl); + pTab->iPK = sqlite3_column_int(pStmt, 1); + } + + unionFinalize(&rc, pStmt, pzErr); + } + + if( rc!=SQLITE_OK ){ + unionDisconnect((sqlite3_vtab*)pTab); + pTab = 0; + } + + *ppVtab = (sqlite3_vtab*)pTab; + return rc; +} + +/* +** xOpen +*/ +static int unionOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + UnionCsr *pCsr; + int rc = SQLITE_OK; + (void)p; /* Suppress harmless warning */ + pCsr = (UnionCsr*)unionMalloc(&rc, sizeof(UnionCsr)); + *ppCursor = &pCsr->base; + return rc; +} + +/* +** xClose +*/ +static int unionClose(sqlite3_vtab_cursor *cur){ + UnionCsr *pCsr = (UnionCsr*)cur; + unionFinalizeCsrStmt(pCsr); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** This function does the work of the xNext() method. Except that, if it +** returns SQLITE_ROW, it should be called again within the same xNext() +** method call. See unionNext() for details. +*/ +static int doUnionNext(UnionCsr *pCsr){ + int rc = SQLITE_OK; + assert( pCsr->pStmt ); + if( sqlite3_step(pCsr->pStmt)!=SQLITE_ROW ){ + UnionTab *pTab = (UnionTab*)pCsr->base.pVtab; + rc = unionFinalizeCsrStmt(pCsr); + if( rc==SQLITE_OK && pTab->bSwarm ){ + pCsr->iTab++; + if( pCsr->iTabnSrc ){ + UnionSrc *pSrc = &pTab->aSrc[pCsr->iTab]; + if( pCsr->iMaxRowid>=pSrc->iMin ){ + /* It is necessary to scan the next table. */ + rc = unionOpenDatabase(pTab, pCsr->iTab, &pTab->base.zErrMsg); + pCsr->pStmt = unionPreparePrintf(&rc, &pTab->base.zErrMsg, pSrc->db, + "SELECT rowid, * FROM %Q %s %lld", + pSrc->zTab, + (pSrc->iMax>pCsr->iMaxRowid ? "WHERE _rowid_ <=" : "-- "), + pCsr->iMaxRowid + ); + if( rc==SQLITE_OK ){ + assert( pCsr->pStmt ); + unionIncrRefcount(pTab, pCsr->iTab); + rc = SQLITE_ROW; + } + } + } + } + } + + return rc; +} + +/* +** xNext +*/ +static int unionNext(sqlite3_vtab_cursor *cur){ + int rc; + do { + rc = doUnionNext((UnionCsr*)cur); + }while( rc==SQLITE_ROW ); + return rc; +} + +/* +** xColumn +*/ +static int unionColumn( + sqlite3_vtab_cursor *cur, + sqlite3_context *ctx, + int i +){ + UnionCsr *pCsr = (UnionCsr*)cur; + sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, i+1)); + return SQLITE_OK; +} + +/* +** xRowid +*/ +static int unionRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + UnionCsr *pCsr = (UnionCsr*)cur; + *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); + return SQLITE_OK; +} + +/* +** xEof +*/ +static int unionEof(sqlite3_vtab_cursor *cur){ + UnionCsr *pCsr = (UnionCsr*)cur; + return pCsr->pStmt==0; +} + +/* +** xFilter +*/ +static int unionFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + UnionTab *pTab = (UnionTab*)(pVtabCursor->pVtab); + UnionCsr *pCsr = (UnionCsr*)pVtabCursor; + int rc = SQLITE_OK; + int i; + char *zSql = 0; + int bZero = 0; + + sqlite3_int64 iMin = SMALLEST_INT64; + sqlite3_int64 iMax = LARGEST_INT64; + + assert( idxNum==0 + || idxNum==SQLITE_INDEX_CONSTRAINT_EQ + || idxNum==SQLITE_INDEX_CONSTRAINT_LE + || idxNum==SQLITE_INDEX_CONSTRAINT_GE + || idxNum==SQLITE_INDEX_CONSTRAINT_LT + || idxNum==SQLITE_INDEX_CONSTRAINT_GT + || idxNum==(SQLITE_INDEX_CONSTRAINT_GE|SQLITE_INDEX_CONSTRAINT_LE) + ); + + (void)idxStr; /* Suppress harmless warning */ + + if( idxNum==SQLITE_INDEX_CONSTRAINT_EQ ){ + assert( argc==1 ); + iMin = iMax = sqlite3_value_int64(argv[0]); + }else{ + + if( idxNum & (SQLITE_INDEX_CONSTRAINT_LE|SQLITE_INDEX_CONSTRAINT_LT) ){ + assert( argc>=1 ); + iMax = sqlite3_value_int64(argv[0]); + if( idxNum & SQLITE_INDEX_CONSTRAINT_LT ){ + if( iMax==SMALLEST_INT64 ){ + bZero = 1; + }else{ + iMax--; + } + } + } + + if( idxNum & (SQLITE_INDEX_CONSTRAINT_GE|SQLITE_INDEX_CONSTRAINT_GT) ){ + assert( argc>=1 ); + iMin = sqlite3_value_int64(argv[argc-1]); + if( idxNum & SQLITE_INDEX_CONSTRAINT_GT ){ + if( iMin==LARGEST_INT64 ){ + bZero = 1; + }else{ + iMin++; + } + } + } + } + + unionFinalizeCsrStmt(pCsr); + if( bZero ){ + return SQLITE_OK; + } + + for(i=0; inSrc; i++){ + UnionSrc *pSrc = &pTab->aSrc[i]; + if( iMin>pSrc->iMax || iMaxiMin ){ + continue; + } + + zSql = sqlite3_mprintf("%z%sSELECT rowid, * FROM %s%q%s%Q" + , zSql + , (zSql ? " UNION ALL " : "") + , (pSrc->zDb ? "'" : "") + , (pSrc->zDb ? pSrc->zDb : "") + , (pSrc->zDb ? "'." : "") + , pSrc->zTab + ); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + break; + } + + if( iMin==iMax ){ + zSql = sqlite3_mprintf("%z WHERE rowid=%lld", zSql, iMin); + }else{ + const char *zWhere = "WHERE"; + if( iMin!=SMALLEST_INT64 && iMin>pSrc->iMin ){ + zSql = sqlite3_mprintf("%z WHERE rowid>=%lld", zSql, iMin); + zWhere = "AND"; + } + if( iMax!=LARGEST_INT64 && iMaxiMax ){ + zSql = sqlite3_mprintf("%z %s rowid<=%lld", zSql, zWhere, iMax); + } + } + + if( pTab->bSwarm ){ + pCsr->iTab = i; + pCsr->iMaxRowid = iMax; + rc = unionOpenDatabase(pTab, i, &pTab->base.zErrMsg); + break; + } + } + + if( zSql==0 ){ + return rc; + }else{ + sqlite3 *db = unionGetDb(pTab, &pTab->aSrc[pCsr->iTab]); + pCsr->pStmt = unionPrepare(&rc, db, zSql, &pTab->base.zErrMsg); + if( pCsr->pStmt ){ + unionIncrRefcount(pTab, pCsr->iTab); + } + sqlite3_free(zSql); + } + if( rc!=SQLITE_OK ) return rc; + return unionNext(pVtabCursor); +} + +/* +** xBestIndex. +** +** This implementation searches for constraints on the rowid field. EQ, +** LE, LT, GE and GT are handled. +** +** If there is an EQ comparison, then idxNum is set to INDEX_CONSTRAINT_EQ. +** In this case the only argument passed to xFilter is the rhs of the == +** operator. +** +** Otherwise, if an LE or LT constraint is found, then the INDEX_CONSTRAINT_LE +** or INDEX_CONSTRAINT_LT (but not both) bit is set in idxNum. The first +** argument to xFilter is the rhs of the <= or < operator. Similarly, if +** an GE or GT constraint is found, then the INDEX_CONSTRAINT_GE or +** INDEX_CONSTRAINT_GT bit is set in idxNum. The rhs of the >= or > operator +** is passed as either the first or second argument to xFilter, depending +** on whether or not there is also a LT|LE constraint. +*/ +static int unionBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + UnionTab *pTab = (UnionTab*)tab; + int iEq = -1; + int iLt = -1; + int iGt = -1; + int i; + + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i]; + if( p->usable && (p->iColumn<0 || p->iColumn==pTab->iPK) ){ + switch( p->op ){ + case SQLITE_INDEX_CONSTRAINT_EQ: + iEq = i; + break; + case SQLITE_INDEX_CONSTRAINT_LE: + case SQLITE_INDEX_CONSTRAINT_LT: + iLt = i; + break; + case SQLITE_INDEX_CONSTRAINT_GE: + case SQLITE_INDEX_CONSTRAINT_GT: + iGt = i; + break; + } + } + } + + if( iEq>=0 ){ + pIdxInfo->estimatedRows = 1; + pIdxInfo->idxFlags = SQLITE_INDEX_SCAN_UNIQUE; + pIdxInfo->estimatedCost = 3.0; + pIdxInfo->idxNum = SQLITE_INDEX_CONSTRAINT_EQ; + pIdxInfo->aConstraintUsage[iEq].argvIndex = 1; + pIdxInfo->aConstraintUsage[iEq].omit = 1; + }else{ + int iCons = 1; + int idxNum = 0; + sqlite3_int64 nRow = 1000000; + if( iLt>=0 ){ + nRow = nRow / 2; + pIdxInfo->aConstraintUsage[iLt].argvIndex = iCons++; + pIdxInfo->aConstraintUsage[iLt].omit = 1; + idxNum |= pIdxInfo->aConstraint[iLt].op; + } + if( iGt>=0 ){ + nRow = nRow / 2; + pIdxInfo->aConstraintUsage[iGt].argvIndex = iCons++; + pIdxInfo->aConstraintUsage[iGt].omit = 1; + idxNum |= pIdxInfo->aConstraint[iGt].op; + } + pIdxInfo->estimatedRows = nRow; + pIdxInfo->estimatedCost = 3.0 * (double)nRow; + pIdxInfo->idxNum = idxNum; + } + + return SQLITE_OK; +} + +/* +** Register the unionvtab virtual table module with database handle db. +*/ +static int createUnionVtab(sqlite3 *db){ + static sqlite3_module unionModule = { + 0, /* iVersion */ + unionConnect, + unionConnect, + unionBestIndex, /* xBestIndex - query planner */ + unionDisconnect, + unionDisconnect, + unionOpen, /* xOpen - open a cursor */ + unionClose, /* xClose - close a cursor */ + unionFilter, /* xFilter - configure scan constraints */ + unionNext, /* xNext - advance a cursor */ + unionEof, /* xEof - check for end of scan */ + unionColumn, /* xColumn - read data */ + unionRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ + }; + int rc; + + rc = sqlite3_create_module(db, "unionvtab", &unionModule, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module(db, "swarmvtab", &unionModule, (void*)db); + } + return rc; +} + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_unionvtab_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Suppress harmless warning */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = createUnionVtab(db); +#endif + return rc; +} ADDED ext/misc/vtablog.c Index: ext/misc/vtablog.c ================================================================== --- /dev/null +++ ext/misc/vtablog.c @@ -0,0 +1,509 @@ +/* +** 2017-08-10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements a virtual table that prints diagnostic information +** on stdout when its key interfaces are called. This is intended for +** interactive analysis and debugging of virtual table interfaces. +** +** Usage example: +** +** .load ./vtablog +** CREATE VIRTUAL TABLE temp.log USING vtablog( +** schema='CREATE TABLE x(a,b,c)', +** rows=25 +** ); +** SELECT * FROM log; +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include +#include +#include +#include + + +/* vtablog_vtab is a subclass of sqlite3_vtab which will +** serve as the underlying representation of a vtablog virtual table +*/ +typedef struct vtablog_vtab vtablog_vtab; +struct vtablog_vtab { + sqlite3_vtab base; /* Base class - must be first */ + int nRow; /* Number of rows in the table */ + int iInst; /* Instance number for this vtablog table */ + int nCursor; /* Number of cursors created */ +}; + +/* vtablog_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result +*/ +typedef struct vtablog_cursor vtablog_cursor; +struct vtablog_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + int iCursor; /* Cursor number */ + sqlite3_int64 iRowid; /* The rowid */ +}; + +/* Skip leading whitespace. Return a pointer to the first non-whitespace +** character, or to the zero terminator if the string has only whitespace */ +static const char *vtablog_skip_whitespace(const char *z){ + while( isspace((unsigned char)z[0]) ) z++; + return z; +} + +/* Remove trailing whitespace from the end of string z[] */ +static void vtablog_trim_whitespace(char *z){ + size_t n = strlen(z); + while( n>0 && isspace((unsigned char)z[n]) ) n--; + z[n] = 0; +} + +/* Dequote the string */ +static void vtablog_dequote(char *z){ + int j; + char cQuote = z[0]; + size_t i, n; + + if( cQuote!='\'' && cQuote!='"' ) return; + n = strlen(z); + if( n<2 || z[n-1]!=z[0] ) return; + for(i=1, j=0; inRow = 10; + if( zNRow ) pNew->nRow = atoi(zNRow); + pNew->iInst = iInst; + } + return rc; +} +static int vtablogCreate( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + return vtablogConnectCreate(db,pAux,argc,argv,ppVtab,pzErr,1); +} +static int vtablogConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + return vtablogConnectCreate(db,pAux,argc,argv,ppVtab,pzErr,0); +} + + +/* +** This method is the destructor for vtablog_cursor objects. +*/ +static int vtablogDisconnect(sqlite3_vtab *pVtab){ + vtablog_vtab *pTab = (vtablog_vtab*)pVtab; + printf("vtablogDisconnect(%d)\n", pTab->iInst); + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** This method is the destructor for vtablog_cursor objects. +*/ +static int vtablogDestroy(sqlite3_vtab *pVtab){ + vtablog_vtab *pTab = (vtablog_vtab*)pVtab; + printf("vtablogDestroy(%d)\n", pTab->iInst); + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new vtablog_cursor object. +*/ +static int vtablogOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + vtablog_vtab *pTab = (vtablog_vtab*)p; + vtablog_cursor *pCur; + printf("vtablogOpen(tab=%d, cursor=%d)\n", pTab->iInst, ++pTab->nCursor); + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->iCursor = pTab->nCursor; + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Destructor for a vtablog_cursor. +*/ +static int vtablogClose(sqlite3_vtab_cursor *cur){ + vtablog_cursor *pCur = (vtablog_cursor*)cur; + vtablog_vtab *pTab = (vtablog_vtab*)cur->pVtab; + printf("vtablogClose(tab=%d, cursor=%d)\n", pTab->iInst, pCur->iCursor); + sqlite3_free(cur); + return SQLITE_OK; +} + + +/* +** Advance a vtablog_cursor to its next row of output. +*/ +static int vtablogNext(sqlite3_vtab_cursor *cur){ + vtablog_cursor *pCur = (vtablog_cursor*)cur; + vtablog_vtab *pTab = (vtablog_vtab*)cur->pVtab; + printf("vtablogNext(tab=%d, cursor=%d) rowid %d -> %d\n", + pTab->iInst, pCur->iCursor, (int)pCur->iRowid, (int)pCur->iRowid+1); + pCur->iRowid++; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the vtablog_cursor +** is currently pointing. +*/ +static int vtablogColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + vtablog_cursor *pCur = (vtablog_cursor*)cur; + vtablog_vtab *pTab = (vtablog_vtab*)cur->pVtab; + char zVal[50]; + + if( i<26 ){ + sqlite3_snprintf(sizeof(zVal),zVal,"%c%d", + "abcdefghijklmnopqrstuvwyz"[i], pCur->iRowid); + }else{ + sqlite3_snprintf(sizeof(zVal),zVal,"{%d}%d", i, pCur->iRowid); + } + printf("vtablogColumn(tab=%d, cursor=%d, i=%d): [%s]\n", + pTab->iInst, pCur->iCursor, i, zVal); + sqlite3_result_text(ctx, zVal, -1, SQLITE_TRANSIENT); + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int vtablogRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + vtablog_cursor *pCur = (vtablog_cursor*)cur; + vtablog_vtab *pTab = (vtablog_vtab*)cur->pVtab; + printf("vtablogRowid(tab=%d, cursor=%d): %d\n", + pTab->iInst, pCur->iCursor, (int)pCur->iRowid); + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int vtablogEof(sqlite3_vtab_cursor *cur){ + vtablog_cursor *pCur = (vtablog_cursor*)cur; + vtablog_vtab *pTab = (vtablog_vtab*)cur->pVtab; + int rc = pCur->iRowid >= pTab->nRow; + printf("vtablogEof(tab=%d, cursor=%d): %d\n", + pTab->iInst, pCur->iCursor, rc); + return rc; +} + +/* +** Output an sqlite3_value object's value as an SQL literal. +*/ +static void vtablogQuote(sqlite3_value *p){ + char z[50]; + switch( sqlite3_value_type(p) ){ + case SQLITE_NULL: { + printf("NULL"); + break; + } + case SQLITE_INTEGER: { + sqlite3_snprintf(50,z,"%lld", sqlite3_value_int64(p)); + printf("%s", z); + break; + } + case SQLITE_FLOAT: { + sqlite3_snprintf(50,z,"%!.20g", sqlite3_value_double(p)); + printf("%s", z); + break; + } + case SQLITE_BLOB: { + int n = sqlite3_value_bytes(p); + const unsigned char *z = (const unsigned char*)sqlite3_value_blob(p); + int i; + printf("x'"); + for(i=0; ipVtab; + printf("vtablogFilter(tab=%d, cursor=%d):\n", pTab->iInst, pCur->iCursor); + pCur->iRowid = 0; + return SQLITE_OK; +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the vtablog virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +*/ +static int vtablogBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + vtablog_vtab *pTab = (vtablog_vtab*)tab; + printf("vtablogBestIndex(tab=%d):\n", pTab->iInst); + pIdxInfo->estimatedCost = (double)500; + pIdxInfo->estimatedRows = 500; + return SQLITE_OK; +} + +/* +** SQLite invokes this method to INSERT, UPDATE, or DELETE content from +** the table. +** +** This implementation does not actually make any changes to the table +** content. It merely logs the fact that the method was invoked +*/ +static int vtablogUpdate( + sqlite3_vtab *tab, + int argc, + sqlite3_value **argv, + sqlite_int64 *pRowid +){ + vtablog_vtab *pTab = (vtablog_vtab*)tab; + int i; + printf("vtablogUpdate(tab=%d):\n", pTab->iInst); + printf(" argc=%d\n", argc); + for(i=0; i1048576] [expr $a1<1200000] \ + [expr $a2>1500000] [expr $a2<1700000] +} {1 1 1 1 1} + +do_test 1.2.1 { + setup_databases + run_rbu_cachesize test.db test.db2 10 1000000 +} {1 SQLITE_FULL} +do_test 1.2.2 { info commands rbu } {} + +do_test 1.3.1 { + setup_databases + run_rbu_cachesize test.db test.db2 10 1300000 +} {1 SQLITE_FULL} +do_test 1.3.2 { info commands rbu } {} + +do_test 1.4.1 { + setup_databases + run_rbu_cachesize test.db test.db2 10 1800000 +} {0 SQLITE_DONE} +do_test 1.4.2 { info commands rbu } {} + +do_test 1.5.1 { + setup_databases + unset -nocomplain ::A + step_rbu_cachesize test.db test.db2 1000 10 2400000 +} {0 SQLITE_DONE} +do_test 1.5.2 { info commands rbu } {} + +do_test 1.6.1 { + setup_databases + unset -nocomplain ::A + step_rbu_cachesize test.db test.db2 1000 10 1400000 +} {1 SQLITE_FULL} +do_test 1.6.2 { info commands rbu } {} + +finish_test + Index: ext/rbu/sqlite3rbu.c ================================================================== --- ext/rbu/sqlite3rbu.c +++ ext/rbu/sqlite3rbu.c @@ -369,35 +369,47 @@ int nFrameAlloc; /* Allocated size of aFrame[] array */ RbuFrame *aFrame; int pgsz; u8 *aBuf; i64 iWalCksum; + i64 szTemp; /* Current size of all temp files in use */ + i64 szTempLimit; /* Total size limit for temp files */ /* Used in RBU vacuum mode only */ int nRbu; /* Number of RBU VFS in the stack */ rbu_file *pRbuFd; /* Fd for main db of dbRbu */ }; /* ** An rbu VFS is implemented using an instance of this structure. +** +** Variable pRbu is only non-NULL for automatically created RBU VFS objects. +** It is NULL for RBU VFS objects created explicitly using +** sqlite3rbu_create_vfs(). It is used to track the total amount of temp +** space used by the RBU handle. */ struct rbu_vfs { sqlite3_vfs base; /* rbu VFS shim methods */ sqlite3_vfs *pRealVfs; /* Underlying VFS */ sqlite3_mutex *mutex; /* Mutex to protect pMain */ + sqlite3rbu *pRbu; /* Owner RBU object */ rbu_file *pMain; /* Linked list of main db files */ }; /* ** Each file opened by an rbu VFS is represented by an instance of ** the following structure. +** +** If this is a temporary file (pRbu!=0 && flags&DELETE_ON_CLOSE), variable +** "sz" is set to the current size of the database file. */ struct rbu_file { sqlite3_file base; /* sqlite3_file methods */ sqlite3_file *pReal; /* Underlying file handle */ rbu_vfs *pRbuVfs; /* Pointer to the rbu_vfs object */ sqlite3rbu *pRbu; /* Pointer to rbu object (rbu target only) */ + i64 sz; /* Size of file in bytes (temp only) */ int openFlags; /* Flags this file was opened with */ u32 iCookie; /* Cookie value for main db files */ u8 iWriteVer; /* "write-version" value for main db files */ u8 bNolock; /* True to fail EXCLUSIVE locks */ @@ -3407,10 +3419,11 @@ p->rc = sqlite3rbu_create_vfs(zRnd, 0); if( p->rc==SQLITE_OK ){ sqlite3_vfs *pVfs = sqlite3_vfs_find(zRnd); assert( pVfs ); p->zVfsName = pVfs->zName; + ((rbu_vfs*)pVfs)->pRbu = p; } } /* ** Destroy the private VFS created for the rbu handle passed as the only @@ -3779,10 +3792,11 @@ } /* Close the open database handle and VFS object. */ sqlite3_close(p->dbRbu); sqlite3_close(p->dbMain); + assert( p->szTemp==0 ); rbuDeleteVfs(p); sqlite3_free(p->aBuf); sqlite3_free(p->aFrame); rbuEditErrmsg(p); @@ -3966,10 +3980,11 @@ ** relinquished). Finally, calls to xSync() on the target database ** file fail with SQLITE_INTERNAL errors. */ static void rbuUnlockShm(rbu_file *p){ + assert( p->openFlags & SQLITE_OPEN_MAIN_DB ); if( p->pRbu ){ int (*xShmLock)(sqlite3_file*,int,int,int) = p->pReal->pMethods->xShmLock; int i; for(i=0; ipRbu->mLock ){ @@ -3977,10 +3992,22 @@ } } p->pRbu->mLock = 0; } } + +/* +*/ +static int rbuUpdateTempSize(rbu_file *pFd, sqlite3_int64 nNew){ + sqlite3rbu *pRbu = pFd->pRbu; + i64 nDiff = nNew - pFd->sz; + pRbu->szTemp += nDiff; + pFd->sz = nNew; + assert( pRbu->szTemp>=0 ); + if( pRbu->szTempLimit && pRbu->szTemp>pRbu->szTempLimit ) return SQLITE_FULL; + return SQLITE_OK; +} /* ** Close an rbu file. */ static int rbuVfsClose(sqlite3_file *pFile){ @@ -4002,10 +4029,13 @@ for(pp=&p->pRbuVfs->pMain; *pp!=p; pp=&((*pp)->pMainNext)); *pp = p->pMainNext; sqlite3_mutex_leave(p->pRbuVfs->mutex); rbuUnlockShm(p); p->pReal->pMethods->xShmUnmap(p->pReal, 0); + } + else if( (p->openFlags & SQLITE_OPEN_DELETEONCLOSE) && p->pRbu ){ + rbuUpdateTempSize(p, 0); } /* Close the underlying file handle */ rc = p->pReal->pMethods->xClose(p->pReal); return rc; @@ -4120,15 +4150,23 @@ if( pRbu && pRbu->eStage==RBU_STAGE_CAPTURE ){ assert( p->openFlags & SQLITE_OPEN_MAIN_DB ); rc = rbuCaptureDbWrite(p->pRbu, iOfst); }else{ - if( pRbu && pRbu->eStage==RBU_STAGE_OAL - && (p->openFlags & SQLITE_OPEN_WAL) - && iOfst>=pRbu->iOalSz - ){ - pRbu->iOalSz = iAmt + iOfst; + if( pRbu ){ + if( pRbu->eStage==RBU_STAGE_OAL + && (p->openFlags & SQLITE_OPEN_WAL) + && iOfst>=pRbu->iOalSz + ){ + pRbu->iOalSz = iAmt + iOfst; + }else if( p->openFlags & SQLITE_OPEN_DELETEONCLOSE ){ + i64 szNew = iAmt+iOfst; + if( szNew>p->sz ){ + rc = rbuUpdateTempSize(p, szNew); + if( rc!=SQLITE_OK ) return rc; + } + } } rc = p->pReal->pMethods->xWrite(p->pReal, zBuf, iAmt, iOfst); if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){ /* These look like magic numbers. But they are stable, as they are part ** of the definition of the SQLite file format, which may not change. */ @@ -4143,10 +4181,14 @@ /* ** Truncate an rbuVfs-file. */ static int rbuVfsTruncate(sqlite3_file *pFile, sqlite_int64 size){ rbu_file *p = (rbu_file*)pFile; + if( (p->openFlags & SQLITE_OPEN_DELETEONCLOSE) && p->pRbu ){ + int rc = rbuUpdateTempSize(p, size); + if( rc!=SQLITE_OK ) return rc; + } return p->pReal->pMethods->xTruncate(p->pReal, size); } /* ** Sync an rbuVfs-file. @@ -4532,10 +4574,12 @@ pFd->pRbu = pDb->pRbu; } pDb->pWalFd = pFd; } } + }else{ + pFd->pRbu = pRbuVfs->pRbu; } if( oflags & SQLITE_OPEN_MAIN_DB && sqlite3_uri_boolean(zName, "rbu_memory", 0) ){ @@ -4608,11 +4652,13 @@ rbu_file *pDb = rbuFindMaindb(pRbuVfs, zPath); if( pDb && pDb->pRbu && pDb->pRbu->eStage==RBU_STAGE_OAL ){ if( *pResOut ){ rc = SQLITE_CANTOPEN; }else{ - *pResOut = 1; + sqlite3_int64 sz = 0; + rc = rbuVfsFileSize(&pDb->base, &sz); + *pResOut = (sz>0); } } } return rc; @@ -4796,10 +4842,24 @@ } } return rc; } + +/* +** Configure the aggregate temp file size limit for this RBU handle. +*/ +sqlite3_int64 sqlite3rbu_temp_size_limit(sqlite3rbu *pRbu, sqlite3_int64 n){ + if( n>=0 ){ + pRbu->szTempLimit = n; + } + return pRbu->szTempLimit; +} + +sqlite3_int64 sqlite3rbu_temp_size(sqlite3rbu *pRbu){ + return pRbu->szTemp; +} /**************************************************************************/ #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU) */ Index: ext/rbu/sqlite3rbu.h ================================================================== --- ext/rbu/sqlite3rbu.h +++ ext/rbu/sqlite3rbu.h @@ -306,11 +306,11 @@ ** IMPORTANT NOTE FOR ZIPVFS USERS: The RBU extension works with all of ** SQLite's built-in VFSs, including the multiplexor VFS. However it does ** not work out of the box with zipvfs. Refer to the comment describing ** the zipvfs_create_vfs() API below for details on using RBU with zipvfs. */ -sqlite3rbu *sqlite3rbu_open( +SQLITE_API sqlite3rbu *sqlite3rbu_open( const char *zTarget, const char *zRbu, const char *zState ); @@ -345,15 +345,37 @@ ** As with sqlite3rbu_open(), Zipvfs users should rever to the comment ** describing the sqlite3rbu_create_vfs() API function below for ** a description of the complications associated with using RBU with ** zipvfs databases. */ -sqlite3rbu *sqlite3rbu_vacuum( +SQLITE_API sqlite3rbu *sqlite3rbu_vacuum( const char *zTarget, const char *zState ); +/* +** Configure a limit for the amount of temp space that may be used by +** the RBU handle passed as the first argument. The new limit is specified +** in bytes by the second parameter. If it is positive, the limit is updated. +** If the second parameter to this function is passed zero, then the limit +** is removed entirely. If the second parameter is negative, the limit is +** not modified (this is useful for querying the current limit). +** +** In all cases the returned value is the current limit in bytes (zero +** indicates unlimited). +** +** If the temp space limit is exceeded during operation, an SQLITE_FULL +** error is returned. +*/ +SQLITE_API sqlite3_int64 sqlite3rbu_temp_size_limit(sqlite3rbu*, sqlite3_int64); + +/* +** Return the current amount of temp file space, in bytes, currently used by +** the RBU handle passed as the only argument. +*/ +SQLITE_API sqlite3_int64 sqlite3rbu_temp_size(sqlite3rbu*); + /* ** Internally, each RBU connection uses a separate SQLite database ** connection to access the target and rbu update databases. This ** API allows the application direct access to these database handles. ** @@ -381,11 +403,11 @@ ** when sqlite3rbu_close() is called. ** ** Database handles returned by this function remain valid until the next ** call to any sqlite3rbu_xxx() function other than sqlite3rbu_db(). */ -sqlite3 *sqlite3rbu_db(sqlite3rbu*, int bRbu); +SQLITE_API sqlite3 *sqlite3rbu_db(sqlite3rbu*, int bRbu); /* ** Do some work towards applying the RBU update to the target db. ** ** Return SQLITE_DONE if the update has been completely applied, or @@ -395,11 +417,11 @@ ** ** Once a call to sqlite3rbu_step() has returned a value other than ** SQLITE_OK, all subsequent calls on the same RBU handle are no-ops ** that immediately return the same value. */ -int sqlite3rbu_step(sqlite3rbu *pRbu); +SQLITE_API int sqlite3rbu_step(sqlite3rbu *pRbu); /* ** Force RBU to save its state to disk. ** ** If a power failure or application crash occurs during an update, following @@ -407,11 +429,11 @@ ** was last saved. In other words, from the most recent successful call to ** sqlite3rbu_close() or this function. ** ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ -int sqlite3rbu_savestate(sqlite3rbu *pRbu); +SQLITE_API int sqlite3rbu_savestate(sqlite3rbu *pRbu); /* ** Close an RBU handle. ** ** If the RBU update has been completely applied, mark the RBU database @@ -427,18 +449,18 @@ ** ** Otherwise, if no error occurs, this function returns SQLITE_OK if the ** update has been partially applied, or SQLITE_DONE if it has been ** completely applied. */ -int sqlite3rbu_close(sqlite3rbu *pRbu, char **pzErrmsg); +SQLITE_API int sqlite3rbu_close(sqlite3rbu *pRbu, char **pzErrmsg); /* ** Return the total number of key-value operations (inserts, deletes or ** updates) that have been performed on the target database since the ** current RBU update was started. */ -sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu); +SQLITE_API sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu); /* ** Obtain permyriadage (permyriadage is to 10000 as percentage is to 100) ** progress indications for the two stages of an RBU update. This API may ** be useful for driving GUI progress indicators and similar. @@ -476,11 +498,11 @@ ** permyriadage progress of the same stage. If the rbu_count table does ** not exist, then (*pnOne) is set to -1 during stage 1. If the rbu_count ** table exists but is not correctly populated, the value of the *pnOne ** output variable during stage 1 is undefined. */ -void sqlite3rbu_bp_progress(sqlite3rbu *pRbu, int *pnOne, int *pnTwo); +SQLITE_API void sqlite3rbu_bp_progress(sqlite3rbu *pRbu, int *pnOne, int*pnTwo); /* ** Obtain an indication as to the current stage of an RBU update or vacuum. ** This function always returns one of the SQLITE_RBU_STATE_XXX constants ** defined in this file. Return values should be interpreted as follows: @@ -514,11 +536,11 @@ #define SQLITE_RBU_STATE_MOVE 2 #define SQLITE_RBU_STATE_CHECKPOINT 3 #define SQLITE_RBU_STATE_DONE 4 #define SQLITE_RBU_STATE_ERROR 5 -int sqlite3rbu_state(sqlite3rbu *pRbu); +SQLITE_API int sqlite3rbu_state(sqlite3rbu *pRbu); /* ** Create an RBU VFS named zName that accesses the underlying file-system ** via existing VFS zParent. Or, if the zParent parameter is passed NULL, ** then the new RBU VFS uses the default system VFS to access the file-system. @@ -558,22 +580,22 @@ ** The overhead of adding the "rbu" VFS to the system is negligible for ** non-RBU users. There is no harm in an application accessing the ** file-system via "rbu" all the time, even if it only uses RBU functionality ** occasionally. */ -int sqlite3rbu_create_vfs(const char *zName, const char *zParent); +SQLITE_API int sqlite3rbu_create_vfs(const char *zName, const char *zParent); /* ** Deregister and destroy an RBU vfs created by an earlier call to ** sqlite3rbu_create_vfs(). ** ** VFS objects are not reference counted. If a VFS object is destroyed ** before all database handles that use it have been closed, the results ** are undefined. */ -void sqlite3rbu_destroy_vfs(const char *zName); +SQLITE_API void sqlite3rbu_destroy_vfs(const char *zName); #ifdef __cplusplus } /* end of the 'extern "C"' block */ #endif #endif /* _SQLITE3RBU_H */ Index: ext/rbu/test_rbu.c ================================================================== --- ext/rbu/test_rbu.c +++ ext/rbu/test_rbu.c @@ -67,20 +67,22 @@ struct RbuCmd { const char *zName; int nArg; const char *zUsage; } aCmd[] = { - {"step", 2, ""}, /* 0 */ - {"close", 2, ""}, /* 1 */ - {"create_rbu_delta", 2, ""}, /* 2 */ - {"savestate", 2, ""}, /* 3 */ - {"dbMain_eval", 3, "SQL"}, /* 4 */ - {"bp_progress", 2, ""}, /* 5 */ - {"db", 3, "RBU"}, /* 6 */ - {"state", 2, ""}, /* 7 */ - {"progress", 2, ""}, /* 8 */ - {"close_no_error", 2, ""}, /* 9 */ + {"step", 2, ""}, /* 0 */ + {"close", 2, ""}, /* 1 */ + {"create_rbu_delta", 2, ""}, /* 2 */ + {"savestate", 2, ""}, /* 3 */ + {"dbMain_eval", 3, "SQL"}, /* 4 */ + {"bp_progress", 2, ""}, /* 5 */ + {"db", 3, "RBU"}, /* 6 */ + {"state", 2, ""}, /* 7 */ + {"progress", 2, ""}, /* 8 */ + {"close_no_error", 2, ""}, /* 9 */ + {"temp_size_limit", 3, "LIMIT"}, /* 10 */ + {"temp_size", 2, ""}, /* 11 */ {0,0,0} }; int iCmd; if( objc<2 ){ @@ -190,10 +192,26 @@ } case 8: /* progress */ { sqlite3_int64 nStep = sqlite3rbu_progress(pRbu); Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nStep)); break; + } + + case 10: /* temp_size_limit */ { + sqlite3_int64 nLimit; + if( Tcl_GetWideIntFromObj(interp, objv[2], &nLimit) ){ + ret = TCL_ERROR; + }else{ + nLimit = sqlite3rbu_temp_size_limit(pRbu, nLimit); + Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nLimit)); + } + break; + } + case 11: /* temp_size */ { + sqlite3_int64 sz = sqlite3rbu_temp_size(pRbu); + Tcl_SetObjResult(interp, Tcl_NewWideIntObj(sz)); + break; } default: /* seems unlikely */ assert( !"cannot happen" ); break; Index: ext/rtree/rtree.c ================================================================== --- ext/rtree/rtree.c +++ ext/rtree/rtree.c @@ -338,26 +338,18 @@ int (*xQueryFunc)(sqlite3_rtree_query_info*); void (*xDestructor)(void*); void *pContext; }; - -/* -** Value for the first field of every RtreeMatchArg object. The MATCH -** operator tests that the first field of a blob operand matches this -** value to avoid operating on invalid blobs (which could cause a segfault). -*/ -#define RTREE_GEOMETRY_MAGIC 0x891245AB - /* ** An instance of this structure (in the form of a BLOB) is returned by ** the SQL functions that sqlite3_rtree_geometry_callback() and ** sqlite3_rtree_query_callback() create, and is read as the right-hand ** operand to the MATCH operator of an R-Tree. */ struct RtreeMatchArg { - u32 magic; /* Always RTREE_GEOMETRY_MAGIC */ + u32 iSize; /* Size of this object */ RtreeGeomCallback cb; /* Info about the callback functions */ int nParam; /* Number of parameters to the SQL function */ sqlite3_value **apSqlParam; /* Original SQL parameter values */ RtreeDValue aParam[1]; /* Values for parameters to the SQL function */ }; @@ -1648,37 +1640,21 @@ ** as the second argument for a MATCH constraint. The value passed as the ** first argument to this function is the right-hand operand to the MATCH ** operator. */ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ - RtreeMatchArg *pBlob; /* BLOB returned by geometry function */ + RtreeMatchArg *pBlob, *pSrc; /* BLOB returned by geometry function */ sqlite3_rtree_query_info *pInfo; /* Callback information */ - int nBlob; /* Size of the geometry function blob */ - int nExpected; /* Expected size of the BLOB */ - - /* Check that value is actually a blob. */ - if( sqlite3_value_type(pValue)!=SQLITE_BLOB ) return SQLITE_ERROR; - - /* Check that the blob is roughly the right size. */ - nBlob = sqlite3_value_bytes(pValue); - if( nBlob<(int)sizeof(RtreeMatchArg) ){ - return SQLITE_ERROR; - } - - pInfo = (sqlite3_rtree_query_info*)sqlite3_malloc( sizeof(*pInfo)+nBlob ); + + pSrc = sqlite3_value_pointer(pValue, "RtreeMatchArg"); + if( pSrc==0 ) return SQLITE_ERROR; + pInfo = (sqlite3_rtree_query_info*) + sqlite3_malloc64( sizeof(*pInfo)+pSrc->iSize ); if( !pInfo ) return SQLITE_NOMEM; memset(pInfo, 0, sizeof(*pInfo)); pBlob = (RtreeMatchArg*)&pInfo[1]; - - memcpy(pBlob, sqlite3_value_blob(pValue), nBlob); - nExpected = (int)(sizeof(RtreeMatchArg) + - pBlob->nParam*sizeof(sqlite3_value*) + - (pBlob->nParam-1)*sizeof(RtreeDValue)); - if( pBlob->magic!=RTREE_GEOMETRY_MAGIC || nBlob!=nExpected ){ - sqlite3_free(pInfo); - return SQLITE_ERROR; - } + memcpy(pBlob, pSrc, pSrc->iSize); pInfo->pContext = pBlob->cb.pContext; pInfo->nParam = pBlob->nParam; pInfo->aParam = pBlob->aParam; pInfo->apSqlParam = pBlob->apSqlParam; @@ -3437,11 +3413,11 @@ ); rc = getIntFromStmt(db, zSql, &pRtree->iNodeSize); if( rc!=SQLITE_OK ){ *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); }else if( pRtree->iNodeSize<(512-64) ){ - rc = SQLITE_CORRUPT; + rc = SQLITE_CORRUPT_VTAB; *pzErr = sqlite3_mprintf("undersize RTree blobs in \"%q_node\"", pRtree->zName); } } @@ -3712,11 +3688,11 @@ pBlob = (RtreeMatchArg *)sqlite3_malloc(nBlob); if( !pBlob ){ sqlite3_result_error_nomem(ctx); }else{ int i; - pBlob->magic = RTREE_GEOMETRY_MAGIC; + pBlob->iSize = nBlob; pBlob->cb = pGeomCtx[0]; pBlob->apSqlParam = (sqlite3_value**)&pBlob->aParam[nArg]; pBlob->nParam = nArg; for(i=0; iapSqlParam[i] = sqlite3_value_dup(aArg[i]); @@ -3729,11 +3705,11 @@ } if( memErr ){ sqlite3_result_error_nomem(ctx); rtreeMatchArgFree(pBlob); }else{ - sqlite3_result_blob(ctx, pBlob, nBlob, rtreeMatchArgFree); + sqlite3_result_pointer(ctx, pBlob, "RtreeMatchArg", rtreeMatchArgFree); } } } /* Index: ext/rtree/rtree1.test ================================================================== --- ext/rtree/rtree1.test +++ ext/rtree/rtree1.test @@ -602,11 +602,10 @@ } do_execsql_test 15.1 { BEGIN; INSERT INTO rt VALUES(1,2,3,4,5); } -breakpoint do_execsql_test 15.2 { DROP TABLE t13; COMMIT; } Index: ext/rtree/rtreeA.test ================================================================== --- ext/rtree/rtreeA.test +++ ext/rtree/rtreeA.test @@ -226,8 +226,12 @@ UPDATE t1_node SET data=x'' WHERE rowid=1; } {} do_catchsql_test rtreeA-7.110 { SELECT * FROM t1 WHERE x1>0 AND x1<100 AND x2>0 AND x2<100; } {1 {undersize RTree blobs in "t1_node"}} +do_test rtreeA-7.120 { + sqlite3_extended_errcode db +} {SQLITE_CORRUPT_VTAB} + finish_test ADDED ext/rtree/rtreeconnect.test Index: ext/rtree/rtreeconnect.test ================================================================== --- /dev/null +++ ext/rtree/rtreeconnect.test @@ -0,0 +1,56 @@ +# 2017 August 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the r-tree extension. Specifically, +# the impact of an SQLITE_SCHEMA error within the rtree module xConnect +# callback. +# + + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +set testprefix rtreeconnect + +ifcapable !rtree { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE r1 USING rtree(id, x1, x2, y1, y2); + CREATE TABLE t1(id, x1, x2, y1, y2); + CREATE TABLE log(l); + + CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN + INSERT INTO r1 VALUES(new.id, new.x1, new.x2, new.y1, new.y2); + INSERT INTO log VALUES('r1: ' || new.id); + END; +} + +db close +sqlite3 db test.db +sqlite3 db2 test.db + +do_test 1.1 { + db eval { INSERT INTO log VALUES('startup'); } + db2 eval { CREATE TABLE newtable(x,y); } +} {} + +do_execsql_test 1.2 { + INSERT INTO t1 VALUES(1, 2, 3, 4, 5); +} + +db2 close +db close + +finish_test Index: ext/session/session1.test ================================================================== --- ext/session/session1.test +++ ext/session/session1.test @@ -534,11 +534,10 @@ execsql " INSERT INTO $tblname VALUES('uvw', 'abc'); DELETE FROM $tblname WHERE a = 'xyz'; " } {} -breakpoint do_changeset_test $tn.10.1.2 S " {INSERT $tblname 0 X. {} {t uvw t abc}} {DELETE $tblname 0 X. {t xyz t def} {}} " do_test $tn.10.1.4 { S delete } {} @@ -575,11 +574,10 @@ execsql " INSERT INTO $tblname VALUES('uvw', 'abc'); DELETE FROM $tblname WHERE a = 'xyz'; " } {} -breakpoint do_changeset_test $tn.10.1.2 S " {INSERT $tblname 0 X. {} {t uvw t abc}} {DELETE $tblname 0 X. {t xyz t def} {}} " do_test $tn.10.1.4 { S delete } {} Index: ext/session/sessionE.test ================================================================== --- ext/session/sessionE.test +++ ext/session/sessionE.test @@ -40,11 +40,10 @@ CREATE TABLE t2(a PRIMARY KEY, b); } do_test 1.1 { sqlite3session S db main S attach * - breakpoint execsql { INSERT INTO t1 VALUES(1, 2); INSERT INTO t2 VALUES(1, 2); } } {} @@ -60,11 +59,10 @@ } do_test 2.1 { sqlite3session S db main S attach t1 S attach t2 - breakpoint execsql { INSERT INTO t1 VALUES(3, 4); INSERT INTO t2 VALUES(3, 4); INSERT INTO t1 VALUES(5, 6); INSERT INTO t2 VALUES(5, 6); Index: main.mk ================================================================== --- main.mk +++ main.mk @@ -342,10 +342,11 @@ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/remember.c \ $(TOP)/ext/misc/series.c \ $(TOP)/ext/misc/spellfix.c \ $(TOP)/ext/misc/totype.c \ + $(TOP)/ext/misc/unionvtab.c \ $(TOP)/ext/misc/wholenumber.c \ $(TOP)/ext/misc/vfslog.c \ $(TOP)/ext/fts5/fts5_tcl.c \ $(TOP)/ext/fts5/fts5_test_mi.c \ $(TOP)/ext/fts5/fts5_test_tok.c @@ -591,10 +592,15 @@ # lemon: $(TOP)/tool/lemon.c $(TOP)/tool/lempar.c $(BCC) -o lemon $(TOP)/tool/lemon.c cp $(TOP)/tool/lempar.c . +# A tool to generate the source-id +# +mksourceid: $(TOP)/tool/mksourceid.c + $(BCC) -o mksourceid $(TOP)/tool/mksourceid.c + # Rules to build individual *.o files from generated *.c files. This # applies to: # # parse.o # opcodes.o @@ -630,11 +636,11 @@ rm -f parse.h ./lemon -s $(OPTS) parse.y mv parse.h parse.h.temp tclsh $(TOP)/tool/addopcodes.tcl parse.h.temp >parse.h -sqlite3.h: $(TOP)/src/sqlite.h.in $(TOP)/manifest.uuid $(TOP)/VERSION $(TOP)/ext/rtree/sqlite3rtree.h +sqlite3.h: $(TOP)/src/sqlite.h.in $(TOP)/manifest mksourceid $(TOP)/VERSION $(TOP)/ext/rtree/sqlite3rtree.h tclsh $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h keywordhash.h: $(TOP)/tool/mkkeywordhash.c $(BCC) -o mkkeywordhash $(OPTS) $(TOP)/tool/mkkeywordhash.c ./mkkeywordhash >keywordhash.h Index: src/alter.c ================================================================== --- src/alter.c +++ src/alter.c @@ -401,22 +401,22 @@ Vdbe *v; #ifndef SQLITE_OMIT_TRIGGER char *zWhere = 0; /* Where clause to locate temp triggers */ #endif VTable *pVTab = 0; /* Non-zero if this is a v-tab with an xRename() */ - int savedDbFlags; /* Saved value of db->flags */ + u32 savedDbFlags; /* Saved value of db->mDbFlags */ - savedDbFlags = db->flags; + savedDbFlags = db->mDbFlags; if( NEVER(db->mallocFailed) ) goto exit_rename_table; assert( pSrc->nSrc==1 ); assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); if( !pTab ) goto exit_rename_table; iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); zDb = db->aDb[iDb].zDbSName; - db->flags |= SQLITE_PreferBuiltin; + db->mDbFlags |= DBFLAG_PreferBuiltin; /* Get a NULL terminated version of the new table name. */ zName = sqlite3NameFromToken(db, pName); if( !zName ) goto exit_rename_table; @@ -577,11 +577,11 @@ reloadTableSchema(pParse, pTab, zName); exit_rename_table: sqlite3SrcListDelete(db, pSrc); sqlite3DbFree(db, zName); - db->flags = savedDbFlags; + db->mDbFlags = savedDbFlags; } /* ** This function is called after an "ALTER TABLE ... ADD" statement ** has been parsed. Argument pColDef contains the text of the new @@ -678,24 +678,24 @@ /* Modify the CREATE TABLE statement. */ zCol = sqlite3DbStrNDup(db, (char*)pColDef->z, pColDef->n); if( zCol ){ char *zEnd = &zCol[pColDef->n-1]; - int savedDbFlags = db->flags; + u32 savedDbFlags = db->mDbFlags; while( zEnd>zCol && (*zEnd==';' || sqlite3Isspace(*zEnd)) ){ *zEnd-- = '\0'; } - db->flags |= SQLITE_PreferBuiltin; + db->mDbFlags |= DBFLAG_PreferBuiltin; sqlite3NestedParse(pParse, "UPDATE \"%w\".%s SET " "sql = substr(sql,1,%d) || ', ' || %Q || substr(sql,%d) " "WHERE type = 'table' AND name = %Q", zDb, MASTER_NAME, pNew->addColOffset, zCol, pNew->addColOffset+1, zTab ); sqlite3DbFree(db, zCol); - db->flags = savedDbFlags; + db->mDbFlags = savedDbFlags; } /* Make sure the schema version is at least 3. But do not upgrade ** from less than 3 to 4, as that will corrupt any preexisting DESC ** index. Index: src/attach.c ================================================================== --- src/attach.c +++ src/attach.c @@ -91,14 +91,10 @@ zErrDyn = sqlite3MPrintf(db, "too many attached databases - max %d", db->aLimit[SQLITE_LIMIT_ATTACHED] ); goto attach_error; } - if( !db->autoCommit ){ - zErrDyn = sqlite3MPrintf(db, "cannot ATTACH database within transaction"); - goto attach_error; - } for(i=0; inDb; i++){ char *z = db->aDb[i].zDbSName; assert( z && zName ); if( sqlite3StrICmp(z, zName)==0 ){ zErrDyn = sqlite3MPrintf(db, "database %s is already in use", zName); @@ -286,15 +282,10 @@ } if( i<2 ){ sqlite3_snprintf(sizeof(zErr),zErr, "cannot detach database %s", zName); goto detach_error; } - if( !db->autoCommit ){ - sqlite3_snprintf(sizeof(zErr), zErr, - "cannot DETACH database within transaction"); - goto detach_error; - } if( sqlite3BtreeIsInReadTrans(pDb->pBt) || sqlite3BtreeIsInBackup(pDb->pBt) ){ sqlite3_snprintf(sizeof(zErr),zErr, "database %s is locked", zName); goto detach_error; } Index: src/auth.c ================================================================== --- src/auth.c +++ src/auth.c @@ -116,15 +116,13 @@ #ifdef SQLITE_USER_AUTHENTICATION ,db->auth.zAuthUser #endif ); if( rc==SQLITE_DENY ){ - if( db->nDb>2 || iDb!=0 ){ - sqlite3ErrorMsg(pParse, "access to %s.%s.%s is prohibited",zDb,zTab,zCol); - }else{ - sqlite3ErrorMsg(pParse, "access to %s.%s is prohibited", zTab, zCol); - } + char *z = sqlite3_mprintf("%s.%s", zTab, zCol); + if( db->nDb>2 || iDb!=0 ) z = sqlite3_mprintf("%s.%z", zDb, z); + sqlite3ErrorMsg(pParse, "access to %z is prohibited", z); pParse->rc = SQLITE_AUTH; }else if( rc!=SQLITE_IGNORE && rc!=SQLITE_OK ){ sqliteAuthBadReturnCode(pParse); } return rc; Index: src/btree.c ================================================================== --- src/btree.c +++ src/btree.c @@ -443,11 +443,13 @@ } } #endif /* SQLITE_OMIT_SHARED_CACHE */ -static void releasePage(MemPage *pPage); /* Forward reference */ +static void releasePage(MemPage *pPage); /* Forward reference */ +static void releasePageOne(MemPage *pPage); /* Forward reference */ +static void releasePageNotNull(MemPage *pPage); /* Forward reference */ /* ***** This routine is used inside of assert() only **** ** ** Verify that the cursor holds the mutex on its BtShared @@ -602,15 +604,17 @@ /* ** Release all of the apPage[] pages for a cursor. */ static void btreeReleaseAllCursorPages(BtCursor *pCur){ int i; - for(i=0; i<=pCur->iPage; i++){ - releasePage(pCur->apPage[i]); - pCur->apPage[i] = 0; + if( pCur->iPage>=0 ){ + for(i=0; iiPage; i++){ + releasePageNotNull(pCur->apPage[i]); + } + releasePageNotNull(pCur->pPage); + pCur->iPage = -1; } - pCur->iPage = -1; } /* ** The cursor passed as the only argument must point to a valid entry ** when this function is called (i.e. have eState==CURSOR_VALID). This @@ -735,11 +739,11 @@ int rc = saveCursorPosition(p); if( SQLITE_OK!=rc ){ return rc; } }else{ - testcase( p->iPage>0 ); + testcase( p->iPage>=0 ); btreeReleaseAllCursorPages(p); } } p = p->pNext; }while( p ); @@ -775,11 +779,11 @@ assert( nKey==(i64)(int)nKey ); pIdxKey = sqlite3VdbeAllocUnpackedRecord(pCur->pKeyInfo); if( pIdxKey==0 ) return SQLITE_NOMEM_BKPT; sqlite3VdbeRecordUnpack(pCur->pKeyInfo, (int)nKey, pKey, pIdxKey); if( pIdxKey->nField==0 ){ - rc = SQLITE_CORRUPT_PGNO(pCur->apPage[pCur->iPage]->pgno); + rc = SQLITE_CORRUPT_BKPT; goto moveto_done; } }else{ pIdxKey = 0; } @@ -838,10 +842,21 @@ ** back to where it ought to be if this routine returns true. */ int sqlite3BtreeCursorHasMoved(BtCursor *pCur){ return pCur->eState!=CURSOR_VALID; } + +/* +** Return a pointer to a fake BtCursor object that will always answer +** false to the sqlite3BtreeCursorHasMoved() routine above. The fake +** cursor returned must not be used with any other Btree interface. +*/ +BtCursor *sqlite3BtreeFakeValidCursor(void){ + static u8 fakeCursor = CURSOR_VALID; + assert( offsetof(BtCursor, eState)==0 ); + return (BtCursor*)&fakeCursor; +} /* ** This routine restores a cursor back to its original position after it ** has been moved by some outside activity (such as a btree rebalance or ** a row having been deleted out from under the cursor). @@ -1389,11 +1404,11 @@ u8 *pAddr; int sz2 = 0; int sz = get2byte(&data[iFree+2]); int top = get2byte(&data[hdr+5]); if( iFree2 ){ - if( iFree+sz>iFree2 ) return SQLITE_CORRUPT_PGNO(pPage->pgno); + assert( iFree+sz<=iFree2 ); /* Verified by pageFindSlot() */ sz2 = get2byte(&data[iFree2+2]); assert( iFree+sz+sz2+iFree2-(iFree+sz) <= usableSize ); memmove(&data[iFree+sz+sz2], &data[iFree+sz], iFree2-(iFree+sz)); sz += sz2; } @@ -1478,28 +1493,22 @@ u8 * const aData = pPg->aData; int iAddr = hdr + 1; int pc = get2byte(&aData[iAddr]); int x; int usableSize = pPg->pBt->usableSize; + int size; /* Size of the free slot */ assert( pc>0 ); - do{ - int size; /* Size of the free slot */ - /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of - ** increasing offset. */ - if( pc>usableSize-4 || pcpgno); - return 0; - } + while( pc<=usableSize-4 ){ /* EVIDENCE-OF: R-22710-53328 The third and fourth bytes of each ** freeblock form a big-endian integer which is the size of the freeblock ** in bytes, including the 4-byte header. */ size = get2byte(&aData[pc+2]); if( (x = size - nByte)>=0 ){ testcase( x==4 ); testcase( x==3 ); - if( pc < pPg->cellOffset+2*pPg->nCell || size+pc > usableSize ){ + if( size+pc > usableSize ){ *pRc = SQLITE_CORRUPT_PGNO(pPg->pgno); return 0; }else if( x<4 ){ /* EVIDENCE-OF: R-11498-58022 In a well-formed b-tree page, the total ** number of bytes in fragments may not exceed 60. */ @@ -1516,11 +1525,15 @@ } return &aData[pc + x]; } iAddr = pc; pc = get2byte(&aData[pc]); - }while( pc ); + if( pcpgno); + } return 0; } /* @@ -1630,27 +1643,21 @@ u16 iPtr; /* Address of ptr to next freeblock */ u16 iFreeBlk; /* Address of the next freeblock */ u8 hdr; /* Page header size. 0 or 100 */ u8 nFrag = 0; /* Reduction in fragmentation */ u16 iOrigSize = iSize; /* Original value of iSize */ - u32 iLast = pPage->pBt->usableSize-4; /* Largest possible freeblock offset */ + u16 x; /* Offset to cell content area */ u32 iEnd = iStart + iSize; /* First byte past the iStart buffer */ unsigned char *data = pPage->aData; /* Page content */ assert( pPage->pBt!=0 ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( CORRUPT_DB || iStart>=pPage->hdrOffset+6+pPage->childPtrSize ); assert( CORRUPT_DB || iEnd <= pPage->pBt->usableSize ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( iSize>=4 ); /* Minimum cell size is 4 */ - assert( iStart<=iLast ); - - /* Overwrite deleted information with zeros when the secure_delete - ** option is enabled */ - if( pPage->pBt->btsFlags & BTS_FAST_SECURE ){ - memset(&data[iStart], 0, iSize); - } + assert( iStart<=pPage->pBt->usableSize-4 ); /* The list of freeblocks must be in ascending order. Find the ** spot on the list where iStart should be inserted. */ hdr = pPage->hdrOffset; @@ -1663,11 +1670,13 @@ if( iFreeBlk==0 ) break; return SQLITE_CORRUPT_PGNO(pPage->pgno); } iPtr = iFreeBlk; } - if( iFreeBlk>iLast ) return SQLITE_CORRUPT_PGNO(pPage->pgno); + if( iFreeBlk>pPage->pBt->usableSize-4 ){ + return SQLITE_CORRUPT_PGNO(pPage->pgno); + } assert( iFreeBlk>iPtr || iFreeBlk==0 ); /* At this point: ** iFreeBlk: First freeblock after iStart, or zero if none ** iPtr: The address of a pointer to iFreeBlk @@ -1699,23 +1708,29 @@ } } if( nFrag>data[hdr+7] ) return SQLITE_CORRUPT_PGNO(pPage->pgno); data[hdr+7] -= nFrag; } - if( iStart==get2byte(&data[hdr+5]) ){ + x = get2byte(&data[hdr+5]); + if( iStart<=x ){ /* The new freeblock is at the beginning of the cell content area, ** so just extend the cell content area rather than create another ** freelist entry */ - if( iPtr!=hdr+1 ) return SQLITE_CORRUPT_PGNO(pPage->pgno); + if( iStartpgno); put2byte(&data[hdr+1], iFreeBlk); put2byte(&data[hdr+5], iEnd); }else{ /* Insert the new freeblock into the freelist */ put2byte(&data[iPtr], iStart); - put2byte(&data[iStart], iFreeBlk); - put2byte(&data[iStart+2], iSize); } + if( pPage->pBt->btsFlags & BTS_FAST_SECURE ){ + /* Overwrite deleted information with zeros when the secure_delete + ** option is enabled */ + memset(&data[iStart], 0, iSize); + } + put2byte(&data[iStart], iFreeBlk); + put2byte(&data[iStart+2], iSize); pPage->nFree += iOrigSize; return SQLITE_OK; } /* @@ -2053,11 +2068,11 @@ int bReadOnly /* True for a read-only page */ ){ int rc; DbPage *pDbPage; assert( sqlite3_mutex_held(pBt->mutex) ); - assert( pCur==0 || ppPage==&pCur->apPage[pCur->iPage] ); + assert( pCur==0 || ppPage==&pCur->pPage ); assert( pCur==0 || bReadOnly==pCur->curPagerFlags ); assert( pCur==0 || pCur->iPage>0 ); if( pgno>btreePagecount(pBt) ){ rc = SQLITE_CORRUPT_BKPT; @@ -2087,19 +2102,24 @@ goto getAndInitPage_error; } return SQLITE_OK; getAndInitPage_error: - if( pCur ) pCur->iPage--; + if( pCur ){ + pCur->iPage--; + pCur->pPage = pCur->apPage[pCur->iPage]; + } testcase( pgno==0 ); assert( pgno!=0 || rc==SQLITE_CORRUPT ); return rc; } /* ** Release a MemPage. This should be called once for each prior ** call to btreeGetPage. +** +** Page1 is a special case and must be released using releasePageOne(). */ static void releasePageNotNull(MemPage *pPage){ assert( pPage->aData ); assert( pPage->pBt ); assert( pPage->pDbPage!=0 ); @@ -2108,10 +2128,20 @@ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); sqlite3PagerUnrefNotNull(pPage->pDbPage); } static void releasePage(MemPage *pPage){ if( pPage ) releasePageNotNull(pPage); +} +static void releasePageOne(MemPage *pPage){ + assert( pPage!=0 ); + assert( pPage->aData ); + assert( pPage->pBt ); + assert( pPage->pDbPage!=0 ); + assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); + assert( sqlite3PagerGetData(pPage->pDbPage)==pPage->aData ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + sqlite3PagerUnrefPageOne(pPage->pDbPage); } /* ** Get an unused page. ** @@ -2987,11 +3017,11 @@ if( rc!=SQLITE_OK ){ goto page1_init_failed; }else{ setDefaultSyncFlag(pBt, SQLITE_DEFAULT_WAL_SYNCHRONOUS+1); if( isOpen==0 ){ - releasePage(pPage1); + releasePageOne(pPage1); return SQLITE_OK; } } rc = SQLITE_NOTADB; }else{ @@ -3034,11 +3064,11 @@ ** of BtShared.pageSize, we have discovered that the page-size is ** actually pageSize. Unlock the database, leave pBt->pPage1 at ** zero and return SQLITE_OK. The caller will call this function ** again with the correct page-size. */ - releasePage(pPage1); + releasePageOne(pPage1); pBt->usableSize = usableSize; pBt->pageSize = pageSize; freeTempSpace(pBt); rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, pageSize-usableSize); @@ -3088,11 +3118,11 @@ pBt->pPage1 = pPage1; pBt->nPage = nPage; return SQLITE_OK; page1_init_failed: - releasePage(pPage1); + releasePageOne(pPage1); pBt->pPage1 = 0; return rc; } #ifndef NDEBUG @@ -3133,11 +3163,11 @@ if( pBt->inTransaction==TRANS_NONE && pBt->pPage1!=0 ){ MemPage *pPage1 = pBt->pPage1; assert( pPage1->aData ); assert( sqlite3PagerRefcount(pBt->pPager)==1 ); pBt->pPage1 = 0; - releasePageNotNull(pPage1); + releasePageOne(pPage1); } } /* ** If pBt points to an empty file then convert that empty file @@ -3985,11 +4015,10 @@ assert( (writeOnly==0 || writeOnly==1) && BTCF_WriteFlag==1 ); if( pBtree ){ sqlite3BtreeEnter(pBtree); for(p=pBtree->pBt->pCursor; p; p=p->pNext){ - int i; if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ){ if( p->eState==CURSOR_VALID || p->eState==CURSOR_SKIPNEXT ){ rc = saveCursorPosition(p); if( rc!=SQLITE_OK ){ (void)sqlite3BtreeTripAllCursors(pBtree, rc, 0); @@ -3999,14 +4028,11 @@ }else{ sqlite3BtreeClearCursor(p); p->eState = CURSOR_FAULT; p->skipNext = errCode; } - for(i=0; i<=p->iPage; i++){ - releasePage(p->apPage[i]); - p->apPage[i] = 0; - } + btreeReleaseAllCursorPages(p); } sqlite3BtreeLeave(pBtree); } return rc; } @@ -4059,11 +4085,11 @@ int nPage = get4byte(28+(u8*)pPage1->aData); testcase( nPage==0 ); if( nPage==0 ) sqlite3PagerPagecount(pBt->pPager, &nPage); testcase( pBt->nPage!=nPage ); pBt->nPage = nPage; - releasePage(pPage1); + releasePageOne(pPage1); } assert( countValidCursors(pBt, 1)==0 ); pBt->inTransaction = TRANS_READ; btreeClearHasContent(pBt); } @@ -4301,14 +4327,12 @@ ** when the last cursor is closed. */ int sqlite3BtreeCloseCursor(BtCursor *pCur){ Btree *pBtree = pCur->pBtree; if( pBtree ){ - int i; BtShared *pBt = pCur->pBt; sqlite3BtreeEnter(pBtree); - sqlite3BtreeClearCursor(pCur); assert( pBt->pCursor!=0 ); if( pBt->pCursor==pCur ){ pBt->pCursor = pCur->pNext; }else{ BtCursor *pPrev = pBt->pCursor; @@ -4318,16 +4342,14 @@ break; } pPrev = pPrev->pNext; }while( ALWAYS(pPrev) ); } - for(i=0; i<=pCur->iPage; i++){ - releasePage(pCur->apPage[i]); - } + btreeReleaseAllCursorPages(pCur); unlockBtreeIfUnused(pBt); sqlite3_free(pCur->aOverflow); - /* sqlite3_free(pCur); */ + sqlite3_free(pCur->pKey); sqlite3BtreeLeave(pBtree); } return SQLITE_OK; } @@ -4340,23 +4362,21 @@ ** Using this cache reduces the number of calls to btreeParseCell(). */ #ifndef NDEBUG static void assertCellInfo(BtCursor *pCur){ CellInfo info; - int iPage = pCur->iPage; memset(&info, 0, sizeof(info)); - btreeParseCell(pCur->apPage[iPage], pCur->ix, &info); + btreeParseCell(pCur->pPage, pCur->ix, &info); assert( CORRUPT_DB || memcmp(&info, &pCur->info, sizeof(info))==0 ); } #else #define assertCellInfo(x) #endif static SQLITE_NOINLINE void getCellInfo(BtCursor *pCur){ if( pCur->info.nSize==0 ){ - int iPage = pCur->iPage; pCur->curFlags |= BTCF_ValidNKey; - btreeParseCell(pCur->apPage[iPage],pCur->ix,&pCur->info); + btreeParseCell(pCur->pPage,pCur->ix,&pCur->info); }else{ assertCellInfo(pCur); } } @@ -4550,11 +4570,11 @@ int eOp /* zero to read. non-zero to write. */ ){ unsigned char *aPayload; int rc = SQLITE_OK; int iIdx = 0; - MemPage *pPage = pCur->apPage[pCur->iPage]; /* Btree page of current entry */ + MemPage *pPage = pCur->pPage; /* Btree page of current entry */ BtShared *pBt = pCur->pBt; /* Btree this cursor belongs to */ #ifdef SQLITE_DIRECT_OVERFLOW_READ unsigned char * const pBufStart = pBuf; /* Start of original out buffer */ #endif @@ -4746,12 +4766,12 @@ ** the available payload. */ int sqlite3BtreePayload(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); - assert( pCur->iPage>=0 && pCur->apPage[pCur->iPage] ); - assert( pCur->ixapPage[pCur->iPage]->nCell ); + assert( pCur->iPage>=0 && pCur->pPage ); + assert( pCur->ixpPage->nCell ); return accessPayload(pCur, offset, amt, (unsigned char*)pBuf, 0); } /* ** This variant of sqlite3BtreePayload() works even if the cursor has not @@ -4805,19 +4825,19 @@ static const void *fetchPayload( BtCursor *pCur, /* Cursor pointing to entry to read from */ u32 *pAmt /* Write the number of available bytes here */ ){ u32 amt; - assert( pCur!=0 && pCur->iPage>=0 && pCur->apPage[pCur->iPage]); + assert( pCur!=0 && pCur->iPage>=0 && pCur->pPage); assert( pCur->eState==CURSOR_VALID ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); assert( cursorOwnsBtShared(pCur) ); - assert( pCur->ixapPage[pCur->iPage]->nCell ); + assert( pCur->ixpPage->nCell ); assert( pCur->info.nSize>0 ); - assert( pCur->info.pPayload>pCur->apPage[pCur->iPage]->aData || CORRUPT_DB ); - assert( pCur->info.pPayloadapPage[pCur->iPage]->aDataEnd ||CORRUPT_DB); - amt = (int)(pCur->apPage[pCur->iPage]->aDataEnd - pCur->info.pPayload); + assert( pCur->info.pPayload>pCur->pPage->aData || CORRUPT_DB ); + assert( pCur->info.pPayloadpPage->aDataEnd ||CORRUPT_DB); + amt = (int)(pCur->pPage->aDataEnd - pCur->info.pPayload); if( pCur->info.nLocalinfo.nLocal; *pAmt = amt; return (void*)pCur->info.pPayload; } @@ -4860,14 +4880,15 @@ if( pCur->iPage>=(BTCURSOR_MAX_DEPTH-1) ){ return SQLITE_CORRUPT_BKPT; } pCur->info.nSize = 0; pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); - pCur->aiIdx[pCur->iPage++] = pCur->ix; + pCur->aiIdx[pCur->iPage] = pCur->ix; + pCur->apPage[pCur->iPage] = pCur->pPage; pCur->ix = 0; - return getAndInitPage(pBt, newPgno, &pCur->apPage[pCur->iPage], - pCur, pCur->curPagerFlags); + pCur->iPage++; + return getAndInitPage(pBt, newPgno, &pCur->pPage, pCur, pCur->curPagerFlags); } #ifdef SQLITE_DEBUG /* ** Page pParent is an internal (non-leaf) tree page. This function @@ -4897,24 +4918,27 @@ ** to the page we are coming from. If we are coming from the ** right-most child page then pCur->idx is set to one more than ** the largest cell index. */ static void moveToParent(BtCursor *pCur){ + MemPage *pLeaf; assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPage>0 ); - assert( pCur->apPage[pCur->iPage] ); + assert( pCur->pPage ); assertParentIndex( pCur->apPage[pCur->iPage-1], pCur->aiIdx[pCur->iPage-1], - pCur->apPage[pCur->iPage]->pgno + pCur->pPage->pgno ); testcase( pCur->aiIdx[pCur->iPage-1] > pCur->apPage[pCur->iPage-1]->nCell ); pCur->info.nSize = 0; pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); pCur->ix = pCur->aiIdx[pCur->iPage-1]; - releasePageNotNull(pCur->apPage[pCur->iPage--]); + pLeaf = pCur->pPage; + pCur->pPage = pCur->apPage[--pCur->iPage]; + releasePageNotNull(pLeaf); } /* ** Move the cursor to point to the root page of its b-tree structure. ** @@ -4922,13 +4946,13 @@ ** to the virtual root page instead of the actual root page. A table has a ** virtual root page when the actual root page contains no cells and a ** single child page. This can only happen with the table rooted at page 1. ** ** If the b-tree structure is empty, the cursor state is set to -** CURSOR_INVALID. Otherwise, the cursor is set to point to the first -** cell located on the root (or virtual root) page and the cursor state -** is set to CURSOR_VALID. +** CURSOR_INVALID and this routine returns SQLITE_EMPTY. Otherwise, +** the cursor is set to point to the first cell located on the root +** (or virtual root) page and the cursor state is set to CURSOR_VALID. ** ** If this function returns successfully, it may be assumed that the ** page-header flags indicate that the [virtual] root-page is the expected ** kind of b-tree page (i.e. if when opening the cursor the caller did not ** specify a KeyInfo structure the flags byte is set to 0x05 or 0x0D, @@ -4942,41 +4966,44 @@ assert( cursorOwnsBtShared(pCur) ); assert( CURSOR_INVALID < CURSOR_REQUIRESEEK ); assert( CURSOR_VALID < CURSOR_REQUIRESEEK ); assert( CURSOR_FAULT > CURSOR_REQUIRESEEK ); - if( pCur->eState>=CURSOR_REQUIRESEEK ){ - if( pCur->eState==CURSOR_FAULT ){ - assert( pCur->skipNext!=SQLITE_OK ); - return pCur->skipNext; - } - sqlite3BtreeClearCursor(pCur); - } + assert( pCur->eState < CURSOR_REQUIRESEEK || pCur->iPage<0 ); + assert( pCur->pgnoRoot>0 || pCur->iPage<0 ); if( pCur->iPage>=0 ){ if( pCur->iPage ){ - do{ - assert( pCur->apPage[pCur->iPage]!=0 ); - releasePageNotNull(pCur->apPage[pCur->iPage--]); - }while( pCur->iPage); + releasePageNotNull(pCur->pPage); + while( --pCur->iPage ){ + releasePageNotNull(pCur->apPage[pCur->iPage]); + } + pCur->pPage = pCur->apPage[0]; goto skip_init; } }else if( pCur->pgnoRoot==0 ){ pCur->eState = CURSOR_INVALID; - return SQLITE_OK; + return SQLITE_EMPTY; }else{ assert( pCur->iPage==(-1) ); - rc = getAndInitPage(pCur->pBtree->pBt, pCur->pgnoRoot, &pCur->apPage[0], + if( pCur->eState>=CURSOR_REQUIRESEEK ){ + if( pCur->eState==CURSOR_FAULT ){ + assert( pCur->skipNext!=SQLITE_OK ); + return pCur->skipNext; + } + sqlite3BtreeClearCursor(pCur); + } + rc = getAndInitPage(pCur->pBtree->pBt, pCur->pgnoRoot, &pCur->pPage, 0, pCur->curPagerFlags); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; - return rc; + return rc; } pCur->iPage = 0; - pCur->curIntKey = pCur->apPage[0]->intKey; + pCur->curIntKey = pCur->pPage->intKey; } - pRoot = pCur->apPage[0]; + pRoot = pCur->pPage; assert( pRoot->pgno==pCur->pgnoRoot ); /* If pCur->pKeyInfo is not NULL, then the caller that opened this cursor ** expected to open it on an index b-tree. Otherwise, if pKeyInfo is ** NULL, the caller expects a table b-tree. If this is not the case, @@ -4987,19 +5014,19 @@ ** if pCur->iPage>=0). But this is not so if the database is corrupted ** in such a way that page pRoot is linked into a second b-tree table ** (or the freelist). */ assert( pRoot->intKey==1 || pRoot->intKey==0 ); if( pRoot->isInit==0 || (pCur->pKeyInfo==0)!=pRoot->intKey ){ - return SQLITE_CORRUPT_PGNO(pCur->apPage[pCur->iPage]->pgno); + return SQLITE_CORRUPT_PGNO(pCur->pPage->pgno); } skip_init: pCur->ix = 0; pCur->info.nSize = 0; pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidNKey|BTCF_ValidOvfl); - pRoot = pCur->apPage[0]; + pRoot = pCur->pPage; if( pRoot->nCell>0 ){ pCur->eState = CURSOR_VALID; }else if( !pRoot->leaf ){ Pgno subpage; if( pRoot->pgno!=1 ) return SQLITE_CORRUPT_BKPT; @@ -5006,10 +5033,11 @@ subpage = get4byte(&pRoot->aData[pRoot->hdrOffset+8]); pCur->eState = CURSOR_VALID; rc = moveToChild(pCur, subpage); }else{ pCur->eState = CURSOR_INVALID; + rc = SQLITE_EMPTY; } return rc; } /* @@ -5024,11 +5052,11 @@ int rc = SQLITE_OK; MemPage *pPage; assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState==CURSOR_VALID ); - while( rc==SQLITE_OK && !(pPage = pCur->apPage[pCur->iPage])->leaf ){ + while( rc==SQLITE_OK && !(pPage = pCur->pPage)->leaf ){ assert( pCur->ixnCell ); pgno = get4byte(findCell(pPage, pCur->ix)); rc = moveToChild(pCur, pgno); } return rc; @@ -5049,11 +5077,11 @@ int rc = SQLITE_OK; MemPage *pPage = 0; assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState==CURSOR_VALID ); - while( !(pPage = pCur->apPage[pCur->iPage])->leaf ){ + while( !(pPage = pCur->pPage)->leaf ){ pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); pCur->ix = pPage->nCell; rc = moveToChild(pCur, pgno); if( rc ) return rc; } @@ -5072,18 +5100,17 @@ assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); rc = moveToRoot(pCur); if( rc==SQLITE_OK ){ - if( pCur->eState==CURSOR_INVALID ){ - assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 ); - *pRes = 1; - }else{ - assert( pCur->apPage[pCur->iPage]->nCell>0 ); - *pRes = 0; - rc = moveToLeftmost(pCur); - } + assert( pCur->pPage->nCell>0 ); + *pRes = 0; + rc = moveToLeftmost(pCur); + }else if( rc==SQLITE_EMPTY ){ + assert( pCur->pgnoRoot==0 || pCur->pPage->nCell==0 ); + *pRes = 1; + rc = SQLITE_OK; } return rc; } /* Move the cursor to the last entry in the table. Return SQLITE_OK @@ -5103,32 +5130,30 @@ ** to the last entry in the b-tree. */ int ii; for(ii=0; iiiPage; ii++){ assert( pCur->aiIdx[ii]==pCur->apPage[ii]->nCell ); } - assert( pCur->ix==pCur->apPage[pCur->iPage]->nCell-1 ); - assert( pCur->apPage[pCur->iPage]->leaf ); + assert( pCur->ix==pCur->pPage->nCell-1 ); + assert( pCur->pPage->leaf ); #endif return SQLITE_OK; } rc = moveToRoot(pCur); if( rc==SQLITE_OK ){ - if( CURSOR_INVALID==pCur->eState ){ - assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 ); - *pRes = 1; - }else{ - assert( pCur->eState==CURSOR_VALID ); - *pRes = 0; - rc = moveToRightmost(pCur); - if( rc==SQLITE_OK ){ - pCur->curFlags |= BTCF_AtLast; - }else{ - pCur->curFlags &= ~BTCF_AtLast; - } - - } + assert( pCur->eState==CURSOR_VALID ); + *pRes = 0; + rc = moveToRightmost(pCur); + if( rc==SQLITE_OK ){ + pCur->curFlags |= BTCF_AtLast; + }else{ + pCur->curFlags &= ~BTCF_AtLast; + } + }else if( rc==SQLITE_EMPTY ){ + assert( pCur->pgnoRoot==0 || pCur->pPage->nCell==0 ); + *pRes = 1; + rc = SQLITE_OK; } return rc; } /* Move the cursor so that it points to an entry near the key @@ -5223,26 +5248,27 @@ xRecordCompare = 0; /* All keys are integers */ } rc = moveToRoot(pCur); if( rc ){ + if( rc==SQLITE_EMPTY ){ + assert( pCur->pgnoRoot==0 || pCur->pPage->nCell==0 ); + *pRes = -1; + return SQLITE_OK; + } return rc; } - assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage] ); - assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->isInit ); - assert( pCur->eState==CURSOR_INVALID || pCur->apPage[pCur->iPage]->nCell>0 ); - if( pCur->eState==CURSOR_INVALID ){ - *pRes = -1; - assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 ); - return SQLITE_OK; - } - assert( pCur->apPage[0]->intKey==pCur->curIntKey ); + assert( pCur->pPage ); + assert( pCur->pPage->isInit ); + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->pPage->nCell > 0 ); + assert( pCur->iPage==0 || pCur->apPage[0]->intKey==pCur->curIntKey ); assert( pCur->curIntKey || pIdxKey ); for(;;){ int lwr, upr, idx, c; Pgno chldPg; - MemPage *pPage = pCur->apPage[pCur->iPage]; + MemPage *pPage = pCur->pPage; u8 *pCell; /* Pointer to current cell in pPage */ /* pPage->nCell must be greater than zero. If this is the root-page ** the cursor would have been INVALID above and this for(;;) loop ** not run. If this is not the root-page, then the moveToChild() routine @@ -5366,11 +5392,11 @@ }else{ assert( c==0 ); *pRes = 0; rc = SQLITE_OK; pCur->ix = (u16)idx; - if( pIdxKey->errCode ) rc = SQLITE_CORRUPT; + if( pIdxKey->errCode ) rc = SQLITE_CORRUPT_BKPT; goto moveto_finish; } if( lwr>upr ) break; assert( lwr+upr>=0 ); idx = (lwr+upr)>>1; /* idx = (lwr+upr)/2 */ @@ -5377,11 +5403,11 @@ } } assert( lwr==upr+1 || (pPage->intKey && !pPage->leaf) ); assert( pPage->isInit ); if( pPage->leaf ){ - assert( pCur->ixapPage[pCur->iPage]->nCell ); + assert( pCur->ixpPage->nCell ); pCur->ix = (u16)idx; *pRes = c; rc = SQLITE_OK; goto moveto_finish; } @@ -5431,13 +5457,14 @@ /* Currently this interface is only called by the OP_IfSmaller ** opcode, and it that case the cursor will always be valid and ** will always point to a leaf node. */ if( NEVER(pCur->eState!=CURSOR_VALID) ) return -1; - if( NEVER(pCur->apPage[pCur->iPage]->leaf==0) ) return -1; + if( NEVER(pCur->pPage->leaf==0) ) return -1; - for(n=1, i=0; i<=pCur->iPage; i++){ + n = pCur->pPage->nCell; + for(i=0; iiPage; i++){ n *= pCur->apPage[i]->nCell; } return n; } @@ -5486,11 +5513,11 @@ } pCur->skipNext = 0; } } - pPage = pCur->apPage[pCur->iPage]; + pPage = pCur->pPage; idx = ++pCur->ix; assert( pPage->isInit ); /* If the database file is corrupt, it is possible for the value of idx ** to be invalid here. This can only occur if a second cursor modifies @@ -5509,11 +5536,11 @@ if( pCur->iPage==0 ){ pCur->eState = CURSOR_INVALID; return SQLITE_DONE; } moveToParent(pCur); - pPage = pCur->apPage[pCur->iPage]; + pPage = pCur->pPage; }while( pCur->ix>=pPage->nCell ); if( pPage->intKey ){ return sqlite3BtreeNext(pCur, 0); }else{ return SQLITE_OK; @@ -5532,11 +5559,11 @@ assert( flags==0 || flags==1 ); assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); pCur->info.nSize = 0; pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); if( pCur->eState!=CURSOR_VALID ) return btreeNext(pCur); - pPage = pCur->apPage[pCur->iPage]; + pPage = pCur->pPage; if( (++pCur->ix)>=pPage->nCell ){ pCur->ix--; return btreeNext(pCur); } if( pPage->leaf ){ @@ -5591,11 +5618,11 @@ } pCur->skipNext = 0; } } - pPage = pCur->apPage[pCur->iPage]; + pPage = pCur->pPage; assert( pPage->isInit ); if( !pPage->leaf ){ int idx = pCur->ix; rc = moveToChild(pCur, get4byte(findCell(pPage, idx))); if( rc ) return rc; @@ -5610,11 +5637,11 @@ } assert( pCur->info.nSize==0 ); assert( (pCur->curFlags & (BTCF_ValidOvfl))==0 ); pCur->ix--; - pPage = pCur->apPage[pCur->iPage]; + pPage = pCur->pPage; if( pPage->intKey && !pPage->leaf ){ rc = sqlite3BtreePrevious(pCur, 0); }else{ rc = SQLITE_OK; } @@ -5628,11 +5655,11 @@ UNUSED_PARAMETER( flags ); /* Used in COMDB2 but not native SQLite */ pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidOvfl|BTCF_ValidNKey); pCur->info.nSize = 0; if( pCur->eState!=CURSOR_VALID || pCur->ix==0 - || pCur->apPage[pCur->iPage]->leaf==0 + || pCur->pPage->leaf==0 ){ return btreePrevious(pCur); } pCur->ix--; return SQLITE_OK; @@ -6124,11 +6151,11 @@ static int clearCell( MemPage *pPage, /* The page that contains the Cell */ unsigned char *pCell, /* First byte of the Cell */ CellInfo *pInfo /* Size information about the cell */ ){ - BtShared *pBt = pPage->pBt; + BtShared *pBt; Pgno ovflPgno; int rc; int nOvfl; u32 ovflPageSize; @@ -6140,10 +6167,11 @@ if( pCell+pInfo->nSize-1 > pPage->aData+pPage->maskPage ){ /* Cell extends past end of page */ return SQLITE_CORRUPT_PGNO(pPage->pgno); } ovflPgno = get4byte(pCell + pInfo->nSize - 4); + pBt = pPage->pBt; assert( pBt->usableSize > 4 ); ovflPageSize = pBt->usableSize - 4; nOvfl = (pInfo->nPayload - pInfo->nLocal + ovflPageSize - 1)/ovflPageSize; assert( nOvfl>0 || (CORRUPT_DB && (pInfo->nPayload + ovflPageSize)pBt; - Pgno pgnoOvfl = 0; + BtShared *pBt; + Pgno pgnoOvfl; int nHeader; assert( sqlite3_mutex_held(pPage->pBt->mutex) ); /* pPage is not necessarily writeable since pCell might be auxiliary ** buffer space that is separate from the pPage buffer area */ - assert( pCellaData || pCell>=&pPage->aData[pBt->pageSize] + assert( pCellaData || pCell>=&pPage->aData[pPage->pBt->pageSize] || sqlite3PagerIswriteable(pPage->pDbPage) ); /* Fill in the header. */ nHeader = pPage->childPtrSize; if( pPage->intKey ){ @@ -6241,29 +6268,40 @@ pSrc = pX->pKey; nHeader += putVarint32(&pCell[nHeader], nPayload); } /* Fill in the payload */ + pPayload = &pCell[nHeader]; if( nPayload<=pPage->maxLocal ){ + /* This is the common case where everything fits on the btree page + ** and no overflow pages are required. */ n = nHeader + nPayload; testcase( n==3 ); testcase( n==4 ); if( n<4 ) n = 4; *pnSize = n; - spaceLeft = nPayload; - pPrior = pCell; - }else{ - int mn = pPage->minLocal; - n = mn + (nPayload - mn) % (pPage->pBt->usableSize - 4); - testcase( n==pPage->maxLocal ); - testcase( n==pPage->maxLocal+1 ); - if( n > pPage->maxLocal ) n = mn; - spaceLeft = n; - *pnSize = n + nHeader + 4; - pPrior = &pCell[nHeader+n]; - } - pPayload = &pCell[nHeader]; + assert( nSrc<=nPayload ); + testcase( nSrcminLocal; + n = mn + (nPayload - mn) % (pPage->pBt->usableSize - 4); + testcase( n==pPage->maxLocal ); + testcase( n==pPage->maxLocal+1 ); + if( n > pPage->maxLocal ) n = mn; + spaceLeft = n; + *pnSize = n + nHeader + 4; + pPrior = &pCell[nHeader+n]; + pToRelease = 0; + pgnoOvfl = 0; + pBt = pPage->pBt; /* At this point variables should be set as follows: ** ** nPayload Total payload size in bytes ** pPayload Begin writing payload here @@ -6285,12 +6323,39 @@ assert( spaceLeft == info.nLocal ); } #endif /* Write the payload into the local Cell and any extra into overflow pages */ - while( nPayload>0 ){ + while( 1 ){ + n = nPayload; + if( n>spaceLeft ) n = spaceLeft; + + /* If pToRelease is not zero than pPayload points into the data area + ** of pToRelease. Make sure pToRelease is still writeable. */ + assert( pToRelease==0 || sqlite3PagerIswriteable(pToRelease->pDbPage) ); + + /* If pPayload is part of the data area of pPage, then make sure pPage + ** is still writeable */ + assert( pPayloadaData || pPayload>=&pPage->aData[pBt->pageSize] + || sqlite3PagerIswriteable(pPage->pDbPage) ); + + if( nSrc>=n ){ + memcpy(pPayload, pSrc, n); + }else if( nSrc>0 ){ + n = nSrc; + memcpy(pPayload, pSrc, n); + }else{ + memset(pPayload, 0, n); + } + nPayload -= n; + if( nPayload<=0 ) break; + pPayload += n; + pSrc += n; + nSrc -= n; + spaceLeft -= n; if( spaceLeft==0 ){ + MemPage *pOvfl = 0; #ifndef SQLITE_OMIT_AUTOVACUUM Pgno pgnoPtrmap = pgnoOvfl; /* Overflow page pointer-map entry page */ if( pBt->autoVacuum ){ do{ pgnoOvfl++; @@ -6339,34 +6404,10 @@ pPrior = pOvfl->aData; put4byte(pPrior, 0); pPayload = &pOvfl->aData[4]; spaceLeft = pBt->usableSize - 4; } - n = nPayload; - if( n>spaceLeft ) n = spaceLeft; - - /* If pToRelease is not zero than pPayload points into the data area - ** of pToRelease. Make sure pToRelease is still writeable. */ - assert( pToRelease==0 || sqlite3PagerIswriteable(pToRelease->pDbPage) ); - - /* If pPayload is part of the data area of pPage, then make sure pPage - ** is still writeable */ - assert( pPayloadaData || pPayload>=&pPage->aData[pBt->pageSize] - || sqlite3PagerIswriteable(pPage->pDbPage) ); - - if( nSrc>0 ){ - if( n>nSrc ) n = nSrc; - assert( pSrc ); - memcpy(pPayload, pSrc, n); - }else{ - memset(pPayload, 0, n); - } - nPayload -= n; - pPayload += n; - pSrc += n; - nSrc -= n; - spaceLeft -= n; } releasePage(pToRelease); return SQLITE_OK; } @@ -6394,11 +6435,11 @@ ptr = &pPage->aCellIdx[2*idx]; pc = get2byte(ptr); hdr = pPage->hdrOffset; testcase( pc==get2byte(&data[hdr+5]) ); testcase( pc+sz==pPage->pBt->usableSize ); - if( pc < (u32)get2byte(&data[hdr+5]) || pc+sz > pPage->pBt->usableSize ){ + if( pc+sz > pPage->pBt->usableSize ){ *pRC = SQLITE_CORRUPT_BKPT; return; } rc = freeSpace(pPage, pc, sz); if( rc ){ @@ -7264,11 +7305,11 @@ + pBt->pageSize; /* aSpace1 */ /* EVIDENCE-OF: R-28375-38319 SQLite will never request a scratch buffer ** that is more than 6 times the database page size. */ assert( szScratch<=6*(int)pBt->pageSize ); - b.apCell = sqlite3ScratchMalloc( szScratch ); + b.apCell = sqlite3StackAllocRaw(0, szScratch ); if( b.apCell==0 ){ rc = SQLITE_NOMEM_BKPT; goto balance_cleanup; } b.szCell = (u16*)&b.apCell[nMaxCells]; @@ -7842,11 +7883,11 @@ /* ** Cleanup before returning. */ balance_cleanup: - sqlite3ScratchFree(b.apCell); + sqlite3StackFree(0, b.apCell); for(i=0; iiPage; - MemPage *pPage = pCur->apPage[iPage]; + MemPage *pPage = pCur->pPage; if( iPage==0 ){ if( pPage->nOverflow ){ /* The root page of the b-tree is overfull. In this case call the ** balance_deeper() function to create a new child for the root-page @@ -7957,11 +7998,13 @@ rc = balance_deeper(pPage, &pCur->apPage[1]); if( rc==SQLITE_OK ){ pCur->iPage = 1; pCur->ix = 0; pCur->aiIdx[0] = 0; - assert( pCur->apPage[1]->nOverflow ); + pCur->apPage[0] = pPage; + pCur->pPage = pCur->apPage[1]; + assert( pCur->pPage->nOverflow ); } }else{ break; } }else if( pPage->nOverflow==0 && pPage->nFree<=nMin ){ @@ -8037,10 +8080,11 @@ /* The next iteration of the do-loop balances the parent page. */ releasePage(pPage); pCur->iPage--; assert( pCur->iPage>=0 ); + pCur->pPage = pCur->apPage[pCur->iPage]; } }while( rc==SQLITE_OK ); if( pFree ){ sqlite3PageFree(pFree); @@ -8168,11 +8212,11 @@ } if( rc ) return rc; } assert( pCur->eState==CURSOR_VALID || (pCur->eState==CURSOR_INVALID && loc) ); - pPage = pCur->apPage[pCur->iPage]; + pPage = pCur->pPage; assert( pPage->intKey || pX->nKey>=0 ); assert( pPage->leaf || !pPage->intKey ); TRACE(("INSERT: table=%d nkey=%lld ndata=%d page=%d %s\n", pCur->pgnoRoot, pX->nKey, pX->nData, pPage->pgno, @@ -8255,14 +8299,14 @@ /* Must make sure nOverflow is reset to zero even if the balance() ** fails. Internal data structure corruption will result otherwise. ** Also, set the cursor state to invalid. This stops saveCursorPosition() ** from trying to save the current position of the cursor. */ - pCur->apPage[pCur->iPage]->nOverflow = 0; + pCur->pPage->nOverflow = 0; pCur->eState = CURSOR_INVALID; if( (flags & BTREE_SAVEPOSITION) && rc==SQLITE_OK ){ - rc = moveToRoot(pCur); + btreeReleaseAllCursorPages(pCur); if( pCur->pKeyInfo ){ assert( pCur->pKey==0 ); pCur->pKey = sqlite3Malloc( pX->nKey ); if( pCur->pKey==0 ){ rc = SQLITE_NOMEM; @@ -8272,11 +8316,11 @@ } pCur->eState = CURSOR_REQUIRESEEK; pCur->nKey = pX->nKey; } } - assert( pCur->apPage[pCur->iPage]->nOverflow==0 ); + assert( pCur->iPage<0 || pCur->pPage->nOverflow==0 ); end_insert: return rc; } @@ -8313,17 +8357,17 @@ assert( pBt->inTransaction==TRANS_WRITE ); assert( (pBt->btsFlags & BTS_READ_ONLY)==0 ); assert( pCur->curFlags & BTCF_WriteFlag ); assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); assert( !hasReadConflicts(p, pCur->pgnoRoot) ); - assert( pCur->ixapPage[pCur->iPage]->nCell ); + assert( pCur->ixpPage->nCell ); assert( pCur->eState==CURSOR_VALID ); assert( (flags & ~(BTREE_SAVEPOSITION | BTREE_AUXDELETE))==0 ); iCellDepth = pCur->iPage; iCellIdx = pCur->ix; - pPage = pCur->apPage[iCellDepth]; + pPage = pCur->pPage; pCell = findCell(pPage, iCellIdx); /* If the bPreserve flag is set to true, then the cursor position must ** be preserved following this delete operation. If the current delete ** will cause a b-tree rebalance, then this is done by saving the cursor @@ -8385,15 +8429,20 @@ ** is currently pointing to the largest entry in the sub-tree headed ** by the child-page of the cell that was just deleted from an internal ** node. The cell from the leaf node needs to be moved to the internal ** node to replace the deleted cell. */ if( !pPage->leaf ){ - MemPage *pLeaf = pCur->apPage[pCur->iPage]; + MemPage *pLeaf = pCur->pPage; int nCell; - Pgno n = pCur->apPage[iCellDepth+1]->pgno; + Pgno n; unsigned char *pTmp; + if( iCellDepthiPage-1 ){ + n = pCur->apPage[iCellDepth+1]->pgno; + }else{ + n = pCur->pPage->pgno; + } pCell = findCell(pLeaf, pLeaf->nCell-1); if( pCell<&pLeaf->aData[4] ) return SQLITE_CORRUPT_BKPT; nCell = pLeaf->xCellSize(pLeaf, pCell); assert( MX_CELL_SIZE(pBt) >= nCell ); pTmp = pBt->pTmpSpace; @@ -8421,20 +8470,23 @@ ** been corrected, so be it. Otherwise, after balancing the leaf node, ** walk the cursor up the tree to the internal node and balance it as ** well. */ rc = balance(pCur); if( rc==SQLITE_OK && pCur->iPage>iCellDepth ){ + releasePageNotNull(pCur->pPage); + pCur->iPage--; while( pCur->iPage>iCellDepth ){ releasePage(pCur->apPage[pCur->iPage--]); } + pCur->pPage = pCur->apPage[pCur->iPage]; rc = balance(pCur); } if( rc==SQLITE_OK ){ if( bSkipnext ){ assert( bPreserve && (pCur->iPage==iCellDepth || CORRUPT_DB) ); - assert( pPage==pCur->apPage[pCur->iPage] || CORRUPT_DB ); + assert( pPage==pCur->pPage || CORRUPT_DB ); assert( (pPage->nCell>0 || CORRUPT_DB) && iCellIdx<=pPage->nCell ); pCur->eState = CURSOR_SKIPNEXT; if( iCellIdx>=pPage->nCell ){ pCur->skipNext = -1; pCur->ix = pPage->nCell-1; @@ -8442,12 +8494,14 @@ pCur->skipNext = 1; } }else{ rc = moveToRoot(pCur); if( bPreserve ){ + btreeReleaseAllCursorPages(pCur); pCur->eState = CURSOR_REQUIRESEEK; } + if( rc==SQLITE_EMPTY ) rc = SQLITE_OK; } } return rc; } @@ -8908,15 +8962,15 @@ */ int sqlite3BtreeCount(BtCursor *pCur, i64 *pnEntry){ i64 nEntry = 0; /* Value to return in *pnEntry */ int rc; /* Return code */ - if( pCur->pgnoRoot==0 ){ + rc = moveToRoot(pCur); + if( rc==SQLITE_EMPTY ){ *pnEntry = 0; return SQLITE_OK; } - rc = moveToRoot(pCur); /* Unless an error occurs, the following loop runs one iteration for each ** page in the B-Tree structure (not including overflow pages). */ while( rc==SQLITE_OK ){ @@ -8925,11 +8979,11 @@ /* If this is a leaf page or the tree is not an int-key tree, then ** this page contains countable entries. Increment the entry counter ** accordingly. */ - pPage = pCur->apPage[pCur->iPage]; + pPage = pCur->pPage; if( pPage->leaf || !pPage->intKey ){ nEntry += pPage->nCell; } /* pPage is a leaf node. This loop navigates the cursor so that it @@ -8948,14 +9002,14 @@ /* All pages of the b-tree have been visited. Return successfully. */ *pnEntry = nEntry; return moveToRoot(pCur); } moveToParent(pCur); - }while ( pCur->ix>=pCur->apPage[pCur->iPage]->nCell ); + }while ( pCur->ix>=pCur->pPage->nCell ); pCur->ix++; - pPage = pCur->apPage[pCur->iPage]; + pPage = pCur->pPage; } /* Descend to the child node of the cell that the cursor currently ** points at. This is the right-child if (iIdx==pPage->nCell). */ @@ -9792,11 +9846,11 @@ } assert( (pCsr->pBt->btsFlags & BTS_READ_ONLY)==0 && pCsr->pBt->inTransaction==TRANS_WRITE ); assert( hasSharedCacheTableLock(pCsr->pBtree, pCsr->pgnoRoot, 0, 2) ); assert( !hasReadConflicts(pCsr->pBtree, pCsr->pgnoRoot) ); - assert( pCsr->apPage[pCsr->iPage]->intKey ); + assert( pCsr->pPage->intKey ); return accessPayload(pCsr, offset, amt, (unsigned char *)z, 1); } /* Index: src/btree.h ================================================================== --- src/btree.h +++ src/btree.h @@ -228,10 +228,11 @@ int iTable, /* Index of root page */ int wrFlag, /* 1 for writing. 0 for read-only */ struct KeyInfo*, /* First argument to compare function */ BtCursor *pCursor /* Space to write cursor structure */ ); +BtCursor *sqlite3BtreeFakeValidCursor(void); int sqlite3BtreeCursorSize(void); void sqlite3BtreeCursorZero(BtCursor*); void sqlite3BtreeCursorHintFlags(BtCursor*, unsigned); #ifdef SQLITE_ENABLE_CURSOR_HINTS void sqlite3BtreeCursorHint(BtCursor*, int, ...); Index: src/btreeInt.h ================================================================== --- src/btreeInt.h +++ src/btreeInt.h @@ -497,34 +497,35 @@ ** eState==SKIPNEXT && skipNext>0: Next sqlite3BtreeNext() is no-op. ** eState==SKIPNEXT && skipNext<0: Next sqlite3BtreePrevious() is no-op. ** eState==FAULT: Cursor fault with skipNext as error code. */ struct BtCursor { + u8 eState; /* One of the CURSOR_XXX constants (see below) */ + u8 curFlags; /* zero or more BTCF_* flags defined below */ + u8 curPagerFlags; /* Flags to send to sqlite3PagerGet() */ + u8 hints; /* As configured by CursorSetHints() */ + int nOvflAlloc; /* Allocated size of aOverflow[] array */ Btree *pBtree; /* The Btree to which this cursor belongs */ BtShared *pBt; /* The BtShared this cursor points to */ BtCursor *pNext; /* Forms a linked list of all cursors */ Pgno *aOverflow; /* Cache of overflow page locations */ CellInfo info; /* A parse of the cell we are pointing at */ i64 nKey; /* Size of pKey, or last integer key */ void *pKey; /* Saved key that was cursor last known position */ Pgno pgnoRoot; /* The root page of this tree */ - int nOvflAlloc; /* Allocated size of aOverflow[] array */ int skipNext; /* Prev() is noop if negative. Next() is noop if positive. ** Error code if eState==CURSOR_FAULT */ - u8 curFlags; /* zero or more BTCF_* flags defined below */ - u8 curPagerFlags; /* Flags to send to sqlite3PagerGet() */ - u8 eState; /* One of the CURSOR_XXX constants (see below) */ - u8 hints; /* As configured by CursorSetHints() */ /* All fields above are zeroed when the cursor is allocated. See ** sqlite3BtreeCursorZero(). Fields that follow must be manually ** initialized. */ i8 iPage; /* Index of current page in apPage */ u8 curIntKey; /* Value of apPage[0]->intKey */ u16 ix; /* Current index for apPage[iPage] */ u16 aiIdx[BTCURSOR_MAX_DEPTH-1]; /* Current index in apPage[i] */ struct KeyInfo *pKeyInfo; /* Arg passed to comparison function */ - MemPage *apPage[BTCURSOR_MAX_DEPTH]; /* Pages from root to current page */ + MemPage *pPage; /* Current page */ + MemPage *apPage[BTCURSOR_MAX_DEPTH-1]; /* Stack of parents of current page */ }; /* ** Legal values for BtCursor.curFlags */ Index: src/build.c ================================================================== --- src/build.c +++ src/build.c @@ -477,11 +477,11 @@ p->pNext = pIndex->pNext; } } freeIndex(db, pIndex); } - db->flags |= SQLITE_InternChanges; + db->mDbFlags |= DBFLAG_SchemaChange; } /* ** Look through the list of open database files in db->aDb[] and if ** any have been closed, remove them from the list. Reallocate the @@ -512,58 +512,57 @@ } } /* ** Reset the schema for the database at index iDb. Also reset the -** TEMP schema. +** TEMP schema. The reset is deferred if db->nSchemaLock is not zero. +** Deferred resets may be run by calling with iDb<0. */ void sqlite3ResetOneSchema(sqlite3 *db, int iDb){ - Db *pDb; + int i; assert( iDbnDb ); - /* Case 1: Reset the single schema identified by iDb */ - pDb = &db->aDb[iDb]; - assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - assert( pDb->pSchema!=0 ); - sqlite3SchemaClear(pDb->pSchema); - - /* If any database other than TEMP is reset, then also reset TEMP - ** since TEMP might be holding triggers that reference tables in the - ** other database. - */ - if( iDb!=1 ){ - pDb = &db->aDb[1]; - assert( pDb->pSchema!=0 ); - sqlite3SchemaClear(pDb->pSchema); - } - return; + if( iDb>=0 ){ + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + DbSetProperty(db, iDb, DB_ResetWanted); + DbSetProperty(db, 1, DB_ResetWanted); + } + + if( db->nSchemaLock==0 ){ + for(i=0; inDb; i++){ + if( DbHasProperty(db, i, DB_ResetWanted) ){ + sqlite3SchemaClear(db->aDb[i].pSchema); + } + } + } } /* ** Erase all schema information from all attached databases (including ** "main" and "temp") for a single database connection. */ void sqlite3ResetAllSchemasOfConnection(sqlite3 *db){ int i; sqlite3BtreeEnterAll(db); + assert( db->nSchemaLock==0 ); for(i=0; inDb; i++){ Db *pDb = &db->aDb[i]; if( pDb->pSchema ){ sqlite3SchemaClear(pDb->pSchema); } } - db->flags &= ~SQLITE_InternChanges; + db->mDbFlags &= ~DBFLAG_SchemaChange; sqlite3VtabUnlockList(db); sqlite3BtreeLeaveAll(db); sqlite3CollapseDatabaseArray(db); } /* ** This routine is called when a commit occurs. */ void sqlite3CommitInternalChanges(sqlite3 *db){ - db->flags &= ~SQLITE_InternChanges; + db->mDbFlags &= ~DBFLAG_SchemaChange; } /* ** Delete memory allocated for the column names of a table or view (the ** Table.aCol[] array). @@ -597,17 +596,20 @@ ** db parameter can be used with db->pnBytesFreed to measure the memory ** used by the Table object. */ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ Index *pIndex, *pNext; - TESTONLY( int nLookaside; ) /* Used to verify lookaside not used for schema */ +#ifdef SQLITE_DEBUG /* Record the number of outstanding lookaside allocations in schema Tables ** prior to doing any free() operations. Since schema Tables do not use ** lookaside, this number should not change. */ - TESTONLY( nLookaside = (db && (pTable->tabFlags & TF_Ephemeral)==0) ? - db->lookaside.nOut : 0 ); + int nLookaside = 0; + if( db && (pTable->tabFlags & TF_Ephemeral)==0 ){ + nLookaside = sqlite3LookasideUsed(db, 0); + } +#endif /* Delete all indices associated with this table. */ for(pIndex = pTable->pIndex; pIndex; pIndex=pNext){ pNext = pIndex->pNext; assert( pIndex->pSchema==pTable->pSchema @@ -637,11 +639,11 @@ sqlite3VtabClear(db, pTable); #endif sqlite3DbFree(db, pTable); /* Verify that no lookaside memory was used by schema tables */ - assert( nLookaside==0 || nLookaside==db->lookaside.nOut ); + assert( nLookaside==0 || nLookaside==sqlite3LookasideUsed(db,0) ); } void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ /* Do not delete the table until the reference count reaches zero. */ if( !pTable ) return; if( ((!db || db->pnBytesFreed==0) && (--pTable->nTabRef)>0) ) return; @@ -663,11 +665,11 @@ assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); testcase( zTabName[0]==0 ); /* Zero-length table names are allowed */ pDb = &db->aDb[iDb]; p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, 0); sqlite3DeleteTable(db, p); - db->flags |= SQLITE_InternChanges; + db->mDbFlags |= DBFLAG_SchemaChange; } /* ** Given a token, return a string that consists of the text of that ** token. Space to hold the returned string @@ -776,11 +778,12 @@ if( iDb<0 ){ sqlite3ErrorMsg(pParse, "unknown database %T", pName1); return -1; } }else{ - assert( db->init.iDb==0 || db->init.busy || (db->flags & SQLITE_Vacuum)!=0); + assert( db->init.iDb==0 || db->init.busy + || (db->mDbFlags & DBFLAG_Vacuum)!=0); iDb = db->init.iDb; *pUnqual = pName1; } return iDb; } @@ -1008,11 +1011,12 @@ if( isView || isVirtual ){ sqlite3VdbeAddOp2(v, OP_Integer, 0, reg2); }else #endif { - pParse->addrCrTab = sqlite3VdbeAddOp2(v, OP_CreateTable, iDb, reg2); + pParse->addrCrTab = + sqlite3VdbeAddOp3(v, OP_CreateBtree, iDb, reg2, BTREE_INTKEY); } sqlite3OpenMasterTable(pParse, iDb); sqlite3VdbeAddOp2(v, OP_NewRowid, 0, reg1); sqlite3VdbeAddOp4(v, OP_Blob, 6, reg3, 0, nullRow, P4_STATIC); sqlite3VdbeAddOp3(v, OP_Insert, 0, reg3, reg1); @@ -1668,21 +1672,20 @@ ** internal schema data structures and the generated VDBE code so that they ** are appropriate for a WITHOUT ROWID table instead of a rowid table. ** Changes include: ** ** (1) Set all columns of the PRIMARY KEY schema object to be NOT NULL. -** (2) Convert the OP_CreateTable into an OP_CreateIndex. There is -** no rowid btree for a WITHOUT ROWID. Instead, the canonical -** data storage is a covering index btree. +** (2) Convert P3 parameter of the OP_CreateBtree from BTREE_INTKEY +** into BTREE_BLOBKEY. ** (3) Bypass the creation of the sqlite_master table entry ** for the PRIMARY KEY as the primary key index is now ** identified by the sqlite_master table entry of the table itself. ** (4) Set the Index.tnum of the PRIMARY KEY Index object in the ** schema to the rootpage from the main table. ** (5) Add all table columns to the PRIMARY KEY Index object ** so that the PRIMARY KEY is a covering index. The surplus -** columns are part of KeyInfo.nXField and are not used for +** columns are part of KeyInfo.nAllField and are not used for ** sorting or lookup or uniqueness checks. ** (6) Replace the rowid tail on all automatically generated UNIQUE ** indices with the PRIMARY KEY columns. ** ** For virtual tables, only (1) is performed. @@ -1707,17 +1710,16 @@ /* The remaining transformations only apply to b-tree tables, not to ** virtual tables */ if( IN_DECLARE_VTAB ) return; - /* Convert the OP_CreateTable opcode that would normally create the - ** root-page for the table into an OP_CreateIndex opcode. The index - ** created will become the PRIMARY KEY index. + /* Convert the P3 operand of the OP_CreateBtree opcode from BTREE_INTKEY + ** into BTREE_BLOBKEY. */ if( pParse->addrCrTab ){ assert( v ); - sqlite3VdbeChangeOpcode(v, pParse->addrCrTab, OP_CreateIndex); + sqlite3VdbeChangeP3(v, pParse->addrCrTab, BTREE_BLOBKEY); } /* Locate the PRIMARY KEY index. Or, if this table was originally ** an INTEGER PRIMARY KEY table, create a new PRIMARY KEY index. */ @@ -1736,19 +1738,10 @@ pPk = sqlite3PrimaryKeyIndex(pTab); pTab->iPKey = -1; }else{ pPk = sqlite3PrimaryKeyIndex(pTab); - /* Bypass the creation of the PRIMARY KEY btree and the sqlite_master - ** table entry. This is only required if currently generating VDBE - ** code for a CREATE TABLE (not when parsing one as part of reading - ** a database schema). */ - if( v ){ - assert( db->init.busy==0 ); - sqlite3VdbeChangeOpcode(v, pPk->tnum, OP_Goto); - } - /* ** Remove all redundant columns from the PRIMARY KEY. For example, change ** "PRIMARY KEY(a,b,a,b,c,b,c,d)" into just "PRIMARY KEY(a,b,c,d)". Later ** code assumes the PRIMARY KEY contains no repeated columns. */ @@ -1763,10 +1756,19 @@ } assert( pPk!=0 ); pPk->isCovering = 1; if( !db->init.imposterTable ) pPk->uniqNotNull = 1; nPk = pPk->nKeyCol; + + /* Bypass the creation of the PRIMARY KEY btree and the sqlite_master + ** table entry. This is only required if currently generating VDBE + ** code for a CREATE TABLE (not when parsing one as part of reading + ** a database schema). */ + if( v && pPk->tnum>0 ){ + assert( db->init.busy==0 ); + sqlite3VdbeChangeOpcode(v, pPk->tnum, OP_Goto); + } /* The root page of the PRIMARY KEY is the table root page */ pPk->tnum = pTab->tnum; /* Update the in-memory representation of all UNIQUE indices by converting @@ -2053,11 +2055,11 @@ assert( p==pOld ); /* Malloc must have failed inside HashInsert() */ sqlite3OomFault(db); return; } pParse->pNewTable = 0; - db->flags |= SQLITE_InternChanges; + db->mDbFlags |= DBFLAG_SchemaChange; #ifndef SQLITE_OMIT_ALTERTABLE if( !p->pSelect ){ const char *zName = (const char *)pParse->sNameToken.z; int nName; @@ -2152,19 +2154,25 @@ Table *pSelTab; /* A fake table from which we get the result set */ Select *pSel; /* Copy of the SELECT that implements the view */ int nErr = 0; /* Number of errors encountered */ int n; /* Temporarily holds the number of cursors assigned */ sqlite3 *db = pParse->db; /* Database connection for malloc errors */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + int rc; +#endif #ifndef SQLITE_OMIT_AUTHORIZATION sqlite3_xauth xAuth; /* Saved xAuth pointer */ #endif assert( pTable ); #ifndef SQLITE_OMIT_VIRTUALTABLE - if( sqlite3VtabCallConnect(pParse, pTable) ){ - return SQLITE_ERROR; + db->nSchemaLock++; + rc = sqlite3VtabCallConnect(pParse, pTable); + db->nSchemaLock--; + if( rc ){ + return 1; } if( IsVirtual(pTable) ) return 0; #endif #ifndef SQLITE_OMIT_VIEW @@ -2833,11 +2841,11 @@ sqlite3UniqueConstraint(pParse, OE_Abort, pIndex); }else{ addr2 = sqlite3VdbeCurrentAddr(v); } sqlite3VdbeAddOp3(v, OP_SorterData, iSorter, regRecord, iIdx); - sqlite3VdbeAddOp3(v, OP_Last, iIdx, 0, -1); + sqlite3VdbeAddOp1(v, OP_SeekEnd, iIdx); sqlite3VdbeAddOp2(v, OP_IdxInsert, iIdx, regRecord); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); sqlite3ReleaseTempReg(pParse, regRecord); sqlite3VdbeAddOp2(v, OP_SorterNext, iSorter, addr2); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addr1); @@ -3322,11 +3330,11 @@ if( p ){ assert( p==pIndex ); /* Malloc must have failed */ sqlite3OomFault(db); goto exit_create_index; } - db->flags |= SQLITE_InternChanges; + db->mDbFlags |= DBFLAG_SchemaChange; if( pTblName!=0 ){ pIndex->tnum = db->init.newTnum; } } @@ -3358,11 +3366,11 @@ ** Index.tnum. This is required in case this index is actually a ** PRIMARY KEY and the table is actually a WITHOUT ROWID table. In ** that case the convertToWithoutRowidTable() routine will replace ** the Noop with a Goto to jump over the VDBE code generated below. */ pIndex->tnum = sqlite3VdbeAddOp0(v, OP_Noop); - sqlite3VdbeAddOp2(v, OP_CreateIndex, iDb, iMem); + sqlite3VdbeAddOp3(v, OP_CreateBtree, iDb, iMem, BTREE_BLOBKEY); /* Gather the complete text of the CREATE INDEX statement into ** the zStmt variable */ if( pStart ){ @@ -3880,22 +3888,24 @@ ** Add an INDEXED BY or NOT INDEXED clause to the most recently added ** element of the source-list passed as the second argument. */ void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pIndexedBy){ assert( pIndexedBy!=0 ); - if( p && ALWAYS(p->nSrc>0) ){ - struct SrcList_item *pItem = &p->a[p->nSrc-1]; + if( p && pIndexedBy->n>0 ){ + struct SrcList_item *pItem; + assert( p->nSrc>0 ); + pItem = &p->a[p->nSrc-1]; assert( pItem->fg.notIndexed==0 ); assert( pItem->fg.isIndexedBy==0 ); assert( pItem->fg.isTabFunc==0 ); if( pIndexedBy->n==1 && !pIndexedBy->z ){ /* A "NOT INDEXED" clause was supplied. See parse.y ** construct "indexed_opt" for details. */ pItem->fg.notIndexed = 1; }else{ pItem->u1.zIndexedBy = sqlite3NameFromToken(pParse->db, pIndexedBy); - pItem->fg.isIndexedBy = (pItem->u1.zIndexedBy!=0); + pItem->fg.isIndexedBy = 1; } } } /* Index: src/callback.c ================================================================== --- src/callback.c +++ src/callback.c @@ -372,21 +372,21 @@ p = p->pNext; } /* If no match is found, search the built-in functions. ** - ** If the SQLITE_PreferBuiltin flag is set, then search the built-in + ** If the DBFLAG_PreferBuiltin flag is set, then search the built-in ** functions even if a prior app-defined function was found. And give ** priority to built-in functions. ** ** Except, if createFlag is true, that means that we are trying to ** install a new function. Whatever FuncDef structure is returned it will ** have fields overwritten with new information appropriate for the ** new function. But the FuncDefs for built-in functions are read-only. ** So we must not search for built-ins when creating a new function. */ - if( !createFlag && (pBest==0 || (db->flags & SQLITE_PreferBuiltin)!=0) ){ + if( !createFlag && (pBest==0 || (db->mDbFlags & DBFLAG_PreferBuiltin)!=0) ){ bestScore = 0; h = (sqlite3UpperToLower[(u8)zName[0]] + nName) % SQLITE_FUNC_HASH_SZ; p = functionSearch(h, zName); while( p ){ int score = matchQuality(p, nArg, enc); @@ -455,12 +455,12 @@ sqlite3HashClear(&temp1); sqlite3HashClear(&pSchema->fkeyHash); pSchema->pSeqTab = 0; if( pSchema->schemaFlags & DB_SchemaLoaded ){ pSchema->iGeneration++; - pSchema->schemaFlags &= ~DB_SchemaLoaded; } + pSchema->schemaFlags &= ~(DB_SchemaLoaded|DB_ResetWanted); } /* ** Find and return the schema associated with a BTree. Create ** a new one if necessary. Index: src/ctime.c ================================================================== --- src/ctime.c +++ src/ctime.c @@ -181,10 +181,13 @@ #if SQLITE_ENABLE_API_ARMOR "ENABLE_API_ARMOR", #endif #if SQLITE_ENABLE_ATOMIC_WRITE "ENABLE_ATOMIC_WRITE", +#endif +#if SQLITE_ENABLE_BATCH_ATOMIC_WRITE + "ENABLE_BATCH_ATOMIC_WRITE", #endif #if SQLITE_ENABLE_CEROD "ENABLE_CEROD", #endif #if SQLITE_ENABLE_COLUMN_METADATA Index: src/date.c ================================================================== --- src/date.c +++ src/date.c @@ -384,11 +384,11 @@ double r; if( parseYyyyMmDd(zDate,p)==0 ){ return 0; }else if( parseHhMmSs(zDate, p)==0 ){ return 0; - }else if( sqlite3StrICmp(zDate,"now")==0){ + }else if( sqlite3StrICmp(zDate,"now")==0 && sqlite3NotPureFunc(context) ){ return setDateTimeToCurrent(context, p); }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8) ){ setRawDateNumber(p, r); return 0; } @@ -667,11 +667,11 @@ /* localtime ** ** Assuming the current time value is UTC (a.k.a. GMT), shift it to ** show local time. */ - if( sqlite3_stricmp(z, "localtime")==0 ){ + if( sqlite3_stricmp(z, "localtime")==0 && sqlite3NotPureFunc(pCtx) ){ computeJD(p); p->iJD += localtimeOffset(p, pCtx, &rc); clearYMD_HMS_TZ(p); } break; @@ -693,11 +693,11 @@ p->rawS = 0; rc = 0; } } #ifndef SQLITE_OMIT_LOCALTIME - else if( sqlite3_stricmp(z, "utc")==0 ){ + else if( sqlite3_stricmp(z, "utc")==0 && sqlite3NotPureFunc(pCtx) ){ if( p->tzSet==0 ){ sqlite3_int64 c1; computeJD(p); c1 = localtimeOffset(p, pCtx, &rc); if( rc==SQLITE_OK ){ @@ -1229,15 +1229,15 @@ ** external linkage. */ void sqlite3RegisterDateTimeFunctions(void){ static FuncDef aDateTimeFuncs[] = { #ifndef SQLITE_OMIT_DATETIME_FUNCS - DFUNCTION(julianday, -1, 0, 0, juliandayFunc ), - DFUNCTION(date, -1, 0, 0, dateFunc ), - DFUNCTION(time, -1, 0, 0, timeFunc ), - DFUNCTION(datetime, -1, 0, 0, datetimeFunc ), - DFUNCTION(strftime, -1, 0, 0, strftimeFunc ), + PURE_DATE(julianday, -1, 0, 0, juliandayFunc ), + PURE_DATE(date, -1, 0, 0, dateFunc ), + PURE_DATE(time, -1, 0, 0, timeFunc ), + PURE_DATE(datetime, -1, 0, 0, datetimeFunc ), + PURE_DATE(strftime, -1, 0, 0, strftimeFunc ), DFUNCTION(current_time, 0, 0, 0, ctimeFunc ), DFUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc), DFUNCTION(current_date, 0, 0, 0, cdateFunc ), #else STR_FUNCTION(current_time, 0, "%H:%M:%S", 0, currentTimeFunc), Index: src/delete.c ================================================================== --- src/delete.c +++ src/delete.c @@ -500,11 +500,15 @@ sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, addrBypass, iKey, nKey); VdbeCoverage(v); } }else if( pPk ){ addrLoop = sqlite3VdbeAddOp1(v, OP_Rewind, iEphCur); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_RowData, iEphCur, iKey); + if( IsVirtual(pTab) ){ + sqlite3VdbeAddOp3(v, OP_Column, iEphCur, 0, iKey); + }else{ + sqlite3VdbeAddOp2(v, OP_RowData, iEphCur, iKey); + } assert( nKey==0 ); /* OP_Found will use a composite key */ }else{ addrLoop = sqlite3VdbeAddOp3(v, OP_RowSetRead, iRowSet, 0, iKey); VdbeCoverage(v); assert( nKey==1 ); @@ -850,14 +854,15 @@ int nCol; if( piPartIdxLabel ){ if( pIdx->pPartIdxWhere ){ *piPartIdxLabel = sqlite3VdbeMakeLabel(v); - pParse->iSelfTab = iDataCur; + pParse->iSelfTab = iDataCur + 1; sqlite3ExprCachePush(pParse); sqlite3ExprIfFalseDup(pParse, pIdx->pPartIdxWhere, *piPartIdxLabel, SQLITE_JUMPIFNULL); + pParse->iSelfTab = 0; }else{ *piPartIdxLabel = 0; } } nCol = (prefixOnly && pIdx->uniqNotNull) ? pIdx->nKeyCol : pIdx->nColumn; Index: src/expr.c ================================================================== --- src/expr.c +++ src/expr.c @@ -773,11 +773,11 @@ int op, /* Expression opcode */ const char *zToken /* Token argument. Might be NULL */ ){ Token x; x.z = zToken; - x.n = zToken ? sqlite3Strlen30(zToken) : 0; + x.n = sqlite3Strlen30(zToken); return sqlite3ExprAlloc(db, op, &x, 0); } /* ** Attach subtrees pLeft and pRight to the Expr node pRoot. @@ -1684,10 +1684,23 @@ m |= pExpr->flags; } } return m; } + +/* +** This is a SELECT-node callback for the expression walker that +** always "fails". By "fail" in this case, we mean set +** pWalker->eCode to zero and abort. +** +** This callback is used by multiple expression walkers. +*/ +int sqlite3SelectWalkFail(Walker *pWalker, Select *NotUsed){ + UNUSED_PARAMETER(NotUsed); + pWalker->eCode = 0; + return WRC_Abort; +} /* ** These routines are Walker callbacks used to check expressions to ** see if they are "constant" for some definition of constant. The ** Walker.eCode value determines the type of "constant" we are looking @@ -1761,25 +1774,20 @@ pWalker->eCode = 0; return WRC_Abort; } /* Fall through */ default: - testcase( pExpr->op==TK_SELECT ); /* selectNodeIsConstant will disallow */ - testcase( pExpr->op==TK_EXISTS ); /* selectNodeIsConstant will disallow */ + testcase( pExpr->op==TK_SELECT ); /* sqlite3SelectWalkFail will disallow */ + testcase( pExpr->op==TK_EXISTS ); /* sqlite3SelectWalkFail will disallow */ return WRC_Continue; } } -static int selectNodeIsConstant(Walker *pWalker, Select *NotUsed){ - UNUSED_PARAMETER(NotUsed); - pWalker->eCode = 0; - return WRC_Abort; -} static int exprIsConst(Expr *p, int initFlag, int iCur){ Walker w; w.eCode = initFlag; w.xExprCallback = exprNodeIsConstant; - w.xSelectCallback = selectNodeIsConstant; + w.xSelectCallback = sqlite3SelectWalkFail; #ifdef SQLITE_DEBUG w.xSelectCallback2 = sqlite3SelectWalkAssert2; #endif w.u.iCur = iCur; sqlite3WalkExpr(&w, p); @@ -1898,11 +1906,11 @@ */ int sqlite3ExprContainsSubquery(Expr *p){ Walker w; w.eCode = 1; w.xExprCallback = sqlite3ExprWalkNoop; - w.xSelectCallback = selectNodeIsConstant; + w.xSelectCallback = sqlite3SelectWalkFail; #ifdef SQLITE_DEBUG w.xSelectCallback2 = sqlite3SelectWalkAssert2; #endif sqlite3WalkExpr(&w, p); return w.eCode==0; @@ -1971,12 +1979,12 @@ case TK_STRING: case TK_FLOAT: case TK_BLOB: return 0; case TK_COLUMN: - assert( p->pTab!=0 ); return ExprHasProperty(p, EP_CanBeNull) || + p->pTab==0 || /* Reference to column of index on expression */ (p->iColumn>=0 && p->pTab->aCol[p->iColumn].notNull==0); default: return 1; } } @@ -3236,12 +3244,13 @@ ){ i16 iTabCol = pIdx->aiColumn[iIdxCol]; if( iTabCol==XN_EXPR ){ assert( pIdx->aColExpr ); assert( pIdx->aColExpr->nExpr>iIdxCol ); - pParse->iSelfTab = iTabCur; + pParse->iSelfTab = iTabCur + 1; sqlite3ExprCodeCopy(pParse, pIdx->aColExpr->a[iIdxCol].pExpr, regOut); + pParse->iSelfTab = 0; }else{ sqlite3ExprCodeGetColumnOfTable(pParse->pVdbe, pIdx->pTable, iTabCur, iTabCol, regOut); } } @@ -3481,17 +3490,17 @@ /* Otherwise, fall thru into the TK_COLUMN case */ } case TK_COLUMN: { int iTab = pExpr->iTable; if( iTab<0 ){ - if( pParse->ckBase>0 ){ + if( pParse->iSelfTab<0 ){ /* Generating CHECK constraints or inserting into partial index */ - return pExpr->iColumn + pParse->ckBase; + return pExpr->iColumn - pParse->iSelfTab; }else{ /* Coding an expression that is part of an index where column names ** in the index refer to the table to which the index belongs */ - iTab = pParse->iSelfTab; + iTab = pParse->iSelfTab - 1; } } return sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, pExpr->iColumn, iTab, target, pExpr->op2); @@ -3824,12 +3833,12 @@ #endif if( pDef->funcFlags & SQLITE_FUNC_NEEDCOLL ){ if( !pColl ) pColl = db->pDfltColl; sqlite3VdbeAddOp4(v, OP_CollSeq, 0, 0, 0, (char *)pColl, P4_COLLSEQ); } - sqlite3VdbeAddOp4(v, OP_Function0, constMask, r1, target, - (char*)pDef, P4_FUNCDEF); + sqlite3VdbeAddOp4(v, pParse->iSelfTab ? OP_PureFunc0 : OP_Function0, + constMask, r1, target, (char*)pDef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, (u8)nFarg); if( nFarg && constMask==0 ){ sqlite3ReleaseTempRange(pParse, r1, nFarg); } return target; Index: src/func.c ================================================================== --- src/func.c +++ src/func.c @@ -863,11 +863,12 @@ } if( zA && zB ){ #ifdef SQLITE_TEST sqlite3_like_count++; #endif - sqlite3_result_int(context, patternCompare(zB, zA, pInfo, escape)==SQLITE_MATCH); + sqlite3_result_int(context, + patternCompare(zB, zA, pInfo, escape)==SQLITE_MATCH); } } /* ** Implementation of the NULLIF(x,y) function. The result is the first @@ -1704,32 +1705,46 @@ } /* ** pExpr points to an expression which implements a function. If ** it is appropriate to apply the LIKE optimization to that function -** then set aWc[0] through aWc[2] to the wildcard characters and -** return TRUE. If the function is not a LIKE-style function then -** return FALSE. +** then set aWc[0] through aWc[2] to the wildcard characters and the +** escape character and then return TRUE. If the function is not a +** LIKE-style function then return FALSE. +** +** The expression "a LIKE b ESCAPE c" is only considered a valid LIKE +** operator if c is a string literal that is exactly one byte in length. +** That one byte is stored in aWc[3]. aWc[3] is set to zero if there is +** no ESCAPE clause. ** ** *pIsNocase is set to true if uppercase and lowercase are equivalent for ** the function (default for LIKE). If the function makes the distinction ** between uppercase and lowercase (as does GLOB) then *pIsNocase is set to ** false. */ int sqlite3IsLikeFunction(sqlite3 *db, Expr *pExpr, int *pIsNocase, char *aWc){ FuncDef *pDef; - if( pExpr->op!=TK_FUNCTION - || !pExpr->x.pList - || pExpr->x.pList->nExpr!=2 - ){ + int nExpr; + if( pExpr->op!=TK_FUNCTION || !pExpr->x.pList ){ return 0; } assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - pDef = sqlite3FindFunction(db, pExpr->u.zToken, 2, SQLITE_UTF8, 0); + nExpr = pExpr->x.pList->nExpr; + pDef = sqlite3FindFunction(db, pExpr->u.zToken, nExpr, SQLITE_UTF8, 0); if( NEVER(pDef==0) || (pDef->funcFlags & SQLITE_FUNC_LIKE)==0 ){ return 0; } + if( nExpr<3 ){ + aWc[3] = 0; + }else{ + Expr *pEscape = pExpr->x.pList->a[2].pExpr; + char *zEscape; + if( pEscape->op!=TK_STRING ) return 0; + zEscape = pEscape->u.zToken; + if( zEscape[0]==0 || zEscape[1]!=0 ) return 0; + aWc[3] = zEscape[0]; + } /* The memcpy() statement assumes that the wildcard characters are ** the first three statements in the compareInfo structure. The ** asserts() that follow verify that assumption */ Index: src/global.c ================================================================== --- src/global.c +++ src/global.c @@ -197,10 +197,11 @@ SQLITE_DEFAULT_MEMSTATUS, /* bMemstat */ 1, /* bCoreMutex */ SQLITE_THREADSAFE==1, /* bFullMutex */ SQLITE_USE_URI, /* bOpenUri */ SQLITE_ALLOW_COVERING_INDEX_SCAN, /* bUseCis */ + 0, /* bSmallMalloc */ 0x7ffffffe, /* mxStrlen */ 0, /* neverCorrupt */ SQLITE_DEFAULT_LOOKASIDE, /* szLookaside, nLookaside */ SQLITE_STMTJRNL_SPILL, /* nStmtSpill */ {0,0,0,0,0,0,0,0}, /* m */ @@ -209,13 +210,10 @@ (void*)0, /* pHeap */ 0, /* nHeap */ 0, 0, /* mnHeap, mxHeap */ SQLITE_DEFAULT_MMAP_SIZE, /* szMmap */ SQLITE_MAX_MMAP_SIZE, /* mxMmap */ - (void*)0, /* pScratch */ - 0, /* szScratch */ - 0, /* nScratch */ (void*)0, /* pPage */ 0, /* szPage */ SQLITE_DEFAULT_PCACHE_INITSZ, /* nPage */ 0, /* mxParserStack */ 0, /* sharedCacheEnabled */ Index: src/insert.c ================================================================== --- src/insert.c +++ src/insert.c @@ -224,11 +224,11 @@ int iDb, /* Index of the database holding pTab */ Table *pTab /* The table we are writing to */ ){ int memId = 0; /* Register holding maximum rowid */ if( (pTab->tabFlags & TF_Autoincrement)!=0 - && (pParse->db->flags & SQLITE_Vacuum)==0 + && (pParse->db->mDbFlags & DBFLAG_Vacuum)==0 ){ Parse *pToplevel = sqlite3ParseToplevel(pParse); AutoincInfo *pInfo; pInfo = pToplevel->pAinc; @@ -1331,11 +1331,11 @@ /* Test all CHECK constraints */ #ifndef SQLITE_OMIT_CHECK if( pTab->pCheck && (db->flags & SQLITE_IgnoreChecks)==0 ){ ExprList *pCheck = pTab->pCheck; - pParse->ckBase = regNewData+1; + pParse->iSelfTab = -(regNewData+1); onError = overrideError!=OE_Default ? overrideError : OE_Abort; for(i=0; inExpr; i++){ int allOk; Expr *pExpr = pCheck->a[i].pExpr; if( aiChng && checkConstraintUnchanged(pExpr, aiChng, pkChng) ) continue; @@ -1351,10 +1351,11 @@ onError, zName, P4_TRANSIENT, P5_ConstraintCheck); } sqlite3VdbeResolveLabel(v, allOk); } + pParse->iSelfTab = 0; } #endif /* !defined(SQLITE_OMIT_CHECK) */ /* If rowid is changing, make sure the new rowid does not previously ** exist in the table. @@ -1495,14 +1496,14 @@ addrUniqueOk = sqlite3VdbeMakeLabel(v); /* Skip partial indices for which the WHERE clause is not true */ if( pIdx->pPartIdxWhere ){ sqlite3VdbeAddOp2(v, OP_Null, 0, aRegIdx[ix]); - pParse->ckBase = regNewData+1; + pParse->iSelfTab = -(regNewData+1); sqlite3ExprIfFalseDup(pParse, pIdx->pPartIdxWhere, addrUniqueOk, SQLITE_JUMPIFNULL); - pParse->ckBase = 0; + pParse->iSelfTab = 0; } /* Create a record for this index entry as it should appear after ** the insert or update. Store that record in the aRegIdx[ix] register */ @@ -1509,13 +1510,13 @@ regIdx = aRegIdx[ix]+1; for(i=0; inColumn; i++){ int iField = pIdx->aiColumn[i]; int x; if( iField==XN_EXPR ){ - pParse->ckBase = regNewData+1; + pParse->iSelfTab = -(regNewData+1); sqlite3ExprCodeCopy(pParse, pIdx->aColExpr->a[i].pExpr, regIdx+i); - pParse->ckBase = 0; + pParse->iSelfTab = 0; VdbeComment((v, "%s column %d", pIdx->zName, i)); }else{ if( iField==XN_ROWID || iField==pTab->iPKey ){ x = regNewData; }else{ @@ -2056,11 +2057,11 @@ } for(i=0; inCol; i++){ Column *pDestCol = &pDest->aCol[i]; Column *pSrcCol = &pSrc->aCol[i]; #ifdef SQLITE_ENABLE_HIDDEN_COLUMNS - if( (db->flags & SQLITE_Vacuum)==0 + if( (db->mDbFlags & DBFLAG_Vacuum)==0 && (pDestCol->colFlags | pSrcCol->colFlags) & COLFLAG_HIDDEN ){ return 0; /* Neither table may have __hidden__ columns */ } #endif @@ -2132,19 +2133,19 @@ regAutoinc = autoIncBegin(pParse, iDbDest, pDest); regData = sqlite3GetTempReg(pParse); regRowid = sqlite3GetTempReg(pParse); sqlite3OpenTable(pParse, iDest, iDbDest, pDest, OP_OpenWrite); assert( HasRowid(pDest) || destHasUniqueIdx ); - if( (db->flags & SQLITE_Vacuum)==0 && ( + if( (db->mDbFlags & DBFLAG_Vacuum)==0 && ( (pDest->iPKey<0 && pDest->pIndex!=0) /* (1) */ || destHasUniqueIdx /* (2) */ || (onError!=OE_Abort && onError!=OE_Rollback) /* (3) */ )){ /* In some circumstances, we are able to run the xfer optimization ** only if the destination table is initially empty. Unless the - ** SQLITE_Vacuum flag is set, this block generates code to make - ** that determination. If SQLITE_Vacuum is set, then the destination + ** DBFLAG_Vacuum flag is set, this block generates code to make + ** that determination. If DBFLAG_Vacuum is set, then the destination ** table is always empty. ** ** Conditions under which the destination must be empty: ** ** (1) There is no INTEGER PRIMARY KEY but there are indices. @@ -2176,12 +2177,12 @@ }else{ addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); assert( (pDest->tabFlags & TF_Autoincrement)==0 ); } sqlite3VdbeAddOp3(v, OP_RowData, iSrc, regData, 1); - if( db->flags & SQLITE_Vacuum ){ - sqlite3VdbeAddOp3(v, OP_Last, iDest, 0, -1); + if( db->mDbFlags & DBFLAG_Vacuum ){ + sqlite3VdbeAddOp1(v, OP_SeekEnd, iDest); insFlags = OPFLAG_NCHANGE|OPFLAG_LASTROWID| OPFLAG_APPEND|OPFLAG_USESEEKRESULT; }else{ insFlags = OPFLAG_NCHANGE|OPFLAG_LASTROWID|OPFLAG_APPEND; } @@ -2208,17 +2209,17 @@ sqlite3VdbeSetP4KeyInfo(pParse, pDestIdx); sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR); VdbeComment((v, "%s", pDestIdx->zName)); addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_RowData, iSrc, regData, 1); - if( db->flags & SQLITE_Vacuum ){ + if( db->mDbFlags & DBFLAG_Vacuum ){ /* This INSERT command is part of a VACUUM operation, which guarantees ** that the destination table is empty. If all indexed columns use ** collation sequence BINARY, then it can also be assumed that the ** index will be populated by inserting keys in strictly sorted ** order. In this case, instead of seeking within the b-tree as part - ** of every OP_IdxInsert opcode, an OP_Last is added before the + ** of every OP_IdxInsert opcode, an OP_SeekEnd is added before the ** OP_IdxInsert to seek to the point within the b-tree where each key ** should be inserted. This is faster. ** ** If any of the indexed columns use a collation sequence other than ** BINARY, this optimization is disabled. This is because the user @@ -2229,11 +2230,11 @@ const char *zColl = pSrcIdx->azColl[i]; if( sqlite3_stricmp(sqlite3StrBINARY, zColl) ) break; } if( i==pSrcIdx->nColumn ){ idxInsFlags = OPFLAG_USESEEKRESULT; - sqlite3VdbeAddOp3(v, OP_Last, iDest, 0, -1); + sqlite3VdbeAddOp1(v, OP_SeekEnd, iDest); } } if( !HasRowid(pSrc) && pDestIdx->idxType==2 ){ idxInsFlags |= OPFLAG_NCHANGE; } Index: src/main.c ================================================================== --- src/main.c +++ src/main.c @@ -437,18 +437,12 @@ ** single argument of type int, interpreted as a boolean, which enables ** or disables the collection of memory allocation statistics. */ sqlite3GlobalConfig.bMemstat = va_arg(ap, int); break; } - case SQLITE_CONFIG_SCRATCH: { - /* EVIDENCE-OF: R-08404-60887 There are three arguments to - ** SQLITE_CONFIG_SCRATCH: A pointer an 8-byte aligned memory buffer from - ** which the scratch allocations will be drawn, the size of each scratch - ** allocation (sz), and the maximum number of scratch allocations (N). */ - sqlite3GlobalConfig.pScratch = va_arg(ap, void*); - sqlite3GlobalConfig.szScratch = va_arg(ap, int); - sqlite3GlobalConfig.nScratch = va_arg(ap, int); + case SQLITE_CONFIG_SMALL_MALLOC: { + sqlite3GlobalConfig.bSmallMalloc = va_arg(ap, int); break; } case SQLITE_CONFIG_PAGECACHE: { /* EVIDENCE-OF: R-18761-36601 There are three arguments to ** SQLITE_CONFIG_PAGECACHE: A pointer to 8-byte aligned memory (pMem), @@ -665,11 +659,12 @@ ** the lookaside memory. */ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ #ifndef SQLITE_OMIT_LOOKASIDE void *pStart; - if( db->lookaside.nOut ){ + + if( sqlite3LookasideUsed(db,0)>0 ){ return SQLITE_BUSY; } /* Free any existing lookaside buffer for this handle before ** allocating a new one so we don't have to have space for ** both at the same time. @@ -693,20 +688,22 @@ if( pStart ) cnt = sqlite3MallocSize(pStart)/sz; }else{ pStart = pBuf; } db->lookaside.pStart = pStart; + db->lookaside.pInit = 0; db->lookaside.pFree = 0; db->lookaside.sz = (u16)sz; if( pStart ){ int i; LookasideSlot *p; assert( sz > (int)sizeof(LookasideSlot*) ); + db->lookaside.nSlot = cnt; p = (LookasideSlot*)pStart; for(i=cnt-1; i>=0; i--){ - p->pNext = db->lookaside.pFree; - db->lookaside.pFree = p; + p->pNext = db->lookaside.pInit; + db->lookaside.pInit = p; p = (LookasideSlot*)&((u8*)p)[sz]; } db->lookaside.pEnd = p; db->lookaside.bDisable = 0; db->lookaside.bMalloced = pBuf==0 ?1:0; @@ -713,10 +710,11 @@ }else{ db->lookaside.pStart = db; db->lookaside.pEnd = db; db->lookaside.bDisable = 1; db->lookaside.bMalloced = 0; + db->lookaside.nSlot = 0; } #endif /* SQLITE_OMIT_LOOKASIDE */ return SQLITE_OK; } @@ -825,11 +823,11 @@ rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ for(i=0; iflags; + u32 oldFlags = db->flags; if( onoff>0 ){ db->flags |= aFlagOp[i].mask; }else if( onoff==0 ){ db->flags &= ~aFlagOp[i].mask; } @@ -1232,11 +1230,11 @@ */ sqlite3DbFree(db, db->aDb[1].pSchema); sqlite3_mutex_leave(db->mutex); db->magic = SQLITE_MAGIC_CLOSED; sqlite3_mutex_free(db->mutex); - assert( db->lookaside.nOut==0 ); /* Fails on a lookaside memory leak */ + assert( sqlite3LookasideUsed(db,0)==0 ); if( db->lookaside.bMalloced ){ sqlite3_free(db->lookaside.pStart); } #ifdef SQLITE_ENABLE_SQLRR SRRecClose(db); @@ -1264,11 +1262,11 @@ ** modified the database schema. If the b-tree mutexes are not taken ** here, then another shared-cache connection might sneak in between ** the database rollback and schema reset, which can cause false ** corruption reports in some cases. */ sqlite3BtreeEnterAll(db); - schemaChange = (db->flags & SQLITE_InternChanges)!=0 && db->init.busy==0; + schemaChange = (db->mDbFlags & DBFLAG_SchemaChange)!=0 && db->init.busy==0; for(i=0; inDb; i++){ Btree *p = db->aDb[i].pBt; if( p ){ if( sqlite3BtreeIsInTrans(p) ){ @@ -1278,11 +1276,11 @@ } } sqlite3VtabRollback(db); sqlite3EndBenignMalloc(); - if( (db->flags&SQLITE_InternChanges)!=0 && db->init.busy==0 ){ + if( (db->mDbFlags&DBFLAG_SchemaChange)!=0 && db->init.busy==0 ){ sqlite3ExpirePreparedStatements(db); sqlite3ResetAllSchemasOfConnection(db); } sqlite3BtreeLeaveAll(db); @@ -2180,11 +2178,12 @@ ** ** If iDb is passed SQLITE_MAX_ATTACHED, then all attached databases are ** checkpointed. If an error is encountered it is returned immediately - ** no attempt is made to checkpoint any remaining databases. ** -** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART. +** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL, RESTART +** or TRUNCATE. */ int sqlite3Checkpoint(sqlite3 *db, int iDb, int eMode, int *pnLog, int *pnCkpt){ int rc = SQLITE_OK; /* Return code */ int i; /* Used to iterate through attached dbs */ int bBusy = 0; /* True if SQLITE_BUSY has been encountered */ @@ -3928,26 +3927,10 @@ rc = (sqlite3KeywordCode((u8*)zWord, n)!=TK_ID) ? SQLITE_N_KEYWORD : 0; break; } #endif - /* sqlite3_test_control(SQLITE_TESTCTRL_SCRATCHMALLOC, sz, &pNew, pFree); - ** - ** Pass pFree into sqlite3ScratchFree(). - ** If sz>0 then allocate a scratch buffer into pNew. - */ - case SQLITE_TESTCTRL_SCRATCHMALLOC: { - void *pFree, **ppNew; - int sz; - sz = va_arg(ap, int); - ppNew = va_arg(ap, void**); - pFree = va_arg(ap, void*); - if( sz ) *ppNew = sqlite3ScratchMalloc(sz); - sqlite3ScratchFree(pFree); - break; - } - /* sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, int onoff); ** ** If parameter onoff is non-zero, configure the wrappers so that all ** subsequent calls to localtime() and variants fail. If onoff is zero, ** undo this setting. Index: src/malloc.c ================================================================== --- src/malloc.c +++ src/malloc.c @@ -30,41 +30,23 @@ UNUSED_PARAMETER(n); return 0; #endif } -/* -** An instance of the following object records the location of -** each unused scratch buffer. -*/ -typedef struct ScratchFreeslot { - struct ScratchFreeslot *pNext; /* Next unused scratch buffer */ -} ScratchFreeslot; - /* ** State information local to the memory allocation subsystem. */ static SQLITE_WSD struct Mem0Global { sqlite3_mutex *mutex; /* Mutex to serialize access */ sqlite3_int64 alarmThreshold; /* The soft heap limit */ - /* - ** Pointers to the end of sqlite3GlobalConfig.pScratch memory - ** (so that a range test can be used to determine if an allocation - ** being freed came from pScratch) and a pointer to the list of - ** unused scratch allocations. - */ - void *pScratchEnd; - ScratchFreeslot *pScratchFree; - u32 nScratchFree; - /* ** True if heap is nearly "full" where "full" is defined by the ** sqlite3_soft_heap_limit() setting. */ int nearlyFull; -} mem0 = { 0, 0, 0, 0, 0, 0 }; +} mem0 = { 0, 0, 0 }; #define mem0 GLOBAL(struct Mem0Global, mem0) /* ** Return the memory allocator mutex. sqlite3_status() needs it. @@ -130,32 +112,10 @@ if( sqlite3GlobalConfig.m.xMalloc==0 ){ sqlite3MemSetDefault(); } memset(&mem0, 0, sizeof(mem0)); mem0.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); - if( sqlite3GlobalConfig.pScratch && sqlite3GlobalConfig.szScratch>=100 - && sqlite3GlobalConfig.nScratch>0 ){ - int i, n, sz; - ScratchFreeslot *pSlot; - sz = ROUNDDOWN8(sqlite3GlobalConfig.szScratch); - sqlite3GlobalConfig.szScratch = sz; - pSlot = (ScratchFreeslot*)sqlite3GlobalConfig.pScratch; - n = sqlite3GlobalConfig.nScratch; - mem0.pScratchFree = pSlot; - mem0.nScratchFree = n; - for(i=0; ipNext = (ScratchFreeslot*)(sz+(char*)pSlot); - pSlot = pSlot->pNext; - } - pSlot->pNext = 0; - mem0.pScratchEnd = (void*)&pSlot[1]; - }else{ - mem0.pScratchEnd = 0; - sqlite3GlobalConfig.pScratch = 0; - sqlite3GlobalConfig.szScratch = 0; - sqlite3GlobalConfig.nScratch = 0; - } if( sqlite3GlobalConfig.pPage==0 || sqlite3GlobalConfig.szPage<512 || sqlite3GlobalConfig.nPage<=0 ){ sqlite3GlobalConfig.pPage = 0; sqlite3GlobalConfig.szPage = 0; } @@ -302,109 +262,10 @@ if( sqlite3_initialize() ) return 0; #endif return sqlite3Malloc(n); } -/* -** Each thread may only have a single outstanding allocation from -** xScratchMalloc(). We verify this constraint in the single-threaded -** case by setting scratchAllocOut to 1 when an allocation -** is outstanding clearing it when the allocation is freed. -*/ -#if SQLITE_THREADSAFE==0 && !defined(NDEBUG) -static int scratchAllocOut = 0; -#endif - - -/* -** Allocate memory that is to be used and released right away. -** This routine is similar to alloca() in that it is not intended -** for situations where the memory might be held long-term. This -** routine is intended to get memory to old large transient data -** structures that would not normally fit on the stack of an -** embedded processor. -*/ -void *sqlite3ScratchMalloc(int n){ - void *p; - assert( n>0 ); - - sqlite3_mutex_enter(mem0.mutex); - sqlite3StatusHighwater(SQLITE_STATUS_SCRATCH_SIZE, n); - if( mem0.nScratchFree && sqlite3GlobalConfig.szScratch>=n ){ - p = mem0.pScratchFree; - mem0.pScratchFree = mem0.pScratchFree->pNext; - mem0.nScratchFree--; - sqlite3StatusUp(SQLITE_STATUS_SCRATCH_USED, 1); - sqlite3_mutex_leave(mem0.mutex); - }else{ - sqlite3_mutex_leave(mem0.mutex); - p = sqlite3Malloc(n); - if( sqlite3GlobalConfig.bMemstat && p ){ - sqlite3_mutex_enter(mem0.mutex); - sqlite3StatusUp(SQLITE_STATUS_SCRATCH_OVERFLOW, sqlite3MallocSize(p)); - sqlite3_mutex_leave(mem0.mutex); - } - sqlite3MemdebugSetType(p, MEMTYPE_SCRATCH); - } - assert( sqlite3_mutex_notheld(mem0.mutex) ); - - -#if SQLITE_THREADSAFE==0 && !defined(NDEBUG) - /* EVIDENCE-OF: R-12970-05880 SQLite will not use more than one scratch - ** buffers per thread. - ** - ** This can only be checked in single-threaded mode. - */ - assert( scratchAllocOut==0 ); - if( p ) scratchAllocOut++; -#endif - - return p; -} -void sqlite3ScratchFree(void *p){ - if( p ){ - -#if SQLITE_THREADSAFE==0 && !defined(NDEBUG) - /* Verify that no more than two scratch allocation per thread - ** is outstanding at one time. (This is only checked in the - ** single-threaded case since checking in the multi-threaded case - ** would be much more complicated.) */ - assert( scratchAllocOut>=1 && scratchAllocOut<=2 ); - scratchAllocOut--; -#endif - - if( SQLITE_WITHIN(p, sqlite3GlobalConfig.pScratch, mem0.pScratchEnd) ){ - /* Release memory from the SQLITE_CONFIG_SCRATCH allocation */ - ScratchFreeslot *pSlot; - pSlot = (ScratchFreeslot*)p; - sqlite3_mutex_enter(mem0.mutex); - pSlot->pNext = mem0.pScratchFree; - mem0.pScratchFree = pSlot; - mem0.nScratchFree++; - assert( mem0.nScratchFree <= (u32)sqlite3GlobalConfig.nScratch ); - sqlite3StatusDown(SQLITE_STATUS_SCRATCH_USED, 1); - sqlite3_mutex_leave(mem0.mutex); - }else{ - /* Release memory back to the heap */ - assert( sqlite3MemdebugHasType(p, MEMTYPE_SCRATCH) ); - assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_SCRATCH) ); - sqlite3MemdebugSetType(p, MEMTYPE_HEAP); - if( sqlite3GlobalConfig.bMemstat ){ - int iSize = sqlite3MallocSize(p); - sqlite3_mutex_enter(mem0.mutex); - sqlite3StatusDown(SQLITE_STATUS_SCRATCH_OVERFLOW, iSize); - sqlite3StatusDown(SQLITE_STATUS_MEMORY_USED, iSize); - sqlite3StatusDown(SQLITE_STATUS_MALLOC_COUNT, 1); - sqlite3GlobalConfig.m.xFree(p); - sqlite3_mutex_leave(mem0.mutex); - }else{ - sqlite3GlobalConfig.m.xFree(p); - } - } - } -} - /* ** TRUE if p is a lookaside memory allocation from db */ #ifndef SQLITE_OMIT_LOOKASIDE static int isLookaside(sqlite3 *db, void *p){ @@ -491,11 +352,10 @@ /* Trash all content in the buffer being freed */ memset(p, 0xaa, db->lookaside.sz); #endif pBuf->pNext = db->lookaside.pFree; db->lookaside.pFree = pBuf; - db->lookaside.nOut--; return; } } assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); @@ -652,20 +512,20 @@ assert( db->pnBytesFreed==0 ); if( db->lookaside.bDisable==0 ){ assert( db->mallocFailed==0 ); if( n>db->lookaside.sz ){ db->lookaside.anStat[1]++; - }else if( (pBuf = db->lookaside.pFree)==0 ){ - db->lookaside.anStat[2]++; - }else{ + }else if( (pBuf = db->lookaside.pFree)!=0 ){ db->lookaside.pFree = pBuf->pNext; - db->lookaside.nOut++; db->lookaside.anStat[0]++; - if( db->lookaside.nOut>db->lookaside.mxOut ){ - db->lookaside.mxOut = db->lookaside.nOut; - } + return (void*)pBuf; + }else if( (pBuf = db->lookaside.pInit)!=0 ){ + db->lookaside.pInit = pBuf->pNext; + db->lookaside.anStat[0]++; return (void*)pBuf; + }else{ + db->lookaside.anStat[2]++; } }else if( db->mallocFailed ){ return 0; } #else Index: src/memjournal.c ================================================================== --- src/memjournal.c +++ src/memjournal.c @@ -94,11 +94,12 @@ u8 *zOut = zBuf; int nRead = iAmt; int iChunkOffset; FileChunk *pChunk; -#ifdef SQLITE_ENABLE_ATOMIC_WRITE +#if defined(SQLITE_ENABLE_ATOMIC_WRITE) \ + || defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) if( (iAmt+iOfst)>p->endpoint.iOffset ){ return SQLITE_IOERR_SHORT_READ; } #endif @@ -213,11 +214,12 @@ ** access writes are not required. The only exception to this is when ** the in-memory journal is being used by a connection using the ** atomic-write optimization. In this case the first 28 bytes of the ** journal file may be written as part of committing the transaction. */ assert( iOfst==p->endpoint.iOffset || iOfst==0 ); -#ifdef SQLITE_ENABLE_ATOMIC_WRITE +#if defined(SQLITE_ENABLE_ATOMIC_WRITE) \ + || defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) if( iOfst==0 && p->pFirst ){ assert( p->nChunkSize>iAmt ); memcpy((u8*)p->pFirst->zChunk, zBuf, iAmt); }else #else @@ -382,21 +384,35 @@ */ void sqlite3MemJournalOpen(sqlite3_file *pJfd){ sqlite3JournalOpen(0, 0, pJfd, 0, -1); } -#ifdef SQLITE_ENABLE_ATOMIC_WRITE +#if defined(SQLITE_ENABLE_ATOMIC_WRITE) \ + || defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) /* ** If the argument p points to a MemJournal structure that is not an ** in-memory-only journal file (i.e. is one that was opened with a +ve -** nSpill parameter), and the underlying file has not yet been created, -** create it now. +** nSpill parameter or as SQLITE_OPEN_MAIN_JOURNAL), and the underlying +** file has not yet been created, create it now. */ -int sqlite3JournalCreate(sqlite3_file *p){ +int sqlite3JournalCreate(sqlite3_file *pJfd){ int rc = SQLITE_OK; - if( p->pMethods==&MemJournalMethods && ((MemJournal*)p)->nSpill>0 ){ - rc = memjrnlCreateFile((MemJournal*)p); + MemJournal *p = (MemJournal*)pJfd; + if( p->pMethod==&MemJournalMethods && ( +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + p->nSpill>0 +#else + /* While this appears to not be possible without ATOMIC_WRITE, the + ** paths are complex, so it seems prudent to leave the test in as + ** a NEVER(), in case our analysis is subtly flawed. */ + NEVER(p->nSpill>0) +#endif +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + || (p->flags & SQLITE_OPEN_MAIN_JOURNAL) +#endif + )){ + rc = memjrnlCreateFile(p); } return rc; } #endif Index: src/os.c ================================================================== --- src/os.c +++ src/os.c @@ -96,11 +96,11 @@ int sqlite3OsTruncate(sqlite3_file *id, i64 size){ return id->pMethods->xTruncate(id, size); } int sqlite3OsSync(sqlite3_file *id, int flags){ DO_OS_MALLOC_TEST(id); - return id->pMethods->xSync(id, flags); + return flags ? id->pMethods->xSync(id, flags) : SQLITE_OK; } int sqlite3OsFileSize(sqlite3_file *id, i64 *pSize){ DO_OS_MALLOC_TEST(id); return id->pMethods->xFileSize(id, pSize); } Index: src/os_unix.c ================================================================== --- src/os_unix.c +++ src/os_unix.c @@ -88,10 +88,11 @@ ** standard include files. */ #include #include #include +#include #include #include #include #include #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 @@ -208,11 +209,11 @@ int h; /* The file descriptor */ unsigned char eFileLock; /* The type of lock held on this fd */ unsigned short int ctrlFlags; /* Behavioral bits. UNIXFILE_* flags */ int lastErrno; /* The unix errno from last I/O error */ void *lockingContext; /* Locking style specific state */ - UnixUnusedFd *pUnused; /* Pre-allocated UnixUnusedFd */ + UnixUnusedFd *pPreallocatedUnused; /* Pre-allocated UnixUnusedFd */ const char *zPath; /* Name of the file */ unixShm *pShm; /* Shared memory segment information */ int szChunk; /* Configured by FCNTL_CHUNK_SIZE */ #if SQLITE_MAX_MMAP_SIZE>0 int nFetchOut; /* Number of outstanding xFetch refs */ @@ -219,14 +220,12 @@ sqlite3_int64 mmapSize; /* Usable size of mapping at pMapRegion */ sqlite3_int64 mmapSizeActual; /* Actual size of mapping at pMapRegion */ sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ void *pMapRegion; /* Memory mapped region */ #endif -#ifdef __QNXNTO__ int sectorSize; /* Device sector size */ int deviceCharacteristics; /* Precomputed device characteristics */ -#endif #if SQLITE_ENABLE_LOCKING_STYLE || defined(__APPLE__) int openFlags; /* The flags specified at open() */ #endif #if SQLITE_ENABLE_DATA_PROTECTION int protFlags; /* Data protection flags from unixOpen */ @@ -613,10 +612,24 @@ ** is the 32-bit version, even if _FILE_OFFSET_BITS=64 is defined. */ #ifdef __ANDROID__ # define lseek lseek64 #endif + +#ifdef __linux__ +/* +** Linux-specific IOCTL magic numbers used for controlling F2FS +*/ +#define F2FS_IOCTL_MAGIC 0xf5 +#define F2FS_IOC_START_ATOMIC_WRITE _IO(F2FS_IOCTL_MAGIC, 1) +#define F2FS_IOC_COMMIT_ATOMIC_WRITE _IO(F2FS_IOCTL_MAGIC, 2) +#define F2FS_IOC_START_VOLATILE_WRITE _IO(F2FS_IOCTL_MAGIC, 3) +#define F2FS_IOC_ABORT_VOLATILE_WRITE _IO(F2FS_IOCTL_MAGIC, 5) +#define F2FS_IOC_GET_FEATURES _IOR(F2FS_IOCTL_MAGIC, 12, u32) +#define F2FS_FEATURE_ATOMIC_WRITE 0x0004 +#endif /* __linux__ */ + /* ** Different Unix systems declare open() in different ways. Same use ** open(const char*,int,mode_t). Others use open(const char*,int,...). ** The difference is important when using a pointer to the function. @@ -785,10 +798,13 @@ { "lstat", (sqlite3_syscall_ptr)lstat, 0 }, #else { "lstat", (sqlite3_syscall_ptr)0, 0 }, #endif #define osLstat ((int(*)(const char*,struct stat*))aSyscall[27].pCurrent) + + { "ioctl", (sqlite3_syscall_ptr)ioctl, 0 }, +#define osIoctl ((int(*)(int,int,...))aSyscall[28].pCurrent) }; /* End of the overrideable system calls */ /* @@ -1390,11 +1406,12 @@ }; /* ** A lists of all unixInodeInfo objects. */ -static unixInodeInfo *inodeList = 0; +static unixInodeInfo *inodeList = 0; /* All unixInodeInfo objects */ +static unsigned int nUnusedFd = 0; /* Total unused file descriptors */ /* ** ** This function - unixLogErrorAtLine(), is only ever called via the macro ** unixLogError(). @@ -1511,10 +1528,11 @@ } #else robust_close(pFile, p->fd, __LINE__); sqlite3_free(p); #endif + nUnusedFd--; } pInode->pUnused = 0; } /* @@ -1543,10 +1561,11 @@ pInode->pNext->pPrev = pInode->pPrev; } sqlite3_free(pInode); } } + assert( inodeList!=0 || nUnusedFd==0 ); } /* ** Given a file descriptor, locate the unixInodeInfo object that ** describes that file descriptor. Create a new one if necessary. The @@ -1612,10 +1631,11 @@ #if OS_VXWORKS fileId.pId = pFile->pId; #else fileId.ino = (u64)statbuf.st_ino; #endif + assert( inodeList!=0 || nUnusedFd==0 ); pInode = inodeList; while( pInode && memcmp(&fileId, &pInode->fileId, sizeof(fileId)) ){ pInode = pInode->pNext; } if( pInode==0 ){ @@ -2049,15 +2069,16 @@ ** Add the file descriptor used by file handle pFile to the corresponding ** pUnused list. */ static void setPendingFd(unixFile *pFile){ unixInodeInfo *pInode = pFile->pInode; - UnixUnusedFd *p = pFile->pUnused; + UnixUnusedFd *p = pFile->pPreallocatedUnused; p->pNext = pInode->pUnused; pInode->pUnused = p; pFile->h = -1; - pFile->pUnused = 0; + pFile->pPreallocatedUnused = 0; + nUnusedFd++; } /* ** Lower the locking level on file descriptor pFile to eFileLock. eFileLock ** must be either NO_LOCK or SHARED_LOCK. @@ -2317,11 +2338,11 @@ pFile->zPath = 0; } #endif OSTRACE(("CLOSE %-3d\n", pFile->h)); OpenCounter(-1); - sqlite3_free(pFile->pUnused); + sqlite3_free(pFile->pPreallocatedUnused); memset(pFile, 0, sizeof(unixFile)); return SQLITE_OK; } /* @@ -2683,11 +2704,11 @@ } } OSTRACE(("TEST WR-LOCK %d %d %d (flock)\n", pFile->h, rc, reserved)); #ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS - if( (rc & SQLITE_IOERR) == SQLITE_IOERR ){ + if( (rc & 0xff) == SQLITE_IOERR ){ rc = SQLITE_OK; reserved=1; } #endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ *pResOut = reserved; @@ -2750,11 +2771,11 @@ pFile->eFileLock = eFileLock; } OSTRACE(("LOCK %d %s %s (flock)\n", pFile->h, azFileLock(eFileLock), rc==SQLITE_OK ? "ok" : "failed")); #ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS - if( (rc & SQLITE_IOERR) == SQLITE_IOERR ){ + if( (rc & 0xff) == SQLITE_IOERR ){ rc = SQLITE_BUSY; } #endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ return rc; } @@ -3287,11 +3308,11 @@ if( failed && (failed2 = afpSetLock(context->dbPath, pFile, SHARED_FIRST + pInode->sharedByte, 1, 1)) ){ /* Can't reestablish the shared lock. Sqlite can't deal, this is ** a critical I/O error */ - rc = ((failed & SQLITE_IOERR) == SQLITE_IOERR) ? failed2 : + rc = ((failed & 0xff) == SQLITE_IOERR) ? failed2 : SQLITE_IOERR_LOCK; goto afp_end_lock; } }else{ rc = failed; @@ -3567,11 +3588,11 @@ assert( amt>0 ); /* If this is a database file (not a journal, master-journal or temp ** file), the bytes in the locking range should never be read or written. */ #if 0 - assert( pFile->pUnused==0 + assert( pFile->pPreallocatedUnused==0 || offset>=PENDING_BYTE+512 || offset+amt<=PENDING_BYTE ); #endif @@ -3680,11 +3701,11 @@ assert( amt>0 ); /* If this is a database file (not a journal, master-journal or temp ** file), the bytes in the locking range should never be read or written. */ #if 0 - assert( pFile->pUnused==0 + assert( pFile->pPreallocatedUnused==0 || offset>=PENDING_BYTE+512 || offset+amt<=PENDING_BYTE ); #endif @@ -4486,10 +4507,25 @@ ** Information and control of an open file handle. */ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ unixFile *pFile = (unixFile*)id; switch( op ){ +#if defined(__linux__) && defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) + case SQLITE_FCNTL_BEGIN_ATOMIC_WRITE: { + int rc = osIoctl(pFile->h, F2FS_IOC_START_ATOMIC_WRITE); + return rc ? SQLITE_IOERR_BEGIN_ATOMIC : SQLITE_OK; + } + case SQLITE_FCNTL_COMMIT_ATOMIC_WRITE: { + int rc = osIoctl(pFile->h, F2FS_IOC_COMMIT_ATOMIC_WRITE); + return rc ? SQLITE_IOERR_COMMIT_ATOMIC : SQLITE_OK; + } + case SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE: { + int rc = osIoctl(pFile->h, F2FS_IOC_ABORT_VOLATILE_WRITE); + return rc ? SQLITE_IOERR_ROLLBACK_ATOMIC : SQLITE_OK; + } +#endif /* __linux__ && SQLITE_ENABLE_BATCH_ATOMIC_WRITE */ + case SQLITE_FCNTL_LOCKSTATE: { *(int*)pArg = pFile->eFileLock; return SQLITE_OK; } case SQLITE_FCNTL_LAST_ERRNO: { @@ -4536,10 +4572,18 @@ i64 newLimit = *(i64*)pArg; int rc = SQLITE_OK; if( newLimit>sqlite3GlobalConfig.mxMmap ){ newLimit = sqlite3GlobalConfig.mxMmap; } + + /* The value of newLimit may be eventually cast to (size_t) and passed + ** to mmap(). Restrict its value to 2GB if (size_t) is not at least a + ** 64-bit type. */ + if( newLimit>0 && sizeof(size_t)<8 ){ + newLimit = (newLimit & 0x7FFFFFFF); + } + *(i64*)pArg = pFile->mmapSizeMax; if( newLimit>=0 && newLimit!=pFile->mmapSizeMax && pFile->nFetchOut==0 ){ pFile->mmapSizeMax = newLimit; if( pFile->mmapSize>0 ){ unixUnmapfile(pFile); @@ -4589,34 +4633,45 @@ } return SQLITE_NOTFOUND; } /* -** Return the sector size in bytes of the underlying block device for -** the specified file. This is almost always 512 bytes, but may be -** larger for some devices. -** -** SQLite code assumes this function cannot fail. It also assumes that -** if two files are created in the same file-system directory (i.e. -** a database and its journal file) that the sector size will be the -** same for both. -*/ -#ifndef __QNXNTO__ -static int unixSectorSize(sqlite3_file *NotUsed){ - UNUSED_PARAMETER(NotUsed); - return SQLITE_DEFAULT_SECTOR_SIZE; -} -#endif - -/* -** The following version of unixSectorSize() is optimized for QNX. -*/ -#ifdef __QNXNTO__ +** If pFd->sectorSize is non-zero when this function is called, it is a +** no-op. Otherwise, the values of pFd->sectorSize and +** pFd->deviceCharacteristics are set according to the file-system +** characteristics. +** +** There are two versions of this function. One for QNX and one for all +** other systems. +*/ +#ifndef __QNXNTO__ +static void setDeviceCharacteristics(unixFile *pFd){ + assert( pFd->deviceCharacteristics==0 || pFd->sectorSize!=0 ); + if( pFd->sectorSize==0 ){ +#if defined(__linux__) && defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) + int res; + u32 f = 0; + + /* Check for support for F2FS atomic batch writes. */ + res = osIoctl(pFd->h, F2FS_IOC_GET_FEATURES, &f); + if( res==0 && (f & F2FS_FEATURE_ATOMIC_WRITE) ){ + pFd->deviceCharacteristics = SQLITE_IOCAP_BATCH_ATOMIC; + } +#endif /* __linux__ && SQLITE_ENABLE_BATCH_ATOMIC_WRITE */ + + /* Set the POWERSAFE_OVERWRITE flag if requested. */ + if( pFd->ctrlFlags & UNIXFILE_PSOW ){ + pFd->deviceCharacteristics |= SQLITE_IOCAP_POWERSAFE_OVERWRITE; + } + + pFd->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE; + } +} +#else #include #include -static int unixSectorSize(sqlite3_file *id){ - unixFile *pFile = (unixFile*)id; +static void setDeviceCharacteristics(unixFile *pFile){ if( pFile->sectorSize == 0 ){ struct statvfs fsInfo; /* Set defaults for non-supported filesystems */ pFile->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE; @@ -4681,13 +4736,28 @@ ** then it isn't valid.*/ if( pFile->sectorSize % 512 != 0 ){ pFile->deviceCharacteristics = 0; pFile->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE; } - return pFile->sectorSize; } -#endif /* __QNXNTO__ */ +#endif + +/* +** Return the sector size in bytes of the underlying block device for +** the specified file. This is almost always 512 bytes, but may be +** larger for some devices. +** +** SQLite code assumes this function cannot fail. It also assumes that +** if two files are created in the same file-system directory (i.e. +** a database and its journal file) that the sector size will be the +** same for both. +*/ +static int unixSectorSize(sqlite3_file *id){ + unixFile *pFd = (unixFile*)id; + setDeviceCharacteristics(pFd); + return pFd->sectorSize; +} /* ** Return the device characteristics for the file. ** ** This VFS is set up to return SQLITE_IOCAP_POWERSAFE_OVERWRITE by default. @@ -4699,20 +4769,13 @@ ** of required I/O for journaling, since a lot of padding is eliminated. ** Hence, while POWERSAFE_OVERWRITE is on by default, there is a file-control ** available to turn it off and URI query parameter available to turn it off. */ static int unixDeviceCharacteristics(sqlite3_file *id){ - unixFile *p = (unixFile*)id; - int rc = 0; -#ifdef __QNXNTO__ - if( p->sectorSize==0 ) unixSectorSize(id); - rc = p->deviceCharacteristics; -#endif - if( p->ctrlFlags & UNIXFILE_PSOW ){ - rc |= SQLITE_IOCAP_POWERSAFE_OVERWRITE; - } - return rc; + unixFile *pFd = (unixFile*)id; + setDeviceCharacteristics(pFd); + return pFd->deviceCharacteristics; } #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 /* @@ -6701,23 +6764,24 @@ ** but because no way to test it is currently available. It is better ** not to risk breaking vxworks support for the sake of such an obscure ** feature. */ #if !OS_VXWORKS struct stat sStat; /* Results of stat() call */ + + unixEnterMutex(); /* A stat() call may fail for various reasons. If this happens, it is ** almost certain that an open() call on the same path will also fail. ** For this reason, if an error occurs in the stat() call here, it is ** ignored and -1 is returned. The caller will try to open a new file ** descriptor on the same path, fail, and return an error to SQLite. ** ** Even if a subsequent open() call does succeed, the consequences of ** not searching for a reusable file descriptor are not dire. */ - if( 0==osStat(zPath, &sStat) ){ + if( nUnusedFd>0 && 0==osStat(zPath, &sStat) ){ unixInodeInfo *pInode; - unixEnterMutex(); pInode = inodeList; while( pInode && (pInode->fileId.dev!=sStat.st_dev || pInode->fileId.ino!=(u64)sStat.st_ino) ){ pInode = pInode->pNext; } @@ -6724,15 +6788,16 @@ if( pInode ){ UnixUnusedFd **pp; for(pp=&pInode->pUnused; *pp && (*pp)->flags!=flags; pp=&((*pp)->pNext)); pUnused = *pp; if( pUnused ){ + nUnusedFd--; *pp = pUnused->pNext; } } - unixLeaveMutex(); } + unixLeaveMutex(); #endif /* if !OS_VXWORKS */ return pUnused; } /* @@ -6953,11 +7018,11 @@ pUnused = sqlite3_malloc64(sizeof(*pUnused)); if( !pUnused ){ return SQLITE_NOMEM_BKPT; } } - p->pUnused = pUnused; + p->pPreallocatedUnused = pUnused; /* Database filenames are double-zero terminated if they are not ** URIs with parameters. Hence, they can always be passed into ** sqlite3_uri_parameter(). */ assert( (flags & SQLITE_OPEN_URI) || zName[strlen(zName)+1]==0 ); @@ -6994,11 +7059,11 @@ mode_t openMode; /* Permissions to create file with */ uid_t uid; /* Userid for the file */ gid_t gid; /* Groupid for the file */ rc = findCreateFileMode(zName, flags, &openMode, &uid, &gid); if( rc!=SQLITE_OK ){ - assert( !p->pUnused ); + assert( !p->pPreallocatedUnused ); assert( eType==SQLITE_OPEN_WAL || eType==SQLITE_OPEN_MAIN_JOURNAL ); return rc; } fd = robust_open(zName, openFlags, openMode); OSTRACE(("OPENX %-3d %s 0%o\n", fd, zName, openFlags)); @@ -7032,13 +7097,13 @@ assert( fd>=0 ); if( pOutFlags ){ *pOutFlags = flags; } - if( p->pUnused ){ - p->pUnused->fd = fd; - p->pUnused->flags = flags; + if( p->pPreallocatedUnused ){ + p->pPreallocatedUnused->fd = fd; + p->pPreallocatedUnused->flags = flags; } if( isDelete ){ #if OS_VXWORKS zPath = zName; @@ -7126,11 +7191,11 @@ rc = fillInUnixFile(pVfs, fd, pFile, zPath, ctrlFlags); open_finished: if( rc!=SQLITE_OK ){ - sqlite3_free(p->pUnused); + sqlite3_free(p->pPreallocatedUnused); } return rc; } @@ -7900,11 +7965,11 @@ memset(&dummyVfs, 0, sizeof(dummyVfs)); dummyVfs.pAppData = (void*)&autolockIoFinder; dummyVfs.zName = "dummy"; pUnused->fd = fd; pUnused->flags = openFlags; - pNew->pUnused = pUnused; + pNew->pPreallocatedUnused = pUnused; rc = fillInUnixFile(&dummyVfs, fd, (sqlite3_file*)pNew, path, 0); if( rc==SQLITE_OK ){ *ppFile = pNew; return SQLITE_OK; @@ -8875,11 +8940,11 @@ }; unsigned int i; /* Loop counter */ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==28 ); + assert( ArraySize(aSyscall)==29 ); /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ sqlite3_vfs_register(&aVfs[i], i==0); } Index: src/os_win.c ================================================================== --- src/os_win.c +++ src/os_win.c @@ -3566,10 +3566,18 @@ i64 newLimit = *(i64*)pArg; int rc = SQLITE_OK; if( newLimit>sqlite3GlobalConfig.mxMmap ){ newLimit = sqlite3GlobalConfig.mxMmap; } + + /* The value of newLimit may be eventually cast to (SIZE_T) and passed + ** to MapViewOfFile(). Restrict its value to 2GB if (SIZE_T) is not at + ** least a 64-bit type. */ + if( newLimit>0 && sizeof(SIZE_T)<8 ){ + newLimit = (newLimit & 0x7FFFFFFF); + } + *(i64*)pArg = pFile->mmapSizeMax; if( newLimit>=0 && newLimit!=pFile->mmapSizeMax && pFile->nFetchOut==0 ){ pFile->mmapSizeMax = newLimit; if( pFile->mmapSize>0 ){ winUnmapfile(pFile); Index: src/pager.c ================================================================== --- src/pager.c +++ src/pager.c @@ -126,12 +126,12 @@ ** ** PAGERID() takes a pointer to a Pager struct as its argument. The ** associated file-descriptor is returned. FILEHANDLEID() takes an sqlite3_file ** struct as its argument. */ -#define PAGERID(p) ((int)(p->fd)) -#define FILEHANDLEID(fd) ((int)fd) +#define PAGERID(p) (SQLITE_PTR_TO_INT(p->fd)) +#define FILEHANDLEID(fd) (SQLITE_PTR_TO_INT(fd)) /* ** The Pager.eState variable stores the current 'state' of a pager. A ** pager may be in any one of the seven states shown in the following ** state diagram. @@ -614,22 +614,33 @@ ** ** The Pager.errCode variable is only ever used in PAGER_ERROR state. It ** is set to zero in all other states. In PAGER_ERROR state, Pager.errCode ** is always set to SQLITE_FULL, SQLITE_IOERR or one of the SQLITE_IOERR_XXX ** sub-codes. +** +** syncFlags, walSyncFlags +** +** syncFlags is either SQLITE_SYNC_NORMAL (0x02) or SQLITE_SYNC_FULL (0x03). +** syncFlags is used for rollback mode. walSyncFlags is used for WAL mode +** and contains the flags used to sync the checkpoint operations in the +** lower two bits, and sync flags used for transaction commits in the WAL +** file in bits 0x04 and 0x08. In other words, to get the correct sync flags +** for checkpoint operations, use (walSyncFlags&0x03) and to get the correct +** sync flags for transaction commit, use ((walSyncFlags>>2)&0x03). Note +** that with synchronous=NORMAL in WAL mode, transaction commit is not synced +** meaning that the 0x04 and 0x08 bits are both zero. */ struct Pager { sqlite3_vfs *pVfs; /* OS functions to use for IO */ u8 exclusiveMode; /* Boolean. True if locking_mode==EXCLUSIVE */ u8 journalMode; /* One of the PAGER_JOURNALMODE_* values */ u8 useJournal; /* Use a rollback journal on this file */ u8 noSync; /* Do not sync the journal if true */ u8 fullSync; /* Do extra syncs of the journal for robustness */ u8 extraSync; /* sync directory after journal delete */ - u8 ckptSyncFlags; /* SYNC_NORMAL or SYNC_FULL for checkpoint */ - u8 walSyncFlags; /* SYNC_NORMAL or SYNC_FULL for wal writes */ u8 syncFlags; /* SYNC_NORMAL or SYNC_FULL otherwise */ + u8 walSyncFlags; /* See description above */ u8 tempFile; /* zFilename is a temporary or immutable file */ u8 noLock; /* Do not lock (except in WAL mode) */ u8 readOnly; /* True for a read-only database */ u8 memDb; /* True to inhibit all file I/O */ @@ -945,10 +956,11 @@ assert( !pagerUseWal(pPager) ); assert( p->eLock>=EXCLUSIVE_LOCK ); assert( isOpen(p->jfd) || p->journalMode==PAGER_JOURNALMODE_OFF || p->journalMode==PAGER_JOURNALMODE_WAL + || (sqlite3OsDeviceCharacteristics(p->fd)&SQLITE_IOCAP_BATCH_ATOMIC) ); assert( pPager->dbOrigSize<=pPager->dbHintSize ); break; case PAGER_WRITER_FINISHED: @@ -956,10 +968,11 @@ assert( pPager->errCode==SQLITE_OK ); assert( !pagerUseWal(pPager) ); assert( isOpen(p->jfd) || p->journalMode==PAGER_JOURNALMODE_OFF || p->journalMode==PAGER_JOURNALMODE_WAL + || (sqlite3OsDeviceCharacteristics(p->fd)&SQLITE_IOCAP_BATCH_ATOMIC) ); break; case PAGER_ERROR: /* There must be at least one outstanding reference to the pager if @@ -1166,51 +1179,62 @@ } return rc; } /* -** This function determines whether or not the atomic-write optimization -** can be used with this pager. The optimization can be used if: +** This function determines whether or not the atomic-write or +** atomic-batch-write optimizations can be used with this pager. The +** atomic-write optimization can be used if: ** ** (a) the value returned by OsDeviceCharacteristics() indicates that ** a database page may be written atomically, and ** (b) the value returned by OsSectorSize() is less than or equal ** to the page size. ** -** The optimization is also always enabled for temporary files. It is -** an error to call this function if pPager is opened on an in-memory -** database. +** If it can be used, then the value returned is the size of the journal +** file when it contains rollback data for exactly one page. ** -** If the optimization cannot be used, 0 is returned. If it can be used, -** then the value returned is the size of the journal file when it -** contains rollback data for exactly one page. +** The atomic-batch-write optimization can be used if OsDeviceCharacteristics() +** returns a value with the SQLITE_IOCAP_BATCH_ATOMIC bit set. -1 is +** returned in this case. +** +** If neither optimization can be used, 0 is returned. */ -#ifdef SQLITE_ENABLE_ATOMIC_WRITE static int jrnlBufferSize(Pager *pPager){ assert( !MEMDB ); - if( !pPager->tempFile ){ - int dc; /* Device characteristics */ - int nSector; /* Sector size */ - int szPage; /* Page size */ - - assert( isOpen(pPager->fd) ); - dc = sqlite3OsDeviceCharacteristics(pPager->fd); - nSector = pPager->sectorSize; - szPage = pPager->pageSize; + +#if defined(SQLITE_ENABLE_ATOMIC_WRITE) \ + || defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) + int dc; /* Device characteristics */ + + assert( isOpen(pPager->fd) ); + dc = sqlite3OsDeviceCharacteristics(pPager->fd); +#endif + +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + if( dc&SQLITE_IOCAP_BATCH_ATOMIC ){ + return -1; + } +#endif + +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + { + int nSector = pPager->sectorSize; + int szPage = pPager->pageSize; assert(SQLITE_IOCAP_ATOMIC512==(512>>8)); assert(SQLITE_IOCAP_ATOMIC64K==(65536>>8)); if( 0==(dc&(SQLITE_IOCAP_ATOMIC|(szPage>>8)) || nSector>szPage) ){ return 0; } } return JOURNAL_HDR_SZ(pPager) + JOURNAL_PG_SZ(pPager); -} -#else -# define jrnlBufferSize(x) 0 #endif + + return 0; +} /* ** If SQLITE_CHECK_PAGES is defined then we do some sanity checking ** on the cache using a hash function. This is used for testing ** and debugging only. @@ -1289,10 +1313,11 @@ if( SQLITE_OK!=(rc = sqlite3OsFileSize(pJrnl, &szJ)) || szJ<16 || SQLITE_OK!=(rc = read32bits(pJrnl, szJ-16, &len)) || len>=nMaster + || len>szJ-16 || len==0 || SQLITE_OK!=(rc = read32bits(pJrnl, szJ-12, &cksum)) || SQLITE_OK!=(rc = sqlite3OsRead(pJrnl, aMagic, 8, szJ-8)) || memcmp(aMagic, aJournalMagic, 8) || SQLITE_OK!=(rc = sqlite3OsRead(pJrnl, zMaster, len, szJ-16-len)) @@ -2010,11 +2035,13 @@ if( pPager->eStateeLockjfd) || pPager->pInJournal==0 ); + assert( isOpen(pPager->jfd) || pPager->pInJournal==0 + || (sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_BATCH_ATOMIC) + ); if( isOpen(pPager->jfd) ){ assert( !pagerUseWal(pPager) ); /* Finalize the journal file. */ if( sqlite3JournalIsInMemory(pPager->jfd) ){ @@ -2973,44 +3000,44 @@ return rc; } /* -** Read the content for page pPg out of the database file and into +** Read the content for page pPg out of the database file (or out of +** the WAL if that is where the most recent copy if found) into ** pPg->pData. A shared lock or greater must be held on the database ** file before this function is called. ** ** If page 1 is read, then the value of Pager.dbFileVers[] is set to ** the value read from the database file. ** ** If an IO error occurs, then the IO error is returned to the caller. ** Otherwise, SQLITE_OK is returned. */ -static int readDbPage(PgHdr *pPg, u32 iFrame){ +static int readDbPage(PgHdr *pPg){ Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */ - Pgno pgno = pPg->pgno; /* Page number to read */ int rc = SQLITE_OK; /* Return code */ - int pgsz = pPager->pageSize; /* Number of bytes to read */ + u32 iFrame = 0; /* Frame of WAL containing pgno */ assert( pPager->eState>=PAGER_READER && !MEMDB ); assert( isOpen(pPager->fd) ); -#ifndef SQLITE_OMIT_WAL + if( pagerUseWal(pPager) ){ + rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame); + if( rc ) return rc; + } if( iFrame ){ - /* Try to pull the page from the write-ahead log. */ - rc = sqlite3WalReadFrame(pPager->pWal, iFrame, pgsz, pPg->pData); - }else -#endif - { - i64 iOffset = (pgno-1)*(i64)pPager->pageSize; - rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); + rc = sqlite3WalReadFrame(pPager->pWal, iFrame,pPager->pageSize,pPg->pData); + }else{ + i64 iOffset = (pPg->pgno-1)*(i64)pPager->pageSize; + rc = sqlite3OsRead(pPager->fd, pPg->pData, pPager->pageSize, iOffset); if( rc==SQLITE_IOERR_SHORT_READ ){ rc = SQLITE_OK; } } - if( pgno==1 ){ + if( pPg->pgno==1 ){ if( rc ){ /* If the read is unsuccessful, set the dbFileVers[] to something ** that will never be a valid file version. dbFileVers[] is a copy ** of bytes 24..39 of the database. Bytes 28..31 should always be ** zero or the size of the database in page. Bytes 32..35 and 35..39 @@ -3026,17 +3053,17 @@ }else{ u8 *dbFileVers = &((u8*)pPg->pData)[24]; memcpy(&pPager->dbFileVers, dbFileVers, sizeof(pPager->dbFileVers)); } } - CODEC1(pPager, pPg->pData, pgno, 3, rc = SQLITE_NOMEM_BKPT); + CODEC1(pPager, pPg->pData, pPg->pgno, 3, rc = SQLITE_NOMEM_BKPT); PAGER_INCR(sqlite3_pager_readdb_count); PAGER_INCR(pPager->nRead); - IOTRACE(("PGIN %p %d\n", pPager, pgno)); + IOTRACE(("PGIN %p %d\n", pPager, pPg->pgno)); PAGERTRACE(("FETCH %d page %d hash(%08x)\n", - PAGERID(pPager), pgno, pager_pagehash(pPg))); + PAGERID(pPager), pPg->pgno, pager_pagehash(pPg))); return rc; } /* @@ -3083,15 +3110,11 @@ pPg = sqlite3PagerLookup(pPager, iPg); if( pPg ){ if( sqlite3PcachePageRefcount(pPg)==1 ){ sqlite3PcacheDrop(pPg); }else{ - u32 iFrame = 0; - rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame); - if( rc==SQLITE_OK ){ - rc = readDbPage(pPg, iFrame); - } + rc = readDbPage(pPg); if( rc==SQLITE_OK ){ pPager->xReiniter(pPg); } sqlite3PagerUnrefNotNull(pPg); } @@ -3593,24 +3616,21 @@ pPager->fullSync = level>=PAGER_SYNCHRONOUS_FULL ?1:0; pPager->extraSync = level==PAGER_SYNCHRONOUS_EXTRA ?1:0; } if( pPager->noSync ){ pPager->syncFlags = 0; - pPager->ckptSyncFlags = 0; }else if( pgFlags & PAGER_FULLFSYNC ){ pPager->syncFlags = SQLITE_SYNC_FULL; - pPager->ckptSyncFlags = SQLITE_SYNC_FULL; - }else if( pgFlags & PAGER_CKPT_FULLFSYNC ){ - pPager->syncFlags = SQLITE_SYNC_NORMAL; - pPager->ckptSyncFlags = SQLITE_SYNC_FULL; }else{ pPager->syncFlags = SQLITE_SYNC_NORMAL; - pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL; } - pPager->walSyncFlags = pPager->syncFlags; + pPager->walSyncFlags = (pPager->syncFlags<<2); if( pPager->fullSync ){ - pPager->walSyncFlags |= WAL_SYNC_TRANSACTIONS; + pPager->walSyncFlags |= pPager->syncFlags; + } + if( (pgFlags & PAGER_CKPT_FULLFSYNC) && !pPager->noSync ){ + pPager->walSyncFlags |= (SQLITE_SYNC_FULL<<2); } if( pgFlags & PAGER_CACHESPILL ){ pPager->doNotSpill &= ~SPILLFLAG_OFF; }else{ pPager->doNotSpill |= SPILLFLAG_OFF; @@ -4109,11 +4129,11 @@ pagerFreeMapHdrs(pPager); /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL assert( db || pPager->pWal==0 ); - sqlite3WalClose(pPager->pWal, db, pPager->ckptSyncFlags, pPager->pageSize, + sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, pPager->pageSize, (db && (db->flags & SQLITE_NoCkptOnClose) ? 0 : pTmp) ); pPager->pWal = 0; #endif pager_reset(pPager); @@ -4577,10 +4597,17 @@ rc = subjournalPageIfRequired(pPg); if( rc==SQLITE_OK ){ rc = pagerWalFrames(pPager, pPg, 0, 0); } }else{ + +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + if( pPager->tempFile==0 ){ + rc = sqlite3JournalCreate(pPager->jfd); + if( rc!=SQLITE_OK ) return pager_error(pPager, rc); + } +#endif /* Sync the journal file if required. */ if( pPg->flags&PGHDR_NEED_SYNC || pPager->eState==PAGER_WRITER_CACHEMOD ){ @@ -4905,25 +4932,20 @@ pPager->changeCountDone = pPager->tempFile; pPager->memDb = (u8)memDb; pPager->readOnly = (u8)readOnly; assert( useJournal || pPager->tempFile ); pPager->noSync = pPager->tempFile; - pPager->fullSync = pPager->noSync ?0:1; - pPager->syncFlags = pPager->noSync ? 0 : SQLITE_SYNC_NORMAL; - pPager->ckptSyncFlags = pPager->syncFlags; if( pPager->noSync ){ assert( pPager->fullSync==0 ); assert( pPager->extraSync==0 ); assert( pPager->syncFlags==0 ); assert( pPager->walSyncFlags==0 ); - assert( pPager->ckptSyncFlags==0 ); }else{ pPager->fullSync = 1; pPager->extraSync = 0; pPager->syncFlags = SQLITE_SYNC_NORMAL; - pPager->walSyncFlags = SQLITE_SYNC_NORMAL | WAL_SYNC_TRANSACTIONS; - pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL; + pPager->walSyncFlags = SQLITE_SYNC_NORMAL | (SQLITE_SYNC_NORMAL<<2); } /* pPager->pFirst = 0; */ /* pPager->pFirstSynced = 0; */ /* pPager->pLast = 0; */ pPager->nExtra = (u16)nExtra; @@ -5347,11 +5369,12 @@ ** Except, in locking_mode=EXCLUSIVE when there is nothing to in ** the rollback journal, the unlock is not performed and there is ** nothing to rollback, so this routine is a no-op. */ static void pagerUnlockIfUnused(Pager *pPager){ - if( pPager->nMmapOut==0 && (sqlite3PcacheRefCount(pPager->pPCache)==0) ){ + if( sqlite3PcacheRefCount(pPager->pPCache)==0 ){ + assert( pPager->nMmapOut==0 ); /* because page1 is never memory mapped */ pagerUnlockAndRollback(pPager); } } /* @@ -5488,18 +5511,13 @@ sqlite3EndBenignMalloc(); } memset(pPg->pData, 0, pPager->pageSize); IOTRACE(("ZERO %p %d\n", pPager, pgno)); }else{ - u32 iFrame = 0; /* Frame to read from WAL file */ - if( pagerUseWal(pPager) ){ - rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); - if( rc!=SQLITE_OK ) goto pager_acquire_err; - } assert( pPg->pPager==pPager ); pPager->aStat[PAGER_STAT_MISS]++; - rc = readDbPage(pPg, iFrame); + rc = readDbPage(pPg); if( rc!=SQLITE_OK ){ goto pager_acquire_err; } } pager_set_pagehash(pPg); @@ -5638,28 +5656,42 @@ } /* ** Release a page reference. ** -** If the number of references to the page drop to zero, then the -** page is added to the LRU list. When all references to all pages -** are released, a rollback occurs and the lock on the database is -** removed. +** The sqlite3PagerUnref() and sqlite3PagerUnrefNotNull() may only be +** used if we know that the page being released is not the last page. +** The btree layer always holds page1 open until the end, so these first +** to routines can be used to release any page other than BtShared.pPage1. +** +** Use sqlite3PagerUnrefPageOne() to release page1. This latter routine +** checks the total number of outstanding pages and if the number of +** pages reaches zero it drops the database lock. */ void sqlite3PagerUnrefNotNull(DbPage *pPg){ - Pager *pPager; + TESTONLY( Pager *pPager = pPg->pPager; ) assert( pPg!=0 ); - pPager = pPg->pPager; if( pPg->flags & PGHDR_MMAP ){ + assert( pPg->pgno!=1 ); /* Page1 is never memory mapped */ pagerReleaseMapPage(pPg); }else{ sqlite3PcacheRelease(pPg); } - pagerUnlockIfUnused(pPager); + /* Do not use this routine to release the last reference to page1 */ + assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); } void sqlite3PagerUnref(DbPage *pPg){ if( pPg ) sqlite3PagerUnrefNotNull(pPg); +} +void sqlite3PagerUnrefPageOne(DbPage *pPg){ + Pager *pPager; + assert( pPg!=0 ); + assert( pPg->pgno==1 ); + assert( (pPg->flags & PGHDR_MMAP)==0 ); /* Page1 is never memory mapped */ + pPager = pPg->pPager; + sqlite3PcacheRelease(pPg); + pagerUnlockIfUnused(pPager); } /* ** This function is called at the start of every write transaction. ** There must already be a RESERVED or EXCLUSIVE lock on the database @@ -6371,10 +6403,25 @@ sqlite3PagerUnref(pPageOne); if( rc==SQLITE_OK ){ sqlite3PcacheCleanAll(pPager->pPCache); } }else{ + /* The bBatch boolean is true if the batch-atomic-write commit method + ** should be used. No rollback journal is created if batch-atomic-write + ** is enabled. + */ + sqlite3_file *fd = pPager->fd; +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + const int bBatch = zMaster==0 /* An SQLITE_IOCAP_BATCH_ATOMIC commit */ + && (sqlite3OsDeviceCharacteristics(fd) & SQLITE_IOCAP_BATCH_ATOMIC) + && !pPager->noSync + && sqlite3JournalIsInMemory(pPager->jfd); +#else +# define bBatch 0 +#endif + +#ifdef SQLITE_ENABLE_ATOMIC_WRITE /* The following block updates the change-counter. Exactly how it ** does this depends on whether or not the atomic-update optimization ** was enabled at compile time, and if this transaction meets the ** runtime criteria to use the operation: ** @@ -6394,37 +6441,44 @@ ** Otherwise, if the optimization is both enabled and applicable, ** then call pager_incr_changecounter() to update the change-counter ** in 'direct' mode. In this case the journal file will never be ** created for this transaction. */ - #ifdef SQLITE_ENABLE_ATOMIC_WRITE - PgHdr *pPg; - assert( isOpen(pPager->jfd) - || pPager->journalMode==PAGER_JOURNALMODE_OFF - || pPager->journalMode==PAGER_JOURNALMODE_WAL - ); - if( !zMaster && isOpen(pPager->jfd) - && pPager->journalOff==jrnlBufferSize(pPager) - && pPager->dbSize>=pPager->dbOrigSize - && (0==(pPg = sqlite3PcacheDirtyList(pPager->pPCache)) || 0==pPg->pDirty) - ){ - /* Update the db file change counter via the direct-write method. The - ** following call will modify the in-memory representation of page 1 - ** to include the updated change counter and then write page 1 - ** directly to the database file. Because of the atomic-write - ** property of the host file-system, this is safe. - */ - rc = pager_incr_changecounter(pPager, 1); - }else{ + if( bBatch==0 ){ + PgHdr *pPg; + assert( isOpen(pPager->jfd) + || pPager->journalMode==PAGER_JOURNALMODE_OFF + || pPager->journalMode==PAGER_JOURNALMODE_WAL + ); + if( !zMaster && isOpen(pPager->jfd) + && pPager->journalOff==jrnlBufferSize(pPager) + && pPager->dbSize>=pPager->dbOrigSize + && (!(pPg = sqlite3PcacheDirtyList(pPager->pPCache)) || 0==pPg->pDirty) + ){ + /* Update the db file change counter via the direct-write method. The + ** following call will modify the in-memory representation of page 1 + ** to include the updated change counter and then write page 1 + ** directly to the database file. Because of the atomic-write + ** property of the host file-system, this is safe. + */ + rc = pager_incr_changecounter(pPager, 1); + }else{ + rc = sqlite3JournalCreate(pPager->jfd); + if( rc==SQLITE_OK ){ + rc = pager_incr_changecounter(pPager, 0); + } + } + } +#else +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + if( zMaster ){ rc = sqlite3JournalCreate(pPager->jfd); - if( rc==SQLITE_OK ){ - rc = pager_incr_changecounter(pPager, 0); - } + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; } - #else +#endif rc = pager_incr_changecounter(pPager, 0); - #endif +#endif if( rc!=SQLITE_OK ) goto commit_phase_one_exit; /* Write the master journal name into the journal file. If a master ** journal file name has already been written to the journal file, ** or if zMaster is NULL (no master journal), then this call is a no-op. @@ -6443,12 +6497,28 @@ ** not the case. In this case it is likely enough that the redundant ** xSync() call will be changed to a no-op by the OS anyhow. */ rc = syncJournal(pPager, 0); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - + + if( bBatch ){ + /* The pager is now in DBMOD state. But regardless of what happens + ** next, attempting to play the journal back into the database would + ** be unsafe. Close it now to make sure that does not happen. */ + sqlite3OsClose(pPager->jfd); + rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_BEGIN_ATOMIC_WRITE, 0); + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + } rc = pager_write_pagelist(pPager,sqlite3PcacheDirtyList(pPager->pPCache)); + if( bBatch ){ + if( rc==SQLITE_OK ){ + rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_COMMIT_ATOMIC_WRITE, 0); + }else{ + sqlite3OsFileControl(fd, SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE, 0); + } + } + if( rc!=SQLITE_OK ){ assert( rc!=SQLITE_IOERR_BLOCKED ); goto commit_phase_one_exit; } sqlite3PcacheCleanAll(pPager->pPCache); @@ -7357,11 +7427,11 @@ int rc = SQLITE_OK; if( pPager->pWal ){ rc = sqlite3WalCheckpoint(pPager->pWal, db, eMode, (eMode==SQLITE_CHECKPOINT_PASSIVE ? 0 : pPager->xBusyHandler), pPager->pBusyHandlerArg, - pPager->ckptSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace, + pPager->walSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace, pnLog, pnCkpt ); } return rc; } @@ -7518,11 +7588,11 @@ ** the database file, the log and log-summary files will be deleted. */ if( rc==SQLITE_OK && pPager->pWal ){ rc = pagerExclusiveLock(pPager); if( rc==SQLITE_OK ){ - rc = sqlite3WalClose(pPager->pWal, db, pPager->ckptSyncFlags, + rc = sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, pPager->pageSize, (u8*)pPager->pTmpSpace); pPager->pWal = 0; pagerFixMaplimit(pPager); /* Ensure that the WAL file is deleted even if the PERSIST_WAL Index: src/pager.h ================================================================== --- src/pager.h +++ src/pager.h @@ -149,10 +149,11 @@ int sqlite3PagerGet(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag); DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno); void sqlite3PagerRef(DbPage*); void sqlite3PagerUnref(DbPage*); void sqlite3PagerUnrefNotNull(DbPage*); +void sqlite3PagerUnrefPageOne(DbPage*); /* Operations on page references. */ int sqlite3PagerWrite(DbPage*); void sqlite3PagerDontWrite(DbPage*); int sqlite3PagerMovepage(Pager*,DbPage*,Pgno,int); Index: src/parse.y ================================================================== --- src/parse.y +++ src/parse.y @@ -189,10 +189,23 @@ } } columnlist ::= columnlist COMMA columnname carglist. columnlist ::= columnname carglist. columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,&A,&Y);} + +// Declare some tokens early in order to influence their values, to +// improve performance and reduce the executable size. The goal here is +// to get the "jump" operations in ISNULL through ESCAPE to have numeric +// values that are early enough so that all jump operations are clustered +// at the beginning, but also so that the comparison tokens NE through GE +// are as large as possible so that they are near to FUNCTION, which is a +// token synthesized by addopcodes.tcl. +// +%token ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST. +%token CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL. +%token OR AND NOT IS MATCH LIKE_KW BETWEEN IN ISNULL NOTNULL NE EQ. +%token GT LE LT GE ESCAPE. // The following directive causes tokens ABORT, AFTER, ASC, etc. to // fallback to ID if they will not parse as their original value. // This obviates the need for the "id" nonterminal. // Index: src/pcache.c ================================================================== --- src/pcache.c +++ src/pcache.c @@ -189,16 +189,13 @@ if( p->pDirty==0 ){ /*OPTIMIZATION-IF-TRUE*/ assert( p->bPurgeable==0 || p->eCreate==1 ); p->eCreate = 2; } } - pPage->pDirtyNext = 0; - pPage->pDirtyPrev = 0; } if( addRemove & PCACHE_DIRTYLIST_ADD ){ - assert( pPage->pDirtyNext==0 && pPage->pDirtyPrev==0 && p->pDirty!=pPage ); - + pPage->pDirtyPrev = 0; pPage->pDirtyNext = p->pDirty; if( pPage->pDirtyNext ){ assert( pPage->pDirtyNext->pDirtyPrev==0 ); pPage->pDirtyNext->pDirtyPrev = pPage; }else{ @@ -511,15 +508,11 @@ assert( p->nRef>0 ); p->pCache->nRefSum--; if( (--p->nRef)==0 ){ if( p->flags&PGHDR_CLEAN ){ pcacheUnpin(p); - }else if( p->pDirtyPrev!=0 ){ /*OPTIMIZATION-IF-FALSE*/ - /* Move the page to the head of the dirty list. If p->pDirtyPrev==0, - ** then page p is already at the head of the dirty list and the - ** following call would be a no-op. Hence the OPTIMIZATION-IF-FALSE - ** tag above. */ + }else{ pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT); } } } Index: src/pcache.h ================================================================== --- src/pcache.h +++ src/pcache.h @@ -41,10 +41,12 @@ ** pCache is grouped with the public elements for efficiency. */ i16 nRef; /* Number of users of this page */ PgHdr *pDirtyNext; /* Next element in list of dirty pages */ PgHdr *pDirtyPrev; /* Previous element in list of dirty pages */ + /* NB: pDirtyNext and pDirtyPrev are undefined if the + ** PgHdr object is not dirty */ }; /* Bit values for PgHdr.flags */ #define PGHDR_CLEAN 0x001 /* Page not on the PCache.pDirty list */ #define PGHDR_DIRTY 0x002 /* Page is on the PCache.pDirty list */ Index: src/pcache1.c ================================================================== --- src/pcache1.c +++ src/pcache1.c @@ -94,19 +94,24 @@ ** in memory. */ struct PgHdr1 { sqlite3_pcache_page page; /* Base class. Must be first. pBuf & pExtra */ unsigned int iKey; /* Key value (page number) */ - u8 isPinned; /* Page in use, not on the LRU list */ u8 isBulkLocal; /* This page from bulk local storage */ u8 isAnchor; /* This is the PGroup.lru element */ PgHdr1 *pNext; /* Next in hash table chain */ PCache1 *pCache; /* Cache that currently owns this page */ PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ }; +/* +** A page is pinned if it is no on the LRU list +*/ +#define PAGE_IS_PINNED(p) ((p)->pLruNext==0) +#define PAGE_IS_UNPINNED(p) ((p)->pLruNext!=0) + /* Each page cache (or PCache) belongs to a PGroup. A PGroup is a set ** of one or more PCaches that are able to recycle each other's unpinned ** pages when they are under memory pressure. A PGroup is an instance of ** the following object. ** @@ -130,11 +135,11 @@ struct PGroup { sqlite3_mutex *mutex; /* MUTEX_STATIC_LRU or NULL */ unsigned int nMaxPage; /* Sum of nMax for purgeable caches */ unsigned int nMinPage; /* Sum of nMin for purgeable caches */ unsigned int mxPinned; /* nMaxpage + 10 - nMinPage */ - unsigned int nCurrentPage; /* Number of purgeable pages allocated */ + unsigned int nPurgeable; /* Number of purgeable pages allocated */ PgHdr1 lru; /* The beginning and end of the LRU list */ }; /* Each page cache is an instance of the following object. Every ** open database file (including each in-memory database and each @@ -144,15 +149,17 @@ ** Pointers to structures of this type are cast and returned as ** opaque sqlite3_pcache* handles. */ struct PCache1 { /* Cache configuration parameters. Page size (szPage) and the purgeable - ** flag (bPurgeable) are set when the cache is created. nMax may be + ** flag (bPurgeable) and the pnPurgeable pointer are all set when the + ** cache is created and are never changed thereafter. nMax may be ** modified at any time by a call to the pcache1Cachesize() method. ** The PGroup mutex must be held when accessing nMax. */ PGroup *pGroup; /* PGroup this cache belongs to */ + unsigned int *pnPurgeable; /* Pointer to pGroup->nPurgeable */ int szPage; /* Size of database content section */ int szExtra; /* sizeof(MemPage)+sizeof(PgHdr) */ int szAlloc; /* Total size of one pcache line */ int bPurgeable; /* True if cache is purgeable */ unsigned int nMin; /* Minimum number of pages reserved */ @@ -243,10 +250,11 @@ */ void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ if( pcache1.isInit ){ PgFreeslot *p; if( pBuf==0 ) sz = n = 0; + if( n==0 ) sz = 0; sz = ROUNDDOWN8(sz); pcache1.szSlot = sz; pcache1.nSlot = pcache1.nFreeSlot = n; pcache1.nReserve = n>90 ? 10 : (n/10 + 1); pcache1.pStart = pBuf; @@ -434,13 +442,11 @@ p->page.pBuf = pPg; p->page.pExtra = &p[1]; p->isBulkLocal = 0; p->isAnchor = 0; } - if( pCache->bPurgeable ){ - pCache->pGroup->nCurrentPage++; - } + (*pCache->pnPurgeable)++; return p; } /* ** Free a page object allocated by pcache1AllocPage(). @@ -457,13 +463,11 @@ pcache1Free(p->page.pBuf); #ifdef SQLITE_PCACHE_SEPARATE_HEADER sqlite3_free(p); #endif } - if( pCache->bPurgeable ){ - pCache->pGroup->nCurrentPage--; - } + (*pCache->pnPurgeable)--; } /* ** Malloc function used by SQLite to obtain space from the buffer configured ** using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no such buffer @@ -554,26 +558,22 @@ ** LRU list, then this function is a no-op. ** ** The PGroup mutex must be held when this function is called. */ static PgHdr1 *pcache1PinPage(PgHdr1 *pPage){ - PCache1 *pCache; - assert( pPage!=0 ); - assert( pPage->isPinned==0 ); - pCache = pPage->pCache; + assert( PAGE_IS_UNPINNED(pPage) ); assert( pPage->pLruNext ); assert( pPage->pLruPrev ); - assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); + assert( sqlite3_mutex_held(pPage->pCache->pGroup->mutex) ); pPage->pLruPrev->pLruNext = pPage->pLruNext; pPage->pLruNext->pLruPrev = pPage->pLruPrev; pPage->pLruNext = 0; pPage->pLruPrev = 0; - pPage->isPinned = 1; assert( pPage->isAnchor==0 ); - assert( pCache->pGroup->lru.isAnchor==1 ); - pCache->nRecyclable--; + assert( pPage->pCache->pGroup->lru.isAnchor==1 ); + pPage->pCache->nRecyclable--; return pPage; } /* @@ -603,15 +603,15 @@ */ static void pcache1EnforceMaxPage(PCache1 *pCache){ PGroup *pGroup = pCache->pGroup; PgHdr1 *p; assert( sqlite3_mutex_held(pGroup->mutex) ); - while( pGroup->nCurrentPage>pGroup->nMaxPage + while( pGroup->nPurgeable>pGroup->nMaxPage && (p=pGroup->lru.pLruPrev)->isAnchor==0 ){ assert( p->pCache->pGroup==pGroup ); - assert( p->isPinned==0 ); + assert( PAGE_IS_UNPINNED(p) ); pcache1PinPage(p); pcache1RemoveFromHash(p, 1); } if( pCache->nPage==0 && pCache->pBulk ){ sqlite3_free(pCache->pBulk); @@ -656,11 +656,11 @@ pp = &pCache->apHash[h]; while( (pPage = *pp)!=0 ){ if( pPage->iKey>=iLimit ){ pCache->nPage--; *pp = pPage->pNext; - if( !pPage->isPinned ) pcache1PinPage(pPage); + if( PAGE_IS_UNPINNED(pPage) ) pcache1PinPage(pPage); pcache1FreePage(pPage); }else{ pp = &pPage->pNext; TESTONLY( if( nPage>=0 ) nPage++; ) } @@ -774,10 +774,14 @@ pcache1ResizeHash(pCache); if( bPurgeable ){ pCache->nMin = 10; pGroup->nMinPage += pCache->nMin; pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; + pCache->pnPurgeable = &pGroup->nPurgeable; + }else{ + static unsigned int dummyCurrentPage; + pCache->pnPurgeable = &dummyCurrentPage; } pcache1LeaveMutex(pGroup); if( pCache->nHash==0 ){ pcache1Destroy((sqlite3_pcache*)pCache); pCache = 0; @@ -875,19 +879,19 @@ && !pGroup->lru.pLruPrev->isAnchor && ((pCache->nPage+1>=pCache->nMax) || pcache1UnderMemoryPressure(pCache)) ){ PCache1 *pOther; pPage = pGroup->lru.pLruPrev; - assert( pPage->isPinned==0 ); + assert( PAGE_IS_UNPINNED(pPage) ); pcache1RemoveFromHash(pPage, 0); pcache1PinPage(pPage); pOther = pPage->pCache; if( pOther->szAlloc != pCache->szAlloc ){ pcache1FreePage(pPage); pPage = 0; }else{ - pGroup->nCurrentPage -= (pOther->bPurgeable - pCache->bPurgeable); + pGroup->nPurgeable -= (pOther->bPurgeable - pCache->bPurgeable); } } /* Step 5. If a usable page buffer has still not been found, ** attempt to allocate a new one. @@ -902,11 +906,10 @@ pPage->iKey = iKey; pPage->pNext = pCache->apHash[h]; pPage->pCache = pCache; pPage->pLruPrev = 0; pPage->pLruNext = 0; - pPage->isPinned = 1; *(void **)pPage->page.pExtra = 0; pCache->apHash[h] = pPage; if( iKey>pCache->iMaxKey ){ pCache->iMaxKey = iKey; } @@ -988,11 +991,11 @@ /* Step 2: If the page was found in the hash table, then return it. ** If the page was not in the hash table and createFlag is 0, abort. ** Otherwise (page not in hash and createFlag!=0) continue with ** subsequent steps to try to create the page. */ if( pPage ){ - if( !pPage->isPinned ){ + if( PAGE_IS_UNPINNED(pPage) ){ return pcache1PinPage(pPage); }else{ return pPage; } }else if( createFlag ){ @@ -1063,22 +1066,21 @@ /* It is an error to call this function if the page is already ** part of the PGroup LRU list. */ assert( pPage->pLruPrev==0 && pPage->pLruNext==0 ); - assert( pPage->isPinned==1 ); + assert( PAGE_IS_PINNED(pPage) ); - if( reuseUnlikely || pGroup->nCurrentPage>pGroup->nMaxPage ){ + if( reuseUnlikely || pGroup->nPurgeable>pGroup->nMaxPage ){ pcache1RemoveFromHash(pPage, 1); }else{ /* Add the page to the PGroup LRU list. */ PgHdr1 **ppFirst = &pGroup->lru.pLruNext; pPage->pLruPrev = &pGroup->lru; (pPage->pLruNext = *ppFirst)->pLruPrev = pPage; *ppFirst = pPage; pCache->nRecyclable++; - pPage->isPinned = 0; } pcache1LeaveMutex(pCache->pGroup); } @@ -1218,11 +1220,11 @@ ){ nFree += pcache1MemSize(p->page.pBuf); #ifdef SQLITE_PCACHE_SEPARATE_HEADER nFree += sqlite3MemSize(p); #endif - assert( p->isPinned==0 ); + assert( PAGE_IS_UNPINNED(p) ); pcache1PinPage(p); pcache1RemoveFromHash(p, 1); } pcache1LeaveMutex(&pcache1.grp); } @@ -1242,14 +1244,14 @@ int *pnRecyclable /* OUT: Total number of pages available for recycling */ ){ PgHdr1 *p; int nRecyclable = 0; for(p=pcache1.grp.lru.pLruNext; p && !p->isAnchor; p=p->pLruNext){ - assert( p->isPinned==0 ); + assert( PAGE_IS_UNPINNED(p) ); nRecyclable++; } - *pnCurrent = pcache1.grp.nCurrentPage; + *pnCurrent = pcache1.grp.nPurgeable; *pnMax = (int)pcache1.grp.nMaxPage; *pnMin = (int)pcache1.grp.nMinPage; *pnRecyclable = nRecyclable; } #endif Index: src/pragma.c ================================================================== --- src/pragma.c +++ src/pragma.c @@ -1232,17 +1232,15 @@ FuncDef *p; pParse->nMem = 2; for(i=0; iu.pHash ){ sqlite3VdbeMultiLoad(v, 1, "si", p->zName, 1); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2); } } for(j=sqliteHashFirst(&db->aFunc); j; j=sqliteHashNext(j)){ p = (FuncDef*)sqliteHashData(j); sqlite3VdbeMultiLoad(v, 1, "si", p->zName, 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2); } } break; #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -1250,21 +1248,19 @@ HashElem *j; pParse->nMem = 1; for(j=sqliteHashFirst(&db->aModule); j; j=sqliteHashNext(j)){ Module *pMod = (Module*)sqliteHashData(j); sqlite3VdbeMultiLoad(v, 1, "s", pMod->zName); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); } } break; #endif /* SQLITE_OMIT_VIRTUALTABLE */ case PragTyp_PRAGMA_LIST: { int i; for(i=0; inCol-1, 3); if( pTab->pCheck && (db->flags & SQLITE_IgnoreChecks)==0 ){ ExprList *pCheck = sqlite3ExprListDup(db, pTab->pCheck, 0); if( db->mallocFailed==0 ){ int addrCkFault = sqlite3VdbeMakeLabel(v); int addrCkOk = sqlite3VdbeMakeLabel(v); char *zErr; int k; - pParse->iSelfTab = iDataCur; + pParse->iSelfTab = iDataCur + 1; sqlite3ExprCachePush(pParse); for(k=pCheck->nExpr-1; k>0; k--){ sqlite3ExprIfFalse(pParse, pCheck->a[k].pExpr, addrCkFault, 0); } sqlite3ExprIfTrue(pParse, pCheck->a[0].pExpr, addrCkOk, SQLITE_JUMPIFNULL); sqlite3VdbeResolveLabel(v, addrCkFault); + pParse->iSelfTab = 0; zErr = sqlite3MPrintf(db, "CHECK constraint failed in %s", pTab->zName); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); integrityCheckResultRow(v, 3); sqlite3VdbeResolveLabel(v, addrCkOk); @@ -2369,14 +2367,18 @@ UNUSED_PARAMETER(idxNum); UNUSED_PARAMETER(idxStr); pragmaVtabCursorClear(pCsr); j = (pTab->pName->mPragFlg & PragFlg_Result1)!=0 ? 0 : 1; for(i=0; iazArg) ); - pCsr->azArg[j] = sqlite3_mprintf("%s", sqlite3_value_text(argv[i])); - if( pCsr->azArg[j]==0 ){ - return SQLITE_NOMEM; + assert( pCsr->azArg[j]==0 ); + if( zText ){ + pCsr->azArg[j] = sqlite3_mprintf("%s", zText); + if( pCsr->azArg[j]==0 ){ + return SQLITE_NOMEM; + } } } sqlite3StrAccumInit(&acc, 0, 0, 0, pTab->db->aLimit[SQLITE_LIMIT_SQL_LENGTH]); sqlite3StrAccumAppendAll(&acc, "PRAGMA "); if( pCsr->azArg[1] ){ Index: src/prepare.c ================================================================== --- src/prepare.c +++ src/prepare.c @@ -86,11 +86,11 @@ db->init.orphanTrigger = 0; TESTONLY(rcp = ) sqlite3_prepare(db, argv[2], -1, &pStmt, 0); rc = db->errCode; assert( (rc&0xFF)==(rcp&0xFF) ); db->init.iDb = saved_iDb; - assert( saved_iDb==0 || (db->flags & SQLITE_Vacuum)!=0 ); + assert( saved_iDb==0 || (db->mDbFlags & DBFLAG_Vacuum)!=0 ); if( SQLITE_OK!=rc ){ if( db->init.orphanTrigger ){ assert( iDb==1 ); }else{ pData->rc = rc; @@ -150,20 +150,22 @@ assert( iDb>=0 && iDbnDb ); assert( db->aDb[iDb].pSchema ); assert( sqlite3_mutex_held(db->mutex) ); assert( iDb==1 || sqlite3BtreeHoldsMutex(db->aDb[iDb].pBt) ); + + db->init.busy = 1; /* Construct the in-memory representation schema tables (sqlite_master or ** sqlite_temp_master) by invoking the parser directly. The appropriate ** table name will be inserted automatically by the parser so we can just ** use the abbreviation "x" here. The parser will also automatically tag ** the schema table as read-only. */ azArg[0] = zMasterName = SCHEMA_TABLE(iDb); azArg[1] = "1"; azArg[2] = "CREATE TABLE x(type text,name text,tbl_name text," - "rootpage integer,sql text)"; + "rootpage int,sql text)"; azArg[3] = 0; initData.db = db; initData.iDb = iDb; initData.rc = SQLITE_OK; initData.pzErrMsg = pzErrMsg; @@ -175,14 +177,14 @@ /* Create a cursor to hold the database open */ pDb = &db->aDb[iDb]; if( pDb->pBt==0 ){ - if( !OMIT_TEMPDB && ALWAYS(iDb==1) ){ - DbSetProperty(db, 1, DB_SchemaLoaded); - } - return SQLITE_OK; + assert( iDb==1 ); + DbSetProperty(db, 1, DB_SchemaLoaded); + rc = SQLITE_OK; + goto error_out; } /* If there is not already a read-only (or read-write) transaction opened ** on the b-tree database, open one now. If a transaction is opened, it ** will be closed before this function returns. */ @@ -337,13 +339,17 @@ sqlite3BtreeCommit(pDb->pBt); } sqlite3BtreeLeave(pDb->pBt); error_out: - if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ - sqlite3OomFault(db); + if( rc ){ + if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ + sqlite3OomFault(db); + } + sqlite3ResetOneSchema(db, iDb); } + db->init.busy = 0; return rc; } /* ** Initialize all database files - the main database file, the file @@ -355,46 +361,33 @@ ** bit is set in the flags field of the Db structure. If the database ** file was of zero-length, then the DB_Empty flag is also set. */ int sqlite3Init(sqlite3 *db, char **pzErrMsg){ int i, rc; - int commit_internal = !(db->flags&SQLITE_InternChanges); + int commit_internal = !(db->mDbFlags&DBFLAG_SchemaChange); assert( sqlite3_mutex_held(db->mutex) ); assert( sqlite3BtreeHoldsMutex(db->aDb[0].pBt) ); assert( db->init.busy==0 ); - rc = SQLITE_OK; - db->init.busy = 1; ENC(db) = SCHEMA_ENC(db); - for(i=0; rc==SQLITE_OK && inDb; i++){ - if( DbHasProperty(db, i, DB_SchemaLoaded) || i==1 ) continue; - rc = sqlite3InitOne(db, i, pzErrMsg); - if( rc ){ - sqlite3ResetOneSchema(db, i); - } - } - - /* Once all the other databases have been initialized, load the schema - ** for the TEMP database. This is loaded last, as the TEMP database - ** schema may contain references to objects in other databases. - */ -#ifndef SQLITE_OMIT_TEMPDB - assert( db->nDb>1 ); - if( rc==SQLITE_OK && !DbHasProperty(db, 1, DB_SchemaLoaded) ){ - rc = sqlite3InitOne(db, 1, pzErrMsg); - if( rc ){ - sqlite3ResetOneSchema(db, 1); - } - } -#endif - - db->init.busy = 0; - if( rc==SQLITE_OK && commit_internal ){ + assert( db->nDb>0 ); + /* Do the main schema first */ + if( !DbHasProperty(db, 0, DB_SchemaLoaded) ){ + rc = sqlite3InitOne(db, 0, pzErrMsg); + if( rc ) return rc; + } + /* All other schemas after the main schema. The "temp" schema must be last */ + for(i=db->nDb-1; i>0; i--){ + if( !DbHasProperty(db, i, DB_SchemaLoaded) ){ + rc = sqlite3InitOne(db, i, pzErrMsg); + if( rc ) return rc; + } + } + if( commit_internal ){ sqlite3CommitInternalChanges(db); } - - return rc; + return SQLITE_OK; } /* ** This routine is a no-op if the database schema is already initialized. ** Otherwise, the schema is loaded. An error code is returned. @@ -495,20 +488,18 @@ /* ** Free all memory allocations in the pParse object */ void sqlite3ParserReset(Parse *pParse){ - if( pParse ){ - sqlite3 *db = pParse->db; - sqlite3DbFree(db, pParse->aLabel); - sqlite3ExprListDelete(db, pParse->pConstExpr); - if( db ){ - assert( db->lookaside.bDisable >= pParse->disableLookaside ); - db->lookaside.bDisable -= pParse->disableLookaside; - } - pParse->disableLookaside = 0; - } + sqlite3 *db = pParse->db; + sqlite3DbFree(db, pParse->aLabel); + sqlite3ExprListDelete(db, pParse->pConstExpr); + if( db ){ + assert( db->lookaside.bDisable >= pParse->disableLookaside ); + db->lookaside.bDisable -= pParse->disableLookaside; + } + pParse->disableLookaside = 0; } /* ** Compile the UTF-8 encoded SQL statement zSql into a statement handle. */ @@ -690,10 +681,11 @@ } sqlite3_mutex_enter(db->mutex); sqlite3BtreeEnterAll(db); rc = sqlite3Prepare(db, zSql, nBytes, prepFlags, pOld, ppStmt, pzTail); if( rc==SQLITE_SCHEMA ){ + sqlite3ResetOneSchema(db, -1); sqlite3_finalize(*ppStmt); rc = sqlite3Prepare(db, zSql, nBytes, prepFlags, pOld, ppStmt, pzTail); } sqlite3BtreeLeaveAll(db); sqlite3_mutex_leave(db->mutex); Index: src/printf.c ================================================================== --- src/printf.c +++ src/printf.c @@ -654,11 +654,11 @@ zExtra = bufpt; } if( precision>=0 ){ for(length=0; lengthzText : 0; i64 szNew = p->nChar; - assert( (p->zText==0 || p->zText==p->zBase)==!isMalloced(p) ); szNew += N + 1; if( szNew+p->nChar<=p->mxAlloc ){ /* Force exponential buffer size growth as long as it does not overflow, ** to avoid having to call this routine too often */ szNew += p->nChar; @@ -822,11 +821,10 @@ void sqlite3AppendChar(StrAccum *p, int N, char c){ testcase( p->nChar + (i64)N > 0x7fffffff ); if( p->nChar+(i64)N >= p->nAlloc && (N = sqlite3StrAccumEnlarge(p, N))<=0 ){ return; } - assert( (p->zText==p->zBase)==!isMalloced(p) ); while( (N--)>0 ) p->zText[p->nChar++] = c; } /* ** The StrAccum "p" is not large enough to accept N new bytes of z[]. @@ -840,11 +838,10 @@ N = sqlite3StrAccumEnlarge(p, N); if( N>0 ){ memcpy(&p->zText[p->nChar], z, N); p->nChar += N; } - assert( (p->zText==0 || p->zText==p->zBase)==!isMalloced(p) ); } /* ** Append N bytes of text from z to the StrAccum object. Increase the ** size of the memory allocation for StrAccum if necessary. @@ -875,23 +872,24 @@ ** Finish off a string by making sure it is zero-terminated. ** Return a pointer to the resulting string. Return a NULL ** pointer if any kind of error was encountered. */ static SQLITE_NOINLINE char *strAccumFinishRealloc(StrAccum *p){ + char *zText; assert( p->mxAlloc>0 && !isMalloced(p) ); - p->zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); - if( p->zText ){ - memcpy(p->zText, p->zBase, p->nChar+1); + zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); + if( zText ){ + memcpy(zText, p->zText, p->nChar+1); p->printfFlags |= SQLITE_PRINTF_MALLOCED; }else{ setStrAccumError(p, STRACCUM_NOMEM); } - return p->zText; + p->zText = zText; + return zText; } char *sqlite3StrAccumFinish(StrAccum *p){ if( p->zText ){ - assert( (p->zText==p->zBase)==!isMalloced(p) ); p->zText[p->nChar] = 0; if( p->mxAlloc>0 && !isMalloced(p) ){ return strAccumFinishRealloc(p); } } @@ -900,11 +898,10 @@ /* ** Reset an StrAccum string. Reclaim all malloced memory. */ void sqlite3StrAccumReset(StrAccum *p){ - assert( (p->zText==0 || p->zText==p->zBase)==!isMalloced(p) ); if( isMalloced(p) ){ sqlite3DbFree(p->db, p->zText); p->printfFlags &= ~SQLITE_PRINTF_MALLOCED; } p->zText = 0; @@ -923,15 +920,15 @@ ** n then no memory allocations ever occur. ** mx: Maximum number of bytes to accumulate. If mx==0 then no memory ** allocations will ever occur. */ void sqlite3StrAccumInit(StrAccum *p, sqlite3 *db, char *zBase, int n, int mx){ - p->zText = p->zBase = zBase; + p->zText = zBase; p->db = db; - p->nChar = 0; p->nAlloc = n; p->mxAlloc = mx; + p->nChar = 0; p->accError = 0; p->printfFlags = 0; } /* Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -560,15 +560,15 @@ sqlite3VdbeAddOp3(v, OP_Compare, regPrevKey, regBase, pSort->nOBSat); pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex); if( pParse->db->mallocFailed ) return; pOp->p2 = nKey + nData; pKI = pOp->p4.pKeyInfo; - memset(pKI->aSortOrder, 0, pKI->nField); /* Makes OP_Jump below testable */ + memset(pKI->aSortOrder, 0, pKI->nKeyField); /* Makes OP_Jump testable */ sqlite3VdbeChangeP4(v, -1, (char*)pKI, P4_KEYINFO); - testcase( pKI->nXField>2 ); + testcase( pKI->nAllField > pKI->nKeyField+2 ); pOp->p4.pKeyInfo = keyInfoFromExprList(pParse, pSort->pOrderBy, nOBSat, - pKI->nXField-1); + pKI->nAllField-pKI->nKeyField-1); addrJmp = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp3(v, OP_Jump, addrJmp+1, 0, addrJmp+1); VdbeCoverage(v); pSort->labelBkOut = sqlite3VdbeMakeLabel(v); pSort->regReturn = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); @@ -1033,12 +1033,12 @@ KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ int nExtra = (N+X)*(sizeof(CollSeq*)+1) - sizeof(CollSeq*); KeyInfo *p = sqlite3DbMallocRawNN(db, sizeof(KeyInfo) + nExtra); if( p ){ p->aSortOrder = (u8*)&p->aColl[N+X]; - p->nField = (u16)N; - p->nXField = (u16)X; + p->nKeyField = (u16)N; + p->nAllField = (u16)(N+X); p->enc = ENC(db); p->db = db; p->nRef = 1; memset(&p[1], 0, nExtra); }else{ @@ -1436,17 +1436,14 @@ if( pS ){ /* The "table" is actually a sub-select or a view in the FROM clause ** of the SELECT statement. Return the declaration type and origin ** data for the result-set column of the sub-select. */ - if( iCol>=0 && ALWAYS(iColpEList->nExpr) ){ + if( iCol>=0 && iColpEList->nExpr ){ /* If iCol is less than zero, then the expression requests the ** rowid of the sub-select or view. This expression is legal (see ** test case misc2.2.2) - it always evaluates to NULL. - ** - ** The ALWAYS() is because iCol>=pS->pEList->nExpr will have been - ** caught already by name resolution. */ NameContext sNC; Expr *p = pS->pEList->a[iCol].pExpr; sNC.pSrcList = pS->pSrc; sNC.pNext = pNC; @@ -1552,22 +1549,10 @@ sqlite3VdbeSetColName(v, i, COLNAME_DECLTYPE, zType, SQLITE_TRANSIENT); } #endif /* !defined(SQLITE_OMIT_DECLTYPE) */ } -/* -** Return the Table objecct in the SrcList that has cursor iCursor. -** Or return NULL if no such Table object exists in the SrcList. -*/ -static Table *tableWithCursor(SrcList *pList, int iCursor){ - int j; - for(j=0; jnSrc; j++){ - if( pList->a[j].iCursor==iCursor ) return pList->a[j].pTab; - } - return 0; -} - /* ** Compute the column names for a SELECT statement. ** ** The only guarantee that SQLite makes about column names is that if the @@ -1597,28 +1582,33 @@ ** then the result column name with the table name ** prefix, ex: TABLE.COLUMN. Otherwise use zSpan. */ static void generateColumnNames( Parse *pParse, /* Parser context */ - SrcList *pTabList, /* The FROM clause of the SELECT */ - ExprList *pEList /* Expressions defining the result set */ + Select *pSelect /* Generate column names for this SELECT statement */ ){ Vdbe *v = pParse->pVdbe; int i; Table *pTab; + SrcList *pTabList; + ExprList *pEList; sqlite3 *db = pParse->db; - int fullName; /* TABLE.COLUMN if no AS clause and is a direct table ref */ - int srcName; /* COLUMN or TABLE.COLUMN if no AS clause and is direct */ + int fullName; /* TABLE.COLUMN if no AS clause and is a direct table ref */ + int srcName; /* COLUMN or TABLE.COLUMN if no AS clause and is direct */ #ifndef SQLITE_OMIT_EXPLAIN /* If this is an EXPLAIN, skip this step */ if( pParse->explain ){ return; } #endif if( pParse->colNamesSet || db->mallocFailed ) return; + /* Column names are determined by the left-most term of a compound select */ + while( pSelect->pPrior ) pSelect = pSelect->pPrior; + pTabList = pSelect->pSrc; + pEList = pSelect->pEList; assert( v!=0 ); assert( pTabList!=0 ); pParse->colNamesSet = 1; fullName = (db->flags & SQLITE_FullColNames)!=0; srcName = (db->flags & SQLITE_ShortColNames)!=0 || fullName; @@ -1625,20 +1615,21 @@ sqlite3VdbeSetNumCols(v, pEList->nExpr); for(i=0; inExpr; i++){ Expr *p = pEList->a[i].pExpr; assert( p!=0 ); + assert( p->op!=TK_AGG_COLUMN ); /* Agg processing has not run yet */ + assert( p->op!=TK_COLUMN || p->pTab!=0 ); /* Covering indexes not yet coded */ if( pEList->a[i].zName ){ /* An AS clause always takes first priority */ char *zName = pEList->a[i].zName; sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_TRANSIENT); - }else if( srcName - && (p->op==TK_COLUMN || p->op==TK_AGG_COLUMN) - && (pTab = tableWithCursor(pTabList, p->iTable))!=0 - ){ + }else if( srcName && p->op==TK_COLUMN ){ char *zCol; int iCol = p->iColumn; + pTab = p->pTab; + assert( pTab!=0 ); if( iCol<0 ) iCol = pTab->iPKey; assert( iCol==-1 || (iCol>=0 && iColnCol) ); if( iCol<0 ){ zCol = "rowid"; }else{ @@ -1719,11 +1710,13 @@ Expr *pColExpr = sqlite3ExprSkipCollate(pEList->a[i].pExpr); while( pColExpr->op==TK_DOT ){ pColExpr = pColExpr->pRight; assert( pColExpr!=0 ); } - if( pColExpr->op==TK_COLUMN && pColExpr->pTab!=0 ){ + if( (pColExpr->op==TK_COLUMN || pColExpr->op==TK_AGG_COLUMN) + && pColExpr->pTab!=0 + ){ /* For columns use the column name name */ int iCol = pColExpr->iColumn; Table *pTab = pColExpr->pTab; if( iCol<0 ) iCol = pTab->iPKey; zName = iCol>=0 ? pTab->aCol[iCol].zName : "rowid"; @@ -2466,15 +2459,10 @@ */ assert( unionTab==dest.iSDParm || dest.eDest!=priorOp ); if( dest.eDest!=priorOp ){ int iCont, iBreak, iStart; assert( p->pEList ); - if( dest.eDest==SRT_Output ){ - Select *pFirst = p; - while( pFirst->pPrior ) pFirst = pFirst->pPrior; - generateColumnNames(pParse, pFirst->pSrc, pFirst->pEList); - } iBreak = sqlite3VdbeMakeLabel(v); iCont = sqlite3VdbeMakeLabel(v); computeLimitRegisters(pParse, p, iBreak); sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); VdbeCoverage(v); iStart = sqlite3VdbeCurrentAddr(v); @@ -2541,15 +2529,10 @@ /* Generate code to take the intersection of the two temporary ** tables. */ assert( p->pEList ); - if( dest.eDest==SRT_Output ){ - Select *pFirst = p; - while( pFirst->pPrior ) pFirst = pFirst->pPrior; - generateColumnNames(pParse, pFirst->pSrc, pFirst->pEList); - } iBreak = sqlite3VdbeMakeLabel(v); iCont = sqlite3VdbeMakeLabel(v); computeLimitRegisters(pParse, p, iBreak); sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak); VdbeCoverage(v); r1 = sqlite3GetTempReg(pParse); @@ -3153,18 +3136,10 @@ /* Jump to the this point in order to terminate the query. */ sqlite3VdbeResolveLabel(v, labelEnd); - /* Set the number of output columns - */ - if( pDest->eDest==SRT_Output ){ - Select *pFirst = pPrior; - while( pFirst->pPrior ) pFirst = pFirst->pPrior; - generateColumnNames(pParse, pFirst->pSrc, pFirst->pEList); - } - /* Reassembly the compound query so that it will be freed correctly ** by the calling function */ if( p->pPrior ){ sqlite3SelectDelete(db, p->pPrior); } @@ -3337,12 +3312,13 @@ ** and (2b) the outer query does not use subqueries other than the one ** FROM-clause subquery that is a candidate for flattening. (2b is ** due to ticket [2f7170d73bf9abf80] from 2015-02-09.) ** ** (3) The subquery is not the right operand of a LEFT JOIN -** or the subquery is not itself a join and the outer query is not -** an aggregate. +** or (a) the subquery is not itself a join and (b) the FROM clause +** of the subquery does not contain a virtual table and (c) the +** outer query is not an aggregate. ** ** (4) The subquery is not DISTINCT. ** ** (**) At one point restrictions (4) and (5) defined a subset of DISTINCT ** sub-queries that were excluded from this optimization. Restriction @@ -3454,11 +3430,10 @@ Select *pParent; /* Current UNION ALL term of the other query */ Select *pSub; /* The inner query or "subquery" */ Select *pSub1; /* Pointer to the rightmost select in sub-query */ SrcList *pSrc; /* The FROM clause of the outer query */ SrcList *pSubSrc; /* The FROM clause of the subquery */ - ExprList *pList; /* The result set of the outer query */ int iParent; /* VDBE cursor number of the pSub result set temp table */ int iNewParent = -1;/* Replacement table for iParent */ int isLeftJoin = 0; /* True if pSub is the right side of a LEFT JOIN */ int i; /* Loop counter */ Expr *pWhere; /* The WHERE clause */ @@ -3543,11 +3518,11 @@ ** ** See also tickets #306, #350, and #3300. */ if( (pSubitem->fg.jointype & JT_OUTER)!=0 ){ isLeftJoin = 1; - if( pSubSrc->nSrc>1 || isAgg ){ + if( pSubSrc->nSrc>1 || isAgg || IsVirtual(pSubSrc->a[0].pTab) ){ return 0; /* Restriction (3) */ } } #ifdef SQLITE_EXTRA_IFNULLROW else if( iFrom>0 && !isAgg ){ @@ -3779,18 +3754,10 @@ ** \_____________________ outer query ______________________________/ ** ** We look at every expression in the outer query and every place we see ** "a" we substitute "x*3" and every place we see "b" we substitute "y+10". */ - pList = pParent->pEList; - for(i=0; inExpr; i++){ - if( pList->a[i].zName==0 ){ - char *zName = sqlite3DbStrDup(db, pList->a[i].zSpan); - sqlite3Dequote(zName); - pList->a[i].zName = zName; - } - } if( pSub->pOrderBy ){ /* At this point, any non-zero iOrderByCol values indicate that the ** ORDER BY column expression is identical to the iOrderByCol'th ** expression returned by SELECT statement pSub. Since these values ** do not necessarily correspond to columns in SELECT statement pParent, @@ -5215,10 +5182,18 @@ if( sqlite3SelectTrace & 0x100 ){ SELECTTRACE(0x100,pParse,p, ("after name resolution:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif + + /* Get a pointer the VDBE under construction, allocating a new VDBE if one + ** does not already exist */ + v = sqlite3GetVdbe(pParse); + if( v==0 ) goto select_end; + if( pDest->eDest==SRT_Output ){ + generateColumnNames(pParse, p); + } /* Try to flatten subqueries in the FROM clause up into the main query */ #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) for(i=0; !p->pPrior && inSrc; i++){ @@ -5251,15 +5226,10 @@ sSort.pOrderBy = p->pOrderBy; } } #endif - /* Get a pointer the VDBE under construction, allocating a new VDBE if one - ** does not already exist */ - v = sqlite3GetVdbe(pParse); - if( v==0 ) goto select_end; - #ifndef SQLITE_OMIT_COMPOUND_SELECT /* Handle compound SELECT statements using the separate multiSelect() ** procedure. */ if( p->pPrior ){ @@ -6055,19 +6025,13 @@ ** successful coding of the SELECT. */ select_end: explainSetInteger(pParse->iSelectId, iRestoreSelectId); - /* Identify column names if results of the SELECT are to be output. - */ - if( rc==SQLITE_OK && pDest->eDest==SRT_Output ){ - generateColumnNames(pParse, pTabList, pEList); - } - sqlite3DbFree(db, sAggInfo.aCol); sqlite3DbFree(db, sAggInfo.aFunc); #if SELECTTRACE_ENABLED SELECTTRACE(1,pParse,p,("end processing\n")); pParse->nSelectIndent--; #endif return rc; } Index: src/shell.c ================================================================== --- src/shell.c +++ src/shell.c @@ -1497,11 +1497,11 @@ sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); } #ifdef _WIN32 -__declspec(dllexport) + #endif int sqlite3_shathree_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi @@ -1609,11 +1609,11 @@ sqlite3_result_int64(context, rc); } #ifdef _WIN32 -__declspec(dllexport) + #endif int sqlite3_fileio_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi @@ -2137,11 +2137,11 @@ #endif return rc; } #ifdef _WIN32 -__declspec(dllexport) + #endif int sqlite3_completion_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi @@ -2229,18 +2229,17 @@ }; /* ** These are the allowed shellFlgs values */ -#define SHFLG_Scratch 0x00000001 /* The --scratch option is used */ -#define SHFLG_Pagecache 0x00000002 /* The --pagecache option is used */ -#define SHFLG_Lookaside 0x00000004 /* Lookaside memory is used */ -#define SHFLG_Backslash 0x00000008 /* The --backslash option is used */ -#define SHFLG_PreserveRowid 0x00000010 /* .dump preserves rowid values */ -#define SHFLG_Newlines 0x00000020 /* .dump --newline flag */ -#define SHFLG_CountChanges 0x00000040 /* .changes setting */ -#define SHFLG_Echo 0x00000080 /* .echo or --echo setting */ +#define SHFLG_Pagecache 0x00000001 /* The --pagecache option is used */ +#define SHFLG_Lookaside 0x00000002 /* Lookaside memory is used */ +#define SHFLG_Backslash 0x00000004 /* The --backslash option is used */ +#define SHFLG_PreserveRowid 0x00000008 /* .dump preserves rowid values */ +#define SHFLG_Newlines 0x00000010 /* .dump --newline flag */ +#define SHFLG_CountChanges 0x00000020 /* .changes setting */ +#define SHFLG_Echo 0x00000040 /* .echo or --echo setting */ /* ** Macros for testing and setting shellFlgs */ #define ShellHasFlag(P,X) (((P)->shellFlgs & (X))!=0) @@ -2657,10 +2656,11 @@ int *aiType /* Column types */ ){ int i; ShellState *p = (ShellState*)pArg; + if( azArg==0 ) return 0; switch( p->cMode ){ case MODE_Line: { int w = 5; if( azArg==0 ) break; for(i=0; in ) appendText(p, "|", 0); for(i=0; izDestTable ){ free(p->zDestTable); p->zDestTable = 0; @@ -3253,22 +3254,14 @@ displayStatLine(pArg, "Number of Pcache Pages Used:", "%lld (max %lld) pages", SQLITE_STATUS_PAGECACHE_USED, bReset); } displayStatLine(pArg, "Number of Pcache Overflow Bytes:", "%lld (max %lld) bytes", SQLITE_STATUS_PAGECACHE_OVERFLOW, bReset); - if( pArg->shellFlgs & SHFLG_Scratch ){ - displayStatLine(pArg, "Number of Scratch Allocations Used:", - "%lld (max %lld)", SQLITE_STATUS_SCRATCH_USED, bReset); - } - displayStatLine(pArg, "Number of Scratch Overflow Bytes:", - "%lld (max %lld) bytes", SQLITE_STATUS_SCRATCH_OVERFLOW, bReset); displayStatLine(pArg, "Largest Allocation:", "%lld bytes", SQLITE_STATUS_MALLOC_SIZE, bReset); displayStatLine(pArg, "Largest Pcache Allocation:", "%lld bytes", SQLITE_STATUS_PAGECACHE_SIZE, bReset); - displayStatLine(pArg, "Largest Scratch Allocation:", - "%lld bytes", SQLITE_STATUS_SCRATCH_SIZE, bReset); #ifdef YYTRACKMAXSTACKDEPTH displayStatLine(pArg, "Deepest Parser Stack:", "%lld (max %lld)", SQLITE_STATUS_PARSER_STACK, bReset); #endif } @@ -3888,11 +3881,11 @@ const char *zType; const char *zSql; ShellState *p = (ShellState *)pArg; UNUSED_PARAMETER(azNotUsed); - if( nArg!=3 ) return 1; + if( nArg!=3 || azArg==0 ) return 0; zTable = azArg[0]; zType = azArg[1]; zSql = azArg[2]; if( strcmp(zTable, "sqlite_sequence")==0 ){ @@ -7285,11 +7278,10 @@ { "assert", SQLITE_TESTCTRL_ASSERT }, { "always", SQLITE_TESTCTRL_ALWAYS }, { "reserve", SQLITE_TESTCTRL_RESERVE }, { "optimizations", SQLITE_TESTCTRL_OPTIMIZATIONS }, { "iskeyword", SQLITE_TESTCTRL_ISKEYWORD }, - { "scratchmalloc", SQLITE_TESTCTRL_SCRATCHMALLOC }, { "byteorder", SQLITE_TESTCTRL_BYTEORDER }, { "never_corrupt", SQLITE_TESTCTRL_NEVER_CORRUPT }, { "imposter", SQLITE_TESTCTRL_IMPOSTER }, }; int testctrl = -1; @@ -7398,11 +7390,10 @@ break; case SQLITE_TESTCTRL_BITVEC_TEST: case SQLITE_TESTCTRL_FAULT_INSTALL: case SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS: - case SQLITE_TESTCTRL_SCRATCHMALLOC: default: utf8_printf(stderr, "Error: CLI support for testctrl %s not implemented\n", azArg[1]); break; @@ -7918,11 +7909,10 @@ #endif " -newline SEP set output row separator. Default: '\\n'\n" " -nullvalue TEXT set text string for NULL values. Default ''\n" " -pagecache SIZE N use N slots of SZ bytes each for page cache memory\n" " -quote set output mode to 'quote'\n" - " -scratch SIZE N use N slots of SZ bytes each for scratch memory\n" " -separator SEP set output column separator. Default: '|'\n" " -stats print memory stats before each finalize\n" " -version show SQLite version\n" " -vfs NAME use NAME as the default VFS\n" #ifdef SQLITE_ENABLE_VFSTRACE @@ -8021,11 +8011,11 @@ setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */ stdin_is_interactive = isatty(0); stdout_is_console = isatty(1); #if USE_SYSTEM_SQLITE+0!=1 - if( strcmp(sqlite3_sourceid(),SQLITE_SOURCE_ID)!=0 ){ + if( strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){ utf8_printf(stderr, "SQLite header and source version mismatch\n%s\n%s\n", sqlite3_sourceid(), SQLITE_SOURCE_ID); exit(1); } #endif @@ -8116,20 +8106,10 @@ if( szHeap>0x7fff0000 ) szHeap = 0x7fff0000; sqlite3_config(SQLITE_CONFIG_HEAP, malloc((int)szHeap), (int)szHeap, 64); #else (void)cmdline_option_value(argc, argv, ++i); #endif - }else if( strcmp(z,"-scratch")==0 ){ - int n, sz; - sz = (int)integerValue(cmdline_option_value(argc,argv,++i)); - if( sz>400000 ) sz = 400000; - if( sz<2500 ) sz = 2500; - n = (int)integerValue(cmdline_option_value(argc,argv,++i)); - if( n>10 ) n = 10; - if( n<1 ) n = 1; - sqlite3_config(SQLITE_CONFIG_SCRATCH, malloc(n*sz+1), sz, n); - data.shellFlgs |= SHFLG_Scratch; }else if( strcmp(z,"-pagecache")==0 ){ int n, sz; sz = (int)integerValue(cmdline_option_value(argc,argv,++i)); if( sz>70000 ) sz = 70000; if( sz<0 ) sz = 0; @@ -8269,12 +8249,10 @@ stdin_is_interactive = 1; }else if( strcmp(z,"-batch")==0 ){ stdin_is_interactive = 0; }else if( strcmp(z,"-heap")==0 ){ i++; - }else if( strcmp(z,"-scratch")==0 ){ - i+=2; }else if( strcmp(z,"-pagecache")==0 ){ i+=2; }else if( strcmp(z,"-lookaside")==0 ){ i+=2; }else if( strcmp(z,"-mmap")==0 ){ @@ -8373,11 +8351,11 @@ #elif HAVE_LINENOISE linenoiseSetCompletionCallback(linenoise_completion); #endif rc = process_input(&data, 0); if( zHistory ){ - shell_stifle_history(100); + shell_stifle_history(2000); shell_write_history(zHistory); free(zHistory); } }else{ rc = process_input(&data, stdin); @@ -8394,6 +8372,5 @@ for(i=0; ishellFlgs & (X))!=0) @@ -1297,10 +1296,11 @@ int *aiType /* Column types */ ){ int i; ShellState *p = (ShellState*)pArg; + if( azArg==0 ) return 0; switch( p->cMode ){ case MODE_Line: { int w = 5; if( azArg==0 ) break; for(i=0; in ) appendText(p, "|", 0); for(i=0; izDestTable ){ free(p->zDestTable); p->zDestTable = 0; @@ -1893,22 +1894,14 @@ displayStatLine(pArg, "Number of Pcache Pages Used:", "%lld (max %lld) pages", SQLITE_STATUS_PAGECACHE_USED, bReset); } displayStatLine(pArg, "Number of Pcache Overflow Bytes:", "%lld (max %lld) bytes", SQLITE_STATUS_PAGECACHE_OVERFLOW, bReset); - if( pArg->shellFlgs & SHFLG_Scratch ){ - displayStatLine(pArg, "Number of Scratch Allocations Used:", - "%lld (max %lld)", SQLITE_STATUS_SCRATCH_USED, bReset); - } - displayStatLine(pArg, "Number of Scratch Overflow Bytes:", - "%lld (max %lld) bytes", SQLITE_STATUS_SCRATCH_OVERFLOW, bReset); displayStatLine(pArg, "Largest Allocation:", "%lld bytes", SQLITE_STATUS_MALLOC_SIZE, bReset); displayStatLine(pArg, "Largest Pcache Allocation:", "%lld bytes", SQLITE_STATUS_PAGECACHE_SIZE, bReset); - displayStatLine(pArg, "Largest Scratch Allocation:", - "%lld bytes", SQLITE_STATUS_SCRATCH_SIZE, bReset); #ifdef YYTRACKMAXSTACKDEPTH displayStatLine(pArg, "Deepest Parser Stack:", "%lld (max %lld)", SQLITE_STATUS_PARSER_STACK, bReset); #endif } @@ -2528,11 +2521,11 @@ const char *zType; const char *zSql; ShellState *p = (ShellState *)pArg; UNUSED_PARAMETER(azNotUsed); - if( nArg!=3 ) return 1; + if( nArg!=3 || azArg==0 ) return 0; zTable = azArg[0]; zType = azArg[1]; zSql = azArg[2]; if( strcmp(zTable, "sqlite_sequence")==0 ){ @@ -5925,11 +5918,10 @@ { "assert", SQLITE_TESTCTRL_ASSERT }, { "always", SQLITE_TESTCTRL_ALWAYS }, { "reserve", SQLITE_TESTCTRL_RESERVE }, { "optimizations", SQLITE_TESTCTRL_OPTIMIZATIONS }, { "iskeyword", SQLITE_TESTCTRL_ISKEYWORD }, - { "scratchmalloc", SQLITE_TESTCTRL_SCRATCHMALLOC }, { "byteorder", SQLITE_TESTCTRL_BYTEORDER }, { "never_corrupt", SQLITE_TESTCTRL_NEVER_CORRUPT }, { "imposter", SQLITE_TESTCTRL_IMPOSTER }, }; int testctrl = -1; @@ -6038,11 +6030,10 @@ break; case SQLITE_TESTCTRL_BITVEC_TEST: case SQLITE_TESTCTRL_FAULT_INSTALL: case SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS: - case SQLITE_TESTCTRL_SCRATCHMALLOC: default: utf8_printf(stderr, "Error: CLI support for testctrl %s not implemented\n", azArg[1]); break; @@ -6558,11 +6549,10 @@ #endif " -newline SEP set output row separator. Default: '\\n'\n" " -nullvalue TEXT set text string for NULL values. Default ''\n" " -pagecache SIZE N use N slots of SZ bytes each for page cache memory\n" " -quote set output mode to 'quote'\n" - " -scratch SIZE N use N slots of SZ bytes each for scratch memory\n" " -separator SEP set output column separator. Default: '|'\n" " -stats print memory stats before each finalize\n" " -version show SQLite version\n" " -vfs NAME use NAME as the default VFS\n" #ifdef SQLITE_ENABLE_VFSTRACE @@ -6661,11 +6651,11 @@ setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */ stdin_is_interactive = isatty(0); stdout_is_console = isatty(1); #if USE_SYSTEM_SQLITE+0!=1 - if( strcmp(sqlite3_sourceid(),SQLITE_SOURCE_ID)!=0 ){ + if( strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){ utf8_printf(stderr, "SQLite header and source version mismatch\n%s\n%s\n", sqlite3_sourceid(), SQLITE_SOURCE_ID); exit(1); } #endif @@ -6756,20 +6746,10 @@ if( szHeap>0x7fff0000 ) szHeap = 0x7fff0000; sqlite3_config(SQLITE_CONFIG_HEAP, malloc((int)szHeap), (int)szHeap, 64); #else (void)cmdline_option_value(argc, argv, ++i); #endif - }else if( strcmp(z,"-scratch")==0 ){ - int n, sz; - sz = (int)integerValue(cmdline_option_value(argc,argv,++i)); - if( sz>400000 ) sz = 400000; - if( sz<2500 ) sz = 2500; - n = (int)integerValue(cmdline_option_value(argc,argv,++i)); - if( n>10 ) n = 10; - if( n<1 ) n = 1; - sqlite3_config(SQLITE_CONFIG_SCRATCH, malloc(n*sz+1), sz, n); - data.shellFlgs |= SHFLG_Scratch; }else if( strcmp(z,"-pagecache")==0 ){ int n, sz; sz = (int)integerValue(cmdline_option_value(argc,argv,++i)); if( sz>70000 ) sz = 70000; if( sz<0 ) sz = 0; @@ -6909,12 +6889,10 @@ stdin_is_interactive = 1; }else if( strcmp(z,"-batch")==0 ){ stdin_is_interactive = 0; }else if( strcmp(z,"-heap")==0 ){ i++; - }else if( strcmp(z,"-scratch")==0 ){ - i+=2; }else if( strcmp(z,"-pagecache")==0 ){ i+=2; }else if( strcmp(z,"-lookaside")==0 ){ i+=2; }else if( strcmp(z,"-mmap")==0 ){ @@ -7013,11 +6991,11 @@ #elif HAVE_LINENOISE linenoiseSetCompletionCallback(linenoise_completion); #endif rc = process_input(&data, 0); if( zHistory ){ - shell_stifle_history(100); + shell_stifle_history(2000); shell_write_history(zHistory); free(zHistory); } }else{ rc = process_input(&data, stdin); Index: src/sqlite.h.in ================================================================== --- src/sqlite.h.in +++ src/sqlite.h.in @@ -113,11 +113,13 @@ ** Fossil configuration management ** system. ^The SQLITE_SOURCE_ID macro evaluates to ** a string which identifies a particular check-in of SQLite ** within its configuration management system. ^The SQLITE_SOURCE_ID ** string contains the date and time of the check-in (UTC) and a SHA1 -** or SHA3-256 hash of the entire source tree. +** or SHA3-256 hash of the entire source tree. If the source code has +** been edited in any way since it was last checked in, then the last +** four hexadecimal digits of the hash may be modified. ** ** See also: [sqlite3_libversion()], ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ @@ -137,23 +139,25 @@ ** the header, and thus ensure that the application is ** compiled with matching library and header files. ** ** 
 ** assert( sqlite3_libversion_number()==SQLITE_VERSION_NUMBER );
-** assert( strcmp(sqlite3_sourceid(),SQLITE_SOURCE_ID)==0 );
+** assert( strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,80)==0 );
 ** assert( strcmp(sqlite3_libversion(),SQLITE_VERSION)==0 );
 **
)^ ** ** ^The sqlite3_version[] string constant contains the text of [SQLITE_VERSION] ** macro. ^The sqlite3_libversion() function returns a pointer to the ** to the sqlite3_version[] string constant. The sqlite3_libversion() ** function is provided for use in DLLs since DLL users usually do not have ** direct access to string constants within the DLL. ^The ** sqlite3_libversion_number() function returns an integer equal to -** [SQLITE_VERSION_NUMBER]. ^The sqlite3_sourceid() function returns +** [SQLITE_VERSION_NUMBER]. ^(The sqlite3_sourceid() function returns ** a pointer to a string constant whose value is the same as the -** [SQLITE_SOURCE_ID] C preprocessor macro. +** [SQLITE_SOURCE_ID] C preprocessor macro. Except if SQLite is built +** using an edited copy of [the amalgamation], then the last four characters +** of the hash might be different from [SQLITE_SOURCE_ID].)^ ** ** See also: [sqlite_version()] and [sqlite_source_id()]. */ SQLITE_EXTERN const char sqlite3_version[]; const char *sqlite3_libversion(void); @@ -430,11 +434,11 @@ #define SQLITE_CORRUPT 11 /* The database disk image is malformed */ #define SQLITE_NOTFOUND 12 /* Unknown opcode in sqlite3_file_control() */ #define SQLITE_FULL 13 /* Insertion failed because database is full */ #define SQLITE_CANTOPEN 14 /* Unable to open the database file */ #define SQLITE_PROTOCOL 15 /* Database lock protocol error */ -#define SQLITE_EMPTY 16 /* Not used */ +#define SQLITE_EMPTY 16 /* Internal use only */ #define SQLITE_SCHEMA 17 /* The database schema changed */ #define SQLITE_TOOBIG 18 /* String or BLOB exceeds size limit */ #define SQLITE_CONSTRAINT 19 /* Abort due to constraint violation */ #define SQLITE_MISMATCH 20 /* Data type mismatch */ #define SQLITE_MISUSE 21 /* Library used incorrectly */ @@ -492,10 +496,13 @@ #define SQLITE_IOERR_MMAP (SQLITE_IOERR | (24<<8)) #define SQLITE_IOERR_GETTEMPPATH (SQLITE_IOERR | (25<<8)) #define SQLITE_IOERR_CONVPATH (SQLITE_IOERR | (26<<8)) #define SQLITE_IOERR_VNODE (SQLITE_IOERR | (27<<8)) #define SQLITE_IOERR_AUTH (SQLITE_IOERR | (28<<8)) +#define SQLITE_IOERR_BEGIN_ATOMIC (SQLITE_IOERR | (29<<8)) +#define SQLITE_IOERR_COMMIT_ATOMIC (SQLITE_IOERR | (30<<8)) +#define SQLITE_IOERR_ROLLBACK_ATOMIC (SQLITE_IOERR | (31<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_BUSY_SNAPSHOT (SQLITE_BUSY | (2<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) #define SQLITE_CANTOPEN_ISDIR (SQLITE_CANTOPEN | (2<<8)) @@ -579,10 +586,15 @@ ** guaranteed to be unchanged. The SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN ** flag indicates that a file cannot be deleted when open. The ** SQLITE_IOCAP_IMMUTABLE flag indicates that the file is on ** read-only media and cannot be changed even by processes with ** elevated privileges. +** +** The SQLITE_IOCAP_BATCH_ATOMIC property means that the underlying +** filesystem supports doing multiple write operations atomically when those +** write operations are bracketed by [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE] and +** [SQLITE_FCNTL_COMMIT_ATOMIC_WRITE]. */ #define SQLITE_IOCAP_ATOMIC 0x00000001 #define SQLITE_IOCAP_ATOMIC512 0x00000002 #define SQLITE_IOCAP_ATOMIC1K 0x00000004 #define SQLITE_IOCAP_ATOMIC2K 0x00000008 @@ -594,10 +606,11 @@ #define SQLITE_IOCAP_SAFE_APPEND 0x00000200 #define SQLITE_IOCAP_SEQUENTIAL 0x00000400 #define SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN 0x00000800 #define SQLITE_IOCAP_POWERSAFE_OVERWRITE 0x00001000 #define SQLITE_IOCAP_IMMUTABLE 0x00002000 +#define SQLITE_IOCAP_BATCH_ATOMIC 0x00004000 /* ** CAPI3REF: File Locking Levels ** ** SQLite uses one of these integer values as the second @@ -728,10 +741,11 @@ **
  • [SQLITE_IOCAP_SAFE_APPEND] **
  • [SQLITE_IOCAP_SEQUENTIAL] **
  • [SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN] **
  • [SQLITE_IOCAP_POWERSAFE_OVERWRITE] **
  • [SQLITE_IOCAP_IMMUTABLE] +**
  • [SQLITE_IOCAP_BATCH_ATOMIC] ** ** ** The SQLITE_IOCAP_ATOMIC property means that all writes of ** any size are atomic. The SQLITE_IOCAP_ATOMICnnn values ** mean that writes of blocks that are nnn bytes in size and @@ -1011,10 +1025,44 @@ ** **
  • [[SQLITE_FCNTL_RBU]] ** The [SQLITE_FCNTL_RBU] opcode is implemented by the special VFS used by ** the RBU extension only. All other VFS should return SQLITE_NOTFOUND for ** this opcode. +** +**
  • [[SQLITE_FCNTL_BEGIN_ATOMIC_WRITE]] +** If the [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE] opcode returns SQLITE_OK, then +** the file descriptor is placed in "batch write mode", which +** means all subsequent write operations will be deferred and done +** atomically at the next [SQLITE_FCNTL_COMMIT_ATOMIC_WRITE]. Systems +** that do not support batch atomic writes will return SQLITE_NOTFOUND. +** ^Following a successful SQLITE_FCNTL_BEGIN_ATOMIC_WRITE and prior to +** the closing [SQLITE_FCNTL_COMMIT_ATOMIC_WRITE] or +** [SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE], SQLite will make +** no VFS interface calls on the same [sqlite3_file] file descriptor +** except for calls to the xWrite method and the xFileControl method +** with [SQLITE_FCNTL_SIZE_HINT]. +** +**
  • [[SQLITE_FCNTL_COMMIT_ATOMIC_WRITE]] +** The [SQLITE_FCNTL_COMMIT_ATOMIC_WRITE] opcode causes all write +** operations since the previous successful call to +** [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE] to be performed atomically. +** This file control returns [SQLITE_OK] if and only if the writes were +** all performed successfully and have been committed to persistent storage. +** ^Regardless of whether or not it is successful, this file control takes +** the file descriptor out of batch write mode so that all subsequent +** write operations are independent. +** ^SQLite will never invoke SQLITE_FCNTL_COMMIT_ATOMIC_WRITE without +** a prior successful call to [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE]. +** +**
  • [[SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE]] +** The [SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE] opcode causes all write +** operations since the previous successful call to +** [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE] to be rolled back. +** ^This file control takes the file descriptor out of batch write mode +** so that all subsequent write operations are independent. +** ^SQLite will never invoke SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE without +** a prior successful call to [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE]. ** */ #define SQLITE_FCNTL_LOCKSTATE 1 #define SQLITE_FCNTL_GET_LOCKPROXYFILE 2 #define SQLITE_FCNTL_SET_LOCKPROXYFILE 3 @@ -1042,10 +1090,13 @@ #define SQLITE_FCNTL_RBU 26 #define SQLITE_FCNTL_VFS_POINTER 27 #define SQLITE_FCNTL_JOURNAL_POINTER 28 #define SQLITE_FCNTL_WIN32_GET_HANDLE 29 #define SQLITE_FCNTL_PDB 30 +#define SQLITE_FCNTL_BEGIN_ATOMIC_WRITE 31 +#define SQLITE_FCNTL_COMMIT_ATOMIC_WRITE 32 +#define SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE 33 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE #define SQLITE_SET_LOCKPROXYFILE SQLITE_FCNTL_SET_LOCKPROXYFILE #define SQLITE_LAST_ERRNO SQLITE_FCNTL_LAST_ERRNO @@ -1611,10 +1662,20 @@ ** The [sqlite3_mem_methods] ** structure is filled with the currently defined memory allocation routines.)^ ** This option can be used to overload the default memory allocation ** routines with a wrapper that simulations memory allocation failure or ** tracks memory usage, for example. +** +** [[SQLITE_CONFIG_SMALL_MALLOC]]
    SQLITE_CONFIG_SMALL_MALLOC
    +**
    ^The SQLITE_CONFIG_SMALL_MALLOC option takes single argument of +** type int, interpreted as a boolean, which if true provides a hint to +** SQLite that it should avoid large memory allocations if possible. +** SQLite will run faster if it is free to make large memory allocations, +** but some application might prefer to run slower in exchange for +** guarantees about memory fragmentation that are possible if large +** allocations are avoided. This hint is normally off. +**
    ** ** [[SQLITE_CONFIG_MEMSTATUS]]
    SQLITE_CONFIG_MEMSTATUS
    **
    ^The SQLITE_CONFIG_MEMSTATUS option takes single argument of type int, ** interpreted as a boolean, which enables or disables the collection of ** memory allocation statistics. ^(When memory allocation statistics are @@ -1629,29 +1690,11 @@ ** compiled with [SQLITE_DEFAULT_MEMSTATUS]=0 in which case memory ** allocation statistics are disabled by default. **
    ** ** [[SQLITE_CONFIG_SCRATCH]]
    SQLITE_CONFIG_SCRATCH
    -**
    ^The SQLITE_CONFIG_SCRATCH option specifies a static memory buffer -** that SQLite can use for scratch memory. ^(There are three arguments -** to SQLITE_CONFIG_SCRATCH: A pointer an 8-byte -** aligned memory buffer from which the scratch allocations will be -** drawn, the size of each scratch allocation (sz), -** and the maximum number of scratch allocations (N).)^ -** The first argument must be a pointer to an 8-byte aligned buffer -** of at least sz*N bytes of memory. -** ^SQLite will not use more than one scratch buffers per thread. -** ^SQLite will never request a scratch buffer that is more than 6 -** times the database page size. -** ^If SQLite needs needs additional -** scratch memory beyond what is provided by this configuration option, then -** [sqlite3_malloc()] will be used to obtain the memory needed.

    -** ^When the application provides any amount of scratch memory using -** SQLITE_CONFIG_SCRATCH, SQLite avoids unnecessary large -** [sqlite3_malloc|heap allocations]. -** This can help [Robson proof|prevent memory allocation failures] due to heap -** fragmentation in low-memory embedded systems. +**

    The SQLITE_CONFIG_SCRATCH option is no longer used. **
    ** ** [[SQLITE_CONFIG_PAGECACHE]]
    SQLITE_CONFIG_PAGECACHE
    **
    ^The SQLITE_CONFIG_PAGECACHE option specifies a memory pool ** that SQLite can use for the database page cache with the default page @@ -1683,12 +1726,11 @@ ** additional cache line.
    ** ** [[SQLITE_CONFIG_HEAP]]
    SQLITE_CONFIG_HEAP
    **
    ^The SQLITE_CONFIG_HEAP option specifies a static memory buffer ** that SQLite will use for all of its dynamic memory allocation needs -** beyond those provided for by [SQLITE_CONFIG_SCRATCH] and -** [SQLITE_CONFIG_PAGECACHE]. +** beyond those provided for by [SQLITE_CONFIG_PAGECACHE]. ** ^The SQLITE_CONFIG_HEAP option is only available if SQLite is compiled ** with either [SQLITE_ENABLE_MEMSYS3] or [SQLITE_ENABLE_MEMSYS5] and returns ** [SQLITE_ERROR] if invoked otherwise. ** ^There are three arguments to SQLITE_CONFIG_HEAP: ** An 8-byte aligned pointer to the memory, @@ -1877,11 +1919,11 @@ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ #define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ #define SQLITE_CONFIG_SERIALIZED 3 /* nil */ #define SQLITE_CONFIG_MALLOC 4 /* sqlite3_mem_methods* */ #define SQLITE_CONFIG_GETMALLOC 5 /* sqlite3_mem_methods* */ -#define SQLITE_CONFIG_SCRATCH 6 /* void*, int sz, int N */ +#define SQLITE_CONFIG_SCRATCH 6 /* No longer used */ #define SQLITE_CONFIG_PAGECACHE 7 /* void*, int sz, int N */ #define SQLITE_CONFIG_HEAP 8 /* void*, int nByte, int min */ #define SQLITE_CONFIG_MEMSTATUS 9 /* boolean */ #define SQLITE_CONFIG_MUTEX 10 /* sqlite3_mutex_methods* */ #define SQLITE_CONFIG_GETMUTEX 11 /* sqlite3_mutex_methods* */ @@ -1898,10 +1940,11 @@ #define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ #define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ #define SQLITE_CONFIG_PCACHE_HDRSZ 24 /* int *psz */ #define SQLITE_CONFIG_PMASZ 25 /* unsigned int szPma */ #define SQLITE_CONFIG_STMTJRNL_SPILL 26 /* int nByte */ +#define SQLITE_CONFIG_SMALL_MALLOC 27 /* boolean */ /* ** CAPI3REF: Database Connection Configuration Options ** ** These constants are the available integer configuration options that @@ -3098,14 +3141,14 @@ ** [[URI filenames in sqlite3_open()]]

    URI Filenames

    ** ** ^If [URI filename] interpretation is enabled, and the filename argument ** begins with "file:", then the filename is interpreted as a URI. ^URI ** filename interpretation is enabled if the [SQLITE_OPEN_URI] flag is -** set in the fourth argument to sqlite3_open_v2(), or if it has +** set in the third argument to sqlite3_open_v2(), or if it has ** been enabled globally using the [SQLITE_CONFIG_URI] option with the ** [sqlite3_config()] method or by the [SQLITE_USE_URI] compile-time option. -** As of SQLite version 3.7.7, URI filename interpretation is turned off +** URI filename interpretation is turned off ** by default, but future releases of SQLite might enable URI filename ** interpretation by default. See "[URI filenames]" for additional ** information. ** ** URI filenames are parsed according to RFC 3986. ^If the URI contains an @@ -3644,11 +3687,11 @@ ); int sqlite3_prepare16_v3( sqlite3 *db, /* Database handle */ const void *zSql, /* SQL statement, UTF-16 encoded */ int nByte, /* Maximum length of zSql in bytes. */ - unsigned int prepFalgs, /* Zero or more SQLITE_PREPARE_ flags */ + unsigned int prepFlags, /* Zero or more SQLITE_PREPARE_ flags */ sqlite3_stmt **ppStmt, /* OUT: Statement handle */ const void **pzTail /* OUT: Pointer to unused portion of zSql */ ); /* @@ -3775,12 +3818,13 @@ ** ** ^The sqlite3_value objects that are passed as parameters into the ** implementation of [application-defined SQL functions] are protected. ** ^The sqlite3_value object returned by ** [sqlite3_column_value()] is unprotected. -** Unprotected sqlite3_value objects may only be used with -** [sqlite3_result_value()] and [sqlite3_bind_value()]. +** Unprotected sqlite3_value objects may only be used as arguments +** to [sqlite3_result_value()], [sqlite3_bind_value()], and +** [sqlite3_value_dup()]. ** The [sqlite3_value_blob | sqlite3_value_type()] family of ** interfaces require protected sqlite3_value objects. */ typedef struct sqlite3_value sqlite3_value; @@ -3882,18 +3926,18 @@ ** Zeroblobs are intended to serve as placeholders for BLOBs whose ** content is later written using ** [sqlite3_blob_open | incremental BLOB I/O] routines. ** ^A negative value for the zeroblob results in a zero-length BLOB. ** -** ^The sqlite3_bind_pointer(S,I,P,T) routine causes the I-th parameter in +** ^The sqlite3_bind_pointer(S,I,P,T,D) routine causes the I-th parameter in ** [prepared statement] S to have an SQL value of NULL, but to also be -** associated with the pointer P of type T. -** ^The sqlite3_bind_pointer() routine can be used to pass -** host-language pointers into [application-defined SQL functions]. -** ^A parameter that is initialized using [sqlite3_bind_pointer()] appears -** to be an ordinary SQL NULL value to everything other than -** [sqlite3_value_pointer()]. The T parameter should be a static string. +** associated with the pointer P of type T. ^D is either a NULL pointer or +** a pointer to a destructor function for P. ^SQLite will invoke the +** destructor D with a single argument of P when it is finished using +** P. The T parameter should be a static string, preferably a string +** literal. The sqlite3_bind_pointer() routine is part of the +** [pointer passing interface] added for SQLite 3.20.0. ** ** ^If any of the sqlite3_bind_*() routines are called with a NULL pointer ** for the [prepared statement] or with a prepared statement for which ** [sqlite3_step()] has been called more recently than [sqlite3_reset()], ** then the call will return [SQLITE_MISUSE]. If any sqlite3_bind_() @@ -3924,11 +3968,11 @@ int sqlite3_bind_text(sqlite3_stmt*,int,const char*,int,void(*)(void*)); int sqlite3_bind_text16(sqlite3_stmt*, int, const void*, int, void(*)(void*)); int sqlite3_bind_text64(sqlite3_stmt*, int, const char*, sqlite3_uint64, void(*)(void*), unsigned char encoding); int sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*); -int sqlite3_bind_pointer(sqlite3_stmt*, int, void*, const char*); +int sqlite3_bind_pointer(sqlite3_stmt*, int, void*, const char*,void(*)(void*)); int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n); int sqlite3_bind_zeroblob64(sqlite3_stmt*, int, sqlite3_uint64); /* ** CAPI3REF: Number Of SQL Parameters @@ -4757,14 +4801,15 @@ ** in the native byte-order of the host machine. ^The ** sqlite3_value_text16be() and sqlite3_value_text16le() interfaces ** extract UTF-16 strings as big-endian and little-endian respectively. ** ** ^If [sqlite3_value] object V was initialized -** using [sqlite3_bind_pointer(S,I,P,X)] or [sqlite3_result_pointer(C,P,X)] +** using [sqlite3_bind_pointer(S,I,P,X,D)] or [sqlite3_result_pointer(C,P,X,D)] ** and if X and Y are strings that compare equal according to strcmp(X,Y), ** then sqlite3_value_pointer(V,Y) will return the pointer P. ^Otherwise, -** sqlite3_value_pointer(V,Y) returns a NULL. +** sqlite3_value_pointer(V,Y) returns a NULL. The sqlite3_bind_pointer() +** routine is part of the [pointer passing interface] added for SQLite 3.20.0. ** ** ^(The sqlite3_value_type(V) interface returns the ** [SQLITE_INTEGER | datatype code] for the initial datatype of the ** [sqlite3_value] object V. The returned value is one of [SQLITE_INTEGER], ** [SQLITE_FLOAT], [SQLITE_TEXT], [SQLITE_BLOB], or [SQLITE_NULL].)^ @@ -5095,18 +5140,20 @@ ** be deallocated after sqlite3_result_value() returns without harm. ** ^A [protected sqlite3_value] object may always be used where an ** [unprotected sqlite3_value] object is required, so either ** kind of [sqlite3_value] object can be used with this interface. ** -** ^The sqlite3_result_pointer(C,P,T) interface sets the result to an +** ^The sqlite3_result_pointer(C,P,T,D) interface sets the result to an ** SQL NULL value, just like [sqlite3_result_null(C)], except that it ** also associates the host-language pointer P or type T with that ** NULL value such that the pointer can be retrieved within an ** [application-defined SQL function] using [sqlite3_value_pointer()]. -** The T parameter should be a static string. -** This mechanism can be used to pass non-SQL values between -** application-defined functions. +** ^If the D parameter is not NULL, then it is a pointer to a destructor +** for the P parameter. ^SQLite invokes D with P as its only argument +** when SQLite is finished with P. The T parameter should be a static +** string and preferably a string literal. The sqlite3_result_pointer() +** routine is part of the [pointer passing interface] added for SQLite 3.20.0. ** ** If these routines are called from within the different thread ** than the one containing the application-defined function that received ** the [sqlite3_context] pointer, the results are undefined. */ @@ -5127,11 +5174,11 @@ void(*)(void*), unsigned char encoding); void sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); void sqlite3_result_text16le(sqlite3_context*, const void*, int,void(*)(void*)); void sqlite3_result_text16be(sqlite3_context*, const void*, int,void(*)(void*)); void sqlite3_result_value(sqlite3_context*, sqlite3_value*); -void sqlite3_result_pointer(sqlite3_context*, void*, const char*); +void sqlite3_result_pointer(sqlite3_context*, void*,const char*,void(*)(void*)); void sqlite3_result_zeroblob(sqlite3_context*, int n); int sqlite3_result_zeroblob64(sqlite3_context*, sqlite3_uint64 n); /* @@ -6959,11 +7006,11 @@ #define SQLITE_TESTCTRL_ASSERT 12 #define SQLITE_TESTCTRL_ALWAYS 13 #define SQLITE_TESTCTRL_RESERVE 14 #define SQLITE_TESTCTRL_OPTIMIZATIONS 15 #define SQLITE_TESTCTRL_ISKEYWORD 16 -#define SQLITE_TESTCTRL_SCRATCHMALLOC 17 +#define SQLITE_TESTCTRL_SCRATCHMALLOC 17 /* NOT USED */ #define SQLITE_TESTCTRL_LOCALTIME_FAULT 18 #define SQLITE_TESTCTRL_EXPLAIN_STMT 19 /* NOT USED */ #define SQLITE_TESTCTRL_ONCE_RESET_THRESHOLD 19 #define SQLITE_TESTCTRL_NEVER_CORRUPT 20 #define SQLITE_TESTCTRL_VDBE_COVERAGE 21 @@ -7018,12 +7065,11 @@ **
    ** [[SQLITE_STATUS_MEMORY_USED]] ^(
    SQLITE_STATUS_MEMORY_USED
    **
    This parameter is the current amount of memory checked out ** using [sqlite3_malloc()], either directly or indirectly. The ** figure includes calls made to [sqlite3_malloc()] by the application -** and internal memory usage by the SQLite library. Scratch memory -** controlled by [SQLITE_CONFIG_SCRATCH] and auxiliary page-cache +** and internal memory usage by the SQLite library. Auxiliary page-cache ** memory controlled by [SQLITE_CONFIG_PAGECACHE] is not included in ** this parameter. The amount returned is the sum of the allocation ** sizes as reported by the xSize method in [sqlite3_mem_methods].
    )^ ** ** [[SQLITE_STATUS_MALLOC_SIZE]] ^(
    SQLITE_STATUS_MALLOC_SIZE
    @@ -7057,33 +7103,18 @@ **
    This parameter records the largest memory allocation request ** handed to [pagecache memory allocator]. Only the value returned in the ** *pHighwater parameter to [sqlite3_status()] is of interest. ** The value written into the *pCurrent parameter is undefined.
    )^ ** -** [[SQLITE_STATUS_SCRATCH_USED]] ^(
    SQLITE_STATUS_SCRATCH_USED
    -**
    This parameter returns the number of allocations used out of the -** [scratch memory allocator] configured using -** [SQLITE_CONFIG_SCRATCH]. The value returned is in allocations, not -** in bytes. Since a single thread may only have one scratch allocation -** outstanding at time, this parameter also reports the number of threads -** using scratch memory at the same time.
    )^ +** [[SQLITE_STATUS_SCRATCH_USED]]
    SQLITE_STATUS_SCRATCH_USED
    +**
    No longer used.
    ** ** [[SQLITE_STATUS_SCRATCH_OVERFLOW]] ^(
    SQLITE_STATUS_SCRATCH_OVERFLOW
    -**
    This parameter returns the number of bytes of scratch memory -** allocation which could not be satisfied by the [SQLITE_CONFIG_SCRATCH] -** buffer and where forced to overflow to [sqlite3_malloc()]. The values -** returned include overflows because the requested allocation was too -** larger (that is, because the requested allocation was larger than the -** "sz" parameter to [SQLITE_CONFIG_SCRATCH]) and because no scratch buffer -** slots were available. -**
    )^ +**
    No longer used.
    ** -** [[SQLITE_STATUS_SCRATCH_SIZE]] ^(
    SQLITE_STATUS_SCRATCH_SIZE
    -**
    This parameter records the largest memory allocation request -** handed to [scratch memory allocator]. Only the value returned in the -** *pHighwater parameter to [sqlite3_status()] is of interest. -** The value written into the *pCurrent parameter is undefined.
    )^ +** [[SQLITE_STATUS_SCRATCH_SIZE]]
    SQLITE_STATUS_SCRATCH_SIZE
    +**
    No longer used.
    ** ** [[SQLITE_STATUS_PARSER_STACK]] ^(
    SQLITE_STATUS_PARSER_STACK
    **
    The *pHighwater parameter records the deepest parser stack. ** The *pCurrent value is undefined. The *pHighwater value is only ** meaningful if SQLite is compiled with [YYTRACKMAXSTACKDEPTH].
    )^ @@ -7092,16 +7123,16 @@ ** New status parameters may be added from time to time. */ #define SQLITE_STATUS_MEMORY_USED 0 #define SQLITE_STATUS_PAGECACHE_USED 1 #define SQLITE_STATUS_PAGECACHE_OVERFLOW 2 -#define SQLITE_STATUS_SCRATCH_USED 3 -#define SQLITE_STATUS_SCRATCH_OVERFLOW 4 +#define SQLITE_STATUS_SCRATCH_USED 3 /* NOT USED */ +#define SQLITE_STATUS_SCRATCH_OVERFLOW 4 /* NOT USED */ #define SQLITE_STATUS_MALLOC_SIZE 5 #define SQLITE_STATUS_PARSER_STACK 6 #define SQLITE_STATUS_PAGECACHE_SIZE 7 -#define SQLITE_STATUS_SCRATCH_SIZE 8 +#define SQLITE_STATUS_SCRATCH_SIZE 8 /* NOT USED */ #define SQLITE_STATUS_MALLOC_COUNT 9 /* ** CAPI3REF: Database Connection Status ** METHOD: sqlite3 Index: src/sqlite3ext.h ================================================================== --- src/sqlite3ext.h +++ src/sqlite3ext.h @@ -244,11 +244,11 @@ int (*stmt_readonly)(sqlite3_stmt*); int (*stricmp)(const char*,const char*); int (*uri_boolean)(const char*,const char*,int); sqlite3_int64 (*uri_int64)(const char*,const char*,sqlite3_int64); const char *(*uri_parameter)(const char*,const char*); - char *(*vsnprintf)(int,char*,const char*,va_list); + char *(*xvsnprintf)(int,char*,const char*,va_list); int (*wal_checkpoint_v2)(sqlite3*,const char*,int,int*,int*); /* Version 3.8.7 and later */ int (*auto_extension)(void(*)(void)); int (*bind_blob64)(sqlite3_stmt*,int,const void*,sqlite3_uint64, void(*)(void*)); @@ -287,12 +287,12 @@ /* Version 3.20.0 and later */ int (*prepare_v3)(sqlite3*,const char*,int,unsigned int, sqlite3_stmt**,const char**); int (*prepare16_v3)(sqlite3*,const void*,int,unsigned int, sqlite3_stmt**,const void**); - int (*bind_pointer)(sqlite3_stmt*,int,void*,const char*); - void (*result_pointer)(sqlite3_context*,void*,const char*); + int (*bind_pointer)(sqlite3_stmt*,int,void*,const char*,void(*)(void*)); + void (*result_pointer)(sqlite3_context*,void*,const char*,void(*)(void*)); void *(*value_pointer)(sqlite3_value*,const char*); }; /* ** This is the function signature used for all extension entry points. It @@ -440,11 +440,11 @@ #define sqlite3_value_text16 sqlite3_api->value_text16 #define sqlite3_value_text16be sqlite3_api->value_text16be #define sqlite3_value_text16le sqlite3_api->value_text16le #define sqlite3_value_type sqlite3_api->value_type #define sqlite3_vmprintf sqlite3_api->vmprintf -#define sqlite3_vsnprintf sqlite3_api->vsnprintf +#define sqlite3_vsnprintf sqlite3_api->xvsnprintf #define sqlite3_overload_function sqlite3_api->overload_function #define sqlite3_prepare_v2 sqlite3_api->prepare_v2 #define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2 #define sqlite3_clear_bindings sqlite3_api->clear_bindings #define sqlite3_bind_zeroblob sqlite3_api->bind_zeroblob @@ -516,11 +516,11 @@ #define sqlite3_stmt_readonly sqlite3_api->stmt_readonly #define sqlite3_stricmp sqlite3_api->stricmp #define sqlite3_uri_boolean sqlite3_api->uri_boolean #define sqlite3_uri_int64 sqlite3_api->uri_int64 #define sqlite3_uri_parameter sqlite3_api->uri_parameter -#define sqlite3_uri_vsnprintf sqlite3_api->vsnprintf +#define sqlite3_uri_vsnprintf sqlite3_api->xvsnprintf #define sqlite3_wal_checkpoint_v2 sqlite3_api->wal_checkpoint_v2 /* Version 3.8.7 and later */ #define sqlite3_auto_extension sqlite3_api->auto_extension #define sqlite3_bind_blob64 sqlite3_api->bind_blob64 #define sqlite3_bind_text64 sqlite3_api->bind_text64 Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -628,10 +628,19 @@ */ #ifndef SQLITE_DEFAULT_PCACHE_INITSZ # define SQLITE_DEFAULT_PCACHE_INITSZ 20 #endif +/* +** The compile-time options SQLITE_MMAP_READWRITE and +** SQLITE_ENABLE_BATCH_ATOMIC_WRITE are not compatible with one another. +** You must choose one or the other (or neither) but not both. +*/ +#if defined(SQLITE_MMAP_READWRITE) && defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) +#error Cannot use both SQLITE_MMAP_READWRITE and SQLITE_ENABLE_BATCH_ATOMIC_WRITE +#endif + /* ** GCC does not define the offsetof() macro so we'll have to do it ** ourselves. */ #ifndef offsetof @@ -1202,10 +1211,11 @@ ** changes and so the view will need to be reset. */ #define DB_SchemaLoaded 0x0001 /* The schema has been loaded */ #define DB_UnresetViews 0x0002 /* Some views have defined column names */ #define DB_Empty 0x0004 /* The file is empty (length 0 bytes) */ +#define DB_ResetWanted 0x0008 /* Reset the schema when nSchemaLock==0 */ /* ** The number of different kinds of things that can be limited ** using the sqlite3_limit() interface. */ @@ -1233,13 +1243,13 @@ */ struct Lookaside { u32 bDisable; /* Only operate the lookaside when zero */ u16 sz; /* Size of each buffer in bytes */ u8 bMalloced; /* True if pStart obtained from sqlite3_malloc() */ - int nOut; /* Number of buffers currently checked out */ - int mxOut; /* Highwater mark for nOut */ - int anStat[3]; /* 0: hits. 1: size misses. 2: full misses */ + u32 nSlot; /* Number of lookaside slots allocated */ + u32 anStat[3]; /* 0: hits. 1: size misses. 2: full misses */ + LookasideSlot *pInit; /* List of buffers not previously used */ LookasideSlot *pFree; /* List of available buffers */ void *pStart; /* First byte of available memory space */ void *pEnd; /* First byte past end of available space */ }; struct LookasideSlot { @@ -1314,13 +1324,15 @@ struct Vdbe *pVdbe; /* List of active virtual machines */ CollSeq *pDfltColl; /* The default collating sequence (BINARY) */ sqlite3_mutex *mutex; /* Connection mutex */ Db *aDb; /* All backends */ int nDb; /* Number of backends currently in use */ - int flags; /* Miscellaneous flags. See below */ + u32 mDbFlags; /* flags recording internal state */ + u32 flags; /* flags settable by pragmas. See below */ i64 lastRowid; /* ROWID of most recent insert (see above) */ i64 szMmap; /* Default mmap_size setting */ + u32 nSchemaLock; /* Do not reset the schema when non-zero */ unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ int iSysErrno; /* Errno value from last system error */ u16 dbOptFlags; /* Flags to enable/disable optimizations */ @@ -1468,31 +1480,32 @@ #define SQLITE_ReverseOrder 0x00001000 /* Reverse unordered SELECTs */ #define SQLITE_RecTriggers 0x00002000 /* Enable recursive triggers */ #define SQLITE_ForeignKeys 0x00004000 /* Enforce foreign key constraints */ #define SQLITE_AutoIndex 0x00008000 /* Enable automatic indexes */ #define SQLITE_LoadExtension 0x00010000 /* Enable load_extension */ -#define SQLITE_EnableTrigger 0x00020000 /* True to enable triggers */ -#define SQLITE_DeferFKs 0x00040000 /* Defer all FK constraints */ -#define SQLITE_QueryOnly 0x00080000 /* Disable database changes */ -#define SQLITE_CellSizeCk 0x00100000 /* Check btree cell sizes on load */ -#define SQLITE_Fts3Tokenizer 0x00200000 /* Enable fts3_tokenizer(2) */ -#define SQLITE_EnableQPSG 0x00400000 /* Query Planner Stability Guarantee */ -/* The next four values are not used by PRAGMAs or by sqlite3_dbconfig() and -** could be factored out into a separate bit vector of the sqlite3 object. */ -#define SQLITE_InternChanges 0x00800000 /* Uncommitted Hash table changes */ -#define SQLITE_LoadExtFunc 0x01000000 /* Enable load_extension() SQL func */ -#define SQLITE_PreferBuiltin 0x02000000 /* Preference to built-in funcs */ -#define SQLITE_Vacuum 0x04000000 /* Currently in a VACUUM */ +#define SQLITE_LoadExtFunc 0x00020000 /* Enable load_extension() SQL func */ +#define SQLITE_EnableTrigger 0x00040000 /* True to enable triggers */ +#define SQLITE_DeferFKs 0x00080000 /* Defer all FK constraints */ +#define SQLITE_QueryOnly 0x00100000 /* Disable database changes */ +#define SQLITE_CellSizeCk 0x00200000 /* Check btree cell sizes on load */ +#define SQLITE_Fts3Tokenizer 0x00400000 /* Enable fts3_tokenizer(2) */ +#define SQLITE_EnableQPSG 0x00800000 /* Query Planner Stability Guarantee */ /* Flags used only if debugging */ #ifdef SQLITE_DEBUG #define SQLITE_SqlTrace 0x08000000 /* Debug print SQL as it executes */ #define SQLITE_VdbeListing 0x10000000 /* Debug listings of VDBE programs */ #define SQLITE_VdbeTrace 0x20000000 /* True to trace VDBE execution */ #define SQLITE_VdbeAddopTrace 0x40000000 /* Trace sqlite3VdbeAddOp() calls */ #define SQLITE_VdbeEQP 0x80000000 /* Debug EXPLAIN QUERY PLAN */ #endif +/* +** Allowed values for sqlite3.mDbFlags +*/ +#define DBFLAG_SchemaChange 0x0001 /* Uncommitted Hash table changes */ +#define DBFLAG_PreferBuiltin 0x0002 /* Preference to built-in funcs */ +#define DBFLAG_Vacuum 0x0004 /* Currently in a VACUUM */ /* ** Bits of the sqlite3.dbOptFlags field that are used by the ** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface to ** selectively disable various optimizations. @@ -1626,11 +1639,18 @@ ** ** DFUNCTION(zName, nArg, iArg, bNC, xFunc) ** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag and ** adds the SQLITE_FUNC_SLOCHNG flag. Used for date & time functions ** and functions like sqlite_version() that can change, but not during -** a single query. +** a single query. The iArg is ignored. The user-data is always set +** to a NULL pointer. The bNC parameter is not used. +** +** PURE_DATE(zName, nArg, iArg, bNC, xFunc) +** Used for "pure" date/time functions, this macro is like DFUNCTION +** except that it does set the SQLITE_FUNC_CONSTANT flags. iArg is +** ignored and the user-data for these functions is set to an +** arbitrary non-NULL pointer. The bNC parameter is not used. ** ** AGGREGATE(zName, nArg, iArg, bNC, xStep, xFinal) ** Used to create an aggregate function definition implemented by ** the C functions xStep and xFinal. The first four parameters ** are interpreted in the same way as the first 4 parameters to @@ -1649,12 +1669,15 @@ SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} } #define VFUNCTION(zName, nArg, iArg, bNC, xFunc) \ {nArg, SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} } #define DFUNCTION(zName, nArg, iArg, bNC, xFunc) \ - {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} } + {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8, \ + 0, 0, xFunc, 0, #zName, {0} } +#define PURE_DATE(zName, nArg, iArg, bNC, xFunc) \ + {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|SQLITE_FUNC_CONSTANT, \ + (void*)&sqlite3Config, 0, xFunc, 0, #zName, {0} } #define FUNCTION2(zName, nArg, iArg, bNC, xFunc, extraFlags) \ {nArg,SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags,\ SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} } #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ @@ -2029,12 +2052,12 @@ ** for the rowid at the end. */ struct KeyInfo { u32 nRef; /* Number of references to this KeyInfo object */ u8 enc; /* Text encoding - one of the SQLITE_UTF* values */ - u16 nField; /* Number of key columns in the index */ - u16 nXField; /* Number of columns beyond the key columns */ + u16 nKeyField; /* Number of key columns in the index */ + u16 nAllField; /* Total columns, including key plus others */ sqlite3 *db; /* The database connection */ u8 *aSortOrder; /* Sort order for each column. */ CollSeq *aColl[1]; /* Collating sequence for each term of the key */ }; @@ -2077,12 +2100,12 @@ KeyInfo *pKeyInfo; /* Collation and sort-order information */ Mem *aMem; /* Values */ u16 nField; /* Number of entries in apMem[] */ i8 default_rc; /* Comparison result if keys are equal */ u8 errCode; /* Error detected by xRecordCompare (CORRUPT or NOMEM) */ - i8 r1; /* Value to return if (lhs > rhs) */ - i8 r2; /* Value to return if (rhs < lhs) */ + i8 r1; /* Value to return if (lhs < rhs) */ + i8 r2; /* Value to return if (lhs > rhs) */ u8 eqSeen; /* True if an equality comparison has been seen */ }; /* @@ -2362,11 +2385,12 @@ i16 iRightJoinTable; /* If EP_FromJoin, the right table of the join */ u8 op2; /* TK_REGISTER: original value of Expr.op ** TK_COLUMN: the value of p5 for OP_Column ** TK_AGG_FUNCTION: nesting depth */ AggInfo *pAggInfo; /* Used by TK_AGG_COLUMN and TK_AGG_FUNCTION */ - Table *pTab; /* Table for TK_COLUMN expressions. */ + Table *pTab; /* Table for TK_COLUMN expressions. Can be NULL + ** for a column of an index on an expression */ }; /* ** The following are the meanings of bits in the Expr.flags field. */ @@ -2951,12 +2975,12 @@ int nErr; /* Number of errors seen */ int nTab; /* Number of previously allocated VDBE cursors */ int nMem; /* Number of memory cells used so far */ int nOpAlloc; /* Number of slots allocated for Vdbe.aOp[] */ int szOpAlloc; /* Bytes of memory space allocated for Vdbe.aOp[] */ - int ckBase; /* Base register of data during check constraints */ - int iSelfTab; /* Table of an index whose exprs are being coded */ + int iSelfTab; /* Table for associated with an index on expr, or negative + ** of the base register during check-constraint eval */ int iCacheLevel; /* ColCache valid when aColCache[].iLevel<=iCacheLevel */ int iCacheCnt; /* Counter used to generate aColCache[].lru values */ int nLabel; /* Number of labels used */ int *aLabel; /* Space to hold the labels */ ExprList *pConstExpr;/* Constant expressions */ @@ -2975,11 +2999,11 @@ TableLock *aTableLock; /* Required table locks for shared-cache mode */ #endif AutoincInfo *pAinc; /* Information about AUTOINCREMENT counters */ Parse *pToplevel; /* Parse structure for main program (or NULL) */ Table *pTriggerTab; /* Table triggers are being coded for */ - int addrCrTab; /* Address of OP_CreateTable opcode on CREATE TABLE */ + int addrCrTab; /* Address of OP_CreateBtree opcode on CREATE TABLE */ u32 nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ u32 oldmask; /* Mask of old.* columns referenced */ u32 newmask; /* Mask of new.* columns referenced */ u8 eTriggerOp; /* TK_UPDATE, TK_INSERT or TK_DELETE */ u8 eOrconf; /* Default ON CONFLICT policy for trigger steps */ @@ -3204,15 +3228,14 @@ ** An objected used to accumulate the text of a string where we ** do not necessarily know how big the string will be in the end. */ struct StrAccum { sqlite3 *db; /* Optional database for lookaside. Can be NULL */ - char *zBase; /* A base allocation. Not from malloc. */ char *zText; /* The string collected so far */ - u32 nChar; /* Length of the string so far */ u32 nAlloc; /* Amount of space allocated in zText */ u32 mxAlloc; /* Maximum allowed allocation. 0 for no malloc usage */ + u32 nChar; /* Length of the string so far */ u8 accError; /* STRACCUM_NOMEM or STRACCUM_TOOBIG */ u8 printfFlags; /* SQLITE_PRINTF flags below */ }; #define STRACCUM_NOMEM 1 #define STRACCUM_TOOBIG 2 @@ -3243,10 +3266,11 @@ int bMemstat; /* True to enable memory status */ int bCoreMutex; /* True to enable core mutexing */ int bFullMutex; /* True to enable full mutexing */ int bOpenUri; /* True to interpret filenames as URIs */ int bUseCis; /* Use covering indices for full-scans */ + int bSmallMalloc; /* Avoid large memory allocations if true */ int mxStrlen; /* Maximum string length */ int neverCorrupt; /* Database is always well-formed */ int szLookaside; /* Default lookaside buffer size */ int nLookaside; /* Default lookaside buffer count */ int nStmtSpill; /* Stmt-journal spill-to-disk threshold */ @@ -3256,13 +3280,10 @@ void *pHeap; /* Heap storage space */ int nHeap; /* Size of pHeap[] */ int mnReq, mxReq; /* Min and max heap requests sizes */ sqlite3_int64 szMmap; /* mmap() space per open file */ sqlite3_int64 mxMmap; /* Maximum value for szMmap */ - void *pScratch; /* Scratch memory */ - int szScratch; /* Size of each scratch buffer */ - int nScratch; /* Number of scratch buffers */ void *pPage; /* Page cache memory */ int szPage; /* Size of each page in pPage[] */ int nPage; /* Number of pages in pPage[] */ int mxParserStack; /* maximum depth of the parser stack */ int sharedCacheEnabled; /* true if shared-cache mode enabled */ @@ -3345,10 +3366,11 @@ int sqlite3WalkSelect(Walker*, Select*); int sqlite3WalkSelectExpr(Walker*, Select*); int sqlite3WalkSelectFrom(Walker*, Select*); int sqlite3ExprWalkNoop(Walker*, Expr*); int sqlite3SelectWalkNoop(Walker*, Select*); +int sqlite3SelectWalkFail(Walker*, Select*); #ifdef SQLITE_DEBUG void sqlite3SelectWalkAssert2(Walker*, Select*); #endif /* @@ -3497,12 +3519,10 @@ void *sqlite3DbRealloc(sqlite3 *, void *, u64); void sqlite3DbFree(sqlite3*, void*); void sqlite3DbFreeNN(sqlite3*, void*); int sqlite3MallocSize(void*); int sqlite3DbMallocSize(sqlite3*, void*); -void *sqlite3ScratchMalloc(int); -void sqlite3ScratchFree(void*); void *sqlite3PageMalloc(int); void sqlite3PageFree(void*); void sqlite3MemSetDefault(void); #ifndef SQLITE_UNTESTABLE void sqlite3BenignMallocHooks(void (*)(void), void (*)(void)); @@ -3554,10 +3574,11 @@ sqlite3_int64 sqlite3StatusValue(int); void sqlite3StatusUp(int, int); void sqlite3StatusDown(int, int); void sqlite3StatusHighwater(int, int); +int sqlite3LookasideUsed(sqlite3*,int*); /* Access to mutexes used by sqlite3_status() */ sqlite3_mutex *sqlite3Pcache1Mutex(void); sqlite3_mutex *sqlite3MallocMutex(void); @@ -4273,11 +4294,12 @@ #define IN_INDEX_LOOP 0x0004 /* IN operator used as a loop */ int sqlite3FindInIndex(Parse *, Expr *, u32, int*, int*); int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int); int sqlite3JournalSize(sqlite3_vfs *); -#ifdef SQLITE_ENABLE_ATOMIC_WRITE +#if defined(SQLITE_ENABLE_ATOMIC_WRITE) \ + || defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) int sqlite3JournalCreate(sqlite3_file *); #endif int sqlite3JournalIsInMemory(sqlite3_file *p); void sqlite3MemJournalOpen(sqlite3_file *); @@ -4359,12 +4381,11 @@ # define sqlite3MemdebugHasType(X,Y) 1 # define sqlite3MemdebugNoType(X,Y) 1 #endif #define MEMTYPE_HEAP 0x01 /* General heap allocations */ #define MEMTYPE_LOOKASIDE 0x02 /* Heap that might have been lookaside */ -#define MEMTYPE_SCRATCH 0x04 /* Scratch allocations */ -#define MEMTYPE_PCACHE 0x08 /* Page cache allocations */ +#define MEMTYPE_PCACHE 0x04 /* Page cache allocations */ #if (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) /* ** An instance of the following structure is used to hold the process ID Index: src/status.c ================================================================== --- src/status.c +++ src/status.c @@ -120,11 +120,10 @@ assert( op>=0 && opwsdStat.mxValue[op] ){ wsdStat.mxValue[op] = newValue; } } @@ -168,10 +167,32 @@ *pCurrent = (int)iCur; *pHighwater = (int)iHwtr; } return rc; } + +/* +** Return the number of LookasideSlot elements on the linked list +*/ +static u32 countLookasideSlots(LookasideSlot *p){ + u32 cnt = 0; + while( p ){ + p = p->pNext; + cnt++; + } + return cnt; +} + +/* +** Count the number of slots of lookaside memory that are outstanding +*/ +int sqlite3LookasideUsed(sqlite3 *db, int *pHighwater){ + u32 nInit = countLookasideSlots(db->lookaside.pInit); + u32 nFree = countLookasideSlots(db->lookaside.pFree); + if( pHighwater ) *pHighwater = db->lookaside.nSlot - nInit; + return db->lookaside.nSlot - (nInit+nFree); +} /* ** Query status information for a single database connection */ int sqlite3_db_status( @@ -188,14 +209,19 @@ } #endif sqlite3_mutex_enter(db->mutex); switch( op ){ case SQLITE_DBSTATUS_LOOKASIDE_USED: { - *pCurrent = db->lookaside.nOut; - *pHighwater = db->lookaside.mxOut; + *pCurrent = sqlite3LookasideUsed(db, pHighwater); if( resetFlag ){ - db->lookaside.mxOut = db->lookaside.nOut; + LookasideSlot *p = db->lookaside.pFree; + if( p ){ + while( p->pNext ) p = p->pNext; + p->pNext = db->lookaside.pInit; + db->lookaside.pInit = db->lookaside.pFree; + db->lookaside.pFree = 0; + } } break; } case SQLITE_DBSTATUS_LOOKASIDE_HIT: Index: src/tclsqlite.c ================================================================== --- src/tclsqlite.c +++ src/tclsqlite.c @@ -3880,32 +3880,43 @@ Tcl_Interp *interp, int argc, const char **argv ){ FILE *in; + int ofst; + int amt; MD5Context ctx; void (*converter)(unsigned char*, char*); unsigned char digest[16]; char zBuf[10240]; - if( argc!=2 ){ + if( argc!=2 && argc!=4 ){ Tcl_AppendResult(interp,"wrong # args: should be \"", argv[0], - " FILENAME\"", (char*)0); + " FILENAME [OFFSET AMT]\"", (char*)0); return TCL_ERROR; } + if( argc==4 ){ + ofst = atoi(argv[2]); + amt = atoi(argv[3]); + }else{ + ofst = 0; + amt = 2147483647; + } in = fopen(argv[1],"rb"); if( in==0 ){ Tcl_AppendResult(interp,"unable to open file \"", argv[1], "\" for reading", (char*)0); return TCL_ERROR; } + fseek(in, ofst, SEEK_SET); MD5Init(&ctx); - for(;;){ + while( amt>0 ){ int n; - n = (int)fread(zBuf, 1, sizeof(zBuf), in); + n = (int)fread(zBuf, 1, sizeof(zBuf)<=amt ? sizeof(zBuf) : amt, in); if( n<=0 ) break; MD5Update(&ctx, (unsigned char*)zBuf, (unsigned)n); + amt -= n; } fclose(in); MD5Final(digest, &ctx); converter = (void(*)(unsigned char*,char*))cd; converter(digest, zBuf); Index: src/test1.c ================================================================== --- src/test1.c +++ src/test1.c @@ -2550,10 +2550,50 @@ rc = sqlite3_delete_database(zFile); Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); return TCL_OK; } + +/* +** Usage: atomic_batch_write PATH +*/ +static int SQLITE_TCLAPI test_atomic_batch_write( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + char *zFile = 0; /* Path to file to test */ + sqlite3 *db = 0; /* Database handle */ + sqlite3_file *pFd = 0; /* SQLite fd open on zFile */ + int bRes = 0; /* Integer result of this command */ + int dc = 0; /* Device-characteristics mask */ + int rc; /* sqlite3_open() return code */ + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "PATH"); + return TCL_ERROR; + } + zFile = Tcl_GetString(objv[1]); + + rc = sqlite3_open(zFile, &db); + if( rc!=SQLITE_OK ){ + Tcl_AppendResult(interp, sqlite3_errmsg(db), 0); + sqlite3_close(db); + return TCL_ERROR; + } + + rc = sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, (void*)&pFd); + dc = pFd->pMethods->xDeviceCharacteristics(pFd); + if( dc & SQLITE_IOCAP_BATCH_ATOMIC ){ + bRes = 1; + } + + Tcl_SetObjResult(interp, Tcl_NewIntObj(bRes)); + sqlite3_close(db); + return TCL_OK; +} /* ** Usage: sqlite3_next_stmt DB STMT ** ** Return the next statment in sequence after STMT. @@ -7109,10 +7149,11 @@ extern int sqlite3_remember_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_series_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_spellfix_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_totype_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_wholenumber_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_unionvtab_init(sqlite3*,char**,const sqlite3_api_routines*); static const struct { const char *zExtName; int (*pInit)(sqlite3*,char**,const sqlite3_api_routines*); } aExtension[] = { { "amatch", sqlite3_amatch_init }, @@ -7128,10 +7169,11 @@ { "regexp", sqlite3_regexp_init }, { "remember", sqlite3_remember_init }, { "series", sqlite3_series_init }, { "spellfix", sqlite3_spellfix_init }, { "totype", sqlite3_totype_init }, + { "unionvtab", sqlite3_unionvtab_init }, { "wholenumber", sqlite3_wholenumber_init }, }; sqlite3 *db; const char *zName; int i, j, rc; @@ -7839,10 +7881,11 @@ { "sqlite3_snapshot_get_blob", test_snapshot_get_blob, 0 }, { "sqlite3_snapshot_open_blob", test_snapshot_open_blob, 0 }, { "sqlite3_snapshot_cmp_blob", test_snapshot_cmp_blob, 0 }, #endif { "sqlite3_delete_database", test_delete_database, 0 }, + { "atomic_batch_write", test_atomic_batch_write, 0 }, }; static int bitmask_size = sizeof(Bitmask)*8; static int longdouble_size = sizeof(LONGDOUBLE_TYPE); int i; extern int sqlite3_sync_count, sqlite3_fullsync_count; Index: src/test6.c ================================================================== --- src/test6.c +++ src/test6.c @@ -734,10 +734,11 @@ { "atomic32k", SQLITE_IOCAP_ATOMIC32K }, { "atomic64k", SQLITE_IOCAP_ATOMIC64K }, { "sequential", SQLITE_IOCAP_SEQUENTIAL }, { "safe_append", SQLITE_IOCAP_SAFE_APPEND }, { "powersafe_overwrite", SQLITE_IOCAP_POWERSAFE_OVERWRITE }, + { "batch-atomic", SQLITE_IOCAP_BATCH_ATOMIC }, { 0, 0 } }; int i; int iDc = 0; @@ -974,11 +975,34 @@ return TCL_ERROR; } devsym_register(iDc, iSectorSize); return TCL_OK; +} + +/* +** tclcmd: sqlite3_crash_on_write N +*/ +static int SQLITE_TCLAPI writeCrashObjCmd( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + void devsym_crash_on_write(int); + int nWrite = 0; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "NWRITE"); + return TCL_ERROR; + } + if( Tcl_GetIntFromObj(interp, objv[1], &nWrite) ){ + return TCL_ERROR; + } + devsym_crash_on_write(nWrite); + return TCL_OK; } /* ** tclcmd: unregister_devsim */ @@ -1066,13 +1090,14 @@ #ifndef SQLITE_OMIT_DISKIO Tcl_CreateObjCommand(interp, "sqlite3_crash_enable", crashEnableCmd, 0, 0); Tcl_CreateObjCommand(interp, "sqlite3_crashparams", crashParamsObjCmd, 0, 0); Tcl_CreateObjCommand(interp, "sqlite3_crash_now", crashNowCmd, 0, 0); Tcl_CreateObjCommand(interp, "sqlite3_simulate_device", devSymObjCmd, 0, 0); + Tcl_CreateObjCommand(interp, "sqlite3_crash_on_write", writeCrashObjCmd,0,0); Tcl_CreateObjCommand(interp, "unregister_devsim", dsUnregisterObjCmd, 0, 0); Tcl_CreateObjCommand(interp, "register_jt_vfs", jtObjCmd, 0, 0); Tcl_CreateObjCommand(interp, "unregister_jt_vfs", jtUnregisterObjCmd, 0, 0); #endif return TCL_OK; } #endif /* SQLITE_TEST */ Index: src/test_blob.c ================================================================== --- src/test_blob.c +++ src/test_blob.c @@ -239,11 +239,11 @@ } if( nByte>0 ){ zBuf = (unsigned char *)Tcl_AttemptAlloc(nByte); if( zBuf==0 ){ - Tcl_AppendResult(interp, "out of memory", 0); + Tcl_AppendResult(interp, "out of memory in " __FILE__, 0); return TCL_ERROR; } } rc = sqlite3_blob_read(pBlob, zBuf, nByte, iOffset); if( rc==SQLITE_OK ){ Index: src/test_devsym.c ================================================================== --- src/test_devsym.c +++ src/test_devsym.c @@ -26,10 +26,11 @@ /* ** Name used to identify this VFS. */ #define DEVSYM_VFS_NAME "devsym" +#define WRITECRASH_NAME "writecrash" typedef struct devsym_file devsym_file; struct devsym_file { sqlite3_file base; sqlite3_file *pReal; @@ -70,65 +71,17 @@ #endif /* SQLITE_OMIT_LOAD_EXTENSION */ static int devsymRandomness(sqlite3_vfs*, int nByte, char *zOut); static int devsymSleep(sqlite3_vfs*, int microseconds); static int devsymCurrentTime(sqlite3_vfs*, double*); -static sqlite3_vfs devsym_vfs = { - 2, /* iVersion */ - sizeof(devsym_file), /* szOsFile */ - DEVSYM_MAX_PATHNAME, /* mxPathname */ - 0, /* pNext */ - DEVSYM_VFS_NAME, /* zName */ - 0, /* pAppData */ - devsymOpen, /* xOpen */ - devsymDelete, /* xDelete */ - devsymAccess, /* xAccess */ - devsymFullPathname, /* xFullPathname */ -#ifndef SQLITE_OMIT_LOAD_EXTENSION - devsymDlOpen, /* xDlOpen */ - devsymDlError, /* xDlError */ - devsymDlSym, /* xDlSym */ - devsymDlClose, /* xDlClose */ -#else - 0, /* xDlOpen */ - 0, /* xDlError */ - 0, /* xDlSym */ - 0, /* xDlClose */ -#endif /* SQLITE_OMIT_LOAD_EXTENSION */ - devsymRandomness, /* xRandomness */ - devsymSleep, /* xSleep */ - devsymCurrentTime, /* xCurrentTime */ - 0, /* xGetLastError */ - 0 /* xCurrentTimeInt64 */ -}; - -static sqlite3_io_methods devsym_io_methods = { - 2, /* iVersion */ - devsymClose, /* xClose */ - devsymRead, /* xRead */ - devsymWrite, /* xWrite */ - devsymTruncate, /* xTruncate */ - devsymSync, /* xSync */ - devsymFileSize, /* xFileSize */ - devsymLock, /* xLock */ - devsymUnlock, /* xUnlock */ - devsymCheckReservedLock, /* xCheckReservedLock */ - devsymFileControl, /* xFileControl */ - devsymSectorSize, /* xSectorSize */ - devsymDeviceCharacteristics, /* xDeviceCharacteristics */ - devsymShmMap, /* xShmMap */ - devsymShmLock, /* xShmLock */ - devsymShmBarrier, /* xShmBarrier */ - devsymShmUnmap /* xShmUnmap */ -}; - struct DevsymGlobal { sqlite3_vfs *pVfs; int iDeviceChar; int iSectorSize; + int nWriteCrash; }; -struct DevsymGlobal g = {0, 0, 512}; +struct DevsymGlobal g = {0, 0, 512, 0}; /* ** Close an devsym-file. */ static int devsymClose(sqlite3_file *pFile){ @@ -269,10 +222,30 @@ const char *zName, sqlite3_file *pFile, int flags, int *pOutFlags ){ +static sqlite3_io_methods devsym_io_methods = { + 2, /* iVersion */ + devsymClose, /* xClose */ + devsymRead, /* xRead */ + devsymWrite, /* xWrite */ + devsymTruncate, /* xTruncate */ + devsymSync, /* xSync */ + devsymFileSize, /* xFileSize */ + devsymLock, /* xLock */ + devsymUnlock, /* xUnlock */ + devsymCheckReservedLock, /* xCheckReservedLock */ + devsymFileControl, /* xFileControl */ + devsymSectorSize, /* xSectorSize */ + devsymDeviceCharacteristics, /* xDeviceCharacteristics */ + devsymShmMap, /* xShmMap */ + devsymShmLock, /* xShmLock */ + devsymShmBarrier, /* xShmBarrier */ + devsymShmUnmap /* xShmUnmap */ +}; + int rc; devsym_file *p = (devsym_file *)pFile; p->pReal = (sqlite3_file *)&p[1]; rc = sqlite3OsOpen(g.pVfs, zName, p->pReal, flags, pOutFlags); if( p->pReal->pMethods ){ @@ -370,21 +343,155 @@ */ static int devsymCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ return g.pVfs->xCurrentTime(g.pVfs, pTimeOut); } +/* +** Return the sector-size in bytes for an writecrash-file. +*/ +static int writecrashSectorSize(sqlite3_file *pFile){ + devsym_file *p = (devsym_file *)pFile; + return sqlite3OsSectorSize(p->pReal); +} + +/* +** Return the device characteristic flags supported by an writecrash-file. +*/ +static int writecrashDeviceCharacteristics(sqlite3_file *pFile){ + devsym_file *p = (devsym_file *)pFile; + return sqlite3OsDeviceCharacteristics(p->pReal); +} + +/* +** Write data to an writecrash-file. +*/ +static int writecrashWrite( + sqlite3_file *pFile, + const void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + devsym_file *p = (devsym_file *)pFile; + if( g.nWriteCrash>0 ){ + g.nWriteCrash--; + if( g.nWriteCrash==0 ) abort(); + } + return sqlite3OsWrite(p->pReal, zBuf, iAmt, iOfst); +} + +/* +** Open an writecrash file handle. +*/ +static int writecrashOpen( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_file *pFile, + int flags, + int *pOutFlags +){ +static sqlite3_io_methods writecrash_io_methods = { + 2, /* iVersion */ + devsymClose, /* xClose */ + devsymRead, /* xRead */ + writecrashWrite, /* xWrite */ + devsymTruncate, /* xTruncate */ + devsymSync, /* xSync */ + devsymFileSize, /* xFileSize */ + devsymLock, /* xLock */ + devsymUnlock, /* xUnlock */ + devsymCheckReservedLock, /* xCheckReservedLock */ + devsymFileControl, /* xFileControl */ + writecrashSectorSize, /* xSectorSize */ + writecrashDeviceCharacteristics, /* xDeviceCharacteristics */ + devsymShmMap, /* xShmMap */ + devsymShmLock, /* xShmLock */ + devsymShmBarrier, /* xShmBarrier */ + devsymShmUnmap /* xShmUnmap */ +}; + + int rc; + devsym_file *p = (devsym_file *)pFile; + p->pReal = (sqlite3_file *)&p[1]; + rc = sqlite3OsOpen(g.pVfs, zName, p->pReal, flags, pOutFlags); + if( p->pReal->pMethods ){ + pFile->pMethods = &writecrash_io_methods; + } + return rc; +} + +static sqlite3_vfs devsym_vfs = { + 2, /* iVersion */ + sizeof(devsym_file), /* szOsFile */ + DEVSYM_MAX_PATHNAME, /* mxPathname */ + 0, /* pNext */ + DEVSYM_VFS_NAME, /* zName */ + 0, /* pAppData */ + devsymOpen, /* xOpen */ + devsymDelete, /* xDelete */ + devsymAccess, /* xAccess */ + devsymFullPathname, /* xFullPathname */ +#ifndef SQLITE_OMIT_LOAD_EXTENSION + devsymDlOpen, /* xDlOpen */ + devsymDlError, /* xDlError */ + devsymDlSym, /* xDlSym */ + devsymDlClose, /* xDlClose */ +#else + 0, /* xDlOpen */ + 0, /* xDlError */ + 0, /* xDlSym */ + 0, /* xDlClose */ +#endif /* SQLITE_OMIT_LOAD_EXTENSION */ + devsymRandomness, /* xRandomness */ + devsymSleep, /* xSleep */ + devsymCurrentTime, /* xCurrentTime */ + 0, /* xGetLastError */ + 0 /* xCurrentTimeInt64 */ +}; + +static sqlite3_vfs writecrash_vfs = { + 2, /* iVersion */ + sizeof(devsym_file), /* szOsFile */ + DEVSYM_MAX_PATHNAME, /* mxPathname */ + 0, /* pNext */ + WRITECRASH_NAME, /* zName */ + 0, /* pAppData */ + writecrashOpen, /* xOpen */ + devsymDelete, /* xDelete */ + devsymAccess, /* xAccess */ + devsymFullPathname, /* xFullPathname */ +#ifndef SQLITE_OMIT_LOAD_EXTENSION + devsymDlOpen, /* xDlOpen */ + devsymDlError, /* xDlError */ + devsymDlSym, /* xDlSym */ + devsymDlClose, /* xDlClose */ +#else + 0, /* xDlOpen */ + 0, /* xDlError */ + 0, /* xDlSym */ + 0, /* xDlClose */ +#endif /* SQLITE_OMIT_LOAD_EXTENSION */ + devsymRandomness, /* xRandomness */ + devsymSleep, /* xSleep */ + devsymCurrentTime, /* xCurrentTime */ + 0, /* xGetLastError */ + 0 /* xCurrentTimeInt64 */ +}; + /* ** This procedure registers the devsym vfs with SQLite. If the argument is ** true, the devsym vfs becomes the new default vfs. It is the only publicly ** available function in this file. */ void devsym_register(int iDeviceChar, int iSectorSize){ + if( g.pVfs==0 ){ g.pVfs = sqlite3_vfs_find(0); devsym_vfs.szOsFile += g.pVfs->szOsFile; + writecrash_vfs.szOsFile += g.pVfs->szOsFile; sqlite3_vfs_register(&devsym_vfs, 0); + sqlite3_vfs_register(&writecrash_vfs, 0); } if( iDeviceChar>=0 ){ g.iDeviceChar = iDeviceChar; }else{ g.iDeviceChar = 0; @@ -400,7 +507,18 @@ sqlite3_vfs_unregister(&devsym_vfs); g.pVfs = 0; g.iDeviceChar = 0; g.iSectorSize = 0; } + +void devsym_crash_on_write(int nWrite){ + if( g.pVfs==0 ){ + g.pVfs = sqlite3_vfs_find(0); + devsym_vfs.szOsFile += g.pVfs->szOsFile; + writecrash_vfs.szOsFile += g.pVfs->szOsFile; + sqlite3_vfs_register(&devsym_vfs, 0); + sqlite3_vfs_register(&writecrash_vfs, 0); + } + g.nWriteCrash = nWrite; +} #endif Index: src/test_malloc.c ================================================================== --- src/test_malloc.c +++ src/test_malloc.c @@ -885,50 +885,10 @@ } return TCL_OK; } -/* -** Usage: sqlite3_config_scratch SIZE N -** -** Set the scratch memory buffer using SQLITE_CONFIG_SCRATCH. -** The buffer is static and is of limited size. N might be -** adjusted downward as needed to accommodate the requested size. -** The revised value of N is returned. -** -** A negative SIZE causes the buffer pointer to be NULL. -*/ -static int SQLITE_TCLAPI test_config_scratch( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - int sz, N, rc; - Tcl_Obj *pResult; - static char *buf = 0; - if( objc!=3 ){ - Tcl_WrongNumArgs(interp, 1, objv, "SIZE N"); - return TCL_ERROR; - } - if( Tcl_GetIntFromObj(interp, objv[1], &sz) ) return TCL_ERROR; - if( Tcl_GetIntFromObj(interp, objv[2], &N) ) return TCL_ERROR; - free(buf); - if( sz<0 ){ - buf = 0; - rc = sqlite3_config(SQLITE_CONFIG_SCRATCH, (void*)0, 0, 0); - }else{ - buf = malloc( sz*N + 1 ); - rc = sqlite3_config(SQLITE_CONFIG_SCRATCH, buf, sz, N); - } - pResult = Tcl_NewObj(); - Tcl_ListObjAppendElement(0, pResult, Tcl_NewIntObj(rc)); - Tcl_ListObjAppendElement(0, pResult, Tcl_NewIntObj(N)); - Tcl_SetObjResult(interp, pResult); - return TCL_OK; -} - /* ** Usage: sqlite3_config_pagecache SIZE N ** ** Set the page-cache memory buffer using SQLITE_CONFIG_PAGECACHE. ** The buffer is static and is of limited size. N might be @@ -1536,11 +1496,10 @@ { "sqlite3_memdebug_fail", test_memdebug_fail ,0 }, { "sqlite3_memdebug_pending", test_memdebug_pending ,0 }, { "sqlite3_memdebug_settitle", test_memdebug_settitle ,0 }, { "sqlite3_memdebug_malloc_count", test_memdebug_malloc_count ,0 }, { "sqlite3_memdebug_log", test_memdebug_log ,0 }, - { "sqlite3_config_scratch", test_config_scratch ,0 }, { "sqlite3_config_pagecache", test_config_pagecache ,0 }, { "sqlite3_config_alt_pcache", test_alt_pcache ,0 }, { "sqlite3_status", test_status ,0 }, { "sqlite3_db_status", test_db_status ,0 }, { "install_malloc_faultsim", test_install_malloc_faultsim ,0 }, Index: src/test_tclvar.c ================================================================== --- src/test_tclvar.c +++ src/test_tclvar.c @@ -13,10 +13,29 @@ ** is not included in the SQLite library. It is used for automated ** testing of the SQLite library. ** ** The emphasis of this file is a virtual table that provides ** access to TCL variables. +** +** The TCLVAR eponymous virtual table has a schema like this: +** +** CREATE TABLE tclvar( +** name TEXT, -- base name of the variable: "x" in "$x(y)" +** arrayname TEXT, -- array index name: "y" in "$x(y)" +** value TEXT, -- the value of the variable +** fullname TEXT, -- the full name of the variable +** PRIMARY KEY(fullname) +** ) WITHOUT ROWID; +** +** DELETE, INSERT, and UPDATE operations use the "fullname" field to +** determine the variable to be modified. Changing "value" to NULL +** deletes the variable. +** +** For SELECT operations, the "name" and "arrayname" fields will always +** match the "fullname" field. For DELETE, INSERT, and UPDATE, the +** "name" and "arrayname" fields are ignored and the variable is modified +** according to "fullname" and "value" only. */ #include "sqliteInt.h" #if defined(INCLUDE_SQLITE_TCL_H) # include "sqlite_tcl.h" #else @@ -65,11 +84,16 @@ sqlite3_vtab **ppVtab, char **pzErr ){ tclvar_vtab *pVtab; static const char zSchema[] = - "CREATE TABLE whatever(name TEXT, arrayname TEXT, value TEXT)"; + "CREATE TABLE x(" + " name TEXT," /* Base name */ + " arrayname TEXT," /* Array index */ + " value TEXT," /* Value */ + " fullname TEXT PRIMARY KEY" /* base(index) name */ + ") WITHOUT ROWID"; pVtab = sqlite3MallocZero( sizeof(*pVtab) ); if( pVtab==0 ) return SQLITE_NOMEM; *ppVtab = &pVtab->base; pVtab->interp = (Tcl_Interp *)pAux; sqlite3_declare_vtab(db, zSchema); @@ -249,10 +273,20 @@ case 2: { Tcl_Obj *pVal = Tcl_GetVar2Ex(interp, z1, *z2?z2:0, TCL_GLOBAL_ONLY); sqlite3_result_text(ctx, Tcl_GetString(pVal), -1, SQLITE_TRANSIENT); break; } + case 3: { + char *z3; + if( p2 ){ + z3 = sqlite3_mprintf("%s(%s)", z1, z2); + sqlite3_result_text(ctx, z3, -1, sqlite3_free); + }else{ + sqlite3_result_text(ctx, z1, -1, SQLITE_TRANSIENT); + } + break; + } } return SQLITE_OK; } static int tclvarRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ @@ -373,10 +407,62 @@ pIdxInfo->idxStr = zStr; pIdxInfo->needToFreeIdxStr = 1; return SQLITE_OK; } + +/* +** Invoked for any UPDATE, INSERT, or DELETE against a tclvar table +*/ +static int tclvarUpdate( + sqlite3_vtab *tab, + int argc, + sqlite3_value **argv, + sqlite_int64 *pRowid +){ + tclvar_vtab *pTab = (tclvar_vtab*)tab; + if( argc==1 ){ + /* A DELETE operation. The variable to be deleted is stored in argv[0] */ + const char *zVar = (const char*)sqlite3_value_text(argv[0]); + Tcl_UnsetVar(pTab->interp, zVar, TCL_GLOBAL_ONLY); + return SQLITE_OK; + } + if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + /* An INSERT operation */ + const char *zValue = (const char*)sqlite3_value_text(argv[4]); + const char *zName; + if( sqlite3_value_type(argv[5])!=SQLITE_TEXT ){ + tab->zErrMsg = sqlite3_mprintf("the 'fullname' column must be TEXT"); + return SQLITE_ERROR; + } + zName = (const char*)sqlite3_value_text(argv[5]); + if( zValue ){ + Tcl_SetVar(pTab->interp, zName, zValue, TCL_GLOBAL_ONLY); + }else{ + Tcl_UnsetVar(pTab->interp, zName, TCL_GLOBAL_ONLY); + } + return SQLITE_OK; + } + if( sqlite3_value_type(argv[0])==SQLITE_TEXT + && sqlite3_value_type(argv[1])==SQLITE_TEXT + ){ + /* An UPDATE operation */ + const char *zOldName = (const char*)sqlite3_value_text(argv[0]); + const char *zNewName = (const char*)sqlite3_value_text(argv[1]); + const char *zValue = (const char*)sqlite3_value_text(argv[4]); + + if( strcmp(zOldName, zNewName)!=0 || zValue==0 ){ + Tcl_UnsetVar(pTab->interp, zOldName, TCL_GLOBAL_ONLY); + } + if( zValue!=0 ){ + Tcl_SetVar(pTab->interp, zNewName, zValue, TCL_GLOBAL_ONLY); + } + return SQLITE_OK; + } + tab->zErrMsg = sqlite3_mprintf("prohibited TCL variable change"); + return SQLITE_ERROR; +} /* ** A virtual table module that provides read-only access to a ** Tcl global variable namespace. */ @@ -392,11 +478,11 @@ tclvarFilter, /* xFilter - configure scan constraints */ tclvarNext, /* xNext - advance a cursor */ tclvarEof, /* xEof - check for end of scan */ tclvarColumn, /* xColumn - read data */ tclvarRowid, /* xRowid - read data */ - 0, /* xUpdate */ + tclvarUpdate, /* xUpdate */ 0, /* xBegin */ 0, /* xSync */ 0, /* xCommit */ 0, /* xRollback */ 0, /* xFindMethod */ Index: src/trigger.c ================================================================== --- src/trigger.c +++ src/trigger.c @@ -583,11 +583,11 @@ Trigger **pp; for(pp=&pTab->pTrigger; *pp!=pTrigger; pp=&((*pp)->pNext)); *pp = (*pp)->pNext; } sqlite3DeleteTrigger(db, pTrigger); - db->flags |= SQLITE_InternChanges; + db->mDbFlags |= DBFLAG_SchemaChange; } } /* ** pEList is the SET clause of an UPDATE statement. Each entry Index: src/update.c ================================================================== --- src/update.c +++ src/update.c @@ -801,23 +801,34 @@ /* Start scanning the virtual table */ pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0,0,WHERE_ONEPASS_DESIRED,0); if( pWInfo==0 ) return; /* Populate the argument registers. */ - sqlite3VdbeAddOp2(v, OP_Rowid, iCsr, regArg); - if( pRowid ){ - sqlite3ExprCode(pParse, pRowid, regArg+1); - }else{ - sqlite3VdbeAddOp2(v, OP_Rowid, iCsr, regArg+1); - } for(i=0; inCol; i++){ if( aXRef[i]>=0 ){ sqlite3ExprCode(pParse, pChanges->a[aXRef[i]].pExpr, regArg+2+i); }else{ sqlite3VdbeAddOp3(v, OP_VColumn, iCsr, i, regArg+2+i); } } + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp2(v, OP_Rowid, iCsr, regArg); + if( pRowid ){ + sqlite3ExprCode(pParse, pRowid, regArg+1); + }else{ + sqlite3VdbeAddOp2(v, OP_Rowid, iCsr, regArg+1); + } + }else{ + Index *pPk; /* PRIMARY KEY index */ + i16 iPk; /* PRIMARY KEY column */ + pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pPk!=0 ); + assert( pPk->nKeyCol==1 ); + iPk = pPk->aiColumn[0]; + sqlite3VdbeAddOp3(v, OP_VColumn, iCsr, iPk, regArg); + sqlite3VdbeAddOp2(v, OP_SCopy, regArg+2+iPk, regArg+1); + } bOnePass = sqlite3WhereOkOnePass(pWInfo, aDummy); if( bOnePass ){ /* If using the onepass strategy, no-op out the OP_OpenEphemeral coded Index: src/util.c ================================================================== --- src/util.c +++ src/util.c @@ -1411,12 +1411,18 @@ LogEst y = 40; if( x<8 ){ if( x<2 ) return 0; while( x<8 ){ y -= 10; x <<= 1; } }else{ +#if GCC_VERSION>=5004000 + int i = 60 - __builtin_clzll(x); + y += i*10; + x >>= i; +#else while( x>255 ){ y += 40; x >>= 4; } /*OPTIMIZATION-IF-TRUE*/ while( x>15 ){ y += 10; x >>= 1; } +#endif } return a[x&7] + y - 10; } #ifndef SQLITE_OMIT_VIRTUALTABLE Index: src/vacuum.c ================================================================== --- src/vacuum.c +++ src/vacuum.c @@ -128,11 +128,12 @@ */ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){ int rc = SQLITE_OK; /* Return code from service routines */ Btree *pMain; /* The database being vacuumed */ Btree *pTemp; /* The temporary database we vacuum into */ - int saved_flags; /* Saved value of the db->flags */ + u16 saved_mDbFlags; /* Saved value of db->mDbFlags */ + u32 saved_flags; /* Saved value of db->flags */ int saved_nChange; /* Saved value of db->nChange */ int saved_nTotalChange; /* Saved value of db->nTotalChange */ u8 saved_mTrace; /* Saved trace settings */ Db *pDb = 0; /* Database to detach at end of vacuum */ int isMemDb; /* True if vacuuming a :memory: database */ @@ -151,15 +152,16 @@ /* Save the current value of the database flags so that it can be ** restored before returning. Then set the writable-schema flag, and ** disable CHECK and foreign key constraints. */ saved_flags = db->flags; + saved_mDbFlags = db->mDbFlags; saved_nChange = db->nChange; saved_nTotalChange = db->nTotalChange; saved_mTrace = db->mTrace; - db->flags |= (SQLITE_WriteSchema | SQLITE_IgnoreChecks - | SQLITE_PreferBuiltin | SQLITE_Vacuum); + db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks; + db->mDbFlags |= DBFLAG_PreferBuiltin | DBFLAG_Vacuum; db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder | SQLITE_CountRows); db->mTrace = 0; zDbMain = db->aDb[iDb].zDbSName; pMain = db->aDb[iDb].pBt; @@ -266,12 +268,12 @@ "||' SELECT*FROM\"%w\".'||quote(name)" "FROM vacuum_db.sqlite_master " "WHERE type='table'AND coalesce(rootpage,1)>0", zDbMain ); - assert( (db->flags & SQLITE_Vacuum)!=0 ); - db->flags &= ~SQLITE_Vacuum; + assert( (db->mDbFlags & DBFLAG_Vacuum)!=0 ); + db->mDbFlags &= ~DBFLAG_Vacuum; if( rc!=SQLITE_OK ) goto end_of_vacuum; /* Copy the triggers, views, and virtual tables from the main database ** over to the temporary database. None of these objects has any ** associated storage, so all we have to do is copy their entries @@ -335,10 +337,11 @@ rc = sqlite3BtreeSetPageSize(pMain, sqlite3BtreeGetPageSize(pTemp), nRes,1); end_of_vacuum: /* Restore the original value of db->flags */ db->init.iDb = 0; + db->mDbFlags = saved_mDbFlags; db->flags = saved_flags; db->nChange = saved_nChange; db->nTotalChange = saved_nTotalChange; db->mTrace = saved_mTrace; sqlite3BtreeSetPageSize(pMain, -1, -1, 1); Index: src/vdbe.c ================================================================== --- src/vdbe.c +++ src/vdbe.c @@ -1589,121 +1589,10 @@ sqlite3VdbeMemSetInt64(&aMem[pOp->p1], 0); } break; } -/* Opcode: Function0 P1 P2 P3 P4 P5 -** Synopsis: r[P3]=func(r[P2@P5]) -** -** Invoke a user function (P4 is a pointer to a FuncDef object that -** defines the function) with P5 arguments taken from register P2 and -** successors. The result of the function is stored in register P3. -** Register P3 must not be one of the function inputs. -** -** P1 is a 32-bit bitmask indicating whether or not each argument to the -** function was determined to be constant at compile time. If the first -** argument was constant then bit 0 of P1 is set. This is used to determine -** whether meta data associated with a user function argument using the -** sqlite3_set_auxdata() API may be safely retained until the next -** invocation of this opcode. -** -** See also: Function, AggStep, AggFinal -*/ -/* Opcode: Function P1 P2 P3 P4 P5 -** Synopsis: r[P3]=func(r[P2@P5]) -** -** Invoke a user function (P4 is a pointer to an sqlite3_context object that -** contains a pointer to the function to be run) with P5 arguments taken -** from register P2 and successors. The result of the function is stored -** in register P3. Register P3 must not be one of the function inputs. -** -** P1 is a 32-bit bitmask indicating whether or not each argument to the -** function was determined to be constant at compile time. If the first -** argument was constant then bit 0 of P1 is set. This is used to determine -** whether meta data associated with a user function argument using the -** sqlite3_set_auxdata() API may be safely retained until the next -** invocation of this opcode. -** -** SQL functions are initially coded as OP_Function0 with P4 pointing -** to a FuncDef object. But on first evaluation, the P4 operand is -** automatically converted into an sqlite3_context object and the operation -** changed to this OP_Function opcode. In this way, the initialization of -** the sqlite3_context object occurs only once, rather than once for each -** evaluation of the function. -** -** See also: Function0, AggStep, AggFinal -*/ -case OP_Function0: { - int n; - sqlite3_context *pCtx; - - assert( pOp->p4type==P4_FUNCDEF ); - n = pOp->p5; - assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); - assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem+1 - p->nCursor)+1) ); - assert( pOp->p3p2 || pOp->p3>=pOp->p2+n ); - pCtx = sqlite3DbMallocRawNN(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*)); - if( pCtx==0 ) goto no_mem; - pCtx->pOut = 0; - pCtx->pFunc = pOp->p4.pFunc; - pCtx->iOp = (int)(pOp - aOp); - pCtx->pVdbe = p; - pCtx->argc = n; - pOp->p4type = P4_FUNCCTX; - pOp->p4.pCtx = pCtx; - pOp->opcode = OP_Function; - /* Fall through into OP_Function */ -} -case OP_Function: { - int i; - sqlite3_context *pCtx; - - assert( pOp->p4type==P4_FUNCCTX ); - pCtx = pOp->p4.pCtx; - - /* If this function is inside of a trigger, the register array in aMem[] - ** might change from one evaluation to the next. The next block of code - ** checks to see if the register array has changed, and if so it - ** reinitializes the relavant parts of the sqlite3_context object */ - pOut = &aMem[pOp->p3]; - if( pCtx->pOut != pOut ){ - pCtx->pOut = pOut; - for(i=pCtx->argc-1; i>=0; i--) pCtx->argv[i] = &aMem[pOp->p2+i]; - } - - memAboutToChange(p, pOut); -#ifdef SQLITE_DEBUG - for(i=0; iargc; i++){ - assert( memIsValid(pCtx->argv[i]) ); - REGISTER_TRACE(pOp->p2+i, pCtx->argv[i]); - } -#endif - MemSetTypeFlag(pOut, MEM_Null); - pCtx->fErrorOrAux = 0; - (*pCtx->pFunc->xSFunc)(pCtx, pCtx->argc, pCtx->argv);/* IMP: R-24505-23230 */ - - /* If the function returned an error, throw an exception */ - if( pCtx->fErrorOrAux ){ - if( pCtx->isError ){ - sqlite3VdbeError(p, "%s", sqlite3_value_text(pOut)); - rc = pCtx->isError; - } - sqlite3VdbeDeleteAuxData(db, &p->pAuxData, pCtx->iOp, pOp->p1); - if( rc ) goto abort_due_to_error; - } - - /* Copy the result of the function into register P3 */ - if( pOut->flags & (MEM_Str|MEM_Blob) ){ - sqlite3VdbeChangeEncoding(pOut, encoding); - if( sqlite3VdbeMemTooBig(pOut) ) goto too_big; - } - - REGISTER_TRACE(pOp->p3, pOut); - UPDATE_MAX_BLOBSIZE(pOut); - break; -} - /* Opcode: BitAnd P1 P2 P3 * * ** Synopsis: r[P3]=r[P1]&r[P2] ** ** Take the bit-wise AND of the values in register P1 and P2 and ** store the result in register P3. @@ -2065,17 +1954,27 @@ } assert( pOp->p4type==P4_COLLSEQ || pOp->p4.pColl==0 ); res = sqlite3MemCompare(pIn3, pIn1, pOp->p4.pColl); } compare_op: - switch( pOp->opcode ){ - case OP_Eq: res2 = res==0; break; - case OP_Ne: res2 = res; break; - case OP_Lt: res2 = res<0; break; - case OP_Le: res2 = res<=0; break; - case OP_Gt: res2 = res>0; break; - default: res2 = res>=0; break; + /* At this point, res is negative, zero, or positive if reg[P1] is + ** less than, equal to, or greater than reg[P3], respectively. Compute + ** the answer to this operator in res2, depending on what the comparison + ** operator actually is. The next block of code depends on the fact + ** that the 6 comparison operators are consecutive integers in this + ** order: NE, EQ, GT, LE, LT, GE */ + assert( OP_Eq==OP_Ne+1 ); assert( OP_Gt==OP_Ne+2 ); assert( OP_Le==OP_Ne+3 ); + assert( OP_Lt==OP_Ne+4 ); assert( OP_Ge==OP_Ne+5 ); + if( res<0 ){ /* ne, eq, gt, le, lt, ge */ + static const unsigned char aLTb[] = { 1, 0, 0, 1, 1, 0 }; + res2 = aLTb[pOp->opcode - OP_Ne]; + }else if( res==0 ){ + static const unsigned char aEQb[] = { 0, 1, 0, 1, 0, 1 }; + res2 = aEQb[pOp->opcode - OP_Ne]; + }else{ + static const unsigned char aGTb[] = { 1, 0, 1, 0, 0, 1 }; + res2 = aGTb[pOp->opcode - OP_Ne]; } /* Undo any changes made by applyAffinity() to the input registers. */ assert( (pIn1->flags & MEM_Dyn) == (flags1 & MEM_Dyn) ); pIn1->flags = flags1; @@ -2083,11 +1982,10 @@ pIn3->flags = flags3; if( pOp->p5 & SQLITE_STOREP2 ){ pOut = &aMem[pOp->p2]; iCompare = res; - res2 = res2!=0; /* For this path res2 must be exactly 0 or 1 */ if( (pOp->p5 & SQLITE_KEEPNULL)!=0 ){ /* The KEEPNULL flag prevents OP_Eq from overwriting a NULL with 1 ** and prevents OP_Ne from overwriting NULL with 0. This flag ** is only used in contexts where either: ** (1) op==OP_Eq && (r[P2]==NULL || r[P2]==0) @@ -2214,11 +2112,11 @@ idx = aPermute ? aPermute[i] : i; assert( memIsValid(&aMem[p1+idx]) ); assert( memIsValid(&aMem[p2+idx]) ); REGISTER_TRACE(p1+idx, &aMem[p1+idx]); REGISTER_TRACE(p2+idx, &aMem[p2+idx]); - assert( inField ); + assert( inKeyField ); pColl = pKeyInfo->aColl[i]; bRev = pKeyInfo->aSortOrder[i]; iCompare = sqlite3MemCompare(&aMem[p1+idx], &aMem[p2+idx], pColl); if( iCompare ){ if( bRev ) iCompare = -iCompare; @@ -2487,13 +2385,11 @@ Mem *pDest; /* Where to write the extracted value */ Mem sMem; /* For storing the record being decoded */ const u8 *zData; /* Part of the record being decoded */ const u8 *zHdr; /* Next unparsed byte of the header */ const u8 *zEndHdr; /* Pointer to first byte after the header */ - u32 offset; /* Offset into the data */ u64 offset64; /* 64-bit offset */ - u32 avail; /* Number of bytes of available data */ u32 t; /* A type code from the record header */ Mem *pReg; /* PseudoTable input register */ pC = p->apCsr[pOp->p1]; p2 = pOp->p2; @@ -2516,15 +2412,17 @@ assert( pC->eCurType!=CURTYPE_SORTER ); if( pC->cacheStatus!=p->cacheCtr ){ /*OPTIMIZATION-IF-FALSE*/ if( pC->nullRow ){ if( pC->eCurType==CURTYPE_PSEUDO ){ - assert( pC->uc.pseudoTableReg>0 ); - pReg = &aMem[pC->uc.pseudoTableReg]; + /* For the special case of as pseudo-cursor, the seekResult field + ** identifies the register that holds the record */ + assert( pC->seekResult>0 ); + pReg = &aMem[pC->seekResult]; assert( pReg->flags & MEM_Blob ); assert( memIsValid(pReg) ); - pC->payloadSize = pC->szRow = avail = pReg->n; + pC->payloadSize = pC->szRow = pReg->n; pC->aRow = (u8*)pReg->z; }else{ sqlite3VdbeMemSetNull(pDest); goto op_column_out; } @@ -2532,27 +2430,23 @@ pCrsr = pC->uc.pCursor; assert( pC->eCurType==CURTYPE_BTREE ); assert( pCrsr ); assert( sqlite3BtreeCursorIsValid(pCrsr) ); pC->payloadSize = sqlite3BtreePayloadSize(pCrsr); - pC->aRow = sqlite3BtreePayloadFetch(pCrsr, &avail); - assert( avail<=65536 ); /* Maximum page size is 64KiB */ - if( pC->payloadSize <= (u32)avail ){ - pC->szRow = pC->payloadSize; - }else if( pC->payloadSize > (u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){ + pC->aRow = sqlite3BtreePayloadFetch(pCrsr, &pC->szRow); + assert( pC->szRow<=pC->payloadSize ); + assert( pC->szRow<=65536 ); /* Maximum page size is 64KiB */ + if( pC->payloadSize > (u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){ goto too_big; - }else{ - pC->szRow = avail; } } pC->cacheStatus = p->cacheCtr; - pC->iHdrOffset = getVarint32(pC->aRow, offset); + pC->iHdrOffset = getVarint32(pC->aRow, aOffset[0]); pC->nHdrParsed = 0; - aOffset[0] = offset; - if( availszRowaRow does not have to hold the entire row, but it does at least ** need to cover the header of the record. If pC->aRow does not contain ** the complete header, then set it to zero, forcing the header to be ** dynamically allocated. */ pC->aRow = 0; @@ -2565,21 +2459,31 @@ ** types use so much data space that there can only be 4096 and 32 of ** them, respectively. So the maximum header length results from a ** 3-byte type for each of the maximum of 32768 columns plus three ** extra bytes for the header length itself. 32768*3 + 3 = 98307. */ - if( offset > 98307 || offset > pC->payloadSize ){ + if( aOffset[0] > 98307 || aOffset[0] > pC->payloadSize ){ rc = SQLITE_CORRUPT_BKPT; goto abort_due_to_error; } - }else if( offset>0 ){ /*OPTIMIZATION-IF-TRUE*/ - /* The following goto is an optimization. It can be omitted and - ** everything will still work. But OP_Column is measurably faster - ** by skipping the subsequent conditional, which is always true. + }else{ + /* This is an optimization. By skipping over the first few tests + ** (ex: pC->nHdrParsed<=p2) in the next section, we achieve a + ** measurable performance gain. + ** + ** This branch is taken even if aOffset[0]==0. Such a record is never + ** generated by SQLite, and could be considered corruption, but we + ** accept it for historical reasons. When aOffset[0]==0, the code this + ** branch jumps to reads past the end of the record, but never more + ** than a few bytes. Even if the record occurs at the end of the page + ** content area, the "page header" comes after the page content and so + ** this overread is harmless. Similar overreads can occur for a corrupt + ** database file. */ zData = pC->aRow; assert( pC->nHdrParsed<=p2 ); /* Conditional skipped */ + testcase( aOffset[0]==0 ); goto op_column_read_header; } } /* Make sure at least the first p2+1 entries of the header have been @@ -2604,10 +2508,11 @@ op_column_read_header: i = pC->nHdrParsed; offset64 = aOffset[i]; zHdr = zData + pC->iHdrOffset; zEndHdr = zData + aOffset[0]; + testcase( zHdr>=zEndHdr ); do{ if( (t = zHdr[0])<0x80 ){ zHdr++; offset64 += sqlite3VdbeOneByteSerialTypeLen(t); }else{ @@ -2624,13 +2529,18 @@ ** (3) the end of the data extends beyond the end of the record. */ if( (zHdr>=zEndHdr && (zHdr>zEndHdr || offset64!=pC->payloadSize)) || (offset64 > pC->payloadSize) ){ - if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem); - rc = SQLITE_CORRUPT_BKPT; - goto abort_due_to_error; + if( aOffset[0]==0 ){ + i = 0; + zHdr = zEndHdr; + }else{ + if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem); + rc = SQLITE_CORRUPT_BKPT; + goto abort_due_to_error; + } } pC->nHdrParsed = i; pC->iHdrOffset = (u32)(zHdr - zData); if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem); @@ -3060,11 +2970,11 @@ rc = p->rc; }else{ int isSchemaChange; iSavepoint = db->nSavepoint - iSavepoint - 1; if( p1==SAVEPOINT_ROLLBACK ){ - isSchemaChange = (db->flags & SQLITE_InternChanges)!=0; + isSchemaChange = (db->mDbFlags & DBFLAG_SchemaChange)!=0; for(ii=0; iinDb; ii++){ rc = sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT_ROLLBACK, isSchemaChange==0); if( rc!=SQLITE_OK ) goto abort_due_to_error; @@ -3079,11 +2989,11 @@ } } if( isSchemaChange ){ sqlite3ExpirePreparedStatements(db); sqlite3ResetAllSchemasOfConnection(db); - db->flags = (db->flags | SQLITE_InternChanges); + db->mDbFlags |= DBFLAG_SchemaChange; } } /* Regardless of whether this is a RELEASE or ROLLBACK, destroy all ** savepoints nested inside of the savepoint being operated on. */ @@ -3359,11 +3269,11 @@ /* See note about index shifting on OP_ReadCookie */ rc = sqlite3BtreeUpdateMeta(pDb->pBt, pOp->p2, pOp->p3); if( pOp->p2==BTREE_SCHEMA_VERSION ){ /* When the schema cookie changes, record the new cookie internally */ pDb->pSchema->schema_cookie = pOp->p3; - db->flags |= SQLITE_InternChanges; + db->mDbFlags |= DBFLAG_SchemaChange; }else if( pOp->p2==BTREE_FILE_FORMAT ){ /* Record changes in the file format */ pDb->pSchema->file_format = pOp->p3; } if( pOp->p1==1 ){ @@ -3498,21 +3408,21 @@ pIn2 = &aMem[p2]; assert( memIsValid(pIn2) ); assert( (pIn2->flags & MEM_Int)!=0 ); sqlite3VdbeMemIntegerify(pIn2); p2 = (int)pIn2->u.i; - /* The p2 value always comes from a prior OP_CreateTable opcode and + /* The p2 value always comes from a prior OP_CreateBtree opcode and ** that opcode will always set the p2 value to 2 or more or else fail. ** If there were a failure, the prepared statement would have halted ** before reaching this instruction. */ assert( p2>=2 ); } if( pOp->p4type==P4_KEYINFO ){ pKeyInfo = pOp->p4.pKeyInfo; assert( pKeyInfo->enc==ENC(db) ); assert( pKeyInfo->db==db ); - nField = pKeyInfo->nField+pKeyInfo->nXField; + nField = pKeyInfo->nAllField; }else if( pOp->p4type==P4_INT32 ){ nField = pOp->p4.i; } assert( pOp->p1>=0 ); assert( nField>=0 ); @@ -3719,12 +3629,17 @@ assert( pOp->p1>=0 ); assert( pOp->p3>=0 ); pCx = allocateCursor(p, pOp->p1, pOp->p3, -1, CURTYPE_PSEUDO); if( pCx==0 ) goto no_mem; pCx->nullRow = 1; - pCx->uc.pseudoTableReg = pOp->p2; + pCx->seekResult = pOp->p2; pCx->isTable = 1; + /* Give this pseudo-cursor a fake BtCursor pointer so that pCx + ** can be safely passed to sqlite3VdbeCursorMoveto(). This avoids a test + ** for pCx->eCurType==CURTYPE_BTREE inside of sqlite3VdbeCursorMoveto() + ** which is a performance optimization */ + pCx->uc.pCursor = sqlite3BtreeFakeValidCursor(); assert( pOp->p5==0 ); break; } /* Opcode: Close P1 * * * * @@ -4886,11 +4801,21 @@ sqlite3BtreeClearCursor(pC->uc.pCursor); } break; } -/* Opcode: Last P1 P2 P3 * * +/* Opcode: SeekEnd P1 * * * * +** +** Position cursor P1 at the end of the btree for the purpose of +** appending a new entry onto the btree. +** +** It is assumed that the cursor is used only for appending and so +** if the cursor is valid, then the cursor must already be pointing +** at the end of the btree and so no changes are made to +** the cursor. +*/ +/* Opcode: Last P1 P2 * * * ** ** The next use of the Rowid or Column or Prev instruction for P1 ** will refer to the last entry in the database table or index. ** If the table or index is empty and P2>0, then jump immediately to P2. ** If P2 is 0 or if the table or index is not empty, fall through @@ -4897,18 +4822,12 @@ ** to the following instruction. ** ** This opcode leaves the cursor configured to move in reverse order, ** from the end toward the beginning. In other words, the cursor is ** configured to use Prev, not Next. -** -** If P3 is -1, then the cursor is positioned at the end of the btree -** for the purpose of appending a new entry onto the btree. In that -** case P2 must be 0. It is assumed that the cursor is used only for -** appending and so if the cursor is valid, then the cursor must already -** be pointing at the end of the btree and so no changes are made to -** the cursor. */ +case OP_SeekEnd: case OP_Last: { /* jump */ VdbeCursor *pC; BtCursor *pCrsr; int res; @@ -4917,26 +4836,28 @@ assert( pC!=0 ); assert( pC->eCurType==CURTYPE_BTREE ); pCrsr = pC->uc.pCursor; res = 0; assert( pCrsr!=0 ); - pC->seekResult = pOp->p3; #ifdef SQLITE_DEBUG - pC->seekOp = OP_Last; + pC->seekOp = pOp->opcode; #endif - if( pOp->p3==0 || !sqlite3BtreeCursorIsValidNN(pCrsr) ){ - rc = sqlite3BtreeLast(pCrsr, &res); - pC->nullRow = (u8)res; - pC->deferredMoveto = 0; - pC->cacheStatus = CACHE_STALE; - if( rc ) goto abort_due_to_error; - if( pOp->p2>0 ){ - VdbeBranchTaken(res!=0,2); - if( res ) goto jump_to_p2; - } - }else{ + if( pOp->opcode==OP_SeekEnd ){ assert( pOp->p2==0 ); + pC->seekResult = -1; + if( sqlite3BtreeCursorIsValidNN(pCrsr) ){ + break; + } + } + rc = sqlite3BtreeLast(pCrsr, &res); + pC->nullRow = (u8)res; + pC->deferredMoveto = 0; + pC->cacheStatus = CACHE_STALE; + if( rc ) goto abort_due_to_error; + if( pOp->p2>0 ){ + VdbeBranchTaken(res!=0,2); + if( res ) goto jump_to_p2; } break; } /* Opcode: IfSmaller P1 P2 P3 * * @@ -5561,54 +5482,32 @@ if( rc ) goto abort_due_to_error; } break; } -/* Opcode: CreateTable P1 P2 * * * -** Synopsis: r[P2]=root iDb=P1 -** -** Allocate a new table in the main database file if P1==0 or in the -** auxiliary database file if P1==1 or in an attached database if -** P1>1. Write the root page number of the new table into -** register P2 -** -** The difference between a table and an index is this: A table must -** have a 4-byte integer key and can have arbitrary data. An index -** has an arbitrary key but no data. -** -** See also: CreateIndex -*/ -/* Opcode: CreateIndex P1 P2 * * * -** Synopsis: r[P2]=root iDb=P1 -** -** Allocate a new index in the main database file if P1==0 or in the -** auxiliary database file if P1==1 or in an attached database if -** P1>1. Write the root page number of the new table into -** register P2. -** -** See documentation on OP_CreateTable for additional information. -*/ -case OP_CreateIndex: /* out2 */ -case OP_CreateTable: { /* out2 */ - int pgno; - int flags; +/* Opcode: CreateBtree P1 P2 P3 * * +** Synopsis: r[P2]=root iDb=P1 flags=P3 +** +** Allocate a new b-tree in the main database file if P1==0 or in the +** TEMP database file if P1==1 or in an attached database if +** P1>1. The P3 argument must be 1 (BTREE_INTKEY) for a rowid table +** it must be 2 (BTREE_BLOBKEY) for a index or WITHOUT ROWID table. +** The root page number of the new b-tree is stored in register P2. +*/ +case OP_CreateBtree: { /* out2 */ + int pgno; Db *pDb; pOut = out2Prerelease(p, pOp); pgno = 0; + assert( pOp->p3==BTREE_INTKEY || pOp->p3==BTREE_BLOBKEY ); assert( pOp->p1>=0 && pOp->p1nDb ); assert( DbMaskTest(p->btreeMask, pOp->p1) ); assert( p->readOnly==0 ); pDb = &db->aDb[pOp->p1]; assert( pDb->pBt!=0 ); - if( pOp->opcode==OP_CreateTable ){ - /* flags = BTREE_INTKEY; */ - flags = BTREE_INTKEY; - }else{ - flags = BTREE_BLOBKEY; - } - rc = sqlite3BtreeCreateTable(pDb->pBt, &pgno, flags); + rc = sqlite3BtreeCreateTable(pDb->pBt, &pgno, pOp->p3); if( rc ) goto abort_due_to_error; pOut->u.i = pgno; break; } @@ -6998,10 +6897,125 @@ } pOut->u.i = sqlite3BtreeMaxPageCount(pBt, newMax); break; } #endif + +/* Opcode: Function0 P1 P2 P3 P4 P5 +** Synopsis: r[P3]=func(r[P2@P5]) +** +** Invoke a user function (P4 is a pointer to a FuncDef object that +** defines the function) with P5 arguments taken from register P2 and +** successors. The result of the function is stored in register P3. +** Register P3 must not be one of the function inputs. +** +** P1 is a 32-bit bitmask indicating whether or not each argument to the +** function was determined to be constant at compile time. If the first +** argument was constant then bit 0 of P1 is set. This is used to determine +** whether meta data associated with a user function argument using the +** sqlite3_set_auxdata() API may be safely retained until the next +** invocation of this opcode. +** +** See also: Function, AggStep, AggFinal +*/ +/* Opcode: Function P1 P2 P3 P4 P5 +** Synopsis: r[P3]=func(r[P2@P5]) +** +** Invoke a user function (P4 is a pointer to an sqlite3_context object that +** contains a pointer to the function to be run) with P5 arguments taken +** from register P2 and successors. The result of the function is stored +** in register P3. Register P3 must not be one of the function inputs. +** +** P1 is a 32-bit bitmask indicating whether or not each argument to the +** function was determined to be constant at compile time. If the first +** argument was constant then bit 0 of P1 is set. This is used to determine +** whether meta data associated with a user function argument using the +** sqlite3_set_auxdata() API may be safely retained until the next +** invocation of this opcode. +** +** SQL functions are initially coded as OP_Function0 with P4 pointing +** to a FuncDef object. But on first evaluation, the P4 operand is +** automatically converted into an sqlite3_context object and the operation +** changed to this OP_Function opcode. In this way, the initialization of +** the sqlite3_context object occurs only once, rather than once for each +** evaluation of the function. +** +** See also: Function0, AggStep, AggFinal +*/ +case OP_PureFunc0: +case OP_Function0: { + int n; + sqlite3_context *pCtx; + + assert( pOp->p4type==P4_FUNCDEF ); + n = pOp->p5; + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); + assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem+1 - p->nCursor)+1) ); + assert( pOp->p3p2 || pOp->p3>=pOp->p2+n ); + pCtx = sqlite3DbMallocRawNN(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*)); + if( pCtx==0 ) goto no_mem; + pCtx->pOut = 0; + pCtx->pFunc = pOp->p4.pFunc; + pCtx->iOp = (int)(pOp - aOp); + pCtx->pVdbe = p; + pCtx->argc = n; + pOp->p4type = P4_FUNCCTX; + pOp->p4.pCtx = pCtx; + assert( OP_PureFunc == OP_PureFunc0+2 ); + assert( OP_Function == OP_Function0+2 ); + pOp->opcode += 2; + /* Fall through into OP_Function */ +} +case OP_PureFunc: +case OP_Function: { + int i; + sqlite3_context *pCtx; + + assert( pOp->p4type==P4_FUNCCTX ); + pCtx = pOp->p4.pCtx; + + /* If this function is inside of a trigger, the register array in aMem[] + ** might change from one evaluation to the next. The next block of code + ** checks to see if the register array has changed, and if so it + ** reinitializes the relavant parts of the sqlite3_context object */ + pOut = &aMem[pOp->p3]; + if( pCtx->pOut != pOut ){ + pCtx->pOut = pOut; + for(i=pCtx->argc-1; i>=0; i--) pCtx->argv[i] = &aMem[pOp->p2+i]; + } + + memAboutToChange(p, pOut); +#ifdef SQLITE_DEBUG + for(i=0; iargc; i++){ + assert( memIsValid(pCtx->argv[i]) ); + REGISTER_TRACE(pOp->p2+i, pCtx->argv[i]); + } +#endif + MemSetTypeFlag(pOut, MEM_Null); + pCtx->fErrorOrAux = 0; + (*pCtx->pFunc->xSFunc)(pCtx, pCtx->argc, pCtx->argv);/* IMP: R-24505-23230 */ + + /* If the function returned an error, throw an exception */ + if( pCtx->fErrorOrAux ){ + if( pCtx->isError ){ + sqlite3VdbeError(p, "%s", sqlite3_value_text(pOut)); + rc = pCtx->isError; + } + sqlite3VdbeDeleteAuxData(db, &p->pAuxData, pCtx->iOp, pOp->p1); + if( rc ) goto abort_due_to_error; + } + + /* Copy the result of the function into register P3 */ + if( pOut->flags & (MEM_Str|MEM_Blob) ){ + sqlite3VdbeChangeEncoding(pOut, encoding); + if( sqlite3VdbeMemTooBig(pOut) ) goto too_big; + } + + REGISTER_TRACE(pOp->p3, pOut); + UPDATE_MAX_BLOBSIZE(pOut); + break; +} /* Opcode: Init P1 P2 * P4 * ** Synopsis: Start at P2 ** Index: src/vdbe.h ================================================================== --- src/vdbe.h +++ src/vdbe.h @@ -250,10 +250,12 @@ RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*); #ifndef SQLITE_OMIT_TRIGGER void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); #endif + +int sqlite3NotPureFunc(sqlite3_context*); /* Use SQLITE_ENABLE_COMMENTS to enable generation of extra comments on ** each VDBE opcode. ** ** Use the SQLITE_ENABLE_MODULE_COMMENTS macro to see some extra no-op Index: src/vdbeInt.h ================================================================== --- src/vdbeInt.h +++ src/vdbeInt.h @@ -94,22 +94,22 @@ ** CACHE_STALE (0) and so setting cacheStatus=CACHE_STALE guarantees that ** the cache is out of date. */ u32 cacheStatus; /* Cache is valid if this matches Vdbe.cacheCtr */ int seekResult; /* Result of previous sqlite3BtreeMoveto() or 0 ** if there have been no prior seeks on the cursor. */ - /* NB: seekResult does not distinguish between "no seeks have ever occurred - ** on this cursor" and "the most recent seek was an exact match". */ + /* seekResult does not distinguish between "no seeks have ever occurred + ** on this cursor" and "the most recent seek was an exact match". + ** For CURTYPE_PSEUDO, seekResult is the register holding the record */ /* When a new VdbeCursor is allocated, only the fields above are zeroed. ** The fields that follow are uninitialized, and must be individually ** initialized prior to first use. */ VdbeCursor *pAltCursor; /* Associated index cursor from which to read */ union { - BtCursor *pCursor; /* CURTYPE_BTREE. Btree cursor */ - sqlite3_vtab_cursor *pVCur; /* CURTYPE_VTAB. Vtab cursor */ - int pseudoTableReg; /* CURTYPE_PSEUDO. Reg holding content. */ - VdbeSorter *pSorter; /* CURTYPE_SORTER. Sorter object */ + BtCursor *pCursor; /* CURTYPE_BTREE or _PSEUDO. Btree cursor */ + sqlite3_vtab_cursor *pVCur; /* CURTYPE_VTAB. Vtab cursor */ + VdbeSorter *pSorter; /* CURTYPE_SORTER. Sorter object */ } uc; KeyInfo *pKeyInfo; /* Info about index keys needed by index cursors */ u32 iHdrOffset; /* Offset to next unparsed byte of the header */ Pgno pgnoRoot; /* Root page of the open btree cursor */ i16 nField; /* Number of fields in the header */ @@ -187,12 +187,12 @@ */ struct sqlite3_value { union MemValue { double r; /* Real value used when MEM_Real is set in flags */ i64 i; /* Integer value used when MEM_Int is set in flags */ - int nZero; /* Used when bit MEM_Zero is set in flags */ - void *pPtr; /* Pointer when flags=MEM_NULL and eSubtype='p' */ + int nZero; /* Extra zero bytes when MEM_Zero and MEM_Blob set */ + const char *zPType; /* Pointer type when MEM_Term|MEM_Subtype|MEM_Null */ FuncDef *pDef; /* Used only when flags==MEM_Agg */ RowSet *pRowSet; /* Used only when flags==MEM_RowSet */ VdbeFrame *pFrame; /* Used when flags==MEM_Frame */ } u; u16 flags; /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */ @@ -220,37 +220,38 @@ /* One or more of the following flags are set to indicate the validOK ** representations of the value stored in the Mem struct. ** ** If the MEM_Null flag is set, then the value is an SQL NULL value. -** No other flags may be set in this case. +** For a pointer type created using sqlite3_bind_pointer() or +** sqlite3_result_pointer() the MEM_Term and MEM_Subtype flags are also set. ** ** If the MEM_Str flag is set then Mem.z points at a string representation. ** Usually this is encoded in the same unicode encoding as the main ** database (see below for exceptions). If the MEM_Term flag is also ** set, then the string is nul terminated. The MEM_Int and MEM_Real ** flags may coexist with the MEM_Str flag. */ -#define MEM_Null 0x0001 /* Value is NULL */ +#define MEM_Null 0x0001 /* Value is NULL (or a pointer) */ #define MEM_Str 0x0002 /* Value is a string */ #define MEM_Int 0x0004 /* Value is an integer */ #define MEM_Real 0x0008 /* Value is a real number */ #define MEM_Blob 0x0010 /* Value is a BLOB */ #define MEM_AffMask 0x001f /* Mask of affinity bits */ #define MEM_RowSet 0x0020 /* Value is a RowSet object */ #define MEM_Frame 0x0040 /* Value is a VdbeFrame object */ #define MEM_Undefined 0x0080 /* Value is undefined */ #define MEM_Cleared 0x0100 /* NULL set by OP_Null, not from data */ -#define MEM_TypeMask 0x81ff /* Mask of type bits */ +#define MEM_TypeMask 0xc1ff /* Mask of type bits */ /* Whenever Mem contains a valid string or blob representation, one of ** the following flags must be set to determine the memory management ** policy for Mem.z. The MEM_Term flag tells us whether or not the ** string is \000 or \u0000 terminated */ -#define MEM_Term 0x0200 /* String rep is nul terminated */ +#define MEM_Term 0x0200 /* String in Mem.z is zero terminated */ #define MEM_Dyn 0x0400 /* Need to call Mem.xDel() on Mem.z */ #define MEM_Static 0x0800 /* Mem.z points to a static string */ #define MEM_Ephem 0x1000 /* Mem.z points to an ephemeral string */ #define MEM_Agg 0x2000 /* Mem.z points to an agg function context */ #define MEM_Zero 0x4000 /* Mem.i contains count of 0s appended to blob */ @@ -474,11 +475,11 @@ #ifdef SQLITE_OMIT_FLOATING_POINT # define sqlite3VdbeMemSetDouble sqlite3VdbeMemSetInt64 #else void sqlite3VdbeMemSetDouble(Mem*, double); #endif -void sqlite3VdbeMemSetPointer(Mem*, void*, const char*); +void sqlite3VdbeMemSetPointer(Mem*, void*, const char*, void(*)(void*)); void sqlite3VdbeMemInit(Mem*,sqlite3*,u16); void sqlite3VdbeMemSetNull(Mem*); void sqlite3VdbeMemSetZeroBlob(Mem*,int); void sqlite3VdbeMemSetRowSet(Mem*); int sqlite3VdbeMemMakeWriteable(Mem*); Index: src/vdbeapi.c ================================================================== --- src/vdbeapi.c +++ src/vdbeapi.c @@ -215,16 +215,17 @@ Mem *pMem = (Mem*)pVal; return ((pMem->flags & MEM_Subtype) ? pMem->eSubtype : 0); } void *sqlite3_value_pointer(sqlite3_value *pVal, const char *zPType){ Mem *p = (Mem*)pVal; - if( p->flags==(MEM_Null|MEM_Subtype|MEM_Term|MEM_Static) + if( (p->flags&(MEM_TypeMask|MEM_Term|MEM_Subtype)) == + (MEM_Null|MEM_Term|MEM_Subtype) && zPType!=0 && p->eSubtype=='p' - && strcmp(p->z, zPType)==0 + && strcmp(p->u.zPType, zPType)==0 ){ - return p->u.pPtr; + return (void*)p->z; }else{ return 0; } } const unsigned char *sqlite3_value_text(sqlite3_value *pVal){ @@ -403,15 +404,21 @@ } void sqlite3_result_null(sqlite3_context *pCtx){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetNull(pCtx->pOut); } -void sqlite3_result_pointer(sqlite3_context *pCtx, void *pPtr, const char *zPT){ +void sqlite3_result_pointer( + sqlite3_context *pCtx, + void *pPtr, + const char *zPType, + void (*xDestructor)(void*) +){ Mem *pOut = pCtx->pOut; assert( sqlite3_mutex_held(pOut->db->mutex) ); - sqlite3VdbeMemSetNull(pOut); - sqlite3VdbeMemSetPointer(pOut, pPtr, zPT); + sqlite3VdbeMemRelease(pOut); + pOut->flags = MEM_Null; + sqlite3VdbeMemSetPointer(pOut, pPtr, zPType, xDestructor); } void sqlite3_result_subtype(sqlite3_context *pCtx, unsigned int eSubtype){ Mem *pOut = pCtx->pOut; assert( sqlite3_mutex_held(pOut->db->mutex) ); pOut->eSubtype = eSubtype & 0xff; @@ -531,11 +538,11 @@ if( pBt ){ int nEntry; sqlite3BtreeEnter(pBt); nEntry = sqlite3PagerWalCallback(sqlite3BtreePager(pBt)); sqlite3BtreeLeave(pBt); - if( db->xWalCallback && nEntry>0 && rc==SQLITE_OK ){ + if( nEntry>0 && db->xWalCallback && rc==SQLITE_OK ){ rc = db->xWalCallback(db->pWalArg, db, db->aDb[i].zDbSName, nEntry); } } } #endif @@ -641,11 +648,11 @@ #ifndef SQLITE_OMIT_TRACE /* If the statement completed successfully, invoke the profile callback */ if( rc!=SQLITE_ROW ) checkProfileCallback(db, p); #endif - if( rc==SQLITE_DONE ){ + if( rc==SQLITE_DONE && db->autoCommit ){ assert( p->rc==SQLITE_OK ); p->rc = doWalCallbacks(db); if( p->rc!=SQLITE_OK ){ rc = SQLITE_ERROR; } @@ -685,11 +692,10 @@ ** sqlite3Step() to do most of the work. If a schema error occurs, ** call sqlite3Reprepare() and try again. */ int sqlite3_step(sqlite3_stmt *pStmt){ int rc = SQLITE_OK; /* Result from sqlite3Step() */ - int rc2 = SQLITE_OK; /* Result from sqlite3Reprepare() */ Vdbe *v = (Vdbe*)pStmt; /* the prepared statement */ int cnt = 0; /* Counter to prevent infinite loop of reprepares */ sqlite3 *db; /* The database connection */ if( vdbeSafetyNotNull(v) ){ @@ -699,36 +705,35 @@ sqlite3_mutex_enter(db->mutex); v->doingRerun = 0; while( (rc = sqlite3Step(v))==SQLITE_SCHEMA && cnt++ < SQLITE_MAX_SCHEMA_RETRY ){ int savedPc = v->pc; - rc2 = rc = sqlite3Reprepare(v); - if( rc!=SQLITE_OK) break; + rc = sqlite3Reprepare(v); + if( rc!=SQLITE_OK ){ + /* This case occurs after failing to recompile an sql statement. + ** The error message from the SQL compiler has already been loaded + ** into the database handle. This block copies the error message + ** from the database handle into the statement and sets the statement + ** program counter to 0 to ensure that when the statement is + ** finalized or reset the parser error message is available via + ** sqlite3_errmsg() and sqlite3_errcode(). + */ + const char *zErr = (const char *)sqlite3_value_text(db->pErr); + sqlite3DbFree(db, v->zErrMsg); + if( !db->mallocFailed ){ + v->zErrMsg = sqlite3DbStrDup(db, zErr); + v->rc = rc = sqlite3ApiExit(db, rc); + } else { + v->zErrMsg = 0; + v->rc = rc = SQLITE_NOMEM_BKPT; + } + break; + } sqlite3_reset(pStmt); if( savedPc>=0 ) v->doingRerun = 1; assert( v->expired==0 ); } - if( rc2!=SQLITE_OK ){ - /* This case occurs after failing to recompile an sql statement. - ** The error message from the SQL compiler has already been loaded - ** into the database handle. This block copies the error message - ** from the database handle into the statement and sets the statement - ** program counter to 0 to ensure that when the statement is - ** finalized or reset the parser error message is available via - ** sqlite3_errmsg() and sqlite3_errcode(). - */ - const char *zErr = (const char *)sqlite3_value_text(db->pErr); - sqlite3DbFree(db, v->zErrMsg); - if( !db->mallocFailed ){ - v->zErrMsg = sqlite3DbStrDup(db, zErr); - v->rc = rc2; - } else { - v->zErrMsg = 0; - v->rc = rc = SQLITE_NOMEM_BKPT; - } - } - rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; } @@ -1427,17 +1432,25 @@ if( rc==SQLITE_OK ){ sqlite3_mutex_leave(p->db->mutex); } return rc; } -int sqlite3_bind_pointer(sqlite3_stmt *pStmt, int i, void *pPtr,const char *zT){ +int sqlite3_bind_pointer( + sqlite3_stmt *pStmt, + int i, + void *pPtr, + const char *zPTtype, + void (*xDestructor)(void*) +){ int rc; Vdbe *p = (Vdbe*)pStmt; rc = vdbeUnbind(p, i); if( rc==SQLITE_OK ){ - sqlite3VdbeMemSetPointer(&p->aVar[i-1], pPtr, zT); + sqlite3VdbeMemSetPointer(&p->aVar[i-1], pPtr, zPTtype, xDestructor); sqlite3_mutex_leave(p->db->mutex); + }else if( xDestructor ){ + xDestructor(pPtr); } return rc; } int sqlite3_bind_text( sqlite3_stmt *pStmt, @@ -1749,11 +1762,11 @@ ){ UnpackedRecord *pRet; /* Return value */ pRet = sqlite3VdbeAllocUnpackedRecord(pKeyInfo); if( pRet ){ - memset(pRet->aMem, 0, sizeof(Mem)*(pKeyInfo->nField+1)); + memset(pRet->aMem, 0, sizeof(Mem)*(pKeyInfo->nKeyField+1)); sqlite3VdbeRecordUnpack(pKeyInfo, nKey, pKey, pRet); } return pRet; } @@ -1822,11 +1835,11 @@ ** This function is called from within a pre-update callback to retrieve ** the number of columns in the row being updated, deleted or inserted. */ int sqlite3_preupdate_count(sqlite3 *db){ PreUpdate *p = db->pPreUpdate; - return (p ? p->keyinfo.nField : 0); + return (p ? p->keyinfo.nKeyField : 0); } #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ #ifdef SQLITE_ENABLE_PREUPDATE_HOOK /* Index: src/vdbeaux.c ================================================================== --- src/vdbeaux.c +++ src/vdbeaux.c @@ -488,11 +488,12 @@ ** * OP_HaltIfNull with P1=SQLITE_CONSTRAINT and P2=OE_Abort. ** * OP_Destroy ** * OP_VUpdate ** * OP_VRename ** * OP_FkCounter with P2==0 (immediate foreign key constraint) -** * OP_CreateTable and OP_InitCoroutine (for CREATE TABLE AS SELECT ...) +** * OP_CreateBtree/BTREE_INTKEY and OP_InitCoroutine +** (for CREATE TABLE AS SELECT ...) ** ** Then check that the value of Parse.mayAbort is true if an ** ABORT may be thrown, or false otherwise. Return true if it does ** match, or false otherwise. This function is intended to be used as ** part of an assert statement in the compiler. Similar to: @@ -516,11 +517,11 @@ && ((pOp->p1&0xff)==SQLITE_CONSTRAINT && pOp->p2==OE_Abort)) ){ hasAbort = 1; break; } - if( opcode==OP_CreateTable ) hasCreateTable = 1; + if( opcode==OP_CreateBtree && pOp->p3==BTREE_INTKEY ) hasCreateTable = 1; if( opcode==OP_InitCoroutine ) hasInitCoroutine = 1; #ifndef SQLITE_OMIT_FOREIGN_KEY if( opcode==OP_FkCounter && pOp->p1==0 && pOp->p2==1 ){ hasFkCounter = 1; } @@ -594,10 +595,31 @@ case OP_Vacuum: case OP_JournalMode: { p->readOnly = 0; p->bIsReader = 1; break; + } + case OP_Next: + case OP_NextIfOpen: + case OP_SorterNext: { + pOp->p4.xAdvance = sqlite3BtreeNext; + pOp->p4type = P4_ADVANCE; + /* The code generator never codes any of these opcodes as a jump + ** to a label. They are always coded as a jump backwards to a + ** known address */ + assert( pOp->p2>=0 ); + break; + } + case OP_Prev: + case OP_PrevIfOpen: { + pOp->p4.xAdvance = sqlite3BtreePrevious; + pOp->p4type = P4_ADVANCE; + /* The code generator never codes any of these opcodes as a jump + ** to a label. They are always coded as a jump backwards to a + ** known address */ + assert( pOp->p2>=0 ); + break; } #ifndef SQLITE_OMIT_VIRTUALTABLE case OP_VUpdate: { if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2; break; @@ -606,31 +628,29 @@ int n; assert( (pOp - p->aOp) >= 3 ); assert( pOp[-1].opcode==OP_Integer ); n = pOp[-1].p1; if( n>nMaxArgs ) nMaxArgs = n; - break; + /* Fall through into the default case */ } #endif - case OP_Next: - case OP_NextIfOpen: - case OP_SorterNext: { - pOp->p4.xAdvance = sqlite3BtreeNext; - pOp->p4type = P4_ADVANCE; - break; - } - case OP_Prev: - case OP_PrevIfOpen: { - pOp->p4.xAdvance = sqlite3BtreePrevious; - pOp->p4type = P4_ADVANCE; + default: { + if( pOp->p2<0 ){ + /* The mkopcodeh.tcl script has so arranged things that the only + ** non-jump opcodes less than SQLITE_MX_JUMP_CODE are guaranteed to + ** have non-negative values for P2. */ + assert( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)!=0 ); + assert( ADDR(pOp->p2)nLabel ); + pOp->p2 = aLabel[ADDR(pOp->p2)]; + } break; } } - if( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)!=0 && pOp->p2<0 ){ - assert( ADDR(pOp->p2)nLabel ); - pOp->p2 = aLabel[ADDR(pOp->p2)]; - } + /* The mkopcodeh.tcl script has so arranged things that the only + ** non-jump opcodes less than SQLITE_MX_JUMP_CODE are guaranteed to + ** have non-negative values for P2. */ + assert( (sqlite3OpcodeProperty[pOp->opcode]&OPFLG_JUMP)==0 || pOp->p2>=0); } if( pOp==p->aOp ) break; pOp--; } sqlite3DbFree(p->db, pParse->aLabel); @@ -1299,12 +1319,12 @@ switch( pOp->p4type ){ case P4_KEYINFO: { int j; KeyInfo *pKeyInfo = pOp->p4.pKeyInfo; assert( pKeyInfo->aSortOrder!=0 ); - sqlite3XPrintf(&x, "k(%d", pKeyInfo->nField); - for(j=0; jnField; j++){ + sqlite3XPrintf(&x, "k(%d", pKeyInfo->nKeyField); + for(j=0; jnKeyField; j++){ CollSeq *pColl = pKeyInfo->aColl[j]; const char *zColl = pColl ? pColl->zName : ""; if( strcmp(zColl, "BINARY")==0 ) zColl = "B"; sqlite3XPrintf(&x, ",%s%s", pKeyInfo->aSortOrder[j] ? "-" : "", zColl); } @@ -3118,23 +3138,22 @@ ** If the cursor is already pointing to the correct row and that row has ** not been deleted out from under the cursor, then this routine is a no-op. */ int sqlite3VdbeCursorMoveto(VdbeCursor **pp, int *piCol){ VdbeCursor *p = *pp; - if( p->eCurType==CURTYPE_BTREE ){ - if( p->deferredMoveto ){ - int iMap; - if( p->aAltMap && (iMap = p->aAltMap[1+*piCol])>0 ){ - *pp = p->pAltCursor; - *piCol = iMap - 1; - return SQLITE_OK; - } - return handleDeferredMoveto(p); - } - if( sqlite3BtreeCursorHasMoved(p->uc.pCursor) ){ - return handleMovedCursor(p); - } + assert( p->eCurType==CURTYPE_BTREE || p->eCurType==CURTYPE_PSEUDO ); + if( p->deferredMoveto ){ + int iMap; + if( p->aAltMap && (iMap = p->aAltMap[1+*piCol])>0 ){ + *pp = p->pAltCursor; + *piCol = iMap - 1; + return SQLITE_OK; + } + return handleDeferredMoveto(p); + } + if( sqlite3BtreeCursorHasMoved(p->uc.pCursor) ){ + return handleMovedCursor(p); } return SQLITE_OK; } /* @@ -3526,17 +3545,17 @@ UnpackedRecord *sqlite3VdbeAllocUnpackedRecord( KeyInfo *pKeyInfo /* Description of the record */ ){ UnpackedRecord *p; /* Unpacked record to return */ int nByte; /* Number of bytes required for *p */ - nByte = ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*(pKeyInfo->nField+1); + nByte = ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*(pKeyInfo->nKeyField+1); p = (UnpackedRecord *)sqlite3DbMallocRaw(pKeyInfo->db, nByte); if( !p ) return 0; p->aMem = (Mem*)&((char*)p)[ROUND8(sizeof(UnpackedRecord))]; assert( pKeyInfo->aSortOrder!=0 ); p->pKeyInfo = pKeyInfo; - p->nField = pKeyInfo->nField + 1; + p->nField = pKeyInfo->nKeyField + 1; return p; } /* ** Given the nKey-byte encoding of a record in pKey[], populate the @@ -3572,11 +3591,11 @@ pMem->z = 0; d += sqlite3VdbeSerialGet(&aKey[d], serial_type, pMem); pMem++; if( (++u)>=p->nField ) break; } - assert( u<=pKeyInfo->nField + 1 ); + assert( u<=pKeyInfo->nKeyField + 1 ); p->nField = u; } #ifdef SQLITE_DEBUG /* @@ -3621,13 +3640,13 @@ /* mem1.u.i = 0; // not needed, here to silence compiler warning */ idx1 = getVarint32(aKey1, szHdr1); if( szHdr1>98307 ) return SQLITE_CORRUPT; d1 = szHdr1; - assert( pKeyInfo->nField+pKeyInfo->nXField>=pPKey2->nField || CORRUPT_DB ); + assert( pKeyInfo->nAllField>=pPKey2->nField || CORRUPT_DB ); assert( pKeyInfo->aSortOrder!=0 ); - assert( pKeyInfo->nField>0 ); + assert( pKeyInfo->nKeyField>0 ); assert( idx1<=szHdr1 || CORRUPT_DB ); do{ u32 serial_type1; /* Read the serial types for the next element in each key. */ @@ -3685,16 +3704,16 @@ #ifdef SQLITE_DEBUG /* ** Count the number of fields (a.k.a. columns) in the record given by ** pKey,nKey. The verify that this count is less than or equal to the -** limit given by pKeyInfo->nField + pKeyInfo->nXField. +** limit given by pKeyInfo->nAllField. ** ** If this constraint is not satisfied, it means that the high-speed ** vdbeRecordCompareInt() and vdbeRecordCompareString() routines will ** not work correctly. If this assert() ever fires, it probably means -** that the KeyInfo.nField or KeyInfo.nXField values were computed +** that the KeyInfo.nKeyField or KeyInfo.nAllField values were computed ** incorrectly. */ static void vdbeAssertFieldCountWithinLimits( int nKey, const void *pKey, /* The record to verify */ const KeyInfo *pKeyInfo /* Compare size with this KeyInfo */ @@ -3711,11 +3730,11 @@ assert( szHdr<=(u32)nKey ); while( idxnField+pKeyInfo->nXField ); + assert( nField <= pKeyInfo->nAllField ); } #else # define vdbeAssertFieldCountWithinLimits(A,B,C) #endif @@ -4016,14 +4035,14 @@ } i = 0; } VVA_ONLY( mem1.szMalloc = 0; ) /* Only needed by assert() statements */ - assert( pPKey2->pKeyInfo->nField+pPKey2->pKeyInfo->nXField>=pPKey2->nField + assert( pPKey2->pKeyInfo->nAllField>=pPKey2->nField || CORRUPT_DB ); assert( pPKey2->pKeyInfo->aSortOrder!=0 ); - assert( pPKey2->pKeyInfo->nField>0 ); + assert( pPKey2->pKeyInfo->nKeyField>0 ); assert( idx1<=szHdr1 || CORRUPT_DB ); do{ u32 serial_type; /* RHS is an integer */ @@ -4352,11 +4371,11 @@ ** is an integer. ** ** The easiest way to enforce this limit is to consider only records with ** 13 fields or less. If the first field is an integer, the maximum legal ** header size is (12*5 + 1 + 1) bytes. */ - if( (p->pKeyInfo->nField + p->pKeyInfo->nXField)<=13 ){ + if( p->pKeyInfo->nAllField<=13 ){ int flags = p->aMem[0].flags; if( p->pKeyInfo->aSortOrder[0] ){ p->r1 = 1; p->r2 = -1; }else{ @@ -4580,10 +4599,32 @@ v->expmask |= 0x80000000; }else{ v->expmask |= ((u32)1 << (iVar-1)); } } + +/* +** Cause a function to throw an error if it was call from OP_PureFunc +** rather than OP_Function. +** +** OP_PureFunc means that the function must be deterministic, and should +** throw an error if it is given inputs that would make it non-deterministic. +** This routine is invoked by date/time functions that use non-deterministic +** features such as 'now'. +*/ +int sqlite3NotPureFunc(sqlite3_context *pCtx){ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + if( pCtx->pVdbe==0 ) return 1; +#endif + if( pCtx->pVdbe->aOp[pCtx->iOp].opcode==OP_PureFunc ){ + sqlite3_result_error(pCtx, + "non-deterministic function in index expression or CHECK constraint", + -1); + return 0; + } + return 1; +} #ifndef SQLITE_OMIT_VIRTUALTABLE /* ** Transfer error message text from an sqlite3_vtab.zErrMsg (text stored ** in memory obtained from sqlite3_malloc) into a Vdbe.zErrMsg (text stored @@ -4665,26 +4706,26 @@ preupdate.pCsr = pCsr; preupdate.op = op; preupdate.iNewReg = iReg; preupdate.keyinfo.db = db; preupdate.keyinfo.enc = ENC(db); - preupdate.keyinfo.nField = pTab->nCol; + preupdate.keyinfo.nKeyField = pTab->nCol; preupdate.keyinfo.aSortOrder = (u8*)&fakeSortOrder; preupdate.iKey1 = iKey1; preupdate.iKey2 = iKey2; preupdate.pTab = pTab; db->pPreUpdate = &preupdate; db->xPreUpdateCallback(db->pPreUpdateArg, db, op, zDb, zTbl, iKey1, iKey2); db->pPreUpdate = 0; sqlite3DbFree(db, preupdate.aRecord); - vdbeFreeUnpacked(db, preupdate.keyinfo.nField+1, preupdate.pUnpacked); - vdbeFreeUnpacked(db, preupdate.keyinfo.nField+1, preupdate.pNewUnpacked); + vdbeFreeUnpacked(db, preupdate.keyinfo.nKeyField+1, preupdate.pUnpacked); + vdbeFreeUnpacked(db, preupdate.keyinfo.nKeyField+1, preupdate.pNewUnpacked); if( preupdate.aNew ){ int i; for(i=0; inField; i++){ sqlite3VdbeMemRelease(&preupdate.aNew[i]); } sqlite3DbFreeNN(db, preupdate.aNew); } } #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ Index: src/vdbeblob.c ================================================================== --- src/vdbeblob.c +++ src/vdbeblob.c @@ -127,12 +127,12 @@ int nAttempt = 0; int iCol; /* Index of zColumn in row-record */ int rc = SQLITE_OK; char *zErr = 0; Table *pTab; - Parse *pParse = 0; Incrblob *pBlob = 0; + Parse sParse; #ifdef SQLITE_ENABLE_API_ARMOR if( ppBlob==0 ){ return SQLITE_MISUSE_BKPT; } @@ -146,41 +146,38 @@ wrFlag = !!wrFlag; /* wrFlag = (wrFlag ? 1 : 0); */ sqlite3_mutex_enter(db->mutex); pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob)); - if( !pBlob ) goto blob_open_out; - pParse = sqlite3StackAllocRaw(db, sizeof(*pParse)); - if( !pParse ) goto blob_open_out; - do { - memset(pParse, 0, sizeof(Parse)); - pParse->db = db; + memset(&sParse, 0, sizeof(Parse)); + if( !pBlob ) goto blob_open_out; + sParse.db = db; sqlite3DbFree(db, zErr); zErr = 0; sqlite3BtreeEnterAll(db); - pTab = sqlite3LocateTable(pParse, 0, zTable, zDb); + pTab = sqlite3LocateTable(&sParse, 0, zTable, zDb); if( pTab && IsVirtual(pTab) ){ pTab = 0; - sqlite3ErrorMsg(pParse, "cannot open virtual table: %s", zTable); + sqlite3ErrorMsg(&sParse, "cannot open virtual table: %s", zTable); } if( pTab && !HasRowid(pTab) ){ pTab = 0; - sqlite3ErrorMsg(pParse, "cannot open table without rowid: %s", zTable); + sqlite3ErrorMsg(&sParse, "cannot open table without rowid: %s", zTable); } #ifndef SQLITE_OMIT_VIEW if( pTab && pTab->pSelect ){ pTab = 0; - sqlite3ErrorMsg(pParse, "cannot open view: %s", zTable); + sqlite3ErrorMsg(&sParse, "cannot open view: %s", zTable); } #endif if( !pTab ){ - if( pParse->zErrMsg ){ + if( sParse.zErrMsg ){ sqlite3DbFree(db, zErr); - zErr = pParse->zErrMsg; - pParse->zErrMsg = 0; + zErr = sParse.zErrMsg; + sParse.zErrMsg = 0; } rc = SQLITE_ERROR; sqlite3BtreeLeaveAll(db); goto blob_open_out; } @@ -240,11 +237,11 @@ sqlite3BtreeLeaveAll(db); goto blob_open_out; } } - pBlob->pStmt = (sqlite3_stmt *)sqlite3VdbeCreate(pParse); + pBlob->pStmt = (sqlite3_stmt *)sqlite3VdbeCreate(&sParse); assert( pBlob->pStmt || db->mallocFailed ); if( pBlob->pStmt ){ /* This VDBE program seeks a btree cursor to the identified ** db/table/row entry. The reason for using a vdbe program instead @@ -313,14 +310,14 @@ */ aOp[1].p4type = P4_INT32; aOp[1].p4.i = pTab->nCol+1; aOp[3].p2 = pTab->nCol; - pParse->nVar = 0; - pParse->nMem = 1; - pParse->nTab = 1; - sqlite3VdbeMakeReady(v, pParse); + sParse.nVar = 0; + sParse.nMem = 1; + sParse.nTab = 1; + sqlite3VdbeMakeReady(v, &sParse); } } pBlob->iCol = iCol; pBlob->db = db; @@ -338,12 +335,11 @@ if( pBlob && pBlob->pStmt ) sqlite3VdbeFinalize((Vdbe *)pBlob->pStmt); sqlite3DbFree(db, pBlob); } sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : 0), zErr); sqlite3DbFree(db, zErr); - sqlite3ParserReset(pParse); - sqlite3StackFree(db, pParse); + sqlite3ParserReset(&sParse); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; } Index: src/vdbemem.c ================================================================== --- src/vdbemem.c +++ src/vdbemem.c @@ -25,11 +25,11 @@ ** This routine is intended for use inside of assert() statements, like ** this: assert( sqlite3VdbeCheckMemInvariants(pMem) ); */ int sqlite3VdbeCheckMemInvariants(Mem *p){ /* If MEM_Dyn is set then Mem.xDel!=0. - ** Mem.xDel is might not be initialized if MEM_Dyn is clear. + ** Mem.xDel might not be initialized if MEM_Dyn is clear. */ assert( (p->flags & MEM_Dyn)==0 || p->xDel!=0 ); /* MEM_Dyn may only be set if Mem.szMalloc==0. In this way we ** ensure that if Mem.szMalloc>0 then it is safe to do @@ -38,13 +38,38 @@ assert( (p->flags & MEM_Dyn)==0 || p->szMalloc==0 ); /* Cannot be both MEM_Int and MEM_Real at the same time */ assert( (p->flags & (MEM_Int|MEM_Real))!=(MEM_Int|MEM_Real) ); - /* Cannot be both MEM_Null and some other type */ - assert( (p->flags & MEM_Null)==0 || - (p->flags & (MEM_Int|MEM_Real|MEM_Str|MEM_Blob))==0 ); + if( p->flags & MEM_Null ){ + /* Cannot be both MEM_Null and some other type */ + assert( (p->flags & (MEM_Int|MEM_Real|MEM_Str|MEM_Blob + |MEM_RowSet|MEM_Frame|MEM_Agg|MEM_Zero))==0 ); + + /* If MEM_Null is set, then either the value is a pure NULL (the usual + ** case) or it is a pointer set using sqlite3_bind_pointer() or + ** sqlite3_result_pointer(). If a pointer, then MEM_Term must also be + ** set. + */ + if( (p->flags & (MEM_Term|MEM_Subtype))==(MEM_Term|MEM_Subtype) ){ + /* This is a pointer type. There may be a flag to indicate what to + ** do with the pointer. */ + assert( ((p->flags&MEM_Dyn)!=0 ? 1 : 0) + + ((p->flags&MEM_Ephem)!=0 ? 1 : 0) + + ((p->flags&MEM_Static)!=0 ? 1 : 0) <= 1 ); + + /* No other bits set */ + assert( (p->flags & ~(MEM_Null|MEM_Term|MEM_Subtype + |MEM_Dyn|MEM_Ephem|MEM_Static))==0 ); + }else{ + /* A pure NULL might have other flags, such as MEM_Static, MEM_Dyn, + ** MEM_Ephem, MEM_Cleared, or MEM_Subtype */ + } + }else{ + /* The MEM_Cleared bit is only allowed on NULLs */ + assert( (p->flags & MEM_Cleared)==0 ); + } /* The szMalloc field holds the correct memory allocation size */ assert( p->szMalloc==0 || p->szMalloc==sqlite3DbMallocSize(p->db,p->zMalloc) ); @@ -703,22 +728,29 @@ pMem->u.i = val; pMem->flags = MEM_Int; } } +/* A no-op destructor */ +static void sqlite3NoopDestructor(void *p){ UNUSED_PARAMETER(p); } + /* ** Set the value stored in *pMem should already be a NULL. ** Also store a pointer to go with it. */ -void sqlite3VdbeMemSetPointer(Mem *pMem, void *pPtr, const char *zPType){ +void sqlite3VdbeMemSetPointer( + Mem *pMem, + void *pPtr, + const char *zPType, + void (*xDestructor)(void*) +){ assert( pMem->flags==MEM_Null ); - if( zPType ){ - pMem->flags = MEM_Null|MEM_Subtype|MEM_Term|MEM_Static; - pMem->u.pPtr = pPtr; - pMem->eSubtype = 'p'; - pMem->z = (char*)zPType; - } + pMem->u.zPType = zPType ? zPType : ""; + pMem->z = pPtr; + pMem->flags = MEM_Null|MEM_Dyn|MEM_Subtype|MEM_Term; + pMem->eSubtype = 'p'; + pMem->xDel = xDestructor ? xDestructor : sqlite3NoopDestructor; } #ifndef SQLITE_OMIT_FLOATING_POINT /* ** Delete any previous value and set the value stored in *pMem to val, @@ -897,11 +929,11 @@ } flags = (enc==0?MEM_Blob:MEM_Str); if( nByte<0 ){ assert( enc!=0 ); if( enc==SQLITE_UTF8 ){ - nByte = sqlite3Strlen30(z); + nByte = 0x7fffffff & (int)strlen(z); if( nByte>iLimit ) nByte = iLimit+1; }else{ for(nByte=0; nByte<=iLimit && (z[nByte] | z[nByte+1]); nByte+=2){} } flags |= MEM_Term; @@ -1129,11 +1161,11 @@ nByte = sizeof(Mem) * nCol + ROUND8(sizeof(UnpackedRecord)); pRec = (UnpackedRecord*)sqlite3DbMallocZero(db, nByte); if( pRec ){ pRec->pKeyInfo = sqlite3KeyInfoOfIndex(p->pParse, pIdx); if( pRec->pKeyInfo ){ - assert( pRec->pKeyInfo->nField+pRec->pKeyInfo->nXField==nCol ); + assert( pRec->pKeyInfo->nAllField==nCol ); assert( pRec->pKeyInfo->enc==ENC(db) ); pRec->aMem = (Mem *)((u8*)pRec + ROUND8(sizeof(UnpackedRecord))); for(i=0; iaMem[i].flags = MEM_Null; pRec->aMem[i].db = db; @@ -1665,11 +1697,11 @@ ** the object. */ void sqlite3Stat4ProbeFree(UnpackedRecord *pRec){ if( pRec ){ int i; - int nCol = pRec->pKeyInfo->nField+pRec->pKeyInfo->nXField; + int nCol = pRec->pKeyInfo->nAllField; Mem *aMem = pRec->aMem; sqlite3 *db = aMem[0].db; for(i=0; ipSorter->pKeyInfo->nField>1 ){ + if( pTask->pSorter->pKeyInfo->nKeyField>1 ){ res = vdbeSorterCompareTail( pTask, pbKey2Cached, pKey1, nKey1, pKey2, nKey2 ); } }else{ @@ -890,11 +890,11 @@ if( *v2 & 0x80 ) res = +1; } } if( res==0 ){ - if( pTask->pSorter->pKeyInfo->nField>1 ){ + if( pTask->pSorter->pKeyInfo->nKeyField>1 ){ res = vdbeSorterCompareTail( pTask, pbKey2Cached, pKey1, nKey1, pKey2, nKey2 ); } }else if( pTask->pSorter->pKeyInfo->aSortOrder[0] ){ @@ -905,11 +905,11 @@ } /* ** Initialize the temporary index cursor just opened as a sorter cursor. ** -** Usually, the sorter module uses the value of (pCsr->pKeyInfo->nField) +** Usually, the sorter module uses the value of (pCsr->pKeyInfo->nKeyField) ** to determine the number of fields that should be compared from the ** records being sorted. However, if the value passed as argument nField ** is non-zero and the sorter is able to guarantee a stable sort, nField ** is used instead. This is used when sorting records for a CREATE INDEX ** statement. In this case, keys are always delivered to the sorter in @@ -958,11 +958,11 @@ } #endif assert( pCsr->pKeyInfo && pCsr->pBtx==0 ); assert( pCsr->eCurType==CURTYPE_SORTER ); - szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nField-1)*sizeof(CollSeq*); + szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nKeyField-1)*sizeof(CollSeq*); sz = sizeof(VdbeSorter) + nWorker * sizeof(SortSubtask); pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sz + szKeyInfo); pCsr->uc.pSorter = pSorter; if( pSorter==0 ){ @@ -970,12 +970,11 @@ }else{ pSorter->pKeyInfo = pKeyInfo = (KeyInfo*)((u8*)pSorter + sz); memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo); pKeyInfo->db = 0; if( nField && nWorker==0 ){ - pKeyInfo->nXField += (pKeyInfo->nField - nField); - pKeyInfo->nField = nField; + pKeyInfo->nKeyField = nField; } pSorter->pgsz = pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); pSorter->nTask = nWorker + 1; pSorter->iPrev = (u8)(nWorker - 1); pSorter->bUseThreads = (pSorter->nTask>1); @@ -999,23 +998,21 @@ mxCache = mxCache * pgsz; } mxCache = MIN(mxCache, SQLITE_MAX_PMASZ); pSorter->mxPmaSize = MAX(pSorter->mnPmaSize, (int)mxCache); - /* EVIDENCE-OF: R-26747-61719 When the application provides any amount of - ** scratch memory using SQLITE_CONFIG_SCRATCH, SQLite avoids unnecessary - ** large heap allocations. - */ - if( sqlite3GlobalConfig.pScratch==0 ){ + /* Avoid large memory allocations if the application has requested + ** SQLITE_CONFIG_SMALL_MALLOC. */ + if( sqlite3GlobalConfig.bSmallMalloc==0 ){ assert( pSorter->iMemory==0 ); pSorter->nMemory = pgsz; pSorter->list.aMemory = (u8*)sqlite3Malloc(pgsz); if( !pSorter->list.aMemory ) rc = SQLITE_NOMEM_BKPT; } } - if( (pKeyInfo->nField+pKeyInfo->nXField)<13 + if( pKeyInfo->nAllField<13 && (pKeyInfo->aColl[0]==0 || pKeyInfo->aColl[0]==db->pDfltColl) ){ pSorter->typeMask = SORTER_TYPE_INTEGER | SORTER_TYPE_TEXT; } } @@ -1326,11 +1323,11 @@ */ static int vdbeSortAllocUnpacked(SortSubtask *pTask){ if( pTask->pUnpacked==0 ){ pTask->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pTask->pSorter->pKeyInfo); if( pTask->pUnpacked==0 ) return SQLITE_NOMEM_BKPT; - pTask->pUnpacked->nField = pTask->pSorter->pKeyInfo->nField; + pTask->pUnpacked->nField = pTask->pSorter->pKeyInfo->nKeyField; pTask->pUnpacked->errCode = 0; } return SQLITE_OK; } Index: src/vdbetrace.c ================================================================== --- src/vdbetrace.c +++ src/vdbetrace.c @@ -80,11 +80,11 @@ int nToken; /* Length of the parameter token */ int i; /* Loop counter */ Mem *pVar; /* Value of a host parameter */ StrAccum out; /* Accumulate the output here */ #ifndef SQLITE_OMIT_UTF16 - Mem utf8; /* Used to convert UTF16 parameters into UTF8 for display */ + Mem utf8; /* Used to convert UTF16 into UTF8 for display */ #endif char zBase[100]; /* Initial working space */ db = p->db; sqlite3StrAccumInit(&out, 0, zBase, sizeof(zBase), Index: src/vtab.c ================================================================== --- src/vtab.c +++ src/vtab.c @@ -642,10 +642,11 @@ }else{ char *zErr = 0; rc = vtabCallConstructor(db, pTab, pMod, pMod->pModule->xConnect, &zErr); if( rc!=SQLITE_OK ){ sqlite3ErrorMsg(pParse, "%s", zErr); + pParse->rc = rc; } sqlite3DbFree(db, zErr); } return rc; @@ -731,14 +732,14 @@ ** valid to call this function from within the xCreate() or xConnect() of a ** virtual table module. */ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ VtabCtx *pCtx; - Parse *pParse; int rc = SQLITE_OK; Table *pTab; char *zErr = 0; + Parse sParse; #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) || zCreateTable==0 ){ return SQLITE_MISUSE_BKPT; } @@ -751,59 +752,59 @@ return SQLITE_MISUSE_BKPT; } pTab = pCtx->pTab; assert( IsVirtual(pTab) ); - pParse = sqlite3StackAllocZero(db, sizeof(*pParse)); - if( pParse==0 ){ - rc = SQLITE_NOMEM_BKPT; - }else{ - pParse->declareVtab = 1; - pParse->db = db; - pParse->nQueryLoop = 1; - - if( SQLITE_OK==sqlite3RunParser(pParse, zCreateTable, &zErr) - && pParse->pNewTable - && !db->mallocFailed - && !pParse->pNewTable->pSelect - && !IsVirtual(pParse->pNewTable) - ){ - if( !pTab->aCol ){ - Table *pNew = pParse->pNewTable; - Index *pIdx; - pTab->aCol = pNew->aCol; - pTab->nCol = pNew->nCol; - pTab->tabFlags |= pNew->tabFlags & (TF_WithoutRowid|TF_NoVisibleRowid); - pNew->nCol = 0; - pNew->aCol = 0; - assert( pTab->pIndex==0 ); - if( !HasRowid(pNew) && pCtx->pVTable->pMod->pModule->xUpdate!=0 ){ - rc = SQLITE_ERROR; - } - pIdx = pNew->pIndex; - if( pIdx ){ - assert( pIdx->pNext==0 ); - pTab->pIndex = pIdx; - pNew->pIndex = 0; - pIdx->pTable = pTab; - } - } - pCtx->bDeclared = 1; - }else{ - sqlite3ErrorWithMsg(db, SQLITE_ERROR, (zErr ? "%s" : 0), zErr); - sqlite3DbFree(db, zErr); - rc = SQLITE_ERROR; - } - pParse->declareVtab = 0; - - if( pParse->pVdbe ){ - sqlite3VdbeFinalize(pParse->pVdbe); - } - sqlite3DeleteTable(db, pParse->pNewTable); - sqlite3ParserReset(pParse); - sqlite3StackFree(db, pParse); - } + memset(&sParse, 0, sizeof(sParse)); + sParse.declareVtab = 1; + sParse.db = db; + sParse.nQueryLoop = 1; + if( SQLITE_OK==sqlite3RunParser(&sParse, zCreateTable, &zErr) + && sParse.pNewTable + && !db->mallocFailed + && !sParse.pNewTable->pSelect + && !IsVirtual(sParse.pNewTable) + ){ + if( !pTab->aCol ){ + Table *pNew = sParse.pNewTable; + Index *pIdx; + pTab->aCol = pNew->aCol; + pTab->nCol = pNew->nCol; + pTab->tabFlags |= pNew->tabFlags & (TF_WithoutRowid|TF_NoVisibleRowid); + pNew->nCol = 0; + pNew->aCol = 0; + assert( pTab->pIndex==0 ); + assert( HasRowid(pNew) || sqlite3PrimaryKeyIndex(pNew)!=0 ); + if( !HasRowid(pNew) + && pCtx->pVTable->pMod->pModule->xUpdate!=0 + && sqlite3PrimaryKeyIndex(pNew)->nKeyCol!=1 + ){ + /* WITHOUT ROWID virtual tables must either be read-only (xUpdate==0) + ** or else must have a single-column PRIMARY KEY */ + rc = SQLITE_ERROR; + } + pIdx = pNew->pIndex; + if( pIdx ){ + assert( pIdx->pNext==0 ); + pTab->pIndex = pIdx; + pNew->pIndex = 0; + pIdx->pTable = pTab; + } + } + pCtx->bDeclared = 1; + }else{ + sqlite3ErrorWithMsg(db, SQLITE_ERROR, (zErr ? "%s" : 0), zErr); + sqlite3DbFree(db, zErr); + rc = SQLITE_ERROR; + } + sParse.declareVtab = 0; + + if( sParse.pVdbe ){ + sqlite3VdbeFinalize(sParse.pVdbe); + } + sqlite3DeleteTable(db, sParse.pNewTable); + sqlite3ParserReset(&sParse); assert( (rc&0xff)==rc ); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; Index: src/wal.c ================================================================== --- src/wal.c +++ src/wal.c @@ -1801,13 +1801,11 @@ u32 nBackfill = pInfo->nBackfill; pInfo->nBackfillAttempted = mxSafeFrame; /* Sync the WAL to disk */ - if( sync_flags ){ - rc = sqlite3OsSync(pWal->pWalFd, sync_flags); - } + rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags)); /* If the database may grow as a result of this checkpoint, hint ** about the eventual size of the db file to the VFS layer. */ if( rc==SQLITE_OK ){ @@ -1844,12 +1842,12 @@ if( rc==SQLITE_OK ){ if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ i64 szDb = pWal->hdr.nPage*(i64)szPage; testcase( IS_BIG_INT(szDb) ); rc = sqlite3OsTruncate(pWal->pDbFd, szDb); - if( rc==SQLITE_OK && sync_flags ){ - rc = sqlite3OsSync(pWal->pDbFd, sync_flags); + if( rc==SQLITE_OK ){ + rc = sqlite3OsSync(pWal->pDbFd, CKPT_SYNC_FLAGS(sync_flags)); } } if( rc==SQLITE_OK ){ pInfo->nBackfill = mxSafeFrame; } @@ -2957,12 +2955,12 @@ rc = sqlite3OsWrite(p->pFd, pContent, iFirstAmt, iOffset); if( rc ) return rc; iOffset += iFirstAmt; iAmt -= iFirstAmt; pContent = (void*)(iFirstAmt + (char*)pContent); - assert( p->syncFlags & (SQLITE_SYNC_NORMAL|SQLITE_SYNC_FULL) ); - rc = sqlite3OsSync(p->pFd, p->syncFlags & SQLITE_SYNC_MASK); + assert( WAL_SYNC_FLAGS(p->syncFlags)!=0 ); + rc = sqlite3OsSync(p->pFd, WAL_SYNC_FLAGS(p->syncFlags)); if( iAmt==0 || rc ) return rc; } rc = sqlite3OsWrite(p->pFd, pContent, iAmt, iOffset); return rc; } @@ -3140,14 +3138,14 @@ /* Sync the header (unless SQLITE_IOCAP_SEQUENTIAL is true or unless ** all syncing is turned off by PRAGMA synchronous=OFF). Otherwise ** an out-of-order write following a WAL restart could result in ** database corruption. See the ticket: ** - ** http://localhost:591/sqlite/info/ff5be73dee + ** https://sqlite.org/src/info/ff5be73dee */ - if( pWal->syncHeader && sync_flags ){ - rc = sqlite3OsSync(pWal->pWalFd, sync_flags & SQLITE_SYNC_MASK); + if( pWal->syncHeader ){ + rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags)); if( rc ) return rc; } } assert( (int)pWal->szPage==szPage ); @@ -3230,11 +3228,11 @@ ** final frame is repeated (with its commit mark) until the next sector ** boundary is crossed. Only the part of the WAL prior to the last ** sector boundary is synced; the part of the last frame that extends ** past the sector boundary is written after the sync. */ - if( isCommit && (sync_flags & WAL_SYNC_TRANSACTIONS)!=0 ){ + if( isCommit && WAL_SYNC_FLAGS(sync_flags)!=0 ){ int bSync = 1; if( pWal->padToSectorBoundary ){ int sectorSize = sqlite3SectorSize(pWal->pWalFd); w.iSyncPoint = ((iOffset+sectorSize-1)/sectorSize)*sectorSize; bSync = (w.iSyncPoint==iOffset); @@ -3251,11 +3249,11 @@ nExtra++; } } if( bSync ){ assert( rc==SQLITE_OK ); - rc = sqlite3OsSync(w.pFd, sync_flags & SQLITE_SYNC_MASK); + rc = sqlite3OsSync(w.pFd, WAL_SYNC_FLAGS(sync_flags)); } } #if defined(SQLITE_WRITE_WALFRAME_PREBUFFERED) free(w.aFrameBuf); Index: src/wal.h ================================================================== --- src/wal.h +++ src/wal.h @@ -17,15 +17,15 @@ #ifndef SQLITE_WAL_H #define SQLITE_WAL_H #include "sqliteInt.h" -/* Additional values that can be added to the sync_flags argument of -** sqlite3WalFrames(): +/* Macros for extracting appropriate sync flags for either transaction +** commits (WAL_SYNC_FLAGS(X)) or for checkpoint ops (CKPT_SYNC_FLAGS(X)): */ -#define WAL_SYNC_TRANSACTIONS 0x20 /* Sync at the end of each transaction */ -#define SQLITE_SYNC_MASK 0x13 /* Mask off the SQLITE_SYNC_* values */ +#define WAL_SYNC_FLAGS(X) ((X)&0x03) +#define CKPT_SYNC_FLAGS(X) (((X)>>2)&0x03) #ifdef SQLITE_OMIT_WAL # define sqlite3WalOpen(x,y,z) 0 # define sqlite3WalLimit(x,y) # define sqlite3WalClose(v,w,x,y,z) 0 Index: src/walker.c ================================================================== --- src/walker.c +++ src/walker.c @@ -38,22 +38,26 @@ */ static SQLITE_NOINLINE int walkExpr(Walker *pWalker, Expr *pExpr){ int rc; testcase( ExprHasProperty(pExpr, EP_TokenOnly) ); testcase( ExprHasProperty(pExpr, EP_Reduced) ); - rc = pWalker->xExprCallback(pWalker, pExpr); - if( rc ) return rc & WRC_Abort; - if( !ExprHasProperty(pExpr,(EP_TokenOnly|EP_Leaf)) ){ - if( pExpr->pLeft && walkExpr(pWalker, pExpr->pLeft) ) return WRC_Abort; - assert( pExpr->x.pList==0 || pExpr->pRight==0 ); - if( pExpr->pRight ){ - if( walkExpr(pWalker, pExpr->pRight) ) return WRC_Abort; - }else if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - if( sqlite3WalkSelect(pWalker, pExpr->x.pSelect) ) return WRC_Abort; - }else if( pExpr->x.pList ){ - if( sqlite3WalkExprList(pWalker, pExpr->x.pList) ) return WRC_Abort; - } + while(1){ + rc = pWalker->xExprCallback(pWalker, pExpr); + if( rc ) return rc & WRC_Abort; + if( !ExprHasProperty(pExpr,(EP_TokenOnly|EP_Leaf)) ){ + if( pExpr->pLeft && walkExpr(pWalker, pExpr->pLeft) ) return WRC_Abort; + assert( pExpr->x.pList==0 || pExpr->pRight==0 ); + if( pExpr->pRight ){ + pExpr = pExpr->pRight; + continue; + }else if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + if( sqlite3WalkSelect(pWalker, pExpr->x.pSelect) ) return WRC_Abort; + }else if( pExpr->x.pList ){ + if( sqlite3WalkExprList(pWalker, pExpr->x.pList) ) return WRC_Abort; + } + } + break; } return WRC_Continue; } int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){ return pExpr ? walkExpr(pWalker,pExpr) : WRC_Continue; Index: src/where.c ================================================================== --- src/where.c +++ src/where.c @@ -606,10 +606,19 @@ Bitmask notReady /* Tables in outer loops of the join */ ){ char aff; if( pTerm->leftCursor!=pSrc->iCursor ) return 0; if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) return 0; + if( (pSrc->fg.jointype & JT_LEFT) + && !ExprHasProperty(pTerm->pExpr, EP_FromJoin) + && (pTerm->eOperator & WO_IS) + ){ + /* Cannot use an IS term from the WHERE clause as an index driver for + ** the RHS of a LEFT JOIN. Such a term can only be used if it is from + ** the ON clause. */ + return 0; + } if( (pTerm->prereqRight & notReady)!=0 ) return 0; if( pTerm->u.leftColumn<0 ) return 0; aff = pSrc->pTab->aCol[pTerm->u.leftColumn].affinity; if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0; testcase( pTerm->pExpr->op==TK_IS ); @@ -3678,11 +3687,11 @@ if( sqlite3ExprCompare(0, pOBExpr,pIndex->aColExpr->a[j].pExpr,iCur) ){ continue; } } - if( iColumn>=0 ){ + if( iColumn!=XN_ROWID ){ pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); if( !pColl ) pColl = db->pDfltColl; if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue; } pLoop->u.btree.nIdxCol = j+1; @@ -4317,10 +4326,11 @@ static int exprIsDeterministic(Expr *p){ Walker w; memset(&w, 0, sizeof(w)); w.eCode = 1; w.xExprCallback = exprNodeIsDeterministic; + w.xSelectCallback = sqlite3SelectWalkFail; sqlite3WalkExpr(&w, p); return w.eCode; } /* @@ -4526,41 +4536,42 @@ if( nTabList==0 ){ if( pOrderBy ) pWInfo->nOBSat = pOrderBy->nExpr; if( wctrlFlags & WHERE_WANT_DISTINCT ){ pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; } - } - - /* Assign a bit from the bitmask to every term in the FROM clause. - ** - ** The N-th term of the FROM clause is assigned a bitmask of 1<nSrc tables in - ** pTabList, not just the first nTabList tables. nTabList is normally - ** equal to pTabList->nSrc but might be shortened to 1 if the - ** WHERE_OR_SUBCLAUSE flag is set. - */ - for(ii=0; iinSrc; ii++){ - createMask(pMaskSet, pTabList->a[ii].iCursor); - sqlite3WhereTabFuncArgs(pParse, &pTabList->a[ii], &pWInfo->sWC); - } -#ifdef SQLITE_DEBUG - { - Bitmask mx = 0; - for(ii=0; iinSrc; ii++){ - Bitmask m = sqlite3WhereGetMask(pMaskSet, pTabList->a[ii].iCursor); - assert( m>=mx ); - mx = m; - } - } -#endif - + }else{ + /* Assign a bit from the bitmask to every term in the FROM clause. + ** + ** The N-th term of the FROM clause is assigned a bitmask of 1<nSrc tables in + ** pTabList, not just the first nTabList tables. nTabList is normally + ** equal to pTabList->nSrc but might be shortened to 1 if the + ** WHERE_OR_SUBCLAUSE flag is set. + */ + ii = 0; + do{ + createMask(pMaskSet, pTabList->a[ii].iCursor); + sqlite3WhereTabFuncArgs(pParse, &pTabList->a[ii], &pWInfo->sWC); + }while( (++ii)nSrc ); + #ifdef SQLITE_DEBUG + { + Bitmask mx = 0; + for(ii=0; iinSrc; ii++){ + Bitmask m = sqlite3WhereGetMask(pMaskSet, pTabList->a[ii].iCursor); + assert( m>=mx ); + mx = m; + } + } + #endif + } + /* Analyze all of the subexpressions. */ sqlite3WhereExprAnalyze(pTabList, &pWInfo->sWC); if( db->mallocFailed ) goto whereBeginError; /* Special case: WHERE terms that do not refer to any tables in the join Index: src/wherecode.c ================================================================== --- src/wherecode.c +++ src/wherecode.c @@ -792,11 +792,11 @@ || pExpr->op==TK_NOTNULL || pExpr->op==TK_CASE ){ pWalker->eCode = 1; }else if( pExpr->op==TK_FUNCTION ){ int d1; - char d2[3]; + char d2[4]; if( 0==sqlite3IsLikeFunction(pWalker->pParse->db, pExpr, &d1, d2) ){ pWalker->eCode = 1; } } @@ -1068,13 +1068,13 @@ return WRC_Continue; } } /* -** For an indexes on expression X, locate every instance of expression X in pExpr -** and change that subexpression into a reference to the appropriate column of -** the index. +** For an indexes on expression X, locate every instance of expression X +** in pExpr and change that subexpression into a reference to the appropriate +** column of the index. */ static void whereIndexExprTrans( Index *pIdx, /* The Index */ int iTabCur, /* Cursor of the table that is being indexed */ int iIdxCur, /* Cursor of the index itself */ Index: src/whereexpr.c ================================================================== --- src/whereexpr.c +++ src/whereexpr.c @@ -192,16 +192,16 @@ Expr *pExpr, /* Test this expression */ Expr **ppPrefix, /* Pointer to TK_STRING expression with pattern prefix */ int *pisComplete, /* True if the only wildcard is % in the last character */ int *pnoCase /* True if uppercase is equivalent to lowercase */ ){ - const char *z = 0; /* String on RHS of LIKE operator */ + const u8 *z = 0; /* String on RHS of LIKE operator */ Expr *pRight, *pLeft; /* Right and left size of LIKE operator */ ExprList *pList; /* List of operands to the LIKE operator */ int c; /* One character in z[] */ int cnt; /* Number of non-wildcard prefix characters */ - char wc[3]; /* Wildcard characters */ + char wc[4]; /* Wildcard characters */ sqlite3 *db = pParse->db; /* Database connection */ sqlite3_value *pVal = 0; int op; /* Opcode of pRight */ int rc; /* Result code to return */ @@ -219,16 +219,16 @@ if( op==TK_VARIABLE && (db->flags & SQLITE_EnableQPSG)==0 ){ Vdbe *pReprepare = pParse->pReprepare; int iCol = pRight->iColumn; pVal = sqlite3VdbeGetBoundValue(pReprepare, iCol, SQLITE_AFF_BLOB); if( pVal && sqlite3_value_type(pVal)==SQLITE_TEXT ){ - z = (char *)sqlite3_value_text(pVal); + z = sqlite3_value_text(pVal); } sqlite3VdbeSetVarmask(pParse->pVdbe, iCol); assert( pRight->op==TK_VARIABLE || pRight->op==TK_REGISTER ); }else if( op==TK_STRING ){ - z = pRight->u.zToken; + z = (u8*)pRight->u.zToken; } if( z ){ /* If the RHS begins with a digit or a minus sign, then the LHS must ** be an ordinary column (not a virtual table column) with TEXT affinity. @@ -244,20 +244,46 @@ ){ sqlite3ValueFree(pVal); return 0; } } + + /* Count the number of prefix characters prior to the first wildcard */ cnt = 0; while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ cnt++; + if( c==wc[3] && z[cnt]!=0 ) cnt++; } + + /* The optimization is possible only if (1) the pattern does not begin + ** with a wildcard and if (2) the non-wildcard prefix does not end with + ** an (illegal 0xff) character. The second condition is necessary so + ** that we can increment the prefix key to find an upper bound for the + ** range search. + */ if( cnt!=0 && 255!=(u8)z[cnt-1] ){ Expr *pPrefix; + + /* A "complete" match if the pattern ends with "*" or "%" */ *pisComplete = c==wc[0] && z[cnt+1]==0; - pPrefix = sqlite3Expr(db, TK_STRING, z); - if( pPrefix ) pPrefix->u.zToken[cnt] = 0; + + /* Get the pattern prefix. Remove all escapes from the prefix. */ + pPrefix = sqlite3Expr(db, TK_STRING, (char*)z); + if( pPrefix ){ + int iFrom, iTo; + char *zNew = pPrefix->u.zToken; + zNew[cnt] = 0; + for(iFrom=iTo=0; iFrompVdbe; sqlite3VdbeSetVarmask(v, pRight->iColumn); if( *pisComplete && pRight->u.zToken[1] ){ /* If the rhs of the LIKE expression is a variable, and the current ADDED test/atomic.test Index: test/atomic.test ================================================================== --- /dev/null +++ test/atomic.test @@ -0,0 +1,41 @@ +# 2015-11-07 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the WITH clause. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix atomic + +db close +if {[atomic_batch_write test.db]==0} { + puts "No f2fs atomic-batch-write support. Skipping tests..." + finish_test + return +} + +reset_db + +do_execsql_test 1.0 { + CREATE TABLE t1(x, y); + BEGIN; + INSERT INTO t1 VALUES(1, 2); +} + +do_test 1.1 { file exists test.db-journal } {0} + +do_execsql_test 1.2 { + COMMIT; +} + + +finish_test Index: test/attach2.test ================================================================== --- test/attach2.test +++ test/attach2.test @@ -372,26 +372,21 @@ } } {} do_test attach2-6.2 { catchsql { ATTACH 'test3.db' as aux2; + DETACH aux2; } -} {1 {cannot ATTACH database within transaction}} +} {0 {}} # EVIDENCE-OF: R-59740-55581 This statement will fail if SQLite is in # the middle of a transaction. # do_test attach2-6.3 { catchsql { DETACH aux; } -} {1 {cannot DETACH database within transaction}} -do_test attach2-6.4 { - execsql { - COMMIT; - DETACH aux; - } -} {} +} {0 {}} db close finish_test Index: test/autoindex1.test ================================================================== --- test/autoindex1.test +++ test/autoindex1.test @@ -517,7 +517,25 @@ do_execsql_test autoindex1-920 { CREATE TABLE t920(x); INSERT INTO t920 VALUES(3),(4),(5); SELECT * FROM t920,(SELECT 0 FROM t920),(VALUES(9)) WHERE 5 IN (x); } {5 0 9 5 0 9 5 0 9} + +#------------------------------------------------------------------------- +# An IS term from the WHERE clause of a LEFT JOIN cannot be used as an +# index driver for the RHS of a LEFT JOIN. Prior to this being fixed, +# the following SELECT count(*) would incorrectly return 1. +# +do_execsql_test autoindex1-1010 { + CREATE TABLE t11(w); + CREATE TABLE t12(y); + INSERT INTO t11 VALUES(NULL); + INSERT INTO t12 VALUES('notnull'); +} +do_execsql_test autoindex1-1020 { + SELECT count(*) FROM t11 LEFT JOIN t12 WHERE t12.y IS t11.w; +} 0 + + + finish_test ADDED test/bigmmap.test Index: test/bigmmap.test ================================================================== --- /dev/null +++ test/bigmmap.test @@ -0,0 +1,104 @@ +# 2017 August 07 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this script testing the ability of SQLite to use mmap +# to access files larger than 4GiB. +# + +if {[file exists skip-big-file]} return +if {$tcl_platform(os)=="Darwin"} return + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix bigmmap + +ifcapable !mmap { + finish_test + return +} + +set mmap_limit 0 +db eval { + SELECT compile_options AS x FROM pragma_compile_options + WHERE x LIKE 'max_mmap_size=%' +} { + regexp {MAX_MMAP_SIZE=([0-9]*)} $x -> mmap_limit +} +if {$mmap_limit < [expr 8 * 1<<30]} { + puts "Skipping bigmmap.test - requires SQLITE_MAX_MMAP_SIZE >= 8G" + finish_test + return +} + + +#------------------------------------------------------------------------- +# Create the database file roughly 8GiB in size. Most pages are unused, +# except that there is a table and index clustered around each 1GiB +# boundary. +# +do_execsql_test 1.0 { + PRAGMA page_size = 4096; + CREATE TABLE t0(a INTEGER PRIMARY KEY, b, c, UNIQUE(b, c)); + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s LIMIT 100 ) + INSERT INTO t0 SELECT i, 't0', randomblob(800) FROM s; +} + +for {set i 1} {$i < 8} {incr i} { + fake_big_file [expr $i*1024] [get_pwd]/test.db + hexio_write test.db 28 [format %.8x [expr ($i*1024*1024*1024/4096) - 5]] + + do_execsql_test 1.$i " + CREATE TABLE t$i (a INTEGER PRIMARY KEY, b, c, UNIQUE(b, c)); + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s LIMIT 100 ) + INSERT INTO t$i SELECT i, 't$i', randomblob(800) FROM s; + " +} + +#------------------------------------------------------------------------- +# Check that data can be retrieved from the db with a variety of +# configured mmap size limits. +# +for {set i 0} {$i < 9} {incr i} { + + # Configure a memory mapping $i GB in size. + # + set val [expr $i*1024*1024*1024] + execsql "PRAGMA main.mmap_size = $val" + do_execsql_test 2.$i.0 { + PRAGMA main.mmap_size + } $val + + for {set t 0} {$t < 8} {incr t} { + do_execsql_test 2.$i.$t.1 " + SELECT count(*) FROM t$t; + SELECT count(b || c) FROM t$t GROUP BY b; + " {100 100} + + do_execsql_test 2.$i.$t.2 " + SELECT * FROM t$t AS o WHERE + NOT EXISTS( SELECT * FROM t$t AS i WHERE a=o.a AND +b=o.b AND +c=o.c ) + ORDER BY b, c; + " {} + + do_eqp_test 2.$i.$t.3 " + SELECT * FROM t$t AS o WHERE + NOT EXISTS( SELECT * FROM t$t AS i WHERE a=o.a AND +b=o.b AND +c=o.c ) + ORDER BY b, c; + " " + 0 0 0 {SCAN TABLE t$t AS o USING COVERING INDEX sqlite_autoindex_t${t}_1} + 0 0 0 {EXECUTE CORRELATED SCALAR SUBQUERY 1} + 1 0 0 {SEARCH TABLE t$t AS i USING INTEGER PRIMARY KEY (rowid=?)} + " + } +} + +finish_test + Index: test/colname.test ================================================================== --- test/colname.test +++ test/colname.test @@ -11,11 +11,10 @@ # This file implements regression tests for SQLite library. # # The focus of this file is testing how SQLite generates the names # of columns in a result set. # -# $Id: colname.test,v 1.7 2009/06/02 15:47:38 drh Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl # Rules (applied in order): @@ -323,7 +322,78 @@ CREATE TABLE "t3893"("x"); INSERT INTO t3893 VALUES(123); SELECT "y"."x" FROM (SELECT "x" FROM "t3893") AS "y"; } } {123} + +# 2017-07-29: Interaction between column naming and query flattening. +# For years now, the query flattener has inserted AS clauses on the +# outer query that were the original SQL text of the column. This caused +# column-name shifts when the query flattener was enhanced, breaking +# legacy applications. See https://sqlite.org/src/info/41c27bc0ff1d3135 +# for details. +# +# To fix this, the column naming logic was moved ahead of the query +# flattener so that column names are assigned before the query flattener +# runs. +# +db close +sqlite3 db :memory: +do_test colname-9.100 { + db eval { + CREATE TABLE t1(a,b); + INSERT INTO t1 VALUES(1,2); + CREATE VIEW v1(x,y) AS SELECT a,b FROM t1; + } + execsql2 {SELECT v1.x, (Y) FROM v1} + # Prior to the fix, this would return: "v1.x 1 (Y) 2" +} {x 1 y 2} +do_test colname-9.110 { + execsql2 {SELECT * FROM v1} +} {x 1 y 2} +do_test colname-9.120 { + db eval { + CREATE VIEW v2(x,y) AS SELECT a,b FROM t1 LIMIT 10; + } + execsql2 {SELECT * FROM v2 WHERE 1} +} {x 1 y 2} +do_test colname-9.130 { + execsql2 {SELECT v2.x, [v2].[y] FROM v2 WHERE 1} +} {x 1 y 2} +do_test colname-9.140 { + execsql2 {SELECT +x, +y FROM v2 WHERE 1} +} {+x 1 +y 2} + +do_test colname-9.200 { + db eval { + CREATE TABLE t2(c,d); + INSERT INTO t2 VALUES(3,4); + CREATE VIEW v3 AS SELECT c AS a, d AS b FROM t2; + } + execsql2 {SELECT t1.a, v3.a AS n FROM t1 LEFT JOIN v3} +} {a 1 n 3} +do_test colname-9.211 { + execsql2 {SELECT t1.a AS n, v3.a FROM t1 JOIN v3} +} {n 1 a 3} +do_test colname-9.210 { + execsql2 {SELECT t1.a, v3.a AS n FROM t1 JOIN v3} +} {a 1 n 3} + +# Make sure the quotation marks get removed from the column names +# when constructing a new table from an aggregate SELECT. +# Email from Juergen Palm on 2017-07-11. +# +do_execsql_test colname-10.100 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1("with space" TEXT); + DROP TABLE IF EXISTS t2; + CREATE TABLE t2 AS SELECT "with space" FROM t1; + PRAGMA table_info(t2); +} {0 {with space} TEXT 0 {} 0} +do_execsql_test colname-10.110 { + DROP TABLE IF EXISTS t3; + CREATE TABLE t3 AS SELECT "with space" FROM t1 GROUP BY 1; + PRAGMA table_info(t3); +} {0 {with space} TEXT 0 {} 0} + finish_test Index: test/csv01.test ================================================================== --- test/csv01.test +++ test/csv01.test @@ -91,22 +91,54 @@ # The rowid column is not visible on a WITHOUT ROWID virtual table do_catchsql_test 3.2 { SELECT rowid, a FROM t3; } {1 {no such column: rowid}} +# Multi-column WITHOUT ROWID virtual tables may not be writable. do_catchsql_test 4.0 { DROP TABLE t3; CREATE VIRTUAL TABLE temp.t4 USING csv_wr( data= '1,2,3,4 5,6,7,8 9,10,11,12 -13,14,15,16 -', +13,14,15,16', columns=4, schema= - 'CREATE TABLE t3(a PRIMARY KEY,b TEXT,c TEXT,d TEXT) WITHOUT ROWID', + 'CREATE TABLE t3(a,b,c,d,PRIMARY KEY(a,b)) WITHOUT ROWID', testflags=1 ); } {1 {vtable constructor failed: t4}} + +# WITHOUT ROWID tables with a single-column PRIMARY KEY may be writable. +do_catchsql_test 4.1 { + DROP TABLE IF EXISTS t4; + CREATE VIRTUAL TABLE temp.t4 USING csv_wr( + data= +'1,2,3,4 +5,6,7,8 +9,10,11,12 +13,14,15,16', + columns=4, + schema= + 'CREATE TABLE t3(a,b,c,d,PRIMARY KEY(b)) WITHOUT ROWID', + testflags=1 + ); +} {0 {}} + +do_catchsql_test 4.2 { + DROP TABLE IF EXISTS t5; + CREATE VIRTUAL TABLE temp.t5 USING csv_wr( + data= + '1,2,3,4 + 5,6,7,8 + 9,10,11,12 + 13,14,15,16', + columns=4, + schema= + 'CREATE TABLE t3(a,b,c,d) WITHOUT ROWID', + testflags=1 + ); +} {1 {vtable constructor failed: t5}} + finish_test ADDED test/date2.test Index: test/date2.test ================================================================== --- /dev/null +++ test/date2.test @@ -0,0 +1,134 @@ +# 2017-07-20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing date and time functions used in +# check constraints and index expressions. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# Skip this whole file if date and time functions are omitted +# at compile-time +# +ifcapable {!datetime} { + finish_test + return +} + +do_execsql_test date2-100 { + CREATE TABLE t1(x, y, CHECK( date(x) BETWEEN '2017-07-01' AND '2017-07-31' )); + INSERT INTO t1(x,y) VALUES('2017-07-20','one'); +} {} +do_catchsql_test date2-110 { + INSERT INTO t1(x,y) VALUES('now','two'); +} {1 {non-deterministic function in index expression or CHECK constraint}} +do_execsql_test date2-120 { + SELECT * FROM t1; +} {2017-07-20 one} +do_catchsql_test date2-130 { + INSERT INTO t1(x,y) VALUES('2017-08-01','two'); +} {1 {CHECK constraint failed: t1}} + +do_execsql_test date2-200 { + CREATE TABLE t2(x,y); + INSERT INTO t2(x,y) VALUES(1, '2017-07-20'), (2, 'xyzzy'); + CREATE INDEX t2y ON t2(date(y)); +} +do_catchsql_test date2-210 { + INSERT INTO t2(x,y) VALUES(3, 'now'); +} {1 {non-deterministic function in index expression or CHECK constraint}} +do_execsql_test date2-220 { + SELECT x, y FROM t2 ORDER BY x; +} {1 2017-07-20 2 xyzzy} + +do_execsql_test date2-300 { + CREATE TABLE t3(a INTEGER PRIMARY KEY,b); + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<1000) + INSERT INTO t3(a,b) SELECT x, julianday('2017-07-01')+x FROM c; + UPDATE t3 SET b='now' WHERE a=500; +} +do_catchsql_test date2-310 { + CREATE INDEX t3b1 ON t3(datetime(b)); +} {1 {non-deterministic function in index expression or CHECK constraint}} +do_catchsql_test date2-320 { + CREATE INDEX t3b1 ON t3(datetime(b)) WHERE typeof(b)='real'; +} {0 {}} +do_execsql_test date2-330 { + EXPLAIN QUERY PLAN + SELECT a FROM t3 + WHERE typeof(b)='real' + AND datetime(b) BETWEEN '2017-07-04' AND '2017-07-08'; +} {/USING INDEX t3b/} +do_execsql_test date2-331 { + SELECT a FROM t3 + WHERE typeof(b)='real' + AND datetime(b) BETWEEN '2017-07-04' AND '2017-07-08' + ORDER BY a; +} {3 4 5 6} + +do_execsql_test date2-400 { + CREATE TABLE t4(a INTEGER PRIMARY KEY,b); + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<1000) + INSERT INTO t4(a,b) SELECT x, julianday('2017-07-01')+x FROM c; + UPDATE t4 SET b='now' WHERE a=500; +} +do_catchsql_test date2-410 { + CREATE INDEX t4b1 ON t4(b) + WHERE date(b) BETWEEN '2017-06-01' AND '2017-08-31'; +} {1 {non-deterministic function in index expression or CHECK constraint}} +do_execsql_test date2-420 { + DELETE FROM t4 WHERE a=500; + CREATE INDEX t4b1 ON t4(b) + WHERE date(b) BETWEEN '2017-06-01' AND '2017-08-31'; +} +do_catchsql_test date2-430 { + INSERT INTO t4(a,b) VALUES(9999,'now'); +} {1 {non-deterministic function in index expression or CHECK constraint}} + +do_execsql_test date2-500 { + CREATE TABLE mods(x); + INSERT INTO mods(x) VALUES + ('+10 days'), + ('-10 days'), + ('+10 hours'), + ('-10 hours'), + ('+10 minutes'), + ('-10 minutes'), + ('+10 seconds'), + ('-10 seconds'), + ('+10 months'), + ('-10 months'), + ('+10 years'), + ('-10 years'), + ('start of month'), + ('start of year'), + ('start of day'), + ('weekday 1'), + ('unixepoch'); + CREATE TABLE t5(y,m); + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<5) + INSERT INTO t5(y,m) SELECT julianday('2017-07-01')+c.x, mods.x FROM c, mods; + CREATE INDEX t5x1 on t5(y) WHERE datetime(y,m) IS NOT NULL; +} +do_catchsql_test date2-510 { + INSERT INTO t5(y,m) VALUES('2017-07-20','localtime'); +} {1 {non-deterministic function in index expression or CHECK constraint}} +do_catchsql_test date2-520 { + INSERT INTO t5(y,m) VALUES('2017-07-20','utc'); +} {1 {non-deterministic function in index expression or CHECK constraint}} + + + + + + +finish_test Index: test/eqp.test ================================================================== --- test/eqp.test +++ test/eqp.test @@ -186,28 +186,28 @@ 1 0 0 {SCAN TABLE t1 AS sub} } do_eqp_test 3.1.2 { SELECT * FROM t1 WHERE (SELECT x FROM t1 AS sub); } { + 0 0 0 {SCAN TABLE t1} 0 0 0 {EXECUTE SCALAR SUBQUERY 1} 1 0 0 {SCAN TABLE t1 AS sub} - 0 0 0 {SCAN TABLE t1} } do_eqp_test 3.1.3 { SELECT * FROM t1 WHERE (SELECT x FROM t1 AS sub ORDER BY y); } { + 0 0 0 {SCAN TABLE t1} 0 0 0 {EXECUTE SCALAR SUBQUERY 1} 1 0 0 {SCAN TABLE t1 AS sub} 1 0 0 {USE TEMP B-TREE FOR ORDER BY} - 0 0 0 {SCAN TABLE t1} } do_eqp_test 3.1.4 { SELECT * FROM t1 WHERE (SELECT x FROM t2 ORDER BY x); } { + 0 0 0 {SCAN TABLE t1} 0 0 0 {EXECUTE SCALAR SUBQUERY 1} 1 0 0 {SCAN TABLE t2 USING COVERING INDEX t2i1} - 0 0 0 {SCAN TABLE t1} } det 3.2.1 { SELECT * FROM (SELECT * FROM t1 ORDER BY x LIMIT 10) ORDER BY y LIMIT 5 } { Index: test/fallocate.test ================================================================== --- test/fallocate.test +++ test/fallocate.test @@ -57,11 +57,13 @@ # causes a database file to grow, the database grows to its previous size # on disk, not to the minimum size required to hold the database image. # do_test fallocate-1.7 { execsql { BEGIN; INSERT INTO t1 VALUES(1, 2); } - if {[permutation] != "inmemory_journal"} { + if {[permutation] != "inmemory_journal" + && [permutation] != "atomic-batch-write" + } { hexio_get_int [hexio_read test.db-journal 16 4] } else { set {} 1024 } } {1024} Index: test/fts3join.test ================================================================== --- test/fts3join.test +++ test/fts3join.test @@ -78,7 +78,27 @@ do_execsql_test 3.2 { SELECT * FROM tt LEFT JOIN vt ON (vt MATCH 'abc') } {1 abc 2 abc} +#------------------------------------------------------------------------- +# Test that queries of the form found in test case 4.2 use an automatic +# index to avoid running multiple fts queries. +# +do_execsql_test 4.1 { + CREATE VIRTUAL TABLE ft4 USING fts3(x); + CREATE TABLE t4(y, z); + CREATE INDEX t4y ON t1(y); +} + +do_eqp_test 4.2 { + SELECT * FROM t4 LEFT JOIN ( + SELECT docid, * FROM ft4 WHERE ft4 MATCH ? + ) AS rr ON t4.rowid=rr.docid + WHERE t4.y = ?; +} { + 1 0 0 {SCAN TABLE ft4 VIRTUAL TABLE INDEX 3:} + 0 0 0 {SCAN TABLE t4} + 0 1 1 {SEARCH SUBQUERY 1 AS rr USING AUTOMATIC COVERING INDEX (docid=?)} +} finish_test Index: test/fts3misc.test ================================================================== --- test/fts3misc.test +++ test/fts3misc.test @@ -219,11 +219,10 @@ WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<50000) INSERT INTO t6 SELECT 'x x x x x x x x x x x' FROM s; COMMIT; } -breakpoint do_execsql_test 6.1 { SELECT rowid FROM t6 WHERE t6 MATCH 'b OR "x a"' } {50001 50002 50003 50004} Index: test/in.test ================================================================== --- test/in.test +++ test/in.test @@ -627,11 +627,10 @@ CREATE INDEX i5 ON b(id); SELECT * FROM a WHERE id NOT IN (SELECT id FROM b); } } {} -breakpoint do_test in-13.15 { catchsql { SELECT 0 WHERE (SELECT 0,0) OR (0 IN (1,2)); } } {1 {sub-select returns 2 columns - expected 1}} Index: test/incrblob2.test ================================================================== --- test/incrblob2.test +++ test/incrblob2.test @@ -329,10 +329,13 @@ set rc [catch { # Prior to 2015-02-07, the following caused a segfault due to # integer overflow. sqlite3_blob_read $rdHandle 2147483647 2147483647 } errmsg] + if {[string match {out of memory in *test_blob.c} $errmsg]} { + set errmsg SQLITE_ERROR + } lappend rc $errmsg } {1 SQLITE_ERROR} } do_test incrblob2-6.2c { set rc [catch { Index: test/indexexpr1.test ================================================================== --- test/indexexpr1.test +++ test/indexexpr1.test @@ -179,16 +179,16 @@ WHERE substr(a,27,3)=='ord' AND d>=29; } {/USING INDEX t1a2/} do_catchsql_test indexexpr1-300 { - CREATE TABLE t2(a,b,c); + CREATE TABLE t2(a,b,c); INSERT INTO t2 VALUES(1,2,3); CREATE INDEX t2x1 ON t2(a,b+random()); } {1 {non-deterministic functions prohibited in index expressions}} do_catchsql_test indexexpr1-301 { - CREATE INDEX t2x1 ON t2(a+julianday('now')); -} {1 {non-deterministic functions prohibited in index expressions}} + CREATE INDEX t2x1 ON t2(julianday('now',a)); +} {1 {non-deterministic function in index expression or CHECK constraint}} do_catchsql_test indexexpr1-310 { CREATE INDEX t2x2 ON t2(a,b+(SELECT 15)); } {1 {subqueries prohibited in index expressions}} do_catchsql_test indexexpr1-320 { CREATE TABLE e1(x,y,UNIQUE(y,substr(x,1,5))); @@ -377,7 +377,29 @@ CREATE TABLE t1300(a INTEGER PRIMARY KEY, b); INSERT INTO t1300 VALUES(1,'coffee'),(2,'COFFEE'),(3,'stress'),(4,'STRESS'); CREATE INDEX t1300bexpr ON t1300( substr(b,4) ); SELECT a FROM t1300 WHERE substr(b,4)='ess' COLLATE nocase ORDER BY +a; } {3 4} + +# Ticket https://sqlite.org/src/tktview/aa98619a +# Assertion fault using an index on a constant +# +do_execsql_test indexexpr1-1400 { + CREATE TABLE t1400(x TEXT); + CREATE INDEX t1400x ON t1400(1); -- Index on a constant + SELECT 1 IN (SELECT 2) FROM t1400; +} {} +do_execsql_test indexexpr1-1410 { + INSERT INTO t1400 VALUES('a'),('b'); + SELECT 1 IN (SELECT 2) FROM t1400; +} {0 0} +do_execsql_test indexexpr1-1420 { + SELECT 1 IN (SELECT 2 UNION ALL SELECT 1) FROM t1400; +} {1 1} +do_execsql_test indexexpr1-1430 { + DROP INDEX t1400x; + CREATE INDEX t1400x ON t1400(abs(15+3)); + SELECT abs(15+3) IN (SELECT 17 UNION ALL SELECT 18) FROM t1; +} {1 1} + finish_test Index: test/indexexpr2.test ================================================================== --- test/indexexpr2.test +++ test/indexexpr2.test @@ -37,7 +37,103 @@ } do_execsql_test 2.1 { SELECT a+1, quote(a+1) FROM t1 ORDER BY 1; } {2 2 3 3 4 4} + +#------------------------------------------------------------------------- +# At one point SQLite was incorrectly using indexes on expressions to +# optimize ORDER BY and GROUP BY clauses even when the collation +# sequences of the query and index did not match (ticket [e20dd54ab0e4]). +# The following tests - 3.* - attempt to verify that this has been fixed. +# + +reset_db +do_execsql_test 3.1.0 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a, b); +} {} + +do_eqp_test 3.1.1 { + SELECT b FROM t1 WHERE b IS NOT NULL AND a IS NULL + GROUP BY b COLLATE nocase + ORDER BY b COLLATE nocase; +} {/USE TEMP B-TREE FOR GROUP BY/} + +do_execsql_test 3.2.0 { + CREATE TABLE t2(x); + + INSERT INTO t2 VALUES('.ABC'); + INSERT INTO t2 VALUES('.abcd'); + INSERT INTO t2 VALUES('.defg'); + INSERT INTO t2 VALUES('.DEF'); +} {} + +do_execsql_test 3.2.1 { + SELECT x FROM t2 ORDER BY substr(x, 2) COLLATE nocase; +} { + .ABC .abcd .DEF .defg +} + +do_execsql_test 3.2.2 { + CREATE INDEX i2 ON t2( substr(x, 2) ); + SELECT x FROM t2 ORDER BY substr(x, 2) COLLATE nocase; +} { + .ABC .abcd .DEF .defg +} + +do_execsql_test 3.3.0 { + CREATE TABLE t3(x); +} + +ifcapable json1 { + do_eqp_test 3.3.1 { + SELECT json_extract(x, '$.b') FROM t2 + WHERE json_extract(x, '$.b') IS NOT NULL AND json_extract(x, '$.a') IS NULL + GROUP BY json_extract(x, '$.b') COLLATE nocase + ORDER BY json_extract(x, '$.b') COLLATE nocase; + } { + 0 0 0 {SCAN TABLE t2} + 0 0 0 {USE TEMP B-TREE FOR GROUP BY} + } + + do_execsql_test 3.3.2 { + CREATE INDEX i3 ON t3(json_extract(x, '$.a'), json_extract(x, '$.b')); + } {} + + do_eqp_test 3.3.3 { + SELECT json_extract(x, '$.b') FROM t3 + WHERE json_extract(x, '$.b') IS NOT NULL AND json_extract(x, '$.a') IS NULL + GROUP BY json_extract(x, '$.b') COLLATE nocase + ORDER BY json_extract(x, '$.b') COLLATE nocase; + } { + 0 0 0 {SEARCH TABLE t3 USING INDEX i3 (=?)} + 0 0 0 {USE TEMP B-TREE FOR GROUP BY} + } +} + +do_execsql_test 3.4.0 { + CREATE TABLE t4(a, b); + INSERT INTO t4 VALUES('.ABC', 1); + INSERT INTO t4 VALUES('.abc', 2); + INSERT INTO t4 VALUES('.ABC', 3); + INSERT INTO t4 VALUES('.abc', 4); +} + +do_execsql_test 3.4.1 { + SELECT * FROM t4 + WHERE substr(a, 2) = 'abc' COLLATE NOCASE + ORDER BY substr(a, 2), b; +} { + .ABC 1 .ABC 3 .abc 2 .abc 4 +} + +do_execsql_test 3.4.2 { + CREATE INDEX i4 ON t4( substr(a, 2) COLLATE NOCASE, b ); + SELECT * FROM t4 + WHERE substr(a, 2) = 'abc' COLLATE NOCASE + ORDER BY substr(a, 2), b; +} { + .ABC 1 .ABC 3 .abc 2 .abc 4 +} finish_test Index: test/kvtest.c ================================================================== --- test/kvtest.c +++ test/kvtest.c @@ -739,31 +739,17 @@ sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &iCur, &iHiwtr, bReset); fprintf(out, "Number of Pcache Overflow Bytes: %d (max %d) bytes\n", iCur, iHiwtr); iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_SCRATCH_USED, &iCur, &iHiwtr, bReset); - fprintf(out, - "Number of Scratch Allocations Used: %d (max %d)\n", - iCur, iHiwtr); - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_SCRATCH_OVERFLOW, &iCur, &iHiwtr, bReset); - fprintf(out, - "Number of Scratch Overflow Bytes: %d (max %d) bytes\n", - iCur, iHiwtr); - iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &iCur, &iHiwtr, bReset); fprintf(out, "Largest Allocation: %d bytes\n", iHiwtr); iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_PAGECACHE_SIZE, &iCur, &iHiwtr, bReset); fprintf(out, "Largest Pcache Allocation: %d bytes\n", iHiwtr); - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_SCRATCH_SIZE, &iCur, &iHiwtr, bReset); - fprintf(out, "Largest Scratch Allocation: %d bytes\n", - iHiwtr); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_USED, &iCur, &iHiwtr, bReset); fprintf(out, "Pager Heap Usage: %d bytes\n", iCur); Index: test/like.test ================================================================== --- test/like.test +++ test/like.test @@ -205,11 +205,11 @@ } queryplan { SELECT x FROM t1 WHERE x LIKE 'abc%' ORDER BY 1; } } {abc abcd nosort {} i1} -do_test like-3.3.101 { +do_test like-3.3.100.cnt { set sqlite_like_count } 0 # The like optimization works even when the pattern is a bound parameter # @@ -1045,7 +1045,55 @@ }] 0] puts -nonewline " ($x ms - want less than 1000) " expr {$x<1000} } {1} } + +# As of 2017-07-27 (3.21.0) the LIKE optimization works with ESCAPE as +# long as the ESCAPE is a single-byte literal. +# +db close +sqlite3 db :memory: +do_execsql_test like-15.100 { + CREATE TABLE t15(x TEXT COLLATE nocase, y, PRIMARY KEY(x)); + INSERT INTO t15(x,y) VALUES + ('abcde',1), ('ab%de',2), ('a_cde',3), + ('uvwxy',11),('uvwx%',12),('uvwx_',13), + ('_bcde',21),('%bcde',22), + ('abcd_',31),('abcd%',32), + ('ab%xy',41); + SELECT y FROM t15 WHERE x LIKE 'ab/%d%' ESCAPE '/'; +} {2} +do_execsql_test like-15.101 { + EXPLAIN QUERY PLAN + SELECT y FROM t15 WHERE x LIKE 'ab/%d%' ESCAPE '/'; +} {/SEARCH/} +do_execsql_test like-15.102 { + EXPLAIN QUERY PLAN + SELECT y FROM t15 WHERE x LIKE 'ab/%d%' ESCAPE '//'; +} {/SCAN/} +do_execsql_test like-15.103 { + EXPLAIN QUERY PLAN + SELECT y FROM t15 WHERE x LIKE 'ab/%d%' ESCAPE ''; +} {/SCAN/} +do_execsql_test like-15.110 { + SELECT y FROM t15 WHERE x LIKE 'abcdx%%' ESCAPE 'x'; +} {32} +do_execsql_test like-15.111 { + SELECT y FROM t15 WHERE x LIKE 'abx%%' ESCAPE 'x' ORDER BY +y +} {2 41} +do_execsql_test like-15.112 { + EXPLAIN QUERY PLAN + SELECT y FROM t15 WHERE x LIKE 'abx%%' ESCAPE 'x' ORDER BY +y +} {/SEARCH/} +do_execsql_test like-15.120 { + SELECT y FROM t15 WHERE x LIKE '/%bc%' ESCAPE '/'; +} {22} +do_execsql_test like-15.121 { + EXPLAIN QUERY PLAN + SELECT y FROM t15 WHERE x LIKE '/%bc%' ESCAPE '/'; +} {/SEARCH/} + + + finish_test Index: test/lookaside.test ================================================================== --- test/lookaside.test +++ test/lookaside.test @@ -31,11 +31,10 @@ test_set_config_pagecache 0 0 catch {db close} sqlite3_shutdown -sqlite3_config_scratch 0 0 sqlite3_initialize autoinstall_test_functions sqlite3 db test.db # Make sure sqlite3_db_config() and sqlite3_db_status are working. Index: test/malloc5.test ================================================================== --- test/malloc5.test +++ test/malloc5.test @@ -118,11 +118,10 @@ execsql { BEGIN; CREATE TABLE def(d, e, f); SELECT * FROM abc; } - breakpoint value_in_range $::pgalloc $::mrange [sqlite3_release_memory 500] } [value_in_range $::pgalloc $::mrange] do_test malloc5-1.7 { # Database should not be locked this time. sqlite3 db2 test.db Index: test/memsubsys1.test ================================================================== --- test/memsubsys1.test +++ test/memsubsys1.test @@ -14,11 +14,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl sqlite3_reset_auto_extension -# This test assumes that no page-cache or scratch buffers are installed +# This test assumes that no page-cache buffers are installed # by default when a new database connection is opened. As a result, it # will not work with the "memsubsys1" permutation. # if {[permutation] == "memsubsys1"} { finish_test @@ -156,16 +156,15 @@ } 20 do_test memsubsys1-3.2.5 { set s_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2] } 0 -# Test 4: Activate both PAGECACHE and SCRATCH. +# Test 4: Activate PAGECACHE # db close sqlite3_shutdown sqlite3_config_pagecache [expr 1024+$xtra_size] 50 -sqlite3_config_scratch 6000 2 sqlite3_initialize reset_highwater_marks build_test_db memsubsys1-4 {PRAGMA page_size=1024} #show_memstats do_test memsubsys1-4.3 { @@ -179,157 +178,16 @@ } do_test memsubsys1-4.5 { set maxreq [lindex [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] 2] expr {$maxreq<7000} } 1 -do_test memsubsys1-4.6 { - set s_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2] -} 1 - -# Test 5: Activate both PAGECACHE and SCRATCH. But make the page size is -# such that the SCRATCH allocations are too small. -# -db close -sqlite3_shutdown -sqlite3_config_pagecache [expr 4096+$xtra_size] 24 -sqlite3_config_scratch 4000 2 -sqlite3_initialize -reset_highwater_marks -build_test_db memsubsys1-5 {PRAGMA page_size=4096} -#show_memstats -do_test memsubsys1-5.3 { - set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2] -} {/^2[34]$/} -set msize [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] -if {[lindex $msize 2]!=0} { - do_test memsubsys1-5.4 { - set maxreq [lindex [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] 2] - expr {$maxreq>4096} - } 1 - do_test memsubsys1-5.5 { - set s_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2] - } 0 - do_test memsubsys1-5.6 { - set s_ovfl [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_OVERFLOW 0] 2] - expr {$s_ovfl>6000} - } 1 -} - -# Test 6: Activate both PAGECACHE and SCRATCH with a 4k page size. -# Make it so that SCRATCH is large enough -# -db close -sqlite3_shutdown -sqlite3_config_pagecache [expr 4096+$xtra_size] 24 -sqlite3_config_scratch 25300 1 -sqlite3_initialize -reset_highwater_marks -build_test_db memsubsys1-6 {PRAGMA page_size=4096} -#show_memstats -do_test memsubsys1-6.3 { - set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2] -} {/^2[34]$/} -#do_test memsubsys1-6.4 { -# set maxreq [lindex [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] 2] -# expr {$maxreq>4096 && $maxreq<=(4096+$xtra_size)} -#} 1 -do_test memsubsys1-6.5 { - set s_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2] -} 1 -do_test memsubsys1-6.6 { - set s_ovfl [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_OVERFLOW 0] 2] -} 0 - -# Test 7: Activate both PAGECACHE and SCRATCH with a 4k page size. -# Set cache_size small so that no PAGECACHE overflow occurs. Verify -# that maximum allocation size is small. -# -db close -sqlite3_shutdown -sqlite3_config_pagecache [expr 4096+$xtra_size] 24 -sqlite3_config_scratch 25300 1 -sqlite3_initialize -reset_highwater_marks -build_test_db memsubsys1-7 { - PRAGMA page_size=4096; - PRAGMA cache_size=10; - PRAGMA temp_store=memory; -} -#show_memstats -do_test memsubsys1-7.3 { - set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2] - expr {$pg_used<24} -} 1 -if !$::sqlite_options(enable_purgeable_pcache) { - do_test memsubsys1-7.4 { - set pg_ovfl [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_OVERFLOW 0] 2] - } 0 - do_test memsubsys1-7.5 { - set maxreq [lindex [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] 2] - expr {$maxreq<4100 + 8200*[nonzero_reserved_bytes]} - } 1 -} -do_test memsubsys1-7.6 { - set s_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2] -} 1 -do_test memsubsys1-7.7 { - set s_ovfl [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_OVERFLOW 0] 2] -} 0 - -# Test 8: Disable PAGECACHE. Make available SCRATCH zero. Verify that -# the SCRATCH overflow logic works. -# -db close -sqlite3_shutdown -sqlite3_config_pagecache 0 0 -sqlite3_config_scratch 25000 0 -sqlite3_initialize -reset_highwater_marks -do_test memsubsys1-8.1 { - set pg_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2] -} 0 -do_test memsubsys1-8.2 { - set s_ovfl [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_OVERFLOW 0] 2] -} 0 -if {[lindex $msize 2]!=0} { - do_test memsubsys1-8.3 { - sqlite3 db :memory: - db eval { - CREATE TABLE t1(x); - INSERT INTO t1 VALUES(zeroblob(400)); - INSERT INTO t1 VALUES(zeroblob(400)); - INSERT INTO t1 SELECT * FROM t1; - INSERT INTO t1 SELECT * FROM t1; - INSERT INTO t1 SELECT * FROM t1; - } - expr {[lindex [sqlite3_status SQLITE_STATUS_SCRATCH_OVERFLOW 0] 2]>0} - } 1 - db close -} -sqlite3_shutdown -sqlite3_config_memstatus 0 -sqlite3_initialize -do_test memsubsys1-8.4 { - sqlite3 db :memory: - db eval { - CREATE TABLE t1(x); - INSERT INTO t1 VALUES(zeroblob(400)); - INSERT INTO t1 VALUES(zeroblob(400)); - INSERT INTO t1 SELECT * FROM t1; - INSERT INTO t1 SELECT * FROM t1; - INSERT INTO t1 SELECT * FROM t1; - SELECT rowid FROM t1; - } -} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16} - db close sqlite3_shutdown sqlite3_config_memstatus 1 -sqlite3_config_scratch 0 0 sqlite3_config_lookaside 100 500 sqlite3_config serialized sqlite3_initialize autoinstall_test_functions test_restore_config_pagecache finish_test Index: test/misc1.test ================================================================== --- test/misc1.test +++ test/misc1.test @@ -477,30 +477,32 @@ # ifcapable curdir { # Make sure a database connection still works after changing the # working directory. # -do_test misc1-14.1 { - file mkdir tempdir - cd tempdir - execsql {BEGIN} - file exists ./test.db-journal -} {0} -do_test misc1-14.2a { - execsql {UPDATE t1 SET a=a||'x' WHERE 0} - file exists ../test.db-journal -} {0} -do_test misc1-14.2b { - execsql {UPDATE t1 SET a=a||'y' WHERE 1} - file exists ../test.db-journal -} {1} -do_test misc1-14.3 { - cd .. - forcedelete tempdir - execsql {COMMIT} - file exists ./test.db-journal -} {0} +if {[atomic_batch_write test.db]==0} { + do_test misc1-14.1 { + file mkdir tempdir + cd tempdir + execsql {BEGIN} + file exists ./test.db-journal + } {0} + do_test misc1-14.2a { + execsql {UPDATE t1 SET a=a||'x' WHERE 0} + file exists ../test.db-journal + } {0} + do_test misc1-14.2b { + execsql {UPDATE t1 SET a=a||'y' WHERE 1} + file exists ../test.db-journal + } {1} + do_test misc1-14.3 { + cd .. + forcedelete tempdir + execsql {COMMIT} + file exists ./test.db-journal + } {0} +} } # A failed create table should not leave the table in the internal # data structures. Ticket #238. # Index: test/ossfuzz.c ================================================================== --- test/ossfuzz.c +++ test/ossfuzz.c @@ -67,10 +67,32 @@ if( iDiff > p->mxInterval ) p->mxInterval = iDiff; p->nCb++; return rc; } #endif + +/* +** Disallow debugging pragmas such as "PRAGMA vdbe_debug" and +** "PRAGMA parser_trace" since they can dramatically increase the +** amount of output without actually testing anything useful. +*/ +static int block_debug_pragmas( + void *Notused, + int eCode, + const char *zArg1, + const char *zArg2, + const char *zArg3, + const char *zArg4 +){ + if( eCode==SQLITE_PRAGMA + && (sqlite3_strnicmp("vdbe_", zArg1, 5)==0 + || sqlite3_stricmp("parser_trace", zArg1)==0) + ){ + return SQLITE_DENY; + } + return SQLITE_OK; +} /* ** Callback for sqlite3_exec(). */ static int exec_handler(void *pCnt, int argc, char **argv, char **namev){ @@ -125,10 +147,13 @@ sqlite3_limit(cx.db, SQLITE_LIMIT_VDBE_OP, 25000); /* Bit 1 of the selector enables foreign key constraints */ sqlite3_db_config(cx.db, SQLITE_DBCONFIG_ENABLE_FKEY, uSelector&1, &rc); uSelector >>= 1; + + /* Do not allow debugging pragma statements that might cause excess output */ + sqlite3_set_authorizer(cx.db, block_debug_pragmas, 0); /* Remaining bits of the selector determine a limit on the number of ** output rows */ execCnt = uSelector + 1; Index: test/permutations.test ================================================================== --- test/permutations.test +++ test/permutations.test @@ -386,10 +386,34 @@ } -files [ test_set $allquicktests -exclude *malloc* *ioerr* *fault* oserror.test \ pager1.test syscall.test sysfault.test tkt3457.test quota* superlock* \ wal* mmap* ] + +test_suite "atomic-batch-write" -prefix "" -description { + Like veryquick.test, but must be run on a file-system that supports + atomic-batch-writes. Tests that depend on the journal file being present + are omitted. +} -files [ + test_set $allquicktests -exclude *malloc* *ioerr* *fault* *bigfile* *_err* \ + *fts5corrupt* *fts5big* *fts5aj* \ + crash8.test delete_db.test \ + exclusive.test journal3.test \ + journal1.test \ + jrnlmode.test jrnlmode2.test \ + lock4.test pager1.test \ + pager3.test sharedA.test \ + symlink.test stmt.test \ + sync.test sync2.test \ + tempdb.test tkt3457.test \ + vacuum5.test wal2.test \ + walmode.test zerodamage.test +] -initialize { + if {[atomic_batch_write test.db]==0} { + error "File system does NOT support atomic-batch-write" + } +} lappend ::testsuitelist xxx #------------------------------------------------------------------------- # Define the coverage related test suites: # @@ -429,59 +453,55 @@ lappend ::testsuitelist xxx #------------------------------------------------------------------------- # Define the permutation test suites: # -# Run some tests using pre-allocated page and scratch blocks. +# Run some tests using pre-allocated page blocks. # # mmap1.test is excluded because a good number of its tests depend on # the page-cache being larger than the database. But this permutation # causes the effective limit on the page-cache to be just 24 pages. # test_suite "memsubsys1" -description { - Tests using pre-allocated page and scratch blocks + Tests using pre-allocated page blocks } -files [ test_set $::allquicktests -exclude ioerr5.test malloc5.test mmap1.test ] -initialize { test_set_config_pagecache 4096 24 catch {db close} sqlite3_shutdown - sqlite3_config_scratch 25000 1 sqlite3_initialize autoinstall_test_functions } -shutdown { test_restore_config_pagecache catch {db close} sqlite3_shutdown - sqlite3_config_scratch 0 0 sqlite3_initialize autoinstall_test_functions } -# Run some tests using pre-allocated page and scratch blocks. This time +# Run some tests using pre-allocated page blocks. This time # the allocations are too small to use in most cases. # # Both ioerr5.test and malloc5.test are excluded because they test the # sqlite3_soft_heap_limit() and sqlite3_release_memory() functionality. # This functionality is disabled if a pre-allocated page block is provided. # test_suite "memsubsys2" -description { - Tests using small pre-allocated page and scratch blocks + Tests using small pre-allocated page blocks } -files [ test_set $::allquicktests -exclude ioerr5.test malloc5.test ] -initialize { test_set_config_pagecache 512 5 catch {db close} sqlite3_shutdown - sqlite3_config_scratch 1000 1 sqlite3_initialize autoinstall_test_functions } -shutdown { test_restore_config_pagecache catch {db close} sqlite3_shutdown - sqlite3_config_scratch 0 0 sqlite3_initialize autoinstall_test_functions } # Run all tests with the lookaside allocator disabled. Index: test/pragma.test ================================================================== --- test/pragma.test +++ test/pragma.test @@ -2036,7 +2036,28 @@ db2 eval { PRAGMA foreign_key_list(t2); } } {0 0 t1 y {} {NO ACTION} {NO ACTION} NONE} +reset_db +do_execsql_test 24.0 { + PRAGMA page_size = 1024; + CREATE TABLE t1(a, b, c); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES('a', 'b', 'c'); + PRAGMA integrity_check; +} {ok} + +set r [db one {SELECT rootpage FROM sqlite_master WHERE name = 't1'}] +db close +hexio_write test.db [expr $r*1024 - 16] 000000000000000701040f0f1f616263 + +sqlite3 db test.db +do_catchsql_test 24.1 { + SELECT * FROM t1; +} {1 {database disk image is malformed}} +do_catchsql_test 24.2 { + PRAGMA integrity_check; +} {1 {database disk image is malformed}} + database_never_corrupt finish_test ADDED test/pragma5.test Index: test/pragma5.test ================================================================== --- /dev/null +++ test/pragma5.test @@ -0,0 +1,64 @@ +# 2017 August 25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# This file implements tests for the PRAGMA command. Specifically, +# those pragmas enabled at build time by setting: +# +# -DSQLITE_INTROSPECTION_PRAGMAS +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix pragma5 + +if { [catch {db one "SELECT count(*) FROM pragma_function_list"}] } { + finish_test + return +} + +db function external external + +do_execsql_test 1.0 { + PRAGMA table_info(pragma_function_list) +} { + 0 name {} 0 {} 0 + 1 builtin {} 0 {} 0 +} +do_execsql_test 1.1 { + SELECT * FROM pragma_function_list WHERE name='upper' +} {upper 1} +do_execsql_test 1.2 { + SELECT * FROM pragma_function_list WHERE name LIKE 'exter%'; +} {external 0} + +ifcapable fts5 { + do_execsql_test 2.0 { + PRAGMA table_info(pragma_module_list) + } { + 0 name {} 0 {} 0 + } + do_execsql_test 2.1 { + SELECT * FROM pragma_module_list WHERE name='fts5' + } {fts5} +} + +do_execsql_test 3.0 { + PRAGMA table_info(pragma_pragma_list) +} { + 0 name {} 0 {} 0 +} +do_execsql_test 3.1 { + SELECT * FROM pragma_pragma_list WHERE name='pragma_list' +} {pragma_list} + + +finish_test Index: test/pushdown.test ================================================================== --- test/pushdown.test +++ test/pushdown.test @@ -76,11 +76,10 @@ set L } {one} do_test 2.2 { set L [list] - breakpoint execsql { SELECT * FROM u1 WHERE 123=( SELECT x FROM u2 WHERE x=a AND f('two') ) AND f('three')=123 } Index: test/rollback.test ================================================================== --- test/rollback.test +++ test/rollback.test @@ -80,10 +80,11 @@ } {SQLITE_OK} if {$tcl_platform(platform) == "unix" && [permutation] ne "onefile" && [permutation] ne "inmemory_journal" + && [permutation] ne "atomic-batch-write" } { do_test rollback-2.1 { execsql { BEGIN; INSERT INTO t3 VALUES('hello world'); Index: test/savepoint.test ================================================================== --- test/savepoint.test +++ test/savepoint.test @@ -613,27 +613,32 @@ # savepoints and ATTACH statements. # # First make sure it is not possible to attach or detach a database while # a savepoint is open (it is not possible if any transaction is open). +# +# UPDATE 2017-07-26: It is not possible to ATTACH and DETACH within a +# a transaction. # do_test savepoint-10.1.1 { catchsql { SAVEPOINT one; ATTACH 'test2.db' AS aux; + DETACH aux; } -} {1 {cannot ATTACH database within transaction}} +} {0 {}} do_test savepoint-10.1.2 { execsql { RELEASE one; ATTACH 'test2.db' AS aux; } catchsql { SAVEPOINT one; DETACH aux; + ATTACH 'test2.db' AS aux; } -} {1 {cannot DETACH database within transaction}} +} {0 {}} do_test savepoint-10.1.3 { execsql { RELEASE one; DETACH aux; } ADDED test/schema6.test Index: test/schema6.test ================================================================== --- /dev/null +++ test/schema6.test @@ -0,0 +1,163 @@ +# 2017-07-30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements tests to show that certain CREATE TABLE statements +# generate identical database files. For example, changes in identifier +# names, white-space, and formatting of the CREATE TABLE statement should +# produce identical table content. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix schema6 + +# Command: check_same_database_content TESTNAME SQL1 SQL2 SQL3 ... +# +# This command creates fresh databases using SQL1 and subsequent arguments +# and checks to make sure the content of all database files is byte-for-byte +# identical. Page 1 of the database files is allowed to be different, since +# page 1 contains the sqlite_master table which is expected to vary. +# +proc check_same_database_content {basename args} { + set i 0 + set hash {} + foreach sql $args { + catch {db close} + forcedelete test.db + sqlite3 db test.db + db eval $sql + set pgsz [db one {PRAGMA page_size}] + db close + set sz [file size test.db] + set thishash [md5file test.db $pgsz [expr {$sz-$pgsz}]] + if {$i==0} { + set hash $thishash + } else { + do_test $basename-$i "set x $thishash" $hash + } + incr i + } +} + +# Command: check_different_database_content TESTNAME SQL1 SQL2 SQL3 ... +# +# This command creates fresh databases using SQL1 and subsequent arguments +# and checks to make sure the content of all database files is different +# in ways other than on page 1. +# +proc check_different_database_content {basename args} { + set i 0 + set hashes {} + foreach sql $args { + forcedelete test.db + sqlite3 db test.db + db eval $sql + set pgsz [db one {PRAGMA page_size}] + db close + set sz [file size test.db] + set thishash [md5file test.db $pgsz [expr {$sz-$pgsz}]] + set j [lsearch $hashes $thishash] + if {$j>=0} { + do_test $basename-$i "set x {$i is the same as $j}" "All are different" + } else { + do_test $basename-$i "set x {All are different}" "All are different" + } + lappend hashes $thishash + incr i + } +} + +check_same_database_content 100 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b UNIQUE); + INSERT INTO t1(a,b) VALUES(123,'Four score and seven years ago...'); +} { + CREATE TABLE t1(xyz INTEGER, abc, PRIMARY KEY(xyz), UNIQUE(abc)); + INSERT INTO t1(xyz,abc) VALUES(123,'Four score and seven years ago...'); +} { + CREATE TABLE t1(xyz INTEGER, abc, UNIQUE(abc), PRIMARY KEY(xyz)); + INSERT INTO t1(xyz,abc) VALUES(123,'Four score and seven years ago...'); +} { + CREATE TABLE t1(a INTEGER PRIMARY KEY ASC, b UNIQUE); + INSERT INTO t1(a,b) VALUES(123,'Four score and seven years ago...'); +} { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE UNIQUE INDEX t1b ON t1(b); + INSERT INTO t1(a,b) VALUES(123,'Four score and seven years ago...'); +} { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1(a,b) VALUES(123,'Four score and seven years ago...'); + CREATE UNIQUE INDEX t1b ON t1(b); +} + +check_same_database_content 110 { + CREATE TABLE t1(a INTEGER PRIMARY KEY UNIQUE, b UNIQUE); + INSERT INTO t1(a,b) VALUES(123,'Four score and seven years ago...'); +} { + CREATE TABLE t1(a INTEGER UNIQUE PRIMARY KEY, b UNIQUE); + INSERT INTO t1(a,b) VALUES(123,'Four score and seven years ago...'); +} { + CREATE TABLE t1(a INTEGER UNIQUE PRIMARY KEY, b UNIQUE, UNIQUE(a)); + INSERT INTO t1(a,b) VALUES(123,'Four score and seven years ago...'); +} { + CREATE TABLE t1(a INTEGER UNIQUE PRIMARY KEY, b); + CREATE UNIQUE INDEX t1b ON t1(b); + INSERT INTO t1(a,b) VALUES(123,'Four score and seven years ago...'); +} { + CREATE TABLE t1(a INTEGER UNIQUE PRIMARY KEY, b); + INSERT INTO t1(a,b) VALUES(123,'Four score and seven years ago...'); + CREATE UNIQUE INDEX t1b ON t1(b); +} + +check_same_database_content 120 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b UNIQUE) WITHOUT ROWID; + INSERT INTO t1(a,b) VALUES(123,'Four score and seven years ago...'); +} { + CREATE TABLE t1(xyz INTEGER, abc, PRIMARY KEY(xyz), UNIQUE(abc))WITHOUT ROWID; + INSERT INTO t1(xyz,abc) VALUES(123,'Four score and seven years ago...'); +} { + CREATE TABLE t1(xyz INTEGER, abc, UNIQUE(abc), PRIMARY KEY(xyz))WITHOUT ROWID; + INSERT INTO t1(xyz,abc) VALUES(123,'Four score and seven years ago...'); +} { + CREATE TABLE t1(a INTEGER PRIMARY KEY ASC, b UNIQUE) WITHOUT ROWID; + INSERT INTO t1(a,b) VALUES(123,'Four score and seven years ago...'); +} { + CREATE TABLE t1(a INTEGER PRIMARY KEY UNIQUE, b UNIQUE) WITHOUT ROWID; + INSERT INTO t1(a,b) VALUES(123,'Four score and seven years ago...'); +} { + CREATE TABLE t1(a INTEGER UNIQUE PRIMARY KEY, b UNIQUE) WITHOUT ROWID; + INSERT INTO t1(a,b) VALUES(123,'Four score and seven years ago...'); +} { + CREATE TABLE t1(a INTEGER UNIQUE PRIMARY KEY, b UNIQUE, UNIQUE(a)) + WITHOUT ROWID; + INSERT INTO t1(a,b) VALUES(123,'Four score and seven years ago...'); +} { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b) WITHOUT ROWID; + CREATE UNIQUE INDEX t1b ON t1(b); + INSERT INTO t1(a,b) VALUES(123,'Four score and seven years ago...'); +} { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b) WITHOUT ROWID; + INSERT INTO t1(a,b) VALUES(123,'Four score and seven years ago...'); + CREATE UNIQUE INDEX t1b ON t1(b); +} + +check_different_database_content 130 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b UNIQUE); + INSERT INTO t1(a,b) VALUES(123,'Four score and seven years ago...'); +} { + CREATE TABLE t1(a INTEGER PRIMARY KEY UNIQUE, b UNIQUE); + INSERT INTO t1(a,b) VALUES(123,'Four score and seven years ago...'); +} { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b UNIQUE) WITHOUT ROWID; + INSERT INTO t1(a,b) VALUES(123,'Four score and seven years ago...'); +} + + +finish_test Index: test/speedtest1.c ================================================================== --- test/speedtest1.c +++ test/speedtest1.c @@ -23,11 +23,10 @@ " --pagesize N Set the page size to N\n" " --pcache N SZ Configure N pages of pagecache each of size SZ bytes\n" " --primarykey Use PRIMARY KEY instead of UNIQUE where appropriate\n" " --repeat N Repeat each SELECT N times (default: 1)\n" " --reprepare Reprepare each statement upon every invocation\n" - " --scratch N SZ Configure scratch memory for N slots of SZ bytes each\n" " --serialized Set serialized threading mode\n" " --singlethread Set single-threaded mode - disables all mutexing\n" " --sqlonly No-op. Only show the SQL that would have been run.\n" " --shrink-memory Invoke sqlite3_db_release_memory() frequently.\n" " --size N Relative test size. Default=100\n" @@ -1647,11 +1646,10 @@ int nLook = -1, szLook = 0; /* --lookaside configuration */ int noSync = 0; /* True for --nosync */ int pageSize = 0; /* Desired page size. 0 means default */ int nPCache = 0, szPCache = 0;/* --pcache configuration */ int doPCache = 0; /* True if --pcache is seen */ - int nScratch = 0, szScratch=0;/* --scratch configuration */ int showStats = 0; /* True for --stats */ int nThread = 0; /* --threads value */ int mmapSize = 0; /* How big of a memory map to use */ const char *zTSet = "main"; /* Which --testset torun */ int doTrace = 0; /* True for --trace */ @@ -1659,11 +1657,10 @@ const char *zDbName = 0; /* Name of the test database */ void *pHeap = 0; /* Allocated heap space */ void *pLook = 0; /* Allocated lookaside space */ void *pPCache = 0; /* Allocated storage for pcache */ - void *pScratch = 0; /* Allocated storage for scratch */ int iCur, iHi; /* Stats values, current and "highwater" */ int i; /* Loop counter */ int rc; /* API return code */ /* Display the version of SQLite being tested */ @@ -1739,15 +1736,10 @@ if( i>=argc-1 ) fatal_error("missing arguments on %s\n", argv[i]); g.nRepeat = integerValue(argv[i+1]); i += 1; }else if( strcmp(z,"reprepare")==0 ){ g.bReprepare = 1; - }else if( strcmp(z,"scratch")==0 ){ - if( i>=argc-2 ) fatal_error("missing arguments on %s\n", argv[i]); - nScratch = integerValue(argv[i+1]); - szScratch = integerValue(argv[i+2]); - i += 2; #if SQLITE_VERSION_NUMBER>=3006000 }else if( strcmp(z,"serialized")==0 ){ sqlite3_config(SQLITE_CONFIG_SERIALIZED); }else if( strcmp(z,"singlethread")==0 ){ sqlite3_config(SQLITE_CONFIG_SINGLETHREAD); @@ -1814,17 +1806,10 @@ nPCache*(sqlite3_int64)szPCache); } rc = sqlite3_config(SQLITE_CONFIG_PAGECACHE, pPCache, szPCache, nPCache); if( rc ) fatal_error("pcache configuration failed: %d\n", rc); } - if( nScratch>0 && szScratch>0 ){ - pScratch = malloc( nScratch*(sqlite3_int64)szScratch ); - if( pScratch==0 ) fatal_error("cannot allocate %lld-byte scratch\n", - nScratch*(sqlite3_int64)szScratch); - rc = sqlite3_config(SQLITE_CONFIG_SCRATCH, pScratch, szScratch, nScratch); - if( rc ) fatal_error("scratch configuration failed: %d\n", rc); - } if( nLook>=0 ){ sqlite3_config(SQLITE_CONFIG_LOOKASIDE, 0, 0); } #endif @@ -1937,18 +1922,14 @@ sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &iCur, &iHi, 0); printf("-- Outstanding Allocations: %d (max %d)\n", iCur,iHi); #endif sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &iCur, &iHi, 0); printf("-- Pcache Overflow Bytes: %d (max %d)\n", iCur,iHi); - sqlite3_status(SQLITE_STATUS_SCRATCH_OVERFLOW, &iCur, &iHi, 0); - printf("-- Scratch Overflow Bytes: %d (max %d)\n", iCur,iHi); sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &iCur, &iHi, 0); printf("-- Largest Allocation: %d bytes\n",iHi); sqlite3_status(SQLITE_STATUS_PAGECACHE_SIZE, &iCur, &iHi, 0); printf("-- Largest Pcache Allocation: %d bytes\n",iHi); - sqlite3_status(SQLITE_STATUS_SCRATCH_SIZE, &iCur, &iHi, 0); - printf("-- Largest Scratch Allocation: %d bytes\n", iHi); } #endif #ifdef __linux__ if( showStats ){ @@ -1957,9 +1938,8 @@ #endif /* Release memory */ free( pLook ); free( pPCache ); - free( pScratch ); free( pHeap ); return 0; } ADDED test/swarmvtab.test Index: test/swarmvtab.test ================================================================== --- /dev/null +++ test/swarmvtab.test @@ -0,0 +1,246 @@ +# 2017-07-15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is the "swarmvtab" extension +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix swarmvtab + +ifcapable !vtab { + finish_test + return +} + +load_static_extension db unionvtab + +set nFile $sqlite_open_file_count + +do_execsql_test 1.0 { + CREATE TABLE t0(a INTEGER PRIMARY KEY, b TEXT); + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<400) + INSERT INTO t0 SELECT i, hex(randomblob(50)) FROM s; + + CREATE TABLE dir(f, t, imin, imax); +} + +do_test 1.1 { + for {set i 0} {$i < 40} {incr i} { + set iMin [expr $i*10 + 1] + set iMax [expr $iMin+9] + + forcedelete "test.db$i" + execsql [subst { + ATTACH 'test.db$i' AS aux; + CREATE TABLE aux.t$i (a INTEGER PRIMARY KEY, b TEXT); + INSERT INTO aux.t$i SELECT * FROM t0 WHERE a BETWEEN $iMin AND $iMax; + DETACH aux; + INSERT INTO dir VALUES('test.db$i', 't$i', $iMin, $iMax); + }] + } + + execsql { + CREATE VIRTUAL TABLE temp.s1 USING swarmvtab('SELECT * FROM dir'); + } +} {} + +do_execsql_test 1.2 { + DROP TABLE s1; +} {} + +do_execsql_test 1.3 { + CREATE VIRTUAL TABLE temp.s1 USING swarmvtab('SELECT * FROM dir'); + SELECT count(*) FROM s1 WHERE rowid<50; +} {49} + +proc do_compare_test {tn where} { + set sql [subst { + SELECT (SELECT group_concat(a || ',' || b, ',') FROM t0 WHERE $where) + IS + (SELECT group_concat(a || ',' || b, ',') FROM s1 WHERE $where) + }] + + uplevel [list do_execsql_test $tn $sql 1] +} + +do_compare_test 1.4.1 "rowid = 700" +do_compare_test 1.4.2 "rowid = -1" +do_compare_test 1.4.3 "rowid = 0" +do_compare_test 1.4.4 "rowid = 55" +do_compare_test 1.4.5 "rowid BETWEEN 20 AND 100" +do_compare_test 1.4.6 "rowid > 350" +do_compare_test 1.4.7 "rowid >= 350" +do_compare_test 1.4.8 "rowid >= 200" +do_compare_test 1.4.9 "1" + +# Multiple simultaneous cursors. +# +do_execsql_test 1.5.1.(5-seconds-or-so) { + SELECT count(*) FROM s1 a, s1 b WHERE b.rowid<=200; +} {80000} +do_execsql_test 1.5.2 { + SELECT count(*) FROM s1 a, s1 b, s1 c + WHERE a.rowid=b.rowid AND b.rowid=c.rowid; +} {400} + +# Empty source tables. +# +do_test 1.6.0 { + for {set i 0} {$i < 20} {incr i} { + sqlite3 db2 test.db$i + db2 eval " DELETE FROM t$i " + db2 close + } + db eval { DELETE FROM t0 WHERE rowid<=200 } +} {} + +do_compare_test 1.6.1 "rowid = 700" +do_compare_test 1.6.2 "rowid = -1" +do_compare_test 1.6.3 "rowid = 0" +do_compare_test 1.6.4 "rowid = 55" +do_compare_test 1.6.5 "rowid BETWEEN 20 AND 100" +do_compare_test 1.6.6 "rowid > 350" +do_compare_test 1.6.7 "rowid >= 350" +do_compare_test 1.6.8 "rowid >= 200" +do_compare_test 1.6.9 "1" +do_compare_test 1.6.10 "rowid >= 5" + +do_test 1.x { + set sqlite_open_file_count +} [expr $nFile+9] + +do_test 1.y { db close } {} + +# Delete all the database files created above. +# +for {set i 0} {$i < 40} {incr i} { forcedelete "test.db$i" } + +#------------------------------------------------------------------------- +# Test some error conditions: +# +# 2.1: Database file does not exist. +# 2.2: Table does not exist. +# 2.3: Table schema does not match. +# 2.4: Syntax error in SELECT statement. +# +reset_db +load_static_extension db unionvtab +do_test 2.0.1 { + db eval { + CREATE TABLE t0(a INTEGER PRIMARY KEY, b TEXT); + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<400) + INSERT INTO t0 SELECT i, hex(randomblob(50)) FROM s; + CREATE TABLE dir(f, t, imin, imax); + } + + for {set i 0} {$i < 40} {incr i} { + set iMin [expr $i*10 + 1] + set iMax [expr $iMin+9] + + forcedelete "test.db$i" + db eval [subst { + ATTACH 'test.db$i' AS aux; + CREATE TABLE aux.t$i (a INTEGER PRIMARY KEY, b TEXT); + INSERT INTO aux.t$i SELECT * FROM t0 WHERE a BETWEEN $iMin AND $iMax; + DETACH aux; + INSERT INTO dir VALUES('test.db$i', 't$i', $iMin, $iMax); + }] + } + execsql { + CREATE VIRTUAL TABLE temp.s1 USING swarmvtab('SELECT * FROM dir'); + } +} {} + +do_test 2.0.2 { + forcedelete test.db5 + + sqlite3 db2 test.db15 + db2 eval { DROP TABLE t15 } + db2 close + + sqlite3 db2 test.db25 + db2 eval { + DROP TABLE t25; + CREATE TABLE t25(x, y, z PRIMARY KEY); + } + db2 close +} {} + +do_catchsql_test 2.1 { + SELECT * FROM s1 WHERE rowid BETWEEN 1 AND 100; +} {1 {unable to open database file}} +do_catchsql_test 2.2 { + SELECT * FROM s1 WHERE rowid BETWEEN 101 AND 200; +} {1 {no such rowid table: t15}} +do_catchsql_test 2.3 { + SELECT * FROM s1 WHERE rowid BETWEEN 201 AND 300; +} {1 {source table schema mismatch}} + +do_catchsql_test 2.4 { + CREATE VIRTUAL TABLE temp.x1 USING swarmvtab('SELECT * FROMdir'); +} {1 {sql error: near "FROMdir": syntax error}} +do_catchsql_test 2.5 { + CREATE VIRTUAL TABLE temp.x1 USING swarmvtab('SELECT * FROMdir', 'fetchdb'); +} {1 {sql error: near "FROMdir": syntax error}} + +for {set i 0} {$i < 40} {incr i} { + forcedelete "test.db$i" +} + +#------------------------------------------------------------------------- +# Test the outcome of the fetch function throwing an exception. +# +proc fetch_db {file} { + error "fetch_db error!" +} + +db func fetch_db fetch_db + +do_catchsql_test 3.1 { + CREATE VIRTUAL TABLE temp.xyz USING swarmvtab( + 'VALUES + ("test.db1", "t1", 1, 10), + ("test.db2", "t1", 11, 20) + ', 'fetch_db_no_such_function' + ); +} {1 {no such function: fetch_db_no_such_function}} + +do_catchsql_test 3.2 { + CREATE VIRTUAL TABLE temp.xyz USING swarmvtab( + 'VALUES + ("test.db1", "t1", 1, 10), + ("test.db2", "t1", 11, 20) + ', 'fetch_db' + ); +} {1 {fetch_db error!}} + +do_execsql_test 3.3.1 { + ATTACH 'test.db1' AS aux; + CREATE TABLE aux.t1(a INTEGER PRIMARY KEY, b); + INSERT INTO aux.t1 VALUES(1, NULL); + INSERT INTO aux.t1 VALUES(2, NULL); + INSERT INTO aux.t1 VALUES(9, NULL); + DETACH aux; + CREATE VIRTUAL TABLE temp.xyz USING swarmvtab( + 'VALUES + ("test.db1", "t1", 1, 10), + ("test.db2", "t1", 11, 20) + ', 'fetch_db' + ); +} {} + +do_catchsql_test 3.3.2 { SELECT * FROM xyz } {1 {fetch_db error!}} + + + +finish_test + ADDED test/swarmvtab2.test Index: test/swarmvtab2.test ================================================================== --- /dev/null +++ test/swarmvtab2.test @@ -0,0 +1,74 @@ +# 2017-07-15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is the "swarmvtab" extension +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix swarmvtab + +ifcapable !vtab { + finish_test + return +} + + +db close +foreach name [glob -nocomplain test*.db] { + forcedelete $name +} +sqlite3 db test.db +load_static_extension db unionvtab +proc create_database {filename} { + sqlite3 dbx $filename + set num [regsub -all {[^0-9]+} $filename {}] + set num [string trimleft $num 0] + set start [expr {$num*1000}] + set end [expr {$start+999}] + dbx eval { + CREATE TABLE t2(a INTEGER PRIMARY KEY,b); + WITH RECURSIVE c(x) AS ( + VALUES($start) UNION ALL SELECT x+1 FROM c WHERE x<$end + ) + INSERT INTO t2(a,b) SELECT x, printf('**%05d**',x) FROM c; + } + dbx close +} +db func create_database create_database +do_execsql_test 100 { + CREATE TABLE t1(filename, tablename, istart, iend); + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<99) + INSERT INTO t1 SELECT printf('test%03d.db',x),'t2',x*1000,x*1000+999 FROM c; + CREATE VIRTUAL TABLE temp.v1 USING swarmvtab( + 'SELECT * FROM t1', 'create_database' + ); +} {} +do_execsql_test 110 { + SELECT b FROM v1 WHERE a=3875; +} {**03875**} +do_test 120 { + lsort [glob -nocomplain test?*.db] +} {test001.db test003.db} +do_execsql_test 130 { + SELECT b FROM v1 WHERE a BETWEEN 3999 AND 4000 ORDER BY a; +} {**03999** **04000**} +do_test 140 { + lsort [glob -nocomplain test?*.db] +} {test001.db test003.db test004.db} +do_execsql_test 150 { + SELECT b FROM v1 WHERE a>=99998; +} {**99998** **99999**} +do_test 160 { + lsort -dictionary [glob -nocomplain test?*.db] +} {test001.db test003.db test004.db test099.db} + +finish_test ADDED test/swarmvtabfault.test Index: test/swarmvtabfault.test ================================================================== --- /dev/null +++ test/swarmvtabfault.test @@ -0,0 +1,61 @@ +# 2017-07-15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is error handling in the swarmvtab extension. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix swarmvtabfault + +ifcapable !vtab { + finish_test + return +} + +proc fetch_db {file} { + forcedelete $file + sqlite3 dbX $file + dbX eval { CREATE TABLE t1(a INTEGER PRIMARY KEY, b) } + dbX close +} + +forcedelete test.db1 +do_execsql_test 1.0 { + ATTACH 'test.db1' AS aux; + CREATE TABLE aux.t1(a INTEGER PRIMARY KEY, b); + INSERT INTO aux.t1 VALUES(1, NULL); + INSERT INTO aux.t1 VALUES(2, NULL); + INSERT INTO aux.t1 VALUES(9, NULL); + DETACH aux; +} {} + +faultsim_save_and_close +do_faultsim_test 1.1 -faults oom* -prep { + faultsim_restore_and_reopen + db func fetch_db fetch_db + load_static_extension db unionvtab + db eval { + CREATE VIRTUAL TABLE temp.xyz USING swarmvtab( + 'VALUES + ("test.db1", "t1", 1, 10), + ("test.db2", "t1", 11, 20) + ', 'fetch_db' + ); + } +} -body { + execsql { SELECT a FROM xyz } +} -test { + faultsim_test_result {0 {1 2 9}} {1 {sql error: out of memory}} +} + +finish_test + Index: test/syscall.test ================================================================== --- test/syscall.test +++ test/syscall.test @@ -59,11 +59,11 @@ foreach s { open close access getcwd stat fstat ftruncate fcntl read pread write pwrite fchmod fallocate pread64 pwrite64 unlink openDirectory mkdir rmdir statvfs fchown geteuid umask mmap munmap mremap - getpagesize readlink lstat + getpagesize readlink lstat ioctl } { if {[test_syscall exists $s]} {lappend syscall_list $s} } do_test 3.1 { lsort [test_syscall list] } [lsort $syscall_list] Index: test/tester.tcl ================================================================== --- test/tester.tcl +++ test/tester.tcl @@ -1268,18 +1268,10 @@ [lindex $x 1] [lindex $x 2] [lindex $y 2]] output1 "Page-cache used: $val" set x [sqlite3_status SQLITE_STATUS_PAGECACHE_OVERFLOW 0] set val [format {now %10d max %10d} [lindex $x 1] [lindex $x 2]] output1 "Page-cache overflow: $val" - set x [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] - set val [format {now %10d max %10d} [lindex $x 1] [lindex $x 2]] - output1 "Scratch memory used: $val" - set x [sqlite3_status SQLITE_STATUS_SCRATCH_OVERFLOW 0] - set y [sqlite3_status SQLITE_STATUS_SCRATCH_SIZE 0] - set val [format {now %10d max %10d max-size %10d} \ - [lindex $x 1] [lindex $x 2] [lindex $y 2]] - output1 "Scratch overflow: $val" ifcapable yytrackmaxstackdepth { set x [sqlite3_status SQLITE_STATUS_PARSER_STACK 0] set val [format { max %10d} [lindex $x 2]] output2 "Parser stack depth: $val" } @@ -1617,10 +1609,58 @@ if {[string length $sql]>0} { puts $f "db eval {" puts $f "$sql" puts $f "}" } + close $f + set r [catch { + exec [info nameofexec] crash.tcl >@stdout + } msg] + + # Windows/ActiveState TCL returns a slightly different + # error message. We map that to the expected message + # so that we don't have to change all of the test + # cases. + if {$::tcl_platform(platform)=="windows"} { + if {$msg=="child killed: unknown signal"} { + set msg "child process exited abnormally" + } + } + + lappend r $msg +} + +# crash_on_write ?-devchar DEVCHAR? CRASHDELAY SQL +# +proc crash_on_write {args} { + + set nArg [llength $args] + if {$nArg<2 || $nArg%2} { + error "bad args: $args" + } + set zSql [lindex $args end] + set nDelay [lindex $args end-1] + + set devchar {} + for {set ii 0} {$ii < $nArg-2} {incr ii 2} { + set opt [lindex $args $ii] + switch -- [lindex $args $ii] { + -devchar { + set devchar [lindex $args [expr $ii+1]] + } + + default { error "unrecognized option: $opt" } + } + } + + set f [open crash.tcl w] + puts $f "sqlite3_crash_on_write $nDelay" + puts $f "sqlite3_test_control_pending_byte $::sqlite_pending_byte" + puts $f "sqlite3 db test.db -vfs writecrash" + puts $f "db eval {$zSql}" + puts $f "set {} {}" + close $f set r [catch { exec [info nameofexec] crash.tcl >@stdout } msg] ADDED test/unionvtab.test Index: test/unionvtab.test ================================================================== --- /dev/null +++ test/unionvtab.test @@ -0,0 +1,458 @@ +# 2017-07-15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is percentile.c extension +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix unionvtab + +ifcapable !vtab { + finish_test + return +} + +load_static_extension db unionvtab + +#------------------------------------------------------------------------- +# Warm body tests. +# +forcedelete test.db2 +do_execsql_test 1.0 { + ATTACH 'test.db2' AS aux; + + CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b TEXT); + CREATE TABLE aux.t3(a INTEGER PRIMARY KEY, b TEXT); + + + INSERT INTO t1 VALUES(1, 'one'), (2, 'two'), (3, 'three'); + INSERT INTO t2 VALUES(10, 'ten'), (11, 'eleven'), (12, 'twelve'); + INSERT INTO t3 VALUES(20, 'twenty'), (21, 'twenty-one'), (22, 'twenty-two'); +} + +do_execsql_test 1.1 { + CREATE VIRTUAL TABLE temp.uuu USING unionvtab( + "VALUES(NULL, 't1', 1, 9), ('main', 't2', 10, 19), ('aux', 't3', 20, 29)" + ); + SELECT * FROM uuu; +} { + 1 one 2 two 3 three + 10 ten 11 eleven 12 twelve + 20 twenty 21 twenty-one 22 twenty-two +} + +do_execsql_test 1.2 { + PRAGMA table_info(uuu); +} { + 0 a INTEGER 0 {} 0 + 1 b TEXT 0 {} 0 +} + +do_execsql_test 1.3 { + SELECT * FROM uuu WHERE rowid = 3; + SELECT * FROM uuu WHERE rowid = 11; +} {3 three 11 eleven} + +do_execsql_test 1.4 { + SELECT * FROM uuu WHERE rowid IN (12, 10, 2); +} {2 two 10 ten 12 twelve} + +do_execsql_test 1.5 { + SELECT * FROM uuu WHERE rowid BETWEEN 3 AND 11; +} {3 three 10 ten 11 eleven} + +do_execsql_test 1.6 { + SELECT * FROM uuu WHERE rowid BETWEEN 11 AND 15; +} {11 eleven 12 twelve} + +do_execsql_test 1.7 { + SELECT * FROM uuu WHERE rowid BETWEEN -46 AND 1500; +} { + 1 one 2 two 3 three + 10 ten 11 eleven 12 twelve + 20 twenty 21 twenty-one 22 twenty-two +} + +do_execsql_test 1.8 { + CREATE TABLE src(db, tbl, min, max); + INSERT INTO src VALUES(NULL, 't1', 1, 9); + INSERT INTO src VALUES('main', 't2', 10, 19); + INSERT INTO src VALUES('aux', 't3', 20, 29); + CREATE VIRTUAL TABLE temp.opp USING unionvtab(src); + SELECT * FROM opp; +} { + 1 one 2 two 3 three + 10 ten 11 eleven 12 twelve + 20 twenty 21 twenty-one 22 twenty-two +} + +do_execsql_test 1.9 { + CREATE VIRTUAL TABLE temp.qll USING unionvtab( + 'SELECT * FROM src WHERE db!=''xyz''' + ); + SELECT * FROM qll WHERE rowid BETWEEN 10 AND 21; +} { + 10 ten 11 eleven 12 twelve + 20 twenty 21 twenty-one +} + +#------------------------------------------------------------------------- +# Error conditions. +# +# 2.1.*: Attempt to create a unionvtab table outside of the TEMP schema. +# 2.2.*: Tables that do not exist. +# 2.3.*: Non rowid tables. +# 2.4.*: Tables with mismatched schemas. +# 2.5.*: A unionvtab table with zero source tables. +# +do_catchsql_test 2.1.1 { + CREATE VIRTUAL TABLE u1 USING unionvtab("VALUES(NULL, 't1', 1, 100)"); +} {1 {unionvtab tables must be created in TEMP schema}} +do_catchsql_test 2.1.2 { + CREATE VIRTUAL TABLE main.u1 USING unionvtab("VALUES('', 't1', 1, 100)"); +} {1 {unionvtab tables must be created in TEMP schema}} +do_catchsql_test 2.1.3 { + CREATE VIRTUAL TABLE aux.u1 USING unionvtab("VALUES('', 't1', 1, 100)"); +} {1 {unionvtab tables must be created in TEMP schema}} + +do_catchsql_test 2.2.1 { + CREATE VIRTUAL TABLE temp.u1 USING unionvtab("VALUES(NULL, 't555', 1, 100)"); +} {1 {no such rowid table: t555}} +do_catchsql_test 2.2.2 { + CREATE VIRTUAL TABLE temp.u1 USING unionvtab("VALUES('aux', 't555', 1, 100)"); +} {1 {no such rowid table: aux.t555}} +do_catchsql_test 2.2.3 { + CREATE VIRTUAL TABLE temp.u1 USING unionvtab("VALUES('xua', 't555', 1, 100)"); +} {1 {no such rowid table: xua.t555}} + +do_execsql_test 2.3.0 { + CREATE TABLE wr1(a, b, c PRIMARY KEY) WITHOUT ROWID; + CREATE VIEW v1 AS SELECT * FROM t1; + CREATE VIEW v2 AS SELECT _rowid_, * FROM t1; + + CREATE TABLE wr2(a, _rowid_ INTEGER, c PRIMARY KEY) WITHOUT ROWID; + CREATE TABLE wr3(a, b, _rowid_ PRIMARY KEY) WITHOUT ROWID; +} +do_catchsql_test 2.3.1 { + CREATE VIRTUAL TABLE temp.u1 USING unionvtab("VALUES('main', 'wr1', 1, 2)"); +} {1 {no such rowid table: main.wr1}} +do_catchsql_test 2.3.2 { + CREATE VIRTUAL TABLE temp.u1 USING unionvtab("VALUES(NULL, 'v1', 1, 2)"); +} {1 {no such rowid table: v1}} +do_catchsql_test 2.3.3 { + CREATE VIRTUAL TABLE temp.u1 USING unionvtab("VALUES(NULL, 'v2', 1, 2)"); +} {1 {no such rowid table: v2}} +do_catchsql_test 2.3.4 { + CREATE VIRTUAL TABLE temp.u1 USING unionvtab("VALUES(NULL, 'wr2', 1, 2)"); +} {1 {no such rowid table: wr2}} +do_catchsql_test 2.3.5 { + CREATE VIRTUAL TABLE temp.u1 USING unionvtab("VALUES(NULL, 'wr3', 1, 2)"); +} {1 {no such rowid table: wr3}} + +do_execsql_test 2.4.0 { + CREATE TABLE x1(a BLOB, b); + CREATE TABLE x2(a BLOB, b); + CREATE TEMP TABLE x3(a BLOB, b); + + CREATE TABLE aux.y1(one, two, three INTEGER PRIMARY KEY); + CREATE TEMP TABLE y2(one, two, three INTEGER PRIMARY KEY); + CREATE TABLE y3(one, two, three INTEGER PRIMARY KEY); +} + +foreach {tn dbs res} { + 1 {x1 x2 x3} {0 {}} + 2 {y1 y2 y3} {0 {}} + 3 {x1 y2 y3} {1 {source table schema mismatch}} + 4 {x1 y2 x3} {1 {source table schema mismatch}} + 5 {x1 x2 y3} {1 {source table schema mismatch}} +} { + set L [list] + set iMin 0 + foreach e $dbs { + set E [split $e .] + if {[llength $E]>1} { + lappend L "('[lindex $E 0]', '[lindex $E 1]', $iMin, $iMin)" + } else { + lappend L "(NULL, '$e', $iMin, $iMin)" + } + incr iMin + } + + set sql "CREATE VIRTUAL TABLE temp.a1 USING unionvtab(\"VALUES [join $L ,]\")" + do_catchsql_test 2.4.$tn " + DROP TABLE IF EXISTS temp.a1; + CREATE VIRTUAL TABLE temp.a1 USING unionvtab(\"VALUES [join $L ,]\"); + " $res +} + +do_catchsql_test 2.5 { + CREATE VIRTUAL TABLE temp.b1 USING unionvtab( + [SELECT 'main', 'b1', 0, 100 WHERE 0] + ) +} {1 {no source tables configured}} + +foreach {tn sql} { + 1 { VALUES('main', 't1', 10, 20), ('main', 't2', 30, 29) } + 2 { VALUES('main', 't1', 10, 20), ('main', 't2', 15, 30) } +} { + do_catchsql_test 2.6.$tn " + CREATE VIRTUAL TABLE temp.a1 USING unionvtab(`$sql`) + " {1 {rowid range mismatch error}} +} + +do_catchsql_test 2.7.1 { + CREATE VIRTUAL TABLE temp.b1 USING unionvtab(1, 2, 3, 4) +} {1 {wrong number of arguments for unionvtab}} + +#------------------------------------------------------------------------- +# +reset_db +load_static_extension db unionvtab +do_execsql_test 3.0 { + CREATE TABLE tbl1(a INTEGER PRIMARY KEY, b); + CREATE TABLE tbl2(a INTEGER PRIMARY KEY, b); + CREATE TABLE tbl3(a INTEGER PRIMARY KEY, b); + + WITH ss(ii) AS ( SELECT 1 UNION ALL SELECT ii+1 FROM ss WHERE ii<100 ) + INSERT INTO tbl1 SELECT ii, '1.' || ii FROM ss; + + WITH ss(ii) AS ( SELECT 1 UNION ALL SELECT ii+1 FROM ss WHERE ii<100 ) + INSERT INTO tbl2 SELECT ii, '2.' || ii FROM ss; + + WITH ss(ii) AS ( SELECT 1 UNION ALL SELECT ii+1 FROM ss WHERE ii<100 ) + INSERT INTO tbl3 SELECT ii, '3.' || ii FROM ss; + + CREATE VIRTUAL TABLE temp.uu USING unionvtab( + "VALUES(NULL,'tbl2', 26, 74), (NULL,'tbl3', 75, 100), (NULL,'tbl1', 1, 25)" + ); +} + +do_execsql_test 3.1 { + SELECT * FROM uu WHERE rowid = 10; +} {10 {1.10}} +do_execsql_test 3.2 { + SELECT * FROM uu WHERE rowid = 25; +} {25 {1.25}} + +do_execsql_test 3.3 { SELECT count(*) FROM uu WHERE rowid <= 24 } {24} + +# The following queries get the "wrong" answers. This is because the +# module assumes that each source table contains rowids from only within +# the range specified. For example, (rowid <= 25) matches 100 rows. This +# is because the module implements (rowid <= 25) as a full table scan +# of tbl1 only. +do_execsql_test 3.4.1 { SELECT count(*) FROM uu WHERE rowid <= 25 } {100} +do_execsql_test 3.4.2 { SELECT count(*) FROM uu WHERE rowid <= 26 } {126} +do_execsql_test 3.4.3 { SELECT count(*) FROM uu WHERE rowid <= 73 } {173} +do_execsql_test 3.4.4 { SELECT count(*) FROM uu WHERE rowid <= 74 } {200} +do_execsql_test 3.4.5 { SELECT count(*) FROM uu WHERE rowid <= 75 } {275} +do_execsql_test 3.4.6 { SELECT count(*) FROM uu WHERE rowid <= 99 } {299} +do_execsql_test 3.4.7 { SELECT count(*) FROM uu WHERE rowid <= 100 } {300} +do_execsql_test 3.4.8 { SELECT count(*) FROM uu WHERE rowid <= 101 } {300} + +do_execsql_test 3.5.1 { SELECT count(*) FROM uu WHERE rowid < 25 } {24} +do_execsql_test 3.5.2 { SELECT count(*) FROM uu WHERE rowid < 26 } {100} +do_execsql_test 3.5.3 { SELECT count(*) FROM uu WHERE rowid < 27 } {126} +do_execsql_test 3.5.4 { SELECT count(*) FROM uu WHERE rowid < 73 } {172} +do_execsql_test 3.5.5 { SELECT count(*) FROM uu WHERE rowid < 74 } {173} +do_execsql_test 3.5.6 { SELECT count(*) FROM uu WHERE rowid < 75 } {200} +do_execsql_test 3.5.7 { SELECT count(*) FROM uu WHERE rowid < 76 } {275} +do_execsql_test 3.5.8 { SELECT count(*) FROM uu WHERE rowid < 99 } {298} +do_execsql_test 3.5.9 { SELECT count(*) FROM uu WHERE rowid < 100 } {299} +do_execsql_test 3.5.10 { SELECT count(*) FROM uu WHERE rowid < 101 } {300} + +do_execsql_test 3.6.1 { SELECT count(*) FROM uu WHERE rowid > 24 } {276} +do_execsql_test 3.6.1 { SELECT count(*) FROM uu WHERE rowid > 25 } {200} +do_execsql_test 3.6.2 { SELECT count(*) FROM uu WHERE rowid > 26 } {174} +do_execsql_test 3.6.3 { SELECT count(*) FROM uu WHERE rowid > 27 } {173} +do_execsql_test 3.6.4 { SELECT count(*) FROM uu WHERE rowid > 73 } {127} +do_execsql_test 3.6.5 { SELECT count(*) FROM uu WHERE rowid > 74 } {100} +do_execsql_test 3.6.6 { SELECT count(*) FROM uu WHERE rowid > 75 } {25} +do_execsql_test 3.6.7 { SELECT count(*) FROM uu WHERE rowid > 76 } {24} +do_execsql_test 3.6.8 { SELECT count(*) FROM uu WHERE rowid > 99 } {1} +do_execsql_test 3.6.9 { SELECT count(*) FROM uu WHERE rowid > 100 } {0} +do_execsql_test 3.6.10 { SELECT count(*) FROM uu WHERE rowid > 101 } {0} + +do_execsql_test 3.7.1 { SELECT count(*) FROM uu WHERE rowid >= 24 } {277} +do_execsql_test 3.7.1 { SELECT count(*) FROM uu WHERE rowid >= 25 } {276} +do_execsql_test 3.7.2 { SELECT count(*) FROM uu WHERE rowid >= 26 } {200} +do_execsql_test 3.7.3 { SELECT count(*) FROM uu WHERE rowid >= 27 } {174} +do_execsql_test 3.7.4 { SELECT count(*) FROM uu WHERE rowid >= 73 } {128} +do_execsql_test 3.7.5 { SELECT count(*) FROM uu WHERE rowid >= 74 } {127} +do_execsql_test 3.7.6 { SELECT count(*) FROM uu WHERE rowid >= 75 } {100} +do_execsql_test 3.7.7 { SELECT count(*) FROM uu WHERE rowid >= 76 } {25} +do_execsql_test 3.7.8 { SELECT count(*) FROM uu WHERE rowid >= 99 } {2} +do_execsql_test 3.7.9 { SELECT count(*) FROM uu WHERE rowid >= 100 } {1} +do_execsql_test 3.7.10 { SELECT count(*) FROM uu WHERE rowid >= 101 } {0} + +set L [expr 9223372036854775807] +set S [expr -9223372036854775808] + +do_execsql_test 3.8.1 { SELECT count(*) FROM uu WHERE rowid >= $S } {300} +do_execsql_test 3.8.2 { SELECT count(*) FROM uu WHERE rowid > $S } {300} +do_execsql_test 3.8.3 { SELECT count(*) FROM uu WHERE rowid <= $S } {0} +do_execsql_test 3.8.4 { SELECT count(*) FROM uu WHERE rowid < $S } {0} + +do_execsql_test 3.9.1 { SELECT count(*) FROM uu WHERE rowid >= $L } {0} +do_execsql_test 3.9.2 { SELECT count(*) FROM uu WHERE rowid > $L } {0} +do_execsql_test 3.9.3 { SELECT count(*) FROM uu WHERE rowid <= $L } {300} +do_execsql_test 3.9.4 { SELECT count(*) FROM uu WHERE rowid < $L } {300} + +do_execsql_test 3.10.1 { SELECT count(*) FROM uu WHERE a < 25 } {24} +do_execsql_test 3.10.2 { SELECT count(*) FROM uu WHERE a < 26 } {100} +do_execsql_test 3.10.3 { SELECT count(*) FROM uu WHERE a < 27 } {126} +do_execsql_test 3.10.4 { SELECT count(*) FROM uu WHERE a < 73 } {172} +do_execsql_test 3.10.5 { SELECT count(*) FROM uu WHERE a < 74 } {173} +do_execsql_test 3.10.6 { SELECT count(*) FROM uu WHERE a < 75 } {200} +do_execsql_test 3.10.7 { SELECT count(*) FROM uu WHERE a < 76 } {275} +do_execsql_test 3.10.8 { SELECT count(*) FROM uu WHERE a < 99 } {298} +do_execsql_test 3.10.9 { SELECT count(*) FROM uu WHERE a < 100 } {299} +do_execsql_test 3.10.10 { SELECT count(*) FROM uu WHERE a < 101 } {300} + + +#------------------------------------------------------------------------- +# +do_execsql_test 4.0 { + CREATE TABLE s1(k INTEGER PRIMARY KEY, v); + INSERT INTO s1 VALUES($S, 'one'); + INSERT INTO s1 VALUES($S+1, 'two'); + INSERT INTO s1 VALUES($S+2, 'three'); + + CREATE TABLE l1(k INTEGER PRIMARY KEY, v); + INSERT INTO l1 VALUES($L, 'six'); + INSERT INTO l1 VALUES($L-1, 'five'); + INSERT INTO l1 VALUES($L-2, 'four'); + + CREATE VIRTUAL TABLE temp.sl USING unionvtab( + "SELECT NULL, 'l1', 0, 9223372036854775807 + UNION ALL + SELECT NULL, 's1', -9223372036854775808, -1" + ); +} + +do_execsql_test 4.1 { + SELECT * FROM sl; +} { + -9223372036854775808 one -9223372036854775807 two -9223372036854775806 three + 9223372036854775805 four 9223372036854775806 five 9223372036854775807 six +} + +foreach {k v} { + -9223372036854775808 one -9223372036854775807 two -9223372036854775806 three + 9223372036854775805 four 9223372036854775806 five 9223372036854775807 six +} { + do_execsql_test 4.2.$v { SELECT * FROM sl WHERE rowid=$k } [list $k $v] +} + +do_execsql_test 4.3.1 { + SELECT * FROM sl WHERE rowid>-9223372036854775808 +} { + -9223372036854775807 two -9223372036854775806 three + 9223372036854775805 four 9223372036854775806 five 9223372036854775807 six +} +do_execsql_test 4.3.2 { + SELECT * FROM sl WHERE rowid>=-9223372036854775808 +} { + -9223372036854775808 one -9223372036854775807 two -9223372036854775806 three + 9223372036854775805 four 9223372036854775806 five 9223372036854775807 six +} +do_execsql_test 4.3.3 { + SELECT * FROM sl WHERE rowid<=-9223372036854775808 +} { + -9223372036854775808 one +} +do_execsql_test 4.3.4 { + SELECT * FROM sl WHERE rowid<-9223372036854775808 +} { } + +do_execsql_test 4.4.1 { + SELECT * FROM sl WHERE rowid<9223372036854775807 +} { + -9223372036854775808 one -9223372036854775807 two -9223372036854775806 three + 9223372036854775805 four 9223372036854775806 five +} +do_execsql_test 4.4.2 { + SELECT * FROM sl WHERE rowid<=9223372036854775807 +} { + -9223372036854775808 one -9223372036854775807 two -9223372036854775806 three + 9223372036854775805 four 9223372036854775806 five 9223372036854775807 six +} +do_execsql_test 4.4.3 { + SELECT * FROM sl WHERE rowid>=9223372036854775807 +} { + 9223372036854775807 six +} +do_execsql_test 4.4.4 { + SELECT * FROM sl WHERE rowid>9223372036854775807 +} { } + +#------------------------------------------------------------------------- +# More than 8 source tables. +# +do_execsql_test 5.0 { + CREATE TABLE c0(one, two INTEGER PRIMARY KEY); + CREATE TABLE c1(one, two INTEGER PRIMARY KEY); + CREATE TABLE c2(one, two INTEGER PRIMARY KEY); + CREATE TABLE c3(one, two INTEGER PRIMARY KEY); + CREATE TABLE c4(one, two INTEGER PRIMARY KEY); + CREATE TABLE c5(one, two INTEGER PRIMARY KEY); + CREATE TABLE c6(one, two INTEGER PRIMARY KEY); + CREATE TABLE c7(one, two INTEGER PRIMARY KEY); + CREATE TABLE c8(one, two INTEGER PRIMARY KEY); + CREATE TABLE c9(one, two INTEGER PRIMARY KEY); + + INSERT INTO c0 VALUES('zero', 0); + INSERT INTO c1 VALUES('one', 1); + INSERT INTO c2 VALUES('two', 2); + INSERT INTO c3 VALUES('three', 3); + INSERT INTO c4 VALUES('four', 4); + INSERT INTO c5 VALUES('five', 5); + INSERT INTO c6 VALUES('six', 6); + INSERT INTO c7 VALUES('seven', 7); + INSERT INTO c8 VALUES('eight', 8); + INSERT INTO c9 VALUES('nine', 9); + + CREATE VIRTUAL TABLE temp.cc USING unionvtab([ + SELECT 'main', 'c9', 9, 9 UNION ALL + SELECT 'main', 'c8', 8, 8 UNION ALL + SELECT 'main', 'c7', 7, 7 UNION ALL + SELECT 'main', 'c6', 6, 6 UNION ALL + SELECT 'main', 'c5', 5, 5 UNION ALL + SELECT 'main', 'c4', 4, 4 UNION ALL + SELECT 'main', 'c3', 3, 3 UNION ALL + SELECT 'main', 'c2', 2, 2 UNION ALL + SELECT 'main', 'c1', 1, 1 UNION ALL + SELECT 'main', 'c0', 0, 0 + ]); + + SELECT sum(two) FROM cc; +} {45} + +do_execsql_test 5.1 { + SELECT one FROM cc WHERE one>='seven' +} {zero two three six seven} + +do_execsql_test 5.2 { + SELECT y.one FROM cc AS x, cc AS y WHERE x.one=y.one AND x.rowid>5 +} {six seven eight nine} + +do_execsql_test 5.3 { + SELECT cc.one FROM c4, cc WHERE cc.rowid>c4.rowid +} {five six seven eight nine} + +do_execsql_test 5.4 { + SELECT * FROM cc WHERE two LIKE '6' +} {six 6} + +finish_test + ADDED test/unionvtabfault.test Index: test/unionvtabfault.test ================================================================== --- /dev/null +++ test/unionvtabfault.test @@ -0,0 +1,84 @@ +# 2017-07-15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is percentile.c extension +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix unionvtabfault + +ifcapable !vtab { + finish_test + return +} + +forcedelete test.db2 +do_execsql_test 1.0 { + ATTACH 'test.db2' AS aux; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b TEXT); + CREATE TABLE aux.t3(a INTEGER PRIMARY KEY, b TEXT); + + INSERT INTO t1 VALUES(1, 'one'), (2, 'two'), (3, 'three'); + INSERT INTO t2 VALUES(10, 'ten'), (11, 'eleven'), (12, 'twelve'); + INSERT INTO t3 VALUES(20, 'twenty'), (21, 'twenty-one'), (22, 'twenty-two'); +} +faultsim_save_and_close + +do_faultsim_test 1.1 -faults * -prep { + faultsim_restore_and_reopen + load_static_extension db unionvtab + execsql { ATTACH 'test.db2' AS aux; } + execsql { CREATE TEMP TABLE xyz(x); } +} -body { + execsql { + CREATE VIRTUAL TABLE temp.uuu USING unionvtab( + "VALUES(NULL, 't1', 1, 9), ('main', 't2', 10, 19), ('aux', 't3', 20, 29)" + ); + } +} -test { + faultsim_test_result {0 {}} \ + {1 {vtable constructor failed: uuu}} \ + {1 {sql error: interrupted}} +} + +faultsim_restore_and_reopen +load_static_extension db unionvtab +execsql { ATTACH 'test.db2' AS aux; } +execsql { CREATE TEMP TABLE xyz(x); } +execsql { + CREATE VIRTUAL TABLE temp.uuu USING unionvtab( + "VALUES(NULL, 't1', 1, 9), ('main', 't2', 10, 19), ('aux', 't3', 20, 29)" + ); +} +do_faultsim_test 1.2 -faults oom* -prep { +} -body { + execsql { SELECT * FROM uuu } +} -test { + faultsim_test_result {0 {1 one 2 two 3 three 10 ten 11 eleven 12 twelve 20 twenty 21 twenty-one 22 twenty-two}} +} + +#------------------------------------------------------------------------- +# Error while registering the two vtab modules. +do_faultsim_test 2.0 -faults * -prep { + catch { db close } + sqlite3 db :memory: +} -body { + load_static_extension db unionvtab +} -test { + faultsim_test_result {0 {}} {1 {initialization of unionvtab failed: }} +} + + + +finish_test + Index: test/vtab2.test ================================================================== --- test/vtab2.test +++ test/vtab2.test @@ -58,19 +58,19 @@ register_tclvar_module [sqlite3_connection_pointer db] do_test vtab2-2.1 { set ::abc 123 execsql { CREATE VIRTUAL TABLE vars USING tclvar; - SELECT * FROM vars WHERE name='abc'; + SELECT name, arrayname, value FROM vars WHERE name='abc'; } } [list abc "" 123] do_test vtab2-2.2 { set A(1) 1 set A(2) 4 set A(3) 9 execsql { - SELECT * FROM vars WHERE name='A'; + SELECT name, arrayname, value FROM vars WHERE name='A'; } } [list A 1 1 A 2 4 A 3 9] unset -nocomplain result unset -nocomplain var set result {} Index: test/vtabE.test ================================================================== --- test/vtabE.test +++ test/vtabE.test @@ -37,11 +37,13 @@ do_test vtabE-1 { db eval { CREATE VIRTUAL TABLE t1 USING tclvar; CREATE VIRTUAL TABLE t2 USING tclvar; CREATE TABLE t3(a INTEGER PRIMARY KEY, b); - SELECT t1.*, t2.*, abs(t3.b + abs(t2.value + abs(t1.value))) + SELECT t1.name, t1.arrayname, t1.value, + t2.name, t2.arrayname, t2.value, + abs(t3.b + abs(t2.value + abs(t1.value))) FROM t1 LEFT JOIN t2 ON t2.name = t1.arrayname LEFT JOIN t3 ON t3.a=t2.value WHERE t1.name = 'vtabE' ORDER BY t1.value, t2.value; } Index: test/vtabH.test ================================================================== --- test/vtabH.test +++ test/vtabH.test @@ -53,11 +53,11 @@ register_tclvar_module db set ::xyz 10 do_execsql_test 2.0 { CREATE VIRTUAL TABLE vars USING tclvar; - SELECT * FROM vars WHERE name = 'xyz'; + SELECT name, arrayname, value FROM vars WHERE name = 'xyz'; } {xyz {} 10} set x1 aback set x2 abaft set x3 abandon ADDED test/vtabJ.test Index: test/vtabJ.test ================================================================== --- /dev/null +++ test/vtabJ.test @@ -0,0 +1,126 @@ +# 2017-08-10 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements tests of writing to WITHOUT ROWID virtual tables +# using the tclvar eponymous virtual table. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix vtabJ + +ifcapable !vtab { + finish_test + return +} + +register_tclvar_module db + +unset -nocomplain vtabJ +do_test 100 { + set vtabJ(1) this + set vtabJ(two) is + set vtabJ(3) {a test} + db eval { + SELECT fullname, value FROM tclvar WHERE name='vtabJ' ORDER BY fullname; + } +} {vtabJ(1) this vtabJ(3) {a test} vtabJ(two) is} + +do_execsql_test 110 { + INSERT INTO tclvar(fullname, value) + VALUES('vtabJ(4)',4),('vtabJ(five)',555); + SELECT fullname, value FROM tclvar WHERE name='vtabJ' ORDER BY fullname; +} {vtabJ(1) this vtabJ(3) {a test} vtabJ(4) 4 vtabJ(five) 555 vtabJ(two) is} +do_test 111 { + set res {} + foreach vname [lsort [array names vtabJ]] { + lappend res vtabJ($vname) $vtabJ($vname) + } + set res +} {vtabJ(1) this vtabJ(3) {a test} vtabJ(4) 4 vtabJ(five) 555 vtabJ(two) is} + +do_test 120 { + db eval { + INSERT INTO tclvar(fullname, value) VALUES('vtabJ(4)',444); + } + set vtabJ(4) +} {444} + +do_test 130 { + db eval { + INSERT INTO tclvar(fullname, value) VALUES('vtabJ(4)',NULL); + } + info exists vtabJ(4) +} {0} + +do_test 140 { + db eval { + UPDATE tclvar SET value=55 WHERE fullname='vtabJ(five)'; + } + set vtabJ(five) +} {55} + +do_test 150 { + db eval { + UPDATE tclvar SET fullname='vtabJ(5)' WHERE fullname='vtabJ(five)'; + } + set vtabJ(5) +} {55} +do_test 151 { + info exists vtabJ(five) +} {0} +do_test 152 { + set res {} + foreach vname [lsort [array names vtabJ]] { + lappend res vtabJ($vname) $vtabJ($vname) + } + set res +} {vtabJ(1) this vtabJ(3) {a test} vtabJ(5) 55 vtabJ(two) is} + +do_execsql_test 160 { + SELECT fullname FROM tclvar WHERE arrayname='two' +} {vtabJ(two)} +do_execsql_test 161 { + DELETE FROM tclvar WHERE arrayname='two'; + SELECT fullname, value FROM tclvar WHERE name='vtabJ' ORDER BY fullname; +} {vtabJ(1) this vtabJ(3) {a test} vtabJ(5) 55} +do_test 162 { + set res {} + foreach vname [lsort [array names vtabJ]] { + lappend res vtabJ($vname) $vtabJ($vname) + } + set res +} {vtabJ(1) this vtabJ(3) {a test} vtabJ(5) 55} + +# Try to trick the module into updating the same variable twice for a +# single UPDATE statement. +# +do_execsql_test 171 { + INSERT INTO tclvar(fullname, value) VALUES('xx', 'a'); + SELECT name, value FROM tclvar where name = 'xx'; +} {xx a} +do_execsql_test 172 { + UPDATE tclvar SET value = value || 't' + WHERE name = 'xx' OR name = 'x'||'x'; + SELECT name, value FROM tclvar where name = 'xx'; +} {xx at} +do_execsql_test 173 { + UPDATE tclvar SET value = value || 't' + WHERE name = 'xx' OR name BETWEEN 'xx' AND 'xx'; + SELECT name, value FROM tclvar where name = 'xx'; +} {xx att} + +do_execsql_test 181 { + DELETE FROM tclvar WHERE name BETWEEN 'xx' AND 'xx' OR name='xx'; + SELECT name, value FROM tclvar where name = 'xx'; +} {} + + +finish_test Index: test/wal2.test ================================================================== --- test/wal2.test +++ test/wal2.test @@ -1230,11 +1230,11 @@ #------------------------------------------------------------------------- # Test that "PRAGMA checkpoint_fullsync" appears to be working. # foreach {tn sql reslist} { 1 { } {10 0 4 0 6 0} - 2 { PRAGMA checkpoint_fullfsync = 1 } {10 4 4 2 6 2} + 2 { PRAGMA checkpoint_fullfsync = 1 } {10 6 4 3 6 3} 3 { PRAGMA checkpoint_fullfsync = 0 } {10 0 4 0 6 0} } { ifcapable default_ckptfullfsync { if {[string trim $sql]==""} continue } @@ -1302,12 +1302,12 @@ 4 {0 1 off} {0 0} {0 0} {0 0} 5 {0 1 normal} {0 1} {0 0} {0 2} 6 {0 1 full} {0 2} {0 1} {0 2} 7 {1 0 off} {0 0} {0 0} {0 0} - 8 {1 0 normal} {1 0} {0 0} {0 2} - 9 {1 0 full} {2 0} {1 0} {0 2} + 8 {1 0 normal} {0 1} {0 0} {0 2} + 9 {1 0 full} {1 1} {1 0} {0 2} 10 {1 1 off} {0 0} {0 0} {0 0} 11 {1 1 normal} {0 1} {0 0} {0 2} 12 {1 1 full} {0 2} {0 1} {0 2} } { Index: test/whereA.test ================================================================== --- test/whereA.test +++ test/whereA.test @@ -155,8 +155,20 @@ do_test whereA-4.6 { count { SELECT x FROM t2 ORDER BY x DESC; } } {2 1 1} + +# Ticket https://sqlite.org/src/tktview/cb91bf4290c211 2017-08-01 +# Assertion fault following PRAGMA reverse_unordered_selects=ON. +# +do_execsql_test whereA-5.1 { + PRAGMA reverse_unordered_selects=on; + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a,b); + INSERT INTO t1 VALUES(1,2); + CREATE INDEX t1b ON t1(b); + SELECT a FROM t1 WHERE b=-99 OR b>1; +} {1} finish_test Index: test/whereF.test ================================================================== --- test/whereF.test +++ test/whereF.test @@ -173,7 +173,17 @@ SELECT count(*) FROM t1, t2 WHERE ( t2.rowid = +t1.rowid OR (t2.f2 = t1.f1 AND t1.f1!=-1) ) } {4} do_test 5.6 { expr [db status vmstep]<200 } 1 + +# 2017-09-04 ticket b899b6042f97f52d +# Segfault on correlated subquery... +# +ifcapable json1 { + do_execsql_test 6.1 { + CREATE TABLE t6(x); + SELECT * FROM t6 WHERE 1 IN (SELECT value FROM json_each(x)); + } {} +} finish_test Index: test/without_rowid1.test ================================================================== --- test/without_rowid1.test +++ test/without_rowid1.test @@ -325,8 +325,22 @@ CREATE TABLE t70b( a INT CHECK( rowid!=33 ), b TEXT PRIMARY KEY ) WITHOUT ROWID; } {1 {no such column: rowid}} + +# 2017-07-30: OSSFuzz discovered that an extra entry was being +# added in the sqlite_master table for an "INTEGER PRIMARY KEY UNIQUE" +# WITHOUT ROWID table. Make sure this has now been fixed. +# +db close +sqlite3 db :memory: +do_execsql_test 8.1 { + CREATE TABLE t1(x INTEGER PRIMARY KEY UNIQUE, b) WITHOUT ROWID; + CREATE INDEX t1x ON t1(x); + INSERT INTO t1(x,b) VALUES('funny','buffalo'); + SELECT type, name, '|' FROM sqlite_master; +} {table t1 | index t1x |} + finish_test Index: test/wordcount.c ================================================================== --- test/wordcount.c +++ test/wordcount.c @@ -631,16 +631,12 @@ printf("%s Memory Used (bytes): %d (max %d)\n", zTag,iCur,iHiwtr); sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &iCur, &iHiwtr, 0); printf("%s Outstanding Allocations: %d (max %d)\n",zTag,iCur,iHiwtr); sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &iCur, &iHiwtr, 0); printf("%s Pcache Overflow Bytes: %d (max %d)\n",zTag,iCur,iHiwtr); - sqlite3_status(SQLITE_STATUS_SCRATCH_OVERFLOW, &iCur, &iHiwtr, 0); - printf("%s Scratch Overflow Bytes: %d (max %d)\n",zTag,iCur,iHiwtr); sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &iCur, &iHiwtr, 0); printf("%s Largest Allocation: %d bytes\n",zTag,iHiwtr); sqlite3_status(SQLITE_STATUS_PAGECACHE_SIZE, &iCur, &iHiwtr, 0); printf("%s Largest Pcache Allocation: %d bytes\n",zTag,iHiwtr); - sqlite3_status(SQLITE_STATUS_SCRATCH_SIZE, &iCur, &iHiwtr, 0); - printf("%s Largest Scratch Allocation: %d bytes\n",zTag,iHiwtr); } return 0; } ADDED test/writecrash.test Index: test/writecrash.test ================================================================== --- /dev/null +++ test/writecrash.test @@ -0,0 +1,68 @@ +# 2009 January 8 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test the outcome of a writer crashing within a call to the VFS +# xWrite function. +# + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix writecrash + +do_not_use_codec + + +if {$tcl_platform(platform)=="windows"} { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b BLOB UNIQUE); + WITH s(i) AS ( + VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100 + ) + INSERT INTO t1 SELECT NULL, randomblob(900) FROM s; +} {} + +set bGo 1 +for {set tn 1} {$bGo} {incr tn} { + +db close +sqlite3 db test.db + + do_test 1.$tn.1 { + set res [crash_on_write $tn { + UPDATE t1 SET b = randomblob(899) WHERE (a%3)==0 + }] + set bGo 0 + if {[string match {1 {child killed:*}} $res]} { + set res {0 {}} + set bGo 1 + } + set res + } {0 {}} + +#db close +#sqlite3 db test.db + + do_execsql_test 1.$tn.2 { PRAGMA integrity_check } {ok} + +db close +sqlite3 db test.db + + do_execsql_test 1.$tn.3 { PRAGMA integrity_check } {ok} +} + + + +finish_test Index: tool/GetTclKit.bat ================================================================== --- tool/GetTclKit.bat +++ tool/GetTclKit.bat @@ -27,10 +27,12 @@ CALL :fn_UnquoteVariable PROCESSOR ) ELSE ( GOTO usage ) +SET PROCESSOR=%PROCESSOR:AMD64=x64% + %_VECHO% Processor = '%PROCESSOR%' SET DUMMY2=%2 IF DEFINED DUMMY2 ( @@ -191,16 +193,23 @@ :skip_sdkEnvironment GOTO no_errors :fn_TclKitX86Variables + REM + REM NOTE: By default, use latest available version of the TclKit SDK + REM for x86. However, the "default" TclKit executable for x86 + REM is still used here because it is the only one "well-known" + REM to be available for download. + REM IF NOT DEFINED TCLKIT_PATCHLEVEL ( - SET TCLKIT_PATCHLEVEL=8.6.4 + SET TCLKIT_PATCHLEVEL=8.6.6 ) SET TCLKIT_VERSION=%TCLKIT_PATCHLEVEL:.=% SET TCLKIT_VERSION=%TCLKIT_VERSION:~0,2% - SET TCLKIT_EXE=tclkit-%TCLKIT_PATCHLEVEL%.exe + REM SET TCLKIT_EXE=tclkit-%TCLKIT_PATCHLEVEL%.exe + SET TCLKIT_EXE=tclkit-8.6.4.exe SET TCLKIT_LIB=libtclkit%TCLKIT_PATCHLEVEL:.=%.lib SET TCLKIT_LIB_STUB=libtclstub%TCLKIT_VERSION:.=%.a SET TCLKIT_SDK=libtclkit-sdk-x86-%TCLKIT_PATCHLEVEL% SET TCLKIT_SDK_ZIP=%TCLKIT_SDK%.zip SET TCLKIT_FILES=%TCLKIT_EXE% @@ -208,24 +217,23 @@ SET TCLKIT_FILES=%TCLKIT_FILES% unzip.exe %TCLKIT_SDK_ZIP% ) GOTO :EOF :fn_TclKitX64Variables + REM + REM NOTE: By default, use latest available version of the TclKit SDK + REM for x64. However, the "default" TclKit executable for x86 + REM is still used here because it is the only one "well-known" + REM to be available for download. + REM IF NOT DEFINED TCLKIT_PATCHLEVEL ( - REM - REM NOTE: By default, use latest available version of the TclKit SDK - REM for x64. However, the "default" TclKit executable for x86 - REM is still used here because it is the only one "well-known" - REM to be available for download. - REM - SET TCLKIT_PATCHLEVEL=8.6.3 - SET TCLKIT_EXE=tclkit-8.6.4.exe - ) ELSE ( - SET TCLKIT_EXE=tclkit-%TCLKIT_PATCHLEVEL%.exe + SET TCLKIT_PATCHLEVEL=8.6.6 ) SET TCLKIT_VERSION=%TCLKIT_PATCHLEVEL:.=% SET TCLKIT_VERSION=%TCLKIT_VERSION:~0,2% + REM SET TCLKIT_EXE=tclkit-%TCLKIT_PATCHLEVEL%.exe + SET TCLKIT_EXE=tclkit-8.6.4.exe SET TCLKIT_LIB=libtclkit%TCLKIT_PATCHLEVEL:.=%.lib SET TCLKIT_LIB_STUB=libtclstub%TCLKIT_VERSION:.=%.a SET TCLKIT_SDK=libtclkit-sdk-x64-%TCLKIT_PATCHLEVEL% SET TCLKIT_SDK_ZIP=%TCLKIT_SDK%.zip SET TCLKIT_FILES=%TCLKIT_EXE% Index: tool/addopcodes.tcl ================================================================== --- tool/addopcodes.tcl +++ tool/addopcodes.tcl @@ -20,18 +20,11 @@ # The following are the extra token codes to be added. SPACE and # ILLEGAL *must* be the last two token codes and they must be in that order. # set extras { - TO_TEXT - TO_BLOB - TO_NUMERIC - TO_INT - TO_REAL ISNOT - END_OF_FILE - UNCLOSED_STRING FUNCTION COLUMN AGG_FUNCTION AGG_COLUMN UMINUS @@ -40,10 +33,12 @@ VECTOR SELECT_COLUMN IF_NULL_ROW ASTERISK SPAN + END_OF_FILE + UNCLOSED_STRING SPACE ILLEGAL } if {[lrange $extras end-1 end]!="SPACE ILLEGAL"} { error "SPACE and ILLEGAL must be the last two token codes and they\ Index: tool/lemon.c ================================================================== --- tool/lemon.c +++ tool/lemon.c @@ -2153,11 +2153,12 @@ WAITING_FOR_DESTRUCTOR_SYMBOL, WAITING_FOR_DATATYPE_SYMBOL, WAITING_FOR_FALLBACK_ID, WAITING_FOR_WILDCARD_ID, WAITING_FOR_CLASS_ID, - WAITING_FOR_CLASS_TOKEN + WAITING_FOR_CLASS_TOKEN, + WAITING_FOR_TOKEN_NAME }; struct pstate { char *filename; /* Name of the input file */ int tokenlineno; /* Linenumber at which current token starts */ int errorcnt; /* Number of errors so far */ @@ -2468,10 +2469,12 @@ }else if( strcmp(x,"type")==0 ){ psp->state = WAITING_FOR_DATATYPE_SYMBOL; }else if( strcmp(x,"fallback")==0 ){ psp->fallback = 0; psp->state = WAITING_FOR_FALLBACK_ID; + }else if( strcmp(x,"token")==0 ){ + psp->state = WAITING_FOR_TOKEN_NAME; }else if( strcmp(x,"wildcard")==0 ){ psp->state = WAITING_FOR_WILDCARD_ID; }else if( strcmp(x,"token_class")==0 ){ psp->state = WAITING_FOR_CLASS_ID; }else{ @@ -2621,10 +2624,30 @@ }else{ sp->fallback = psp->fallback; psp->gp->has_fallback = 1; } } + break; + case WAITING_FOR_TOKEN_NAME: + /* Tokens do not have to be declared before use. But they can be + ** in order to control their assigned integer number. The number for + ** each token is assigned when it is first seen. So by including + ** + ** %token ONE TWO THREE + ** + ** early in the grammar file, that assigns small consecutive values + ** to each of the tokens ONE TWO and THREE. + */ + if( x[0]=='.' ){ + psp->state = WAITING_FOR_DECL_OR_RULE; + }else if( !ISUPPER(x[0]) ){ + ErrorMsg(psp->filename, psp->tokenlineno, + "%%token argument \"%s\" should be a token", x); + psp->errorcnt++; + }else{ + (void)Symbol_new(x); + } break; case WAITING_FOR_WILDCARD_ID: if( x[0]=='.' ){ psp->state = WAITING_FOR_DECL_OR_RULE; }else if( !ISUPPER(x[0]) ){ Index: tool/mkopcodeh.tcl ================================================================== --- tool/mkopcodeh.tcl +++ tool/mkopcodeh.tcl @@ -200,23 +200,21 @@ } if {$i>$max} {set max $i} set name $def($i) puts -nonewline [format {#define %-16s %3d} $name $i] set com {} + if {$jump($name)} { + lappend com "jump" + } if {[info exists sameas($i)]} { - set com "same as $sameas($i)" + lappend com "same as $sameas($i)" } if {[info exists synopsis($name)]} { - set x $synopsis($name) - if {$com==""} { - set com "synopsis: $x" - } else { - append com ", synopsis: $x" - } - } - if {$com!=""} { - puts -nonewline [format " /* %-42s */" $com] + lappend com "synopsis: $synopsis($name)" + } + if {[llength $com]} { + puts -nonewline [format " /* %-42s */" [join $com {, }]] } puts "" } if {$max>255} { Index: tool/mkshellc.tcl ================================================================== --- tool/mkshellc.tcl +++ tool/mkshellc.tcl @@ -23,19 +23,21 @@ ** To modify this program, get a copy of the canonical SQLite source tree, ** edit the src/shell.c.in" and/or some of the other files that are included ** by "src/shell.c.in", then rerun the tool/mkshellc.tcl script. */} set in [open $topdir/src/shell.c.in rb] -while {![eof $in]} { +while {1} { set lx [gets $in] + if {[eof $in]} break; if {[regexp {^INCLUDE } $lx]} { set cfile [lindex $lx 1] puts $out "/************************* Begin $cfile ******************/" set in2 [open $topdir/src/$cfile rb] while {![eof $in2]} { set lx [gets $in2] if {[regexp {^#include "sqlite} $lx]} continue + set lx [string map [list __declspec(dllexport) {}] $lx] puts $out $lx } close $in2 puts $out "/************************* End $cfile ********************/" continue ADDED tool/mksourceid.c Index: tool/mksourceid.c ================================================================== --- /dev/null +++ tool/mksourceid.c @@ -0,0 +1,852 @@ +/* +** Run this program with a single argument which is the name of the +** Fossil "manifest" file for a project, and this program will emit on +** standard output the "source id" for for the program. +** +** (1) The "source id" is the date of check-in together with the +** SHA3 hash of the manifest file. +** +** (2) All individual file hashes in the manifest are verified. If any +** source file has changed, the SHA3 hash ends with "modified". +** +*/ +#include +#include +#include +#include +#include + +/* Portable 64-bit unsigned integers */ +#if defined(_MSC_VER) || defined(__BORLANDC__) + typedef unsigned __int64 u64; +#else + typedef unsigned long long int u64; +#endif + + +/* +** Macros to determine whether the machine is big or little endian, +** and whether or not that determination is run-time or compile-time. +** +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DBYTEORDER=0 is set, then byte-order is determined +** at run-time. +*/ +#ifndef BYTEORDER +# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__arm__) +# define BYTEORDER 1234 +# elif defined(sparc) || defined(__ppc__) +# define BYTEORDER 4321 +# else +# define BYTEORDER 0 +# endif +#endif + + + +/* +** State structure for a SHA3 hash in progress +*/ +typedef struct SHA3Context SHA3Context; +struct SHA3Context { + union { + u64 s[25]; /* Keccak state. 5x5 lines of 64 bits each */ + unsigned char x[1600]; /* ... or 1600 bytes */ + } u; + unsigned nRate; /* Bytes of input accepted per Keccak iteration */ + unsigned nLoaded; /* Input bytes loaded into u.x[] so far this cycle */ + unsigned ixMask; /* Insert next input into u.x[nLoaded^ixMask]. */ +}; + +/* +** A single step of the Keccak mixing function for a 1600-bit state +*/ +static void KeccakF1600Step(SHA3Context *p){ + int i; + u64 B0, B1, B2, B3, B4; + u64 C0, C1, C2, C3, C4; + u64 D0, D1, D2, D3, D4; + static const u64 RC[] = { + 0x0000000000000001ULL, 0x0000000000008082ULL, + 0x800000000000808aULL, 0x8000000080008000ULL, + 0x000000000000808bULL, 0x0000000080000001ULL, + 0x8000000080008081ULL, 0x8000000000008009ULL, + 0x000000000000008aULL, 0x0000000000000088ULL, + 0x0000000080008009ULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x800000000000008bULL, + 0x8000000000008089ULL, 0x8000000000008003ULL, + 0x8000000000008002ULL, 0x8000000000000080ULL, + 0x000000000000800aULL, 0x800000008000000aULL, + 0x8000000080008081ULL, 0x8000000000008080ULL, + 0x0000000080000001ULL, 0x8000000080008008ULL + }; +# define A00 (p->u.s[0]) +# define A01 (p->u.s[1]) +# define A02 (p->u.s[2]) +# define A03 (p->u.s[3]) +# define A04 (p->u.s[4]) +# define A10 (p->u.s[5]) +# define A11 (p->u.s[6]) +# define A12 (p->u.s[7]) +# define A13 (p->u.s[8]) +# define A14 (p->u.s[9]) +# define A20 (p->u.s[10]) +# define A21 (p->u.s[11]) +# define A22 (p->u.s[12]) +# define A23 (p->u.s[13]) +# define A24 (p->u.s[14]) +# define A30 (p->u.s[15]) +# define A31 (p->u.s[16]) +# define A32 (p->u.s[17]) +# define A33 (p->u.s[18]) +# define A34 (p->u.s[19]) +# define A40 (p->u.s[20]) +# define A41 (p->u.s[21]) +# define A42 (p->u.s[22]) +# define A43 (p->u.s[23]) +# define A44 (p->u.s[24]) +# define ROL64(a,x) ((a<>(64-x))) + + for(i=0; i<24; i+=4){ + C0 = A00^A10^A20^A30^A40; + C1 = A01^A11^A21^A31^A41; + C2 = A02^A12^A22^A32^A42; + C3 = A03^A13^A23^A33^A43; + C4 = A04^A14^A24^A34^A44; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A11^D1), 44); + B2 = ROL64((A22^D2), 43); + B3 = ROL64((A33^D3), 21); + B4 = ROL64((A44^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i]; + A11 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A20^D0), 3); + B3 = ROL64((A31^D1), 45); + B4 = ROL64((A42^D2), 61); + B0 = ROL64((A03^D3), 28); + B1 = ROL64((A14^D4), 20); + A20 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A40^D0), 18); + B0 = ROL64((A01^D1), 1); + B1 = ROL64((A12^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A34^D4), 8); + A40 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A10^D0), 36); + B2 = ROL64((A21^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A43^D3), 56); + B0 = ROL64((A04^D4), 27); + A10 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A30^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A02^D2), 62); + B1 = ROL64((A13^D3), 55); + B2 = ROL64((A24^D4), 39); + A30 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + C0 = A00^A20^A40^A10^A30; + C1 = A11^A31^A01^A21^A41; + C2 = A22^A42^A12^A32^A02; + C3 = A33^A03^A23^A43^A13; + C4 = A44^A14^A34^A04^A24; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A31^D1), 44); + B2 = ROL64((A12^D2), 43); + B3 = ROL64((A43^D3), 21); + B4 = ROL64((A24^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+1]; + A31 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A40^D0), 3); + B3 = ROL64((A21^D1), 45); + B4 = ROL64((A02^D2), 61); + B0 = ROL64((A33^D3), 28); + B1 = ROL64((A14^D4), 20); + A40 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A30^D0), 18); + B0 = ROL64((A11^D1), 1); + B1 = ROL64((A42^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A04^D4), 8); + A30 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A20^D0), 36); + B2 = ROL64((A01^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A13^D3), 56); + B0 = ROL64((A44^D4), 27); + A20 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A10^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A22^D2), 62); + B1 = ROL64((A03^D3), 55); + B2 = ROL64((A34^D4), 39); + A10 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + C0 = A00^A40^A30^A20^A10; + C1 = A31^A21^A11^A01^A41; + C2 = A12^A02^A42^A32^A22; + C3 = A43^A33^A23^A13^A03; + C4 = A24^A14^A04^A44^A34; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A21^D1), 44); + B2 = ROL64((A42^D2), 43); + B3 = ROL64((A13^D3), 21); + B4 = ROL64((A34^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+2]; + A21 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A30^D0), 3); + B3 = ROL64((A01^D1), 45); + B4 = ROL64((A22^D2), 61); + B0 = ROL64((A43^D3), 28); + B1 = ROL64((A14^D4), 20); + A30 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A10^D0), 18); + B0 = ROL64((A31^D1), 1); + B1 = ROL64((A02^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A44^D4), 8); + A10 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A40^D0), 36); + B2 = ROL64((A11^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A03^D3), 56); + B0 = ROL64((A24^D4), 27); + A40 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A20^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A12^D2), 62); + B1 = ROL64((A33^D3), 55); + B2 = ROL64((A04^D4), 39); + A20 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + C0 = A00^A30^A10^A40^A20; + C1 = A21^A01^A31^A11^A41; + C2 = A42^A22^A02^A32^A12; + C3 = A13^A43^A23^A03^A33; + C4 = A34^A14^A44^A24^A04; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A01^D1), 44); + B2 = ROL64((A02^D2), 43); + B3 = ROL64((A03^D3), 21); + B4 = ROL64((A04^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+3]; + A01 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A10^D0), 3); + B3 = ROL64((A11^D1), 45); + B4 = ROL64((A12^D2), 61); + B0 = ROL64((A13^D3), 28); + B1 = ROL64((A14^D4), 20); + A10 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A20^D0), 18); + B0 = ROL64((A21^D1), 1); + B1 = ROL64((A22^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A24^D4), 8); + A20 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A30^D0), 36); + B2 = ROL64((A31^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A33^D3), 56); + B0 = ROL64((A34^D4), 27); + A30 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A40^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A42^D2), 62); + B1 = ROL64((A43^D3), 55); + B2 = ROL64((A44^D4), 39); + A40 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + } +} + +/* +** Initialize a new hash. iSize determines the size of the hash +** in bits and should be one of 224, 256, 384, or 512. Or iSize +** can be zero to use the default hash size of 256 bits. +*/ +static void SHA3Init(SHA3Context *p, int iSize){ + memset(p, 0, sizeof(*p)); + if( iSize>=128 && iSize<=512 ){ + p->nRate = (1600 - ((iSize + 31)&~31)*2)/8; + }else{ + p->nRate = (1600 - 2*256)/8; + } +#if BYTEORDER==1234 + /* Known to be little-endian at compile-time. No-op */ +#elif BYTEORDER==4321 + p->ixMask = 7; /* Big-endian */ +#else + { + static unsigned int one = 1; + if( 1==*(unsigned char*)&one ){ + /* Little endian. No byte swapping. */ + p->ixMask = 0; + }else{ + /* Big endian. Byte swap. */ + p->ixMask = 7; + } + } +#endif +} + +/* +** Make consecutive calls to the SHA3Update function to add new content +** to the hash +*/ +static void SHA3Update( + SHA3Context *p, + const unsigned char *aData, + unsigned int nData +){ + unsigned int i = 0; +#if BYTEORDER==1234 + if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)0)&7)==0 ){ + for(; i+7u.s[p->nLoaded/8] ^= *(u64*)&aData[i]; + p->nLoaded += 8; + if( p->nLoaded>=p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } + } +#endif + for(; iu.x[p->nLoaded] ^= aData[i]; +#elif BYTEORDER==4321 + p->u.x[p->nLoaded^0x07] ^= aData[i]; +#else + p->u.x[p->nLoaded^p->ixMask] ^= aData[i]; +#endif + p->nLoaded++; + if( p->nLoaded==p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } +} + +/* +** After all content has been added, invoke SHA3Final() to compute +** the final hash. The function returns a pointer to the binary +** hash value. +*/ +static unsigned char *SHA3Final(SHA3Context *p){ + unsigned int i; + if( p->nLoaded==p->nRate-1 ){ + const unsigned char c1 = 0x86; + SHA3Update(p, &c1, 1); + }else{ + const unsigned char c2 = 0x06; + const unsigned char c3 = 0x80; + SHA3Update(p, &c2, 1); + p->nLoaded = p->nRate - 1; + SHA3Update(p, &c3, 1); + } + for(i=0; inRate; i++){ + p->u.x[i+p->nRate] = p->u.x[i^p->ixMask]; + } + return &p->u.x[p->nRate]; +} + +/* +** Convert a digest into base-16. digest should be declared as +** "unsigned char digest[20]" in the calling function. The SHA3 +** digest is stored in the first 20 bytes. zBuf should +** be "char zBuf[41]". +*/ +static void DigestToBase16(unsigned char *digest, char *zBuf, int nByte){ + static const char zEncode[] = "0123456789abcdef"; + int ix; + + for(ix=0; ix>4)&0xf]; + *zBuf++ = zEncode[*digest++ & 0xf]; + } + *zBuf = '\0'; +} + + +/* +** Compute the SHA3 checksum of a file on disk. Store the resulting +** checksum in the blob pCksum. pCksum is assumed to be initialized. +** +** Return the number of errors. +*/ +static int sha3sum_file(const char *zFilename, int iSize, char *pCksum){ + FILE *in; + SHA3Context ctx; + char zBuf[10240]; + + in = fopen(zFilename,"rb"); + if( in==0 ){ + return 1; + } + SHA3Init(&ctx, iSize); + for(;;){ + int n = (int)fread(zBuf, 1, sizeof(zBuf), in); + if( n<=0 ) break; + SHA3Update(&ctx, (unsigned char*)zBuf, (unsigned)n); + } + fclose(in); + DigestToBase16(SHA3Final(&ctx), pCksum, iSize/8); + return 0; +} + +/* +** The SHA1 implementation below is adapted from: +** +** $NetBSD: sha1.c,v 1.6 2009/11/06 20:31:18 joerg Exp $ +** $OpenBSD: sha1.c,v 1.9 1997/07/23 21:12:32 kstailey Exp $ +** +** SHA-1 in C +** By Steve Reid +** 100% Public Domain +*/ +typedef struct SHA1Context SHA1Context; +struct SHA1Context { + unsigned int state[5]; + unsigned int count[2]; + unsigned char buffer[64]; +}; + +/* + * blk0() and blk() perform the initial expand. + * I got the idea of expanding during the round function from SSLeay + * + * blk0le() for little-endian and blk0be() for big-endian. + */ +#if __GNUC__ && (defined(__i386__) || defined(__x86_64__)) +/* + * GCC by itself only generates left rotates. Use right rotates if + * possible to be kinder to dinky implementations with iterative rotate + * instructions. + */ +#define SHA_ROT(op, x, k) \ + ({ unsigned int y; asm(op " %1,%0" : "=r" (y) : "I" (k), "0" (x)); y; }) +#define rol(x,k) SHA_ROT("roll", x, k) +#define ror(x,k) SHA_ROT("rorl", x, k) + +#else +/* Generic C equivalent */ +#define SHA_ROT(x,l,r) ((x) << (l) | (x) >> (r)) +#define rol(x,k) SHA_ROT(x,k,32-(k)) +#define ror(x,k) SHA_ROT(x,32-(k),k) +#endif + + + + + +#define blk0le(i) (block[i] = (ror(block[i],8)&0xFF00FF00) \ + |(rol(block[i],8)&0x00FF00FF)) +#define blk0be(i) block[i] +#define blk(i) (block[i&15] = rol(block[(i+13)&15]^block[(i+8)&15] \ + ^block[(i+2)&15]^block[i&15],1)) + +/* + * (R0+R1), R2, R3, R4 are the different operations (rounds) used in SHA1 + * + * Rl0() for little-endian and Rb0() for big-endian. Endianness is + * determined at run-time. + */ +#define Rl0(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk0le(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define Rb0(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk0be(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define R1(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define R2(v,w,x,y,z,i) \ + z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=ror(w,2); +#define R3(v,w,x,y,z,i) \ + z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=ror(w,2); +#define R4(v,w,x,y,z,i) \ + z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=ror(w,2); + +/* + * Hash a single 512-bit block. This is the core of the algorithm. + */ +#define a qq[0] +#define b qq[1] +#define c qq[2] +#define d qq[3] +#define e qq[4] + +static void SHA1Transform( + unsigned int state[5], + const unsigned char buffer[64] +){ + unsigned int qq[5]; /* a, b, c, d, e; */ + static int one = 1; + unsigned int block[16]; + memcpy(block, buffer, 64); + memcpy(qq,state,5*sizeof(unsigned int)); + + /* Copy context->state[] to working vars */ + /* + a = state[0]; + b = state[1]; + c = state[2]; + d = state[3]; + e = state[4]; + */ + + /* 4 rounds of 20 operations each. Loop unrolled. */ + if( 1 == *(unsigned char*)&one ){ + Rl0(a,b,c,d,e, 0); Rl0(e,a,b,c,d, 1); Rl0(d,e,a,b,c, 2); Rl0(c,d,e,a,b, 3); + Rl0(b,c,d,e,a, 4); Rl0(a,b,c,d,e, 5); Rl0(e,a,b,c,d, 6); Rl0(d,e,a,b,c, 7); + Rl0(c,d,e,a,b, 8); Rl0(b,c,d,e,a, 9); Rl0(a,b,c,d,e,10); Rl0(e,a,b,c,d,11); + Rl0(d,e,a,b,c,12); Rl0(c,d,e,a,b,13); Rl0(b,c,d,e,a,14); Rl0(a,b,c,d,e,15); + }else{ + Rb0(a,b,c,d,e, 0); Rb0(e,a,b,c,d, 1); Rb0(d,e,a,b,c, 2); Rb0(c,d,e,a,b, 3); + Rb0(b,c,d,e,a, 4); Rb0(a,b,c,d,e, 5); Rb0(e,a,b,c,d, 6); Rb0(d,e,a,b,c, 7); + Rb0(c,d,e,a,b, 8); Rb0(b,c,d,e,a, 9); Rb0(a,b,c,d,e,10); Rb0(e,a,b,c,d,11); + Rb0(d,e,a,b,c,12); Rb0(c,d,e,a,b,13); Rb0(b,c,d,e,a,14); Rb0(a,b,c,d,e,15); + } + R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19); + R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23); + R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27); + R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31); + R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35); + R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39); + R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43); + R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47); + R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51); + R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55); + R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59); + R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63); + R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67); + R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71); + R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75); + R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79); + + /* Add the working vars back into context.state[] */ + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + state[4] += e; +} + + +/* + * SHA1Init - Initialize new context + */ +static void SHA1Init(SHA1Context *context){ + /* SHA1 initialization constants */ + context->state[0] = 0x67452301; + context->state[1] = 0xEFCDAB89; + context->state[2] = 0x98BADCFE; + context->state[3] = 0x10325476; + context->state[4] = 0xC3D2E1F0; + context->count[0] = context->count[1] = 0; +} + + +/* + * Run your data through this. + */ +static void SHA1Update( + SHA1Context *context, + const unsigned char *data, + unsigned int len +){ + unsigned int i, j; + + j = context->count[0]; + if ((context->count[0] += len << 3) < j) + context->count[1] += (len>>29)+1; + j = (j >> 3) & 63; + if ((j + len) > 63) { + (void)memcpy(&context->buffer[j], data, (i = 64-j)); + SHA1Transform(context->state, context->buffer); + for ( ; i + 63 < len; i += 64) + SHA1Transform(context->state, &data[i]); + j = 0; + } else { + i = 0; + } + (void)memcpy(&context->buffer[j], &data[i], len - i); +} + + +/* + * Add padding and return the message digest. + */ +static void SHA1Final(unsigned char *digest, SHA1Context *context){ + unsigned int i; + unsigned char finalcount[8]; + + for (i = 0; i < 8; i++) { + finalcount[i] = (unsigned char)((context->count[(i >= 4 ? 0 : 1)] + >> ((3-(i & 3)) * 8) ) & 255); /* Endian independent */ + } + SHA1Update(context, (const unsigned char *)"\200", 1); + while ((context->count[0] & 504) != 448) + SHA1Update(context, (const unsigned char *)"\0", 1); + SHA1Update(context, finalcount, 8); /* Should cause a SHA1Transform() */ + + if (digest) { + for (i = 0; i < 20; i++) + digest[i] = (unsigned char) + ((context->state[i>>2] >> ((3-(i & 3)) * 8) ) & 255); + } +} + + +/* +** Compute the SHA1 checksum of a file on disk. Store the resulting +** checksum in the blob pCksum. pCksum is assumed to be initialized. +** +** Return the number of errors. +*/ +static int sha1sum_file(const char *zFilename, char *pCksum){ + FILE *in; + SHA1Context ctx; + unsigned char zResult[20]; + char zBuf[10240]; + + in = fopen(zFilename,"rb"); + if( in==0 ){ + return 1; + } + SHA1Init(&ctx); + for(;;){ + int n = (int)fread(zBuf, 1, sizeof(zBuf), in); + if( n<=0 ) break; + SHA1Update(&ctx, (unsigned char*)zBuf, (unsigned)n); + } + fclose(in); + SHA1Final(zResult, &ctx); + DigestToBase16(zResult, pCksum, 20); + return 0; +} + +/* +** Print a usage comment and quit. +*/ +static void usage(const char *argv0){ + fprintf(stderr, + "Usage: %s manifest\n" + "Options:\n" + " -v Diagnostic output\n" + , argv0); + exit(1); +} + +/* +** Find the first whitespace character in a string. Set that whitespace +** to a \000 terminator and return a pointer to the next character. +*/ +static char *nextToken(char *z){ + while( *z && !isspace(*z) ) z++; + if( *z==0 ) return z; + *z = 0; + return &z[1]; +} + + +int main(int argc, char **argv){ + const char *zManifest = 0; + int i; + int bVerbose = 0; + FILE *in; + int allValid = 1; + int rc; + char zDate[50]; + char zHash[100]; + char zLine[1000]; + + for(i=1; i"]*} $line]} continue regsub -- --VERS-- $line $zVersion line regsub -- --VERSION-NUMBER-- $line $nVersion line - regsub -- --SOURCE-ID-- $line "$zDate $zUuid" line + regsub -- --SOURCE-ID-- $line "$zSourceId" line if {[regexp $varpattern $line] && ![regexp {^ *typedef} $line]} { set line "SQLITE_API $line" } else { if {[regexp $declpattern1 $line all rettype funcname rest] || \ Index: tool/speed-check.sh ================================================================== --- tool/speed-check.sh +++ tool/speed-check.sh @@ -145,10 +145,12 @@ fi size sqlite3.o | tee -a summary-$NAME.txt wc sqlite3.c if test $doCachegrind -eq 1; then cg_anno.tcl cachegrind.out.* >cout-$NAME.txt + echo '*****************************************************' >>cout-$NAME.txt + sed 's/^[0-9=-]\{9\}/==00000==/' summary-$NAME.txt >>cout-$NAME.txt fi if test $doExplain -eq 1; then ./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt fi if test "$NAME" != "trunk"; then