From c555d90f4f56e9cc443da9ed2d97457f2d968bf7 Mon Sep 17 00:00:00 2001 From: Xinchen Hui Date: Fri, 18 Dec 2015 23:49:59 +0800 Subject: Implemented FR #71159 (Upgraed bundled SQLite lib to 3.9.2) --- ext/sqlite3/libsqlite/sqlite3.c | 92023 +++++++++++++++++++++++------------ ext/sqlite3/libsqlite/sqlite3.h | 684 +- ext/sqlite3/libsqlite/sqlite3ext.h | 23 +- 3 files changed, 60872 insertions(+), 31858 deletions(-) (limited to 'ext/sqlite3/libsqlite') diff --git a/ext/sqlite3/libsqlite/sqlite3.c b/ext/sqlite3/libsqlite/sqlite3.c index dff4538dd3..0ae407dd83 100644 --- a/ext/sqlite3/libsqlite/sqlite3.c +++ b/ext/sqlite3/libsqlite/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.8.10.2. By combining all the individual C code files into this +** version 3.9.2. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -158,6 +158,13 @@ # define _LARGEFILE_SOURCE 1 #endif +/* What version of GCC is being used. 0 means GCC is not being used */ +#ifdef __GNUC__ +# define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__) +#else +# define GCC_VERSION 0 +#endif + /* Needed for various definitions... */ #if defined(__GNUC__) && !defined(_GNU_SOURCE) # define _GNU_SOURCE @@ -230,7 +237,7 @@ ** ** The official C-language API documentation for SQLite is derived ** from comments in this file. This file is the authoritative source -** on how SQLite interfaces are suppose to operate. +** on how SQLite interfaces are supposed to operate. ** ** The name of this file under configuration management is "sqlite.h.in". ** The makefile makes some minor changes to this file (such as inserting @@ -318,9 +325,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.8.10.2" -#define SQLITE_VERSION_NUMBER 3008010 -#define SQLITE_SOURCE_ID "2015-05-20 18:17:19 2ef4f3a5b1d1d0c4338f8243d40a2452cc1f7fe4" +#define SQLITE_VERSION "3.9.2" +#define SQLITE_VERSION_NUMBER 3009002 +#define SQLITE_SOURCE_ID "2015-11-02 18:31:45 bda77dda9697c463c3d0704014d51627fceee328" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -331,7 +338,7 @@ extern "C" { ** but are associated with the library instead of the header file. ^(Cautious ** programmers might include assert() statements in their application to ** verify that values returned by these interfaces match the macros in -** the header, and thus insure that the application is +** the header, and thus ensure that the application is ** compiled with matching library and header files. ** ** 
@@ -581,7 +588,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**);
 ** Restrictions:
 **
 ** 
    
-** 
  •  The application must insure that the 1st parameter to sqlite3_exec()
+** 
  •  The application must ensure that the 1st parameter to sqlite3_exec()
 **      is a valid and open [database connection].
 ** 
  •  The application must not close the [database connection] specified by
 **      the 1st parameter to sqlite3_exec() while sqlite3_exec() is running.
@@ -684,6 +691,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_exec(
 #define SQLITE_IOERR_MMAP              (SQLITE_IOERR | (24<<8))
 #define SQLITE_IOERR_GETTEMPPATH       (SQLITE_IOERR | (25<<8))
 #define SQLITE_IOERR_CONVPATH          (SQLITE_IOERR | (26<<8))
+#define SQLITE_IOERR_VNODE             (SQLITE_IOERR | (27<<8))
 #define SQLITE_LOCKED_SHAREDCACHE      (SQLITE_LOCKED |  (1<<8))
 #define SQLITE_BUSY_RECOVERY           (SQLITE_BUSY   |  (1<<8))
 #define SQLITE_BUSY_SNAPSHOT           (SQLITE_BUSY   |  (2<<8))
@@ -1170,6 +1178,14 @@ struct sqlite3_io_methods {
 ** circumstances in order to fix a problem with priority inversion.
 ** Applications should not use this file-control.
 **
+** 
  • [[SQLITE_FCNTL_ZIPVFS]]
+** The [SQLITE_FCNTL_ZIPVFS] opcode is implemented by zipvfs only. All other
+** VFS should return SQLITE_NOTFOUND for this opcode.
+**
+** 
  • [[SQLITE_FCNTL_RBU]]
+** The [SQLITE_FCNTL_RBU] opcode is implemented by the special VFS used by
+** the RBU extension only.  All other VFS should return SQLITE_NOTFOUND for
+** this opcode.  
 ** 

 */
 #define SQLITE_FCNTL_LOCKSTATE               1
@@ -1195,6 +1211,8 @@ struct sqlite3_io_methods {
 #define SQLITE_FCNTL_COMMIT_PHASETWO        22
 #define SQLITE_FCNTL_WIN32_SET_HANDLE       23
 #define SQLITE_FCNTL_WAL_BLOCK              24
+#define SQLITE_FCNTL_ZIPVFS                 25
+#define SQLITE_FCNTL_RBU                    26
 
 /* deprecated names */
 #define SQLITE_GET_LOCKPROXYFILE      SQLITE_FCNTL_GET_LOCKPROXYFILE
@@ -1563,9 +1581,11 @@ SQLITE_API int SQLITE_STDCALL sqlite3_os_end(void);
 ** applications and so this routine is usually not necessary.  It is
 ** provided to support rare applications with unusual needs.
 **
-** The sqlite3_config() interface is not threadsafe.  The application
-** must insure that no other SQLite interfaces are invoked by other
-** threads while sqlite3_config() is running.  Furthermore, sqlite3_config()
+** The sqlite3_config() interface is not threadsafe. The application
+** must ensure that no other SQLite interfaces are invoked by other
+** threads while sqlite3_config() is running.
+**
+** The sqlite3_config() interface
 ** may only be invoked prior to library initialization using
 ** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()].
 ** ^If sqlite3_config() is called after [sqlite3_initialize()] and before
@@ -3570,7 +3590,8 @@ SQLITE_API int SQLITE_STDCALL sqlite3_stmt_readonly(sqlite3_stmt *pStmt);
 **
 ** ^The sqlite3_stmt_busy(S) interface returns true (non-zero) if the
 ** [prepared statement] S has been stepped at least once using 
-** [sqlite3_step(S)] but has not run to completion and/or has not 
+** [sqlite3_step(S)] but has neither run to completion (returned
+** [SQLITE_DONE] from [sqlite3_step(S)]) nor
 ** been reset using [sqlite3_reset(S)].  ^The sqlite3_stmt_busy(S)
 ** interface returns false if S is a NULL pointer.  If S is not a 
 ** NULL pointer and is not a pointer to a valid [prepared statement]
@@ -3597,7 +3618,9 @@ SQLITE_API int SQLITE_STDCALL sqlite3_stmt_busy(sqlite3_stmt*);
 ** Some interfaces require a protected sqlite3_value.  Other interfaces
 ** will accept either a protected or an unprotected sqlite3_value.
 ** Every interface that accepts sqlite3_value arguments specifies
-** whether or not it requires a protected sqlite3_value.
+** whether or not it requires a protected sqlite3_value.  The
+** [sqlite3_value_dup()] interface can be used to construct a new 
+** protected sqlite3_value from an unprotected sqlite3_value.
 **
 ** The terms "protected" and "unprotected" refer to whether or not
 ** a mutex is held.  An internal mutex is held for a protected
@@ -3757,6 +3780,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_bind_text64(sqlite3_stmt*, int, const char
                          void(*)(void*), unsigned char encoding);
 SQLITE_API int SQLITE_STDCALL sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*);
 SQLITE_API int SQLITE_STDCALL sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n);
+SQLITE_API int SQLITE_STDCALL sqlite3_bind_zeroblob64(sqlite3_stmt*, int, sqlite3_uint64);
 
 /*
 ** CAPI3REF: Number Of SQL Parameters
@@ -3820,7 +3844,7 @@ SQLITE_API const char *SQLITE_STDCALL sqlite3_bind_parameter_name(sqlite3_stmt*,
 **
 ** See also: [sqlite3_bind_blob|sqlite3_bind()],
 ** [sqlite3_bind_parameter_count()], and
-** [sqlite3_bind_parameter_index()].
+** [sqlite3_bind_parameter_name()].
 */
 SQLITE_API int SQLITE_STDCALL sqlite3_bind_parameter_index(sqlite3_stmt*, const char *zName);
 
@@ -4100,8 +4124,6 @@ SQLITE_API int SQLITE_STDCALL sqlite3_data_count(sqlite3_stmt *pStmt);
 ** KEYWORDS: {column access functions}
 ** METHOD: sqlite3_stmt
 **
-** These routines form the "result set" interface.
-**
 ** ^These routines return information about a single column of the current
 ** result row of a query.  ^In every case the first argument is a pointer
 ** to the [prepared statement] that is being evaluated (the [sqlite3_stmt*]
@@ -4161,13 +4183,14 @@ SQLITE_API int SQLITE_STDCALL sqlite3_data_count(sqlite3_stmt *pStmt);
 ** even empty strings, are always zero-terminated.  ^The return
 ** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer.
 **
-** ^The object returned by [sqlite3_column_value()] is an
-** [unprotected sqlite3_value] object.  An unprotected sqlite3_value object
-** may only be used with [sqlite3_bind_value()] and [sqlite3_result_value()].
+** Warning: ^The object returned by [sqlite3_column_value()] is an
+** [unprotected sqlite3_value] object.  In a multithreaded environment,
+** an unprotected sqlite3_value object may only be used safely with
+** [sqlite3_bind_value()] and [sqlite3_result_value()].
 ** If the [unprotected sqlite3_value] object returned by
 ** [sqlite3_column_value()] is used in any other way, including calls
 ** to routines like [sqlite3_value_int()], [sqlite3_value_text()],
-** or [sqlite3_value_bytes()], then the behavior is undefined.
+** or [sqlite3_value_bytes()], the behavior is not threadsafe.
 **
 ** These routines attempt to convert the value where appropriate.  ^For
 ** example, if the internal representation is FLOAT and a text result
@@ -4198,12 +4221,6 @@ SQLITE_API int SQLITE_STDCALL sqlite3_data_count(sqlite3_stmt *pStmt);
 ** 
 **
)^ ** -** The table above makes reference to standard C library functions atoi() -** and atof(). SQLite does not really use these functions. It has its -** own equivalent internal routines. The atoi() and atof() names are -** used in the table for brevity and because they are familiar to most -** C programmers. -** ** Note that when type conversions occur, pointers returned by prior ** calls to sqlite3_column_blob(), sqlite3_column_text(), and/or ** sqlite3_column_text16() may be invalidated. @@ -4228,7 +4245,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_data_count(sqlite3_stmt *pStmt); ** of conversion are done in place when it is possible, but sometimes they ** are not possible and in those cases prior pointers are invalidated. ** -** The safest and easiest to remember policy is to invoke these routines +** The safest policy is to invoke these routines ** in one of the following ways: ** ** ** ** ^The first two constants (SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) @@ -6476,6 +6570,9 @@ SQLITE_API int SQLITE_STDCALL sqlite3_mutex_notheld(sqlite3_mutex*); #define SQLITE_MUTEX_STATIC_APP1 8 /* For use by application */ #define SQLITE_MUTEX_STATIC_APP2 9 /* For use by application */ #define SQLITE_MUTEX_STATIC_APP3 10 /* For use by application */ +#define SQLITE_MUTEX_STATIC_VFS1 11 /* For use by built-in VFS */ +#define SQLITE_MUTEX_STATIC_VFS2 12 /* For use by extension VFS */ +#define SQLITE_MUTEX_STATIC_VFS3 13 /* For use by application VFS */ /* ** CAPI3REF: Retrieve the mutex for a database connection @@ -7889,7 +7986,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_vtab_on_conflict(sqlite3 *); ** ** See also: [sqlite3_stmt_scanstatus_reset()] */ -SQLITE_API SQLITE_EXPERIMENTAL int SQLITE_STDCALL sqlite3_stmt_scanstatus( +SQLITE_API int SQLITE_STDCALL sqlite3_stmt_scanstatus( sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ int idx, /* Index of loop to report on */ int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ @@ -7905,7 +8002,7 @@ SQLITE_API SQLITE_EXPERIMENTAL int SQLITE_STDCALL sqlite3_stmt_scanstatus( ** This API is only available if the library is built with pre-processor ** symbol [SQLITE_ENABLE_STMT_SCANSTATUS] defined. */ -SQLITE_API SQLITE_EXPERIMENTAL void SQLITE_STDCALL sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); +SQLITE_API void SQLITE_STDCALL sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); /* @@ -8020,6 +8117,8 @@ struct sqlite3_rtree_query_info { int eParentWithin; /* Visibility of parent node */ int eWithin; /* OUT: Visiblity */ sqlite3_rtree_dbl rScore; /* OUT: Write the score here */ + /* The following fields are only available in 3.8.11 and later */ + sqlite3_value **apSqlParam; /* Original SQL values of parameters */ }; /* @@ -8036,6 +8135,526 @@ struct sqlite3_rtree_query_info { #endif /* ifndef _SQLITE3RTREE_H_ */ +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Interfaces to extend FTS5. Using the interfaces defined in this file, +** FTS5 may be extended with: +** +** * custom tokenizers, and +** * custom auxiliary functions. +*/ + + +#ifndef _FTS5_H +#define _FTS5_H + + +#if 0 +extern "C" { +#endif + +/************************************************************************* +** CUSTOM AUXILIARY FUNCTIONS +** +** Virtual table implementations may overload SQL functions by implementing +** the sqlite3_module.xFindFunction() method. +*/ + +typedef struct Fts5ExtensionApi Fts5ExtensionApi; +typedef struct Fts5Context Fts5Context; +typedef struct Fts5PhraseIter Fts5PhraseIter; + +typedef void (*fts5_extension_function)( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +); + +struct Fts5PhraseIter { + const unsigned char *a; + const unsigned char *b; +}; + +/* +** EXTENSION API FUNCTIONS +** +** xUserData(pFts): +** Return a copy of the context pointer the extension function was +** registered with. +** +** xColumnTotalSize(pFts, iCol, pnToken): +** If parameter iCol is less than zero, set output variable *pnToken +** to the total number of tokens in the FTS5 table. Or, if iCol is +** non-negative but less than the number of columns in the table, return +** the total number of tokens in column iCol, considering all rows in +** the FTS5 table. +** +** If parameter iCol is greater than or equal to the number of columns +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. +** an OOM condition or IO error), an appropriate SQLite error code is +** returned. +** +** xColumnCount(pFts): +** Return the number of columns in the table. +** +** xColumnSize(pFts, iCol, pnToken): +** If parameter iCol is less than zero, set output variable *pnToken +** to the total number of tokens in the current row. Or, if iCol is +** non-negative but less than the number of columns in the table, set +** *pnToken to the number of tokens in column iCol of the current row. +** +** If parameter iCol is greater than or equal to the number of columns +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. +** an OOM condition or IO error), an appropriate SQLite error code is +** returned. +** +** xColumnText: +** This function attempts to retrieve the text of column iCol of the +** current document. If successful, (*pz) is set to point to a buffer +** containing the text in utf-8 encoding, (*pn) is set to the size in bytes +** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, +** if an error occurs, an SQLite error code is returned and the final values +** of (*pz) and (*pn) are undefined. +** +** xPhraseCount: +** Returns the number of phrases in the current query expression. +** +** xPhraseSize: +** Returns the number of tokens in phrase iPhrase of the query. Phrases +** are numbered starting from zero. +** +** xInstCount: +** Set *pnInst to the total number of occurrences of all phrases within +** the query within the current row. Return SQLITE_OK if successful, or +** an error code (i.e. SQLITE_NOMEM) if an error occurs. +** +** xInst: +** Query for the details of phrase match iIdx within the current row. +** Phrase matches are numbered starting from zero, so the iIdx argument +** should be greater than or equal to zero and smaller than the value +** output by xInstCount(). +** +** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) +** if an error occurs. +** +** xRowid: +** Returns the rowid of the current row. +** +** xTokenize: +** Tokenize text using the tokenizer belonging to the FTS5 table. +** +** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback): +** This API function is used to query the FTS table for phrase iPhrase +** of the current query. Specifically, a query equivalent to: +** +** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid +** +** with $p set to a phrase equivalent to the phrase iPhrase of the +** current query is executed. For each row visited, the callback function +** passed as the fourth argument is invoked. The context and API objects +** passed to the callback function may be used to access the properties of +** each matched row. Invoking Api.xUserData() returns a copy of the pointer +** passed as the third argument to pUserData. +** +** If the callback function returns any value other than SQLITE_OK, the +** query is abandoned and the xQueryPhrase function returns immediately. +** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK. +** Otherwise, the error code is propagated upwards. +** +** If the query runs to completion without incident, SQLITE_OK is returned. +** Or, if some error occurs before the query completes or is aborted by +** the callback, an SQLite error code is returned. +** +** +** xSetAuxdata(pFts5, pAux, xDelete) +** +** Save the pointer passed as the second argument as the extension functions +** "auxiliary data". The pointer may then be retrieved by the current or any +** future invocation of the same fts5 extension function made as part of +** of the same MATCH query using the xGetAuxdata() API. +** +** Each extension function is allocated a single auxiliary data slot for +** each FTS query (MATCH expression). If the extension function is invoked +** more than once for a single FTS query, then all invocations share a +** single auxiliary data context. +** +** If there is already an auxiliary data pointer when this function is +** invoked, then it is replaced by the new pointer. If an xDelete callback +** was specified along with the original pointer, it is invoked at this +** point. +** +** The xDelete callback, if one is specified, is also invoked on the +** auxiliary data pointer after the FTS5 query has finished. +** +** If an error (e.g. an OOM condition) occurs within this function, an +** the auxiliary data is set to NULL and an error code returned. If the +** xDelete parameter was not NULL, it is invoked on the auxiliary data +** pointer before returning. +** +** +** xGetAuxdata(pFts5, bClear) +** +** Returns the current auxiliary data pointer for the fts5 extension +** function. See the xSetAuxdata() method for details. +** +** If the bClear argument is non-zero, then the auxiliary data is cleared +** (set to NULL) before this function returns. In this case the xDelete, +** if any, is not invoked. +** +** +** xRowCount(pFts5, pnRow) +** +** This function is used to retrieve the total number of rows in the table. +** In other words, the same value that would be returned by: +** +** SELECT count(*) FROM ftstable; +** +** xPhraseFirst() +** This function is used, along with type Fts5PhraseIter and the xPhraseNext +** method, to iterate through all instances of a single query phrase within +** the current row. This is the same information as is accessible via the +** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient +** to use, this API may be faster under some circumstances. To iterate +** through instances of phrase iPhrase, use the following code: +** +** Fts5PhraseIter iter; +** int iCol, iOff; +** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); +** iOff>=0; +** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) +** ){ +** // An instance of phrase iPhrase at offset iOff of column iCol +** } +** +** The Fts5PhraseIter structure is defined above. Applications should not +** modify this structure directly - it should only be used as shown above +** with the xPhraseFirst() and xPhraseNext() API methods. +** +** xPhraseNext() +** See xPhraseFirst above. +*/ +struct Fts5ExtensionApi { + int iVersion; /* Currently always set to 1 */ + + void *(*xUserData)(Fts5Context*); + + int (*xColumnCount)(Fts5Context*); + int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow); + int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken); + + int (*xTokenize)(Fts5Context*, + const char *pText, int nText, /* Text to tokenize */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ + ); + + int (*xPhraseCount)(Fts5Context*); + int (*xPhraseSize)(Fts5Context*, int iPhrase); + + int (*xInstCount)(Fts5Context*, int *pnInst); + int (*xInst)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff); + + sqlite3_int64 (*xRowid)(Fts5Context*); + int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn); + int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken); + + int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData, + int(*)(const Fts5ExtensionApi*,Fts5Context*,void*) + ); + int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); + void *(*xGetAuxdata)(Fts5Context*, int bClear); + + void (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); + void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff); +}; + +/* +** CUSTOM AUXILIARY FUNCTIONS +*************************************************************************/ + +/************************************************************************* +** CUSTOM TOKENIZERS +** +** Applications may also register custom tokenizer types. A tokenizer +** is registered by providing fts5 with a populated instance of the +** following structure. All structure methods must be defined, setting +** any member of the fts5_tokenizer struct to NULL leads to undefined +** behaviour. The structure methods are expected to function as follows: +** +** xCreate: +** This function is used to allocate and inititalize a tokenizer instance. +** A tokenizer instance is required to actually tokenize text. +** +** The first argument passed to this function is a copy of the (void*) +** pointer provided by the application when the fts5_tokenizer object +** was registered with FTS5 (the third argument to xCreateTokenizer()). +** The second and third arguments are an array of nul-terminated strings +** containing the tokenizer arguments, if any, specified following the +** tokenizer name as part of the CREATE VIRTUAL TABLE statement used +** to create the FTS5 table. +** +** The final argument is an output variable. If successful, (*ppOut) +** should be set to point to the new tokenizer handle and SQLITE_OK +** returned. If an error occurs, some value other than SQLITE_OK should +** be returned. In this case, fts5 assumes that the final value of *ppOut +** is undefined. +** +** xDelete: +** This function is invoked to delete a tokenizer handle previously +** allocated using xCreate(). Fts5 guarantees that this function will +** be invoked exactly once for each successful call to xCreate(). +** +** xTokenize: +** This function is expected to tokenize the nText byte string indicated +** by argument pText. pText may or may not be nul-terminated. The first +** argument passed to this function is a pointer to an Fts5Tokenizer object +** returned by an earlier call to xCreate(). +** +** The second argument indicates the reason that FTS5 is requesting +** tokenization of the supplied text. This is always one of the following +** four values: +** +** +** +** For each token in the input string, the supplied callback xToken() must +** be invoked. The first argument to it should be a copy of the pointer +** passed as the second argument to xTokenize(). The third and fourth +** arguments are a pointer to a buffer containing the token text, and the +** size of the token in bytes. The 4th and 5th arguments are the byte offsets +** of the first byte of and first byte immediately following the text from +** which the token is derived within the input. +** +** The second argument passed to the xToken() callback ("tflags") should +** normally be set to 0. The exception is if the tokenizer supports +** synonyms. In this case see the discussion below for details. +** +** FTS5 assumes the xToken() callback is invoked for each token in the +** order that they occur within the input text. +** +** If an xToken() callback returns any value other than SQLITE_OK, then +** the tokenization should be abandoned and the xTokenize() method should +** immediately return a copy of the xToken() return value. Or, if the +** input buffer is exhausted, xTokenize() should return SQLITE_OK. Finally, +** if an error occurs with the xTokenize() implementation itself, it +** may abandon the tokenization and return any error code other than +** SQLITE_OK or SQLITE_DONE. +** +** SYNONYM SUPPORT +** +** Custom tokenizers may also support synonyms. Consider a case in which a +** user wishes to query for a phrase such as "first place". Using the +** built-in tokenizers, the FTS5 query 'first + place' will match instances +** of "first place" within the document set, but not alternative forms +** such as "1st place". In some applications, it would be better to match +** all instances of "first place" or "1st place" regardless of which form +** the user specified in the MATCH query text. +** +** There are several ways to approach this in FTS5: +** +**
  1. By mapping all synonyms to a single token. In this case, the +** In the above example, this means that the tokenizer returns the +** same token for inputs "first" and "1st". Say that token is in +** fact "first", so that when the user inserts the document "I won +** 1st place" entries are added to the index for tokens "i", "won", +** "first" and "place". If the user then queries for '1st + place', +** the tokenizer substitutes "first" for "1st" and the query works +** as expected. +** +**
  2. By adding multiple synonyms for a single term to the FTS index. +** In this case, when tokenizing query text, the tokenizer may +** provide multiple synonyms for a single term within the document. +** FTS5 then queries the index for each synonym individually. For +** example, faced with the query: +** +** +** ... MATCH 'first place' +** +** the tokenizer offers both "1st" and "first" as synonyms for the +** first token in the MATCH query and FTS5 effectively runs a query +** similar to: +** +** +** ... MATCH '(first OR 1st) place' +** +** except that, for the purposes of auxiliary functions, the query +** still appears to contain just two phrases - "(first OR 1st)" +** being treated as a single phrase. +** +**
  3. By adding multiple synonyms for a single term to the FTS index. +** Using this method, when tokenizing document text, the tokenizer +** provides multiple synonyms for each token. So that when a +** document such as "I won first place" is tokenized, entries are +** added to the FTS index for "i", "won", "first", "1st" and +** "place". +** +** This way, even if the tokenizer does not provide synonyms +** when tokenizing query text (it should not - to do would be +** inefficient), it doesn't matter if the user queries for +** 'first + place' or '1st + place', as there are entires in the +** FTS index corresponding to both forms of the first token. +**
+** +** Whether it is parsing document or query text, any call to xToken that +** specifies a tflags argument with the FTS5_TOKEN_COLOCATED bit +** is considered to supply a synonym for the previous token. For example, +** when parsing the document "I won first place", a tokenizer that supports +** synonyms would call xToken() 5 times, as follows: +** +** +** xToken(pCtx, 0, "i", 1, 0, 1); +** xToken(pCtx, 0, "won", 3, 2, 5); +** xToken(pCtx, 0, "first", 5, 6, 11); +** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11); +** xToken(pCtx, 0, "place", 5, 12, 17); +** +** +** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time +** xToken() is called. Multiple synonyms may be specified for a single token +** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence. +** There is no limit to the number of synonyms that may be provided for a +** single token. +** +** In many cases, method (1) above is the best approach. It does not add +** extra data to the FTS index or require FTS5 to query for multiple terms, +** so it is efficient in terms of disk space and query speed. However, it +** does not support prefix queries very well. If, as suggested above, the +** token "first" is subsituted for "1st" by the tokenizer, then the query: +** +** +** ... MATCH '1s*' +** +** will not match documents that contain the token "1st" (as the tokenizer +** will probably not map "1s" to any prefix of "first"). +** +** For full prefix support, method (3) may be preferred. In this case, +** because the index contains entries for both "first" and "1st", prefix +** queries such as 'fi*' or '1s*' will match correctly. However, because +** extra entries are added to the FTS index, this method uses more space +** within the database. +** +** Method (2) offers a midpoint between (1) and (3). Using this method, +** a query such as '1s*' will match documents that contain the literal +** token "1st", but not "first" (assuming the tokenizer is not able to +** provide synonyms for prefixes). However, a non-prefix query like '1st' +** will match against "1st" and "first". This method does not require +** extra disk space, as no extra entries are added to the FTS index. +** On the other hand, it may require more CPU cycles to run MATCH queries, +** as separate queries of the FTS index are required for each synonym. +** +** When using methods (2) or (3), it is important that the tokenizer only +** provide synonyms when tokenizing document text (method (2)) or query +** text (method (3)), not both. Doing so will not cause any errors, but is +** inefficient. +*/ +typedef struct Fts5Tokenizer Fts5Tokenizer; +typedef struct fts5_tokenizer fts5_tokenizer; +struct fts5_tokenizer { + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + void (*xDelete)(Fts5Tokenizer*); + int (*xTokenize)(Fts5Tokenizer*, + void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ + const char *pText, int nText, + int (*xToken)( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ + ) + ); +}; + +/* Flags that may be passed as the third argument to xTokenize() */ +#define FTS5_TOKENIZE_QUERY 0x0001 +#define FTS5_TOKENIZE_PREFIX 0x0002 +#define FTS5_TOKENIZE_DOCUMENT 0x0004 +#define FTS5_TOKENIZE_AUX 0x0008 + +/* Flags that may be passed by the tokenizer implementation back to FTS5 +** as the third argument to the supplied xToken callback. */ +#define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */ + +/* +** END OF CUSTOM TOKENIZERS +*************************************************************************/ + +/************************************************************************* +** FTS5 EXTENSION REGISTRATION API +*/ +typedef struct fts5_api fts5_api; +struct fts5_api { + int iVersion; /* Currently always set to 2 */ + + /* Create a new tokenizer */ + int (*xCreateTokenizer)( + fts5_api *pApi, + const char *zName, + void *pContext, + fts5_tokenizer *pTokenizer, + void (*xDestroy)(void*) + ); + + /* Find an existing tokenizer */ + int (*xFindTokenizer)( + fts5_api *pApi, + const char *zName, + void **ppContext, + fts5_tokenizer *pTokenizer + ); + + /* Create a new auxiliary function */ + int (*xCreateFunction)( + fts5_api *pApi, + const char *zName, + void *pContext, + fts5_extension_function xFunction, + void (*xDestroy)(void*) + ); +}; + +/* +** END OF REGISTRATION API +*************************************************************************/ + +#if 0 +} /* end of the 'extern "C"' block */ +#endif + +#endif /* _FTS5_H */ + + /************** End of sqlite3.h *********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -8325,6 +8944,24 @@ struct sqlite3_rtree_query_info { # define SQLITE_NOINLINE #endif +/* +** Make sure that the compiler intrinsics we desire are enabled when +** compiling with an appropriate version of MSVC unless prevented by +** the SQLITE_DISABLE_INTRINSIC define. +*/ +#if !defined(SQLITE_DISABLE_INTRINSIC) +# if defined(_MSC_VER) && _MSC_VER>=1300 +# if !defined(_WIN32_WCE) +# include +# pragma intrinsic(_byteswap_ushort) +# pragma intrinsic(_byteswap_ulong) +# pragma intrinsic(_ReadWriteBarrier) +# else +# include +# endif +# endif +#endif + /* ** The SQLITE_THREADSAFE macro must be defined as 0, 1, or 2. ** 0 means mutexes are permanently disable and the library is never @@ -8902,6 +9539,16 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); # define SQLITE_MAX_WORKER_THREADS SQLITE_DEFAULT_WORKER_THREADS #endif +/* +** The default initial allocation for the pagecache when using separate +** pagecaches for each database connection. A positive number is the +** number of pages. A negative number N translations means that a buffer +** of -1024*N bytes is allocated and used for as many pages as it will hold. +*/ +#ifndef SQLITE_DEFAULT_PCACHE_INITSZ +# define SQLITE_DEFAULT_PCACHE_INITSZ 100 +#endif + /* ** GCC does not define the offsetof() macro so we'll have to do it @@ -9137,7 +9784,9 @@ SQLITE_PRIVATE const int sqlite3one; # if defined(__linux__) \ || defined(_WIN32) \ || (defined(__APPLE__) && defined(__MACH__)) \ - || defined(__sun) + || defined(__sun) \ + || defined(__FreeBSD__) \ + || defined(__DragonFly__) # define SQLITE_MAX_MMAP_SIZE 0x7fff0000 /* 2147418112 */ # else # define SQLITE_MAX_MMAP_SIZE 0 @@ -9527,7 +10176,7 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( ); SQLITE_PRIVATE int sqlite3BtreeCursorHasMoved(BtCursor*); SQLITE_PRIVATE int sqlite3BtreeCursorRestore(BtCursor*, int*); -SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor*); +SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor*, int); SQLITE_PRIVATE int sqlite3BtreeInsert(BtCursor*, const void *pKey, i64 nKey, const void *pData, int nData, int nZero, int bias, int seekResult); @@ -9667,13 +10316,14 @@ struct VdbeOp { int p1; /* First operand */ int p2; /* Second parameter (often the jump destination) */ int p3; /* The third parameter */ - union { /* fourth parameter */ + union p4union { /* fourth parameter */ int i; /* Integer value if p4type==P4_INT32 */ void *p; /* Generic pointer */ char *z; /* Pointer to data for string (char array) types */ i64 *pI64; /* Used when p4type is P4_INT64 */ double *pReal; /* Used when p4type is P4_REAL */ FuncDef *pFunc; /* Used when p4type is P4_FUNCDEF */ + sqlite3_context *pCtx; /* Used when p4type is P4_FUNCCTX */ CollSeq *pColl; /* Used when p4type is P4_COLLSEQ */ Mem *pMem; /* Used when p4type is P4_MEM */ VTable *pVtab; /* Used when p4type is P4_VTAB */ @@ -9740,6 +10390,7 @@ typedef struct VdbeOpList VdbeOpList; #define P4_INTARRAY (-15) /* P4 is a vector of 32-bit integers */ #define P4_SUBPROGRAM (-18) /* P4 is a pointer to a SubProgram structure */ #define P4_ADVANCE (-19) /* P4 is a pointer to BtreeNext() or BtreePrev() */ +#define P4_FUNCCTX (-20) /* P4 is a pointer to an sqlite3_context object */ /* Error message codes for OP_Halt */ #define P5_ConstraintNotNull 1 @@ -9782,42 +10433,42 @@ typedef struct VdbeOpList VdbeOpList; /************** Begin file opcodes.h *****************************************/ /* Automatically generated. Do not edit */ /* See the mkopcodeh.awk script for details */ -#define OP_Function 1 /* synopsis: r[P3]=func(r[P2@P5]) */ -#define OP_Savepoint 2 -#define OP_AutoCommit 3 -#define OP_Transaction 4 -#define OP_SorterNext 5 -#define OP_PrevIfOpen 6 -#define OP_NextIfOpen 7 -#define OP_Prev 8 -#define OP_Next 9 -#define OP_AggStep 10 /* synopsis: accum=r[P3] step(r[P2@P5]) */ -#define OP_Checkpoint 11 -#define OP_JournalMode 12 -#define OP_Vacuum 13 -#define OP_VFilter 14 /* synopsis: iplan=r[P3] zplan='P4' */ -#define OP_VUpdate 15 /* synopsis: data=r[P3@P2] */ -#define OP_Goto 16 -#define OP_Gosub 17 -#define OP_Return 18 +#define OP_Savepoint 1 +#define OP_AutoCommit 2 +#define OP_Transaction 3 +#define OP_SorterNext 4 +#define OP_PrevIfOpen 5 +#define OP_NextIfOpen 6 +#define OP_Prev 7 +#define OP_Next 8 +#define OP_Checkpoint 9 +#define OP_JournalMode 10 +#define OP_Vacuum 11 +#define OP_VFilter 12 /* synopsis: iplan=r[P3] zplan='P4' */ +#define OP_VUpdate 13 /* synopsis: data=r[P3@P2] */ +#define OP_Goto 14 +#define OP_Gosub 15 +#define OP_Return 16 +#define OP_InitCoroutine 17 +#define OP_EndCoroutine 18 #define OP_Not 19 /* same as TK_NOT, synopsis: r[P2]= !r[P1] */ -#define OP_InitCoroutine 20 -#define OP_EndCoroutine 21 -#define OP_Yield 22 -#define OP_HaltIfNull 23 /* synopsis: if r[P3]=null halt */ -#define OP_Halt 24 -#define OP_Integer 25 /* synopsis: r[P2]=P1 */ -#define OP_Int64 26 /* synopsis: r[P2]=P4 */ -#define OP_String 27 /* synopsis: r[P2]='P4' (len=P1) */ -#define OP_Null 28 /* synopsis: r[P2..P3]=NULL */ -#define OP_SoftNull 29 /* synopsis: r[P1]=NULL */ -#define OP_Blob 30 /* synopsis: r[P2]=P4 (len=P1) */ -#define OP_Variable 31 /* synopsis: r[P2]=parameter(P1,P4) */ -#define OP_Move 32 /* synopsis: r[P2@P3]=r[P1@P3] */ -#define OP_Copy 33 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ -#define OP_SCopy 34 /* synopsis: r[P2]=r[P1] */ -#define OP_ResultRow 35 /* synopsis: output=r[P1@P2] */ -#define OP_CollSeq 36 +#define OP_Yield 20 +#define OP_HaltIfNull 21 /* synopsis: if r[P3]=null halt */ +#define OP_Halt 22 +#define OP_Integer 23 /* synopsis: r[P2]=P1 */ +#define OP_Int64 24 /* synopsis: r[P2]=P4 */ +#define OP_String 25 /* synopsis: r[P2]='P4' (len=P1) */ +#define OP_Null 26 /* synopsis: r[P2..P3]=NULL */ +#define OP_SoftNull 27 /* synopsis: r[P1]=NULL */ +#define OP_Blob 28 /* synopsis: r[P2]=P4 (len=P1) */ +#define OP_Variable 29 /* synopsis: r[P2]=parameter(P1,P4) */ +#define OP_Move 30 /* synopsis: r[P2@P3]=r[P1@P3] */ +#define OP_Copy 31 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ +#define OP_SCopy 32 /* synopsis: r[P2]=r[P1] */ +#define OP_ResultRow 33 /* synopsis: output=r[P1@P2] */ +#define OP_CollSeq 34 +#define OP_Function0 35 /* synopsis: r[P3]=func(r[P2@P5]) */ +#define OP_Function 36 /* synopsis: r[P3]=func(r[P2@P5]) */ #define OP_AddImm 37 /* synopsis: r[P1]=r[P1]+P2 */ #define OP_MustBeInt 38 #define OP_RealAffinity 39 @@ -9843,20 +10494,20 @@ typedef struct VdbeOpList VdbeOpList; #define OP_SequenceTest 59 /* synopsis: if( cursor[P1].ctr++ ) pc = P2 */ #define OP_OpenPseudo 60 /* synopsis: P3 columns in r[P2] */ #define OP_Close 61 -#define OP_SeekLT 62 /* synopsis: key=r[P3@P4] */ -#define OP_SeekLE 63 /* synopsis: key=r[P3@P4] */ -#define OP_SeekGE 64 /* synopsis: key=r[P3@P4] */ -#define OP_SeekGT 65 /* synopsis: key=r[P3@P4] */ -#define OP_Seek 66 /* synopsis: intkey=r[P2] */ -#define OP_NoConflict 67 /* synopsis: key=r[P3@P4] */ -#define OP_NotFound 68 /* synopsis: key=r[P3@P4] */ -#define OP_Found 69 /* synopsis: key=r[P3@P4] */ -#define OP_NotExists 70 /* synopsis: intkey=r[P3] */ +#define OP_ColumnsUsed 62 +#define OP_SeekLT 63 /* synopsis: key=r[P3@P4] */ +#define OP_SeekLE 64 /* synopsis: key=r[P3@P4] */ +#define OP_SeekGE 65 /* synopsis: key=r[P3@P4] */ +#define OP_SeekGT 66 /* synopsis: key=r[P3@P4] */ +#define OP_Seek 67 /* synopsis: intkey=r[P2] */ +#define OP_NoConflict 68 /* synopsis: key=r[P3@P4] */ +#define OP_NotFound 69 /* synopsis: key=r[P3@P4] */ +#define OP_Found 70 /* synopsis: key=r[P3@P4] */ #define OP_Or 71 /* same as TK_OR, synopsis: r[P3]=(r[P1] || r[P2]) */ #define OP_And 72 /* same as TK_AND, synopsis: r[P3]=(r[P1] && r[P2]) */ -#define OP_Sequence 73 /* synopsis: r[P2]=cursor[P1].ctr++ */ -#define OP_NewRowid 74 /* synopsis: r[P2]=rowid */ -#define OP_Insert 75 /* synopsis: intkey=r[P3] data=r[P2] */ +#define OP_NotExists 73 /* synopsis: intkey=r[P3] */ +#define OP_Sequence 74 /* synopsis: r[P2]=cursor[P1].ctr++ */ +#define OP_NewRowid 75 /* synopsis: r[P2]=rowid */ #define OP_IsNull 76 /* same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */ #define OP_NotNull 77 /* same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */ #define OP_Ne 78 /* same as TK_NE, synopsis: if r[P1]!=r[P3] goto P2 */ @@ -9865,7 +10516,7 @@ typedef struct VdbeOpList VdbeOpList; #define OP_Le 81 /* same as TK_LE, synopsis: if r[P1]<=r[P3] goto P2 */ #define OP_Lt 82 /* same as TK_LT, synopsis: if r[P1]=r[P3] goto P2 */ -#define OP_InsertInt 84 /* synopsis: intkey=P3 data=r[P2] */ +#define OP_Insert 84 /* synopsis: intkey=r[P3] data=r[P2] */ #define OP_BitAnd 85 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ #define OP_BitOr 86 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ #define OP_ShiftLeft 87 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<0 goto P2 */ -#define OP_IfNeg 138 /* synopsis: r[P1]+=P3, if r[P1]<0 goto P2 */ -#define OP_IfNotZero 139 /* synopsis: if r[P1]!=0 then r[P1]+=P3, goto P2 */ -#define OP_DecrJumpZero 140 /* synopsis: if (--r[P1])==0 goto P2 */ -#define OP_JumpZeroIncr 141 /* synopsis: if (r[P1]++)==0 ) goto P2 */ -#define OP_AggFinal 142 /* synopsis: accum=r[P1] N=P2 */ -#define OP_IncrVacuum 143 -#define OP_Expire 144 -#define OP_TableLock 145 /* synopsis: iDb=P1 root=P2 write=P3 */ -#define OP_VBegin 146 -#define OP_VCreate 147 -#define OP_VDestroy 148 -#define OP_VOpen 149 -#define OP_VColumn 150 /* synopsis: r[P3]=vcolumn(P2) */ -#define OP_VNext 151 -#define OP_VRename 152 -#define OP_Pagecount 153 -#define OP_MaxPgcnt 154 -#define OP_Init 155 /* synopsis: Start at P2 */ -#define OP_Noop 156 -#define OP_Explain 157 +#define OP_Param 134 +#define OP_FkCounter 135 /* synopsis: fkctr[P1]+=P2 */ +#define OP_FkIfZero 136 /* synopsis: if fkctr[P1]==0 goto P2 */ +#define OP_MemMax 137 /* synopsis: r[P1]=max(r[P1],r[P2]) */ +#define OP_IfPos 138 /* synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ +#define OP_SetIfNotPos 139 /* synopsis: if r[P1]<=0 then r[P2]=P3 */ +#define OP_IfNotZero 140 /* synopsis: if r[P1]!=0 then r[P1]-=P3, goto P2 */ +#define OP_DecrJumpZero 141 /* synopsis: if (--r[P1])==0 goto P2 */ +#define OP_JumpZeroIncr 142 /* synopsis: if (r[P1]++)==0 ) goto P2 */ +#define OP_AggStep0 143 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggStep 144 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggFinal 145 /* synopsis: accum=r[P1] N=P2 */ +#define OP_IncrVacuum 146 +#define OP_Expire 147 +#define OP_TableLock 148 /* synopsis: iDb=P1 root=P2 write=P3 */ +#define OP_VBegin 149 +#define OP_VCreate 150 +#define OP_VDestroy 151 +#define OP_VOpen 152 +#define OP_VColumn 153 /* synopsis: r[P3]=vcolumn(P2) */ +#define OP_VNext 154 +#define OP_VRename 155 +#define OP_Pagecount 156 +#define OP_MaxPgcnt 157 +#define OP_Init 158 /* synopsis: Start at P2 */ +#define OP_Noop 159 +#define OP_Explain 160 /* Properties such as "out2" or "jump" that are specified in @@ -9952,26 +10606,27 @@ typedef struct VdbeOpList VdbeOpList; #define OPFLG_OUT2 0x0010 /* out2: P2 is an output */ #define OPFLG_OUT3 0x0020 /* out3: P3 is an output */ #define OPFLG_INITIALIZER {\ -/* 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01,\ -/* 8 */ 0x01, 0x01, 0x00, 0x00, 0x10, 0x00, 0x01, 0x00,\ -/* 16 */ 0x01, 0x01, 0x02, 0x12, 0x01, 0x02, 0x03, 0x08,\ -/* 24 */ 0x00, 0x10, 0x10, 0x10, 0x10, 0x00, 0x10, 0x10,\ -/* 32 */ 0x00, 0x00, 0x10, 0x00, 0x00, 0x02, 0x03, 0x02,\ +/* 0 */ 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01,\ +/* 8 */ 0x01, 0x00, 0x10, 0x00, 0x01, 0x00, 0x01, 0x01,\ +/* 16 */ 0x02, 0x01, 0x02, 0x12, 0x03, 0x08, 0x00, 0x10,\ +/* 24 */ 0x10, 0x10, 0x10, 0x00, 0x10, 0x10, 0x00, 0x00,\ +/* 32 */ 0x10, 0x00, 0x00, 0x00, 0x00, 0x02, 0x03, 0x02,\ /* 40 */ 0x02, 0x00, 0x00, 0x01, 0x01, 0x03, 0x03, 0x00,\ /* 48 */ 0x00, 0x00, 0x10, 0x10, 0x08, 0x00, 0x00, 0x00,\ -/* 56 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x09,\ -/* 64 */ 0x09, 0x09, 0x04, 0x09, 0x09, 0x09, 0x09, 0x26,\ -/* 72 */ 0x26, 0x10, 0x10, 0x00, 0x03, 0x03, 0x0b, 0x0b,\ +/* 56 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09,\ +/* 64 */ 0x09, 0x09, 0x09, 0x04, 0x09, 0x09, 0x09, 0x26,\ +/* 72 */ 0x26, 0x09, 0x10, 0x10, 0x03, 0x03, 0x0b, 0x0b,\ /* 80 */ 0x0b, 0x0b, 0x0b, 0x0b, 0x00, 0x26, 0x26, 0x26,\ /* 88 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x00,\ -/* 96 */ 0x12, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10,\ -/* 104 */ 0x00, 0x01, 0x01, 0x01, 0x01, 0x04, 0x04, 0x00,\ -/* 112 */ 0x10, 0x01, 0x01, 0x01, 0x01, 0x10, 0x00, 0x00,\ -/* 120 */ 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 128 */ 0x06, 0x23, 0x0b, 0x01, 0x10, 0x10, 0x00, 0x01,\ -/* 136 */ 0x04, 0x03, 0x03, 0x03, 0x03, 0x03, 0x00, 0x01,\ -/* 144 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,\ -/* 152 */ 0x00, 0x10, 0x10, 0x01, 0x00, 0x00,} +/* 96 */ 0x12, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 104 */ 0x10, 0x00, 0x01, 0x01, 0x01, 0x01, 0x04, 0x04,\ +/* 112 */ 0x00, 0x10, 0x01, 0x01, 0x01, 0x01, 0x10, 0x00,\ +/* 120 */ 0x00, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 128 */ 0x00, 0x06, 0x23, 0x0b, 0x01, 0x10, 0x10, 0x00,\ +/* 136 */ 0x01, 0x04, 0x03, 0x06, 0x03, 0x03, 0x03, 0x00,\ +/* 144 */ 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 152 */ 0x00, 0x00, 0x01, 0x00, 0x10, 0x10, 0x01, 0x00,\ +/* 160 */ 0x00,} /************** End of opcodes.h *********************************************/ /************** Continuing where we left off in vdbe.h ***********************/ @@ -9984,11 +10639,16 @@ SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse*); SQLITE_PRIVATE int sqlite3VdbeAddOp0(Vdbe*,int); SQLITE_PRIVATE int sqlite3VdbeAddOp1(Vdbe*,int,int); SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe*,int,int,int); +SQLITE_PRIVATE int sqlite3VdbeGoto(Vdbe*,int); +SQLITE_PRIVATE int sqlite3VdbeLoadString(Vdbe*,int,const char*); +SQLITE_PRIVATE void sqlite3VdbeMultiLoad(Vdbe*,int,const char*,...); SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int); SQLITE_PRIVATE int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int); +SQLITE_PRIVATE int sqlite3VdbeAddOp4Dup8(Vdbe*,int,int,int,int,const u8*,int); SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int); SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp, int iLineno); SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*); +SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe*, u32 addr, u8); SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, u32 addr, int P1); SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, u32 addr, int P2); SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, u32 addr, int P3); @@ -10232,6 +10892,9 @@ SQLITE_PRIVATE int sqlite3PagerReadFileheader(Pager*, int, unsigned char*); /* Functions used to configure a Pager object. */ SQLITE_PRIVATE void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager*, u32*, int); +#ifdef SQLITE_HAS_CODEC +SQLITE_PRIVATE void sqlite3PagerAlignReserve(Pager*,Pager*); +#endif SQLITE_PRIVATE int sqlite3PagerMaxPageCount(Pager*, int); SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager*, int); SQLITE_PRIVATE void sqlite3PagerSetMmapLimit(Pager *, sqlite3_int64); @@ -10287,7 +10950,9 @@ SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager); /* Functions used to query pager state and configuration. */ SQLITE_PRIVATE u8 sqlite3PagerIsreadonly(Pager*); SQLITE_PRIVATE u32 sqlite3PagerDataVersion(Pager*); -SQLITE_PRIVATE int sqlite3PagerRefcount(Pager*); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3PagerRefcount(Pager*); +#endif SQLITE_PRIVATE int sqlite3PagerMemUsed(Pager*); SQLITE_PRIVATE const char *sqlite3PagerFilename(Pager*, int); SQLITE_PRIVATE const sqlite3_vfs *sqlite3PagerVfs(Pager*); @@ -10378,14 +11043,14 @@ struct PgHdr { }; /* Bit values for PgHdr.flags */ -#define PGHDR_DIRTY 0x002 /* Page has changed */ -#define PGHDR_NEED_SYNC 0x004 /* Fsync the rollback journal before - ** writing this page to the database */ -#define PGHDR_NEED_READ 0x008 /* Content is unread */ -#define PGHDR_REUSE_UNLIKELY 0x010 /* A hint that reuse is unlikely */ -#define PGHDR_DONT_WRITE 0x020 /* Do not write content to disk */ - -#define PGHDR_MMAP 0x040 /* This is an mmap page object */ +#define PGHDR_CLEAN 0x001 /* Page not on the PCache.pDirty list */ +#define PGHDR_DIRTY 0x002 /* Page is on the PCache.pDirty list */ +#define PGHDR_WRITEABLE 0x004 /* Journaled and ready to modify */ +#define PGHDR_NEED_SYNC 0x008 /* Fsync the rollback journal before + ** writing this page to the database */ +#define PGHDR_NEED_READ 0x010 /* Content is unread */ +#define PGHDR_DONT_WRITE 0x020 /* Do not write content to disk */ +#define PGHDR_MMAP 0x040 /* This is an mmap page object */ /* Initialize and shutdown the page cache subsystem */ SQLITE_PRIVATE int sqlite3PcacheInitialize(void); @@ -11169,6 +11834,7 @@ struct sqlite3 { #define SQLITE_QueryOnly 0x02000000 /* Disable database changes */ #define SQLITE_VdbeEQP 0x04000000 /* Debug EXPLAIN QUERY PLAN */ #define SQLITE_Vacuum 0x08000000 /* Currently in a VACUUM */ +#define SQLITE_CellSizeCk 0x10000000 /* Check btree cell sizes on load */ /* @@ -11263,18 +11929,20 @@ struct FuncDestructor { ** values must correspond to OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG. There ** are assert() statements in the code to verify this. */ -#define SQLITE_FUNC_ENCMASK 0x003 /* SQLITE_UTF8, SQLITE_UTF16BE or UTF16LE */ -#define SQLITE_FUNC_LIKE 0x004 /* Candidate for the LIKE optimization */ -#define SQLITE_FUNC_CASE 0x008 /* Case-sensitive LIKE-type function */ -#define SQLITE_FUNC_EPHEM 0x010 /* Ephemeral. Delete with VDBE */ -#define SQLITE_FUNC_NEEDCOLL 0x020 /* sqlite3GetFuncCollSeq() might be called */ -#define SQLITE_FUNC_LENGTH 0x040 /* Built-in length() function */ -#define SQLITE_FUNC_TYPEOF 0x080 /* Built-in typeof() function */ -#define SQLITE_FUNC_COUNT 0x100 /* Built-in count(*) aggregate */ -#define SQLITE_FUNC_COALESCE 0x200 /* Built-in coalesce() or ifnull() */ -#define SQLITE_FUNC_UNLIKELY 0x400 /* Built-in unlikely() function */ -#define SQLITE_FUNC_CONSTANT 0x800 /* Constant inputs give a constant output */ -#define SQLITE_FUNC_MINMAX 0x1000 /* True for min() and max() aggregates */ +#define SQLITE_FUNC_ENCMASK 0x0003 /* SQLITE_UTF8, SQLITE_UTF16BE or UTF16LE */ +#define SQLITE_FUNC_LIKE 0x0004 /* Candidate for the LIKE optimization */ +#define SQLITE_FUNC_CASE 0x0008 /* Case-sensitive LIKE-type function */ +#define SQLITE_FUNC_EPHEM 0x0010 /* Ephemeral. Delete with VDBE */ +#define SQLITE_FUNC_NEEDCOLL 0x0020 /* sqlite3GetFuncCollSeq() might be called*/ +#define SQLITE_FUNC_LENGTH 0x0040 /* Built-in length() function */ +#define SQLITE_FUNC_TYPEOF 0x0080 /* Built-in typeof() function */ +#define SQLITE_FUNC_COUNT 0x0100 /* Built-in count(*) aggregate */ +#define SQLITE_FUNC_COALESCE 0x0200 /* Built-in coalesce() or ifnull() */ +#define SQLITE_FUNC_UNLIKELY 0x0400 /* Built-in unlikely() function */ +#define SQLITE_FUNC_CONSTANT 0x0800 /* Constant inputs give a constant output */ +#define SQLITE_FUNC_MINMAX 0x1000 /* True for min() and max() aggregates */ +#define SQLITE_FUNC_SLOCHNG 0x2000 /* "Slow Change". Value constant during a + ** single query - might change over time */ /* ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are @@ -11290,6 +11958,12 @@ struct FuncDestructor { ** VFUNCTION(zName, nArg, iArg, bNC, xFunc) ** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag. ** +** DFUNCTION(zName, nArg, iArg, bNC, xFunc) +** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag and +** adds the SQLITE_FUNC_SLOCHNG flag. Used for date & time functions +** and functions like sqlite_version() that can change, but not during +** a single query. +** ** AGGREGATE(zName, nArg, iArg, bNC, xStep, xFinal) ** Used to create an aggregate function definition implemented by ** the C functions xStep and xFinal. The first four parameters @@ -11310,11 +11984,14 @@ struct FuncDestructor { #define VFUNCTION(zName, nArg, iArg, bNC, xFunc) \ {nArg, SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} +#define DFUNCTION(zName, nArg, iArg, bNC, xFunc) \ + {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} #define FUNCTION2(zName, nArg, iArg, bNC, xFunc, extraFlags) \ {nArg,SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags,\ SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ - {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ + {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ pArg, 0, xFunc, 0, 0, #zName, 0, 0} #define LIKEFUNC(zName, nArg, arg, flags) \ {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|flags, \ @@ -11358,6 +12035,7 @@ struct Module { const char *zName; /* Name passed to create_module() */ void *pAux; /* pAux passed to create_module() */ void (*xDestroy)(void *); /* Module destructor function */ + Table *pEpoTab; /* Eponymous table for this module */ }; /* @@ -11403,6 +12081,7 @@ struct CollSeq { */ #define SQLITE_SO_ASC 0 /* Sort in ascending order */ #define SQLITE_SO_DESC 1 /* Sort in ascending order */ +#define SQLITE_SO_UNDEFINED -1 /* No sort order specified */ /* ** Column affinity types. @@ -11416,9 +12095,9 @@ struct CollSeq { ** used as the P4 operand, they will be more readable. ** ** Note also that the numeric types are grouped together so that testing -** for a numeric type is a single comparison. And the NONE type is first. +** for a numeric type is a single comparison. And the BLOB type is first. */ -#define SQLITE_AFF_NONE 'A' +#define SQLITE_AFF_BLOB 'A' #define SQLITE_AFF_TEXT 'B' #define SQLITE_AFF_NUMERIC 'C' #define SQLITE_AFF_INTEGER 'D' @@ -11509,9 +12188,8 @@ struct Table { Select *pSelect; /* NULL for tables. Points to definition if a view. */ FKey *pFKey; /* Linked list of all foreign keys in this table */ char *zColAff; /* String defining the affinity of each column */ -#ifndef SQLITE_OMIT_CHECK ExprList *pCheck; /* All CHECK constraints */ -#endif + /* ... also used as column name list in a VIEW */ int tnum; /* Root BTree page for this table */ i16 iPKey; /* If not negative, use aCol[iPKey] as the rowid */ i16 nCol; /* Number of columns in this table */ @@ -11528,7 +12206,7 @@ struct Table { #endif #ifndef SQLITE_OMIT_VIRTUALTABLE int nModuleArg; /* Number of arguments to the module */ - char **azModuleArg; /* Text of all module args. [0] is module name */ + char **azModuleArg; /* 0: module 1: schema 2: vtab name 3...: args */ VTable *pVTable; /* List of VTable objects. */ #endif Trigger *pTrigger; /* List of triggers stored in pSchema */ @@ -11550,8 +12228,9 @@ struct Table { #define TF_HasPrimaryKey 0x04 /* Table has a primary key */ #define TF_Autoincrement 0x08 /* Integer primary key is autoincrement */ #define TF_Virtual 0x10 /* Is a virtual table */ -#define TF_WithoutRowid 0x20 /* No rowid used. PRIMARY KEY is the key */ -#define TF_OOOHidden 0x40 /* Out-of-Order hidden columns */ +#define TF_WithoutRowid 0x20 /* No rowid. PRIMARY KEY is the key */ +#define TF_NoVisibleRowid 0x40 /* No user-visible "rowid" column */ +#define TF_OOOHidden 0x80 /* Out-of-Order hidden columns */ /* @@ -11569,6 +12248,7 @@ struct Table { /* Does the table have a rowid */ #define HasRowid(X) (((X)->tabFlags & TF_WithoutRowid)==0) +#define VisibleRowid(X) (((X)->tabFlags & TF_NoVisibleRowid)==0) /* ** Each foreign key constraint is an instance of the following structure. @@ -11727,6 +12407,14 @@ struct UnpackedRecord { ** and the value of Index.onError indicate the which conflict resolution ** algorithm to employ whenever an attempt is made to insert a non-unique ** element. +** +** While parsing a CREATE TABLE or CREATE INDEX statement in order to +** generate VDBE code (as opposed to parsing one read from an sqlite_master +** table as part of parsing an existing database schema), transient instances +** of this structure may be created. In this case the Index.tnum variable is +** used to store the address of a VDBE instruction, not a database page +** number (it cannot - the database page is not allocated until the VDBE +** program is executed). See convertToWithoutRowidTable() for details. */ struct Index { char *zName; /* Name of this index */ @@ -11739,6 +12427,7 @@ struct Index { u8 *aSortOrder; /* for each column: True==DESC, False==ASC */ char **azColl; /* Array of collation sequence names for index */ Expr *pPartIdxWhere; /* WHERE clause for partial indices */ + ExprList *aColExpr; /* Column expressions */ int tnum; /* DB Page containing root of this index */ LogEst szIdxRow; /* Estimated average row size in bytes */ u16 nKeyCol; /* Number of columns forming the key */ @@ -11773,6 +12462,12 @@ struct Index { /* Return true if index X is a UNIQUE index */ #define IsUniqueIndex(X) ((X)->onError!=OE_None) +/* The Index.aiColumn[] values are normally positive integer. But +** there are some negative values that have special meaning: +*/ +#define XN_ROWID (-1) /* Indexed column is the rowid */ +#define XN_EXPR (-2) /* Indexed column is an expression */ + /* ** Each sample stored in the sqlite_stat3 table is represented in memory ** using a structure of this type. See documentation at the top of the @@ -11988,9 +12683,10 @@ struct Expr { #define EP_MemToken 0x010000 /* Need to sqlite3DbFree() Expr.zToken */ #define EP_NoReduce 0x020000 /* Cannot EXPRDUP_REDUCE this Expr */ #define EP_Unlikely 0x040000 /* unlikely() or likelihood() function */ -#define EP_ConstFunc 0x080000 /* Node is a SQLITE_FUNC_CONSTANT function */ +#define EP_ConstFunc 0x080000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */ #define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */ #define EP_Subquery 0x200000 /* Tree contains a TK_SELECT operator */ +#define EP_Alias 0x400000 /* Is an alias for a result set column */ /* ** Combinations of two or more EP_* flags @@ -12153,11 +12849,15 @@ struct SrcList { int addrFillSub; /* Address of subroutine to manifest a subquery */ int regReturn; /* Register holding return address of addrFillSub */ int regResult; /* Registers holding results of a co-routine */ - u8 jointype; /* Type of join between this able and the previous */ - unsigned notIndexed :1; /* True if there is a NOT INDEXED clause */ - unsigned isCorrelated :1; /* True if sub-query is correlated */ - unsigned viaCoroutine :1; /* Implemented as a co-routine */ - unsigned isRecursive :1; /* True for recursive reference in WITH */ + struct { + u8 jointype; /* Type of join between this able and the previous */ + unsigned notIndexed :1; /* True if there is a NOT INDEXED clause */ + unsigned isIndexedBy :1; /* True if there is an INDEXED BY clause */ + unsigned isTabFunc :1; /* True if table-valued-function syntax */ + unsigned isCorrelated :1; /* True if sub-query is correlated */ + unsigned viaCoroutine :1; /* Implemented as a co-routine */ + unsigned isRecursive :1; /* True for recursive reference in WITH */ + } fg; #ifndef SQLITE_OMIT_EXPLAIN u8 iSelectId; /* If pSelect!=0, the id of the sub-select in EQP */ #endif @@ -12165,8 +12865,11 @@ struct SrcList { Expr *pOn; /* The ON clause of a join */ IdList *pUsing; /* The USING clause of a join */ Bitmask colUsed; /* Bit N (1<" clause */ - Index *pIndex; /* Index structure corresponding to zIndex, if any */ + union { + char *zIndexedBy; /* Identifier from "INDEXED BY " clause */ + ExprList *pFuncArg; /* Arguments to table-valued-function */ + } u1; + Index *pIBIndex; /* Index structure corresponding to u1.zIndexedBy */ } a[1]; /* One entry for each identifier on the list */ }; @@ -12200,6 +12903,7 @@ struct SrcList { #define WHERE_WANT_DISTINCT 0x0400 /* All output needs to be distinct */ #define WHERE_SORTBYGROUP 0x0800 /* Support sqlite3WhereIsSorted() */ #define WHERE_REOPEN_IDX 0x1000 /* Try to use OP_ReopenIdx */ +#define WHERE_ONEPASS_MULTIROW 0x2000 /* ONEPASS is ok with multiple rows */ /* Allowed return values from sqlite3WhereIsDistinct() */ @@ -12252,6 +12956,7 @@ struct NameContext { #define NC_IsCheck 0x0004 /* True if resolving names in a CHECK constraint */ #define NC_InAggFunc 0x0008 /* True if analyzing arguments to an agg func */ #define NC_PartIdx 0x0010 /* True if resolving a partial index WHERE */ +#define NC_IdxExpr 0x0020 /* True if resolving columns of CREATE INDEX */ #define NC_MinMaxAgg 0x1000 /* min/max aggregates seen. See note above */ /* @@ -12301,19 +13006,20 @@ struct Select { ** "Select Flag". */ #define SF_Distinct 0x0001 /* Output should be DISTINCT */ -#define SF_Resolved 0x0002 /* Identifiers have been resolved */ -#define SF_Aggregate 0x0004 /* Contains aggregate functions */ -#define SF_UsesEphemeral 0x0008 /* Uses the OpenEphemeral opcode */ -#define SF_Expanded 0x0010 /* sqlite3SelectExpand() called on this */ -#define SF_HasTypeInfo 0x0020 /* FROM subqueries have Table metadata */ -#define SF_Compound 0x0040 /* Part of a compound query */ -#define SF_Values 0x0080 /* Synthesized from VALUES clause */ -#define SF_MultiValue 0x0100 /* Single VALUES term with multiple rows */ -#define SF_NestedFrom 0x0200 /* Part of a parenthesized FROM clause */ -#define SF_MaybeConvert 0x0400 /* Need convertCompoundSelectToSubquery() */ -#define SF_Recursive 0x0800 /* The recursive part of a recursive CTE */ +#define SF_All 0x0002 /* Includes the ALL keyword */ +#define SF_Resolved 0x0004 /* Identifiers have been resolved */ +#define SF_Aggregate 0x0008 /* Contains aggregate functions */ +#define SF_UsesEphemeral 0x0010 /* Uses the OpenEphemeral opcode */ +#define SF_Expanded 0x0020 /* sqlite3SelectExpand() called on this */ +#define SF_HasTypeInfo 0x0040 /* FROM subqueries have Table metadata */ +#define SF_Compound 0x0080 /* Part of a compound query */ +#define SF_Values 0x0100 /* Synthesized from VALUES clause */ +#define SF_MultiValue 0x0200 /* Single VALUES term with multiple rows */ +#define SF_NestedFrom 0x0400 /* Part of a parenthesized FROM clause */ +#define SF_MaybeConvert 0x0800 /* Need convertCompoundSelectToSubquery() */ #define SF_MinMaxAgg 0x1000 /* Aggregate containing min() or max() */ -#define SF_Converted 0x2000 /* By convertCompoundSelectToSubquery() */ +#define SF_Recursive 0x2000 /* The recursive part of a recursive CTE */ +#define SF_Converted 0x4000 /* By convertCompoundSelectToSubquery() */ /* @@ -12520,7 +13226,7 @@ struct Parse { int nOpAlloc; /* Number of slots allocated for Vdbe.aOp[] */ int iFixedOp; /* Never back out opcodes iFixedOp-1 or earlier */ int ckBase; /* Base register of data during check constraints */ - int iPartIdxTab; /* Table corresponding to a partial index */ + int iSelfTab; /* Table of an index whose exprs are being coded */ int iCacheLevel; /* ColCache valid when aColCache[].iLevel<=iCacheLevel */ int iCacheCnt; /* Counter used to generate aColCache[].lru values */ int nLabel; /* Number of labels used */ @@ -12555,7 +13261,6 @@ struct Parse { Parse *pToplevel; /* Parse structure for main program (or NULL) */ Table *pTriggerTab; /* Table triggers are being coded for */ int addrCrTab; /* Address of OP_CreateTable opcode on CREATE TABLE */ - int addrSkipPK; /* Address of instruction to skip PRIMARY KEY index */ u32 nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ u32 oldmask; /* Mask of old.* columns referenced */ u32 newmask; /* Mask of new.* columns referenced */ @@ -12891,7 +13596,7 @@ struct With { char *zName; /* Name of this CTE */ ExprList *pCols; /* List of explicit column names, or NULL */ Select *pSelect; /* The definition of this CTE */ - const char *zErr; /* Error message for circular references */ + const char *zCteErr; /* Error message for circular references */ } a[1]; }; @@ -12971,7 +13676,9 @@ SQLITE_PRIVATE int sqlite3CantopenError(int); # define sqlite3Isxdigit(x) isxdigit((unsigned char)(x)) # define sqlite3Tolower(x) tolower((unsigned char)(x)) #endif +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS SQLITE_PRIVATE int sqlite3IsIdChar(u8); +#endif /* ** Internal function prototypes @@ -12999,7 +13706,9 @@ SQLITE_PRIVATE void sqlite3ScratchFree(void*); SQLITE_PRIVATE void *sqlite3PageMalloc(int); SQLITE_PRIVATE void sqlite3PageFree(void*); SQLITE_PRIVATE void sqlite3MemSetDefault(void); +#ifndef SQLITE_OMIT_BUILTIN_TEST SQLITE_PRIVATE void sqlite3BenignMallocHooks(void (*)(void), void (*)(void)); +#endif SQLITE_PRIVATE int sqlite3HeapNearlyFull(void); /* @@ -13035,6 +13744,11 @@ SQLITE_PRIVATE sqlite3_mutex *sqlite3MutexAlloc(int); SQLITE_PRIVATE int sqlite3MutexInit(void); SQLITE_PRIVATE int sqlite3MutexEnd(void); #endif +#if !defined(SQLITE_MUTEX_OMIT) && !defined(SQLITE_MUTEX_NOOP) +SQLITE_PRIVATE void sqlite3MemoryBarrier(void); +#else +# define sqlite3MemoryBarrier() +#endif SQLITE_PRIVATE sqlite3_int64 sqlite3StatusValue(int); SQLITE_PRIVATE void sqlite3StatusUp(int, int); @@ -13067,7 +13781,6 @@ SQLITE_PRIVATE void sqlite3VXPrintf(StrAccum*, u32, const char*, va_list); SQLITE_PRIVATE void sqlite3XPrintf(StrAccum*, u32, const char*, ...); SQLITE_PRIVATE char *sqlite3MPrintf(sqlite3*,const char*, ...); SQLITE_PRIVATE char *sqlite3VMPrintf(sqlite3*,const char*, va_list); -SQLITE_PRIVATE char *sqlite3MAppendf(sqlite3*,char*,const char*,...); #if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) SQLITE_PRIVATE void sqlite3DebugPrintf(const char*, ...); #endif @@ -13076,17 +13789,13 @@ SQLITE_PRIVATE void *sqlite3TestTextToPtr(const char*); #endif #if defined(SQLITE_DEBUG) -SQLITE_PRIVATE TreeView *sqlite3TreeViewPush(TreeView*,u8); -SQLITE_PRIVATE void sqlite3TreeViewPop(TreeView*); -SQLITE_PRIVATE void sqlite3TreeViewLine(TreeView*, const char*, ...); -SQLITE_PRIVATE void sqlite3TreeViewItem(TreeView*, const char*, u8); SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView*, const Expr*, u8); SQLITE_PRIVATE void sqlite3TreeViewExprList(TreeView*, const ExprList*, u8, const char*); SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView*, const Select*, u8); #endif -SQLITE_PRIVATE void sqlite3SetString(char **, sqlite3*, const char*, ...); +SQLITE_PRIVATE void sqlite3SetString(char **, sqlite3*, const char*); SQLITE_PRIVATE void sqlite3ErrorMsg(Parse*, const char*, ...); SQLITE_PRIVATE int sqlite3Dequote(char*); SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char*, int); @@ -13106,6 +13815,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse*,ExprList*, Token*); SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse*, Expr*); SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3*, Expr*); SQLITE_PRIVATE ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*); +SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList*,int); SQLITE_PRIVATE void sqlite3ExprListSetName(Parse*,ExprList*,Token*,int); SQLITE_PRIVATE void sqlite3ExprListSetSpan(Parse*,ExprList*,ExprSpan*); SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3*, ExprList*); @@ -13118,6 +13828,8 @@ SQLITE_PRIVATE void sqlite3ResetOneSchema(sqlite3*,int); SQLITE_PRIVATE void sqlite3CollapseDatabaseArray(sqlite3*); SQLITE_PRIVATE void sqlite3BeginParse(Parse*,int); SQLITE_PRIVATE void sqlite3CommitInternalChanges(sqlite3*); +SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3*,Table*); +SQLITE_PRIVATE int sqlite3ColumnsFromExprList(Parse*,ExprList*,i16*,Column**); SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse*,Select*); SQLITE_PRIVATE void sqlite3OpenMasterTable(Parse *, int); SQLITE_PRIVATE Index *sqlite3PrimaryKeyIndex(Table*); @@ -13144,11 +13856,14 @@ SQLITE_PRIVATE int sqlite3FaultSim(int); SQLITE_PRIVATE Bitvec *sqlite3BitvecCreate(u32); SQLITE_PRIVATE int sqlite3BitvecTest(Bitvec*, u32); +SQLITE_PRIVATE int sqlite3BitvecTestNotNull(Bitvec*, u32); SQLITE_PRIVATE int sqlite3BitvecSet(Bitvec*, u32); SQLITE_PRIVATE void sqlite3BitvecClear(Bitvec*, u32, void*); SQLITE_PRIVATE void sqlite3BitvecDestroy(Bitvec*); SQLITE_PRIVATE u32 sqlite3BitvecSize(Bitvec*); +#ifndef SQLITE_OMIT_BUILTIN_TEST SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int,int*); +#endif SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3*, void*, unsigned int); SQLITE_PRIVATE void sqlite3RowSetClear(RowSet*); @@ -13156,7 +13871,7 @@ SQLITE_PRIVATE void sqlite3RowSetInsert(RowSet*, i64); SQLITE_PRIVATE int sqlite3RowSetTest(RowSet*, int iBatch, i64); SQLITE_PRIVATE int sqlite3RowSetNext(RowSet*, i64*); -SQLITE_PRIVATE void sqlite3CreateView(Parse*,Token*,Token*,Token*,Select*,int,int); +SQLITE_PRIVATE void sqlite3CreateView(Parse*,Token*,Token*,Token*,ExprList*,Select*,int,int); #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse*,Table*); @@ -13186,6 +13901,7 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListAppend(sqlite3*, SrcList*, Token*, Token*) SQLITE_PRIVATE SrcList *sqlite3SrcListAppendFromTerm(Parse*, SrcList*, Token*, Token*, Token*, Select*, Expr*, IdList*); SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *, SrcList *, Token *); +SQLITE_PRIVATE void sqlite3SrcListFuncArgs(Parse*, SrcList*, ExprList*); SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *, struct SrcList_item *); SQLITE_PRIVATE void sqlite3SrcListShiftJoinType(SrcList*); SQLITE_PRIVATE void sqlite3SrcListAssignCursors(Parse*, SrcList*); @@ -13216,6 +13932,10 @@ SQLITE_PRIVATE int sqlite3WhereIsSorted(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereContinueLabel(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereBreakLabel(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereOkOnePass(WhereInfo*, int*); +#define ONEPASS_OFF 0 /* Use of ONEPASS not allowed */ +#define ONEPASS_SINGLE 1 /* ONEPASS valid for a single row update */ +#define ONEPASS_MULTI 2 /* ONEPASS is valid for multiple rows */ +SQLITE_PRIVATE void sqlite3ExprCodeLoadIndexColumn(Parse*, Index*, int, int, int); SQLITE_PRIVATE int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8); SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse*, int, int, int); @@ -13231,11 +13951,13 @@ SQLITE_PRIVATE void sqlite3ExprCodeAtInit(Parse*, Expr*, int, u8); SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse*, Expr*, int*); SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse*, Expr*, int); SQLITE_PRIVATE void sqlite3ExprCodeAndCache(Parse*, Expr*, int); -SQLITE_PRIVATE int sqlite3ExprCodeExprList(Parse*, ExprList*, int, u8); +SQLITE_PRIVATE int sqlite3ExprCodeExprList(Parse*, ExprList*, int, int, u8); #define SQLITE_ECEL_DUP 0x01 /* Deep, not shallow copies */ #define SQLITE_ECEL_FACTOR 0x02 /* Factor out constant terms */ +#define SQLITE_ECEL_REF 0x04 /* Use ExprList.u.x.iOrderByCol */ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse*, Expr*, int, int); SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse*, Expr*, int, int); +SQLITE_PRIVATE void sqlite3ExprIfFalseDup(Parse*, Expr*, int, int); SQLITE_PRIVATE Table *sqlite3FindTable(sqlite3*,const char*, const char*); SQLITE_PRIVATE Table *sqlite3LocateTable(Parse*,int isView,const char*, const char*); SQLITE_PRIVATE Table *sqlite3LocateTableItem(Parse*,int isView,struct SrcList_item *); @@ -13252,8 +13974,10 @@ SQLITE_PRIVATE void sqlite3ExprAnalyzeAggregates(NameContext*, Expr*); SQLITE_PRIVATE void sqlite3ExprAnalyzeAggList(NameContext*,ExprList*); SQLITE_PRIVATE int sqlite3FunctionUsesThisSrc(Expr*, SrcList*); SQLITE_PRIVATE Vdbe *sqlite3GetVdbe(Parse*); +#ifndef SQLITE_OMIT_BUILTIN_TEST SQLITE_PRIVATE void sqlite3PrngSaveState(void); SQLITE_PRIVATE void sqlite3PrngRestoreState(void); +#endif SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3*,int); SQLITE_PRIVATE void sqlite3CodeVerifySchema(Parse*, int); SQLITE_PRIVATE void sqlite3CodeVerifyNamedSchema(Parse*, const char *zDb); @@ -13271,8 +13995,9 @@ SQLITE_PRIVATE int sqlite3ExprIsInteger(Expr*, int*); SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr*); SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); SQLITE_PRIVATE int sqlite3IsRowid(const char*); -SQLITE_PRIVATE void sqlite3GenerateRowDelete(Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8); -SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*); +SQLITE_PRIVATE void sqlite3GenerateRowDelete( + Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8,int); +SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int); SQLITE_PRIVATE int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int); SQLITE_PRIVATE void sqlite3ResolvePartIdxLabel(Parse*,int); SQLITE_PRIVATE void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int, @@ -13330,6 +14055,7 @@ SQLITE_PRIVATE void sqlite3DeleteTrigger(sqlite3*, Trigger*); SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTrigger(sqlite3*,int,const char*); SQLITE_PRIVATE u32 sqlite3TriggerColmask(Parse*,Trigger*,ExprList*,int,int,Table*,int); # define sqlite3ParseToplevel(p) ((p)->pToplevel ? (p)->pToplevel : (p)) +# define sqlite3IsToplevel(p) ((p)->pToplevel==0) #else # define sqlite3TriggersExist(B,C,D,E,F) 0 # define sqlite3DeleteTrigger(A,B) @@ -13339,6 +14065,7 @@ SQLITE_PRIVATE u32 sqlite3TriggerColmask(Parse*,Trigger*,ExprList*,int,int,Tab # define sqlite3CodeRowTriggerDirect(A,B,C,D,E,F) # define sqlite3TriggerList(X, Y) 0 # define sqlite3ParseToplevel(p) p +# define sqlite3IsToplevel(p) 1 # define sqlite3TriggerColmask(A,B,C,D,E,F,G) 0 #endif @@ -13402,7 +14129,7 @@ SQLITE_PRIVATE int sqlite3VarintLen(u64 v); #define putVarint sqlite3PutVarint -SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *, Index *); +SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3*, Index*); SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe*, Table*, int); SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2); SQLITE_PRIVATE int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity); @@ -13471,8 +14198,10 @@ SQLITE_PRIVATE void sqlite3NestedParse(Parse*, const char*, ...); SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3*); SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *, Expr *, int, int); SQLITE_PRIVATE void sqlite3SelectPrep(Parse*, Select*, NameContext*); +SQLITE_PRIVATE void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p); SQLITE_PRIVATE int sqlite3MatchSpanName(const char*, const char*, const char*, const char*); SQLITE_PRIVATE int sqlite3ResolveExprNames(NameContext*, Expr*); +SQLITE_PRIVATE int sqlite3ResolveExprListNames(NameContext*, ExprList*); SQLITE_PRIVATE void sqlite3ResolveSelectNames(Parse*, Select*, NameContext*); SQLITE_PRIVATE void sqlite3ResolveSelfReference(Parse*,Table*,int,Expr*,ExprList*); SQLITE_PRIVATE int sqlite3ResolveOrderGroupBy(Parse*, Select*, ExprList*, const char*); @@ -13581,6 +14310,8 @@ SQLITE_PRIVATE void sqlite3VtabImportErrmsg(Vdbe*, sqlite3_vtab*); SQLITE_PRIVATE VTable *sqlite3GetVTable(sqlite3*, Table*); # define sqlite3VtabInSync(db) ((db)->nVTrans>0 && (db)->aVTrans==0) #endif +SQLITE_PRIVATE int sqlite3VtabEponymousTableInit(Parse*,Module*); +SQLITE_PRIVATE void sqlite3VtabEponymousTableClear(sqlite3*,Module*); SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse*,Table*); SQLITE_PRIVATE void sqlite3VtabBeginParse(Parse*, Token*, Token*, Token*, int); SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse*, Token*); @@ -13783,6 +14514,10 @@ SQLITE_PRIVATE int sqlite3ThreadCreate(SQLiteThread**,void*(*)(void*),void*); SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread*, void**); #endif +#if defined(SQLITE_ENABLE_DBSTAT_VTAB) || defined(SQLITE_TEST) +SQLITE_PRIVATE int sqlite3DbstatRegister(sqlite3*); +#endif + #endif /* _SQLITEINT_H_ */ /************** End of sqliteInt.h *******************************************/ @@ -13801,6 +14536,7 @@ SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread*, void**); ** ** This file contains definitions of global variables and constants. */ +/* #include "sqliteInt.h" */ /* An array to map all upper-case characters into their corresponding ** lower-case character. @@ -13974,7 +14710,7 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* nScratch */ (void*)0, /* pPage */ 0, /* szPage */ - 0, /* nPage */ + SQLITE_DEFAULT_PCACHE_INITSZ, /* nPage */ 0, /* mxParserStack */ 0, /* sharedCacheEnabled */ SQLITE_SORTER_PMASZ, /* szPma */ @@ -14040,6 +14776,7 @@ SQLITE_PRIVATE const Token sqlite3IntTokens[] = { SQLITE_PRIVATE int sqlite3PendingByte = 0x40000000; #endif +/* #include "opcodes.h" */ /* ** Properties of opcodes. The OPFLG_INITIALIZER macro is ** created by mkopcodeh.awk during compilation. Data is obtained @@ -14068,6 +14805,7 @@ SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[] = OPFLG_INITIALIZER; #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +/* #include "sqliteInt.h" */ /* ** An array of names of all compile-time options. This array should @@ -14147,12 +14885,18 @@ static const char * const azCompileOpt[] = { #if SQLITE_ENABLE_FTS4 "ENABLE_FTS4", #endif +#if SQLITE_ENABLE_FTS5 + "ENABLE_FTS5", +#endif #if SQLITE_ENABLE_ICU "ENABLE_ICU", #endif #if SQLITE_ENABLE_IOTRACE "ENABLE_IOTRACE", #endif +#if SQLITE_ENABLE_JSON1 + "ENABLE_JSON1", +#endif #if SQLITE_ENABLE_LOAD_EXTENSION "ENABLE_LOAD_EXTENSION", #endif @@ -14497,6 +15241,7 @@ SQLITE_API const char *SQLITE_STDCALL sqlite3_compileoption_get(int N){ ** This module implements the sqlite3_status() interface and related ** functionality. */ +/* #include "sqliteInt.h" */ /************** Include vdbeInt.h in the middle of status.c ******************/ /************** Begin file vdbeInt.h *****************************************/ /* @@ -14584,6 +15329,9 @@ struct VdbeCursor { i64 seqCount; /* Sequence counter */ i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ VdbeSorter *pSorter; /* Sorter object for OP_SorterOpen cursors */ +#ifdef SQLITE_ENABLE_COLUMN_USED_MASK + u64 maskUsed; /* Mask of columns used by this cursor */ +#endif /* Cached information about the header for the data record that the ** cursor is currently pointing to. Only valid if cacheStatus matches @@ -14673,6 +15421,7 @@ struct Mem { } u; u16 flags; /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */ u8 enc; /* SQLITE_UTF8, SQLITE_UTF16BE, SQLITE_UTF16LE */ + u8 eSubtype; /* Subtype for this value */ int n; /* Number of characters in string value, excluding '\0' */ char *z; /* String or BLOB value */ /* ShallowCopy only needs to copy the information above */ @@ -14687,6 +15436,12 @@ struct Mem { #endif }; +/* +** Size of struct Mem not including the Mem.zMalloc member or anything that +** follows. +*/ +#define MEMCELLSIZE offsetof(Mem,zMalloc) + /* One or more of the following flags are set to indicate the validOK ** representations of the value stored in the Mem struct. ** @@ -14771,14 +15526,16 @@ struct AuxData { ** (Mem) which are only defined there. */ struct sqlite3_context { - Mem *pOut; /* The return value is stored here */ - FuncDef *pFunc; /* Pointer to function information */ - Mem *pMem; /* Memory cell used to store aggregate context */ - Vdbe *pVdbe; /* The VM that owns this context */ - int iOp; /* Instruction number of OP_Function */ - int isError; /* Error code returned by the function. */ - u8 skipFlag; /* Skip accumulator loading if true */ - u8 fErrorOrAux; /* isError!=0 or pVdbe->pAuxData modified */ + Mem *pOut; /* The return value is stored here */ + FuncDef *pFunc; /* Pointer to function information */ + Mem *pMem; /* Memory cell used to store aggregate context */ + Vdbe *pVdbe; /* The VM that owns this context */ + int iOp; /* Instruction number of OP_Function */ + int isError; /* Error code returned by the function. */ + u8 skipFlag; /* Skip accumulator loading if true */ + u8 fErrorOrAux; /* isError!=0 or pVdbe->pAuxData modified */ + u8 argc; /* Number of arguments */ + sqlite3_value *argv[1]; /* Argument set */ }; /* @@ -14892,6 +15649,7 @@ struct Vdbe { /* ** Function prototypes */ +SQLITE_PRIVATE void sqlite3VdbeError(Vdbe*, const char *, ...); SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *, VdbeCursor*); void sqliteVdbePopStack(Vdbe*,int); SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor*); @@ -15368,6 +16126,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_status( ** Willmann-Bell, Inc ** Richmond, Virginia (USA) */ +/* #include "sqliteInt.h" */ /* #include */ /* #include */ #include @@ -15679,7 +16438,7 @@ static void computeYMD(DateTime *p){ A = Z + 1 + A - (A/4); B = A + 1524; C = (int)((B - 122.1)/365.25); - D = (36525*C)/100; + D = (36525*(C&32767))/100; E = (int)((B-D)/30.6001); X1 = (int)(30.6001*E); p->D = B - D - X1; @@ -16439,14 +17198,14 @@ static void currentTimeFunc( SQLITE_PRIVATE void sqlite3RegisterDateTimeFunctions(void){ static SQLITE_WSD FuncDef aDateTimeFuncs[] = { #ifndef SQLITE_OMIT_DATETIME_FUNCS - FUNCTION(julianday, -1, 0, 0, juliandayFunc ), - FUNCTION(date, -1, 0, 0, dateFunc ), - FUNCTION(time, -1, 0, 0, timeFunc ), - FUNCTION(datetime, -1, 0, 0, datetimeFunc ), - FUNCTION(strftime, -1, 0, 0, strftimeFunc ), - FUNCTION(current_time, 0, 0, 0, ctimeFunc ), - FUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc), - FUNCTION(current_date, 0, 0, 0, cdateFunc ), + DFUNCTION(julianday, -1, 0, 0, juliandayFunc ), + DFUNCTION(date, -1, 0, 0, dateFunc ), + DFUNCTION(time, -1, 0, 0, timeFunc ), + DFUNCTION(datetime, -1, 0, 0, datetimeFunc ), + DFUNCTION(strftime, -1, 0, 0, strftimeFunc ), + DFUNCTION(current_time, 0, 0, 0, ctimeFunc ), + DFUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc), + DFUNCTION(current_date, 0, 0, 0, cdateFunc ), #else STR_FUNCTION(current_time, 0, "%H:%M:%S", 0, currentTimeFunc), STR_FUNCTION(current_date, 0, "%Y-%m-%d", 0, currentTimeFunc), @@ -16480,6 +17239,7 @@ SQLITE_PRIVATE void sqlite3RegisterDateTimeFunctions(void){ ** architectures. */ #define _SQLITE_OS_C_ 1 +/* #include "sqliteInt.h" */ #undef _SQLITE_OS_C_ /* @@ -16886,6 +17646,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_vfs_unregister(sqlite3_vfs *pVfs){ ** during a hash table resize is a benign fault. */ +/* #include "sqliteInt.h" */ #ifndef SQLITE_OMIT_BUILTIN_TEST @@ -16967,6 +17728,7 @@ SQLITE_PRIVATE void sqlite3EndBenignMalloc(void){ ** are merely placeholders. Real drivers must be substituted using ** sqlite3_config() before SQLite will operate. */ +/* #include "sqliteInt.h" */ /* ** This version of the memory allocator is the default. It is @@ -17053,6 +17815,7 @@ SQLITE_PRIVATE void sqlite3MemSetDefault(void){ ** be necessary when compiling for Delphi, ** for example. */ +/* #include "sqliteInt.h" */ /* ** This version of the memory allocator is the default. It is @@ -17328,6 +18091,7 @@ SQLITE_PRIVATE void sqlite3MemSetDefault(void){ ** This file contains implementations of the low-level memory allocation ** routines specified in the sqlite3_mem_methods object. */ +/* #include "sqliteInt.h" */ /* ** This version of the memory allocator is used only if the @@ -17862,6 +18626,7 @@ SQLITE_PRIVATE int sqlite3MemdebugMallocCount(){ ** This version of the memory allocation subsystem is included ** in the build only if SQLITE_ENABLE_MEMSYS3 is defined. */ +/* #include "sqliteInt.h" */ /* ** This version of the memory allocator is only built into the library @@ -18576,6 +19341,7 @@ SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys3(void){ ** The sqlite3_status() logic tracks the maximum values of n and M so ** that an application can, at any time, verify this constraint. */ +/* #include "sqliteInt.h" */ /* ** This version of the memory allocator is used only when @@ -19119,6 +19885,7 @@ SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys5(void){ ** ** This file contains code that is common across all mutex implementations. */ +/* #include "sqliteInt.h" */ #if defined(SQLITE_DEBUG) && !defined(SQLITE_MUTEX_OMIT) /* @@ -19127,7 +19894,7 @@ SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys5(void){ ** allocate a mutex while the system is uninitialized. */ static SQLITE_WSD int mutexIsInit = 0; -#endif /* SQLITE_DEBUG */ +#endif /* SQLITE_DEBUG && !defined(SQLITE_MUTEX_OMIT) */ #ifndef SQLITE_MUTEX_OMIT @@ -19150,11 +19917,18 @@ SQLITE_PRIVATE int sqlite3MutexInit(void){ }else{ pFrom = sqlite3NoopMutex(); } - memcpy(pTo, pFrom, offsetof(sqlite3_mutex_methods, xMutexAlloc)); - memcpy(&pTo->xMutexFree, &pFrom->xMutexFree, - sizeof(*pTo) - offsetof(sqlite3_mutex_methods, xMutexFree)); + pTo->xMutexInit = pFrom->xMutexInit; + pTo->xMutexEnd = pFrom->xMutexEnd; + pTo->xMutexFree = pFrom->xMutexFree; + pTo->xMutexEnter = pFrom->xMutexEnter; + pTo->xMutexTry = pFrom->xMutexTry; + pTo->xMutexLeave = pFrom->xMutexLeave; + pTo->xMutexHeld = pFrom->xMutexHeld; + pTo->xMutexNotheld = pFrom->xMutexNotheld; + sqlite3MemoryBarrier(); pTo->xMutexAlloc = pFrom->xMutexAlloc; } + assert( sqlite3GlobalConfig.mutex.xMutexInit ); rc = sqlite3GlobalConfig.mutex.xMutexInit(); #ifdef SQLITE_DEBUG @@ -19189,6 +19963,7 @@ SQLITE_API sqlite3_mutex *SQLITE_STDCALL sqlite3_mutex_alloc(int id){ if( id<=SQLITE_MUTEX_RECURSIVE && sqlite3_initialize() ) return 0; if( id>SQLITE_MUTEX_RECURSIVE && sqlite3MutexInit() ) return 0; #endif + assert( sqlite3GlobalConfig.mutex.xMutexAlloc ); return sqlite3GlobalConfig.mutex.xMutexAlloc(id); } @@ -19197,6 +19972,7 @@ SQLITE_PRIVATE sqlite3_mutex *sqlite3MutexAlloc(int id){ return 0; } assert( GLOBAL(int, mutexIsInit) ); + assert( sqlite3GlobalConfig.mutex.xMutexAlloc ); return sqlite3GlobalConfig.mutex.xMutexAlloc(id); } @@ -19205,6 +19981,7 @@ SQLITE_PRIVATE sqlite3_mutex *sqlite3MutexAlloc(int id){ */ SQLITE_API void SQLITE_STDCALL sqlite3_mutex_free(sqlite3_mutex *p){ if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexFree ); sqlite3GlobalConfig.mutex.xMutexFree(p); } } @@ -19215,6 +19992,7 @@ SQLITE_API void SQLITE_STDCALL sqlite3_mutex_free(sqlite3_mutex *p){ */ SQLITE_API void SQLITE_STDCALL sqlite3_mutex_enter(sqlite3_mutex *p){ if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexEnter ); sqlite3GlobalConfig.mutex.xMutexEnter(p); } } @@ -19226,6 +20004,7 @@ SQLITE_API void SQLITE_STDCALL sqlite3_mutex_enter(sqlite3_mutex *p){ SQLITE_API int SQLITE_STDCALL sqlite3_mutex_try(sqlite3_mutex *p){ int rc = SQLITE_OK; if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexTry ); return sqlite3GlobalConfig.mutex.xMutexTry(p); } return rc; @@ -19239,6 +20018,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_mutex_try(sqlite3_mutex *p){ */ SQLITE_API void SQLITE_STDCALL sqlite3_mutex_leave(sqlite3_mutex *p){ if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexLeave ); sqlite3GlobalConfig.mutex.xMutexLeave(p); } } @@ -19249,9 +20029,11 @@ SQLITE_API void SQLITE_STDCALL sqlite3_mutex_leave(sqlite3_mutex *p){ ** intended for use inside assert() statements. */ SQLITE_API int SQLITE_STDCALL sqlite3_mutex_held(sqlite3_mutex *p){ + assert( p==0 || sqlite3GlobalConfig.mutex.xMutexHeld ); return p==0 || sqlite3GlobalConfig.mutex.xMutexHeld(p); } SQLITE_API int SQLITE_STDCALL sqlite3_mutex_notheld(sqlite3_mutex *p){ + assert( p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld ); return p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld(p); } #endif @@ -19287,6 +20069,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_mutex_notheld(sqlite3_mutex *p){ ** that does error checking on mutexes to make sure they are being ** called correctly. */ +/* #include "sqliteInt.h" */ #ifndef SQLITE_MUTEX_OMIT @@ -19368,7 +20151,7 @@ static int debugMutexEnd(void){ return SQLITE_OK; } ** that means that a mutex could not be allocated. */ static sqlite3_mutex *debugMutexAlloc(int id){ - static sqlite3_debug_mutex aStatic[SQLITE_MUTEX_STATIC_APP3 - 1]; + static sqlite3_debug_mutex aStatic[SQLITE_MUTEX_STATIC_VFS3 - 1]; sqlite3_debug_mutex *pNew = 0; switch( id ){ case SQLITE_MUTEX_FAST: @@ -19490,6 +20273,7 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ ************************************************************************* ** This file contains the C functions that implement mutexes for pthreads */ +/* #include "sqliteInt.h" */ /* ** The code in this file is only used if we are compiling threadsafe @@ -19558,6 +20342,19 @@ static int pthreadMutexNotheld(sqlite3_mutex *p){ } #endif +/* +** Try to provide a memory barrier operation, needed for initialization +** and also for the implementation of xShmBarrier in the VFS in cases +** where SQLite is compiled without mutexes. +*/ +SQLITE_PRIVATE void sqlite3MemoryBarrier(void){ +#if defined(SQLITE_MEMORY_BARRIER) + SQLITE_MEMORY_BARRIER; +#elif defined(__GNUC__) && GCC_VERSION>=4001000 + __sync_synchronize(); +#endif +} + /* ** Initialize and deinitialize the mutex subsystem. */ @@ -19583,6 +20380,9 @@ static int pthreadMutexEnd(void){ return SQLITE_OK; } **
  • SQLITE_MUTEX_STATIC_APP1 **
  • SQLITE_MUTEX_STATIC_APP2 **
  • SQLITE_MUTEX_STATIC_APP3 +**
  • SQLITE_MUTEX_STATIC_VFS1 +**
  • SQLITE_MUTEX_STATIC_VFS2 +**
  • SQLITE_MUTEX_STATIC_VFS3 ** ** ** The first two constants cause sqlite3_mutex_alloc() to create @@ -19611,6 +20411,9 @@ static int pthreadMutexEnd(void){ return SQLITE_OK; } */ static sqlite3_mutex *pthreadMutexAlloc(int iType){ static sqlite3_mutex staticMutexes[] = { + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, @@ -19858,6 +20661,7 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ ************************************************************************* ** This file contains the C functions that implement mutexes for Win32. */ +/* #include "sqliteInt.h" */ #if SQLITE_OS_WIN /* @@ -20213,10 +21017,31 @@ static int winMutexNotheld(sqlite3_mutex *p){ } #endif +/* +** Try to provide a memory barrier operation, needed for initialization +** and also for the xShmBarrier method of the VFS in cases when SQLite is +** compiled without mutexes (SQLITE_THREADSAFE=0). +*/ +SQLITE_PRIVATE void sqlite3MemoryBarrier(void){ +#if defined(SQLITE_MEMORY_BARRIER) + SQLITE_MEMORY_BARRIER; +#elif defined(__GNUC__) + __sync_synchronize(); +#elif !defined(SQLITE_DISABLE_INTRINSIC) && \ + defined(_MSC_VER) && _MSC_VER>=1300 + _ReadWriteBarrier(); +#elif defined(MemoryBarrier) + MemoryBarrier(); +#endif +} + /* ** Initialize and deinitialize the mutex subsystem. */ static sqlite3_mutex winMutex_staticMutexes[] = { + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, @@ -20296,6 +21121,9 @@ static int winMutexEnd(void){ **
  • SQLITE_MUTEX_STATIC_APP1 **
  • SQLITE_MUTEX_STATIC_APP2 **
  • SQLITE_MUTEX_STATIC_APP3 +**
  • SQLITE_MUTEX_STATIC_VFS1 +**
  • SQLITE_MUTEX_STATIC_VFS2 +**
  • SQLITE_MUTEX_STATIC_VFS3 ** ** ** The first two constants cause sqlite3_mutex_alloc() to create @@ -20527,6 +21355,7 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ ** ** Memory allocation functions used throughout sqlite. */ +/* #include "sqliteInt.h" */ /* #include */ /* @@ -20559,16 +21388,7 @@ typedef struct ScratchFreeslot { */ static SQLITE_WSD struct Mem0Global { sqlite3_mutex *mutex; /* Mutex to serialize access */ - - /* - ** The alarm callback and its arguments. The mem0.mutex lock will - ** be held while the callback is running. Recursive calls into - ** the memory subsystem are allowed, but no new callbacks will be - ** issued. - */ - sqlite3_int64 alarmThreshold; - void (*alarmCallback)(void*, sqlite3_int64,int); - void *alarmArg; + sqlite3_int64 alarmThreshold; /* The soft heap limit */ /* ** Pointers to the end of sqlite3GlobalConfig.pScratch memory @@ -20585,7 +21405,7 @@ static SQLITE_WSD struct Mem0Global { ** sqlite3_soft_heap_limit() setting. */ int nearlyFull; -} mem0 = { 0, 0, 0, 0, 0, 0, 0, 0 }; +} mem0 = { 0, 0, 0, 0, 0, 0 }; #define mem0 GLOBAL(struct Mem0Global, mem0) @@ -20596,50 +21416,21 @@ SQLITE_PRIVATE sqlite3_mutex *sqlite3MallocMutex(void){ return mem0.mutex; } -/* -** This routine runs when the memory allocator sees that the -** total memory allocation is about to exceed the soft heap -** limit. -*/ -static void softHeapLimitEnforcer( - void *NotUsed, - sqlite3_int64 NotUsed2, - int allocSize -){ - UNUSED_PARAMETER2(NotUsed, NotUsed2); - sqlite3_release_memory(allocSize); -} - -/* -** Change the alarm callback -*/ -static int sqlite3MemoryAlarm( - void(*xCallback)(void *pArg, sqlite3_int64 used,int N), - void *pArg, - sqlite3_int64 iThreshold -){ - sqlite3_int64 nUsed; - sqlite3_mutex_enter(mem0.mutex); - mem0.alarmCallback = xCallback; - mem0.alarmArg = pArg; - mem0.alarmThreshold = iThreshold; - nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); - mem0.nearlyFull = (iThreshold>0 && iThreshold<=nUsed); - sqlite3_mutex_leave(mem0.mutex); - return SQLITE_OK; -} - #ifndef SQLITE_OMIT_DEPRECATED /* -** Deprecated external interface. Internal/core SQLite code -** should call sqlite3MemoryAlarm. +** Deprecated external interface. It used to set an alarm callback +** that was invoked when memory usage grew too large. Now it is a +** no-op. */ SQLITE_API int SQLITE_STDCALL sqlite3_memory_alarm( void(*xCallback)(void *pArg, sqlite3_int64 used,int N), void *pArg, sqlite3_int64 iThreshold ){ - return sqlite3MemoryAlarm(xCallback, pArg, iThreshold); + (void)xCallback; + (void)pArg; + (void)iThreshold; + return SQLITE_OK; } #endif @@ -20650,19 +21441,21 @@ SQLITE_API int SQLITE_STDCALL sqlite3_memory_alarm( SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_soft_heap_limit64(sqlite3_int64 n){ sqlite3_int64 priorLimit; sqlite3_int64 excess; + sqlite3_int64 nUsed; #ifndef SQLITE_OMIT_AUTOINIT int rc = sqlite3_initialize(); if( rc ) return -1; #endif sqlite3_mutex_enter(mem0.mutex); priorLimit = mem0.alarmThreshold; - sqlite3_mutex_leave(mem0.mutex); - if( n<0 ) return priorLimit; - if( n>0 ){ - sqlite3MemoryAlarm(softHeapLimitEnforcer, 0, n); - }else{ - sqlite3MemoryAlarm(0, 0, 0); + if( n<0 ){ + sqlite3_mutex_leave(mem0.mutex); + return priorLimit; } + mem0.alarmThreshold = n; + nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + mem0.nearlyFull = (n>0 && n<=nUsed); + sqlite3_mutex_leave(mem0.mutex); excess = sqlite3_memory_used() - n; if( excess>0 ) sqlite3_release_memory((int)(excess & 0x7fffffff)); return priorLimit; @@ -20707,10 +21500,9 @@ SQLITE_PRIVATE int sqlite3MallocInit(void){ sqlite3GlobalConfig.nScratch = 0; } if( sqlite3GlobalConfig.pPage==0 || sqlite3GlobalConfig.szPage<512 - || sqlite3GlobalConfig.nPage<1 ){ + || sqlite3GlobalConfig.nPage<=0 ){ sqlite3GlobalConfig.pPage = 0; sqlite3GlobalConfig.szPage = 0; - sqlite3GlobalConfig.nPage = 0; } rc = sqlite3GlobalConfig.m.xInit(sqlite3GlobalConfig.m.pAppData); if( rc!=SQLITE_OK ) memset(&mem0, 0, sizeof(mem0)); @@ -20740,10 +21532,8 @@ SQLITE_PRIVATE void sqlite3MallocEnd(void){ ** Return the amount of memory currently checked out. */ SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_memory_used(void){ - int n, mx; - sqlite3_int64 res; - sqlite3_status(SQLITE_STATUS_MEMORY_USED, &n, &mx, 0); - res = (sqlite3_int64)n; /* Work around bug in Borland C. Ticket #3216 */ + sqlite3_int64 res, mx; + sqlite3_status64(SQLITE_STATUS_MEMORY_USED, &res, &mx, 0); return res; } @@ -20753,30 +21543,19 @@ SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_memory_used(void){ ** or since the most recent reset. */ SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_memory_highwater(int resetFlag){ - int n, mx; - sqlite3_int64 res; - sqlite3_status(SQLITE_STATUS_MEMORY_USED, &n, &mx, resetFlag); - res = (sqlite3_int64)mx; /* Work around bug in Borland C. Ticket #3216 */ - return res; + sqlite3_int64 res, mx; + sqlite3_status64(SQLITE_STATUS_MEMORY_USED, &res, &mx, resetFlag); + return mx; } /* ** Trigger the alarm */ static void sqlite3MallocAlarm(int nByte){ - void (*xCallback)(void*,sqlite3_int64,int); - sqlite3_int64 nowUsed; - void *pArg; - if( mem0.alarmCallback==0 ) return; - xCallback = mem0.alarmCallback; - nowUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); - pArg = mem0.alarmArg; - mem0.alarmCallback = 0; + if( mem0.alarmThreshold<=0 ) return; sqlite3_mutex_leave(mem0.mutex); - xCallback(pArg, nowUsed, nByte); + sqlite3_release_memory(nByte); sqlite3_mutex_enter(mem0.mutex); - mem0.alarmCallback = xCallback; - mem0.alarmArg = pArg; } /* @@ -20789,7 +21568,7 @@ static int mallocWithAlarm(int n, void **pp){ assert( sqlite3_mutex_held(mem0.mutex) ); nFull = sqlite3GlobalConfig.m.xRoundup(n); sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, n); - if( mem0.alarmCallback!=0 ){ + if( mem0.alarmThreshold>0 ){ sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); if( nUsed >= mem0.alarmThreshold - nFull ){ mem0.nearlyFull = 1; @@ -20800,7 +21579,7 @@ static int mallocWithAlarm(int n, void **pp){ } p = sqlite3GlobalConfig.m.xMalloc(nFull); #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - if( p==0 && mem0.alarmCallback ){ + if( p==0 && mem0.alarmThreshold>0 ){ sqlite3MallocAlarm(nFull); p = sqlite3GlobalConfig.m.xMalloc(nFull); } @@ -20975,19 +21754,20 @@ SQLITE_PRIVATE int sqlite3MallocSize(void *p){ return sqlite3GlobalConfig.m.xSize(p); } SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3 *db, void *p){ - if( db==0 ){ - assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); - assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); - return sqlite3MallocSize(p); - }else{ - assert( sqlite3_mutex_held(db->mutex) ); - if( isLookaside(db, p) ){ - return db->lookaside.sz; + if( db==0 || !isLookaside(db,p) ){ +#if SQLITE_DEBUG + if( db==0 ){ + assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); }else{ assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); - return sqlite3GlobalConfig.m.xSize(p); } +#endif + return sqlite3GlobalConfig.m.xSize(p); + }else{ + assert( sqlite3_mutex_held(db->mutex) ); + return db->lookaside.sz; } } SQLITE_API sqlite3_uint64 SQLITE_STDCALL sqlite3_msize(void *p){ @@ -21088,7 +21868,7 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, u64 nBytes){ sqlite3MallocAlarm(nDiff); } pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); - if( pNew==0 && mem0.alarmCallback ){ + if( pNew==0 && mem0.alarmThreshold>0 ){ sqlite3MallocAlarm((int)nBytes); pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); } @@ -21289,19 +22069,11 @@ SQLITE_PRIVATE char *sqlite3DbStrNDup(sqlite3 *db, const char *z, u64 n){ } /* -** Create a string from the zFromat argument and the va_list that follows. -** Store the string in memory obtained from sqliteMalloc() and make *pz -** point to that string. +** Free any prior content in *pz and replace it with a copy of zNew. */ -SQLITE_PRIVATE void sqlite3SetString(char **pz, sqlite3 *db, const char *zFormat, ...){ - va_list ap; - char *z; - - va_start(ap, zFormat); - z = sqlite3VMPrintf(db, zFormat, ap); - va_end(ap); +SQLITE_PRIVATE void sqlite3SetString(char **pz, sqlite3 *db, const char *zNew){ sqlite3DbFree(db, *pz); - *pz = z; + *pz = sqlite3DbStrDup(db, zNew); } /* @@ -21322,17 +22094,16 @@ static SQLITE_NOINLINE int apiOomError(sqlite3 *db){ ** function. However, if a malloc() failure has occurred since the previous ** invocation SQLITE_NOMEM is returned instead. ** -** If the first argument, db, is not NULL and a malloc() error has occurred, -** then the connection error-code (the value returned by sqlite3_errcode()) -** is set to SQLITE_NOMEM. +** If an OOM as occurred, then the connection error-code (the value +** returned by sqlite3_errcode()) is set to SQLITE_NOMEM. */ SQLITE_PRIVATE int sqlite3ApiExit(sqlite3* db, int rc){ - /* If the db handle is not NULL, then we must hold the connection handle - ** mutex here. Otherwise the read (and possible write) of db->mallocFailed + /* If the db handle must hold the connection handle mutex here. + ** Otherwise the read (and possible write) of db->mallocFailed ** is unsafe, as is the call to sqlite3Error(). */ - assert( !db || sqlite3_mutex_held(db->mutex) ); - if( db==0 ) return rc & 0xff; + assert( db!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); if( db->mallocFailed || rc==SQLITE_IOERR_NOMEM ){ return apiOomError(db); } @@ -21343,18 +22114,16 @@ SQLITE_PRIVATE int sqlite3ApiExit(sqlite3* db, int rc){ /************** Begin file printf.c ******************************************/ /* ** The "printf" code that follows dates from the 1980's. It is in -** the public domain. The original comments are included here for -** completeness. They are very out-of-date but might be useful as -** an historical reference. Most of the "enhancements" have been backed -** out so that the functionality is now the same as standard printf(). +** the public domain. ** ************************************************************************** ** ** This file contains code for a set of "printf"-like routines. These ** routines format strings much like the printf() from the standard C ** library, though the implementation here has enhancements to support -** SQLlite. +** SQLite. */ +/* #include "sqliteInt.h" */ /* ** Conversion types fall into various categories as defined by the @@ -21813,21 +22582,16 @@ SQLITE_PRIVATE void sqlite3VXPrintf( if( realvalue>0.0 ){ LONGDOUBLE_TYPE scale = 1.0; while( realvalue>=1e100*scale && exp<=350 ){ scale *= 1e100;exp+=100;} - while( realvalue>=1e64*scale && exp<=350 ){ scale *= 1e64; exp+=64; } - while( realvalue>=1e8*scale && exp<=350 ){ scale *= 1e8; exp+=8; } + while( realvalue>=1e10*scale && exp<=350 ){ scale *= 1e10; exp+=10; } while( realvalue>=10.0*scale && exp<=350 ){ scale *= 10.0; exp++; } realvalue /= scale; while( realvalue<1e-8 ){ realvalue *= 1e8; exp-=8; } while( realvalue<1.0 ){ realvalue *= 10.0; exp--; } if( exp>350 ){ - if( prefix=='-' ){ - bufpt = "-Inf"; - }else if( prefix=='+' ){ - bufpt = "+Inf"; - }else{ - bufpt = "Inf"; - } - length = sqlite3Strlen30(bufpt); + bufpt = buf; + buf[0] = prefix; + memcpy(buf+(prefix!=0),"Inf",4); + length = 3+(prefix!=0); break; } } @@ -21976,12 +22740,13 @@ SQLITE_PRIVATE void sqlite3VXPrintf( case etDYNSTRING: if( bArgList ){ bufpt = getTextArg(pArgList); + xtype = etSTRING; }else{ bufpt = va_arg(ap,char*); } if( bufpt==0 ){ bufpt = ""; - }else if( xtype==etDYNSTRING && !bArgList ){ + }else if( xtype==etDYNSTRING ){ zExtra = bufpt; } if( precision>=0 ){ @@ -21990,9 +22755,9 @@ SQLITE_PRIVATE void sqlite3VXPrintf( length = sqlite3Strlen30(bufpt); } break; - case etSQLESCAPE: - case etSQLESCAPE2: - case etSQLESCAPE3: { + case etSQLESCAPE: /* Escape ' characters */ + case etSQLESCAPE2: /* Escape ' and enclose in '...' */ + case etSQLESCAPE3: { /* Escape " characters */ int i, j, k, n, isnull; int needQuote; char ch; @@ -22011,7 +22776,7 @@ SQLITE_PRIVATE void sqlite3VXPrintf( if( ch==q ) n++; } needQuote = !isnull && xtype==etSQLESCAPE2; - n += i + 1 + needQuote*2; + n += i + 3; if( n>etBUFSIZE ){ bufpt = zExtra = sqlite3Malloc( n ); if( bufpt==0 ){ @@ -22273,24 +23038,6 @@ SQLITE_PRIVATE char *sqlite3MPrintf(sqlite3 *db, const char *zFormat, ...){ return z; } -/* -** Like sqlite3MPrintf(), but call sqlite3DbFree() on zStr after formatting -** the string and before returning. This routine is intended to be used -** to modify an existing string. For example: -** -** x = sqlite3MPrintf(db, x, "prefix %s suffix", x); -** -*/ -SQLITE_PRIVATE char *sqlite3MAppendf(sqlite3 *db, char *zStr, const char *zFormat, ...){ - va_list ap; - char *z; - va_start(ap, zFormat); - z = sqlite3VMPrintf(db, zFormat, ap); - va_end(ap); - sqlite3DbFree(db, zStr); - return z; -} - /* ** Print into memory obtained from sqlite3_malloc(). Omit the internal ** %-conversion extensions. @@ -22375,6 +23122,11 @@ SQLITE_API char *SQLITE_CDECL sqlite3_snprintf(int n, char *zBuf, const char *zF ** sqlite3_log() must render into a static buffer. It cannot dynamically ** allocate memory because it might be called while the memory allocator ** mutex is held. +** +** sqlite3VXPrintf() might ask for *temporary* memory allocations for +** certain format characters (%q) or for very large precisions or widths. +** Care must be taken that any sqlite3_log() calls that occur while the +** memory mutex is held do not use these mechanisms. */ static void renderLogMsg(int iErrCode, const char *zFormat, va_list ap){ StrAccum acc; /* String accumulator */ @@ -22418,22 +23170,47 @@ SQLITE_PRIVATE void sqlite3DebugPrintf(const char *zFormat, ...){ } #endif -#ifdef SQLITE_DEBUG -/************************************************************************* -** Routines for implementing the "TreeView" display of hierarchical -** data structures for debugging. + +/* +** variable-argument wrapper around sqlite3VXPrintf(). The bFlags argument +** can contain the bit SQLITE_PRINTF_INTERNAL enable internal formats. +*/ +SQLITE_PRIVATE void sqlite3XPrintf(StrAccum *p, u32 bFlags, const char *zFormat, ...){ + va_list ap; + va_start(ap,zFormat); + sqlite3VXPrintf(p, bFlags, zFormat, ap); + va_end(ap); +} + +/************** End of printf.c **********************************************/ +/************** Begin file treeview.c ****************************************/ +/* +** 2015-06-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** The main entry points (coded elsewhere) are: -** sqlite3TreeViewExpr(0, pExpr, 0); -** sqlite3TreeViewExprList(0, pList, 0, 0); -** sqlite3TreeViewSelect(0, pSelect, 0); -** Insert calls to those routines while debugging in order to display -** a diagram of Expr, ExprList, and Select objects. +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** +************************************************************************* +** +** This file contains C code to implement the TreeView debugging routines. +** These routines print a parse tree to standard output for debugging and +** analysis. +** +** The interfaces in this file is only available when compiling +** with SQLITE_DEBUG. +*/ +/* #include "sqliteInt.h" */ +#ifdef SQLITE_DEBUG + +/* +** Add a new subitem to the tree. The moreToFollow flag indicates that this +** is not the last item in the tree. */ -/* Add a new subitem to the tree. The moreToFollow flag indicates that this -** is not the last item in the tree. */ -SQLITE_PRIVATE TreeView *sqlite3TreeViewPush(TreeView *p, u8 moreToFollow){ +static TreeView *sqlite3TreeViewPush(TreeView *p, u8 moreToFollow){ if( p==0 ){ p = sqlite3_malloc64( sizeof(*p) ); if( p==0 ) return 0; @@ -22445,15 +23222,21 @@ SQLITE_PRIVATE TreeView *sqlite3TreeViewPush(TreeView *p, u8 moreToFollow){ if( p->iLevelbLine) ) p->bLine[p->iLevel] = moreToFollow; return p; } -/* Finished with one layer of the tree */ -SQLITE_PRIVATE void sqlite3TreeViewPop(TreeView *p){ + +/* +** Finished with one layer of the tree +*/ +static void sqlite3TreeViewPop(TreeView *p){ if( p==0 ) return; p->iLevel--; if( p->iLevel<0 ) sqlite3_free(p); } -/* Generate a single line of output for the tree, with a prefix that contains -** all the appropriate tree lines */ -SQLITE_PRIVATE void sqlite3TreeViewLine(TreeView *p, const char *zFormat, ...){ + +/* +** Generate a single line of output for the tree, with a prefix that contains +** all the appropriate tree lines +*/ +static void sqlite3TreeViewLine(TreeView *p, const char *zFormat, ...){ va_list ap; int i; StrAccum acc; @@ -22473,24 +23256,378 @@ SQLITE_PRIVATE void sqlite3TreeViewLine(TreeView *p, const char *zFormat, ...){ fprintf(stdout,"%s", zBuf); fflush(stdout); } -/* Shorthand for starting a new tree item that consists of a single label */ -SQLITE_PRIVATE void sqlite3TreeViewItem(TreeView *p, const char *zLabel, u8 moreToFollow){ - p = sqlite3TreeViewPush(p, moreToFollow); + +/* +** Shorthand for starting a new tree item that consists of a single label +*/ +static void sqlite3TreeViewItem(TreeView *p, const char *zLabel,u8 moreFollows){ + p = sqlite3TreeViewPush(p, moreFollows); sqlite3TreeViewLine(p, "%s", zLabel); } -#endif /* SQLITE_DEBUG */ + /* -** variable-argument wrapper around sqlite3VXPrintf(). +** Generate a human-readable description of a the Select object. */ -SQLITE_PRIVATE void sqlite3XPrintf(StrAccum *p, u32 bFlags, const char *zFormat, ...){ - va_list ap; - va_start(ap,zFormat); - sqlite3VXPrintf(p, bFlags, zFormat, ap); - va_end(ap); +SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ + int n = 0; + int cnt = 0; + pView = sqlite3TreeViewPush(pView, moreToFollow); + do{ + sqlite3TreeViewLine(pView, "SELECT%s%s (0x%p) selFlags=0x%x", + ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), + ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), p, p->selFlags + ); + if( cnt++ ) sqlite3TreeViewPop(pView); + if( p->pPrior ){ + n = 1000; + }else{ + n = 0; + if( p->pSrc && p->pSrc->nSrc ) n++; + if( p->pWhere ) n++; + if( p->pGroupBy ) n++; + if( p->pHaving ) n++; + if( p->pOrderBy ) n++; + if( p->pLimit ) n++; + if( p->pOffset ) n++; + } + sqlite3TreeViewExprList(pView, p->pEList, (n--)>0, "result-set"); + if( p->pSrc && p->pSrc->nSrc ){ + int i; + pView = sqlite3TreeViewPush(pView, (n--)>0); + sqlite3TreeViewLine(pView, "FROM"); + for(i=0; ipSrc->nSrc; i++){ + struct SrcList_item *pItem = &p->pSrc->a[i]; + StrAccum x; + char zLine[100]; + sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); + sqlite3XPrintf(&x, 0, "{%d,*}", pItem->iCursor); + if( pItem->zDatabase ){ + sqlite3XPrintf(&x, 0, " %s.%s", pItem->zDatabase, pItem->zName); + }else if( pItem->zName ){ + sqlite3XPrintf(&x, 0, " %s", pItem->zName); + } + if( pItem->pTab ){ + sqlite3XPrintf(&x, 0, " tabname=%Q", pItem->pTab->zName); + } + if( pItem->zAlias ){ + sqlite3XPrintf(&x, 0, " (AS %s)", pItem->zAlias); + } + if( pItem->fg.jointype & JT_LEFT ){ + sqlite3XPrintf(&x, 0, " LEFT-JOIN"); + } + sqlite3StrAccumFinish(&x); + sqlite3TreeViewItem(pView, zLine, ipSrc->nSrc-1); + if( pItem->pSelect ){ + sqlite3TreeViewSelect(pView, pItem->pSelect, 0); + } + if( pItem->fg.isTabFunc ){ + sqlite3TreeViewExprList(pView, pItem->u1.pFuncArg, 0, "func-args:"); + } + sqlite3TreeViewPop(pView); + } + sqlite3TreeViewPop(pView); + } + if( p->pWhere ){ + sqlite3TreeViewItem(pView, "WHERE", (n--)>0); + sqlite3TreeViewExpr(pView, p->pWhere, 0); + sqlite3TreeViewPop(pView); + } + if( p->pGroupBy ){ + sqlite3TreeViewExprList(pView, p->pGroupBy, (n--)>0, "GROUPBY"); + } + if( p->pHaving ){ + sqlite3TreeViewItem(pView, "HAVING", (n--)>0); + sqlite3TreeViewExpr(pView, p->pHaving, 0); + sqlite3TreeViewPop(pView); + } + if( p->pOrderBy ){ + sqlite3TreeViewExprList(pView, p->pOrderBy, (n--)>0, "ORDERBY"); + } + if( p->pLimit ){ + sqlite3TreeViewItem(pView, "LIMIT", (n--)>0); + sqlite3TreeViewExpr(pView, p->pLimit, 0); + sqlite3TreeViewPop(pView); + } + if( p->pOffset ){ + sqlite3TreeViewItem(pView, "OFFSET", (n--)>0); + sqlite3TreeViewExpr(pView, p->pOffset, 0); + sqlite3TreeViewPop(pView); + } + if( p->pPrior ){ + const char *zOp = "UNION"; + switch( p->op ){ + case TK_ALL: zOp = "UNION ALL"; break; + case TK_INTERSECT: zOp = "INTERSECT"; break; + case TK_EXCEPT: zOp = "EXCEPT"; break; + } + sqlite3TreeViewItem(pView, zOp, 1); + } + p = p->pPrior; + }while( p!=0 ); + sqlite3TreeViewPop(pView); } -/************** End of printf.c **********************************************/ +/* +** Generate a human-readable explanation of an expression tree. +*/ +SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ + const char *zBinOp = 0; /* Binary operator */ + const char *zUniOp = 0; /* Unary operator */ + char zFlgs[30]; + pView = sqlite3TreeViewPush(pView, moreToFollow); + if( pExpr==0 ){ + sqlite3TreeViewLine(pView, "nil"); + sqlite3TreeViewPop(pView); + return; + } + if( pExpr->flags ){ + sqlite3_snprintf(sizeof(zFlgs),zFlgs," flags=0x%x",pExpr->flags); + }else{ + zFlgs[0] = 0; + } + switch( pExpr->op ){ + case TK_AGG_COLUMN: { + sqlite3TreeViewLine(pView, "AGG{%d:%d}%s", + pExpr->iTable, pExpr->iColumn, zFlgs); + break; + } + case TK_COLUMN: { + if( pExpr->iTable<0 ){ + /* This only happens when coding check constraints */ + sqlite3TreeViewLine(pView, "COLUMN(%d)%s", pExpr->iColumn, zFlgs); + }else{ + sqlite3TreeViewLine(pView, "{%d:%d}%s", + pExpr->iTable, pExpr->iColumn, zFlgs); + } + break; + } + case TK_INTEGER: { + if( pExpr->flags & EP_IntValue ){ + sqlite3TreeViewLine(pView, "%d", pExpr->u.iValue); + }else{ + sqlite3TreeViewLine(pView, "%s", pExpr->u.zToken); + } + break; + } +#ifndef SQLITE_OMIT_FLOATING_POINT + case TK_FLOAT: { + sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken); + break; + } +#endif + case TK_STRING: { + sqlite3TreeViewLine(pView,"%Q", pExpr->u.zToken); + break; + } + case TK_NULL: { + sqlite3TreeViewLine(pView,"NULL"); + break; + } +#ifndef SQLITE_OMIT_BLOB_LITERAL + case TK_BLOB: { + sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken); + break; + } +#endif + case TK_VARIABLE: { + sqlite3TreeViewLine(pView,"VARIABLE(%s,%d)", + pExpr->u.zToken, pExpr->iColumn); + break; + } + case TK_REGISTER: { + sqlite3TreeViewLine(pView,"REGISTER(%d)", pExpr->iTable); + break; + } + case TK_ID: { + sqlite3TreeViewLine(pView,"ID \"%w\"", pExpr->u.zToken); + break; + } +#ifndef SQLITE_OMIT_CAST + case TK_CAST: { + /* Expressions of the form: CAST(pLeft AS token) */ + sqlite3TreeViewLine(pView,"CAST %Q", pExpr->u.zToken); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + break; + } +#endif /* SQLITE_OMIT_CAST */ + case TK_LT: zBinOp = "LT"; break; + case TK_LE: zBinOp = "LE"; break; + case TK_GT: zBinOp = "GT"; break; + case TK_GE: zBinOp = "GE"; break; + case TK_NE: zBinOp = "NE"; break; + case TK_EQ: zBinOp = "EQ"; break; + case TK_IS: zBinOp = "IS"; break; + case TK_ISNOT: zBinOp = "ISNOT"; break; + case TK_AND: zBinOp = "AND"; break; + case TK_OR: zBinOp = "OR"; break; + case TK_PLUS: zBinOp = "ADD"; break; + case TK_STAR: zBinOp = "MUL"; break; + case TK_MINUS: zBinOp = "SUB"; break; + case TK_REM: zBinOp = "REM"; break; + case TK_BITAND: zBinOp = "BITAND"; break; + case TK_BITOR: zBinOp = "BITOR"; break; + case TK_SLASH: zBinOp = "DIV"; break; + case TK_LSHIFT: zBinOp = "LSHIFT"; break; + case TK_RSHIFT: zBinOp = "RSHIFT"; break; + case TK_CONCAT: zBinOp = "CONCAT"; break; + case TK_DOT: zBinOp = "DOT"; break; + + case TK_UMINUS: zUniOp = "UMINUS"; break; + case TK_UPLUS: zUniOp = "UPLUS"; break; + case TK_BITNOT: zUniOp = "BITNOT"; break; + case TK_NOT: zUniOp = "NOT"; break; + case TK_ISNULL: zUniOp = "ISNULL"; break; + case TK_NOTNULL: zUniOp = "NOTNULL"; break; + + case TK_COLLATE: { + sqlite3TreeViewLine(pView, "COLLATE %Q", pExpr->u.zToken); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + break; + } + + case TK_AGG_FUNCTION: + case TK_FUNCTION: { + ExprList *pFarg; /* List of function arguments */ + if( ExprHasProperty(pExpr, EP_TokenOnly) ){ + pFarg = 0; + }else{ + pFarg = pExpr->x.pList; + } + if( pExpr->op==TK_AGG_FUNCTION ){ + sqlite3TreeViewLine(pView, "AGG_FUNCTION%d %Q", + pExpr->op2, pExpr->u.zToken); + }else{ + sqlite3TreeViewLine(pView, "FUNCTION %Q", pExpr->u.zToken); + } + if( pFarg ){ + sqlite3TreeViewExprList(pView, pFarg, 0, 0); + } + break; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_EXISTS: { + sqlite3TreeViewLine(pView, "EXISTS-expr"); + sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); + break; + } + case TK_SELECT: { + sqlite3TreeViewLine(pView, "SELECT-expr"); + sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); + break; + } + case TK_IN: { + sqlite3TreeViewLine(pView, "IN"); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); + }else{ + sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, 0); + } + break; + } +#endif /* SQLITE_OMIT_SUBQUERY */ + + /* + ** x BETWEEN y AND z + ** + ** This is equivalent to + ** + ** x>=y AND x<=z + ** + ** X is stored in pExpr->pLeft. + ** Y is stored in pExpr->pList->a[0].pExpr. + ** Z is stored in pExpr->pList->a[1].pExpr. + */ + case TK_BETWEEN: { + Expr *pX = pExpr->pLeft; + Expr *pY = pExpr->x.pList->a[0].pExpr; + Expr *pZ = pExpr->x.pList->a[1].pExpr; + sqlite3TreeViewLine(pView, "BETWEEN"); + sqlite3TreeViewExpr(pView, pX, 1); + sqlite3TreeViewExpr(pView, pY, 1); + sqlite3TreeViewExpr(pView, pZ, 0); + break; + } + case TK_TRIGGER: { + /* If the opcode is TK_TRIGGER, then the expression is a reference + ** to a column in the new.* or old.* pseudo-tables available to + ** trigger programs. In this case Expr.iTable is set to 1 for the + ** new.* pseudo-table, or 0 for the old.* pseudo-table. Expr.iColumn + ** is set to the column of the pseudo-table to read, or to -1 to + ** read the rowid field. + */ + sqlite3TreeViewLine(pView, "%s(%d)", + pExpr->iTable ? "NEW" : "OLD", pExpr->iColumn); + break; + } + case TK_CASE: { + sqlite3TreeViewLine(pView, "CASE"); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); + sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, 0); + break; + } +#ifndef SQLITE_OMIT_TRIGGER + case TK_RAISE: { + const char *zType = "unk"; + switch( pExpr->affinity ){ + case OE_Rollback: zType = "rollback"; break; + case OE_Abort: zType = "abort"; break; + case OE_Fail: zType = "fail"; break; + case OE_Ignore: zType = "ignore"; break; + } + sqlite3TreeViewLine(pView, "RAISE %s(%Q)", zType, pExpr->u.zToken); + break; + } +#endif + default: { + sqlite3TreeViewLine(pView, "op=%d", pExpr->op); + break; + } + } + if( zBinOp ){ + sqlite3TreeViewLine(pView, "%s%s", zBinOp, zFlgs); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); + sqlite3TreeViewExpr(pView, pExpr->pRight, 0); + }else if( zUniOp ){ + sqlite3TreeViewLine(pView, "%s%s", zUniOp, zFlgs); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + } + sqlite3TreeViewPop(pView); +} + +/* +** Generate a human-readable explanation of an expression list. +*/ +SQLITE_PRIVATE void sqlite3TreeViewExprList( + TreeView *pView, + const ExprList *pList, + u8 moreToFollow, + const char *zLabel +){ + int i; + pView = sqlite3TreeViewPush(pView, moreToFollow); + if( zLabel==0 || zLabel[0]==0 ) zLabel = "LIST"; + if( pList==0 ){ + sqlite3TreeViewLine(pView, "%s (empty)", zLabel); + }else{ + sqlite3TreeViewLine(pView, "%s", zLabel); + for(i=0; inExpr; i++){ + int j = pList->a[i].u.x.iOrderByCol; + if( j ){ + sqlite3TreeViewPush(pView, 0); + sqlite3TreeViewLine(pView, "iOrderByCol=%d", j); + } + sqlite3TreeViewExpr(pView, pList->a[i].pExpr, inExpr-1); + if( j ) sqlite3TreeViewPop(pView); + } + } + sqlite3TreeViewPop(pView); +} + +#endif /* SQLITE_DEBUG */ + +/************** End of treeview.c ********************************************/ /************** Begin file random.c ******************************************/ /* ** 2001 September 15 @@ -22509,6 +23646,7 @@ SQLITE_PRIVATE void sqlite3XPrintf(StrAccum *p, u32 bFlags, const char *zFormat, ** Random numbers are used by some of the database backends in order ** to generate random integer keys for tables or random filenames. */ +/* #include "sqliteInt.h" */ /* All threads share a single random number generator. @@ -22655,7 +23793,9 @@ SQLITE_PRIVATE void sqlite3PrngRestoreState(void){ ** of multiple cores can do so, while also allowing applications to stay ** single-threaded if desired. */ +/* #include "sqliteInt.h" */ #if SQLITE_OS_WIN +/* # include "os_win.h" */ #endif #if SQLITE_MAX_WORKER_THREADS>0 @@ -22695,6 +23835,10 @@ SQLITE_PRIVATE int sqlite3ThreadCreate( memset(p, 0, sizeof(*p)); p->xTask = xTask; p->pIn = pIn; + /* If the SQLITE_TESTCTRL_FAULT_INSTALL callback is registered to a + ** function that returns SQLITE_ERROR when passed the argument 200, that + ** forces worker threads to run sequentially and deterministically + ** for testing purposes. */ if( sqlite3FaultSim(200) ){ rc = 1; }else{ @@ -22779,7 +23923,12 @@ SQLITE_PRIVATE int sqlite3ThreadCreate( *ppThread = 0; p = sqlite3Malloc(sizeof(*p)); if( p==0 ) return SQLITE_NOMEM; - if( sqlite3GlobalConfig.bCoreMutex==0 ){ + /* If the SQLITE_TESTCTRL_FAULT_INSTALL callback is registered to a + ** function that returns SQLITE_ERROR when passed the argument 200, that + ** forces worker threads to run sequentially and deterministically + ** (via the sqlite3FaultSim() term of the conditional) for testing + ** purposes. */ + if( sqlite3GlobalConfig.bCoreMutex==0 || sqlite3FaultSim(200) ){ memset(p, 0, sizeof(*p)); }else{ p->xTask = xTask; @@ -22807,7 +23956,7 @@ SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ assert( ppOut!=0 ); if( NEVER(p==0) ) return SQLITE_NOMEM; if( p->xTask==0 ){ - assert( p->id==GetCurrentThreadId() ); + /* assert( p->id==GetCurrentThreadId() ); */ rc = WAIT_OBJECT_0; assert( p->tid==0 ); }else{ @@ -22929,7 +24078,9 @@ SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ ** 0xfe 0xff big-endian utf-16 follows ** */ +/* #include "sqliteInt.h" */ /* #include */ +/* #include "vdbeInt.h" */ #ifndef SQLITE_AMALGAMATION /* @@ -23442,6 +24593,7 @@ SQLITE_PRIVATE void sqlite3UtfSelfTest(void){ ** strings, and stuff like that. ** */ +/* #include "sqliteInt.h" */ /* #include */ #if HAVE_ISNAN || SQLITE_HAVE_ISNAN # include @@ -23531,10 +24683,8 @@ SQLITE_PRIVATE int sqlite3IsNaN(double x){ ** than 1GiB) the value returned might be less than the true string length. */ SQLITE_PRIVATE int sqlite3Strlen30(const char *z){ - const char *z2 = z; if( z==0 ) return 0; - while( *z2 ){ z2++; } - return 0x3fffffff & (int)(z2 - z); + return 0x3fffffff & (int)strlen(z); } /* @@ -24493,11 +25643,8 @@ SQLITE_PRIVATE u8 sqlite3GetVarint32(const unsigned char *p, u32 *v){ ** 64-bit integer. */ SQLITE_PRIVATE int sqlite3VarintLen(u64 v){ - int i = 0; - do{ - i++; - v >>= 7; - }while( v!=0 && ALWAYS(i<9) ); + int i; + for(i=1; (v >>= 7)!=0; i++){ assert( i<9 ); } return i; } @@ -24506,14 +25653,40 @@ SQLITE_PRIVATE int sqlite3VarintLen(u64 v){ ** Read or write a four-byte big-endian integer value. */ SQLITE_PRIVATE u32 sqlite3Get4byte(const u8 *p){ +#if SQLITE_BYTEORDER==4321 + u32 x; + memcpy(&x,p,4); + return x; +#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ + && defined(__GNUC__) && GCC_VERSION>=4003000 + u32 x; + memcpy(&x,p,4); + return __builtin_bswap32(x); +#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ + && defined(_MSC_VER) && _MSC_VER>=1300 + u32 x; + memcpy(&x,p,4); + return _byteswap_ulong(x); +#else testcase( p[0]&0x80 ); return ((unsigned)p[0]<<24) | (p[1]<<16) | (p[2]<<8) | p[3]; +#endif } SQLITE_PRIVATE void sqlite3Put4byte(unsigned char *p, u32 v){ +#if SQLITE_BYTEORDER==4321 + memcpy(p,&v,4); +#elif SQLITE_BYTEORDER==1234 && defined(__GNUC__) && GCC_VERSION>=4003000 + u32 x = __builtin_bswap32(v); + memcpy(p,&x,4); +#elif SQLITE_BYTEORDER==1234 && defined(_MSC_VER) && _MSC_VER>=1300 + u32 x = _byteswap_ulong(v); + memcpy(p,&x,4); +#else p[0] = (u8)(v>>24); p[1] = (u8)(v>>16); p[2] = (u8)(v>>8); p[3] = (u8)v; +#endif } @@ -24816,6 +25989,7 @@ SQLITE_PRIVATE u64 sqlite3LogEstToInt(LogEst x){ ** This is the implementation of generic hash-tables ** used in SQLite. */ +/* #include "sqliteInt.h" */ /* #include */ /* Turn bulk memory into a hash table object by initializing the @@ -25081,42 +26255,42 @@ SQLITE_PRIVATE void *sqlite3HashInsert(Hash *pH, const char *pKey, void *data){ #endif SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ static const char *const azName[] = { "?", - /* 1 */ "Function" OpHelp("r[P3]=func(r[P2@P5])"), - /* 2 */ "Savepoint" OpHelp(""), - /* 3 */ "AutoCommit" OpHelp(""), - /* 4 */ "Transaction" OpHelp(""), - /* 5 */ "SorterNext" OpHelp(""), - /* 6 */ "PrevIfOpen" OpHelp(""), - /* 7 */ "NextIfOpen" OpHelp(""), - /* 8 */ "Prev" OpHelp(""), - /* 9 */ "Next" OpHelp(""), - /* 10 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 11 */ "Checkpoint" OpHelp(""), - /* 12 */ "JournalMode" OpHelp(""), - /* 13 */ "Vacuum" OpHelp(""), - /* 14 */ "VFilter" OpHelp("iplan=r[P3] zplan='P4'"), - /* 15 */ "VUpdate" OpHelp("data=r[P3@P2]"), - /* 16 */ "Goto" OpHelp(""), - /* 17 */ "Gosub" OpHelp(""), - /* 18 */ "Return" OpHelp(""), + /* 1 */ "Savepoint" OpHelp(""), + /* 2 */ "AutoCommit" OpHelp(""), + /* 3 */ "Transaction" OpHelp(""), + /* 4 */ "SorterNext" OpHelp(""), + /* 5 */ "PrevIfOpen" OpHelp(""), + /* 6 */ "NextIfOpen" OpHelp(""), + /* 7 */ "Prev" OpHelp(""), + /* 8 */ "Next" OpHelp(""), + /* 9 */ "Checkpoint" OpHelp(""), + /* 10 */ "JournalMode" OpHelp(""), + /* 11 */ "Vacuum" OpHelp(""), + /* 12 */ "VFilter" OpHelp("iplan=r[P3] zplan='P4'"), + /* 13 */ "VUpdate" OpHelp("data=r[P3@P2]"), + /* 14 */ "Goto" OpHelp(""), + /* 15 */ "Gosub" OpHelp(""), + /* 16 */ "Return" OpHelp(""), + /* 17 */ "InitCoroutine" OpHelp(""), + /* 18 */ "EndCoroutine" OpHelp(""), /* 19 */ "Not" OpHelp("r[P2]= !r[P1]"), - /* 20 */ "InitCoroutine" OpHelp(""), - /* 21 */ "EndCoroutine" OpHelp(""), - /* 22 */ "Yield" OpHelp(""), - /* 23 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), - /* 24 */ "Halt" OpHelp(""), - /* 25 */ "Integer" OpHelp("r[P2]=P1"), - /* 26 */ "Int64" OpHelp("r[P2]=P4"), - /* 27 */ "String" OpHelp("r[P2]='P4' (len=P1)"), - /* 28 */ "Null" OpHelp("r[P2..P3]=NULL"), - /* 29 */ "SoftNull" OpHelp("r[P1]=NULL"), - /* 30 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), - /* 31 */ "Variable" OpHelp("r[P2]=parameter(P1,P4)"), - /* 32 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), - /* 33 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), - /* 34 */ "SCopy" OpHelp("r[P2]=r[P1]"), - /* 35 */ "ResultRow" OpHelp("output=r[P1@P2]"), - /* 36 */ "CollSeq" OpHelp(""), + /* 20 */ "Yield" OpHelp(""), + /* 21 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), + /* 22 */ "Halt" OpHelp(""), + /* 23 */ "Integer" OpHelp("r[P2]=P1"), + /* 24 */ "Int64" OpHelp("r[P2]=P4"), + /* 25 */ "String" OpHelp("r[P2]='P4' (len=P1)"), + /* 26 */ "Null" OpHelp("r[P2..P3]=NULL"), + /* 27 */ "SoftNull" OpHelp("r[P1]=NULL"), + /* 28 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), + /* 29 */ "Variable" OpHelp("r[P2]=parameter(P1,P4)"), + /* 30 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), + /* 31 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), + /* 32 */ "SCopy" OpHelp("r[P2]=r[P1]"), + /* 33 */ "ResultRow" OpHelp("output=r[P1@P2]"), + /* 34 */ "CollSeq" OpHelp(""), + /* 35 */ "Function0" OpHelp("r[P3]=func(r[P2@P5])"), + /* 36 */ "Function" OpHelp("r[P3]=func(r[P2@P5])"), /* 37 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), /* 38 */ "MustBeInt" OpHelp(""), /* 39 */ "RealAffinity" OpHelp(""), @@ -25142,20 +26316,20 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 59 */ "SequenceTest" OpHelp("if( cursor[P1].ctr++ ) pc = P2"), /* 60 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), /* 61 */ "Close" OpHelp(""), - /* 62 */ "SeekLT" OpHelp("key=r[P3@P4]"), - /* 63 */ "SeekLE" OpHelp("key=r[P3@P4]"), - /* 64 */ "SeekGE" OpHelp("key=r[P3@P4]"), - /* 65 */ "SeekGT" OpHelp("key=r[P3@P4]"), - /* 66 */ "Seek" OpHelp("intkey=r[P2]"), - /* 67 */ "NoConflict" OpHelp("key=r[P3@P4]"), - /* 68 */ "NotFound" OpHelp("key=r[P3@P4]"), - /* 69 */ "Found" OpHelp("key=r[P3@P4]"), - /* 70 */ "NotExists" OpHelp("intkey=r[P3]"), + /* 62 */ "ColumnsUsed" OpHelp(""), + /* 63 */ "SeekLT" OpHelp("key=r[P3@P4]"), + /* 64 */ "SeekLE" OpHelp("key=r[P3@P4]"), + /* 65 */ "SeekGE" OpHelp("key=r[P3@P4]"), + /* 66 */ "SeekGT" OpHelp("key=r[P3@P4]"), + /* 67 */ "Seek" OpHelp("intkey=r[P2]"), + /* 68 */ "NoConflict" OpHelp("key=r[P3@P4]"), + /* 69 */ "NotFound" OpHelp("key=r[P3@P4]"), + /* 70 */ "Found" OpHelp("key=r[P3@P4]"), /* 71 */ "Or" OpHelp("r[P3]=(r[P1] || r[P2])"), /* 72 */ "And" OpHelp("r[P3]=(r[P1] && r[P2])"), - /* 73 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), - /* 74 */ "NewRowid" OpHelp("r[P2]=rowid"), - /* 75 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), + /* 73 */ "NotExists" OpHelp("intkey=r[P3]"), + /* 74 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), + /* 75 */ "NewRowid" OpHelp("r[P2]=rowid"), /* 76 */ "IsNull" OpHelp("if r[P1]==NULL goto P2"), /* 77 */ "NotNull" OpHelp("if r[P1]!=NULL goto P2"), /* 78 */ "Ne" OpHelp("if r[P1]!=r[P3] goto P2"), @@ -25164,7 +26338,7 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 81 */ "Le" OpHelp("if r[P1]<=r[P3] goto P2"), /* 82 */ "Lt" OpHelp("if r[P1]=r[P3] goto P2"), - /* 84 */ "InsertInt" OpHelp("intkey=P3 data=r[P2]"), + /* 84 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), /* 85 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), /* 86 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), /* 87 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<0 goto P2"), - /* 138 */ "IfNeg" OpHelp("r[P1]+=P3, if r[P1]<0 goto P2"), - /* 139 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]+=P3, goto P2"), - /* 140 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), - /* 141 */ "JumpZeroIncr" OpHelp("if (r[P1]++)==0 ) goto P2"), - /* 142 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), - /* 143 */ "IncrVacuum" OpHelp(""), - /* 144 */ "Expire" OpHelp(""), - /* 145 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), - /* 146 */ "VBegin" OpHelp(""), - /* 147 */ "VCreate" OpHelp(""), - /* 148 */ "VDestroy" OpHelp(""), - /* 149 */ "VOpen" OpHelp(""), - /* 150 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), - /* 151 */ "VNext" OpHelp(""), - /* 152 */ "VRename" OpHelp(""), - /* 153 */ "Pagecount" OpHelp(""), - /* 154 */ "MaxPgcnt" OpHelp(""), - /* 155 */ "Init" OpHelp("Start at P2"), - /* 156 */ "Noop" OpHelp(""), - /* 157 */ "Explain" OpHelp(""), + /* 134 */ "Param" OpHelp(""), + /* 135 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), + /* 136 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), + /* 137 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), + /* 138 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), + /* 139 */ "SetIfNotPos" OpHelp("if r[P1]<=0 then r[P2]=P3"), + /* 140 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]-=P3, goto P2"), + /* 141 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), + /* 142 */ "JumpZeroIncr" OpHelp("if (r[P1]++)==0 ) goto P2"), + /* 143 */ "AggStep0" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 144 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 145 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), + /* 146 */ "IncrVacuum" OpHelp(""), + /* 147 */ "Expire" OpHelp(""), + /* 148 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), + /* 149 */ "VBegin" OpHelp(""), + /* 150 */ "VCreate" OpHelp(""), + /* 151 */ "VDestroy" OpHelp(""), + /* 152 */ "VOpen" OpHelp(""), + /* 153 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), + /* 154 */ "VNext" OpHelp(""), + /* 155 */ "VRename" OpHelp(""), + /* 156 */ "Pagecount" OpHelp(""), + /* 157 */ "MaxPgcnt" OpHelp(""), + /* 158 */ "Init" OpHelp("Start at P2"), + /* 159 */ "Noop" OpHelp(""), + /* 160 */ "Explain" OpHelp(""), }; return azName[i]; } @@ -25290,6 +26467,7 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ ** * Definitions of sqlite3_vfs objects for all locking methods ** plus implementations of sqlite3_os_init() and sqlite3_os_end(). */ +/* #include "sqliteInt.h" */ #if SQLITE_OS_UNIX /* This file is used on unix only */ /* @@ -26073,14 +27251,14 @@ static int robust_open(const char *z, int f, mode_t m){ ** unixEnterLeave() */ static void unixEnterMutex(void){ - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); } static void unixLeaveMutex(void){ - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); } #ifdef SQLITE_DEBUG static int unixMutexHeld(void) { - return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); } #endif @@ -28590,7 +29768,6 @@ static int seekAndRead(unixFile *id, sqlite3_int64 offset, void *pBuf, int cnt){ TIMER_START; assert( cnt==(cnt&0x1ffff) ); assert( id->h>2 ); - cnt &= 0x1ffff; do{ #if defined(USE_PREAD) got = osPread(id->h, pBuf, cnt, offset); @@ -28657,7 +29834,7 @@ static int unixRead( #endif #if SQLITE_MAX_MMAP_SIZE>0 - /* Deal with as much of this read request as possible by transferring + /* Deal with as much of this read request as possible by transfering ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ @@ -28792,7 +29969,7 @@ static int unixWrite( #endif #if SQLITE_MAX_MMAP_SIZE>0 - /* Deal with as much of this write request as possible by transferring + /* Deal with as much of this write request as possible by transfering ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ @@ -28807,8 +29984,8 @@ static int unixWrite( } } #endif - - while( amt>0 && (wrote = seekAndWrite(pFile, offset, pBuf, amt))>0 ){ + + while( (wrote = seekAndWrite(pFile, offset, pBuf, amt))0 ){ amt -= wrote; offset += wrote; pBuf = &((char*)pBuf)[wrote]; @@ -28816,7 +29993,7 @@ static int unixWrite( SimulateIOError(( wrote=(-1), amt=1 )); SimulateDiskfullError(( wrote=0, amt=1 )); - if( amt>0 ){ + if( amt>wrote ){ if( wrote<0 && pFile->lastErrno!=ENOSPC ){ /* lastErrno set by seekAndWrite */ return SQLITE_IOERR_WRITE; @@ -30105,7 +31282,8 @@ static void unixShmBarrier( sqlite3_file *fd /* Database file holding the shared memory */ ){ UNUSED_PARAMETER(fd); - unixEnterMutex(); + sqlite3MemoryBarrier(); /* compiler-defined memory barrier */ + unixEnterMutex(); /* Also mutex, for redundancy */ unixLeaveMutex(); } @@ -33024,6 +34202,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_os_end(void){ ** ** This file contains code that is specific to Windows. */ +/* #include "sqliteInt.h" */ #if SQLITE_OS_WIN /* This file is used for Windows only */ /* @@ -33232,6 +34411,7 @@ SQLITE_API int sqlite3_open_file_count = 0; /* ** Include the header file for the Windows VFS. */ +/* #include "os_win.h" */ /* ** Compiling and using WAL mode requires several APIs that are only @@ -35733,7 +36913,7 @@ static int winRead( pFile->h, pBuf, amt, offset, pFile->locktype)); #if SQLITE_MAX_MMAP_SIZE>0 - /* Deal with as much of this read request as possible by transferring + /* Deal with as much of this read request as possible by transfering ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ @@ -35811,7 +36991,7 @@ static int winWrite( pFile->h, pBuf, amt, offset, pFile->locktype)); #if SQLITE_MAX_MMAP_SIZE>0 - /* Deal with as much of this write request as possible by transferring + /* Deal with as much of this write request as possible by transfering ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ @@ -36223,6 +37403,12 @@ static int winLock(sqlite3_file *id, int locktype){ return SQLITE_OK; } + /* Do not allow any kind of write-lock on a read-only database + */ + if( (pFile->ctrlFlags & WINFILE_RDONLY)!=0 && locktype>=RESERVED_LOCK ){ + return SQLITE_IOERR_LOCK; + } + /* Make sure the locking sequence is correct */ assert( pFile->locktype!=NO_LOCK || locktype==SHARED_LOCK ); @@ -36592,14 +37778,14 @@ static SYSTEM_INFO winSysInfo; ** winShmLeaveMutex() */ static void winShmEnterMutex(void){ - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); } static void winShmLeaveMutex(void){ - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); } #ifndef NDEBUG static int winShmMutexHeld(void) { - return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); } #endif @@ -37052,8 +38238,8 @@ static void winShmBarrier( sqlite3_file *fd /* Database holding the shared memory */ ){ UNUSED_PARAMETER(fd); - /* MemoryBarrier(); // does not work -- do not know why not */ - winShmEnterMutex(); + sqlite3MemoryBarrier(); /* compiler-defined memory barrier */ + winShmEnterMutex(); /* Also mutex, for redundancy */ winShmLeaveMutex(); } @@ -38619,14 +39805,14 @@ static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ UUID id; memset(&id, 0, sizeof(UUID)); osUuidCreate(&id); - memcpy(zBuf, &id, sizeof(UUID)); + memcpy(&zBuf[n], &id, sizeof(UUID)); n += sizeof(UUID); } if( sizeof(UUID)<=nBuf-n ){ UUID id; memset(&id, 0, sizeof(UUID)); osUuidCreateSequential(&id); - memcpy(zBuf, &id, sizeof(UUID)); + memcpy(&zBuf[n], &id, sizeof(UUID)); n += sizeof(UUID); } #endif @@ -38879,6 +40065,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_os_end(void){ ** start of a transaction, and is thus usually less than a few thousand, ** but can be as large as 2 billion for a really big database. */ +/* #include "sqliteInt.h" */ /* Size of the Bitvec structure in bytes. */ #define BITVEC_SZ 512 @@ -38970,10 +40157,10 @@ SQLITE_PRIVATE Bitvec *sqlite3BitvecCreate(u32 iSize){ ** If p is NULL (if the bitmap has not been created) or if ** i is out of range, then return false. */ -SQLITE_PRIVATE int sqlite3BitvecTest(Bitvec *p, u32 i){ - if( p==0 ) return 0; - if( i>p->iSize || i==0 ) return 0; +SQLITE_PRIVATE int sqlite3BitvecTestNotNull(Bitvec *p, u32 i){ + assert( p!=0 ); i--; + if( i>=p->iSize ) return 0; while( p->iDivisor ){ u32 bin = i/p->iDivisor; i = i%p->iDivisor; @@ -38993,6 +40180,9 @@ SQLITE_PRIVATE int sqlite3BitvecTest(Bitvec *p, u32 i){ return 0; } } +SQLITE_PRIVATE int sqlite3BitvecTest(Bitvec *p, u32 i){ + return p!=0 && sqlite3BitvecTestNotNull(p,i); +} /* ** Set the i-th bit. Return 0 on success and an error code if @@ -39265,6 +40455,7 @@ bitvec_end: ************************************************************************* ** This file implements that page cache. */ +/* #include "sqliteInt.h" */ /* ** A complete page cache is an instance of this structure. @@ -39272,7 +40463,7 @@ bitvec_end: struct PCache { PgHdr *pDirty, *pDirtyTail; /* List of dirty pages in LRU order */ PgHdr *pSynced; /* Last synced page in dirty page list */ - int nRef; /* Number of referenced pages */ + int nRefSum; /* Sum of ref counts over all pages */ int szCache; /* Configured cache size */ int szPage; /* Size of every page in this cache */ int szExtra; /* Size of extra space for each page */ @@ -39281,7 +40472,6 @@ struct PCache { int (*xStress)(void*,PgHdr*); /* Call to try make a page clean */ void *pStress; /* Argument to xStress */ sqlite3_pcache *pCache; /* Pluggable cache module */ - PgHdr *pPage1; /* Reference to page 1 */ }; /********************************** Linked List Management ********************/ @@ -39359,9 +40549,6 @@ static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){ */ static void pcacheUnpin(PgHdr *p){ if( p->pCache->bPurgeable ){ - if( p->pgno==1 ){ - p->pCache->pPage1 = 0; - } sqlite3GlobalConfig.pcache2.xUnpin(p->pCache->pCache, p->pPage, 0); } } @@ -39441,7 +40628,7 @@ SQLITE_PRIVATE int sqlite3PcacheOpen( ** are no outstanding page references when this function is called. */ SQLITE_PRIVATE int sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ - assert( pCache->nRef==0 && pCache->pDirty==0 ); + assert( pCache->nRefSum==0 && pCache->pDirty==0 ); if( pCache->szPage ){ sqlite3_pcache *pNew; pNew = sqlite3GlobalConfig.pcache2.xCreate( @@ -39454,7 +40641,6 @@ SQLITE_PRIVATE int sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); } pCache->pCache = pNew; - pCache->pPage1 = 0; pCache->szPage = szPage; } return SQLITE_OK; @@ -39579,13 +40765,14 @@ static SQLITE_NOINLINE PgHdr *pcacheFetchFinishWithInit( assert( pPage!=0 ); pPgHdr = (PgHdr*)pPage->pExtra; assert( pPgHdr->pPage==0 ); - memset(pPgHdr, 0, sizeof(PgHdr)); + memset(pPgHdr, 0, sizeof(PgHdr)); pPgHdr->pPage = pPage; pPgHdr->pData = pPage->pBuf; pPgHdr->pExtra = (void *)&pPgHdr[1]; memset(pPgHdr->pExtra, 0, pCache->szExtra); pPgHdr->pCache = pCache; pPgHdr->pgno = pgno; + pPgHdr->flags = PGHDR_CLEAN; return sqlite3PcacheFetchFinish(pCache,pgno,pPage); } @@ -39602,19 +40789,14 @@ SQLITE_PRIVATE PgHdr *sqlite3PcacheFetchFinish( ){ PgHdr *pPgHdr; - if( pPage==0 ) return 0; + assert( pPage!=0 ); pPgHdr = (PgHdr *)pPage->pExtra; if( !pPgHdr->pPage ){ return pcacheFetchFinishWithInit(pCache, pgno, pPage); } - if( 0==pPgHdr->nRef ){ - pCache->nRef++; - } + pCache->nRefSum++; pPgHdr->nRef++; - if( pgno==1 ){ - pCache->pPage1 = pPgHdr; - } return pPgHdr; } @@ -39624,10 +40806,9 @@ SQLITE_PRIVATE PgHdr *sqlite3PcacheFetchFinish( */ SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ assert( p->nRef>0 ); - p->nRef--; - if( p->nRef==0 ){ - p->pCache->nRef--; - if( (p->flags&PGHDR_DIRTY)==0 ){ + p->pCache->nRefSum--; + if( (--p->nRef)==0 ){ + if( p->flags&PGHDR_CLEAN ){ pcacheUnpin(p); }else if( p->pDirtyPrev!=0 ){ /* Move the page to the head of the dirty list. */ @@ -39642,6 +40823,7 @@ SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ SQLITE_PRIVATE void sqlite3PcacheRef(PgHdr *p){ assert(p->nRef>0); p->nRef++; + p->pCache->nRefSum++; } /* @@ -39654,10 +40836,7 @@ SQLITE_PRIVATE void sqlite3PcacheDrop(PgHdr *p){ if( p->flags&PGHDR_DIRTY ){ pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); } - p->pCache->nRef--; - if( p->pgno==1 ){ - p->pCache->pPage1 = 0; - } + p->pCache->nRefSum--; sqlite3GlobalConfig.pcache2.xUnpin(p->pCache->pCache, p->pPage, 1); } @@ -39666,11 +40845,14 @@ SQLITE_PRIVATE void sqlite3PcacheDrop(PgHdr *p){ ** make it so. */ SQLITE_PRIVATE void sqlite3PcacheMakeDirty(PgHdr *p){ - p->flags &= ~PGHDR_DONT_WRITE; assert( p->nRef>0 ); - if( 0==(p->flags & PGHDR_DIRTY) ){ - p->flags |= PGHDR_DIRTY; - pcacheManageDirtyList(p, PCACHE_DIRTYLIST_ADD); + if( p->flags & (PGHDR_CLEAN|PGHDR_DONT_WRITE) ){ + p->flags &= ~PGHDR_DONT_WRITE; + if( p->flags & PGHDR_CLEAN ){ + p->flags ^= (PGHDR_DIRTY|PGHDR_CLEAN); + assert( (p->flags & (PGHDR_DIRTY|PGHDR_CLEAN))==PGHDR_DIRTY ); + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_ADD); + } } } @@ -39680,8 +40862,10 @@ SQLITE_PRIVATE void sqlite3PcacheMakeDirty(PgHdr *p){ */ SQLITE_PRIVATE void sqlite3PcacheMakeClean(PgHdr *p){ if( (p->flags & PGHDR_DIRTY) ){ + assert( (p->flags & PGHDR_CLEAN)==0 ); pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); - p->flags &= ~(PGHDR_DIRTY|PGHDR_NEED_SYNC); + p->flags &= ~(PGHDR_DIRTY|PGHDR_NEED_SYNC|PGHDR_WRITEABLE); + p->flags |= PGHDR_CLEAN; if( p->nRef==0 ){ pcacheUnpin(p); } @@ -39748,9 +40932,14 @@ SQLITE_PRIVATE void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ sqlite3PcacheMakeClean(p); } } - if( pgno==0 && pCache->pPage1 ){ - memset(pCache->pPage1->pData, 0, pCache->szPage); - pgno = 1; + if( pgno==0 && pCache->nRefSum ){ + sqlite3_pcache_page *pPage1; + pPage1 = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache,1,0); + if( ALWAYS(pPage1) ){ /* Page 1 is always available in cache, because + ** pCache->nRefSum>0 */ + memset(pPage1->pBuf, 0, pCache->szPage); + pgno = 1; + } } sqlite3GlobalConfig.pcache2.xTruncate(pCache->pCache, pgno+1); } @@ -39853,10 +41042,13 @@ SQLITE_PRIVATE PgHdr *sqlite3PcacheDirtyList(PCache *pCache){ } /* -** Return the total number of referenced pages held by the cache. +** Return the total number of references to all pages held by the cache. +** +** This is not the total number of pages referenced, but the sum of the +** reference count for all pages. */ SQLITE_PRIVATE int sqlite3PcacheRefCount(PCache *pCache){ - return pCache->nRef; + return pCache->nRefSum; } /* @@ -39941,14 +41133,96 @@ SQLITE_PRIVATE void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHd ** of the SQLITE_CONFIG_PAGECACHE and sqlite3_release_memory() features. ** If the default page cache implementation is overridden, then neither of ** these two features are available. +** +** A Page cache line looks like this: +** +** ------------------------------------------------------------- +** | database page content | PgHdr1 | MemPage | PgHdr | +** ------------------------------------------------------------- +** +** The database page content is up front (so that buffer overreads tend to +** flow harmlessly into the PgHdr1, MemPage, and PgHdr extensions). MemPage +** is the extension added by the btree.c module containing information such +** as the database page number and how that database page is used. PgHdr +** is added by the pcache.c layer and contains information used to keep track +** of which pages are "dirty". PgHdr1 is an extension added by this +** module (pcache1.c). The PgHdr1 header is a subclass of sqlite3_pcache_page. +** PgHdr1 contains information needed to look up a page by its page number. +** The superclass sqlite3_pcache_page.pBuf points to the start of the +** database page content and sqlite3_pcache_page.pExtra points to PgHdr. +** +** The size of the extension (MemPage+PgHdr+PgHdr1) can be determined at +** runtime using sqlite3_config(SQLITE_CONFIG_PCACHE_HDRSZ, &size). The +** sizes of the extensions sum to 272 bytes on x64 for 3.8.10, but this +** size can vary according to architecture, compile-time options, and +** SQLite library version number. +** +** If SQLITE_PCACHE_SEPARATE_HEADER is defined, then the extension is obtained +** using a separate memory allocation from the database page content. This +** seeks to overcome the "clownshoe" problem (also called "internal +** fragmentation" in academic literature) of allocating a few bytes more +** than a power of two with the memory allocator rounding up to the next +** power of two, and leaving the rounded-up space unused. +** +** This module tracks pointers to PgHdr1 objects. Only pcache.c communicates +** with this module. Information is passed back and forth as PgHdr1 pointers. +** +** The pcache.c and pager.c modules deal pointers to PgHdr objects. +** The btree.c module deals with pointers to MemPage objects. +** +** SOURCE OF PAGE CACHE MEMORY: +** +** Memory for a page might come from any of three sources: +** +** (1) The general-purpose memory allocator - sqlite3Malloc() +** (2) Global page-cache memory provided using sqlite3_config() with +** SQLITE_CONFIG_PAGECACHE. +** (3) PCache-local bulk allocation. +** +** The third case is a chunk of heap memory (defaulting to 100 pages worth) +** that is allocated when the page cache is created. The size of the local +** bulk allocation can be adjusted using +** +** sqlite3_config(SQLITE_CONFIG_PAGECACHE, 0, 0, N). +** +** If N is positive, then N pages worth of memory are allocated using a single +** sqlite3Malloc() call and that memory is used for the first N pages allocated. +** Or if N is negative, then -1024*N bytes of memory are allocated and used +** for as many pages as can be accomodated. +** +** Only one of (2) or (3) can be used. Once the memory available to (2) or +** (3) is exhausted, subsequent allocations fail over to the general-purpose +** memory allocator (1). +** +** Earlier versions of SQLite used only methods (1) and (2). But experiments +** show that method (3) with N==100 provides about a 5% performance boost for +** common workloads. */ - +/* #include "sqliteInt.h" */ typedef struct PCache1 PCache1; typedef struct PgHdr1 PgHdr1; typedef struct PgFreeslot PgFreeslot; typedef struct PGroup PGroup; +/* +** Each cache entry is represented by an instance of the following +** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of +** PgHdr1.pCache->szPage bytes is allocated directly before this structure +** in memory. +*/ +struct PgHdr1 { + sqlite3_pcache_page page; /* Base class. Must be first. pBuf & pExtra */ + unsigned int iKey; /* Key value (page number) */ + u8 isPinned; /* Page in use, not on the LRU list */ + u8 isBulkLocal; /* This page from bulk local storage */ + u8 isAnchor; /* This is the PGroup.lru element */ + PgHdr1 *pNext; /* Next in hash table chain */ + PCache1 *pCache; /* Cache that currently owns this page */ + PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ + PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ +}; + /* Each page cache (or PCache) belongs to a PGroup. A PGroup is a set ** of one or more PCaches that are able to recycle each other's unpinned ** pages when they are under memory pressure. A PGroup is an instance of @@ -39977,7 +41251,7 @@ struct PGroup { unsigned int nMinPage; /* Sum of nMin for purgeable caches */ unsigned int mxPinned; /* nMaxpage + 10 - nMinPage */ unsigned int nCurrentPage; /* Number of purgeable pages allocated */ - PgHdr1 *pLruHead, *pLruTail; /* LRU list of unpinned pages */ + PgHdr1 lru; /* The beginning and end of the LRU list */ }; /* Each page cache is an instance of the following object. Every @@ -39995,8 +41269,9 @@ struct PCache1 { ** The PGroup mutex must be held when accessing nMax. */ PGroup *pGroup; /* PGroup this cache belongs to */ - int szPage; /* Size of allocated pages in bytes */ - int szExtra; /* Size of extra space in bytes */ + int szPage; /* Size of database content section */ + int szExtra; /* sizeof(MemPage)+sizeof(PgHdr) */ + int szAlloc; /* Total size of one pcache line */ int bPurgeable; /* True if cache is purgeable */ unsigned int nMin; /* Minimum number of pages reserved */ unsigned int nMax; /* Configured "cache_size" value */ @@ -40010,27 +41285,13 @@ struct PCache1 { unsigned int nPage; /* Total number of pages in apHash */ unsigned int nHash; /* Number of slots in apHash[] */ PgHdr1 **apHash; /* Hash table for fast lookup by key */ + PgHdr1 *pFree; /* List of unused pcache-local pages */ + void *pBulk; /* Bulk memory used by pcache-local */ }; /* -** Each cache entry is represented by an instance of the following -** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of -** PgHdr1.pCache->szPage bytes is allocated directly before this structure -** in memory. -*/ -struct PgHdr1 { - sqlite3_pcache_page page; - unsigned int iKey; /* Key value (page number) */ - u8 isPinned; /* Page in use, not on the LRU list */ - PgHdr1 *pNext; /* Next in hash table chain */ - PCache1 *pCache; /* Cache that currently owns this page */ - PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ - PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ -}; - -/* -** Free slots in the allocator used to divide up the buffer provided using -** the SQLITE_CONFIG_PAGECACHE mechanism. +** Free slots in the allocator used to divide up the global page cache +** buffer provided using the SQLITE_CONFIG_PAGECACHE mechanism. */ struct PgFreeslot { PgFreeslot *pNext; /* Next free slot */ @@ -40048,10 +41309,12 @@ static SQLITE_WSD struct PCacheGlobal { ** The nFreeSlot and pFree values do require mutex protection. */ int isInit; /* True if initialized */ + int separateCache; /* Use a new PGroup for each PCache */ + int nInitPage; /* Initial bulk allocation size */ int szSlot; /* Size of each free slot */ int nSlot; /* The number of pcache slots */ int nReserve; /* Try to keep nFreeSlot above this */ - void *pStart, *pEnd; /* Bounds of pagecache malloc range */ + void *pStart, *pEnd; /* Bounds of global page cache memory */ /* Above requires no mutex. Use mutex below for variable that follow. */ sqlite3_mutex *mutex; /* Mutex for accessing the following: */ PgFreeslot *pFree; /* Free page blocks */ @@ -40073,12 +41336,20 @@ static SQLITE_WSD struct PCacheGlobal { /* ** Macros to enter and leave the PCache LRU mutex. */ -#define pcache1EnterMutex(X) sqlite3_mutex_enter((X)->mutex) -#define pcache1LeaveMutex(X) sqlite3_mutex_leave((X)->mutex) +#if !defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) || SQLITE_THREADSAFE==0 +# define pcache1EnterMutex(X) assert((X)->mutex==0) +# define pcache1LeaveMutex(X) assert((X)->mutex==0) +# define PCACHE1_MIGHT_USE_GROUP_MUTEX 0 +#else +# define pcache1EnterMutex(X) sqlite3_mutex_enter((X)->mutex) +# define pcache1LeaveMutex(X) sqlite3_mutex_leave((X)->mutex) +# define PCACHE1_MIGHT_USE_GROUP_MUTEX 1 +#endif /******************************************************************************/ /******** Page Allocation/SQLITE_CONFIG_PCACHE Related Functions **************/ + /* ** This function is called during initialization if a static buffer is ** supplied to use for the page-cache by passing the SQLITE_CONFIG_PAGECACHE @@ -40091,6 +41362,7 @@ static SQLITE_WSD struct PCacheGlobal { SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ if( pcache1.isInit ){ PgFreeslot *p; + if( pBuf==0 ) sz = n = 0; sz = ROUNDDOWN8(sz); pcache1.szSlot = sz; pcache1.nSlot = pcache1.nFreeSlot = n; @@ -40108,6 +41380,44 @@ SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ } } +/* +** Try to initialize the pCache->pFree and pCache->pBulk fields. Return +** true if pCache->pFree ends up containing one or more free pages. +*/ +static int pcache1InitBulk(PCache1 *pCache){ + i64 szBulk; + char *zBulk; + if( pcache1.nInitPage==0 ) return 0; + /* Do not bother with a bulk allocation if the cache size very small */ + if( pCache->nMax<3 ) return 0; + sqlite3BeginBenignMalloc(); + if( pcache1.nInitPage>0 ){ + szBulk = pCache->szAlloc * (i64)pcache1.nInitPage; + }else{ + szBulk = -1024 * (i64)pcache1.nInitPage; + } + if( szBulk > pCache->szAlloc*(i64)pCache->nMax ){ + szBulk = pCache->szAlloc*pCache->nMax; + } + zBulk = pCache->pBulk = sqlite3Malloc( szBulk ); + sqlite3EndBenignMalloc(); + if( zBulk ){ + int nBulk = sqlite3MallocSize(zBulk)/pCache->szAlloc; + int i; + for(i=0; iszPage]; + pX->page.pBuf = zBulk; + pX->page.pExtra = &pX[1]; + pX->isBulkLocal = 1; + pX->isAnchor = 0; + pX->pNext = pCache->pFree; + pCache->pFree = pX; + zBulk += pCache->szAlloc; + } + } + return pCache->pFree!=0; +} + /* ** Malloc function used within this file to allocate space from the buffer ** configured using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no @@ -40155,9 +41465,9 @@ static void *pcache1Alloc(int nByte){ /* ** Free an allocated buffer obtained from pcache1Alloc(). */ -static int pcache1Free(void *p){ +static void pcache1Free(void *p){ int nFreed = 0; - if( p==0 ) return 0; + if( p==0 ) return; if( p>=pcache1.pStart && ppGroup->mutex) ); - pcache1LeaveMutex(pCache->pGroup); + if( pCache->pFree || (pCache->nPage==0 && pcache1InitBulk(pCache)) ){ + p = pCache->pFree; + pCache->pFree = p->pNext; + p->pNext = 0; + }else{ +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + /* The group mutex must be released before pcache1Alloc() is called. This + ** is because it might call sqlite3_release_memory(), which assumes that + ** this mutex is not held. */ + assert( pcache1.separateCache==0 ); + assert( pCache->pGroup==&pcache1.grp ); + pcache1LeaveMutex(pCache->pGroup); +#endif + if( benignMalloc ){ sqlite3BeginBenignMalloc(); } #ifdef SQLITE_PCACHE_SEPARATE_HEADER - pPg = pcache1Alloc(pCache->szPage); - p = sqlite3Malloc(sizeof(PgHdr1) + pCache->szExtra); - if( !pPg || !p ){ - pcache1Free(pPg); - sqlite3_free(p); - pPg = 0; - } + pPg = pcache1Alloc(pCache->szPage); + p = sqlite3Malloc(sizeof(PgHdr1) + pCache->szExtra); + if( !pPg || !p ){ + pcache1Free(pPg); + sqlite3_free(p); + pPg = 0; + } #else - pPg = pcache1Alloc(ROUND8(sizeof(PgHdr1)) + pCache->szPage + pCache->szExtra); - p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; + pPg = pcache1Alloc(pCache->szAlloc); + p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; #endif - pcache1EnterMutex(pCache->pGroup); - - if( pPg ){ + if( benignMalloc ){ sqlite3EndBenignMalloc(); } +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + pcache1EnterMutex(pCache->pGroup); +#endif + if( pPg==0 ) return 0; p->page.pBuf = pPg; p->page.pExtra = &p[1]; - if( pCache->bPurgeable ){ - pCache->pGroup->nCurrentPage++; - } - return p; + p->isBulkLocal = 0; + p->isAnchor = 0; } - return 0; + if( pCache->bPurgeable ){ + pCache->pGroup->nCurrentPage++; + } + return p; } /* ** Free a page object allocated by pcache1AllocPage(). -** -** The pointer is allowed to be NULL, which is prudent. But it turns out -** that the current implementation happens to never call this routine -** with a NULL pointer, so we mark the NULL test with ALWAYS(). */ static void pcache1FreePage(PgHdr1 *p){ - if( ALWAYS(p) ){ - PCache1 *pCache = p->pCache; - assert( sqlite3_mutex_held(p->pCache->pGroup->mutex) ); + PCache1 *pCache; + assert( p!=0 ); + pCache = p->pCache; + assert( sqlite3_mutex_held(p->pCache->pGroup->mutex) ); + if( p->isBulkLocal ){ + p->pNext = pCache->pFree; + pCache->pFree = p; + }else{ pcache1Free(p->page.pBuf); #ifdef SQLITE_PCACHE_SEPARATE_HEADER sqlite3_free(p); #endif - if( pCache->bPurgeable ){ - pCache->pGroup->nCurrentPage--; - } + } + if( pCache->bPurgeable ){ + pCache->pGroup->nCurrentPage--; } } @@ -40350,41 +41673,35 @@ static void pcache1ResizeHash(PCache1 *p){ ** ** The PGroup mutex must be held when this function is called. */ -static void pcache1PinPage(PgHdr1 *pPage){ +static PgHdr1 *pcache1PinPage(PgHdr1 *pPage){ PCache1 *pCache; - PGroup *pGroup; assert( pPage!=0 ); assert( pPage->isPinned==0 ); pCache = pPage->pCache; - pGroup = pCache->pGroup; - assert( pPage->pLruNext || pPage==pGroup->pLruTail ); - assert( pPage->pLruPrev || pPage==pGroup->pLruHead ); - assert( sqlite3_mutex_held(pGroup->mutex) ); - if( pPage->pLruPrev ){ - pPage->pLruPrev->pLruNext = pPage->pLruNext; - }else{ - pGroup->pLruHead = pPage->pLruNext; - } - if( pPage->pLruNext ){ - pPage->pLruNext->pLruPrev = pPage->pLruPrev; - }else{ - pGroup->pLruTail = pPage->pLruPrev; - } + assert( pPage->pLruNext ); + assert( pPage->pLruPrev ); + assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); + pPage->pLruPrev->pLruNext = pPage->pLruNext; + pPage->pLruNext->pLruPrev = pPage->pLruPrev; pPage->pLruNext = 0; pPage->pLruPrev = 0; pPage->isPinned = 1; + assert( pPage->isAnchor==0 ); + assert( pCache->pGroup->lru.isAnchor==1 ); pCache->nRecyclable--; + return pPage; } /* ** Remove the page supplied as an argument from the hash table ** (PCache1.apHash structure) that it is currently stored in. +** Also free the page if freePage is true. ** ** The PGroup mutex must be held when this function is called. */ -static void pcache1RemoveFromHash(PgHdr1 *pPage){ +static void pcache1RemoveFromHash(PgHdr1 *pPage, int freeFlag){ unsigned int h; PCache1 *pCache = pPage->pCache; PgHdr1 **pp; @@ -40395,21 +41712,28 @@ static void pcache1RemoveFromHash(PgHdr1 *pPage){ *pp = (*pp)->pNext; pCache->nPage--; + if( freeFlag ) pcache1FreePage(pPage); } /* ** If there are currently more than nMaxPage pages allocated, try ** to recycle pages to reduce the number allocated to nMaxPage. */ -static void pcache1EnforceMaxPage(PGroup *pGroup){ +static void pcache1EnforceMaxPage(PCache1 *pCache){ + PGroup *pGroup = pCache->pGroup; + PgHdr1 *p; assert( sqlite3_mutex_held(pGroup->mutex) ); - while( pGroup->nCurrentPage>pGroup->nMaxPage && pGroup->pLruTail ){ - PgHdr1 *p = pGroup->pLruTail; + while( pGroup->nCurrentPage>pGroup->nMaxPage + && (p=pGroup->lru.pLruPrev)->isAnchor==0 + ){ assert( p->pCache->pGroup==pGroup ); assert( p->isPinned==0 ); pcache1PinPage(p); - pcache1RemoveFromHash(p); - pcache1FreePage(p); + pcache1RemoveFromHash(p, 1); + } + if( pCache->nPage==0 && pCache->pBulk ){ + sqlite3_free(pCache->pBulk); + pCache->pBulk = pCache->pFree = 0; } } @@ -40455,10 +41779,45 @@ static int pcache1Init(void *NotUsed){ UNUSED_PARAMETER(NotUsed); assert( pcache1.isInit==0 ); memset(&pcache1, 0, sizeof(pcache1)); + + + /* + ** The pcache1.separateCache variable is true if each PCache has its own + ** private PGroup (mode-1). pcache1.separateCache is false if the single + ** PGroup in pcache1.grp is used for all page caches (mode-2). + ** + ** * Always use a unified cache (mode-2) if ENABLE_MEMORY_MANAGEMENT + ** + ** * Use a unified cache in single-threaded applications that have + ** configured a start-time buffer for use as page-cache memory using + ** sqlite3_config(SQLITE_CONFIG_PAGECACHE, pBuf, sz, N) with non-NULL + ** pBuf argument. + ** + ** * Otherwise use separate caches (mode-1) + */ +#if defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) + pcache1.separateCache = 0; +#elif SQLITE_THREADSAFE + pcache1.separateCache = sqlite3GlobalConfig.pPage==0 + || sqlite3GlobalConfig.bCoreMutex>0; +#else + pcache1.separateCache = sqlite3GlobalConfig.pPage==0; +#endif + +#if SQLITE_THREADSAFE if( sqlite3GlobalConfig.bCoreMutex ){ pcache1.grp.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU); pcache1.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_PMEM); } +#endif + if( pcache1.separateCache + && sqlite3GlobalConfig.nPage!=0 + && sqlite3GlobalConfig.pPage==0 + ){ + pcache1.nInitPage = sqlite3GlobalConfig.nPage; + }else{ + pcache1.nInitPage = 0; + } pcache1.grp.mxPinned = 10; pcache1.isInit = 1; return SQLITE_OK; @@ -40488,39 +41847,26 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ PGroup *pGroup; /* The group the new page cache will belong to */ int sz; /* Bytes of memory required to allocate the new cache */ - /* - ** The separateCache variable is true if each PCache has its own private - ** PGroup. In other words, separateCache is true for mode (1) where no - ** mutexing is required. - ** - ** * Always use a unified cache (mode-2) if ENABLE_MEMORY_MANAGEMENT - ** - ** * Always use a unified cache in single-threaded applications - ** - ** * Otherwise (if multi-threaded and ENABLE_MEMORY_MANAGEMENT is off) - ** use separate caches (mode-1) - */ -#if defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) || SQLITE_THREADSAFE==0 - const int separateCache = 0; -#else - int separateCache = sqlite3GlobalConfig.bCoreMutex>0; -#endif - assert( (szPage & (szPage-1))==0 && szPage>=512 && szPage<=65536 ); assert( szExtra < 300 ); - sz = sizeof(PCache1) + sizeof(PGroup)*separateCache; + sz = sizeof(PCache1) + sizeof(PGroup)*pcache1.separateCache; pCache = (PCache1 *)sqlite3MallocZero(sz); if( pCache ){ - if( separateCache ){ + if( pcache1.separateCache ){ pGroup = (PGroup*)&pCache[1]; pGroup->mxPinned = 10; }else{ pGroup = &pcache1.grp; } + if( pGroup->lru.isAnchor==0 ){ + pGroup->lru.isAnchor = 1; + pGroup->lru.pLruPrev = pGroup->lru.pLruNext = &pGroup->lru; + } pCache->pGroup = pGroup; pCache->szPage = szPage; pCache->szExtra = szExtra; + pCache->szAlloc = szPage + szExtra + ROUND8(sizeof(PgHdr1)); pCache->bPurgeable = (bPurgeable ? 1 : 0); pcache1EnterMutex(pGroup); pcache1ResizeHash(pCache); @@ -40552,7 +41898,7 @@ static void pcache1Cachesize(sqlite3_pcache *p, int nMax){ pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; pCache->nMax = nMax; pCache->n90pct = pCache->nMax*9/10; - pcache1EnforceMaxPage(pGroup); + pcache1EnforceMaxPage(pCache); pcache1LeaveMutex(pGroup); } } @@ -40570,7 +41916,7 @@ static void pcache1Shrink(sqlite3_pcache *p){ pcache1EnterMutex(pGroup); savedMaxPage = pGroup->nMaxPage; pGroup->nMaxPage = 0; - pcache1EnforceMaxPage(pGroup); + pcache1EnforceMaxPage(pCache); pGroup->nMaxPage = savedMaxPage; pcache1LeaveMutex(pGroup); } @@ -40623,26 +41969,17 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( assert( pCache->nHash>0 && pCache->apHash ); /* Step 4. Try to recycle a page. */ - if( pCache->bPurgeable && pGroup->pLruTail && ( - (pCache->nPage+1>=pCache->nMax) - || pGroup->nCurrentPage>=pGroup->nMaxPage - || pcache1UnderMemoryPressure(pCache) - )){ + if( pCache->bPurgeable + && !pGroup->lru.pLruPrev->isAnchor + && ((pCache->nPage+1>=pCache->nMax) || pcache1UnderMemoryPressure(pCache)) + ){ PCache1 *pOther; - pPage = pGroup->pLruTail; + pPage = pGroup->lru.pLruPrev; assert( pPage->isPinned==0 ); - pcache1RemoveFromHash(pPage); + pcache1RemoveFromHash(pPage, 0); pcache1PinPage(pPage); pOther = pPage->pCache; - - /* We want to verify that szPage and szExtra are the same for pOther - ** and pCache. Assert that we can verify this by comparing sums. */ - assert( (pCache->szPage & (pCache->szPage-1))==0 && pCache->szPage>=512 ); - assert( pCache->szExtra<512 ); - assert( (pOther->szPage & (pOther->szPage-1))==0 && pOther->szPage>=512 ); - assert( pOther->szExtra<512 ); - - if( pOther->szPage+pOther->szExtra != pCache->szPage+pCache->szExtra ){ + if( pOther->szAlloc != pCache->szAlloc ){ pcache1FreePage(pPage); pPage = 0; }else{ @@ -40654,9 +41991,7 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( ** attempt to allocate a new one. */ if( !pPage ){ - if( createFlag==1 ) sqlite3BeginBenignMalloc(); - pPage = pcache1AllocPage(pCache); - if( createFlag==1 ) sqlite3EndBenignMalloc(); + pPage = pcache1AllocPage(pCache, createFlag==1); } if( pPage ){ @@ -40730,8 +42065,13 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( ** proceed to step 5. ** ** 5. Otherwise, allocate and return a new page buffer. +** +** There are two versions of this routine. pcache1FetchWithMutex() is +** the general case. pcache1FetchNoMutex() is a faster implementation for +** the common case where pGroup->mutex is NULL. The pcache1Fetch() wrapper +** invokes the appropriate routine. */ -static sqlite3_pcache_page *pcache1Fetch( +static PgHdr1 *pcache1FetchNoMutex( sqlite3_pcache *p, unsigned int iKey, int createFlag @@ -40739,28 +42079,66 @@ static sqlite3_pcache_page *pcache1Fetch( PCache1 *pCache = (PCache1 *)p; PgHdr1 *pPage = 0; - assert( offsetof(PgHdr1,page)==0 ); - assert( pCache->bPurgeable || createFlag!=1 ); - assert( pCache->bPurgeable || pCache->nMin==0 ); - assert( pCache->bPurgeable==0 || pCache->nMin==10 ); - assert( pCache->nMin==0 || pCache->bPurgeable ); - assert( pCache->nHash>0 ); - pcache1EnterMutex(pCache->pGroup); - /* Step 1: Search the hash table for an existing entry. */ pPage = pCache->apHash[iKey % pCache->nHash]; while( pPage && pPage->iKey!=iKey ){ pPage = pPage->pNext; } - /* Step 2: Abort if no existing page is found and createFlag is 0 */ + /* Step 2: If the page was found in the hash table, then return it. + ** If the page was not in the hash table and createFlag is 0, abort. + ** Otherwise (page not in hash and createFlag!=0) continue with + ** subsequent steps to try to create the page. */ if( pPage ){ - if( !pPage->isPinned ) pcache1PinPage(pPage); + if( !pPage->isPinned ){ + return pcache1PinPage(pPage); + }else{ + return pPage; + } }else if( createFlag ){ /* Steps 3, 4, and 5 implemented by this subroutine */ - pPage = pcache1FetchStage2(pCache, iKey, createFlag); + return pcache1FetchStage2(pCache, iKey, createFlag); + }else{ + return 0; } +} +#if PCACHE1_MIGHT_USE_GROUP_MUTEX +static PgHdr1 *pcache1FetchWithMutex( + sqlite3_pcache *p, + unsigned int iKey, + int createFlag +){ + PCache1 *pCache = (PCache1 *)p; + PgHdr1 *pPage; + + pcache1EnterMutex(pCache->pGroup); + pPage = pcache1FetchNoMutex(p, iKey, createFlag); assert( pPage==0 || pCache->iMaxKey>=iKey ); pcache1LeaveMutex(pCache->pGroup); - return (sqlite3_pcache_page*)pPage; + return pPage; +} +#endif +static sqlite3_pcache_page *pcache1Fetch( + sqlite3_pcache *p, + unsigned int iKey, + int createFlag +){ +#if PCACHE1_MIGHT_USE_GROUP_MUTEX || defined(SQLITE_DEBUG) + PCache1 *pCache = (PCache1 *)p; +#endif + + assert( offsetof(PgHdr1,page)==0 ); + assert( pCache->bPurgeable || createFlag!=1 ); + assert( pCache->bPurgeable || pCache->nMin==0 ); + assert( pCache->bPurgeable==0 || pCache->nMin==10 ); + assert( pCache->nMin==0 || pCache->bPurgeable ); + assert( pCache->nHash>0 ); +#if PCACHE1_MIGHT_USE_GROUP_MUTEX + if( pCache->pGroup->mutex ){ + return (sqlite3_pcache_page*)pcache1FetchWithMutex(p, iKey, createFlag); + }else +#endif + { + return (sqlite3_pcache_page*)pcache1FetchNoMutex(p, iKey, createFlag); + } } @@ -40785,22 +42163,16 @@ static void pcache1Unpin( ** part of the PGroup LRU list. */ assert( pPage->pLruPrev==0 && pPage->pLruNext==0 ); - assert( pGroup->pLruHead!=pPage && pGroup->pLruTail!=pPage ); assert( pPage->isPinned==1 ); if( reuseUnlikely || pGroup->nCurrentPage>pGroup->nMaxPage ){ - pcache1RemoveFromHash(pPage); - pcache1FreePage(pPage); + pcache1RemoveFromHash(pPage, 1); }else{ /* Add the page to the PGroup LRU list. */ - if( pGroup->pLruHead ){ - pGroup->pLruHead->pLruPrev = pPage; - pPage->pLruNext = pGroup->pLruHead; - pGroup->pLruHead = pPage; - }else{ - pGroup->pLruTail = pPage; - pGroup->pLruHead = pPage; - } + PgHdr1 **ppFirst = &pGroup->lru.pLruNext; + pPage->pLruPrev = &pGroup->lru; + (pPage->pLruNext = *ppFirst)->pLruPrev = pPage; + *ppFirst = pPage; pCache->nRecyclable++; pPage->isPinned = 0; } @@ -40877,8 +42249,9 @@ static void pcache1Destroy(sqlite3_pcache *p){ assert( pGroup->nMinPage >= pCache->nMin ); pGroup->nMinPage -= pCache->nMin; pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; - pcache1EnforceMaxPage(pGroup); + pcache1EnforceMaxPage(pCache); pcache1LeaveMutex(pGroup); + sqlite3_free(pCache->pBulk); sqlite3_free(pCache->apHash); sqlite3_free(pCache); } @@ -40934,18 +42307,20 @@ SQLITE_PRIVATE int sqlite3PcacheReleaseMemory(int nReq){ int nFree = 0; assert( sqlite3_mutex_notheld(pcache1.grp.mutex) ); assert( sqlite3_mutex_notheld(pcache1.mutex) ); - if( pcache1.pStart==0 ){ + if( sqlite3GlobalConfig.nPage==0 ){ PgHdr1 *p; pcache1EnterMutex(&pcache1.grp); - while( (nReq<0 || nFreeisAnchor==0 + ){ nFree += pcache1MemSize(p->page.pBuf); #ifdef SQLITE_PCACHE_SEPARATE_HEADER nFree += sqlite3MemSize(p); #endif assert( p->isPinned==0 ); pcache1PinPage(p); - pcache1RemoveFromHash(p); - pcache1FreePage(p); + pcache1RemoveFromHash(p, 1); } pcache1LeaveMutex(&pcache1.grp); } @@ -40966,7 +42341,7 @@ SQLITE_PRIVATE void sqlite3PcacheStats( ){ PgHdr1 *p; int nRecyclable = 0; - for(p=pcache1.grp.pLruHead; p; p=p->pLruNext){ + for(p=pcache1.grp.lru.pLruNext; p && !p->isAnchor; p=p->pLruNext){ assert( p->isPinned==0 ); nRecyclable++; } @@ -41041,6 +42416,7 @@ SQLITE_PRIVATE void sqlite3PcacheStats( ** There is an added cost of O(N) when switching between TEST and ** SMALLEST primitives. */ +/* #include "sqliteInt.h" */ /* @@ -41510,6 +42886,7 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 ** another is writing. */ #ifndef SQLITE_OMIT_DISKIO +/* #include "sqliteInt.h" */ /************** Include wal.h in the middle of pager.c ***********************/ /************** Begin file wal.h *********************************************/ /* @@ -41531,6 +42908,7 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 #ifndef _WAL_H_ #define _WAL_H_ +/* #include "sqliteInt.h" */ /* Additional values that can be added to the sync_flags argument of ** sqlite3WalFrames(): @@ -42086,9 +43464,9 @@ struct PagerSavepoint { /* ** Bits of the Pager.doNotSpill flag. See further description below. */ -#define SPILLFLAG_OFF 0x01 /* Never spill cache. Set via pragma */ -#define SPILLFLAG_ROLLBACK 0x02 /* Current rolling back, so do not spill */ -#define SPILLFLAG_NOSYNC 0x04 /* Spill is ok, but do not sync */ +#define SPILLFLAG_OFF 0x01 /* Never spill cache. Set via pragma */ +#define SPILLFLAG_ROLLBACK 0x02 /* Current rolling back, so do not spill */ +#define SPILLFLAG_NOSYNC 0x04 /* Spill is ok, but do not sync */ /* ** An open page cache is an instance of struct Pager. A description of @@ -42170,11 +43548,11 @@ struct PagerSavepoint { ** while it is being traversed by code in pager_playback(). The SPILLFLAG_OFF ** case is a user preference. ** -** If the SPILLFLAG_NOSYNC bit is set, writing to the database from pagerStress() -** is permitted, but syncing the journal file is not. This flag is set -** by sqlite3PagerWrite() when the file-system sector-size is larger than -** the database page-size in order to prevent a journal sync from happening -** in between the journalling of two pages on the same sector. +** If the SPILLFLAG_NOSYNC bit is set, writing to the database from +** pagerStress() is permitted, but syncing the journal file is not. +** This flag is set by sqlite3PagerWrite() when the file-system sector-size +** is larger than the database page-size in order to prevent a journal sync +** from happening in between the journalling of two pages on the same sector. ** ** subjInMemory ** @@ -42277,7 +43655,7 @@ struct Pager { u8 doNotSpill; /* Do not spill the cache when non-zero */ u8 subjInMemory; /* True to use in-memory sub-journals */ u8 bUseFetch; /* True to use xFetch() */ - u8 hasBeenUsed; /* True if any content previously read from this pager*/ + u8 hasHeldSharedLock; /* True if a shared lock has ever been held */ Pgno dbSize; /* Number of pages in the database */ Pgno dbOrigSize; /* dbSize before the current transaction */ Pgno dbFileSize; /* Number of pages in the database file */ @@ -42438,7 +43816,7 @@ static const unsigned char aJournalMagic[] = { ** ** if( pPager->jfd->pMethods ){ ... */ -#define isOpen(pFd) ((pFd)->pMethods) +#define isOpen(pFd) ((pFd)->pMethods!=0) /* ** Return true if this pager uses a write-ahead log instead of the usual @@ -42661,19 +44039,21 @@ static int subjRequiresPage(PgHdr *pPg){ int i; for(i=0; inSavepoint; i++){ p = &pPager->aSavepoint[i]; - if( p->nOrig>=pgno && 0==sqlite3BitvecTest(p->pInSavepoint, pgno) ){ + if( p->nOrig>=pgno && 0==sqlite3BitvecTestNotNull(p->pInSavepoint, pgno) ){ return 1; } } return 0; } +#ifdef SQLITE_DEBUG /* ** Return true if the page is already in the journal file. */ static int pageInJournal(Pager *pPager, PgHdr *pPg){ return sqlite3BitvecTest(pPager->pInJournal, pPg->pgno); } +#endif /* ** Read a 32-bit integer from the given file descriptor. Store the integer @@ -43285,7 +44665,8 @@ static int writeMasterJournal(Pager *pPager, const char *zMaster){ || (0 != (rc = sqlite3OsWrite(pPager->jfd, zMaster, nMaster, iHdrOff+4))) || (0 != (rc = write32bits(pPager->jfd, iHdrOff+4+nMaster, nMaster))) || (0 != (rc = write32bits(pPager->jfd, iHdrOff+4+nMaster+4, cksum))) - || (0 != (rc = sqlite3OsWrite(pPager->jfd, aJournalMagic, 8, iHdrOff+4+nMaster+8))) + || (0 != (rc = sqlite3OsWrite(pPager->jfd, aJournalMagic, 8, + iHdrOff+4+nMaster+8))) ){ return rc; } @@ -43743,6 +45124,20 @@ static void pagerReportSize(Pager *pPager){ # define pagerReportSize(X) /* No-op if we do not support a codec */ #endif +#ifdef SQLITE_HAS_CODEC +/* +** Make sure the number of reserved bits is the same in the destination +** pager as it is in the source. This comes up when a VACUUM changes the +** number of reserved bits to the "optimal" amount. +*/ +SQLITE_PRIVATE void sqlite3PagerAlignReserve(Pager *pDest, Pager *pSrc){ + if( pDest->nReserve!=pSrc->nReserve ){ + pDest->nReserve = pSrc->nReserve; + pagerReportSize(pDest); + } +} +#endif + /* ** Read a single page from either the journal file (if isMainJrnl==1) or ** from the sub-journal (if isMainJrnl==0) and playback that page. @@ -43845,7 +45240,7 @@ static int pager_playback_one_page( } } - /* If this page has already been played by before during the current + /* If this page has already been played back before during the current ** rollback, then don't bother to play it back again. */ if( pDone && (rc = sqlite3BitvecSet(pDone, pgno))!=SQLITE_OK ){ @@ -44779,11 +46174,10 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ assert( pPager->eLock>=SHARED_LOCK ); nPage = sqlite3WalDbsize(pPager->pWal); - /* If the database size was not available from the WAL sub-system, - ** determine it based on the size of the database file. If the size - ** of the database file is not an integer multiple of the page-size, - ** round down to the nearest page. Except, any file larger than 0 - ** bytes in size is considered to contain at least one page. + /* If the number of pages in the database is not available from the + ** WAL sub-system, determine the page counte based on the size of + ** the database file. If the size of the database file is not an + ** integer multiple of the page-size, round up the result. */ if( nPage==0 ){ i64 n = 0; /* Size of db file in bytes */ @@ -45948,8 +47342,6 @@ static int openSubJournal(Pager *pPager){ /* ** Append a record of the current state of page pPg to the sub-journal. -** It is the callers responsibility to use subjRequiresPage() to check -** that it is really required before calling this function. ** ** If successful, set the bit corresponding to pPg->pgno in the bitvecs ** for all open savepoints before returning. @@ -45996,6 +47388,13 @@ static int subjournalPage(PgHdr *pPg){ } return rc; } +static int subjournalPageIfRequired(PgHdr *pPg){ + if( subjRequiresPage(pPg) ){ + return subjournalPage(pPg); + }else{ + return SQLITE_OK; + } +} /* ** This function is called by the pcache layer when it has reached some @@ -46053,9 +47452,7 @@ static int pagerStress(void *p, PgHdr *pPg){ pPg->pDirty = 0; if( pagerUseWal(pPager) ){ /* Write a single frame for this page to the log. */ - if( subjRequiresPage(pPg) ){ - rc = subjournalPage(pPg); - } + rc = subjournalPageIfRequired(pPg); if( rc==SQLITE_OK ){ rc = pagerWalFrames(pPager, pPg, 0, 0); } @@ -46068,39 +47465,6 @@ static int pagerStress(void *p, PgHdr *pPg){ rc = syncJournal(pPager, 1); } - /* If the page number of this page is larger than the current size of - ** the database image, it may need to be written to the sub-journal. - ** This is because the call to pager_write_pagelist() below will not - ** actually write data to the file in this case. - ** - ** Consider the following sequence of events: - ** - ** BEGIN; - ** - ** - ** SAVEPOINT sp; - ** - ** pagerStress(page X) - ** ROLLBACK TO sp; - ** - ** If (X>Y), then when pagerStress is called page X will not be written - ** out to the database file, but will be dropped from the cache. Then, - ** following the "ROLLBACK TO sp" statement, reading page X will read - ** data from the database file. This will be the copy of page X as it - ** was when the transaction started, not as it was when "SAVEPOINT sp" - ** was executed. - ** - ** The solution is to write the current data for page X into the - ** sub-journal file now (if it is not already there), so that it will - ** be restored to its current value when the "ROLLBACK TO sp" is - ** executed. - */ - if( NEVER( - rc==SQLITE_OK && pPg->pgno>pPager->dbSize && subjRequiresPage(pPg) - ) ){ - rc = subjournalPage(pPg); - } - /* Write the contents of the page out to the database file. */ if( rc==SQLITE_OK ){ assert( (pPg->flags&PGHDR_NEED_SYNC)==0 ); @@ -46356,7 +47720,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( act_like_temp_file: tempFile = 1; pPager->eState = PAGER_READER; /* Pretend we already have a lock */ - pPager->eLock = EXCLUSIVE_LOCK; /* Pretend we are in EXCLUSIVE locking mode */ + pPager->eLock = EXCLUSIVE_LOCK; /* Pretend we are in EXCLUSIVE mode */ pPager->noLock = 1; /* Do no locking */ readOnly = (vfsFlags&SQLITE_OPEN_READONLY); } @@ -46375,7 +47739,7 @@ act_like_temp_file: assert( nExtra<1000 ); nExtra = ROUND8(nExtra); rc = sqlite3PcacheOpen(szPageDflt, nExtra, !memDb, - !memDb?pagerStress:0, (void *)pPager, pPager->pPCache); + !memDb?pagerStress:0, (void *)pPager, pPager->pPCache); } /* If an error occurred above, free the Pager structure and close the file. @@ -46755,14 +48119,14 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ ); } - if( !pPager->tempFile && pPager->hasBeenUsed ){ + if( !pPager->tempFile && pPager->hasHeldSharedLock ){ /* The shared-lock has just been acquired then check to ** see if the database has been modified. If the database has changed, - ** flush the cache. The pPager->hasBeenUsed flag prevents this from + ** flush the cache. The hasHeldSharedLock flag prevents this from ** occurring on the very first access to a file, in order to save a ** single unnecessary sqlite3OsRead() call at the start-up. ** - ** Database changes is detected by looking at 15 bytes beginning + ** Database changes are detected by looking at 15 bytes beginning ** at offset 24 into the file. The first 4 of these 16 bytes are ** a 32-bit counter that is incremented with each change. The ** other bytes change randomly with each file change when @@ -46828,6 +48192,7 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ assert( pPager->eState==PAGER_OPEN ); }else{ pPager->eState = PAGER_READER; + pPager->hasHeldSharedLock = 1; } return rc; } @@ -46911,21 +48276,25 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY ** flag was specified by the caller. And so long as the db is not a ** temporary or in-memory database. */ - const int bMmapOk = (pgno!=1 && USEFETCH(pPager) + const int bMmapOk = (pgno>1 && USEFETCH(pPager) && (pPager->eState==PAGER_READER || (flags & PAGER_GET_READONLY)) #ifdef SQLITE_HAS_CODEC && pPager->xCodec==0 #endif ); + /* Optimization note: Adding the "pgno<=1" term before "pgno==0" here + ** allows the compiler optimizer to reuse the results of the "pgno>1" + ** test in the previous statement, and avoid testing pgno==0 in the + ** common case where pgno is large. */ + if( pgno<=1 && pgno==0 ){ + return SQLITE_CORRUPT_BKPT; + } assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); assert( noContent==0 || bMmapOk==0 ); - if( pgno==0 ){ - return SQLITE_CORRUPT_BKPT; - } - pPager->hasBeenUsed = 1; + assert( pPager->hasHeldSharedLock==1 ); /* If the pager is in the error state, return an error immediately. ** Otherwise, request the page from the PCache layer. */ @@ -46970,9 +48339,14 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( if( pBase==0 ){ rc = sqlite3PcacheFetchStress(pPager->pPCache, pgno, &pBase); if( rc!=SQLITE_OK ) goto pager_acquire_err; + if( pBase==0 ){ + pPg = *ppPage = 0; + rc = SQLITE_NOMEM; + goto pager_acquire_err; + } } pPg = *ppPage = sqlite3PcacheFetchFinish(pPager->pPCache, pgno, pBase); - if( pPg==0 ) rc = SQLITE_NOMEM; + assert( pPg!=0 ); } } @@ -46983,10 +48357,11 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( pPg = 0; goto pager_acquire_err; } - assert( (*ppPage)->pgno==pgno ); - assert( (*ppPage)->pPager==pPager || (*ppPage)->pPager==0 ); + assert( pPg==(*ppPage) ); + assert( pPg->pgno==pgno ); + assert( pPg->pPager==pPager || pPg->pPager==0 ); - if( (*ppPage)->pPager && !noContent ){ + if( pPg->pPager && !noContent ){ /* In this case the pcache already contains an initialized copy of ** the page. Return without further ado. */ assert( pgno<=PAGER_MAX_PGNO && pgno!=PAGER_MJ_PGNO(pPager) ); @@ -46997,7 +48372,6 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( /* The pager cache has created a new page. Its content needs to ** be initialized. */ - pPg = *ppPage; pPg->pPager = pPager; /* The maximum page number is 2^31. Return SQLITE_CORRUPT if a page @@ -47075,7 +48449,8 @@ SQLITE_PRIVATE DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ assert( pgno!=0 ); assert( pPager->pPCache!=0 ); pPage = sqlite3PcacheFetch(pPager->pPCache, pgno, 0); - assert( pPage==0 || pPager->hasBeenUsed ); + assert( pPage==0 || pPager->hasHeldSharedLock ); + if( pPage==0 ) return 0; return sqlite3PcacheFetchFinish(pPager->pPCache, pgno, pPage); } @@ -47279,6 +48654,59 @@ SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory return rc; } +/* +** Write page pPg onto the end of the rollback journal. +*/ +static SQLITE_NOINLINE int pagerAddPageToRollbackJournal(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + int rc; + u32 cksum; + char *pData2; + i64 iOff = pPager->journalOff; + + /* We should never write to the journal file the page that + ** contains the database locks. The following assert verifies + ** that we do not. */ + assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) ); + + assert( pPager->journalHdr<=pPager->journalOff ); + CODEC2(pPager, pPg->pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2); + cksum = pager_cksum(pPager, (u8*)pData2); + + /* Even if an IO or diskfull error occurs while journalling the + ** page in the block above, set the need-sync flag for the page. + ** Otherwise, when the transaction is rolled back, the logic in + ** playback_one_page() will think that the page needs to be restored + ** in the database file. And if an IO error occurs while doing so, + ** then corruption may follow. + */ + pPg->flags |= PGHDR_NEED_SYNC; + + rc = write32bits(pPager->jfd, iOff, pPg->pgno); + if( rc!=SQLITE_OK ) return rc; + rc = sqlite3OsWrite(pPager->jfd, pData2, pPager->pageSize, iOff+4); + if( rc!=SQLITE_OK ) return rc; + rc = write32bits(pPager->jfd, iOff+pPager->pageSize+4, cksum); + if( rc!=SQLITE_OK ) return rc; + + IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, + pPager->journalOff, pPager->pageSize)); + PAGER_INCR(sqlite3_pager_writej_count); + PAGERTRACE(("JOURNAL %d page %d needSync=%d hash(%08x)\n", + PAGERID(pPager), pPg->pgno, + ((pPg->flags&PGHDR_NEED_SYNC)?1:0), pager_pagehash(pPg))); + + pPager->journalOff += 8 + pPager->pageSize; + pPager->nRec++; + assert( pPager->pInJournal!=0 ); + rc = sqlite3BitvecSet(pPager->pInJournal, pPg->pgno); + testcase( rc==SQLITE_NOMEM ); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + rc |= addToSavepointBitvecs(pPager, pPg->pgno); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + return rc; +} + /* ** Mark a single data page as writeable. The page is written into the ** main journal or sub-journal as required. If the page is written into @@ -47289,7 +48717,6 @@ SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory static int pager_write(PgHdr *pPg){ Pager *pPager = pPg->pPager; int rc = SQLITE_OK; - int inJournal; /* This routine is not called unless a write-transaction has already ** been started. The journal file may or may not be open at this point. @@ -47302,7 +48729,6 @@ static int pager_write(PgHdr *pPg){ assert( assert_pager_state(pPager) ); assert( pPager->errCode==0 ); assert( pPager->readOnly==0 ); - CHECK_PAGE(pPg); /* The journal file needs to be opened. Higher level routines have already @@ -47321,91 +48747,48 @@ static int pager_write(PgHdr *pPg){ assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); assert( assert_pager_state(pPager) ); - /* Mark the page as dirty. If the page has already been written - ** to the journal then we can return right away. - */ + /* Mark the page that is about to be modified as dirty. */ sqlite3PcacheMakeDirty(pPg); - inJournal = pageInJournal(pPager, pPg); - if( inJournal && (pPager->nSavepoint==0 || !subjRequiresPage(pPg)) ){ - assert( !pagerUseWal(pPager) ); - }else{ - - /* The transaction journal now exists and we have a RESERVED or an - ** EXCLUSIVE lock on the main database file. Write the current page to - ** the transaction journal if it is not there already. - */ - if( !inJournal && !pagerUseWal(pPager) ){ - assert( pagerUseWal(pPager)==0 ); - if( pPg->pgno<=pPager->dbOrigSize && isOpen(pPager->jfd) ){ - u32 cksum; - char *pData2; - i64 iOff = pPager->journalOff; - - /* We should never write to the journal file the page that - ** contains the database locks. The following assert verifies - ** that we do not. */ - assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) ); - - assert( pPager->journalHdr<=pPager->journalOff ); - CODEC2(pPager, pPg->pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2); - cksum = pager_cksum(pPager, (u8*)pData2); - - /* Even if an IO or diskfull error occurs while journalling the - ** page in the block above, set the need-sync flag for the page. - ** Otherwise, when the transaction is rolled back, the logic in - ** playback_one_page() will think that the page needs to be restored - ** in the database file. And if an IO error occurs while doing so, - ** then corruption may follow. - */ - pPg->flags |= PGHDR_NEED_SYNC; - rc = write32bits(pPager->jfd, iOff, pPg->pgno); - if( rc!=SQLITE_OK ) return rc; - rc = sqlite3OsWrite(pPager->jfd, pData2, pPager->pageSize, iOff+4); - if( rc!=SQLITE_OK ) return rc; - rc = write32bits(pPager->jfd, iOff+pPager->pageSize+4, cksum); - if( rc!=SQLITE_OK ) return rc; - - IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, - pPager->journalOff, pPager->pageSize)); - PAGER_INCR(sqlite3_pager_writej_count); - PAGERTRACE(("JOURNAL %d page %d needSync=%d hash(%08x)\n", - PAGERID(pPager), pPg->pgno, - ((pPg->flags&PGHDR_NEED_SYNC)?1:0), pager_pagehash(pPg))); - - pPager->journalOff += 8 + pPager->pageSize; - pPager->nRec++; - assert( pPager->pInJournal!=0 ); - rc = sqlite3BitvecSet(pPager->pInJournal, pPg->pgno); - testcase( rc==SQLITE_NOMEM ); - assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); - rc |= addToSavepointBitvecs(pPager, pPg->pgno); - if( rc!=SQLITE_OK ){ - assert( rc==SQLITE_NOMEM ); - return rc; - } - }else{ - if( pPager->eState!=PAGER_WRITER_DBMOD ){ - pPg->flags |= PGHDR_NEED_SYNC; - } - PAGERTRACE(("APPEND %d page %d needSync=%d\n", - PAGERID(pPager), pPg->pgno, - ((pPg->flags&PGHDR_NEED_SYNC)?1:0))); + /* If a rollback journal is in use, them make sure the page that is about + ** to change is in the rollback journal, or if the page is a new page off + ** then end of the file, make sure it is marked as PGHDR_NEED_SYNC. + */ + assert( (pPager->pInJournal!=0) == isOpen(pPager->jfd) ); + if( pPager->pInJournal!=0 + && sqlite3BitvecTestNotNull(pPager->pInJournal, pPg->pgno)==0 + ){ + assert( pagerUseWal(pPager)==0 ); + if( pPg->pgno<=pPager->dbOrigSize ){ + rc = pagerAddPageToRollbackJournal(pPg); + if( rc!=SQLITE_OK ){ + return rc; } - } - - /* If the statement journal is open and the page is not in it, - ** then write the current page to the statement journal. Note that - ** the statement journal format differs from the standard journal format - ** in that it omits the checksums and the header. - */ - if( pPager->nSavepoint>0 && subjRequiresPage(pPg) ){ - rc = subjournalPage(pPg); + }else{ + if( pPager->eState!=PAGER_WRITER_DBMOD ){ + pPg->flags |= PGHDR_NEED_SYNC; + } + PAGERTRACE(("APPEND %d page %d needSync=%d\n", + PAGERID(pPager), pPg->pgno, + ((pPg->flags&PGHDR_NEED_SYNC)?1:0))); } } - /* Update the database size and return. + /* The PGHDR_DIRTY bit is set above when the page was added to the dirty-list + ** and before writing the page into the rollback journal. Wait until now, + ** after the page has been successfully journalled, before setting the + ** PGHDR_WRITEABLE bit that indicates that the page can be safely modified. */ + pPg->flags |= PGHDR_WRITEABLE; + + /* If the statement journal is open and the page is not in it, + ** then write the page into the statement journal. + */ + if( pPager->nSavepoint>0 ){ + rc = subjournalPageIfRequired(pPg); + } + + /* Update the database size and return. */ if( pPager->dbSizepgno ){ pPager->dbSize = pPg->pgno; } @@ -47420,17 +48803,17 @@ static int pager_write(PgHdr *pPg){ ** a write. ** ** Usually, the sector size is less than or equal to the page size, in which -** case pages can be individually written. This routine only runs in the exceptional -** case where the page size is smaller than the sector size. +** case pages can be individually written. This routine only runs in the +** exceptional case where the page size is smaller than the sector size. */ static SQLITE_NOINLINE int pagerWriteLargeSector(PgHdr *pPg){ - int rc = SQLITE_OK; /* Return code */ - Pgno nPageCount; /* Total number of pages in database file */ - Pgno pg1; /* First page of the sector pPg is located on. */ - int nPage = 0; /* Number of pages starting at pg1 to journal */ - int ii; /* Loop counter */ - int needSync = 0; /* True if any page has PGHDR_NEED_SYNC */ - Pager *pPager = pPg->pPager; /* The pager that owns pPg */ + int rc = SQLITE_OK; /* Return code */ + Pgno nPageCount; /* Total number of pages in database file */ + Pgno pg1; /* First page of the sector pPg is located on. */ + int nPage = 0; /* Number of pages starting at pg1 to journal */ + int ii; /* Loop counter */ + int needSync = 0; /* True if any page has PGHDR_NEED_SYNC */ + Pager *pPager = pPg->pPager; /* The pager that owns pPg */ Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); /* Set the doNotSpill NOSYNC bit to 1. This is because we cannot allow @@ -47518,11 +48901,15 @@ static SQLITE_NOINLINE int pagerWriteLargeSector(PgHdr *pPg){ ** as appropriate. Otherwise, SQLITE_OK. */ SQLITE_PRIVATE int sqlite3PagerWrite(PgHdr *pPg){ + Pager *pPager = pPg->pPager; assert( (pPg->flags & PGHDR_MMAP)==0 ); - assert( pPg->pPager->eState>=PAGER_WRITER_LOCKED ); - assert( pPg->pPager->eState!=PAGER_ERROR ); - assert( assert_pager_state(pPg->pPager) ); - if( pPg->pPager->sectorSize > (u32)pPg->pPager->pageSize ){ + assert( pPager->eState>=PAGER_WRITER_LOCKED ); + assert( pPager->eState!=PAGER_ERROR ); + assert( assert_pager_state(pPager) ); + if( (pPg->flags & PGHDR_WRITEABLE)!=0 && pPager->dbSize>=pPg->pgno ){ + if( pPager->nSavepoint ) return subjournalPageIfRequired(pPg); + return SQLITE_OK; + }else if( pPager->sectorSize > (u32)pPager->pageSize ){ return pagerWriteLargeSector(pPg); }else{ return pager_write(pPg); @@ -47536,7 +48923,7 @@ SQLITE_PRIVATE int sqlite3PagerWrite(PgHdr *pPg){ */ #ifndef NDEBUG SQLITE_PRIVATE int sqlite3PagerIswriteable(DbPage *pPg){ - return pPg->flags&PGHDR_DIRTY; + return pPg->flags & PGHDR_WRITEABLE; } #endif @@ -47560,6 +48947,7 @@ SQLITE_PRIVATE void sqlite3PagerDontWrite(PgHdr *pPg){ PAGERTRACE(("DONT_WRITE page %d of %d\n", pPg->pgno, PAGERID(pPager))); IOTRACE(("CLEAN %p %d\n", pPager, pPg->pgno)) pPg->flags |= PGHDR_DONT_WRITE; + pPg->flags &= ~PGHDR_WRITEABLE; pager_set_pagehash(pPg); } } @@ -48026,12 +49414,14 @@ SQLITE_PRIVATE u8 sqlite3PagerIsreadonly(Pager *pPager){ return pPager->readOnly; } +#ifdef SQLITE_DEBUG /* -** Return the number of references to the pager. +** Return the sum of the reference counts for all pages held by pPager. */ SQLITE_PRIVATE int sqlite3PagerRefcount(Pager *pPager){ return sqlite3PcacheRefCount(pPager->pPCache); } +#endif /* ** Return the approximate number of bytes of memory currently @@ -48114,54 +49504,62 @@ SQLITE_PRIVATE int sqlite3PagerIsMemdb(Pager *pPager){ ** occurs while opening the sub-journal file, then an IO error code is ** returned. Otherwise, SQLITE_OK. */ -SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ +static SQLITE_NOINLINE int pagerOpenSavepoint(Pager *pPager, int nSavepoint){ int rc = SQLITE_OK; /* Return code */ int nCurrent = pPager->nSavepoint; /* Current number of savepoints */ + int ii; /* Iterator variable */ + PagerSavepoint *aNew; /* New Pager.aSavepoint array */ assert( pPager->eState>=PAGER_WRITER_LOCKED ); assert( assert_pager_state(pPager) ); + assert( nSavepoint>nCurrent && pPager->useJournal ); - if( nSavepoint>nCurrent && pPager->useJournal ){ - int ii; /* Iterator variable */ - PagerSavepoint *aNew; /* New Pager.aSavepoint array */ + /* Grow the Pager.aSavepoint array using realloc(). Return SQLITE_NOMEM + ** if the allocation fails. Otherwise, zero the new portion in case a + ** malloc failure occurs while populating it in the for(...) loop below. + */ + aNew = (PagerSavepoint *)sqlite3Realloc( + pPager->aSavepoint, sizeof(PagerSavepoint)*nSavepoint + ); + if( !aNew ){ + return SQLITE_NOMEM; + } + memset(&aNew[nCurrent], 0, (nSavepoint-nCurrent) * sizeof(PagerSavepoint)); + pPager->aSavepoint = aNew; - /* Grow the Pager.aSavepoint array using realloc(). Return SQLITE_NOMEM - ** if the allocation fails. Otherwise, zero the new portion in case a - ** malloc failure occurs while populating it in the for(...) loop below. - */ - aNew = (PagerSavepoint *)sqlite3Realloc( - pPager->aSavepoint, sizeof(PagerSavepoint)*nSavepoint - ); - if( !aNew ){ + /* Populate the PagerSavepoint structures just allocated. */ + for(ii=nCurrent; iidbSize; + if( isOpen(pPager->jfd) && pPager->journalOff>0 ){ + aNew[ii].iOffset = pPager->journalOff; + }else{ + aNew[ii].iOffset = JOURNAL_HDR_SZ(pPager); + } + aNew[ii].iSubRec = pPager->nSubRec; + aNew[ii].pInSavepoint = sqlite3BitvecCreate(pPager->dbSize); + if( !aNew[ii].pInSavepoint ){ return SQLITE_NOMEM; } - memset(&aNew[nCurrent], 0, (nSavepoint-nCurrent) * sizeof(PagerSavepoint)); - pPager->aSavepoint = aNew; - - /* Populate the PagerSavepoint structures just allocated. */ - for(ii=nCurrent; iidbSize; - if( isOpen(pPager->jfd) && pPager->journalOff>0 ){ - aNew[ii].iOffset = pPager->journalOff; - }else{ - aNew[ii].iOffset = JOURNAL_HDR_SZ(pPager); - } - aNew[ii].iSubRec = pPager->nSubRec; - aNew[ii].pInSavepoint = sqlite3BitvecCreate(pPager->dbSize); - if( !aNew[ii].pInSavepoint ){ - return SQLITE_NOMEM; - } - if( pagerUseWal(pPager) ){ - sqlite3WalSavepoint(pPager->pWal, aNew[ii].aWalData); - } - pPager->nSavepoint = ii+1; + if( pagerUseWal(pPager) ){ + sqlite3WalSavepoint(pPager->pWal, aNew[ii].aWalData); } - assert( pPager->nSavepoint==nSavepoint ); - assertTruncateConstraint(pPager); + pPager->nSavepoint = ii+1; } - + assert( pPager->nSavepoint==nSavepoint ); + assertTruncateConstraint(pPager); return rc; } +SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ + assert( pPager->eState>=PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); + + if( nSavepoint>pPager->nSavepoint && pPager->useJournal ){ + return pagerOpenSavepoint(pPager, nSavepoint); + }else{ + return SQLITE_OK; + } +} + /* ** This function is called to rollback or release (commit) a savepoint. @@ -48392,9 +49790,8 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i ** one or more savepoint bitvecs. This is the reason this function ** may return SQLITE_NOMEM. */ - if( pPg->flags&PGHDR_DIRTY - && subjRequiresPage(pPg) - && SQLITE_OK!=(rc = subjournalPage(pPg)) + if( (pPg->flags & PGHDR_DIRTY)!=0 + && SQLITE_OK!=(rc = subjournalPageIfRequired(pPg)) ){ return rc; } @@ -49145,6 +50542,7 @@ SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager){ */ #ifndef SQLITE_OMIT_WAL +/* #include "wal.h" */ /* ** Trace output macros @@ -49330,6 +50728,7 @@ struct Wal { u8 syncHeader; /* Fsync the WAL header if true */ u8 padToSectorBoundary; /* Pad transactions out to the next sector */ WalIndexHdr hdr; /* Wal-index header for current transaction */ + u32 minFrame; /* Ignore wal frames before this one */ const char *zWalName; /* Name of WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ #ifdef SQLITE_DEBUG @@ -49550,9 +50949,9 @@ static void walIndexWriteHdr(Wal *pWal){ pWal->hdr.isInit = 1; pWal->hdr.iVersion = WALINDEX_MAX_VERSION; walChecksumBytes(1, (u8*)&pWal->hdr, nCksum, 0, pWal->hdr.aCksum); - memcpy((void *)&aHdr[1], (void *)&pWal->hdr, sizeof(WalIndexHdr)); + memcpy((void*)&aHdr[1], (const void*)&pWal->hdr, sizeof(WalIndexHdr)); walShmBarrier(pWal); - memcpy((void *)&aHdr[0], (void *)&pWal->hdr, sizeof(WalIndexHdr)); + memcpy((void*)&aHdr[0], (const void*)&pWal->hdr, sizeof(WalIndexHdr)); } /* @@ -49854,13 +51253,13 @@ static void walCleanupHash(Wal *pWal){ ** via the hash table even after the cleanup. */ if( iLimit ){ - int i; /* Loop counter */ + int j; /* Loop counter */ int iKey; /* Hash key */ - for(i=1; i<=iLimit; i++){ - for(iKey=walHash(aPgno[i]); aHash[iKey]; iKey=walNextHash(iKey)){ - if( aHash[iKey]==i ) break; + for(j=1; j<=iLimit; j++){ + for(iKey=walHash(aPgno[j]); aHash[iKey]; iKey=walNextHash(iKey)){ + if( aHash[iKey]==j ) break; } - assert( aHash[iKey]==i ); + assert( aHash[iKey]==j ); } } #endif /* SQLITE_ENABLE_EXPENSIVE_ASSERT */ @@ -50362,7 +51761,7 @@ static void walMergesort( int nMerge = 0; /* Number of elements in list aMerge */ ht_slot *aMerge = 0; /* List to be merged */ int iList; /* Index into input list */ - int iSub = 0; /* Index into aSub array */ + u32 iSub = 0; /* Index into aSub array */ struct Sublist aSub[13]; /* Array of sub-lists */ memset(aSub, 0, sizeof(aSub)); @@ -50373,7 +51772,9 @@ static void walMergesort( nMerge = 1; aMerge = &aList[iList]; for(iSub=0; iList & (1<aList && p->nList<=(1<aList==&aList[iList&~((2<aList, p->nList, &aMerge, &nMerge, aBuffer); @@ -50384,7 +51785,9 @@ static void walMergesort( for(iSub++; iSubnList<=(1<aList==&aList[nList&~((2<aList, p->nList, &aMerge, &nMerge, aBuffer); @@ -51194,12 +52597,27 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ ** pWal->hdr.mxFrame risks reading a corrupted snapshot. So, retry ** instead. ** - ** This does not guarantee that the copy of the wal-index header is up to - ** date before proceeding. That would not be possible without somehow - ** blocking writers. It only guarantees that a dangerous checkpoint or - ** log-wrap (either of which would require an exclusive lock on - ** WAL_READ_LOCK(mxI)) has not occurred since the snapshot was valid. + ** Before checking that the live wal-index header has not changed + ** since it was read, set Wal.minFrame to the first frame in the wal + ** file that has not yet been checkpointed. This client will not need + ** to read any frames earlier than minFrame from the wal file - they + ** can be safely read directly from the database file. + ** + ** Because a ShmBarrier() call is made between taking the copy of + ** nBackfill and checking that the wal-header in shared-memory still + ** matches the one cached in pWal->hdr, it is guaranteed that the + ** checkpointer that set nBackfill was not working with a wal-index + ** header newer than that cached in pWal->hdr. If it were, that could + ** cause a problem. The checkpointer could omit to checkpoint + ** a version of page X that lies before pWal->minFrame (call that version + ** A) on the basis that there is a newer version (version B) of the same + ** page later in the wal file. But if version B happens to like past + ** frame pWal->hdr.mxFrame - then the client would incorrectly assume + ** that it can read version A from the database file. However, since + ** we can guarantee that the checkpointer that set nBackfill could not + ** see any pages past pWal->hdr.mxFrame, this problem does not come up. */ + pWal->minFrame = pInfo->nBackfill+1; walShmBarrier(pWal); if( pInfo->aReadMark[mxI]!=mxReadMark || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) @@ -51270,6 +52688,7 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( u32 iRead = 0; /* If !=0, WAL frame to return data from */ u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ int iHash; /* Used to loop through N hash tables */ + int iMinHash; /* This routine is only be called from within a read transaction. */ assert( pWal->readLock>=0 || pWal->lockError ); @@ -51310,7 +52729,8 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( ** This condition filters out entries that were added to the hash ** table after the current read-transaction had started. */ - for(iHash=walFramePage(iLast); iHash>=0 && iRead==0; iHash--){ + iMinHash = walFramePage(pWal->minFrame); + for(iHash=walFramePage(iLast); iHash>=iMinHash && iRead==0; iHash--){ volatile ht_slot *aHash; /* Pointer to hash table */ volatile u32 *aPgno; /* Pointer to array of page numbers */ u32 iZero; /* Frame number corresponding to aPgno[0] */ @@ -51325,7 +52745,7 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( nCollide = HASHTABLE_NSLOT; for(iKey=walHash(pgno); aHash[iKey]; iKey=walNextHash(iKey)){ u32 iFrame = aHash[iKey] + iZero; - if( iFrame<=iLast && aPgno[aHash[iKey]]==pgno ){ + if( iFrame<=iLast && iFrame>=pWal->minFrame && aPgno[aHash[iKey]]==pgno ){ assert( iFrame>iRead || CORRUPT_DB ); iRead = iFrame; } @@ -51342,7 +52762,8 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( { u32 iRead2 = 0; u32 iTest; - for(iTest=iLast; iTest>0; iTest--){ + assert( pWal->minFrame>0 ); + for(iTest=iLast; iTest>=pWal->minFrame; iTest--){ if( walFramePgno(pWal, iTest)==pgno ){ iRead2 = iTest; break; @@ -52295,6 +53716,7 @@ SQLITE_PRIVATE int sqlite3WalFramesize(Wal *pWal){ ** 4 Number of leaf pointers on this page ** * zero or more pages numbers of leaves */ +/* #include "sqliteInt.h" */ /* The following value is the maximum cell size assuming a maximum page @@ -52312,6 +53734,7 @@ SQLITE_PRIVATE int sqlite3WalFramesize(Wal *pWal){ /* Forward declarations */ typedef struct MemPage MemPage; typedef struct BtLock BtLock; +typedef struct CellInfo CellInfo; /* ** This is a magic string that appears at the beginning of every @@ -52375,7 +53798,10 @@ struct MemPage { u8 *aData; /* Pointer to disk image of the page data */ u8 *aDataEnd; /* One byte past the end of usable data */ u8 *aCellIdx; /* The cell index area */ + u8 *aDataOfst; /* Same as aData for leaves. aData+4 for interior */ DbPage *pDbPage; /* Pager page handle */ + u16 (*xCellSize)(MemPage*,u8*); /* cellSizePtr method */ + void (*xParseCell)(MemPage*,u8*,CellInfo*); /* btreeParseCell method */ Pgno pgno; /* Page number for this page */ }; @@ -52431,6 +53857,7 @@ struct Btree { u8 inTrans; /* TRANS_NONE, TRANS_READ or TRANS_WRITE */ u8 sharable; /* True if we can share pBt with another db */ u8 locked; /* True if db currently has pBt locked */ + u8 hasIncrblobCur; /* True if there are one or more Incrblob cursors */ int wantToLock; /* Number of nested calls to sqlite3BtreeEnter() */ int nBackup; /* Number of backup operations reading this btree */ u32 iDataVersion; /* Combines with pBt->pPager->iDataVersion */ @@ -52541,7 +53968,6 @@ struct BtShared { ** about a cell. The parseCellPtr() function fills in this structure ** based on information extract from the raw disk page. */ -typedef struct CellInfo CellInfo; struct CellInfo { i64 nKey; /* The key for INTKEY tables, or nPayload otherwise */ u8 *pPayload; /* Pointer to the start of payload */ @@ -52584,8 +54010,7 @@ struct CellInfo { struct BtCursor { Btree *pBtree; /* The Btree to which this cursor belongs */ BtShared *pBt; /* The BtShared this cursor points to */ - BtCursor *pNext, *pPrev; /* Forms a linked list of all cursors */ - struct KeyInfo *pKeyInfo; /* Argument passed to comparison function */ + BtCursor *pNext; /* Forms a linked list of all cursors */ Pgno *aOverflow; /* Cache of overflow page locations */ CellInfo info; /* A parse of the cell we are pointing at */ i64 nKey; /* Size of pKey, or last integer key */ @@ -52595,9 +54020,16 @@ struct BtCursor { int skipNext; /* Prev() is noop if negative. Next() is noop if positive. ** Error code if eState==CURSOR_FAULT */ u8 curFlags; /* zero or more BTCF_* flags defined below */ + u8 curPagerFlags; /* Flags to send to sqlite3PagerAcquire() */ u8 eState; /* One of the CURSOR_XXX constants (see below) */ - u8 hints; /* As configured by CursorSetHints() */ - i16 iPage; /* Index of current page in apPage */ + u8 hints; /* As configured by CursorSetHints() */ + /* All fields above are zeroed when the cursor is allocated. See + ** sqlite3BtreeCursorZero(). Fields that follow must be manually + ** initialized. */ + i8 iPage; /* Index of current page in apPage */ + u8 curIntKey; /* Value of apPage[0]->intKey */ + struct KeyInfo *pKeyInfo; /* Argument passed to comparison function */ + void *padding1; /* Make object size a multiple of 16 */ u16 aiIdx[BTCURSOR_MAX_DEPTH]; /* Current index in apPage[i] */ MemPage *apPage[BTCURSOR_MAX_DEPTH]; /* Pages from root to current page */ }; @@ -52610,6 +54042,7 @@ struct BtCursor { #define BTCF_ValidOvfl 0x04 /* True if aOverflow is valid */ #define BTCF_AtLast 0x08 /* Cursor is pointing ot the last entry */ #define BTCF_Incrblob 0x10 /* True if an incremental I/O handle */ +#define BTCF_Multiple 0x20 /* Maybe another cursor on the same btree */ /* ** Potential values for BtCursor.eState. @@ -52752,6 +54185,7 @@ struct IntegrityCk { const char *zPfx; /* Error message prefix */ int v1, v2; /* Values for up to two %d fields in zPfx */ StrAccum errMsg; /* Accumulate the error message text here */ + u32 *heap; /* Min-heap used for analyzing cell coverage */ }; /* @@ -52762,6 +54196,23 @@ struct IntegrityCk { #define get4byte sqlite3Get4byte #define put4byte sqlite3Put4byte +/* +** get2byteAligned(), unlike get2byte(), requires that its argument point to a +** two-byte aligned address. get2bytea() is only used for accessing the +** cell addresses in a btree header. +*/ +#if SQLITE_BYTEORDER==4321 +# define get2byteAligned(x) (*(u16*)(x)) +#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ + && GCC_VERSION>=4008000 +# define get2byteAligned(x) __builtin_bswap16(*(u16*)(x)) +#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ + && defined(_MSC_VER) && _MSC_VER>=1300 +# define get2byteAligned(x) _byteswap_ushort(*(u16*)(x)) +#else +# define get2byteAligned(x) ((x)[0]<<8 | (x)[1]) +#endif + /************** End of btreeInt.h ********************************************/ /************** Continuing where we left off in btmutex.c ********************/ #ifndef SQLITE_OMIT_SHARED_CACHE @@ -53065,6 +54516,7 @@ SQLITE_PRIVATE void sqlite3BtreeEnterAll(sqlite3 *db){ ** See the header comment on "btreeInt.h" for additional information. ** Including a description of file format and an overview of operation. */ +/* #include "btreeInt.h" */ /* ** The header string that appears at the beginning of every @@ -53541,13 +54993,15 @@ static void invalidateIncrblobCursors( int isClearTable /* True if all rows are being deleted */ ){ BtCursor *p; - BtShared *pBt = pBtree->pBt; + if( pBtree->hasIncrblobCur==0 ) return; assert( sqlite3BtreeHoldsMutex(pBtree) ); - for(p=pBt->pCursor; p; p=p->pNext){ - if( (p->curFlags & BTCF_Incrblob)!=0 - && (isClearTable || p->info.nKey==iRow) - ){ - p->eState = CURSOR_INVALID; + pBtree->hasIncrblobCur = 0; + for(p=pBtree->pBt->pCursor; p; p=p->pNext){ + if( (p->curFlags & BTCF_Incrblob)!=0 ){ + pBtree->hasIncrblobCur = 1; + if( isClearTable || p->info.nKey==iRow ){ + p->eState = CURSOR_INVALID; + } } } } @@ -53640,26 +55094,25 @@ static void btreeReleaseAllCursorPages(BtCursor *pCur){ pCur->iPage = -1; } - /* -** Save the current cursor position in the variables BtCursor.nKey -** and BtCursor.pKey. The cursor's state is set to CURSOR_REQUIRESEEK. +** The cursor passed as the only argument must point to a valid entry +** when this function is called (i.e. have eState==CURSOR_VALID). This +** function saves the current cursor key in variables pCur->nKey and +** pCur->pKey. SQLITE_OK is returned if successful or an SQLite error +** code otherwise. ** -** The caller must ensure that the cursor is valid (has eState==CURSOR_VALID) -** prior to calling this routine. +** If the cursor is open on an intkey table, then the integer key +** (the rowid) is stored in pCur->nKey and pCur->pKey is left set to +** NULL. If the cursor is open on a non-intkey table, then pCur->pKey is +** set to point to a malloced buffer pCur->nKey bytes in size containing +** the key. */ -static int saveCursorPosition(BtCursor *pCur){ +static int saveCursorKey(BtCursor *pCur){ int rc; - - assert( CURSOR_VALID==pCur->eState || CURSOR_SKIPNEXT==pCur->eState ); + assert( CURSOR_VALID==pCur->eState ); assert( 0==pCur->pKey ); assert( cursorHoldsMutex(pCur) ); - if( pCur->eState==CURSOR_SKIPNEXT ){ - pCur->eState = CURSOR_VALID; - }else{ - pCur->skipNext = 0; - } rc = sqlite3BtreeKeySize(pCur, &pCur->nKey); assert( rc==SQLITE_OK ); /* KeySize() cannot fail */ @@ -53667,9 +55120,8 @@ static int saveCursorPosition(BtCursor *pCur){ ** stores the integer key in pCur->nKey. In this case this value is ** all that is required. Otherwise, if pCur is not open on an intKey ** table, then malloc space for and store the pCur->nKey bytes of key - ** data. - */ - if( 0==pCur->apPage[0]->intKey ){ + ** data. */ + if( 0==pCur->curIntKey ){ void *pKey = sqlite3Malloc( pCur->nKey ); if( pKey ){ rc = sqlite3BtreeKey(pCur, 0, (int)pCur->nKey, pKey); @@ -53682,14 +55134,37 @@ static int saveCursorPosition(BtCursor *pCur){ rc = SQLITE_NOMEM; } } - assert( !pCur->apPage[0]->intKey || !pCur->pKey ); + assert( !pCur->curIntKey || !pCur->pKey ); + return rc; +} + +/* +** Save the current cursor position in the variables BtCursor.nKey +** and BtCursor.pKey. The cursor's state is set to CURSOR_REQUIRESEEK. +** +** The caller must ensure that the cursor is valid (has eState==CURSOR_VALID) +** prior to calling this routine. +*/ +static int saveCursorPosition(BtCursor *pCur){ + int rc; + + assert( CURSOR_VALID==pCur->eState || CURSOR_SKIPNEXT==pCur->eState ); + assert( 0==pCur->pKey ); + assert( cursorHoldsMutex(pCur) ); + + if( pCur->eState==CURSOR_SKIPNEXT ){ + pCur->eState = CURSOR_VALID; + }else{ + pCur->skipNext = 0; + } + rc = saveCursorKey(pCur); if( rc==SQLITE_OK ){ btreeReleaseAllCursorPages(pCur); pCur->eState = CURSOR_REQUIRESEEK; } - invalidateOverflowCache(pCur); + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl|BTCF_AtLast); return rc; } @@ -53704,6 +55179,15 @@ static int SQLITE_NOINLINE saveCursorsOnList(BtCursor*,Pgno,BtCursor*); ** routine is called just before cursor pExcept is used to modify the ** table, for example in BtreeDelete() or BtreeInsert(). ** +** If there are two or more cursors on the same btree, then all such +** cursors should have their BTCF_Multiple flag set. The btreeCursor() +** routine enforces that rule. This routine only needs to be called in +** the uncommon case when pExpect has the BTCF_Multiple flag set. +** +** If pExpect!=NULL and if no other cursors are found on the same root-page, +** then the BTCF_Multiple flag on pExpect is cleared, to avoid another +** pointless call to this routine. +** ** Implementation note: This routine merely checks to see if any cursors ** need to be saved. It calls out to saveCursorsOnList() in the (unusual) ** event that cursors are in need to being saved. @@ -53715,7 +55199,9 @@ static int saveAllCursors(BtShared *pBt, Pgno iRoot, BtCursor *pExcept){ for(p=pBt->pCursor; p; p=p->pNext){ if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ) break; } - return p ? saveCursorsOnList(p, iRoot, pExcept) : SQLITE_OK; + if( p ) return saveCursorsOnList(p, iRoot, pExcept); + if( pExcept ) pExcept->curFlags &= ~BTCF_Multiple; + return SQLITE_OK; } /* This helper routine to saveAllCursors does the actual work of saving @@ -54003,39 +55489,88 @@ static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ ** the page, 1 means the second cell, and so forth) return a pointer ** to the cell content. ** +** findCellPastPtr() does the same except it skips past the initial +** 4-byte child pointer found on interior pages, if there is one. +** ** This routine works only for pages that do not contain overflow cells. */ #define findCell(P,I) \ - ((P)->aData + ((P)->maskPage & get2byte(&(P)->aCellIdx[2*(I)]))) -#define findCellv2(D,M,O,I) (D+(M&get2byte(D+(O+2*(I))))) + ((P)->aData + ((P)->maskPage & get2byteAligned(&(P)->aCellIdx[2*(I)]))) +#define findCellPastPtr(P,I) \ + ((P)->aDataOfst + ((P)->maskPage & get2byteAligned(&(P)->aCellIdx[2*(I)]))) /* -** This a more complex version of findCell() that works for -** pages that do contain overflow cells. +** This is common tail processing for btreeParseCellPtr() and +** btreeParseCellPtrIndex() for the case when the cell does not fit entirely +** on a single B-tree page. Make necessary adjustments to the CellInfo +** structure. */ -static u8 *findOverflowCell(MemPage *pPage, int iCell){ - int i; - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - for(i=pPage->nOverflow-1; i>=0; i--){ - int k; - k = pPage->aiOvfl[i]; - if( k<=iCell ){ - if( k==iCell ){ - return pPage->apOvfl[i]; - } - iCell--; - } +static SQLITE_NOINLINE void btreeParseCellAdjustSizeForOverflow( + MemPage *pPage, /* Page containing the cell */ + u8 *pCell, /* Pointer to the cell text. */ + CellInfo *pInfo /* Fill in this structure */ +){ + /* If the payload will not fit completely on the local page, we have + ** to decide how much to store locally and how much to spill onto + ** overflow pages. The strategy is to minimize the amount of unused + ** space on overflow pages while keeping the amount of local storage + ** in between minLocal and maxLocal. + ** + ** Warning: changing the way overflow payload is distributed in any + ** way will result in an incompatible file format. + */ + int minLocal; /* Minimum amount of payload held locally */ + int maxLocal; /* Maximum amount of payload held locally */ + int surplus; /* Overflow payload available for local storage */ + + minLocal = pPage->minLocal; + maxLocal = pPage->maxLocal; + surplus = minLocal + (pInfo->nPayload - minLocal)%(pPage->pBt->usableSize-4); + testcase( surplus==maxLocal ); + testcase( surplus==maxLocal+1 ); + if( surplus <= maxLocal ){ + pInfo->nLocal = (u16)surplus; + }else{ + pInfo->nLocal = (u16)minLocal; } - return findCell(pPage, iCell); + pInfo->iOverflow = (u16)(&pInfo->pPayload[pInfo->nLocal] - pCell); + pInfo->nSize = pInfo->iOverflow + 4; } /* -** Parse a cell content block and fill in the CellInfo structure. There -** are two versions of this function. btreeParseCell() takes a -** cell index as the second argument and btreeParseCellPtr() -** takes a pointer to the body of the cell as its second argument. +** The following routines are implementations of the MemPage.xParseCell() +** method. +** +** Parse a cell content block and fill in the CellInfo structure. +** +** btreeParseCellPtr() => table btree leaf nodes +** btreeParseCellNoPayload() => table btree internal nodes +** btreeParseCellPtrIndex() => index btree nodes +** +** There is also a wrapper function btreeParseCell() that works for +** all MemPage types and that references the cell by index rather than +** by pointer. */ +static void btreeParseCellPtrNoPayload( + MemPage *pPage, /* Page containing the cell */ + u8 *pCell, /* Pointer to the cell text. */ + CellInfo *pInfo /* Fill in this structure */ +){ + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->leaf==0 ); + assert( pPage->noPayload ); + assert( pPage->childPtrSize==4 ); +#ifndef SQLITE_DEBUG + UNUSED_PARAMETER(pPage); +#endif + pInfo->nSize = 4 + getVarint(&pCell[4], (u64*)&pInfo->nKey); + pInfo->nPayload = 0; + pInfo->nLocal = 0; + pInfo->iOverflow = 0; + pInfo->pPayload = 0; + return; +} static void btreeParseCellPtr( MemPage *pPage, /* Page containing the cell */ u8 *pCell, /* Pointer to the cell text. */ @@ -54043,26 +55578,93 @@ static void btreeParseCellPtr( ){ u8 *pIter; /* For scanning through pCell */ u32 nPayload; /* Number of bytes of cell payload */ + u64 iKey; /* Extracted Key value */ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( pPage->leaf==0 || pPage->leaf==1 ); - if( pPage->intKeyLeaf ){ - assert( pPage->childPtrSize==0 ); - pIter = pCell + getVarint32(pCell, nPayload); - pIter += getVarint(pIter, (u64*)&pInfo->nKey); - }else if( pPage->noPayload ){ - assert( pPage->childPtrSize==4 ); - pInfo->nSize = 4 + getVarint(&pCell[4], (u64*)&pInfo->nKey); - pInfo->nPayload = 0; - pInfo->nLocal = 0; + assert( pPage->intKeyLeaf || pPage->noPayload ); + assert( pPage->noPayload==0 ); + assert( pPage->intKeyLeaf ); + assert( pPage->childPtrSize==0 ); + pIter = pCell; + + /* The next block of code is equivalent to: + ** + ** pIter += getVarint32(pIter, nPayload); + ** + ** The code is inlined to avoid a function call. + */ + nPayload = *pIter; + if( nPayload>=0x80 ){ + u8 *pEnd = &pIter[8]; + nPayload &= 0x7f; + do{ + nPayload = (nPayload<<7) | (*++pIter & 0x7f); + }while( (*pIter)>=0x80 && pIternKey); + ** + ** The code is inlined to avoid a function call. + */ + iKey = *pIter; + if( iKey>=0x80 ){ + u8 *pEnd = &pIter[7]; + iKey &= 0x7f; + while(1){ + iKey = (iKey<<7) | (*++pIter & 0x7f); + if( (*pIter)<0x80 ) break; + if( pIter>=pEnd ){ + iKey = (iKey<<8) | *++pIter; + break; + } + } + } + pIter++; + + pInfo->nKey = *(i64*)&iKey; + pInfo->nPayload = nPayload; + pInfo->pPayload = pIter; + testcase( nPayload==pPage->maxLocal ); + testcase( nPayload==pPage->maxLocal+1 ); + if( nPayload<=pPage->maxLocal ){ + /* This is the (easy) common case where the entire payload fits + ** on the local page. No overflow is required. + */ + pInfo->nSize = nPayload + (u16)(pIter - pCell); + if( pInfo->nSize<4 ) pInfo->nSize = 4; + pInfo->nLocal = (u16)nPayload; pInfo->iOverflow = 0; - pInfo->pPayload = 0; - return; }else{ - pIter = pCell + pPage->childPtrSize; - pIter += getVarint32(pIter, nPayload); - pInfo->nKey = nPayload; + btreeParseCellAdjustSizeForOverflow(pPage, pCell, pInfo); } +} +static void btreeParseCellPtrIndex( + MemPage *pPage, /* Page containing the cell */ + u8 *pCell, /* Pointer to the cell text. */ + CellInfo *pInfo /* Fill in this structure */ +){ + u8 *pIter; /* For scanning through pCell */ + u32 nPayload; /* Number of bytes of cell payload */ + + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->leaf==0 || pPage->leaf==1 ); + assert( pPage->intKeyLeaf==0 ); + assert( pPage->noPayload==0 ); + pIter = pCell + pPage->childPtrSize; + nPayload = *pIter; + if( nPayload>=0x80 ){ + u8 *pEnd = &pIter[8]; + nPayload &= 0x7f; + do{ + nPayload = (nPayload<<7) | (*++pIter & 0x7f); + }while( *(pIter)>=0x80 && pIternKey = nPayload; pInfo->nPayload = nPayload; pInfo->pPayload = pIter; testcase( nPayload==pPage->maxLocal ); @@ -54076,31 +55678,7 @@ static void btreeParseCellPtr( pInfo->nLocal = (u16)nPayload; pInfo->iOverflow = 0; }else{ - /* If the payload will not fit completely on the local page, we have - ** to decide how much to store locally and how much to spill onto - ** overflow pages. The strategy is to minimize the amount of unused - ** space on overflow pages while keeping the amount of local storage - ** in between minLocal and maxLocal. - ** - ** Warning: changing the way overflow payload is distributed in any - ** way will result in an incompatible file format. - */ - int minLocal; /* Minimum amount of payload held locally */ - int maxLocal; /* Maximum amount of payload held locally */ - int surplus; /* Overflow payload available for local storage */ - - minLocal = pPage->minLocal; - maxLocal = pPage->maxLocal; - surplus = minLocal + (nPayload - minLocal)%(pPage->pBt->usableSize - 4); - testcase( surplus==maxLocal ); - testcase( surplus==maxLocal+1 ); - if( surplus <= maxLocal ){ - pInfo->nLocal = (u16)surplus; - }else{ - pInfo->nLocal = (u16)minLocal; - } - pInfo->iOverflow = (u16)(&pInfo->pPayload[pInfo->nLocal] - pCell); - pInfo->nSize = pInfo->iOverflow + 4; + btreeParseCellAdjustSizeForOverflow(pPage, pCell, pInfo); } } static void btreeParseCell( @@ -54108,14 +55686,20 @@ static void btreeParseCell( int iCell, /* The cell index. First cell is 0 */ CellInfo *pInfo /* Fill in this structure */ ){ - btreeParseCellPtr(pPage, findCell(pPage, iCell), pInfo); + pPage->xParseCell(pPage, findCell(pPage, iCell), pInfo); } /* +** The following routines are implementations of the MemPage.xCellSize +** method. +** ** Compute the total number of bytes that a Cell needs in the cell ** data area of the btree-page. The return number includes the cell ** data header and the local payload, but not any overflow page or ** the space used by the cell pointer. +** +** cellSizePtrNoPayload() => table internal nodes +** cellSizePtr() => all index nodes & table leaf nodes */ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ u8 *pIter = pCell + pPage->childPtrSize; /* For looping over bytes of pCell */ @@ -54128,18 +55712,13 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of ** this function verifies that this invariant is not violated. */ CellInfo debuginfo; - btreeParseCellPtr(pPage, pCell, &debuginfo); + pPage->xParseCell(pPage, pCell, &debuginfo); #endif - if( pPage->noPayload ){ - pEnd = &pIter[9]; - while( (*pIter++)&0x80 && pIterchildPtrSize==4 ); - return (u16)(pIter - pCell); - } + assert( pPage->noPayload==0 ); nSize = *pIter; if( nSize>=0x80 ){ - pEnd = &pIter[9]; + pEnd = &pIter[8]; nSize &= 0x7f; do{ nSize = (nSize<<7) | (*++pIter & 0x7f); @@ -54171,12 +55750,34 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ assert( nSize==debuginfo.nSize || CORRUPT_DB ); return (u16)nSize; } +static u16 cellSizePtrNoPayload(MemPage *pPage, u8 *pCell){ + u8 *pIter = pCell + 4; /* For looping over bytes of pCell */ + u8 *pEnd; /* End mark for a varint */ + +#ifdef SQLITE_DEBUG + /* The value returned by this function should always be the same as + ** the (CellInfo.nSize) value found by doing a full parse of the + ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of + ** this function verifies that this invariant is not violated. */ + CellInfo debuginfo; + pPage->xParseCell(pPage, pCell, &debuginfo); +#else + UNUSED_PARAMETER(pPage); +#endif + + assert( pPage->childPtrSize==4 ); + pEnd = pIter + 9; + while( (*pIter++)&0x80 && pIterxCellSize(pPage, findCell(pPage, iCell)); } #endif @@ -54190,7 +55791,7 @@ static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){ CellInfo info; if( *pRC ) return; assert( pCell!=0 ); - btreeParseCellPtr(pPage, pCell, &info); + pPage->xParseCell(pPage, pCell, &info); if( info.iOverflow ){ Pgno ovfl = get4byte(&pCell[info.iOverflow]); ptrmapPut(pPage->pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, pRC); @@ -54247,26 +55848,18 @@ static int defragmentPage(MemPage *pPage){ pc = get2byte(pAddr); testcase( pc==iCellFirst ); testcase( pc==iCellLast ); -#if !defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) /* These conditions have already been verified in btreeInitPage() - ** if SQLITE_ENABLE_OVERSIZE_CELL_CHECK is defined + ** if PRAGMA cell_size_check=ON. */ if( pciCellLast ){ return SQLITE_CORRUPT_BKPT; } -#endif assert( pc>=iCellFirst && pc<=iCellLast ); - size = cellSizePtr(pPage, &src[pc]); + size = pPage->xCellSize(pPage, &src[pc]); cbrk -= size; -#if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) - if( cbrkusableSize ){ return SQLITE_CORRUPT_BKPT; } -#endif assert( cbrk+size<=usableSize && cbrk>=iCellFirst ); testcase( cbrk+size==usableSize ); testcase( pc+size==usableSize ); @@ -54304,18 +55897,20 @@ static int defragmentPage(MemPage *pPage){ ** This function may detect corruption within pPg. If corruption is ** detected then *pRc is set to SQLITE_CORRUPT and NULL is returned. ** -** If a slot of at least nByte bytes is found but cannot be used because -** there are already at least 60 fragmented bytes on the page, return NULL. -** In this case, if pbDefrag parameter is not NULL, set *pbDefrag to true. +** Slots on the free list that are between 1 and 3 bytes larger than nByte +** will be ignored if adding the extra space to the fragmentation count +** causes the fragmentation count to exceed 60. */ -static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc, int *pbDefrag){ +static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ const int hdr = pPg->hdrOffset; u8 * const aData = pPg->aData; - int iAddr; - int pc; + int iAddr = hdr + 1; + int pc = get2byte(&aData[iAddr]); + int x; int usableSize = pPg->pBt->usableSize; - for(iAddr=hdr+1; (pc = get2byte(&aData[iAddr]))>0; iAddr=pc){ + assert( pc>0 ); + do{ int size; /* Size of the free slot */ /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of ** increasing offset. */ @@ -54327,24 +55922,21 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc, int *pbDefrag){ ** freeblock form a big-endian integer which is the size of the freeblock ** in bytes, including the 4-byte header. */ size = get2byte(&aData[pc+2]); - if( size>=nByte ){ - int x = size - nByte; + if( (x = size - nByte)>=0 ){ testcase( x==4 ); testcase( x==3 ); - if( x<4 ){ + if( pc < pPg->cellOffset+2*pPg->nCell || size+pc > usableSize ){ + *pRc = SQLITE_CORRUPT_BKPT; + return 0; + }else if( x<4 ){ /* EVIDENCE-OF: R-11498-58022 In a well-formed b-tree page, the total ** number of bytes in fragments may not exceed 60. */ - if( aData[hdr+7]>=60 ){ - if( pbDefrag ) *pbDefrag = 1; - return 0; - } + if( aData[hdr+7]>57 ) return 0; + /* Remove the slot from the free-list. Update the number of ** fragmented bytes within the page. */ memcpy(&aData[iAddr], &aData[pc], 2); aData[hdr+7] += (u8)x; - }else if( size+pc > usableSize ){ - *pRc = SQLITE_CORRUPT_BKPT; - return 0; }else{ /* The slot remains on the free-list. Reduce its size to account ** for the portion used by the new allocation. */ @@ -54352,7 +55944,9 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc, int *pbDefrag){ } return &aData[pc + x]; } - } + iAddr = pc; + pc = get2byte(&aData[pc]); + }while( pc ); return 0; } @@ -54393,8 +55987,15 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ ** then the cell content offset of an empty page wants to be 65536. ** However, that integer is too large to be stored in a 2-byte unsigned ** integer, so a value of 0 is used in its place. */ - top = get2byteNotZero(&data[hdr+5]); - if( gap>top ) return SQLITE_CORRUPT_BKPT; + top = get2byte(&data[hdr+5]); + assert( top<=(int)pPage->pBt->usableSize ); /* Prevent by getAndInitPage() */ + if( gap>top ){ + if( top==0 && pPage->pBt->usableSize==65536 ){ + top = 65536; + }else{ + return SQLITE_CORRUPT_BKPT; + } + } /* If there is enough space between gap and top for one more cell pointer ** array entry offset, and if the freelist is not empty, then search the @@ -54403,15 +56004,14 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ testcase( gap+2==top ); testcase( gap+1==top ); testcase( gap==top ); - if( gap+2<=top && (data[hdr+1] || data[hdr+2]) ){ - int bDefrag = 0; - u8 *pSpace = pageFindSlot(pPage, nByte, &rc, &bDefrag); - if( rc ) return rc; - if( bDefrag ) goto defragment_page; + if( (data[hdr+2] || data[hdr+1]) && gap+2<=top ){ + u8 *pSpace = pageFindSlot(pPage, nByte, &rc); if( pSpace ){ assert( pSpace>=data && (pSpace - data)<65536 ); *pIdx = (int)(pSpace - data); return SQLITE_OK; + }else if( rc ){ + return rc; } } @@ -54420,7 +56020,6 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ */ testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ - defragment_page: assert( pPage->nCell>0 || CORRUPT_DB ); rc = defragmentPage(pPage); if( rc ) return rc; @@ -54467,7 +56066,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ assert( pPage->pBt!=0 ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - assert( iStart>=pPage->hdrOffset+6+pPage->childPtrSize ); + assert( CORRUPT_DB || iStart>=pPage->hdrOffset+6+pPage->childPtrSize ); assert( CORRUPT_DB || iEnd <= pPage->pBt->usableSize ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( iSize>=4 ); /* Minimum cell size is 4 */ @@ -54496,7 +56095,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ /* At this point: ** iFreeBlk: First freeblock after iStart, or zero if none - ** iPtr: The address of a pointer iFreeBlk + ** iPtr: The address of a pointer to iFreeBlk ** ** Check to see if iFreeBlk should be coalesced onto the end of iStart. */ @@ -54504,6 +56103,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ nFrag = iFreeBlk - iEnd; if( iEnd>iFreeBlk ) return SQLITE_CORRUPT_BKPT; iEnd = iFreeBlk + get2byte(&data[iFreeBlk+2]); + if( iEnd > pPage->pBt->usableSize ) return SQLITE_CORRUPT_BKPT; iSize = iEnd - iStart; iFreeBlk = get2byte(&data[iFreeBlk]); } @@ -54561,6 +56161,7 @@ static int decodeFlags(MemPage *pPage, int flagByte){ pPage->leaf = (u8)(flagByte>>3); assert( PTF_LEAF == 1<<3 ); flagByte &= ~PTF_LEAF; pPage->childPtrSize = 4-4*pPage->leaf; + pPage->xCellSize = cellSizePtr; pBt = pPage->pBt; if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ /* EVIDENCE-OF: R-03640-13415 A value of 5 means the page is an interior @@ -54570,8 +56171,16 @@ static int decodeFlags(MemPage *pPage, int flagByte){ ** table b-tree page. */ assert( (PTF_LEAFDATA|PTF_INTKEY|PTF_LEAF)==13 ); pPage->intKey = 1; - pPage->intKeyLeaf = pPage->leaf; - pPage->noPayload = !pPage->leaf; + if( pPage->leaf ){ + pPage->intKeyLeaf = 1; + pPage->noPayload = 0; + pPage->xParseCell = btreeParseCellPtr; + }else{ + pPage->intKeyLeaf = 0; + pPage->noPayload = 1; + pPage->xCellSize = cellSizePtrNoPayload; + pPage->xParseCell = btreeParseCellPtrNoPayload; + } pPage->maxLocal = pBt->maxLeaf; pPage->minLocal = pBt->minLeaf; }else if( flagByte==PTF_ZERODATA ){ @@ -54584,6 +56193,7 @@ static int decodeFlags(MemPage *pPage, int flagByte){ pPage->intKey = 0; pPage->intKeyLeaf = 0; pPage->noPayload = 0; + pPage->xParseCell = btreeParseCellPtrIndex; pPage->maxLocal = pBt->maxLocal; pPage->minLocal = pBt->minLocal; }else{ @@ -54607,6 +56217,7 @@ static int decodeFlags(MemPage *pPage, int flagByte){ static int btreeInitPage(MemPage *pPage){ assert( pPage->pBt!=0 ); + assert( pPage->pBt->db!=0 ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( pPage->pgno==sqlite3PagerPagenumber(pPage->pDbPage) ); assert( pPage == sqlite3PagerGetExtra(pPage->pDbPage) ); @@ -54638,6 +56249,7 @@ static int btreeInitPage(MemPage *pPage){ pPage->cellOffset = cellOffset = hdr + 8 + pPage->childPtrSize; pPage->aDataEnd = &data[usableSize]; pPage->aCellIdx = &data[cellOffset]; + pPage->aDataOfst = &data[pPage->childPtrSize]; /* EVIDENCE-OF: R-58015-48175 The two-byte integer at offset 5 designates ** the start of the cell content area. A zero value for this integer is ** interpreted as 65536. */ @@ -54665,20 +56277,19 @@ static int btreeInitPage(MemPage *pPage){ */ iCellFirst = cellOffset + 2*pPage->nCell; iCellLast = usableSize - 4; -#if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) - { + if( pBt->db->flags & SQLITE_CellSizeCk ){ int i; /* Index into the cell pointer array */ int sz; /* Size of a cell */ if( !pPage->leaf ) iCellLast--; for(i=0; inCell; i++){ - pc = get2byte(&data[cellOffset+i*2]); + pc = get2byteAligned(&data[cellOffset+i*2]); testcase( pc==iCellFirst ); testcase( pc==iCellLast ); if( pciCellLast ){ return SQLITE_CORRUPT_BKPT; } - sz = cellSizePtr(pPage, &data[pc]); + sz = pPage->xCellSize(pPage, &data[pc]); testcase( pc+sz==usableSize ); if( pc+sz>usableSize ){ return SQLITE_CORRUPT_BKPT; @@ -54686,7 +56297,6 @@ static int btreeInitPage(MemPage *pPage){ } if( !pPage->leaf ) iCellLast++; } -#endif /* Compute the total free space on the page ** EVIDENCE-OF: R-23588-34450 The two-byte integer at offset 1 gives the @@ -54759,6 +56369,7 @@ static void zeroPage(MemPage *pPage, int flags){ pPage->cellOffset = first; pPage->aDataEnd = &data[pBt->usableSize]; pPage->aCellIdx = &data[first]; + pPage->aDataOfst = &data[pPage->childPtrSize]; pPage->nOverflow = 0; assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); pPage->maskPage = (u16)(pBt->pageSize - 1); @@ -54777,16 +56388,16 @@ static MemPage *btreePageFromDbPage(DbPage *pDbPage, Pgno pgno, BtShared *pBt){ pPage->pDbPage = pDbPage; pPage->pBt = pBt; pPage->pgno = pgno; - pPage->hdrOffset = pPage->pgno==1 ? 100 : 0; + pPage->hdrOffset = pgno==1 ? 100 : 0; return pPage; } /* ** Get a page from the pager. Initialize the MemPage.pBt and -** MemPage.aData elements if needed. +** MemPage.aData elements if needed. See also: btreeGetUnusedPage(). ** -** If the noContent flag is set, it means that we do not care about -** the content of the page at this time. So do not go to the disk +** If the PAGER_GET_NOCONTENT flag is set, it means that we do not care +** about the content of the page at this time. So do not go to the disk ** to fetch the content. Just fill in the content with zeros for now. ** If in the future we call sqlite3PagerWrite() on this page, that ** means we have started to be concerned about content and the disk @@ -54838,35 +56449,62 @@ SQLITE_PRIVATE u32 sqlite3BtreeLastPage(Btree *p){ } /* -** Get a page from the pager and initialize it. This routine is just a -** convenience wrapper around separate calls to btreeGetPage() and -** btreeInitPage(). +** Get a page from the pager and initialize it. ** -** If an error occurs, then the value *ppPage is set to is undefined. It +** If pCur!=0 then the page is being fetched as part of a moveToChild() +** call. Do additional sanity checking on the page in this case. +** And if the fetch fails, this routine must decrement pCur->iPage. +** +** The page is fetched as read-write unless pCur is not NULL and is +** a read-only cursor. +** +** If an error occurs, then *ppPage is undefined. It ** may remain unchanged, or it may be set to an invalid value. */ static int getAndInitPage( BtShared *pBt, /* The database file */ Pgno pgno, /* Number of the page to get */ MemPage **ppPage, /* Write the page pointer here */ - int bReadonly /* PAGER_GET_READONLY or 0 */ + BtCursor *pCur, /* Cursor to receive the page, or NULL */ + int bReadOnly /* True for a read-only page */ ){ int rc; + DbPage *pDbPage; assert( sqlite3_mutex_held(pBt->mutex) ); - assert( bReadonly==PAGER_GET_READONLY || bReadonly==0 ); + assert( pCur==0 || ppPage==&pCur->apPage[pCur->iPage] ); + assert( pCur==0 || bReadOnly==pCur->curPagerFlags ); + assert( pCur==0 || pCur->iPage>0 ); if( pgno>btreePagecount(pBt) ){ rc = SQLITE_CORRUPT_BKPT; - }else{ - rc = btreeGetPage(pBt, pgno, ppPage, bReadonly); - if( rc==SQLITE_OK && (*ppPage)->isInit==0 ){ - rc = btreeInitPage(*ppPage); - if( rc!=SQLITE_OK ){ - releasePage(*ppPage); - } + goto getAndInitPage_error; + } + rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, bReadOnly); + if( rc ){ + goto getAndInitPage_error; + } + *ppPage = btreePageFromDbPage(pDbPage, pgno, pBt); + if( (*ppPage)->isInit==0 ){ + rc = btreeInitPage(*ppPage); + if( rc!=SQLITE_OK ){ + releasePage(*ppPage); + goto getAndInitPage_error; } } + /* If obtaining a child page for a cursor, we must verify that the page is + ** compatible with the root page. */ + if( pCur + && ((*ppPage)->nCell<1 || (*ppPage)->intKey!=pCur->curIntKey) + ){ + rc = SQLITE_CORRUPT_BKPT; + releasePage(*ppPage); + goto getAndInitPage_error; + } + return SQLITE_OK; + +getAndInitPage_error: + if( pCur ) pCur->iPage--; testcase( pgno==0 ); assert( pgno!=0 || rc==SQLITE_CORRUPT ); return rc; @@ -54876,18 +56514,49 @@ static int getAndInitPage( ** Release a MemPage. This should be called once for each prior ** call to btreeGetPage. */ +static void releasePageNotNull(MemPage *pPage){ + assert( pPage->aData ); + assert( pPage->pBt ); + assert( pPage->pDbPage!=0 ); + assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); + assert( sqlite3PagerGetData(pPage->pDbPage)==pPage->aData ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + sqlite3PagerUnrefNotNull(pPage->pDbPage); +} static void releasePage(MemPage *pPage){ - if( pPage ){ - assert( pPage->aData ); - assert( pPage->pBt ); - assert( pPage->pDbPage!=0 ); - assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); - assert( sqlite3PagerGetData(pPage->pDbPage)==pPage->aData ); - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - sqlite3PagerUnrefNotNull(pPage->pDbPage); + if( pPage ) releasePageNotNull(pPage); +} + +/* +** Get an unused page. +** +** This works just like btreeGetPage() with the addition: +** +** * If the page is already in use for some other purpose, immediately +** release it and return an SQLITE_CURRUPT error. +** * Make sure the isInit flag is clear +*/ +static int btreeGetUnusedPage( + BtShared *pBt, /* The btree */ + Pgno pgno, /* Number of the page to fetch */ + MemPage **ppPage, /* Return the page in this parameter */ + int flags /* PAGER_GET_NOCONTENT or PAGER_GET_READONLY */ +){ + int rc = btreeGetPage(pBt, pgno, ppPage, flags); + if( rc==SQLITE_OK ){ + if( sqlite3PagerPageRefcount((*ppPage)->pDbPage)>1 ){ + releasePage(*ppPage); + *ppPage = 0; + return SQLITE_CORRUPT_BKPT; + } + (*ppPage)->isInit = 0; + }else{ + *ppPage = 0; } + return rc; } + /* ** During a rollback, when the pager reloads information into the cache ** so that the cache is restored to its original state at the start of @@ -55830,7 +57499,7 @@ static void unlockBtreeIfUnused(BtShared *pBt){ assert( pPage1->aData ); assert( sqlite3PagerRefcount(pBt->pPager)==1 ); pBt->pPage1 = 0; - releasePage(pPage1); + releasePageNotNull(pPage1); } } @@ -56135,15 +57804,17 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ u8 isInitOrig = pPage->isInit; int i; int nCell; + int rc; - btreeInitPage(pPage); + rc = btreeInitPage(pPage); + if( rc ) return rc; nCell = pPage->nCell; for(i=0; ixParseCell(pPage, pCell, &info); if( info.iOverflow && pCell+info.iOverflow+3<=pPage->aData+pPage->maskPage && iFrom==get4byte(&pCell[info.iOverflow]) @@ -56442,7 +58113,7 @@ SQLITE_PRIVATE int sqlite3BtreeIncrVacuum(Btree *p){ static int autoVacuumCommit(BtShared *pBt){ int rc = SQLITE_OK; Pager *pPager = pBt->pPager; - VVA_ONLY( int nRef = sqlite3PagerRefcount(pPager) ); + VVA_ONLY( int nRef = sqlite3PagerRefcount(pPager); ) assert( sqlite3_mutex_held(pBt->mutex) ); invalidateAllOverflowCache(pBt); @@ -56884,6 +58555,7 @@ static int btreeCursor( BtCursor *pCur /* Space for new cursor */ ){ BtShared *pBt = p->pBt; /* Shared b-tree handle */ + BtCursor *pX; /* Looping over other all cursors */ assert( sqlite3BtreeHoldsMutex(p) ); assert( wrFlag==0 || wrFlag==1 ); @@ -56899,10 +58571,8 @@ static int btreeCursor( assert( p->inTrans>TRANS_NONE ); assert( wrFlag==0 || p->inTrans==TRANS_WRITE ); assert( pBt->pPage1 && pBt->pPage1->aData ); + assert( wrFlag==0 || (pBt->btsFlags & BTS_READ_ONLY)==0 ); - if( NEVER(wrFlag && (pBt->btsFlags & BTS_READ_ONLY)!=0) ){ - return SQLITE_READONLY; - } if( wrFlag ){ allocateTempSpace(pBt); if( pBt->pTmpSpace==0 ) return SQLITE_NOMEM; @@ -56921,10 +58591,16 @@ static int btreeCursor( pCur->pBt = pBt; assert( wrFlag==0 || wrFlag==BTCF_WriteFlag ); pCur->curFlags = wrFlag; - pCur->pNext = pBt->pCursor; - if( pCur->pNext ){ - pCur->pNext->pPrev = pCur; + pCur->curPagerFlags = wrFlag ? 0 : PAGER_GET_READONLY; + /* If there are two or more cursors on the same btree, then all such + ** cursors *must* have the BTCF_Multiple flag set. */ + for(pX=pBt->pCursor; pX; pX=pX->pNext){ + if( pX->pgnoRoot==(Pgno)iTable ){ + pX->curFlags |= BTCF_Multiple; + pCur->curFlags |= BTCF_Multiple; + } } + pCur->pNext = pBt->pCursor; pBt->pCursor = pCur; pCur->eState = CURSOR_INVALID; return SQLITE_OK; @@ -56937,9 +58613,13 @@ SQLITE_PRIVATE int sqlite3BtreeCursor( BtCursor *pCur /* Write new cursor here */ ){ int rc; - sqlite3BtreeEnter(p); - rc = btreeCursor(p, iTable, wrFlag, pKeyInfo, pCur); - sqlite3BtreeLeave(p); + if( iTable<1 ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + sqlite3BtreeEnter(p); + rc = btreeCursor(p, iTable, wrFlag, pKeyInfo, pCur); + sqlite3BtreeLeave(p); + } return rc; } @@ -56978,13 +58658,18 @@ SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor *pCur){ BtShared *pBt = pCur->pBt; sqlite3BtreeEnter(pBtree); sqlite3BtreeClearCursor(pCur); - if( pCur->pPrev ){ - pCur->pPrev->pNext = pCur->pNext; - }else{ + assert( pBt->pCursor!=0 ); + if( pBt->pCursor==pCur ){ pBt->pCursor = pCur->pNext; - } - if( pCur->pNext ){ - pCur->pNext->pPrev = pCur->pPrev; + }else{ + BtCursor *pPrev = pBt->pCursor; + do{ + if( pPrev->pNext==pCur ){ + pPrev->pNext = pCur->pNext; + break; + } + pPrev = pPrev->pNext; + }while( ALWAYS(pPrev) ); } for(i=0; i<=pCur->iPage; i++){ releasePage(pCur->apPage[i]); @@ -57004,13 +58689,6 @@ SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor *pCur){ ** ** BtCursor.info is a cache of the information in the current cell. ** Using this cache reduces the number of calls to btreeParseCell(). -** -** 2007-06-25: There is a bug in some versions of MSVC that cause the -** compiler to crash when getCellInfo() is implemented as a macro. -** But there is a measureable speed advantage to using the macro on gcc -** (when less compiler optimizations like -Os or -O0 are used and the -** compiler is not doing aggressive inlining.) So we use a real function -** for MSVC and a macro for everything else. Ticket #2457. */ #ifndef NDEBUG static void assertCellInfo(BtCursor *pCur){ @@ -57023,28 +58701,15 @@ SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor *pCur){ #else #define assertCellInfo(x) #endif -#ifdef _MSC_VER - /* Use a real function in MSVC to work around bugs in that compiler. */ - static void getCellInfo(BtCursor *pCur){ - if( pCur->info.nSize==0 ){ - int iPage = pCur->iPage; - btreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); - pCur->curFlags |= BTCF_ValidNKey; - }else{ - assertCellInfo(pCur); - } - } -#else /* if not _MSC_VER */ - /* Use a macro in all other compilers so that the function is inlined */ -#define getCellInfo(pCur) \ - if( pCur->info.nSize==0 ){ \ - int iPage = pCur->iPage; \ - btreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); \ - pCur->curFlags |= BTCF_ValidNKey; \ - }else{ \ - assertCellInfo(pCur); \ +static SQLITE_NOINLINE void getCellInfo(BtCursor *pCur){ + if( pCur->info.nSize==0 ){ + int iPage = pCur->iPage; + pCur->curFlags |= BTCF_ValidNKey; + btreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); + }else{ + assertCellInfo(pCur); } -#endif /* _MSC_VER */ +} #ifndef NDEBUG /* The next routine used only within assert() statements */ /* @@ -57550,9 +59215,6 @@ SQLITE_PRIVATE const void *sqlite3BtreeDataFetch(BtCursor *pCur, u32 *pAmt){ ** vice-versa). */ static int moveToChild(BtCursor *pCur, u32 newPgno){ - int rc; - int i = pCur->iPage; - MemPage *pNewPage; BtShared *pBt = pCur->pBt; assert( cursorHoldsMutex(pCur) ); @@ -57562,19 +59224,12 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){ if( pCur->iPage>=(BTCURSOR_MAX_DEPTH-1) ){ return SQLITE_CORRUPT_BKPT; } - rc = getAndInitPage(pBt, newPgno, &pNewPage, - (pCur->curFlags & BTCF_WriteFlag)==0 ? PAGER_GET_READONLY : 0); - if( rc ) return rc; - pCur->apPage[i+1] = pNewPage; - pCur->aiIdx[i+1] = 0; - pCur->iPage++; - pCur->info.nSize = 0; pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); - if( pNewPage->nCell<1 || pNewPage->intKey!=pCur->apPage[i]->intKey ){ - return SQLITE_CORRUPT_BKPT; - } - return SQLITE_OK; + pCur->iPage++; + pCur->aiIdx[pCur->iPage] = 0; + return getAndInitPage(pBt, newPgno, &pCur->apPage[pCur->iPage], + pCur, pCur->curPagerFlags); } #if SQLITE_DEBUG @@ -57618,11 +59273,9 @@ static void moveToParent(BtCursor *pCur){ pCur->apPage[pCur->iPage]->pgno ); testcase( pCur->aiIdx[pCur->iPage-1] > pCur->apPage[pCur->iPage-1]->nCell ); - - releasePage(pCur->apPage[pCur->iPage]); - pCur->iPage--; pCur->info.nSize = 0; pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + releasePageNotNull(pCur->apPage[pCur->iPage--]); } /* @@ -57663,18 +59316,23 @@ static int moveToRoot(BtCursor *pCur){ } if( pCur->iPage>=0 ){ - while( pCur->iPage ) releasePage(pCur->apPage[pCur->iPage--]); + while( pCur->iPage ){ + assert( pCur->apPage[pCur->iPage]!=0 ); + releasePageNotNull(pCur->apPage[pCur->iPage--]); + } }else if( pCur->pgnoRoot==0 ){ pCur->eState = CURSOR_INVALID; return SQLITE_OK; }else{ + assert( pCur->iPage==(-1) ); rc = getAndInitPage(pCur->pBtree->pBt, pCur->pgnoRoot, &pCur->apPage[0], - (pCur->curFlags & BTCF_WriteFlag)==0 ? PAGER_GET_READONLY : 0); + 0, pCur->curPagerFlags); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; return rc; } pCur->iPage = 0; + pCur->curIntKey = pCur->apPage[0]->intKey; } pRoot = pCur->apPage[0]; assert( pRoot->pgno==pCur->pgnoRoot ); @@ -57877,7 +59535,7 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( /* If the cursor is already positioned at the point we are trying ** to move to, then just return without doing any work */ if( pCur->eState==CURSOR_VALID && (pCur->curFlags & BTCF_ValidNKey)!=0 - && pCur->apPage[0]->intKey + && pCur->curIntKey ){ if( pCur->info.nKey==intKey ){ *pRes = 0; @@ -57912,7 +59570,8 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 ); return SQLITE_OK; } - assert( pCur->apPage[0]->intKey || pIdxKey ); + assert( pCur->apPage[0]->intKey==pCur->curIntKey ); + assert( pCur->curIntKey || pIdxKey ); for(;;){ int lwr, upr, idx, c; Pgno chldPg; @@ -57935,7 +59594,7 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( if( xRecordCompare==0 ){ for(;;){ i64 nCellKey; - pCell = findCell(pPage, idx) + pPage->childPtrSize; + pCell = findCellPastPtr(pPage, idx); if( pPage->intKeyLeaf ){ while( 0x80 <= *(pCell++) ){ if( pCell>=pPage->aDataEnd ) return SQLITE_CORRUPT_BKPT; @@ -57967,8 +59626,8 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( } }else{ for(;;){ - int nCell; - pCell = findCell(pPage, idx) + pPage->childPtrSize; + int nCell; /* Size of the pCell cell in bytes */ + pCell = findCellPastPtr(pPage, idx); /* The maximum supported page-size is 65536 bytes. This means that ** the maximum number of record bytes stored on an index B-Tree @@ -57996,12 +59655,25 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( /* The record flows over onto one or more overflow pages. In ** this case the whole cell needs to be parsed, a buffer allocated ** and accessPayload() used to retrieve the record into the - ** buffer before VdbeRecordCompare() can be called. */ + ** buffer before VdbeRecordCompare() can be called. + ** + ** If the record is corrupt, the xRecordCompare routine may read + ** up to two varints past the end of the buffer. An extra 18 + ** bytes of padding is allocated at the end of the buffer in + ** case this happens. */ void *pCellKey; u8 * const pCellBody = pCell - pPage->childPtrSize; - btreeParseCellPtr(pPage, pCellBody, &pCur->info); + pPage->xParseCell(pPage, pCellBody, &pCur->info); nCell = (int)pCur->info.nKey; - pCellKey = sqlite3Malloc( nCell ); + testcase( nCell<0 ); /* True if key size is 2^32 or more */ + testcase( nCell==0 ); /* Invalid key size: 0x80 0x80 0x00 */ + testcase( nCell==1 ); /* Invalid key size: 0x80 0x80 0x01 */ + testcase( nCell==2 ); /* Minimum legal index key size */ + if( nCell<2 ){ + rc = SQLITE_CORRUPT_BKPT; + goto moveto_finish; + } + pCellKey = sqlite3Malloc( nCell+18 ); if( pCellKey==0 ){ rc = SQLITE_NOMEM; goto moveto_finish; @@ -58294,8 +59966,7 @@ SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ ** sqlite3PagerUnref() on the new page when it is done. ** ** SQLITE_OK is returned on success. Any other return value indicates -** an error. *ppPage and *pPgno are undefined in the event of an error. -** Do not invoke sqlite3PagerUnref() on *ppPage if an error is returned. +** an error. *ppPage is set to NULL in the event of an error. ** ** If the "nearby" parameter is not 0, then an effort is made to ** locate a page close to the page number "nearby". This can be used in an @@ -58338,6 +60009,7 @@ static int allocateBtreePage( /* There are pages on the freelist. Reuse one of those pages. */ Pgno iTrunk; u8 searchList = 0; /* If the free-list must be searched for 'nearby' */ + u32 nSearch = 0; /* Count of the number of search attempts */ /* If eMode==BTALLOC_EXACT and a query of the pointer-map ** shows that the page 'nearby' is somewhere on the free-list, then @@ -58386,10 +60058,10 @@ static int allocateBtreePage( iTrunk = get4byte(&pPage1->aData[32]); } testcase( iTrunk==mxPage ); - if( iTrunk>mxPage ){ + if( iTrunk>mxPage || nSearch++ > n ){ rc = SQLITE_CORRUPT_BKPT; }else{ - rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); + rc = btreeGetUnusedPage(pBt, iTrunk, &pTrunk, 0); } if( rc ){ pTrunk = 0; @@ -58454,7 +60126,7 @@ static int allocateBtreePage( goto end_allocate_page; } testcase( iNewTrunk==mxPage ); - rc = btreeGetPage(pBt, iNewTrunk, &pNewTrunk, 0); + rc = btreeGetUnusedPage(pBt, iNewTrunk, &pNewTrunk, 0); if( rc!=SQLITE_OK ){ goto end_allocate_page; } @@ -58534,11 +60206,12 @@ static int allocateBtreePage( } put4byte(&aData[4], k-1); noContent = !btreeGetHasContent(pBt, *pPgno)? PAGER_GET_NOCONTENT : 0; - rc = btreeGetPage(pBt, *pPgno, ppPage, noContent); + rc = btreeGetUnusedPage(pBt, *pPgno, ppPage, noContent); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite((*ppPage)->pDbPage); if( rc!=SQLITE_OK ){ releasePage(*ppPage); + *ppPage = 0; } } searchList = 0; @@ -58582,7 +60255,7 @@ static int allocateBtreePage( MemPage *pPg = 0; TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", pBt->nPage)); assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetPage(pBt, pBt->nPage, &pPg, bNoContent); + rc = btreeGetUnusedPage(pBt, pBt->nPage, &pPg, bNoContent); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite(pPg->pDbPage); releasePage(pPg); @@ -58596,11 +60269,12 @@ static int allocateBtreePage( *pPgno = pBt->nPage; assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetPage(pBt, *pPgno, ppPage, bNoContent); + rc = btreeGetUnusedPage(pBt, *pPgno, ppPage, bNoContent); if( rc ) return rc; rc = sqlite3PagerWrite((*ppPage)->pDbPage); if( rc!=SQLITE_OK ){ releasePage(*ppPage); + *ppPage = 0; } TRACE(("ALLOCATE: %d from end of file\n", *pPgno)); } @@ -58610,17 +60284,8 @@ static int allocateBtreePage( end_allocate_page: releasePage(pTrunk); releasePage(pPrevTrunk); - if( rc==SQLITE_OK ){ - if( sqlite3PagerPageRefcount((*ppPage)->pDbPage)>1 ){ - releasePage(*ppPage); - *ppPage = 0; - return SQLITE_CORRUPT_BKPT; - } - (*ppPage)->isInit = 0; - }else{ - *ppPage = 0; - } - assert( rc!=SQLITE_OK || sqlite3PagerIswriteable((*ppPage)->pDbPage) ); + assert( rc!=SQLITE_OK || sqlite3PagerPageRefcount((*ppPage)->pDbPage)<=1 ); + assert( rc!=SQLITE_OK || (*ppPage)->isInit==0 ); return rc; } @@ -58645,9 +60310,10 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ int nFree; /* Initial number of pages on free-list */ assert( sqlite3_mutex_held(pBt->mutex) ); - assert( iPage>1 ); + assert( CORRUPT_DB || iPage>1 ); assert( !pMemPage || pMemPage->pgno==iPage ); + if( iPage<2 ) return SQLITE_CORRUPT_BKPT; if( pMemPage ){ pPage = pMemPage; sqlite3PagerRef(pPage->pDbPage); @@ -58787,7 +60453,7 @@ static int clearCell( u32 ovflPageSize; assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - btreeParseCellPtr(pPage, pCell, &info); + pPage->xParseCell(pPage, pCell, &info); *pnSize = info.nSize; if( info.iOverflow==0 ){ return SQLITE_OK; /* No overflow pages. Return without doing anything */ @@ -58799,7 +60465,9 @@ static int clearCell( assert( pBt->usableSize > 4 ); ovflPageSize = pBt->usableSize - 4; nOvfl = (info.nPayload - info.nLocal + ovflPageSize - 1)/ovflPageSize; - assert( ovflPgno==0 || nOvfl>0 ); + assert( nOvfl>0 || + (CORRUPT_DB && (info.nPayload + ovflPageSize)0x7fffffff || pKey==0) ){ - return SQLITE_CORRUPT_BKPT; - } + assert( nKey<=0x7fffffff && pKey!=0 ); nPayload = (int)nKey; pSrc = pKey; nSrc = (int)nKey; @@ -58939,7 +60605,7 @@ static int fillInCell( #if SQLITE_DEBUG { CellInfo info; - btreeParseCellPtr(pPage, pCell, &info); + pPage->xParseCell(pPage, pCell, &info); assert( nHeader=(int)(info.pPayload - pCell) ); assert( info.nKey==nKey ); assert( *pnSize == info.nSize ); @@ -59054,7 +60720,7 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ if( *pRC ) return; assert( idx>=0 && idxnCell ); - assert( sz==cellSize(pPage, idx) ); + assert( CORRUPT_DB || sz==cellSize(pPage, idx) ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); data = pPage->aData; @@ -59109,10 +60775,8 @@ static void insertCell( ){ int idx = 0; /* Where to write new cell content in data[] */ int j; /* Loop counter */ - int end; /* First byte past the last cell pointer in data[] */ - int ins; /* Index in data[] where new cell pointer is inserted */ - int cellOffset; /* Address of first cell pointer in data[] */ u8 *data; /* The content of the whole page */ + u8 *pIns; /* The point in pPage->aCellIdx[] where no cell inserted */ if( *pRC ) return; @@ -59127,7 +60791,7 @@ static void insertCell( ** wanted to be less than 4 but got rounded up to 4 on the leaf, then size ** might be less than 8 (leaf-size + pointer) on the interior node. Hence ** the term after the || in the following assert(). */ - assert( sz==cellSizePtr(pPage, pCell) || (sz==8 && iChild>0) ); + assert( sz==pPage->xCellSize(pPage, pCell) || (sz==8 && iChild>0) ); if( pPage->nOverflow || sz+2>pPage->nFree ){ if( pTemp ){ memcpy(pTemp, pCell, sz); @@ -59140,6 +60804,14 @@ static void insertCell( assert( j<(int)(sizeof(pPage->apOvfl)/sizeof(pPage->apOvfl[0])) ); pPage->apOvfl[j] = pCell; pPage->aiOvfl[j] = (u16)i; + + /* When multiple overflows occur, they are always sequential and in + ** sorted order. This invariants arise because multiple overflows can + ** only occur when inserting divider cells into the parent page during + ** balancing, and the dividers are adjacent and sorted. + */ + assert( j==0 || pPage->aiOvfl[j-1]<(u16)i ); /* Overflows in sorted order */ + assert( j==0 || i==pPage->aiOvfl[j-1]+1 ); /* Overflows are sequential */ }else{ int rc = sqlite3PagerWrite(pPage->pDbPage); if( rc!=SQLITE_OK ){ @@ -59148,24 +60820,26 @@ static void insertCell( } assert( sqlite3PagerIswriteable(pPage->pDbPage) ); data = pPage->aData; - cellOffset = pPage->cellOffset; - end = cellOffset + 2*pPage->nCell; - ins = cellOffset + 2*i; + assert( &data[pPage->cellOffset]==pPage->aCellIdx ); rc = allocateSpace(pPage, sz, &idx); if( rc ){ *pRC = rc; return; } - /* The allocateSpace() routine guarantees the following two properties - ** if it returns success */ - assert( idx >= end+2 ); + /* The allocateSpace() routine guarantees the following properties + ** if it returns successfully */ + assert( idx >= 0 ); + assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB ); assert( idx+sz <= (int)pPage->pBt->usableSize ); - pPage->nCell++; pPage->nFree -= (u16)(2 + sz); memcpy(&data[idx], pCell, sz); if( iChild ){ put4byte(&data[idx], iChild); } - memmove(&data[ins+2], &data[ins], end-ins); - put2byte(&data[ins], idx); - put2byte(&data[pPage->hdrOffset+3], pPage->nCell); + pIns = pPage->aCellIdx + i*2; + memmove(pIns+2, pIns, 2*(pPage->nCell - i)); + put2byte(pIns, idx); + pPage->nCell++; + /* increment the cell count */ + if( (++data[pPage->hdrOffset+4])==0 ) data[pPage->hdrOffset+3]++; + assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell ); #ifndef SQLITE_OMIT_AUTOVACUUM if( pPage->pBt->autoVacuum ){ /* The cell may contain a pointer to an overflow page. If so, write @@ -59177,6 +60851,52 @@ static void insertCell( } } +/* +** A CellArray object contains a cache of pointers and sizes for a +** consecutive sequence of cells that might be held multiple pages. +*/ +typedef struct CellArray CellArray; +struct CellArray { + int nCell; /* Number of cells in apCell[] */ + MemPage *pRef; /* Reference page */ + u8 **apCell; /* All cells begin balanced */ + u16 *szCell; /* Local size of all cells in apCell[] */ +}; + +/* +** Make sure the cell sizes at idx, idx+1, ..., idx+N-1 have been +** computed. +*/ +static void populateCellCache(CellArray *p, int idx, int N){ + assert( idx>=0 && idx+N<=p->nCell ); + while( N>0 ){ + assert( p->apCell[idx]!=0 ); + if( p->szCell[idx]==0 ){ + p->szCell[idx] = p->pRef->xCellSize(p->pRef, p->apCell[idx]); + }else{ + assert( CORRUPT_DB || + p->szCell[idx]==p->pRef->xCellSize(p->pRef, p->apCell[idx]) ); + } + idx++; + N--; + } +} + +/* +** Return the size of the Nth element of the cell array +*/ +static SQLITE_NOINLINE u16 computeCellSize(CellArray *p, int N){ + assert( N>=0 && NnCell ); + assert( p->szCell[N]==0 ); + p->szCell[N] = p->pRef->xCellSize(p->pRef, p->apCell[N]); + return p->szCell[N]; +} +static u16 cachedCellSize(CellArray *p, int N){ + assert( N>=0 && NnCell ); + if( p->szCell[N] ) return p->szCell[N]; + return computeCellSize(p, N); +} + /* ** Array apCell[] contains pointers to nCell b-tree page cells. The ** szCell[] array contains the size in bytes of each cell. This function @@ -59190,7 +60910,7 @@ static void insertCell( ** The MemPage.nFree field is invalidated by this function. It is the ** responsibility of the caller to set it correctly. */ -static void rebuildPage( +static int rebuildPage( MemPage *pPg, /* Edit this page */ int nCell, /* Final number of cells on page */ u8 **apCell, /* Array of cells */ @@ -59215,10 +60935,12 @@ static void rebuildPage( pCell = &pTmp[pCell - aData]; } pData -= szCell[i]; - memcpy(pData, pCell, szCell[i]); put2byte(pCellptr, (pData - aData)); pCellptr += 2; - assert( szCell[i]==cellSizePtr(pPg, pCell) ); + if( pData < pCellptr ) return SQLITE_CORRUPT_BKPT; + memcpy(pData, pCell, szCell[i]); + assert( szCell[i]==pPg->xCellSize(pPg, pCell) || CORRUPT_DB ); + testcase( szCell[i]!=pPg->xCellSize(pPg,pCell) ); } /* The pPg->nFree field is now set incorrectly. The caller will fix it. */ @@ -59229,6 +60951,7 @@ static void rebuildPage( put2byte(&aData[hdr+3], pPg->nCell); put2byte(&aData[hdr+5], pData - aData); aData[hdr+7] = 0x00; + return SQLITE_OK; } /* @@ -59261,25 +60984,31 @@ static int pageInsertArray( u8 *pBegin, /* End of cell-pointer array */ u8 **ppData, /* IN/OUT: Page content -area pointer */ u8 *pCellptr, /* Pointer to cell-pointer area */ + int iFirst, /* Index of first cell to add */ int nCell, /* Number of cells to add to pPg */ - u8 **apCell, /* Array of cells */ - u16 *szCell /* Array of cell sizes */ + CellArray *pCArray /* Array of cells */ ){ int i; u8 *aData = pPg->aData; u8 *pData = *ppData; - const int bFreelist = aData[1] || aData[2]; + int iEnd = iFirst + nCell; assert( CORRUPT_DB || pPg->hdrOffset==0 ); /* Never called on page 1 */ - for(i=0; iapCell[i] will never overlap on a well-formed + ** database. But they might for a corrupt database. Hence use memmove() + ** since memcpy() sends SIGABORT with overlapping buffers on OpenBSD */ + assert( (pSlot+sz)<=pCArray->apCell[i] + || pSlot>=(pCArray->apCell[i]+sz) + || CORRUPT_DB ); + memmove(pSlot, pCArray->apCell[i], sz); put2byte(pCellptr, (pSlot - aData)); pCellptr += 2; } @@ -59298,22 +61027,27 @@ static int pageInsertArray( */ static int pageFreeArray( MemPage *pPg, /* Page to edit */ + int iFirst, /* First cell to delete */ int nCell, /* Cells to delete */ - u8 **apCell, /* Array of cells */ - u16 *szCell /* Array of cell sizes */ + CellArray *pCArray /* Array of cells */ ){ u8 * const aData = pPg->aData; u8 * const pEnd = &aData[pPg->pBt->usableSize]; u8 * const pStart = &aData[pPg->hdrOffset + 8 + pPg->childPtrSize]; int nRet = 0; int i; + int iEnd = iFirst + nCell; u8 *pFree = 0; int szFree = 0; - for(i=0; iapCell[i]; if( pCell>=pStart && pCellszCell[i]; assert( sz>0 ); if( pFree!=(pCell + sz) ){ if( pFree ){ assert( pFree>aData && (pFree - aData)<65536 ); @@ -59348,13 +61082,12 @@ static int pageFreeArray( ** The pPg->nFree field is invalid when this function returns. It is the ** responsibility of the caller to set it correctly. */ -static void editPage( +static int editPage( MemPage *pPg, /* Edit this page */ int iOld, /* Index of first cell currently on page */ int iNew, /* Index of new first cell on page */ int nNew, /* Final number of cells on page */ - u8 **apCell, /* Array of cells */ - u16 *szCell /* Array of cell sizes */ + CellArray *pCArray /* Array of cells and sizes */ ){ u8 * const aData = pPg->aData; const int hdr = pPg->hdrOffset; @@ -59373,16 +61106,12 @@ static void editPage( /* Remove cells from the start and end of the page */ if( iOldaCellIdx, &pPg->aCellIdx[nShift*2], nCell*2); nCell -= nShift; } if( iNewEnd < iOldEnd ){ - nCell -= pageFreeArray( - pPg, iOldEnd-iNewEnd, &apCell[iNewEnd], &szCell[iNewEnd] - ); + nCell -= pageFreeArray(pPg, iNewEnd, iOldEnd - iNewEnd, pCArray); } pData = &aData[get2byteNotZero(&aData[hdr+5])]; @@ -59396,7 +61125,7 @@ static void editPage( memmove(&pCellptr[nAdd*2], pCellptr, nCell*2); if( pageInsertArray( pPg, pBegin, &pData, pCellptr, - nAdd, &apCell[iNew], &szCell[iNew] + iNew, nAdd, pCArray ) ) goto editpage_fail; nCell += nAdd; } @@ -59410,7 +61139,7 @@ static void editPage( nCell++; if( pageInsertArray( pPg, pBegin, &pData, pCellptr, - 1, &apCell[iCell + iNew], &szCell[iCell + iNew] + iCell+iNew, 1, pCArray ) ) goto editpage_fail; } } @@ -59419,7 +61148,7 @@ static void editPage( pCellptr = &pPg->aCellIdx[nCell*2]; if( pageInsertArray( pPg, pBegin, &pData, pCellptr, - nNew-nCell, &apCell[iNew+nCell], &szCell[iNew+nCell] + iNew+nCell, nNew-nCell, pCArray ) ) goto editpage_fail; pPg->nCell = nNew; @@ -59430,19 +61159,21 @@ static void editPage( #ifdef SQLITE_DEBUG for(i=0; iaCellIdx[i*2]); + u8 *pCell = pCArray->apCell[i+iNew]; + int iOff = get2byteAligned(&pPg->aCellIdx[i*2]); if( pCell>=aData && pCell<&aData[pPg->pBt->usableSize] ){ pCell = &pTmp[pCell - aData]; } - assert( 0==memcmp(pCell, &aData[iOff], szCell[i+iNew]) ); + assert( 0==memcmp(pCell, &aData[iOff], + pCArray->pRef->xCellSize(pCArray->pRef, pCArray->apCell[i+iNew])) ); } #endif - return; + return SQLITE_OK; editpage_fail: /* Unable to edit this page. Rebuild it from scratch instead. */ - rebuildPage(pPg, nNew, &apCell[iNew], &szCell[iNew]); + populateCellCache(pCArray, iNew, nNew); + return rebuildPage(pPg, nNew, &pCArray->apCell[iNew], &pCArray->szCell[iNew]); } /* @@ -59508,13 +61239,14 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ u8 *pOut = &pSpace[4]; u8 *pCell = pPage->apOvfl[0]; - u16 szCell = cellSizePtr(pPage, pCell); + u16 szCell = pPage->xCellSize(pPage, pCell); u8 *pStop; assert( sqlite3PagerIswriteable(pNew->pDbPage) ); assert( pPage->aData[0]==(PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF) ); zeroPage(pNew, PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF); - rebuildPage(pNew, 1, &pCell, &szCell); + rc = rebuildPage(pNew, 1, &pCell, &szCell); + if( NEVER(rc) ) return rc; pNew->nFree = pBt->usableSize - pNew->cellOffset - 2 - szCell; /* If this is an auto-vacuum database, update the pointer map @@ -59587,7 +61319,7 @@ static int ptrmapCheckPages(MemPage **apPage, int nPage){ u8 *z; z = findCell(pPage, j); - btreeParseCellPtr(pPage, z, &info); + pPage->xParseCell(pPage, z, &info); if( info.iOverflow ){ Pgno ovfl = get4byte(&z[info.iOverflow]); ptrmapGet(pBt, ovfl, &e, &n); @@ -59718,7 +61450,6 @@ static int balance_nonroot( int bBulk /* True if this call is part of a bulk load */ ){ BtShared *pBt; /* The whole database */ - int nCell = 0; /* Number of cells in apCell[] */ int nMaxCells = 0; /* Allocated size of apCell, szCell, aFrom. */ int nNew = 0; /* Number of pages in apNew[] */ int nOld; /* Number of pages in apOld[] */ @@ -59729,7 +61460,6 @@ static int balance_nonroot( int leafData; /* True if pPage is a leaf of a LEAFDATA tree */ int usableSpace; /* Bytes in pPage beyond the header */ int pageFlags; /* Value of pPage->aData[0] */ - int subtotal; /* Subtotal of bytes in cells on one page */ int iSpace1 = 0; /* First unused byte of aSpace1[] */ int iOvflSpace = 0; /* First unused byte of aOvflSpace[] */ int szScratch; /* Size of scratch memory requested */ @@ -59737,19 +61467,20 @@ static int balance_nonroot( MemPage *apNew[NB+2]; /* pPage and up to NB siblings after balancing */ u8 *pRight; /* Location in parent of right-sibling pointer */ u8 *apDiv[NB-1]; /* Divider cells in pParent */ - int cntNew[NB+2]; /* Index in aCell[] of cell after i-th page */ - int cntOld[NB+2]; /* Old index in aCell[] after i-th page */ + int cntNew[NB+2]; /* Index in b.paCell[] of cell after i-th page */ + int cntOld[NB+2]; /* Old index in b.apCell[] */ int szNew[NB+2]; /* Combined size of cells placed on i-th page */ - u8 **apCell = 0; /* All cells begin balanced */ - u16 *szCell; /* Local size of all cells in apCell[] */ u8 *aSpace1; /* Space for copies of dividers cells */ Pgno pgno; /* Temp var to store a page number in */ u8 abDone[NB+2]; /* True after i'th new page is populated */ Pgno aPgno[NB+2]; /* Page numbers of new pages before shuffling */ Pgno aPgOrder[NB+2]; /* Copy of aPgno[] used for sorting pages */ u16 aPgFlags[NB+2]; /* flags field of new pages before shuffling */ + CellArray b; /* Parsed information on cells being balanced */ memset(abDone, 0, sizeof(abDone)); + b.nCell = 0; + b.apCell = 0; pBt = pParent->pBt; assert( sqlite3_mutex_held(pBt->mutex) ); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); @@ -59803,7 +61534,7 @@ static int balance_nonroot( } pgno = get4byte(pRight); while( 1 ){ - rc = getAndInitPage(pBt, pgno, &apOld[i], 0); + rc = getAndInitPage(pBt, pgno, &apOld[i], 0, 0); if( rc ){ memset(apOld, 0, (i+1)*sizeof(MemPage*)); goto balance_cleanup; @@ -59814,12 +61545,12 @@ static int balance_nonroot( if( i+nxDiv==pParent->aiOvfl[0] && pParent->nOverflow ){ apDiv[i] = pParent->apOvfl[0]; pgno = get4byte(apDiv[i]); - szNew[i] = cellSizePtr(pParent, apDiv[i]); + szNew[i] = pParent->xCellSize(pParent, apDiv[i]); pParent->nOverflow = 0; }else{ apDiv[i] = findCell(pParent, i+nxDiv-pParent->nOverflow); pgno = get4byte(apDiv[i]); - szNew[i] = cellSizePtr(pParent, apDiv[i]); + szNew[i] = pParent->xCellSize(pParent, apDiv[i]); /* Drop the cell from the parent page. apDiv[i] still points to ** the cell within the parent, even though it has been dropped. @@ -59858,130 +61589,201 @@ static int balance_nonroot( ** Allocate space for memory structures */ szScratch = - nMaxCells*sizeof(u8*) /* apCell */ - + nMaxCells*sizeof(u16) /* szCell */ + nMaxCells*sizeof(u8*) /* b.apCell */ + + nMaxCells*sizeof(u16) /* b.szCell */ + pBt->pageSize; /* aSpace1 */ /* EVIDENCE-OF: R-28375-38319 SQLite will never request a scratch buffer ** that is more than 6 times the database page size. */ assert( szScratch<=6*(int)pBt->pageSize ); - apCell = sqlite3ScratchMalloc( szScratch ); - if( apCell==0 ){ + b.apCell = sqlite3ScratchMalloc( szScratch ); + if( b.apCell==0 ){ rc = SQLITE_NOMEM; goto balance_cleanup; } - szCell = (u16*)&apCell[nMaxCells]; - aSpace1 = (u8*)&szCell[nMaxCells]; + b.szCell = (u16*)&b.apCell[nMaxCells]; + aSpace1 = (u8*)&b.szCell[nMaxCells]; assert( EIGHT_BYTE_ALIGNMENT(aSpace1) ); /* ** Load pointers to all cells on sibling pages and the divider cells - ** into the local apCell[] array. Make copies of the divider cells + ** into the local b.apCell[] array. Make copies of the divider cells ** into space obtained from aSpace1[]. The divider cells have already ** been removed from pParent. ** ** If the siblings are on leaf pages, then the child pointers of the ** divider cells are stripped from the cells before they are copied - ** into aSpace1[]. In this way, all cells in apCell[] are without + ** into aSpace1[]. In this way, all cells in b.apCell[] are without ** child pointers. If siblings are not leaves, then all cell in - ** apCell[] include child pointers. Either way, all cells in apCell[] + ** b.apCell[] include child pointers. Either way, all cells in b.apCell[] ** are alike. ** ** leafCorrection: 4 if pPage is a leaf. 0 if pPage is not a leaf. ** leafData: 1 if pPage holds key+data and pParent holds only keys. */ - leafCorrection = apOld[0]->leaf*4; - leafData = apOld[0]->intKeyLeaf; + b.pRef = apOld[0]; + leafCorrection = b.pRef->leaf*4; + leafData = b.pRef->intKeyLeaf; for(i=0; inCell; + u8 *aData = pOld->aData; + u16 maskPage = pOld->maskPage; + u8 *piCell = aData + pOld->cellOffset; + u8 *piEnd; + + /* Verify that all sibling pages are of the same "type" (table-leaf, + ** table-interior, index-leaf, or index-interior). + */ + if( pOld->aData[0]!=apOld[0]->aData[0] ){ + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + } - limit = pOld->nCell+pOld->nOverflow; + /* Load b.apCell[] with pointers to all cells in pOld. If pOld + ** constains overflow cells, include them in the b.apCell[] array + ** in the correct spot. + ** + ** Note that when there are multiple overflow cells, it is always the + ** case that they are sequential and adjacent. This invariant arises + ** because multiple overflows can only occurs when inserting divider + ** cells into a parent on a prior balance, and divider cells are always + ** adjacent and are inserted in order. There is an assert() tagged + ** with "NOTE 1" in the overflow cell insertion loop to prove this + ** invariant. + ** + ** This must be done in advance. Once the balance starts, the cell + ** offset section of the btree page will be overwritten and we will no + ** long be able to find the cells if a pointer to each cell is not saved + ** first. + */ + memset(&b.szCell[b.nCell], 0, sizeof(b.szCell[0])*limit); if( pOld->nOverflow>0 ){ + memset(&b.szCell[b.nCell+limit], 0, sizeof(b.szCell[0])*pOld->nOverflow); + limit = pOld->aiOvfl[0]; for(j=0; jaData; - u16 maskPage = pOld->maskPage; - u16 cellOffset = pOld->cellOffset; - for(j=0; jnOverflow; k++){ + assert( k==0 || pOld->aiOvfl[k-1]+1==pOld->aiOvfl[k] );/* NOTE 1 */ + b.apCell[b.nCell] = pOld->apOvfl[k]; + b.nCell++; } - } - cntOld[i] = nCell; + } + piEnd = aData + pOld->cellOffset + 2*pOld->nCell; + while( piCellmaxLocal+23 ); assert( iSpace1 <= (int)pBt->pageSize ); memcpy(pTemp, apDiv[i], sz); - apCell[nCell] = pTemp+leafCorrection; + b.apCell[b.nCell] = pTemp+leafCorrection; assert( leafCorrection==0 || leafCorrection==4 ); - szCell[nCell] = szCell[nCell] - leafCorrection; + b.szCell[b.nCell] = b.szCell[b.nCell] - leafCorrection; if( !pOld->leaf ){ assert( leafCorrection==0 ); assert( pOld->hdrOffset==0 ); /* The right pointer of the child page pOld becomes the left ** pointer of the divider cell */ - memcpy(apCell[nCell], &pOld->aData[8], 4); + memcpy(b.apCell[b.nCell], &pOld->aData[8], 4); }else{ assert( leafCorrection==4 ); - if( szCell[nCell]<4 ){ + while( b.szCell[b.nCell]<4 ){ /* Do not allow any cells smaller than 4 bytes. If a smaller cell ** does exist, pad it with 0x00 bytes. */ - assert( szCell[nCell]==3 ); - assert( apCell[nCell]==&aSpace1[iSpace1-3] ); + assert( b.szCell[b.nCell]==3 || CORRUPT_DB ); + assert( b.apCell[b.nCell]==&aSpace1[iSpace1-3] || CORRUPT_DB ); aSpace1[iSpace1++] = 0x00; - szCell[nCell] = 4; + b.szCell[b.nCell]++; } } - nCell++; + b.nCell++; } } /* - ** Figure out the number of pages needed to hold all nCell cells. + ** Figure out the number of pages needed to hold all b.nCell cells. ** Store this number in "k". Also compute szNew[] which is the total ** size of all cells on the i-th page and cntNew[] which is the index - ** in apCell[] of the cell that divides page i from page i+1. - ** cntNew[k] should equal nCell. + ** in b.apCell[] of the cell that divides page i from page i+1. + ** cntNew[k] should equal b.nCell. ** ** Values computed by this block: ** ** k: The total number of sibling pages ** szNew[i]: Spaced used on the i-th sibling page. - ** cntNew[i]: Index in apCell[] and szCell[] for the first cell to + ** cntNew[i]: Index in b.apCell[] and b.szCell[] for the first cell to ** the right of the i-th sibling page. ** usableSpace: Number of bytes of space available on each sibling. ** */ usableSpace = pBt->usableSize - 12 + leafCorrection; - for(subtotal=k=i=0; i usableSpace ){ - szNew[k] = subtotal - szCell[i] - 2; - cntNew[k] = i; - if( leafData ){ i--; } - subtotal = 0; - k++; - if( k>NB+1 ){ rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } - } - } - szNew[k] = subtotal; - cntNew[k] = nCell; - k++; + for(i=0; inFree; + if( szNew[i]<0 ){ rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } + for(j=0; jnOverflow; j++){ + szNew[i] += 2 + p->xCellSize(p, p->apOvfl[j]); + } + cntNew[i] = cntOld[i]; + } + k = nOld; + for(i=0; iusableSpace ){ + if( i+1>=k ){ + k = i+2; + if( k>NB+2 ){ rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } + szNew[k-1] = 0; + cntNew[k-1] = b.nCell; + } + sz = 2 + cachedCellSize(&b, cntNew[i]-1); + szNew[i] -= sz; + if( !leafData ){ + if( cntNew[i]usableSpace ) break; + szNew[i] += sz; + cntNew[i]++; + if( !leafData ){ + if( cntNew[i]=b.nCell ){ + k = i+1; + }else if( cntNew[i] <= (i>0 ? cntNew[i-1] : 0) ){ + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + } + } /* ** The packing computed by the previous block is biased toward the siblings @@ -60002,19 +61804,27 @@ static int balance_nonroot( r = cntNew[i-1] - 1; d = r + 1 - leafData; - assert( d szLeft-(b.szCell[r]+2)) ){ + break; + } + szRight += b.szCell[d] + 2; + szLeft -= b.szCell[r] + 2; + cntNew[i-1] = r; + r--; + d--; + }while( r>=0 ); szNew[i] = szRight; szNew[i-1] = szLeft; + if( cntNew[i-1] <= (i>1 ? cntNew[i-2] : 0) ){ + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + } } /* Sanity check: For a non-corrupt database file one of the follwing @@ -60034,10 +61844,6 @@ static int balance_nonroot( /* ** Allocate k new pages. Reuse old pages where possible. */ - if( apOld[0]->pgno<=1 ){ - rc = SQLITE_CORRUPT_BKPT; - goto balance_cleanup; - } pageFlags = apOld[0]->aData[0]; for(i=0; inCell + pOld->nOverflow + !leafData; @@ -60185,9 +61991,10 @@ static int balance_nonroot( if( !leafCorrection ){ ptrmapPut(pBt, get4byte(pCell), PTRMAP_BTREE, pNew->pgno, &rc); } - if( szCell[i]>pNew->minLocal ){ + if( cachedCellSize(&b,i)>pNew->minLocal ){ ptrmapPutOvflPtr(pNew, pCell, &rc); } + if( rc ) goto balance_cleanup; } } } @@ -60201,20 +62008,21 @@ static int balance_nonroot( j = cntNew[i]; assert( jleaf ){ memcpy(&pNew->aData[8], pCell, 4); }else if( leafData ){ /* If the tree is a leaf-data tree, and the siblings are leaves, - ** then there is no divider cell in apCell[]. Instead, the divider + ** then there is no divider cell in b.apCell[]. Instead, the divider ** cell consists of the integer key for the right-most cell of ** the sibling-page assembled above only. */ CellInfo info; j--; - btreeParseCellPtr(pNew, apCell[j], &info); + pNew->xParseCell(pNew, b.apCell[j], &info); pCell = pTemp; sz = 4 + putVarint(&pCell[4], info.nKey); pTemp = 0; @@ -60231,9 +62039,9 @@ static int balance_nonroot( ** cells are at least 4 bytes. It only happens in b-trees used ** to evaluate "IN (SELECT ...)" and similar clauses. */ - if( szCell[j]==4 ){ + if( b.szCell[j]==4 ){ assert(leafCorrection==4); - sz = cellSizePtr(pParent, pCell); + sz = pParent->xCellSize(pParent, pCell); } } iOvflSpace += sz; @@ -60289,12 +62097,13 @@ static int balance_nonroot( iNew = iOld = 0; nNewCell = cntNew[0]; }else{ - iOld = iPgnFree = usableSpace-szNew[iPg]; assert( apNew[iPg]->nOverflow==0 ); @@ -60324,7 +62133,7 @@ static int balance_nonroot( ** by smaller than the child due to the database header, and so all the ** free space needs to be up front. */ - assert( nNew==1 ); + assert( nNew==1 || CORRUPT_DB ); rc = defragmentPage(apNew[0]); testcase( rc!=SQLITE_OK ); assert( apNew[0]->nFree == @@ -60345,7 +62154,7 @@ static int balance_nonroot( assert( pParent->isInit ); TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n", - nOld, nNew, nCell)); + nOld, nNew, b.nCell)); /* Free any old pages that were not reused as new pages. */ @@ -60368,7 +62177,7 @@ static int balance_nonroot( ** Cleanup before returning. */ balance_cleanup: - sqlite3ScratchFree(apCell); + sqlite3ScratchFree(b.apCell); for(i=0; ipgnoRoot, pCur); - if( rc ) return rc; + if( pCur->curFlags & BTCF_Multiple ){ + rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); + if( rc ) return rc; + } if( pCur->pKeyInfo==0 ){ + assert( pKey==0 ); /* If this is an insert into a table b-tree, invalidate any incrblob ** cursors open on the row being replaced */ invalidateIncrblobCursors(p, nKey, 0); /* If the cursor is currently on the last row and we are appending a - ** new row onto the end, set the "loc" to avoid an unnecessary btreeMoveto() - ** call */ + ** new row onto the end, set the "loc" to avoid an unnecessary + ** btreeMoveto() call */ if( (pCur->curFlags&BTCF_ValidNKey)!=0 && nKey>0 && pCur->info.nKey==nKey-1 ){ - loc = -1; + loc = -1; + }else if( loc==0 ){ + rc = sqlite3BtreeMovetoUnpacked(pCur, 0, nKey, appendBias, &loc); + if( rc ) return rc; } - } - - if( !loc ){ + }else if( loc==0 ){ rc = btreeMoveto(pCur, pKey, nKey, appendBias, &loc); if( rc ) return rc; } @@ -60678,7 +62491,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( assert( newCell!=0 ); rc = fillInCell(pPage, newCell, pKey, nKey, pData, nData, nZero, &szNew); if( rc ) goto end_insert; - assert( szNew==cellSizePtr(pPage, newCell) ); + assert( szNew==pPage->xCellSize(pPage, newCell) ); assert( szNew <= MX_CELL_SIZE(pBt) ); idx = pCur->aiIdx[pCur->iPage]; if( loc==0 ){ @@ -60743,10 +62556,15 @@ end_insert: } /* -** Delete the entry that the cursor is pointing to. The cursor -** is left pointing at an arbitrary location. +** Delete the entry that the cursor is pointing to. +** +** If the second parameter is zero, then the cursor is left pointing at an +** arbitrary location after the delete. If it is non-zero, then the cursor +** is left in a state such that the next call to BtreeNext() or BtreePrev() +** moves it to the same row as it would if the call to BtreeDelete() had +** been omitted. */ -SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ +SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, int bPreserve){ Btree *p = pCur->pBtree; BtShared *pBt = p->pBt; int rc; /* Return code */ @@ -60755,6 +62573,7 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ int iCellIdx; /* Index of cell to delete */ int iCellDepth; /* Depth of node containing pCell */ u16 szCell; /* Size of the cell being deleted */ + int bSkipnext = 0; /* Leaf cursor in SKIPNEXT state */ assert( cursorHoldsMutex(pCur) ); assert( pBt->inTransaction==TRANS_WRITE ); @@ -60762,12 +62581,8 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ assert( pCur->curFlags & BTCF_WriteFlag ); assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); assert( !hasReadConflicts(p, pCur->pgnoRoot) ); - - if( NEVER(pCur->aiIdx[pCur->iPage]>=pCur->apPage[pCur->iPage]->nCell) - || NEVER(pCur->eState!=CURSOR_VALID) - ){ - return SQLITE_ERROR; /* Something has gone awry. */ - } + assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); + assert( pCur->eState==CURSOR_VALID ); iCellDepth = pCur->iPage; iCellIdx = pCur->aiIdx[iCellDepth]; @@ -60788,12 +62603,11 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ } /* Save the positions of any other cursors open on this table before - ** making any modifications. Make the page containing the entry to be - ** deleted writable. Then free any overflow pages associated with the - ** entry and finally remove the cell itself from within the page. - */ - rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); - if( rc ) return rc; + ** making any modifications. */ + if( pCur->curFlags & BTCF_Multiple ){ + rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); + if( rc ) return rc; + } /* If this is a delete operation to remove a row from a table b-tree, ** invalidate any incrblob cursors open on the row being deleted. */ @@ -60801,6 +62615,31 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ invalidateIncrblobCursors(p, pCur->info.nKey, 0); } + /* If the bPreserve flag is set to true, then the cursor position must + ** be preserved following this delete operation. If the current delete + ** will cause a b-tree rebalance, then this is done by saving the cursor + ** key and leaving the cursor in CURSOR_REQUIRESEEK state before + ** returning. + ** + ** Or, if the current delete will not cause a rebalance, then the cursor + ** will be left in CURSOR_SKIPNEXT state pointing to the entry immediately + ** before or after the deleted entry. In this case set bSkipnext to true. */ + if( bPreserve ){ + if( !pPage->leaf + || (pPage->nFree+cellSizePtr(pPage,pCell)+2)>(int)(pBt->usableSize*2/3) + ){ + /* A b-tree rebalance will be required after deleting this entry. + ** Save the cursor key. */ + rc = saveCursorKey(pCur); + if( rc ) return rc; + }else{ + bSkipnext = 1; + } + } + + /* Make the page containing the entry to be deleted writable. Then free any + ** overflow pages associated with the entry and finally remove the cell + ** itself from within the page. */ rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ) return rc; rc = clearCell(pPage, pCell, &szCell); @@ -60819,7 +62658,8 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ unsigned char *pTmp; pCell = findCell(pLeaf, pLeaf->nCell-1); - nCell = cellSizePtr(pLeaf, pCell); + if( pCell<&pLeaf->aData[4] ) return SQLITE_CORRUPT_BKPT; + nCell = pLeaf->xCellSize(pLeaf, pCell); assert( MX_CELL_SIZE(pBt) >= nCell ); pTmp = pBt->pTmpSpace; assert( pTmp!=0 ); @@ -60853,7 +62693,23 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ } if( rc==SQLITE_OK ){ - moveToRoot(pCur); + if( bSkipnext ){ + assert( bPreserve && pCur->iPage==iCellDepth ); + assert( pPage==pCur->apPage[pCur->iPage] ); + assert( (pPage->nCell>0 || CORRUPT_DB) && iCellIdx<=pPage->nCell ); + pCur->eState = CURSOR_SKIPNEXT; + if( iCellIdx>=pPage->nCell ){ + pCur->skipNext = -1; + pCur->aiIdx[iCellDepth] = pPage->nCell-1; + }else{ + pCur->skipNext = 1; + } + }else{ + rc = moveToRoot(pCur); + if( bPreserve ){ + pCur->eState = CURSOR_REQUIRESEEK; + } + } } return rc; } @@ -60911,7 +62767,8 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ pgnoRoot==PENDING_BYTE_PAGE(pBt) ){ pgnoRoot++; } - assert( pgnoRoot>=3 ); + assert( pgnoRoot>=3 || CORRUPT_DB ); + testcase( pgnoRoot<3 ); /* Allocate a page. The page that currently resides at pgnoRoot will ** be moved to the allocated page (unless the allocated page happens @@ -61040,7 +62897,7 @@ static int clearDatabasePage( if( pgno>btreePagecount(pBt) ){ return SQLITE_CORRUPT_BKPT; } - rc = getAndInitPage(pBt, pgno, &pPage, 0); + rc = getAndInitPage(pBt, pgno, &pPage, 0, 0); if( rc ) return rc; if( pPage->bBusy ){ rc = SQLITE_CORRUPT_BKPT; @@ -61061,7 +62918,8 @@ static int clearDatabasePage( rc = clearDatabasePage(pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange); if( rc ) goto cleardatabasepage_out; }else if( pnChange ){ - assert( pPage->intKey ); + assert( pPage->intKey || CORRUPT_DB ); + testcase( !pPage->intKey ); *pnChange += pPage->nCell; } if( freePageFlag ){ @@ -61416,7 +63274,6 @@ static void checkAppendMsg( ... ){ va_list ap; - char zBuf[200]; if( !pCheck->mxErr ) return; pCheck->mxErr--; pCheck->nErr++; @@ -61425,8 +63282,7 @@ static void checkAppendMsg( sqlite3StrAccumAppend(&pCheck->errMsg, "\n", 1); } if( pCheck->zPfx ){ - sqlite3_snprintf(sizeof(zBuf), zBuf, pCheck->zPfx, pCheck->v1, pCheck->v2); - sqlite3StrAccumAppendAll(&pCheck->errMsg, zBuf); + sqlite3XPrintf(&pCheck->errMsg, 0, pCheck->zPfx, pCheck->v1, pCheck->v2); } sqlite3VXPrintf(&pCheck->errMsg, 1, zFormat, ap); va_end(ap); @@ -61575,6 +63431,10 @@ static void checkList( #endif iPage = get4byte(pOvflData); sqlite3PagerUnref(pOvflPage); + + if( isFreeList && N<(iPage!=0) ){ + checkAppendMsg(pCheck, "free-page count in header is too small"); + } } } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -61640,35 +63500,42 @@ static int btreeHeapPull(u32 *aHeap, u32 *pOut){ ** ** 1. Make sure that cells and freeblocks do not overlap ** but combine to completely cover the page. -** NO 2. Make sure cell keys are in order. -** NO 3. Make sure no key is less than or equal to zLowerBound. -** NO 4. Make sure no key is greater than or equal to zUpperBound. -** 5. Check the integrity of overflow pages. -** 6. Recursively call checkTreePage on all children. -** 7. Verify that the depth of all children is the same. -** 8. Make sure this page is at least 33% full or else it is -** the root of the tree. +** 2. Make sure integer cell keys are in order. +** 3. Check the integrity of overflow pages. +** 4. Recursively call checkTreePage on all children. +** 5. Verify that the depth of all children is the same. */ static int checkTreePage( IntegrityCk *pCheck, /* Context for the sanity check */ int iPage, /* Page number of the page to check */ - i64 *pnParentMinKey, - i64 *pnParentMaxKey + i64 *piMinKey, /* Write minimum integer primary key here */ + i64 maxKey /* Error if integer primary key greater than this */ ){ - MemPage *pPage; - int i, rc, depth, d2, pgno, cnt; - int hdr, cellStart; - int nCell; - u8 *data; - BtShared *pBt; - int usableSize; - u32 *heap = 0; - u32 x, prev = 0; - i64 nMinKey = 0; - i64 nMaxKey = 0; + MemPage *pPage = 0; /* The page being analyzed */ + int i; /* Loop counter */ + int rc; /* Result code from subroutine call */ + int depth = -1, d2; /* Depth of a subtree */ + int pgno; /* Page number */ + int nFrag; /* Number of fragmented bytes on the page */ + int hdr; /* Offset to the page header */ + int cellStart; /* Offset to the start of the cell pointer array */ + int nCell; /* Number of cells */ + int doCoverageCheck = 1; /* True if cell coverage checking should be done */ + int keyCanBeEqual = 1; /* True if IPK can be equal to maxKey + ** False if IPK must be strictly less than maxKey */ + u8 *data; /* Page content */ + u8 *pCell; /* Cell content */ + u8 *pCellIdx; /* Next element of the cell pointer array */ + BtShared *pBt; /* The BtShared object that owns pPage */ + u32 pc; /* Address of a cell */ + u32 usableSize; /* Usable size of the page */ + u32 contentOffset; /* Offset to the start of the cell content area */ + u32 *heap = 0; /* Min-heap used for checking cell coverage */ + u32 x, prev = 0; /* Next and previous entry on the min-heap */ const char *saved_zPfx = pCheck->zPfx; int saved_v1 = pCheck->v1; int saved_v2 = pCheck->v2; + u8 savedIsInit = 0; /* Check that the page exists */ @@ -61681,54 +63548,95 @@ static int checkTreePage( if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0))!=0 ){ checkAppendMsg(pCheck, "unable to get the page. error code=%d", rc); - depth = -1; goto end_of_check; } /* Clear MemPage.isInit to make sure the corruption detection code in ** btreeInitPage() is executed. */ + savedIsInit = pPage->isInit; pPage->isInit = 0; if( (rc = btreeInitPage(pPage))!=0 ){ assert( rc==SQLITE_CORRUPT ); /* The only possible error from InitPage */ checkAppendMsg(pCheck, "btreeInitPage() returns error code %d", rc); - releasePage(pPage); - depth = -1; goto end_of_check; } + data = pPage->aData; + hdr = pPage->hdrOffset; - /* Check out all the cells. - */ - depth = 0; - for(i=0; inCell && pCheck->mxErr; i++){ - u8 *pCell; - u32 sz; + /* Set up for cell analysis */ + pCheck->zPfx = "On tree page %d cell %d: "; + contentOffset = get2byteNotZero(&data[hdr+5]); + assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */ + + /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the + ** number of cells on the page. */ + nCell = get2byte(&data[hdr+3]); + assert( pPage->nCell==nCell ); + + /* EVIDENCE-OF: R-23882-45353 The cell pointer array of a b-tree page + ** immediately follows the b-tree page header. */ + cellStart = hdr + 12 - 4*pPage->leaf; + assert( pPage->aCellIdx==&data[cellStart] ); + pCellIdx = &data[cellStart + 2*(nCell-1)]; + + if( !pPage->leaf ){ + /* Analyze the right-child page of internal pages */ + pgno = get4byte(&data[hdr+8]); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + pCheck->zPfx = "On page %d at right child: "; + checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage); + } +#endif + depth = checkTreePage(pCheck, pgno, &maxKey, maxKey); + keyCanBeEqual = 0; + }else{ + /* For leaf pages, the coverage check will occur in the same loop + ** as the other cell checks, so initialize the heap. */ + heap = pCheck->heap; + heap[0] = 0; + } + + /* EVIDENCE-OF: R-02776-14802 The cell pointer array consists of K 2-byte + ** integer offsets to the cell contents. */ + for(i=nCell-1; i>=0 && pCheck->mxErr; i--){ CellInfo info; - /* Check payload overflow pages - */ - pCheck->zPfx = "On tree page %d cell %d: "; - pCheck->v1 = iPage; + /* Check cell size */ pCheck->v2 = i; - pCell = findCell(pPage,i); - btreeParseCellPtr(pPage, pCell, &info); - sz = info.nPayload; - /* For intKey pages, check that the keys are in order. - */ + assert( pCellIdx==&data[cellStart + i*2] ); + pc = get2byteAligned(pCellIdx); + pCellIdx -= 2; + if( pcusableSize-4 ){ + checkAppendMsg(pCheck, "Offset %d out of range %d..%d", + pc, contentOffset, usableSize-4); + doCoverageCheck = 0; + continue; + } + pCell = &data[pc]; + pPage->xParseCell(pPage, pCell, &info); + if( pc+info.nSize>usableSize ){ + checkAppendMsg(pCheck, "Extends off end of page"); + doCoverageCheck = 0; + continue; + } + + /* Check for integer primary key out of range */ if( pPage->intKey ){ - if( i==0 ){ - nMinKey = nMaxKey = info.nKey; - }else if( info.nKey <= nMaxKey ){ - checkAppendMsg(pCheck, - "Rowid %lld out of order (previous was %lld)", info.nKey, nMaxKey); + if( keyCanBeEqual ? (info.nKey > maxKey) : (info.nKey >= maxKey) ){ + checkAppendMsg(pCheck, "Rowid %lld out of order", info.nKey); } - nMaxKey = info.nKey; + maxKey = info.nKey; } - if( (sz>info.nLocal) - && (&pCell[info.iOverflow]<=&pPage->aData[pBt->usableSize]) - ){ - int nPage = (sz - info.nLocal + usableSize - 5)/(usableSize - 4); - Pgno pgnoOvfl = get4byte(&pCell[info.iOverflow]); + + /* Check the content overflow list */ + if( info.nPayload>info.nLocal ){ + int nPage; /* Number of pages on the overflow chain */ + Pgno pgnoOvfl; /* First page of the overflow chain */ + assert( pc + info.iOverflow <= usableSize ); + nPage = (info.nPayload - info.nLocal + usableSize - 5)/(usableSize - 4); + pgnoOvfl = get4byte(&pCell[info.iOverflow]); #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ checkPtrmap(pCheck, pgnoOvfl, PTRMAP_OVERFLOW1, iPage); @@ -61737,118 +63645,57 @@ static int checkTreePage( checkList(pCheck, 0, pgnoOvfl, nPage); } - /* Check sanity of left child page. - */ if( !pPage->leaf ){ + /* Check sanity of left child page for internal pages */ pgno = get4byte(pCell); #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage); } #endif - d2 = checkTreePage(pCheck, pgno, &nMinKey, i==0?NULL:&nMaxKey); - if( i>0 && d2!=depth ){ + d2 = checkTreePage(pCheck, pgno, &maxKey, maxKey); + keyCanBeEqual = 0; + if( d2!=depth ){ checkAppendMsg(pCheck, "Child page depth differs"); + depth = d2; } - depth = d2; - } - } - - if( !pPage->leaf ){ - pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); - pCheck->zPfx = "On page %d at right child: "; - pCheck->v1 = iPage; -#ifndef SQLITE_OMIT_AUTOVACUUM - if( pBt->autoVacuum ){ - checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage); - } -#endif - checkTreePage(pCheck, pgno, NULL, !pPage->nCell?NULL:&nMaxKey); - } - - /* For intKey leaf pages, check that the min/max keys are in order - ** with any left/parent/right pages. - */ - pCheck->zPfx = "Page %d: "; - pCheck->v1 = iPage; - if( pPage->leaf && pPage->intKey ){ - /* if we are a left child page */ - if( pnParentMinKey ){ - /* if we are the left most child page */ - if( !pnParentMaxKey ){ - if( nMaxKey > *pnParentMinKey ){ - checkAppendMsg(pCheck, - "Rowid %lld out of order (max larger than parent min of %lld)", - nMaxKey, *pnParentMinKey); - } - }else{ - if( nMinKey <= *pnParentMinKey ){ - checkAppendMsg(pCheck, - "Rowid %lld out of order (min less than parent min of %lld)", - nMinKey, *pnParentMinKey); - } - if( nMaxKey > *pnParentMaxKey ){ - checkAppendMsg(pCheck, - "Rowid %lld out of order (max larger than parent max of %lld)", - nMaxKey, *pnParentMaxKey); - } - *pnParentMinKey = nMaxKey; - } - /* else if we're a right child page */ - } else if( pnParentMaxKey ){ - if( nMinKey <= *pnParentMaxKey ){ - checkAppendMsg(pCheck, - "Rowid %lld out of order (min less than parent max of %lld)", - nMinKey, *pnParentMaxKey); - } + }else{ + /* Populate the coverage-checking heap for leaf pages */ + btreeHeapInsert(heap, (pc<<16)|(pc+info.nSize-1)); } } + *piMinKey = maxKey; /* Check for complete coverage of the page */ - data = pPage->aData; - hdr = pPage->hdrOffset; - heap = (u32*)sqlite3PageMalloc( pBt->pageSize ); pCheck->zPfx = 0; - if( heap==0 ){ - pCheck->mallocFailed = 1; - }else{ - int contentOffset = get2byteNotZero(&data[hdr+5]); - assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */ - heap[0] = 0; - btreeHeapInsert(heap, contentOffset-1); - /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the - ** number of cells on the page. */ - nCell = get2byte(&data[hdr+3]); - /* EVIDENCE-OF: R-23882-45353 The cell pointer array of a b-tree page - ** immediately follows the b-tree page header. */ - cellStart = hdr + 12 - 4*pPage->leaf; - /* EVIDENCE-OF: R-02776-14802 The cell pointer array consists of K 2-byte - ** integer offsets to the cell contents. */ - for(i=0; i=usableSize ){ - pCheck->zPfx = 0; - checkAppendMsg(pCheck, - "Corruption detected in cell %d on page %d",i,iPage); - }else{ + if( doCoverageCheck && pCheck->mxErr>0 ){ + /* For leaf pages, the min-heap has already been initialized and the + ** cells have already been inserted. But for internal pages, that has + ** not yet been done, so do it now */ + if( !pPage->leaf ){ + heap = pCheck->heap; + heap[0] = 0; + for(i=nCell-1; i>=0; i--){ + u32 size; + pc = get2byteAligned(&data[cellStart+i*2]); + size = pPage->xCellSize(pPage, &data[pc]); btreeHeapInsert(heap, (pc<<16)|(pc+size-1)); } } - /* EVIDENCE-OF: R-20690-50594 The second field of the b-tree page header + /* Add the freeblocks to the min-heap + ** + ** EVIDENCE-OF: R-20690-50594 The second field of the b-tree page header ** is the offset of the first freeblock, or zero if there are no - ** freeblocks on the page. */ + ** freeblocks on the page. + */ i = get2byte(&data[hdr+1]); while( i>0 ){ int size, j; - assert( i<=usableSize-4 ); /* Enforced by btreeInitPage() */ + assert( (u32)i<=usableSize-4 ); /* Enforced by btreeInitPage() */ size = get2byte(&data[i+2]); - assert( i+size<=usableSize ); /* Enforced by btreeInitPage() */ - btreeHeapInsert(heap, (i<<16)|(i+size-1)); + assert( (u32)(i+size)<=usableSize ); /* Enforced by btreeInitPage() */ + btreeHeapInsert(heap, (((u32)i)<<16)|(i+size-1)); /* EVIDENCE-OF: R-58208-19414 The first 2 bytes of a freeblock are a ** big-endian integer which is the offset in the b-tree page of the next ** freeblock in the chain, or zero if the freeblock is the last on the @@ -61857,39 +63704,50 @@ static int checkTreePage( /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of ** increasing offset. */ assert( j==0 || j>i+size ); /* Enforced by btreeInitPage() */ - assert( j<=usableSize-4 ); /* Enforced by btreeInitPage() */ + assert( (u32)j<=usableSize-4 ); /* Enforced by btreeInitPage() */ i = j; } - cnt = 0; - assert( heap[0]>0 ); - assert( (heap[1]>>16)==0 ); - btreeHeapPull(heap,&prev); + /* Analyze the min-heap looking for overlap between cells and/or + ** freeblocks, and counting the number of untracked bytes in nFrag. + ** + ** Each min-heap entry is of the form: (start_address<<16)|end_address. + ** There is an implied first entry the covers the page header, the cell + ** pointer index, and the gap between the cell pointer index and the start + ** of cell content. + ** + ** The loop below pulls entries from the min-heap in order and compares + ** the start_address against the previous end_address. If there is an + ** overlap, that means bytes are used multiple times. If there is a gap, + ** that gap is added to the fragmentation count. + */ + nFrag = 0; + prev = contentOffset - 1; /* Implied first min-heap entry */ while( btreeHeapPull(heap,&x) ){ - if( (prev&0xffff)+1>(x>>16) ){ + if( (prev&0xffff)>=(x>>16) ){ checkAppendMsg(pCheck, "Multiple uses for byte %u of page %d", x>>16, iPage); break; }else{ - cnt += (x>>16) - (prev&0xffff) - 1; + nFrag += (x>>16) - (prev&0xffff) - 1; prev = x; } } - cnt += usableSize - (prev&0xffff) - 1; + nFrag += usableSize - (prev&0xffff) - 1; /* EVIDENCE-OF: R-43263-13491 The total number of bytes in all fragments ** is stored in the fifth field of the b-tree page header. ** EVIDENCE-OF: R-07161-27322 The one-byte integer at offset 7 gives the ** number of fragmented free bytes within the cell content area. */ - if( heap[0]==0 && cnt!=data[hdr+7] ){ + if( heap[0]==0 && nFrag!=data[hdr+7] ){ checkAppendMsg(pCheck, "Fragmentation of %d bytes reported as %d on page %d", - cnt, data[hdr+7], iPage); + nFrag, data[hdr+7], iPage); } } - sqlite3PageFree(heap); - releasePage(pPage); end_of_check: + if( !doCoverageCheck ) pPage->isInit = savedIsInit; + releasePage(pPage); pCheck->zPfx = saved_zPfx; pCheck->v1 = saved_v1; pCheck->v2 = saved_v2; @@ -61919,14 +63777,15 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( int *pnErr /* Write number of errors seen to this variable */ ){ Pgno i; - int nRef; IntegrityCk sCheck; BtShared *pBt = p->pBt; + int savedDbFlags = pBt->db->flags; char zErr[100]; + VVA_ONLY( int nRef ); sqlite3BtreeEnter(p); assert( p->inTrans>TRANS_NONE && pBt->inTransaction>TRANS_NONE ); - nRef = sqlite3PagerRefcount(pBt->pPager); + assert( (nRef = sqlite3PagerRefcount(pBt->pPager))>=0 ); sCheck.pBt = pBt; sCheck.pPager = pBt->pPager; sCheck.nPage = btreePagecount(sCheck.pBt); @@ -61936,21 +63795,26 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( sCheck.zPfx = 0; sCheck.v1 = 0; sCheck.v2 = 0; - *pnErr = 0; + sCheck.aPgRef = 0; + sCheck.heap = 0; + sqlite3StrAccumInit(&sCheck.errMsg, 0, zErr, sizeof(zErr), SQLITE_MAX_LENGTH); if( sCheck.nPage==0 ){ - sqlite3BtreeLeave(p); - return 0; + goto integrity_ck_cleanup; } sCheck.aPgRef = sqlite3MallocZero((sCheck.nPage / 8)+ 1); if( !sCheck.aPgRef ){ - *pnErr = 1; - sqlite3BtreeLeave(p); - return 0; + sCheck.mallocFailed = 1; + goto integrity_ck_cleanup; } + sCheck.heap = (u32*)sqlite3PageMalloc( pBt->pageSize ); + if( sCheck.heap==0 ){ + sCheck.mallocFailed = 1; + goto integrity_ck_cleanup; + } + i = PENDING_BYTE_PAGE(pBt); if( i<=sCheck.nPage ) setPageReferenced(&sCheck, i); - sqlite3StrAccumInit(&sCheck.errMsg, 0, zErr, sizeof(zErr), SQLITE_MAX_LENGTH); /* Check the integrity of the freelist */ @@ -61961,17 +63825,19 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( /* Check all the tables. */ + testcase( pBt->db->flags & SQLITE_CellSizeCk ); + pBt->db->flags &= ~SQLITE_CellSizeCk; for(i=0; (int)iautoVacuum && aRoot[i]>1 ){ checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0); } #endif - sCheck.zPfx = "List of tree roots: "; - checkTreePage(&sCheck, aRoot[i], NULL, NULL); - sCheck.zPfx = 0; + checkTreePage(&sCheck, aRoot[i], ¬Used, LARGEST_INT64); } + pBt->db->flags = savedDbFlags; /* Make sure every page in the file is referenced */ @@ -61995,28 +63861,20 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( #endif } - /* Make sure this analysis did not leave any unref() pages. - ** This is an internal consistency check; an integrity check - ** of the integrity check. - */ - if( NEVER(nRef != sqlite3PagerRefcount(pBt->pPager)) ){ - checkAppendMsg(&sCheck, - "Outstanding page count goes from %d to %d during this analysis", - nRef, sqlite3PagerRefcount(pBt->pPager) - ); - } - /* Clean up and report errors. */ - sqlite3BtreeLeave(p); +integrity_ck_cleanup: + sqlite3PageFree(sCheck.heap); sqlite3_free(sCheck.aPgRef); if( sCheck.mallocFailed ){ sqlite3StrAccumReset(&sCheck.errMsg); - *pnErr = sCheck.nErr+1; - return 0; + sCheck.nErr++; } *pnErr = sCheck.nErr; if( sCheck.nErr==0 ) sqlite3StrAccumReset(&sCheck.errMsg); + /* Make sure this analysis did not leave any unref() pages. */ + assert( nRef==sqlite3PagerRefcount(pBt->pPager) ); + sqlite3BtreeLeave(p); return sqlite3StrAccumFinish(&sCheck.errMsg); } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -62227,6 +64085,7 @@ SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void */ SQLITE_PRIVATE void sqlite3BtreeIncrblobCursor(BtCursor *pCur){ pCur->curFlags |= BTCF_Incrblob; + pCur->pBtree->hasIncrblobCur = 1; } #endif @@ -62312,6 +64171,8 @@ SQLITE_PRIVATE int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); ** This file contains the implementation of the sqlite3_backup_XXX() ** API functions and the related features. */ +/* #include "sqliteInt.h" */ +/* #include "btreeInt.h" */ /* ** Structure allocated for each backup operation. @@ -62983,9 +64844,13 @@ SQLITE_API int SQLITE_STDCALL sqlite3_backup_pagecount(sqlite3_backup *p){ ** corresponding to the source database is held when this function is ** called. */ -SQLITE_PRIVATE void sqlite3BackupUpdate(sqlite3_backup *pBackup, Pgno iPage, const u8 *aData){ - sqlite3_backup *p; /* Iterator variable */ - for(p=pBackup; p; p=p->pNext){ +static SQLITE_NOINLINE void backupUpdate( + sqlite3_backup *p, + Pgno iPage, + const u8 *aData +){ + assert( p!=0 ); + do{ assert( sqlite3_mutex_held(p->pSrc->pBt->mutex) ); if( !isFatalError(p->rc) && iPageiNext ){ /* The backup process p has already copied page iPage. But now it @@ -63002,7 +64867,10 @@ SQLITE_PRIVATE void sqlite3BackupUpdate(sqlite3_backup *pBackup, Pgno iPage, con p->rc = rc; } } - } + }while( (p = p->pNext)!=0 ); +} +SQLITE_PRIVATE void sqlite3BackupUpdate(sqlite3_backup *pBackup, Pgno iPage, const u8 *aData){ + if( pBackup ) backupUpdate(pBackup, iPage, aData); } /* @@ -63060,6 +64928,10 @@ SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ b.pDest = pTo; b.iNext = 1; +#ifdef SQLITE_HAS_CODEC + sqlite3PagerAlignReserve(sqlite3BtreePager(pTo), sqlite3BtreePager(pFrom)); +#endif + /* 0x7FFFFFFF is the hard limit for the number of pages in a database ** file. By passing this as the number of pages to copy to ** sqlite3_backup_step(), we can guarantee that the copy finishes @@ -63103,6 +64975,8 @@ copy_finished: ** only within the VDBE. Interface routines refer to a Mem using the ** name sqlite_value */ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ #ifdef SQLITE_DEBUG /* @@ -63674,7 +65548,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){ SQLITE_PRIVATE void sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ if( pMem->flags & MEM_Null ) return; switch( aff ){ - case SQLITE_AFF_NONE: { /* Really a cast to BLOB */ + case SQLITE_AFF_BLOB: { /* Really a cast to BLOB */ if( (pMem->flags & MEM_Blob)==0 ){ sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding); assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); @@ -63856,10 +65730,6 @@ SQLITE_PRIVATE void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ } #endif /* SQLITE_DEBUG */ -/* -** Size of struct Mem not including the Mem.zMalloc member. -*/ -#define MEMCELLSIZE offsetof(Mem,zMalloc) /* ** Make an shallow copy of pFrom into pTo. Prior contents of @@ -63867,10 +65737,15 @@ SQLITE_PRIVATE void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ ** pFrom->z is used, then pTo->z points to the same thing as pFrom->z ** and flags gets srcType (either MEM_Ephem or MEM_Static). */ +static SQLITE_NOINLINE void vdbeClrCopy(Mem *pTo, const Mem *pFrom, int eType){ + vdbeMemClearExternAndSetNull(pTo); + assert( !VdbeMemDynamic(pTo) ); + sqlite3VdbeMemShallowCopy(pTo, pFrom, eType); +} SQLITE_PRIVATE void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int srcType){ assert( (pFrom->flags & MEM_RowSet)==0 ); assert( pTo->db==pFrom->db ); - if( VdbeMemDynamic(pTo) ) vdbeMemClearExternAndSetNull(pTo); + if( VdbeMemDynamic(pTo) ){ vdbeClrCopy(pTo,pFrom,srcType); return; } memcpy(pTo, pFrom, MEMCELLSIZE); if( (pFrom->flags&MEM_Static)==0 ){ pTo->flags &= ~(MEM_Dyn|MEM_Static|MEM_Ephem); @@ -63886,7 +65761,10 @@ SQLITE_PRIVATE void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int sr SQLITE_PRIVATE int sqlite3VdbeMemCopy(Mem *pTo, const Mem *pFrom){ int rc = SQLITE_OK; - assert( pTo->db==pFrom->db ); + /* The pFrom==0 case in the following assert() is when an sqlite3_value + ** from sqlite3_value_dup() is used as the argument + ** to sqlite3_result_value(). */ + assert( pTo->db==pFrom->db || pFrom->db==0 ); assert( (pFrom->flags & MEM_RowSet)==0 ); if( VdbeMemDynamic(pTo) ) vdbeMemClearExternAndSetNull(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); @@ -64033,6 +65911,32 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( ** If this routine fails for any reason (malloc returns NULL or unable ** to read from the disk) then the pMem is left in an inconsistent state. */ +static SQLITE_NOINLINE int vdbeMemFromBtreeResize( + BtCursor *pCur, /* Cursor pointing at record to retrieve. */ + u32 offset, /* Offset from the start of data to return bytes from. */ + u32 amt, /* Number of bytes to return. */ + int key, /* If true, retrieve from the btree key, not data. */ + Mem *pMem /* OUT: Return data in this Mem structure. */ +){ + int rc; + pMem->flags = MEM_Null; + if( SQLITE_OK==(rc = sqlite3VdbeMemClearAndResize(pMem, amt+2)) ){ + if( key ){ + rc = sqlite3BtreeKey(pCur, offset, amt, pMem->z); + }else{ + rc = sqlite3BtreeData(pCur, offset, amt, pMem->z); + } + if( rc==SQLITE_OK ){ + pMem->z[amt] = 0; + pMem->z[amt+1] = 0; + pMem->flags = MEM_Blob|MEM_Term; + pMem->n = (int)amt; + }else{ + sqlite3VdbeMemRelease(pMem); + } + } + return rc; +} SQLITE_PRIVATE int sqlite3VdbeMemFromBtree( BtCursor *pCur, /* Cursor pointing at record to retrieve. */ u32 offset, /* Offset from the start of data to return bytes from. */ @@ -64062,22 +65966,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemFromBtree( pMem->flags = MEM_Blob|MEM_Ephem; pMem->n = (int)amt; }else{ - pMem->flags = MEM_Null; - if( SQLITE_OK==(rc = sqlite3VdbeMemClearAndResize(pMem, amt+2)) ){ - if( key ){ - rc = sqlite3BtreeKey(pCur, offset, amt, pMem->z); - }else{ - rc = sqlite3BtreeData(pCur, offset, amt, pMem->z); - } - if( rc==SQLITE_OK ){ - pMem->z[amt] = 0; - pMem->z[amt+1] = 0; - pMem->flags = MEM_Blob|MEM_Term; - pMem->n = (int)amt; - }else{ - sqlite3VdbeMemRelease(pMem); - } - } + rc = vdbeMemFromBtreeResize(pCur, offset, amt, key, pMem); } return rc; @@ -64226,7 +66115,7 @@ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ ** to be a scalar SQL function. If ** ** * all function arguments are SQL literals, -** * the SQLITE_FUNC_CONSTANT function flag is set, and +** * one of the SQLITE_FUNC_CONSTANT or _SLOCHNG function flags is set, and ** * the SQLITE_FUNC_NEEDCOLL function flag is not set, ** ** then this routine attempts to invoke the SQL function. Assuming no @@ -64267,7 +66156,7 @@ static int valueFromFunction( nName = sqlite3Strlen30(p->u.zToken); pFunc = sqlite3FindFunction(db, p->u.zToken, nName, nVal, enc, 0); assert( pFunc ); - if( (pFunc->funcFlags & SQLITE_FUNC_CONSTANT)==0 + if( (pFunc->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 || (pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL) ){ return SQLITE_OK; @@ -64398,7 +66287,7 @@ static int valueFromExpr( if( zVal==0 ) goto no_mem; sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC); } - if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE_AFF_NONE ){ + if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE_AFF_BLOB ){ sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, SQLITE_UTF8); }else{ sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); @@ -64765,19 +66654,28 @@ SQLITE_PRIVATE void sqlite3ValueFree(sqlite3_value *v){ } /* -** Return the number of bytes in the sqlite3_value object assuming -** that it uses the encoding "enc" +** The sqlite3ValueBytes() routine returns the number of bytes in the +** sqlite3_value object assuming that it uses the encoding "enc". +** The valueBytes() routine is a helper function. */ +static SQLITE_NOINLINE int valueBytes(sqlite3_value *pVal, u8 enc){ + return valueToText(pVal, enc)!=0 ? pVal->n : 0; +} SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value *pVal, u8 enc){ Mem *p = (Mem*)pVal; - if( (p->flags & MEM_Blob)!=0 || sqlite3ValueText(pVal, enc) ){ + assert( (p->flags & MEM_Null)==0 || (p->flags & (MEM_Str|MEM_Blob))==0 ); + if( (p->flags & MEM_Str)!=0 && pVal->enc==enc ){ + return p->n; + } + if( (p->flags & MEM_Blob)!=0 ){ if( p->flags & MEM_Zero ){ return p->n + p->u.nZero; }else{ return p->n; } } - return 0; + if( p->flags & MEM_Null ) return 0; + return valueBytes(pVal, enc); } /************** End of vdbemem.c *********************************************/ @@ -64796,6 +66694,8 @@ SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value *pVal, u8 enc){ ** This file contains code used for creating, destroying, and populating ** a VDBE (or an "sqlite3_stmt" as it is known to the outside world.) */ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ /* ** Create a new virtual database engine. @@ -64820,6 +66720,17 @@ SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse *pParse){ return p; } +/* +** Change the error string stored in Vdbe.zErrMsg +*/ +SQLITE_PRIVATE void sqlite3VdbeError(Vdbe *p, const char *zFormat, ...){ + va_list ap; + sqlite3DbFree(p->db, p->zErrMsg); + va_start(ap, zFormat); + p->zErrMsg = sqlite3VMPrintf(p->db, zFormat, ap); + va_end(ap); +} + /* ** Remember the SQL string for a prepared statement. */ @@ -64839,7 +66750,7 @@ SQLITE_PRIVATE void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, int isPrepa */ SQLITE_API const char *SQLITE_STDCALL sqlite3_sql(sqlite3_stmt *pStmt){ Vdbe *p = (Vdbe *)pStmt; - return (p && p->isPrepareV2) ? p->zSql : 0; + return p ? p->zSql : 0; } /* @@ -64986,6 +66897,44 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe *p, int op, int p1, int p2){ return sqlite3VdbeAddOp3(p, op, p1, p2, 0); } +/* Generate code for an unconditional jump to instruction iDest +*/ +SQLITE_PRIVATE int sqlite3VdbeGoto(Vdbe *p, int iDest){ + return sqlite3VdbeAddOp3(p, OP_Goto, 0, iDest, 0); +} + +/* Generate code to cause the string zStr to be loaded into +** register iDest +*/ +SQLITE_PRIVATE int sqlite3VdbeLoadString(Vdbe *p, int iDest, const char *zStr){ + return sqlite3VdbeAddOp4(p, OP_String8, 0, iDest, 0, zStr, 0); +} + +/* +** Generate code that initializes multiple registers to string or integer +** constants. The registers begin with iDest and increase consecutively. +** One register is initialized for each characgter in zTypes[]. For each +** "s" character in zTypes[], the register is a string if the argument is +** not NULL, or OP_Null if the value is a null pointer. For each "i" character +** in zTypes[], the register is initialized to an integer. +*/ +SQLITE_PRIVATE void sqlite3VdbeMultiLoad(Vdbe *p, int iDest, const char *zTypes, ...){ + va_list ap; + int i; + char c; + va_start(ap, zTypes); + for(i=0; (c = zTypes[i])!=0; i++){ + if( c=='s' ){ + const char *z = va_arg(ap, const char*); + int addr = sqlite3VdbeAddOp2(p, z==0 ? OP_Null : OP_String8, 0, iDest++); + if( z ) sqlite3VdbeChangeP4(p, addr, z, 0); + }else{ + assert( c=='i' ); + sqlite3VdbeAddOp2(p, OP_Integer, va_arg(ap, int), iDest++); + } + } + va_end(ap); +} /* ** Add an opcode that includes the p4 value as a pointer. @@ -65004,6 +66953,24 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp4( return addr; } +/* +** Add an opcode that includes the p4 value with a P4_INT64 or +** P4_REAL type. +*/ +SQLITE_PRIVATE int sqlite3VdbeAddOp4Dup8( + Vdbe *p, /* Add the opcode to this VM */ + int op, /* The new opcode */ + int p1, /* The P1 operand */ + int p2, /* The P2 operand */ + int p3, /* The P3 operand */ + const u8 *zP4, /* The P4 operand */ + int p4type /* P4 operand type */ +){ + char *p4copy = sqlite3DbMallocRaw(sqlite3VdbeDb(p), 8); + if( p4copy ) memcpy(p4copy, zP4, 8); + return sqlite3VdbeAddOp4(p, op, p1, p2, p3, p4copy, p4type); +} + /* ** Add an OP_ParseSchema opcode. This routine is broken out from ** sqlite3VdbeAddOp4() since it needs to also needs to mark all btrees @@ -65073,7 +67040,8 @@ SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe *v, int x){ int j = -1-x; assert( v->magic==VDBE_MAGIC_INIT ); assert( jnLabel ); - if( ALWAYS(j>=0) && p->aLabel ){ + assert( j>=0 ); + if( p->aLabel ){ p->aLabel[j] = v->nOp; } p->iFixedOp = v->nOp - 1; @@ -65168,6 +67136,7 @@ static Op *opIterNext(VdbeOpIter *p){ ** * OP_VUpdate ** * OP_VRename ** * OP_FkCounter with P2==0 (immediate foreign key constraint) +** * OP_CreateTable and OP_InitCoroutine (for CREATE TABLE AS SELECT ...) ** ** Then check that the value of Parse.mayAbort is true if an ** ABORT may be thrown, or false otherwise. Return true if it does @@ -65179,6 +67148,8 @@ static Op *opIterNext(VdbeOpIter *p){ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ int hasAbort = 0; int hasFkCounter = 0; + int hasCreateTable = 0; + int hasInitCoroutine = 0; Op *pOp; VdbeOpIter sIter; memset(&sIter, 0, sizeof(sIter)); @@ -65193,6 +67164,8 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ hasAbort = 1; break; } + if( opcode==OP_CreateTable ) hasCreateTable = 1; + if( opcode==OP_InitCoroutine ) hasInitCoroutine = 1; #ifndef SQLITE_OMIT_FOREIGN_KEY if( opcode==OP_FkCounter && pOp->p1==0 && pOp->p2==1 ){ hasFkCounter = 1; @@ -65206,22 +67179,27 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ ** through all opcodes and hasAbort may be set incorrectly. Return ** true for this case to prevent the assert() in the callers frame ** from failing. */ - return ( v->db->mallocFailed || hasAbort==mayAbort || hasFkCounter ); + return ( v->db->mallocFailed || hasAbort==mayAbort || hasFkCounter + || (hasCreateTable && hasInitCoroutine) ); } #endif /* SQLITE_DEBUG - the sqlite3AssertMayAbort() function */ /* -** Loop through the program looking for P2 values that are negative -** on jump instructions. Each such value is a label. Resolve the -** label by setting the P2 value to its correct non-zero value. +** This routine is called after all opcodes have been inserted. It loops +** through all the opcodes and fixes up some details. +** +** (1) For each jump instruction with a negative P2 value (a label) +** resolve the P2 value to an actual address. +** +** (2) Compute the maximum number of arguments used by any SQL function +** and store that value in *pMaxFuncArgs. ** -** This routine is called once after all opcodes have been inserted. +** (3) Update the Vdbe.readOnly and Vdbe.bIsReader flags to accurately +** indicate what the prepared statement actually does. ** -** Variable *pMaxFuncArgs is set to the maximum value of any P2 argument -** to an OP_Function, OP_AggStep or OP_VFilter opcode. This is used by -** sqlite3VdbeMakeReady() to size the Vdbe.apArg[] array. +** (4) Initialize the p4.xAdvance pointer on opcodes that use it. ** -** The Op.opflags field is set on all opcodes. +** (5) Reclaim the memory allocated for storing labels. */ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ int i; @@ -65237,11 +67215,6 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ /* NOTE: Be sure to update mkopcodeh.awk when adding or removing ** cases from this switch! */ switch( opcode ){ - case OP_Function: - case OP_AggStep: { - if( pOp->p5>nMaxArgs ) nMaxArgs = pOp->p5; - break; - } case OP_Transaction: { if( pOp->p2!=0 ) p->readOnly = 0; /* fall thru */ @@ -65339,46 +67312,44 @@ SQLITE_PRIVATE VdbeOp *sqlite3VdbeTakeOpArray(Vdbe *p, int *pnOp, int *pnMaxArg) ** address of the first operation added. */ SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe *p, int nOp, VdbeOpList const *aOp, int iLineno){ - int addr; + int addr, i; + VdbeOp *pOut; + assert( nOp>0 ); assert( p->magic==VDBE_MAGIC_INIT ); if( p->nOp + nOp > p->pParse->nOpAlloc && growOpArray(p, nOp) ){ return 0; } addr = p->nOp; - if( ALWAYS(nOp>0) ){ - int i; - VdbeOpList const *pIn = aOp; - for(i=0; ip2; - VdbeOp *pOut = &p->aOp[i+addr]; - pOut->opcode = pIn->opcode; - pOut->p1 = pIn->p1; - if( p2<0 ){ - assert( sqlite3OpcodeProperty[pOut->opcode] & OPFLG_JUMP ); - pOut->p2 = addr + ADDR(p2); - }else{ - pOut->p2 = p2; - } - pOut->p3 = pIn->p3; - pOut->p4type = P4_NOTUSED; - pOut->p4.p = 0; - pOut->p5 = 0; + pOut = &p->aOp[addr]; + for(i=0; ip2; + pOut->opcode = aOp->opcode; + pOut->p1 = aOp->p1; + if( p2<0 ){ + assert( sqlite3OpcodeProperty[pOut->opcode] & OPFLG_JUMP ); + pOut->p2 = addr + ADDR(p2); + }else{ + pOut->p2 = p2; + } + pOut->p3 = aOp->p3; + pOut->p4type = P4_NOTUSED; + pOut->p4.p = 0; + pOut->p5 = 0; #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS - pOut->zComment = 0; + pOut->zComment = 0; #endif #ifdef SQLITE_VDBE_COVERAGE - pOut->iSrcLine = iLineno+i; + pOut->iSrcLine = iLineno+i; #else - (void)iLineno; + (void)iLineno; #endif #ifdef SQLITE_DEBUG - if( p->db->flags & SQLITE_VdbeAddopTrace ){ - sqlite3VdbePrintOp(0, i+addr, &p->aOp[i+addr]); - } -#endif + if( p->db->flags & SQLITE_VdbeAddopTrace ){ + sqlite3VdbePrintOp(0, i+addr, &p->aOp[i+addr]); } - p->nOp += nOp; +#endif } + p->nOp += nOp; return addr; } @@ -65411,49 +67382,23 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatus( /* -** Change the value of the P1 operand for a specific instruction. -** This routine is useful when a large program is loaded from a -** static array using sqlite3VdbeAddOpList but we want to make a -** few minor changes to the program. +** Change the value of the opcode, or P1, P2, P3, or P5 operands +** for a specific instruction. */ +SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe *p, u32 addr, u8 iNewOpcode){ + sqlite3VdbeGetOp(p,addr)->opcode = iNewOpcode; +} SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe *p, u32 addr, int val){ - assert( p!=0 ); - if( ((u32)p->nOp)>addr ){ - p->aOp[addr].p1 = val; - } + sqlite3VdbeGetOp(p,addr)->p1 = val; } - -/* -** Change the value of the P2 operand for a specific instruction. -** This routine is useful for setting a jump destination. -*/ SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe *p, u32 addr, int val){ - assert( p!=0 ); - if( ((u32)p->nOp)>addr ){ - p->aOp[addr].p2 = val; - } + sqlite3VdbeGetOp(p,addr)->p2 = val; } - -/* -** Change the value of the P3 operand for a specific instruction. -*/ SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe *p, u32 addr, int val){ - assert( p!=0 ); - if( ((u32)p->nOp)>addr ){ - p->aOp[addr].p3 = val; - } + sqlite3VdbeGetOp(p,addr)->p3 = val; } - -/* -** Change the value of the P5 operand for the most recently -** added operation. -*/ -SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u8 val){ - assert( p!=0 ); - if( p->aOp ){ - assert( p->nOp>0 ); - p->aOp[p->nOp-1].p5 = val; - } +SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u8 p5){ + sqlite3VdbeGetOp(p,-1)->p5 = p5; } /* @@ -65461,8 +67406,8 @@ SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u8 val){ ** the address of the next instruction to be coded. */ SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe *p, int addr){ - sqlite3VdbeChangeP2(p, addr, p->nOp); p->pParse->iFixedOp = p->nOp - 1; + sqlite3VdbeChangeP2(p, addr, p->nOp); } @@ -65485,6 +67430,10 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ if( p4 ){ assert( db ); switch( p4type ){ + case P4_FUNCCTX: { + freeEphemeralFunction(db, ((sqlite3_context*)p4)->pFunc); + /* Fall through into the next case */ + } case P4_REAL: case P4_INT64: case P4_DYNAMIC: @@ -65843,8 +67792,9 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ zColl = "B"; n = 1; } - if( i+n>nTemp-6 ){ + if( i+n>nTemp-7 ){ memcpy(&zTemp[i],",...",4); + i += 4; break; } zTemp[i++] = ','; @@ -65869,6 +67819,13 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ sqlite3_snprintf(nTemp, zTemp, "%s(%d)", pDef->zName, pDef->nArg); break; } +#ifdef SQLITE_DEBUG + case P4_FUNCCTX: { + FuncDef *pDef = pOp->p4.pCtx->pFunc; + sqlite3_snprintf(nTemp, zTemp, "%s(%d)", pDef->zName, pDef->nArg); + break; + } +#endif case P4_INT64: { sqlite3_snprintf(nTemp, zTemp, "%lld", *pOp->p4.pI64); break; @@ -65989,12 +67946,11 @@ SQLITE_PRIVATE void sqlite3VdbeEnter(Vdbe *p){ /* ** Unlock all of the btrees previously locked by a call to sqlite3VdbeEnter(). */ -SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe *p){ +static SQLITE_NOINLINE void vdbeLeave(Vdbe *p){ int i; sqlite3 *db; Db *aDb; int nDb; - if( DbMaskAllZero(p->lockMask) ) return; /* The common case */ db = p->db; aDb = db->aDb; nDb = db->nDb; @@ -66004,6 +67960,10 @@ SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe *p){ } } } +SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe *p){ + if( DbMaskAllZero(p->lockMask) ) return; /* The common case */ + vdbeLeave(p); +} #endif #if defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) @@ -66176,7 +68136,7 @@ SQLITE_PRIVATE int sqlite3VdbeList( }else if( db->u1.isInterrupted ){ p->rc = SQLITE_INTERRUPT; rc = SQLITE_ERROR; - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3ErrStr(p->rc)); + sqlite3VdbeError(p, sqlite3ErrStr(p->rc)); }else{ char *zP4; Op *pOp; @@ -67079,7 +69039,7 @@ SQLITE_PRIVATE int sqlite3VdbeCheckFk(Vdbe *p, int deferred){ ){ p->rc = SQLITE_CONSTRAINT_FOREIGNKEY; p->errorAction = OE_Abort; - sqlite3SetString(&p->zErrMsg, db, "FOREIGN KEY constraint failed"); + sqlite3VdbeError(p, "FOREIGN KEY constraint failed"); return SQLITE_ERROR; } return SQLITE_OK; @@ -67700,6 +69660,13 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format){ return ((n*2) + 12 + ((flags&MEM_Str)!=0)); } +/* +** The sizes for serial types less than 12 +*/ +static const u8 sqlite3SmallTypeSizes[] = { + 0, 1, 2, 3, 4, 6, 8, 8, 0, 0, 0, 0 +}; + /* ** Return the length of the data corresponding to the supplied serial-type. */ @@ -67707,8 +69674,7 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialTypeLen(u32 serial_type){ if( serial_type>=12 ){ return (serial_type-12)/2; }else{ - static const u8 aSize[] = { 0, 1, 2, 3, 4, 6, 8, 8, 0, 0, 0, 0 }; - return aSize[serial_type]; + return sqlite3SmallTypeSizes[serial_type]; } } @@ -67792,7 +69758,7 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(u8 *buf, Mem *pMem, u32 serial_type){ }else{ v = pMem->u.i; } - len = i = sqlite3VdbeSerialTypeLen(serial_type); + len = i = sqlite3SmallTypeSizes[serial_type]; assert( i>0 ); do{ buf[--i] = (u8)(v&0xFF); @@ -68077,6 +70043,7 @@ static int vdbeRecordCompareDebug( /* mem1.u.i = 0; // not needed, here to silence compiler warning */ idx1 = getVarint32(aKey1, szHdr1); + if( szHdr1>98307 ) return SQLITE_CORRUPT; d1 = szHdr1; assert( pKeyInfo->nField+pKeyInfo->nXField>=pPKey2->nField || CORRUPT_DB ); assert( pKeyInfo->aSortOrder!=0 ); @@ -68422,7 +70389,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( if( pRhs->flags & MEM_Int ){ serial_type = aKey1[idx1]; testcase( serial_type==12 ); - if( serial_type>=12 ){ + if( serial_type>=10 ){ rc = +1; }else if( serial_type==0 ){ rc = -1; @@ -68448,7 +70415,11 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( /* RHS is real */ else if( pRhs->flags & MEM_Real ){ serial_type = aKey1[idx1]; - if( serial_type>=12 ){ + if( serial_type>=10 ){ + /* Serial types 12 or greater are strings and blobs (greater than + ** numbers). Types 10 and 11 are currently "reserved for future + ** use", so it doesn't really matter what the results of comparing + ** them to numberic values are. */ rc = +1; }else if( serial_type==0 ){ rc = -1; @@ -68817,7 +70788,7 @@ SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){ if( unlikely(typeRowid<1 || typeRowid>9 || typeRowid==7) ){ goto idx_rowid_corruption; } - lenRowid = sqlite3VdbeSerialTypeLen(typeRowid); + lenRowid = sqlite3SmallTypeSizes[typeRowid]; testcase( (u32)m.n==szHdr+lenRowid ); if( unlikely((u32)m.nstartTime>0 ); + assert( db->xProfile!=0 ); + assert( db->init.busy==0 ); + assert( p->zSql!=0 ); + sqlite3OsCurrentTimeInt64(db->pVfs, &iNow); + db->xProfile(db->pProfileArg, p->zSql, (iNow - p->startTime)*1000000); + p->startTime = 0; +} +/* +** The checkProfileCallback(DB,P) macro checks to see if a profile callback +** is needed, and it invokes the callback if it is needed. +*/ +# define checkProfileCallback(DB,P) \ + if( ((P)->startTime)>0 ){ invokeProfileCallback(DB,P); } +#else +# define checkProfileCallback(DB,P) /*no-op*/ +#endif + /* ** The following routine destroys a virtual machine that is created by ** the sqlite3_compile() routine. The integer returned is an SQLITE_ @@ -69048,6 +71046,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_finalize(sqlite3_stmt *pStmt){ sqlite3 *db = v->db; if( vdbeSafety(v) ) return SQLITE_MISUSE_BKPT; sqlite3_mutex_enter(db->mutex); + checkProfileCallback(db, v); rc = sqlite3VdbeFinalize(v); rc = sqlite3ApiExit(db, rc); sqlite3LeaveMutexAndCloseZombie(db); @@ -69069,12 +71068,14 @@ SQLITE_API int SQLITE_STDCALL sqlite3_reset(sqlite3_stmt *pStmt){ rc = SQLITE_OK; }else{ Vdbe *v = (Vdbe*)pStmt; - sqlite3_mutex_enter(v->db->mutex); + sqlite3 *db = v->db; + sqlite3_mutex_enter(db->mutex); + checkProfileCallback(db, v); rc = sqlite3VdbeReset(v); sqlite3VdbeRewind(v); - assert( (rc & (v->db->errMask))==rc ); - rc = sqlite3ApiExit(v->db, rc); - sqlite3_mutex_leave(v->db->mutex); + assert( (rc & (db->errMask))==rc ); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); } return rc; } @@ -69109,7 +71110,10 @@ SQLITE_API int SQLITE_STDCALL sqlite3_clear_bindings(sqlite3_stmt *pStmt){ SQLITE_API const void *SQLITE_STDCALL sqlite3_value_blob(sqlite3_value *pVal){ Mem *p = (Mem*)pVal; if( p->flags & (MEM_Blob|MEM_Str) ){ - sqlite3VdbeMemExpandBlob(p); + if( sqlite3VdbeMemExpandBlob(p)!=SQLITE_OK ){ + assert( p->flags==MEM_Null && p->z==0 ); + return 0; + } p->flags |= MEM_Blob; return p->n ? p->z : 0; }else{ @@ -69131,6 +71135,9 @@ SQLITE_API int SQLITE_STDCALL sqlite3_value_int(sqlite3_value *pVal){ SQLITE_API sqlite_int64 SQLITE_STDCALL sqlite3_value_int64(sqlite3_value *pVal){ return sqlite3VdbeIntValue((Mem*)pVal); } +SQLITE_API unsigned int SQLITE_STDCALL sqlite3_value_subtype(sqlite3_value *pVal){ + return ((Mem*)pVal)->eSubtype; +} SQLITE_API const unsigned char *SQLITE_STDCALL sqlite3_value_text(sqlite3_value *pVal){ return (const unsigned char *)sqlite3ValueText(pVal, SQLITE_UTF8); } @@ -69187,6 +71194,36 @@ SQLITE_API int SQLITE_STDCALL sqlite3_value_type(sqlite3_value* pVal){ return aType[pVal->flags&MEM_AffMask]; } +/* Make a copy of an sqlite3_value object +*/ +SQLITE_API sqlite3_value *SQLITE_STDCALL sqlite3_value_dup(const sqlite3_value *pOrig){ + sqlite3_value *pNew; + if( pOrig==0 ) return 0; + pNew = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return 0; + memset(pNew, 0, sizeof(*pNew)); + memcpy(pNew, pOrig, MEMCELLSIZE); + pNew->flags &= ~MEM_Dyn; + pNew->db = 0; + if( pNew->flags&(MEM_Str|MEM_Blob) ){ + pNew->flags &= ~(MEM_Static|MEM_Dyn); + pNew->flags |= MEM_Ephem; + if( sqlite3VdbeMemMakeWriteable(pNew)!=SQLITE_OK ){ + sqlite3ValueFree(pNew); + pNew = 0; + } + } + return pNew; +} + +/* Destroy an sqlite3_value object previously obtained from +** sqlite3_value_dup(). +*/ +SQLITE_API void SQLITE_STDCALL sqlite3_value_free(sqlite3_value *pOld){ + sqlite3ValueFree(pOld); +} + + /**************************** sqlite3_result_ ******************************* ** The following routines are used by user-defined functions to specify ** the function result. @@ -69279,6 +71316,10 @@ SQLITE_API void SQLITE_STDCALL sqlite3_result_null(sqlite3_context *pCtx){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetNull(pCtx->pOut); } +SQLITE_API void SQLITE_STDCALL sqlite3_result_subtype(sqlite3_context *pCtx, unsigned int eSubtype){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + pCtx->pOut->eSubtype = eSubtype & 0xff; +} SQLITE_API void SQLITE_STDCALL sqlite3_result_text( sqlite3_context *pCtx, const char *z, @@ -69341,6 +71382,15 @@ SQLITE_API void SQLITE_STDCALL sqlite3_result_zeroblob(sqlite3_context *pCtx, in assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetZeroBlob(pCtx->pOut, n); } +SQLITE_API int SQLITE_STDCALL sqlite3_result_zeroblob64(sqlite3_context *pCtx, u64 n){ + Mem *pOut = pCtx->pOut; + assert( sqlite3_mutex_held(pOut->db->mutex) ); + if( n>(u64)pOut->db->aLimit[SQLITE_LIMIT_LENGTH] ){ + return SQLITE_TOOBIG; + } + sqlite3VdbeMemSetZeroBlob(pCtx->pOut, (int)n); + return SQLITE_OK; +} SQLITE_API void SQLITE_STDCALL sqlite3_result_error_code(sqlite3_context *pCtx, int errCode){ pCtx->isError = errCode; pCtx->fErrorOrAux = 1; @@ -69395,6 +71445,7 @@ static int doWalCallbacks(sqlite3 *db){ return rc; } + /* ** Execute the statement pStmt, either until a row of data is ready, the ** statement is completely executed or an error occurs. @@ -69463,8 +71514,10 @@ static int sqlite3Step(Vdbe *p){ ); #ifndef SQLITE_OMIT_TRACE - if( db->xProfile && !db->init.busy ){ + if( db->xProfile && !db->init.busy && p->zSql ){ sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime); + }else{ + assert( p->startTime==0 ); } #endif @@ -69488,13 +71541,8 @@ static int sqlite3Step(Vdbe *p){ } #ifndef SQLITE_OMIT_TRACE - /* Invoke the profile callback if there is one - */ - if( rc!=SQLITE_ROW && db->xProfile && !db->init.busy && p->zSql ){ - sqlite3_int64 iNow; - sqlite3OsCurrentTimeInt64(db->pVfs, &iNow); - db->xProfile(db->pProfileArg, p->zSql, (iNow - p->startTime)*1000000); - } + /* If the statement completed successfully, invoke the profile callback */ + if( rc!=SQLITE_ROW ) checkProfileCallback(db, p); #endif if( rc==SQLITE_DONE ){ @@ -69518,7 +71566,7 @@ end_of_step: ** were called on statement p. */ assert( rc==SQLITE_ROW || rc==SQLITE_DONE || rc==SQLITE_ERROR - || rc==SQLITE_BUSY || rc==SQLITE_MISUSE + || (rc&0xff)==SQLITE_BUSY || rc==SQLITE_MISUSE ); assert( (p->rc!=SQLITE_ROW && p->rc!=SQLITE_DONE) || p->rc==p->rcApp ); if( p->isPrepareV2 && rc!=SQLITE_ROW && rc!=SQLITE_DONE ){ @@ -69603,7 +71651,7 @@ SQLITE_API void *SQLITE_STDCALL sqlite3_user_data(sqlite3_context *p){ ** application defined function. */ SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3_context_db_handle(sqlite3_context *p){ - assert( p && p->pFunc ); + assert( p && p->pOut ); return p->pOut->db; } @@ -69812,18 +71860,19 @@ static const Mem *columnNullValue(void){ #endif = { /* .u = */ {0}, - /* .flags = */ MEM_Null, - /* .enc = */ 0, - /* .n = */ 0, - /* .z = */ 0, - /* .zMalloc = */ 0, - /* .szMalloc = */ 0, - /* .iPadding1 = */ 0, - /* .db = */ 0, - /* .xDel = */ 0, + /* .flags = */ (u16)MEM_Null, + /* .enc = */ (u8)0, + /* .eSubtype = */ (u8)0, + /* .n = */ (int)0, + /* .z = */ (char*)0, + /* .zMalloc = */ (char*)0, + /* .szMalloc = */ (int)0, + /* .uTemp = */ (u32)0, + /* .db = */ (sqlite3*)0, + /* .xDel = */ (void(*)(void*))0, #ifdef SQLITE_DEBUG - /* .pScopyFrom = */ 0, - /* .pFiller = */ 0, + /* .pScopyFrom = */ (Mem*)0, + /* .pFiller = */ (void*)0, #endif }; return &nullMem; @@ -70322,6 +72371,20 @@ SQLITE_API int SQLITE_STDCALL sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, } return rc; } +SQLITE_API int SQLITE_STDCALL sqlite3_bind_zeroblob64(sqlite3_stmt *pStmt, int i, sqlite3_uint64 n){ + int rc; + Vdbe *p = (Vdbe *)pStmt; + sqlite3_mutex_enter(p->db->mutex); + if( n>(u64)p->db->aLimit[SQLITE_LIMIT_LENGTH] ){ + rc = SQLITE_TOOBIG; + }else{ + assert( (n & 0x7FFFFFFF)==n ); + rc = sqlite3_bind_zeroblob(pStmt, i, n); + } + rc = sqlite3ApiExit(p->db, rc); + sqlite3_mutex_leave(p->db->mutex); + return rc; +} /* ** Return the number of wildcards that can be potentially bound to. @@ -70571,6 +72634,8 @@ SQLITE_API void SQLITE_STDCALL sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt ** ** The Vdbe parse-tree explainer is also found here. */ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ #ifndef SQLITE_OMIT_TRACE @@ -70763,6 +72828,8 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( ** in this file for details. If in doubt, do not deviate from existing ** commenting and indentation practices when changing or adding code. */ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ /* ** Invoke this macro on memory cells just prior to changing the @@ -71013,7 +73080,7 @@ static void applyNumericAffinity(Mem *pRec, int bTryForInt){ ** SQLITE_AFF_TEXT: ** Convert pRec to a text representation. ** -** SQLITE_AFF_NONE: +** SQLITE_AFF_BLOB: ** No-op. pRec is unchanged. */ static void applyAffinity( @@ -71403,7 +73470,7 @@ SQLITE_PRIVATE int sqlite3VdbeExec( ** sqlite3_column_text16() failed. */ goto no_mem; } - assert( p->rc==SQLITE_OK || p->rc==SQLITE_BUSY ); + assert( p->rc==SQLITE_OK || (p->rc&0xff)==SQLITE_BUSY ); assert( p->bIsReader || p->readOnly!=0 ); p->rc = SQLITE_OK; p->iCurrentTime = 0; @@ -71414,13 +73481,9 @@ SQLITE_PRIVATE int sqlite3VdbeExec( sqlite3VdbeIOTraceSql(p); #ifndef SQLITE_OMIT_PROGRESS_CALLBACK if( db->xProgress ){ + u32 iPrior = p->aCounter[SQLITE_STMTSTATUS_VM_STEP]; assert( 0 < db->nProgressOps ); - nProgressLimit = (unsigned)p->aCounter[SQLITE_STMTSTATUS_VM_STEP]; - if( nProgressLimit==0 ){ - nProgressLimit = db->nProgressOps; - }else{ - nProgressLimit %= (unsigned)db->nProgressOps; - } + nProgressLimit = db->nProgressOps - (iPrior % db->nProgressOps); } #endif #ifdef SQLITE_DEBUG @@ -71800,12 +73863,11 @@ case OP_Halt: { assert( zType!=0 || pOp->p4.z!=0 ); zLogFmt = "abort at %d in [%s]: %s"; if( zType && pOp->p4.z ){ - sqlite3SetString(&p->zErrMsg, db, "%s constraint failed: %s", - zType, pOp->p4.z); + sqlite3VdbeError(p, "%s constraint failed: %s", zType, pOp->p4.z); }else if( pOp->p4.z ){ - sqlite3SetString(&p->zErrMsg, db, "%s", pOp->p4.z); + sqlite3VdbeError(p, "%s", pOp->p4.z); }else{ - sqlite3SetString(&p->zErrMsg, db, "%s constraint failed", zType); + sqlite3VdbeError(p, "%s constraint failed", zType); } sqlite3_log(pOp->p1, zLogFmt, pcx, p->zSql, p->zErrMsg); } @@ -72383,10 +74445,10 @@ case OP_CollSeq: { break; } -/* Opcode: Function P1 P2 P3 P4 P5 +/* Opcode: Function0 P1 P2 P3 P4 P5 ** Synopsis: r[P3]=func(r[P2@P5]) ** -** Invoke a user function (P4 is a pointer to a Function structure that +** Invoke a user function (P4 is a pointer to a FuncDef object that ** defines the function) with P5 arguments taken from register P2 and ** successors. The result of the function is stored in register P3. ** Register P3 must not be one of the function inputs. @@ -72398,59 +74460,100 @@ case OP_CollSeq: { ** sqlite3_set_auxdata() API may be safely retained until the next ** invocation of this opcode. ** -** See also: AggStep and AggFinal +** See also: Function, AggStep, AggFinal */ -case OP_Function: { - int i; - Mem *pArg; - sqlite3_context ctx; - sqlite3_value **apVal; +/* Opcode: Function P1 P2 P3 P4 P5 +** Synopsis: r[P3]=func(r[P2@P5]) +** +** Invoke a user function (P4 is a pointer to an sqlite3_context object that +** contains a pointer to the function to be run) with P5 arguments taken +** from register P2 and successors. The result of the function is stored +** in register P3. Register P3 must not be one of the function inputs. +** +** P1 is a 32-bit bitmask indicating whether or not each argument to the +** function was determined to be constant at compile time. If the first +** argument was constant then bit 0 of P1 is set. This is used to determine +** whether meta data associated with a user function argument using the +** sqlite3_set_auxdata() API may be safely retained until the next +** invocation of this opcode. +** +** SQL functions are initially coded as OP_Function0 with P4 pointing +** to a FuncDef object. But on first evaluation, the P4 operand is +** automatically converted into an sqlite3_context object and the operation +** changed to this OP_Function opcode. In this way, the initialization of +** the sqlite3_context object occurs only once, rather than once for each +** evaluation of the function. +** +** See also: Function0, AggStep, AggFinal +*/ +case OP_Function0: { int n; + sqlite3_context *pCtx; + assert( pOp->p4type==P4_FUNCDEF ); n = pOp->p5; - apVal = p->apArg; - assert( apVal || n==0 ); assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); - ctx.pOut = &aMem[pOp->p3]; - memAboutToChange(p, ctx.pOut); - assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem-p->nCursor)+1) ); assert( pOp->p3p2 || pOp->p3>=pOp->p2+n ); - pArg = &aMem[pOp->p2]; - for(i=0; ip2+i, pArg); + pCtx = sqlite3DbMallocRaw(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*)); + if( pCtx==0 ) goto no_mem; + pCtx->pOut = 0; + pCtx->pFunc = pOp->p4.pFunc; + pCtx->iOp = (int)(pOp - aOp); + pCtx->pVdbe = p; + pCtx->argc = n; + pOp->p4type = P4_FUNCCTX; + pOp->p4.pCtx = pCtx; + pOp->opcode = OP_Function; + /* Fall through into OP_Function */ +} +case OP_Function: { + int i; + sqlite3_context *pCtx; + + assert( pOp->p4type==P4_FUNCCTX ); + pCtx = pOp->p4.pCtx; + + /* If this function is inside of a trigger, the register array in aMem[] + ** might change from one evaluation to the next. The next block of code + ** checks to see if the register array has changed, and if so it + ** reinitializes the relavant parts of the sqlite3_context object */ + pOut = &aMem[pOp->p3]; + if( pCtx->pOut != pOut ){ + pCtx->pOut = pOut; + for(i=pCtx->argc-1; i>=0; i--) pCtx->argv[i] = &aMem[pOp->p2+i]; } - assert( pOp->p4type==P4_FUNCDEF ); - ctx.pFunc = pOp->p4.pFunc; - ctx.iOp = (int)(pOp - aOp); - ctx.pVdbe = p; - MemSetTypeFlag(ctx.pOut, MEM_Null); - ctx.fErrorOrAux = 0; + memAboutToChange(p, pCtx->pOut); +#ifdef SQLITE_DEBUG + for(i=0; iargc; i++){ + assert( memIsValid(pCtx->argv[i]) ); + REGISTER_TRACE(pOp->p2+i, pCtx->argv[i]); + } +#endif + MemSetTypeFlag(pCtx->pOut, MEM_Null); + pCtx->fErrorOrAux = 0; db->lastRowid = lastRowid; - (*ctx.pFunc->xFunc)(&ctx, n, apVal); /* IMP: R-24505-23230 */ + (*pCtx->pFunc->xFunc)(pCtx, pCtx->argc, pCtx->argv); /* IMP: R-24505-23230 */ lastRowid = db->lastRowid; /* Remember rowid changes made by xFunc */ /* If the function returned an error, throw an exception */ - if( ctx.fErrorOrAux ){ - if( ctx.isError ){ - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(ctx.pOut)); - rc = ctx.isError; + if( pCtx->fErrorOrAux ){ + if( pCtx->isError ){ + sqlite3VdbeError(p, "%s", sqlite3_value_text(pCtx->pOut)); + rc = pCtx->isError; } - sqlite3VdbeDeleteAuxData(p, (int)(pOp - aOp), pOp->p1); + sqlite3VdbeDeleteAuxData(p, pCtx->iOp, pOp->p1); } /* Copy the result of the function into register P3 */ - sqlite3VdbeChangeEncoding(ctx.pOut, encoding); - if( sqlite3VdbeMemTooBig(ctx.pOut) ){ - goto too_big; + if( pOut->flags & (MEM_Str|MEM_Blob) ){ + sqlite3VdbeChangeEncoding(pCtx->pOut, encoding); + if( sqlite3VdbeMemTooBig(pCtx->pOut) ) goto too_big; } - REGISTER_TRACE(pOp->p3, ctx.pOut); - UPDATE_MAX_BLOBSIZE(ctx.pOut); + REGISTER_TRACE(pOp->p3, pCtx->pOut); + UPDATE_MAX_BLOBSIZE(pCtx->pOut); break; } @@ -72613,9 +74716,9 @@ case OP_RealAffinity: { /* in1 */ ** A NULL value is not changed by this routine. It remains NULL. */ case OP_Cast: { /* in1 */ - assert( pOp->p2>=SQLITE_AFF_NONE && pOp->p2<=SQLITE_AFF_REAL ); + assert( pOp->p2>=SQLITE_AFF_BLOB && pOp->p2<=SQLITE_AFF_REAL ); testcase( pOp->p2==SQLITE_AFF_TEXT ); - testcase( pOp->p2==SQLITE_AFF_NONE ); + testcase( pOp->p2==SQLITE_AFF_BLOB ); testcase( pOp->p2==SQLITE_AFF_NUMERIC ); testcase( pOp->p2==SQLITE_AFF_INTEGER ); testcase( pOp->p2==SQLITE_AFF_REAL ); @@ -73426,7 +75529,7 @@ case OP_Affinity: { ** The mapping from character to affinity is given by the SQLITE_AFF_ ** macros defined in sqliteInt.h. ** -** If P4 is NULL then all index fields have the affinity NONE. +** If P4 is NULL then all index fields have the affinity BLOB. */ case OP_MakeRecord: { u8 *zNewRecord; /* A buffer to hold the data for the new record */ @@ -73498,7 +75601,7 @@ case OP_MakeRecord: { len = sqlite3VdbeSerialTypeLen(serial_type); if( pRec->flags & MEM_Zero ){ if( nData ){ - sqlite3VdbeMemExpandBlob(pRec); + if( sqlite3VdbeMemExpandBlob(pRec) ) goto no_mem; }else{ nZero += pRec->u.nZero; len -= pRec->u.nZero; @@ -73624,8 +75727,7 @@ case OP_Savepoint: { /* A new savepoint cannot be created if there are active write ** statements (i.e. open read/write incremental blob handles). */ - sqlite3SetString(&p->zErrMsg, db, "cannot open savepoint - " - "SQL statements in progress"); + sqlite3VdbeError(p, "cannot open savepoint - SQL statements in progress"); rc = SQLITE_BUSY; }else{ nName = sqlite3Strlen30(zName); @@ -73676,15 +75778,14 @@ case OP_Savepoint: { iSavepoint++; } if( !pSavepoint ){ - sqlite3SetString(&p->zErrMsg, db, "no such savepoint: %s", zName); + sqlite3VdbeError(p, "no such savepoint: %s", zName); rc = SQLITE_ERROR; }else if( db->nVdbeWrite>0 && p1==SAVEPOINT_RELEASE ){ /* It is not possible to release (commit) a savepoint if there are ** active write statements. */ - sqlite3SetString(&p->zErrMsg, db, - "cannot release savepoint - SQL statements in progress" - ); + sqlite3VdbeError(p, "cannot release savepoint - " + "SQL statements in progress"); rc = SQLITE_BUSY; }else{ @@ -73790,23 +75891,12 @@ case OP_AutoCommit: { assert( db->nVdbeActive>0 ); /* At least this one VM is active */ assert( p->bIsReader ); -#if 0 - if( turnOnAC && iRollback && db->nVdbeActive>1 ){ - /* If this instruction implements a ROLLBACK and other VMs are - ** still running, and a transaction is active, return an error indicating - ** that the other VMs must complete first. - */ - sqlite3SetString(&p->zErrMsg, db, "cannot rollback transaction - " - "SQL statements in progress"); - rc = SQLITE_BUSY; - }else -#endif if( turnOnAC && !iRollback && db->nVdbeWrite>0 ){ /* If this instruction implements a COMMIT and other VMs are writing ** return an error indicating that the other VMs must complete first. */ - sqlite3SetString(&p->zErrMsg, db, "cannot commit transaction - " - "SQL statements in progress"); + sqlite3VdbeError(p, "cannot commit transaction - " + "SQL statements in progress"); rc = SQLITE_BUSY; }else if( desiredAutoCommit!=db->autoCommit ){ if( iRollback ){ @@ -73817,12 +75907,12 @@ case OP_AutoCommit: { goto vdbe_return; }else{ db->autoCommit = (u8)desiredAutoCommit; - if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){ - p->pc = (int)(pOp - aOp); - db->autoCommit = (u8)(1-desiredAutoCommit); - p->rc = rc = SQLITE_BUSY; - goto vdbe_return; - } + } + if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){ + p->pc = (int)(pOp - aOp); + db->autoCommit = (u8)(1-desiredAutoCommit); + p->rc = rc = SQLITE_BUSY; + goto vdbe_return; } assert( db->nStatement==0 ); sqlite3CloseSavepoints(db); @@ -73833,7 +75923,7 @@ case OP_AutoCommit: { } goto vdbe_return; }else{ - sqlite3SetString(&p->zErrMsg, db, + sqlite3VdbeError(p, (!desiredAutoCommit)?"cannot start a transaction within a transaction":( (iRollback)?"cannot rollback - no transaction is active": "cannot commit - no transaction is active")); @@ -73894,9 +75984,11 @@ case OP_Transaction: { if( pBt ){ rc = sqlite3BtreeBeginTrans(pBt, pOp->p2); - if( rc==SQLITE_BUSY ){ + testcase( rc==SQLITE_BUSY_SNAPSHOT ); + testcase( rc==SQLITE_BUSY_RECOVERY ); + if( (rc&0xff)==SQLITE_BUSY ){ p->pc = (int)(pOp - aOp); - p->rc = rc = SQLITE_BUSY; + p->rc = rc; goto vdbe_return; } if( rc!=SQLITE_OK ){ @@ -74357,6 +76449,26 @@ case OP_Close: { break; } +#ifdef SQLITE_ENABLE_COLUMN_USED_MASK +/* Opcode: ColumnsUsed P1 * * P4 * +** +** This opcode (which only exists if SQLite was compiled with +** SQLITE_ENABLE_COLUMN_USED_MASK) identifies which columns of the +** table or index for cursor P1 are used. P4 is a 64-bit integer +** (P4_INT64) in which the first 63 bits are one for each of the +** first 63 columns of the table or index that are actually used +** by the cursor. The high-order bit is set if any column after +** the 64th is used. +*/ +case OP_ColumnsUsed: { + VdbeCursor *pC; + pC = p->apCsr[pOp->p1]; + assert( pC->pCursor ); + pC->maskUsed = *(u64*)pOp->p4.pI64; + break; +} +#endif + /* Opcode: SeekGE P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] ** @@ -74755,9 +76867,10 @@ case OP_Found: { /* jump, in3 */ ** ** P1 is the index of a cursor open on an SQL table btree (with integer ** keys). P3 is an integer rowid. If P1 does not contain a record with -** rowid P3 then jump immediately to P2. If P1 does contain a record -** with rowid P3 then leave the cursor pointing at that record and fall -** through to the next instruction. +** rowid P3 then jump immediately to P2. Or, if P2 is 0, raise an +** SQLITE_CORRUPT error. If P1 does contain a record with rowid P3 then +** leave the cursor pointing at that record and fall through to the next +** instruction. ** ** The OP_NotFound opcode performs the same operation on index btrees ** (with arbitrary multi-value keys). @@ -74789,13 +76902,21 @@ case OP_NotExists: { /* jump, in3 */ res = 0; iKey = pIn3->u.i; rc = sqlite3BtreeMovetoUnpacked(pCrsr, 0, iKey, 0, &res); + assert( rc==SQLITE_OK || res==0 ); pC->movetoTarget = iKey; /* Used by OP_Delete */ pC->nullRow = 0; pC->cacheStatus = CACHE_STALE; pC->deferredMoveto = 0; VdbeBranchTaken(res!=0,2); pC->seekResult = res; - if( res!=0 ) goto jump_to_p2; + if( res!=0 ){ + assert( rc==SQLITE_OK ); + if( pOp->p2==0 ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + goto jump_to_p2; + } + } break; } @@ -74845,9 +76966,8 @@ case OP_NewRowid: { /* out2 */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - if( NEVER(pC->pCursor==0) ){ - /* The zero initialization above is all that is needed */ - }else{ + assert( pC->pCursor!=0 ); + { /* The next rowid or record number (different terms for the same ** thing) is obtained in a two-step algorithm. ** @@ -75062,14 +77182,15 @@ case OP_InsertInt: { break; } -/* Opcode: Delete P1 P2 * P4 * +/* Opcode: Delete P1 P2 * P4 P5 ** ** Delete the record at which the P1 cursor is currently pointing. ** -** The cursor will be left pointing at either the next or the previous -** record in the table. If it is left pointing at the next record, then -** the next Next instruction will be a no-op. Hence it is OK to delete -** a record from within a Next loop. +** If the P5 parameter is non-zero, the cursor will be left pointing at +** either the next or the previous record in the table. If it is left +** pointing at the next record, then the next Next instruction will be a +** no-op. As a result, in this case it is OK to delete a record from within a +** Next loop. If P5 is zero, then the cursor is left in an undefined state. ** ** If the OPFLAG_NCHANGE flag of P2 is set, then the row change count is ** incremented (otherwise not). @@ -75084,6 +77205,7 @@ case OP_InsertInt: { */ case OP_Delete: { VdbeCursor *pC; + u8 hasUpdateCallback; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; @@ -75091,22 +77213,27 @@ case OP_Delete: { assert( pC->pCursor!=0 ); /* Only valid for real tables, no pseudotables */ assert( pC->deferredMoveto==0 ); + hasUpdateCallback = db->xUpdateCallback && pOp->p4.z && pC->isTable; + if( pOp->p5 && hasUpdateCallback ){ + sqlite3BtreeKeySize(pC->pCursor, &pC->movetoTarget); + } + #ifdef SQLITE_DEBUG /* The seek operation that positioned the cursor prior to OP_Delete will ** have also set the pC->movetoTarget field to the rowid of the row that ** is being deleted */ - if( pOp->p4.z && pC->isTable ){ + if( pOp->p4.z && pC->isTable && pOp->p5==0 ){ i64 iKey = 0; sqlite3BtreeKeySize(pC->pCursor, &iKey); assert( pC->movetoTarget==iKey ); } #endif - rc = sqlite3BtreeDelete(pC->pCursor); + rc = sqlite3BtreeDelete(pC->pCursor, pOp->p5); pC->cacheStatus = CACHE_STALE; /* Invoke the update-hook if required. */ - if( rc==SQLITE_OK && db->xUpdateCallback && pOp->p4.z && pC->isTable ){ + if( rc==SQLITE_OK && hasUpdateCallback ){ db->xUpdateCallback(db->pUpdateArg, SQLITE_DELETE, db->aDb[pC->iDb].zName, pOp->p4.z, pC->movetoTarget); assert( pC->iDb>=0 ); @@ -75586,7 +77713,6 @@ next_tail: case OP_SorterInsert: /* in2 */ case OP_IdxInsert: { /* in2 */ VdbeCursor *pC; - BtCursor *pCrsr; int nKey; const char *zKey; @@ -75596,18 +77722,17 @@ case OP_IdxInsert: { /* in2 */ assert( isSorter(pC)==(pOp->opcode==OP_SorterInsert) ); pIn2 = &aMem[pOp->p2]; assert( pIn2->flags & MEM_Blob ); - pCrsr = pC->pCursor; if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; - assert( pCrsr!=0 ); + assert( pC->pCursor!=0 ); assert( pC->isTable==0 ); rc = ExpandBlob(pIn2); if( rc==SQLITE_OK ){ - if( isSorter(pC) ){ + if( pOp->opcode==OP_SorterInsert ){ rc = sqlite3VdbeSorterWrite(pC, pIn2); }else{ nKey = pIn2->n; zKey = pIn2->z; - rc = sqlite3BtreeInsert(pCrsr, zKey, nKey, "", 0, 0, pOp->p3, + rc = sqlite3BtreeInsert(pC->pCursor, zKey, nKey, "", 0, 0, pOp->p3, ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0) ); assert( pC->deferredMoveto==0 ); @@ -75647,7 +77772,7 @@ case OP_IdxDelete: { #endif rc = sqlite3BtreeMovetoUnpacked(pCrsr, &r, 0, 0, &res); if( rc==SQLITE_OK && res==0 ){ - rc = sqlite3BtreeDelete(pCrsr); + rc = sqlite3BtreeDelete(pCrsr, 0); } assert( pC->deferredMoveto==0 ); pC->cacheStatus = CACHE_STALE; @@ -76266,7 +78391,7 @@ case OP_Program: { /* jump */ if( p->nFrame>=db->aLimit[SQLITE_LIMIT_TRIGGER_DEPTH] ){ rc = SQLITE_ERROR; - sqlite3SetString(&p->zErrMsg, db, "too many levels of trigger recursion"); + sqlite3VdbeError(p, "too many levels of trigger recursion"); break; } @@ -76445,12 +78570,12 @@ case OP_MemMax: { /* in2 */ } #endif /* SQLITE_OMIT_AUTOINCREMENT */ -/* Opcode: IfPos P1 P2 * * * -** Synopsis: if r[P1]>0 goto P2 +/* Opcode: IfPos P1 P2 P3 * * +** Synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 ** ** Register P1 must contain an integer. -** If the value of register P1 is 1 or greater, jump to P2 and -** add the literal value P3 to register P1. +** If the value of register P1 is 1 or greater, subtract P3 from the +** value in P1 and jump to P2. ** ** If the initial value of register P1 is less than 1, then the ** value is unchanged and control passes through to the next instruction. @@ -76459,38 +78584,44 @@ case OP_IfPos: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); VdbeBranchTaken( pIn1->u.i>0, 2); - if( pIn1->u.i>0 ) goto jump_to_p2; + if( pIn1->u.i>0 ){ + pIn1->u.i -= pOp->p3; + goto jump_to_p2; + } break; } -/* Opcode: IfNeg P1 P2 P3 * * -** Synopsis: r[P1]+=P3, if r[P1]<0 goto P2 +/* Opcode: SetIfNotPos P1 P2 P3 * * +** Synopsis: if r[P1]<=0 then r[P2]=P3 ** -** Register P1 must contain an integer. Add literal P3 to the value in -** register P1 then if the value of register P1 is less than zero, jump to P2. +** Register P1 must contain an integer. +** If the value of register P1 is not positive (if it is less than 1) then +** set the value of register P2 to be the integer P3. */ -case OP_IfNeg: { /* jump, in1 */ +case OP_SetIfNotPos: { /* in1, in2 */ pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); - pIn1->u.i += pOp->p3; - VdbeBranchTaken(pIn1->u.i<0, 2); - if( pIn1->u.i<0 ) goto jump_to_p2; + if( pIn1->u.i<=0 ){ + pOut = out2Prerelease(p, pOp); + pOut->u.i = pOp->p3; + } break; } /* Opcode: IfNotZero P1 P2 P3 * * -** Synopsis: if r[P1]!=0 then r[P1]+=P3, goto P2 +** Synopsis: if r[P1]!=0 then r[P1]-=P3, goto P2 ** ** Register P1 must contain an integer. If the content of register P1 is -** initially nonzero, then add P3 to P1 and jump to P2. If register P1 is -** initially zero, leave it unchanged and fall through. +** initially nonzero, then subtract P3 from the value in register P1 and +** jump to P2. If register P1 is initially zero, leave it unchanged +** and fall through. */ case OP_IfNotZero: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); VdbeBranchTaken(pIn1->u.i<0, 2); if( pIn1->u.i ){ - pIn1->u.i += pOp->p3; + pIn1->u.i -= pOp->p3; goto jump_to_p2; } break; @@ -76527,57 +78658,101 @@ case OP_JumpZeroIncr: { /* jump, in1 */ break; } -/* Opcode: AggStep * P2 P3 P4 P5 +/* Opcode: AggStep0 * P2 P3 P4 P5 ** Synopsis: accum=r[P3] step(r[P2@P5]) ** ** Execute the step function for an aggregate. The ** function has P5 arguments. P4 is a pointer to the FuncDef -** structure that specifies the function. Use register -** P3 as the accumulator. +** structure that specifies the function. Register P3 is the +** accumulator. ** ** The P5 arguments are taken from register P2 and its ** successors. */ -case OP_AggStep: { +/* Opcode: AggStep * P2 P3 P4 P5 +** Synopsis: accum=r[P3] step(r[P2@P5]) +** +** Execute the step function for an aggregate. The +** function has P5 arguments. P4 is a pointer to an sqlite3_context +** object that is used to run the function. Register P3 is +** as the accumulator. +** +** The P5 arguments are taken from register P2 and its +** successors. +** +** This opcode is initially coded as OP_AggStep0. On first evaluation, +** the FuncDef stored in P4 is converted into an sqlite3_context and +** the opcode is changed. In this way, the initialization of the +** sqlite3_context only happens once, instead of on each call to the +** step function. +*/ +case OP_AggStep0: { int n; + sqlite3_context *pCtx; + + assert( pOp->p4type==P4_FUNCDEF ); + n = pOp->p5; + assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); + assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem-p->nCursor)+1) ); + assert( pOp->p3p2 || pOp->p3>=pOp->p2+n ); + pCtx = sqlite3DbMallocRaw(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*)); + if( pCtx==0 ) goto no_mem; + pCtx->pMem = 0; + pCtx->pFunc = pOp->p4.pFunc; + pCtx->iOp = (int)(pOp - aOp); + pCtx->pVdbe = p; + pCtx->argc = n; + pOp->p4type = P4_FUNCCTX; + pOp->p4.pCtx = pCtx; + pOp->opcode = OP_AggStep; + /* Fall through into OP_AggStep */ +} +case OP_AggStep: { int i; + sqlite3_context *pCtx; Mem *pMem; - Mem *pRec; Mem t; - sqlite3_context ctx; - sqlite3_value **apVal; - n = pOp->p5; - assert( n>=0 ); - pRec = &aMem[pOp->p2]; - apVal = p->apArg; - assert( apVal || n==0 ); - for(i=0; ip4type==P4_FUNCCTX ); + pCtx = pOp->p4.pCtx; + pMem = &aMem[pOp->p3]; + + /* If this function is inside of a trigger, the register array in aMem[] + ** might change from one evaluation to the next. The next block of code + ** checks to see if the register array has changed, and if so it + ** reinitializes the relavant parts of the sqlite3_context object */ + if( pCtx->pMem != pMem ){ + pCtx->pMem = pMem; + for(i=pCtx->argc-1; i>=0; i--) pCtx->argv[i] = &aMem[pOp->p2+i]; } - ctx.pFunc = pOp->p4.pFunc; - assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); - ctx.pMem = pMem = &aMem[pOp->p3]; + +#ifdef SQLITE_DEBUG + for(i=0; iargc; i++){ + assert( memIsValid(pCtx->argv[i]) ); + REGISTER_TRACE(pOp->p2+i, pCtx->argv[i]); + } +#endif + pMem->n++; sqlite3VdbeMemInit(&t, db, MEM_Null); - ctx.pOut = &t; - ctx.isError = 0; - ctx.pVdbe = p; - ctx.iOp = (int)(pOp - aOp); - ctx.skipFlag = 0; - (ctx.pFunc->xStep)(&ctx, n, apVal); /* IMP: R-24505-23230 */ - if( ctx.isError ){ - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&t)); - rc = ctx.isError; + pCtx->pOut = &t; + pCtx->fErrorOrAux = 0; + pCtx->skipFlag = 0; + (pCtx->pFunc->xStep)(pCtx,pCtx->argc,pCtx->argv); /* IMP: R-24505-23230 */ + if( pCtx->fErrorOrAux ){ + if( pCtx->isError ){ + sqlite3VdbeError(p, "%s", sqlite3_value_text(&t)); + rc = pCtx->isError; + } + sqlite3VdbeMemRelease(&t); + }else{ + assert( t.flags==MEM_Null ); } - if( ctx.skipFlag ){ + if( pCtx->skipFlag ){ assert( pOp[-1].opcode==OP_CollSeq ); i = pOp[-1].p1; if( i ) sqlite3VdbeMemSetInt64(&aMem[i], 1); } - sqlite3VdbeMemRelease(&t); break; } @@ -76601,7 +78776,7 @@ case OP_AggFinal: { assert( (pMem->flags & ~(MEM_Null|MEM_Agg))==0 ); rc = sqlite3VdbeMemFinalize(pMem, pOp->p4.pFunc); if( rc ){ - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(pMem)); + sqlite3VdbeError(p, "%s", sqlite3_value_text(pMem)); } sqlite3VdbeChangeEncoding(pMem, encoding); UPDATE_MAX_BLOBSIZE(pMem); @@ -76706,7 +78881,7 @@ case OP_JournalMode: { /* out2 */ ){ if( !db->autoCommit || db->nVdbeRead>1 ){ rc = SQLITE_ERROR; - sqlite3SetString(&p->zErrMsg, db, + sqlite3VdbeError(p, "cannot change %s wal mode from within a transaction", (eNew==PAGER_JOURNALMODE_WAL ? "into" : "out of") ); @@ -76837,7 +79012,7 @@ case OP_TableLock: { rc = sqlite3BtreeLockTable(db->aDb[p1].pBt, pOp->p2, isWriteLock); if( (rc&0xFF)==SQLITE_LOCKED ){ const char *z = pOp->p4.z; - sqlite3SetString(&p->zErrMsg, db, "database table is locked: %s", z); + sqlite3VdbeError(p, "database table is locked: %s", z); } } break; @@ -77385,7 +79560,7 @@ vdbe_return: ** is encountered. */ too_big: - sqlite3SetString(&p->zErrMsg, db, "string or blob too big"); + sqlite3VdbeError(p, "string or blob too big"); rc = SQLITE_TOOBIG; goto vdbe_error_halt; @@ -77393,7 +79568,7 @@ too_big: */ no_mem: db->mallocFailed = 1; - sqlite3SetString(&p->zErrMsg, db, "out of memory"); + sqlite3VdbeError(p, "out of memory"); rc = SQLITE_NOMEM; goto vdbe_error_halt; @@ -77404,7 +79579,7 @@ abort_due_to_error: assert( p->zErrMsg==0 ); if( db->mallocFailed ) rc = SQLITE_NOMEM; if( rc!=SQLITE_IOERR_NOMEM ){ - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3ErrStr(rc)); + sqlite3VdbeError(p, "%s", sqlite3ErrStr(rc)); } goto vdbe_error_halt; @@ -77415,7 +79590,7 @@ abort_due_to_interrupt: assert( db->u1.isInterrupted ); rc = SQLITE_INTERRUPT; p->rc = rc; - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3ErrStr(rc)); + sqlite3VdbeError(p, "%s", sqlite3ErrStr(rc)); goto vdbe_error_halt; } @@ -77437,6 +79612,8 @@ abort_due_to_interrupt: ** This file contains code used to implement incremental BLOB I/O. */ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ #ifndef SQLITE_OMIT_INCRBLOB @@ -77669,7 +79846,8 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_open( for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int j; for(j=0; jnKeyCol; j++){ - if( pIdx->aiColumn[j]==iCol ){ + /* FIXME: Be smarter about indexes that use expressions */ + if( pIdx->aiColumn[j]==iCol || pIdx->aiColumn[j]==XN_EXPR ){ zFault = "indexed"; } } @@ -78035,6 +80213,8 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_i ** thread to merge the output of each of the others to a single PMA for ** the main thread to read from. */ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ /* ** If SQLITE_DEBUG_SORTER_THREADS is defined, this module outputs various @@ -80651,6 +82831,7 @@ SQLITE_PRIVATE int sqlite3VdbeSorterCompare( ** 2) The sqlite3JournalCreate() function is called. */ #ifdef SQLITE_ENABLE_ATOMIC_WRITE +/* #include "sqliteInt.h" */ /* @@ -80898,6 +83079,7 @@ SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *pVfs){ ** The in-memory rollback journal is used to journal transactions for ** ":memory:" databases and when the journal_mode=MEMORY pragma is used. */ +/* #include "sqliteInt.h" */ /* Forward references to internal structures */ typedef struct MemJournal MemJournal; @@ -81153,6 +83335,7 @@ SQLITE_PRIVATE int sqlite3MemJournalSize(void){ ** This file contains routines used for walking the parser tree for ** an SQL statement. */ +/* #include "sqliteInt.h" */ /* #include */ /* #include */ @@ -81245,6 +83428,11 @@ SQLITE_PRIVATE int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){ if( sqlite3WalkSelect(pWalker, pItem->pSelect) ){ return WRC_Abort; } + if( pItem->fg.isTabFunc + && sqlite3WalkExprList(pWalker, pItem->u1.pFuncArg) + ){ + return WRC_Abort; + } } } return WRC_Continue; @@ -81311,6 +83499,7 @@ SQLITE_PRIVATE int sqlite3WalkSelect(Walker *pWalker, Select *p){ ** resolve all identifiers by associating them with a particular ** table and column. */ +/* #include "sqliteInt.h" */ /* #include */ /* #include */ @@ -81341,30 +83530,6 @@ static void incrAggFunctionDepth(Expr *pExpr, int N){ ** Turn the pExpr expression into an alias for the iCol-th column of the ** result set in pEList. ** -** If the result set column is a simple column reference, then this routine -** makes an exact copy. But for any other kind of expression, this -** routine make a copy of the result set column as the argument to the -** TK_AS operator. The TK_AS operator causes the expression to be -** evaluated just once and then reused for each alias. -** -** The reason for suppressing the TK_AS term when the expression is a simple -** column reference is so that the column reference will be recognized as -** usable by indices within the WHERE clause processing logic. -** -** The TK_AS operator is inhibited if zType[0]=='G'. This means -** that in a GROUP BY clause, the expression is evaluated twice. Hence: -** -** SELECT random()%5 AS x, count(*) FROM tab GROUP BY x -** -** Is equivalent to: -** -** SELECT random()%5 AS x, count(*) FROM tab GROUP BY random()%5 -** -** The result of random()%5 in the GROUP BY clause is probably different -** from the result in the result-set. On the other hand Standard SQL does -** not allow the GROUP BY clause to contain references to result-set columns. -** So this should never come up in well-formed queries. -** ** If the reference is followed by a COLLATE operator, then make sure ** the COLLATE operator is preserved. For example: ** @@ -81398,19 +83563,11 @@ static void resolveAlias( db = pParse->db; pDup = sqlite3ExprDup(db, pOrig, 0); if( pDup==0 ) return; - if( pOrig->op!=TK_COLUMN && zType[0]!='G' ){ - incrAggFunctionDepth(pDup, nSubquery); - pDup = sqlite3PExpr(pParse, TK_AS, pDup, 0, 0); - if( pDup==0 ) return; - ExprSetProperty(pDup, EP_Skip); - if( pEList->a[iCol].u.x.iAlias==0 ){ - pEList->a[iCol].u.x.iAlias = (u16)(++pParse->nAlias); - } - pDup->iTable = pEList->a[iCol].u.x.iAlias; - } + if( zType[0]!='G' ) incrAggFunctionDepth(pDup, nSubquery); if( pExpr->op==TK_COLLATE ){ pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken); } + ExprSetProperty(pDup, EP_Alias); /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This ** prevents ExprDelete() from deleting the Expr structure itself, @@ -81602,7 +83759,7 @@ static int lookupName( ** USING clause, then skip this match. */ if( cnt==1 ){ - if( pItem->jointype & JT_NATURAL ) continue; + if( pItem->fg.jointype & JT_NATURAL ) continue; if( nameInUsingClause(pItem->pUsing, zCol) ) continue; } cnt++; @@ -81617,8 +83774,8 @@ static int lookupName( pExpr->iTable = pMatch->iCursor; pExpr->pTab = pMatch->pTab; /* RIGHT JOIN not (yet) supported */ - assert( (pMatch->jointype & JT_RIGHT)==0 ); - if( (pMatch->jointype & JT_LEFT)!=0 ){ + assert( (pMatch->fg.jointype & JT_RIGHT)==0 ); + if( (pMatch->fg.jointype & JT_LEFT)!=0 ){ ExprSetProperty(pExpr, EP_CanBeNull); } pSchema = pExpr->pTab->pSchema; @@ -81654,7 +83811,7 @@ static int lookupName( break; } } - if( iCol>=pTab->nCol && sqlite3IsRowid(zCol) && HasRowid(pTab) ){ + if( iCol>=pTab->nCol && sqlite3IsRowid(zCol) && VisibleRowid(pTab) ){ /* IMP: R-51414-32910 */ /* IMP: R-44911-55124 */ iCol = -1; @@ -81683,8 +83840,13 @@ static int lookupName( /* ** Perhaps the name is a reference to the ROWID */ - if( cnt==0 && cntTab==1 && pMatch && sqlite3IsRowid(zCol) - && HasRowid(pMatch->pTab) ){ + if( cnt==0 + && cntTab==1 + && pMatch + && (pNC->ncFlags & NC_IdxExpr)==0 + && sqlite3IsRowid(zCol) + && VisibleRowid(pMatch->pTab) + ){ cnt = 1; pExpr->iColumn = -1; /* IMP: R-44911-55124 */ pExpr->affinity = SQLITE_AFF_INTEGER; @@ -81703,9 +83865,9 @@ static int lookupName( ** resolved by the time the WHERE clause is resolved. ** ** The ability to use an output result-set column in the WHERE, GROUP BY, - ** or HAVING clauses, or as part of a larger expression in the ORDRE BY + ** or HAVING clauses, or as part of a larger expression in the ORDER BY ** clause is not standard SQL. This is a (goofy) SQLite extension, that - ** is supported for backwards compatibility only. TO DO: Issue a warning + ** is supported for backwards compatibility only. Hence, we issue a warning ** on sqlite3_log() whenever the capability is used. */ if( (pEList = pNC->pEList)!=0 @@ -81802,7 +83964,7 @@ static int lookupName( lookupname_end: if( cnt==1 ){ assert( pNC!=0 ); - if( pExpr->op!=TK_AS ){ + if( !ExprHasProperty(pExpr, EP_Alias) ){ sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); } /* Increment the nRef value on all name contexts from TopNC up to @@ -81843,36 +84005,25 @@ SQLITE_PRIVATE Expr *sqlite3CreateColumnExpr(sqlite3 *db, SrcList *pSrc, int iSr } /* -** Report an error that an expression is not valid for a partial index WHERE -** clause. +** Report an error that an expression is not valid for some set of +** pNC->ncFlags values determined by validMask. */ -static void notValidPartIdxWhere( +static void notValid( Parse *pParse, /* Leave error message here */ NameContext *pNC, /* The name context */ - const char *zMsg /* Type of error */ + const char *zMsg, /* Type of error */ + int validMask /* Set of contexts for which prohibited */ ){ - if( (pNC->ncFlags & NC_PartIdx)!=0 ){ - sqlite3ErrorMsg(pParse, "%s prohibited in partial index WHERE clauses", - zMsg); - } -} - + assert( (validMask&~(NC_IsCheck|NC_PartIdx|NC_IdxExpr))==0 ); + if( (pNC->ncFlags & validMask)!=0 ){ + const char *zIn = "partial index WHERE clauses"; + if( pNC->ncFlags & NC_IdxExpr ) zIn = "index expressions"; #ifndef SQLITE_OMIT_CHECK -/* -** Report an error that an expression is not valid for a CHECK constraint. -*/ -static void notValidCheckConstraint( - Parse *pParse, /* Leave error message here */ - NameContext *pNC, /* The name context */ - const char *zMsg /* Type of error */ -){ - if( (pNC->ncFlags & NC_IsCheck)!=0 ){ - sqlite3ErrorMsg(pParse,"%s prohibited in CHECK constraints", zMsg); + else if( pNC->ncFlags & NC_IsCheck ) zIn = "CHECK constraints"; +#endif + sqlite3ErrorMsg(pParse, "%s prohibited in %s", zMsg, zIn); } } -#else -# define notValidCheckConstraint(P,N,M) -#endif /* ** Expression p should encode a floating point value between 1.0 and 0.0. @@ -81957,6 +84108,8 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ Expr *pRight; /* if( pSrcList==0 ) break; */ + notValid(pParse, pNC, "the \".\" operator", NC_IdxExpr); + /*notValid(pParse, pNC, "the \".\" operator", NC_PartIdx|NC_IsCheck, 1);*/ pRight = pExpr->pRight; if( pRight->op==TK_ID ){ zDb = 0; @@ -81986,7 +84139,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ u8 enc = ENC(pParse->db); /* The database encoding */ assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - notValidPartIdxWhere(pParse, pNC, "functions"); + notValid(pParse, pNC, "functions", NC_PartIdx); zId = pExpr->u.zToken; nId = sqlite3Strlen30(zId); pDef = sqlite3FindFunction(pParse->db, zId, nId, n, enc, 0); @@ -82034,9 +84187,18 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ return WRC_Prune; } #endif - if( pDef->funcFlags & SQLITE_FUNC_CONSTANT ){ + if( pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG) ){ + /* For the purposes of the EP_ConstFunc flag, date and time + ** functions and other functions that change slowly are considered + ** constant because they are constant for the duration of one query */ ExprSetProperty(pExpr,EP_ConstFunc); } + if( (pDef->funcFlags & SQLITE_FUNC_CONSTANT)==0 ){ + /* Date/time functions that use 'now', and other functions like + ** sqlite_version() that might change over time cannot be used + ** in an index. */ + notValid(pParse, pNC, "non-deterministic functions", NC_IdxExpr); + } } if( is_agg && (pNC->ncFlags & NC_AllowAgg)==0 ){ sqlite3ErrorMsg(pParse, "misuse of aggregate function %.*s()", nId,zId); @@ -82082,8 +84244,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_IN ); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ int nRef = pNC->nRef; - notValidCheckConstraint(pParse, pNC, "subqueries"); - notValidPartIdxWhere(pParse, pNC, "subqueries"); + notValid(pParse, pNC, "subqueries", NC_IsCheck|NC_PartIdx|NC_IdxExpr); sqlite3WalkSelect(pWalker, pExpr->x.pSelect); assert( pNC->nRef>=nRef ); if( nRef!=pNC->nRef ){ @@ -82093,8 +84254,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ break; } case TK_VARIABLE: { - notValidCheckConstraint(pParse, pNC, "parameters"); - notValidPartIdxWhere(pParse, pNC, "parameters"); + notValid(pParse, pNC, "parameters", NC_IsCheck|NC_PartIdx|NC_IdxExpr); break; } } @@ -82438,7 +84598,6 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ int isCompound; /* True if p is a compound select */ int nCompound; /* Number of compound terms processed so far */ Parse *pParse; /* Parsing context */ - ExprList *pEList; /* Result set expression list */ int i; /* Loop counter */ ExprList *pGroupBy; /* The GROUP BY clause */ Select *pLeftmost; /* Left-most of SELECT of a compound */ @@ -82511,7 +84670,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** parent contexts. After resolving references to expressions in ** pItem->pSelect, check if this value has changed. If so, then ** SELECT statement pItem->pSelect must be correlated. Set the - ** pItem->isCorrelated flag if this is the case. */ + ** pItem->fg.isCorrelated flag if this is the case. */ for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef += pNC->nRef; if( pItem->zName ) pParse->zAuthContext = pItem->zName; @@ -82520,8 +84679,8 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ if( pParse->nErr || db->mallocFailed ) return WRC_Abort; for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef -= pNC->nRef; - assert( pItem->isCorrelated==0 && nRef<=0 ); - pItem->isCorrelated = (nRef!=0); + assert( pItem->fg.isCorrelated==0 && nRef<=0 ); + pItem->fg.isCorrelated = (nRef!=0); } } @@ -82533,14 +84692,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ sNC.pNext = pOuterNC; /* Resolve names in the result set. */ - pEList = p->pEList; - assert( pEList!=0 ); - for(i=0; inExpr; i++){ - Expr *pX = pEList->a[i].pExpr; - if( sqlite3ResolveExprNames(&sNC, pX) ){ - return WRC_Abort; - } - } + if( sqlite3ResolveExprListNames(&sNC, p->pEList) ) return WRC_Abort; /* If there are no aggregate functions in the result-set, and no GROUP BY ** expression, do not allow aggregates in any of the other expressions. @@ -82573,6 +84725,16 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort; if( sqlite3ResolveExprNames(&sNC, p->pWhere) ) return WRC_Abort; + /* Resolve names in table-valued-function arguments */ + for(i=0; ipSrc->nSrc; i++){ + struct SrcList_item *pItem = &p->pSrc->a[i]; + if( pItem->fg.isTabFunc + && sqlite3ResolveExprListNames(&sNC, pItem->u1.pFuncArg) + ){ + return WRC_Abort; + } + } + /* The ORDER BY and GROUP BY clauses may not refer to terms in ** outer queries */ @@ -82627,6 +84789,13 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ } } + /* If this is part of a compound SELECT, check that it has the right + ** number of expressions in the select list. */ + if( p->pNext && p->pEList->nExpr!=p->pNext->pEList->nExpr ){ + sqlite3SelectWrongNumTermsError(pParse, p->pNext); + return WRC_Abort; + } + /* Advance to the next term of the compound */ p = p->pPrior; @@ -82729,6 +84898,22 @@ SQLITE_PRIVATE int sqlite3ResolveExprNames( return ExprHasProperty(pExpr, EP_Error); } +/* +** Resolve all names for all expression in an expression list. This is +** just like sqlite3ResolveExprNames() except that it works for an expression +** list rather than a single expression. +*/ +SQLITE_PRIVATE int sqlite3ResolveExprListNames( + NameContext *pNC, /* Namespace to resolve expressions in. */ + ExprList *pList /* The expression list to be analyzed. */ +){ + int i; + assert( pList!=0 ); + for(i=0; inExpr; i++){ + if( sqlite3ResolveExprNames(pNC, pList->a[i].pExpr) ) return WRC_Abort; + } + return WRC_Continue; +} /* ** Resolve all names in all expressions of a SELECT and in all @@ -82772,15 +84957,14 @@ SQLITE_PRIVATE void sqlite3ResolveSelectNames( SQLITE_PRIVATE void sqlite3ResolveSelfReference( Parse *pParse, /* Parsing context */ Table *pTab, /* The table being referenced */ - int type, /* NC_IsCheck or NC_PartIdx */ + int type, /* NC_IsCheck or NC_PartIdx or NC_IdxExpr */ Expr *pExpr, /* Expression to resolve. May be NULL. */ ExprList *pList /* Expression list to resolve. May be NUL. */ ){ SrcList sSrc; /* Fake SrcList for pParse->pNewTable */ NameContext sNC; /* Name context for pParse->pNewTable */ - int i; /* Loop counter */ - assert( type==NC_IsCheck || type==NC_PartIdx ); + assert( type==NC_IsCheck || type==NC_PartIdx || type==NC_IdxExpr ); memset(&sNC, 0, sizeof(sNC)); memset(&sSrc, 0, sizeof(sSrc)); sSrc.nSrc = 1; @@ -82791,13 +84975,7 @@ SQLITE_PRIVATE void sqlite3ResolveSelfReference( sNC.pSrcList = &sSrc; sNC.ncFlags = type; if( sqlite3ResolveExprNames(&sNC, pExpr) ) return; - if( pList ){ - for(i=0; inExpr; i++){ - if( sqlite3ResolveExprNames(&sNC, pList->a[i].pExpr) ){ - return; - } - } - } + if( pList ) sqlite3ResolveExprListNames(&sNC, pList); } /************** End of resolve.c *********************************************/ @@ -82816,6 +84994,7 @@ SQLITE_PRIVATE void sqlite3ResolveSelfReference( ** This file contains routines used for analyzing expressions and ** for generating VDBE code that evaluates expressions in SQLite. */ +/* #include "sqliteInt.h" */ /* ** Return the 'affinity' of the expression pExpr if any. @@ -82894,7 +85073,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprAddCollateString(Parse *pParse, Expr *pExpr, con } /* -** Skip over any TK_COLLATE or TK_AS operators and any unlikely() +** Skip over any TK_COLLATE operators and any unlikely() ** or likelihood() function at the root of an expression. */ SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr *pExpr){ @@ -82905,7 +85084,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr *pExpr){ assert( pExpr->op==TK_FUNCTION ); pExpr = pExpr->x.pList->a[0].pExpr; }else{ - assert( pExpr->op==TK_COLLATE || pExpr->op==TK_AS ); + assert( pExpr->op==TK_COLLATE ); pExpr = pExpr->pLeft; } } @@ -82994,13 +85173,13 @@ SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2){ if( sqlite3IsNumericAffinity(aff1) || sqlite3IsNumericAffinity(aff2) ){ return SQLITE_AFF_NUMERIC; }else{ - return SQLITE_AFF_NONE; + return SQLITE_AFF_BLOB; } }else if( !aff1 && !aff2 ){ /* Neither side of the comparison is a column. Compare the ** results directly. */ - return SQLITE_AFF_NONE; + return SQLITE_AFF_BLOB; }else{ /* One side is a column, the other is not. Use the columns affinity. */ assert( aff1==0 || aff2==0 ); @@ -83024,7 +85203,7 @@ static char comparisonAffinity(Expr *pExpr){ }else if( ExprHasProperty(pExpr, EP_xIsSelect) ){ aff = sqlite3CompareAffinity(pExpr->x.pSelect->pEList->a[0].pExpr, aff); }else if( !aff ){ - aff = SQLITE_AFF_NONE; + aff = SQLITE_AFF_BLOB; } return aff; } @@ -83038,7 +85217,7 @@ static char comparisonAffinity(Expr *pExpr){ SQLITE_PRIVATE int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity){ char aff = comparisonAffinity(pExpr); switch( aff ){ - case SQLITE_AFF_NONE: + case SQLITE_AFF_BLOB: return 1; case SQLITE_AFF_TEXT: return idx_affinity==SQLITE_AFF_TEXT; @@ -83236,7 +85415,7 @@ SQLITE_PRIVATE void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ ** is responsible for making sure the node eventually gets freed. ** ** If dequote is true, then the token (if it exists) is dequoted. -** If dequote is false, no dequoting is performance. The deQuote +** If dequote is false, no dequoting is performed. The deQuote ** parameter is ignored if pToken is NULL or if the token does not ** appear to be quoted. If the quotes were of the form "..." (double-quotes) ** then the EP_DblQuoted flag is set on the expression node. @@ -83837,16 +86016,18 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ pNewItem->zDatabase = sqlite3DbStrDup(db, pOldItem->zDatabase); pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias); - pNewItem->jointype = pOldItem->jointype; + pNewItem->fg = pOldItem->fg; pNewItem->iCursor = pOldItem->iCursor; pNewItem->addrFillSub = pOldItem->addrFillSub; pNewItem->regReturn = pOldItem->regReturn; - pNewItem->isCorrelated = pOldItem->isCorrelated; - pNewItem->viaCoroutine = pOldItem->viaCoroutine; - pNewItem->isRecursive = pOldItem->isRecursive; - pNewItem->zIndex = sqlite3DbStrDup(db, pOldItem->zIndex); - pNewItem->notIndexed = pOldItem->notIndexed; - pNewItem->pIndex = pOldItem->pIndex; + if( pNewItem->fg.isIndexedBy ){ + pNewItem->u1.zIndexedBy = sqlite3DbStrDup(db, pOldItem->u1.zIndexedBy); + } + pNewItem->pIBIndex = pOldItem->pIBIndex; + if( pNewItem->fg.isTabFunc ){ + pNewItem->u1.pFuncArg = + sqlite3ExprListDup(db, pOldItem->u1.pFuncArg, flags); + } pTab = pNewItem->pTab = pOldItem->pTab; if( pTab ){ pTab->nRef++; @@ -83961,6 +86142,20 @@ no_mem: return 0; } +/* +** Set the sort order for the last element on the given ExprList. +*/ +SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList *p, int iSortOrder){ + if( p==0 ) return; + assert( SQLITE_SO_UNDEFINED<0 && SQLITE_SO_ASC>=0 && SQLITE_SO_DESC>0 ); + assert( p->nExpr>0 ); + if( iSortOrder<0 ){ + assert( p->a[p->nExpr-1].sortOrder==SQLITE_SO_ASC ); + return; + } + p->a[p->nExpr-1].sortOrder = (u8)iSortOrder; +} + /* ** Set the ExprList.a[].zName element of the most recently added item ** on the expression list. @@ -84071,7 +86266,7 @@ SQLITE_PRIVATE u32 sqlite3ExprListFlags(const ExprList *pList){ ** ** sqlite3ExprIsConstant() pWalker->eCode==1 ** sqlite3ExprIsConstantNotJoin() pWalker->eCode==2 -** sqlite3ExprRefOneTableOnly() pWalker->eCode==3 +** sqlite3ExprIsTableConstant() pWalker->eCode==3 ** sqlite3ExprIsConstantOrFunction() pWalker->eCode==4 or 5 ** ** In all cases, the callbacks set Walker.eCode=0 and abort if the expression @@ -84179,7 +86374,7 @@ SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr *p){ } /* -** Walk an expression tree. Return non-zero if the expression constant +** Walk an expression tree. Return non-zero if the expression is constant ** for any single row of the table with cursor iCur. In other words, the ** expression must not refer to any non-deterministic function nor any ** table other than iCur. @@ -84285,7 +86480,7 @@ SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr *p){ */ SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr *p, char aff){ u8 op; - if( aff==SQLITE_AFF_NONE ) return 1; + if( aff==SQLITE_AFF_BLOB ) return 1; while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ p = p->pLeft; } op = p->op; if( op==TK_REGISTER ) op = p->op2; @@ -84382,13 +86577,13 @@ SQLITE_PRIVATE int sqlite3CodeOnce(Parse *pParse){ ** to be set to NULL if iCur contains one or more NULL values. */ static void sqlite3SetHasNullFlag(Vdbe *v, int iCur, int regHasNull){ - int j1; + int addr1; sqlite3VdbeAddOp2(v, OP_Integer, 0, regHasNull); - j1 = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Column, iCur, 0, regHasNull); sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); VdbeComment((v, "first_entry_in(%d)", iCur)); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); } @@ -84736,7 +86931,7 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( int r1, r2, r3; if( !affinity ){ - affinity = SQLITE_AFF_NONE; + affinity = SQLITE_AFF_BLOB; } if( pKeyInfo ){ assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); @@ -84932,7 +87127,7 @@ static void sqlite3ExprCodeIN( } if( regCkNull ){ sqlite3VdbeAddOp2(v, OP_IsNull, regCkNull, destIfNull); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); + sqlite3VdbeGoto(v, destIfFalse); } sqlite3VdbeResolveLabel(v, labelOk); sqlite3ReleaseTempReg(pParse, regCkNull); @@ -84950,7 +87145,7 @@ static void sqlite3ExprCodeIN( int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, r1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull); + sqlite3VdbeGoto(v, destIfNull); sqlite3VdbeJumpHere(v, addr1); } } @@ -84988,7 +87183,7 @@ static void sqlite3ExprCodeIN( ** the presence of a NULL on the RHS makes a difference in the ** outcome. */ - int j1; + int addr1; /* First check to see if the LHS is contained in the RHS. If so, ** then the answer is TRUE the presence of NULLs in the RHS does @@ -84996,12 +87191,12 @@ static void sqlite3ExprCodeIN( ** answer is NULL if the RHS contains NULLs and the answer is ** FALSE if the RHS is NULL-free. */ - j1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1); + addr1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_IsNull, rRhsHasNull, destIfNull); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeGoto(v, destIfFalse); + sqlite3VdbeJumpHere(v, addr1); } } } @@ -85011,17 +87206,6 @@ static void sqlite3ExprCodeIN( } #endif /* SQLITE_OMIT_SUBQUERY */ -/* -** Duplicate an 8-byte value -*/ -static char *dup8bytes(Vdbe *v, const char *in){ - char *out = sqlite3DbMallocRaw(sqlite3VdbeDb(v), 8); - if( out ){ - memcpy(out, in, 8); - } - return out; -} - #ifndef SQLITE_OMIT_FLOATING_POINT /* ** Generate an instruction that will put the floating point @@ -85034,12 +87218,10 @@ static char *dup8bytes(Vdbe *v, const char *in){ static void codeReal(Vdbe *v, const char *z, int negateFlag, int iMem){ if( ALWAYS(z!=0) ){ double value; - char *zV; sqlite3AtoF(z, &value, sqlite3Strlen30(z), SQLITE_UTF8); assert( !sqlite3IsNaN(value) ); /* The new AtoF never returns NaN */ if( negateFlag ) value = -value; - zV = dup8bytes(v, (char*)&value); - sqlite3VdbeAddOp4(v, OP_Real, 0, iMem, 0, zV, P4_REAL); + sqlite3VdbeAddOp4Dup8(v, OP_Real, 0, iMem, 0, (u8*)&value, P4_REAL); } } #endif @@ -85065,10 +87247,8 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ assert( z!=0 ); c = sqlite3DecOrHexToI64(z, &value); if( c==0 || (c==2 && negFlag) ){ - char *zV; if( negFlag ){ value = c==2 ? SMALLEST_INT64 : -value; } - zV = dup8bytes(v, (char*)&value); - sqlite3VdbeAddOp4(v, OP_Int64, 0, iMem, 0, zV, P4_INT64); + sqlite3VdbeAddOp4Dup8(v, OP_Int64, 0, iMem, 0, (u8*)&value, P4_INT64); }else{ #ifdef SQLITE_OMIT_FLOATING_POINT sqlite3ErrorMsg(pParse, "oversized integer: %s%s", negFlag ? "-" : "", z); @@ -85234,6 +87414,28 @@ static void sqlite3ExprCachePinRegister(Parse *pParse, int iReg){ } } +/* Generate code that will load into register regOut a value that is +** appropriate for the iIdxCol-th column of index pIdx. +*/ +SQLITE_PRIVATE void sqlite3ExprCodeLoadIndexColumn( + Parse *pParse, /* The parsing context */ + Index *pIdx, /* The index whose column is to be loaded */ + int iTabCur, /* Cursor pointing to a table row */ + int iIdxCol, /* The column of the index to be loaded */ + int regOut /* Store the index column value in this register */ +){ + i16 iTabCol = pIdx->aiColumn[iIdxCol]; + if( iTabCol==XN_EXPR ){ + assert( pIdx->aColExpr ); + assert( pIdx->aColExpr->nExpr>iIdxCol ); + pParse->iSelfTab = iTabCur; + sqlite3ExprCode(pParse, pIdx->aColExpr->a[iIdxCol].pExpr, regOut); + }else{ + sqlite3ExprCodeGetColumnOfTable(pParse->pVdbe, pIdx->pTable, iTabCur, + iTabCol, regOut); + } +} + /* ** Generate code to extract the value of the iCol-th column of a table. */ @@ -85419,8 +87621,9 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) inReg = pExpr->iColumn + pParse->ckBase; break; }else{ - /* Deleting from a partial index */ - iTab = pParse->iPartIdxTab; + /* Coding an expression that is part of an index where column names + ** in the index refer to the table to which the index belongs */ + iTab = pParse->iSelfTab; } } inReg = sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, @@ -85441,7 +87644,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) #endif case TK_STRING: { assert( !ExprHasProperty(pExpr, EP_IntValue) ); - sqlite3VdbeAddOp4(v, OP_String8, 0, target, 0, pExpr->u.zToken, 0); + sqlite3VdbeLoadString(v, target, pExpr->u.zToken); break; } case TK_NULL: { @@ -85480,10 +87683,6 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) inReg = pExpr->iTable; break; } - case TK_AS: { - inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); - break; - } #ifndef SQLITE_OMIT_CAST case TK_CAST: { /* Expressions of the form: CAST(pLeft AS token) */ @@ -85673,7 +87872,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) */ if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){ assert( nFarg>=1 ); - sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target); + inReg = sqlite3ExprCodeTarget(pParse, pFarg->a[0].pExpr, target); break; } @@ -85714,7 +87913,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) } sqlite3ExprCachePush(pParse); /* Ticket 2ea2425d34be */ - sqlite3ExprCodeExprList(pParse, pFarg, r1, + sqlite3ExprCodeExprList(pParse, pFarg, r1, 0, SQLITE_ECEL_DUP|SQLITE_ECEL_FACTOR); sqlite3ExprCachePop(pParse); /* Ticket 2ea2425d34be */ }else{ @@ -85743,7 +87942,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) if( !pColl ) pColl = db->pDfltColl; sqlite3VdbeAddOp4(v, OP_CollSeq, 0, 0, 0, (char *)pColl, P4_COLLSEQ); } - sqlite3VdbeAddOp4(v, OP_Function, constMask, r1, target, + sqlite3VdbeAddOp4(v, OP_Function0, constMask, r1, target, (char*)pDef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, (u8)nFarg); if( nFarg && constMask==0 ){ @@ -85938,7 +88137,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) sqlite3ExprIfFalse(pParse, pTest, nextCase, SQLITE_JUMPIFNULL); testcase( aListelem[i+1].pExpr->op==TK_COLUMN ); sqlite3ExprCode(pParse, aListelem[i+1].pExpr, target); - sqlite3VdbeAddOp2(v, OP_Goto, 0, endLabel); + sqlite3VdbeGoto(v, endLabel); sqlite3ExprCachePop(pParse); sqlite3VdbeResolveLabel(v, nextCase); } @@ -86114,268 +88313,6 @@ SQLITE_PRIVATE void sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr, int targ exprToRegister(pExpr, iMem); } -#ifdef SQLITE_DEBUG -/* -** Generate a human-readable explanation of an expression tree. -*/ -SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ - const char *zBinOp = 0; /* Binary operator */ - const char *zUniOp = 0; /* Unary operator */ - pView = sqlite3TreeViewPush(pView, moreToFollow); - if( pExpr==0 ){ - sqlite3TreeViewLine(pView, "nil"); - sqlite3TreeViewPop(pView); - return; - } - switch( pExpr->op ){ - case TK_AGG_COLUMN: { - sqlite3TreeViewLine(pView, "AGG{%d:%d}", - pExpr->iTable, pExpr->iColumn); - break; - } - case TK_COLUMN: { - if( pExpr->iTable<0 ){ - /* This only happens when coding check constraints */ - sqlite3TreeViewLine(pView, "COLUMN(%d)", pExpr->iColumn); - }else{ - sqlite3TreeViewLine(pView, "{%d:%d}", - pExpr->iTable, pExpr->iColumn); - } - break; - } - case TK_INTEGER: { - if( pExpr->flags & EP_IntValue ){ - sqlite3TreeViewLine(pView, "%d", pExpr->u.iValue); - }else{ - sqlite3TreeViewLine(pView, "%s", pExpr->u.zToken); - } - break; - } -#ifndef SQLITE_OMIT_FLOATING_POINT - case TK_FLOAT: { - sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken); - break; - } -#endif - case TK_STRING: { - sqlite3TreeViewLine(pView,"%Q", pExpr->u.zToken); - break; - } - case TK_NULL: { - sqlite3TreeViewLine(pView,"NULL"); - break; - } -#ifndef SQLITE_OMIT_BLOB_LITERAL - case TK_BLOB: { - sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken); - break; - } -#endif - case TK_VARIABLE: { - sqlite3TreeViewLine(pView,"VARIABLE(%s,%d)", - pExpr->u.zToken, pExpr->iColumn); - break; - } - case TK_REGISTER: { - sqlite3TreeViewLine(pView,"REGISTER(%d)", pExpr->iTable); - break; - } - case TK_AS: { - sqlite3TreeViewLine(pView,"AS %Q", pExpr->u.zToken); - sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); - break; - } - case TK_ID: { - sqlite3TreeViewLine(pView,"ID \"%w\"", pExpr->u.zToken); - break; - } -#ifndef SQLITE_OMIT_CAST - case TK_CAST: { - /* Expressions of the form: CAST(pLeft AS token) */ - sqlite3TreeViewLine(pView,"CAST %Q", pExpr->u.zToken); - sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); - break; - } -#endif /* SQLITE_OMIT_CAST */ - case TK_LT: zBinOp = "LT"; break; - case TK_LE: zBinOp = "LE"; break; - case TK_GT: zBinOp = "GT"; break; - case TK_GE: zBinOp = "GE"; break; - case TK_NE: zBinOp = "NE"; break; - case TK_EQ: zBinOp = "EQ"; break; - case TK_IS: zBinOp = "IS"; break; - case TK_ISNOT: zBinOp = "ISNOT"; break; - case TK_AND: zBinOp = "AND"; break; - case TK_OR: zBinOp = "OR"; break; - case TK_PLUS: zBinOp = "ADD"; break; - case TK_STAR: zBinOp = "MUL"; break; - case TK_MINUS: zBinOp = "SUB"; break; - case TK_REM: zBinOp = "REM"; break; - case TK_BITAND: zBinOp = "BITAND"; break; - case TK_BITOR: zBinOp = "BITOR"; break; - case TK_SLASH: zBinOp = "DIV"; break; - case TK_LSHIFT: zBinOp = "LSHIFT"; break; - case TK_RSHIFT: zBinOp = "RSHIFT"; break; - case TK_CONCAT: zBinOp = "CONCAT"; break; - case TK_DOT: zBinOp = "DOT"; break; - - case TK_UMINUS: zUniOp = "UMINUS"; break; - case TK_UPLUS: zUniOp = "UPLUS"; break; - case TK_BITNOT: zUniOp = "BITNOT"; break; - case TK_NOT: zUniOp = "NOT"; break; - case TK_ISNULL: zUniOp = "ISNULL"; break; - case TK_NOTNULL: zUniOp = "NOTNULL"; break; - - case TK_COLLATE: { - sqlite3TreeViewLine(pView, "COLLATE %Q", pExpr->u.zToken); - sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); - break; - } - - case TK_AGG_FUNCTION: - case TK_FUNCTION: { - ExprList *pFarg; /* List of function arguments */ - if( ExprHasProperty(pExpr, EP_TokenOnly) ){ - pFarg = 0; - }else{ - pFarg = pExpr->x.pList; - } - if( pExpr->op==TK_AGG_FUNCTION ){ - sqlite3TreeViewLine(pView, "AGG_FUNCTION%d %Q", - pExpr->op2, pExpr->u.zToken); - }else{ - sqlite3TreeViewLine(pView, "FUNCTION %Q", pExpr->u.zToken); - } - if( pFarg ){ - sqlite3TreeViewExprList(pView, pFarg, 0, 0); - } - break; - } -#ifndef SQLITE_OMIT_SUBQUERY - case TK_EXISTS: { - sqlite3TreeViewLine(pView, "EXISTS-expr"); - sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); - break; - } - case TK_SELECT: { - sqlite3TreeViewLine(pView, "SELECT-expr"); - sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); - break; - } - case TK_IN: { - sqlite3TreeViewLine(pView, "IN"); - sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); - }else{ - sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, 0); - } - break; - } -#endif /* SQLITE_OMIT_SUBQUERY */ - - /* - ** x BETWEEN y AND z - ** - ** This is equivalent to - ** - ** x>=y AND x<=z - ** - ** X is stored in pExpr->pLeft. - ** Y is stored in pExpr->pList->a[0].pExpr. - ** Z is stored in pExpr->pList->a[1].pExpr. - */ - case TK_BETWEEN: { - Expr *pX = pExpr->pLeft; - Expr *pY = pExpr->x.pList->a[0].pExpr; - Expr *pZ = pExpr->x.pList->a[1].pExpr; - sqlite3TreeViewLine(pView, "BETWEEN"); - sqlite3TreeViewExpr(pView, pX, 1); - sqlite3TreeViewExpr(pView, pY, 1); - sqlite3TreeViewExpr(pView, pZ, 0); - break; - } - case TK_TRIGGER: { - /* If the opcode is TK_TRIGGER, then the expression is a reference - ** to a column in the new.* or old.* pseudo-tables available to - ** trigger programs. In this case Expr.iTable is set to 1 for the - ** new.* pseudo-table, or 0 for the old.* pseudo-table. Expr.iColumn - ** is set to the column of the pseudo-table to read, or to -1 to - ** read the rowid field. - */ - sqlite3TreeViewLine(pView, "%s(%d)", - pExpr->iTable ? "NEW" : "OLD", pExpr->iColumn); - break; - } - case TK_CASE: { - sqlite3TreeViewLine(pView, "CASE"); - sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); - sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, 0); - break; - } -#ifndef SQLITE_OMIT_TRIGGER - case TK_RAISE: { - const char *zType = "unk"; - switch( pExpr->affinity ){ - case OE_Rollback: zType = "rollback"; break; - case OE_Abort: zType = "abort"; break; - case OE_Fail: zType = "fail"; break; - case OE_Ignore: zType = "ignore"; break; - } - sqlite3TreeViewLine(pView, "RAISE %s(%Q)", zType, pExpr->u.zToken); - break; - } -#endif - default: { - sqlite3TreeViewLine(pView, "op=%d", pExpr->op); - break; - } - } - if( zBinOp ){ - sqlite3TreeViewLine(pView, "%s", zBinOp); - sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); - sqlite3TreeViewExpr(pView, pExpr->pRight, 0); - }else if( zUniOp ){ - sqlite3TreeViewLine(pView, "%s", zUniOp); - sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); - } - sqlite3TreeViewPop(pView); -} -#endif /* SQLITE_DEBUG */ - -#ifdef SQLITE_DEBUG -/* -** Generate a human-readable explanation of an expression list. -*/ -SQLITE_PRIVATE void sqlite3TreeViewExprList( - TreeView *pView, - const ExprList *pList, - u8 moreToFollow, - const char *zLabel -){ - int i; - pView = sqlite3TreeViewPush(pView, moreToFollow); - if( zLabel==0 || zLabel[0]==0 ) zLabel = "LIST"; - if( pList==0 ){ - sqlite3TreeViewLine(pView, "%s (empty)", zLabel); - }else{ - sqlite3TreeViewLine(pView, "%s", zLabel); - for(i=0; inExpr; i++){ - sqlite3TreeViewExpr(pView, pList->a[i].pExpr, inExpr-1); -#if 0 - if( pList->a[i].zName ){ - sqlite3ExplainPrintf(pOut, " AS %s", pList->a[i].zName); - } - if( pList->a[i].bSpanIsTab ){ - sqlite3ExplainPrintf(pOut, " (%s)", pList->a[i].zSpan); - } -#endif - } - } - sqlite3TreeViewPop(pView); -} -#endif /* SQLITE_DEBUG */ - /* ** Generate code that pushes the value of every element of the given ** expression list into a sequence of registers beginning at target. @@ -86392,11 +88329,13 @@ SQLITE_PRIVATE int sqlite3ExprCodeExprList( Parse *pParse, /* Parsing context */ ExprList *pList, /* The expression list to be coded */ int target, /* Where to write results */ + int srcReg, /* Source registers if SQLITE_ECEL_REF */ u8 flags /* SQLITE_ECEL_* flags */ ){ struct ExprList_item *pItem; - int i, n; + int i, j, n; u8 copyOp = (flags & SQLITE_ECEL_DUP) ? OP_Copy : OP_SCopy; + Vdbe *v = pParse->pVdbe; assert( pList!=0 ); assert( target>0 ); assert( pParse->pVdbe!=0 ); /* Never gets this far otherwise */ @@ -86404,13 +88343,14 @@ SQLITE_PRIVATE int sqlite3ExprCodeExprList( if( !ConstFactorOk(pParse) ) flags &= ~SQLITE_ECEL_FACTOR; for(pItem=pList->a, i=0; ipExpr; - if( (flags & SQLITE_ECEL_FACTOR)!=0 && sqlite3ExprIsConstant(pExpr) ){ + if( (flags & SQLITE_ECEL_REF)!=0 && (j = pList->a[i].u.x.iOrderByCol)>0 ){ + sqlite3VdbeAddOp2(v, copyOp, j+srcReg-1, target+i); + }else if( (flags & SQLITE_ECEL_FACTOR)!=0 && sqlite3ExprIsConstant(pExpr) ){ sqlite3ExprCodeAtInit(pParse, pExpr, target+i, 0); }else{ int inReg = sqlite3ExprCodeTarget(pParse, pExpr, target+i); if( inReg!=target+i ){ VdbeOp *pOp; - Vdbe *v = pParse->pVdbe; if( copyOp==OP_Copy && (pOp=sqlite3VdbeGetOp(v, -1))->opcode==OP_Copy && pOp->p1+pOp->p3+1==inReg @@ -86587,14 +88527,14 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int int destIfFalse = sqlite3VdbeMakeLabel(v); int destIfNull = jumpIfNull ? dest : destIfFalse; sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull); - sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); + sqlite3VdbeGoto(v, dest); sqlite3VdbeResolveLabel(v, destIfFalse); break; } #endif default: { if( exprAlwaysTrue(pExpr) ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); + sqlite3VdbeGoto(v, dest); }else if( exprAlwaysFalse(pExpr) ){ /* No-op */ }else{ @@ -86750,7 +88690,7 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int #endif default: { if( exprAlwaysFalse(pExpr) ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); + sqlite3VdbeGoto(v, dest); }else if( exprAlwaysTrue(pExpr) ){ /* no-op */ }else{ @@ -86767,6 +88707,21 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int sqlite3ReleaseTempReg(pParse, regFree2); } +/* +** Like sqlite3ExprIfFalse() except that a copy is made of pExpr before +** code generation, and that copy is deleted after code generation. This +** ensures that the original pExpr is unchanged. +*/ +SQLITE_PRIVATE void sqlite3ExprIfFalseDup(Parse *pParse, Expr *pExpr, int dest,int jumpIfNull){ + sqlite3 *db = pParse->db; + Expr *pCopy = sqlite3ExprDup(db, pExpr, 0); + if( db->mallocFailed==0 ){ + sqlite3ExprIfFalse(pParse, pCopy, dest, jumpIfNull); + } + sqlite3ExprDelete(db, pCopy); +} + + /* ** Do a deep comparison of two expression trees. Return 0 if the two ** expressions are completely identical. Return 1 if they differ only @@ -86811,7 +88766,9 @@ SQLITE_PRIVATE int sqlite3ExprCompare(Expr *pA, Expr *pB, int iTab){ return 2; } if( pA->op!=TK_COLUMN && ALWAYS(pA->op!=TK_AGG_COLUMN) && pA->u.zToken ){ - if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ + if( pA->op==TK_FUNCTION ){ + if( sqlite3StrICmp(pA->u.zToken,pB->u.zToken)!=0 ) return 2; + }else if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ return pA->op==TK_COLLATE ? 1 : 2; } } @@ -87240,6 +89197,7 @@ SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse *pParse){ ** This file contains C code routines that used to generate VDBE code ** that implements the ALTER TABLE command. */ +/* #include "sqliteInt.h" */ /* ** The code in this file only exists if we are not omitting the @@ -87718,7 +89676,7 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( #ifndef SQLITE_OMIT_VIRTUALTABLE if( pVTab ){ int i = ++pParse->nMem; - sqlite3VdbeAddOp4(v, OP_String8, 0, i, 0, zName, 0); + sqlite3VdbeLoadString(v, i, zName); sqlite3VdbeAddOp4(v, OP_VRename, i, 0, 0,(const char*)pVTab, P4_VTAB); sqlite3MayAbort(pParse); } @@ -87829,14 +89787,14 @@ SQLITE_PRIVATE void sqlite3MinimumFileFormat(Parse *pParse, int iDb, int minForm if( ALWAYS(v) ){ int r1 = sqlite3GetTempReg(pParse); int r2 = sqlite3GetTempReg(pParse); - int j1; + int addr1; sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, r1, BTREE_FILE_FORMAT); sqlite3VdbeUsesBtree(v, iDb); sqlite3VdbeAddOp2(v, OP_Integer, minFormat, r2); - j1 = sqlite3VdbeAddOp3(v, OP_Ge, r2, 0, r1); + addr1 = sqlite3VdbeAddOp3(v, OP_Ge, r2, 0, r1); sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, r2); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); sqlite3ReleaseTempReg(pParse, r1); sqlite3ReleaseTempReg(pParse, r2); } @@ -87919,7 +89877,7 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ if( pDflt ){ sqlite3_value *pVal = 0; int rc; - rc = sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_NONE, &pVal); + rc = sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_BLOB, &pVal); assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); if( rc!=SQLITE_OK ){ db->mallocFailed = 1; @@ -88202,6 +90160,7 @@ exit_begin_add_column: ** integer in the equivalent columns in sqlite_stat4. */ #ifndef SQLITE_OMIT_ANALYZE +/* #include "sqliteInt.h" */ #if defined(SQLITE_ENABLE_STAT4) # define IsStat4 1 @@ -89004,7 +90963,7 @@ static void callStatGet(Vdbe *v, int regStat4, int iParam, int regOut){ #else UNUSED_PARAMETER( iParam ); #endif - sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4, regOut); + sqlite3VdbeAddOp3(v, OP_Function0, 0, regStat4, regOut); sqlite3VdbeChangeP4(v, -1, (char*)&statGetFuncdef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, 1 + IsStat34); } @@ -89075,7 +91034,7 @@ static void analyzeOneTable( iIdxCur = iTab++; pParse->nTab = MAX(pParse->nTab, iTab); sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); - sqlite3VdbeAddOp4(v, OP_String8, 0, regTabname, 0, pTab->zName, 0); + sqlite3VdbeLoadString(v, regTabname, pTab->zName); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int nCol; /* Number of columns in pIdx. "N" */ @@ -89097,7 +91056,7 @@ static void analyzeOneTable( } /* Populate the register containing the index name. */ - sqlite3VdbeAddOp4(v, OP_String8, 0, regIdxname, 0, zIdxName, 0); + sqlite3VdbeLoadString(v, regIdxname, zIdxName); VdbeComment((v, "Analysis for %s.%s", pTab->zName, zIdxName)); /* @@ -89159,7 +91118,7 @@ static void analyzeOneTable( #endif sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat4+1); sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regStat4+2); - sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4+1, regStat4); + sqlite3VdbeAddOp3(v, OP_Function0, 0, regStat4+1, regStat4); sqlite3VdbeChangeP4(v, -1, (char*)&statInitFuncdef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, 2+IsStat34); @@ -89211,7 +91170,7 @@ static void analyzeOneTable( VdbeCoverage(v); } sqlite3VdbeAddOp2(v, OP_Integer, nColTest, regChng); - sqlite3VdbeAddOp2(v, OP_Goto, 0, endDistinctTest); + sqlite3VdbeGoto(v, endDistinctTest); /* @@ -89247,6 +91206,7 @@ static void analyzeOneTable( regKey = sqlite3GetTempRange(pParse, pPk->nKeyCol); for(j=0; jnKeyCol; j++){ k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); + assert( k>=0 && knCol ); sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, regKey+j); VdbeComment((v, "%s", pTab->aCol[pPk->aiColumn[j]].zName)); } @@ -89255,7 +91215,7 @@ static void analyzeOneTable( } #endif assert( regChng==(regStat4+1) ); - sqlite3VdbeAddOp3(v, OP_Function, 1, regStat4, regTemp); + sqlite3VdbeAddOp3(v, OP_Function0, 1, regStat4, regTemp); sqlite3VdbeChangeP4(v, -1, (char*)&statPushFuncdef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, 2+IsStat34); sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v); @@ -89296,12 +91256,10 @@ static void analyzeOneTable( ** be taken */ VdbeCoverageNeverTaken(v); #ifdef SQLITE_ENABLE_STAT3 - sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, - pIdx->aiColumn[0], regSample); + sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iTabCur, 0, regSample); #else for(i=0; iaiColumn[i]; - sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regCol+i); + sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iTabCur, i, regCol+i); } sqlite3VdbeAddOp3(v, OP_MakeRecord, regCol, nCol, regSample); #endif @@ -89967,6 +91925,7 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ ************************************************************************* ** This file contains code used to implement the ATTACH and DETACH commands. */ +/* #include "sqliteInt.h" */ #ifndef SQLITE_OMIT_ATTACH /* @@ -90314,7 +92273,7 @@ static void codeAttach( assert( v || db->mallocFailed ); if( v ){ - sqlite3VdbeAddOp3(v, OP_Function, 0, regArgs+3-pFunc->nArg, regArgs+3); + sqlite3VdbeAddOp3(v, OP_Function0, 0, regArgs+3-pFunc->nArg, regArgs+3); assert( pFunc->nArg==-1 || (pFunc->nArg&0xff)==pFunc->nArg ); sqlite3VdbeChangeP5(v, (u8)(pFunc->nArg)); sqlite3VdbeChangeP4(v, -1, (char *)pFunc, P4_FUNCDEF); @@ -90556,6 +92515,7 @@ SQLITE_PRIVATE int sqlite3FixTriggerStep( ** systems that do not need this facility may omit it by recompiling ** the library with -DSQLITE_OMIT_AUTHORIZATION=1 */ +/* #include "sqliteInt.h" */ /* ** All of the code in this file may be omitted by defining a single @@ -90826,6 +92786,7 @@ SQLITE_PRIVATE void sqlite3AuthContextPop(AuthContext *pContext){ ** COMMIT ** ROLLBACK */ +/* #include "sqliteInt.h" */ /* ** This routine is called when a new SQL statement is beginning to @@ -90995,6 +92956,8 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ db->aDb[iDb].pSchema->iGeneration /* P4 */ ); if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1); + VdbeComment((v, + "usesStmtJournal=%d", pParse->mayAbort && pParse->isMultiWrite)); } #ifndef SQLITE_OMIT_VIRTUALTABLE for(i=0; inVtabLock; i++){ @@ -91024,7 +92987,7 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ } /* Finally, jump back to the beginning of the executable code. */ - sqlite3VdbeAddOp2(v, OP_Goto, 0, 1); + sqlite3VdbeGoto(v, 1); } } @@ -91159,6 +93122,17 @@ SQLITE_PRIVATE Table *sqlite3LocateTable( p = sqlite3FindTable(pParse->db, zName, zDbase); if( p==0 ){ const char *zMsg = isView ? "no such view" : "no such table"; +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( sqlite3FindDbName(pParse->db, zDbase)<1 ){ + /* If zName is the not the name of a table in the schema created using + ** CREATE, then check to see if it is the name of an virtual table that + ** can be an eponymous virtual table. */ + Module *pMod = (Module*)sqlite3HashFind(&pParse->db->aModule, zName); + if( pMod && sqlite3VtabEponymousTableInit(pParse, pMod) ){ + return pMod->pEpoTab; + } + } +#endif if( zDbase ){ sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName); }else{ @@ -91166,7 +93140,7 @@ SQLITE_PRIVATE Table *sqlite3LocateTable( } pParse->checkSchema = 1; } -#if SQLITE_USER_AUTHENICATION +#if SQLITE_USER_AUTHENTICATION else if( pParse->db->auth.authLevelpPartIdxWhere); + sqlite3ExprListDelete(db, p->aColExpr); sqlite3DbFree(db, p->zColAff); if( p->isResized ) sqlite3DbFree(db, p->azColl); #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 @@ -91363,7 +93338,7 @@ SQLITE_PRIVATE void sqlite3CommitInternalChanges(sqlite3 *db){ ** Delete memory allocated for the column names of a table or view (the ** Table.aCol[] array). */ -static void sqliteDeleteColumnNames(sqlite3 *db, Table *pTable){ +SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3 *db, Table *pTable){ int i; Column *pCol; assert( pTable!=0 ); @@ -91430,13 +93405,11 @@ SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ /* Delete the Table structure itself. */ - sqliteDeleteColumnNames(db, pTable); + sqlite3DeleteColumnNames(db, pTable); sqlite3DbFree(db, pTable->zName); sqlite3DbFree(db, pTable->zColAff); sqlite3SelectDelete(db, pTable->pSelect); -#ifndef SQLITE_OMIT_CHECK sqlite3ExprListDelete(db, pTable->pCheck); -#endif #ifndef SQLITE_OMIT_VIRTUALTABLE sqlite3VtabClear(db, pTable); #endif @@ -91776,10 +93749,12 @@ SQLITE_PRIVATE void sqlite3StartTable( ** now. */ if( !db->init.busy && (v = sqlite3GetVdbe(pParse))!=0 ){ - int j1; + int addr1; int fileFormat; int reg1, reg2, reg3; - sqlite3BeginWriteOperation(pParse, 0, iDb); + /* nullRow[] is an OP_Record encoding of a row containing 5 NULLs */ + static const char nullRow[] = { 6, 0, 0, 0, 0, 0 }; + sqlite3BeginWriteOperation(pParse, 1, iDb); #ifndef SQLITE_OMIT_VIRTUALTABLE if( isVirtual ){ @@ -91795,14 +93770,14 @@ SQLITE_PRIVATE void sqlite3StartTable( reg3 = ++pParse->nMem; sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, reg3, BTREE_FILE_FORMAT); sqlite3VdbeUsesBtree(v, iDb); - j1 = sqlite3VdbeAddOp1(v, OP_If, reg3); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_If, reg3); VdbeCoverage(v); fileFormat = (db->flags & SQLITE_LegacyFileFmt)!=0 ? 1 : SQLITE_MAX_FILE_FORMAT; sqlite3VdbeAddOp2(v, OP_Integer, fileFormat, reg3); sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, reg3); sqlite3VdbeAddOp2(v, OP_Integer, ENC(db), reg3); sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_TEXT_ENCODING, reg3); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); /* This just creates a place-holder record in the sqlite_master table. ** The record created does not contain anything yet. It will be replaced @@ -91823,7 +93798,7 @@ SQLITE_PRIVATE void sqlite3StartTable( } sqlite3OpenMasterTable(pParse, iDb); sqlite3VdbeAddOp2(v, OP_NewRowid, 0, reg1); - sqlite3VdbeAddOp2(v, OP_Null, 0, reg3); + sqlite3VdbeAddOp4(v, OP_Blob, 6, reg3, 0, nullRow, P4_STATIC); sqlite3VdbeAddOp3(v, OP_Insert, 0, reg3, reg1); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3VdbeAddOp0(v, OP_Close); @@ -91895,10 +93870,10 @@ SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName){ pCol->zName = z; /* If there is no type specified, columns have the default affinity - ** 'NONE'. If there is a type specified, then sqlite3AddColumnType() will + ** 'BLOB'. If there is a type specified, then sqlite3AddColumnType() will ** be called next to set pCol->affinity correctly. */ - pCol->affinity = SQLITE_AFF_NONE; + pCol->affinity = SQLITE_AFF_BLOB; pCol->szEst = 1; p->nCol++; } @@ -91933,7 +93908,7 @@ SQLITE_PRIVATE void sqlite3AddNotNull(Parse *pParse, int onError){ ** 'CHAR' | SQLITE_AFF_TEXT ** 'CLOB' | SQLITE_AFF_TEXT ** 'TEXT' | SQLITE_AFF_TEXT -** 'BLOB' | SQLITE_AFF_NONE +** 'BLOB' | SQLITE_AFF_BLOB ** 'REAL' | SQLITE_AFF_REAL ** 'FLOA' | SQLITE_AFF_REAL ** 'DOUB' | SQLITE_AFF_REAL @@ -91959,7 +93934,7 @@ SQLITE_PRIVATE char sqlite3AffinityType(const char *zIn, u8 *pszEst){ aff = SQLITE_AFF_TEXT; }else if( h==(('b'<<24)+('l'<<16)+('o'<<8)+'b') /* BLOB */ && (aff==SQLITE_AFF_NUMERIC || aff==SQLITE_AFF_REAL) ){ - aff = SQLITE_AFF_NONE; + aff = SQLITE_AFF_BLOB; if( zIn[0]=='(' ) zChar = zIn; #ifndef SQLITE_OMIT_FLOATING_POINT }else if( h==(('r'<<24)+('e'<<16)+('a'<<8)+'l') /* REAL */ @@ -92060,6 +94035,30 @@ SQLITE_PRIVATE void sqlite3AddDefaultValue(Parse *pParse, ExprSpan *pSpan){ sqlite3ExprDelete(db, pSpan->pExpr); } +/* +** Backwards Compatibility Hack: +** +** Historical versions of SQLite accepted strings as column names in +** indexes and PRIMARY KEY constraints and in UNIQUE constraints. Example: +** +** CREATE TABLE xyz(a,b,c,d,e,PRIMARY KEY('a'),UNIQUE('b','c' COLLATE trim) +** CREATE INDEX abc ON xyz('c','d' DESC,'e' COLLATE nocase DESC); +** +** This is goofy. But to preserve backwards compatibility we continue to +** accept it. This routine does the necessary conversion. It converts +** the expression given in its argument from a TK_STRING into a TK_ID +** if the expression is just a TK_STRING with an optional COLLATE clause. +** If the epxression is anything other than TK_STRING, the expression is +** unchanged. +*/ +static void sqlite3StringToId(Expr *p){ + if( p->op==TK_STRING ){ + p->op = TK_ID; + }else if( p->op==TK_COLLATE && p->pLeft->op==TK_STRING ){ + p->pLeft->op = TK_ID; + } +} + /* ** Designate the PRIMARY KEY for the table. pList is a list of names ** of columns that form the primary key. If pList is NULL, then the @@ -92104,18 +94103,24 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( }else{ nTerm = pList->nExpr; for(i=0; inCol; iCol++){ - if( sqlite3StrICmp(pList->a[i].zName, pTab->aCol[iCol].zName)==0 ){ - pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY; - zType = pTab->aCol[iCol].zType; - break; + Expr *pCExpr = sqlite3ExprSkipCollate(pList->a[i].pExpr); + assert( pCExpr!=0 ); + sqlite3StringToId(pCExpr); + if( pCExpr->op==TK_ID ){ + const char *zCName = pCExpr->u.zToken; + for(iCol=0; iColnCol; iCol++){ + if( sqlite3StrICmp(zCName, pTab->aCol[iCol].zName)==0 ){ + pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY; + zType = pTab->aCol[iCol].zType; + break; + } } } } } if( nTerm==1 && zType && sqlite3StrICmp(zType, "INTEGER")==0 - && sortOrder==SQLITE_SO_ASC + && sortOrder!=SQLITE_SO_DESC ){ pTab->iPKey = iCol; pTab->keyConf = (u8)onError; @@ -92128,14 +94133,11 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( "INTEGER PRIMARY KEY"); #endif }else{ - Vdbe *v = pParse->pVdbe; Index *p; - if( v ) pParse->addrSkipPK = sqlite3VdbeAddOp0(v, OP_Noop); p = sqlite3CreateIndex(pParse, 0, 0, 0, pList, onError, 0, 0, sortOrder, 0); if( p ){ p->idxType = SQLITE_IDXTYPE_PRIMARYKEY; - if( v ) sqlite3VdbeJumpHere(v, pParse->addrSkipPK); } pList = 0; } @@ -92354,7 +94356,7 @@ static char *createTableStmt(sqlite3 *db, Table *p){ zStmt[k++] = '('; for(pCol=p->aCol, i=0; inCol; i++, pCol++){ static const char * const azType[] = { - /* SQLITE_AFF_NONE */ "", + /* SQLITE_AFF_BLOB */ "", /* SQLITE_AFF_TEXT */ " TEXT", /* SQLITE_AFF_NUMERIC */ " NUM", /* SQLITE_AFF_INTEGER */ " INT", @@ -92367,17 +94369,17 @@ static char *createTableStmt(sqlite3 *db, Table *p){ k += sqlite3Strlen30(&zStmt[k]); zSep = zSep2; identPut(zStmt, &k, pCol->zName); - assert( pCol->affinity-SQLITE_AFF_NONE >= 0 ); - assert( pCol->affinity-SQLITE_AFF_NONE < ArraySize(azType) ); - testcase( pCol->affinity==SQLITE_AFF_NONE ); + assert( pCol->affinity-SQLITE_AFF_BLOB >= 0 ); + assert( pCol->affinity-SQLITE_AFF_BLOB < ArraySize(azType) ); + testcase( pCol->affinity==SQLITE_AFF_BLOB ); testcase( pCol->affinity==SQLITE_AFF_TEXT ); testcase( pCol->affinity==SQLITE_AFF_NUMERIC ); testcase( pCol->affinity==SQLITE_AFF_INTEGER ); testcase( pCol->affinity==SQLITE_AFF_REAL ); - zType = azType[pCol->affinity - SQLITE_AFF_NONE]; + zType = azType[pCol->affinity - SQLITE_AFF_BLOB]; len = sqlite3Strlen30(zType); - assert( pCol->affinity==SQLITE_AFF_NONE + assert( pCol->affinity==SQLITE_AFF_BLOB || pCol->affinity==sqlite3AffinityType(zType, 0) ); memcpy(&zStmt[k], zType, len); k += len; @@ -92485,15 +94487,7 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ */ if( pParse->addrCrTab ){ assert( v ); - sqlite3VdbeGetOp(v, pParse->addrCrTab)->opcode = OP_CreateIndex; - } - - /* Bypass the creation of the PRIMARY KEY btree and the sqlite_master - ** table entry. - */ - if( pParse->addrSkipPK ){ - assert( v ); - sqlite3VdbeGetOp(v, pParse->addrSkipPK)->opcode = OP_Goto; + sqlite3VdbeChangeOpcode(v, pParse->addrCrTab, OP_CreateIndex); } /* Locate the PRIMARY KEY index. Or, if this table was originally @@ -92501,10 +94495,12 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ */ if( pTab->iPKey>=0 ){ ExprList *pList; - pList = sqlite3ExprListAppend(pParse, 0, 0); + Token ipkToken; + ipkToken.z = pTab->aCol[pTab->iPKey].zName; + ipkToken.n = sqlite3Strlen30(ipkToken.z); + pList = sqlite3ExprListAppend(pParse, 0, + sqlite3ExprAlloc(db, TK_ID, &ipkToken, 0)); if( pList==0 ) return; - pList->a[0].zName = sqlite3DbStrDup(pParse->db, - pTab->aCol[pTab->iPKey].zName); pList->a[0].sortOrder = pParse->iPkSortOrder; assert( pParse->pNewTable==pTab ); pPk = sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0); @@ -92513,6 +94509,16 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ pTab->iPKey = -1; }else{ pPk = sqlite3PrimaryKeyIndex(pTab); + + /* Bypass the creation of the PRIMARY KEY btree and the sqlite_master + ** table entry. This is only required if currently generating VDBE + ** code for a CREATE TABLE (not when parsing one as part of reading + ** a database schema). */ + if( v ){ + assert( db->init.busy==0 ); + sqlite3VdbeChangeOpcode(v, pPk->tnum, OP_Goto); + } + /* ** Remove all redundant columns from the PRIMARY KEY. For example, change ** "PRIMARY KEY(a,b,a,b,c,b,c,d)" into just "PRIMARY KEY(a,b,c,d)". Later @@ -92620,9 +94626,10 @@ SQLITE_PRIVATE void sqlite3EndTable( int iDb; /* Database in which the table lives */ Index *pIdx; /* An implied index of the table */ - if( (pEnd==0 && pSelect==0) || db->mallocFailed ){ + if( pEnd==0 && pSelect==0 ){ return; } + assert( !db->mallocFailed ); p = pParse->pNewTable; if( p==0 ) return; @@ -92648,7 +94655,7 @@ SQLITE_PRIVATE void sqlite3EndTable( if( (p->tabFlags & TF_HasPrimaryKey)==0 ){ sqlite3ErrorMsg(pParse, "PRIMARY KEY missing on table %s", p->zName); }else{ - p->tabFlags |= TF_WithoutRowid; + p->tabFlags |= TF_WithoutRowid | TF_NoVisibleRowid; convertToWithoutRowidTable(pParse, p); } } @@ -92716,26 +94723,46 @@ SQLITE_PRIVATE void sqlite3EndTable( ** be redundant. */ if( pSelect ){ - SelectDest dest; - Table *pSelTab; - + SelectDest dest; /* Where the SELECT should store results */ + int regYield; /* Register holding co-routine entry-point */ + int addrTop; /* Top of the co-routine */ + int regRec; /* A record to be insert into the new table */ + int regRowid; /* Rowid of the next row to insert */ + int addrInsLoop; /* Top of the loop for inserting rows */ + Table *pSelTab; /* A table that describes the SELECT results */ + + regYield = ++pParse->nMem; + regRec = ++pParse->nMem; + regRowid = ++pParse->nMem; assert(pParse->nTab==1); + sqlite3MayAbort(pParse); sqlite3VdbeAddOp3(v, OP_OpenWrite, 1, pParse->regRoot, iDb); sqlite3VdbeChangeP5(v, OPFLAG_P2ISREG); pParse->nTab = 2; - sqlite3SelectDestInit(&dest, SRT_Table, 1); + addrTop = sqlite3VdbeCurrentAddr(v) + 1; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); + sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); sqlite3Select(pParse, pSelect, &dest); + sqlite3VdbeAddOp1(v, OP_EndCoroutine, regYield); + sqlite3VdbeJumpHere(v, addrTop - 1); + if( pParse->nErr ) return; + pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect); + if( pSelTab==0 ) return; + assert( p->aCol==0 ); + p->nCol = pSelTab->nCol; + p->aCol = pSelTab->aCol; + pSelTab->nCol = 0; + pSelTab->aCol = 0; + sqlite3DeleteTable(db, pSelTab); + addrInsLoop = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_MakeRecord, dest.iSdst, dest.nSdst, regRec); + sqlite3TableAffinity(v, p, 0); + sqlite3VdbeAddOp2(v, OP_NewRowid, 1, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, 1, regRec, regRowid); + sqlite3VdbeGoto(v, addrInsLoop); + sqlite3VdbeJumpHere(v, addrInsLoop); sqlite3VdbeAddOp1(v, OP_Close, 1); - if( pParse->nErr==0 ){ - pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect); - if( pSelTab==0 ) return; - assert( p->aCol==0 ); - p->nCol = pSelTab->nCol; - p->aCol = pSelTab->aCol; - pSelTab->nCol = 0; - pSelTab->aCol = 0; - sqlite3DeleteTable(db, pSelTab); - } } /* Compute the complete text of the CREATE statement */ @@ -92830,6 +94857,7 @@ SQLITE_PRIVATE void sqlite3CreateView( Token *pBegin, /* The CREATE token that begins the statement */ Token *pName1, /* The token that holds the name of the view */ Token *pName2, /* The token that holds the name of the view */ + ExprList *pCNames, /* Optional list of view column names */ Select *pSelect, /* A SELECT statement that will become the new view */ int isTemp, /* TRUE for a TEMPORARY view */ int noErr /* Suppress error messages if VIEW already exists */ @@ -92845,22 +94873,15 @@ SQLITE_PRIVATE void sqlite3CreateView( if( pParse->nVar>0 ){ sqlite3ErrorMsg(pParse, "parameters are not allowed in views"); - sqlite3SelectDelete(db, pSelect); - return; + goto create_view_fail; } sqlite3StartTable(pParse, pName1, pName2, isTemp, 1, 0, noErr); p = pParse->pNewTable; - if( p==0 || pParse->nErr ){ - sqlite3SelectDelete(db, pSelect); - return; - } + if( p==0 || pParse->nErr ) goto create_view_fail; sqlite3TwoPartName(pParse, pName1, pName2, &pName); iDb = sqlite3SchemaToIndex(db, p->pSchema); sqlite3FixInit(&sFix, pParse, iDb, "view", pName); - if( sqlite3FixSelect(&sFix, pSelect) ){ - sqlite3SelectDelete(db, pSelect); - return; - } + if( sqlite3FixSelect(&sFix, pSelect) ) goto create_view_fail; /* Make a copy of the entire SELECT statement that defines the view. ** This will force all the Expr.token.z values to be dynamically @@ -92868,30 +94889,31 @@ SQLITE_PRIVATE void sqlite3CreateView( ** they will persist after the current sqlite3_exec() call returns. */ p->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); - sqlite3SelectDelete(db, pSelect); - if( db->mallocFailed ){ - return; - } - if( !db->init.busy ){ - sqlite3ViewGetColumnNames(pParse, p); - } + p->pCheck = sqlite3ExprListDup(db, pCNames, EXPRDUP_REDUCE); + if( db->mallocFailed ) goto create_view_fail; /* Locate the end of the CREATE VIEW statement. Make sEnd point to ** the end. */ sEnd = pParse->sLastToken; - if( ALWAYS(sEnd.z[0]!=0) && sEnd.z[0]!=';' ){ + assert( sEnd.z[0]!=0 ); + if( sEnd.z[0]!=';' ){ sEnd.z += sEnd.n; } sEnd.n = 0; n = (int)(sEnd.z - pBegin->z); + assert( n>0 ); z = pBegin->z; - while( ALWAYS(n>0) && sqlite3Isspace(z[n-1]) ){ n--; } + while( sqlite3Isspace(z[n-1]) ){ n--; } sEnd.z = &z[n-1]; sEnd.n = 1; /* Use sqlite3EndTable() to add the view to the SQLITE_MASTER table */ sqlite3EndTable(pParse, 0, &sEnd, 0, 0); + +create_view_fail: + sqlite3SelectDelete(db, pSelect); + sqlite3ExprListDelete(db, pCNames); return; } #endif /* SQLITE_OMIT_VIEW */ @@ -92909,6 +94931,7 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ int n; /* Temporarily holds the number of cursors assigned */ sqlite3 *db = pParse->db; /* Database connection for malloc errors */ sqlite3_xauth xAuth; /* Saved xAuth pointer */ + u8 bEnabledLA; /* Saved db->lookaside.bEnabled state */ assert( pTable ); @@ -92954,40 +94977,46 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ ** statement that defines the view. */ assert( pTable->pSelect ); - pSel = sqlite3SelectDup(db, pTable->pSelect, 0); - if( pSel ){ - u8 enableLookaside = db->lookaside.bEnabled; - n = pParse->nTab; - sqlite3SrcListAssignCursors(pParse, pSel->pSrc); - pTable->nCol = -1; + bEnabledLA = db->lookaside.bEnabled; + if( pTable->pCheck ){ db->lookaside.bEnabled = 0; + sqlite3ColumnsFromExprList(pParse, pTable->pCheck, + &pTable->nCol, &pTable->aCol); + }else{ + pSel = sqlite3SelectDup(db, pTable->pSelect, 0); + if( pSel ){ + n = pParse->nTab; + sqlite3SrcListAssignCursors(pParse, pSel->pSrc); + pTable->nCol = -1; + db->lookaside.bEnabled = 0; #ifndef SQLITE_OMIT_AUTHORIZATION - xAuth = db->xAuth; - db->xAuth = 0; - pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); - db->xAuth = xAuth; + xAuth = db->xAuth; + db->xAuth = 0; + pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); + db->xAuth = xAuth; #else - pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); + pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); #endif - db->lookaside.bEnabled = enableLookaside; - pParse->nTab = n; - if( pSelTab ){ - assert( pTable->aCol==0 ); - pTable->nCol = pSelTab->nCol; - pTable->aCol = pSelTab->aCol; - pSelTab->nCol = 0; - pSelTab->aCol = 0; - sqlite3DeleteTable(db, pSelTab); - assert( sqlite3SchemaMutexHeld(db, 0, pTable->pSchema) ); - pTable->pSchema->schemaFlags |= DB_UnresetViews; - }else{ - pTable->nCol = 0; + pParse->nTab = n; + if( pSelTab ){ + assert( pTable->aCol==0 ); + pTable->nCol = pSelTab->nCol; + pTable->aCol = pSelTab->aCol; + pSelTab->nCol = 0; + pSelTab->aCol = 0; + sqlite3DeleteTable(db, pSelTab); + assert( sqlite3SchemaMutexHeld(db, 0, pTable->pSchema) ); + }else{ + pTable->nCol = 0; + nErr++; + } + sqlite3SelectDelete(db, pSel); + } else { nErr++; } - sqlite3SelectDelete(db, pSel); - } else { - nErr++; } + db->lookaside.bEnabled = bEnabledLA; + pTable->pSchema->schemaFlags |= DB_UnresetViews; #endif /* SQLITE_OMIT_VIEW */ return nErr; } @@ -93004,7 +95033,7 @@ static void sqliteViewResetAll(sqlite3 *db, int idx){ for(i=sqliteHashFirst(&db->aDb[idx].pSchema->tblHash); i;i=sqliteHashNext(i)){ Table *pTab = sqliteHashData(i); if( pTab->pSelect ){ - sqliteDeleteColumnNames(db, pTab); + sqlite3DeleteColumnNames(db, pTab); pTab->aCol = 0; pTab->nCol = 0; } @@ -93559,7 +95588,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ assert( pKey!=0 || db->mallocFailed || pParse->nErr ); if( IsUniqueIndex(pIndex) && pKey!=0 ){ int j2 = sqlite3VdbeCurrentAddr(v) + 3; - sqlite3VdbeAddOp2(v, OP_Goto, 0, j2); + sqlite3VdbeGoto(v, j2); addr2 = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp4Int(v, OP_SorterCompare, iSorter, j2, regRecord, pIndex->nKeyCol); VdbeCoverage(v); @@ -93656,7 +95685,6 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( int iDb; /* Index of the database that is being written */ Token *pName = 0; /* Unqualified name of the index to create */ struct ExprList_item *pListItem; /* For looping over pList */ - const Column *pTabCol; /* A column in the table */ int nExtra = 0; /* Space allocated for zExtra[] */ int nExtraCol; /* Number of extra columns needed */ char *zExtra = 0; /* Extra space after the Index object */ @@ -93811,11 +95839,16 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( ** So create a fake list to simulate this. */ if( pList==0 ){ - pList = sqlite3ExprListAppend(pParse, 0, 0); + Token prevCol; + prevCol.z = pTab->aCol[pTab->nCol-1].zName; + prevCol.n = sqlite3Strlen30(prevCol.z); + pList = sqlite3ExprListAppend(pParse, 0, + sqlite3ExprAlloc(db, TK_ID, &prevCol, 0)); if( pList==0 ) goto exit_create_index; - pList->a[0].zName = sqlite3DbStrDup(pParse->db, - pTab->aCol[pTab->nCol-1].zName); - pList->a[0].sortOrder = (u8)sortOrder; + assert( pList->nExpr==1 ); + sqlite3ExprListSetSortOrder(pList, sortOrder); + }else{ + sqlite3ExprListCheckLength(pParse, pList, "index"); } /* Figure out how many bytes of space are required to store explicitly @@ -93823,8 +95856,8 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( */ for(i=0; inExpr; i++){ Expr *pExpr = pList->a[i].pExpr; - if( pExpr ){ - assert( pExpr->op==TK_COLLATE ); + assert( pExpr!=0 ); + if( pExpr->op==TK_COLLATE ){ nExtra += (1 + sqlite3Strlen30(pExpr->u.zToken)); } } @@ -93865,35 +95898,54 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( sortOrderMask = 0; /* Ignore DESC */ } - /* Scan the names of the columns of the table to be indexed and - ** load the column indices into the Index structure. Report an error - ** if any column is not found. + /* Analyze the list of expressions that form the terms of the index and + ** report any errors. In the common case where the expression is exactly + ** a table column, store that column in aiColumn[]. For general expressions, + ** populate pIndex->aColExpr and store XN_EXPR (-2) in aiColumn[]. ** - ** TODO: Add a test to make sure that the same column is not named - ** more than once within the same index. Only the first instance of - ** the column will ever be used by the optimizer. Note that using the - ** same column more than once cannot be an error because that would - ** break backwards compatibility - it needs to be a warning. + ** TODO: Issue a warning if two or more columns of the index are identical. + ** TODO: Issue a warning if the table primary key is used as part of the + ** index key. */ for(i=0, pListItem=pList->a; inExpr; i++, pListItem++){ - const char *zColName = pListItem->zName; - int requestedSortOrder; + Expr *pCExpr; /* The i-th index expression */ + int requestedSortOrder; /* ASC or DESC on the i-th expression */ char *zColl; /* Collation sequence name */ - for(j=0, pTabCol=pTab->aCol; jnCol; j++, pTabCol++){ - if( sqlite3StrICmp(zColName, pTabCol->zName)==0 ) break; - } - if( j>=pTab->nCol ){ - sqlite3ErrorMsg(pParse, "table %s has no column named %s", - pTab->zName, zColName); - pParse->checkSchema = 1; - goto exit_create_index; + sqlite3StringToId(pListItem->pExpr); + sqlite3ResolveSelfReference(pParse, pTab, NC_IdxExpr, pListItem->pExpr, 0); + if( pParse->nErr ) goto exit_create_index; + pCExpr = sqlite3ExprSkipCollate(pListItem->pExpr); + if( pCExpr->op!=TK_COLUMN ){ + if( pTab==pParse->pNewTable ){ + sqlite3ErrorMsg(pParse, "expressions prohibited in PRIMARY KEY and " + "UNIQUE constraints"); + goto exit_create_index; + } + if( pIndex->aColExpr==0 ){ + ExprList *pCopy = sqlite3ExprListDup(db, pList, 0); + pIndex->aColExpr = pCopy; + if( !db->mallocFailed ){ + assert( pCopy!=0 ); + pListItem = &pCopy->a[i]; + } + } + j = XN_EXPR; + pIndex->aiColumn[i] = XN_EXPR; + pIndex->uniqNotNull = 0; + }else{ + j = pCExpr->iColumn; + assert( j<=0x7fff ); + if( j<0 ){ + j = pTab->iPKey; + }else if( pTab->aCol[j].notNull==0 ){ + pIndex->uniqNotNull = 0; + } + pIndex->aiColumn[i] = (i16)j; } - assert( j<=0x7fff ); - pIndex->aiColumn[i] = (i16)j; - if( pListItem->pExpr ){ + zColl = 0; + if( pListItem->pExpr->op==TK_COLLATE ){ int nColl; - assert( pListItem->pExpr->op==TK_COLLATE ); zColl = pListItem->pExpr->u.zToken; nColl = sqlite3Strlen30(zColl) + 1; assert( nExtra>=nColl ); @@ -93901,21 +95953,26 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( zColl = zExtra; zExtra += nColl; nExtra -= nColl; - }else{ + }else if( j>=0 ){ zColl = pTab->aCol[j].zColl; - if( !zColl ) zColl = "BINARY"; } + if( !zColl ) zColl = "BINARY"; if( !db->init.busy && !sqlite3LocateCollSeq(pParse, zColl) ){ goto exit_create_index; } pIndex->azColl[i] = zColl; requestedSortOrder = pListItem->sortOrder & sortOrderMask; pIndex->aSortOrder[i] = (u8)requestedSortOrder; - if( pTab->aCol[j].notNull==0 ) pIndex->uniqNotNull = 0; } + + /* Append the table key to the end of the index. For WITHOUT ROWID + ** tables (when pPk!=0) this will be the declared PRIMARY KEY. For + ** normal tables (when pPk==0) this will be the rowid. + */ if( pPk ){ for(j=0; jnKeyCol; j++){ int x = pPk->aiColumn[j]; + assert( x>=0 ); if( hasColumn(pIndex->aiColumn, pIndex->nKeyCol, x) ){ pIndex->nColumn--; }else{ @@ -93927,7 +95984,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( } assert( i==pIndex->nColumn ); }else{ - pIndex->aiColumn[i] = -1; + pIndex->aiColumn[i] = XN_ROWID; pIndex->azColl[i] = "BINARY"; } sqlite3DefaultRowEst(pIndex); @@ -93966,6 +96023,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( for(k=0; knKeyCol; k++){ const char *z1; const char *z2; + assert( pIdx->aiColumn[k]>=0 ); if( pIdx->aiColumn[k]!=pIndex->aiColumn[k] ) break; z1 = pIdx->azColl[k]; z2 = pIndex->azColl[k]; @@ -93997,6 +96055,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( /* Link the new Index structure to its table and to the other ** in-memory database structures. */ + assert( pParse->nErr==0 ); if( db->init.busy ){ Index *p; assert( sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); @@ -94026,7 +96085,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( ** has just been created, it contains no data and the index initialization ** step can be skipped. */ - else if( pParse->nErr==0 && (HasRowid(pTab) || pTblName!=0) ){ + else if( HasRowid(pTab) || pTblName!=0 ){ Vdbe *v; char *zStmt; int iMem = ++pParse->nMem; @@ -94034,10 +96093,15 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( v = sqlite3GetVdbe(pParse); if( v==0 ) goto exit_create_index; - - /* Create the rootpage for the index - */ sqlite3BeginWriteOperation(pParse, 1, iDb); + + /* Create the rootpage for the index using CreateIndex. But before + ** doing so, code a Noop instruction and store its address in + ** Index.tnum. This is required in case this index is actually a + ** PRIMARY KEY and the table is actually a WITHOUT ROWID table. In + ** that case the convertToWithoutRowidTable() routine will replace + ** the Noop with a Goto to jump over the VDBE code generated below. */ + pIndex->tnum = sqlite3VdbeAddOp0(v, OP_Noop); sqlite3VdbeAddOp2(v, OP_CreateIndex, iDb, iMem); /* Gather the complete text of the CREATE INDEX statement into @@ -94077,6 +96141,8 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName)); sqlite3VdbeAddOp1(v, OP_Expire, 0); } + + sqlite3VdbeJumpHere(v, pIndex->tnum); } /* When adding an index to the list of indices for a table, make @@ -94479,7 +96545,8 @@ SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ sqlite3DbFree(db, pItem->zDatabase); sqlite3DbFree(db, pItem->zName); sqlite3DbFree(db, pItem->zAlias); - sqlite3DbFree(db, pItem->zIndex); + if( pItem->fg.isIndexedBy ) sqlite3DbFree(db, pItem->u1.zIndexedBy); + if( pItem->fg.isTabFunc ) sqlite3ExprListDelete(db, pItem->u1.pFuncArg); sqlite3DeleteTable(db, pItem->pTab); sqlite3SelectDelete(db, pItem->pSelect); sqlite3ExprDelete(db, pItem->pOn); @@ -94552,17 +96619,37 @@ SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pI assert( pIndexedBy!=0 ); if( p && ALWAYS(p->nSrc>0) ){ struct SrcList_item *pItem = &p->a[p->nSrc-1]; - assert( pItem->notIndexed==0 && pItem->zIndex==0 ); + assert( pItem->fg.notIndexed==0 ); + assert( pItem->fg.isIndexedBy==0 ); + assert( pItem->fg.isTabFunc==0 ); if( pIndexedBy->n==1 && !pIndexedBy->z ){ /* A "NOT INDEXED" clause was supplied. See parse.y ** construct "indexed_opt" for details. */ - pItem->notIndexed = 1; + pItem->fg.notIndexed = 1; }else{ - pItem->zIndex = sqlite3NameFromToken(pParse->db, pIndexedBy); + pItem->u1.zIndexedBy = sqlite3NameFromToken(pParse->db, pIndexedBy); + pItem->fg.isIndexedBy = (pItem->u1.zIndexedBy!=0); } } } +/* +** Add the list of function arguments to the SrcList entry for a +** table-valued-function. +*/ +SQLITE_PRIVATE void sqlite3SrcListFuncArgs(Parse *pParse, SrcList *p, ExprList *pList){ + if( p && pList ){ + struct SrcList_item *pItem = &p->a[p->nSrc-1]; + assert( pItem->fg.notIndexed==0 ); + assert( pItem->fg.isIndexedBy==0 ); + assert( pItem->fg.isTabFunc==0 ); + pItem->u1.pFuncArg = pList; + pItem->fg.isTabFunc = 1; + }else{ + sqlite3ExprListDelete(pParse->db, pList); + } +} + /* ** When building up a FROM clause in the parser, the join operator ** is initially attached to the left operand. But the code generator @@ -94582,9 +96669,9 @@ SQLITE_PRIVATE void sqlite3SrcListShiftJoinType(SrcList *p){ if( p ){ int i; for(i=p->nSrc-1; i>0; i--){ - p->a[i].jointype = p->a[i-1].jointype; + p->a[i].fg.jointype = p->a[i-1].fg.jointype; } - p->a[0].jointype = 0; + p->a[0].fg.jointype = 0; } } @@ -94828,12 +96915,16 @@ SQLITE_PRIVATE void sqlite3UniqueConstraint( Table *pTab = pIdx->pTable; sqlite3StrAccumInit(&errMsg, pParse->db, 0, 0, 200); - for(j=0; jnKeyCol; j++){ - char *zCol = pTab->aCol[pIdx->aiColumn[j]].zName; - if( j ) sqlite3StrAccumAppend(&errMsg, ", ", 2); - sqlite3StrAccumAppendAll(&errMsg, pTab->zName); - sqlite3StrAccumAppend(&errMsg, ".", 1); - sqlite3StrAccumAppendAll(&errMsg, zCol); + if( pIdx->aColExpr ){ + sqlite3XPrintf(&errMsg, 0, "index '%q'", pIdx->zName); + }else{ + for(j=0; jnKeyCol; j++){ + char *zCol; + assert( pIdx->aiColumn[j]>=0 ); + zCol = pTab->aCol[pIdx->aiColumn[j]].zName; + if( j ) sqlite3StrAccumAppend(&errMsg, ", ", 2); + sqlite3XPrintf(&errMsg, 0, "%s.%s", pTab->zName, zCol); + } } zErr = sqlite3StrAccumFinish(&errMsg); sqlite3HaltConstraint(pParse, @@ -95078,7 +97169,7 @@ SQLITE_PRIVATE With *sqlite3WithAdd( pNew->a[pNew->nCte].pSelect = pQuery; pNew->a[pNew->nCte].pCols = pArglist; pNew->a[pNew->nCte].zName = zName; - pNew->a[pNew->nCte].zErr = 0; + pNew->a[pNew->nCte].zCteErr = 0; pNew->nCte++; } @@ -95120,6 +97211,7 @@ SQLITE_PRIVATE void sqlite3WithDelete(sqlite3 *db, With *pWith){ ** of user defined functions and collation sequences. */ +/* #include "sqliteInt.h" */ /* ** Invoke the 'collation needed' callback to request a collation sequence @@ -95597,6 +97689,7 @@ SQLITE_PRIVATE Schema *sqlite3SchemaGet(sqlite3 *db, Btree *pBt){ ** This file contains C code routines that are called by the parser ** in order to generate code for DELETE FROM statements. */ +/* #include "sqliteInt.h" */ /* ** While a SrcList can in general represent multiple tables and subqueries @@ -95819,7 +97912,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( int iDb; /* Database number */ int memCnt = -1; /* Memory cell used for change counting */ int rcauth; /* Value returned by authorization callback */ - int okOnePass; /* True for one-pass algorithm without the FIFO */ + int eOnePass; /* ONEPASS_OFF or _SINGLE or _MULTI */ int aiCurOnePass[2]; /* The write cursors opened by WHERE_ONEPASS */ u8 *aToOpen = 0; /* Open cursor iTabCur+j if aToOpen[j] is true */ Index *pPk; /* The PRIMARY KEY index on the table */ @@ -95831,12 +97924,12 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( int iRowSet = 0; /* Register for rowset of rows to delete */ int addrBypass = 0; /* Address of jump over the delete logic */ int addrLoop = 0; /* Top of the delete loop */ - int addrDelete = 0; /* Jump directly to the delete logic */ int addrEphOpen = 0; /* Instruction to open the Ephemeral table */ #ifndef SQLITE_OMIT_TRIGGER int isView; /* True if attempting to delete from a view */ Trigger *pTrigger; /* List of table triggers, if required */ + int bComplex; /* True if there are either triggers or FKs */ #endif memset(&sContext, 0, sizeof(sContext)); @@ -95860,9 +97953,11 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( #ifndef SQLITE_OMIT_TRIGGER pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); isView = pTab->pSelect!=0; + bComplex = pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0); #else # define pTrigger 0 # define isView 0 +# define bComplex 0 #endif #ifdef SQLITE_OMIT_VIEW # undef isView @@ -95943,8 +98038,10 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( ** It is easier just to erase the whole table. Prior to version 3.6.5, ** this optimization caused the row change count (the value returned by ** API function sqlite3_count_changes) to be set incorrectly. */ - if( rcauth==SQLITE_OK && pWhere==0 && !pTrigger && !IsVirtual(pTab) - && 0==sqlite3FkRequired(pParse, pTab, 0, 0) + if( rcauth==SQLITE_OK + && pWhere==0 + && !bComplex + && !IsVirtual(pTab) ){ assert( !isView ); sqlite3TableLock(pParse, iDb, pTab->tnum, 1, pTab->zName); @@ -95959,6 +98056,8 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( }else #endif /* SQLITE_OMIT_TRUNCATE_OPTIMIZATION */ { + u16 wcf = WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK; + wcf |= (bComplex ? 0 : WHERE_ONEPASS_MULTIROW); if( HasRowid(pTab) ){ /* For a rowid table, initialize the RowSet to an empty set */ pPk = 0; @@ -95979,13 +98078,18 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( } /* Construct a query to find the rowid or primary key for every row - ** to be deleted, based on the WHERE clause. + ** to be deleted, based on the WHERE clause. Set variable eOnePass + ** to indicate the strategy used to implement this delete: + ** + ** ONEPASS_OFF: Two-pass approach - use a FIFO for rowids/PK values. + ** ONEPASS_SINGLE: One-pass approach - at most one row deleted. + ** ONEPASS_MULTI: One-pass approach - any number of rows may be deleted. */ - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, - WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK, - iTabCur+1); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, wcf, iTabCur+1); if( pWInfo==0 ) goto delete_from_cleanup; - okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + assert( IsVirtual(pTab)==0 || eOnePass!=ONEPASS_MULTI ); + assert( IsVirtual(pTab) || bComplex || eOnePass!=ONEPASS_OFF ); /* Keep track of the number of rows to be deleted */ if( db->flags & SQLITE_CountRows ){ @@ -95995,6 +98099,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( /* Extract the rowid or primary key for the current row */ if( pPk ){ for(i=0; iaiColumn[i]>=0 ); sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, pPk->aiColumn[i], iPk+i); } @@ -96005,11 +98110,10 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( if( iKey>pParse->nMem ) pParse->nMem = iKey; } - if( okOnePass ){ - /* For ONEPASS, no need to store the rowid/primary-key. There is only + if( eOnePass!=ONEPASS_OFF ){ + /* For ONEPASS, no need to store the rowid/primary-key. There is only ** one, so just keep it in its register(s) and fall through to the - ** delete code. - */ + ** delete code. */ nKey = nPk; /* OP_Found will use an unpacked key */ aToOpen = sqlite3DbMallocRaw(db, nIdx+2); if( aToOpen==0 ){ @@ -96021,27 +98125,27 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iTabCur] = 0; if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iTabCur] = 0; if( addrEphOpen ) sqlite3VdbeChangeToNoop(v, addrEphOpen); - addrDelete = sqlite3VdbeAddOp0(v, OP_Goto); /* Jump to DELETE logic */ - }else if( pPk ){ - /* Construct a composite key for the row to be deleted and remember it */ - iKey = ++pParse->nMem; - nKey = 0; /* Zero tells OP_Found to use a composite key */ - sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, iKey, - sqlite3IndexAffinityStr(v, pPk), nPk); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iEphCur, iKey); }else{ - /* Get the rowid of the row to be deleted and remember it in the RowSet */ - nKey = 1; /* OP_Seek always uses a single rowid */ - sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, iKey); + if( pPk ){ + /* Add the PK key for this row to the temporary table */ + iKey = ++pParse->nMem; + nKey = 0; /* Zero tells OP_Found to use a composite key */ + sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, iKey, + sqlite3IndexAffinityStr(pParse->db, pPk), nPk); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iEphCur, iKey); + }else{ + /* Add the rowid of the row to be deleted to the RowSet */ + nKey = 1; /* OP_Seek always uses a single rowid */ + sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, iKey); + } } - /* End of the WHERE loop */ - sqlite3WhereEnd(pWInfo); - if( okOnePass ){ - /* Bypass the delete logic below if the WHERE loop found zero rows */ + /* If this DELETE cannot use the ONEPASS strategy, this is the + ** end of the WHERE loop */ + if( eOnePass!=ONEPASS_OFF ){ addrBypass = sqlite3VdbeMakeLabel(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrBypass); - sqlite3VdbeJumpHere(v, addrDelete); + }else{ + sqlite3WhereEnd(pWInfo); } /* Unless this is a view, open cursors for the table we are @@ -96050,21 +98154,24 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( ** triggers. */ if( !isView ){ + int iAddrOnce = 0; + if( eOnePass==ONEPASS_MULTI ){ + iAddrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v); + } testcase( IsVirtual(pTab) ); sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, iTabCur, aToOpen, &iDataCur, &iIdxCur); assert( pPk || IsVirtual(pTab) || iDataCur==iTabCur ); assert( pPk || IsVirtual(pTab) || iIdxCur==iDataCur+1 ); + if( eOnePass==ONEPASS_MULTI ) sqlite3VdbeJumpHere(v, iAddrOnce); } /* Set up a loop over the rowids/primary-keys that were found in the ** where-clause loop above. */ - if( okOnePass ){ - /* Just one row. Hence the top-of-loop is a no-op */ + if( eOnePass!=ONEPASS_OFF ){ assert( nKey==nPk ); /* OP_Found will use an unpacked key */ - assert( !IsVirtual(pTab) ); - if( aToOpen[iDataCur-iTabCur] ){ + if( !IsVirtual(pTab) && aToOpen[iDataCur-iTabCur] ){ assert( pPk!=0 || pTab->pSelect!=0 ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, addrBypass, iKey, nKey); VdbeCoverage(v); @@ -96086,23 +98193,32 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( sqlite3VtabMakeWritable(pParse, pTab); sqlite3VdbeAddOp4(v, OP_VUpdate, 0, 1, iKey, pVTab, P4_VTAB); sqlite3VdbeChangeP5(v, OE_Abort); + assert( eOnePass==ONEPASS_OFF || eOnePass==ONEPASS_SINGLE ); sqlite3MayAbort(pParse); + if( eOnePass==ONEPASS_SINGLE && sqlite3IsToplevel(pParse) ){ + pParse->isMultiWrite = 0; + } }else #endif { int count = (pParse->nested==0); /* True to count changes */ + int iIdxNoSeek = -1; + if( bComplex==0 && aiCurOnePass[1]!=iDataCur ){ + iIdxNoSeek = aiCurOnePass[1]; + } sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - iKey, nKey, count, OE_Default, okOnePass); + iKey, nKey, count, OE_Default, eOnePass, iIdxNoSeek); } /* End of the loop over all rowids/primary-keys. */ - if( okOnePass ){ + if( eOnePass!=ONEPASS_OFF ){ sqlite3VdbeResolveLabel(v, addrBypass); + sqlite3WhereEnd(pWInfo); }else if( pPk ){ sqlite3VdbeAddOp2(v, OP_Next, iEphCur, addrLoop+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addrLoop); }else{ - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrLoop); + sqlite3VdbeGoto(v, addrLoop); sqlite3VdbeJumpHere(v, addrLoop); } @@ -96169,6 +98285,25 @@ delete_from_cleanup: ** sequence of nPk memory cells starting at iPk. If nPk==0 that means ** that a search record formed from OP_MakeRecord is contained in the ** single memory location iPk. +** +** eMode: +** Parameter eMode may be passed either ONEPASS_OFF (0), ONEPASS_SINGLE, or +** ONEPASS_MULTI. If eMode is not ONEPASS_OFF, then the cursor +** iDataCur already points to the row to delete. If eMode is ONEPASS_OFF +** then this function must seek iDataCur to the entry identified by iPk +** and nPk before reading from it. +** +** If eMode is ONEPASS_MULTI, then this call is being made as part +** of a ONEPASS delete that affects multiple rows. In this case, if +** iIdxNoSeek is a valid cursor number (>=0), then its position should +** be preserved following the delete operation. Or, if iIdxNoSeek is not +** a valid cursor number, the position of iDataCur should be preserved +** instead. +** +** iIdxNoSeek: +** If iIdxNoSeek is a valid cursor number (>=0), then it identifies an +** index cursor (from within array of cursors starting at iIdxCur) that +** already points to the index entry to be deleted. */ SQLITE_PRIVATE void sqlite3GenerateRowDelete( Parse *pParse, /* Parsing context */ @@ -96180,7 +98315,8 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( i16 nPk, /* Number of PRIMARY KEY memory cells */ u8 count, /* If non-zero, increment the row change counter */ u8 onconf, /* Default ON CONFLICT policy for triggers */ - u8 bNoSeek /* iDataCur is already pointing to the row to delete */ + u8 eMode, /* ONEPASS_OFF, _SINGLE, or _MULTI. See above */ + int iIdxNoSeek /* Cursor number of cursor that does not need seeking */ ){ Vdbe *v = pParse->pVdbe; /* Vdbe */ int iOld = 0; /* First register in OLD.* array */ @@ -96197,7 +98333,7 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( ** not attempt to delete it or fire any DELETE triggers. */ iLabel = sqlite3VdbeMakeLabel(v); opSeek = HasRowid(pTab) ? OP_NotExists : OP_NotFound; - if( !bNoSeek ){ + if( eMode==ONEPASS_OFF ){ sqlite3VdbeAddOp4Int(v, opSeek, iDataCur, iLabel, iPk, nPk); VdbeCoverageIf(v, opSeek==OP_NotExists); VdbeCoverageIf(v, opSeek==OP_NotFound); @@ -96257,11 +98393,15 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( ** a view (in which case the only effect of the DELETE statement is to ** fire the INSTEAD OF triggers). */ if( pTab->pSelect==0 ){ - sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, 0); + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,iIdxNoSeek); sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, (count?OPFLAG_NCHANGE:0)); if( count ){ sqlite3VdbeChangeP4(v, -1, pTab->zName, P4_TRANSIENT); } + if( iIdxNoSeek>=0 ){ + sqlite3VdbeAddOp1(v, OP_Delete, iIdxNoSeek); + } + sqlite3VdbeChangeP5(v, eMode==ONEPASS_MULTI); } /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to @@ -96304,7 +98444,8 @@ SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete( Table *pTab, /* Table containing the row to be deleted */ int iDataCur, /* Cursor of table holding data. */ int iIdxCur, /* First index cursor */ - int *aRegIdx /* Only delete if aRegIdx!=0 && aRegIdx[i]>0 */ + int *aRegIdx, /* Only delete if aRegIdx!=0 && aRegIdx[i]>0 */ + int iIdxNoSeek /* Do not delete from this cursor */ ){ int i; /* Index loop counter */ int r1 = -1; /* Register holding an index key */ @@ -96320,11 +98461,12 @@ SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete( assert( iIdxCur+i!=iDataCur || pPk==pIdx ); if( aRegIdx!=0 && aRegIdx[i]==0 ) continue; if( pIdx==pPk ) continue; + if( iIdxCur+i==iIdxNoSeek ) continue; VdbeModuleComment((v, "GenRowIdxDel for %s", pIdx->zName)); r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 1, - &iPartIdxLabel, pPrior, r1); + &iPartIdxLabel, pPrior, r1); sqlite3VdbeAddOp3(v, OP_IdxDelete, iIdxCur+i, r1, - pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); + pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); sqlite3ResolvePartIdxLabel(pParse, iPartIdxLabel); pPrior = pIdx; } @@ -96373,17 +98515,16 @@ SQLITE_PRIVATE int sqlite3GenerateIndexKey( ){ Vdbe *v = pParse->pVdbe; int j; - Table *pTab = pIdx->pTable; int regBase; int nCol; if( piPartIdxLabel ){ if( pIdx->pPartIdxWhere ){ *piPartIdxLabel = sqlite3VdbeMakeLabel(v); - pParse->iPartIdxTab = iDataCur; + pParse->iSelfTab = iDataCur; sqlite3ExprCachePush(pParse); - sqlite3ExprIfFalse(pParse, pIdx->pPartIdxWhere, *piPartIdxLabel, - SQLITE_JUMPIFNULL); + sqlite3ExprIfFalseDup(pParse, pIdx->pPartIdxWhere, *piPartIdxLabel, + SQLITE_JUMPIFNULL); }else{ *piPartIdxLabel = 0; } @@ -96392,9 +98533,14 @@ SQLITE_PRIVATE int sqlite3GenerateIndexKey( regBase = sqlite3GetTempRange(pParse, nCol); if( pPrior && (regBase!=regPrior || pPrior->pPartIdxWhere) ) pPrior = 0; for(j=0; jaiColumn[j]==pIdx->aiColumn[j] ) continue; - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, pIdx->aiColumn[j], - regBase+j); + if( pPrior + && pPrior->aiColumn[j]==pIdx->aiColumn[j] + && pPrior->aiColumn[j]!=XN_EXPR + ){ + /* This column was already computed by the previous index */ + continue; + } + sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iDataCur, j, regBase+j); /* If the column affinity is REAL but the number is an integer, then it ** might be stored in the table as an integer (using a compact ** representation) then converted to REAL by an OP_RealAffinity opcode. @@ -96439,8 +98585,10 @@ SQLITE_PRIVATE void sqlite3ResolvePartIdxLabel(Parse *pParse, int iLabel){ ** functions of SQLite. (Some function, and in particular the date and ** time functions, are implemented separately.) */ +/* #include "sqliteInt.h" */ /* #include */ /* #include */ +/* #include "vdbeInt.h" */ /* ** Return the collating function associated with a function. @@ -96999,17 +99147,15 @@ struct compareInfo { /* ** For LIKE and GLOB matching on EBCDIC machines, assume that every -** character is exactly one byte in size. Also, all characters are -** able to participate in upper-case-to-lower-case mappings in EBCDIC -** whereas only characters less than 0x80 do in ASCII. +** character is exactly one byte in size. Also, provde the Utf8Read() +** macro for fast reading of the next character in the common case where +** the next character is ASCII. */ #if defined(SQLITE_EBCDIC) # define sqlite3Utf8Read(A) (*((*A)++)) -# define GlobUpperToLower(A) A = sqlite3UpperToLower[A] -# define GlobUpperToLowerAscii(A) A = sqlite3UpperToLower[A] +# define Utf8Read(A) (*(A++)) #else -# define GlobUpperToLower(A) if( A<=0x7f ){ A = sqlite3UpperToLower[A]; } -# define GlobUpperToLowerAscii(A) A = sqlite3UpperToLower[A] +# define Utf8Read(A) (A[0]<0x80?*(A++):sqlite3Utf8Read(&A)) #endif static const struct compareInfo globInfo = { '*', '?', '[', 0 }; @@ -97051,7 +99197,7 @@ static const struct compareInfo likeInfoAlt = { '%', '_', 0, 0 }; ** Ec Where E is the "esc" character and c is any other ** character, including '%', '_', and esc, match exactly c. ** -** The comments through this routine usually assume glob matching. +** The comments within this routine usually assume glob matching. ** ** This routine is usually quick, but can be N**2 in the worst case. */ @@ -97075,13 +99221,12 @@ static int patternCompare( */ matchOther = esc ? esc : pInfo->matchSet; - while( (c = sqlite3Utf8Read(&zPattern))!=0 ){ + while( (c = Utf8Read(zPattern))!=0 ){ if( c==matchAll ){ /* Match "*" */ /* Skip over multiple "*" characters in the pattern. If there ** are also "?" characters, skip those as well, but consume a ** single character of the input string for each "?" skipped */ - while( (c=sqlite3Utf8Read(&zPattern)) == matchAll - || c == matchOne ){ + while( (c=Utf8Read(zPattern)) == matchAll || c == matchOne ){ if( c==matchOne && sqlite3Utf8Read(&zString)==0 ){ return 0; } @@ -97126,7 +99271,7 @@ static int patternCompare( if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; } }else{ - while( (c2 = sqlite3Utf8Read(&zString))!=0 ){ + while( (c2 = Utf8Read(zString))!=0 ){ if( c2!=c ) continue; if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; } @@ -97172,7 +99317,7 @@ static int patternCompare( continue; } } - c2 = sqlite3Utf8Read(&zString); + c2 = Utf8Read(zString); if( c==c2 ) continue; if( noCase && c<0x80 && c2<0x80 && sqlite3Tolower(c)==sqlite3Tolower(c2) ){ continue; @@ -97549,16 +99694,14 @@ static void zeroblobFunc( sqlite3_value **argv ){ i64 n; - sqlite3 *db = sqlite3_context_db_handle(context); + int rc; assert( argc==1 ); UNUSED_PARAMETER(argc); n = sqlite3_value_int64(argv[0]); - testcase( n==db->aLimit[SQLITE_LIMIT_LENGTH] ); - testcase( n==db->aLimit[SQLITE_LIMIT_LENGTH]+1 ); - if( n>db->aLimit[SQLITE_LIMIT_LENGTH] ){ - sqlite3_result_error_toobig(context); - }else{ - sqlite3_result_zeroblob(context, (int)n); /* IMP: R-00293-64994 */ + if( n<0 ) n = 0; + rc = sqlite3_result_zeroblob64(context, n); /* IMP: R-00293-64994 */ + if( rc ){ + sqlite3_result_error_code(context, rc); } } @@ -98166,15 +100309,15 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ VFUNCTION(random, 0, 0, 0, randomFunc ), VFUNCTION(randomblob, 1, 0, 0, randomBlob ), FUNCTION(nullif, 2, 0, 1, nullifFunc ), - FUNCTION(sqlite_version, 0, 0, 0, versionFunc ), - FUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), + DFUNCTION(sqlite_version, 0, 0, 0, versionFunc ), + DFUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), FUNCTION(sqlite_log, 2, 0, 0, errlogFunc ), #if SQLITE_USER_AUTHENTICATION FUNCTION(sqlite_crypt, 2, 0, 0, sqlite3CryptFunc ), #endif #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS - FUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), - FUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), + DFUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), + DFUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), #endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ FUNCTION(quote, 1, 0, 0, quoteFunc ), VFUNCTION(last_insert_rowid, 0, 0, 0, last_insert_rowid), @@ -98186,8 +100329,8 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ FUNCTION(soundex, 1, 0, 0, soundexFunc ), #endif #ifndef SQLITE_OMIT_LOAD_EXTENSION - FUNCTION(load_extension, 1, 0, 0, loadExt ), - FUNCTION(load_extension, 2, 0, 0, loadExt ), + VFUNCTION(load_extension, 1, 0, 0, loadExt ), + VFUNCTION(load_extension, 2, 0, 0, loadExt ), #endif AGGREGATE(sum, 1, 0, 0, sumStep, sumFinalize ), AGGREGATE(total, 1, 0, 0, sumStep, totalFinalize ), @@ -98239,6 +100382,7 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ ** This file contains code used by the compiler to add foreign key ** support to compiled SQL statements. */ +/* #include "sqliteInt.h" */ #ifndef SQLITE_OMIT_FOREIGN_KEY #ifndef SQLITE_OMIT_TRIGGER @@ -98479,6 +100623,8 @@ SQLITE_PRIVATE int sqlite3FkLocateIndex( char *zDfltColl; /* Def. collation for column */ char *zIdxCol; /* Name of indexed column */ + if( iCol<0 ) break; /* No foreign keys against expression indexes */ + /* If the index uses a collation sequence that is different from ** the default collation sequence for the column, this index is ** unusable. Bail out early in this case. */ @@ -98601,7 +100747,7 @@ static void fkLookupParent( sqlite3OpenTable(pParse, iCur, iDb, pTab, OP_OpenRead); sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, regTemp); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk); + sqlite3VdbeGoto(v, iOk); sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); sqlite3VdbeJumpHere(v, iMustBeInt); sqlite3ReleaseTempReg(pParse, regTemp); @@ -98631,6 +100777,7 @@ static void fkLookupParent( for(i=0; iaiColumn[i]+1+regData; + assert( pIdx->aiColumn[i]>=0 ); assert( aiCol[i]!=pTab->iPKey ); if( pIdx->aiColumn[i]==pTab->iPKey ){ /* The parent key is a composite key that includes the IPK column */ @@ -98639,11 +100786,11 @@ static void fkLookupParent( sqlite3VdbeAddOp3(v, OP_Ne, iChild, iJump, iParent); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); } - sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk); + sqlite3VdbeGoto(v, iOk); } sqlite3VdbeAddOp4(v, OP_MakeRecord, regTemp, nCol, regRec, - sqlite3IndexAffinityStr(v,pIdx), nCol); + sqlite3IndexAffinityStr(pParse->db,pIdx), nCol); sqlite3VdbeAddOp4Int(v, OP_Found, iCur, iOk, regRec, 0); VdbeCoverage(v); sqlite3ReleaseTempReg(pParse, regRec); @@ -98839,6 +100986,7 @@ static void fkScanChildren( assert( pIdx!=0 ); for(i=0; inKeyCol; i++){ i16 iCol = pIdx->aiColumn[i]; + assert( iCol>=0 ); pLeft = exprTableRegister(pParse, pTab, regData, iCol); pRight = exprTableColumn(db, pTab, pSrc->a[0].iCursor, iCol); pEq = sqlite3PExpr(pParse, TK_EQ, pLeft, pRight, 0); @@ -99158,6 +101306,7 @@ SQLITE_PRIVATE void sqlite3FkCheck( if( aiCol[i]==pTab->iPKey ){ aiCol[i] = -1; } + assert( pIdx==0 || pIdx->aiColumn[i]>=0 ); #ifndef SQLITE_OMIT_AUTHORIZATION /* Request permission to read the parent key columns. If the ** authorization callback returns SQLITE_IGNORE, behave as if any @@ -99289,7 +101438,10 @@ SQLITE_PRIVATE u32 sqlite3FkOldmask( Index *pIdx = 0; sqlite3FkLocateIndex(pParse, pTab, p, &pIdx, 0); if( pIdx ){ - for(i=0; inKeyCol; i++) mask |= COLUMN_MASK(pIdx->aiColumn[i]); + for(i=0; inKeyCol; i++){ + assert( pIdx->aiColumn[i]>=0 ); + mask |= COLUMN_MASK(pIdx->aiColumn[i]); + } } } } @@ -99412,6 +101564,7 @@ static Trigger *fkActionTrigger( iFromCol = aiCol ? aiCol[i] : pFKey->aCol[0].iFrom; assert( iFromCol>=0 ); assert( pIdx!=0 || (pTab->iPKey>=0 && pTab->iPKeynCol) ); + assert( pIdx==0 || pIdx->aiColumn[i]>=0 ); tToCol.z = pTab->aCol[pIdx ? pIdx->aiColumn[i] : pTab->iPKey].zName; tFromCol.z = pFKey->pFrom->aCol[iFromCol].zName; @@ -99643,6 +101796,7 @@ SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *db, Table *pTab){ ** This file contains C code routines that are called by the parser ** to handle INSERT statements in SQLite. */ +/* #include "sqliteInt.h" */ /* ** Generate code that will @@ -99672,7 +101826,7 @@ SQLITE_PRIVATE void sqlite3OpenTable( }else{ Index *pPk = sqlite3PrimaryKeyIndex(pTab); assert( pPk!=0 ); - assert( pPk->tnum=pTab->tnum ); + assert( pPk->tnum==pTab->tnum ); sqlite3VdbeAddOp3(v, opcode, iCur, pPk->tnum, iDb); sqlite3VdbeSetP4KeyInfo(pParse, pPk); VdbeComment((v, "%s", pTab->zName)); @@ -99686,7 +101840,7 @@ SQLITE_PRIVATE void sqlite3OpenTable( ** ** Character Column affinity ** ------------------------------ -** 'A' NONE +** 'A' BLOB ** 'B' TEXT ** 'C' NUMERIC ** 'D' INTEGER @@ -99699,7 +101853,7 @@ SQLITE_PRIVATE void sqlite3OpenTable( ** is managed along with the rest of the Index structure. It will be ** released when sqlite3DeleteIndex() is called. */ -SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ +SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ if( !pIdx->zColAff ){ /* The first time a column affinity string for a particular index is ** required, it is allocated and populated here. It is then stored as @@ -99711,7 +101865,6 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ */ int n; Table *pTab = pIdx->pTable; - sqlite3 *db = sqlite3VdbeDb(v); pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+1); if( !pIdx->zColAff ){ db->mallocFailed = 1; @@ -99719,7 +101872,18 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ } for(n=0; nnColumn; n++){ i16 x = pIdx->aiColumn[n]; - pIdx->zColAff[n] = x<0 ? SQLITE_AFF_INTEGER : pTab->aCol[x].affinity; + if( x>=0 ){ + pIdx->zColAff[n] = pTab->aCol[x].affinity; + }else if( x==XN_ROWID ){ + pIdx->zColAff[n] = SQLITE_AFF_INTEGER; + }else{ + char aff; + assert( x==XN_EXPR ); + assert( pIdx->aColExpr!=0 ); + aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); + if( aff==0 ) aff = SQLITE_AFF_BLOB; + pIdx->zColAff[n] = aff; + } } pIdx->zColAff[n] = 0; } @@ -99729,9 +101893,9 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ /* ** Compute the affinity string for table pTab, if it has not already been -** computed. As an optimization, omit trailing SQLITE_AFF_NONE affinities. +** computed. As an optimization, omit trailing SQLITE_AFF_BLOB affinities. ** -** If the affinity exists (if it is no entirely SQLITE_AFF_NONE values) and +** If the affinity exists (if it is no entirely SQLITE_AFF_BLOB values) and ** if iReg>0 then code an OP_Affinity opcode that will set the affinities ** for register iReg and following. Or if affinities exists and iReg==0, ** then just set the P4 operand of the previous opcode (which should be @@ -99741,7 +101905,7 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ ** ** Character Column affinity ** ------------------------------ -** 'A' NONE +** 'A' BLOB ** 'B' TEXT ** 'C' NUMERIC ** 'D' INTEGER @@ -99763,7 +101927,7 @@ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ } do{ zColAff[i--] = 0; - }while( i>=0 && zColAff[i]==SQLITE_AFF_NONE ); + }while( i>=0 && zColAff[i]==SQLITE_AFF_BLOB ); pTab->zColAff = zColAff; } i = sqlite3Strlen30(zColAff); @@ -99880,7 +102044,7 @@ SQLITE_PRIVATE void sqlite3AutoincrementBegin(Parse *pParse){ /* This routine is never called during trigger-generation. It is ** only called from the top-level */ assert( pParse->pTriggerTab==0 ); - assert( pParse==sqlite3ParseToplevel(pParse) ); + assert( sqlite3IsToplevel(pParse) ); assert( v ); /* We failed long ago if this is not so */ for(p = pParse->pAinc; p; p = p->pNext){ @@ -99890,14 +102054,14 @@ SQLITE_PRIVATE void sqlite3AutoincrementBegin(Parse *pParse){ sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenRead); sqlite3VdbeAddOp3(v, OP_Null, 0, memId, memId+1); addr = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp4(v, OP_String8, 0, memId-1, 0, p->pTab->zName, 0); + sqlite3VdbeLoadString(v, memId-1, p->pTab->zName); sqlite3VdbeAddOp2(v, OP_Rewind, 0, addr+9); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Column, 0, 0, memId); sqlite3VdbeAddOp3(v, OP_Ne, memId-1, addr+7, memId); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); sqlite3VdbeAddOp2(v, OP_Rowid, 0, memId+1); sqlite3VdbeAddOp3(v, OP_Column, 0, 1, memId); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addr+9); + sqlite3VdbeGoto(v, addr+9); sqlite3VdbeAddOp2(v, OP_Next, 0, addr+2); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Integer, 0, memId); sqlite3VdbeAddOp0(v, OP_Close); @@ -99933,16 +102097,16 @@ SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse){ assert( v ); for(p = pParse->pAinc; p; p = p->pNext){ Db *pDb = &db->aDb[p->iDb]; - int j1; + int addr1; int iRec; int memId = p->regCtr; iRec = sqlite3GetTempReg(pParse); assert( sqlite3SchemaMutexHeld(db, 0, pDb->pSchema) ); sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenWrite); - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, memId+1); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, memId+1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_NewRowid, 0, memId+1); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp3(v, OP_MakeRecord, memId-1, 2, iRec); sqlite3VdbeAddOp3(v, OP_Insert, 0, iRec, memId+1); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); @@ -100321,7 +102485,7 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3VdbeAddOp3(v, OP_MakeRecord, regFromSelect, nColumn, regRec); sqlite3VdbeAddOp2(v, OP_NewRowid, srcTab, regTempRowid); sqlite3VdbeAddOp3(v, OP_Insert, srcTab, regRec, regTempRowid); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrL); + sqlite3VdbeGoto(v, addrL); sqlite3VdbeJumpHere(v, addrL); sqlite3ReleaseTempReg(pParse, regRec); sqlite3ReleaseTempReg(pParse, regTempRowid); @@ -100335,11 +102499,13 @@ SQLITE_PRIVATE void sqlite3Insert( sNC.pParse = pParse; srcTab = -1; assert( useTempTable==0 ); - nColumn = pList ? pList->nExpr : 0; - for(i=0; ia[i].pExpr) ){ + if( pList ){ + nColumn = pList->nExpr; + if( sqlite3ResolveExprListNames(&sNC, pList) ){ goto insert_cleanup; } + }else{ + nColumn = 0; } } @@ -100432,7 +102598,7 @@ SQLITE_PRIVATE void sqlite3Insert( if( ipkColumn<0 ){ sqlite3VdbeAddOp2(v, OP_Integer, -1, regCols); }else{ - int j1; + int addr1; assert( !withoutRowid ); if( useTempTable ){ sqlite3VdbeAddOp3(v, OP_Column, srcTab, ipkColumn, regCols); @@ -100440,9 +102606,9 @@ SQLITE_PRIVATE void sqlite3Insert( assert( pSelect==0 ); /* Otherwise useTempTable is true */ sqlite3ExprCode(pParse, pList->a[ipkColumn].pExpr, regCols); } - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regCols); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, regCols); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Integer, -1, regCols); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp1(v, OP_MustBeInt, regCols); VdbeCoverage(v); } @@ -100516,14 +102682,14 @@ SQLITE_PRIVATE void sqlite3Insert( ** to generate a unique primary key value. */ if( !appendFlag ){ - int j1; + int addr1; if( !IsVirtual(pTab) ){ - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regRowid); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, regRowid); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_NewRowid, iDataCur, regRowid, regAutoinc); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); }else{ - j1 = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_IsNull, regRowid, j1+2); VdbeCoverage(v); + addr1 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_IsNull, regRowid, addr1+2); VdbeCoverage(v); } sqlite3VdbeAddOp1(v, OP_MustBeInt, regRowid); VdbeCoverage(v); } @@ -100620,7 +102786,7 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3VdbeJumpHere(v, addrInsTop); sqlite3VdbeAddOp1(v, OP_Close, srcTab); }else if( pSelect ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrCont); + sqlite3VdbeGoto(v, addrCont); sqlite3VdbeJumpHere(v, addrInsTop); } @@ -100777,7 +102943,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( int ix; /* Index loop counter */ int nCol; /* Number of columns */ int onError; /* Conflict resolution strategy */ - int j1; /* Address of jump instruction */ + int addr1; /* Address of jump instruction */ int seenReplace = 0; /* True if REPLACE is used to resolve INT PK conflict */ int nPkField; /* Number of fields in PRIMARY KEY. 1 for ROWID tables */ int ipkTop = 0; /* Top of the rowid change constraint check */ @@ -100848,9 +103014,10 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( } default: { assert( onError==OE_Replace ); - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regNewData+1+i); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, regNewData+1+i); + VdbeCoverage(v); sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regNewData+1+i); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); break; } } @@ -100867,7 +103034,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( int allOk = sqlite3VdbeMakeLabel(v); sqlite3ExprIfTrue(pParse, pCheck->a[i].pExpr, allOk, SQLITE_JUMPIFNULL); if( onError==OE_Ignore ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); + sqlite3VdbeGoto(v, ignoreDest); }else{ char *zName = pCheck->a[i].zName; if( zName==0 ) zName = pTab->zName; @@ -100965,17 +103132,20 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( if( pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0) ){ sqlite3MultiWrite(pParse); sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - regNewData, 1, 0, OE_Replace, 1); - }else if( pTab->pIndex ){ - sqlite3MultiWrite(pParse); - sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, 0); + regNewData, 1, 0, OE_Replace, + ONEPASS_SINGLE, -1); + }else{ + if( pTab->pIndex ){ + sqlite3MultiWrite(pParse); + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,-1); + } } seenReplace = 1; break; } case OE_Ignore: { /*assert( seenReplace==0 );*/ - sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); + sqlite3VdbeGoto(v, ignoreDest); break; } } @@ -101011,8 +103181,8 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( if( pIdx->pPartIdxWhere ){ sqlite3VdbeAddOp2(v, OP_Null, 0, aRegIdx[ix]); pParse->ckBase = regNewData+1; - sqlite3ExprIfFalse(pParse, pIdx->pPartIdxWhere, addrUniqueOk, - SQLITE_JUMPIFNULL); + sqlite3ExprIfFalseDup(pParse, pIdx->pPartIdxWhere, addrUniqueOk, + SQLITE_JUMPIFNULL); pParse->ckBase = 0; } @@ -101023,15 +103193,22 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( for(i=0; inColumn; i++){ int iField = pIdx->aiColumn[i]; int x; - if( iField<0 || iField==pTab->iPKey ){ - if( regRowid==regIdx+i ) continue; /* ROWID already in regIdx+i */ - x = regNewData; - regRowid = pIdx->pPartIdxWhere ? -1 : regIdx+i; + if( iField==XN_EXPR ){ + pParse->ckBase = regNewData+1; + sqlite3ExprCode(pParse, pIdx->aColExpr->a[i].pExpr, regIdx+i); + pParse->ckBase = 0; + VdbeComment((v, "%s column %d", pIdx->zName, i)); }else{ - x = iField + regNewData + 1; + if( iField==XN_ROWID || iField==pTab->iPKey ){ + if( regRowid==regIdx+i ) continue; /* ROWID already in regIdx+i */ + x = regNewData; + regRowid = pIdx->pPartIdxWhere ? -1 : regIdx+i; + }else{ + x = iField + regNewData + 1; + } + sqlite3VdbeAddOp2(v, OP_SCopy, x, regIdx+i); + VdbeComment((v, "%s", iField<0 ? "rowid" : pTab->aCol[iField].zName)); } - sqlite3VdbeAddOp2(v, OP_SCopy, x, regIdx+i); - VdbeComment((v, "%s", iField<0 ? "rowid" : pTab->aCol[iField].zName)); } sqlite3VdbeAddOp3(v, OP_MakeRecord, regIdx, pIdx->nColumn, aRegIdx[ix]); VdbeComment((v, "for %s", pIdx->zName)); @@ -101081,6 +103258,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( ** store it in registers regR..regR+nPk-1 */ if( pIdx!=pPk ){ for(i=0; inKeyCol; i++){ + assert( pPk->aiColumn[i]>=0 ); x = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[i]); sqlite3VdbeAddOp3(v, OP_Column, iThisCur, x, regR+i); VdbeComment((v, "%s.%s", pTab->zName, @@ -101102,6 +103280,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( for(i=0; inKeyCol; i++){ char *p4 = (char*)sqlite3LocateCollSeq(pParse, pPk->azColl[i]); x = pPk->aiColumn[i]; + assert( x>=0 ); if( i==(pPk->nKeyCol-1) ){ addrJump = addrUniqueOk; op = OP_Eq; @@ -101128,7 +103307,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( break; } case OE_Ignore: { - sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); + sqlite3VdbeGoto(v, ignoreDest); break; } default: { @@ -101139,7 +103318,8 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); } sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - regR, nPkField, 0, OE_Replace, pIdx==pPk); + regR, nPkField, 0, OE_Replace, + (pIdx==pPk ? ONEPASS_SINGLE : ONEPASS_OFF), -1); seenReplace = 1; break; } @@ -101149,7 +103329,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( if( regR!=regIdx ) sqlite3ReleaseTempRange(pParse, regR, nPkField); } if( ipkTop ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, ipkTop+1); + sqlite3VdbeGoto(v, ipkTop+1); sqlite3VdbeJumpHere(v, ipkBottom); } @@ -101352,6 +103532,13 @@ static int xferCompatibleIndex(Index *pDest, Index *pSrc){ if( pSrc->aiColumn[i]!=pDest->aiColumn[i] ){ return 0; /* Different columns indexed */ } + if( pSrc->aiColumn[i]==XN_EXPR ){ + assert( pSrc->aColExpr!=0 && pDest->aColExpr!=0 ); + if( sqlite3ExprCompare(pSrc->aColExpr->a[i].pExpr, + pDest->aColExpr->a[i].pExpr, -1)!=0 ){ + return 0; /* Different expressions in the index */ + } + } if( pSrc->aSortOrder[i]!=pDest->aSortOrder[i] ){ return 0; /* Different sort orders */ } @@ -101595,7 +103782,7 @@ static int xferOptimization( ** (3) onError is something other than OE_Abort and OE_Rollback. */ addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iDest, 0); VdbeCoverage(v); - emptyDestTest = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); + emptyDestTest = sqlite3VdbeAddOp0(v, OP_Goto); sqlite3VdbeJumpHere(v, addr1); } if( HasRowid(pSrc) ){ @@ -101626,7 +103813,7 @@ static int xferOptimization( sqlite3TableLock(pParse, iDbSrc, pSrc->tnum, 0, pSrc->zName); } for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ - u8 useSeekResult = 0; + u8 idxInsFlags = 0; for(pSrcIdx=pSrc->pIndex; ALWAYS(pSrcIdx); pSrcIdx=pSrcIdx->pNext){ if( xferCompatibleIndex(pDestIdx, pSrcIdx) ) break; } @@ -101661,12 +103848,15 @@ static int xferOptimization( if( sqlite3_stricmp("BINARY", zColl) ) break; } if( i==pSrcIdx->nColumn ){ - useSeekResult = OPFLAG_USESEEKRESULT; + idxInsFlags = OPFLAG_USESEEKRESULT; sqlite3VdbeAddOp3(v, OP_Last, iDest, 0, -1); } } + if( !HasRowid(pSrc) && pDestIdx->idxType==2 ){ + idxInsFlags |= OPFLAG_NCHANGE; + } sqlite3VdbeAddOp3(v, OP_IdxInsert, iDest, regData, 1); - sqlite3VdbeChangeP5(v, useSeekResult); + sqlite3VdbeChangeP5(v, idxInsFlags); sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); @@ -101705,6 +103895,7 @@ static int xferOptimization( ** accessed by users of the library. */ +/* #include "sqliteInt.h" */ /* ** Execute SQL code. Return one of the SQLITE_ success/failure @@ -101873,6 +104064,7 @@ exec_out: */ #ifndef _SQLITE3EXT_H_ #define _SQLITE3EXT_H_ +/* #include "sqlite3.h" */ typedef struct sqlite3_api_routines sqlite3_api_routines; @@ -102122,6 +104314,14 @@ struct sqlite3_api_routines { void (*result_text64)(sqlite3_context*,const char*,sqlite3_uint64, void(*)(void*), unsigned char); int (*strglob)(const char*,const char*); + /* Version 3.8.11 and later */ + sqlite3_value *(*value_dup)(const sqlite3_value*); + void (*value_free)(sqlite3_value*); + int (*result_zeroblob64)(sqlite3_context*,sqlite3_uint64); + int (*bind_zeroblob64)(sqlite3_stmt*, int, sqlite3_uint64); + /* Version 3.9.0 and later */ + unsigned int (*value_subtype)(sqlite3_value*); + void (*result_subtype)(sqlite3_context*,unsigned int); }; /* @@ -102135,7 +104335,7 @@ struct sqlite3_api_routines { ** the API. So the redefinition macros are only valid if the ** SQLITE_CORE macros is undefined. */ -#ifndef SQLITE_CORE +#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) #define sqlite3_aggregate_context sqlite3_api->aggregate_context #ifndef SQLITE_OMIT_DEPRECATED #define sqlite3_aggregate_count sqlite3_api->aggregate_count @@ -102262,6 +104462,7 @@ struct sqlite3_api_routines { #define sqlite3_value_text16le sqlite3_api->value_text16le #define sqlite3_value_type sqlite3_api->value_type #define sqlite3_vmprintf sqlite3_api->vmprintf +#define sqlite3_vsnprintf sqlite3_api->vsnprintf #define sqlite3_overload_function sqlite3_api->overload_function #define sqlite3_prepare_v2 sqlite3_api->prepare_v2 #define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2 @@ -102352,9 +104553,17 @@ struct sqlite3_api_routines { #define sqlite3_result_blob64 sqlite3_api->result_blob64 #define sqlite3_result_text64 sqlite3_api->result_text64 #define sqlite3_strglob sqlite3_api->strglob -#endif /* SQLITE_CORE */ - -#ifndef SQLITE_CORE +/* Version 3.8.11 and later */ +#define sqlite3_value_dup sqlite3_api->value_dup +#define sqlite3_value_free sqlite3_api->value_free +#define sqlite3_result_zeroblob64 sqlite3_api->result_zeroblob64 +#define sqlite3_bind_zeroblob64 sqlite3_api->bind_zeroblob64 +/* Version 3.9.0 and later */ +#define sqlite3_value_subtype sqlite3_api->value_subtype +#define sqlite3_result_subtype sqlite3_api->result_subtype +#endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ + +#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) /* This case when the file really is being compiled as a loadable ** extension */ # define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api=0; @@ -102373,6 +104582,7 @@ struct sqlite3_api_routines { /************** End of sqlite3ext.h ******************************************/ /************** Continuing where we left off in loadext.c ********************/ +/* #include "sqliteInt.h" */ /* #include */ #ifndef SQLITE_OMIT_LOAD_EXTENSION @@ -102757,7 +104967,15 @@ static const sqlite3_api_routines sqlite3Apis = { sqlite3_reset_auto_extension, sqlite3_result_blob64, sqlite3_result_text64, - sqlite3_strglob + sqlite3_strglob, + /* Version 3.8.11 and later */ + (sqlite3_value*(*)(const sqlite3_value*))sqlite3_value_dup, + sqlite3_value_free, + sqlite3_result_zeroblob64, + sqlite3_bind_zeroblob64, + /* Version 3.9.0 and later */ + sqlite3_value_subtype, + sqlite3_result_subtype }; /* @@ -102962,7 +105180,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_enable_load_extension(sqlite3 *db, int ono ** dummy pointer. */ #ifdef SQLITE_OMIT_LOAD_EXTENSION -static const sqlite3_api_routines sqlite3Apis = { 0 }; +static const sqlite3_api_routines sqlite3Apis; #endif @@ -103139,6 +105357,7 @@ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ ************************************************************************* ** This file contains code used to implement the PRAGMA command. */ +/* #include "sqliteInt.h" */ #if !defined(SQLITE_ENABLE_LOCKING_STYLE) # if defined(__APPLE__) @@ -103245,7 +105464,7 @@ static const struct sPragmaNames { #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) { /* zName: */ "cache_size", /* ePragTyp: */ PragTyp_CACHE_SIZE, - /* ePragFlag: */ PragFlag_NeedSchema, + /* ePragFlag: */ 0, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) @@ -103258,6 +105477,10 @@ static const struct sPragmaNames { /* ePragTyp: */ PragTyp_CASE_SENSITIVE_LIKE, /* ePragFlag: */ 0, /* iArg: */ 0 }, + { /* zName: */ "cell_size_check", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_CellSizeCk }, #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) { /* zName: */ "checkpoint_fullfsync", /* ePragTyp: */ PragTyp_FLAG, @@ -103615,7 +105838,7 @@ static const struct sPragmaNames { /* iArg: */ SQLITE_WriteSchema|SQLITE_RecoveryMode }, #endif }; -/* Number of pragmas: 59 on by default, 72 total. */ +/* Number of pragmas: 60 on by default, 73 total. */ /************** End of pragma.h **********************************************/ /************** Continuing where we left off in pragma.c *********************/ @@ -103749,20 +105972,46 @@ static int changeTempStorage(Parse *pParse, const char *zStorageType){ } #endif /* SQLITE_PAGER_PRAGMAS */ +/* +** Set the names of the first N columns to the values in azCol[] +*/ +static void setAllColumnNames( + Vdbe *v, /* The query under construction */ + int N, /* Number of columns */ + const char **azCol /* Names of columns */ +){ + int i; + sqlite3VdbeSetNumCols(v, N); + for(i=0; inMem; - i64 *pI64 = sqlite3DbMallocRaw(pParse->db, sizeof(value)); - if( pI64 ){ - memcpy(pI64, &value, sizeof(value)); +static void returnSingleInt(Vdbe *v, const char *zLabel, i64 value){ + sqlite3VdbeAddOp4Dup8(v, OP_Int64, 0, 1, 0, (const u8*)&value, P4_INT64); + setOneColumnName(v, zLabel); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); +} + +/* +** Generate code to return a single text value. +*/ +static void returnSingleText( + Vdbe *v, /* Prepared statement under construction */ + const char *zLabel, /* Name of the result column */ + const char *zValue /* Value to be returned */ +){ + if( zValue ){ + sqlite3VdbeLoadString(v, 1, (const char*)zValue); + setOneColumnName(v, zLabel); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); } - sqlite3VdbeAddOp4(v, OP_Int64, 0, nMem, 0, (char*)pI64, P4_INT64); - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLabel, SQLITE_STATIC); - sqlite3VdbeAddOp2(v, OP_ResultRow, nMem, 1); } @@ -103926,14 +106175,8 @@ SQLITE_PRIVATE void sqlite3Pragma( db->busyHandler.nBusy = 0; rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_PRAGMA, (void*)aFcntl); if( rc==SQLITE_OK ){ - if( aFcntl[0] ){ - int nMem = ++pParse->nMem; - sqlite3VdbeAddOp4(v, OP_String8, 0, nMem, 0, aFcntl[0], 0); - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "result", SQLITE_STATIC); - sqlite3VdbeAddOp2(v, OP_ResultRow, nMem, 1); - sqlite3_free(aFcntl[0]); - } + returnSingleText(v, "result", aFcntl[0]); + sqlite3_free(aFcntl[0]); goto pragma_out; } if( rc!=SQLITE_NOTFOUND ){ @@ -104003,8 +106246,7 @@ SQLITE_PRIVATE void sqlite3Pragma( int addr; sqlite3VdbeUsesBtree(v, iDb); if( !zRight ){ - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cache_size", SQLITE_STATIC); + setOneColumnName(v, "cache_size"); pParse->nMem += 2; addr = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize,iLn); sqlite3VdbeChangeP1(v, addr, iDb); @@ -104038,7 +106280,7 @@ SQLITE_PRIVATE void sqlite3Pragma( assert( pBt!=0 ); if( !zRight ){ int size = ALWAYS(pBt) ? sqlite3BtreeGetPageSize(pBt) : 0; - returnSingleInt(pParse, "page_size", size); + returnSingleInt(v, "page_size", size); }else{ /* Malloc may fail when setting the page-size, as there is an internal ** buffer that the pager module resizes using sqlite3_realloc(). @@ -104073,7 +106315,7 @@ SQLITE_PRIVATE void sqlite3Pragma( } } b = sqlite3BtreeSecureDelete(pBt, b); - returnSingleInt(pParse, "secure_delete", b); + returnSingleInt(v, "secure_delete", b); break; } @@ -104152,10 +106394,7 @@ SQLITE_PRIVATE void sqlite3Pragma( if( eMode==PAGER_LOCKINGMODE_EXCLUSIVE ){ zRet = "exclusive"; } - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "locking_mode", SQLITE_STATIC); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, zRet, 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + returnSingleText(v, "locking_mode", zRet); break; } @@ -104168,9 +106407,7 @@ SQLITE_PRIVATE void sqlite3Pragma( int eMode; /* One of the PAGER_JOURNALMODE_XXX symbols */ int ii; /* Loop counter */ - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "journal_mode", SQLITE_STATIC); - + setOneColumnName(v, "journal_mode"); if( zRight==0 ){ /* If there is no "=MODE" part of the pragma, do a query for the ** current mode */ @@ -104216,7 +106453,7 @@ SQLITE_PRIVATE void sqlite3Pragma( if( iLimit<-1 ) iLimit = -1; } iLimit = sqlite3PagerJournalSizeLimit(pPager, iLimit); - returnSingleInt(pParse, "journal_size_limit", iLimit); + returnSingleInt(v, "journal_size_limit", iLimit); break; } @@ -104234,7 +106471,7 @@ SQLITE_PRIVATE void sqlite3Pragma( Btree *pBt = pDb->pBt; assert( pBt!=0 ); if( !zRight ){ - returnSingleInt(pParse, "auto_vacuum", sqlite3BtreeGetAutoVacuum(pBt)); + returnSingleInt(v, "auto_vacuum", sqlite3BtreeGetAutoVacuum(pBt)); }else{ int eAuto = getAutoVacuum(zRight); assert( eAuto>=0 && eAuto<=2 ); @@ -104311,11 +106548,13 @@ SQLITE_PRIVATE void sqlite3Pragma( case PragTyp_CACHE_SIZE: { assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( !zRight ){ - returnSingleInt(pParse, "cache_size", pDb->pSchema->cache_size); + if( sqlite3ReadSchema(pParse) ) goto pragma_out; + returnSingleInt(v, "cache_size", pDb->pSchema->cache_size); }else{ int size = sqlite3Atoi(zRight); pDb->pSchema->cache_size = size; sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); + if( sqlite3ReadSchema(pParse) ) goto pragma_out; } break; } @@ -104356,7 +106595,7 @@ SQLITE_PRIVATE void sqlite3Pragma( rc = SQLITE_OK; #endif if( rc==SQLITE_OK ){ - returnSingleInt(pParse, "mmap_size", sz); + returnSingleInt(v, "mmap_size", sz); }else if( rc!=SQLITE_NOTFOUND ){ pParse->nErr++; pParse->rc = rc; @@ -104377,7 +106616,7 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_TEMP_STORE: { if( !zRight ){ - returnSingleInt(pParse, "temp_store", db->temp_store); + returnSingleInt(v, "temp_store", db->temp_store); }else{ changeTempStorage(pParse, zRight); } @@ -104396,13 +106635,7 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_TEMP_STORE_DIRECTORY: { if( !zRight ){ - if( sqlite3_temp_directory ){ - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, - "temp_store_directory", SQLITE_STATIC); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, sqlite3_temp_directory, 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - } + returnSingleText(v, "temp_store_directory", sqlite3_temp_directory); }else{ #ifndef SQLITE_OMIT_WSD if( zRight[0] ){ @@ -104446,13 +106679,7 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_DATA_STORE_DIRECTORY: { if( !zRight ){ - if( sqlite3_data_directory ){ - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, - "data_store_directory", SQLITE_STATIC); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, sqlite3_data_directory, 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - } + returnSingleText(v, "data_store_directory", sqlite3_data_directory); }else{ #ifndef SQLITE_OMIT_WSD if( zRight[0] ){ @@ -104491,14 +106718,7 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3_file *pFile = sqlite3PagerFile(pPager); sqlite3OsFileControlHint(pFile, SQLITE_GET_LOCKPROXYFILE, &proxy_file_path); - - if( proxy_file_path ){ - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, - "lock_proxy_file", SQLITE_STATIC); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, proxy_file_path, 0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - } + returnSingleText(v, "lock_proxy_file", proxy_file_path); }else{ Pager *pPager = sqlite3BtreePager(pDb->pBt); sqlite3_file *pFile = sqlite3PagerFile(pPager); @@ -104530,7 +106750,7 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_SYNCHRONOUS: { if( !zRight ){ - returnSingleInt(pParse, "synchronous", pDb->safety_level-1); + returnSingleInt(v, "synchronous", pDb->safety_level-1); }else{ if( !db->autoCommit ){ sqlite3ErrorMsg(pParse, @@ -104549,7 +106769,7 @@ SQLITE_PRIVATE void sqlite3Pragma( #ifndef SQLITE_OMIT_FLAG_PRAGMAS case PragTyp_FLAG: { if( zRight==0 ){ - returnSingleInt(pParse, pPragma->zName, (db->flags & pPragma->iArg)!=0 ); + returnSingleInt(v, pPragma->zName, (db->flags & pPragma->iArg)!=0 ); }else{ int mask = pPragma->iArg; /* Mask of bits to set or clear. */ if( db->autoCommit==0 ){ @@ -104599,35 +106819,22 @@ SQLITE_PRIVATE void sqlite3Pragma( Table *pTab; pTab = sqlite3FindTable(db, zRight, zDb); if( pTab ){ + static const char *azCol[] = { + "cid", "name", "type", "notnull", "dflt_value", "pk" + }; int i, k; int nHidden = 0; Column *pCol; Index *pPk = sqlite3PrimaryKeyIndex(pTab); - sqlite3VdbeSetNumCols(v, 6); pParse->nMem = 6; sqlite3CodeVerifySchema(pParse, iDb); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cid", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "type", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "notnull", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 4, COLNAME_NAME, "dflt_value", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 5, COLNAME_NAME, "pk", SQLITE_STATIC); + setAllColumnNames(v, 6, azCol); assert( 6==ArraySize(azCol) ); sqlite3ViewGetColumnNames(pParse, pTab); for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ if( IsHiddenColumn(pCol) ){ nHidden++; continue; } - sqlite3VdbeAddOp2(v, OP_Integer, i-nHidden, 1); - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pCol->zName, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, - pCol->zType ? pCol->zType : "", 0); - sqlite3VdbeAddOp2(v, OP_Integer, (pCol->notNull ? 1 : 0), 4); - if( pCol->zDflt ){ - sqlite3VdbeAddOp4(v, OP_String8, 0, 5, 0, (char*)pCol->zDflt, 0); - }else{ - sqlite3VdbeAddOp2(v, OP_Null, 0, 5); - } if( (pCol->colFlags & COLFLAG_PRIMKEY)==0 ){ k = 0; }else if( pPk==0 ){ @@ -104635,7 +106842,13 @@ SQLITE_PRIVATE void sqlite3Pragma( }else{ for(k=1; k<=pTab->nCol && pPk->aiColumn[k-1]!=i; k++){} } - sqlite3VdbeAddOp2(v, OP_Integer, k, 6); + sqlite3VdbeMultiLoad(v, 1, "issisi", + i-nHidden, + pCol->zName, + pCol->zType ? pCol->zType : "", + pCol->notNull ? 1 : 0, + pCol->zDflt, + k); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 6); } } @@ -104643,31 +106856,26 @@ SQLITE_PRIVATE void sqlite3Pragma( break; case PragTyp_STATS: { + static const char *azCol[] = { "table", "index", "width", "height" }; Index *pIdx; HashElem *i; v = sqlite3GetVdbe(pParse); - sqlite3VdbeSetNumCols(v, 4); pParse->nMem = 4; sqlite3CodeVerifySchema(pParse, iDb); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "table", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "index", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "width", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "height", SQLITE_STATIC); + setAllColumnNames(v, 4, azCol); assert( 4==ArraySize(azCol) ); for(i=sqliteHashFirst(&pDb->pSchema->tblHash); i; i=sqliteHashNext(i)){ Table *pTab = sqliteHashData(i); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, pTab->zName, 0); - sqlite3VdbeAddOp2(v, OP_Null, 0, 2); - sqlite3VdbeAddOp2(v, OP_Integer, - (int)sqlite3LogEstToInt(pTab->szTabRow), 3); - sqlite3VdbeAddOp2(v, OP_Integer, - (int)sqlite3LogEstToInt(pTab->nRowLogEst), 4); + sqlite3VdbeMultiLoad(v, 1, "ssii", + pTab->zName, + 0, + (int)sqlite3LogEstToInt(pTab->szTabRow), + (int)sqlite3LogEstToInt(pTab->nRowLogEst)); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); - sqlite3VdbeAddOp2(v, OP_Integer, - (int)sqlite3LogEstToInt(pIdx->szIdxRow), 3); - sqlite3VdbeAddOp2(v, OP_Integer, - (int)sqlite3LogEstToInt(pIdx->aiRowLogEst[0]), 4); + sqlite3VdbeMultiLoad(v, 2, "sii", + pIdx->zName, + (int)sqlite3LogEstToInt(pIdx->szIdxRow), + (int)sqlite3LogEstToInt(pIdx->aiRowLogEst[0])); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); } } @@ -104679,6 +106887,9 @@ SQLITE_PRIVATE void sqlite3Pragma( Table *pTab; pIdx = sqlite3FindIndex(db, zRight, zDb); if( pIdx ){ + static const char *azCol[] = { + "seqno", "cid", "name", "desc", "coll", "key" + }; int i; int mx; if( pPragma->iArg ){ @@ -104691,29 +106902,18 @@ SQLITE_PRIVATE void sqlite3Pragma( pParse->nMem = 3; } pTab = pIdx->pTable; - sqlite3VdbeSetNumCols(v, pParse->nMem); sqlite3CodeVerifySchema(pParse, iDb); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seqno", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "cid", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "name", SQLITE_STATIC); - if( pPragma->iArg ){ - sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "desc", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 4, COLNAME_NAME, "coll", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 5, COLNAME_NAME, "key", SQLITE_STATIC); - } + assert( pParse->nMem<=ArraySize(azCol) ); + setAllColumnNames(v, pParse->nMem, azCol); for(i=0; iaiColumn[i]; - sqlite3VdbeAddOp2(v, OP_Integer, i, 1); - sqlite3VdbeAddOp2(v, OP_Integer, cnum, 2); - if( cnum<0 ){ - sqlite3VdbeAddOp2(v, OP_Null, 0, 3); - }else{ - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pTab->aCol[cnum].zName, 0); - } + sqlite3VdbeMultiLoad(v, 1, "iis", i, cnum, + cnum<0 ? 0 : pTab->aCol[cnum].zName); if( pPragma->iArg ){ - sqlite3VdbeAddOp2(v, OP_Integer, pIdx->aSortOrder[i], 4); - sqlite3VdbeAddOp4(v, OP_String8, 0, 5, 0, pIdx->azColl[i], 0); - sqlite3VdbeAddOp2(v, OP_Integer, inKeyCol, 6); + sqlite3VdbeMultiLoad(v, 4, "isi", + pIdx->aSortOrder[i], + pIdx->azColl[i], + inKeyCol); } sqlite3VdbeAddOp2(v, OP_ResultRow, 1, pParse->nMem); } @@ -104727,22 +106927,21 @@ SQLITE_PRIVATE void sqlite3Pragma( int i; pTab = sqlite3FindTable(db, zRight, zDb); if( pTab ){ + static const char *azCol[] = { + "seq", "name", "unique", "origin", "partial" + }; v = sqlite3GetVdbe(pParse); - sqlite3VdbeSetNumCols(v, 5); pParse->nMem = 5; sqlite3CodeVerifySchema(pParse, iDb); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "unique", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "origin", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 4, COLNAME_NAME, "partial", SQLITE_STATIC); + setAllColumnNames(v, 5, azCol); assert( 5==ArraySize(azCol) ); for(pIdx=pTab->pIndex, i=0; pIdx; pIdx=pIdx->pNext, i++){ const char *azOrigin[] = { "c", "u", "pk" }; - sqlite3VdbeAddOp2(v, OP_Integer, i, 1); - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); - sqlite3VdbeAddOp2(v, OP_Integer, IsUniqueIndex(pIdx), 3); - sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, azOrigin[pIdx->idxType], 0); - sqlite3VdbeAddOp2(v, OP_Integer, pIdx->pPartIdxWhere!=0, 5); + sqlite3VdbeMultiLoad(v, 1, "isisi", + i, + pIdx->zName, + IsUniqueIndex(pIdx), + azOrigin[pIdx->idxType], + pIdx->pPartIdxWhere!=0); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 5); } } @@ -104750,35 +106949,31 @@ SQLITE_PRIVATE void sqlite3Pragma( break; case PragTyp_DATABASE_LIST: { + static const char *azCol[] = { "seq", "name", "file" }; int i; - sqlite3VdbeSetNumCols(v, 3); pParse->nMem = 3; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "file", SQLITE_STATIC); + setAllColumnNames(v, 3, azCol); assert( 3==ArraySize(azCol) ); for(i=0; inDb; i++){ if( db->aDb[i].pBt==0 ) continue; assert( db->aDb[i].zName!=0 ); - sqlite3VdbeAddOp2(v, OP_Integer, i, 1); - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, db->aDb[i].zName, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, - sqlite3BtreeGetFilename(db->aDb[i].pBt), 0); + sqlite3VdbeMultiLoad(v, 1, "iss", + i, + db->aDb[i].zName, + sqlite3BtreeGetFilename(db->aDb[i].pBt)); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); } } break; case PragTyp_COLLATION_LIST: { + static const char *azCol[] = { "seq", "name" }; int i = 0; HashElem *p; - sqlite3VdbeSetNumCols(v, 2); pParse->nMem = 2; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); + setAllColumnNames(v, 2, azCol); assert( 2==ArraySize(azCol) ); for(p=sqliteHashFirst(&db->aCollSeq); p; p=sqliteHashNext(p)){ CollSeq *pColl = (CollSeq *)sqliteHashData(p); - sqlite3VdbeAddOp2(v, OP_Integer, i++, 1); - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pColl->zName, 0); + sqlite3VdbeMultiLoad(v, 1, "is", i++, pColl->zName); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2); } } @@ -104794,33 +106989,26 @@ SQLITE_PRIVATE void sqlite3Pragma( v = sqlite3GetVdbe(pParse); pFK = pTab->pFKey; if( pFK ){ + static const char *azCol[] = { + "id", "seq", "table", "from", "to", "on_update", "on_delete", + "match" + }; int i = 0; - sqlite3VdbeSetNumCols(v, 8); pParse->nMem = 8; sqlite3CodeVerifySchema(pParse, iDb); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "id", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "seq", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "table", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "from", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 4, COLNAME_NAME, "to", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 5, COLNAME_NAME, "on_update", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 6, COLNAME_NAME, "on_delete", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 7, COLNAME_NAME, "match", SQLITE_STATIC); + setAllColumnNames(v, 8, azCol); assert( 8==ArraySize(azCol) ); while(pFK){ int j; for(j=0; jnCol; j++){ - char *zCol = pFK->aCol[j].zCol; - char *zOnDelete = (char *)actionName(pFK->aAction[0]); - char *zOnUpdate = (char *)actionName(pFK->aAction[1]); - sqlite3VdbeAddOp2(v, OP_Integer, i, 1); - sqlite3VdbeAddOp2(v, OP_Integer, j, 2); - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pFK->zTo, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, - pTab->aCol[pFK->aCol[j].iFrom].zName, 0); - sqlite3VdbeAddOp4(v, zCol ? OP_String8 : OP_Null, 0, 5, 0, zCol, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 6, 0, zOnUpdate, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 7, 0, zOnDelete, 0); - sqlite3VdbeAddOp4(v, OP_String8, 0, 8, 0, "NONE", 0); + sqlite3VdbeMultiLoad(v, 1, "iissssss", + i, + j, + pFK->zTo, + pTab->aCol[pFK->aCol[j].iFrom].zName, + pFK->aCol[j].zCol, + actionName(pFK->aAction[1]), /* ON UPDATE */ + actionName(pFK->aAction[0]), /* ON DELETE */ + "NONE"); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 8); } ++i; @@ -104849,17 +107037,14 @@ SQLITE_PRIVATE void sqlite3Pragma( int addrTop; /* Top of a loop checking foreign keys */ int addrOk; /* Jump here if the key is OK */ int *aiCols; /* child to parent column mapping */ + static const char *azCol[] = { "table", "rowid", "parent", "fkid" }; regResult = pParse->nMem+1; pParse->nMem += 4; regKey = ++pParse->nMem; regRow = ++pParse->nMem; v = sqlite3GetVdbe(pParse); - sqlite3VdbeSetNumCols(v, 4); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "table", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "rowid", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "parent", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "fkid", SQLITE_STATIC); + setAllColumnNames(v, 4, azCol); assert( 4==ArraySize(azCol) ); sqlite3CodeVerifySchema(pParse, iDb); k = sqliteHashFirst(&db->aDb[iDb].pSchema->tblHash); while( k ){ @@ -104874,8 +107059,7 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); if( pTab->nCol+regRow>pParse->nMem ) pParse->nMem = pTab->nCol + regRow; sqlite3OpenTable(pParse, 0, iDb, pTab, OP_OpenRead); - sqlite3VdbeAddOp4(v, OP_String8, 0, regResult, 0, pTab->zName, - P4_TRANSIENT); + sqlite3VdbeLoadString(v, regResult, pTab->zName); for(i=1, pFK=pTab->pFKey; pFK; i++, pFK=pFK->pNextFrom){ pParent = sqlite3FindTable(db, pFK->zTo, zDb); if( pParent==0 ) continue; @@ -104920,7 +107104,7 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp2(v, OP_Rowid, 0, regRow); } sqlite3VdbeAddOp3(v, OP_NotExists, i, 0, regRow); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrOk); + sqlite3VdbeGoto(v, addrOk); sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); }else{ for(j=0; jnCol; j++){ @@ -104930,15 +107114,13 @@ SQLITE_PRIVATE void sqlite3Pragma( } if( pParent ){ sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, pFK->nCol, regKey, - sqlite3IndexAffinityStr(v,pIdx), pFK->nCol); + sqlite3IndexAffinityStr(db,pIdx), pFK->nCol); sqlite3VdbeAddOp4Int(v, OP_Found, i, addrOk, regKey, 0); VdbeCoverage(v); } } sqlite3VdbeAddOp2(v, OP_Rowid, 0, regResult+1); - sqlite3VdbeAddOp4(v, OP_String8, 0, regResult+2, 0, - pFK->zTo, P4_TRANSIENT); - sqlite3VdbeAddOp2(v, OP_Integer, i-1, regResult+3); + sqlite3VdbeMultiLoad(v, regResult+2, "si", pFK->zTo, i-1); sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, 4); sqlite3VdbeResolveLabel(v, addrOk); sqlite3DbFree(db, aiCols); @@ -104992,8 +107174,9 @@ SQLITE_PRIVATE void sqlite3Pragma( */ static const int iLn = VDBE_OFFSET_LINENO(2); static const VdbeOpList endCode[] = { - { OP_IfNeg, 1, 0, 0}, /* 0 */ - { OP_String8, 0, 3, 0}, /* 1 */ + { OP_AddImm, 1, 0, 0}, /* 0 */ + { OP_If, 1, 0, 0}, /* 1 */ + { OP_String8, 0, 3, 0}, /* 2 */ { OP_ResultRow, 3, 1, 0}, }; @@ -105014,8 +107197,7 @@ SQLITE_PRIVATE void sqlite3Pragma( /* Initialize the VDBE program */ pParse->nMem = 6; - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "integrity_check", SQLITE_STATIC); + setOneColumnName(v, "integrity_check"); /* Set the maximum error count */ mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; @@ -105137,13 +107319,11 @@ SQLITE_PRIVATE void sqlite3Pragma( jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, ckUniq, r1, pIdx->nColumn); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, "row ", P4_STATIC); + sqlite3VdbeLoadString(v, 3, "row "); sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); - sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, - " missing from index ", P4_STATIC); + sqlite3VdbeLoadString(v, 4, " missing from index "); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); - jmp5 = sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, - pIdx->zName, P4_TRANSIENT); + jmp5 = sqlite3VdbeLoadString(v, 4, pIdx->zName); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); jmp4 = sqlite3VdbeAddOp1(v, OP_IfPos, 1); VdbeCoverage(v); @@ -105158,20 +107338,19 @@ SQLITE_PRIVATE void sqlite3Pragma( int kk; for(kk=0; kknKeyCol; kk++){ int iCol = pIdx->aiColumn[kk]; - assert( iCol>=0 && iColnCol ); - if( pTab->aCol[iCol].notNull ) continue; + assert( iCol!=XN_ROWID && iColnCol ); + if( iCol>=0 && pTab->aCol[iCol].notNull ) continue; sqlite3VdbeAddOp2(v, OP_IsNull, r1+kk, uniqOk); VdbeCoverage(v); } jmp6 = sqlite3VdbeAddOp1(v, OP_Next, iIdxCur+j); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, uniqOk); + sqlite3VdbeGoto(v, uniqOk); sqlite3VdbeJumpHere(v, jmp6); sqlite3VdbeAddOp4Int(v, OP_IdxGT, iIdxCur+j, uniqOk, r1, pIdx->nKeyCol); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, - "non-unique entry in index ", P4_STATIC); - sqlite3VdbeAddOp2(v, OP_Goto, 0, jmp5); + sqlite3VdbeLoadString(v, 3, "non-unique entry in index "); + sqlite3VdbeGoto(v, jmp5); sqlite3VdbeResolveLabel(v, uniqOk); } sqlite3VdbeJumpHere(v, jmp4); @@ -105180,8 +107359,7 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); VdbeCoverage(v); sqlite3VdbeJumpHere(v, loopTop-1); #ifndef SQLITE_OMIT_BTREECOUNT - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, - "wrong # of entries in index ", P4_STATIC); + sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ if( pPk==pIdx ) continue; addr = sqlite3VdbeCurrentAddr(v); @@ -105191,7 +107369,7 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp3(v, OP_Eq, 8+j, addr+8, 3); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pIdx->zName, P4_TRANSIENT); + sqlite3VdbeLoadString(v, 3, pIdx->zName); sqlite3VdbeAddOp3(v, OP_Concat, 3, 2, 7); sqlite3VdbeAddOp2(v, OP_ResultRow, 7, 1); } @@ -105199,9 +107377,9 @@ SQLITE_PRIVATE void sqlite3Pragma( } } addr = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn); - sqlite3VdbeChangeP3(v, addr, -mxErr); - sqlite3VdbeJumpHere(v, addr); - sqlite3VdbeChangeP4(v, addr+1, "ok", P4_STATIC); + sqlite3VdbeChangeP2(v, addr, -mxErr); + sqlite3VdbeJumpHere(v, addr+1); + sqlite3VdbeChangeP4(v, addr+2, "ok", P4_STATIC); } break; #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -105247,14 +107425,10 @@ SQLITE_PRIVATE void sqlite3Pragma( const struct EncName *pEnc; if( !zRight ){ /* "PRAGMA encoding" */ if( sqlite3ReadSchema(pParse) ) goto pragma_out; - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "encoding", SQLITE_STATIC); - sqlite3VdbeAddOp2(v, OP_String8, 0, 1); assert( encnames[SQLITE_UTF8].enc==SQLITE_UTF8 ); assert( encnames[SQLITE_UTF16LE].enc==SQLITE_UTF16LE ); assert( encnames[SQLITE_UTF16BE].enc==SQLITE_UTF16BE ); - sqlite3VdbeChangeP4(v, -1, encnames[ENC(pParse->db)].zName, P4_STATIC); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + returnSingleText(v, "encoding", encnames[ENC(pParse->db)].zName); }else{ /* "PRAGMA encoding = XXX" */ /* Only change the value of sqlite.enc if the database handle is not ** initialized. If the main database exists, the new sqlite.enc value @@ -105355,11 +107529,10 @@ SQLITE_PRIVATE void sqlite3Pragma( case PragTyp_COMPILE_OPTIONS: { int i = 0; const char *zOpt; - sqlite3VdbeSetNumCols(v, 1); pParse->nMem = 1; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "compile_option", SQLITE_STATIC); + setOneColumnName(v, "compile_option"); while( (zOpt = sqlite3_compileoption_get(i++))!=0 ){ - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, zOpt, 0); + sqlite3VdbeLoadString(v, 1, zOpt); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); } } @@ -105373,6 +107546,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** Checkpoint the database. */ case PragTyp_WAL_CHECKPOINT: { + static const char *azCol[] = { "busy", "log", "checkpointed" }; int iBt = (pId2->z?iDb:SQLITE_MAX_ATTACHED); int eMode = SQLITE_CHECKPOINT_PASSIVE; if( zRight ){ @@ -105384,12 +107558,8 @@ SQLITE_PRIVATE void sqlite3Pragma( eMode = SQLITE_CHECKPOINT_TRUNCATE; } } - sqlite3VdbeSetNumCols(v, 3); + setAllColumnNames(v, 3, azCol); assert( 3==ArraySize(azCol) ); pParse->nMem = 3; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "busy", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "log", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "checkpointed", SQLITE_STATIC); - sqlite3VdbeAddOp3(v, OP_Checkpoint, iBt, eMode, 1); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); } @@ -105407,7 +107577,7 @@ SQLITE_PRIVATE void sqlite3Pragma( if( zRight ){ sqlite3_wal_autocheckpoint(db, sqlite3Atoi(zRight)); } - returnSingleInt(pParse, "wal_autocheckpoint", + returnSingleInt(v, "wal_autocheckpoint", db->xWalCallback==sqlite3WalDefaultHook ? SQLITE_PTR_TO_INT(db->pWalArg) : 0); } @@ -105440,7 +107610,7 @@ SQLITE_PRIVATE void sqlite3Pragma( if( zRight ){ sqlite3_busy_timeout(db, sqlite3Atoi(zRight)); } - returnSingleInt(pParse, "timeout", db->busyTimeout); + returnSingleInt(v, "timeout", db->busyTimeout); break; } @@ -105460,7 +107630,7 @@ SQLITE_PRIVATE void sqlite3Pragma( if( zRight && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK ){ sqlite3_soft_heap_limit64(N); } - returnSingleInt(pParse, "soft_heap_limit", sqlite3_soft_heap_limit64(-1)); + returnSingleInt(v, "soft_heap_limit", sqlite3_soft_heap_limit64(-1)); break; } @@ -105479,7 +107649,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ){ sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, (int)(N&0x7fffffff)); } - returnSingleInt(pParse, "threads", + returnSingleInt(v, "threads", sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, -1)); break; } @@ -105492,17 +107662,15 @@ SQLITE_PRIVATE void sqlite3Pragma( static const char *const azLockName[] = { "unlocked", "shared", "reserved", "pending", "exclusive" }; + static const char *azCol[] = { "database", "status" }; int i; - sqlite3VdbeSetNumCols(v, 2); + setAllColumnNames(v, 2, azCol); assert( 2==ArraySize(azCol) ); pParse->nMem = 2; - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "database", SQLITE_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "status", SQLITE_STATIC); for(i=0; inDb; i++){ Btree *pBt; const char *zState = "unknown"; int j; if( db->aDb[i].zName==0 ) continue; - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, db->aDb[i].zName, P4_STATIC); pBt = db->aDb[i].pBt; if( pBt==0 || sqlite3BtreePager(pBt)==0 ){ zState = "closed"; @@ -105510,7 +107678,7 @@ SQLITE_PRIVATE void sqlite3Pragma( SQLITE_FCNTL_LOCKSTATE, &j)==SQLITE_OK ){ zState = azLockName[j]; } - sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, zState, P4_STATIC); + sqlite3VdbeMultiLoad(v, 1, "ss", db->aDb[i].zName, zState); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2); } break; @@ -105586,6 +107754,7 @@ pragma_out: ** interface, and routines that contribute to loading the database schema ** from disk. */ +/* #include "sqliteInt.h" */ /* ** Fill the InitData structure with an error message that indicates @@ -105598,13 +107767,13 @@ static void corruptSchema( ){ sqlite3 *db = pData->db; if( !db->mallocFailed && (db->flags & SQLITE_RecoveryMode)==0 ){ + char *z; if( zObj==0 ) zObj = "?"; - sqlite3SetString(pData->pzErrMsg, db, - "malformed database schema (%s)", zObj); - if( zExtra ){ - *pData->pzErrMsg = sqlite3MAppendf(db, *pData->pzErrMsg, - "%s - %s", *pData->pzErrMsg, zExtra); - } + z = sqlite3_mprintf("malformed database schema (%s)", zObj); + if( z && zExtra ) z = sqlite3_mprintf("%z - %s", z, zExtra); + sqlite3DbFree(db, *pData->pzErrMsg); + *pData->pzErrMsg = z; + if( z==0 ) db->mallocFailed = 1; } pData->rc = db->mallocFailed ? SQLITE_NOMEM : SQLITE_CORRUPT_BKPT; } @@ -105796,7 +107965,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ if( !sqlite3BtreeIsInReadTrans(pDb->pBt) ){ rc = sqlite3BtreeBeginTrans(pDb->pBt, 0); if( rc!=SQLITE_OK ){ - sqlite3SetString(pzErrMsg, db, "%s", sqlite3ErrStr(rc)); + sqlite3SetString(pzErrMsg, db, sqlite3ErrStr(rc)); goto initone_error_out; } openedTransaction = 1; @@ -106480,6 +108649,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_prepare16_v2( ** This file contains C code routines that are called by the parser ** to handle SELECT statements in SQLite. */ +/* #include "sqliteInt.h" */ /* ** Trace output macros @@ -106488,7 +108658,8 @@ SQLITE_API int SQLITE_STDCALL sqlite3_prepare16_v2( /***/ int sqlite3SelectTrace = 0; # define SELECTTRACE(K,P,S,X) \ if(sqlite3SelectTrace&(K)) \ - sqlite3DebugPrintf("%*s%s.%p: ",(P)->nSelectIndent*2-2,"",(S)->zSelName,(S)),\ + sqlite3DebugPrintf("%*s%s.%p: ",(P)->nSelectIndent*2-2,"",\ + (S)->zSelName,(S)),\ sqlite3DebugPrintf X #else # define SELECTTRACE(K,P,S,X) @@ -106832,6 +109003,12 @@ static void setJoinExpr(Expr *p, int iTable){ assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); ExprSetVVAProperty(p, EP_NoReduce); p->iRightJoinTable = (i16)iTable; + if( p->op==TK_FUNCTION && p->x.pList ){ + int i; + for(i=0; ix.pList->nExpr; i++){ + setJoinExpr(p->x.pList->a[i].pExpr, iTable); + } + } setJoinExpr(p->pLeft, iTable); p = p->pRight; } @@ -106866,12 +109043,12 @@ static int sqliteProcessJoin(Parse *pParse, Select *p){ int isOuter; if( NEVER(pLeftTab==0 || pRightTab==0) ) continue; - isOuter = (pRight->jointype & JT_OUTER)!=0; + isOuter = (pRight->fg.jointype & JT_OUTER)!=0; /* When the NATURAL keyword is present, add WHERE clause terms for ** every column that the two tables have in common. */ - if( pRight->jointype & JT_NATURAL ){ + if( pRight->fg.jointype & JT_NATURAL ){ if( pRight->pOn || pRight->pUsing ){ sqlite3ErrorMsg(pParse, "a NATURAL join may not have " "an ON or USING clause", 0); @@ -106956,6 +109133,7 @@ static void pushOntoSorter( SortCtx *pSort, /* Information about the ORDER BY clause */ Select *pSelect, /* The whole SELECT statement */ int regData, /* First register holding data to be sorted */ + int regOrigData, /* First register holding data before packing */ int nData, /* Number of elements in the data array */ int nPrefixReg /* No. of reg prior to regData available for use */ ){ @@ -106969,6 +109147,7 @@ static void pushOntoSorter( int op; /* Opcode to add sorter record to sorter */ assert( bSeq==0 || bSeq==1 ); + assert( nData==1 || regData==regOrigData ); if( nPrefixReg ){ assert( nPrefixReg==nExpr+bSeq ); regBase = regData - nExpr - bSeq; @@ -106976,7 +109155,8 @@ static void pushOntoSorter( regBase = pParse->nMem + 1; pParse->nMem += nBase; } - sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, SQLITE_ECEL_DUP); + sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, regOrigData, + SQLITE_ECEL_DUP|SQLITE_ECEL_REF); if( bSeq ){ sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); } @@ -107036,7 +109216,7 @@ static void pushOntoSorter( }else{ iLimit = pSelect->iLimit; } - addr = sqlite3VdbeAddOp3(v, OP_IfNotZero, iLimit, 0, -1); VdbeCoverage(v); + addr = sqlite3VdbeAddOp3(v, OP_IfNotZero, iLimit, 0, 1); VdbeCoverage(v); sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor); sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor); sqlite3VdbeJumpHere(v, addr); @@ -107052,11 +109232,8 @@ static void codeOffset( int iContinue /* Jump here to skip the current record */ ){ if( iOffset>0 ){ - int addr; - addr = sqlite3VdbeAddOp3(v, OP_IfNeg, iOffset, 0, -1); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, iContinue); - VdbeComment((v, "skip OFFSET records")); - sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeAddOp3(v, OP_IfPos, iOffset, iContinue, 1); VdbeCoverage(v); + VdbeComment((v, "OFFSET")); } } @@ -107180,8 +109357,13 @@ static void selectInnerLoop( /* If the destination is an EXISTS(...) expression, the actual ** values returned by the SELECT are not required. */ - sqlite3ExprCodeExprList(pParse, pEList, regResult, - (eDest==SRT_Output||eDest==SRT_Coroutine)?SQLITE_ECEL_DUP:0); + u8 ecelFlags; + if( eDest==SRT_Mem || eDest==SRT_Output || eDest==SRT_Coroutine ){ + ecelFlags = SQLITE_ECEL_DUP; + }else{ + ecelFlags = 0; + } + sqlite3ExprCodeExprList(pParse, pEList, regResult, 0, ecelFlags); } /* If the DISTINCT keyword was present on the SELECT statement @@ -107236,7 +109418,8 @@ static void selectInnerLoop( default: { assert( pDistinct->eTnctType==WHERE_DISTINCT_UNORDERED ); - codeDistinct(pParse, pDistinct->tabTnct, iContinue, nResultCol, regResult); + codeDistinct(pParse, pDistinct->tabTnct, iContinue, nResultCol, + regResult); break; } } @@ -107278,6 +109461,8 @@ static void selectInnerLoop( int r1 = sqlite3GetTempRange(pParse, nPrefixReg+1); testcase( eDest==SRT_Table ); testcase( eDest==SRT_EphemTab ); + testcase( eDest==SRT_Fifo ); + testcase( eDest==SRT_DistFifo ); sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1+nPrefixReg); #ifndef SQLITE_OMIT_CTE if( eDest==SRT_DistFifo ){ @@ -107287,13 +109472,14 @@ static void selectInnerLoop( ** current row to the index and proceed with writing it to the ** output table as well. */ int addr = sqlite3VdbeCurrentAddr(v) + 4; - sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0); VdbeCoverage(v); + sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0); + VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r1); assert( pSort==0 ); } #endif if( pSort ){ - pushOntoSorter(pParse, pSort, p, r1+nPrefixReg, 1, nPrefixReg); + pushOntoSorter(pParse, pSort, p, r1+nPrefixReg,regResult,1,nPrefixReg); }else{ int r2 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); @@ -107319,7 +109505,7 @@ static void selectInnerLoop( ** ORDER BY in this case since the order of entries in the set ** does not matter. But there might be a LIMIT clause, in which ** case the order does matter */ - pushOntoSorter(pParse, pSort, p, regResult, 1, nPrefixReg); + pushOntoSorter(pParse, pSort, p, regResult, regResult, 1, nPrefixReg); }else{ int r1 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult,1,r1, &pDest->affSdst, 1); @@ -107345,7 +109531,7 @@ static void selectInnerLoop( case SRT_Mem: { assert( nResultCol==1 ); if( pSort ){ - pushOntoSorter(pParse, pSort, p, regResult, 1, nPrefixReg); + pushOntoSorter(pParse, pSort, p, regResult, regResult, 1, nPrefixReg); }else{ assert( regResult==iParm ); /* The LIMIT clause will jump out of the loop for us */ @@ -107359,7 +109545,8 @@ static void selectInnerLoop( testcase( eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); if( pSort ){ - pushOntoSorter(pParse, pSort, p, regResult, nResultCol, nPrefixReg); + pushOntoSorter(pParse, pSort, p, regResult, regResult, nResultCol, + nPrefixReg); }else if( eDest==SRT_Coroutine ){ sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); }else{ @@ -107533,7 +109720,6 @@ static KeyInfo *keyInfoFromExprList( return pInfo; } -#ifndef SQLITE_OMIT_COMPOUND_SELECT /* ** Name of the connection operator, used for error messages. */ @@ -107547,7 +109733,6 @@ static const char *selectOpName(int id){ } return z; } -#endif /* SQLITE_OMIT_COMPOUND_SELECT */ #ifndef SQLITE_OMIT_EXPLAIN /* @@ -107655,7 +109840,7 @@ static void generateSortTail( if( pSort->labelBkOut ){ sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrBreak); + sqlite3VdbeGoto(v, addrBreak); sqlite3VdbeResolveLabel(v, pSort->labelBkOut); } iTab = pSort->iECursor; @@ -107693,10 +109878,7 @@ static void generateSortTail( VdbeComment((v, "%s", aOutEx[i].zName ? aOutEx[i].zName : aOutEx[i].zSpan)); } switch( eDest ){ - case SRT_Table: case SRT_EphemTab: { - testcase( eDest==SRT_Table ); - testcase( eDest==SRT_EphemTab ); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, regRowid); sqlite3VdbeAddOp3(v, OP_Insert, iParm, regRow, regRowid); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); @@ -107773,28 +109955,27 @@ static void generateSortTail( */ #ifdef SQLITE_ENABLE_COLUMN_METADATA # define columnType(A,B,C,D,E,F) columnTypeImpl(A,B,C,D,E,F) +#else /* if !defined(SQLITE_ENABLE_COLUMN_METADATA) */ +# define columnType(A,B,C,D,E,F) columnTypeImpl(A,B,F) +#endif static const char *columnTypeImpl( NameContext *pNC, Expr *pExpr, +#ifdef SQLITE_ENABLE_COLUMN_METADATA const char **pzOrigDb, const char **pzOrigTab, const char **pzOrigCol, +#endif u8 *pEstWidth ){ - char const *zOrigDb = 0; - char const *zOrigTab = 0; - char const *zOrigCol = 0; -#else /* if !defined(SQLITE_ENABLE_COLUMN_METADATA) */ -# define columnType(A,B,C,D,E,F) columnTypeImpl(A,B,F) -static const char *columnTypeImpl( - NameContext *pNC, - Expr *pExpr, - u8 *pEstWidth -){ -#endif /* !defined(SQLITE_ENABLE_COLUMN_METADATA) */ char const *zType = 0; int j; u8 estWidth = 1; +#ifdef SQLITE_ENABLE_COLUMN_METADATA + char const *zOrigDb = 0; + char const *zOrigTab = 0; + char const *zOrigCol = 0; +#endif if( NEVER(pExpr==0) || pNC->pSrcList==0 ) return 0; switch( pExpr->op ){ @@ -107847,10 +110028,13 @@ static const char *columnTypeImpl( ** of the SELECT statement. Return the declaration type and origin ** data for the result-set column of the sub-select. */ - if( iCol>=0 && iColpEList->nExpr ){ + if( iCol>=0 && ALWAYS(iColpEList->nExpr) ){ /* If iCol is less than zero, then the expression requests the ** rowid of the sub-select or view. This expression is legal (see ** test case misc2.2.2) - it always evaluates to NULL. + ** + ** The ALWAYS() is because iCol>=pS->pEList->nExpr will have been + ** caught already by name resolution. */ NameContext sNC; Expr *p = pS->pEList->a[iCol].pExpr; @@ -108041,7 +110225,7 @@ static void generateColumnNames( ** Return SQLITE_OK on success. If a memory allocation error occurs, ** store NULL in *paCol and 0 in *pnCol and return SQLITE_NOMEM. */ -static int selectColumnsFromExprList( +SQLITE_PRIVATE int sqlite3ColumnsFromExprList( Parse *pParse, /* Parsing context */ ExprList *pEList, /* Expr list from which to derive column names */ i16 *pnCol, /* Write the number of columns here */ @@ -108168,11 +110352,12 @@ static void selectAddColumnTypeAndCollation( for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ p = a[i].pExpr; if( pCol->zType==0 ){ - pCol->zType = sqlite3DbStrDup(db, columnType(&sNC, p,0,0,0, &pCol->szEst)); + pCol->zType = sqlite3DbStrDup(db, + columnType(&sNC, p,0,0,0, &pCol->szEst)); } szAll += pCol->szEst; pCol->affinity = sqlite3ExprAffinity(p); - if( pCol->affinity==0 ) pCol->affinity = SQLITE_AFF_NONE; + if( pCol->affinity==0 ) pCol->affinity = SQLITE_AFF_BLOB; pColl = sqlite3ExprCollSeq(pParse, p); if( pColl && pCol->zColl==0 ){ pCol->zColl = sqlite3DbStrDup(db, pColl->zName); @@ -108207,7 +110392,7 @@ SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ pTab->nRef = 1; pTab->zName = 0; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); - selectColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); + sqlite3ColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); selectAddColumnTypeAndCollation(pParse, pTab, pSelect); pTab->iPKey = -1; if( db->mallocFailed ){ @@ -108264,7 +110449,7 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ Vdbe *v = 0; int iLimit = 0; int iOffset; - int addr1, n; + int n; if( p->iLimit ) return; /* @@ -108283,7 +110468,7 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit); VdbeComment((v, "LIMIT counter")); if( n==0 ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak); + sqlite3VdbeGoto(v, iBreak); }else if( n>=0 && p->nSelectRow>(u64)n ){ p->nSelectRow = n; } @@ -108299,14 +110484,10 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ sqlite3ExprCode(pParse, p->pOffset, iOffset); sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset); VdbeCoverage(v); VdbeComment((v, "OFFSET counter")); - addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iOffset); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Integer, 0, iOffset); - sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeAddOp3(v, OP_SetIfNotPos, iOffset, iOffset, 0); sqlite3VdbeAddOp3(v, OP_Add, iLimit, iOffset, iOffset+1); VdbeComment((v, "LIMIT+OFFSET")); - addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iLimit); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Integer, -1, iOffset+1); - sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeAddOp3(v, OP_SetIfNotPos, iLimit, iOffset+1, -1); } } } @@ -108328,7 +110509,10 @@ static CollSeq *multiSelectCollSeq(Parse *pParse, Select *p, int iCol){ pRet = 0; } assert( iCol>=0 ); - if( pRet==0 && iColpEList->nExpr ){ + /* iCol must be less than p->pEList->nExpr. Otherwise an error would + ** have been thrown during name resolution and we would not have gotten + ** this far */ + if( pRet==0 && ALWAYS(iColpEList->nExpr) ){ pRet = sqlite3ExprCollSeq(pParse, p->pEList->a[iCol].pExpr); } return pRet; @@ -108383,7 +110567,7 @@ static KeyInfo *multiSelectOrderByKeyInfo(Parse *pParse, Select *p, int nExtra){ ** ** ** There is exactly one reference to the recursive-table in the FROM clause -** of recursive-query, marked with the SrcList->a[].isRecursive flag. +** of recursive-query, marked with the SrcList->a[].fg.isRecursive flag. ** ** The setup-query runs once to generate an initial set of rows that go ** into a Queue table. Rows are extracted from the Queue table one by @@ -108448,7 +110632,7 @@ static void generateWithRecursiveQuery( /* Locate the cursor number of the Current table */ for(i=0; ALWAYS(inSrc); i++){ - if( pSrc->a[i].isRecursive ){ + if( pSrc->a[i].fg.isRecursive ){ iCurrent = pSrc->a[i].iCursor; break; } @@ -108518,13 +110702,17 @@ static void generateWithRecursiveQuery( /* Execute the recursive SELECT taking the single row in Current as ** the value for the recursive-table. Store the results in the Queue. */ - p->pPrior = 0; - sqlite3Select(pParse, p, &destQueue); - assert( p->pPrior==0 ); - p->pPrior = pSetup; + if( p->selFlags & SF_Aggregate ){ + sqlite3ErrorMsg(pParse, "recursive aggregate queries not supported"); + }else{ + p->pPrior = 0; + sqlite3Select(pParse, p, &destQueue); + assert( p->pPrior==0 ); + p->pPrior = pSetup; + } /* Keep running the loop until the Queue is empty */ - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop); + sqlite3VdbeGoto(v, addrTop); sqlite3VdbeResolveLabel(v, addrBreak); end_of_recursive_query: @@ -108543,19 +110731,6 @@ static int multiSelectOrderBy( SelectDest *pDest /* What to do with query results */ ); -/* -** Error message for when two or more terms of a compound select have different -** size result sets. -*/ -static void selectWrongNumTermsError(Parse *pParse, Select *p){ - if( p->selFlags & SF_Values ){ - sqlite3ErrorMsg(pParse, "all VALUES must have the same number of terms"); - }else{ - sqlite3ErrorMsg(pParse, "SELECTs to the left and right of %s" - " do not have the same number of result columns", selectOpName(p->op)); - } -} - /* ** Handle the special case of a compound-select that originates from a ** VALUES clause. By handling this as a special case, we avoid deep @@ -108573,7 +110748,6 @@ static int multiSelectValues( SelectDest *pDest /* What to do with query results */ ){ Select *pPrior; - int nExpr = p->pEList->nExpr; int nRow = 1; int rc = 0; assert( p->selFlags & SF_MultiValue ); @@ -108582,10 +110756,7 @@ static int multiSelectValues( assert( p->op==TK_ALL || (p->op==TK_SELECT && p->pPrior==0) ); assert( p->pLimit==0 ); assert( p->pOffset==0 ); - if( p->pEList->nExpr!=nExpr ){ - selectWrongNumTermsError(pParse, p); - return 1; - } + assert( p->pNext==0 || p->pEList->nExpr==p->pNext->pEList->nExpr ); if( p->pPrior==0 ) break; assert( p->pPrior->pNext==p ); p = p->pPrior; @@ -108694,11 +110865,7 @@ static int multiSelect( ** in their result sets. */ assert( p->pEList && pPrior->pEList ); - if( p->pEList->nExpr!=pPrior->pEList->nExpr ){ - selectWrongNumTermsError(pParse, p); - rc = 1; - goto multi_select_end; - } + assert( p->pEList->nExpr==pPrior->pEList->nExpr ); #ifndef SQLITE_OMIT_CTE if( p->selFlags & SF_Recursive ){ @@ -108736,6 +110903,11 @@ static int multiSelect( if( p->iLimit ){ addr = sqlite3VdbeAddOp1(v, OP_IfNot, p->iLimit); VdbeCoverage(v); VdbeComment((v, "Jump ahead if LIMIT reached")); + if( p->iOffset ){ + sqlite3VdbeAddOp3(v, OP_SetIfNotPos, p->iOffset, p->iOffset, 0); + sqlite3VdbeAddOp3(v, OP_Add, p->iLimit, p->iOffset, p->iOffset+1); + sqlite3VdbeAddOp3(v, OP_SetIfNotPos, p->iLimit, p->iOffset+1, -1); + } } explainSetInteger(iSub2, pParse->iNextSelectId); rc = sqlite3Select(pParse, p, &dest); @@ -108990,6 +111162,19 @@ multi_select_end: } #endif /* SQLITE_OMIT_COMPOUND_SELECT */ +/* +** Error message for when two or more terms of a compound select have different +** size result sets. +*/ +SQLITE_PRIVATE void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p){ + if( p->selFlags & SF_Values ){ + sqlite3ErrorMsg(pParse, "all VALUES must have the same number of terms"); + }else{ + sqlite3ErrorMsg(pParse, "SELECTs to the left and right of %s" + " do not have the same number of result columns", selectOpName(p->op)); + } +} + /* ** Code an output subroutine for a coroutine implementation of a ** SELECT statment. @@ -109030,12 +111215,12 @@ static int generateOutputSubroutine( /* Suppress duplicates for UNION, EXCEPT, and INTERSECT */ if( regPrev ){ - int j1, j2; - j1 = sqlite3VdbeAddOp1(v, OP_IfNot, regPrev); VdbeCoverage(v); - j2 = sqlite3VdbeAddOp4(v, OP_Compare, pIn->iSdst, regPrev+1, pIn->nSdst, + int addr1, addr2; + addr1 = sqlite3VdbeAddOp1(v, OP_IfNot, regPrev); VdbeCoverage(v); + addr2 = sqlite3VdbeAddOp4(v, OP_Compare, pIn->iSdst, regPrev+1, pIn->nSdst, (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO); - sqlite3VdbeAddOp3(v, OP_Jump, j2+2, iContinue, j2+2); VdbeCoverage(v); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeAddOp3(v, OP_Jump, addr2+2, iContinue, addr2+2); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp3(v, OP_Copy, pIn->iSdst, regPrev+1, pIn->nSdst-1); sqlite3VdbeAddOp2(v, OP_Integer, 1, regPrev); } @@ -109045,15 +111230,14 @@ static int generateOutputSubroutine( */ codeOffset(v, p->iOffset, iContinue); + assert( pDest->eDest!=SRT_Exists ); + assert( pDest->eDest!=SRT_Table ); switch( pDest->eDest ){ /* Store the result as data using a unique key. */ - case SRT_Table: case SRT_EphemTab: { int r1 = sqlite3GetTempReg(pParse); int r2 = sqlite3GetTempReg(pParse); - testcase( pDest->eDest==SRT_Table ); - testcase( pDest->eDest==SRT_EphemTab ); sqlite3VdbeAddOp3(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, r1); sqlite3VdbeAddOp2(v, OP_NewRowid, pDest->iSDParm, r2); sqlite3VdbeAddOp3(v, OP_Insert, pDest->iSDParm, r1, r2); @@ -109081,16 +111265,6 @@ static int generateOutputSubroutine( break; } -#if 0 /* Never occurs on an ORDER BY query */ - /* If any row exist in the result set, record that fact and abort. - */ - case SRT_Exists: { - sqlite3VdbeAddOp2(v, OP_Integer, 1, pDest->iSDParm); - /* The LIMIT clause will terminate the loop for us */ - break; - } -#endif - /* If this is a scalar select that is part of an expression, then ** store the results in the appropriate memory cell and break out ** of the scan loop. @@ -109263,7 +111437,7 @@ static int multiSelectOrderBy( int savedOffset; /* Saved value of p->iOffset */ int labelCmpr; /* Label for the start of the merge algorithm */ int labelEnd; /* Label for the end of the overall SELECT stmt */ - int j1; /* Jump instructions that get retargetted */ + int addr1; /* Jump instructions that get retargetted */ int op; /* One of TK_ALL, TK_UNION, TK_EXCEPT, TK_INTERSECT */ KeyInfo *pKeyDup = 0; /* Comparison information for duplicate removal */ KeyInfo *pKeyMerge; /* Comparison information for merging rows */ @@ -109328,9 +111502,7 @@ static int multiSelectOrderBy( struct ExprList_item *pItem; for(i=0, pItem=pOrderBy->a; iu.x.iOrderByCol>0 ); - /* assert( pItem->u.x.iOrderByCol<=p->pEList->nExpr ) is also true - ** but only for well-formed SELECT statements. */ - testcase( pItem->u.x.iOrderByCol > p->pEList->nExpr ); + assert( pItem->u.x.iOrderByCol<=p->pEList->nExpr ); aPermute[i] = pItem->u.x.iOrderByCol - 1; } pKeyMerge = multiSelectOrderByKeyInfo(pParse, p, 1); @@ -109401,19 +111573,19 @@ static int multiSelectOrderBy( ** left of the compound operator - the "A" select. */ addrSelectA = sqlite3VdbeCurrentAddr(v) + 1; - j1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA); + addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA); VdbeComment((v, "left SELECT")); pPrior->iLimit = regLimitA; explainSetInteger(iSub1, pParse->iNextSelectId); sqlite3Select(pParse, pPrior, &destA); sqlite3VdbeAddOp1(v, OP_EndCoroutine, regAddrA); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); /* Generate a coroutine to evaluate the SELECT statement on ** the right - the "B" select */ addrSelectB = sqlite3VdbeCurrentAddr(v) + 1; - j1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrB, 0, addrSelectB); + addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrB, 0, addrSelectB); VdbeComment((v, "right SELECT")); savedLimit = p->iLimit; savedOffset = p->iOffset; @@ -109454,7 +111626,7 @@ static int multiSelectOrderBy( addrEofA = sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); addrEofA_noB = sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, labelEnd); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEofA); + sqlite3VdbeGoto(v, addrEofA); p->nSelectRow += pPrior->nSelectRow; } @@ -109468,7 +111640,7 @@ static int multiSelectOrderBy( VdbeNoopComment((v, "eof-B subroutine")); addrEofB = sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, labelEnd); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEofB); + sqlite3VdbeGoto(v, addrEofB); } /* Generate code to handle the case of AB @@ -109500,11 +111672,11 @@ static int multiSelectOrderBy( sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); } sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, labelCmpr); + sqlite3VdbeGoto(v, labelCmpr); /* This code runs once to initialize everything. */ - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA_noB); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); @@ -109547,7 +111719,7 @@ static int multiSelectOrderBy( #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* Forward Declarations */ static void substExprList(sqlite3*, ExprList*, int, ExprList*); -static void substSelect(sqlite3*, Select *, int, ExprList *); +static void substSelect(sqlite3*, Select *, int, ExprList*, int); /* ** Scan through the expression pExpr. Replace every reference to @@ -109584,7 +111756,7 @@ static Expr *substExpr( pExpr->pLeft = substExpr(db, pExpr->pLeft, iTable, pEList); pExpr->pRight = substExpr(db, pExpr->pRight, iTable, pEList); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - substSelect(db, pExpr->x.pSelect, iTable, pEList); + substSelect(db, pExpr->x.pSelect, iTable, pEList, 1); }else{ substExprList(db, pExpr->x.pList, iTable, pEList); } @@ -109607,25 +111779,28 @@ static void substSelect( sqlite3 *db, /* Report malloc errors here */ Select *p, /* SELECT statement in which to make substitutions */ int iTable, /* Table to be replaced */ - ExprList *pEList /* Substitute values */ + ExprList *pEList, /* Substitute values */ + int doPrior /* Do substitutes on p->pPrior too */ ){ SrcList *pSrc; struct SrcList_item *pItem; int i; if( !p ) return; - substExprList(db, p->pEList, iTable, pEList); - substExprList(db, p->pGroupBy, iTable, pEList); - substExprList(db, p->pOrderBy, iTable, pEList); - p->pHaving = substExpr(db, p->pHaving, iTable, pEList); - p->pWhere = substExpr(db, p->pWhere, iTable, pEList); - substSelect(db, p->pPrior, iTable, pEList); - pSrc = p->pSrc; - assert( pSrc ); /* Even for (SELECT 1) we have: pSrc!=0 but pSrc->nSrc==0 */ - if( ALWAYS(pSrc) ){ + do{ + substExprList(db, p->pEList, iTable, pEList); + substExprList(db, p->pGroupBy, iTable, pEList); + substExprList(db, p->pOrderBy, iTable, pEList); + p->pHaving = substExpr(db, p->pHaving, iTable, pEList); + p->pWhere = substExpr(db, p->pWhere, iTable, pEList); + pSrc = p->pSrc; + assert( pSrc!=0 ); for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ - substSelect(db, pItem->pSelect, iTable, pEList); + substSelect(db, pItem->pSelect, iTable, pEList, 1); + if( pItem->fg.isTabFunc ){ + substExprList(db, pItem->u1.pFuncArg, iTable, pEList); + } } - } + }while( doPrior && (p = p->pPrior)!=0 ); } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ @@ -109689,8 +111864,8 @@ static void substSelect( ** ** (**) Restriction (10) was removed from the code on 2005-02-05 but we ** accidently carried the comment forward until 2014-09-15. Original -** text: "The subquery does not use aggregates or the outer query does not -** use LIMIT." +** text: "The subquery does not use aggregates or the outer query +** does not use LIMIT." ** ** (11) The subquery and the outer query do not both have ORDER BY clauses. ** @@ -109777,7 +111952,7 @@ static int flattenSubquery( int subqueryIsAgg /* True if the subquery uses aggregate functions */ ){ const char *zSavedAuthContext = pParse->zAuthContext; - Select *pParent; + Select *pParent; /* Current UNION ALL term of the other query */ Select *pSub; /* The inner query or "subquery" */ Select *pSub1; /* Pointer to the rightmost select in sub-query */ SrcList *pSrc; /* The FROM clause of the outer query */ @@ -109880,7 +112055,7 @@ static int flattenSubquery( ** is fraught with danger. Best to avoid the whole thing. If the ** subquery is the right term of a LEFT JOIN, then do not flatten. */ - if( (pSubitem->jointype & JT_OUTER)!=0 ){ + if( (pSubitem->fg.jointype & JT_OUTER)!=0 ){ return 0; } @@ -109900,10 +112075,10 @@ static int flattenSubquery( testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ); testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Aggregate ); assert( pSub->pSrc!=0 ); + assert( pSub->pEList->nExpr==pSub1->pEList->nExpr ); if( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))!=0 || (pSub1->pPrior && pSub1->op!=TK_ALL) || pSub1->pSrc->nSrc<1 - || pSub->pEList->nExpr!=pSub1->pEList->nExpr ){ return 0; } @@ -110051,7 +112226,7 @@ static int flattenSubquery( if( pSrc ){ assert( pParent==p ); /* First time through the loop */ - jointype = pSubitem->jointype; + jointype = pSubitem->fg.jointype; }else{ assert( pParent!=p ); /* 2nd and subsequent times through the loop */ pSrc = pParent->pSrc = sqlite3SrcListAppend(db, 0, 0, 0); @@ -110072,9 +112247,9 @@ static int flattenSubquery( ** ** The outer query has 3 slots in its FROM clause. One slot of the ** outer query (the middle slot) is used by the subquery. The next - ** block of code will expand the out query to 4 slots. The middle - ** slot is expanded to two slots in order to make space for the - ** two elements in the FROM clause of the subquery. + ** block of code will expand the outer query FROM clause to 4 slots. + ** The middle slot is expanded to two slots in order to make space + ** for the two elements in the FROM clause of the subquery. */ if( nSubSrc>1 ){ pParent->pSrc = pSrc = sqlite3SrcListEnlarge(db, pSrc, nSubSrc-1,iFrom+1); @@ -110091,7 +112266,7 @@ static int flattenSubquery( pSrc->a[i+iFrom] = pSubSrc->a[i]; memset(&pSubSrc->a[i], 0, sizeof(pSubSrc->a[i])); } - pSrc->a[iFrom].jointype = jointype; + pSrc->a[iFrom].fg.jointype = jointype; /* Now begin substituting subquery result set expressions for ** references to the iParent in the outer query. @@ -110113,17 +112288,12 @@ static int flattenSubquery( pList->a[i].zName = zName; } } - substExprList(db, pParent->pEList, iParent, pSub->pEList); - if( isAgg ){ - substExprList(db, pParent->pGroupBy, iParent, pSub->pEList); - pParent->pHaving = substExpr(db, pParent->pHaving, iParent, pSub->pEList); - } if( pSub->pOrderBy ){ /* At this point, any non-zero iOrderByCol values indicate that the ** ORDER BY column expression is identical to the iOrderByCol'th ** expression returned by SELECT statement pSub. Since these values ** do not necessarily correspond to columns in SELECT statement pParent, - ** zero them before transferring the ORDER BY clause. + ** zero them before transfering the ORDER BY clause. ** ** Not doing this may cause an error if a subsequent call to this ** function attempts to flatten a compound sub-query into pParent @@ -110137,27 +112307,20 @@ static int flattenSubquery( assert( pSub->pPrior==0 ); pParent->pOrderBy = pOrderBy; pSub->pOrderBy = 0; - }else if( pParent->pOrderBy ){ - substExprList(db, pParent->pOrderBy, iParent, pSub->pEList); - } - if( pSub->pWhere ){ - pWhere = sqlite3ExprDup(db, pSub->pWhere, 0); - }else{ - pWhere = 0; } + pWhere = sqlite3ExprDup(db, pSub->pWhere, 0); if( subqueryIsAgg ){ assert( pParent->pHaving==0 ); pParent->pHaving = pParent->pWhere; pParent->pWhere = pWhere; - pParent->pHaving = substExpr(db, pParent->pHaving, iParent, pSub->pEList); pParent->pHaving = sqlite3ExprAnd(db, pParent->pHaving, sqlite3ExprDup(db, pSub->pHaving, 0)); assert( pParent->pGroupBy==0 ); pParent->pGroupBy = sqlite3ExprListDup(db, pSub->pGroupBy, 0); }else{ - pParent->pWhere = substExpr(db, pParent->pWhere, iParent, pSub->pEList); pParent->pWhere = sqlite3ExprAnd(db, pParent->pWhere, pWhere); } + substSelect(db, pParent, iParent, pSub->pEList, 0); /* The flattened query is distinct if either the inner or the ** outer query is distinct. @@ -110183,7 +112346,7 @@ static int flattenSubquery( #if SELECTTRACE_ENABLED if( sqlite3SelectTrace & 0x100 ){ - sqlite3DebugPrintf("After flattening:\n"); + SELECTTRACE(0x100,pParse,p,("After flattening:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -110192,6 +112355,77 @@ static int flattenSubquery( } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ + + +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) +/* +** Make copies of relevant WHERE clause terms of the outer query into +** the WHERE clause of subquery. Example: +** +** SELECT * FROM (SELECT a AS x, c-d AS y FROM t1) WHERE x=5 AND y=10; +** +** Transformed into: +** +** SELECT * FROM (SELECT a AS x, c-d AS y FROM t1 WHERE a=5 AND c-d=10) +** WHERE x=5 AND y=10; +** +** The hope is that the terms added to the inner query will make it more +** efficient. +** +** Do not attempt this optimization if: +** +** (1) The inner query is an aggregate. (In that case, we'd really want +** to copy the outer WHERE-clause terms onto the HAVING clause of the +** inner query. But they probably won't help there so do not bother.) +** +** (2) The inner query is the recursive part of a common table expression. +** +** (3) The inner query has a LIMIT clause (since the changes to the WHERE +** close would change the meaning of the LIMIT). +** +** (4) The inner query is the right operand of a LEFT JOIN. (The caller +** enforces this restriction since this routine does not have enough +** information to know.) +** +** (5) The WHERE clause expression originates in the ON or USING clause +** of a LEFT JOIN. +** +** Return 0 if no changes are made and non-zero if one or more WHERE clause +** terms are duplicated into the subquery. +*/ +static int pushDownWhereTerms( + sqlite3 *db, /* The database connection (for malloc()) */ + Select *pSubq, /* The subquery whose WHERE clause is to be augmented */ + Expr *pWhere, /* The WHERE clause of the outer query */ + int iCursor /* Cursor number of the subquery */ +){ + Expr *pNew; + int nChng = 0; + if( pWhere==0 ) return 0; + if( (pSubq->selFlags & (SF_Aggregate|SF_Recursive))!=0 ){ + return 0; /* restrictions (1) and (2) */ + } + if( pSubq->pLimit!=0 ){ + return 0; /* restriction (3) */ + } + while( pWhere->op==TK_AND ){ + nChng += pushDownWhereTerms(db, pSubq, pWhere->pRight, iCursor); + pWhere = pWhere->pLeft; + } + if( ExprHasProperty(pWhere,EP_FromJoin) ) return 0; /* restriction 5 */ + if( sqlite3ExprIsTableConstant(pWhere, iCursor) ){ + nChng++; + while( pSubq ){ + pNew = sqlite3ExprDup(db, pWhere, 0); + pNew = substExpr(db, pNew, iCursor, pSubq->pEList); + pSubq->pWhere = sqlite3ExprAnd(db, pSubq->pWhere, pNew); + pSubq = pSubq->pPrior; + } + } + return nChng; +} +#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ + /* ** Based on the contents of the AggInfo structure indicated by the first ** argument, this function checks if the following are true: @@ -110275,20 +112509,20 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ ** pFrom->pIndex and return SQLITE_OK. */ SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *pParse, struct SrcList_item *pFrom){ - if( pFrom->pTab && pFrom->zIndex ){ + if( pFrom->pTab && pFrom->fg.isIndexedBy ){ Table *pTab = pFrom->pTab; - char *zIndex = pFrom->zIndex; + char *zIndexedBy = pFrom->u1.zIndexedBy; Index *pIdx; for(pIdx=pTab->pIndex; - pIdx && sqlite3StrICmp(pIdx->zName, zIndex); + pIdx && sqlite3StrICmp(pIdx->zName, zIndexedBy); pIdx=pIdx->pNext ); if( !pIdx ){ - sqlite3ErrorMsg(pParse, "no such index: %s", zIndex, 0); + sqlite3ErrorMsg(pParse, "no such index: %s", zIndexedBy, 0); pParse->checkSchema = 1; return SQLITE_ERROR; } - pFrom->pIndex = pIdx; + pFrom->pIBIndex = pIdx; } return SQLITE_OK; } @@ -110449,12 +112683,12 @@ static int withExpand( int bMayRecursive; /* True if compound joined by UNION [ALL] */ With *pSavedWith; /* Initial value of pParse->pWith */ - /* If pCte->zErr is non-NULL at this point, then this is an illegal + /* If pCte->zCteErr is non-NULL at this point, then this is an illegal ** recursive reference to CTE pCte. Leave an error in pParse and return - ** early. If pCte->zErr is NULL, then this is not a recursive reference. + ** early. If pCte->zCteErr is NULL, then this is not a recursive reference. ** In this case, proceed. */ - if( pCte->zErr ){ - sqlite3ErrorMsg(pParse, pCte->zErr, pCte->zName); + if( pCte->zCteErr ){ + sqlite3ErrorMsg(pParse, pCte->zCteErr, pCte->zName); return SQLITE_ERROR; } @@ -110465,7 +112699,7 @@ static int withExpand( pTab->zName = sqlite3DbStrDup(db, pCte->zName); pTab->iPKey = -1; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); - pTab->tabFlags |= TF_Ephemeral; + pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid; pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); if( db->mallocFailed ) return SQLITE_NOMEM; assert( pFrom->pSelect ); @@ -110483,7 +112717,7 @@ static int withExpand( && 0==sqlite3StrICmp(pItem->zName, pCte->zName) ){ pItem->pTab = pTab; - pItem->isRecursive = 1; + pItem->fg.isRecursive = 1; pTab->nRef++; pSel->selFlags |= SF_Recursive; } @@ -110499,7 +112733,7 @@ static int withExpand( } assert( pTab->nRef==1 || ((pSel->selFlags&SF_Recursive) && pTab->nRef==2 )); - pCte->zErr = "circular reference: %s"; + pCte->zCteErr = "circular reference: %s"; pSavedWith = pParse->pWith; pParse->pWith = pWith; sqlite3WalkSelect(pWalker, bMayRecursive ? pSel->pPrior : pSel); @@ -110517,16 +112751,16 @@ static int withExpand( pEList = pCte->pCols; } - selectColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); + sqlite3ColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); if( bMayRecursive ){ if( pSel->selFlags & SF_Recursive ){ - pCte->zErr = "multiple recursive references: %s"; + pCte->zCteErr = "multiple recursive references: %s"; }else{ - pCte->zErr = "recursive reference in a subquery: %s"; + pCte->zCteErr = "recursive reference in a subquery: %s"; } sqlite3WalkSelect(pWalker, pSel); } - pCte->zErr = 0; + pCte->zCteErr = 0; pParse->pWith = pSavedWith; } @@ -110613,17 +112847,9 @@ static int selectExpander(Walker *pWalker, Select *p){ */ for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ Table *pTab; - assert( pFrom->isRecursive==0 || pFrom->pTab ); - if( pFrom->isRecursive ) continue; - if( pFrom->pTab!=0 ){ - /* This statement has already been prepared. There is no need - ** to go further. */ - assert( i==0 ); -#ifndef SQLITE_OMIT_CTE - selectPopWith(pWalker, p); -#endif - return WRC_Prune; - } + assert( pFrom->fg.isRecursive==0 || pFrom->pTab!=0 ); + if( pFrom->fg.isRecursive ) continue; + assert( pFrom->pTab==0 ); #ifndef SQLITE_OMIT_CTE if( withExpand(pWalker, pFrom) ) return WRC_Abort; if( pFrom->pTab ) {} else @@ -110640,7 +112866,7 @@ static int selectExpander(Walker *pWalker, Select *p){ pTab->nRef = 1; pTab->zName = sqlite3MPrintf(db, "sqlite_sq_%p", (void*)pTab); while( pSel->pPrior ){ pSel = pSel->pPrior; } - selectColumnsFromExprList(pParse, pSel->pEList, &pTab->nCol, &pTab->aCol); + sqlite3ColumnsFromExprList(pParse, pSel->pEList,&pTab->nCol,&pTab->aCol); pTab->iPKey = -1; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral; @@ -110659,12 +112885,19 @@ static int selectExpander(Walker *pWalker, Select *p){ pTab->nRef++; #if !defined(SQLITE_OMIT_VIEW) || !defined (SQLITE_OMIT_VIRTUALTABLE) if( pTab->pSelect || IsVirtual(pTab) ){ - /* We reach here if the named table is a really a view */ + i16 nCol; if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort; assert( pFrom->pSelect==0 ); + if( pFrom->fg.isTabFunc && !IsVirtual(pTab) ){ + sqlite3ErrorMsg(pParse, "'%s' is not a function", pTab->zName); + return WRC_Abort; + } pFrom->pSelect = sqlite3SelectDup(db, pTab->pSelect, 0); sqlite3SelectSetName(pFrom->pSelect, pTab->zName); + nCol = pTab->nCol; + pTab->nCol = -1; sqlite3WalkSelect(pWalker, pFrom->pSelect); + pTab->nCol = nCol; } #endif } @@ -110710,13 +112943,6 @@ static int selectExpander(Walker *pWalker, Select *p){ int longNames = (flags & SQLITE_FullColNames)!=0 && (flags & SQLITE_ShortColNames)==0; - /* When processing FROM-clause subqueries, it is always the case - ** that full_column_names=OFF and short_column_names=ON. The - ** sqlite3ResultSetOfSelect() routine makes it so. */ - assert( (p->selFlags & SF_NestedFrom)==0 - || ((flags & SQLITE_FullColNames)==0 && - (flags & SQLITE_ShortColNames)!=0) ); - for(k=0; knExpr; k++){ pE = a[k].pExpr; pRight = pE->pRight; @@ -110784,7 +113010,7 @@ static int selectExpander(Walker *pWalker, Select *p){ tableSeen = 1; if( i>0 && zTName==0 ){ - if( (pFrom->jointype & JT_NATURAL)!=0 + if( (pFrom->fg.jointype & JT_NATURAL)!=0 && tableAndColumnIndex(pTabList, i, zName, 0, 0) ){ /* In a NATURAL join, omit the join columns from the @@ -110919,19 +113145,19 @@ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ struct SrcList_item *pFrom; assert( p->selFlags & SF_Resolved ); - if( (p->selFlags & SF_HasTypeInfo)==0 ){ - p->selFlags |= SF_HasTypeInfo; - pParse = pWalker->pParse; - pTabList = p->pSrc; - for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ - Table *pTab = pFrom->pTab; - if( ALWAYS(pTab!=0) && (pTab->tabFlags & TF_Ephemeral)!=0 ){ - /* A sub-query in the FROM clause of a SELECT */ - Select *pSel = pFrom->pSelect; - if( pSel ){ - while( pSel->pPrior ) pSel = pSel->pPrior; - selectAddColumnTypeAndCollation(pParse, pTab, pSel); - } + assert( (p->selFlags & SF_HasTypeInfo)==0 ); + p->selFlags |= SF_HasTypeInfo; + pParse = pWalker->pParse; + pTabList = p->pSrc; + for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ + Table *pTab = pFrom->pTab; + assert( pTab!=0 ); + if( (pTab->tabFlags & TF_Ephemeral)!=0 ){ + /* A sub-query in the FROM clause of a SELECT */ + Select *pSel = pFrom->pSelect; + if( pSel ){ + while( pSel->pPrior ) pSel = pSel->pPrior; + selectAddColumnTypeAndCollation(pParse, pTab, pSel); } } } @@ -111070,7 +113296,7 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ if( pList ){ nArg = pList->nExpr; regAgg = sqlite3GetTempRange(pParse, nArg); - sqlite3ExprCodeExprList(pParse, pList, regAgg, SQLITE_ECEL_DUP); + sqlite3ExprCodeExprList(pParse, pList, regAgg, 0, SQLITE_ECEL_DUP); }else{ nArg = 0; regAgg = 0; @@ -111095,7 +113321,7 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem; sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, (char *)pColl, P4_COLLSEQ); } - sqlite3VdbeAddOp4(v, OP_AggStep, 0, regAgg, pF->iMem, + sqlite3VdbeAddOp4(v, OP_AggStep0, 0, regAgg, pF->iMem, (void*)pF->pFunc, P4_FUNCDEF); sqlite3VdbeChangeP5(v, (u8)nArg); sqlite3ExprCacheAffinityChange(pParse, regAgg, nArg); @@ -111178,7 +113404,7 @@ SQLITE_PRIVATE int sqlite3Select( WhereInfo *pWInfo; /* Return from sqlite3WhereBegin() */ Vdbe *v; /* The virtual machine under construction */ int isAgg; /* True for select lists like "count(*)" */ - ExprList *pEList; /* List of columns to extract. */ + ExprList *pEList = 0; /* List of columns to extract. */ SrcList *pTabList; /* List of tables to select from */ Expr *pWhere; /* The WHERE clause. May be NULL */ ExprList *pGroupBy; /* The GROUP BY clause. May be NULL */ @@ -111228,12 +113454,11 @@ SQLITE_PRIVATE int sqlite3Select( memset(&sSort, 0, sizeof(sSort)); sSort.pOrderBy = p->pOrderBy; pTabList = p->pSrc; - pEList = p->pEList; if( pParse->nErr || db->mallocFailed ){ goto select_end; } + assert( p->pEList!=0 ); isAgg = (p->selFlags & SF_Aggregate)!=0; - assert( pEList!=0 ); #if SELECTTRACE_ENABLED if( sqlite3SelectTrace & 0x100 ){ SELECTTRACE(0x100,pParse,p, ("after name resolution:\n")); @@ -111242,29 +113467,77 @@ SQLITE_PRIVATE int sqlite3Select( #endif - /* Begin generating code. - */ - v = sqlite3GetVdbe(pParse); - if( v==0 ) goto select_end; - /* If writing to memory or generating a set ** only a single column may be output. */ #ifndef SQLITE_OMIT_SUBQUERY - if( checkForMultiColumnSelectError(pParse, pDest, pEList->nExpr) ){ + if( checkForMultiColumnSelectError(pParse, pDest, p->pEList->nExpr) ){ goto select_end; } #endif - /* Generate code for all sub-queries in the FROM clause + /* Try to flatten subqueries in the FROM clause up into the main query */ #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) for(i=0; !p->pPrior && inSrc; i++){ struct SrcList_item *pItem = &pTabList->a[i]; - SelectDest dest; Select *pSub = pItem->pSelect; int isAggSub; + Table *pTab = pItem->pTab; + if( pSub==0 ) continue; + + /* Catch mismatch in the declared columns of a view and the number of + ** columns in the SELECT on the RHS */ + if( pTab->nCol!=pSub->pEList->nExpr ){ + sqlite3ErrorMsg(pParse, "expected %d columns for '%s' but got %d", + pTab->nCol, pTab->zName, pSub->pEList->nExpr); + goto select_end; + } + + isAggSub = (pSub->selFlags & SF_Aggregate)!=0; + if( flattenSubquery(pParse, p, i, isAgg, isAggSub) ){ + /* This subquery can be absorbed into its parent. */ + if( isAggSub ){ + isAgg = 1; + p->selFlags |= SF_Aggregate; + } + i = -1; + } + pTabList = p->pSrc; + if( db->mallocFailed ) goto select_end; + if( !IgnorableOrderby(pDest) ){ + sSort.pOrderBy = p->pOrderBy; + } + } +#endif + + /* Get a pointer the VDBE under construction, allocating a new VDBE if one + ** does not already exist */ + v = sqlite3GetVdbe(pParse); + if( v==0 ) goto select_end; + +#ifndef SQLITE_OMIT_COMPOUND_SELECT + /* Handle compound SELECT statements using the separate multiSelect() + ** procedure. + */ + if( p->pPrior ){ + rc = multiSelect(pParse, p, pDest); + explainSetInteger(pParse->iSelectId, iRestoreSelectId); +#if SELECTTRACE_ENABLED + SELECTTRACE(1,pParse,p,("end compound-select processing\n")); + pParse->nSelectIndent--; +#endif + return rc; + } +#endif + /* Generate code for all sub-queries in the FROM clause + */ +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) + for(i=0; inSrc; i++){ + struct SrcList_item *pItem = &pTabList->a[i]; + SelectDest dest; + Select *pSub = pItem->pSelect; if( pSub==0 ) continue; /* Sometimes the code for a subquery will be generated more than @@ -111274,7 +113547,7 @@ SQLITE_PRIVATE int sqlite3Select( ** is sufficient, though the subroutine to manifest the view does need ** to be invoked again. */ if( pItem->addrFillSub ){ - if( pItem->viaCoroutine==0 ){ + if( pItem->fg.viaCoroutine==0 ){ sqlite3VdbeAddOp2(v, OP_Gosub, pItem->regReturn, pItem->addrFillSub); } continue; @@ -111289,16 +113562,25 @@ SQLITE_PRIVATE int sqlite3Select( */ pParse->nHeight += sqlite3SelectExprHeight(p); - isAggSub = (pSub->selFlags & SF_Aggregate)!=0; - if( flattenSubquery(pParse, p, i, isAgg, isAggSub) ){ - /* This subquery can be absorbed into its parent. */ - if( isAggSub ){ - isAgg = 1; - p->selFlags |= SF_Aggregate; + /* Make copies of constant WHERE-clause terms in the outer query down + ** inside the subquery. This can help the subquery to run more efficiently. + */ + if( (pItem->fg.jointype & JT_OUTER)==0 + && pushDownWhereTerms(db, pSub, p->pWhere, pItem->iCursor) + ){ +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x100 ){ + SELECTTRACE(0x100,pParse,p,("After WHERE-clause push-down:\n")); + sqlite3TreeViewSelect(0, p, 0); } - i = -1; - }else if( pTabList->nSrc==1 - && OptimizationEnabled(db, SQLITE_SubqCoroutine) +#endif + } + + /* Generate code to implement the subquery + */ + if( pTabList->nSrc==1 + && (p->selFlags & SF_All)==0 + && OptimizationEnabled(db, SQLITE_SubqCoroutine) ){ /* Implement a co-routine that will return a single row of the result ** set on each invocation. @@ -111312,7 +113594,7 @@ SQLITE_PRIVATE int sqlite3Select( explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); - pItem->viaCoroutine = 1; + pItem->fg.viaCoroutine = 1; pItem->regResult = dest.iSdst; sqlite3VdbeAddOp1(v, OP_EndCoroutine, pItem->regReturn); sqlite3VdbeJumpHere(v, addrTop-1); @@ -111330,7 +113612,7 @@ SQLITE_PRIVATE int sqlite3Select( pItem->regReturn = ++pParse->nMem; topAddr = sqlite3VdbeAddOp2(v, OP_Integer, 0, pItem->regReturn); pItem->addrFillSub = topAddr+1; - if( pItem->isCorrelated==0 ){ + if( pItem->fg.isCorrelated==0 ){ /* If the subquery is not correlated and if we are not inside of ** a trigger, then we only need to compute the value of the subquery ** once. */ @@ -111349,33 +113631,23 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3VdbeChangeP1(v, topAddr, retAddr); sqlite3ClearTempRegCache(pParse); } - if( /*pParse->nErr ||*/ db->mallocFailed ){ - goto select_end; - } + if( db->mallocFailed ) goto select_end; pParse->nHeight -= sqlite3SelectExprHeight(p); - pTabList = p->pSrc; - if( !IgnorableOrderby(pDest) ){ - sSort.pOrderBy = p->pOrderBy; - } } - pEList = p->pEList; #endif + + /* Various elements of the SELECT copied into local variables for + ** convenience */ + pEList = p->pEList; pWhere = p->pWhere; pGroupBy = p->pGroupBy; pHaving = p->pHaving; sDistinct.isTnct = (p->selFlags & SF_Distinct)!=0; -#ifndef SQLITE_OMIT_COMPOUND_SELECT - /* If there is are a sequence of queries, do the earlier ones first. - */ - if( p->pPrior ){ - rc = multiSelect(pParse, p, pDest); - explainSetInteger(pParse->iSelectId, iRestoreSelectId); #if SELECTTRACE_ENABLED - SELECTTRACE(1,pParse,p,("end compound-select processing\n")); - pParse->nSelectIndent--; -#endif - return rc; + if( sqlite3SelectTrace & 0x400 ){ + SELECTTRACE(0x400,pParse,p,("After all FROM-clause analysis:\n")); + sqlite3TreeViewSelect(0, p, 0); } #endif @@ -111395,23 +113667,23 @@ SQLITE_PRIVATE int sqlite3Select( ** BY and DISTINCT, and an index or separate temp-table for the other. */ if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct - && sqlite3ExprListCompare(sSort.pOrderBy, p->pEList, -1)==0 + && sqlite3ExprListCompare(sSort.pOrderBy, pEList, -1)==0 ){ p->selFlags &= ~SF_Distinct; - p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); - pGroupBy = p->pGroupBy; + pGroupBy = p->pGroupBy = sqlite3ExprListDup(db, pEList, 0); /* Notice that even thought SF_Distinct has been cleared from p->selFlags, ** the sDistinct.isTnct is still set. Hence, isTnct represents the ** original setting of the SF_Distinct flag, not the current setting */ assert( sDistinct.isTnct ); } - /* If there is an ORDER BY clause, then this sorting - ** index might end up being unused if the data can be - ** extracted in pre-sorted order. If that is the case, then the - ** OP_OpenEphemeral instruction will be changed to an OP_Noop once - ** we figure out that the sorting index is not needed. The addrSortIndex - ** variable is used to facilitate that change. + /* If there is an ORDER BY clause, then create an ephemeral index to + ** do the sorting. But this sorting ephemeral index might end up + ** being unused if the data can be extracted in pre-sorted order. + ** If that is the case, then the OP_OpenEphemeral instruction will be + ** changed to an OP_Noop once we figure out that the sorting index is + ** not needed. The sSort.addrSortIndex variable is used to facilitate + ** that change. */ if( sSort.pOrderBy ){ KeyInfo *pKeyInfo; @@ -111438,18 +113710,18 @@ SQLITE_PRIVATE int sqlite3Select( p->nSelectRow = LARGEST_INT64; computeLimitRegisters(pParse, p, iEnd); if( p->iLimit==0 && sSort.addrSortIndex>=0 ){ - sqlite3VdbeGetOp(v, sSort.addrSortIndex)->opcode = OP_SorterOpen; + sqlite3VdbeChangeOpcode(v, sSort.addrSortIndex, OP_SorterOpen); sSort.sortFlags |= SORTFLAG_UseSorter; } - /* Open a virtual index to use for the distinct set. + /* Open an ephemeral index to use for the distinct set. */ if( p->selFlags & SF_Distinct ){ sDistinct.tabTnct = pParse->nTab++; sDistinct.addrTnct = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, - sDistinct.tabTnct, 0, 0, - (char*)keyInfoFromExprList(pParse, p->pEList,0,0), - P4_KEYINFO); + sDistinct.tabTnct, 0, 0, + (char*)keyInfoFromExprList(pParse, p->pEList,0,0), + P4_KEYINFO); sqlite3VdbeChangeP5(v, BTREE_UNORDERED); sDistinct.eTnctType = WHERE_DISTINCT_UNORDERED; }else{ @@ -111527,11 +113799,10 @@ SQLITE_PRIVATE int sqlite3Select( p->nSelectRow = 1; } - /* If there is both a GROUP BY and an ORDER BY clause and they are ** identical, then it may be possible to disable the ORDER BY clause ** on the grounds that the GROUP BY will cause elements to come out - ** in the correct order. It also may not - the GROUP BY may use a + ** in the correct order. It also may not - the GROUP BY might use a ** database index that causes rows to be grouped together as required ** but not actually sorted. Either way, record the fact that the ** ORDER BY and GROUP BY clauses are the same by setting the orderByGrp @@ -111574,7 +113845,7 @@ SQLITE_PRIVATE int sqlite3Select( */ if( pGroupBy ){ KeyInfo *pKeyInfo; /* Keying information for the group by clause */ - int j1; /* A-vs-B comparision jump */ + int addr1; /* A-vs-B comparision jump */ int addrOutputRow; /* Start of subroutine that outputs a result row */ int regOutputRow; /* Return address register for output subroutine */ int addrSetAbort; /* Set the abort flag and return */ @@ -111655,7 +113926,7 @@ SQLITE_PRIVATE int sqlite3Select( } regBase = sqlite3GetTempRange(pParse, nCol); sqlite3ExprCacheClear(pParse); - sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0); + sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0, 0); j = nGroupBy; for(i=0; inExpr; j++){ if( groupBySort ){ @@ -111721,8 +113993,8 @@ SQLITE_PRIVATE int sqlite3Select( } sqlite3VdbeAddOp4(v, OP_Compare, iAMem, iBMem, pGroupBy->nExpr, (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO); - j1 = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp3(v, OP_Jump, j1+1, 0, j1+1); VdbeCoverage(v); + addr1 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_Jump, addr1+1, 0, addr1+1); VdbeCoverage(v); /* Generate code that runs whenever the GROUP BY changes. ** Changes in the GROUP BY are detected by the previous code @@ -111744,7 +114016,7 @@ SQLITE_PRIVATE int sqlite3Select( /* Update the aggregate accumulators based on the content of ** the current row */ - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); updateAccumulator(pParse, &sAggInfo); sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag); VdbeComment((v, "indicate data in accumulator")); @@ -111766,7 +114038,7 @@ SQLITE_PRIVATE int sqlite3Select( /* Jump over the subroutines */ - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEnd); + sqlite3VdbeGoto(v, addrEnd); /* Generate a subroutine that outputs a single row of the result ** set. This subroutine first looks at the iUseFlag. If iUseFlag @@ -111781,7 +114053,8 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); sqlite3VdbeResolveLabel(v, addrOutputRow); addrOutputRow = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_IfPos, iUseFlag, addrOutputRow+2); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_IfPos, iUseFlag, addrOutputRow+2); + VdbeCoverage(v); VdbeComment((v, "Groupby result generator entry point")); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); finalizeAggFunctions(pParse, &sAggInfo); @@ -111919,7 +114192,7 @@ SQLITE_PRIVATE int sqlite3Select( updateAccumulator(pParse, &sAggInfo); assert( pMinMax==0 || pMinMax->nExpr==1 ); if( sqlite3WhereIsOrdered(pWInfo)>0 ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3WhereBreakLabel(pWInfo)); + sqlite3VdbeGoto(v, sqlite3WhereBreakLabel(pWInfo)); VdbeComment((v, "%s() by index", (flag==WHERE_ORDERBY_MIN?"min":"max"))); } @@ -111945,7 +114218,8 @@ SQLITE_PRIVATE int sqlite3Select( ** and send them to the callback one by one. */ if( sSort.pOrderBy ){ - explainTempTable(pParse, sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY"); + explainTempTable(pParse, + sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY"); generateSortTail(pParse, p, &sSort, pEList->nExpr, pDest); } @@ -111978,100 +114252,6 @@ select_end: return rc; } -#ifdef SQLITE_DEBUG -/* -** Generate a human-readable description of a the Select object. -*/ -SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ - int n = 0; - pView = sqlite3TreeViewPush(pView, moreToFollow); - sqlite3TreeViewLine(pView, "SELECT%s%s (0x%p)", - ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), - ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), p - ); - if( p->pSrc && p->pSrc->nSrc ) n++; - if( p->pWhere ) n++; - if( p->pGroupBy ) n++; - if( p->pHaving ) n++; - if( p->pOrderBy ) n++; - if( p->pLimit ) n++; - if( p->pOffset ) n++; - if( p->pPrior ) n++; - sqlite3TreeViewExprList(pView, p->pEList, (n--)>0, "result-set"); - if( p->pSrc && p->pSrc->nSrc ){ - int i; - pView = sqlite3TreeViewPush(pView, (n--)>0); - sqlite3TreeViewLine(pView, "FROM"); - for(i=0; ipSrc->nSrc; i++){ - struct SrcList_item *pItem = &p->pSrc->a[i]; - StrAccum x; - char zLine[100]; - sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); - sqlite3XPrintf(&x, 0, "{%d,*}", pItem->iCursor); - if( pItem->zDatabase ){ - sqlite3XPrintf(&x, 0, " %s.%s", pItem->zDatabase, pItem->zName); - }else if( pItem->zName ){ - sqlite3XPrintf(&x, 0, " %s", pItem->zName); - } - if( pItem->pTab ){ - sqlite3XPrintf(&x, 0, " tabname=%Q", pItem->pTab->zName); - } - if( pItem->zAlias ){ - sqlite3XPrintf(&x, 0, " (AS %s)", pItem->zAlias); - } - if( pItem->jointype & JT_LEFT ){ - sqlite3XPrintf(&x, 0, " LEFT-JOIN"); - } - sqlite3StrAccumFinish(&x); - sqlite3TreeViewItem(pView, zLine, ipSrc->nSrc-1); - if( pItem->pSelect ){ - sqlite3TreeViewSelect(pView, pItem->pSelect, 0); - } - sqlite3TreeViewPop(pView); - } - sqlite3TreeViewPop(pView); - } - if( p->pWhere ){ - sqlite3TreeViewItem(pView, "WHERE", (n--)>0); - sqlite3TreeViewExpr(pView, p->pWhere, 0); - sqlite3TreeViewPop(pView); - } - if( p->pGroupBy ){ - sqlite3TreeViewExprList(pView, p->pGroupBy, (n--)>0, "GROUPBY"); - } - if( p->pHaving ){ - sqlite3TreeViewItem(pView, "HAVING", (n--)>0); - sqlite3TreeViewExpr(pView, p->pHaving, 0); - sqlite3TreeViewPop(pView); - } - if( p->pOrderBy ){ - sqlite3TreeViewExprList(pView, p->pOrderBy, (n--)>0, "ORDERBY"); - } - if( p->pLimit ){ - sqlite3TreeViewItem(pView, "LIMIT", (n--)>0); - sqlite3TreeViewExpr(pView, p->pLimit, 0); - sqlite3TreeViewPop(pView); - } - if( p->pOffset ){ - sqlite3TreeViewItem(pView, "OFFSET", (n--)>0); - sqlite3TreeViewExpr(pView, p->pOffset, 0); - sqlite3TreeViewPop(pView); - } - if( p->pPrior ){ - const char *zOp = "UNION"; - switch( p->op ){ - case TK_ALL: zOp = "UNION ALL"; break; - case TK_INTERSECT: zOp = "INTERSECT"; break; - case TK_EXCEPT: zOp = "EXCEPT"; break; - } - sqlite3TreeViewItem(pView, zOp, (n--)>0); - sqlite3TreeViewSelect(pView, p->pPrior, 0); - sqlite3TreeViewPop(pView); - } - sqlite3TreeViewPop(pView); -} -#endif /* SQLITE_DEBUG */ - /************** End of select.c **********************************************/ /************** Begin file table.c *******************************************/ /* @@ -112092,6 +114272,7 @@ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 m ** These routines are in a separate files so that they will not be linked ** if they are not used. */ +/* #include "sqliteInt.h" */ /* #include */ /* #include */ @@ -112288,6 +114469,7 @@ SQLITE_API void SQLITE_STDCALL sqlite3_free_table( ************************************************************************* ** This file contains the implementation for TRIGGERs */ +/* #include "sqliteInt.h" */ #ifndef SQLITE_OMIT_TRIGGER /* @@ -113411,6 +115593,7 @@ SQLITE_PRIVATE u32 sqlite3TriggerColmask( ** This file contains C code routines that are called by the parser ** to handle UPDATE statements. */ +/* #include "sqliteInt.h" */ #ifndef SQLITE_OMIT_VIRTUALTABLE /* Forward declaration */ @@ -113532,9 +115715,9 @@ SQLITE_PRIVATE void sqlite3Update( /* Register Allocations */ int regRowCount = 0; /* A count of rows changed */ - int regOldRowid; /* The old rowid */ - int regNewRowid; /* The new rowid */ - int regNew; /* Content of the NEW.* table in triggers */ + int regOldRowid = 0; /* The old rowid */ + int regNewRowid = 0; /* The new rowid */ + int regNew = 0; /* Content of the NEW.* table in triggers */ int regOld = 0; /* Content of OLD.* table in triggers */ int regRowSet = 0; /* Rowset of rows to be updated */ int regKey = 0; /* composite PRIMARY KEY value */ @@ -113670,7 +115853,9 @@ SQLITE_PRIVATE void sqlite3Update( /* There is one entry in the aRegIdx[] array for each index on the table ** being updated. Fill in aRegIdx[] with a register number that will hold - ** the key for accessing each index. + ** the key for accessing each index. + ** + ** FIXME: Be smarter about omitting indexes that use expressions. */ for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ int reg; @@ -113679,7 +115864,8 @@ SQLITE_PRIVATE void sqlite3Update( }else{ reg = 0; for(i=0; inKeyCol; i++){ - if( aXRef[pIdx->aiColumn[i]]>=0 ){ + i16 iIdxCol = pIdx->aiColumn[i]; + if( iIdxCol<0 || aXRef[iIdxCol]>=0 ){ reg = ++pParse->nMem; break; } @@ -113695,29 +115881,20 @@ SQLITE_PRIVATE void sqlite3Update( if( pParse->nested==0 ) sqlite3VdbeCountChanges(v); sqlite3BeginWriteOperation(pParse, 1, iDb); -#ifndef SQLITE_OMIT_VIRTUALTABLE - /* Virtual tables must be handled separately */ - if( IsVirtual(pTab) ){ - updateVirtualTable(pParse, pTabList, pTab, pChanges, pRowidExpr, aXRef, - pWhere, onError); - pWhere = 0; - pTabList = 0; - goto update_cleanup; - } -#endif - /* Allocate required registers. */ - regRowSet = ++pParse->nMem; - regOldRowid = regNewRowid = ++pParse->nMem; - if( chngPk || pTrigger || hasFK ){ - regOld = pParse->nMem + 1; + if( !IsVirtual(pTab) ){ + regRowSet = ++pParse->nMem; + regOldRowid = regNewRowid = ++pParse->nMem; + if( chngPk || pTrigger || hasFK ){ + regOld = pParse->nMem + 1; + pParse->nMem += pTab->nCol; + } + if( chngKey || pTrigger || hasFK ){ + regNewRowid = ++pParse->nMem; + } + regNew = pParse->nMem + 1; pParse->nMem += pTab->nCol; } - if( chngKey || pTrigger || hasFK ){ - regNewRowid = ++pParse->nMem; - } - regNew = pParse->nMem + 1; - pParse->nMem += pTab->nCol; /* Start the view context. */ if( isView ){ @@ -113740,6 +115917,15 @@ SQLITE_PRIVATE void sqlite3Update( goto update_cleanup; } +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* Virtual tables must be handled separately */ + if( IsVirtual(pTab) ){ + updateVirtualTable(pParse, pTabList, pTab, pChanges, pRowidExpr, aXRef, + pWhere, onError); + goto update_cleanup; + } +#endif + /* Begin the database scan */ if( HasRowid(pTab) ){ @@ -113779,6 +115965,7 @@ SQLITE_PRIVATE void sqlite3Update( if( pWInfo==0 ) goto update_cleanup; okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); for(i=0; iaiColumn[i]>=0 ); sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, pPk->aiColumn[i], iPk+i); } @@ -113788,7 +115975,7 @@ SQLITE_PRIVATE void sqlite3Update( regKey = iPk; }else{ sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, regKey, - sqlite3IndexAffinityStr(v, pPk), nPk); + sqlite3IndexAffinityStr(db, pPk), nPk); sqlite3VdbeAddOp2(v, OP_IdxInsert, iEph, regKey); } sqlite3WhereEnd(pWInfo); @@ -113901,7 +116088,6 @@ SQLITE_PRIVATE void sqlite3Update( newmask = sqlite3TriggerColmask( pParse, pTrigger, pChanges, 1, TRIGGER_BEFORE, pTab, onError ); - /*sqlite3VdbeAddOp3(v, OP_Null, 0, regNew, regNew+pTab->nCol-1);*/ for(i=0; inCol; i++){ if( i==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i); @@ -113959,7 +116145,7 @@ SQLITE_PRIVATE void sqlite3Update( } if( !isView ){ - int j1 = 0; /* Address of jump instruction */ + int addr1 = 0; /* Address of jump instruction */ int bReplace = 0; /* True if REPLACE conflict resolution might happen */ /* Do constraint checks. */ @@ -113975,20 +116161,20 @@ SQLITE_PRIVATE void sqlite3Update( /* Delete the index entries associated with the current record. */ if( bReplace || chngKey ){ if( pPk ){ - j1 = sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, 0, regKey, nKey); + addr1 = sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, 0, regKey, nKey); }else{ - j1 = sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, 0, regOldRowid); + addr1 = sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, 0, regOldRowid); } VdbeCoverageNeverTaken(v); } - sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, aRegIdx); + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, aRegIdx, -1); /* If changing the record number, delete the old record. */ if( hasFK || chngKey || pPk!=0 ){ sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, 0); } if( bReplace || chngKey ){ - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); } if( hasFK ){ @@ -114025,7 +116211,7 @@ SQLITE_PRIVATE void sqlite3Update( sqlite3VdbeResolveLabel(v, labelContinue); sqlite3VdbeAddOp2(v, OP_Next, iEph, addrTop); VdbeCoverage(v); }else{ - sqlite3VdbeAddOp2(v, OP_Goto, 0, labelContinue); + sqlite3VdbeGoto(v, labelContinue); } sqlite3VdbeResolveLabel(v, labelBreak); @@ -114079,21 +116265,23 @@ update_cleanup: /* ** Generate code for an UPDATE of a virtual table. ** -** The strategy is that we create an ephemeral table that contains +** There are two possible strategies - the default and the special +** "onepass" strategy. Onepass is only used if the virtual table +** implementation indicates that pWhere may match at most one row. +** +** The default strategy is to create an ephemeral table that contains ** for each row to be changed: ** ** (A) The original rowid of that row. -** (B) The revised rowid for the row. (note1) +** (B) The revised rowid for the row. ** (C) The content of every column in the row. ** -** Then we loop over this ephemeral table and for each row in -** the ephemeral table call VUpdate. +** Then loop through the contents of this ephemeral table executing a +** VUpdate for each row. When finished, drop the ephemeral table. ** -** When finished, drop the ephemeral table. -** -** (note1) Actually, if we know in advance that (A) is always the same -** as (B) we only store (A), then duplicate (A) when pulling -** it out of the ephemeral table before calling VUpdate. +** The "onepass" strategy does not use an ephemeral table. Instead, it +** stores the same values (A, B and C above) in a register array and +** makes a single invocation of VUpdate. */ static void updateVirtualTable( Parse *pParse, /* The parsing context */ @@ -114106,68 +116294,96 @@ static void updateVirtualTable( int onError /* ON CONFLICT strategy */ ){ Vdbe *v = pParse->pVdbe; /* Virtual machine under construction */ - ExprList *pEList = 0; /* The result set of the SELECT statement */ - Select *pSelect = 0; /* The SELECT statement */ - Expr *pExpr; /* Temporary expression */ int ephemTab; /* Table holding the result of the SELECT */ int i; /* Loop counter */ - int addr; /* Address of top of loop */ - int iReg; /* First register in set passed to OP_VUpdate */ sqlite3 *db = pParse->db; /* Database connection */ const char *pVTab = (const char*)sqlite3GetVTable(db, pTab); - SelectDest dest; - - /* Construct the SELECT statement that will find the new values for - ** all updated rows. - */ - pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ID, "_rowid_")); + WhereInfo *pWInfo; + int nArg = 2 + pTab->nCol; /* Number of arguments to VUpdate */ + int regArg; /* First register in VUpdate arg array */ + int regRec; /* Register in which to assemble record */ + int regRowid; /* Register for ephem table rowid */ + int iCsr = pSrc->a[0].iCursor; /* Cursor used for virtual table scan */ + int aDummy[2]; /* Unused arg for sqlite3WhereOkOnePass() */ + int bOnePass; /* True to use onepass strategy */ + int addr; /* Address of OP_OpenEphemeral */ + + /* Allocate nArg registers to martial the arguments to VUpdate. Then + ** create and open the ephemeral table in which the records created from + ** these arguments will be temporarily stored. */ + assert( v ); + ephemTab = pParse->nTab++; + addr= sqlite3VdbeAddOp2(v, OP_OpenEphemeral, ephemTab, nArg); + regArg = pParse->nMem + 1; + pParse->nMem += nArg; + regRec = ++pParse->nMem; + regRowid = ++pParse->nMem; + + /* Start scanning the virtual table */ + pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0,0,WHERE_ONEPASS_DESIRED,0); + if( pWInfo==0 ) return; + + /* Populate the argument registers. */ + sqlite3VdbeAddOp2(v, OP_Rowid, iCsr, regArg); if( pRowid ){ - pEList = sqlite3ExprListAppend(pParse, pEList, - sqlite3ExprDup(db, pRowid, 0)); + sqlite3ExprCode(pParse, pRowid, regArg+1); + }else{ + sqlite3VdbeAddOp2(v, OP_Rowid, iCsr, regArg+1); } - assert( pTab->iPKey<0 ); for(i=0; inCol; i++){ if( aXRef[i]>=0 ){ - pExpr = sqlite3ExprDup(db, pChanges->a[aXRef[i]].pExpr, 0); + sqlite3ExprCode(pParse, pChanges->a[aXRef[i]].pExpr, regArg+2+i); }else{ - pExpr = sqlite3Expr(db, TK_ID, pTab->aCol[i].zName); + sqlite3VdbeAddOp3(v, OP_VColumn, iCsr, i, regArg+2+i); } - pEList = sqlite3ExprListAppend(pParse, pEList, pExpr); } - pSelect = sqlite3SelectNew(pParse, pEList, pSrc, pWhere, 0, 0, 0, 0, 0, 0); - - /* Create the ephemeral table into which the update results will - ** be stored. - */ - assert( v ); - ephemTab = pParse->nTab++; - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, ephemTab, pTab->nCol+1+(pRowid!=0)); - sqlite3VdbeChangeP5(v, BTREE_UNORDERED); - /* fill the ephemeral table - */ - sqlite3SelectDestInit(&dest, SRT_Table, ephemTab); - sqlite3Select(pParse, pSelect, &dest); + bOnePass = sqlite3WhereOkOnePass(pWInfo, aDummy); - /* Generate code to scan the ephemeral table and call VUpdate. */ - iReg = ++pParse->nMem; - pParse->nMem += pTab->nCol+1; - addr = sqlite3VdbeAddOp2(v, OP_Rewind, ephemTab, 0); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_Column, ephemTab, 0, iReg); - sqlite3VdbeAddOp3(v, OP_Column, ephemTab, (pRowid?1:0), iReg+1); - for(i=0; inCol; i++){ - sqlite3VdbeAddOp3(v, OP_Column, ephemTab, i+1+(pRowid!=0), iReg+2+i); + if( bOnePass ){ + /* If using the onepass strategy, no-op out the OP_OpenEphemeral coded + ** above. Also, if this is a top-level parse (not a trigger), clear the + ** multi-write flag so that the VM does not open a statement journal */ + sqlite3VdbeChangeToNoop(v, addr); + if( sqlite3IsToplevel(pParse) ){ + pParse->isMultiWrite = 0; + } + }else{ + /* Create a record from the argument register contents and insert it into + ** the ephemeral table. */ + sqlite3VdbeAddOp3(v, OP_MakeRecord, regArg, nArg, regRec); + sqlite3VdbeAddOp2(v, OP_NewRowid, ephemTab, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, ephemTab, regRec, regRowid); + } + + + if( bOnePass==0 ){ + /* End the virtual table scan */ + sqlite3WhereEnd(pWInfo); + + /* Begin scannning through the ephemeral table. */ + addr = sqlite3VdbeAddOp1(v, OP_Rewind, ephemTab); VdbeCoverage(v); + + /* Extract arguments from the current row of the ephemeral table and + ** invoke the VUpdate method. */ + for(i=0; inCol+2, iReg, pVTab, P4_VTAB); + sqlite3VdbeAddOp4(v, OP_VUpdate, 0, nArg, regArg, pVTab, P4_VTAB); sqlite3VdbeChangeP5(v, onError==OE_Default ? OE_Abort : onError); sqlite3MayAbort(pParse); - sqlite3VdbeAddOp2(v, OP_Next, ephemTab, addr+1); VdbeCoverage(v); - sqlite3VdbeJumpHere(v, addr); - sqlite3VdbeAddOp2(v, OP_Close, ephemTab, 0); - /* Cleanup */ - sqlite3SelectDelete(db, pSelect); + /* End of the ephemeral table scan. Or, if using the onepass strategy, + ** jump to here if the scan visited zero rows. */ + if( bOnePass==0 ){ + sqlite3VdbeAddOp2(v, OP_Next, ephemTab, addr+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeAddOp2(v, OP_Close, ephemTab, 0); + }else{ + sqlite3WhereEnd(pWInfo); + } } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -114189,6 +116405,8 @@ static void updateVirtualTable( ** Most of the code in this file may be omitted by defining the ** SQLITE_OMIT_VACUUM macro. */ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ #if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH) /* @@ -114561,6 +116779,7 @@ end_of_vacuum: ** This file contains code used to help implement virtual tables. */ #ifndef SQLITE_OMIT_VIRTUALTABLE +/* #include "sqliteInt.h" */ /* ** Before a virtual table xCreate() or xConnect() method is invoked, the @@ -114606,6 +116825,7 @@ static int createModule( pMod->pModule = pModule; pMod->pAux = pAux; pMod->xDestroy = xDestroy; + pMod->pEpoTab = 0; pDel = (Module *)sqlite3HashInsert(&db->aModule,zCopy,(void*)pMod); assert( pDel==0 || pDel==pMod ); if( pDel ){ @@ -114833,23 +117053,17 @@ SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table *p){ ** deleted. */ static void addModuleArgument(sqlite3 *db, Table *pTable, char *zArg){ - int i = pTable->nModuleArg++; - int nBytes = sizeof(char *)*(1+pTable->nModuleArg); + int nBytes = sizeof(char *)*(2+pTable->nModuleArg); char **azModuleArg; azModuleArg = sqlite3DbRealloc(db, pTable->azModuleArg, nBytes); if( azModuleArg==0 ){ - int j; - for(j=0; jazModuleArg[j]); - } sqlite3DbFree(db, zArg); - sqlite3DbFree(db, pTable->azModuleArg); - pTable->nModuleArg = 0; }else{ + int i = pTable->nModuleArg++; azModuleArg[i] = zArg; azModuleArg[i+1] = 0; + pTable->azModuleArg = azModuleArg; } - pTable->azModuleArg = azModuleArg; } /* @@ -114976,7 +117190,7 @@ SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere); iReg = ++pParse->nMem; - sqlite3VdbeAddOp4(v, OP_String8, 0, iReg, 0, pTab->zName, 0); + sqlite3VdbeLoadString(v, iReg, pTab->zName); sqlite3VdbeAddOp2(v, OP_VCreate, iDb, iReg); } @@ -115252,7 +117466,7 @@ SQLITE_PRIVATE int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, ** invoke it now. If the module has not been registered, return an ** error. Otherwise, do nothing. */ - if( !pMod ){ + if( pMod==0 || pMod->pModule->xCreate==0 || pMod->pModule->xDestroy==0 ){ *pzErr = sqlite3MPrintf(db, "no such module: %s", zMod); rc = SQLITE_ERROR; }else{ @@ -115354,6 +117568,7 @@ SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zName); if( ALWAYS(pTab!=0 && pTab->pVTable!=0) ){ VTable *p; + int (*xDestroy)(sqlite3_vtab *); for(p=pTab->pVTable; p; p=p->pNext){ assert( p->pVtab ); if( p->pVtab->nRef>0 ){ @@ -115361,7 +117576,9 @@ SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab } } p = vtabDisconnectAll(db, pTab); - rc = p->pMod->pModule->xDestroy(p->pVtab); + xDestroy = p->pMod->pModule->xDestroy; + assert( xDestroy!=0 ); /* Checked before the virtual table is created */ + rc = xDestroy(p->pVtab); /* Remove the sqlite3_vtab* from the aVTrans[] array, if applicable */ if( rc==SQLITE_OK ){ assert( pTab->pVTable==p && p->pNext==0 ); @@ -115385,8 +117602,10 @@ SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab static void callFinaliser(sqlite3 *db, int offset){ int i; if( db->aVTrans ){ + VTable **aVTrans = db->aVTrans; + db->aVTrans = 0; for(i=0; inVTrans; i++){ - VTable *pVTab = db->aVTrans[i]; + VTable *pVTab = aVTrans[i]; sqlite3_vtab *p = pVTab->pVtab; if( p ){ int (*x)(sqlite3_vtab *); @@ -115396,9 +117615,8 @@ static void callFinaliser(sqlite3 *db, int offset){ pVTab->iSavepoint = 0; sqlite3VtabUnlock(pVTab); } - sqlite3DbFree(db, db->aVTrans); + sqlite3DbFree(db, aVTrans); db->nVTrans = 0; - db->aVTrans = 0; } } @@ -115486,7 +117704,9 @@ SQLITE_PRIVATE int sqlite3VtabBegin(sqlite3 *db, VTable *pVTab){ if( rc==SQLITE_OK ){ rc = pModule->xBegin(pVTab->pVtab); if( rc==SQLITE_OK ){ + int iSvpt = db->nStatement + db->nSavepoint; addToVTrans(db, pVTab); + if( iSvpt ) rc = sqlite3VtabSavepoint(db, SAVEPOINT_BEGIN, iSvpt-1); } } } @@ -115639,6 +117859,67 @@ SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse *pParse, Table *pTab){ } } +/* +** Check to see if virtual tale module pMod can be have an eponymous +** virtual table instance. If it can, create one if one does not already +** exist. Return non-zero if the eponymous virtual table instance exists +** when this routine returns, and return zero if it does not exist. +** +** An eponymous virtual table instance is one that is named after its +** module, and more importantly, does not require a CREATE VIRTUAL TABLE +** statement in order to come into existance. Eponymous virtual table +** instances always exist. They cannot be DROP-ed. +** +** Any virtual table module for which xConnect and xCreate are the same +** method can have an eponymous virtual table instance. +*/ +SQLITE_PRIVATE int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){ + const sqlite3_module *pModule = pMod->pModule; + Table *pTab; + char *zErr = 0; + int nName; + int rc; + sqlite3 *db = pParse->db; + if( pMod->pEpoTab ) return 1; + if( pModule->xCreate!=0 && pModule->xCreate!=pModule->xConnect ) return 0; + nName = sqlite3Strlen30(pMod->zName) + 1; + pTab = sqlite3DbMallocZero(db, sizeof(Table) + nName); + if( pTab==0 ) return 0; + pMod->pEpoTab = pTab; + pTab->zName = (char*)&pTab[1]; + memcpy(pTab->zName, pMod->zName, nName); + pTab->nRef = 1; + pTab->pSchema = db->aDb[0].pSchema; + pTab->tabFlags |= TF_Virtual; + pTab->nModuleArg = 0; + pTab->iPKey = -1; + addModuleArgument(db, pTab, sqlite3DbStrDup(db, pTab->zName)); + addModuleArgument(db, pTab, 0); + addModuleArgument(db, pTab, sqlite3DbStrDup(db, pTab->zName)); + rc = vtabCallConstructor(db, pTab, pMod, pModule->xConnect, &zErr); + if( rc ){ + sqlite3ErrorMsg(pParse, "%s", zErr); + sqlite3DbFree(db, zErr); + sqlite3VtabEponymousTableClear(db, pMod); + return 0; + } + return 1; +} + +/* +** Erase the eponymous virtual table instance associated with +** virtual table module pMod, if it exists. +*/ +SQLITE_PRIVATE void sqlite3VtabEponymousTableClear(sqlite3 *db, Module *pMod){ + Table *pTab = pMod->pEpoTab; + if( pTab!=0 ){ + sqlite3DeleteColumnNames(db, pTab); + sqlite3VtabClear(db, pTab); + sqlite3DbFree(db, pTab); + pMod->pEpoTab = 0; + } +} + /* ** Return the ON CONFLICT resolution mode in effect for the virtual ** table update operation currently in progress. @@ -115698,9 +117979,9 @@ SQLITE_API int SQLITE_CDECL sqlite3_vtab_config(sqlite3 *db, int op, ...){ #endif /* SQLITE_OMIT_VIRTUALTABLE */ /************** End of vtab.c ************************************************/ -/************** Begin file where.c *******************************************/ +/************** Begin file wherecode.c ***************************************/ /* -** 2001 September 15 +** 2015-06-06 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -115711,13 +117992,15 @@ SQLITE_API int SQLITE_CDECL sqlite3_vtab_config(sqlite3 *db, int op, ...){ ** ************************************************************************* ** This module contains C code that generates VDBE code used to process -** the WHERE clause of SQL statements. This module is responsible for -** generating the code that loops through a table looking for applicable -** rows. Indices are selected and used to speed the search when doing -** so is applicable. Because this module is responsible for selecting -** indices, you might also think of this module as the "query optimizer". +** the WHERE clause of SQL statements. +** +** This file was split off from where.c on 2015-06-06 in order to reduce the +** size of where.c and make it easier to edit. This file contains the routines +** that actually generate the bulk of the WHERE loop code. The original where.c +** file retains the code that does query planning and analysis. */ -/************** Include whereInt.h in the middle of where.c ******************/ +/* #include "sqliteInt.h" */ +/************** Include whereInt.h in the middle of wherecode.c **************/ /************** Begin file whereInt.h ****************************************/ /* ** 2013-11-12 @@ -115740,7 +118023,7 @@ SQLITE_API int SQLITE_CDECL sqlite3_vtab_config(sqlite3 *db, int op, ...){ ** Trace output macros */ #if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) -/***/ int sqlite3WhereTrace = 0; +/***/ int sqlite3WhereTrace; #endif #if defined(SQLITE_DEBUG) \ && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE)) @@ -115882,10 +118165,6 @@ struct WhereOrSet { WhereOrCost a[N_OR_COST]; /* Set of best costs */ }; - -/* Forward declaration of methods */ -static int whereLoopResize(sqlite3*, WhereLoop*, int); - /* ** Each instance of this object holds a sequence of WhereLoop objects ** that implement some or all of a query plan. @@ -116001,6 +118280,7 @@ struct WhereTerm { #define TERM_LIKEOPT 0x100 /* Virtual terms from the LIKE optimization */ #define TERM_LIKECOND 0x200 /* Conditionally this LIKE operator term */ #define TERM_LIKE 0x400 /* The original LIKE operator */ +#define TERM_IS 0x800 /* Term.pExpr is an IS operator */ /* ** An instance of the WhereScan object is used as an iterator for locating @@ -116010,12 +118290,14 @@ struct WhereScan { WhereClause *pOrigWC; /* Original, innermost WhereClause */ WhereClause *pWC; /* WhereClause currently being scanned */ char *zCollName; /* Required collating sequence, if not NULL */ + Expr *pIdxExpr; /* Search for this index expression */ char idxaff; /* Must match this affinity, if zCollName!=NULL */ unsigned char nEquiv; /* Number of entries in aEquiv[] */ unsigned char iEquiv; /* Next unused slot in aEquiv[] */ u32 opMask; /* Acceptable operators */ int k; /* Resume scanning at this->pWC->a[this->k] */ - int aEquiv[22]; /* Cursor,Column pairs for equivalence classes */ + int aiCur[11]; /* Cursors in the equivalence class */ + i16 aiColumn[11]; /* Corresponding column number in the eq-class */ }; /* @@ -116092,6 +118374,11 @@ struct WhereMaskSet { int ix[BMS]; /* Cursor assigned to each bit */ }; +/* +** Initialize a WhereMaskSet object +*/ +#define initMaskSet(P) (P)->n=0 + /* ** This object is a convenience wrapper holding all information needed ** to construct WhereLoop objects for a particular query. @@ -116129,7 +118416,7 @@ struct WhereInfo { u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ i8 nOBSat; /* Number of ORDER BY terms satisfied by indices */ u8 sorted; /* True if really sorted (not just grouped) */ - u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE/DELETE */ + u8 eOnePass; /* ONEPASS_OFF, or _SINGLE, or _MULTI */ u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ u8 eDistinct; /* One of the WHERE_DISTINCT_* values below */ u8 nLevel; /* Number of nested loop */ @@ -116143,27 +118430,85 @@ struct WhereInfo { WhereLevel a[1]; /* Information about each nest loop in WHERE */ }; +/* +** Private interfaces - callable only by other where.c routines. +** +** where.c: +*/ +SQLITE_PRIVATE Bitmask sqlite3WhereGetMask(WhereMaskSet*,int); +SQLITE_PRIVATE WhereTerm *sqlite3WhereFindTerm( + WhereClause *pWC, /* The WHERE clause to be searched */ + int iCur, /* Cursor number of LHS */ + int iColumn, /* Column number of LHS */ + Bitmask notReady, /* RHS must not overlap with this mask */ + u32 op, /* Mask of WO_xx values describing operator */ + Index *pIdx /* Must be compatible with this index, if not NULL */ +); + +/* wherecode.c: */ +#ifndef SQLITE_OMIT_EXPLAIN +SQLITE_PRIVATE int sqlite3WhereExplainOneScan( + Parse *pParse, /* Parse context */ + SrcList *pTabList, /* Table list this loop refers to */ + WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ + int iLevel, /* Value for "level" column of output */ + int iFrom, /* Value for "from" column of output */ + u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ +); +#else +# define sqlite3WhereExplainOneScan(u,v,w,x,y,z) 0 +#endif /* SQLITE_OMIT_EXPLAIN */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +SQLITE_PRIVATE void sqlite3WhereAddScanStatus( + Vdbe *v, /* Vdbe to add scanstatus entry to */ + SrcList *pSrclist, /* FROM clause pLvl reads data from */ + WhereLevel *pLvl, /* Level to add scanstatus() entry for */ + int addrExplain /* Address of OP_Explain (or 0) */ +); +#else +# define sqlite3WhereAddScanStatus(a, b, c, d) ((void)d) +#endif +SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( + WhereInfo *pWInfo, /* Complete information about the WHERE clause */ + int iLevel, /* Which level of pWInfo->a[] should be coded */ + Bitmask notReady /* Which tables are currently available */ +); + +/* whereexpr.c: */ +SQLITE_PRIVATE void sqlite3WhereClauseInit(WhereClause*,WhereInfo*); +SQLITE_PRIVATE void sqlite3WhereClauseClear(WhereClause*); +SQLITE_PRIVATE void sqlite3WhereSplit(WhereClause*,Expr*,u8); +SQLITE_PRIVATE Bitmask sqlite3WhereExprUsage(WhereMaskSet*, Expr*); +SQLITE_PRIVATE Bitmask sqlite3WhereExprListUsage(WhereMaskSet*, ExprList*); +SQLITE_PRIVATE void sqlite3WhereExprAnalyze(SrcList*, WhereClause*); +SQLITE_PRIVATE void sqlite3WhereTabFuncArgs(Parse*, struct SrcList_item*, WhereClause*); + + + + + /* ** Bitmasks for the operators on WhereTerm objects. These are all ** operators that are of interest to the query planner. An ** OR-ed combination of these values can be used when searching for ** particular WhereTerms within a WhereClause. */ -#define WO_IN 0x001 -#define WO_EQ 0x002 +#define WO_IN 0x0001 +#define WO_EQ 0x0002 #define WO_LT (WO_EQ<<(TK_LT-TK_EQ)) #define WO_LE (WO_EQ<<(TK_LE-TK_EQ)) #define WO_GT (WO_EQ<<(TK_GT-TK_EQ)) #define WO_GE (WO_EQ<<(TK_GE-TK_EQ)) -#define WO_MATCH 0x040 -#define WO_ISNULL 0x080 -#define WO_OR 0x100 /* Two or more OR-connected terms */ -#define WO_AND 0x200 /* Two or more AND-connected terms */ -#define WO_EQUIV 0x400 /* Of the form A==B, both columns */ -#define WO_NOOP 0x800 /* This term does not restrict search space */ +#define WO_MATCH 0x0040 +#define WO_IS 0x0080 +#define WO_ISNULL 0x0100 +#define WO_OR 0x0200 /* Two or more OR-connected terms */ +#define WO_AND 0x0400 /* Two or more AND-connected terms */ +#define WO_EQUIV 0x0800 /* Of the form A==B, both columns */ +#define WO_NOOP 0x1000 /* This term does not restrict search space */ -#define WO_ALL 0xfff /* Mask of all possible WO_* values */ -#define WO_SINGLE 0x0ff /* Mask of all non-compound WO_* values */ +#define WO_ALL 0x1fff /* Mask of all possible WO_* values */ +#define WO_SINGLE 0x01ff /* Mask of all non-compound WO_* values */ /* ** These are definitions of bits in the WhereLoop.wsFlags field. @@ -116191,4420 +118536,4746 @@ struct WhereInfo { #define WHERE_PARTIALIDX 0x00020000 /* The automatic index is partial */ /************** End of whereInt.h ********************************************/ -/************** Continuing where we left off in where.c **********************/ - -/* -** Return the estimated number of output rows from a WHERE clause -*/ -SQLITE_PRIVATE u64 sqlite3WhereOutputRowCount(WhereInfo *pWInfo){ - return sqlite3LogEstToInt(pWInfo->nRowOut); -} - -/* -** Return one of the WHERE_DISTINCT_xxxxx values to indicate how this -** WHERE clause returns outputs for DISTINCT processing. -*/ -SQLITE_PRIVATE int sqlite3WhereIsDistinct(WhereInfo *pWInfo){ - return pWInfo->eDistinct; -} - -/* -** Return TRUE if the WHERE clause returns rows in ORDER BY order. -** Return FALSE if the output needs to be sorted. -*/ -SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ - return pWInfo->nOBSat; -} +/************** Continuing where we left off in wherecode.c ******************/ +#ifndef SQLITE_OMIT_EXPLAIN /* -** Return the VDBE address or label to jump to in order to continue -** immediately with the next row of a WHERE clause. +** This routine is a helper for explainIndexRange() below +** +** pStr holds the text of an expression that we are building up one term +** at a time. This routine adds a new term to the end of the expression. +** Terms are separated by AND so add the "AND" text for second and subsequent +** terms only. */ -SQLITE_PRIVATE int sqlite3WhereContinueLabel(WhereInfo *pWInfo){ - assert( pWInfo->iContinue!=0 ); - return pWInfo->iContinue; +static void explainAppendTerm( + StrAccum *pStr, /* The text expression being built */ + int iTerm, /* Index of this term. First is zero */ + const char *zColumn, /* Name of the column */ + const char *zOp /* Name of the operator */ +){ + if( iTerm ) sqlite3StrAccumAppend(pStr, " AND ", 5); + sqlite3StrAccumAppendAll(pStr, zColumn); + sqlite3StrAccumAppend(pStr, zOp, 1); + sqlite3StrAccumAppend(pStr, "?", 1); } /* -** Return the VDBE address or label to jump to in order to break -** out of a WHERE loop. +** Return the name of the i-th column of the pIdx index. */ -SQLITE_PRIVATE int sqlite3WhereBreakLabel(WhereInfo *pWInfo){ - return pWInfo->iBreak; +static const char *explainIndexColumnName(Index *pIdx, int i){ + i = pIdx->aiColumn[i]; + if( i==XN_EXPR ) return ""; + if( i==XN_ROWID ) return "rowid"; + return pIdx->pTable->aCol[i].zName; } /* -** Return TRUE if an UPDATE or DELETE statement can operate directly on -** the rowids returned by a WHERE clause. Return FALSE if doing an -** UPDATE or DELETE might change subsequent WHERE clause results. +** Argument pLevel describes a strategy for scanning table pTab. This +** function appends text to pStr that describes the subset of table +** rows scanned by the strategy in the form of an SQL expression. ** -** If the ONEPASS optimization is used (if this routine returns true) -** then also write the indices of open cursors used by ONEPASS -** into aiCur[0] and aiCur[1]. iaCur[0] gets the cursor of the data -** table and iaCur[1] gets the cursor used by an auxiliary index. -** Either value may be -1, indicating that cursor is not used. -** Any cursors returned will have been opened for writing. +** For example, if the query: ** -** aiCur[0] and aiCur[1] both get -1 if the where-clause logic is -** unable to use the ONEPASS optimization. +** SELECT * FROM t1 WHERE a=1 AND b>2; +** +** is run and there is an index on (a, b), then this function returns a +** string similar to: +** +** "a=? AND b>?" */ -SQLITE_PRIVATE int sqlite3WhereOkOnePass(WhereInfo *pWInfo, int *aiCur){ - memcpy(aiCur, pWInfo->aiCurOnePass, sizeof(int)*2); - return pWInfo->okOnePass; -} +static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop){ + Index *pIndex = pLoop->u.btree.pIndex; + u16 nEq = pLoop->u.btree.nEq; + u16 nSkip = pLoop->nSkip; + int i, j; -/* -** Move the content of pSrc into pDest -*/ -static void whereOrMove(WhereOrSet *pDest, WhereOrSet *pSrc){ - pDest->n = pSrc->n; - memcpy(pDest->a, pSrc->a, pDest->n*sizeof(pDest->a[0])); -} + if( nEq==0 && (pLoop->wsFlags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ) return; + sqlite3StrAccumAppend(pStr, " (", 2); + for(i=0; i=nSkip ? "%s=?" : "ANY(%s)", z); + } -/* -** Try to insert a new prerequisite/cost entry into the WhereOrSet pSet. -** -** The new entry might overwrite an existing entry, or it might be -** appended, or it might be discarded. Do whatever is the right thing -** so that pSet keeps the N_OR_COST best entries seen so far. -*/ -static int whereOrInsert( - WhereOrSet *pSet, /* The WhereOrSet to be updated */ - Bitmask prereq, /* Prerequisites of the new entry */ - LogEst rRun, /* Run-cost of the new entry */ - LogEst nOut /* Number of outputs for the new entry */ -){ - u16 i; - WhereOrCost *p; - for(i=pSet->n, p=pSet->a; i>0; i--, p++){ - if( rRun<=p->rRun && (prereq & p->prereq)==prereq ){ - goto whereOrInsert_done; - } - if( p->rRun<=rRun && (p->prereq & prereq)==p->prereq ){ - return 0; - } + j = i; + if( pLoop->wsFlags&WHERE_BTM_LIMIT ){ + const char *z = explainIndexColumnName(pIndex, i); + explainAppendTerm(pStr, i++, z, ">"); } - if( pSet->na[pSet->n++]; - p->nOut = nOut; - }else{ - p = pSet->a; - for(i=1; in; i++){ - if( p->rRun>pSet->a[i].rRun ) p = pSet->a + i; - } - if( p->rRun<=rRun ) return 0; + if( pLoop->wsFlags&WHERE_TOP_LIMIT ){ + const char *z = explainIndexColumnName(pIndex, j); + explainAppendTerm(pStr, i, z, "<"); } -whereOrInsert_done: - p->prereq = prereq; - p->rRun = rRun; - if( p->nOut>nOut ) p->nOut = nOut; - return 1; + sqlite3StrAccumAppend(pStr, ")", 1); } /* -** Initialize a preallocated WhereClause structure. +** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN +** command, or if either SQLITE_DEBUG or SQLITE_ENABLE_STMT_SCANSTATUS was +** defined at compile-time. If it is not a no-op, a single OP_Explain opcode +** is added to the output to describe the table scan strategy in pLevel. +** +** If an OP_Explain opcode is added to the VM, its address is returned. +** Otherwise, if no OP_Explain is coded, zero is returned. */ -static void whereClauseInit( - WhereClause *pWC, /* The WhereClause to be initialized */ - WhereInfo *pWInfo /* The WHERE processing context */ +SQLITE_PRIVATE int sqlite3WhereExplainOneScan( + Parse *pParse, /* Parse context */ + SrcList *pTabList, /* Table list this loop refers to */ + WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ + int iLevel, /* Value for "level" column of output */ + int iFrom, /* Value for "from" column of output */ + u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ ){ - pWC->pWInfo = pWInfo; - pWC->pOuter = 0; - pWC->nTerm = 0; - pWC->nSlot = ArraySize(pWC->aStatic); - pWC->a = pWC->aStatic; -} - -/* Forward reference */ -static void whereClauseClear(WhereClause*); + int ret = 0; +#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) + if( pParse->explain==2 ) +#endif + { + struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom]; + Vdbe *v = pParse->pVdbe; /* VM being constructed */ + sqlite3 *db = pParse->db; /* Database handle */ + int iId = pParse->iSelectId; /* Select id (left-most output column) */ + int isSearch; /* True for a SEARCH. False for SCAN. */ + WhereLoop *pLoop; /* The controlling WhereLoop object */ + u32 flags; /* Flags that describe this loop */ + char *zMsg; /* Text to add to EQP output */ + StrAccum str; /* EQP output string */ + char zBuf[100]; /* Initial space for EQP output string */ -/* -** Deallocate all memory associated with a WhereOrInfo object. -*/ -static void whereOrInfoDelete(sqlite3 *db, WhereOrInfo *p){ - whereClauseClear(&p->wc); - sqlite3DbFree(db, p); -} + pLoop = pLevel->pWLoop; + flags = pLoop->wsFlags; + if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return 0; -/* -** Deallocate all memory associated with a WhereAndInfo object. -*/ -static void whereAndInfoDelete(sqlite3 *db, WhereAndInfo *p){ - whereClauseClear(&p->wc); - sqlite3DbFree(db, p); -} + isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 + || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0)) + || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); -/* -** Deallocate a WhereClause structure. The WhereClause structure -** itself is not freed. This routine is the inverse of whereClauseInit(). -*/ -static void whereClauseClear(WhereClause *pWC){ - int i; - WhereTerm *a; - sqlite3 *db = pWC->pWInfo->pParse->db; - for(i=pWC->nTerm-1, a=pWC->a; i>=0; i--, a++){ - if( a->wtFlags & TERM_DYNAMIC ){ - sqlite3ExprDelete(db, a->pExpr); + sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); + sqlite3StrAccumAppendAll(&str, isSearch ? "SEARCH" : "SCAN"); + if( pItem->pSelect ){ + sqlite3XPrintf(&str, 0, " SUBQUERY %d", pItem->iSelectId); + }else{ + sqlite3XPrintf(&str, 0, " TABLE %s", pItem->zName); } - if( a->wtFlags & TERM_ORINFO ){ - whereOrInfoDelete(db, a->u.pOrInfo); - }else if( a->wtFlags & TERM_ANDINFO ){ - whereAndInfoDelete(db, a->u.pAndInfo); + + if( pItem->zAlias ){ + sqlite3XPrintf(&str, 0, " AS %s", pItem->zAlias); } - } - if( pWC->a!=pWC->aStatic ){ - sqlite3DbFree(db, pWC->a); - } -} + if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ + const char *zFmt = 0; + Index *pIdx; -/* -** Add a single new WhereTerm entry to the WhereClause object pWC. -** The new WhereTerm object is constructed from Expr p and with wtFlags. -** The index in pWC->a[] of the new WhereTerm is returned on success. -** 0 is returned if the new WhereTerm could not be added due to a memory -** allocation error. The memory allocation failure will be recorded in -** the db->mallocFailed flag so that higher-level functions can detect it. -** -** This routine will increase the size of the pWC->a[] array as necessary. -** -** If the wtFlags argument includes TERM_DYNAMIC, then responsibility -** for freeing the expression p is assumed by the WhereClause object pWC. -** This is true even if this routine fails to allocate a new WhereTerm. -** -** WARNING: This routine might reallocate the space used to store -** WhereTerms. All pointers to WhereTerms should be invalidated after -** calling this routine. Such pointers may be reinitialized by referencing -** the pWC->a[] array. -*/ -static int whereClauseInsert(WhereClause *pWC, Expr *p, u16 wtFlags){ - WhereTerm *pTerm; - int idx; - testcase( wtFlags & TERM_VIRTUAL ); - if( pWC->nTerm>=pWC->nSlot ){ - WhereTerm *pOld = pWC->a; - sqlite3 *db = pWC->pWInfo->pParse->db; - pWC->a = sqlite3DbMallocRaw(db, sizeof(pWC->a[0])*pWC->nSlot*2 ); - if( pWC->a==0 ){ - if( wtFlags & TERM_DYNAMIC ){ - sqlite3ExprDelete(db, p); + assert( pLoop->u.btree.pIndex!=0 ); + pIdx = pLoop->u.btree.pIndex; + assert( !(flags&WHERE_AUTO_INDEX) || (flags&WHERE_IDX_ONLY) ); + if( !HasRowid(pItem->pTab) && IsPrimaryKeyIndex(pIdx) ){ + if( isSearch ){ + zFmt = "PRIMARY KEY"; + } + }else if( flags & WHERE_PARTIALIDX ){ + zFmt = "AUTOMATIC PARTIAL COVERING INDEX"; + }else if( flags & WHERE_AUTO_INDEX ){ + zFmt = "AUTOMATIC COVERING INDEX"; + }else if( flags & WHERE_IDX_ONLY ){ + zFmt = "COVERING INDEX %s"; + }else{ + zFmt = "INDEX %s"; } - pWC->a = pOld; - return 0; + if( zFmt ){ + sqlite3StrAccumAppend(&str, " USING ", 7); + sqlite3XPrintf(&str, 0, zFmt, pIdx->zName); + explainIndexRange(&str, pLoop); + } + }else if( (flags & WHERE_IPK)!=0 && (flags & WHERE_CONSTRAINT)!=0 ){ + const char *zRangeOp; + if( flags&(WHERE_COLUMN_EQ|WHERE_COLUMN_IN) ){ + zRangeOp = "="; + }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){ + zRangeOp = ">? AND rowid<"; + }else if( flags&WHERE_BTM_LIMIT ){ + zRangeOp = ">"; + }else{ + assert( flags&WHERE_TOP_LIMIT); + zRangeOp = "<"; + } + sqlite3XPrintf(&str, 0, " USING INTEGER PRIMARY KEY (rowid%s?)",zRangeOp); } - memcpy(pWC->a, pOld, sizeof(pWC->a[0])*pWC->nTerm); - if( pOld!=pWC->aStatic ){ - sqlite3DbFree(db, pOld); +#ifndef SQLITE_OMIT_VIRTUALTABLE + else if( (flags & WHERE_VIRTUALTABLE)!=0 ){ + sqlite3XPrintf(&str, 0, " VIRTUAL TABLE INDEX %d:%s", + pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr); } - pWC->nSlot = sqlite3DbMallocSize(db, pWC->a)/sizeof(pWC->a[0]); - memset(&pWC->a[pWC->nTerm], 0, sizeof(pWC->a[0])*(pWC->nSlot-pWC->nTerm)); - } - pTerm = &pWC->a[idx = pWC->nTerm++]; - if( p && ExprHasProperty(p, EP_Unlikely) ){ - pTerm->truthProb = sqlite3LogEst(p->iTable) - 270; - }else{ - pTerm->truthProb = 1; +#endif +#ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS + if( pLoop->nOut>=10 ){ + sqlite3XPrintf(&str, 0, " (~%llu rows)", sqlite3LogEstToInt(pLoop->nOut)); + }else{ + sqlite3StrAccumAppend(&str, " (~1 row)", 9); + } +#endif + zMsg = sqlite3StrAccumFinish(&str); + ret = sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg,P4_DYNAMIC); } - pTerm->pExpr = sqlite3ExprSkipCollate(p); - pTerm->wtFlags = wtFlags; - pTerm->pWC = pWC; - pTerm->iParent = -1; - return idx; + return ret; } +#endif /* SQLITE_OMIT_EXPLAIN */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS /* -** This routine identifies subexpressions in the WHERE clause where -** each subexpression is separated by the AND operator or some other -** operator specified in the op parameter. The WhereClause structure -** is filled with pointers to subexpressions. For example: -** -** WHERE a=='hello' AND coalesce(b,11)<10 AND (c+12!=d OR c==22) -** \________/ \_______________/ \________________/ -** slot[0] slot[1] slot[2] -** -** The original WHERE clause in pExpr is unaltered. All this routine -** does is make slot[] entries point to substructure within pExpr. +** Configure the VM passed as the first argument with an +** sqlite3_stmt_scanstatus() entry corresponding to the scan used to +** implement level pLvl. Argument pSrclist is a pointer to the FROM +** clause that the scan reads data from. ** -** In the previous sentence and in the diagram, "slot[]" refers to -** the WhereClause.a[] array. The slot[] array grows as needed to contain -** all terms of the WHERE clause. +** If argument addrExplain is not 0, it must be the address of an +** OP_Explain instruction that describes the same loop. */ -static void whereSplit(WhereClause *pWC, Expr *pExpr, u8 op){ - Expr *pE2 = sqlite3ExprSkipCollate(pExpr); - pWC->op = op; - if( pE2==0 ) return; - if( pE2->op!=op ){ - whereClauseInsert(pWC, pExpr, 0); +SQLITE_PRIVATE void sqlite3WhereAddScanStatus( + Vdbe *v, /* Vdbe to add scanstatus entry to */ + SrcList *pSrclist, /* FROM clause pLvl reads data from */ + WhereLevel *pLvl, /* Level to add scanstatus() entry for */ + int addrExplain /* Address of OP_Explain (or 0) */ +){ + const char *zObj = 0; + WhereLoop *pLoop = pLvl->pWLoop; + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ + zObj = pLoop->u.btree.pIndex->zName; }else{ - whereSplit(pWC, pE2->pLeft, op); - whereSplit(pWC, pE2->pRight, op); + zObj = pSrclist->a[pLvl->iFrom].zName; } + sqlite3VdbeScanStatus( + v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj + ); } +#endif -/* -** Initialize a WhereMaskSet object -*/ -#define initMaskSet(P) (P)->n=0 /* -** Return the bitmask for the given cursor number. Return 0 if -** iCursor is not in the set. +** Disable a term in the WHERE clause. Except, do not disable the term +** if it controls a LEFT OUTER JOIN and it did not originate in the ON +** or USING clause of that join. +** +** Consider the term t2.z='ok' in the following queries: +** +** (1) SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.x WHERE t2.z='ok' +** (2) SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.x AND t2.z='ok' +** (3) SELECT * FROM t1, t2 WHERE t1.a=t2.x AND t2.z='ok' +** +** The t2.z='ok' is disabled in the in (2) because it originates +** in the ON clause. The term is disabled in (3) because it is not part +** of a LEFT OUTER JOIN. In (1), the term is not disabled. +** +** Disabling a term causes that term to not be tested in the inner loop +** of the join. Disabling is an optimization. When terms are satisfied +** by indices, we disable them to prevent redundant tests in the inner +** loop. We would get the correct results if nothing were ever disabled, +** but joins might run a little slower. The trick is to disable as much +** as we can without disabling too much. If we disabled in (1), we'd get +** the wrong answer. See ticket #813. +** +** If all the children of a term are disabled, then that term is also +** automatically disabled. In this way, terms get disabled if derived +** virtual terms are tested first. For example: +** +** x GLOB 'abc*' AND x>='abc' AND x<'acd' +** \___________/ \______/ \_____/ +** parent child1 child2 +** +** Only the parent term was in the original WHERE clause. The child1 +** and child2 terms were added by the LIKE optimization. If both of +** the virtual child terms are valid, then testing of the parent can be +** skipped. +** +** Usually the parent term is marked as TERM_CODED. But if the parent +** term was originally TERM_LIKE, then the parent gets TERM_LIKECOND instead. +** The TERM_LIKECOND marking indicates that the term should be coded inside +** a conditional such that is only evaluated on the second pass of a +** LIKE-optimization loop, when scanning BLOBs instead of strings. */ -static Bitmask getMask(WhereMaskSet *pMaskSet, int iCursor){ - int i; - assert( pMaskSet->n<=(int)sizeof(Bitmask)*8 ); - for(i=0; in; i++){ - if( pMaskSet->ix[i]==iCursor ){ - return MASKBIT(i); +static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){ + int nLoop = 0; + while( pTerm + && (pTerm->wtFlags & TERM_CODED)==0 + && (pLevel->iLeftJoin==0 || ExprHasProperty(pTerm->pExpr, EP_FromJoin)) + && (pLevel->notReady & pTerm->prereqAll)==0 + ){ + if( nLoop && (pTerm->wtFlags & TERM_LIKE)!=0 ){ + pTerm->wtFlags |= TERM_LIKECOND; + }else{ + pTerm->wtFlags |= TERM_CODED; } + if( pTerm->iParent<0 ) break; + pTerm = &pTerm->pWC->a[pTerm->iParent]; + pTerm->nChild--; + if( pTerm->nChild!=0 ) break; + nLoop++; } - return 0; } /* -** Create a new mask for cursor iCursor. +** Code an OP_Affinity opcode to apply the column affinity string zAff +** to the n registers starting at base. ** -** There is one cursor per table in the FROM clause. The number of -** tables in the FROM clause is limited by a test early in the -** sqlite3WhereBegin() routine. So we know that the pMaskSet->ix[] -** array will never overflow. -*/ -static void createMask(WhereMaskSet *pMaskSet, int iCursor){ - assert( pMaskSet->n < ArraySize(pMaskSet->ix) ); - pMaskSet->ix[pMaskSet->n++] = iCursor; -} - -/* -** These routines walk (recursively) an expression tree and generate -** a bitmask indicating which tables are used in that expression -** tree. +** As an optimization, SQLITE_AFF_BLOB entries (which are no-ops) at the +** beginning and end of zAff are ignored. If all entries in zAff are +** SQLITE_AFF_BLOB, then no code gets generated. +** +** This routine makes its own copy of zAff so that the caller is free +** to modify zAff after this routine returns. */ -static Bitmask exprListTableUsage(WhereMaskSet*, ExprList*); -static Bitmask exprSelectTableUsage(WhereMaskSet*, Select*); -static Bitmask exprTableUsage(WhereMaskSet *pMaskSet, Expr *p){ - Bitmask mask = 0; - if( p==0 ) return 0; - if( p->op==TK_COLUMN ){ - mask = getMask(pMaskSet, p->iTable); - return mask; +static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){ + Vdbe *v = pParse->pVdbe; + if( zAff==0 ){ + assert( pParse->db->mallocFailed ); + return; } - mask = exprTableUsage(pMaskSet, p->pRight); - mask |= exprTableUsage(pMaskSet, p->pLeft); - if( ExprHasProperty(p, EP_xIsSelect) ){ - mask |= exprSelectTableUsage(pMaskSet, p->x.pSelect); - }else{ - mask |= exprListTableUsage(pMaskSet, p->x.pList); + assert( v!=0 ); + + /* Adjust base and n to skip over SQLITE_AFF_BLOB entries at the beginning + ** and end of the affinity string. + */ + while( n>0 && zAff[0]==SQLITE_AFF_BLOB ){ + n--; + base++; + zAff++; } - return mask; -} -static Bitmask exprListTableUsage(WhereMaskSet *pMaskSet, ExprList *pList){ - int i; - Bitmask mask = 0; - if( pList ){ - for(i=0; inExpr; i++){ - mask |= exprTableUsage(pMaskSet, pList->a[i].pExpr); - } + while( n>1 && zAff[n-1]==SQLITE_AFF_BLOB ){ + n--; } - return mask; -} -static Bitmask exprSelectTableUsage(WhereMaskSet *pMaskSet, Select *pS){ - Bitmask mask = 0; - while( pS ){ - SrcList *pSrc = pS->pSrc; - mask |= exprListTableUsage(pMaskSet, pS->pEList); - mask |= exprListTableUsage(pMaskSet, pS->pGroupBy); - mask |= exprListTableUsage(pMaskSet, pS->pOrderBy); - mask |= exprTableUsage(pMaskSet, pS->pWhere); - mask |= exprTableUsage(pMaskSet, pS->pHaving); - if( ALWAYS(pSrc!=0) ){ - int i; - for(i=0; inSrc; i++){ - mask |= exprSelectTableUsage(pMaskSet, pSrc->a[i].pSelect); - mask |= exprTableUsage(pMaskSet, pSrc->a[i].pOn); - } - } - pS = pS->pPrior; + + /* Code the OP_Affinity opcode if there is anything left to do. */ + if( n>0 ){ + sqlite3VdbeAddOp2(v, OP_Affinity, base, n); + sqlite3VdbeChangeP4(v, -1, zAff, n); + sqlite3ExprCacheAffinityChange(pParse, base, n); } - return mask; } -/* -** Return TRUE if the given operator is one of the operators that is -** allowed for an indexable WHERE clause term. The allowed operators are -** "=", "<", ">", "<=", ">=", "IN", and "IS NULL" -*/ -static int allowedOp(int op){ - assert( TK_GT>TK_EQ && TK_GTTK_EQ && TK_LTTK_EQ && TK_LE=TK_EQ && op<=TK_GE) || op==TK_ISNULL; -} /* -** Commute a comparison operator. Expressions of the form "X op Y" -** are converted into "Y op X". +** Generate code for a single equality term of the WHERE clause. An equality +** term can be either X=expr or X IN (...). pTerm is the term to be +** coded. ** -** If left/right precedence rules come into play when determining the -** collating sequence, then COLLATE operators are adjusted to ensure -** that the collating sequence does not change. For example: -** "Y collate NOCASE op X" becomes "X op Y" because any collation sequence on -** the left hand side of a comparison overrides any collation sequence -** attached to the right. For the same reason the EP_Collate flag -** is not commuted. +** The current value for the constraint is left in register iReg. +** +** For a constraint of the form X=expr, the expression is evaluated and its +** result is left on the stack. For constraints of the form X IN (...) +** this routine sets up a loop that will iterate over all values of X. */ -static void exprCommute(Parse *pParse, Expr *pExpr){ - u16 expRight = (pExpr->pRight->flags & EP_Collate); - u16 expLeft = (pExpr->pLeft->flags & EP_Collate); - assert( allowedOp(pExpr->op) && pExpr->op!=TK_IN ); - if( expRight==expLeft ){ - /* Either X and Y both have COLLATE operator or neither do */ - if( expRight ){ - /* Both X and Y have COLLATE operators. Make sure X is always - ** used by clearing the EP_Collate flag from Y. */ - pExpr->pRight->flags &= ~EP_Collate; - }else if( sqlite3ExprCollSeq(pParse, pExpr->pLeft)!=0 ){ - /* Neither X nor Y have COLLATE operators, but X has a non-default - ** collating sequence. So add the EP_Collate marker on X to cause - ** it to be searched first. */ - pExpr->pLeft->flags |= EP_Collate; - } - } - SWAP(Expr*,pExpr->pRight,pExpr->pLeft); - if( pExpr->op>=TK_GT ){ - assert( TK_LT==TK_GT+2 ); - assert( TK_GE==TK_LE+2 ); - assert( TK_GT>TK_EQ ); - assert( TK_GTop>=TK_GT && pExpr->op<=TK_GE ); - pExpr->op = ((pExpr->op-TK_GT)^2)+TK_GT; - } -} +static int codeEqualityTerm( + Parse *pParse, /* The parsing context */ + WhereTerm *pTerm, /* The term of the WHERE clause to be coded */ + WhereLevel *pLevel, /* The level of the FROM clause we are working on */ + int iEq, /* Index of the equality term within this level */ + int bRev, /* True for reverse-order IN operations */ + int iTarget /* Attempt to leave results in this register */ +){ + Expr *pX = pTerm->pExpr; + Vdbe *v = pParse->pVdbe; + int iReg; /* Register holding results */ -/* -** Translate from TK_xx operator to WO_xx bitmask. -*/ -static u16 operatorMask(int op){ - u16 c; - assert( allowedOp(op) ); - if( op==TK_IN ){ - c = WO_IN; - }else if( op==TK_ISNULL ){ - c = WO_ISNULL; + assert( iTarget>0 ); + if( pX->op==TK_EQ || pX->op==TK_IS ){ + iReg = sqlite3ExprCodeTarget(pParse, pX->pRight, iTarget); + }else if( pX->op==TK_ISNULL ){ + iReg = iTarget; + sqlite3VdbeAddOp2(v, OP_Null, 0, iReg); +#ifndef SQLITE_OMIT_SUBQUERY }else{ - assert( (WO_EQ<<(op-TK_EQ)) < 0x7fff ); - c = (u16)(WO_EQ<<(op-TK_EQ)); - } - assert( op!=TK_ISNULL || c==WO_ISNULL ); - assert( op!=TK_IN || c==WO_IN ); - assert( op!=TK_EQ || c==WO_EQ ); - assert( op!=TK_LT || c==WO_LT ); - assert( op!=TK_LE || c==WO_LE ); - assert( op!=TK_GT || c==WO_GT ); - assert( op!=TK_GE || c==WO_GE ); - return c; -} - -/* -** Advance to the next WhereTerm that matches according to the criteria -** established when the pScan object was initialized by whereScanInit(). -** Return NULL if there are no more matching WhereTerms. -*/ -static WhereTerm *whereScanNext(WhereScan *pScan){ - int iCur; /* The cursor on the LHS of the term */ - int iColumn; /* The column on the LHS of the term. -1 for IPK */ - Expr *pX; /* An expression being tested */ - WhereClause *pWC; /* Shorthand for pScan->pWC */ - WhereTerm *pTerm; /* The term being tested */ - int k = pScan->k; /* Where to start scanning */ + int eType; + int iTab; + struct InLoop *pIn; + WhereLoop *pLoop = pLevel->pWLoop; - while( pScan->iEquiv<=pScan->nEquiv ){ - iCur = pScan->aEquiv[pScan->iEquiv-2]; - iColumn = pScan->aEquiv[pScan->iEquiv-1]; - while( (pWC = pScan->pWC)!=0 ){ - for(pTerm=pWC->a+k; knTerm; k++, pTerm++){ - if( pTerm->leftCursor==iCur - && pTerm->u.leftColumn==iColumn - && (pScan->iEquiv<=2 || !ExprHasProperty(pTerm->pExpr, EP_FromJoin)) - ){ - if( (pTerm->eOperator & WO_EQUIV)!=0 - && pScan->nEquivaEquiv) - ){ - int j; - pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight); - assert( pX->op==TK_COLUMN ); - for(j=0; jnEquiv; j+=2){ - if( pScan->aEquiv[j]==pX->iTable - && pScan->aEquiv[j+1]==pX->iColumn ){ - break; - } - } - if( j==pScan->nEquiv ){ - pScan->aEquiv[j] = pX->iTable; - pScan->aEquiv[j+1] = pX->iColumn; - pScan->nEquiv += 2; - } - } - if( (pTerm->eOperator & pScan->opMask)!=0 ){ - /* Verify the affinity and collating sequence match */ - if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){ - CollSeq *pColl; - Parse *pParse = pWC->pWInfo->pParse; - pX = pTerm->pExpr; - if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ - continue; - } - assert(pX->pLeft); - pColl = sqlite3BinaryCompareCollSeq(pParse, - pX->pLeft, pX->pRight); - if( pColl==0 ) pColl = pParse->db->pDfltColl; - if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){ - continue; - } - } - if( (pTerm->eOperator & WO_EQ)!=0 - && (pX = pTerm->pExpr->pRight)->op==TK_COLUMN - && pX->iTable==pScan->aEquiv[0] - && pX->iColumn==pScan->aEquiv[1] - ){ - continue; - } - pScan->k = k+1; - return pTerm; - } - } + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 + && pLoop->u.btree.pIndex!=0 + && pLoop->u.btree.pIndex->aSortOrder[iEq] + ){ + testcase( iEq==0 ); + testcase( bRev ); + bRev = !bRev; + } + assert( pX->op==TK_IN ); + iReg = iTarget; + eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0); + if( eType==IN_INDEX_INDEX_DESC ){ + testcase( bRev ); + bRev = !bRev; + } + iTab = pX->iTable; + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0); + VdbeCoverageIf(v, bRev); + VdbeCoverageIf(v, !bRev); + assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 ); + pLoop->wsFlags |= WHERE_IN_ABLE; + if( pLevel->u.in.nIn==0 ){ + pLevel->addrNxt = sqlite3VdbeMakeLabel(v); + } + pLevel->u.in.nIn++; + pLevel->u.in.aInLoop = + sqlite3DbReallocOrFree(pParse->db, pLevel->u.in.aInLoop, + sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn); + pIn = pLevel->u.in.aInLoop; + if( pIn ){ + pIn += pLevel->u.in.nIn - 1; + pIn->iCur = iTab; + if( eType==IN_INDEX_ROWID ){ + pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iReg); + }else{ + pIn->addrInTop = sqlite3VdbeAddOp3(v, OP_Column, iTab, 0, iReg); } - pScan->pWC = pScan->pWC->pOuter; - k = 0; + pIn->eEndLoopOp = bRev ? OP_PrevIfOpen : OP_NextIfOpen; + sqlite3VdbeAddOp1(v, OP_IsNull, iReg); VdbeCoverage(v); + }else{ + pLevel->u.in.nIn = 0; } - pScan->pWC = pScan->pOrigWC; - k = 0; - pScan->iEquiv += 2; +#endif } - return 0; + disableTerm(pLevel, pTerm); + return iReg; } /* -** Initialize a WHERE clause scanner object. Return a pointer to the -** first match. Return NULL if there are no matches. +** Generate code that will evaluate all == and IN constraints for an +** index scan. ** -** The scanner will be searching the WHERE clause pWC. It will look -** for terms of the form "X " where X is column iColumn of table -** iCur. The must be one of the operators described by opMask. +** For example, consider table t1(a,b,c,d,e,f) with index i1(a,b,c). +** Suppose the WHERE clause is this: a==5 AND b IN (1,2,3) AND c>5 AND c<10 +** The index has as many as three equality constraints, but in this +** example, the third "c" value is an inequality. So only two +** constraints are coded. This routine will generate code to evaluate +** a==5 and b IN (1,2,3). The current values for a and b will be stored +** in consecutive registers and the index of the first register is returned. ** -** If the search is for X and the WHERE clause contains terms of the -** form X=Y then this routine might also return terms of the form -** "Y ". The number of levels of transitivity is limited, -** but is enough to handle most commonly occurring SQL statements. +** In the example above nEq==2. But this subroutine works for any value +** of nEq including 0. If nEq==0, this routine is nearly a no-op. +** The only thing it does is allocate the pLevel->iMem memory cell and +** compute the affinity string. ** -** If X is not the INTEGER PRIMARY KEY then X must be compatible with -** index pIdx. +** The nExtraReg parameter is 0 or 1. It is 0 if all WHERE clause constraints +** are == or IN and are covered by the nEq. nExtraReg is 1 if there is +** an inequality constraint (such as the "c>=5 AND c<10" in the example) that +** occurs after the nEq quality constraints. +** +** This routine allocates a range of nEq+nExtraReg memory cells and returns +** the index of the first memory cell in that range. The code that +** calls this routine will use that memory range to store keys for +** start and termination conditions of the loop. +** key value of the loop. If one or more IN operators appear, then +** this routine allocates an additional nEq memory cells for internal +** use. +** +** Before returning, *pzAff is set to point to a buffer containing a +** copy of the column affinity string of the index allocated using +** sqlite3DbMalloc(). Except, entries in the copy of the string associated +** with equality constraints that use BLOB or NONE affinity are set to +** SQLITE_AFF_BLOB. This is to deal with SQL such as the following: +** +** CREATE TABLE t1(a TEXT PRIMARY KEY, b); +** SELECT ... FROM t1 AS t2, t1 WHERE t1.a = t2.b; +** +** In the example above, the index on t1(a) has TEXT affinity. But since +** the right hand side of the equality constraint (t2.b) has BLOB/NONE affinity, +** no conversion should be attempted before using a t2.b value as part of +** a key to search the index. Hence the first byte in the returned affinity +** string in this example would be set to SQLITE_AFF_BLOB. */ -static WhereTerm *whereScanInit( - WhereScan *pScan, /* The WhereScan object being initialized */ - WhereClause *pWC, /* The WHERE clause to be scanned */ - int iCur, /* Cursor to scan for */ - int iColumn, /* Column to scan for */ - u32 opMask, /* Operator(s) to scan for */ - Index *pIdx /* Must be compatible with this index */ +static int codeAllEqualityTerms( + Parse *pParse, /* Parsing context */ + WhereLevel *pLevel, /* Which nested loop of the FROM we are coding */ + int bRev, /* Reverse the order of IN operators */ + int nExtraReg, /* Number of extra registers to allocate */ + char **pzAff /* OUT: Set to point to affinity string */ ){ - int j; + u16 nEq; /* The number of == or IN constraints to code */ + u16 nSkip; /* Number of left-most columns to skip */ + Vdbe *v = pParse->pVdbe; /* The vm under construction */ + Index *pIdx; /* The index being used for this loop */ + WhereTerm *pTerm; /* A single constraint term */ + WhereLoop *pLoop; /* The WhereLoop object */ + int j; /* Loop counter */ + int regBase; /* Base register */ + int nReg; /* Number of registers to allocate */ + char *zAff; /* Affinity string to return */ - /* memset(pScan, 0, sizeof(*pScan)); */ - pScan->pOrigWC = pWC; - pScan->pWC = pWC; - if( pIdx && iColumn>=0 ){ - pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity; - for(j=0; pIdx->aiColumn[j]!=iColumn; j++){ - if( NEVER(j>pIdx->nColumn) ) return 0; - } - pScan->zCollName = pIdx->azColl[j]; - }else{ - pScan->idxaff = 0; - pScan->zCollName = 0; + /* This module is only called on query plans that use an index. */ + pLoop = pLevel->pWLoop; + assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); + nEq = pLoop->u.btree.nEq; + nSkip = pLoop->nSkip; + pIdx = pLoop->u.btree.pIndex; + assert( pIdx!=0 ); + + /* Figure out how many memory cells we will need then allocate them. + */ + regBase = pParse->nMem + 1; + nReg = pLoop->u.btree.nEq + nExtraReg; + pParse->nMem += nReg; + + zAff = sqlite3DbStrDup(pParse->db,sqlite3IndexAffinityStr(pParse->db,pIdx)); + if( !zAff ){ + pParse->db->mallocFailed = 1; } - pScan->opMask = opMask; - pScan->k = 0; - pScan->aEquiv[0] = iCur; - pScan->aEquiv[1] = iColumn; - pScan->nEquiv = 2; - pScan->iEquiv = 2; - return whereScanNext(pScan); -} -/* -** Search for a term in the WHERE clause that is of the form "X " -** where X is a reference to the iColumn of table iCur and is one of -** the WO_xx operator codes specified by the op parameter. -** Return a pointer to the term. Return 0 if not found. -** -** The term returned might by Y= if there is another constraint in -** the WHERE clause that specifies that X=Y. Any such constraints will be -** identified by the WO_EQUIV bit in the pTerm->eOperator field. The -** aEquiv[] array holds X and all its equivalents, with each SQL variable -** taking up two slots in aEquiv[]. The first slot is for the cursor number -** and the second is for the column number. There are 22 slots in aEquiv[] -** so that means we can look for X plus up to 10 other equivalent values. -** Hence a search for X will return if X=A1 and A1=A2 and A2=A3 -** and ... and A9=A10 and A10=. -** -** If there are multiple terms in the WHERE clause of the form "X " -** then try for the one with no dependencies on - in other words where -** is a constant expression of some kind. Only return entries of -** the form "X Y" where Y is a column in another table if no terms of -** the form "X " exist. If no terms with a constant RHS -** exist, try to return a term that does not use WO_EQUIV. -*/ -static WhereTerm *findTerm( - WhereClause *pWC, /* The WHERE clause to be searched */ - int iCur, /* Cursor number of LHS */ - int iColumn, /* Column number of LHS */ - Bitmask notReady, /* RHS must not overlap with this mask */ - u32 op, /* Mask of WO_xx values describing operator */ - Index *pIdx /* Must be compatible with this index, if not NULL */ -){ - WhereTerm *pResult = 0; - WhereTerm *p; - WhereScan scan; + if( nSkip ){ + int iIdxCur = pLevel->iIdxCur; + sqlite3VdbeAddOp1(v, (bRev?OP_Last:OP_Rewind), iIdxCur); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); + VdbeComment((v, "begin skip-scan on %s", pIdx->zName)); + j = sqlite3VdbeAddOp0(v, OP_Goto); + pLevel->addrSkip = sqlite3VdbeAddOp4Int(v, (bRev?OP_SeekLT:OP_SeekGT), + iIdxCur, 0, regBase, nSkip); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); + sqlite3VdbeJumpHere(v, j); + for(j=0; jaiColumn[j]==XN_EXPR ); + VdbeComment((v, "%s", explainIndexColumnName(pIdx, j))); + } + } - p = whereScanInit(&scan, pWC, iCur, iColumn, op, pIdx); - while( p ){ - if( (p->prereqRight & notReady)==0 ){ - if( p->prereqRight==0 && (p->eOperator&WO_EQ)!=0 ){ - return p; + /* Evaluate the equality constraints + */ + assert( zAff==0 || (int)strlen(zAff)>=nEq ); + for(j=nSkip; jaLTerm[j]; + assert( pTerm!=0 ); + /* The following testcase is true for indices with redundant columns. + ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ + testcase( (pTerm->wtFlags & TERM_CODED)!=0 ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + r1 = codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, regBase+j); + if( r1!=regBase+j ){ + if( nReg==1 ){ + sqlite3ReleaseTempReg(pParse, regBase); + regBase = r1; + }else{ + sqlite3VdbeAddOp2(v, OP_SCopy, r1, regBase+j); + } + } + testcase( pTerm->eOperator & WO_ISNULL ); + testcase( pTerm->eOperator & WO_IN ); + if( (pTerm->eOperator & (WO_ISNULL|WO_IN))==0 ){ + Expr *pRight = pTerm->pExpr->pRight; + if( (pTerm->wtFlags & TERM_IS)==0 && sqlite3ExprCanBeNull(pRight) ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regBase+j, pLevel->addrBrk); + VdbeCoverage(v); + } + if( zAff ){ + if( sqlite3CompareAffinity(pRight, zAff[j])==SQLITE_AFF_BLOB ){ + zAff[j] = SQLITE_AFF_BLOB; + } + if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[j]) ){ + zAff[j] = SQLITE_AFF_BLOB; + } } - if( pResult==0 ) pResult = p; } - p = whereScanNext(&scan); } - return pResult; + *pzAff = zAff; + return regBase; } -/* Forward reference */ -static void exprAnalyze(SrcList*, WhereClause*, int); - /* -** Call exprAnalyze on all terms in a WHERE clause. +** If the most recently coded instruction is a constant range contraint +** that originated from the LIKE optimization, then change the P3 to be +** pLoop->iLikeRepCntr and set P5. +** +** The LIKE optimization trys to evaluate "x LIKE 'abc%'" as a range +** expression: "x>='ABC' AND x<'abd'". But this requires that the range +** scan loop run twice, once for strings and a second time for BLOBs. +** The OP_String opcodes on the second pass convert the upper and lower +** bound string contants to blobs. This routine makes the necessary changes +** to the OP_String opcodes for that to happen. */ -static void exprAnalyzeAll( - SrcList *pTabList, /* the FROM clause */ - WhereClause *pWC /* the WHERE clause to be analyzed */ +static void whereLikeOptimizationStringFixup( + Vdbe *v, /* prepared statement under construction */ + WhereLevel *pLevel, /* The loop that contains the LIKE operator */ + WhereTerm *pTerm /* The upper or lower bound just coded */ ){ - int i; - for(i=pWC->nTerm-1; i>=0; i--){ - exprAnalyze(pTabList, pWC, i); + if( pTerm->wtFlags & TERM_LIKEOPT ){ + VdbeOp *pOp; + assert( pLevel->iLikeRepCntr>0 ); + pOp = sqlite3VdbeGetOp(v, -1); + assert( pOp!=0 ); + assert( pOp->opcode==OP_String8 + || pTerm->pWC->pWInfo->pParse->db->mallocFailed ); + pOp->p3 = pLevel->iLikeRepCntr; + pOp->p5 = 1; } } -#ifndef SQLITE_OMIT_LIKE_OPTIMIZATION + /* -** Check to see if the given expression is a LIKE or GLOB operator that -** can be optimized using inequality constraints. Return TRUE if it is -** so and false if not. -** -** In order for the operator to be optimizible, the RHS must be a string -** literal that does not begin with a wildcard. The LHS must be a column -** that may only be NULL, a string, or a BLOB, never a number. (This means -** that virtual tables cannot participate in the LIKE optimization.) If the -** collating sequence for the column on the LHS must be appropriate for -** the operator. +** Generate code for the start of the iLevel-th loop in the WHERE clause +** implementation described by pWInfo. */ -static int isLikeOrGlob( - Parse *pParse, /* Parsing and code generating context */ - Expr *pExpr, /* Test this expression */ - Expr **ppPrefix, /* Pointer to TK_STRING expression with pattern prefix */ - int *pisComplete, /* True if the only wildcard is % in the last character */ - int *pnoCase /* True if uppercase is equivalent to lowercase */ +SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( + WhereInfo *pWInfo, /* Complete information about the WHERE clause */ + int iLevel, /* Which level of pWInfo->a[] should be coded */ + Bitmask notReady /* Which tables are currently available */ ){ - const char *z = 0; /* String on RHS of LIKE operator */ - Expr *pRight, *pLeft; /* Right and left size of LIKE operator */ - ExprList *pList; /* List of operands to the LIKE operator */ - int c; /* One character in z[] */ - int cnt; /* Number of non-wildcard prefix characters */ - char wc[3]; /* Wildcard characters */ - sqlite3 *db = pParse->db; /* Database connection */ - sqlite3_value *pVal = 0; - int op; /* Opcode of pRight */ - - if( !sqlite3IsLikeFunction(db, pExpr, pnoCase, wc) ){ - return 0; - } -#ifdef SQLITE_EBCDIC - if( *pnoCase ) return 0; -#endif - pList = pExpr->x.pList; - pLeft = pList->a[1].pExpr; - if( pLeft->op!=TK_COLUMN - || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT - || IsVirtual(pLeft->pTab) /* Value might be numeric */ - ){ - /* IMP: R-02065-49465 The left-hand side of the LIKE or GLOB operator must - ** be the name of an indexed column with TEXT affinity. */ - return 0; - } - assert( pLeft->iColumn!=(-1) ); /* Because IPK never has AFF_TEXT */ + int j, k; /* Loop counters */ + int iCur; /* The VDBE cursor for the table */ + int addrNxt; /* Where to jump to continue with the next IN case */ + int omitTable; /* True if we use the index only */ + int bRev; /* True if we need to scan in reverse order */ + WhereLevel *pLevel; /* The where level to be coded */ + WhereLoop *pLoop; /* The WhereLoop object being coded */ + WhereClause *pWC; /* Decomposition of the entire WHERE clause */ + WhereTerm *pTerm; /* A WHERE clause term */ + Parse *pParse; /* Parsing context */ + sqlite3 *db; /* Database connection */ + Vdbe *v; /* The prepared stmt under constructions */ + struct SrcList_item *pTabItem; /* FROM clause term being coded */ + int addrBrk; /* Jump here to break out of the loop */ + int addrCont; /* Jump here to continue with next cycle */ + int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ + int iReleaseReg = 0; /* Temp register to free before returning */ - pRight = sqlite3ExprSkipCollate(pList->a[0].pExpr); - op = pRight->op; - if( op==TK_VARIABLE ){ - Vdbe *pReprepare = pParse->pReprepare; - int iCol = pRight->iColumn; - pVal = sqlite3VdbeGetBoundValue(pReprepare, iCol, SQLITE_AFF_NONE); - if( pVal && sqlite3_value_type(pVal)==SQLITE_TEXT ){ - z = (char *)sqlite3_value_text(pVal); - } - sqlite3VdbeSetVarmask(pParse->pVdbe, iCol); - assert( pRight->op==TK_VARIABLE || pRight->op==TK_REGISTER ); - }else if( op==TK_STRING ){ - z = pRight->u.zToken; - } - if( z ){ - cnt = 0; - while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ - cnt++; - } - if( cnt!=0 && 255!=(u8)z[cnt-1] ){ - Expr *pPrefix; - *pisComplete = c==wc[0] && z[cnt+1]==0; - pPrefix = sqlite3Expr(db, TK_STRING, z); - if( pPrefix ) pPrefix->u.zToken[cnt] = 0; - *ppPrefix = pPrefix; - if( op==TK_VARIABLE ){ - Vdbe *v = pParse->pVdbe; - sqlite3VdbeSetVarmask(v, pRight->iColumn); - if( *pisComplete && pRight->u.zToken[1] ){ - /* If the rhs of the LIKE expression is a variable, and the current - ** value of the variable means there is no need to invoke the LIKE - ** function, then no OP_Variable will be added to the program. - ** This causes problems for the sqlite3_bind_parameter_name() - ** API. To work around them, add a dummy OP_Variable here. - */ - int r1 = sqlite3GetTempReg(pParse); - sqlite3ExprCodeTarget(pParse, pRight, r1); - sqlite3VdbeChangeP3(v, sqlite3VdbeCurrentAddr(v)-1, 0); - sqlite3ReleaseTempReg(pParse, r1); - } - } - }else{ - z = 0; - } - } + pParse = pWInfo->pParse; + v = pParse->pVdbe; + pWC = &pWInfo->sWC; + db = pParse->db; + pLevel = &pWInfo->a[iLevel]; + pLoop = pLevel->pWLoop; + pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; + iCur = pTabItem->iCursor; + pLevel->notReady = notReady & ~sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); + bRev = (pWInfo->revMask>>iLevel)&1; + omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0 + && (pWInfo->wctrlFlags & WHERE_FORCE_TABLE)==0; + VdbeModuleComment((v, "Begin WHERE-loop%d: %s",iLevel,pTabItem->pTab->zName)); - sqlite3ValueFree(pVal); - return (z!=0); -} -#endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ + /* Create labels for the "break" and "continue" instructions + ** for the current loop. Jump to addrBrk to break out of a loop. + ** Jump to cont to go immediately to the next iteration of the + ** loop. + ** + ** When there is an IN operator, we also have a "addrNxt" label that + ** means to continue with the next IN value combination. When + ** there are no IN operators in the constraints, the "addrNxt" label + ** is the same as "addrBrk". + */ + addrBrk = pLevel->addrBrk = pLevel->addrNxt = sqlite3VdbeMakeLabel(v); + addrCont = pLevel->addrCont = sqlite3VdbeMakeLabel(v); + + /* If this is the right table of a LEFT OUTER JOIN, allocate and + ** initialize a memory cell that records if this table matches any + ** row of the left table of the join. + */ + if( pLevel->iFrom>0 && (pTabItem[0].fg.jointype & JT_LEFT)!=0 ){ + pLevel->iLeftJoin = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin); + VdbeComment((v, "init LEFT JOIN no-match flag")); + } + /* Special case of a FROM clause subquery implemented as a co-routine */ + if( pTabItem->fg.viaCoroutine ){ + int regYield = pTabItem->regReturn; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); + pLevel->p2 = sqlite3VdbeAddOp2(v, OP_Yield, regYield, addrBrk); + VdbeCoverage(v); + VdbeComment((v, "next row of \"%s\"", pTabItem->pTab->zName)); + pLevel->op = OP_Goto; + }else #ifndef SQLITE_OMIT_VIRTUALTABLE -/* -** Check to see if the given expression is of the form -** -** column MATCH expr -** -** If it is then return TRUE. If not, return FALSE. -*/ -static int isMatchOfColumn( - Expr *pExpr /* Test this expression */ -){ - ExprList *pList; + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ + /* Case 1: The table is a virtual-table. Use the VFilter and VNext + ** to access the data. + */ + int iReg; /* P3 Value for OP_VFilter */ + int addrNotFound; + int nConstraint = pLoop->nLTerm; - if( pExpr->op!=TK_FUNCTION ){ - return 0; - } - if( sqlite3StrICmp(pExpr->u.zToken,"match")!=0 ){ - return 0; - } - pList = pExpr->x.pList; - if( pList->nExpr!=2 ){ - return 0; - } - if( pList->a[1].pExpr->op != TK_COLUMN ){ - return 0; - } - return 1; -} + sqlite3ExprCachePush(pParse); + iReg = sqlite3GetTempRange(pParse, nConstraint+2); + addrNotFound = pLevel->addrBrk; + for(j=0; jaLTerm[j]; + if( pTerm==0 ) continue; + if( pTerm->eOperator & WO_IN ){ + codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget); + addrNotFound = pLevel->addrNxt; + }else{ + sqlite3ExprCode(pParse, pTerm->pExpr->pRight, iTarget); + } + } + sqlite3VdbeAddOp2(v, OP_Integer, pLoop->u.vtab.idxNum, iReg); + sqlite3VdbeAddOp2(v, OP_Integer, nConstraint, iReg+1); + sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg, + pLoop->u.vtab.idxStr, + pLoop->u.vtab.needFree ? P4_MPRINTF : P4_STATIC); + VdbeCoverage(v); + pLoop->u.vtab.needFree = 0; + for(j=0; ju.vtab.omitMask>>j)&1 ){ + disableTerm(pLevel, pLoop->aLTerm[j]); + } + } + pLevel->p1 = iCur; + pLevel->op = pWInfo->eOnePass ? OP_Noop : OP_VNext; + pLevel->p2 = sqlite3VdbeCurrentAddr(v); + sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2); + sqlite3ExprCachePop(pParse); + }else #endif /* SQLITE_OMIT_VIRTUALTABLE */ -/* -** If the pBase expression originated in the ON or USING clause of -** a join, then transfer the appropriate markings over to derived. -*/ -static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ - if( pDerived ){ - pDerived->flags |= pBase->flags & EP_FromJoin; - pDerived->iRightJoinTable = pBase->iRightJoinTable; - } -} - -/* -** Mark term iChild as being a child of term iParent -*/ -static void markTermAsChild(WhereClause *pWC, int iChild, int iParent){ - pWC->a[iChild].iParent = iParent; - pWC->a[iChild].truthProb = pWC->a[iParent].truthProb; - pWC->a[iParent].nChild++; -} + if( (pLoop->wsFlags & WHERE_IPK)!=0 + && (pLoop->wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_EQ))!=0 + ){ + /* Case 2: We can directly reference a single row using an + ** equality comparison against the ROWID field. Or + ** we reference multiple rows using a "rowid IN (...)" + ** construct. + */ + assert( pLoop->u.btree.nEq==1 ); + pTerm = pLoop->aLTerm[0]; + assert( pTerm!=0 ); + assert( pTerm->pExpr!=0 ); + assert( omitTable==0 ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + iReleaseReg = ++pParse->nMem; + iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, bRev, iReleaseReg); + if( iRowidReg!=iReleaseReg ) sqlite3ReleaseTempReg(pParse, iReleaseReg); + addrNxt = pLevel->addrNxt; + sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addrNxt, iRowidReg); + VdbeCoverage(v); + sqlite3ExprCacheAffinityChange(pParse, iRowidReg, 1); + sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); + VdbeComment((v, "pk")); + pLevel->op = OP_Noop; + }else if( (pLoop->wsFlags & WHERE_IPK)!=0 + && (pLoop->wsFlags & WHERE_COLUMN_RANGE)!=0 + ){ + /* Case 3: We have an inequality comparison against the ROWID field. + */ + int testOp = OP_Noop; + int start; + int memEndValue = 0; + WhereTerm *pStart, *pEnd; -/* -** Return the N-th AND-connected subterm of pTerm. Or if pTerm is not -** a conjunction, then return just pTerm when N==0. If N is exceeds -** the number of available subterms, return NULL. -*/ -static WhereTerm *whereNthSubterm(WhereTerm *pTerm, int N){ - if( pTerm->eOperator!=WO_AND ){ - return N==0 ? pTerm : 0; - } - if( Nu.pAndInfo->wc.nTerm ){ - return &pTerm->u.pAndInfo->wc.a[N]; - } - return 0; -} + assert( omitTable==0 ); + j = 0; + pStart = pEnd = 0; + if( pLoop->wsFlags & WHERE_BTM_LIMIT ) pStart = pLoop->aLTerm[j++]; + if( pLoop->wsFlags & WHERE_TOP_LIMIT ) pEnd = pLoop->aLTerm[j++]; + assert( pStart!=0 || pEnd!=0 ); + if( bRev ){ + pTerm = pStart; + pStart = pEnd; + pEnd = pTerm; + } + if( pStart ){ + Expr *pX; /* The expression that defines the start bound */ + int r1, rTemp; /* Registers for holding the start boundary */ -/* -** Subterms pOne and pTwo are contained within WHERE clause pWC. The -** two subterms are in disjunction - they are OR-ed together. -** -** If these two terms are both of the form: "A op B" with the same -** A and B values but different operators and if the operators are -** compatible (if one is = and the other is <, for example) then -** add a new virtual AND term to pWC that is the combination of the -** two. -** -** Some examples: -** -** x x<=y -** x=y OR x=y --> x=y -** x<=y OR x x<=y -** -** The following is NOT generated: -** -** xy --> x!=y -*/ -static void whereCombineDisjuncts( - SrcList *pSrc, /* the FROM clause */ - WhereClause *pWC, /* The complete WHERE clause */ - WhereTerm *pOne, /* First disjunct */ - WhereTerm *pTwo /* Second disjunct */ -){ - u16 eOp = pOne->eOperator | pTwo->eOperator; - sqlite3 *db; /* Database connection (for malloc) */ - Expr *pNew; /* New virtual expression */ - int op; /* Operator for the combined expression */ - int idxNew; /* Index in pWC of the next virtual term */ + /* The following constant maps TK_xx codes into corresponding + ** seek opcodes. It depends on a particular ordering of TK_xx + */ + const u8 aMoveOp[] = { + /* TK_GT */ OP_SeekGT, + /* TK_LE */ OP_SeekLE, + /* TK_LT */ OP_SeekLT, + /* TK_GE */ OP_SeekGE + }; + assert( TK_LE==TK_GT+1 ); /* Make sure the ordering.. */ + assert( TK_LT==TK_GT+2 ); /* ... of the TK_xx values... */ + assert( TK_GE==TK_GT+3 ); /* ... is correcct. */ - if( (pOne->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE))==0 ) return; - if( (pTwo->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE))==0 ) return; - if( (eOp & (WO_EQ|WO_LT|WO_LE))!=eOp - && (eOp & (WO_EQ|WO_GT|WO_GE))!=eOp ) return; - assert( pOne->pExpr->pLeft!=0 && pOne->pExpr->pRight!=0 ); - assert( pTwo->pExpr->pLeft!=0 && pTwo->pExpr->pRight!=0 ); - if( sqlite3ExprCompare(pOne->pExpr->pLeft, pTwo->pExpr->pLeft, -1) ) return; - if( sqlite3ExprCompare(pOne->pExpr->pRight, pTwo->pExpr->pRight, -1) )return; - /* If we reach this point, it means the two subterms can be combined */ - if( (eOp & (eOp-1))!=0 ){ - if( eOp & (WO_LT|WO_LE) ){ - eOp = WO_LE; + assert( (pStart->wtFlags & TERM_VNULL)==0 ); + testcase( pStart->wtFlags & TERM_VIRTUAL ); + pX = pStart->pExpr; + assert( pX!=0 ); + testcase( pStart->leftCursor!=iCur ); /* transitive constraints */ + r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp); + sqlite3VdbeAddOp3(v, aMoveOp[pX->op-TK_GT], iCur, addrBrk, r1); + VdbeComment((v, "pk")); + VdbeCoverageIf(v, pX->op==TK_GT); + VdbeCoverageIf(v, pX->op==TK_LE); + VdbeCoverageIf(v, pX->op==TK_LT); + VdbeCoverageIf(v, pX->op==TK_GE); + sqlite3ExprCacheAffinityChange(pParse, r1, 1); + sqlite3ReleaseTempReg(pParse, rTemp); + disableTerm(pLevel, pStart); }else{ - assert( eOp & (WO_GT|WO_GE) ); - eOp = WO_GE; + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrBrk); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); } - } - db = pWC->pWInfo->pParse->db; - pNew = sqlite3ExprDup(db, pOne->pExpr, 0); - if( pNew==0 ) return; - for(op=TK_EQ; eOp!=(WO_EQ<<(op-TK_EQ)); op++){ assert( opop = op; - idxNew = whereClauseInsert(pWC, pNew, TERM_VIRTUAL|TERM_DYNAMIC); - exprAnalyze(pSrc, pWC, idxNew); -} + if( pEnd ){ + Expr *pX; + pX = pEnd->pExpr; + assert( pX!=0 ); + assert( (pEnd->wtFlags & TERM_VNULL)==0 ); + testcase( pEnd->leftCursor!=iCur ); /* Transitive constraints */ + testcase( pEnd->wtFlags & TERM_VIRTUAL ); + memEndValue = ++pParse->nMem; + sqlite3ExprCode(pParse, pX->pRight, memEndValue); + if( pX->op==TK_LT || pX->op==TK_GT ){ + testOp = bRev ? OP_Le : OP_Ge; + }else{ + testOp = bRev ? OP_Lt : OP_Gt; + } + disableTerm(pLevel, pEnd); + } + start = sqlite3VdbeCurrentAddr(v); + pLevel->op = bRev ? OP_Prev : OP_Next; + pLevel->p1 = iCur; + pLevel->p2 = start; + assert( pLevel->p5==0 ); + if( testOp!=OP_Noop ){ + iRowidReg = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Rowid, iCur, iRowidReg); + sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); + sqlite3VdbeAddOp3(v, testOp, memEndValue, addrBrk, iRowidReg); + VdbeCoverageIf(v, testOp==OP_Le); + VdbeCoverageIf(v, testOp==OP_Lt); + VdbeCoverageIf(v, testOp==OP_Ge); + VdbeCoverageIf(v, testOp==OP_Gt); + sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC | SQLITE_JUMPIFNULL); + } + }else if( pLoop->wsFlags & WHERE_INDEXED ){ + /* Case 4: A scan using an index. + ** + ** The WHERE clause may contain zero or more equality + ** terms ("==" or "IN" operators) that refer to the N + ** left-most columns of the index. It may also contain + ** inequality constraints (>, <, >= or <=) on the indexed + ** column that immediately follows the N equalities. Only + ** the right-most column can be an inequality - the rest must + ** use the "==" and "IN" operators. For example, if the + ** index is on (x,y,z), then the following clauses are all + ** optimized: + ** + ** x=5 + ** x=5 AND y=10 + ** x=5 AND y<10 + ** x=5 AND y>5 AND y<10 + ** x=5 AND y=5 AND z<=10 + ** + ** The z<10 term of the following cannot be used, only + ** the x=5 term: + ** + ** x=5 AND z<10 + ** + ** N may be zero if there are inequality constraints. + ** If there are no inequality constraints, then N is at + ** least one. + ** + ** This case is also used when there are no WHERE clause + ** constraints but an index is selected anyway, in order + ** to force the output order to conform to an ORDER BY. + */ + static const u8 aStartOp[] = { + 0, + 0, + OP_Rewind, /* 2: (!start_constraints && startEq && !bRev) */ + OP_Last, /* 3: (!start_constraints && startEq && bRev) */ + OP_SeekGT, /* 4: (start_constraints && !startEq && !bRev) */ + OP_SeekLT, /* 5: (start_constraints && !startEq && bRev) */ + OP_SeekGE, /* 6: (start_constraints && startEq && !bRev) */ + OP_SeekLE /* 7: (start_constraints && startEq && bRev) */ + }; + static const u8 aEndOp[] = { + OP_IdxGE, /* 0: (end_constraints && !bRev && !endEq) */ + OP_IdxGT, /* 1: (end_constraints && !bRev && endEq) */ + OP_IdxLE, /* 2: (end_constraints && bRev && !endEq) */ + OP_IdxLT, /* 3: (end_constraints && bRev && endEq) */ + }; + u16 nEq = pLoop->u.btree.nEq; /* Number of == or IN terms */ + int regBase; /* Base register holding constraint values */ + WhereTerm *pRangeStart = 0; /* Inequality constraint at range start */ + WhereTerm *pRangeEnd = 0; /* Inequality constraint at range end */ + int startEq; /* True if range start uses ==, >= or <= */ + int endEq; /* True if range end uses ==, >= or <= */ + int start_constraints; /* Start of range is constrained */ + int nConstraint; /* Number of constraint terms */ + Index *pIdx; /* The index we will be using */ + int iIdxCur; /* The VDBE cursor for the index */ + int nExtraReg = 0; /* Number of extra registers needed */ + int op; /* Instruction opcode */ + char *zStartAff; /* Affinity for start of range constraint */ + char cEndAff = 0; /* Affinity for end of range constraint */ + u8 bSeekPastNull = 0; /* True to seek past initial nulls */ + u8 bStopAtNull = 0; /* Add condition to terminate at NULLs */ -#if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) -/* -** Analyze a term that consists of two or more OR-connected -** subterms. So in: -** -** ... WHERE (a=5) AND (b=7 OR c=9 OR d=13) AND (d=13) -** ^^^^^^^^^^^^^^^^^^^^ -** -** This routine analyzes terms such as the middle term in the above example. -** A WhereOrTerm object is computed and attached to the term under -** analysis, regardless of the outcome of the analysis. Hence: -** -** WhereTerm.wtFlags |= TERM_ORINFO -** WhereTerm.u.pOrInfo = a dynamically allocated WhereOrTerm object -** -** The term being analyzed must have two or more of OR-connected subterms. -** A single subterm might be a set of AND-connected sub-subterms. -** Examples of terms under analysis: -** -** (A) t1.x=t2.y OR t1.x=t2.z OR t1.y=15 OR t1.z=t3.a+5 -** (B) x=expr1 OR expr2=x OR x=expr3 -** (C) t1.x=t2.y OR (t1.x=t2.z AND t1.y=15) -** (D) x=expr1 OR (y>11 AND y<22 AND z LIKE '*hello*') -** (E) (p.a=1 AND q.b=2 AND r.c=3) OR (p.x=4 AND q.y=5 AND r.z=6) -** (F) x>A OR (x=A AND y>=B) -** -** CASE 1: -** -** If all subterms are of the form T.C=expr for some single column of C and -** a single table T (as shown in example B above) then create a new virtual -** term that is an equivalent IN expression. In other words, if the term -** being analyzed is: -** -** x = expr1 OR expr2 = x OR x = expr3 -** -** then create a new virtual term like this: -** -** x IN (expr1,expr2,expr3) -** -** CASE 2: -** -** If there are exactly two disjuncts one side has x>A and the other side -** has x=A (for the same x and A) then add a new virtual conjunct term to the -** WHERE clause of the form "x>=A". Example: -** -** x>A OR (x=A AND y>B) adds: x>=A -** -** The added conjunct can sometimes be helpful in query planning. -** -** CASE 3: -** -** If all subterms are indexable by a single table T, then set -** -** WhereTerm.eOperator = WO_OR -** WhereTerm.u.pOrInfo->indexable |= the cursor number for table T -** -** A subterm is "indexable" if it is of the form -** "T.C " where C is any column of table T and -** is one of "=", "<", "<=", ">", ">=", "IS NULL", or "IN". -** A subterm is also indexable if it is an AND of two or more -** subsubterms at least one of which is indexable. Indexable AND -** subterms have their eOperator set to WO_AND and they have -** u.pAndInfo set to a dynamically allocated WhereAndTerm object. -** -** From another point of view, "indexable" means that the subterm could -** potentially be used with an index if an appropriate index exists. -** This analysis does not consider whether or not the index exists; that -** is decided elsewhere. This analysis only looks at whether subterms -** appropriate for indexing exist. -** -** All examples A through E above satisfy case 2. But if a term -** also satisfies case 1 (such as B) we know that the optimizer will -** always prefer case 1, so in that case we pretend that case 2 is not -** satisfied. -** -** It might be the case that multiple tables are indexable. For example, -** (E) above is indexable on tables P, Q, and R. -** -** Terms that satisfy case 2 are candidates for lookup by using -** separate indices to find rowids for each subterm and composing -** the union of all rowids using a RowSet object. This is similar -** to "bitmap indices" in other database engines. -** -** OTHERWISE: -** -** If neither case 1 nor case 2 apply, then leave the eOperator set to -** zero. This term is not useful for search. -*/ -static void exprAnalyzeOrTerm( - SrcList *pSrc, /* the FROM clause */ - WhereClause *pWC, /* the complete WHERE clause */ - int idxTerm /* Index of the OR-term to be analyzed */ -){ - WhereInfo *pWInfo = pWC->pWInfo; /* WHERE clause processing context */ - Parse *pParse = pWInfo->pParse; /* Parser context */ - sqlite3 *db = pParse->db; /* Database connection */ - WhereTerm *pTerm = &pWC->a[idxTerm]; /* The term to be analyzed */ - Expr *pExpr = pTerm->pExpr; /* The expression of the term */ - int i; /* Loop counters */ - WhereClause *pOrWc; /* Breakup of pTerm into subterms */ - WhereTerm *pOrTerm; /* A Sub-term within the pOrWc */ - WhereOrInfo *pOrInfo; /* Additional information associated with pTerm */ - Bitmask chngToIN; /* Tables that might satisfy case 1 */ - Bitmask indexable; /* Tables that are indexable, satisfying case 2 */ + pIdx = pLoop->u.btree.pIndex; + iIdxCur = pLevel->iIdxCur; + assert( nEq>=pLoop->nSkip ); - /* - ** Break the OR clause into its separate subterms. The subterms are - ** stored in a WhereClause structure containing within the WhereOrInfo - ** object that is attached to the original OR clause term. - */ - assert( (pTerm->wtFlags & (TERM_DYNAMIC|TERM_ORINFO|TERM_ANDINFO))==0 ); - assert( pExpr->op==TK_OR ); - pTerm->u.pOrInfo = pOrInfo = sqlite3DbMallocZero(db, sizeof(*pOrInfo)); - if( pOrInfo==0 ) return; - pTerm->wtFlags |= TERM_ORINFO; - pOrWc = &pOrInfo->wc; - whereClauseInit(pOrWc, pWInfo); - whereSplit(pOrWc, pExpr, TK_OR); - exprAnalyzeAll(pSrc, pOrWc); - if( db->mallocFailed ) return; - assert( pOrWc->nTerm>=2 ); + /* If this loop satisfies a sort order (pOrderBy) request that + ** was passed to this function to implement a "SELECT min(x) ..." + ** query, then the caller will only allow the loop to run for + ** a single iteration. This means that the first row returned + ** should not have a NULL value stored in 'x'. If column 'x' is + ** the first one after the nEq equality constraints in the index, + ** this requires some special handling. + */ + assert( pWInfo->pOrderBy==0 + || pWInfo->pOrderBy->nExpr==1 + || (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 ); + if( (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)!=0 + && pWInfo->nOBSat>0 + && (pIdx->nKeyCol>nEq) + ){ + assert( pLoop->nSkip==0 ); + bSeekPastNull = 1; + nExtraReg = 1; + } - /* - ** Compute the set of tables that might satisfy cases 1 or 2. - */ - indexable = ~(Bitmask)0; - chngToIN = ~(Bitmask)0; - for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0 && indexable; i--, pOrTerm++){ - if( (pOrTerm->eOperator & WO_SINGLE)==0 ){ - WhereAndInfo *pAndInfo; - assert( (pOrTerm->wtFlags & (TERM_ANDINFO|TERM_ORINFO))==0 ); - chngToIN = 0; - pAndInfo = sqlite3DbMallocRaw(db, sizeof(*pAndInfo)); - if( pAndInfo ){ - WhereClause *pAndWC; - WhereTerm *pAndTerm; - int j; - Bitmask b = 0; - pOrTerm->u.pAndInfo = pAndInfo; - pOrTerm->wtFlags |= TERM_ANDINFO; - pOrTerm->eOperator = WO_AND; - pAndWC = &pAndInfo->wc; - whereClauseInit(pAndWC, pWC->pWInfo); - whereSplit(pAndWC, pOrTerm->pExpr, TK_AND); - exprAnalyzeAll(pSrc, pAndWC); - pAndWC->pOuter = pWC; - testcase( db->mallocFailed ); - if( !db->mallocFailed ){ - for(j=0, pAndTerm=pAndWC->a; jnTerm; j++, pAndTerm++){ - assert( pAndTerm->pExpr ); - if( allowedOp(pAndTerm->pExpr->op) ){ - b |= getMask(&pWInfo->sMaskSet, pAndTerm->leftCursor); - } - } - } - indexable &= b; - } - }else if( pOrTerm->wtFlags & TERM_COPIED ){ - /* Skip this term for now. We revisit it when we process the - ** corresponding TERM_VIRTUAL term */ - }else{ - Bitmask b; - b = getMask(&pWInfo->sMaskSet, pOrTerm->leftCursor); - if( pOrTerm->wtFlags & TERM_VIRTUAL ){ - WhereTerm *pOther = &pOrWc->a[pOrTerm->iParent]; - b |= getMask(&pWInfo->sMaskSet, pOther->leftCursor); + /* Find any inequality constraint terms for the start and end + ** of the range. + */ + j = nEq; + if( pLoop->wsFlags & WHERE_BTM_LIMIT ){ + pRangeStart = pLoop->aLTerm[j++]; + nExtraReg = 1; + /* Like optimization range constraints always occur in pairs */ + assert( (pRangeStart->wtFlags & TERM_LIKEOPT)==0 || + (pLoop->wsFlags & WHERE_TOP_LIMIT)!=0 ); + } + if( pLoop->wsFlags & WHERE_TOP_LIMIT ){ + pRangeEnd = pLoop->aLTerm[j++]; + nExtraReg = 1; + if( (pRangeEnd->wtFlags & TERM_LIKEOPT)!=0 ){ + assert( pRangeStart!=0 ); /* LIKE opt constraints */ + assert( pRangeStart->wtFlags & TERM_LIKEOPT ); /* occur in pairs */ + pLevel->iLikeRepCntr = ++pParse->nMem; + testcase( bRev ); + testcase( pIdx->aSortOrder[nEq]==SQLITE_SO_DESC ); + sqlite3VdbeAddOp2(v, OP_Integer, + bRev ^ (pIdx->aSortOrder[nEq]==SQLITE_SO_DESC), + pLevel->iLikeRepCntr); + VdbeComment((v, "LIKE loop counter")); + pLevel->addrLikeRep = sqlite3VdbeCurrentAddr(v); } - indexable &= b; - if( (pOrTerm->eOperator & WO_EQ)==0 ){ - chngToIN = 0; - }else{ - chngToIN &= b; + if( pRangeStart==0 + && (j = pIdx->aiColumn[nEq])>=0 + && pIdx->pTable->aCol[j].notNull==0 + ){ + bSeekPastNull = 1; } } - } + assert( pRangeEnd==0 || (pRangeEnd->wtFlags & TERM_VNULL)==0 ); - /* - ** Record the set of tables that satisfy case 3. The set might be - ** empty. - */ - pOrInfo->indexable = indexable; - pTerm->eOperator = indexable==0 ? 0 : WO_OR; + /* Generate code to evaluate all constraint terms using == or IN + ** and store the values of those terms in an array of registers + ** starting at regBase. + */ + regBase = codeAllEqualityTerms(pParse,pLevel,bRev,nExtraReg,&zStartAff); + assert( zStartAff==0 || sqlite3Strlen30(zStartAff)>=nEq ); + if( zStartAff ) cEndAff = zStartAff[nEq]; + addrNxt = pLevel->addrNxt; - /* For a two-way OR, attempt to implementation case 2. - */ - if( indexable && pOrWc->nTerm==2 ){ - int iOne = 0; - WhereTerm *pOne; - while( (pOne = whereNthSubterm(&pOrWc->a[0],iOne++))!=0 ){ - int iTwo = 0; - WhereTerm *pTwo; - while( (pTwo = whereNthSubterm(&pOrWc->a[1],iTwo++))!=0 ){ - whereCombineDisjuncts(pSrc, pWC, pOne, pTwo); - } + /* If we are doing a reverse order scan on an ascending index, or + ** a forward order scan on a descending index, interchange the + ** start and end terms (pRangeStart and pRangeEnd). + */ + if( (nEqnKeyCol && bRev==(pIdx->aSortOrder[nEq]==SQLITE_SO_ASC)) + || (bRev && pIdx->nKeyCol==nEq) + ){ + SWAP(WhereTerm *, pRangeEnd, pRangeStart); + SWAP(u8, bSeekPastNull, bStopAtNull); } - } - /* - ** chngToIN holds a set of tables that *might* satisfy case 1. But - ** we have to do some additional checking to see if case 1 really - ** is satisfied. - ** - ** chngToIN will hold either 0, 1, or 2 bits. The 0-bit case means - ** that there is no possibility of transforming the OR clause into an - ** IN operator because one or more terms in the OR clause contain - ** something other than == on a column in the single table. The 1-bit - ** case means that every term of the OR clause is of the form - ** "table.column=expr" for some single table. The one bit that is set - ** will correspond to the common table. We still need to check to make - ** sure the same column is used on all terms. The 2-bit case is when - ** the all terms are of the form "table1.column=table2.column". It - ** might be possible to form an IN operator with either table1.column - ** or table2.column as the LHS if either is common to every term of - ** the OR clause. - ** - ** Note that terms of the form "table.column1=table.column2" (the - ** same table on both sizes of the ==) cannot be optimized. - */ - if( chngToIN ){ - int okToChngToIN = 0; /* True if the conversion to IN is valid */ - int iColumn = -1; /* Column index on lhs of IN operator */ - int iCursor = -1; /* Table cursor common to all terms */ - int j = 0; /* Loop counter */ + testcase( pRangeStart && (pRangeStart->eOperator & WO_LE)!=0 ); + testcase( pRangeStart && (pRangeStart->eOperator & WO_GE)!=0 ); + testcase( pRangeEnd && (pRangeEnd->eOperator & WO_LE)!=0 ); + testcase( pRangeEnd && (pRangeEnd->eOperator & WO_GE)!=0 ); + startEq = !pRangeStart || pRangeStart->eOperator & (WO_LE|WO_GE); + endEq = !pRangeEnd || pRangeEnd->eOperator & (WO_LE|WO_GE); + start_constraints = pRangeStart || nEq>0; - /* Search for a table and column that appears on one side or the - ** other of the == operator in every subterm. That table and column - ** will be recorded in iCursor and iColumn. There might not be any - ** such table and column. Set okToChngToIN if an appropriate table - ** and column is found but leave okToChngToIN false if not found. - */ - for(j=0; j<2 && !okToChngToIN; j++){ - pOrTerm = pOrWc->a; - for(i=pOrWc->nTerm-1; i>=0; i--, pOrTerm++){ - assert( pOrTerm->eOperator & WO_EQ ); - pOrTerm->wtFlags &= ~TERM_OR_OK; - if( pOrTerm->leftCursor==iCursor ){ - /* This is the 2-bit case and we are on the second iteration and - ** current term is from the first iteration. So skip this term. */ - assert( j==1 ); - continue; - } - if( (chngToIN & getMask(&pWInfo->sMaskSet, pOrTerm->leftCursor))==0 ){ - /* This term must be of the form t1.a==t2.b where t2 is in the - ** chngToIN set but t1 is not. This term will be either preceded - ** or follwed by an inverted copy (t2.b==t1.a). Skip this term - ** and use its inversion. */ - testcase( pOrTerm->wtFlags & TERM_COPIED ); - testcase( pOrTerm->wtFlags & TERM_VIRTUAL ); - assert( pOrTerm->wtFlags & (TERM_COPIED|TERM_VIRTUAL) ); - continue; - } - iColumn = pOrTerm->u.leftColumn; - iCursor = pOrTerm->leftCursor; - break; - } - if( i<0 ){ - /* No candidate table+column was found. This can only occur - ** on the second iteration */ - assert( j==1 ); - assert( IsPowerOfTwo(chngToIN) ); - assert( chngToIN==getMask(&pWInfo->sMaskSet, iCursor) ); - break; + /* Seek the index cursor to the start of the range. */ + nConstraint = nEq; + if( pRangeStart ){ + Expr *pRight = pRangeStart->pExpr->pRight; + sqlite3ExprCode(pParse, pRight, regBase+nEq); + whereLikeOptimizationStringFixup(v, pLevel, pRangeStart); + if( (pRangeStart->wtFlags & TERM_VNULL)==0 + && sqlite3ExprCanBeNull(pRight) + ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); + VdbeCoverage(v); } - testcase( j==1 ); - - /* We have found a candidate table and column. Check to see if that - ** table and column is common to every term in the OR clause */ - okToChngToIN = 1; - for(; i>=0 && okToChngToIN; i--, pOrTerm++){ - assert( pOrTerm->eOperator & WO_EQ ); - if( pOrTerm->leftCursor!=iCursor ){ - pOrTerm->wtFlags &= ~TERM_OR_OK; - }else if( pOrTerm->u.leftColumn!=iColumn ){ - okToChngToIN = 0; - }else{ - int affLeft, affRight; - /* If the right-hand side is also a column, then the affinities - ** of both right and left sides must be such that no type - ** conversions are required on the right. (Ticket #2249) - */ - affRight = sqlite3ExprAffinity(pOrTerm->pExpr->pRight); - affLeft = sqlite3ExprAffinity(pOrTerm->pExpr->pLeft); - if( affRight!=0 && affRight!=affLeft ){ - okToChngToIN = 0; - }else{ - pOrTerm->wtFlags |= TERM_OR_OK; - } + if( zStartAff ){ + if( sqlite3CompareAffinity(pRight, zStartAff[nEq])==SQLITE_AFF_BLOB){ + /* Since the comparison is to be performed with no conversions + ** applied to the operands, set the affinity to apply to pRight to + ** SQLITE_AFF_BLOB. */ + zStartAff[nEq] = SQLITE_AFF_BLOB; } - } + if( sqlite3ExprNeedsNoAffinityChange(pRight, zStartAff[nEq]) ){ + zStartAff[nEq] = SQLITE_AFF_BLOB; + } + } + nConstraint++; + testcase( pRangeStart->wtFlags & TERM_VIRTUAL ); + }else if( bSeekPastNull ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); + nConstraint++; + startEq = 0; + start_constraints = 1; } + codeApplyAffinity(pParse, regBase, nConstraint - bSeekPastNull, zStartAff); + op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev]; + assert( op!=0 ); + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); + VdbeCoverage(v); + VdbeCoverageIf(v, op==OP_Rewind); testcase( op==OP_Rewind ); + VdbeCoverageIf(v, op==OP_Last); testcase( op==OP_Last ); + VdbeCoverageIf(v, op==OP_SeekGT); testcase( op==OP_SeekGT ); + VdbeCoverageIf(v, op==OP_SeekGE); testcase( op==OP_SeekGE ); + VdbeCoverageIf(v, op==OP_SeekLE); testcase( op==OP_SeekLE ); + VdbeCoverageIf(v, op==OP_SeekLT); testcase( op==OP_SeekLT ); - /* At this point, okToChngToIN is true if original pTerm satisfies - ** case 1. In that case, construct a new virtual term that is - ** pTerm converted into an IN operator. + /* Load the value for the inequality constraint at the end of the + ** range (if any). */ - if( okToChngToIN ){ - Expr *pDup; /* A transient duplicate expression */ - ExprList *pList = 0; /* The RHS of the IN operator */ - Expr *pLeft = 0; /* The LHS of the IN operator */ - Expr *pNew; /* The complete IN operator */ - - for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0; i--, pOrTerm++){ - if( (pOrTerm->wtFlags & TERM_OR_OK)==0 ) continue; - assert( pOrTerm->eOperator & WO_EQ ); - assert( pOrTerm->leftCursor==iCursor ); - assert( pOrTerm->u.leftColumn==iColumn ); - pDup = sqlite3ExprDup(db, pOrTerm->pExpr->pRight, 0); - pList = sqlite3ExprListAppend(pWInfo->pParse, pList, pDup); - pLeft = pOrTerm->pExpr->pLeft; + nConstraint = nEq; + if( pRangeEnd ){ + Expr *pRight = pRangeEnd->pExpr->pRight; + sqlite3ExprCacheRemove(pParse, regBase+nEq, 1); + sqlite3ExprCode(pParse, pRight, regBase+nEq); + whereLikeOptimizationStringFixup(v, pLevel, pRangeEnd); + if( (pRangeEnd->wtFlags & TERM_VNULL)==0 + && sqlite3ExprCanBeNull(pRight) + ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); + VdbeCoverage(v); } - assert( pLeft!=0 ); - pDup = sqlite3ExprDup(db, pLeft, 0); - pNew = sqlite3PExpr(pParse, TK_IN, pDup, 0, 0); - if( pNew ){ - int idxNew; - transferJoinMarkings(pNew, pExpr); - assert( !ExprHasProperty(pNew, EP_xIsSelect) ); - pNew->x.pList = pList; - idxNew = whereClauseInsert(pWC, pNew, TERM_VIRTUAL|TERM_DYNAMIC); - testcase( idxNew==0 ); - exprAnalyze(pSrc, pWC, idxNew); - pTerm = &pWC->a[idxTerm]; - markTermAsChild(pWC, idxNew, idxTerm); - }else{ - sqlite3ExprListDelete(db, pList); + if( sqlite3CompareAffinity(pRight, cEndAff)!=SQLITE_AFF_BLOB + && !sqlite3ExprNeedsNoAffinityChange(pRight, cEndAff) + ){ + codeApplyAffinity(pParse, regBase+nEq, 1, &cEndAff); } - pTerm->eOperator = WO_NOOP; /* case 1 trumps case 3 */ + nConstraint++; + testcase( pRangeEnd->wtFlags & TERM_VIRTUAL ); + }else if( bStopAtNull ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); + endEq = 0; + nConstraint++; } - } -} -#endif /* !SQLITE_OMIT_OR_OPTIMIZATION && !SQLITE_OMIT_SUBQUERY */ + sqlite3DbFree(db, zStartAff); -/* -** The input to this routine is an WhereTerm structure with only the -** "pExpr" field filled in. The job of this routine is to analyze the -** subexpression and populate all the other fields of the WhereTerm -** structure. -** -** If the expression is of the form " X" it gets commuted -** to the standard form of "X ". -** -** If the expression is of the form "X Y" where both X and Y are -** columns, then the original expression is unchanged and a new virtual -** term of the form "Y X" is added to the WHERE clause and -** analyzed separately. The original term is marked with TERM_COPIED -** and the new term is marked with TERM_DYNAMIC (because it's pExpr -** needs to be freed with the WhereClause) and TERM_VIRTUAL (because it -** is a commuted copy of a prior term.) The original term has nChild=1 -** and the copy has idxParent set to the index of the original term. -*/ -static void exprAnalyze( - SrcList *pSrc, /* the FROM clause */ - WhereClause *pWC, /* the WHERE clause */ - int idxTerm /* Index of the term to be analyzed */ -){ - WhereInfo *pWInfo = pWC->pWInfo; /* WHERE clause processing context */ - WhereTerm *pTerm; /* The term to be analyzed */ - WhereMaskSet *pMaskSet; /* Set of table index masks */ - Expr *pExpr; /* The expression to be analyzed */ - Bitmask prereqLeft; /* Prerequesites of the pExpr->pLeft */ - Bitmask prereqAll; /* Prerequesites of pExpr */ - Bitmask extraRight = 0; /* Extra dependencies on LEFT JOIN */ - Expr *pStr1 = 0; /* RHS of LIKE/GLOB operator */ - int isComplete = 0; /* RHS of LIKE/GLOB ends with wildcard */ - int noCase = 0; /* uppercase equivalent to lowercase */ - int op; /* Top-level operator. pExpr->op */ - Parse *pParse = pWInfo->pParse; /* Parsing context */ - sqlite3 *db = pParse->db; /* Database connection */ + /* Top of the loop body */ + pLevel->p2 = sqlite3VdbeCurrentAddr(v); - if( db->mallocFailed ){ - return; - } - pTerm = &pWC->a[idxTerm]; - pMaskSet = &pWInfo->sMaskSet; - pExpr = pTerm->pExpr; - assert( pExpr->op!=TK_AS && pExpr->op!=TK_COLLATE ); - prereqLeft = exprTableUsage(pMaskSet, pExpr->pLeft); - op = pExpr->op; - if( op==TK_IN ){ - assert( pExpr->pRight==0 ); - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - pTerm->prereqRight = exprSelectTableUsage(pMaskSet, pExpr->x.pSelect); - }else{ - pTerm->prereqRight = exprListTableUsage(pMaskSet, pExpr->x.pList); - } - }else if( op==TK_ISNULL ){ - pTerm->prereqRight = 0; - }else{ - pTerm->prereqRight = exprTableUsage(pMaskSet, pExpr->pRight); - } - prereqAll = exprTableUsage(pMaskSet, pExpr); - if( ExprHasProperty(pExpr, EP_FromJoin) ){ - Bitmask x = getMask(pMaskSet, pExpr->iRightJoinTable); - prereqAll |= x; - extraRight = x-1; /* ON clause terms may not be used with an index - ** on left table of a LEFT JOIN. Ticket #3015 */ - } - pTerm->prereqAll = prereqAll; - pTerm->leftCursor = -1; - pTerm->iParent = -1; - pTerm->eOperator = 0; - if( allowedOp(op) ){ - Expr *pLeft = sqlite3ExprSkipCollate(pExpr->pLeft); - Expr *pRight = sqlite3ExprSkipCollate(pExpr->pRight); - u16 opMask = (pTerm->prereqRight & prereqLeft)==0 ? WO_ALL : WO_EQUIV; - if( pLeft->op==TK_COLUMN ){ - pTerm->leftCursor = pLeft->iTable; - pTerm->u.leftColumn = pLeft->iColumn; - pTerm->eOperator = operatorMask(op) & opMask; + /* Check if the index cursor is past the end of the range. */ + if( nConstraint ){ + op = aEndOp[bRev*2 + endEq]; + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); + testcase( op==OP_IdxGT ); VdbeCoverageIf(v, op==OP_IdxGT ); + testcase( op==OP_IdxGE ); VdbeCoverageIf(v, op==OP_IdxGE ); + testcase( op==OP_IdxLT ); VdbeCoverageIf(v, op==OP_IdxLT ); + testcase( op==OP_IdxLE ); VdbeCoverageIf(v, op==OP_IdxLE ); } - if( pRight && pRight->op==TK_COLUMN ){ - WhereTerm *pNew; - Expr *pDup; - u16 eExtraOp = 0; /* Extra bits for pNew->eOperator */ - if( pTerm->leftCursor>=0 ){ - int idxNew; - pDup = sqlite3ExprDup(db, pExpr, 0); - if( db->mallocFailed ){ - sqlite3ExprDelete(db, pDup); - return; - } - idxNew = whereClauseInsert(pWC, pDup, TERM_VIRTUAL|TERM_DYNAMIC); - if( idxNew==0 ) return; - pNew = &pWC->a[idxNew]; - markTermAsChild(pWC, idxNew, idxTerm); - pTerm = &pWC->a[idxTerm]; - pTerm->wtFlags |= TERM_COPIED; - if( pExpr->op==TK_EQ - && !ExprHasProperty(pExpr, EP_FromJoin) - && OptimizationEnabled(db, SQLITE_Transitive) - ){ - pTerm->eOperator |= WO_EQUIV; - eExtraOp = WO_EQUIV; - } + + /* Seek the table cursor, if required */ + disableTerm(pLevel, pRangeStart); + disableTerm(pLevel, pRangeEnd); + if( omitTable ){ + /* pIdx is a covering index. No need to access the main table. */ + }else if( HasRowid(pIdx->pTable) ){ + iRowidReg = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg); + sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); + if( pWInfo->eOnePass!=ONEPASS_OFF ){ + sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, iRowidReg); + VdbeCoverage(v); }else{ - pDup = pExpr; - pNew = pTerm; + sqlite3VdbeAddOp2(v, OP_Seek, iCur, iRowidReg); /* Deferred seek */ } - exprCommute(pParse, pDup); - pLeft = sqlite3ExprSkipCollate(pDup->pLeft); - pNew->leftCursor = pLeft->iTable; - pNew->u.leftColumn = pLeft->iColumn; - testcase( (prereqLeft | extraRight) != prereqLeft ); - pNew->prereqRight = prereqLeft | extraRight; - pNew->prereqAll = prereqAll; - pNew->eOperator = (operatorMask(pDup->op) + eExtraOp) & opMask; + }else if( iCur!=iIdxCur ){ + Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable); + iRowidReg = sqlite3GetTempRange(pParse, pPk->nKeyCol); + for(j=0; jnKeyCol; j++){ + k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, iRowidReg+j); + } + sqlite3VdbeAddOp4Int(v, OP_NotFound, iCur, addrCont, + iRowidReg, pPk->nKeyCol); VdbeCoverage(v); } - } -#ifndef SQLITE_OMIT_BETWEEN_OPTIMIZATION - /* If a term is the BETWEEN operator, create two new virtual terms - ** that define the range that the BETWEEN implements. For example: - ** - ** a BETWEEN b AND c - ** - ** is converted into: - ** - ** (a BETWEEN b AND c) AND (a>=b) AND (a<=c) - ** - ** The two new terms are added onto the end of the WhereClause object. - ** The new terms are "dynamic" and are children of the original BETWEEN - ** term. That means that if the BETWEEN term is coded, the children are - ** skipped. Or, if the children are satisfied by an index, the original - ** BETWEEN term is skipped. - */ - else if( pExpr->op==TK_BETWEEN && pWC->op==TK_AND ){ - ExprList *pList = pExpr->x.pList; - int i; - static const u8 ops[] = {TK_GE, TK_LE}; - assert( pList!=0 ); - assert( pList->nExpr==2 ); - for(i=0; i<2; i++){ - Expr *pNewExpr; - int idxNew; - pNewExpr = sqlite3PExpr(pParse, ops[i], - sqlite3ExprDup(db, pExpr->pLeft, 0), - sqlite3ExprDup(db, pList->a[i].pExpr, 0), 0); - transferJoinMarkings(pNewExpr, pExpr); - idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); - testcase( idxNew==0 ); - exprAnalyze(pSrc, pWC, idxNew); - pTerm = &pWC->a[idxTerm]; - markTermAsChild(pWC, idxNew, idxTerm); + /* Record the instruction used to terminate the loop. Disable + ** WHERE clause terms made redundant by the index range scan. + */ + if( pLoop->wsFlags & WHERE_ONEROW ){ + pLevel->op = OP_Noop; + }else if( bRev ){ + pLevel->op = OP_Prev; + }else{ + pLevel->op = OP_Next; } - } -#endif /* SQLITE_OMIT_BETWEEN_OPTIMIZATION */ + pLevel->p1 = iIdxCur; + pLevel->p3 = (pLoop->wsFlags&WHERE_UNQ_WANTED)!=0 ? 1:0; + if( (pLoop->wsFlags & WHERE_CONSTRAINT)==0 ){ + pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + }else{ + assert( pLevel->p5==0 ); + } + }else -#if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) - /* Analyze a term that is composed of two or more subterms connected by - ** an OR operator. - */ - else if( pExpr->op==TK_OR ){ - assert( pWC->op==TK_AND ); - exprAnalyzeOrTerm(pSrc, pWC, idxTerm); - pTerm = &pWC->a[idxTerm]; - } -#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ +#ifndef SQLITE_OMIT_OR_OPTIMIZATION + if( pLoop->wsFlags & WHERE_MULTI_OR ){ + /* Case 5: Two or more separately indexed terms connected by OR + ** + ** Example: + ** + ** CREATE TABLE t1(a,b,c,d); + ** CREATE INDEX i1 ON t1(a); + ** CREATE INDEX i2 ON t1(b); + ** CREATE INDEX i3 ON t1(c); + ** + ** SELECT * FROM t1 WHERE a=5 OR b=7 OR (c=11 AND d=13) + ** + ** In the example, there are three indexed terms connected by OR. + ** The top of the loop looks like this: + ** + ** Null 1 # Zero the rowset in reg 1 + ** + ** Then, for each indexed term, the following. The arguments to + ** RowSetTest are such that the rowid of the current row is inserted + ** into the RowSet. If it is already present, control skips the + ** Gosub opcode and jumps straight to the code generated by WhereEnd(). + ** + ** sqlite3WhereBegin() + ** RowSetTest # Insert rowid into rowset + ** Gosub 2 A + ** sqlite3WhereEnd() + ** + ** Following the above, code to terminate the loop. Label A, the target + ** of the Gosub above, jumps to the instruction right after the Goto. + ** + ** Null 1 # Zero the rowset in reg 1 + ** Goto B # The loop is finished. + ** + ** A: # Return data, whatever. + ** + ** Return 2 # Jump back to the Gosub + ** + ** B: + ** + ** Added 2014-05-26: If the table is a WITHOUT ROWID table, then + ** use an ephemeral index instead of a RowSet to record the primary + ** keys of the rows we have already seen. + ** + */ + WhereClause *pOrWc; /* The OR-clause broken out into subterms */ + SrcList *pOrTab; /* Shortened table list or OR-clause generation */ + Index *pCov = 0; /* Potential covering index (or NULL) */ + int iCovCur = pParse->nTab++; /* Cursor used for index scans (if any) */ -#ifndef SQLITE_OMIT_LIKE_OPTIMIZATION - /* Add constraints to reduce the search space on a LIKE or GLOB - ** operator. - ** - ** A like pattern of the form "x LIKE 'aBc%'" is changed into constraints - ** - ** x>='ABC' AND x<'abd' AND x LIKE 'aBc%' - ** - ** The last character of the prefix "abc" is incremented to form the - ** termination condition "abd". If case is not significant (the default - ** for LIKE) then the lower-bound is made all uppercase and the upper- - ** bound is made all lowercase so that the bounds also work when comparing - ** BLOBs. - */ - if( pWC->op==TK_AND - && isLikeOrGlob(pParse, pExpr, &pStr1, &isComplete, &noCase) - ){ - Expr *pLeft; /* LHS of LIKE/GLOB operator */ - Expr *pStr2; /* Copy of pStr1 - RHS of LIKE/GLOB operator */ - Expr *pNewExpr1; - Expr *pNewExpr2; - int idxNew1; - int idxNew2; - const char *zCollSeqName; /* Name of collating sequence */ - const u16 wtFlags = TERM_LIKEOPT | TERM_VIRTUAL | TERM_DYNAMIC; + int regReturn = ++pParse->nMem; /* Register used with OP_Gosub */ + int regRowset = 0; /* Register for RowSet object */ + int regRowid = 0; /* Register holding rowid */ + int iLoopBody = sqlite3VdbeMakeLabel(v); /* Start of loop body */ + int iRetInit; /* Address of regReturn init */ + int untestedTerms = 0; /* Some terms not completely tested */ + int ii; /* Loop counter */ + u16 wctrlFlags; /* Flags for sub-WHERE clause */ + Expr *pAndExpr = 0; /* An ".. AND (...)" expression */ + Table *pTab = pTabItem->pTab; + + pTerm = pLoop->aLTerm[0]; + assert( pTerm!=0 ); + assert( pTerm->eOperator & WO_OR ); + assert( (pTerm->wtFlags & TERM_ORINFO)!=0 ); + pOrWc = &pTerm->u.pOrInfo->wc; + pLevel->op = OP_Return; + pLevel->p1 = regReturn; - pLeft = pExpr->x.pList->a[1].pExpr; - pStr2 = sqlite3ExprDup(db, pStr1, 0); + /* Set up a new SrcList in pOrTab containing the table being scanned + ** by this loop in the a[0] slot and all notReady tables in a[1..] slots. + ** This becomes the SrcList in the recursive call to sqlite3WhereBegin(). + */ + if( pWInfo->nLevel>1 ){ + int nNotReady; /* The number of notReady tables */ + struct SrcList_item *origSrc; /* Original list of tables */ + nNotReady = pWInfo->nLevel - iLevel - 1; + pOrTab = sqlite3StackAllocRaw(db, + sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0])); + if( pOrTab==0 ) return notReady; + pOrTab->nAlloc = (u8)(nNotReady + 1); + pOrTab->nSrc = pOrTab->nAlloc; + memcpy(pOrTab->a, pTabItem, sizeof(*pTabItem)); + origSrc = pWInfo->pTabList->a; + for(k=1; k<=nNotReady; k++){ + memcpy(&pOrTab->a[k], &origSrc[pLevel[k].iFrom], sizeof(pOrTab->a[k])); + } + }else{ + pOrTab = pWInfo->pTabList; + } - /* Convert the lower bound to upper-case and the upper bound to - ** lower-case (upper-case is less than lower-case in ASCII) so that - ** the range constraints also work for BLOBs + /* Initialize the rowset register to contain NULL. An SQL NULL is + ** equivalent to an empty rowset. Or, create an ephemeral index + ** capable of holding primary keys in the case of a WITHOUT ROWID. + ** + ** Also initialize regReturn to contain the address of the instruction + ** immediately following the OP_Return at the bottom of the loop. This + ** is required in a few obscure LEFT JOIN cases where control jumps + ** over the top of the loop into the body of it. In this case the + ** correct response for the end-of-loop code (the OP_Return) is to + ** fall through to the next instruction, just as an OP_Next does if + ** called on an uninitialized cursor. */ - if( noCase && !pParse->db->mallocFailed ){ - int i; - char c; - pTerm->wtFlags |= TERM_LIKE; - for(i=0; (c = pStr1->u.zToken[i])!=0; i++){ - pStr1->u.zToken[i] = sqlite3Toupper(c); - pStr2->u.zToken[i] = sqlite3Tolower(c); + if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ + if( HasRowid(pTab) ){ + regRowset = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Null, 0, regRowset); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + regRowset = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, regRowset, pPk->nKeyCol); + sqlite3VdbeSetP4KeyInfo(pParse, pPk); } + regRowid = ++pParse->nMem; } + iRetInit = sqlite3VdbeAddOp2(v, OP_Integer, 0, regReturn); - if( !db->mallocFailed ){ - u8 c, *pC; /* Last character before the first wildcard */ - pC = (u8*)&pStr2->u.zToken[sqlite3Strlen30(pStr2->u.zToken)-1]; - c = *pC; - if( noCase ){ - /* The point is to increment the last character before the first - ** wildcard. But if we increment '@', that will push it into the - ** alphabetic range where case conversions will mess up the - ** inequality. To avoid this, make sure to also run the full - ** LIKE on all candidate expressions by clearing the isComplete flag - */ - if( c=='A'-1 ) isComplete = 0; - c = sqlite3UpperToLower[c]; + /* If the original WHERE clause is z of the form: (x1 OR x2 OR ...) AND y + ** Then for every term xN, evaluate as the subexpression: xN AND z + ** That way, terms in y that are factored into the disjunction will + ** be picked up by the recursive calls to sqlite3WhereBegin() below. + ** + ** Actually, each subexpression is converted to "xN AND w" where w is + ** the "interesting" terms of z - terms that did not originate in the + ** ON or USING clause of a LEFT JOIN, and terms that are usable as + ** indices. + ** + ** This optimization also only applies if the (x1 OR x2 OR ...) term + ** is not contained in the ON clause of a LEFT JOIN. + ** See ticket http://www.sqlite.org/src/info/f2369304e4 + */ + if( pWC->nTerm>1 ){ + int iTerm; + for(iTerm=0; iTermnTerm; iTerm++){ + Expr *pExpr = pWC->a[iTerm].pExpr; + if( &pWC->a[iTerm] == pTerm ) continue; + if( ExprHasProperty(pExpr, EP_FromJoin) ) continue; + if( (pWC->a[iTerm].wtFlags & TERM_VIRTUAL)!=0 ) continue; + if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue; + testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO ); + pExpr = sqlite3ExprDup(db, pExpr, 0); + pAndExpr = sqlite3ExprAnd(db, pAndExpr, pExpr); + } + if( pAndExpr ){ + pAndExpr = sqlite3PExpr(pParse, TK_AND, 0, pAndExpr, 0); } - *pC = c + 1; } - zCollSeqName = noCase ? "NOCASE" : "BINARY"; - pNewExpr1 = sqlite3ExprDup(db, pLeft, 0); - pNewExpr1 = sqlite3PExpr(pParse, TK_GE, - sqlite3ExprAddCollateString(pParse,pNewExpr1,zCollSeqName), - pStr1, 0); - transferJoinMarkings(pNewExpr1, pExpr); - idxNew1 = whereClauseInsert(pWC, pNewExpr1, wtFlags); - testcase( idxNew1==0 ); - exprAnalyze(pSrc, pWC, idxNew1); - pNewExpr2 = sqlite3ExprDup(db, pLeft, 0); - pNewExpr2 = sqlite3PExpr(pParse, TK_LT, - sqlite3ExprAddCollateString(pParse,pNewExpr2,zCollSeqName), - pStr2, 0); - transferJoinMarkings(pNewExpr2, pExpr); - idxNew2 = whereClauseInsert(pWC, pNewExpr2, wtFlags); - testcase( idxNew2==0 ); - exprAnalyze(pSrc, pWC, idxNew2); - pTerm = &pWC->a[idxTerm]; - if( isComplete ){ - markTermAsChild(pWC, idxNew1, idxTerm); - markTermAsChild(pWC, idxNew2, idxTerm); + + /* Run a separate WHERE clause for each term of the OR clause. After + ** eliminating duplicates from other WHERE clauses, the action for each + ** sub-WHERE clause is to to invoke the main loop body as a subroutine. + */ + wctrlFlags = WHERE_OMIT_OPEN_CLOSE + | WHERE_FORCE_TABLE + | WHERE_ONETABLE_ONLY + | WHERE_NO_AUTOINDEX; + for(ii=0; iinTerm; ii++){ + WhereTerm *pOrTerm = &pOrWc->a[ii]; + if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ + WhereInfo *pSubWInfo; /* Info for single OR-term scan */ + Expr *pOrExpr = pOrTerm->pExpr; /* Current OR clause term */ + int jmp1 = 0; /* Address of jump operation */ + if( pAndExpr && !ExprHasProperty(pOrExpr, EP_FromJoin) ){ + pAndExpr->pLeft = pOrExpr; + pOrExpr = pAndExpr; + } + /* Loop through table entries that match term pOrTerm. */ + WHERETRACE(0xffff, ("Subplan for OR-clause:\n")); + pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, + wctrlFlags, iCovCur); + assert( pSubWInfo || pParse->nErr || db->mallocFailed ); + if( pSubWInfo ){ + WhereLoop *pSubLoop; + int addrExplain = sqlite3WhereExplainOneScan( + pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0 + ); + sqlite3WhereAddScanStatus(v, pOrTab, &pSubWInfo->a[0], addrExplain); + + /* This is the sub-WHERE clause body. First skip over + ** duplicate rows from prior sub-WHERE clauses, and record the + ** rowid (or PRIMARY KEY) for the current row so that the same + ** row will be skipped in subsequent sub-WHERE clauses. + */ + if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ + int r; + int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); + if( HasRowid(pTab) ){ + r = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, regRowid, 0); + jmp1 = sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, 0, + r,iSet); + VdbeCoverage(v); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + int nPk = pPk->nKeyCol; + int iPk; + + /* Read the PK into an array of temp registers. */ + r = sqlite3GetTempRange(pParse, nPk); + for(iPk=0; iPkaiColumn[iPk]; + int rx; + rx = sqlite3ExprCodeGetColumn(pParse, pTab, iCol, iCur,r+iPk,0); + if( rx!=r+iPk ){ + sqlite3VdbeAddOp2(v, OP_SCopy, rx, r+iPk); + } + } + + /* Check if the temp table already contains this key. If so, + ** the row has already been included in the result set and + ** can be ignored (by jumping past the Gosub below). Otherwise, + ** insert the key into the temp table and proceed with processing + ** the row. + ** + ** Use some of the same optimizations as OP_RowSetTest: If iSet + ** is zero, assume that the key cannot already be present in + ** the temp table. And if iSet is -1, assume that there is no + ** need to insert the key into the temp table, as it will never + ** be tested for. */ + if( iSet ){ + jmp1 = sqlite3VdbeAddOp4Int(v, OP_Found, regRowset, 0, r, nPk); + VdbeCoverage(v); + } + if( iSet>=0 ){ + sqlite3VdbeAddOp3(v, OP_MakeRecord, r, nPk, regRowid); + sqlite3VdbeAddOp3(v, OP_IdxInsert, regRowset, regRowid, 0); + if( iSet ) sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + } + + /* Release the array of temp registers */ + sqlite3ReleaseTempRange(pParse, r, nPk); + } + } + + /* Invoke the main loop body as a subroutine */ + sqlite3VdbeAddOp2(v, OP_Gosub, regReturn, iLoopBody); + + /* Jump here (skipping the main loop body subroutine) if the + ** current sub-WHERE row is a duplicate from prior sub-WHEREs. */ + if( jmp1 ) sqlite3VdbeJumpHere(v, jmp1); + + /* The pSubWInfo->untestedTerms flag means that this OR term + ** contained one or more AND term from a notReady table. The + ** terms from the notReady table could not be tested and will + ** need to be tested later. + */ + if( pSubWInfo->untestedTerms ) untestedTerms = 1; + + /* If all of the OR-connected terms are optimized using the same + ** index, and the index is opened using the same cursor number + ** by each call to sqlite3WhereBegin() made by this loop, it may + ** be possible to use that index as a covering index. + ** + ** If the call to sqlite3WhereBegin() above resulted in a scan that + ** uses an index, and this is either the first OR-connected term + ** processed or the index is the same as that used by all previous + ** terms, set pCov to the candidate covering index. Otherwise, set + ** pCov to NULL to indicate that no candidate covering index will + ** be available. + */ + pSubLoop = pSubWInfo->a[0].pWLoop; + assert( (pSubLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); + if( (pSubLoop->wsFlags & WHERE_INDEXED)!=0 + && (ii==0 || pSubLoop->u.btree.pIndex==pCov) + && (HasRowid(pTab) || !IsPrimaryKeyIndex(pSubLoop->u.btree.pIndex)) + ){ + assert( pSubWInfo->a[0].iIdxCur==iCovCur ); + pCov = pSubLoop->u.btree.pIndex; + wctrlFlags |= WHERE_REOPEN_IDX; + }else{ + pCov = 0; + } + + /* Finish the loop through table entries that match term pOrTerm. */ + sqlite3WhereEnd(pSubWInfo); + } + } + } + pLevel->u.pCovidx = pCov; + if( pCov ) pLevel->iIdxCur = iCovCur; + if( pAndExpr ){ + pAndExpr->pLeft = 0; + sqlite3ExprDelete(db, pAndExpr); + } + sqlite3VdbeChangeP1(v, iRetInit, sqlite3VdbeCurrentAddr(v)); + sqlite3VdbeGoto(v, pLevel->addrBrk); + sqlite3VdbeResolveLabel(v, iLoopBody); + + if( pWInfo->nLevel>1 ) sqlite3StackFree(db, pOrTab); + if( !untestedTerms ) disableTerm(pLevel, pTerm); + }else +#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ + + { + /* Case 6: There is no usable index. We must do a complete + ** scan of the entire table. + */ + static const u8 aStep[] = { OP_Next, OP_Prev }; + static const u8 aStart[] = { OP_Rewind, OP_Last }; + assert( bRev==0 || bRev==1 ); + if( pTabItem->fg.isRecursive ){ + /* Tables marked isRecursive have only a single row that is stored in + ** a pseudo-cursor. No need to Rewind or Next such cursors. */ + pLevel->op = OP_Noop; + }else{ + pLevel->op = aStep[bRev]; + pLevel->p1 = iCur; + pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); + pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; } } -#endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ -#ifndef SQLITE_OMIT_VIRTUALTABLE - /* Add a WO_MATCH auxiliary term to the constraint set if the - ** current expression is of the form: column MATCH expr. - ** This information is used by the xBestIndex methods of - ** virtual tables. The native query optimizer does not attempt - ** to do anything with MATCH functions. - */ - if( isMatchOfColumn(pExpr) ){ - int idxNew; - Expr *pRight, *pLeft; - WhereTerm *pNewTerm; - Bitmask prereqColumn, prereqExpr; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + pLevel->addrVisit = sqlite3VdbeCurrentAddr(v); +#endif - pRight = pExpr->x.pList->a[0].pExpr; - pLeft = pExpr->x.pList->a[1].pExpr; - prereqExpr = exprTableUsage(pMaskSet, pRight); - prereqColumn = exprTableUsage(pMaskSet, pLeft); - if( (prereqExpr & prereqColumn)==0 ){ - Expr *pNewExpr; - pNewExpr = sqlite3PExpr(pParse, TK_MATCH, - 0, sqlite3ExprDup(db, pRight, 0), 0); - idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); - testcase( idxNew==0 ); - pNewTerm = &pWC->a[idxNew]; - pNewTerm->prereqRight = prereqExpr; - pNewTerm->leftCursor = pLeft->iTable; - pNewTerm->u.leftColumn = pLeft->iColumn; - pNewTerm->eOperator = WO_MATCH; - markTermAsChild(pWC, idxNew, idxTerm); - pTerm = &pWC->a[idxTerm]; - pTerm->wtFlags |= TERM_COPIED; - pNewTerm->prereqAll = pTerm->prereqAll; + /* Insert code to test every subexpression that can be completely + ** computed using the current set of tables. + */ + for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ + Expr *pE; + int skipLikeAddr = 0; + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + testcase( pTerm->wtFlags & TERM_CODED ); + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ + testcase( pWInfo->untestedTerms==0 + && (pWInfo->wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ); + pWInfo->untestedTerms = 1; + continue; + } + pE = pTerm->pExpr; + assert( pE!=0 ); + if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){ + continue; + } + if( pTerm->wtFlags & TERM_LIKECOND ){ + assert( pLevel->iLikeRepCntr>0 ); + skipLikeAddr = sqlite3VdbeAddOp1(v, OP_IfNot, pLevel->iLikeRepCntr); + VdbeCoverage(v); } + sqlite3ExprIfFalse(pParse, pE, addrCont, SQLITE_JUMPIFNULL); + if( skipLikeAddr ) sqlite3VdbeJumpHere(v, skipLikeAddr); + pTerm->wtFlags |= TERM_CODED; } -#endif /* SQLITE_OMIT_VIRTUALTABLE */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - /* When sqlite_stat3 histogram data is available an operator of the - ** form "x IS NOT NULL" can sometimes be evaluated more efficiently - ** as "x>NULL" if x is not an INTEGER PRIMARY KEY. So construct a - ** virtual term of that form. + /* Insert code to test for implied constraints based on transitivity + ** of the "==" operator. ** - ** Note that the virtual term must be tagged with TERM_VNULL. This - ** TERM_VNULL tag will suppress the not-null check at the beginning - ** of the loop. Without the TERM_VNULL flag, the not-null check at - ** the start of the loop will prevent any results from being returned. + ** Example: If the WHERE clause contains "t1.a=t2.b" and "t2.b=123" + ** and we are coding the t1 loop and the t2 loop has not yet coded, + ** then we cannot use the "t1.a=t2.b" constraint, but we can code + ** the implied "t1.a=123" constraint. */ - if( pExpr->op==TK_NOTNULL - && pExpr->pLeft->op==TK_COLUMN - && pExpr->pLeft->iColumn>=0 - && OptimizationEnabled(db, SQLITE_Stat34) - ){ - Expr *pNewExpr; - Expr *pLeft = pExpr->pLeft; - int idxNew; - WhereTerm *pNewTerm; - - pNewExpr = sqlite3PExpr(pParse, TK_GT, - sqlite3ExprDup(db, pLeft, 0), - sqlite3PExpr(pParse, TK_NULL, 0, 0, 0), 0); - - idxNew = whereClauseInsert(pWC, pNewExpr, - TERM_VIRTUAL|TERM_DYNAMIC|TERM_VNULL); - if( idxNew ){ - pNewTerm = &pWC->a[idxNew]; - pNewTerm->prereqRight = 0; - pNewTerm->leftCursor = pLeft->iTable; - pNewTerm->u.leftColumn = pLeft->iColumn; - pNewTerm->eOperator = WO_GT; - markTermAsChild(pWC, idxNew, idxTerm); - pTerm = &pWC->a[idxTerm]; - pTerm->wtFlags |= TERM_COPIED; - pNewTerm->prereqAll = pTerm->prereqAll; + for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ + Expr *pE, *pEAlt; + WhereTerm *pAlt; + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) continue; + if( (pTerm->eOperator & WO_EQUIV)==0 ) continue; + if( pTerm->leftCursor!=iCur ) continue; + if( pLevel->iLeftJoin ) continue; + pE = pTerm->pExpr; + assert( !ExprHasProperty(pE, EP_FromJoin) ); + assert( (pTerm->prereqRight & pLevel->notReady)!=0 ); + pAlt = sqlite3WhereFindTerm(pWC, iCur, pTerm->u.leftColumn, notReady, + WO_EQ|WO_IN|WO_IS, 0); + if( pAlt==0 ) continue; + if( pAlt->wtFlags & (TERM_CODED) ) continue; + testcase( pAlt->eOperator & WO_EQ ); + testcase( pAlt->eOperator & WO_IS ); + testcase( pAlt->eOperator & WO_IN ); + VdbeModuleComment((v, "begin transitive constraint")); + pEAlt = sqlite3StackAllocRaw(db, sizeof(*pEAlt)); + if( pEAlt ){ + *pEAlt = *pAlt->pExpr; + pEAlt->pLeft = pE->pLeft; + sqlite3ExprIfFalse(pParse, pEAlt, addrCont, SQLITE_JUMPIFNULL); + sqlite3StackFree(db, pEAlt); } } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ - /* Prevent ON clause terms of a LEFT JOIN from being used to drive - ** an index for tables to the left of the join. + /* For a LEFT OUTER JOIN, generate code that will record the fact that + ** at least one row of the right table has matched the left table. */ - pTerm->prereqRight |= extraRight; -} - -/* -** This function searches pList for an entry that matches the iCol-th column -** of index pIdx. -** -** If such an expression is found, its index in pList->a[] is returned. If -** no expression is found, -1 is returned. -*/ -static int findIndexCol( - Parse *pParse, /* Parse context */ - ExprList *pList, /* Expression list to search */ - int iBase, /* Cursor for table associated with pIdx */ - Index *pIdx, /* Index to match column of */ - int iCol /* Column of index to match */ -){ - int i; - const char *zColl = pIdx->azColl[iCol]; - - for(i=0; inExpr; i++){ - Expr *p = sqlite3ExprSkipCollate(pList->a[i].pExpr); - if( p->op==TK_COLUMN - && p->iColumn==pIdx->aiColumn[iCol] - && p->iTable==iBase - ){ - CollSeq *pColl = sqlite3ExprCollSeq(pParse, pList->a[i].pExpr); - if( pColl && 0==sqlite3StrICmp(pColl->zName, zColl) ){ - return i; + if( pLevel->iLeftJoin ){ + pLevel->addrFirst = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_Integer, 1, pLevel->iLeftJoin); + VdbeComment((v, "record LEFT JOIN hit")); + sqlite3ExprCacheClear(pParse); + for(pTerm=pWC->a, j=0; jnTerm; j++, pTerm++){ + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + testcase( pTerm->wtFlags & TERM_CODED ); + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ + assert( pWInfo->untestedTerms ); + continue; } + assert( pTerm->pExpr ); + sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); + pTerm->wtFlags |= TERM_CODED; } } - return -1; + return pLevel->notReady; } +/************** End of wherecode.c *******************************************/ +/************** Begin file whereexpr.c ***************************************/ /* -** Return true if the DISTINCT expression-list passed as the third argument -** is redundant. +** 2015-06-08 ** -** A DISTINCT list is redundant if the database contains some subset of -** columns that are unique and non-null. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This module contains C code that generates VDBE code used to process +** the WHERE clause of SQL statements. +** +** This file was originally part of where.c but was split out to improve +** readability and editabiliity. This file contains utility routines for +** analyzing Expr objects in the WHERE clause. */ -static int isDistinctRedundant( - Parse *pParse, /* Parsing context */ - SrcList *pTabList, /* The FROM clause */ - WhereClause *pWC, /* The WHERE clause */ - ExprList *pDistinct /* The result set that needs to be DISTINCT */ -){ - Table *pTab; - Index *pIdx; - int i; - int iBase; +/* #include "sqliteInt.h" */ +/* #include "whereInt.h" */ - /* If there is more than one table or sub-select in the FROM clause of - ** this query, then it will not be possible to show that the DISTINCT - ** clause is redundant. */ - if( pTabList->nSrc!=1 ) return 0; - iBase = pTabList->a[0].iCursor; - pTab = pTabList->a[0].pTab; +/* Forward declarations */ +static void exprAnalyze(SrcList*, WhereClause*, int); - /* If any of the expressions is an IPK column on table iBase, then return - ** true. Note: The (p->iTable==iBase) part of this test may be false if the - ** current SELECT is a correlated sub-query. - */ - for(i=0; inExpr; i++){ - Expr *p = sqlite3ExprSkipCollate(pDistinct->a[i].pExpr); - if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1; - } +/* +** Deallocate all memory associated with a WhereOrInfo object. +*/ +static void whereOrInfoDelete(sqlite3 *db, WhereOrInfo *p){ + sqlite3WhereClauseClear(&p->wc); + sqlite3DbFree(db, p); +} - /* Loop through all indices on the table, checking each to see if it makes - ** the DISTINCT qualifier redundant. It does so if: - ** - ** 1. The index is itself UNIQUE, and - ** - ** 2. All of the columns in the index are either part of the pDistinct - ** list, or else the WHERE clause contains a term of the form "col=X", - ** where X is a constant value. The collation sequences of the - ** comparison and select-list expressions must match those of the index. - ** - ** 3. All of those index columns for which the WHERE clause does not - ** contain a "col=X" term are subject to a NOT NULL constraint. - */ - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( !IsUniqueIndex(pIdx) ) continue; - for(i=0; inKeyCol; i++){ - i16 iCol = pIdx->aiColumn[i]; - if( 0==findTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx) ){ - int iIdxCol = findIndexCol(pParse, pDistinct, iBase, pIdx, i); - if( iIdxCol<0 || pTab->aCol[iCol].notNull==0 ){ - break; - } +/* +** Deallocate all memory associated with a WhereAndInfo object. +*/ +static void whereAndInfoDelete(sqlite3 *db, WhereAndInfo *p){ + sqlite3WhereClauseClear(&p->wc); + sqlite3DbFree(db, p); +} + +/* +** Add a single new WhereTerm entry to the WhereClause object pWC. +** The new WhereTerm object is constructed from Expr p and with wtFlags. +** The index in pWC->a[] of the new WhereTerm is returned on success. +** 0 is returned if the new WhereTerm could not be added due to a memory +** allocation error. The memory allocation failure will be recorded in +** the db->mallocFailed flag so that higher-level functions can detect it. +** +** This routine will increase the size of the pWC->a[] array as necessary. +** +** If the wtFlags argument includes TERM_DYNAMIC, then responsibility +** for freeing the expression p is assumed by the WhereClause object pWC. +** This is true even if this routine fails to allocate a new WhereTerm. +** +** WARNING: This routine might reallocate the space used to store +** WhereTerms. All pointers to WhereTerms should be invalidated after +** calling this routine. Such pointers may be reinitialized by referencing +** the pWC->a[] array. +*/ +static int whereClauseInsert(WhereClause *pWC, Expr *p, u16 wtFlags){ + WhereTerm *pTerm; + int idx; + testcase( wtFlags & TERM_VIRTUAL ); + if( pWC->nTerm>=pWC->nSlot ){ + WhereTerm *pOld = pWC->a; + sqlite3 *db = pWC->pWInfo->pParse->db; + pWC->a = sqlite3DbMallocRaw(db, sizeof(pWC->a[0])*pWC->nSlot*2 ); + if( pWC->a==0 ){ + if( wtFlags & TERM_DYNAMIC ){ + sqlite3ExprDelete(db, p); } + pWC->a = pOld; + return 0; } - if( i==pIdx->nKeyCol ){ - /* This index implies that the DISTINCT qualifier is redundant. */ - return 1; + memcpy(pWC->a, pOld, sizeof(pWC->a[0])*pWC->nTerm); + if( pOld!=pWC->aStatic ){ + sqlite3DbFree(db, pOld); } + pWC->nSlot = sqlite3DbMallocSize(db, pWC->a)/sizeof(pWC->a[0]); + memset(&pWC->a[pWC->nTerm], 0, sizeof(pWC->a[0])*(pWC->nSlot-pWC->nTerm)); } - - return 0; + pTerm = &pWC->a[idx = pWC->nTerm++]; + if( p && ExprHasProperty(p, EP_Unlikely) ){ + pTerm->truthProb = sqlite3LogEst(p->iTable) - 270; + }else{ + pTerm->truthProb = 1; + } + pTerm->pExpr = sqlite3ExprSkipCollate(p); + pTerm->wtFlags = wtFlags; + pTerm->pWC = pWC; + pTerm->iParent = -1; + return idx; } - /* -** Estimate the logarithm of the input value to base 2. +** Return TRUE if the given operator is one of the operators that is +** allowed for an indexable WHERE clause term. The allowed operators are +** "=", "<", ">", "<=", ">=", "IN", and "IS NULL" */ -static LogEst estLog(LogEst N){ - return N<=10 ? 0 : sqlite3LogEst(N) - 33; +static int allowedOp(int op){ + assert( TK_GT>TK_EQ && TK_GTTK_EQ && TK_LTTK_EQ && TK_LE=TK_EQ && op<=TK_GE) || op==TK_ISNULL || op==TK_IS; } /* -** Two routines for printing the content of an sqlite3_index_info -** structure. Used for testing and debugging only. If neither -** SQLITE_TEST or SQLITE_DEBUG are defined, then these routines -** are no-ops. +** Commute a comparison operator. Expressions of the form "X op Y" +** are converted into "Y op X". +** +** If left/right precedence rules come into play when determining the +** collating sequence, then COLLATE operators are adjusted to ensure +** that the collating sequence does not change. For example: +** "Y collate NOCASE op X" becomes "X op Y" because any collation sequence on +** the left hand side of a comparison overrides any collation sequence +** attached to the right. For the same reason the EP_Collate flag +** is not commuted. */ -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED) -static void TRACE_IDX_INPUTS(sqlite3_index_info *p){ - int i; - if( !sqlite3WhereTrace ) return; - for(i=0; inConstraint; i++){ - sqlite3DebugPrintf(" constraint[%d]: col=%d termid=%d op=%d usabled=%d\n", - i, - p->aConstraint[i].iColumn, - p->aConstraint[i].iTermOffset, - p->aConstraint[i].op, - p->aConstraint[i].usable); - } - for(i=0; inOrderBy; i++){ - sqlite3DebugPrintf(" orderby[%d]: col=%d desc=%d\n", - i, - p->aOrderBy[i].iColumn, - p->aOrderBy[i].desc); +static void exprCommute(Parse *pParse, Expr *pExpr){ + u16 expRight = (pExpr->pRight->flags & EP_Collate); + u16 expLeft = (pExpr->pLeft->flags & EP_Collate); + assert( allowedOp(pExpr->op) && pExpr->op!=TK_IN ); + if( expRight==expLeft ){ + /* Either X and Y both have COLLATE operator or neither do */ + if( expRight ){ + /* Both X and Y have COLLATE operators. Make sure X is always + ** used by clearing the EP_Collate flag from Y. */ + pExpr->pRight->flags &= ~EP_Collate; + }else if( sqlite3ExprCollSeq(pParse, pExpr->pLeft)!=0 ){ + /* Neither X nor Y have COLLATE operators, but X has a non-default + ** collating sequence. So add the EP_Collate marker on X to cause + ** it to be searched first. */ + pExpr->pLeft->flags |= EP_Collate; + } } -} -static void TRACE_IDX_OUTPUTS(sqlite3_index_info *p){ - int i; - if( !sqlite3WhereTrace ) return; - for(i=0; inConstraint; i++){ - sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n", - i, - p->aConstraintUsage[i].argvIndex, - p->aConstraintUsage[i].omit); + SWAP(Expr*,pExpr->pRight,pExpr->pLeft); + if( pExpr->op>=TK_GT ){ + assert( TK_LT==TK_GT+2 ); + assert( TK_GE==TK_LE+2 ); + assert( TK_GT>TK_EQ ); + assert( TK_GTop>=TK_GT && pExpr->op<=TK_GE ); + pExpr->op = ((pExpr->op-TK_GT)^2)+TK_GT; } - sqlite3DebugPrintf(" idxNum=%d\n", p->idxNum); - sqlite3DebugPrintf(" idxStr=%s\n", p->idxStr); - sqlite3DebugPrintf(" orderByConsumed=%d\n", p->orderByConsumed); - sqlite3DebugPrintf(" estimatedCost=%g\n", p->estimatedCost); - sqlite3DebugPrintf(" estimatedRows=%lld\n", p->estimatedRows); } -#else -#define TRACE_IDX_INPUTS(A) -#define TRACE_IDX_OUTPUTS(A) -#endif -#ifndef SQLITE_OMIT_AUTOMATIC_INDEX /* -** Return TRUE if the WHERE clause term pTerm is of a form where it -** could be used with an index to access pSrc, assuming an appropriate -** index existed. +** Translate from TK_xx operator to WO_xx bitmask. */ -static int termCanDriveIndex( - WhereTerm *pTerm, /* WHERE clause term to check */ - struct SrcList_item *pSrc, /* Table we are trying to access */ - Bitmask notReady /* Tables in outer loops of the join */ -){ - char aff; - if( pTerm->leftCursor!=pSrc->iCursor ) return 0; - if( (pTerm->eOperator & WO_EQ)==0 ) return 0; - if( (pTerm->prereqRight & notReady)!=0 ) return 0; - if( pTerm->u.leftColumn<0 ) return 0; - aff = pSrc->pTab->aCol[pTerm->u.leftColumn].affinity; - if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0; - return 1; +static u16 operatorMask(int op){ + u16 c; + assert( allowedOp(op) ); + if( op==TK_IN ){ + c = WO_IN; + }else if( op==TK_ISNULL ){ + c = WO_ISNULL; + }else if( op==TK_IS ){ + c = WO_IS; + }else{ + assert( (WO_EQ<<(op-TK_EQ)) < 0x7fff ); + c = (u16)(WO_EQ<<(op-TK_EQ)); + } + assert( op!=TK_ISNULL || c==WO_ISNULL ); + assert( op!=TK_IN || c==WO_IN ); + assert( op!=TK_EQ || c==WO_EQ ); + assert( op!=TK_LT || c==WO_LT ); + assert( op!=TK_LE || c==WO_LE ); + assert( op!=TK_GT || c==WO_GT ); + assert( op!=TK_GE || c==WO_GE ); + assert( op!=TK_IS || c==WO_IS ); + return c; } -#endif -#ifndef SQLITE_OMIT_AUTOMATIC_INDEX +#ifndef SQLITE_OMIT_LIKE_OPTIMIZATION /* -** Generate code to construct the Index object for an automatic index -** and to set up the WhereLevel object pLevel so that the code generator -** makes use of the automatic index. +** Check to see if the given expression is a LIKE or GLOB operator that +** can be optimized using inequality constraints. Return TRUE if it is +** so and false if not. +** +** In order for the operator to be optimizible, the RHS must be a string +** literal that does not begin with a wildcard. The LHS must be a column +** that may only be NULL, a string, or a BLOB, never a number. (This means +** that virtual tables cannot participate in the LIKE optimization.) The +** collating sequence for the column on the LHS must be appropriate for +** the operator. */ -static void constructAutomaticIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to get the next index */ - Bitmask notReady, /* Mask of cursors that are not available */ - WhereLevel *pLevel /* Write new index here */ +static int isLikeOrGlob( + Parse *pParse, /* Parsing and code generating context */ + Expr *pExpr, /* Test this expression */ + Expr **ppPrefix, /* Pointer to TK_STRING expression with pattern prefix */ + int *pisComplete, /* True if the only wildcard is % in the last character */ + int *pnoCase /* True if uppercase is equivalent to lowercase */ ){ - int nKeyCol; /* Number of columns in the constructed index */ - WhereTerm *pTerm; /* A single term of the WHERE clause */ - WhereTerm *pWCEnd; /* End of pWC->a[] */ - Index *pIdx; /* Object describing the transient index */ - Vdbe *v; /* Prepared statement under construction */ - int addrInit; /* Address of the initialization bypass jump */ - Table *pTable; /* The table being indexed */ - int addrTop; /* Top of the index fill loop */ - int regRecord; /* Register holding an index record */ - int n; /* Column counter */ - int i; /* Loop counter */ - int mxBitCol; /* Maximum column in pSrc->colUsed */ - CollSeq *pColl; /* Collating sequence to on a column */ - WhereLoop *pLoop; /* The Loop object */ - char *zNotUsed; /* Extra space on the end of pIdx */ - Bitmask idxCols; /* Bitmap of columns used for indexing */ - Bitmask extraCols; /* Bitmap of additional columns */ - u8 sentWarning = 0; /* True if a warnning has been issued */ - Expr *pPartial = 0; /* Partial Index Expression */ - int iContinue = 0; /* Jump here to skip excluded rows */ + const char *z = 0; /* String on RHS of LIKE operator */ + Expr *pRight, *pLeft; /* Right and left size of LIKE operator */ + ExprList *pList; /* List of operands to the LIKE operator */ + int c; /* One character in z[] */ + int cnt; /* Number of non-wildcard prefix characters */ + char wc[3]; /* Wildcard characters */ + sqlite3 *db = pParse->db; /* Database connection */ + sqlite3_value *pVal = 0; + int op; /* Opcode of pRight */ - /* Generate code to skip over the creation and initialization of the - ** transient index on 2nd and subsequent iterations of the loop. */ - v = pParse->pVdbe; - assert( v!=0 ); - addrInit = sqlite3CodeOnce(pParse); VdbeCoverage(v); + if( !sqlite3IsLikeFunction(db, pExpr, pnoCase, wc) ){ + return 0; + } +#ifdef SQLITE_EBCDIC + if( *pnoCase ) return 0; +#endif + pList = pExpr->x.pList; + pLeft = pList->a[1].pExpr; + if( pLeft->op!=TK_COLUMN + || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT + || IsVirtual(pLeft->pTab) /* Value might be numeric */ + ){ + /* IMP: R-02065-49465 The left-hand side of the LIKE or GLOB operator must + ** be the name of an indexed column with TEXT affinity. */ + return 0; + } + assert( pLeft->iColumn!=(-1) ); /* Because IPK never has AFF_TEXT */ - /* Count the number of columns that will be added to the index - ** and used to match WHERE clause constraints */ - nKeyCol = 0; - pTable = pSrc->pTab; - pWCEnd = &pWC->a[pWC->nTerm]; - pLoop = pLevel->pWLoop; - idxCols = 0; - for(pTerm=pWC->a; pTermpExpr; - assert( !ExprHasProperty(pExpr, EP_FromJoin) /* prereq always non-zero */ - || pExpr->iRightJoinTable!=pSrc->iCursor /* for the right-hand */ - || pLoop->prereq!=0 ); /* table of a LEFT JOIN */ - if( pLoop->prereq==0 - && (pTerm->wtFlags & TERM_VIRTUAL)==0 - && !ExprHasProperty(pExpr, EP_FromJoin) - && sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor) ){ - pPartial = sqlite3ExprAnd(pParse->db, pPartial, - sqlite3ExprDup(pParse->db, pExpr, 0)); + pRight = sqlite3ExprSkipCollate(pList->a[0].pExpr); + op = pRight->op; + if( op==TK_VARIABLE ){ + Vdbe *pReprepare = pParse->pReprepare; + int iCol = pRight->iColumn; + pVal = sqlite3VdbeGetBoundValue(pReprepare, iCol, SQLITE_AFF_BLOB); + if( pVal && sqlite3_value_type(pVal)==SQLITE_TEXT ){ + z = (char *)sqlite3_value_text(pVal); } - if( termCanDriveIndex(pTerm, pSrc, notReady) ){ - int iCol = pTerm->u.leftColumn; - Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); - testcase( iCol==BMS ); - testcase( iCol==BMS-1 ); - if( !sentWarning ){ - sqlite3_log(SQLITE_WARNING_AUTOINDEX, - "automatic index on %s(%s)", pTable->zName, - pTable->aCol[iCol].zName); - sentWarning = 1; - } - if( (idxCols & cMask)==0 ){ - if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ){ - goto end_auto_index_create; + sqlite3VdbeSetVarmask(pParse->pVdbe, iCol); + assert( pRight->op==TK_VARIABLE || pRight->op==TK_REGISTER ); + }else if( op==TK_STRING ){ + z = pRight->u.zToken; + } + if( z ){ + cnt = 0; + while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ + cnt++; + } + if( cnt!=0 && 255!=(u8)z[cnt-1] ){ + Expr *pPrefix; + *pisComplete = c==wc[0] && z[cnt+1]==0; + pPrefix = sqlite3Expr(db, TK_STRING, z); + if( pPrefix ) pPrefix->u.zToken[cnt] = 0; + *ppPrefix = pPrefix; + if( op==TK_VARIABLE ){ + Vdbe *v = pParse->pVdbe; + sqlite3VdbeSetVarmask(v, pRight->iColumn); + if( *pisComplete && pRight->u.zToken[1] ){ + /* If the rhs of the LIKE expression is a variable, and the current + ** value of the variable means there is no need to invoke the LIKE + ** function, then no OP_Variable will be added to the program. + ** This causes problems for the sqlite3_bind_parameter_name() + ** API. To work around them, add a dummy OP_Variable here. + */ + int r1 = sqlite3GetTempReg(pParse); + sqlite3ExprCodeTarget(pParse, pRight, r1); + sqlite3VdbeChangeP3(v, sqlite3VdbeCurrentAddr(v)-1, 0); + sqlite3ReleaseTempReg(pParse, r1); } - pLoop->aLTerm[nKeyCol++] = pTerm; - idxCols |= cMask; } + }else{ + z = 0; } } - assert( nKeyCol>0 ); - pLoop->u.btree.nEq = pLoop->nLTerm = nKeyCol; - pLoop->wsFlags = WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WHERE_INDEXED - | WHERE_AUTO_INDEX; - /* Count the number of additional columns needed to create a - ** covering index. A "covering index" is an index that contains all - ** columns that are needed by the query. With a covering index, the - ** original table never needs to be accessed. Automatic indices must - ** be a covering index because the index will not be updated if the - ** original table changes and the index and table cannot both be used - ** if they go out of sync. - */ - extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); - mxBitCol = MIN(BMS-1,pTable->nCol); - testcase( pTable->nCol==BMS-1 ); - testcase( pTable->nCol==BMS-2 ); - for(i=0; iop!=TK_FUNCTION ){ + return 0; } - if( pSrc->colUsed & MASKBIT(BMS-1) ){ - nKeyCol += pTable->nCol - BMS + 1; + if( sqlite3StrICmp(pExpr->u.zToken,"match")!=0 ){ + return 0; } - - /* Construct the Index object to describe this index */ - pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed); - if( pIdx==0 ) goto end_auto_index_create; - pLoop->u.btree.pIndex = pIdx; - pIdx->zName = "auto-index"; - pIdx->pTable = pTable; - n = 0; - idxCols = 0; - for(pTerm=pWC->a; pTermu.leftColumn; - Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); - testcase( iCol==BMS-1 ); - testcase( iCol==BMS ); - if( (idxCols & cMask)==0 ){ - Expr *pX = pTerm->pExpr; - idxCols |= cMask; - pIdx->aiColumn[n] = pTerm->u.leftColumn; - pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); - pIdx->azColl[n] = pColl ? pColl->zName : "BINARY"; - n++; - } - } + pList = pExpr->x.pList; + if( pList->nExpr!=2 ){ + return 0; } - assert( (u32)n==pLoop->u.btree.nEq ); - - /* Add additional columns needed to make the automatic index into - ** a covering index */ - for(i=0; iaiColumn[n] = i; - pIdx->azColl[n] = "BINARY"; - n++; - } - } - if( pSrc->colUsed & MASKBIT(BMS-1) ){ - for(i=BMS-1; inCol; i++){ - pIdx->aiColumn[n] = i; - pIdx->azColl[n] = "BINARY"; - n++; - } - } - assert( n==nKeyCol ); - pIdx->aiColumn[n] = -1; - pIdx->azColl[n] = "BINARY"; - - /* Create the automatic index */ - assert( pLevel->iIdxCur>=0 ); - pLevel->iIdxCur = pParse->nTab++; - sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1); - sqlite3VdbeSetP4KeyInfo(pParse, pIdx); - VdbeComment((v, "for %s", pTable->zName)); - - /* Fill the automatic index with content */ - sqlite3ExprCachePush(pParse); - addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); - if( pPartial ){ - iContinue = sqlite3VdbeMakeLabel(v); - sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL); - pLoop->wsFlags |= WHERE_PARTIALIDX; + if( pList->a[1].pExpr->op != TK_COLUMN ){ + return 0; } - regRecord = sqlite3GetTempReg(pParse); - sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0); - sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); - sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); - if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); - sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); - sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); - sqlite3VdbeJumpHere(v, addrTop); - sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3ExprCachePop(pParse); - - /* Jump here when skipping the initialization */ - sqlite3VdbeJumpHere(v, addrInit); - -end_auto_index_create: - sqlite3ExprDelete(pParse->db, pPartial); + return 1; } -#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ +#endif /* SQLITE_OMIT_VIRTUALTABLE */ -#ifndef SQLITE_OMIT_VIRTUALTABLE /* -** Allocate and populate an sqlite3_index_info structure. It is the -** responsibility of the caller to eventually release the structure -** by passing the pointer returned by this function to sqlite3_free(). +** If the pBase expression originated in the ON or USING clause of +** a join, then transfer the appropriate markings over to derived. */ -static sqlite3_index_info *allocateIndexInfo( - Parse *pParse, - WhereClause *pWC, - struct SrcList_item *pSrc, - ExprList *pOrderBy -){ - int i, j; - int nTerm; - struct sqlite3_index_constraint *pIdxCons; - struct sqlite3_index_orderby *pIdxOrderBy; - struct sqlite3_index_constraint_usage *pUsage; - WhereTerm *pTerm; - int nOrderBy; - sqlite3_index_info *pIdxInfo; - - /* Count the number of possible WHERE clause constraints referring - ** to this virtual table */ - for(i=nTerm=0, pTerm=pWC->a; inTerm; i++, pTerm++){ - if( pTerm->leftCursor != pSrc->iCursor ) continue; - assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); - testcase( pTerm->eOperator & WO_IN ); - testcase( pTerm->eOperator & WO_ISNULL ); - testcase( pTerm->eOperator & WO_ALL ); - if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV))==0 ) continue; - if( pTerm->wtFlags & TERM_VNULL ) continue; - nTerm++; - } - - /* If the ORDER BY clause contains only columns in the current - ** virtual table then allocate space for the aOrderBy part of - ** the sqlite3_index_info structure. - */ - nOrderBy = 0; - if( pOrderBy ){ - int n = pOrderBy->nExpr; - for(i=0; ia[i].pExpr; - if( pExpr->op!=TK_COLUMN || pExpr->iTable!=pSrc->iCursor ) break; - } - if( i==n){ - nOrderBy = n; - } - } - - /* Allocate the sqlite3_index_info structure - */ - pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo) - + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm - + sizeof(*pIdxOrderBy)*nOrderBy ); - if( pIdxInfo==0 ){ - sqlite3ErrorMsg(pParse, "out of memory"); - return 0; +static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ + if( pDerived ){ + pDerived->flags |= pBase->flags & EP_FromJoin; + pDerived->iRightJoinTable = pBase->iRightJoinTable; } +} - /* Initialize the structure. The sqlite3_index_info structure contains - ** many fields that are declared "const" to prevent xBestIndex from - ** changing them. We have to do some funky casting in order to - ** initialize those fields. - */ - pIdxCons = (struct sqlite3_index_constraint*)&pIdxInfo[1]; - pIdxOrderBy = (struct sqlite3_index_orderby*)&pIdxCons[nTerm]; - pUsage = (struct sqlite3_index_constraint_usage*)&pIdxOrderBy[nOrderBy]; - *(int*)&pIdxInfo->nConstraint = nTerm; - *(int*)&pIdxInfo->nOrderBy = nOrderBy; - *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint = pIdxCons; - *(struct sqlite3_index_orderby**)&pIdxInfo->aOrderBy = pIdxOrderBy; - *(struct sqlite3_index_constraint_usage**)&pIdxInfo->aConstraintUsage = - pUsage; +/* +** Mark term iChild as being a child of term iParent +*/ +static void markTermAsChild(WhereClause *pWC, int iChild, int iParent){ + pWC->a[iChild].iParent = iParent; + pWC->a[iChild].truthProb = pWC->a[iParent].truthProb; + pWC->a[iParent].nChild++; +} - for(i=j=0, pTerm=pWC->a; inTerm; i++, pTerm++){ - u8 op; - if( pTerm->leftCursor != pSrc->iCursor ) continue; - assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); - testcase( pTerm->eOperator & WO_IN ); - testcase( pTerm->eOperator & WO_ISNULL ); - testcase( pTerm->eOperator & WO_ALL ); - if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV))==0 ) continue; - if( pTerm->wtFlags & TERM_VNULL ) continue; - pIdxCons[j].iColumn = pTerm->u.leftColumn; - pIdxCons[j].iTermOffset = i; - op = (u8)pTerm->eOperator & WO_ALL; - if( op==WO_IN ) op = WO_EQ; - pIdxCons[j].op = op; - /* The direct assignment in the previous line is possible only because - ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The - ** following asserts verify this fact. */ - assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ ); - assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT ); - assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE ); - assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT ); - assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE ); - assert( WO_MATCH==SQLITE_INDEX_CONSTRAINT_MATCH ); - assert( pTerm->eOperator & (WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) ); - j++; +/* +** Return the N-th AND-connected subterm of pTerm. Or if pTerm is not +** a conjunction, then return just pTerm when N==0. If N is exceeds +** the number of available subterms, return NULL. +*/ +static WhereTerm *whereNthSubterm(WhereTerm *pTerm, int N){ + if( pTerm->eOperator!=WO_AND ){ + return N==0 ? pTerm : 0; } - for(i=0; ia[i].pExpr; - pIdxOrderBy[i].iColumn = pExpr->iColumn; - pIdxOrderBy[i].desc = pOrderBy->a[i].sortOrder; + if( Nu.pAndInfo->wc.nTerm ){ + return &pTerm->u.pAndInfo->wc.a[N]; } - - return pIdxInfo; + return 0; } /* -** The table object reference passed as the second argument to this function -** must represent a virtual table. This function invokes the xBestIndex() -** method of the virtual table with the sqlite3_index_info object that -** comes in as the 3rd argument to this function. +** Subterms pOne and pTwo are contained within WHERE clause pWC. The +** two subterms are in disjunction - they are OR-ed together. ** -** If an error occurs, pParse is populated with an error message and a -** non-zero value is returned. Otherwise, 0 is returned and the output -** part of the sqlite3_index_info structure is left populated. +** If these two terms are both of the form: "A op B" with the same +** A and B values but different operators and if the operators are +** compatible (if one is = and the other is <, for example) then +** add a new virtual AND term to pWC that is the combination of the +** two. ** -** Whether or not an error is returned, it is the responsibility of the -** caller to eventually free p->idxStr if p->needToFreeIdxStr indicates -** that this is required. +** Some examples: +** +** x x<=y +** x=y OR x=y --> x=y +** x<=y OR x x<=y +** +** The following is NOT generated: +** +** xy --> x!=y */ -static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ - sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab; - int i; - int rc; - - TRACE_IDX_INPUTS(p); - rc = pVtab->pModule->xBestIndex(pVtab, p); - TRACE_IDX_OUTPUTS(p); +static void whereCombineDisjuncts( + SrcList *pSrc, /* the FROM clause */ + WhereClause *pWC, /* The complete WHERE clause */ + WhereTerm *pOne, /* First disjunct */ + WhereTerm *pTwo /* Second disjunct */ +){ + u16 eOp = pOne->eOperator | pTwo->eOperator; + sqlite3 *db; /* Database connection (for malloc) */ + Expr *pNew; /* New virtual expression */ + int op; /* Operator for the combined expression */ + int idxNew; /* Index in pWC of the next virtual term */ - if( rc!=SQLITE_OK ){ - if( rc==SQLITE_NOMEM ){ - pParse->db->mallocFailed = 1; - }else if( !pVtab->zErrMsg ){ - sqlite3ErrorMsg(pParse, "%s", sqlite3ErrStr(rc)); + if( (pOne->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE))==0 ) return; + if( (pTwo->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE))==0 ) return; + if( (eOp & (WO_EQ|WO_LT|WO_LE))!=eOp + && (eOp & (WO_EQ|WO_GT|WO_GE))!=eOp ) return; + assert( pOne->pExpr->pLeft!=0 && pOne->pExpr->pRight!=0 ); + assert( pTwo->pExpr->pLeft!=0 && pTwo->pExpr->pRight!=0 ); + if( sqlite3ExprCompare(pOne->pExpr->pLeft, pTwo->pExpr->pLeft, -1) ) return; + if( sqlite3ExprCompare(pOne->pExpr->pRight, pTwo->pExpr->pRight, -1) )return; + /* If we reach this point, it means the two subterms can be combined */ + if( (eOp & (eOp-1))!=0 ){ + if( eOp & (WO_LT|WO_LE) ){ + eOp = WO_LE; }else{ - sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg); - } - } - sqlite3_free(pVtab->zErrMsg); - pVtab->zErrMsg = 0; - - for(i=0; inConstraint; i++){ - if( !p->aConstraint[i].usable && p->aConstraintUsage[i].argvIndex>0 ){ - sqlite3ErrorMsg(pParse, - "table %s: xBestIndex returned an invalid plan", pTab->zName); + assert( eOp & (WO_GT|WO_GE) ); + eOp = WO_GE; } } - - return pParse->nErr; + db = pWC->pWInfo->pParse->db; + pNew = sqlite3ExprDup(db, pOne->pExpr, 0); + if( pNew==0 ) return; + for(op=TK_EQ; eOp!=(WO_EQ<<(op-TK_EQ)); op++){ assert( opop = op; + idxNew = whereClauseInsert(pWC, pNew, TERM_VIRTUAL|TERM_DYNAMIC); + exprAnalyze(pSrc, pWC, idxNew); } -#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) /* -** Estimate the location of a particular key among all keys in an -** index. Store the results in aStat as follows: +** Analyze a term that consists of two or more OR-connected +** subterms. So in: ** -** aStat[0] Est. number of rows less than pRec -** aStat[1] Est. number of rows equal to pRec +** ... WHERE (a=5) AND (b=7 OR c=9 OR d=13) AND (d=13) +** ^^^^^^^^^^^^^^^^^^^^ ** -** Return the index of the sample that is the smallest sample that -** is greater than or equal to pRec. Note that this index is not an index -** into the aSample[] array - it is an index into a virtual set of samples -** based on the contents of aSample[] and the number of fields in record -** pRec. +** This routine analyzes terms such as the middle term in the above example. +** A WhereOrTerm object is computed and attached to the term under +** analysis, regardless of the outcome of the analysis. Hence: +** +** WhereTerm.wtFlags |= TERM_ORINFO +** WhereTerm.u.pOrInfo = a dynamically allocated WhereOrTerm object +** +** The term being analyzed must have two or more of OR-connected subterms. +** A single subterm might be a set of AND-connected sub-subterms. +** Examples of terms under analysis: +** +** (A) t1.x=t2.y OR t1.x=t2.z OR t1.y=15 OR t1.z=t3.a+5 +** (B) x=expr1 OR expr2=x OR x=expr3 +** (C) t1.x=t2.y OR (t1.x=t2.z AND t1.y=15) +** (D) x=expr1 OR (y>11 AND y<22 AND z LIKE '*hello*') +** (E) (p.a=1 AND q.b=2 AND r.c=3) OR (p.x=4 AND q.y=5 AND r.z=6) +** (F) x>A OR (x=A AND y>=B) +** +** CASE 1: +** +** If all subterms are of the form T.C=expr for some single column of C and +** a single table T (as shown in example B above) then create a new virtual +** term that is an equivalent IN expression. In other words, if the term +** being analyzed is: +** +** x = expr1 OR expr2 = x OR x = expr3 +** +** then create a new virtual term like this: +** +** x IN (expr1,expr2,expr3) +** +** CASE 2: +** +** If there are exactly two disjuncts and one side has x>A and the other side +** has x=A (for the same x and A) then add a new virtual conjunct term to the +** WHERE clause of the form "x>=A". Example: +** +** x>A OR (x=A AND y>B) adds: x>=A +** +** The added conjunct can sometimes be helpful in query planning. +** +** CASE 3: +** +** If all subterms are indexable by a single table T, then set +** +** WhereTerm.eOperator = WO_OR +** WhereTerm.u.pOrInfo->indexable |= the cursor number for table T +** +** A subterm is "indexable" if it is of the form +** "T.C " where C is any column of table T and +** is one of "=", "<", "<=", ">", ">=", "IS NULL", or "IN". +** A subterm is also indexable if it is an AND of two or more +** subsubterms at least one of which is indexable. Indexable AND +** subterms have their eOperator set to WO_AND and they have +** u.pAndInfo set to a dynamically allocated WhereAndTerm object. +** +** From another point of view, "indexable" means that the subterm could +** potentially be used with an index if an appropriate index exists. +** This analysis does not consider whether or not the index exists; that +** is decided elsewhere. This analysis only looks at whether subterms +** appropriate for indexing exist. +** +** All examples A through E above satisfy case 3. But if a term +** also satisfies case 1 (such as B) we know that the optimizer will +** always prefer case 1, so in that case we pretend that case 3 is not +** satisfied. +** +** It might be the case that multiple tables are indexable. For example, +** (E) above is indexable on tables P, Q, and R. +** +** Terms that satisfy case 3 are candidates for lookup by using +** separate indices to find rowids for each subterm and composing +** the union of all rowids using a RowSet object. This is similar +** to "bitmap indices" in other database engines. +** +** OTHERWISE: +** +** If none of cases 1, 2, or 3 apply, then leave the eOperator set to +** zero. This term is not useful for search. */ -static int whereKeyStats( - Parse *pParse, /* Database connection */ - Index *pIdx, /* Index to consider domain of */ - UnpackedRecord *pRec, /* Vector of values to consider */ - int roundUp, /* Round up if true. Round down if false */ - tRowcnt *aStat /* OUT: stats written here */ +static void exprAnalyzeOrTerm( + SrcList *pSrc, /* the FROM clause */ + WhereClause *pWC, /* the complete WHERE clause */ + int idxTerm /* Index of the OR-term to be analyzed */ ){ - IndexSample *aSample = pIdx->aSample; - int iCol; /* Index of required stats in anEq[] etc. */ - int i; /* Index of first sample >= pRec */ - int iSample; /* Smallest sample larger than or equal to pRec */ - int iMin = 0; /* Smallest sample not yet tested */ - int iTest; /* Next sample to test */ - int res; /* Result of comparison operation */ - int nField; /* Number of fields in pRec */ - tRowcnt iLower = 0; /* anLt[] + anEq[] of largest sample pRec is > */ + WhereInfo *pWInfo = pWC->pWInfo; /* WHERE clause processing context */ + Parse *pParse = pWInfo->pParse; /* Parser context */ + sqlite3 *db = pParse->db; /* Database connection */ + WhereTerm *pTerm = &pWC->a[idxTerm]; /* The term to be analyzed */ + Expr *pExpr = pTerm->pExpr; /* The expression of the term */ + int i; /* Loop counters */ + WhereClause *pOrWc; /* Breakup of pTerm into subterms */ + WhereTerm *pOrTerm; /* A Sub-term within the pOrWc */ + WhereOrInfo *pOrInfo; /* Additional information associated with pTerm */ + Bitmask chngToIN; /* Tables that might satisfy case 1 */ + Bitmask indexable; /* Tables that are indexable, satisfying case 2 */ -#ifndef SQLITE_DEBUG - UNUSED_PARAMETER( pParse ); -#endif - assert( pRec!=0 ); - assert( pIdx->nSample>0 ); - assert( pRec->nField>0 && pRec->nField<=pIdx->nSampleCol ); + /* + ** Break the OR clause into its separate subterms. The subterms are + ** stored in a WhereClause structure containing within the WhereOrInfo + ** object that is attached to the original OR clause term. + */ + assert( (pTerm->wtFlags & (TERM_DYNAMIC|TERM_ORINFO|TERM_ANDINFO))==0 ); + assert( pExpr->op==TK_OR ); + pTerm->u.pOrInfo = pOrInfo = sqlite3DbMallocZero(db, sizeof(*pOrInfo)); + if( pOrInfo==0 ) return; + pTerm->wtFlags |= TERM_ORINFO; + pOrWc = &pOrInfo->wc; + sqlite3WhereClauseInit(pOrWc, pWInfo); + sqlite3WhereSplit(pOrWc, pExpr, TK_OR); + sqlite3WhereExprAnalyze(pSrc, pOrWc); + if( db->mallocFailed ) return; + assert( pOrWc->nTerm>=2 ); - /* Do a binary search to find the first sample greater than or equal - ** to pRec. If pRec contains a single field, the set of samples to search - ** is simply the aSample[] array. If the samples in aSample[] contain more - ** than one fields, all fields following the first are ignored. - ** - ** If pRec contains N fields, where N is more than one, then as well as the - ** samples in aSample[] (truncated to N fields), the search also has to - ** consider prefixes of those samples. For example, if the set of samples - ** in aSample is: - ** - ** aSample[0] = (a, 5) - ** aSample[1] = (a, 10) - ** aSample[2] = (b, 5) - ** aSample[3] = (c, 100) - ** aSample[4] = (c, 105) - ** - ** Then the search space should ideally be the samples above and the - ** unique prefixes [a], [b] and [c]. But since that is hard to organize, - ** the code actually searches this set: - ** - ** 0: (a) - ** 1: (a, 5) - ** 2: (a, 10) - ** 3: (a, 10) - ** 4: (b) - ** 5: (b, 5) - ** 6: (c) - ** 7: (c, 100) - ** 8: (c, 105) - ** 9: (c, 105) + /* + ** Compute the set of tables that might satisfy cases 1 or 3. + */ + indexable = ~(Bitmask)0; + chngToIN = ~(Bitmask)0; + for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0 && indexable; i--, pOrTerm++){ + if( (pOrTerm->eOperator & WO_SINGLE)==0 ){ + WhereAndInfo *pAndInfo; + assert( (pOrTerm->wtFlags & (TERM_ANDINFO|TERM_ORINFO))==0 ); + chngToIN = 0; + pAndInfo = sqlite3DbMallocRaw(db, sizeof(*pAndInfo)); + if( pAndInfo ){ + WhereClause *pAndWC; + WhereTerm *pAndTerm; + int j; + Bitmask b = 0; + pOrTerm->u.pAndInfo = pAndInfo; + pOrTerm->wtFlags |= TERM_ANDINFO; + pOrTerm->eOperator = WO_AND; + pAndWC = &pAndInfo->wc; + sqlite3WhereClauseInit(pAndWC, pWC->pWInfo); + sqlite3WhereSplit(pAndWC, pOrTerm->pExpr, TK_AND); + sqlite3WhereExprAnalyze(pSrc, pAndWC); + pAndWC->pOuter = pWC; + testcase( db->mallocFailed ); + if( !db->mallocFailed ){ + for(j=0, pAndTerm=pAndWC->a; jnTerm; j++, pAndTerm++){ + assert( pAndTerm->pExpr ); + if( allowedOp(pAndTerm->pExpr->op) ){ + b |= sqlite3WhereGetMask(&pWInfo->sMaskSet, pAndTerm->leftCursor); + } + } + } + indexable &= b; + } + }else if( pOrTerm->wtFlags & TERM_COPIED ){ + /* Skip this term for now. We revisit it when we process the + ** corresponding TERM_VIRTUAL term */ + }else{ + Bitmask b; + b = sqlite3WhereGetMask(&pWInfo->sMaskSet, pOrTerm->leftCursor); + if( pOrTerm->wtFlags & TERM_VIRTUAL ){ + WhereTerm *pOther = &pOrWc->a[pOrTerm->iParent]; + b |= sqlite3WhereGetMask(&pWInfo->sMaskSet, pOther->leftCursor); + } + indexable &= b; + if( (pOrTerm->eOperator & WO_EQ)==0 ){ + chngToIN = 0; + }else{ + chngToIN &= b; + } + } + } + + /* + ** Record the set of tables that satisfy case 3. The set might be + ** empty. + */ + pOrInfo->indexable = indexable; + pTerm->eOperator = indexable==0 ? 0 : WO_OR; + + /* For a two-way OR, attempt to implementation case 2. + */ + if( indexable && pOrWc->nTerm==2 ){ + int iOne = 0; + WhereTerm *pOne; + while( (pOne = whereNthSubterm(&pOrWc->a[0],iOne++))!=0 ){ + int iTwo = 0; + WhereTerm *pTwo; + while( (pTwo = whereNthSubterm(&pOrWc->a[1],iTwo++))!=0 ){ + whereCombineDisjuncts(pSrc, pWC, pOne, pTwo); + } + } + } + + /* + ** chngToIN holds a set of tables that *might* satisfy case 1. But + ** we have to do some additional checking to see if case 1 really + ** is satisfied. ** - ** For each sample in the aSample[] array, N samples are present in the - ** effective sample array. In the above, samples 0 and 1 are based on - ** sample aSample[0]. Samples 2 and 3 on aSample[1] etc. + ** chngToIN will hold either 0, 1, or 2 bits. The 0-bit case means + ** that there is no possibility of transforming the OR clause into an + ** IN operator because one or more terms in the OR clause contain + ** something other than == on a column in the single table. The 1-bit + ** case means that every term of the OR clause is of the form + ** "table.column=expr" for some single table. The one bit that is set + ** will correspond to the common table. We still need to check to make + ** sure the same column is used on all terms. The 2-bit case is when + ** the all terms are of the form "table1.column=table2.column". It + ** might be possible to form an IN operator with either table1.column + ** or table2.column as the LHS if either is common to every term of + ** the OR clause. ** - ** Often, sample i of each block of N effective samples has (i+1) fields. - ** Except, each sample may be extended to ensure that it is greater than or - ** equal to the previous sample in the array. For example, in the above, - ** sample 2 is the first sample of a block of N samples, so at first it - ** appears that it should be 1 field in size. However, that would make it - ** smaller than sample 1, so the binary search would not work. As a result, - ** it is extended to two fields. The duplicates that this creates do not - ** cause any problems. + ** Note that terms of the form "table.column1=table.column2" (the + ** same table on both sizes of the ==) cannot be optimized. */ - nField = pRec->nField; - iCol = 0; - iSample = pIdx->nSample * nField; - do{ - int iSamp; /* Index in aSample[] of test sample */ - int n; /* Number of fields in test sample */ + if( chngToIN ){ + int okToChngToIN = 0; /* True if the conversion to IN is valid */ + int iColumn = -1; /* Column index on lhs of IN operator */ + int iCursor = -1; /* Table cursor common to all terms */ + int j = 0; /* Loop counter */ - iTest = (iMin+iSample)/2; - iSamp = iTest / nField; - if( iSamp>0 ){ - /* The proposed effective sample is a prefix of sample aSample[iSamp]. - ** Specifically, the shortest prefix of at least (1 + iTest%nField) - ** fields that is greater than the previous effective sample. */ - for(n=(iTest % nField) + 1; na; + for(i=pOrWc->nTerm-1; i>=0; i--, pOrTerm++){ + assert( pOrTerm->eOperator & WO_EQ ); + pOrTerm->wtFlags &= ~TERM_OR_OK; + if( pOrTerm->leftCursor==iCursor ){ + /* This is the 2-bit case and we are on the second iteration and + ** current term is from the first iteration. So skip this term. */ + assert( j==1 ); + continue; + } + if( (chngToIN & sqlite3WhereGetMask(&pWInfo->sMaskSet, + pOrTerm->leftCursor))==0 ){ + /* This term must be of the form t1.a==t2.b where t2 is in the + ** chngToIN set but t1 is not. This term will be either preceded + ** or follwed by an inverted copy (t2.b==t1.a). Skip this term + ** and use its inversion. */ + testcase( pOrTerm->wtFlags & TERM_COPIED ); + testcase( pOrTerm->wtFlags & TERM_VIRTUAL ); + assert( pOrTerm->wtFlags & (TERM_COPIED|TERM_VIRTUAL) ); + continue; + } + iColumn = pOrTerm->u.leftColumn; + iCursor = pOrTerm->leftCursor; + break; } - }else{ - n = iTest + 1; - } + if( i<0 ){ + /* No candidate table+column was found. This can only occur + ** on the second iteration */ + assert( j==1 ); + assert( IsPowerOfTwo(chngToIN) ); + assert( chngToIN==sqlite3WhereGetMask(&pWInfo->sMaskSet, iCursor) ); + break; + } + testcase( j==1 ); - pRec->nField = n; - res = sqlite3VdbeRecordCompare(aSample[iSamp].n, aSample[iSamp].p, pRec); - if( res<0 ){ - iLower = aSample[iSamp].anLt[n-1] + aSample[iSamp].anEq[n-1]; - iMin = iTest+1; - }else if( res==0 && n=0 && okToChngToIN; i--, pOrTerm++){ + assert( pOrTerm->eOperator & WO_EQ ); + if( pOrTerm->leftCursor!=iCursor ){ + pOrTerm->wtFlags &= ~TERM_OR_OK; + }else if( pOrTerm->u.leftColumn!=iColumn ){ + okToChngToIN = 0; + }else{ + int affLeft, affRight; + /* If the right-hand side is also a column, then the affinities + ** of both right and left sides must be such that no type + ** conversions are required on the right. (Ticket #2249) + */ + affRight = sqlite3ExprAffinity(pOrTerm->pExpr->pRight); + affLeft = sqlite3ExprAffinity(pOrTerm->pExpr->pLeft); + if( affRight!=0 && affRight!=affLeft ){ + okToChngToIN = 0; + }else{ + pOrTerm->wtFlags |= TERM_OR_OK; + } + } + } } - }while( res && iMindb->mallocFailed==0 ){ - if( res==0 ){ - /* If (res==0) is true, then pRec must be equal to sample i. */ - assert( inSample ); - assert( iCol==nField-1 ); - pRec->nField = nField; - assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec) - || pParse->db->mallocFailed - ); - }else{ - /* Unless i==pIdx->nSample, indicating that pRec is larger than - ** all samples in the aSample[] array, pRec must be smaller than the - ** (iCol+1) field prefix of sample i. */ - assert( i<=pIdx->nSample && i>=0 ); - pRec->nField = iCol+1; - assert( i==pIdx->nSample - || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0 - || pParse->db->mallocFailed ); + /* At this point, okToChngToIN is true if original pTerm satisfies + ** case 1. In that case, construct a new virtual term that is + ** pTerm converted into an IN operator. + */ + if( okToChngToIN ){ + Expr *pDup; /* A transient duplicate expression */ + ExprList *pList = 0; /* The RHS of the IN operator */ + Expr *pLeft = 0; /* The LHS of the IN operator */ + Expr *pNew; /* The complete IN operator */ - /* if i==0 and iCol==0, then record pRec is smaller than all samples - ** in the aSample[] array. Otherwise, if (iCol>0) then pRec must - ** be greater than or equal to the (iCol) field prefix of sample i. - ** If (i>0), then pRec must also be greater than sample (i-1). */ - if( iCol>0 ){ - pRec->nField = iCol; - assert( sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)<=0 - || pParse->db->mallocFailed ); + for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0; i--, pOrTerm++){ + if( (pOrTerm->wtFlags & TERM_OR_OK)==0 ) continue; + assert( pOrTerm->eOperator & WO_EQ ); + assert( pOrTerm->leftCursor==iCursor ); + assert( pOrTerm->u.leftColumn==iColumn ); + pDup = sqlite3ExprDup(db, pOrTerm->pExpr->pRight, 0); + pList = sqlite3ExprListAppend(pWInfo->pParse, pList, pDup); + pLeft = pOrTerm->pExpr->pLeft; } - if( i>0 ){ - pRec->nField = nField; - assert( sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0 - || pParse->db->mallocFailed ); + assert( pLeft!=0 ); + pDup = sqlite3ExprDup(db, pLeft, 0); + pNew = sqlite3PExpr(pParse, TK_IN, pDup, 0, 0); + if( pNew ){ + int idxNew; + transferJoinMarkings(pNew, pExpr); + assert( !ExprHasProperty(pNew, EP_xIsSelect) ); + pNew->x.pList = pList; + idxNew = whereClauseInsert(pWC, pNew, TERM_VIRTUAL|TERM_DYNAMIC); + testcase( idxNew==0 ); + exprAnalyze(pSrc, pWC, idxNew); + pTerm = &pWC->a[idxTerm]; + markTermAsChild(pWC, idxNew, idxTerm); + }else{ + sqlite3ExprListDelete(db, pList); } + pTerm->eOperator = WO_NOOP; /* case 1 trumps case 3 */ } } -#endif /* ifdef SQLITE_DEBUG */ +} +#endif /* !SQLITE_OMIT_OR_OPTIMIZATION && !SQLITE_OMIT_SUBQUERY */ - if( res==0 ){ - /* Record pRec is equal to sample i */ - assert( iCol==nField-1 ); - aStat[0] = aSample[i].anLt[iCol]; - aStat[1] = aSample[i].anEq[iCol]; - }else{ - /* At this point, the (iCol+1) field prefix of aSample[i] is the first - ** sample that is greater than pRec. Or, if i==pIdx->nSample then pRec - ** is larger than all samples in the array. */ - tRowcnt iUpper, iGap; - if( i>=pIdx->nSample ){ - iUpper = sqlite3LogEstToInt(pIdx->aiRowLogEst[0]); - }else{ - iUpper = aSample[i].anLt[iCol]; - } +/* +** We already know that pExpr is a binary operator where both operands are +** column references. This routine checks to see if pExpr is an equivalence +** relation: +** 1. The SQLITE_Transitive optimization must be enabled +** 2. Must be either an == or an IS operator +** 3. Not originating in the ON clause of an OUTER JOIN +** 4. The affinities of A and B must be compatible +** 5a. Both operands use the same collating sequence OR +** 5b. The overall collating sequence is BINARY +** If this routine returns TRUE, that means that the RHS can be substituted +** for the LHS anyplace else in the WHERE clause where the LHS column occurs. +** This is an optimization. No harm comes from returning 0. But if 1 is +** returned when it should not be, then incorrect answers might result. +*/ +static int termIsEquivalence(Parse *pParse, Expr *pExpr){ + char aff1, aff2; + CollSeq *pColl; + const char *zColl1, *zColl2; + if( !OptimizationEnabled(pParse->db, SQLITE_Transitive) ) return 0; + if( pExpr->op!=TK_EQ && pExpr->op!=TK_IS ) return 0; + if( ExprHasProperty(pExpr, EP_FromJoin) ) return 0; + aff1 = sqlite3ExprAffinity(pExpr->pLeft); + aff2 = sqlite3ExprAffinity(pExpr->pRight); + if( aff1!=aff2 + && (!sqlite3IsNumericAffinity(aff1) || !sqlite3IsNumericAffinity(aff2)) + ){ + return 0; + } + pColl = sqlite3BinaryCompareCollSeq(pParse, pExpr->pLeft, pExpr->pRight); + if( pColl==0 || sqlite3StrICmp(pColl->zName, "BINARY")==0 ) return 1; + pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft); + /* Since pLeft and pRight are both a column references, their collating + ** sequence should always be defined. */ + zColl1 = ALWAYS(pColl) ? pColl->zName : 0; + pColl = sqlite3ExprCollSeq(pParse, pExpr->pRight); + zColl2 = ALWAYS(pColl) ? pColl->zName : 0; + return sqlite3StrICmp(zColl1, zColl2)==0; +} - if( iLower>=iUpper ){ - iGap = 0; - }else{ - iGap = iUpper - iLower; - } - if( roundUp ){ - iGap = (iGap*2)/3; - }else{ - iGap = iGap/3; +/* +** Recursively walk the expressions of a SELECT statement and generate +** a bitmask indicating which tables are used in that expression +** tree. +*/ +static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){ + Bitmask mask = 0; + while( pS ){ + SrcList *pSrc = pS->pSrc; + mask |= sqlite3WhereExprListUsage(pMaskSet, pS->pEList); + mask |= sqlite3WhereExprListUsage(pMaskSet, pS->pGroupBy); + mask |= sqlite3WhereExprListUsage(pMaskSet, pS->pOrderBy); + mask |= sqlite3WhereExprUsage(pMaskSet, pS->pWhere); + mask |= sqlite3WhereExprUsage(pMaskSet, pS->pHaving); + if( ALWAYS(pSrc!=0) ){ + int i; + for(i=0; inSrc; i++){ + mask |= exprSelectUsage(pMaskSet, pSrc->a[i].pSelect); + mask |= sqlite3WhereExprUsage(pMaskSet, pSrc->a[i].pOn); + } } - aStat[0] = iLower + iGap; - aStat[1] = pIdx->aAvgEq[iCol]; + pS = pS->pPrior; } - - /* Restore the pRec->nField value before returning. */ - pRec->nField = nField; - return i; + return mask; } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ /* -** If it is not NULL, pTerm is a term that provides an upper or lower -** bound on a range scan. Without considering pTerm, it is estimated -** that the scan will visit nNew rows. This function returns the number -** estimated to be visited after taking pTerm into account. +** Expression pExpr is one operand of a comparison operator that might +** be useful for indexing. This routine checks to see if pExpr appears +** in any index. Return TRUE (1) if pExpr is an indexed term and return +** FALSE (0) if not. If TRUE is returned, also set *piCur to the cursor +** number of the table that is indexed and *piColumn to the column number +** of the column that is indexed, or -2 if an expression is being indexed. ** -** If the user explicitly specified a likelihood() value for this term, -** then the return value is the likelihood multiplied by the number of -** input rows. Otherwise, this function assumes that an "IS NOT NULL" term -** has a likelihood of 0.50, and any other term a likelihood of 0.25. +** If pExpr is a TK_COLUMN column reference, then this routine always returns +** true even if that particular column is not indexed, because the column +** might be added to an automatic index later. */ -static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ - LogEst nRet = nNew; - if( pTerm ){ - if( pTerm->truthProb<=0 ){ - nRet += pTerm->truthProb; - }else if( (pTerm->wtFlags & TERM_VNULL)==0 ){ - nRet -= 20; assert( 20==sqlite3LogEst(4) ); +static int exprMightBeIndexed( + SrcList *pFrom, /* The FROM clause */ + Bitmask mPrereq, /* Bitmask of FROM clause terms referenced by pExpr */ + Expr *pExpr, /* An operand of a comparison operator */ + int *piCur, /* Write the referenced table cursor number here */ + int *piColumn /* Write the referenced table column number here */ +){ + Index *pIdx; + int i; + int iCur; + if( pExpr->op==TK_COLUMN ){ + *piCur = pExpr->iTable; + *piColumn = pExpr->iColumn; + return 1; + } + if( mPrereq==0 ) return 0; /* No table references */ + if( (mPrereq&(mPrereq-1))!=0 ) return 0; /* Refs more than one table */ + for(i=0; mPrereq>1; i++, mPrereq>>=1){} + iCur = pFrom->a[i].iCursor; + for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->aColExpr==0 ) continue; + for(i=0; inKeyCol; i++){ + if( pIdx->aiColumn[i]!=(-2) ) continue; + if( sqlite3ExprCompare(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){ + *piCur = iCur; + *piColumn = -2; + return 1; + } } } - return nRet; + return 0; } -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 -/* -** This function is called to estimate the number of rows visited by a -** range-scan on a skip-scan index. For example: -** -** CREATE INDEX i1 ON t1(a, b, c); -** SELECT * FROM t1 WHERE a=? AND c BETWEEN ? AND ?; -** -** Value pLoop->nOut is currently set to the estimated number of rows -** visited for scanning (a=? AND b=?). This function reduces that estimate -** by some factor to account for the (c BETWEEN ? AND ?) expression based -** on the stat4 data for the index. this scan will be peformed multiple -** times (once for each (a,b) combination that matches a=?) is dealt with -** by the caller. -** -** It does this by scanning through all stat4 samples, comparing values -** extracted from pLower and pUpper with the corresponding column in each -** sample. If L and U are the number of samples found to be less than or -** equal to the values extracted from pLower and pUpper respectively, and -** N is the total number of samples, the pLoop->nOut value is adjusted -** as follows: -** -** nOut = nOut * ( min(U - L, 1) / N ) -** -** If pLower is NULL, or a value cannot be extracted from the term, L is -** set to zero. If pUpper is NULL, or a value cannot be extracted from it, -** U is set to N. +/* +** The input to this routine is an WhereTerm structure with only the +** "pExpr" field filled in. The job of this routine is to analyze the +** subexpression and populate all the other fields of the WhereTerm +** structure. ** -** Normally, this function sets *pbDone to 1 before returning. However, -** if no value can be extracted from either pLower or pUpper (and so the -** estimate of the number of rows delivered remains unchanged), *pbDone -** is left as is. +** If the expression is of the form " X" it gets commuted +** to the standard form of "X ". ** -** If an error occurs, an SQLite error code is returned. Otherwise, -** SQLITE_OK. +** If the expression is of the form "X Y" where both X and Y are +** columns, then the original expression is unchanged and a new virtual +** term of the form "Y X" is added to the WHERE clause and +** analyzed separately. The original term is marked with TERM_COPIED +** and the new term is marked with TERM_DYNAMIC (because it's pExpr +** needs to be freed with the WhereClause) and TERM_VIRTUAL (because it +** is a commuted copy of a prior term.) The original term has nChild=1 +** and the copy has idxParent set to the index of the original term. */ -static int whereRangeSkipScanEst( - Parse *pParse, /* Parsing & code generating context */ - WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */ - WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ - WhereLoop *pLoop, /* Update the .nOut value of this loop */ - int *pbDone /* Set to true if at least one expr. value extracted */ +static void exprAnalyze( + SrcList *pSrc, /* the FROM clause */ + WhereClause *pWC, /* the WHERE clause */ + int idxTerm /* Index of the term to be analyzed */ ){ - Index *p = pLoop->u.btree.pIndex; - int nEq = pLoop->u.btree.nEq; - sqlite3 *db = pParse->db; - int nLower = -1; - int nUpper = p->nSample+1; - int rc = SQLITE_OK; - int iCol = p->aiColumn[nEq]; - u8 aff = iCol>=0 ? p->pTable->aCol[iCol].affinity : SQLITE_AFF_INTEGER; - CollSeq *pColl; - - sqlite3_value *p1 = 0; /* Value extracted from pLower */ - sqlite3_value *p2 = 0; /* Value extracted from pUpper */ - sqlite3_value *pVal = 0; /* Value extracted from record */ + WhereInfo *pWInfo = pWC->pWInfo; /* WHERE clause processing context */ + WhereTerm *pTerm; /* The term to be analyzed */ + WhereMaskSet *pMaskSet; /* Set of table index masks */ + Expr *pExpr; /* The expression to be analyzed */ + Bitmask prereqLeft; /* Prerequesites of the pExpr->pLeft */ + Bitmask prereqAll; /* Prerequesites of pExpr */ + Bitmask extraRight = 0; /* Extra dependencies on LEFT JOIN */ + Expr *pStr1 = 0; /* RHS of LIKE/GLOB operator */ + int isComplete = 0; /* RHS of LIKE/GLOB ends with wildcard */ + int noCase = 0; /* uppercase equivalent to lowercase */ + int op; /* Top-level operator. pExpr->op */ + Parse *pParse = pWInfo->pParse; /* Parsing context */ + sqlite3 *db = pParse->db; /* Database connection */ - pColl = sqlite3LocateCollSeq(pParse, p->azColl[nEq]); - if( pLower ){ - rc = sqlite3Stat4ValueFromExpr(pParse, pLower->pExpr->pRight, aff, &p1); - nLower = 0; + if( db->mallocFailed ){ + return; } - if( pUpper && rc==SQLITE_OK ){ - rc = sqlite3Stat4ValueFromExpr(pParse, pUpper->pExpr->pRight, aff, &p2); - nUpper = p2 ? 0 : p->nSample; + pTerm = &pWC->a[idxTerm]; + pMaskSet = &pWInfo->sMaskSet; + pExpr = pTerm->pExpr; + assert( pExpr->op!=TK_AS && pExpr->op!=TK_COLLATE ); + prereqLeft = sqlite3WhereExprUsage(pMaskSet, pExpr->pLeft); + op = pExpr->op; + if( op==TK_IN ){ + assert( pExpr->pRight==0 ); + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + pTerm->prereqRight = exprSelectUsage(pMaskSet, pExpr->x.pSelect); + }else{ + pTerm->prereqRight = sqlite3WhereExprListUsage(pMaskSet, pExpr->x.pList); + } + }else if( op==TK_ISNULL ){ + pTerm->prereqRight = 0; + }else{ + pTerm->prereqRight = sqlite3WhereExprUsage(pMaskSet, pExpr->pRight); } + prereqAll = sqlite3WhereExprUsage(pMaskSet, pExpr); + if( ExprHasProperty(pExpr, EP_FromJoin) ){ + Bitmask x = sqlite3WhereGetMask(pMaskSet, pExpr->iRightJoinTable); + prereqAll |= x; + extraRight = x-1; /* ON clause terms may not be used with an index + ** on left table of a LEFT JOIN. Ticket #3015 */ + } + pTerm->prereqAll = prereqAll; + pTerm->leftCursor = -1; + pTerm->iParent = -1; + pTerm->eOperator = 0; + if( allowedOp(op) ){ + int iCur, iColumn; + Expr *pLeft = sqlite3ExprSkipCollate(pExpr->pLeft); + Expr *pRight = sqlite3ExprSkipCollate(pExpr->pRight); + u16 opMask = (pTerm->prereqRight & prereqLeft)==0 ? WO_ALL : WO_EQUIV; + if( exprMightBeIndexed(pSrc, prereqLeft, pLeft, &iCur, &iColumn) ){ + pTerm->leftCursor = iCur; + pTerm->u.leftColumn = iColumn; + pTerm->eOperator = operatorMask(op) & opMask; + } + if( op==TK_IS ) pTerm->wtFlags |= TERM_IS; + if( pRight + && exprMightBeIndexed(pSrc, pTerm->prereqRight, pRight, &iCur, &iColumn) + ){ + WhereTerm *pNew; + Expr *pDup; + u16 eExtraOp = 0; /* Extra bits for pNew->eOperator */ + if( pTerm->leftCursor>=0 ){ + int idxNew; + pDup = sqlite3ExprDup(db, pExpr, 0); + if( db->mallocFailed ){ + sqlite3ExprDelete(db, pDup); + return; + } + idxNew = whereClauseInsert(pWC, pDup, TERM_VIRTUAL|TERM_DYNAMIC); + if( idxNew==0 ) return; + pNew = &pWC->a[idxNew]; + markTermAsChild(pWC, idxNew, idxTerm); + if( op==TK_IS ) pNew->wtFlags |= TERM_IS; + pTerm = &pWC->a[idxTerm]; + pTerm->wtFlags |= TERM_COPIED; - if( p1 || p2 ){ - int i; - int nDiff; - for(i=0; rc==SQLITE_OK && inSample; i++){ - rc = sqlite3Stat4Column(db, p->aSample[i].p, p->aSample[i].n, nEq, &pVal); - if( rc==SQLITE_OK && p1 ){ - int res = sqlite3MemCompare(p1, pVal, pColl); - if( res>=0 ) nLower++; - } - if( rc==SQLITE_OK && p2 ){ - int res = sqlite3MemCompare(p2, pVal, pColl); - if( res>=0 ) nUpper++; + if( termIsEquivalence(pParse, pDup) ){ + pTerm->eOperator |= WO_EQUIV; + eExtraOp = WO_EQUIV; + } + }else{ + pDup = pExpr; + pNew = pTerm; } + exprCommute(pParse, pDup); + pNew->leftCursor = iCur; + pNew->u.leftColumn = iColumn; + testcase( (prereqLeft | extraRight) != prereqLeft ); + pNew->prereqRight = prereqLeft | extraRight; + pNew->prereqAll = prereqAll; + pNew->eOperator = (operatorMask(pDup->op) + eExtraOp) & opMask; } - nDiff = (nUpper - nLower); - if( nDiff<=0 ) nDiff = 1; + } - /* If there is both an upper and lower bound specified, and the - ** comparisons indicate that they are close together, use the fallback - ** method (assume that the scan visits 1/64 of the rows) for estimating - ** the number of rows visited. Otherwise, estimate the number of rows - ** using the method described in the header comment for this function. */ - if( nDiff!=1 || pUpper==0 || pLower==0 ){ - int nAdjust = (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff)); - pLoop->nOut -= nAdjust; - *pbDone = 1; - WHERETRACE(0x10, ("range skip-scan regions: %u..%u adjust=%d est=%d\n", - nLower, nUpper, nAdjust*-1, pLoop->nOut)); +#ifndef SQLITE_OMIT_BETWEEN_OPTIMIZATION + /* If a term is the BETWEEN operator, create two new virtual terms + ** that define the range that the BETWEEN implements. For example: + ** + ** a BETWEEN b AND c + ** + ** is converted into: + ** + ** (a BETWEEN b AND c) AND (a>=b) AND (a<=c) + ** + ** The two new terms are added onto the end of the WhereClause object. + ** The new terms are "dynamic" and are children of the original BETWEEN + ** term. That means that if the BETWEEN term is coded, the children are + ** skipped. Or, if the children are satisfied by an index, the original + ** BETWEEN term is skipped. + */ + else if( pExpr->op==TK_BETWEEN && pWC->op==TK_AND ){ + ExprList *pList = pExpr->x.pList; + int i; + static const u8 ops[] = {TK_GE, TK_LE}; + assert( pList!=0 ); + assert( pList->nExpr==2 ); + for(i=0; i<2; i++){ + Expr *pNewExpr; + int idxNew; + pNewExpr = sqlite3PExpr(pParse, ops[i], + sqlite3ExprDup(db, pExpr->pLeft, 0), + sqlite3ExprDup(db, pList->a[i].pExpr, 0), 0); + transferJoinMarkings(pNewExpr, pExpr); + idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); + testcase( idxNew==0 ); + exprAnalyze(pSrc, pWC, idxNew); + pTerm = &pWC->a[idxTerm]; + markTermAsChild(pWC, idxNew, idxTerm); } + } +#endif /* SQLITE_OMIT_BETWEEN_OPTIMIZATION */ - }else{ - assert( *pbDone==0 ); +#if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) + /* Analyze a term that is composed of two or more subterms connected by + ** an OR operator. + */ + else if( pExpr->op==TK_OR ){ + assert( pWC->op==TK_AND ); + exprAnalyzeOrTerm(pSrc, pWC, idxTerm); + pTerm = &pWC->a[idxTerm]; } +#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ - sqlite3ValueFree(p1); - sqlite3ValueFree(p2); - sqlite3ValueFree(pVal); +#ifndef SQLITE_OMIT_LIKE_OPTIMIZATION + /* Add constraints to reduce the search space on a LIKE or GLOB + ** operator. + ** + ** A like pattern of the form "x LIKE 'aBc%'" is changed into constraints + ** + ** x>='ABC' AND x<'abd' AND x LIKE 'aBc%' + ** + ** The last character of the prefix "abc" is incremented to form the + ** termination condition "abd". If case is not significant (the default + ** for LIKE) then the lower-bound is made all uppercase and the upper- + ** bound is made all lowercase so that the bounds also work when comparing + ** BLOBs. + */ + if( pWC->op==TK_AND + && isLikeOrGlob(pParse, pExpr, &pStr1, &isComplete, &noCase) + ){ + Expr *pLeft; /* LHS of LIKE/GLOB operator */ + Expr *pStr2; /* Copy of pStr1 - RHS of LIKE/GLOB operator */ + Expr *pNewExpr1; + Expr *pNewExpr2; + int idxNew1; + int idxNew2; + const char *zCollSeqName; /* Name of collating sequence */ + const u16 wtFlags = TERM_LIKEOPT | TERM_VIRTUAL | TERM_DYNAMIC; - return rc; -} -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ + pLeft = pExpr->x.pList->a[1].pExpr; + pStr2 = sqlite3ExprDup(db, pStr1, 0); -/* -** This function is used to estimate the number of rows that will be visited -** by scanning an index for a range of values. The range may have an upper -** bound, a lower bound, or both. The WHERE clause terms that set the upper -** and lower bounds are represented by pLower and pUpper respectively. For -** example, assuming that index p is on t1(a): -** -** ... FROM t1 WHERE a > ? AND a < ? ... -** |_____| |_____| -** | | -** pLower pUpper -** -** If either of the upper or lower bound is not present, then NULL is passed in -** place of the corresponding WhereTerm. -** -** The value in (pBuilder->pNew->u.btree.nEq) is the number of the index -** column subject to the range constraint. Or, equivalently, the number of -** equality constraints optimized by the proposed index scan. For example, -** assuming index p is on t1(a, b), and the SQL query is: -** -** ... FROM t1 WHERE a = ? AND b > ? AND b < ? ... -** -** then nEq is set to 1 (as the range restricted column, b, is the second -** left-most column of the index). Or, if the query is: -** -** ... FROM t1 WHERE a > ? AND a < ? ... -** -** then nEq is set to 0. -** -** When this function is called, *pnOut is set to the sqlite3LogEst() of the -** number of rows that the index scan is expected to visit without -** considering the range constraints. If nEq is 0, then *pnOut is the number of -** rows in the index. Assuming no error occurs, *pnOut is adjusted (reduced) -** to account for the range constraints pLower and pUpper. -** -** In the absence of sqlite_stat4 ANALYZE data, or if such data cannot be -** used, a single range inequality reduces the search space by a factor of 4. -** and a pair of constraints (x>? AND x123" Might be NULL */ - WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ - WhereLoop *pLoop /* Modify the .nOut and maybe .rRun fields */ -){ - int rc = SQLITE_OK; - int nOut = pLoop->nOut; - LogEst nNew; - -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - Index *p = pLoop->u.btree.pIndex; - int nEq = pLoop->u.btree.nEq; - - if( p->nSample>0 && nEqnSampleCol ){ - if( nEq==pBuilder->nRecValid ){ - UnpackedRecord *pRec = pBuilder->pRec; - tRowcnt a[2]; - u8 aff; - - /* Variable iLower will be set to the estimate of the number of rows in - ** the index that are less than the lower bound of the range query. The - ** lower bound being the concatenation of $P and $L, where $P is the - ** key-prefix formed by the nEq values matched against the nEq left-most - ** columns of the index, and $L is the value in pLower. - ** - ** Or, if pLower is NULL or $L cannot be extracted from it (because it - ** is not a simple variable or literal value), the lower bound of the - ** range is $P. Due to a quirk in the way whereKeyStats() works, even - ** if $L is available, whereKeyStats() is called for both ($P) and - ** ($P:$L) and the larger of the two returned values is used. - ** - ** Similarly, iUpper is to be set to the estimate of the number of rows - ** less than the upper bound of the range query. Where the upper bound - ** is either ($P) or ($P:$U). Again, even if $U is available, both values - ** of iUpper are requested of whereKeyStats() and the smaller used. - ** - ** The number of rows between the two bounds is then just iUpper-iLower. - */ - tRowcnt iLower; /* Rows less than the lower bound */ - tRowcnt iUpper; /* Rows less than the upper bound */ - int iLwrIdx = -2; /* aSample[] for the lower bound */ - int iUprIdx = -1; /* aSample[] for the upper bound */ - - if( pRec ){ - testcase( pRec->nField!=pBuilder->nRecValid ); - pRec->nField = pBuilder->nRecValid; - } - if( nEq==p->nKeyCol ){ - aff = SQLITE_AFF_INTEGER; - }else{ - aff = p->pTable->aCol[p->aiColumn[nEq]].affinity; - } - /* Determine iLower and iUpper using ($P) only. */ - if( nEq==0 ){ - iLower = 0; - iUpper = p->nRowEst0; - }else{ - /* Note: this call could be optimized away - since the same values must - ** have been requested when testing key $P in whereEqualScanEst(). */ - whereKeyStats(pParse, p, pRec, 0, a); - iLower = a[0]; - iUpper = a[0] + a[1]; - } - - assert( pLower==0 || (pLower->eOperator & (WO_GT|WO_GE))!=0 ); - assert( pUpper==0 || (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); - assert( p->aSortOrder!=0 ); - if( p->aSortOrder[nEq] ){ - /* The roles of pLower and pUpper are swapped for a DESC index */ - SWAP(WhereTerm*, pLower, pUpper); - } - - /* If possible, improve on the iLower estimate using ($P:$L). */ - if( pLower ){ - int bOk; /* True if value is extracted from pExpr */ - Expr *pExpr = pLower->pExpr->pRight; - rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); - if( rc==SQLITE_OK && bOk ){ - tRowcnt iNew; - iLwrIdx = whereKeyStats(pParse, p, pRec, 0, a); - iNew = a[0] + ((pLower->eOperator & (WO_GT|WO_LE)) ? a[1] : 0); - if( iNew>iLower ) iLower = iNew; - nOut--; - pLower = 0; - } - } - - /* If possible, improve on the iUpper estimate using ($P:$U). */ - if( pUpper ){ - int bOk; /* True if value is extracted from pExpr */ - Expr *pExpr = pUpper->pExpr->pRight; - rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); - if( rc==SQLITE_OK && bOk ){ - tRowcnt iNew; - iUprIdx = whereKeyStats(pParse, p, pRec, 1, a); - iNew = a[0] + ((pUpper->eOperator & (WO_GT|WO_LE)) ? a[1] : 0); - if( iNewdb->mallocFailed ){ + int i; + char c; + pTerm->wtFlags |= TERM_LIKE; + for(i=0; (c = pStr1->u.zToken[i])!=0; i++){ + pStr1->u.zToken[i] = sqlite3Toupper(c); + pStr2->u.zToken[i] = sqlite3Tolower(c); } + } - pBuilder->pRec = pRec; - if( rc==SQLITE_OK ){ - if( iUpper>iLower ){ - nNew = sqlite3LogEst(iUpper - iLower); - /* TUNING: If both iUpper and iLower are derived from the same - ** sample, then assume they are 4x more selective. This brings - ** the estimated selectivity more in line with what it would be - ** if estimated without the use of STAT3/4 tables. */ - if( iLwrIdx==iUprIdx ) nNew -= 20; assert( 20==sqlite3LogEst(4) ); - }else{ - nNew = 10; assert( 10==sqlite3LogEst(2) ); - } - if( nNewmallocFailed ){ + u8 c, *pC; /* Last character before the first wildcard */ + pC = (u8*)&pStr2->u.zToken[sqlite3Strlen30(pStr2->u.zToken)-1]; + c = *pC; + if( noCase ){ + /* The point is to increment the last character before the first + ** wildcard. But if we increment '@', that will push it into the + ** alphabetic range where case conversions will mess up the + ** inequality. To avoid this, make sure to also run the full + ** LIKE on all candidate expressions by clearing the isComplete flag + */ + if( c=='A'-1 ) isComplete = 0; + c = sqlite3UpperToLower[c]; } - }else{ - int bDone = 0; - rc = whereRangeSkipScanEst(pParse, pLower, pUpper, pLoop, &bDone); - if( bDone ) return rc; + *pC = c + 1; + } + zCollSeqName = noCase ? "NOCASE" : "BINARY"; + pNewExpr1 = sqlite3ExprDup(db, pLeft, 0); + pNewExpr1 = sqlite3PExpr(pParse, TK_GE, + sqlite3ExprAddCollateString(pParse,pNewExpr1,zCollSeqName), + pStr1, 0); + transferJoinMarkings(pNewExpr1, pExpr); + idxNew1 = whereClauseInsert(pWC, pNewExpr1, wtFlags); + testcase( idxNew1==0 ); + exprAnalyze(pSrc, pWC, idxNew1); + pNewExpr2 = sqlite3ExprDup(db, pLeft, 0); + pNewExpr2 = sqlite3PExpr(pParse, TK_LT, + sqlite3ExprAddCollateString(pParse,pNewExpr2,zCollSeqName), + pStr2, 0); + transferJoinMarkings(pNewExpr2, pExpr); + idxNew2 = whereClauseInsert(pWC, pNewExpr2, wtFlags); + testcase( idxNew2==0 ); + exprAnalyze(pSrc, pWC, idxNew2); + pTerm = &pWC->a[idxTerm]; + if( isComplete ){ + markTermAsChild(pWC, idxNew1, idxTerm); + markTermAsChild(pWC, idxNew2, idxTerm); } } -#else - UNUSED_PARAMETER(pParse); - UNUSED_PARAMETER(pBuilder); - assert( pLower || pUpper ); -#endif - assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 ); - nNew = whereRangeAdjust(pLower, nOut); - nNew = whereRangeAdjust(pUpper, nNew); +#endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ - /* TUNING: If there is both an upper and lower limit and neither limit - ** has an application-defined likelihood(), assume the range is - ** reduced by an additional 75%. This means that, by default, an open-ended - ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the - ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to - ** match 1/64 of the index. */ - if( pLower && pLower->truthProb>0 && pUpper && pUpper->truthProb>0 ){ - nNew -= 20; - } +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* Add a WO_MATCH auxiliary term to the constraint set if the + ** current expression is of the form: column MATCH expr. + ** This information is used by the xBestIndex methods of + ** virtual tables. The native query optimizer does not attempt + ** to do anything with MATCH functions. + */ + if( isMatchOfColumn(pExpr) ){ + int idxNew; + Expr *pRight, *pLeft; + WhereTerm *pNewTerm; + Bitmask prereqColumn, prereqExpr; - nOut -= (pLower!=0) + (pUpper!=0); - if( nNew<10 ) nNew = 10; - if( nNewnOut>nOut ){ - WHERETRACE(0x10,("Range scan lowers nOut from %d to %d\n", - pLoop->nOut, nOut)); + pRight = pExpr->x.pList->a[0].pExpr; + pLeft = pExpr->x.pList->a[1].pExpr; + prereqExpr = sqlite3WhereExprUsage(pMaskSet, pRight); + prereqColumn = sqlite3WhereExprUsage(pMaskSet, pLeft); + if( (prereqExpr & prereqColumn)==0 ){ + Expr *pNewExpr; + pNewExpr = sqlite3PExpr(pParse, TK_MATCH, + 0, sqlite3ExprDup(db, pRight, 0), 0); + idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); + testcase( idxNew==0 ); + pNewTerm = &pWC->a[idxNew]; + pNewTerm->prereqRight = prereqExpr; + pNewTerm->leftCursor = pLeft->iTable; + pNewTerm->u.leftColumn = pLeft->iColumn; + pNewTerm->eOperator = WO_MATCH; + markTermAsChild(pWC, idxNew, idxTerm); + pTerm = &pWC->a[idxTerm]; + pTerm->wtFlags |= TERM_COPIED; + pNewTerm->prereqAll = pTerm->prereqAll; + } } -#endif - pLoop->nOut = (LogEst)nOut; - return rc; -} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 -/* -** Estimate the number of rows that will be returned based on -** an equality constraint x=VALUE and where that VALUE occurs in -** the histogram data. This only works when x is the left-most -** column of an index and sqlite_stat3 histogram data is available -** for that index. When pExpr==NULL that means the constraint is -** "x IS NULL" instead of "x=VALUE". -** -** Write the estimated row count into *pnRow and return SQLITE_OK. -** If unable to make an estimate, leave *pnRow unchanged and return -** non-zero. -** -** This routine can fail if it is unable to load a collating sequence -** required for string comparison, or if unable to allocate memory -** for a UTF conversion required for comparison. The error is stored -** in the pParse structure. -*/ -static int whereEqualScanEst( - Parse *pParse, /* Parsing & code generating context */ - WhereLoopBuilder *pBuilder, - Expr *pExpr, /* Expression for VALUE in the x=VALUE constraint */ - tRowcnt *pnRow /* Write the revised row estimate here */ -){ - Index *p = pBuilder->pNew->u.btree.pIndex; - int nEq = pBuilder->pNew->u.btree.nEq; - UnpackedRecord *pRec = pBuilder->pRec; - u8 aff; /* Column affinity */ - int rc; /* Subfunction return code */ - tRowcnt a[2]; /* Statistics */ - int bOk; - - assert( nEq>=1 ); - assert( nEq<=p->nColumn ); - assert( p->aSample!=0 ); - assert( p->nSample>0 ); - assert( pBuilder->nRecValidNULL" if x is not an INTEGER PRIMARY KEY. So construct a + ** virtual term of that form. + ** + ** Note that the virtual term must be tagged with TERM_VNULL. + */ + if( pExpr->op==TK_NOTNULL + && pExpr->pLeft->op==TK_COLUMN + && pExpr->pLeft->iColumn>=0 + && OptimizationEnabled(db, SQLITE_Stat34) + ){ + Expr *pNewExpr; + Expr *pLeft = pExpr->pLeft; + int idxNew; + WhereTerm *pNewTerm; - /* If values are not available for all fields of the index to the left - ** of this one, no estimate can be made. Return SQLITE_NOTFOUND. */ - if( pBuilder->nRecValid<(nEq-1) ){ - return SQLITE_NOTFOUND; - } + pNewExpr = sqlite3PExpr(pParse, TK_GT, + sqlite3ExprDup(db, pLeft, 0), + sqlite3PExpr(pParse, TK_NULL, 0, 0, 0), 0); - /* This is an optimization only. The call to sqlite3Stat4ProbeSetValue() - ** below would return the same value. */ - if( nEq>=p->nColumn ){ - *pnRow = 1; - return SQLITE_OK; + idxNew = whereClauseInsert(pWC, pNewExpr, + TERM_VIRTUAL|TERM_DYNAMIC|TERM_VNULL); + if( idxNew ){ + pNewTerm = &pWC->a[idxNew]; + pNewTerm->prereqRight = 0; + pNewTerm->leftCursor = pLeft->iTable; + pNewTerm->u.leftColumn = pLeft->iColumn; + pNewTerm->eOperator = WO_GT; + markTermAsChild(pWC, idxNew, idxTerm); + pTerm = &pWC->a[idxTerm]; + pTerm->wtFlags |= TERM_COPIED; + pNewTerm->prereqAll = pTerm->prereqAll; + } } +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ - aff = p->pTable->aCol[p->aiColumn[nEq-1]].affinity; - rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq-1, &bOk); - pBuilder->pRec = pRec; - if( rc!=SQLITE_OK ) return rc; - if( bOk==0 ) return SQLITE_NOTFOUND; - pBuilder->nRecValid = nEq; - - whereKeyStats(pParse, p, pRec, 0, a); - WHERETRACE(0x10,("equality scan regions: %d\n", (int)a[1])); - *pnRow = a[1]; - - return rc; + /* Prevent ON clause terms of a LEFT JOIN from being used to drive + ** an index for tables to the left of the join. + */ + pTerm->prereqRight |= extraRight; } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +/*************************************************************************** +** Routines with file scope above. Interface to the rest of the where.c +** subsystem follows. +***************************************************************************/ + /* -** Estimate the number of rows that will be returned based on -** an IN constraint where the right-hand side of the IN operator -** is a list of values. Example: +** This routine identifies subexpressions in the WHERE clause where +** each subexpression is separated by the AND operator or some other +** operator specified in the op parameter. The WhereClause structure +** is filled with pointers to subexpressions. For example: ** -** WHERE x IN (1,2,3,4) +** WHERE a=='hello' AND coalesce(b,11)<10 AND (c+12!=d OR c==22) +** \________/ \_______________/ \________________/ +** slot[0] slot[1] slot[2] ** -** Write the estimated row count into *pnRow and return SQLITE_OK. -** If unable to make an estimate, leave *pnRow unchanged and return -** non-zero. +** The original WHERE clause in pExpr is unaltered. All this routine +** does is make slot[] entries point to substructure within pExpr. ** -** This routine can fail if it is unable to load a collating sequence -** required for string comparison, or if unable to allocate memory -** for a UTF conversion required for comparison. The error is stored -** in the pParse structure. +** In the previous sentence and in the diagram, "slot[]" refers to +** the WhereClause.a[] array. The slot[] array grows as needed to contain +** all terms of the WHERE clause. */ -static int whereInScanEst( - Parse *pParse, /* Parsing & code generating context */ - WhereLoopBuilder *pBuilder, - ExprList *pList, /* The value list on the RHS of "x IN (v1,v2,v3,...)" */ - tRowcnt *pnRow /* Write the revised row estimate here */ -){ - Index *p = pBuilder->pNew->u.btree.pIndex; - i64 nRow0 = sqlite3LogEstToInt(p->aiRowLogEst[0]); - int nRecValid = pBuilder->nRecValid; - int rc = SQLITE_OK; /* Subfunction return code */ - tRowcnt nEst; /* Number of rows for a single term */ - tRowcnt nRowEst = 0; /* New estimate of the number of rows */ - int i; /* Loop counter */ - - assert( p->aSample!=0 ); - for(i=0; rc==SQLITE_OK && inExpr; i++){ - nEst = nRow0; - rc = whereEqualScanEst(pParse, pBuilder, pList->a[i].pExpr, &nEst); - nRowEst += nEst; - pBuilder->nRecValid = nRecValid; +SQLITE_PRIVATE void sqlite3WhereSplit(WhereClause *pWC, Expr *pExpr, u8 op){ + Expr *pE2 = sqlite3ExprSkipCollate(pExpr); + pWC->op = op; + if( pE2==0 ) return; + if( pE2->op!=op ){ + whereClauseInsert(pWC, pExpr, 0); + }else{ + sqlite3WhereSplit(pWC, pE2->pLeft, op); + sqlite3WhereSplit(pWC, pE2->pRight, op); } +} - if( rc==SQLITE_OK ){ - if( nRowEst > nRow0 ) nRowEst = nRow0; - *pnRow = nRowEst; - WHERETRACE(0x10,("IN row estimate: est=%d\n", nRowEst)); - } - assert( pBuilder->nRecValid==nRecValid ); - return rc; +/* +** Initialize a preallocated WhereClause structure. +*/ +SQLITE_PRIVATE void sqlite3WhereClauseInit( + WhereClause *pWC, /* The WhereClause to be initialized */ + WhereInfo *pWInfo /* The WHERE processing context */ +){ + pWC->pWInfo = pWInfo; + pWC->pOuter = 0; + pWC->nTerm = 0; + pWC->nSlot = ArraySize(pWC->aStatic); + pWC->a = pWC->aStatic; } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ /* -** Disable a term in the WHERE clause. Except, do not disable the term -** if it controls a LEFT OUTER JOIN and it did not originate in the ON -** or USING clause of that join. -** -** Consider the term t2.z='ok' in the following queries: -** -** (1) SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.x WHERE t2.z='ok' -** (2) SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.x AND t2.z='ok' -** (3) SELECT * FROM t1, t2 WHERE t1.a=t2.x AND t2.z='ok' -** -** The t2.z='ok' is disabled in the in (2) because it originates -** in the ON clause. The term is disabled in (3) because it is not part -** of a LEFT OUTER JOIN. In (1), the term is not disabled. -** -** Disabling a term causes that term to not be tested in the inner loop -** of the join. Disabling is an optimization. When terms are satisfied -** by indices, we disable them to prevent redundant tests in the inner -** loop. We would get the correct results if nothing were ever disabled, -** but joins might run a little slower. The trick is to disable as much -** as we can without disabling too much. If we disabled in (1), we'd get -** the wrong answer. See ticket #813. -** -** If all the children of a term are disabled, then that term is also -** automatically disabled. In this way, terms get disabled if derived -** virtual terms are tested first. For example: -** -** x GLOB 'abc*' AND x>='abc' AND x<'acd' -** \___________/ \______/ \_____/ -** parent child1 child2 -** -** Only the parent term was in the original WHERE clause. The child1 -** and child2 terms were added by the LIKE optimization. If both of -** the virtual child terms are valid, then testing of the parent can be -** skipped. -** -** Usually the parent term is marked as TERM_CODED. But if the parent -** term was originally TERM_LIKE, then the parent gets TERM_LIKECOND instead. -** The TERM_LIKECOND marking indicates that the term should be coded inside -** a conditional such that is only evaluated on the second pass of a -** LIKE-optimization loop, when scanning BLOBs instead of strings. +** Deallocate a WhereClause structure. The WhereClause structure +** itself is not freed. This routine is the inverse of sqlite3WhereClauseInit(). */ -static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){ - int nLoop = 0; - while( pTerm - && (pTerm->wtFlags & TERM_CODED)==0 - && (pLevel->iLeftJoin==0 || ExprHasProperty(pTerm->pExpr, EP_FromJoin)) - && (pLevel->notReady & pTerm->prereqAll)==0 - ){ - if( nLoop && (pTerm->wtFlags & TERM_LIKE)!=0 ){ - pTerm->wtFlags |= TERM_LIKECOND; - }else{ - pTerm->wtFlags |= TERM_CODED; +SQLITE_PRIVATE void sqlite3WhereClauseClear(WhereClause *pWC){ + int i; + WhereTerm *a; + sqlite3 *db = pWC->pWInfo->pParse->db; + for(i=pWC->nTerm-1, a=pWC->a; i>=0; i--, a++){ + if( a->wtFlags & TERM_DYNAMIC ){ + sqlite3ExprDelete(db, a->pExpr); } - if( pTerm->iParent<0 ) break; - pTerm = &pTerm->pWC->a[pTerm->iParent]; - pTerm->nChild--; - if( pTerm->nChild!=0 ) break; - nLoop++; + if( a->wtFlags & TERM_ORINFO ){ + whereOrInfoDelete(db, a->u.pOrInfo); + }else if( a->wtFlags & TERM_ANDINFO ){ + whereAndInfoDelete(db, a->u.pAndInfo); + } + } + if( pWC->a!=pWC->aStatic ){ + sqlite3DbFree(db, pWC->a); } } + /* -** Code an OP_Affinity opcode to apply the column affinity string zAff -** to the n registers starting at base. -** -** As an optimization, SQLITE_AFF_NONE entries (which are no-ops) at the -** beginning and end of zAff are ignored. If all entries in zAff are -** SQLITE_AFF_NONE, then no code gets generated. -** -** This routine makes its own copy of zAff so that the caller is free -** to modify zAff after this routine returns. +** These routines walk (recursively) an expression tree and generate +** a bitmask indicating which tables are used in that expression +** tree. */ -static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){ - Vdbe *v = pParse->pVdbe; - if( zAff==0 ){ - assert( pParse->db->mallocFailed ); - return; - } - assert( v!=0 ); - - /* Adjust base and n to skip over SQLITE_AFF_NONE entries at the beginning - ** and end of the affinity string. - */ - while( n>0 && zAff[0]==SQLITE_AFF_NONE ){ - n--; - base++; - zAff++; +SQLITE_PRIVATE Bitmask sqlite3WhereExprUsage(WhereMaskSet *pMaskSet, Expr *p){ + Bitmask mask = 0; + if( p==0 ) return 0; + if( p->op==TK_COLUMN ){ + mask = sqlite3WhereGetMask(pMaskSet, p->iTable); + return mask; } - while( n>1 && zAff[n-1]==SQLITE_AFF_NONE ){ - n--; + mask = sqlite3WhereExprUsage(pMaskSet, p->pRight); + mask |= sqlite3WhereExprUsage(pMaskSet, p->pLeft); + if( ExprHasProperty(p, EP_xIsSelect) ){ + mask |= exprSelectUsage(pMaskSet, p->x.pSelect); + }else{ + mask |= sqlite3WhereExprListUsage(pMaskSet, p->x.pList); } - - /* Code the OP_Affinity opcode if there is anything left to do. */ - if( n>0 ){ - sqlite3VdbeAddOp2(v, OP_Affinity, base, n); - sqlite3VdbeChangeP4(v, -1, zAff, n); - sqlite3ExprCacheAffinityChange(pParse, base, n); + return mask; +} +SQLITE_PRIVATE Bitmask sqlite3WhereExprListUsage(WhereMaskSet *pMaskSet, ExprList *pList){ + int i; + Bitmask mask = 0; + if( pList ){ + for(i=0; inExpr; i++){ + mask |= sqlite3WhereExprUsage(pMaskSet, pList->a[i].pExpr); + } } + return mask; } /* -** Generate code for a single equality term of the WHERE clause. An equality -** term can be either X=expr or X IN (...). pTerm is the term to be -** coded. -** -** The current value for the constraint is left in register iReg. +** Call exprAnalyze on all terms in a WHERE clause. ** -** For a constraint of the form X=expr, the expression is evaluated and its -** result is left on the stack. For constraints of the form X IN (...) -** this routine sets up a loop that will iterate over all values of X. +** Note that exprAnalyze() might add new virtual terms onto the +** end of the WHERE clause. We do not want to analyze these new +** virtual terms, so start analyzing at the end and work forward +** so that the added virtual terms are never processed. */ -static int codeEqualityTerm( - Parse *pParse, /* The parsing context */ - WhereTerm *pTerm, /* The term of the WHERE clause to be coded */ - WhereLevel *pLevel, /* The level of the FROM clause we are working on */ - int iEq, /* Index of the equality term within this level */ - int bRev, /* True for reverse-order IN operations */ - int iTarget /* Attempt to leave results in this register */ +SQLITE_PRIVATE void sqlite3WhereExprAnalyze( + SrcList *pTabList, /* the FROM clause */ + WhereClause *pWC /* the WHERE clause to be analyzed */ ){ - Expr *pX = pTerm->pExpr; - Vdbe *v = pParse->pVdbe; - int iReg; /* Register holding results */ - - assert( iTarget>0 ); - if( pX->op==TK_EQ ){ - iReg = sqlite3ExprCodeTarget(pParse, pX->pRight, iTarget); - }else if( pX->op==TK_ISNULL ){ - iReg = iTarget; - sqlite3VdbeAddOp2(v, OP_Null, 0, iReg); -#ifndef SQLITE_OMIT_SUBQUERY - }else{ - int eType; - int iTab; - struct InLoop *pIn; - WhereLoop *pLoop = pLevel->pWLoop; + int i; + for(i=pWC->nTerm-1; i>=0; i--){ + exprAnalyze(pTabList, pWC, i); + } +} - if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 - && pLoop->u.btree.pIndex!=0 - && pLoop->u.btree.pIndex->aSortOrder[iEq] - ){ - testcase( iEq==0 ); - testcase( bRev ); - bRev = !bRev; - } - assert( pX->op==TK_IN ); - iReg = iTarget; - eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0); - if( eType==IN_INDEX_INDEX_DESC ){ - testcase( bRev ); - bRev = !bRev; - } - iTab = pX->iTable; - sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0); - VdbeCoverageIf(v, bRev); - VdbeCoverageIf(v, !bRev); - assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 ); - pLoop->wsFlags |= WHERE_IN_ABLE; - if( pLevel->u.in.nIn==0 ){ - pLevel->addrNxt = sqlite3VdbeMakeLabel(v); - } - pLevel->u.in.nIn++; - pLevel->u.in.aInLoop = - sqlite3DbReallocOrFree(pParse->db, pLevel->u.in.aInLoop, - sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn); - pIn = pLevel->u.in.aInLoop; - if( pIn ){ - pIn += pLevel->u.in.nIn - 1; - pIn->iCur = iTab; - if( eType==IN_INDEX_ROWID ){ - pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iReg); - }else{ - pIn->addrInTop = sqlite3VdbeAddOp3(v, OP_Column, iTab, 0, iReg); - } - pIn->eEndLoopOp = bRev ? OP_PrevIfOpen : OP_NextIfOpen; - sqlite3VdbeAddOp1(v, OP_IsNull, iReg); VdbeCoverage(v); - }else{ - pLevel->u.in.nIn = 0; +/* +** For table-valued-functions, transform the function arguments into +** new WHERE clause terms. +** +** Each function argument translates into an equality constraint against +** a HIDDEN column in the table. +*/ +SQLITE_PRIVATE void sqlite3WhereTabFuncArgs( + Parse *pParse, /* Parsing context */ + struct SrcList_item *pItem, /* The FROM clause term to process */ + WhereClause *pWC /* Xfer function arguments to here */ +){ + Table *pTab; + int j, k; + ExprList *pArgs; + Expr *pColRef; + Expr *pTerm; + if( pItem->fg.isTabFunc==0 ) return; + pTab = pItem->pTab; + assert( pTab!=0 ); + pArgs = pItem->u1.pFuncArg; + assert( pArgs!=0 ); + for(j=k=0; jnExpr; j++){ + while( knCol && (pTab->aCol[k].colFlags & COLFLAG_HIDDEN)==0 ){ k++; } + if( k>=pTab->nCol ){ + sqlite3ErrorMsg(pParse, "too many arguments on %s() - max %d", + pTab->zName, j); + return; } -#endif + pColRef = sqlite3PExpr(pParse, TK_COLUMN, 0, 0, 0); + if( pColRef==0 ) return; + pColRef->iTable = pItem->iCursor; + pColRef->iColumn = k++; + pColRef->pTab = pTab; + pTerm = sqlite3PExpr(pParse, TK_EQ, pColRef, + sqlite3ExprDup(pParse->db, pArgs->a[j].pExpr, 0), 0); + whereClauseInsert(pWC, pTerm, TERM_DYNAMIC); } - disableTerm(pLevel, pTerm); - return iReg; } +/************** End of whereexpr.c *******************************************/ +/************** Begin file where.c *******************************************/ /* -** Generate code that will evaluate all == and IN constraints for an -** index scan. -** -** For example, consider table t1(a,b,c,d,e,f) with index i1(a,b,c). -** Suppose the WHERE clause is this: a==5 AND b IN (1,2,3) AND c>5 AND c<10 -** The index has as many as three equality constraints, but in this -** example, the third "c" value is an inequality. So only two -** constraints are coded. This routine will generate code to evaluate -** a==5 and b IN (1,2,3). The current values for a and b will be stored -** in consecutive registers and the index of the first register is returned. -** -** In the example above nEq==2. But this subroutine works for any value -** of nEq including 0. If nEq==0, this routine is nearly a no-op. -** The only thing it does is allocate the pLevel->iMem memory cell and -** compute the affinity string. -** -** The nExtraReg parameter is 0 or 1. It is 0 if all WHERE clause constraints -** are == or IN and are covered by the nEq. nExtraReg is 1 if there is -** an inequality constraint (such as the "c>=5 AND c<10" in the example) that -** occurs after the nEq quality constraints. -** -** This routine allocates a range of nEq+nExtraReg memory cells and returns -** the index of the first memory cell in that range. The code that -** calls this routine will use that memory range to store keys for -** start and termination conditions of the loop. -** key value of the loop. If one or more IN operators appear, then -** this routine allocates an additional nEq memory cells for internal -** use. +** 2001 September 15 ** -** Before returning, *pzAff is set to point to a buffer containing a -** copy of the column affinity string of the index allocated using -** sqlite3DbMalloc(). Except, entries in the copy of the string associated -** with equality constraints that use NONE affinity are set to -** SQLITE_AFF_NONE. This is to deal with SQL such as the following: +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** CREATE TABLE t1(a TEXT PRIMARY KEY, b); -** SELECT ... FROM t1 AS t2, t1 WHERE t1.a = t2.b; +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** -** In the example above, the index on t1(a) has TEXT affinity. But since -** the right hand side of the equality constraint (t2.b) has NONE affinity, -** no conversion should be attempted before using a t2.b value as part of -** a key to search the index. Hence the first byte in the returned affinity -** string in this example would be set to SQLITE_AFF_NONE. +************************************************************************* +** This module contains C code that generates VDBE code used to process +** the WHERE clause of SQL statements. This module is responsible for +** generating the code that loops through a table looking for applicable +** rows. Indices are selected and used to speed the search when doing +** so is applicable. Because this module is responsible for selecting +** indices, you might also think of this module as the "query optimizer". */ -static int codeAllEqualityTerms( - Parse *pParse, /* Parsing context */ - WhereLevel *pLevel, /* Which nested loop of the FROM we are coding */ - int bRev, /* Reverse the order of IN operators */ - int nExtraReg, /* Number of extra registers to allocate */ - char **pzAff /* OUT: Set to point to affinity string */ -){ - u16 nEq; /* The number of == or IN constraints to code */ - u16 nSkip; /* Number of left-most columns to skip */ - Vdbe *v = pParse->pVdbe; /* The vm under construction */ - Index *pIdx; /* The index being used for this loop */ - WhereTerm *pTerm; /* A single constraint term */ - WhereLoop *pLoop; /* The WhereLoop object */ - int j; /* Loop counter */ - int regBase; /* Base register */ - int nReg; /* Number of registers to allocate */ - char *zAff; /* Affinity string to return */ +/* #include "sqliteInt.h" */ +/* #include "whereInt.h" */ - /* This module is only called on query plans that use an index. */ - pLoop = pLevel->pWLoop; - assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); - nEq = pLoop->u.btree.nEq; - nSkip = pLoop->nSkip; - pIdx = pLoop->u.btree.pIndex; - assert( pIdx!=0 ); +/* Forward declaration of methods */ +static int whereLoopResize(sqlite3*, WhereLoop*, int); - /* Figure out how many memory cells we will need then allocate them. - */ - regBase = pParse->nMem + 1; - nReg = pLoop->u.btree.nEq + nExtraReg; - pParse->nMem += nReg; +/* Test variable that can be set to enable WHERE tracing */ +#if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) +/***/ int sqlite3WhereTrace = 0; +#endif - zAff = sqlite3DbStrDup(pParse->db, sqlite3IndexAffinityStr(v, pIdx)); - if( !zAff ){ - pParse->db->mallocFailed = 1; - } - if( nSkip ){ - int iIdxCur = pLevel->iIdxCur; - sqlite3VdbeAddOp1(v, (bRev?OP_Last:OP_Rewind), iIdxCur); - VdbeCoverageIf(v, bRev==0); - VdbeCoverageIf(v, bRev!=0); - VdbeComment((v, "begin skip-scan on %s", pIdx->zName)); - j = sqlite3VdbeAddOp0(v, OP_Goto); - pLevel->addrSkip = sqlite3VdbeAddOp4Int(v, (bRev?OP_SeekLT:OP_SeekGT), - iIdxCur, 0, regBase, nSkip); - VdbeCoverageIf(v, bRev==0); - VdbeCoverageIf(v, bRev!=0); - sqlite3VdbeJumpHere(v, j); - for(j=0; jaiColumn[j]>=0 ); - VdbeComment((v, "%s", pIdx->pTable->aCol[pIdx->aiColumn[j]].zName)); - } - } +/* +** Return the estimated number of output rows from a WHERE clause +*/ +SQLITE_PRIVATE u64 sqlite3WhereOutputRowCount(WhereInfo *pWInfo){ + return sqlite3LogEstToInt(pWInfo->nRowOut); +} - /* Evaluate the equality constraints - */ - assert( zAff==0 || (int)strlen(zAff)>=nEq ); - for(j=nSkip; jaLTerm[j]; - assert( pTerm!=0 ); - /* The following testcase is true for indices with redundant columns. - ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ - testcase( (pTerm->wtFlags & TERM_CODED)!=0 ); - testcase( pTerm->wtFlags & TERM_VIRTUAL ); - r1 = codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, regBase+j); - if( r1!=regBase+j ){ - if( nReg==1 ){ - sqlite3ReleaseTempReg(pParse, regBase); - regBase = r1; - }else{ - sqlite3VdbeAddOp2(v, OP_SCopy, r1, regBase+j); - } - } - testcase( pTerm->eOperator & WO_ISNULL ); - testcase( pTerm->eOperator & WO_IN ); - if( (pTerm->eOperator & (WO_ISNULL|WO_IN))==0 ){ - Expr *pRight = pTerm->pExpr->pRight; - if( sqlite3ExprCanBeNull(pRight) ){ - sqlite3VdbeAddOp2(v, OP_IsNull, regBase+j, pLevel->addrBrk); - VdbeCoverage(v); - } - if( zAff ){ - if( sqlite3CompareAffinity(pRight, zAff[j])==SQLITE_AFF_NONE ){ - zAff[j] = SQLITE_AFF_NONE; - } - if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[j]) ){ - zAff[j] = SQLITE_AFF_NONE; - } - } - } - } - *pzAff = zAff; - return regBase; +/* +** Return one of the WHERE_DISTINCT_xxxxx values to indicate how this +** WHERE clause returns outputs for DISTINCT processing. +*/ +SQLITE_PRIVATE int sqlite3WhereIsDistinct(WhereInfo *pWInfo){ + return pWInfo->eDistinct; } -#ifndef SQLITE_OMIT_EXPLAIN /* -** This routine is a helper for explainIndexRange() below -** -** pStr holds the text of an expression that we are building up one term -** at a time. This routine adds a new term to the end of the expression. -** Terms are separated by AND so add the "AND" text for second and subsequent -** terms only. +** Return TRUE if the WHERE clause returns rows in ORDER BY order. +** Return FALSE if the output needs to be sorted. */ -static void explainAppendTerm( - StrAccum *pStr, /* The text expression being built */ - int iTerm, /* Index of this term. First is zero */ - const char *zColumn, /* Name of the column */ - const char *zOp /* Name of the operator */ -){ - if( iTerm ) sqlite3StrAccumAppend(pStr, " AND ", 5); - sqlite3StrAccumAppendAll(pStr, zColumn); - sqlite3StrAccumAppend(pStr, zOp, 1); - sqlite3StrAccumAppend(pStr, "?", 1); +SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ + return pWInfo->nOBSat; } /* -** Argument pLevel describes a strategy for scanning table pTab. This -** function appends text to pStr that describes the subset of table -** rows scanned by the strategy in the form of an SQL expression. -** -** For example, if the query: -** -** SELECT * FROM t1 WHERE a=1 AND b>2; +** Return the VDBE address or label to jump to in order to continue +** immediately with the next row of a WHERE clause. +*/ +SQLITE_PRIVATE int sqlite3WhereContinueLabel(WhereInfo *pWInfo){ + assert( pWInfo->iContinue!=0 ); + return pWInfo->iContinue; +} + +/* +** Return the VDBE address or label to jump to in order to break +** out of a WHERE loop. +*/ +SQLITE_PRIVATE int sqlite3WhereBreakLabel(WhereInfo *pWInfo){ + return pWInfo->iBreak; +} + +/* +** Return ONEPASS_OFF (0) if an UPDATE or DELETE statement is unable to +** operate directly on the rowis returned by a WHERE clause. Return +** ONEPASS_SINGLE (1) if the statement can operation directly because only +** a single row is to be changed. Return ONEPASS_MULTI (2) if the one-pass +** optimization can be used on multiple ** -** is run and there is an index on (a, b), then this function returns a -** string similar to: +** If the ONEPASS optimization is used (if this routine returns true) +** then also write the indices of open cursors used by ONEPASS +** into aiCur[0] and aiCur[1]. iaCur[0] gets the cursor of the data +** table and iaCur[1] gets the cursor used by an auxiliary index. +** Either value may be -1, indicating that cursor is not used. +** Any cursors returned will have been opened for writing. ** -** "a=? AND b>?" +** aiCur[0] and aiCur[1] both get -1 if the where-clause logic is +** unable to use the ONEPASS optimization. */ -static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop, Table *pTab){ - Index *pIndex = pLoop->u.btree.pIndex; - u16 nEq = pLoop->u.btree.nEq; - u16 nSkip = pLoop->nSkip; - int i, j; - Column *aCol = pTab->aCol; - i16 *aiColumn = pIndex->aiColumn; - - if( nEq==0 && (pLoop->wsFlags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ) return; - sqlite3StrAccumAppend(pStr, " (", 2); - for(i=0; i=nSkip ){ - explainAppendTerm(pStr, i, z, "="); - }else{ - if( i ) sqlite3StrAccumAppend(pStr, " AND ", 5); - sqlite3XPrintf(pStr, 0, "ANY(%s)", z); - } +SQLITE_PRIVATE int sqlite3WhereOkOnePass(WhereInfo *pWInfo, int *aiCur){ + memcpy(aiCur, pWInfo->aiCurOnePass, sizeof(int)*2); +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace && pWInfo->eOnePass!=ONEPASS_OFF ){ + sqlite3DebugPrintf("%s cursors: %d %d\n", + pWInfo->eOnePass==ONEPASS_SINGLE ? "ONEPASS_SINGLE" : "ONEPASS_MULTI", + aiCur[0], aiCur[1]); } +#endif + return pWInfo->eOnePass; +} - j = i; - if( pLoop->wsFlags&WHERE_BTM_LIMIT ){ - char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName; - explainAppendTerm(pStr, i++, z, ">"); - } - if( pLoop->wsFlags&WHERE_TOP_LIMIT ){ - char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName; - explainAppendTerm(pStr, i, z, "<"); - } - sqlite3StrAccumAppend(pStr, ")", 1); +/* +** Move the content of pSrc into pDest +*/ +static void whereOrMove(WhereOrSet *pDest, WhereOrSet *pSrc){ + pDest->n = pSrc->n; + memcpy(pDest->a, pSrc->a, pDest->n*sizeof(pDest->a[0])); } /* -** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN -** command, or if either SQLITE_DEBUG or SQLITE_ENABLE_STMT_SCANSTATUS was -** defined at compile-time. If it is not a no-op, a single OP_Explain opcode -** is added to the output to describe the table scan strategy in pLevel. +** Try to insert a new prerequisite/cost entry into the WhereOrSet pSet. ** -** If an OP_Explain opcode is added to the VM, its address is returned. -** Otherwise, if no OP_Explain is coded, zero is returned. +** The new entry might overwrite an existing entry, or it might be +** appended, or it might be discarded. Do whatever is the right thing +** so that pSet keeps the N_OR_COST best entries seen so far. */ -static int explainOneScan( - Parse *pParse, /* Parse context */ - SrcList *pTabList, /* Table list this loop refers to */ - WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ - int iLevel, /* Value for "level" column of output */ - int iFrom, /* Value for "from" column of output */ - u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ +static int whereOrInsert( + WhereOrSet *pSet, /* The WhereOrSet to be updated */ + Bitmask prereq, /* Prerequisites of the new entry */ + LogEst rRun, /* Run-cost of the new entry */ + LogEst nOut /* Number of outputs for the new entry */ ){ - int ret = 0; -#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) - if( pParse->explain==2 ) -#endif - { - struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom]; - Vdbe *v = pParse->pVdbe; /* VM being constructed */ - sqlite3 *db = pParse->db; /* Database handle */ - int iId = pParse->iSelectId; /* Select id (left-most output column) */ - int isSearch; /* True for a SEARCH. False for SCAN. */ - WhereLoop *pLoop; /* The controlling WhereLoop object */ - u32 flags; /* Flags that describe this loop */ - char *zMsg; /* Text to add to EQP output */ - StrAccum str; /* EQP output string */ - char zBuf[100]; /* Initial space for EQP output string */ - - pLoop = pLevel->pWLoop; - flags = pLoop->wsFlags; - if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return 0; - - isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 - || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0)) - || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); - - sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); - sqlite3StrAccumAppendAll(&str, isSearch ? "SEARCH" : "SCAN"); - if( pItem->pSelect ){ - sqlite3XPrintf(&str, 0, " SUBQUERY %d", pItem->iSelectId); - }else{ - sqlite3XPrintf(&str, 0, " TABLE %s", pItem->zName); - } - - if( pItem->zAlias ){ - sqlite3XPrintf(&str, 0, " AS %s", pItem->zAlias); + u16 i; + WhereOrCost *p; + for(i=pSet->n, p=pSet->a; i>0; i--, p++){ + if( rRun<=p->rRun && (prereq & p->prereq)==prereq ){ + goto whereOrInsert_done; } - if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ - const char *zFmt = 0; - Index *pIdx; - - assert( pLoop->u.btree.pIndex!=0 ); - pIdx = pLoop->u.btree.pIndex; - assert( !(flags&WHERE_AUTO_INDEX) || (flags&WHERE_IDX_ONLY) ); - if( !HasRowid(pItem->pTab) && IsPrimaryKeyIndex(pIdx) ){ - if( isSearch ){ - zFmt = "PRIMARY KEY"; - } - }else if( flags & WHERE_PARTIALIDX ){ - zFmt = "AUTOMATIC PARTIAL COVERING INDEX"; - }else if( flags & WHERE_AUTO_INDEX ){ - zFmt = "AUTOMATIC COVERING INDEX"; - }else if( flags & WHERE_IDX_ONLY ){ - zFmt = "COVERING INDEX %s"; - }else{ - zFmt = "INDEX %s"; - } - if( zFmt ){ - sqlite3StrAccumAppend(&str, " USING ", 7); - sqlite3XPrintf(&str, 0, zFmt, pIdx->zName); - explainIndexRange(&str, pLoop, pItem->pTab); - } - }else if( (flags & WHERE_IPK)!=0 && (flags & WHERE_CONSTRAINT)!=0 ){ - const char *zRange; - if( flags&(WHERE_COLUMN_EQ|WHERE_COLUMN_IN) ){ - zRange = "(rowid=?)"; - }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){ - zRange = "(rowid>? AND rowid?)"; - }else{ - assert( flags&WHERE_TOP_LIMIT); - zRange = "(rowidrRun<=rRun && (p->prereq & prereq)==p->prereq ){ + return 0; } -#ifndef SQLITE_OMIT_VIRTUALTABLE - else if( (flags & WHERE_VIRTUALTABLE)!=0 ){ - sqlite3XPrintf(&str, 0, " VIRTUAL TABLE INDEX %d:%s", - pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr); + } + if( pSet->na[pSet->n++]; + p->nOut = nOut; + }else{ + p = pSet->a; + for(i=1; in; i++){ + if( p->rRun>pSet->a[i].rRun ) p = pSet->a + i; } -#endif -#ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS - if( pLoop->nOut>=10 ){ - sqlite3XPrintf(&str, 0, " (~%llu rows)", sqlite3LogEstToInt(pLoop->nOut)); - }else{ - sqlite3StrAccumAppend(&str, " (~1 row)", 9); + if( p->rRun<=rRun ) return 0; + } +whereOrInsert_done: + p->prereq = prereq; + p->rRun = rRun; + if( p->nOut>nOut ) p->nOut = nOut; + return 1; +} + +/* +** Return the bitmask for the given cursor number. Return 0 if +** iCursor is not in the set. +*/ +SQLITE_PRIVATE Bitmask sqlite3WhereGetMask(WhereMaskSet *pMaskSet, int iCursor){ + int i; + assert( pMaskSet->n<=(int)sizeof(Bitmask)*8 ); + for(i=0; in; i++){ + if( pMaskSet->ix[i]==iCursor ){ + return MASKBIT(i); } -#endif - zMsg = sqlite3StrAccumFinish(&str); - ret = sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg,P4_DYNAMIC); } - return ret; + return 0; } -#else -# define explainOneScan(u,v,w,x,y,z) 0 -#endif /* SQLITE_OMIT_EXPLAIN */ -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS /* -** Configure the VM passed as the first argument with an -** sqlite3_stmt_scanstatus() entry corresponding to the scan used to -** implement level pLvl. Argument pSrclist is a pointer to the FROM -** clause that the scan reads data from. +** Create a new mask for cursor iCursor. ** -** If argument addrExplain is not 0, it must be the address of an -** OP_Explain instruction that describes the same loop. +** There is one cursor per table in the FROM clause. The number of +** tables in the FROM clause is limited by a test early in the +** sqlite3WhereBegin() routine. So we know that the pMaskSet->ix[] +** array will never overflow. */ -static void addScanStatus( - Vdbe *v, /* Vdbe to add scanstatus entry to */ - SrcList *pSrclist, /* FROM clause pLvl reads data from */ - WhereLevel *pLvl, /* Level to add scanstatus() entry for */ - int addrExplain /* Address of OP_Explain (or 0) */ +static void createMask(WhereMaskSet *pMaskSet, int iCursor){ + assert( pMaskSet->n < ArraySize(pMaskSet->ix) ); + pMaskSet->ix[pMaskSet->n++] = iCursor; +} + +/* +** Advance to the next WhereTerm that matches according to the criteria +** established when the pScan object was initialized by whereScanInit(). +** Return NULL if there are no more matching WhereTerms. +*/ +static WhereTerm *whereScanNext(WhereScan *pScan){ + int iCur; /* The cursor on the LHS of the term */ + i16 iColumn; /* The column on the LHS of the term. -1 for IPK */ + Expr *pX; /* An expression being tested */ + WhereClause *pWC; /* Shorthand for pScan->pWC */ + WhereTerm *pTerm; /* The term being tested */ + int k = pScan->k; /* Where to start scanning */ + + while( pScan->iEquiv<=pScan->nEquiv ){ + iCur = pScan->aiCur[pScan->iEquiv-1]; + iColumn = pScan->aiColumn[pScan->iEquiv-1]; + if( iColumn==XN_EXPR && pScan->pIdxExpr==0 ) return 0; + while( (pWC = pScan->pWC)!=0 ){ + for(pTerm=pWC->a+k; knTerm; k++, pTerm++){ + if( pTerm->leftCursor==iCur + && pTerm->u.leftColumn==iColumn + && (iColumn!=XN_EXPR + || sqlite3ExprCompare(pTerm->pExpr->pLeft,pScan->pIdxExpr,iCur)==0) + && (pScan->iEquiv<=1 || !ExprHasProperty(pTerm->pExpr, EP_FromJoin)) + ){ + if( (pTerm->eOperator & WO_EQUIV)!=0 + && pScan->nEquivaiCur) + && (pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight))->op==TK_COLUMN + ){ + int j; + for(j=0; jnEquiv; j++){ + if( pScan->aiCur[j]==pX->iTable + && pScan->aiColumn[j]==pX->iColumn ){ + break; + } + } + if( j==pScan->nEquiv ){ + pScan->aiCur[j] = pX->iTable; + pScan->aiColumn[j] = pX->iColumn; + pScan->nEquiv++; + } + } + if( (pTerm->eOperator & pScan->opMask)!=0 ){ + /* Verify the affinity and collating sequence match */ + if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){ + CollSeq *pColl; + Parse *pParse = pWC->pWInfo->pParse; + pX = pTerm->pExpr; + if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ + continue; + } + assert(pX->pLeft); + pColl = sqlite3BinaryCompareCollSeq(pParse, + pX->pLeft, pX->pRight); + if( pColl==0 ) pColl = pParse->db->pDfltColl; + if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){ + continue; + } + } + if( (pTerm->eOperator & (WO_EQ|WO_IS))!=0 + && (pX = pTerm->pExpr->pRight)->op==TK_COLUMN + && pX->iTable==pScan->aiCur[0] + && pX->iColumn==pScan->aiColumn[0] + ){ + testcase( pTerm->eOperator & WO_IS ); + continue; + } + pScan->k = k+1; + return pTerm; + } + } + } + pScan->pWC = pScan->pWC->pOuter; + k = 0; + } + pScan->pWC = pScan->pOrigWC; + k = 0; + pScan->iEquiv++; + } + return 0; +} + +/* +** Initialize a WHERE clause scanner object. Return a pointer to the +** first match. Return NULL if there are no matches. +** +** The scanner will be searching the WHERE clause pWC. It will look +** for terms of the form "X " where X is column iColumn of table +** iCur. The must be one of the operators described by opMask. +** +** If the search is for X and the WHERE clause contains terms of the +** form X=Y then this routine might also return terms of the form +** "Y ". The number of levels of transitivity is limited, +** but is enough to handle most commonly occurring SQL statements. +** +** If X is not the INTEGER PRIMARY KEY then X must be compatible with +** index pIdx. +*/ +static WhereTerm *whereScanInit( + WhereScan *pScan, /* The WhereScan object being initialized */ + WhereClause *pWC, /* The WHERE clause to be scanned */ + int iCur, /* Cursor to scan for */ + int iColumn, /* Column to scan for */ + u32 opMask, /* Operator(s) to scan for */ + Index *pIdx /* Must be compatible with this index */ ){ - const char *zObj = 0; - WhereLoop *pLoop = pLvl->pWLoop; - if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ - zObj = pLoop->u.btree.pIndex->zName; + int j = 0; + + /* memset(pScan, 0, sizeof(*pScan)); */ + pScan->pOrigWC = pWC; + pScan->pWC = pWC; + pScan->pIdxExpr = 0; + if( pIdx ){ + j = iColumn; + iColumn = pIdx->aiColumn[j]; + if( iColumn==XN_EXPR ) pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr; + } + if( pIdx && iColumn>=0 ){ + pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity; + pScan->zCollName = pIdx->azColl[j]; }else{ - zObj = pSrclist->a[pLvl->iFrom].zName; + pScan->idxaff = 0; + pScan->zCollName = 0; } - sqlite3VdbeScanStatus( - v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj - ); + pScan->opMask = opMask; + pScan->k = 0; + pScan->aiCur[0] = iCur; + pScan->aiColumn[0] = iColumn; + pScan->nEquiv = 1; + pScan->iEquiv = 1; + return whereScanNext(pScan); } -#else -# define addScanStatus(a, b, c, d) ((void)d) -#endif /* -** If the most recently coded instruction is a constant range contraint -** that originated from the LIKE optimization, then change the P3 to be -** pLoop->iLikeRepCntr and set P5. +** Search for a term in the WHERE clause that is of the form "X " +** where X is a reference to the iColumn of table iCur and is one of +** the WO_xx operator codes specified by the op parameter. +** Return a pointer to the term. Return 0 if not found. ** -** The LIKE optimization trys to evaluate "x LIKE 'abc%'" as a range -** expression: "x>='ABC' AND x<'abd'". But this requires that the range -** scan loop run twice, once for strings and a second time for BLOBs. -** The OP_String opcodes on the second pass convert the upper and lower -** bound string contants to blobs. This routine makes the necessary changes -** to the OP_String opcodes for that to happen. +** If pIdx!=0 then search for terms matching the iColumn-th column of pIdx +** rather than the iColumn-th column of table iCur. +** +** The term returned might by Y= if there is another constraint in +** the WHERE clause that specifies that X=Y. Any such constraints will be +** identified by the WO_EQUIV bit in the pTerm->eOperator field. The +** aiCur[]/iaColumn[] arrays hold X and all its equivalents. There are 11 +** slots in aiCur[]/aiColumn[] so that means we can look for X plus up to 10 +** other equivalent values. Hence a search for X will return if X=A1 +** and A1=A2 and A2=A3 and ... and A9=A10 and A10=. +** +** If there are multiple terms in the WHERE clause of the form "X " +** then try for the one with no dependencies on - in other words where +** is a constant expression of some kind. Only return entries of +** the form "X Y" where Y is a column in another table if no terms of +** the form "X " exist. If no terms with a constant RHS +** exist, try to return a term that does not use WO_EQUIV. */ -static void whereLikeOptimizationStringFixup( - Vdbe *v, /* prepared statement under construction */ - WhereLevel *pLevel, /* The loop that contains the LIKE operator */ - WhereTerm *pTerm /* The upper or lower bound just coded */ +SQLITE_PRIVATE WhereTerm *sqlite3WhereFindTerm( + WhereClause *pWC, /* The WHERE clause to be searched */ + int iCur, /* Cursor number of LHS */ + int iColumn, /* Column number of LHS */ + Bitmask notReady, /* RHS must not overlap with this mask */ + u32 op, /* Mask of WO_xx values describing operator */ + Index *pIdx /* Must be compatible with this index, if not NULL */ ){ - if( pTerm->wtFlags & TERM_LIKEOPT ){ - VdbeOp *pOp; - assert( pLevel->iLikeRepCntr>0 ); - pOp = sqlite3VdbeGetOp(v, -1); - assert( pOp!=0 ); - assert( pOp->opcode==OP_String8 - || pTerm->pWC->pWInfo->pParse->db->mallocFailed ); - pOp->p3 = pLevel->iLikeRepCntr; - pOp->p5 = 1; + WhereTerm *pResult = 0; + WhereTerm *p; + WhereScan scan; + + p = whereScanInit(&scan, pWC, iCur, iColumn, op, pIdx); + op &= WO_EQ|WO_IS; + while( p ){ + if( (p->prereqRight & notReady)==0 ){ + if( p->prereqRight==0 && (p->eOperator&op)!=0 ){ + testcase( p->eOperator & WO_IS ); + return p; + } + if( pResult==0 ) pResult = p; + } + p = whereScanNext(&scan); } + return pResult; } /* -** Generate code for the start of the iLevel-th loop in the WHERE clause -** implementation described by pWInfo. +** This function searches pList for an entry that matches the iCol-th column +** of index pIdx. +** +** If such an expression is found, its index in pList->a[] is returned. If +** no expression is found, -1 is returned. */ -static Bitmask codeOneLoopStart( - WhereInfo *pWInfo, /* Complete information about the WHERE clause */ - int iLevel, /* Which level of pWInfo->a[] should be coded */ - Bitmask notReady /* Which tables are currently available */ +static int findIndexCol( + Parse *pParse, /* Parse context */ + ExprList *pList, /* Expression list to search */ + int iBase, /* Cursor for table associated with pIdx */ + Index *pIdx, /* Index to match column of */ + int iCol /* Column of index to match */ ){ - int j, k; /* Loop counters */ - int iCur; /* The VDBE cursor for the table */ - int addrNxt; /* Where to jump to continue with the next IN case */ - int omitTable; /* True if we use the index only */ - int bRev; /* True if we need to scan in reverse order */ - WhereLevel *pLevel; /* The where level to be coded */ - WhereLoop *pLoop; /* The WhereLoop object being coded */ - WhereClause *pWC; /* Decomposition of the entire WHERE clause */ - WhereTerm *pTerm; /* A WHERE clause term */ - Parse *pParse; /* Parsing context */ - sqlite3 *db; /* Database connection */ - Vdbe *v; /* The prepared stmt under constructions */ - struct SrcList_item *pTabItem; /* FROM clause term being coded */ - int addrBrk; /* Jump here to break out of the loop */ - int addrCont; /* Jump here to continue with next cycle */ - int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ - int iReleaseReg = 0; /* Temp register to free before returning */ + int i; + const char *zColl = pIdx->azColl[iCol]; - pParse = pWInfo->pParse; - v = pParse->pVdbe; - pWC = &pWInfo->sWC; - db = pParse->db; - pLevel = &pWInfo->a[iLevel]; - pLoop = pLevel->pWLoop; - pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; - iCur = pTabItem->iCursor; - pLevel->notReady = notReady & ~getMask(&pWInfo->sMaskSet, iCur); - bRev = (pWInfo->revMask>>iLevel)&1; - omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0 - && (pWInfo->wctrlFlags & WHERE_FORCE_TABLE)==0; - VdbeModuleComment((v, "Begin WHERE-loop%d: %s",iLevel,pTabItem->pTab->zName)); + for(i=0; inExpr; i++){ + Expr *p = sqlite3ExprSkipCollate(pList->a[i].pExpr); + if( p->op==TK_COLUMN + && p->iColumn==pIdx->aiColumn[iCol] + && p->iTable==iBase + ){ + CollSeq *pColl = sqlite3ExprCollSeq(pParse, pList->a[i].pExpr); + if( pColl && 0==sqlite3StrICmp(pColl->zName, zColl) ){ + return i; + } + } + } - /* Create labels for the "break" and "continue" instructions - ** for the current loop. Jump to addrBrk to break out of a loop. - ** Jump to cont to go immediately to the next iteration of the - ** loop. - ** - ** When there is an IN operator, we also have a "addrNxt" label that - ** means to continue with the next IN value combination. When - ** there are no IN operators in the constraints, the "addrNxt" label - ** is the same as "addrBrk". - */ - addrBrk = pLevel->addrBrk = pLevel->addrNxt = sqlite3VdbeMakeLabel(v); - addrCont = pLevel->addrCont = sqlite3VdbeMakeLabel(v); + return -1; +} + +/* +** Return TRUE if the iCol-th column of index pIdx is NOT NULL +*/ +static int indexColumnNotNull(Index *pIdx, int iCol){ + int j; + assert( pIdx!=0 ); + assert( iCol>=0 && iColnColumn ); + j = pIdx->aiColumn[iCol]; + if( j>=0 ){ + return pIdx->pTable->aCol[j].notNull; + }else if( j==(-1) ){ + return 1; + }else{ + assert( j==(-2) ); + return 0; /* Assume an indexed expression can always yield a NULL */ - /* If this is the right table of a LEFT OUTER JOIN, allocate and - ** initialize a memory cell that records if this table matches any - ** row of the left table of the join. - */ - if( pLevel->iFrom>0 && (pTabItem[0].jointype & JT_LEFT)!=0 ){ - pLevel->iLeftJoin = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin); - VdbeComment((v, "init LEFT JOIN no-match flag")); } +} - /* Special case of a FROM clause subquery implemented as a co-routine */ - if( pTabItem->viaCoroutine ){ - int regYield = pTabItem->regReturn; - sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); - pLevel->p2 = sqlite3VdbeAddOp2(v, OP_Yield, regYield, addrBrk); - VdbeCoverage(v); - VdbeComment((v, "next row of \"%s\"", pTabItem->pTab->zName)); - pLevel->op = OP_Goto; - }else +/* +** Return true if the DISTINCT expression-list passed as the third argument +** is redundant. +** +** A DISTINCT list is redundant if any subset of the columns in the +** DISTINCT list are collectively unique and individually non-null. +*/ +static int isDistinctRedundant( + Parse *pParse, /* Parsing context */ + SrcList *pTabList, /* The FROM clause */ + WhereClause *pWC, /* The WHERE clause */ + ExprList *pDistinct /* The result set that needs to be DISTINCT */ +){ + Table *pTab; + Index *pIdx; + int i; + int iBase; -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ - /* Case 1: The table is a virtual-table. Use the VFilter and VNext - ** to access the data. - */ - int iReg; /* P3 Value for OP_VFilter */ - int addrNotFound; - int nConstraint = pLoop->nLTerm; + /* If there is more than one table or sub-select in the FROM clause of + ** this query, then it will not be possible to show that the DISTINCT + ** clause is redundant. */ + if( pTabList->nSrc!=1 ) return 0; + iBase = pTabList->a[0].iCursor; + pTab = pTabList->a[0].pTab; - sqlite3ExprCachePush(pParse); - iReg = sqlite3GetTempRange(pParse, nConstraint+2); - addrNotFound = pLevel->addrBrk; - for(j=0; jaLTerm[j]; - if( pTerm==0 ) continue; - if( pTerm->eOperator & WO_IN ){ - codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget); - addrNotFound = pLevel->addrNxt; - }else{ - sqlite3ExprCode(pParse, pTerm->pExpr->pRight, iTarget); + /* If any of the expressions is an IPK column on table iBase, then return + ** true. Note: The (p->iTable==iBase) part of this test may be false if the + ** current SELECT is a correlated sub-query. + */ + for(i=0; inExpr; i++){ + Expr *p = sqlite3ExprSkipCollate(pDistinct->a[i].pExpr); + if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1; + } + + /* Loop through all indices on the table, checking each to see if it makes + ** the DISTINCT qualifier redundant. It does so if: + ** + ** 1. The index is itself UNIQUE, and + ** + ** 2. All of the columns in the index are either part of the pDistinct + ** list, or else the WHERE clause contains a term of the form "col=X", + ** where X is a constant value. The collation sequences of the + ** comparison and select-list expressions must match those of the index. + ** + ** 3. All of those index columns for which the WHERE clause does not + ** contain a "col=X" term are subject to a NOT NULL constraint. + */ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( !IsUniqueIndex(pIdx) ) continue; + for(i=0; inKeyCol; i++){ + if( 0==sqlite3WhereFindTerm(pWC, iBase, i, ~(Bitmask)0, WO_EQ, pIdx) ){ + if( findIndexCol(pParse, pDistinct, iBase, pIdx, i)<0 ) break; + if( indexColumnNotNull(pIdx, i)==0 ) break; } } - sqlite3VdbeAddOp2(v, OP_Integer, pLoop->u.vtab.idxNum, iReg); - sqlite3VdbeAddOp2(v, OP_Integer, nConstraint, iReg+1); - sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg, - pLoop->u.vtab.idxStr, - pLoop->u.vtab.needFree ? P4_MPRINTF : P4_STATIC); - VdbeCoverage(v); - pLoop->u.vtab.needFree = 0; - for(j=0; ju.vtab.omitMask>>j)&1 ){ - disableTerm(pLevel, pLoop->aLTerm[j]); - } + if( i==pIdx->nKeyCol ){ + /* This index implies that the DISTINCT qualifier is redundant. */ + return 1; } - pLevel->op = OP_VNext; - pLevel->p1 = iCur; - pLevel->p2 = sqlite3VdbeCurrentAddr(v); - sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2); - sqlite3ExprCachePop(pParse); - }else -#endif /* SQLITE_OMIT_VIRTUALTABLE */ + } - if( (pLoop->wsFlags & WHERE_IPK)!=0 - && (pLoop->wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_EQ))!=0 - ){ - /* Case 2: We can directly reference a single row using an - ** equality comparison against the ROWID field. Or - ** we reference multiple rows using a "rowid IN (...)" - ** construct. - */ - assert( pLoop->u.btree.nEq==1 ); - pTerm = pLoop->aLTerm[0]; - assert( pTerm!=0 ); - assert( pTerm->pExpr!=0 ); - assert( omitTable==0 ); - testcase( pTerm->wtFlags & TERM_VIRTUAL ); - iReleaseReg = ++pParse->nMem; - iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, bRev, iReleaseReg); - if( iRowidReg!=iReleaseReg ) sqlite3ReleaseTempReg(pParse, iReleaseReg); - addrNxt = pLevel->addrNxt; - sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addrNxt, iRowidReg); - VdbeCoverage(v); - sqlite3ExprCacheAffinityChange(pParse, iRowidReg, 1); - sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); - VdbeComment((v, "pk")); - pLevel->op = OP_Noop; - }else if( (pLoop->wsFlags & WHERE_IPK)!=0 - && (pLoop->wsFlags & WHERE_COLUMN_RANGE)!=0 - ){ - /* Case 3: We have an inequality comparison against the ROWID field. - */ - int testOp = OP_Noop; - int start; - int memEndValue = 0; - WhereTerm *pStart, *pEnd; + return 0; +} - assert( omitTable==0 ); - j = 0; - pStart = pEnd = 0; - if( pLoop->wsFlags & WHERE_BTM_LIMIT ) pStart = pLoop->aLTerm[j++]; - if( pLoop->wsFlags & WHERE_TOP_LIMIT ) pEnd = pLoop->aLTerm[j++]; - assert( pStart!=0 || pEnd!=0 ); - if( bRev ){ - pTerm = pStart; - pStart = pEnd; - pEnd = pTerm; - } - if( pStart ){ - Expr *pX; /* The expression that defines the start bound */ - int r1, rTemp; /* Registers for holding the start boundary */ - /* The following constant maps TK_xx codes into corresponding - ** seek opcodes. It depends on a particular ordering of TK_xx - */ - const u8 aMoveOp[] = { - /* TK_GT */ OP_SeekGT, - /* TK_LE */ OP_SeekLE, - /* TK_LT */ OP_SeekLT, - /* TK_GE */ OP_SeekGE - }; - assert( TK_LE==TK_GT+1 ); /* Make sure the ordering.. */ - assert( TK_LT==TK_GT+2 ); /* ... of the TK_xx values... */ - assert( TK_GE==TK_GT+3 ); /* ... is correcct. */ +/* +** Estimate the logarithm of the input value to base 2. +*/ +static LogEst estLog(LogEst N){ + return N<=10 ? 0 : sqlite3LogEst(N) - 33; +} - assert( (pStart->wtFlags & TERM_VNULL)==0 ); - testcase( pStart->wtFlags & TERM_VIRTUAL ); - pX = pStart->pExpr; - assert( pX!=0 ); - testcase( pStart->leftCursor!=iCur ); /* transitive constraints */ - r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp); - sqlite3VdbeAddOp3(v, aMoveOp[pX->op-TK_GT], iCur, addrBrk, r1); - VdbeComment((v, "pk")); - VdbeCoverageIf(v, pX->op==TK_GT); - VdbeCoverageIf(v, pX->op==TK_LE); - VdbeCoverageIf(v, pX->op==TK_LT); - VdbeCoverageIf(v, pX->op==TK_GE); - sqlite3ExprCacheAffinityChange(pParse, r1, 1); - sqlite3ReleaseTempReg(pParse, rTemp); - disableTerm(pLevel, pStart); - }else{ - sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrBrk); - VdbeCoverageIf(v, bRev==0); - VdbeCoverageIf(v, bRev!=0); - } - if( pEnd ){ - Expr *pX; - pX = pEnd->pExpr; - assert( pX!=0 ); - assert( (pEnd->wtFlags & TERM_VNULL)==0 ); - testcase( pEnd->leftCursor!=iCur ); /* Transitive constraints */ - testcase( pEnd->wtFlags & TERM_VIRTUAL ); - memEndValue = ++pParse->nMem; - sqlite3ExprCode(pParse, pX->pRight, memEndValue); - if( pX->op==TK_LT || pX->op==TK_GT ){ - testOp = bRev ? OP_Le : OP_Ge; +/* +** Convert OP_Column opcodes to OP_Copy in previously generated code. +** +** This routine runs over generated VDBE code and translates OP_Column +** opcodes into OP_Copy when the table is being accessed via co-routine +** instead of via table lookup. +** +** If the bIncrRowid parameter is 0, then any OP_Rowid instructions on +** cursor iTabCur are transformed into OP_Null. Or, if bIncrRowid is non-zero, +** then each OP_Rowid is transformed into an instruction to increment the +** value stored in its output register. +*/ +static void translateColumnToCopy( + Vdbe *v, /* The VDBE containing code to translate */ + int iStart, /* Translate from this opcode to the end */ + int iTabCur, /* OP_Column/OP_Rowid references to this table */ + int iRegister, /* The first column is in this register */ + int bIncrRowid /* If non-zero, transform OP_rowid to OP_AddImm(1) */ +){ + VdbeOp *pOp = sqlite3VdbeGetOp(v, iStart); + int iEnd = sqlite3VdbeCurrentAddr(v); + for(; iStartp1!=iTabCur ) continue; + if( pOp->opcode==OP_Column ){ + pOp->opcode = OP_Copy; + pOp->p1 = pOp->p2 + iRegister; + pOp->p2 = pOp->p3; + pOp->p3 = 0; + }else if( pOp->opcode==OP_Rowid ){ + if( bIncrRowid ){ + /* Increment the value stored in the P2 operand of the OP_Rowid. */ + pOp->opcode = OP_AddImm; + pOp->p1 = pOp->p2; + pOp->p2 = 1; }else{ - testOp = bRev ? OP_Lt : OP_Gt; + pOp->opcode = OP_Null; + pOp->p1 = 0; + pOp->p3 = 0; } - disableTerm(pLevel, pEnd); - } - start = sqlite3VdbeCurrentAddr(v); - pLevel->op = bRev ? OP_Prev : OP_Next; - pLevel->p1 = iCur; - pLevel->p2 = start; - assert( pLevel->p5==0 ); - if( testOp!=OP_Noop ){ - iRowidReg = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Rowid, iCur, iRowidReg); - sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); - sqlite3VdbeAddOp3(v, testOp, memEndValue, addrBrk, iRowidReg); - VdbeCoverageIf(v, testOp==OP_Le); - VdbeCoverageIf(v, testOp==OP_Lt); - VdbeCoverageIf(v, testOp==OP_Ge); - VdbeCoverageIf(v, testOp==OP_Gt); - sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC | SQLITE_JUMPIFNULL); } - }else if( pLoop->wsFlags & WHERE_INDEXED ){ - /* Case 4: A scan using an index. - ** - ** The WHERE clause may contain zero or more equality - ** terms ("==" or "IN" operators) that refer to the N - ** left-most columns of the index. It may also contain - ** inequality constraints (>, <, >= or <=) on the indexed - ** column that immediately follows the N equalities. Only - ** the right-most column can be an inequality - the rest must - ** use the "==" and "IN" operators. For example, if the - ** index is on (x,y,z), then the following clauses are all - ** optimized: - ** - ** x=5 - ** x=5 AND y=10 - ** x=5 AND y<10 - ** x=5 AND y>5 AND y<10 - ** x=5 AND y=5 AND z<=10 - ** - ** The z<10 term of the following cannot be used, only - ** the x=5 term: - ** - ** x=5 AND z<10 - ** - ** N may be zero if there are inequality constraints. - ** If there are no inequality constraints, then N is at - ** least one. - ** - ** This case is also used when there are no WHERE clause - ** constraints but an index is selected anyway, in order - ** to force the output order to conform to an ORDER BY. - */ - static const u8 aStartOp[] = { - 0, - 0, - OP_Rewind, /* 2: (!start_constraints && startEq && !bRev) */ - OP_Last, /* 3: (!start_constraints && startEq && bRev) */ - OP_SeekGT, /* 4: (start_constraints && !startEq && !bRev) */ - OP_SeekLT, /* 5: (start_constraints && !startEq && bRev) */ - OP_SeekGE, /* 6: (start_constraints && startEq && !bRev) */ - OP_SeekLE /* 7: (start_constraints && startEq && bRev) */ - }; - static const u8 aEndOp[] = { - OP_IdxGE, /* 0: (end_constraints && !bRev && !endEq) */ - OP_IdxGT, /* 1: (end_constraints && !bRev && endEq) */ - OP_IdxLE, /* 2: (end_constraints && bRev && !endEq) */ - OP_IdxLT, /* 3: (end_constraints && bRev && endEq) */ - }; - u16 nEq = pLoop->u.btree.nEq; /* Number of == or IN terms */ - int regBase; /* Base register holding constraint values */ - WhereTerm *pRangeStart = 0; /* Inequality constraint at range start */ - WhereTerm *pRangeEnd = 0; /* Inequality constraint at range end */ - int startEq; /* True if range start uses ==, >= or <= */ - int endEq; /* True if range end uses ==, >= or <= */ - int start_constraints; /* Start of range is constrained */ - int nConstraint; /* Number of constraint terms */ - Index *pIdx; /* The index we will be using */ - int iIdxCur; /* The VDBE cursor for the index */ - int nExtraReg = 0; /* Number of extra registers needed */ - int op; /* Instruction opcode */ - char *zStartAff; /* Affinity for start of range constraint */ - char cEndAff = 0; /* Affinity for end of range constraint */ - u8 bSeekPastNull = 0; /* True to seek past initial nulls */ - u8 bStopAtNull = 0; /* Add condition to terminate at NULLs */ + } +} - pIdx = pLoop->u.btree.pIndex; - iIdxCur = pLevel->iIdxCur; - assert( nEq>=pLoop->nSkip ); +/* +** Two routines for printing the content of an sqlite3_index_info +** structure. Used for testing and debugging only. If neither +** SQLITE_TEST or SQLITE_DEBUG are defined, then these routines +** are no-ops. +*/ +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED) +static void TRACE_IDX_INPUTS(sqlite3_index_info *p){ + int i; + if( !sqlite3WhereTrace ) return; + for(i=0; inConstraint; i++){ + sqlite3DebugPrintf(" constraint[%d]: col=%d termid=%d op=%d usabled=%d\n", + i, + p->aConstraint[i].iColumn, + p->aConstraint[i].iTermOffset, + p->aConstraint[i].op, + p->aConstraint[i].usable); + } + for(i=0; inOrderBy; i++){ + sqlite3DebugPrintf(" orderby[%d]: col=%d desc=%d\n", + i, + p->aOrderBy[i].iColumn, + p->aOrderBy[i].desc); + } +} +static void TRACE_IDX_OUTPUTS(sqlite3_index_info *p){ + int i; + if( !sqlite3WhereTrace ) return; + for(i=0; inConstraint; i++){ + sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n", + i, + p->aConstraintUsage[i].argvIndex, + p->aConstraintUsage[i].omit); + } + sqlite3DebugPrintf(" idxNum=%d\n", p->idxNum); + sqlite3DebugPrintf(" idxStr=%s\n", p->idxStr); + sqlite3DebugPrintf(" orderByConsumed=%d\n", p->orderByConsumed); + sqlite3DebugPrintf(" estimatedCost=%g\n", p->estimatedCost); + sqlite3DebugPrintf(" estimatedRows=%lld\n", p->estimatedRows); +} +#else +#define TRACE_IDX_INPUTS(A) +#define TRACE_IDX_OUTPUTS(A) +#endif - /* If this loop satisfies a sort order (pOrderBy) request that - ** was passed to this function to implement a "SELECT min(x) ..." - ** query, then the caller will only allow the loop to run for - ** a single iteration. This means that the first row returned - ** should not have a NULL value stored in 'x'. If column 'x' is - ** the first one after the nEq equality constraints in the index, - ** this requires some special handling. - */ - assert( pWInfo->pOrderBy==0 - || pWInfo->pOrderBy->nExpr==1 - || (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 ); - if( (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)!=0 - && pWInfo->nOBSat>0 - && (pIdx->nKeyCol>nEq) - ){ - assert( pLoop->nSkip==0 ); - bSeekPastNull = 1; - nExtraReg = 1; - } +#ifndef SQLITE_OMIT_AUTOMATIC_INDEX +/* +** Return TRUE if the WHERE clause term pTerm is of a form where it +** could be used with an index to access pSrc, assuming an appropriate +** index existed. +*/ +static int termCanDriveIndex( + WhereTerm *pTerm, /* WHERE clause term to check */ + struct SrcList_item *pSrc, /* Table we are trying to access */ + Bitmask notReady /* Tables in outer loops of the join */ +){ + char aff; + if( pTerm->leftCursor!=pSrc->iCursor ) return 0; + if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) return 0; + if( (pTerm->prereqRight & notReady)!=0 ) return 0; + if( pTerm->u.leftColumn<0 ) return 0; + aff = pSrc->pTab->aCol[pTerm->u.leftColumn].affinity; + if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0; + testcase( pTerm->pExpr->op==TK_IS ); + return 1; +} +#endif - /* Find any inequality constraint terms for the start and end - ** of the range. - */ - j = nEq; - if( pLoop->wsFlags & WHERE_BTM_LIMIT ){ - pRangeStart = pLoop->aLTerm[j++]; - nExtraReg = 1; - /* Like optimization range constraints always occur in pairs */ - assert( (pRangeStart->wtFlags & TERM_LIKEOPT)==0 || - (pLoop->wsFlags & WHERE_TOP_LIMIT)!=0 ); + +#ifndef SQLITE_OMIT_AUTOMATIC_INDEX +/* +** Generate code to construct the Index object for an automatic index +** and to set up the WhereLevel object pLevel so that the code generator +** makes use of the automatic index. +*/ +static void constructAutomaticIndex( + Parse *pParse, /* The parsing context */ + WhereClause *pWC, /* The WHERE clause */ + struct SrcList_item *pSrc, /* The FROM clause term to get the next index */ + Bitmask notReady, /* Mask of cursors that are not available */ + WhereLevel *pLevel /* Write new index here */ +){ + int nKeyCol; /* Number of columns in the constructed index */ + WhereTerm *pTerm; /* A single term of the WHERE clause */ + WhereTerm *pWCEnd; /* End of pWC->a[] */ + Index *pIdx; /* Object describing the transient index */ + Vdbe *v; /* Prepared statement under construction */ + int addrInit; /* Address of the initialization bypass jump */ + Table *pTable; /* The table being indexed */ + int addrTop; /* Top of the index fill loop */ + int regRecord; /* Register holding an index record */ + int n; /* Column counter */ + int i; /* Loop counter */ + int mxBitCol; /* Maximum column in pSrc->colUsed */ + CollSeq *pColl; /* Collating sequence to on a column */ + WhereLoop *pLoop; /* The Loop object */ + char *zNotUsed; /* Extra space on the end of pIdx */ + Bitmask idxCols; /* Bitmap of columns used for indexing */ + Bitmask extraCols; /* Bitmap of additional columns */ + u8 sentWarning = 0; /* True if a warnning has been issued */ + Expr *pPartial = 0; /* Partial Index Expression */ + int iContinue = 0; /* Jump here to skip excluded rows */ + struct SrcList_item *pTabItem; /* FROM clause term being indexed */ + int addrCounter; /* Address where integer counter is initialized */ + int regBase; /* Array of registers where record is assembled */ + + /* Generate code to skip over the creation and initialization of the + ** transient index on 2nd and subsequent iterations of the loop. */ + v = pParse->pVdbe; + assert( v!=0 ); + addrInit = sqlite3CodeOnce(pParse); VdbeCoverage(v); + + /* Count the number of columns that will be added to the index + ** and used to match WHERE clause constraints */ + nKeyCol = 0; + pTable = pSrc->pTab; + pWCEnd = &pWC->a[pWC->nTerm]; + pLoop = pLevel->pWLoop; + idxCols = 0; + for(pTerm=pWC->a; pTermpExpr; + assert( !ExprHasProperty(pExpr, EP_FromJoin) /* prereq always non-zero */ + || pExpr->iRightJoinTable!=pSrc->iCursor /* for the right-hand */ + || pLoop->prereq!=0 ); /* table of a LEFT JOIN */ + if( pLoop->prereq==0 + && (pTerm->wtFlags & TERM_VIRTUAL)==0 + && !ExprHasProperty(pExpr, EP_FromJoin) + && sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor) ){ + pPartial = sqlite3ExprAnd(pParse->db, pPartial, + sqlite3ExprDup(pParse->db, pExpr, 0)); } - if( pLoop->wsFlags & WHERE_TOP_LIMIT ){ - pRangeEnd = pLoop->aLTerm[j++]; - nExtraReg = 1; - if( (pRangeEnd->wtFlags & TERM_LIKEOPT)!=0 ){ - assert( pRangeStart!=0 ); /* LIKE opt constraints */ - assert( pRangeStart->wtFlags & TERM_LIKEOPT ); /* occur in pairs */ - pLevel->iLikeRepCntr = ++pParse->nMem; - testcase( bRev ); - testcase( pIdx->aSortOrder[nEq]==SQLITE_SO_DESC ); - sqlite3VdbeAddOp2(v, OP_Integer, - bRev ^ (pIdx->aSortOrder[nEq]==SQLITE_SO_DESC), - pLevel->iLikeRepCntr); - VdbeComment((v, "LIKE loop counter")); - pLevel->addrLikeRep = sqlite3VdbeCurrentAddr(v); + if( termCanDriveIndex(pTerm, pSrc, notReady) ){ + int iCol = pTerm->u.leftColumn; + Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); + testcase( iCol==BMS ); + testcase( iCol==BMS-1 ); + if( !sentWarning ){ + sqlite3_log(SQLITE_WARNING_AUTOINDEX, + "automatic index on %s(%s)", pTable->zName, + pTable->aCol[iCol].zName); + sentWarning = 1; } - if( pRangeStart==0 - && (j = pIdx->aiColumn[nEq])>=0 - && pIdx->pTable->aCol[j].notNull==0 - ){ - bSeekPastNull = 1; + if( (idxCols & cMask)==0 ){ + if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ){ + goto end_auto_index_create; + } + pLoop->aLTerm[nKeyCol++] = pTerm; + idxCols |= cMask; } } - assert( pRangeEnd==0 || (pRangeEnd->wtFlags & TERM_VNULL)==0 ); - - /* Generate code to evaluate all constraint terms using == or IN - ** and store the values of those terms in an array of registers - ** starting at regBase. - */ - regBase = codeAllEqualityTerms(pParse,pLevel,bRev,nExtraReg,&zStartAff); - assert( zStartAff==0 || sqlite3Strlen30(zStartAff)>=nEq ); - if( zStartAff ) cEndAff = zStartAff[nEq]; - addrNxt = pLevel->addrNxt; - - /* If we are doing a reverse order scan on an ascending index, or - ** a forward order scan on a descending index, interchange the - ** start and end terms (pRangeStart and pRangeEnd). - */ - if( (nEqnKeyCol && bRev==(pIdx->aSortOrder[nEq]==SQLITE_SO_ASC)) - || (bRev && pIdx->nKeyCol==nEq) - ){ - SWAP(WhereTerm *, pRangeEnd, pRangeStart); - SWAP(u8, bSeekPastNull, bStopAtNull); - } + } + assert( nKeyCol>0 ); + pLoop->u.btree.nEq = pLoop->nLTerm = nKeyCol; + pLoop->wsFlags = WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WHERE_INDEXED + | WHERE_AUTO_INDEX; - testcase( pRangeStart && (pRangeStart->eOperator & WO_LE)!=0 ); - testcase( pRangeStart && (pRangeStart->eOperator & WO_GE)!=0 ); - testcase( pRangeEnd && (pRangeEnd->eOperator & WO_LE)!=0 ); - testcase( pRangeEnd && (pRangeEnd->eOperator & WO_GE)!=0 ); - startEq = !pRangeStart || pRangeStart->eOperator & (WO_LE|WO_GE); - endEq = !pRangeEnd || pRangeEnd->eOperator & (WO_LE|WO_GE); - start_constraints = pRangeStart || nEq>0; + /* Count the number of additional columns needed to create a + ** covering index. A "covering index" is an index that contains all + ** columns that are needed by the query. With a covering index, the + ** original table never needs to be accessed. Automatic indices must + ** be a covering index because the index will not be updated if the + ** original table changes and the index and table cannot both be used + ** if they go out of sync. + */ + extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); + mxBitCol = MIN(BMS-1,pTable->nCol); + testcase( pTable->nCol==BMS-1 ); + testcase( pTable->nCol==BMS-2 ); + for(i=0; icolUsed & MASKBIT(BMS-1) ){ + nKeyCol += pTable->nCol - BMS + 1; + } - /* Seek the index cursor to the start of the range. */ - nConstraint = nEq; - if( pRangeStart ){ - Expr *pRight = pRangeStart->pExpr->pRight; - sqlite3ExprCode(pParse, pRight, regBase+nEq); - whereLikeOptimizationStringFixup(v, pLevel, pRangeStart); - if( (pRangeStart->wtFlags & TERM_VNULL)==0 - && sqlite3ExprCanBeNull(pRight) - ){ - sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); - VdbeCoverage(v); + /* Construct the Index object to describe this index */ + pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed); + if( pIdx==0 ) goto end_auto_index_create; + pLoop->u.btree.pIndex = pIdx; + pIdx->zName = "auto-index"; + pIdx->pTable = pTable; + n = 0; + idxCols = 0; + for(pTerm=pWC->a; pTermu.leftColumn; + Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); + testcase( iCol==BMS-1 ); + testcase( iCol==BMS ); + if( (idxCols & cMask)==0 ){ + Expr *pX = pTerm->pExpr; + idxCols |= cMask; + pIdx->aiColumn[n] = pTerm->u.leftColumn; + pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); + pIdx->azColl[n] = pColl ? pColl->zName : "BINARY"; + n++; } - if( zStartAff ){ - if( sqlite3CompareAffinity(pRight, zStartAff[nEq])==SQLITE_AFF_NONE){ - /* Since the comparison is to be performed with no conversions - ** applied to the operands, set the affinity to apply to pRight to - ** SQLITE_AFF_NONE. */ - zStartAff[nEq] = SQLITE_AFF_NONE; - } - if( sqlite3ExprNeedsNoAffinityChange(pRight, zStartAff[nEq]) ){ - zStartAff[nEq] = SQLITE_AFF_NONE; - } - } - nConstraint++; - testcase( pRangeStart->wtFlags & TERM_VIRTUAL ); - }else if( bSeekPastNull ){ - sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); - nConstraint++; - startEq = 0; - start_constraints = 1; } - codeApplyAffinity(pParse, regBase, nConstraint - bSeekPastNull, zStartAff); - op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev]; - assert( op!=0 ); - sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); - VdbeCoverage(v); - VdbeCoverageIf(v, op==OP_Rewind); testcase( op==OP_Rewind ); - VdbeCoverageIf(v, op==OP_Last); testcase( op==OP_Last ); - VdbeCoverageIf(v, op==OP_SeekGT); testcase( op==OP_SeekGT ); - VdbeCoverageIf(v, op==OP_SeekGE); testcase( op==OP_SeekGE ); - VdbeCoverageIf(v, op==OP_SeekLE); testcase( op==OP_SeekLE ); - VdbeCoverageIf(v, op==OP_SeekLT); testcase( op==OP_SeekLT ); + } + assert( (u32)n==pLoop->u.btree.nEq ); - /* Load the value for the inequality constraint at the end of the - ** range (if any). - */ - nConstraint = nEq; - if( pRangeEnd ){ - Expr *pRight = pRangeEnd->pExpr->pRight; - sqlite3ExprCacheRemove(pParse, regBase+nEq, 1); - sqlite3ExprCode(pParse, pRight, regBase+nEq); - whereLikeOptimizationStringFixup(v, pLevel, pRangeEnd); - if( (pRangeEnd->wtFlags & TERM_VNULL)==0 - && sqlite3ExprCanBeNull(pRight) - ){ - sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); - VdbeCoverage(v); - } - if( sqlite3CompareAffinity(pRight, cEndAff)!=SQLITE_AFF_NONE - && !sqlite3ExprNeedsNoAffinityChange(pRight, cEndAff) - ){ - codeApplyAffinity(pParse, regBase+nEq, 1, &cEndAff); - } - nConstraint++; - testcase( pRangeEnd->wtFlags & TERM_VIRTUAL ); - }else if( bStopAtNull ){ - sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); - endEq = 0; - nConstraint++; + /* Add additional columns needed to make the automatic index into + ** a covering index */ + for(i=0; iaiColumn[n] = i; + pIdx->azColl[n] = "BINARY"; + n++; } - sqlite3DbFree(db, zStartAff); + } + if( pSrc->colUsed & MASKBIT(BMS-1) ){ + for(i=BMS-1; inCol; i++){ + pIdx->aiColumn[n] = i; + pIdx->azColl[n] = "BINARY"; + n++; + } + } + assert( n==nKeyCol ); + pIdx->aiColumn[n] = XN_ROWID; + pIdx->azColl[n] = "BINARY"; - /* Top of the loop body */ - pLevel->p2 = sqlite3VdbeCurrentAddr(v); + /* Create the automatic index */ + assert( pLevel->iIdxCur>=0 ); + pLevel->iIdxCur = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + VdbeComment((v, "for %s", pTable->zName)); - /* Check if the index cursor is past the end of the range. */ - if( nConstraint ){ - op = aEndOp[bRev*2 + endEq]; - sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); - testcase( op==OP_IdxGT ); VdbeCoverageIf(v, op==OP_IdxGT ); - testcase( op==OP_IdxGE ); VdbeCoverageIf(v, op==OP_IdxGE ); - testcase( op==OP_IdxLT ); VdbeCoverageIf(v, op==OP_IdxLT ); - testcase( op==OP_IdxLE ); VdbeCoverageIf(v, op==OP_IdxLE ); - } + /* Fill the automatic index with content */ + sqlite3ExprCachePush(pParse); + pTabItem = &pWC->pWInfo->pTabList->a[pLevel->iFrom]; + if( pTabItem->fg.viaCoroutine ){ + int regYield = pTabItem->regReturn; + addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0); + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); + addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield); + VdbeCoverage(v); + VdbeComment((v, "next row of \"%s\"", pTabItem->pTab->zName)); + }else{ + addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); + } + if( pPartial ){ + iContinue = sqlite3VdbeMakeLabel(v); + sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL); + pLoop->wsFlags |= WHERE_PARTIALIDX; + } + regRecord = sqlite3GetTempReg(pParse); + regBase = sqlite3GenerateIndexKey( + pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0 + ); + sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); + if( pTabItem->fg.viaCoroutine ){ + sqlite3VdbeChangeP2(v, addrCounter, regBase+n); + translateColumnToCopy(v, addrTop, pLevel->iTabCur, pTabItem->regResult, 1); + sqlite3VdbeGoto(v, addrTop); + pTabItem->fg.viaCoroutine = 0; + }else{ + sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); + } + sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); + sqlite3VdbeJumpHere(v, addrTop); + sqlite3ReleaseTempReg(pParse, regRecord); + sqlite3ExprCachePop(pParse); + + /* Jump here when skipping the initialization */ + sqlite3VdbeJumpHere(v, addrInit); - /* Seek the table cursor, if required */ - disableTerm(pLevel, pRangeStart); - disableTerm(pLevel, pRangeEnd); - if( omitTable ){ - /* pIdx is a covering index. No need to access the main table. */ - }else if( HasRowid(pIdx->pTable) ){ - iRowidReg = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg); - sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); - sqlite3VdbeAddOp2(v, OP_Seek, iCur, iRowidReg); /* Deferred seek */ - }else if( iCur!=iIdxCur ){ - Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable); - iRowidReg = sqlite3GetTempRange(pParse, pPk->nKeyCol); - for(j=0; jnKeyCol; j++){ - k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); - sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, iRowidReg+j); - } - sqlite3VdbeAddOp4Int(v, OP_NotFound, iCur, addrCont, - iRowidReg, pPk->nKeyCol); VdbeCoverage(v); - } +end_auto_index_create: + sqlite3ExprDelete(pParse->db, pPartial); +} +#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ - /* Record the instruction used to terminate the loop. Disable - ** WHERE clause terms made redundant by the index range scan. - */ - if( pLoop->wsFlags & WHERE_ONEROW ){ - pLevel->op = OP_Noop; - }else if( bRev ){ - pLevel->op = OP_Prev; - }else{ - pLevel->op = OP_Next; +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Allocate and populate an sqlite3_index_info structure. It is the +** responsibility of the caller to eventually release the structure +** by passing the pointer returned by this function to sqlite3_free(). +*/ +static sqlite3_index_info *allocateIndexInfo( + Parse *pParse, + WhereClause *pWC, + Bitmask mUnusable, /* Ignore terms with these prereqs */ + struct SrcList_item *pSrc, + ExprList *pOrderBy +){ + int i, j; + int nTerm; + struct sqlite3_index_constraint *pIdxCons; + struct sqlite3_index_orderby *pIdxOrderBy; + struct sqlite3_index_constraint_usage *pUsage; + WhereTerm *pTerm; + int nOrderBy; + sqlite3_index_info *pIdxInfo; + + /* Count the number of possible WHERE clause constraints referring + ** to this virtual table */ + for(i=nTerm=0, pTerm=pWC->a; inTerm; i++, pTerm++){ + if( pTerm->leftCursor != pSrc->iCursor ) continue; + if( pTerm->prereqRight & mUnusable ) continue; + assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); + testcase( pTerm->eOperator & WO_IN ); + testcase( pTerm->eOperator & WO_ISNULL ); + testcase( pTerm->eOperator & WO_IS ); + testcase( pTerm->eOperator & WO_ALL ); + if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV|WO_IS))==0 ) continue; + if( pTerm->wtFlags & TERM_VNULL ) continue; + assert( pTerm->u.leftColumn>=(-1) ); + nTerm++; + } + + /* If the ORDER BY clause contains only columns in the current + ** virtual table then allocate space for the aOrderBy part of + ** the sqlite3_index_info structure. + */ + nOrderBy = 0; + if( pOrderBy ){ + int n = pOrderBy->nExpr; + for(i=0; ia[i].pExpr; + if( pExpr->op!=TK_COLUMN || pExpr->iTable!=pSrc->iCursor ) break; } - pLevel->p1 = iIdxCur; - pLevel->p3 = (pLoop->wsFlags&WHERE_UNQ_WANTED)!=0 ? 1:0; - if( (pLoop->wsFlags & WHERE_CONSTRAINT)==0 ){ - pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; - }else{ - assert( pLevel->p5==0 ); + if( i==n){ + nOrderBy = n; } - }else + } -#ifndef SQLITE_OMIT_OR_OPTIMIZATION - if( pLoop->wsFlags & WHERE_MULTI_OR ){ - /* Case 5: Two or more separately indexed terms connected by OR - ** - ** Example: - ** - ** CREATE TABLE t1(a,b,c,d); - ** CREATE INDEX i1 ON t1(a); - ** CREATE INDEX i2 ON t1(b); - ** CREATE INDEX i3 ON t1(c); - ** - ** SELECT * FROM t1 WHERE a=5 OR b=7 OR (c=11 AND d=13) - ** - ** In the example, there are three indexed terms connected by OR. - ** The top of the loop looks like this: - ** - ** Null 1 # Zero the rowset in reg 1 - ** - ** Then, for each indexed term, the following. The arguments to - ** RowSetTest are such that the rowid of the current row is inserted - ** into the RowSet. If it is already present, control skips the - ** Gosub opcode and jumps straight to the code generated by WhereEnd(). - ** - ** sqlite3WhereBegin() - ** RowSetTest # Insert rowid into rowset - ** Gosub 2 A - ** sqlite3WhereEnd() - ** - ** Following the above, code to terminate the loop. Label A, the target - ** of the Gosub above, jumps to the instruction right after the Goto. - ** - ** Null 1 # Zero the rowset in reg 1 - ** Goto B # The loop is finished. - ** - ** A: # Return data, whatever. - ** - ** Return 2 # Jump back to the Gosub - ** - ** B: - ** - ** Added 2014-05-26: If the table is a WITHOUT ROWID table, then - ** use an ephemeral index instead of a RowSet to record the primary - ** keys of the rows we have already seen. - ** - */ - WhereClause *pOrWc; /* The OR-clause broken out into subterms */ - SrcList *pOrTab; /* Shortened table list or OR-clause generation */ - Index *pCov = 0; /* Potential covering index (or NULL) */ - int iCovCur = pParse->nTab++; /* Cursor used for index scans (if any) */ - - int regReturn = ++pParse->nMem; /* Register used with OP_Gosub */ - int regRowset = 0; /* Register for RowSet object */ - int regRowid = 0; /* Register holding rowid */ - int iLoopBody = sqlite3VdbeMakeLabel(v); /* Start of loop body */ - int iRetInit; /* Address of regReturn init */ - int untestedTerms = 0; /* Some terms not completely tested */ - int ii; /* Loop counter */ - u16 wctrlFlags; /* Flags for sub-WHERE clause */ - Expr *pAndExpr = 0; /* An ".. AND (...)" expression */ - Table *pTab = pTabItem->pTab; - - pTerm = pLoop->aLTerm[0]; - assert( pTerm!=0 ); - assert( pTerm->eOperator & WO_OR ); - assert( (pTerm->wtFlags & TERM_ORINFO)!=0 ); - pOrWc = &pTerm->u.pOrInfo->wc; - pLevel->op = OP_Return; - pLevel->p1 = regReturn; - - /* Set up a new SrcList in pOrTab containing the table being scanned - ** by this loop in the a[0] slot and all notReady tables in a[1..] slots. - ** This becomes the SrcList in the recursive call to sqlite3WhereBegin(). - */ - if( pWInfo->nLevel>1 ){ - int nNotReady; /* The number of notReady tables */ - struct SrcList_item *origSrc; /* Original list of tables */ - nNotReady = pWInfo->nLevel - iLevel - 1; - pOrTab = sqlite3StackAllocRaw(db, - sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0])); - if( pOrTab==0 ) return notReady; - pOrTab->nAlloc = (u8)(nNotReady + 1); - pOrTab->nSrc = pOrTab->nAlloc; - memcpy(pOrTab->a, pTabItem, sizeof(*pTabItem)); - origSrc = pWInfo->pTabList->a; - for(k=1; k<=nNotReady; k++){ - memcpy(&pOrTab->a[k], &origSrc[pLevel[k].iFrom], sizeof(pOrTab->a[k])); - } - }else{ - pOrTab = pWInfo->pTabList; - } + /* Allocate the sqlite3_index_info structure + */ + pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo) + + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm + + sizeof(*pIdxOrderBy)*nOrderBy ); + if( pIdxInfo==0 ){ + sqlite3ErrorMsg(pParse, "out of memory"); + return 0; + } - /* Initialize the rowset register to contain NULL. An SQL NULL is - ** equivalent to an empty rowset. Or, create an ephemeral index - ** capable of holding primary keys in the case of a WITHOUT ROWID. - ** - ** Also initialize regReturn to contain the address of the instruction - ** immediately following the OP_Return at the bottom of the loop. This - ** is required in a few obscure LEFT JOIN cases where control jumps - ** over the top of the loop into the body of it. In this case the - ** correct response for the end-of-loop code (the OP_Return) is to - ** fall through to the next instruction, just as an OP_Next does if - ** called on an uninitialized cursor. - */ - if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ - if( HasRowid(pTab) ){ - regRowset = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Null, 0, regRowset); - }else{ - Index *pPk = sqlite3PrimaryKeyIndex(pTab); - regRowset = pParse->nTab++; - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, regRowset, pPk->nKeyCol); - sqlite3VdbeSetP4KeyInfo(pParse, pPk); - } - regRowid = ++pParse->nMem; - } - iRetInit = sqlite3VdbeAddOp2(v, OP_Integer, 0, regReturn); + /* Initialize the structure. The sqlite3_index_info structure contains + ** many fields that are declared "const" to prevent xBestIndex from + ** changing them. We have to do some funky casting in order to + ** initialize those fields. + */ + pIdxCons = (struct sqlite3_index_constraint*)&pIdxInfo[1]; + pIdxOrderBy = (struct sqlite3_index_orderby*)&pIdxCons[nTerm]; + pUsage = (struct sqlite3_index_constraint_usage*)&pIdxOrderBy[nOrderBy]; + *(int*)&pIdxInfo->nConstraint = nTerm; + *(int*)&pIdxInfo->nOrderBy = nOrderBy; + *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint = pIdxCons; + *(struct sqlite3_index_orderby**)&pIdxInfo->aOrderBy = pIdxOrderBy; + *(struct sqlite3_index_constraint_usage**)&pIdxInfo->aConstraintUsage = + pUsage; - /* If the original WHERE clause is z of the form: (x1 OR x2 OR ...) AND y - ** Then for every term xN, evaluate as the subexpression: xN AND z - ** That way, terms in y that are factored into the disjunction will - ** be picked up by the recursive calls to sqlite3WhereBegin() below. - ** - ** Actually, each subexpression is converted to "xN AND w" where w is - ** the "interesting" terms of z - terms that did not originate in the - ** ON or USING clause of a LEFT JOIN, and terms that are usable as - ** indices. - ** - ** This optimization also only applies if the (x1 OR x2 OR ...) term - ** is not contained in the ON clause of a LEFT JOIN. - ** See ticket http://www.sqlite.org/src/info/f2369304e4 - */ - if( pWC->nTerm>1 ){ - int iTerm; - for(iTerm=0; iTermnTerm; iTerm++){ - Expr *pExpr = pWC->a[iTerm].pExpr; - if( &pWC->a[iTerm] == pTerm ) continue; - if( ExprHasProperty(pExpr, EP_FromJoin) ) continue; - if( (pWC->a[iTerm].wtFlags & TERM_VIRTUAL)!=0 ) continue; - if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue; - testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO ); - pExpr = sqlite3ExprDup(db, pExpr, 0); - pAndExpr = sqlite3ExprAnd(db, pAndExpr, pExpr); - } - if( pAndExpr ){ - pAndExpr = sqlite3PExpr(pParse, TK_AND, 0, pAndExpr, 0); - } - } + for(i=j=0, pTerm=pWC->a; inTerm; i++, pTerm++){ + u8 op; + if( pTerm->leftCursor != pSrc->iCursor ) continue; + if( pTerm->prereqRight & mUnusable ) continue; + assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); + testcase( pTerm->eOperator & WO_IN ); + testcase( pTerm->eOperator & WO_IS ); + testcase( pTerm->eOperator & WO_ISNULL ); + testcase( pTerm->eOperator & WO_ALL ); + if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV|WO_IS))==0 ) continue; + if( pTerm->wtFlags & TERM_VNULL ) continue; + assert( pTerm->u.leftColumn>=(-1) ); + pIdxCons[j].iColumn = pTerm->u.leftColumn; + pIdxCons[j].iTermOffset = i; + op = (u8)pTerm->eOperator & WO_ALL; + if( op==WO_IN ) op = WO_EQ; + pIdxCons[j].op = op; + /* The direct assignment in the previous line is possible only because + ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The + ** following asserts verify this fact. */ + assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ ); + assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT ); + assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE ); + assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT ); + assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE ); + assert( WO_MATCH==SQLITE_INDEX_CONSTRAINT_MATCH ); + assert( pTerm->eOperator & (WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) ); + j++; + } + for(i=0; ia[i].pExpr; + pIdxOrderBy[i].iColumn = pExpr->iColumn; + pIdxOrderBy[i].desc = pOrderBy->a[i].sortOrder; + } - /* Run a separate WHERE clause for each term of the OR clause. After - ** eliminating duplicates from other WHERE clauses, the action for each - ** sub-WHERE clause is to to invoke the main loop body as a subroutine. - */ - wctrlFlags = WHERE_OMIT_OPEN_CLOSE - | WHERE_FORCE_TABLE - | WHERE_ONETABLE_ONLY - | WHERE_NO_AUTOINDEX; - for(ii=0; iinTerm; ii++){ - WhereTerm *pOrTerm = &pOrWc->a[ii]; - if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ - WhereInfo *pSubWInfo; /* Info for single OR-term scan */ - Expr *pOrExpr = pOrTerm->pExpr; /* Current OR clause term */ - int j1 = 0; /* Address of jump operation */ - if( pAndExpr && !ExprHasProperty(pOrExpr, EP_FromJoin) ){ - pAndExpr->pLeft = pOrExpr; - pOrExpr = pAndExpr; - } - /* Loop through table entries that match term pOrTerm. */ - WHERETRACE(0xffff, ("Subplan for OR-clause:\n")); - pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, - wctrlFlags, iCovCur); - assert( pSubWInfo || pParse->nErr || db->mallocFailed ); - if( pSubWInfo ){ - WhereLoop *pSubLoop; - int addrExplain = explainOneScan( - pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0 - ); - addScanStatus(v, pOrTab, &pSubWInfo->a[0], addrExplain); + return pIdxInfo; +} - /* This is the sub-WHERE clause body. First skip over - ** duplicate rows from prior sub-WHERE clauses, and record the - ** rowid (or PRIMARY KEY) for the current row so that the same - ** row will be skipped in subsequent sub-WHERE clauses. - */ - if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ - int r; - int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); - if( HasRowid(pTab) ){ - r = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, regRowid, 0); - j1 = sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, 0, r,iSet); - VdbeCoverage(v); - }else{ - Index *pPk = sqlite3PrimaryKeyIndex(pTab); - int nPk = pPk->nKeyCol; - int iPk; +/* +** The table object reference passed as the second argument to this function +** must represent a virtual table. This function invokes the xBestIndex() +** method of the virtual table with the sqlite3_index_info object that +** comes in as the 3rd argument to this function. +** +** If an error occurs, pParse is populated with an error message and a +** non-zero value is returned. Otherwise, 0 is returned and the output +** part of the sqlite3_index_info structure is left populated. +** +** Whether or not an error is returned, it is the responsibility of the +** caller to eventually free p->idxStr if p->needToFreeIdxStr indicates +** that this is required. +*/ +static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ + sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab; + int i; + int rc; - /* Read the PK into an array of temp registers. */ - r = sqlite3GetTempRange(pParse, nPk); - for(iPk=0; iPkaiColumn[iPk]; - sqlite3ExprCodeGetColumn(pParse, pTab, iCol, iCur, r+iPk, 0); - } + TRACE_IDX_INPUTS(p); + rc = pVtab->pModule->xBestIndex(pVtab, p); + TRACE_IDX_OUTPUTS(p); - /* Check if the temp table already contains this key. If so, - ** the row has already been included in the result set and - ** can be ignored (by jumping past the Gosub below). Otherwise, - ** insert the key into the temp table and proceed with processing - ** the row. - ** - ** Use some of the same optimizations as OP_RowSetTest: If iSet - ** is zero, assume that the key cannot already be present in - ** the temp table. And if iSet is -1, assume that there is no - ** need to insert the key into the temp table, as it will never - ** be tested for. */ - if( iSet ){ - j1 = sqlite3VdbeAddOp4Int(v, OP_Found, regRowset, 0, r, nPk); - VdbeCoverage(v); - } - if( iSet>=0 ){ - sqlite3VdbeAddOp3(v, OP_MakeRecord, r, nPk, regRowid); - sqlite3VdbeAddOp3(v, OP_IdxInsert, regRowset, regRowid, 0); - if( iSet ) sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); - } + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ){ + pParse->db->mallocFailed = 1; + }else if( !pVtab->zErrMsg ){ + sqlite3ErrorMsg(pParse, "%s", sqlite3ErrStr(rc)); + }else{ + sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg); + } + } + sqlite3_free(pVtab->zErrMsg); + pVtab->zErrMsg = 0; - /* Release the array of temp registers */ - sqlite3ReleaseTempRange(pParse, r, nPk); - } - } + for(i=0; inConstraint; i++){ + if( !p->aConstraint[i].usable && p->aConstraintUsage[i].argvIndex>0 ){ + sqlite3ErrorMsg(pParse, + "table %s: xBestIndex returned an invalid plan", pTab->zName); + } + } - /* Invoke the main loop body as a subroutine */ - sqlite3VdbeAddOp2(v, OP_Gosub, regReturn, iLoopBody); + return pParse->nErr; +} +#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ - /* Jump here (skipping the main loop body subroutine) if the - ** current sub-WHERE row is a duplicate from prior sub-WHEREs. */ - if( j1 ) sqlite3VdbeJumpHere(v, j1); +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +/* +** Estimate the location of a particular key among all keys in an +** index. Store the results in aStat as follows: +** +** aStat[0] Est. number of rows less than pRec +** aStat[1] Est. number of rows equal to pRec +** +** Return the index of the sample that is the smallest sample that +** is greater than or equal to pRec. Note that this index is not an index +** into the aSample[] array - it is an index into a virtual set of samples +** based on the contents of aSample[] and the number of fields in record +** pRec. +*/ +static int whereKeyStats( + Parse *pParse, /* Database connection */ + Index *pIdx, /* Index to consider domain of */ + UnpackedRecord *pRec, /* Vector of values to consider */ + int roundUp, /* Round up if true. Round down if false */ + tRowcnt *aStat /* OUT: stats written here */ +){ + IndexSample *aSample = pIdx->aSample; + int iCol; /* Index of required stats in anEq[] etc. */ + int i; /* Index of first sample >= pRec */ + int iSample; /* Smallest sample larger than or equal to pRec */ + int iMin = 0; /* Smallest sample not yet tested */ + int iTest; /* Next sample to test */ + int res; /* Result of comparison operation */ + int nField; /* Number of fields in pRec */ + tRowcnt iLower = 0; /* anLt[] + anEq[] of largest sample pRec is > */ - /* The pSubWInfo->untestedTerms flag means that this OR term - ** contained one or more AND term from a notReady table. The - ** terms from the notReady table could not be tested and will - ** need to be tested later. - */ - if( pSubWInfo->untestedTerms ) untestedTerms = 1; +#ifndef SQLITE_DEBUG + UNUSED_PARAMETER( pParse ); +#endif + assert( pRec!=0 ); + assert( pIdx->nSample>0 ); + assert( pRec->nField>0 && pRec->nField<=pIdx->nSampleCol ); - /* If all of the OR-connected terms are optimized using the same - ** index, and the index is opened using the same cursor number - ** by each call to sqlite3WhereBegin() made by this loop, it may - ** be possible to use that index as a covering index. - ** - ** If the call to sqlite3WhereBegin() above resulted in a scan that - ** uses an index, and this is either the first OR-connected term - ** processed or the index is the same as that used by all previous - ** terms, set pCov to the candidate covering index. Otherwise, set - ** pCov to NULL to indicate that no candidate covering index will - ** be available. - */ - pSubLoop = pSubWInfo->a[0].pWLoop; - assert( (pSubLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); - if( (pSubLoop->wsFlags & WHERE_INDEXED)!=0 - && (ii==0 || pSubLoop->u.btree.pIndex==pCov) - && (HasRowid(pTab) || !IsPrimaryKeyIndex(pSubLoop->u.btree.pIndex)) - ){ - assert( pSubWInfo->a[0].iIdxCur==iCovCur ); - pCov = pSubLoop->u.btree.pIndex; - wctrlFlags |= WHERE_REOPEN_IDX; - }else{ - pCov = 0; - } + /* Do a binary search to find the first sample greater than or equal + ** to pRec. If pRec contains a single field, the set of samples to search + ** is simply the aSample[] array. If the samples in aSample[] contain more + ** than one fields, all fields following the first are ignored. + ** + ** If pRec contains N fields, where N is more than one, then as well as the + ** samples in aSample[] (truncated to N fields), the search also has to + ** consider prefixes of those samples. For example, if the set of samples + ** in aSample is: + ** + ** aSample[0] = (a, 5) + ** aSample[1] = (a, 10) + ** aSample[2] = (b, 5) + ** aSample[3] = (c, 100) + ** aSample[4] = (c, 105) + ** + ** Then the search space should ideally be the samples above and the + ** unique prefixes [a], [b] and [c]. But since that is hard to organize, + ** the code actually searches this set: + ** + ** 0: (a) + ** 1: (a, 5) + ** 2: (a, 10) + ** 3: (a, 10) + ** 4: (b) + ** 5: (b, 5) + ** 6: (c) + ** 7: (c, 100) + ** 8: (c, 105) + ** 9: (c, 105) + ** + ** For each sample in the aSample[] array, N samples are present in the + ** effective sample array. In the above, samples 0 and 1 are based on + ** sample aSample[0]. Samples 2 and 3 on aSample[1] etc. + ** + ** Often, sample i of each block of N effective samples has (i+1) fields. + ** Except, each sample may be extended to ensure that it is greater than or + ** equal to the previous sample in the array. For example, in the above, + ** sample 2 is the first sample of a block of N samples, so at first it + ** appears that it should be 1 field in size. However, that would make it + ** smaller than sample 1, so the binary search would not work. As a result, + ** it is extended to two fields. The duplicates that this creates do not + ** cause any problems. + */ + nField = pRec->nField; + iCol = 0; + iSample = pIdx->nSample * nField; + do{ + int iSamp; /* Index in aSample[] of test sample */ + int n; /* Number of fields in test sample */ - /* Finish the loop through table entries that match term pOrTerm. */ - sqlite3WhereEnd(pSubWInfo); - } + iTest = (iMin+iSample)/2; + iSamp = iTest / nField; + if( iSamp>0 ){ + /* The proposed effective sample is a prefix of sample aSample[iSamp]. + ** Specifically, the shortest prefix of at least (1 + iTest%nField) + ** fields that is greater than the previous effective sample. */ + for(n=(iTest % nField) + 1; nu.pCovidx = pCov; - if( pCov ) pLevel->iIdxCur = iCovCur; - if( pAndExpr ){ - pAndExpr->pLeft = 0; - sqlite3ExprDelete(db, pAndExpr); - } - sqlite3VdbeChangeP1(v, iRetInit, sqlite3VdbeCurrentAddr(v)); - sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrBrk); - sqlite3VdbeResolveLabel(v, iLoopBody); - - if( pWInfo->nLevel>1 ) sqlite3StackFree(db, pOrTab); - if( !untestedTerms ) disableTerm(pLevel, pTerm); - }else -#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ - - { - /* Case 6: There is no usable index. We must do a complete - ** scan of the entire table. - */ - static const u8 aStep[] = { OP_Next, OP_Prev }; - static const u8 aStart[] = { OP_Rewind, OP_Last }; - assert( bRev==0 || bRev==1 ); - if( pTabItem->isRecursive ){ - /* Tables marked isRecursive have only a single row that is stored in - ** a pseudo-cursor. No need to Rewind or Next such cursors. */ - pLevel->op = OP_Noop; }else{ - pLevel->op = aStep[bRev]; - pLevel->p1 = iCur; - pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); - VdbeCoverageIf(v, bRev==0); - VdbeCoverageIf(v, bRev!=0); - pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + n = iTest + 1; } - } - -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - pLevel->addrVisit = sqlite3VdbeCurrentAddr(v); -#endif - /* Insert code to test every subexpression that can be completely - ** computed using the current set of tables. - */ - for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ - Expr *pE; - int skipLikeAddr = 0; - testcase( pTerm->wtFlags & TERM_VIRTUAL ); - testcase( pTerm->wtFlags & TERM_CODED ); - if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ - testcase( pWInfo->untestedTerms==0 - && (pWInfo->wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ); - pWInfo->untestedTerms = 1; - continue; - } - pE = pTerm->pExpr; - assert( pE!=0 ); - if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){ - continue; - } - if( pTerm->wtFlags & TERM_LIKECOND ){ - assert( pLevel->iLikeRepCntr>0 ); - skipLikeAddr = sqlite3VdbeAddOp1(v, OP_IfNot, pLevel->iLikeRepCntr); - VdbeCoverage(v); + pRec->nField = n; + res = sqlite3VdbeRecordCompare(aSample[iSamp].n, aSample[iSamp].p, pRec); + if( res<0 ){ + iLower = aSample[iSamp].anLt[n-1] + aSample[iSamp].anEq[n-1]; + iMin = iTest+1; + }else if( res==0 && nwtFlags |= TERM_CODED; - } + }while( res && iMina, j=pWC->nTerm; j>0; j--, pTerm++){ - Expr *pE, *pEAlt; - WhereTerm *pAlt; - if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( pTerm->eOperator!=(WO_EQUIV|WO_EQ) ) continue; - if( pTerm->leftCursor!=iCur ) continue; - if( pLevel->iLeftJoin ) continue; - pE = pTerm->pExpr; - assert( !ExprHasProperty(pE, EP_FromJoin) ); - assert( (pTerm->prereqRight & pLevel->notReady)!=0 ); - pAlt = findTerm(pWC, iCur, pTerm->u.leftColumn, notReady, WO_EQ|WO_IN, 0); - if( pAlt==0 ) continue; - if( pAlt->wtFlags & (TERM_CODED) ) continue; - testcase( pAlt->eOperator & WO_EQ ); - testcase( pAlt->eOperator & WO_IN ); - VdbeModuleComment((v, "begin transitive constraint")); - pEAlt = sqlite3StackAllocRaw(db, sizeof(*pEAlt)); - if( pEAlt ){ - *pEAlt = *pAlt->pExpr; - pEAlt->pLeft = pE->pLeft; - sqlite3ExprIfFalse(pParse, pEAlt, addrCont, SQLITE_JUMPIFNULL); - sqlite3StackFree(db, pEAlt); - } - } +#ifdef SQLITE_DEBUG + /* The following assert statements check that the binary search code + ** above found the right answer. This block serves no purpose other + ** than to invoke the asserts. */ + if( pParse->db->mallocFailed==0 ){ + if( res==0 ){ + /* If (res==0) is true, then pRec must be equal to sample i. */ + assert( inSample ); + assert( iCol==nField-1 ); + pRec->nField = nField; + assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec) + || pParse->db->mallocFailed + ); + }else{ + /* Unless i==pIdx->nSample, indicating that pRec is larger than + ** all samples in the aSample[] array, pRec must be smaller than the + ** (iCol+1) field prefix of sample i. */ + assert( i<=pIdx->nSample && i>=0 ); + pRec->nField = iCol+1; + assert( i==pIdx->nSample + || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0 + || pParse->db->mallocFailed ); - /* For a LEFT OUTER JOIN, generate code that will record the fact that - ** at least one row of the right table has matched the left table. - */ - if( pLevel->iLeftJoin ){ - pLevel->addrFirst = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_Integer, 1, pLevel->iLeftJoin); - VdbeComment((v, "record LEFT JOIN hit")); - sqlite3ExprCacheClear(pParse); - for(pTerm=pWC->a, j=0; jnTerm; j++, pTerm++){ - testcase( pTerm->wtFlags & TERM_VIRTUAL ); - testcase( pTerm->wtFlags & TERM_CODED ); - if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ - assert( pWInfo->untestedTerms ); - continue; + /* if i==0 and iCol==0, then record pRec is smaller than all samples + ** in the aSample[] array. Otherwise, if (iCol>0) then pRec must + ** be greater than or equal to the (iCol) field prefix of sample i. + ** If (i>0), then pRec must also be greater than sample (i-1). */ + if( iCol>0 ){ + pRec->nField = iCol; + assert( sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)<=0 + || pParse->db->mallocFailed ); + } + if( i>0 ){ + pRec->nField = nField; + assert( sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0 + || pParse->db->mallocFailed ); } - assert( pTerm->pExpr ); - sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); - pTerm->wtFlags |= TERM_CODED; } } +#endif /* ifdef SQLITE_DEBUG */ - return pLevel->notReady; -} - -#ifdef WHERETRACE_ENABLED -/* -** Print the content of a WhereTerm object -*/ -static void whereTermPrint(WhereTerm *pTerm, int iTerm){ - if( pTerm==0 ){ - sqlite3DebugPrintf("TERM-%-3d NULL\n", iTerm); + if( res==0 ){ + /* Record pRec is equal to sample i */ + assert( iCol==nField-1 ); + aStat[0] = aSample[i].anLt[iCol]; + aStat[1] = aSample[i].anEq[iCol]; }else{ - char zType[4]; - memcpy(zType, "...", 4); - if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V'; - if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E'; - if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) zType[2] = 'L'; - sqlite3DebugPrintf("TERM-%-3d %p %s cursor=%-3d prob=%-3d op=0x%03x\n", - iTerm, pTerm, zType, pTerm->leftCursor, pTerm->truthProb, - pTerm->eOperator); - sqlite3TreeViewExpr(0, pTerm->pExpr, 0); - } -} -#endif - -#ifdef WHERETRACE_ENABLED -/* -** Print a WhereLoop object for debugging purposes -*/ -static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ - WhereInfo *pWInfo = pWC->pWInfo; - int nb = 1+(pWInfo->pTabList->nSrc+7)/8; - struct SrcList_item *pItem = pWInfo->pTabList->a + p->iTab; - Table *pTab = pItem->pTab; - sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, - p->iTab, nb, p->maskSelf, nb, p->prereq); - sqlite3DebugPrintf(" %12s", - pItem->zAlias ? pItem->zAlias : pTab->zName); - if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ - const char *zName; - if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){ - if( strncmp(zName, "sqlite_autoindex_", 17)==0 ){ - int i = sqlite3Strlen30(zName) - 1; - while( zName[i]!='_' ) i--; - zName += i; - } - sqlite3DebugPrintf(".%-16s %2d", zName, p->u.btree.nEq); + /* At this point, the (iCol+1) field prefix of aSample[i] is the first + ** sample that is greater than pRec. Or, if i==pIdx->nSample then pRec + ** is larger than all samples in the array. */ + tRowcnt iUpper, iGap; + if( i>=pIdx->nSample ){ + iUpper = sqlite3LogEstToInt(pIdx->aiRowLogEst[0]); }else{ - sqlite3DebugPrintf("%20s",""); + iUpper = aSample[i].anLt[iCol]; } - }else{ - char *z; - if( p->u.vtab.idxStr ){ - z = sqlite3_mprintf("(%d,\"%s\",%x)", - p->u.vtab.idxNum, p->u.vtab.idxStr, p->u.vtab.omitMask); + + if( iLower>=iUpper ){ + iGap = 0; }else{ - z = sqlite3_mprintf("(%d,%x)", p->u.vtab.idxNum, p->u.vtab.omitMask); + iGap = iUpper - iLower; } - sqlite3DebugPrintf(" %-19s", z); - sqlite3_free(z); - } - if( p->wsFlags & WHERE_SKIPSCAN ){ - sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->nSkip); - }else{ - sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm); - } - sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut); - if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){ - int i; - for(i=0; inLTerm; i++){ - whereTermPrint(p->aLTerm[i], i); + if( roundUp ){ + iGap = (iGap*2)/3; + }else{ + iGap = iGap/3; } + aStat[0] = iLower + iGap; + aStat[1] = pIdx->aAvgEq[iCol]; } -} -#endif -/* -** Convert bulk memory into a valid WhereLoop that can be passed -** to whereLoopClear harmlessly. -*/ -static void whereLoopInit(WhereLoop *p){ - p->aLTerm = p->aLTermSpace; - p->nLTerm = 0; - p->nLSlot = ArraySize(p->aLTermSpace); - p->wsFlags = 0; + /* Restore the pRec->nField value before returning. */ + pRec->nField = nField; + return i; } +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ /* -** Clear the WhereLoop.u union. Leave WhereLoop.pLTerm intact. +** If it is not NULL, pTerm is a term that provides an upper or lower +** bound on a range scan. Without considering pTerm, it is estimated +** that the scan will visit nNew rows. This function returns the number +** estimated to be visited after taking pTerm into account. +** +** If the user explicitly specified a likelihood() value for this term, +** then the return value is the likelihood multiplied by the number of +** input rows. Otherwise, this function assumes that an "IS NOT NULL" term +** has a likelihood of 0.50, and any other term a likelihood of 0.25. */ -static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){ - if( p->wsFlags & (WHERE_VIRTUALTABLE|WHERE_AUTO_INDEX) ){ - if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 && p->u.vtab.needFree ){ - sqlite3_free(p->u.vtab.idxStr); - p->u.vtab.needFree = 0; - p->u.vtab.idxStr = 0; - }else if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 && p->u.btree.pIndex!=0 ){ - sqlite3DbFree(db, p->u.btree.pIndex->zColAff); - sqlite3DbFree(db, p->u.btree.pIndex); - p->u.btree.pIndex = 0; +static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ + LogEst nRet = nNew; + if( pTerm ){ + if( pTerm->truthProb<=0 ){ + nRet += pTerm->truthProb; + }else if( (pTerm->wtFlags & TERM_VNULL)==0 ){ + nRet -= 20; assert( 20==sqlite3LogEst(4) ); } } + return nRet; } -/* -** Deallocate internal memory used by a WhereLoop object -*/ -static void whereLoopClear(sqlite3 *db, WhereLoop *p){ - if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFree(db, p->aLTerm); - whereLoopClearUnion(db, p); - whereLoopInit(p); -} - -/* -** Increase the memory allocation for pLoop->aLTerm[] to be at least n. -*/ -static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){ - WhereTerm **paNew; - if( p->nLSlot>=n ) return SQLITE_OK; - n = (n+7)&~7; - paNew = sqlite3DbMallocRaw(db, sizeof(p->aLTerm[0])*n); - if( paNew==0 ) return SQLITE_NOMEM; - memcpy(paNew, p->aLTerm, sizeof(p->aLTerm[0])*p->nLSlot); - if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFree(db, p->aLTerm); - p->aLTerm = paNew; - p->nLSlot = n; - return SQLITE_OK; -} +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* -** Transfer content from the second pLoop into the first. +** Return the affinity for a single column of an index. */ -static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){ - whereLoopClearUnion(db, pTo); - if( whereLoopResize(db, pTo, pFrom->nLTerm) ){ - memset(&pTo->u, 0, sizeof(pTo->u)); - return SQLITE_NOMEM; - } - memcpy(pTo, pFrom, WHERE_LOOP_XFER_SZ); - memcpy(pTo->aLTerm, pFrom->aLTerm, pTo->nLTerm*sizeof(pTo->aLTerm[0])); - if( pFrom->wsFlags & WHERE_VIRTUALTABLE ){ - pFrom->u.vtab.needFree = 0; - }else if( (pFrom->wsFlags & WHERE_AUTO_INDEX)!=0 ){ - pFrom->u.btree.pIndex = 0; +static char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){ + assert( iCol>=0 && iColnColumn ); + if( !pIdx->zColAff ){ + if( sqlite3IndexAffinityStr(db, pIdx)==0 ) return SQLITE_AFF_BLOB; } - return SQLITE_OK; -} - -/* -** Delete a WhereLoop object -*/ -static void whereLoopDelete(sqlite3 *db, WhereLoop *p){ - whereLoopClear(db, p); - sqlite3DbFree(db, p); + return pIdx->zColAff[iCol]; } +#endif -/* -** Free a WhereInfo structure -*/ -static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ - if( ALWAYS(pWInfo) ){ - int i; - for(i=0; inLevel; i++){ - WhereLevel *pLevel = &pWInfo->a[i]; - if( pLevel->pWLoop && (pLevel->pWLoop->wsFlags & WHERE_IN_ABLE) ){ - sqlite3DbFree(db, pLevel->u.in.aInLoop); - } - } - whereClauseClear(&pWInfo->sWC); - while( pWInfo->pLoops ){ - WhereLoop *p = pWInfo->pLoops; - pWInfo->pLoops = p->pNextLoop; - whereLoopDelete(db, p); - } - sqlite3DbFree(db, pWInfo); - } -} -/* -** Return TRUE if all of the following are true: +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +/* +** This function is called to estimate the number of rows visited by a +** range-scan on a skip-scan index. For example: ** -** (1) X has the same or lower cost that Y -** (2) X is a proper subset of Y -** (3) X skips at least as many columns as Y +** CREATE INDEX i1 ON t1(a, b, c); +** SELECT * FROM t1 WHERE a=? AND c BETWEEN ? AND ?; ** -** By "proper subset" we mean that X uses fewer WHERE clause terms -** than Y and that every WHERE clause term used by X is also used -** by Y. +** Value pLoop->nOut is currently set to the estimated number of rows +** visited for scanning (a=? AND b=?). This function reduces that estimate +** by some factor to account for the (c BETWEEN ? AND ?) expression based +** on the stat4 data for the index. this scan will be peformed multiple +** times (once for each (a,b) combination that matches a=?) is dealt with +** by the caller. ** -** If X is a proper subset of Y then Y is a better choice and ought -** to have a lower cost. This routine returns TRUE when that cost -** relationship is inverted and needs to be adjusted. The third rule -** was added because if X uses skip-scan less than Y it still might -** deserve a lower cost even if it is a proper subset of Y. +** It does this by scanning through all stat4 samples, comparing values +** extracted from pLower and pUpper with the corresponding column in each +** sample. If L and U are the number of samples found to be less than or +** equal to the values extracted from pLower and pUpper respectively, and +** N is the total number of samples, the pLoop->nOut value is adjusted +** as follows: +** +** nOut = nOut * ( min(U - L, 1) / N ) +** +** If pLower is NULL, or a value cannot be extracted from the term, L is +** set to zero. If pUpper is NULL, or a value cannot be extracted from it, +** U is set to N. +** +** Normally, this function sets *pbDone to 1 before returning. However, +** if no value can be extracted from either pLower or pUpper (and so the +** estimate of the number of rows delivered remains unchanged), *pbDone +** is left as is. +** +** If an error occurs, an SQLite error code is returned. Otherwise, +** SQLITE_OK. */ -static int whereLoopCheaperProperSubset( - const WhereLoop *pX, /* First WhereLoop to compare */ - const WhereLoop *pY /* Compare against this WhereLoop */ +static int whereRangeSkipScanEst( + Parse *pParse, /* Parsing & code generating context */ + WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */ + WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ + WhereLoop *pLoop, /* Update the .nOut value of this loop */ + int *pbDone /* Set to true if at least one expr. value extracted */ ){ - int i, j; - if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){ - return 0; /* X is not a subset of Y */ + Index *p = pLoop->u.btree.pIndex; + int nEq = pLoop->u.btree.nEq; + sqlite3 *db = pParse->db; + int nLower = -1; + int nUpper = p->nSample+1; + int rc = SQLITE_OK; + u8 aff = sqlite3IndexColumnAffinity(db, p, nEq); + CollSeq *pColl; + + sqlite3_value *p1 = 0; /* Value extracted from pLower */ + sqlite3_value *p2 = 0; /* Value extracted from pUpper */ + sqlite3_value *pVal = 0; /* Value extracted from record */ + + pColl = sqlite3LocateCollSeq(pParse, p->azColl[nEq]); + if( pLower ){ + rc = sqlite3Stat4ValueFromExpr(pParse, pLower->pExpr->pRight, aff, &p1); + nLower = 0; } - if( pY->nSkip > pX->nSkip ) return 0; - if( pX->rRun >= pY->rRun ){ - if( pX->rRun > pY->rRun ) return 0; /* X costs more than Y */ - if( pX->nOut > pY->nOut ) return 0; /* X costs more than Y */ + if( pUpper && rc==SQLITE_OK ){ + rc = sqlite3Stat4ValueFromExpr(pParse, pUpper->pExpr->pRight, aff, &p2); + nUpper = p2 ? 0 : p->nSample; } - for(i=pX->nLTerm-1; i>=0; i--){ - if( pX->aLTerm[i]==0 ) continue; - for(j=pY->nLTerm-1; j>=0; j--){ - if( pY->aLTerm[j]==pX->aLTerm[i] ) break; + + if( p1 || p2 ){ + int i; + int nDiff; + for(i=0; rc==SQLITE_OK && inSample; i++){ + rc = sqlite3Stat4Column(db, p->aSample[i].p, p->aSample[i].n, nEq, &pVal); + if( rc==SQLITE_OK && p1 ){ + int res = sqlite3MemCompare(p1, pVal, pColl); + if( res>=0 ) nLower++; + } + if( rc==SQLITE_OK && p2 ){ + int res = sqlite3MemCompare(p2, pVal, pColl); + if( res>=0 ) nUpper++; + } } - if( j<0 ) return 0; /* X not a subset of Y since term X[i] not used by Y */ + nDiff = (nUpper - nLower); + if( nDiff<=0 ) nDiff = 1; + + /* If there is both an upper and lower bound specified, and the + ** comparisons indicate that they are close together, use the fallback + ** method (assume that the scan visits 1/64 of the rows) for estimating + ** the number of rows visited. Otherwise, estimate the number of rows + ** using the method described in the header comment for this function. */ + if( nDiff!=1 || pUpper==0 || pLower==0 ){ + int nAdjust = (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff)); + pLoop->nOut -= nAdjust; + *pbDone = 1; + WHERETRACE(0x10, ("range skip-scan regions: %u..%u adjust=%d est=%d\n", + nLower, nUpper, nAdjust*-1, pLoop->nOut)); + } + + }else{ + assert( *pbDone==0 ); } - return 1; /* All conditions meet */ + + sqlite3ValueFree(p1); + sqlite3ValueFree(p2); + sqlite3ValueFree(pVal); + + return rc; } +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ /* -** Try to adjust the cost of WhereLoop pTemplate upwards or downwards so -** that: +** This function is used to estimate the number of rows that will be visited +** by scanning an index for a range of values. The range may have an upper +** bound, a lower bound, or both. The WHERE clause terms that set the upper +** and lower bounds are represented by pLower and pUpper respectively. For +** example, assuming that index p is on t1(a): ** -** (1) pTemplate costs less than any other WhereLoops that are a proper -** subset of pTemplate +** ... FROM t1 WHERE a > ? AND a < ? ... +** |_____| |_____| +** | | +** pLower pUpper ** -** (2) pTemplate costs more than any other WhereLoops for which pTemplate -** is a proper subset. +** If either of the upper or lower bound is not present, then NULL is passed in +** place of the corresponding WhereTerm. ** -** To say "WhereLoop X is a proper subset of Y" means that X uses fewer -** WHERE clause terms than Y and that every WHERE clause term used by X is -** also used by Y. -*/ -static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ - if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; - for(; p; p=p->pNextLoop){ - if( p->iTab!=pTemplate->iTab ) continue; - if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; - if( whereLoopCheaperProperSubset(p, pTemplate) ){ - /* Adjust pTemplate cost downward so that it is cheaper than its - ** subset p. */ - WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", - pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut-1)); - pTemplate->rRun = p->rRun; - pTemplate->nOut = p->nOut - 1; - }else if( whereLoopCheaperProperSubset(pTemplate, p) ){ - /* Adjust pTemplate cost upward so that it is costlier than p since - ** pTemplate is a proper subset of p */ - WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", - pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut+1)); - pTemplate->rRun = p->rRun; - pTemplate->nOut = p->nOut + 1; - } - } -} - -/* -** Search the list of WhereLoops in *ppPrev looking for one that can be -** supplanted by pTemplate. +** The value in (pBuilder->pNew->u.btree.nEq) is the number of the index +** column subject to the range constraint. Or, equivalently, the number of +** equality constraints optimized by the proposed index scan. For example, +** assuming index p is on t1(a, b), and the SQL query is: ** -** Return NULL if the WhereLoop list contains an entry that can supplant -** pTemplate, in other words if pTemplate does not belong on the list. +** ... FROM t1 WHERE a = ? AND b > ? AND b < ? ... ** -** If pX is a WhereLoop that pTemplate can supplant, then return the -** link that points to pX. +** then nEq is set to 1 (as the range restricted column, b, is the second +** left-most column of the index). Or, if the query is: ** -** If pTemplate cannot supplant any existing element of the list but needs -** to be added to the list, then return a pointer to the tail of the list. +** ... FROM t1 WHERE a > ? AND a < ? ... +** +** then nEq is set to 0. +** +** When this function is called, *pnOut is set to the sqlite3LogEst() of the +** number of rows that the index scan is expected to visit without +** considering the range constraints. If nEq is 0, then *pnOut is the number of +** rows in the index. Assuming no error occurs, *pnOut is adjusted (reduced) +** to account for the range constraints pLower and pUpper. +** +** In the absence of sqlite_stat4 ANALYZE data, or if such data cannot be +** used, a single range inequality reduces the search space by a factor of 4. +** and a pair of constraints (x>? AND x123" Might be NULL */ + WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ + WhereLoop *pLoop /* Modify the .nOut and maybe .rRun fields */ ){ - WhereLoop *p; - for(p=(*ppPrev); p; ppPrev=&p->pNextLoop, p=*ppPrev){ - if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ){ - /* If either the iTab or iSortIdx values for two WhereLoop are different - ** then those WhereLoops need to be considered separately. Neither is - ** a candidate to replace the other. */ - continue; - } - /* In the current implementation, the rSetup value is either zero - ** or the cost of building an automatic index (NlogN) and the NlogN - ** is the same for compatible WhereLoops. */ - assert( p->rSetup==0 || pTemplate->rSetup==0 - || p->rSetup==pTemplate->rSetup ); + int rc = SQLITE_OK; + int nOut = pLoop->nOut; + LogEst nNew; - /* whereLoopAddBtree() always generates and inserts the automatic index - ** case first. Hence compatible candidate WhereLoops never have a larger - ** rSetup. Call this SETUP-INVARIANT */ - assert( p->rSetup>=pTemplate->rSetup ); +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + Index *p = pLoop->u.btree.pIndex; + int nEq = pLoop->u.btree.nEq; + + if( p->nSample>0 && nEqnSampleCol ){ + if( nEq==pBuilder->nRecValid ){ + UnpackedRecord *pRec = pBuilder->pRec; + tRowcnt a[2]; + u8 aff; + + /* Variable iLower will be set to the estimate of the number of rows in + ** the index that are less than the lower bound of the range query. The + ** lower bound being the concatenation of $P and $L, where $P is the + ** key-prefix formed by the nEq values matched against the nEq left-most + ** columns of the index, and $L is the value in pLower. + ** + ** Or, if pLower is NULL or $L cannot be extracted from it (because it + ** is not a simple variable or literal value), the lower bound of the + ** range is $P. Due to a quirk in the way whereKeyStats() works, even + ** if $L is available, whereKeyStats() is called for both ($P) and + ** ($P:$L) and the larger of the two returned values is used. + ** + ** Similarly, iUpper is to be set to the estimate of the number of rows + ** less than the upper bound of the range query. Where the upper bound + ** is either ($P) or ($P:$U). Again, even if $U is available, both values + ** of iUpper are requested of whereKeyStats() and the smaller used. + ** + ** The number of rows between the two bounds is then just iUpper-iLower. + */ + tRowcnt iLower; /* Rows less than the lower bound */ + tRowcnt iUpper; /* Rows less than the upper bound */ + int iLwrIdx = -2; /* aSample[] for the lower bound */ + int iUprIdx = -1; /* aSample[] for the upper bound */ + + if( pRec ){ + testcase( pRec->nField!=pBuilder->nRecValid ); + pRec->nField = pBuilder->nRecValid; + } + aff = sqlite3IndexColumnAffinity(pParse->db, p, nEq); + assert( nEq!=p->nKeyCol || aff==SQLITE_AFF_INTEGER ); + /* Determine iLower and iUpper using ($P) only. */ + if( nEq==0 ){ + iLower = 0; + iUpper = p->nRowEst0; + }else{ + /* Note: this call could be optimized away - since the same values must + ** have been requested when testing key $P in whereEqualScanEst(). */ + whereKeyStats(pParse, p, pRec, 0, a); + iLower = a[0]; + iUpper = a[0] + a[1]; + } + + assert( pLower==0 || (pLower->eOperator & (WO_GT|WO_GE))!=0 ); + assert( pUpper==0 || (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); + assert( p->aSortOrder!=0 ); + if( p->aSortOrder[nEq] ){ + /* The roles of pLower and pUpper are swapped for a DESC index */ + SWAP(WhereTerm*, pLower, pUpper); + } + + /* If possible, improve on the iLower estimate using ($P:$L). */ + if( pLower ){ + int bOk; /* True if value is extracted from pExpr */ + Expr *pExpr = pLower->pExpr->pRight; + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); + if( rc==SQLITE_OK && bOk ){ + tRowcnt iNew; + iLwrIdx = whereKeyStats(pParse, p, pRec, 0, a); + iNew = a[0] + ((pLower->eOperator & (WO_GT|WO_LE)) ? a[1] : 0); + if( iNew>iLower ) iLower = iNew; + nOut--; + pLower = 0; + } + } + + /* If possible, improve on the iUpper estimate using ($P:$U). */ + if( pUpper ){ + int bOk; /* True if value is extracted from pExpr */ + Expr *pExpr = pUpper->pExpr->pRight; + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); + if( rc==SQLITE_OK && bOk ){ + tRowcnt iNew; + iUprIdx = whereKeyStats(pParse, p, pRec, 1, a); + iNew = a[0] + ((pUpper->eOperator & (WO_GT|WO_LE)) ? a[1] : 0); + if( iNewpRec = pRec; + if( rc==SQLITE_OK ){ + if( iUpper>iLower ){ + nNew = sqlite3LogEst(iUpper - iLower); + /* TUNING: If both iUpper and iLower are derived from the same + ** sample, then assume they are 4x more selective. This brings + ** the estimated selectivity more in line with what it would be + ** if estimated without the use of STAT3/4 tables. */ + if( iLwrIdx==iUprIdx ) nNew -= 20; assert( 20==sqlite3LogEst(4) ); + }else{ + nNew = 10; assert( 10==sqlite3LogEst(2) ); + } + if( nNewwtFlags & TERM_VNULL)==0 ); + nNew = whereRangeAdjust(pLower, nOut); + nNew = whereRangeAdjust(pUpper, nNew); + + /* TUNING: If there is both an upper and lower limit and neither limit + ** has an application-defined likelihood(), assume the range is + ** reduced by an additional 75%. This means that, by default, an open-ended + ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the + ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to + ** match 1/64 of the index. */ + if( pLower && pLower->truthProb>0 && pUpper && pUpper->truthProb>0 ){ + nNew -= 20; + } + + nOut -= (pLower!=0) + (pUpper!=0); + if( nNew<10 ) nNew = 10; + if( nNewnOut>nOut ){ + WHERETRACE(0x10,("Range scan lowers nOut from %d to %d\n", + pLoop->nOut, nOut)); + } +#endif + pLoop->nOut = (LogEst)nOut; + return rc; +} + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +/* +** Estimate the number of rows that will be returned based on +** an equality constraint x=VALUE and where that VALUE occurs in +** the histogram data. This only works when x is the left-most +** column of an index and sqlite_stat3 histogram data is available +** for that index. When pExpr==NULL that means the constraint is +** "x IS NULL" instead of "x=VALUE". +** +** Write the estimated row count into *pnRow and return SQLITE_OK. +** If unable to make an estimate, leave *pnRow unchanged and return +** non-zero. +** +** This routine can fail if it is unable to load a collating sequence +** required for string comparison, or if unable to allocate memory +** for a UTF conversion required for comparison. The error is stored +** in the pParse structure. +*/ +static int whereEqualScanEst( + Parse *pParse, /* Parsing & code generating context */ + WhereLoopBuilder *pBuilder, + Expr *pExpr, /* Expression for VALUE in the x=VALUE constraint */ + tRowcnt *pnRow /* Write the revised row estimate here */ +){ + Index *p = pBuilder->pNew->u.btree.pIndex; + int nEq = pBuilder->pNew->u.btree.nEq; + UnpackedRecord *pRec = pBuilder->pRec; + u8 aff; /* Column affinity */ + int rc; /* Subfunction return code */ + tRowcnt a[2]; /* Statistics */ + int bOk; + + assert( nEq>=1 ); + assert( nEq<=p->nColumn ); + assert( p->aSample!=0 ); + assert( p->nSample>0 ); + assert( pBuilder->nRecValidnRecValid<(nEq-1) ){ + return SQLITE_NOTFOUND; + } + + /* This is an optimization only. The call to sqlite3Stat4ProbeSetValue() + ** below would return the same value. */ + if( nEq>=p->nColumn ){ + *pnRow = 1; + return SQLITE_OK; + } + + aff = sqlite3IndexColumnAffinity(pParse->db, p, nEq-1); + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq-1, &bOk); + pBuilder->pRec = pRec; + if( rc!=SQLITE_OK ) return rc; + if( bOk==0 ) return SQLITE_NOTFOUND; + pBuilder->nRecValid = nEq; + + whereKeyStats(pParse, p, pRec, 0, a); + WHERETRACE(0x10,("equality scan regions: %d\n", (int)a[1])); + *pnRow = a[1]; + + return rc; +} +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +/* +** Estimate the number of rows that will be returned based on +** an IN constraint where the right-hand side of the IN operator +** is a list of values. Example: +** +** WHERE x IN (1,2,3,4) +** +** Write the estimated row count into *pnRow and return SQLITE_OK. +** If unable to make an estimate, leave *pnRow unchanged and return +** non-zero. +** +** This routine can fail if it is unable to load a collating sequence +** required for string comparison, or if unable to allocate memory +** for a UTF conversion required for comparison. The error is stored +** in the pParse structure. +*/ +static int whereInScanEst( + Parse *pParse, /* Parsing & code generating context */ + WhereLoopBuilder *pBuilder, + ExprList *pList, /* The value list on the RHS of "x IN (v1,v2,v3,...)" */ + tRowcnt *pnRow /* Write the revised row estimate here */ +){ + Index *p = pBuilder->pNew->u.btree.pIndex; + i64 nRow0 = sqlite3LogEstToInt(p->aiRowLogEst[0]); + int nRecValid = pBuilder->nRecValid; + int rc = SQLITE_OK; /* Subfunction return code */ + tRowcnt nEst; /* Number of rows for a single term */ + tRowcnt nRowEst = 0; /* New estimate of the number of rows */ + int i; /* Loop counter */ + + assert( p->aSample!=0 ); + for(i=0; rc==SQLITE_OK && inExpr; i++){ + nEst = nRow0; + rc = whereEqualScanEst(pParse, pBuilder, pList->a[i].pExpr, &nEst); + nRowEst += nEst; + pBuilder->nRecValid = nRecValid; + } + + if( rc==SQLITE_OK ){ + if( nRowEst > nRow0 ) nRowEst = nRow0; + *pnRow = nRowEst; + WHERETRACE(0x10,("IN row estimate: est=%d\n", nRowEst)); + } + assert( pBuilder->nRecValid==nRecValid ); + return rc; +} +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ + + +#ifdef WHERETRACE_ENABLED +/* +** Print the content of a WhereTerm object +*/ +static void whereTermPrint(WhereTerm *pTerm, int iTerm){ + if( pTerm==0 ){ + sqlite3DebugPrintf("TERM-%-3d NULL\n", iTerm); + }else{ + char zType[4]; + memcpy(zType, "...", 4); + if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V'; + if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E'; + if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) zType[2] = 'L'; + sqlite3DebugPrintf( + "TERM-%-3d %p %s cursor=%-3d prob=%-3d op=0x%03x wtFlags=0x%04x\n", + iTerm, pTerm, zType, pTerm->leftCursor, pTerm->truthProb, + pTerm->eOperator, pTerm->wtFlags); + sqlite3TreeViewExpr(0, pTerm->pExpr, 0); + } +} +#endif + +#ifdef WHERETRACE_ENABLED +/* +** Print a WhereLoop object for debugging purposes +*/ +static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ + WhereInfo *pWInfo = pWC->pWInfo; + int nb = 1+(pWInfo->pTabList->nSrc+7)/8; + struct SrcList_item *pItem = pWInfo->pTabList->a + p->iTab; + Table *pTab = pItem->pTab; + sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, + p->iTab, nb, p->maskSelf, nb, p->prereq); + sqlite3DebugPrintf(" %12s", + pItem->zAlias ? pItem->zAlias : pTab->zName); + if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ + const char *zName; + if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){ + if( strncmp(zName, "sqlite_autoindex_", 17)==0 ){ + int i = sqlite3Strlen30(zName) - 1; + while( zName[i]!='_' ) i--; + zName += i; + } + sqlite3DebugPrintf(".%-16s %2d", zName, p->u.btree.nEq); + }else{ + sqlite3DebugPrintf("%20s",""); + } + }else{ + char *z; + if( p->u.vtab.idxStr ){ + z = sqlite3_mprintf("(%d,\"%s\",%x)", + p->u.vtab.idxNum, p->u.vtab.idxStr, p->u.vtab.omitMask); + }else{ + z = sqlite3_mprintf("(%d,%x)", p->u.vtab.idxNum, p->u.vtab.omitMask); + } + sqlite3DebugPrintf(" %-19s", z); + sqlite3_free(z); + } + if( p->wsFlags & WHERE_SKIPSCAN ){ + sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->nSkip); + }else{ + sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm); + } + sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut); + if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){ + int i; + for(i=0; inLTerm; i++){ + whereTermPrint(p->aLTerm[i], i); + } + } +} +#endif + +/* +** Convert bulk memory into a valid WhereLoop that can be passed +** to whereLoopClear harmlessly. +*/ +static void whereLoopInit(WhereLoop *p){ + p->aLTerm = p->aLTermSpace; + p->nLTerm = 0; + p->nLSlot = ArraySize(p->aLTermSpace); + p->wsFlags = 0; +} + +/* +** Clear the WhereLoop.u union. Leave WhereLoop.pLTerm intact. +*/ +static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){ + if( p->wsFlags & (WHERE_VIRTUALTABLE|WHERE_AUTO_INDEX) ){ + if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 && p->u.vtab.needFree ){ + sqlite3_free(p->u.vtab.idxStr); + p->u.vtab.needFree = 0; + p->u.vtab.idxStr = 0; + }else if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 && p->u.btree.pIndex!=0 ){ + sqlite3DbFree(db, p->u.btree.pIndex->zColAff); + sqlite3DbFree(db, p->u.btree.pIndex); + p->u.btree.pIndex = 0; + } + } +} + +/* +** Deallocate internal memory used by a WhereLoop object +*/ +static void whereLoopClear(sqlite3 *db, WhereLoop *p){ + if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFree(db, p->aLTerm); + whereLoopClearUnion(db, p); + whereLoopInit(p); +} + +/* +** Increase the memory allocation for pLoop->aLTerm[] to be at least n. +*/ +static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){ + WhereTerm **paNew; + if( p->nLSlot>=n ) return SQLITE_OK; + n = (n+7)&~7; + paNew = sqlite3DbMallocRaw(db, sizeof(p->aLTerm[0])*n); + if( paNew==0 ) return SQLITE_NOMEM; + memcpy(paNew, p->aLTerm, sizeof(p->aLTerm[0])*p->nLSlot); + if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFree(db, p->aLTerm); + p->aLTerm = paNew; + p->nLSlot = n; + return SQLITE_OK; +} + +/* +** Transfer content from the second pLoop into the first. +*/ +static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){ + whereLoopClearUnion(db, pTo); + if( whereLoopResize(db, pTo, pFrom->nLTerm) ){ + memset(&pTo->u, 0, sizeof(pTo->u)); + return SQLITE_NOMEM; + } + memcpy(pTo, pFrom, WHERE_LOOP_XFER_SZ); + memcpy(pTo->aLTerm, pFrom->aLTerm, pTo->nLTerm*sizeof(pTo->aLTerm[0])); + if( pFrom->wsFlags & WHERE_VIRTUALTABLE ){ + pFrom->u.vtab.needFree = 0; + }else if( (pFrom->wsFlags & WHERE_AUTO_INDEX)!=0 ){ + pFrom->u.btree.pIndex = 0; + } + return SQLITE_OK; +} + +/* +** Delete a WhereLoop object +*/ +static void whereLoopDelete(sqlite3 *db, WhereLoop *p){ + whereLoopClear(db, p); + sqlite3DbFree(db, p); +} + +/* +** Free a WhereInfo structure +*/ +static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ + if( ALWAYS(pWInfo) ){ + int i; + for(i=0; inLevel; i++){ + WhereLevel *pLevel = &pWInfo->a[i]; + if( pLevel->pWLoop && (pLevel->pWLoop->wsFlags & WHERE_IN_ABLE) ){ + sqlite3DbFree(db, pLevel->u.in.aInLoop); + } + } + sqlite3WhereClauseClear(&pWInfo->sWC); + while( pWInfo->pLoops ){ + WhereLoop *p = pWInfo->pLoops; + pWInfo->pLoops = p->pNextLoop; + whereLoopDelete(db, p); + } + sqlite3DbFree(db, pWInfo); + } +} + +/* +** Return TRUE if all of the following are true: +** +** (1) X has the same or lower cost that Y +** (2) X is a proper subset of Y +** (3) X skips at least as many columns as Y +** +** By "proper subset" we mean that X uses fewer WHERE clause terms +** than Y and that every WHERE clause term used by X is also used +** by Y. +** +** If X is a proper subset of Y then Y is a better choice and ought +** to have a lower cost. This routine returns TRUE when that cost +** relationship is inverted and needs to be adjusted. The third rule +** was added because if X uses skip-scan less than Y it still might +** deserve a lower cost even if it is a proper subset of Y. +*/ +static int whereLoopCheaperProperSubset( + const WhereLoop *pX, /* First WhereLoop to compare */ + const WhereLoop *pY /* Compare against this WhereLoop */ +){ + int i, j; + if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){ + return 0; /* X is not a subset of Y */ + } + if( pY->nSkip > pX->nSkip ) return 0; + if( pX->rRun >= pY->rRun ){ + if( pX->rRun > pY->rRun ) return 0; /* X costs more than Y */ + if( pX->nOut > pY->nOut ) return 0; /* X costs more than Y */ + } + for(i=pX->nLTerm-1; i>=0; i--){ + if( pX->aLTerm[i]==0 ) continue; + for(j=pY->nLTerm-1; j>=0; j--){ + if( pY->aLTerm[j]==pX->aLTerm[i] ) break; + } + if( j<0 ) return 0; /* X not a subset of Y since term X[i] not used by Y */ + } + return 1; /* All conditions meet */ +} + +/* +** Try to adjust the cost of WhereLoop pTemplate upwards or downwards so +** that: +** +** (1) pTemplate costs less than any other WhereLoops that are a proper +** subset of pTemplate +** +** (2) pTemplate costs more than any other WhereLoops for which pTemplate +** is a proper subset. +** +** To say "WhereLoop X is a proper subset of Y" means that X uses fewer +** WHERE clause terms than Y and that every WHERE clause term used by X is +** also used by Y. +*/ +static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ + if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; + for(; p; p=p->pNextLoop){ + if( p->iTab!=pTemplate->iTab ) continue; + if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; + if( whereLoopCheaperProperSubset(p, pTemplate) ){ + /* Adjust pTemplate cost downward so that it is cheaper than its + ** subset p. */ + WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", + pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut-1)); + pTemplate->rRun = p->rRun; + pTemplate->nOut = p->nOut - 1; + }else if( whereLoopCheaperProperSubset(pTemplate, p) ){ + /* Adjust pTemplate cost upward so that it is costlier than p since + ** pTemplate is a proper subset of p */ + WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", + pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut+1)); + pTemplate->rRun = p->rRun; + pTemplate->nOut = p->nOut + 1; + } + } +} + +/* +** Search the list of WhereLoops in *ppPrev looking for one that can be +** supplanted by pTemplate. +** +** Return NULL if the WhereLoop list contains an entry that can supplant +** pTemplate, in other words if pTemplate does not belong on the list. +** +** If pX is a WhereLoop that pTemplate can supplant, then return the +** link that points to pX. +** +** If pTemplate cannot supplant any existing element of the list but needs +** to be added to the list, then return a pointer to the tail of the list. +*/ +static WhereLoop **whereLoopFindLesser( + WhereLoop **ppPrev, + const WhereLoop *pTemplate +){ + WhereLoop *p; + for(p=(*ppPrev); p; ppPrev=&p->pNextLoop, p=*ppPrev){ + if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ){ + /* If either the iTab or iSortIdx values for two WhereLoop are different + ** then those WhereLoops need to be considered separately. Neither is + ** a candidate to replace the other. */ + continue; + } + /* In the current implementation, the rSetup value is either zero + ** or the cost of building an automatic index (NlogN) and the NlogN + ** is the same for compatible WhereLoops. */ + assert( p->rSetup==0 || pTemplate->rSetup==0 + || p->rSetup==pTemplate->rSetup ); + + /* whereLoopAddBtree() always generates and inserts the automatic index + ** case first. Hence compatible candidate WhereLoops never have a larger + ** rSetup. Call this SETUP-INVARIANT */ + assert( p->rSetup>=pTemplate->rSetup ); /* Any loop using an appliation-defined index (or PRIMARY KEY or ** UNIQUE constraint) with one or more == constraints is better @@ -120680,18 +123351,20 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ** and prereqs. */ if( pBuilder->pOrSet!=0 ){ + if( pTemplate->nLTerm ){ #if WHERETRACE_ENABLED - u16 n = pBuilder->pOrSet->n; - int x = + u16 n = pBuilder->pOrSet->n; + int x = #endif - whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun, + whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun, pTemplate->nOut); #if WHERETRACE_ENABLED /* 0x8 */ - if( sqlite3WhereTrace & 0x8 ){ - sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n); - whereLoopPrint(pTemplate, pBuilder->pWC); - } + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n); + whereLoopPrint(pTemplate, pBuilder->pWC); + } #endif + } return SQLITE_OK; } @@ -120824,8 +123497,9 @@ static void whereLoopOutputAdjust( /* In the absence of explicit truth probabilities, use heuristics to ** guess a reasonable truth probability. */ pLoop->nOut--; - if( pTerm->eOperator&WO_EQ ){ + if( pTerm->eOperator&(WO_EQ|WO_IS) ){ Expr *pRight = pTerm->pExpr->pRight; + testcase( pTerm->pExpr->op==TK_IS ); if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){ k = 10; }else{ @@ -120880,7 +123554,6 @@ static int whereLoopAddBtreeIndex( u16 saved_nSkip; /* Original value of pNew->nSkip */ u32 saved_wsFlags; /* Original value of pNew->wsFlags */ LogEst saved_nOut; /* Original value of pNew->nOut */ - int iCol; /* Index of the column in the table */ int rc = SQLITE_OK; /* Return code */ LogEst rSize; /* Number of rows in the table */ LogEst rLogSize; /* Logarithm of table size */ @@ -120893,24 +123566,23 @@ static int whereLoopAddBtreeIndex( assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 ); if( pNew->wsFlags & WHERE_BTM_LIMIT ){ opMask = WO_LT|WO_LE; - }else if( pProbe->tnum<=0 || (pSrc->jointype & JT_LEFT)!=0 ){ + }else if( /*pProbe->tnum<=0 ||*/ (pSrc->fg.jointype & JT_LEFT)!=0 ){ opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE; }else{ - opMask = WO_EQ|WO_IN|WO_ISNULL|WO_GT|WO_GE|WO_LT|WO_LE; + opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS; } if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); assert( pNew->u.btree.nEqnColumn ); - iCol = pProbe->aiColumn[pNew->u.btree.nEq]; - pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, - opMask, pProbe); saved_nEq = pNew->u.btree.nEq; saved_nSkip = pNew->nSkip; saved_nLTerm = pNew->nLTerm; saved_wsFlags = pNew->wsFlags; saved_prereq = pNew->prereq; saved_nOut = pNew->nOut; + pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, saved_nEq, + opMask, pProbe); pNew->rSetup = 0; rSize = pProbe->aiRowLogEst[0]; rLogSize = estLog(rSize); @@ -120923,7 +123595,7 @@ static int whereLoopAddBtreeIndex( int nRecValid = pBuilder->nRecValid; #endif if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) - && (iCol<0 || pSrc->pTab->aCol[iCol].notNull) + && indexColumnNotNull(pProbe, saved_nEq) ){ continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */ } @@ -120959,9 +123631,13 @@ static int whereLoopAddBtreeIndex( assert( nIn>0 ); /* RHS always has 2 or more terms... The parser ** changes "x IN (?)" into "x=?". */ - }else if( eOp & (WO_EQ) ){ + }else if( eOp & (WO_EQ|WO_IS) ){ + int iCol = pProbe->aiColumn[saved_nEq]; pNew->wsFlags |= WHERE_COLUMN_EQ; - if( iCol<0 || (nInMul==0 && pNew->u.btree.nEq==pProbe->nKeyCol-1) ){ + assert( saved_nEq==pNew->u.btree.nEq ); + if( iCol==XN_ROWID + || (iCol>0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1) + ){ if( iCol>=0 && pProbe->uniqNotNull==0 ){ pNew->wsFlags |= WHERE_UNQ_WANTED; }else{ @@ -121009,10 +123685,10 @@ static int whereLoopAddBtreeIndex( whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew); }else{ int nEq = ++pNew->u.btree.nEq; - assert( eOp & (WO_ISNULL|WO_EQ|WO_IN) ); + assert( eOp & (WO_ISNULL|WO_EQ|WO_IN|WO_IS) ); assert( pNew->nOut==saved_nOut ); - if( pTerm->truthProb<=0 && iCol>=0 ){ + if( pTerm->truthProb<=0 && pProbe->aiColumn[saved_nEq]>=0 ){ assert( (eOp & WO_IN) || nIn==0 ); testcase( eOp & WO_IN ); pNew->nOut += pTerm->truthProb; @@ -121026,8 +123702,9 @@ static int whereLoopAddBtreeIndex( && ((eOp & WO_IN)==0 || !ExprHasProperty(pTerm->pExpr, EP_xIsSelect)) ){ Expr *pExpr = pTerm->pExpr; - if( (eOp & (WO_EQ|WO_ISNULL))!=0 ){ + if( (eOp & (WO_EQ|WO_ISNULL|WO_IS))!=0 ){ testcase( eOp & WO_EQ ); + testcase( eOp & WO_IS ); testcase( eOp & WO_ISNULL ); rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut); }else{ @@ -121146,18 +123823,25 @@ static int indexMightHelpWithOrderBy( int iCursor ){ ExprList *pOB; + ExprList *aColExpr; int ii, jj; if( pIndex->bUnordered ) return 0; if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0; for(ii=0; iinExpr; ii++){ Expr *pExpr = sqlite3ExprSkipCollate(pOB->a[ii].pExpr); - if( pExpr->op!=TK_COLUMN ) return 0; - if( pExpr->iTable==iCursor ){ + if( pExpr->op==TK_COLUMN && pExpr->iTable==iCursor ){ if( pExpr->iColumn<0 ) return 1; for(jj=0; jjnKeyCol; jj++){ if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1; } + }else if( (aColExpr = pIndex->aColExpr)!=0 ){ + for(jj=0; jjnKeyCol; jj++){ + if( pIndex->aiColumn[jj]!=XN_EXPR ) continue; + if( sqlite3ExprCompare(pExpr,aColExpr->a[jj].pExpr,iCursor)==0 ){ + return 1; + } + } } } return 0; @@ -121187,6 +123871,10 @@ static Bitmask columnsInIndex(Index *pIdx){ static int whereUsablePartialIndex(int iTab, WhereClause *pWC, Expr *pWhere){ int i; WhereTerm *pTerm; + while( pWhere->op==TK_AND ){ + if( !whereUsablePartialIndex(iTab,pWC,pWhere->pLeft) ) return 0; + pWhere = pWhere->pRight; + } for(i=0, pTerm=pWC->a; inTerm; i++, pTerm++){ Expr *pExpr = pTerm->pExpr; if( sqlite3ExprImpliesExpr(pExpr, pWhere, iTab) @@ -121262,9 +123950,9 @@ static int whereLoopAddBtree( pWC = pBuilder->pWC; assert( !IsVirtual(pSrc->pTab) ); - if( pSrc->pIndex ){ + if( pSrc->pIBIndex ){ /* An INDEXED BY clause specifies a particular index to use */ - pProbe = pSrc->pIndex; + pProbe = pSrc->pIBIndex; }else if( !HasRowid(pTab) ){ pProbe = pTab->pIndex; }else{ @@ -121284,7 +123972,7 @@ static int whereLoopAddBtree( aiRowEstPk[0] = pTab->nRowLogEst; aiRowEstPk[1] = 0; pFirst = pSrc->pTab->pIndex; - if( pSrc->notIndexed==0 ){ + if( pSrc->fg.notIndexed==0 ){ /* The real indices of the table are only considered if the ** NOT INDEXED qualifier is omitted from the FROM clause */ sPk.pNext = pFirst; @@ -121296,15 +123984,14 @@ static int whereLoopAddBtree( #ifndef SQLITE_OMIT_AUTOMATIC_INDEX /* Automatic indexes */ - if( !pBuilder->pOrSet + if( !pBuilder->pOrSet /* Not part of an OR optimization */ && (pWInfo->wctrlFlags & WHERE_NO_AUTOINDEX)==0 && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0 - && pSrc->pIndex==0 - && !pSrc->viaCoroutine - && !pSrc->notIndexed - && HasRowid(pTab) - && !pSrc->isCorrelated - && !pSrc->isRecursive + && pSrc->pIBIndex==0 /* Has no INDEXED BY clause */ + && !pSrc->fg.notIndexed /* Has no NOT INDEXED clause */ + && HasRowid(pTab) /* Not WITHOUT ROWID table. (FIXME: Why not?) */ + && !pSrc->fg.isCorrelated /* Not a correlated subquery */ + && !pSrc->fg.isRecursive /* Not a recursive common table expression. */ ){ /* Generate auto-index WhereLoops */ WhereTerm *pTerm; @@ -121425,7 +124112,7 @@ static int whereLoopAddBtree( /* If there was an INDEXED BY clause, then only that one index is ** considered. */ - if( pSrc->pIndex ) break; + if( pSrc->pIBIndex ) break; } return rc; } @@ -121434,10 +124121,32 @@ static int whereLoopAddBtree( /* ** Add all WhereLoop objects for a table of the join identified by ** pBuilder->pNew->iTab. That table is guaranteed to be a virtual table. +** +** If there are no LEFT or CROSS JOIN joins in the query, both mExtra and +** mUnusable are set to 0. Otherwise, mExtra is a mask of all FROM clause +** entries that occur before the virtual table in the FROM clause and are +** separated from it by at least one LEFT or CROSS JOIN. Similarly, the +** mUnusable mask contains all FROM clause entries that occur after the +** virtual table and are separated from it by at least one LEFT or +** CROSS JOIN. +** +** For example, if the query were: +** +** ... FROM t1, t2 LEFT JOIN t3, t4, vt CROSS JOIN t5, t6; +** +** then mExtra corresponds to (t1, t2) and mUnusable to (t5, t6). +** +** All the tables in mExtra must be scanned before the current virtual +** table. So any terms for which all prerequisites are satisfied by +** mExtra may be specified as "usable" in all calls to xBestIndex. +** Conversely, all tables in mUnusable must be scanned after the current +** virtual table, so any terms for which the prerequisites overlap with +** mUnusable should always be configured as "not-usable" for xBestIndex. */ static int whereLoopAddVirtual( WhereLoopBuilder *pBuilder, /* WHERE clause information */ - Bitmask mExtra + Bitmask mExtra, /* Tables that must be scanned before this one */ + Bitmask mUnusable /* Tables that must be scanned after this one */ ){ WhereInfo *pWInfo; /* WHERE analysis context */ Parse *pParse; /* The parsing context */ @@ -121458,6 +124167,7 @@ static int whereLoopAddVirtual( WhereLoop *pNew; int rc = SQLITE_OK; + assert( (mExtra & mUnusable)==0 ); pWInfo = pBuilder->pWInfo; pParse = pWInfo->pParse; db = pParse->db; @@ -121466,7 +124176,7 @@ static int whereLoopAddVirtual( pSrc = &pWInfo->pTabList->a[pNew->iTab]; pTab = pSrc->pTab; assert( IsVirtual(pTab) ); - pIdxInfo = allocateIndexInfo(pParse, pWC, pSrc, pBuilder->pOrderBy); + pIdxInfo = allocateIndexInfo(pParse, pWC, mUnusable, pSrc,pBuilder->pOrderBy); if( pIdxInfo==0 ) return SQLITE_NOMEM; pNew->prereq = 0; pNew->rSetup = 0; @@ -121496,7 +124206,7 @@ static int whereLoopAddVirtual( if( (pTerm->eOperator & WO_IN)!=0 ){ seenIn = 1; } - if( pTerm->prereqRight!=0 ){ + if( (pTerm->prereqRight & ~mExtra)!=0 ){ seenVar = 1; }else if( (pTerm->eOperator & WO_IN)==0 ){ pIdxCons->usable = 1; @@ -121504,7 +124214,7 @@ static int whereLoopAddVirtual( break; case 1: /* Constants with IN operators */ assert( seenIn ); - pIdxCons->usable = (pTerm->prereqRight==0); + pIdxCons->usable = (pTerm->prereqRight & ~mExtra)==0; break; case 2: /* Variables without IN */ assert( seenVar ); @@ -121524,6 +124234,7 @@ static int whereLoopAddVirtual( pIdxInfo->orderByConsumed = 0; pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2; pIdxInfo->estimatedRows = 25; + pIdxInfo->idxFlags = 0; rc = vtabBestIndex(pParse, pTab, pIdxInfo); if( rc ) goto whereLoopAddVtab_exit; pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; @@ -121569,6 +124280,7 @@ static int whereLoopAddVirtual( ** (2) Multiple outputs from a single IN value will not merge ** together. */ pIdxInfo->orderByConsumed = 0; + pIdxInfo->idxFlags &= ~SQLITE_INDEX_SCAN_UNIQUE; } } } @@ -121584,6 +124296,14 @@ static int whereLoopAddVirtual( pNew->rSetup = 0; pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost); pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows); + + /* Set the WHERE_ONEROW flag if the xBestIndex() method indicated + ** that the scan will visit at most one row. Clear it otherwise. */ + if( pIdxInfo->idxFlags & SQLITE_INDEX_SCAN_UNIQUE ){ + pNew->wsFlags |= WHERE_ONEROW; + }else{ + pNew->wsFlags &= ~WHERE_ONEROW; + } whereLoopInsert(pBuilder, pNew); if( pNew->u.vtab.needFree ){ sqlite3_free(pNew->u.vtab.idxStr); @@ -121603,7 +124323,11 @@ whereLoopAddVtab_exit: ** Add WhereLoop entries to handle OR terms. This works for either ** btrees or virtual tables. */ -static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ +static int whereLoopAddOr( + WhereLoopBuilder *pBuilder, + Bitmask mExtra, + Bitmask mUnusable +){ WhereInfo *pWInfo = pBuilder->pWInfo; WhereClause *pWC; WhereLoop *pNew; @@ -121662,14 +124386,14 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ #endif #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pItem->pTab) ){ - rc = whereLoopAddVirtual(&sSubBuild, mExtra); + rc = whereLoopAddVirtual(&sSubBuild, mExtra, mUnusable); }else #endif { rc = whereLoopAddBtree(&sSubBuild, mExtra); } if( rc==SQLITE_OK ){ - rc = whereLoopAddOr(&sSubBuild, mExtra); + rc = whereLoopAddOr(&sSubBuild, mExtra, mUnusable); } assert( rc==SQLITE_OK || sCur.n==0 ); if( sCur.n==0 ){ @@ -121731,33 +124455,43 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ int iTab; SrcList *pTabList = pWInfo->pTabList; struct SrcList_item *pItem; + struct SrcList_item *pEnd = &pTabList->a[pWInfo->nLevel]; sqlite3 *db = pWInfo->pParse->db; - int nTabList = pWInfo->nLevel; int rc = SQLITE_OK; - u8 priorJoinType = 0; WhereLoop *pNew; + u8 priorJointype = 0; /* Loop over the tables in the join, from left to right */ pNew = pBuilder->pNew; whereLoopInit(pNew); - for(iTab=0, pItem=pTabList->a; iTaba; pItemiTab = iTab; - pNew->maskSelf = getMask(&pWInfo->sMaskSet, pItem->iCursor); - if( ((pItem->jointype|priorJoinType) & (JT_LEFT|JT_CROSS))!=0 ){ + pNew->maskSelf = sqlite3WhereGetMask(&pWInfo->sMaskSet, pItem->iCursor); + if( ((pItem->fg.jointype|priorJointype) & (JT_LEFT|JT_CROSS))!=0 ){ + /* This condition is true when pItem is the FROM clause term on the + ** right-hand-side of a LEFT or CROSS JOIN. */ mExtra = mPrior; } - priorJoinType = pItem->jointype; + priorJointype = pItem->fg.jointype; if( IsVirtual(pItem->pTab) ){ - rc = whereLoopAddVirtual(pBuilder, mExtra); + struct SrcList_item *p; + for(p=&pItem[1]; pfg.jointype & (JT_LEFT|JT_CROSS)) ){ + mUnusable |= sqlite3WhereGetMask(&pWInfo->sMaskSet, p->iCursor); + } + } + rc = whereLoopAddVirtual(pBuilder, mExtra, mUnusable); }else{ rc = whereLoopAddBtree(pBuilder, mExtra); } if( rc==SQLITE_OK ){ - rc = whereLoopAddOr(pBuilder, mExtra); + rc = whereLoopAddOr(pBuilder, mExtra, mUnusable); } mPrior |= pNew->maskSelf; if( rc || db->mallocFailed ) break; } + whereLoopClear(db, pNew); return rc; } @@ -121863,10 +124597,10 @@ static i8 wherePathSatisfiesOrderBy( pOBExpr = sqlite3ExprSkipCollate(pOrderBy->a[i].pExpr); if( pOBExpr->op!=TK_COLUMN ) continue; if( pOBExpr->iTable!=iCur ) continue; - pTerm = findTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn, - ~ready, WO_EQ|WO_ISNULL, 0); + pTerm = sqlite3WhereFindTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn, + ~ready, WO_EQ|WO_ISNULL|WO_IS, 0); if( pTerm==0 ) continue; - if( (pTerm->eOperator&WO_EQ)!=0 && pOBExpr->iColumn>=0 ){ + if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 && pOBExpr->iColumn>=0 ){ const char *z1, *z2; pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); if( !pColl ) pColl = db->pDfltColl; @@ -121875,6 +124609,7 @@ static i8 wherePathSatisfiesOrderBy( if( !pColl ) pColl = db->pDfltColl; z2 = pColl->zName; if( sqlite3StrICmp(z1, z2)!=0 ) continue; + testcase( pTerm->pExpr->op==TK_IS ); } obSat |= MASKBIT(i); } @@ -121890,7 +124625,8 @@ static i8 wherePathSatisfiesOrderBy( nKeyCol = pIndex->nKeyCol; nColumn = pIndex->nColumn; assert( nColumn==nKeyCol+1 || !HasRowid(pIndex->pTable) ); - assert( pIndex->aiColumn[nColumn-1]==(-1) || !HasRowid(pIndex->pTable)); + assert( pIndex->aiColumn[nColumn-1]==XN_ROWID + || !HasRowid(pIndex->pTable)); isOrderDistinct = IsUniqueIndex(pIndex); } @@ -121905,7 +124641,7 @@ static i8 wherePathSatisfiesOrderBy( /* Skip over == and IS NULL terms */ if( ju.btree.nEq && pLoop->nSkip==0 - && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL))!=0 + && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL|WO_IS))!=0 ){ if( i & WO_ISNULL ){ testcase( isOrderDistinct ); @@ -121922,7 +124658,7 @@ static i8 wherePathSatisfiesOrderBy( revIdx = pIndex->aSortOrder[j]; if( iColumn==pIndex->pTable->iPKey ) iColumn = -1; }else{ - iColumn = -1; + iColumn = XN_ROWID; revIdx = 0; } @@ -121948,9 +124684,15 @@ static i8 wherePathSatisfiesOrderBy( testcase( wctrlFlags & WHERE_GROUPBY ); testcase( wctrlFlags & WHERE_DISTINCTBY ); if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0; - if( pOBExpr->op!=TK_COLUMN ) continue; - if( pOBExpr->iTable!=iCur ) continue; - if( pOBExpr->iColumn!=iColumn ) continue; + if( iColumn>=(-1) ){ + if( pOBExpr->op!=TK_COLUMN ) continue; + if( pOBExpr->iTable!=iCur ) continue; + if( pOBExpr->iColumn!=iColumn ) continue; + }else{ + if( sqlite3ExprCompare(pOBExpr,pIndex->aColExpr->a[j].pExpr,iCur) ){ + continue; + } + } if( iColumn>=0 ){ pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); if( !pColl ) pColl = db->pDfltColl; @@ -121999,7 +124741,7 @@ static i8 wherePathSatisfiesOrderBy( Bitmask mTerm; if( MASKBIT(i) & obSat ) continue; p = pOrderBy->a[i].pExpr; - mTerm = exprTableUsage(&pWInfo->sMaskSet,p); + mTerm = sqlite3WhereExprUsage(&pWInfo->sMaskSet,p); if( mTerm==0 && !sqlite3ExprIsConstant(p) ) continue; if( (mTerm&~orderDistinctMask)==0 ){ obSat |= MASKBIT(i); @@ -122472,14 +125214,15 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pItem = pWInfo->pTabList->a; pTab = pItem->pTab; if( IsVirtual(pTab) ) return 0; - if( pItem->zIndex ) return 0; + if( pItem->fg.isIndexedBy ) return 0; iCur = pItem->iCursor; pWC = &pWInfo->sWC; pLoop = pBuilder->pNew; pLoop->wsFlags = 0; pLoop->nSkip = 0; - pTerm = findTerm(pWC, iCur, -1, 0, WO_EQ, 0); + pTerm = sqlite3WhereFindTerm(pWC, iCur, -1, 0, WO_EQ|WO_IS, 0); if( pTerm ){ + testcase( pTerm->eOperator & WO_IS ); pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW; pLoop->aLTerm[0] = pTerm; pLoop->nLTerm = 1; @@ -122488,14 +125231,17 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pLoop->rRun = 33; /* 33==sqlite3LogEst(10) */ }else{ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + int opMask; assert( pLoop->aLTermSpace==pLoop->aLTerm ); if( !IsUniqueIndex(pIdx) || pIdx->pPartIdxWhere!=0 || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace) ) continue; + opMask = pIdx->uniqNotNull ? (WO_EQ|WO_IS) : WO_EQ; for(j=0; jnKeyCol; j++){ - pTerm = findTerm(pWC, iCur, pIdx->aiColumn[j], 0, WO_EQ, pIdx); + pTerm = sqlite3WhereFindTerm(pWC, iCur, j, 0, opMask, pIdx); if( pTerm==0 ) break; + testcase( pTerm->eOperator & WO_IS ); pLoop->aLTerm[j] = pTerm; } if( j!=pIdx->nKeyCol ) continue; @@ -122514,7 +125260,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ if( pLoop->wsFlags ){ pLoop->nOut = (LogEst)1; pWInfo->a[0].pWLoop = pLoop; - pLoop->maskSelf = getMask(&pWInfo->sMaskSet, iCur); + pLoop->maskSelf = sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); pWInfo->a[0].iTabCur = iCur; pWInfo->nRowOut = 1; if( pWInfo->pOrderBy ) pWInfo->nOBSat = pWInfo->pOrderBy->nExpr; @@ -122639,6 +125385,10 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( sqlite3 *db; /* Database connection */ int rc; /* Return code */ + assert( (wctrlFlags & WHERE_ONEPASS_MULTIROW)==0 || ( + (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 + && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 + )); /* Variable initialization */ db = pParse->db; @@ -122694,6 +125444,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(v); pWInfo->wctrlFlags = wctrlFlags; pWInfo->savedNQueryLoop = pParse->nQueryLoop; + assert( pWInfo->eOnePass==ONEPASS_OFF ); /* ONEPASS defaults to OFF */ pMaskSet = &pWInfo->sMaskSet; sWLB.pWInfo = pWInfo; sWLB.pWC = &pWInfo->sWC; @@ -122708,8 +125459,8 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** subexpression is separated by an AND operator. */ initMaskSet(pMaskSet); - whereClauseInit(&pWInfo->sWC, pWInfo); - whereSplit(&pWInfo->sWC, pWhere, TK_AND); + sqlite3WhereClauseInit(&pWInfo->sWC, pWInfo); + sqlite3WhereSplit(&pWInfo->sWC, pWhere, TK_AND); /* Special case: a WHERE clause that is constant. Evaluate the ** expression and either jump over all of the code or fall thru. @@ -122733,14 +125484,12 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( /* Assign a bit from the bitmask to every term in the FROM clause. ** - ** When assigning bitmask values to FROM clause cursors, it must be - ** the case that if X is the bitmask for the N-th FROM clause term then - ** the bitmask for all FROM clause terms to the left of the N-th term - ** is (X-1). An expression from the ON clause of a LEFT JOIN can use - ** its Expr.iRightJoinTable value to find the bitmask of the right table - ** of the join. Subtracting one from the right table bitmask gives a - ** bitmask for all tables to the left of the join. Knowing the bitmask - ** for all tables to the left of a left join is important. Ticket #3015. + ** The N-th term of the FROM clause is assigned a bitmask of 1<nSrc tables in ** pTabList, not just the first nTabList tables. nTabList is normally @@ -122749,27 +125498,18 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( */ for(ii=0; iinSrc; ii++){ createMask(pMaskSet, pTabList->a[ii].iCursor); + sqlite3WhereTabFuncArgs(pParse, &pTabList->a[ii], &pWInfo->sWC); } -#ifndef NDEBUG - { - Bitmask toTheLeft = 0; - for(ii=0; iinSrc; ii++){ - Bitmask m = getMask(pMaskSet, pTabList->a[ii].iCursor); - assert( (m-1)==toTheLeft ); - toTheLeft |= m; - } +#ifdef SQLITE_DEBUG + for(ii=0; iinSrc; ii++){ + Bitmask m = sqlite3WhereGetMask(pMaskSet, pTabList->a[ii].iCursor); + assert( m==MASKBIT(ii) ); } #endif - /* Analyze all of the subexpressions. Note that exprAnalyze() might - ** add new virtual terms onto the end of the WHERE clause. We do not - ** want to analyze these virtual terms, so start analyzing at the end - ** and work forward so that the added virtual terms are never processed. - */ - exprAnalyzeAll(pTabList, &pWInfo->sWC); - if( db->mallocFailed ){ - goto whereBeginError; - } + /* Analyze all of the subexpressions. */ + sqlite3WhereExprAnalyze(pTabList, &pWInfo->sWC); + if( db->mallocFailed ) goto whereBeginError; if( wctrlFlags & WHERE_WANT_DISTINCT ){ if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pResultSet) ){ @@ -122783,10 +125523,10 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( } /* Construct the WhereLoop objects */ - WHERETRACE(0xffff,("*** Optimizer Start ***\n")); + WHERETRACE(0xffff,("*** Optimizer Start *** (wctrlFlags: 0x%x)\n", + wctrlFlags)); #if defined(WHERETRACE_ENABLED) - /* Display all terms of the WHERE clause */ - if( sqlite3WhereTrace & 0x100 ){ + if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */ int i; for(i=0; inTerm; i++){ whereTermPrint(&sWLB.pWC->a[i], i); @@ -122798,13 +125538,12 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( rc = whereLoopAddAll(&sWLB); if( rc ) goto whereBeginError; - /* Display all of the WhereLoop objects if wheretrace is enabled */ -#ifdef WHERETRACE_ENABLED /* !=0 */ - if( sqlite3WhereTrace ){ +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace ){ /* Display all of the WhereLoop objects */ WhereLoop *p; int i; - static char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz" - "ABCDEFGHIJKLMNOPQRSTUVWYXZ"; + static const char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz" + "ABCDEFGHIJKLMNOPQRSTUVWYXZ"; for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){ p->cId = zLabel[i%sizeof(zLabel)]; whereLoopPrint(p, sWLB.pWC); @@ -122825,7 +125564,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( if( pParse->nErr || NEVER(db->mallocFailed) ){ goto whereBeginError; } -#ifdef WHERETRACE_ENABLED /* !=0 */ +#ifdef WHERETRACE_ENABLED if( sqlite3WhereTrace ){ sqlite3DebugPrintf("---- Solution nRow=%d", pWInfo->nRowOut); if( pWInfo->nOBSat>0 ){ @@ -122856,12 +125595,14 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( && pResultSet!=0 && OptimizationEnabled(db, SQLITE_OmitNoopJoin) ){ - Bitmask tabUsed = exprListTableUsage(pMaskSet, pResultSet); - if( sWLB.pOrderBy ) tabUsed |= exprListTableUsage(pMaskSet, sWLB.pOrderBy); + Bitmask tabUsed = sqlite3WhereExprListUsage(pMaskSet, pResultSet); + if( sWLB.pOrderBy ){ + tabUsed |= sqlite3WhereExprListUsage(pMaskSet, sWLB.pOrderBy); + } while( pWInfo->nLevel>=2 ){ WhereTerm *pTerm, *pEnd; pLoop = pWInfo->a[pWInfo->nLevel-1].pWLoop; - if( (pWInfo->pTabList->a[pLoop->iTab].jointype & JT_LEFT)==0 ) break; + if( (pWInfo->pTabList->a[pLoop->iTab].fg.jointype & JT_LEFT)==0 ) break; if( (wctrlFlags & WHERE_WANT_DISTINCT)==0 && (pLoop->wsFlags & WHERE_ONEROW)==0 ){ @@ -122888,21 +125629,25 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( /* If the caller is an UPDATE or DELETE statement that is requesting ** to use a one-pass algorithm, determine if this is appropriate. ** The one-pass algorithm only works if the WHERE clause constrains - ** the statement to update a single row. + ** the statement to update or delete a single row. */ assert( (wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || pWInfo->nLevel==1 ); - if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 - && (pWInfo->a[0].pWLoop->wsFlags & WHERE_ONEROW)!=0 ){ - pWInfo->okOnePass = 1; - if( HasRowid(pTabList->a[0].pTab) ){ - pWInfo->a[0].pWLoop->wsFlags &= ~WHERE_IDX_ONLY; + if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 ){ + int wsFlags = pWInfo->a[0].pWLoop->wsFlags; + int bOnerow = (wsFlags & WHERE_ONEROW)!=0; + if( bOnerow || ( (wctrlFlags & WHERE_ONEPASS_MULTIROW) + && 0==(wsFlags & WHERE_VIRTUALTABLE) + )){ + pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI; + if( HasRowid(pTabList->a[0].pTab) ){ + pWInfo->a[0].pWLoop->wsFlags &= ~WHERE_IDX_ONLY; + } } } /* Open all tables in the pTabList and any indices selected for ** searching those tables. */ - notReady = ~(Bitmask)0; for(ii=0, pLevel=pWInfo->a; iiwsFlags & WHERE_IDX_ONLY)==0 && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 ){ int op = OP_OpenRead; - if( pWInfo->okOnePass ){ + if( pWInfo->eOnePass!=ONEPASS_OFF ){ op = OP_OpenWrite; pWInfo->aiCurOnePass[0] = pTabItem->iCursor; }; sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op); assert( pTabItem->iCursor==pLevel->iTabCur ); - testcase( !pWInfo->okOnePass && pTab->nCol==BMS-1 ); - testcase( !pWInfo->okOnePass && pTab->nCol==BMS ); - if( !pWInfo->okOnePass && pTab->nColeOnePass==ONEPASS_OFF && pTab->nCol==BMS-1 ); + testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS ); + if( pWInfo->eOnePass==ONEPASS_OFF && pTab->nColcolUsed; int n = 0; for(; b; b=b>>1, n++){} @@ -122943,6 +125688,10 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( SQLITE_INT_TO_PTR(n), P4_INT32); assert( n<=pTab->nCol ); } +#ifdef SQLITE_ENABLE_COLUMN_USED_MASK + sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, pTabItem->iCursor, 0, 0, + (const u8*)&pTabItem->colUsed, P4_INT64); +#endif }else{ sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); } @@ -122959,7 +125708,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** WITHOUT ROWID table. No need for a separate index */ iIndexCur = pLevel->iTabCur; op = 0; - }else if( pWInfo->okOnePass ){ + }else if( pWInfo->eOnePass!=ONEPASS_OFF ){ Index *pJ = pTabItem->pTab->pIndex; iIndexCur = iIdxCur; assert( wctrlFlags & WHERE_ONEPASS_DESIRED ); @@ -122988,10 +125737,24 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ); /* Hint to COMDB2 */ } VdbeComment((v, "%s", pIx->zName)); +#ifdef SQLITE_ENABLE_COLUMN_USED_MASK + { + u64 colUsed = 0; + int ii, jj; + for(ii=0; iinColumn; ii++){ + jj = pIx->aiColumn[ii]; + if( jj<0 ) continue; + if( jj>63 ) jj = 63; + if( (pTabItem->colUsed & MASKBIT(jj))==0 ) continue; + colUsed |= ((u64)1)<<(ii<63 ? ii : 63); + } + sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, iIndexCur, 0, 0, + (u8*)&colUsed, P4_INT64); + } +#endif /* SQLITE_ENABLE_COLUMN_USED_MASK */ } } if( iDb>=0 ) sqlite3CodeVerifySchema(pParse, iDb); - notReady &= ~getMask(&pWInfo->sMaskSet, pTabItem->iCursor); } pWInfo->iTop = sqlite3VdbeCurrentAddr(v); if( db->mallocFailed ) goto whereBeginError; @@ -123013,14 +125776,14 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( if( db->mallocFailed ) goto whereBeginError; } #endif - addrExplain = explainOneScan( + addrExplain = sqlite3WhereExplainOneScan( pParse, pTabList, pLevel, ii, pLevel->iFrom, wctrlFlags ); pLevel->addrBody = sqlite3VdbeCurrentAddr(v); - notReady = codeOneLoopStart(pWInfo, ii, notReady); + notReady = sqlite3WhereCodeOneLoopStart(pWInfo, ii, notReady); pWInfo->iContinue = pLevel->addrCont; if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_ONETABLE_ONLY)==0 ){ - addScanStatus(v, pTabList, pLevel, addrExplain); + sqlite3WhereAddScanStatus(v, pTabList, pLevel, addrExplain); } } @@ -123082,7 +125845,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ } sqlite3VdbeResolveLabel(v, pLevel->addrBrk); if( pLevel->addrSkip ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrSkip); + sqlite3VdbeGoto(v, pLevel->addrSkip); VdbeComment((v, "next skip-scan on %s", pLoop->u.btree.pIndex->zName)); sqlite3VdbeJumpHere(v, pLevel->addrSkip); sqlite3VdbeJumpHere(v, pLevel->addrSkip-2); @@ -123110,7 +125873,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ if( pLevel->op==OP_Return ){ sqlite3VdbeAddOp2(v, OP_Gosub, pLevel->p1, pLevel->addrFirst); }else{ - sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrFirst); + sqlite3VdbeGoto(v, pLevel->addrFirst); } sqlite3VdbeJumpHere(v, addr); } @@ -123134,26 +125897,12 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ pLoop = pLevel->pWLoop; /* For a co-routine, change all OP_Column references to the table of - ** the co-routine into OP_SCopy of result contained in a register. + ** the co-routine into OP_Copy of result contained in a register. ** OP_Rowid becomes OP_Null. */ - if( pTabItem->viaCoroutine && !db->mallocFailed ){ - last = sqlite3VdbeCurrentAddr(v); - k = pLevel->addrBody; - pOp = sqlite3VdbeGetOp(v, k); - for(; kp1!=pLevel->iTabCur ) continue; - if( pOp->opcode==OP_Column ){ - pOp->opcode = OP_Copy; - pOp->p1 = pOp->p2 + pTabItem->regResult; - pOp->p2 = pOp->p3; - pOp->p3 = 0; - }else if( pOp->opcode==OP_Rowid ){ - pOp->opcode = OP_Null; - pOp->p1 = 0; - pOp->p3 = 0; - } - } + if( pTabItem->fg.viaCoroutine && !db->mallocFailed ){ + translateColumnToCopy(v, pLevel->addrBody, pLevel->iTabCur, + pTabItem->regResult, 0); continue; } @@ -123167,7 +125916,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ && (pWInfo->wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 ){ int ws = pLoop->wsFlags; - if( !pWInfo->okOnePass && (ws & WHERE_IDX_ONLY)==0 ){ + if( pWInfo->eOnePass==ONEPASS_OFF && (ws & WHERE_IDX_ONLY)==0 ){ sqlite3VdbeAddOp1(v, OP_Close, pTabItem->iCursor); } if( (ws & WHERE_INDEXED)!=0 @@ -123194,7 +125943,10 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ }else if( pLoop->wsFlags & WHERE_MULTI_OR ){ pIdx = pLevel->u.pCovidx; } - if( pIdx && !db->mallocFailed ){ + if( pIdx + && (pWInfo->eOnePass==ONEPASS_OFF || !HasRowid(pIdx->pTable)) + && !db->mallocFailed + ){ last = sqlite3VdbeCurrentAddr(v); k = pLevel->addrBody; pOp = sqlite3VdbeGetOp(v, k); @@ -123206,6 +125958,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ if( !HasRowid(pTab) ){ Index *pPk = sqlite3PrimaryKeyIndex(pTab); x = pPk->aiColumn[x]; + assert( x>=0 ); } x = sqlite3ColumnOfIndex(pIdx, x); if( x>=0 ){ @@ -123243,6 +125996,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ ** in the input grammar file. */ /* #include */ +/* #include "sqliteInt.h" */ /* ** Disable all error recovery processing in the parser push-down @@ -123384,6 +126138,29 @@ struct AttachKey { int type; Token key; }; pOut->zStart = pPreOp->z; pOut->zEnd = pOperand->zEnd; } + + /* Add a single new term to an ExprList that is used to store a + ** list of identifiers. Report an error if the ID list contains + ** a COLLATE clause or an ASC or DESC keyword, except ignore the + ** error while parsing a legacy schema. + */ + static ExprList *parserAddExprIdListTerm( + Parse *pParse, + ExprList *pPrior, + Token *pIdToken, + int hasCollate, + int sortOrder + ){ + ExprList *p = sqlite3ExprListAppend(pParse, pPrior, 0); + if( (hasCollate || sortOrder!=SQLITE_SO_UNDEFINED) + && pParse->db->init.busy==0 + ){ + sqlite3ErrorMsg(pParse, "syntax error after column name \"%.*s\"", + pIdToken->n, pIdToken->z); + } + sqlite3ExprListSetName(pParse, p, pIdToken, 1); + return p; + } /* Next is all token values, in a form suitable for use by makeheaders. ** This section will be null unless lemon is run with the -m switch. */ @@ -123428,10 +126205,17 @@ struct AttachKey { int type; Token key; }; ** sqlite3ParserARG_PDECL A parameter declaration for the %extra_argument ** sqlite3ParserARG_STORE Code to store %extra_argument into yypParser ** sqlite3ParserARG_FETCH Code to extract %extra_argument from yypParser -** YYNSTATE the combined number of states. -** YYNRULE the number of rules in the grammar ** YYERRORSYMBOL is the code number of the error symbol. If not ** defined, then do no error processing. +** YYNSTATE the combined number of states. +** YYNRULE the number of rules in the grammar +** YY_MAX_SHIFT Maximum value for shift actions +** YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions +** YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions +** YY_MIN_REDUCE Maximum value for reduce actions +** YY_ERROR_ACTION The yy_action[] code for syntax error +** YY_ACCEPT_ACTION The yy_action[] code for accept +** YY_NO_ACTION The yy_action[] code for no-op */ #define YYCODETYPE unsigned char #define YYNOCODE 254 @@ -123464,12 +126248,17 @@ typedef union { #define sqlite3ParserARG_PDECL ,Parse *pParse #define sqlite3ParserARG_FETCH Parse *pParse = yypParser->pParse #define sqlite3ParserARG_STORE yypParser->pParse = pParse -#define YYNSTATE 642 -#define YYNRULE 327 #define YYFALLBACK 1 -#define YY_NO_ACTION (YYNSTATE+YYNRULE+2) -#define YY_ACCEPT_ACTION (YYNSTATE+YYNRULE+1) -#define YY_ERROR_ACTION (YYNSTATE+YYNRULE) +#define YYNSTATE 436 +#define YYNRULE 328 +#define YY_MAX_SHIFT 435 +#define YY_MIN_SHIFTREDUCE 649 +#define YY_MAX_SHIFTREDUCE 976 +#define YY_MIN_REDUCE 977 +#define YY_MAX_REDUCE 1304 +#define YY_ERROR_ACTION 1305 +#define YY_ACCEPT_ACTION 1306 +#define YY_NO_ACTION 1307 /* The yyzerominor constant is used to initialize instances of ** YYMINORTYPE objects to zero. */ @@ -123496,16 +126285,20 @@ static const YYMINORTYPE yyzerominor = { 0 }; ** Suppose the action integer is N. Then the action is determined as ** follows ** -** 0 <= N < YYNSTATE Shift N. That is, push the lookahead +** 0 <= N <= YY_MAX_SHIFT Shift N. That is, push the lookahead ** token onto the stack and goto state N. ** -** YYNSTATE <= N < YYNSTATE+YYNRULE Reduce by rule N-YYNSTATE. +** N between YY_MIN_SHIFTREDUCE Shift to an arbitrary state then +** and YY_MAX_SHIFTREDUCE reduce by rule N-YY_MIN_SHIFTREDUCE. ** -** N == YYNSTATE+YYNRULE A syntax error has occurred. +** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE +** and YY_MAX_REDUCE + +** N == YY_ERROR_ACTION A syntax error has occurred. ** -** N == YYNSTATE+YYNRULE+1 The parser accepts its input. +** N == YY_ACCEPT_ACTION The parser accepts its input. ** -** N == YYNSTATE+YYNRULE+2 No such action. Denotes unused +** N == YY_NO_ACTION No such action. Denotes unused ** slots in the yy_action[] table. ** ** The action table is constructed as a single large table named yy_action[]. @@ -123535,463 +126328,446 @@ static const YYMINORTYPE yyzerominor = { 0 }; ** shifting non-terminals after a reduce. ** yy_default[] Default action for each state. */ -#define YY_ACTTAB_COUNT (1497) +#define YY_ACTTAB_COUNT (1501) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 306, 212, 432, 955, 639, 191, 955, 295, 559, 88, - /* 10 */ 88, 88, 88, 81, 86, 86, 86, 86, 85, 85, - /* 20 */ 84, 84, 84, 83, 330, 185, 184, 183, 635, 635, - /* 30 */ 292, 606, 606, 88, 88, 88, 88, 683, 86, 86, - /* 40 */ 86, 86, 85, 85, 84, 84, 84, 83, 330, 16, - /* 50 */ 436, 597, 89, 90, 80, 600, 599, 601, 601, 87, - /* 60 */ 87, 88, 88, 88, 88, 684, 86, 86, 86, 86, - /* 70 */ 85, 85, 84, 84, 84, 83, 330, 306, 559, 84, - /* 80 */ 84, 84, 83, 330, 65, 86, 86, 86, 86, 85, - /* 90 */ 85, 84, 84, 84, 83, 330, 635, 635, 634, 633, - /* 100 */ 182, 682, 550, 379, 376, 375, 17, 322, 606, 606, - /* 110 */ 371, 198, 479, 91, 374, 82, 79, 165, 85, 85, - /* 120 */ 84, 84, 84, 83, 330, 598, 635, 635, 107, 89, - /* 130 */ 90, 80, 600, 599, 601, 601, 87, 87, 88, 88, - /* 140 */ 88, 88, 186, 86, 86, 86, 86, 85, 85, 84, - /* 150 */ 84, 84, 83, 330, 306, 594, 594, 142, 328, 327, - /* 160 */ 484, 249, 344, 238, 635, 635, 634, 633, 585, 448, - /* 170 */ 526, 525, 229, 388, 1, 394, 450, 584, 449, 635, - /* 180 */ 635, 635, 635, 319, 395, 606, 606, 199, 157, 273, - /* 190 */ 382, 268, 381, 187, 635, 635, 634, 633, 311, 555, - /* 200 */ 266, 593, 593, 266, 347, 588, 89, 90, 80, 600, - /* 210 */ 599, 601, 601, 87, 87, 88, 88, 88, 88, 478, - /* 220 */ 86, 86, 86, 86, 85, 85, 84, 84, 84, 83, - /* 230 */ 330, 306, 272, 536, 634, 633, 146, 610, 197, 310, - /* 240 */ 575, 182, 482, 271, 379, 376, 375, 506, 21, 634, - /* 250 */ 633, 634, 633, 635, 635, 374, 611, 574, 548, 440, - /* 260 */ 111, 563, 606, 606, 634, 633, 324, 479, 608, 608, - /* 270 */ 608, 300, 435, 573, 119, 407, 210, 162, 562, 883, - /* 280 */ 592, 592, 306, 89, 90, 80, 600, 599, 601, 601, - /* 290 */ 87, 87, 88, 88, 88, 88, 506, 86, 86, 86, - /* 300 */ 86, 85, 85, 84, 84, 84, 83, 330, 620, 111, - /* 310 */ 635, 635, 361, 606, 606, 358, 249, 349, 248, 433, - /* 320 */ 243, 479, 586, 634, 633, 195, 611, 93, 119, 221, - /* 330 */ 575, 497, 534, 534, 89, 90, 80, 600, 599, 601, - /* 340 */ 601, 87, 87, 88, 88, 88, 88, 574, 86, 86, - /* 350 */ 86, 86, 85, 85, 84, 84, 84, 83, 330, 306, - /* 360 */ 77, 429, 638, 573, 589, 530, 240, 230, 242, 105, - /* 370 */ 249, 349, 248, 515, 588, 208, 460, 529, 564, 173, - /* 380 */ 634, 633, 970, 144, 430, 2, 424, 228, 380, 557, - /* 390 */ 606, 606, 190, 153, 159, 158, 514, 51, 632, 631, - /* 400 */ 630, 71, 536, 432, 954, 196, 610, 954, 614, 45, - /* 410 */ 18, 89, 90, 80, 600, 599, 601, 601, 87, 87, - /* 420 */ 88, 88, 88, 88, 261, 86, 86, 86, 86, 85, - /* 430 */ 85, 84, 84, 84, 83, 330, 306, 608, 608, 608, - /* 440 */ 542, 424, 402, 385, 241, 506, 451, 320, 211, 543, - /* 450 */ 164, 436, 386, 293, 451, 587, 108, 496, 111, 334, - /* 460 */ 391, 591, 424, 614, 27, 452, 453, 606, 606, 72, - /* 470 */ 257, 70, 259, 452, 339, 342, 564, 582, 68, 415, - /* 480 */ 469, 328, 327, 62, 614, 45, 110, 393, 89, 90, - /* 490 */ 80, 600, 599, 601, 601, 87, 87, 88, 88, 88, - /* 500 */ 88, 152, 86, 86, 86, 86, 85, 85, 84, 84, - /* 510 */ 84, 83, 330, 306, 110, 499, 520, 538, 402, 389, - /* 520 */ 424, 110, 566, 500, 593, 593, 454, 82, 79, 165, - /* 530 */ 424, 591, 384, 564, 340, 615, 188, 162, 424, 350, - /* 540 */ 616, 424, 614, 44, 606, 606, 445, 582, 300, 434, - /* 550 */ 151, 19, 614, 9, 568, 580, 348, 615, 469, 567, - /* 560 */ 614, 26, 616, 614, 45, 89, 90, 80, 600, 599, - /* 570 */ 601, 601, 87, 87, 88, 88, 88, 88, 411, 86, - /* 580 */ 86, 86, 86, 85, 85, 84, 84, 84, 83, 330, - /* 590 */ 306, 579, 110, 578, 521, 282, 433, 398, 400, 255, - /* 600 */ 486, 82, 79, 165, 487, 164, 82, 79, 165, 488, - /* 610 */ 488, 364, 387, 424, 544, 544, 509, 350, 362, 155, - /* 620 */ 191, 606, 606, 559, 642, 640, 333, 82, 79, 165, - /* 630 */ 305, 564, 507, 312, 357, 614, 45, 329, 596, 595, - /* 640 */ 194, 337, 89, 90, 80, 600, 599, 601, 601, 87, - /* 650 */ 87, 88, 88, 88, 88, 424, 86, 86, 86, 86, - /* 660 */ 85, 85, 84, 84, 84, 83, 330, 306, 20, 323, - /* 670 */ 150, 263, 211, 543, 421, 596, 595, 614, 22, 424, - /* 680 */ 193, 424, 284, 424, 391, 424, 509, 424, 577, 424, - /* 690 */ 186, 335, 424, 559, 424, 313, 120, 546, 606, 606, - /* 700 */ 67, 614, 47, 614, 50, 614, 48, 614, 100, 614, - /* 710 */ 99, 614, 101, 576, 614, 102, 614, 109, 326, 89, - /* 720 */ 90, 80, 600, 599, 601, 601, 87, 87, 88, 88, - /* 730 */ 88, 88, 424, 86, 86, 86, 86, 85, 85, 84, - /* 740 */ 84, 84, 83, 330, 306, 424, 311, 424, 585, 54, - /* 750 */ 424, 516, 517, 590, 614, 112, 424, 584, 424, 572, - /* 760 */ 424, 195, 424, 571, 424, 67, 424, 614, 94, 614, - /* 770 */ 98, 424, 614, 97, 264, 606, 606, 195, 614, 46, - /* 780 */ 614, 96, 614, 30, 614, 49, 614, 115, 614, 114, - /* 790 */ 418, 229, 388, 614, 113, 306, 89, 90, 80, 600, - /* 800 */ 599, 601, 601, 87, 87, 88, 88, 88, 88, 424, - /* 810 */ 86, 86, 86, 86, 85, 85, 84, 84, 84, 83, - /* 820 */ 330, 119, 424, 590, 110, 372, 606, 606, 195, 53, - /* 830 */ 250, 614, 29, 195, 472, 438, 729, 190, 302, 498, - /* 840 */ 14, 523, 641, 2, 614, 43, 306, 89, 90, 80, - /* 850 */ 600, 599, 601, 601, 87, 87, 88, 88, 88, 88, - /* 860 */ 424, 86, 86, 86, 86, 85, 85, 84, 84, 84, - /* 870 */ 83, 330, 424, 613, 964, 964, 354, 606, 606, 420, - /* 880 */ 312, 64, 614, 42, 391, 355, 283, 437, 301, 255, - /* 890 */ 414, 410, 495, 492, 614, 28, 471, 306, 89, 90, - /* 900 */ 80, 600, 599, 601, 601, 87, 87, 88, 88, 88, - /* 910 */ 88, 424, 86, 86, 86, 86, 85, 85, 84, 84, - /* 920 */ 84, 83, 330, 424, 110, 110, 110, 110, 606, 606, - /* 930 */ 110, 254, 13, 614, 41, 532, 531, 283, 481, 531, - /* 940 */ 457, 284, 119, 561, 356, 614, 40, 284, 306, 89, - /* 950 */ 78, 80, 600, 599, 601, 601, 87, 87, 88, 88, - /* 960 */ 88, 88, 424, 86, 86, 86, 86, 85, 85, 84, - /* 970 */ 84, 84, 83, 330, 110, 424, 341, 220, 555, 606, - /* 980 */ 606, 351, 555, 318, 614, 95, 413, 255, 83, 330, - /* 990 */ 284, 284, 255, 640, 333, 356, 255, 614, 39, 306, - /* 1000 */ 356, 90, 80, 600, 599, 601, 601, 87, 87, 88, - /* 1010 */ 88, 88, 88, 424, 86, 86, 86, 86, 85, 85, - /* 1020 */ 84, 84, 84, 83, 330, 424, 317, 316, 141, 465, - /* 1030 */ 606, 606, 219, 619, 463, 614, 10, 417, 462, 255, - /* 1040 */ 189, 510, 553, 351, 207, 363, 161, 614, 38, 315, - /* 1050 */ 218, 255, 255, 80, 600, 599, 601, 601, 87, 87, - /* 1060 */ 88, 88, 88, 88, 424, 86, 86, 86, 86, 85, - /* 1070 */ 85, 84, 84, 84, 83, 330, 76, 419, 255, 3, - /* 1080 */ 878, 461, 424, 247, 331, 331, 614, 37, 217, 76, - /* 1090 */ 419, 390, 3, 216, 215, 422, 4, 331, 331, 424, - /* 1100 */ 547, 12, 424, 545, 614, 36, 424, 541, 422, 424, - /* 1110 */ 540, 424, 214, 424, 408, 424, 539, 403, 605, 605, - /* 1120 */ 237, 614, 25, 119, 614, 24, 588, 408, 614, 45, - /* 1130 */ 118, 614, 35, 614, 34, 614, 33, 614, 23, 588, - /* 1140 */ 60, 223, 603, 602, 513, 378, 73, 74, 140, 139, - /* 1150 */ 424, 110, 265, 75, 426, 425, 59, 424, 610, 73, - /* 1160 */ 74, 549, 402, 404, 424, 373, 75, 426, 425, 604, - /* 1170 */ 138, 610, 614, 11, 392, 76, 419, 181, 3, 614, - /* 1180 */ 32, 271, 369, 331, 331, 493, 614, 31, 149, 608, - /* 1190 */ 608, 608, 607, 15, 422, 365, 614, 8, 137, 489, - /* 1200 */ 136, 190, 608, 608, 608, 607, 15, 485, 176, 135, - /* 1210 */ 7, 252, 477, 408, 174, 133, 175, 474, 57, 56, - /* 1220 */ 132, 130, 119, 76, 419, 588, 3, 468, 245, 464, - /* 1230 */ 171, 331, 331, 125, 123, 456, 447, 122, 446, 104, - /* 1240 */ 336, 231, 422, 166, 154, 73, 74, 332, 116, 431, - /* 1250 */ 121, 309, 75, 426, 425, 222, 106, 610, 308, 637, - /* 1260 */ 204, 408, 629, 627, 628, 6, 200, 428, 427, 290, - /* 1270 */ 203, 622, 201, 588, 62, 63, 289, 66, 419, 399, - /* 1280 */ 3, 401, 288, 92, 143, 331, 331, 287, 608, 608, - /* 1290 */ 608, 607, 15, 73, 74, 227, 422, 325, 69, 416, - /* 1300 */ 75, 426, 425, 612, 412, 610, 192, 61, 569, 209, - /* 1310 */ 396, 226, 278, 225, 383, 408, 527, 558, 276, 533, - /* 1320 */ 552, 528, 321, 523, 370, 508, 180, 588, 494, 179, - /* 1330 */ 366, 117, 253, 269, 522, 503, 608, 608, 608, 607, - /* 1340 */ 15, 551, 502, 58, 274, 524, 178, 73, 74, 304, - /* 1350 */ 501, 368, 303, 206, 75, 426, 425, 491, 360, 610, - /* 1360 */ 213, 177, 483, 131, 345, 298, 297, 296, 202, 294, - /* 1370 */ 480, 490, 466, 134, 172, 129, 444, 346, 470, 128, - /* 1380 */ 314, 459, 103, 127, 126, 148, 124, 167, 443, 235, - /* 1390 */ 608, 608, 608, 607, 15, 442, 439, 623, 234, 299, - /* 1400 */ 145, 583, 291, 377, 581, 160, 119, 156, 270, 636, - /* 1410 */ 971, 169, 279, 626, 520, 625, 473, 624, 170, 621, - /* 1420 */ 618, 119, 168, 55, 409, 423, 537, 609, 286, 285, - /* 1430 */ 405, 570, 560, 556, 5, 52, 458, 554, 147, 267, - /* 1440 */ 519, 504, 518, 406, 262, 239, 260, 512, 343, 511, - /* 1450 */ 258, 353, 565, 256, 224, 251, 359, 277, 275, 476, - /* 1460 */ 475, 246, 352, 244, 467, 455, 236, 233, 232, 307, - /* 1470 */ 441, 281, 205, 163, 397, 280, 535, 505, 330, 617, - /* 1480 */ 971, 971, 971, 971, 367, 971, 971, 971, 971, 971, - /* 1490 */ 971, 971, 971, 971, 971, 971, 338, + /* 0 */ 311, 1306, 145, 651, 2, 192, 652, 338, 780, 92, + /* 10 */ 92, 92, 92, 85, 90, 90, 90, 90, 89, 89, + /* 20 */ 88, 88, 88, 87, 335, 88, 88, 88, 87, 335, + /* 30 */ 327, 856, 856, 92, 92, 92, 92, 776, 90, 90, + /* 40 */ 90, 90, 89, 89, 88, 88, 88, 87, 335, 86, + /* 50 */ 83, 166, 93, 94, 84, 868, 871, 860, 860, 91, + /* 60 */ 91, 92, 92, 92, 92, 335, 90, 90, 90, 90, + /* 70 */ 89, 89, 88, 88, 88, 87, 335, 311, 780, 90, + /* 80 */ 90, 90, 90, 89, 89, 88, 88, 88, 87, 335, + /* 90 */ 123, 808, 689, 689, 689, 689, 112, 230, 430, 257, + /* 100 */ 809, 698, 430, 86, 83, 166, 324, 55, 856, 856, + /* 110 */ 201, 158, 276, 387, 271, 386, 188, 689, 689, 828, + /* 120 */ 833, 49, 944, 269, 833, 49, 123, 87, 335, 93, + /* 130 */ 94, 84, 868, 871, 860, 860, 91, 91, 92, 92, + /* 140 */ 92, 92, 342, 90, 90, 90, 90, 89, 89, 88, + /* 150 */ 88, 88, 87, 335, 311, 328, 333, 332, 701, 408, + /* 160 */ 394, 69, 690, 691, 690, 691, 715, 910, 251, 354, + /* 170 */ 250, 698, 704, 430, 908, 430, 909, 89, 89, 88, + /* 180 */ 88, 88, 87, 335, 391, 856, 856, 690, 691, 183, + /* 190 */ 95, 340, 384, 381, 380, 833, 31, 833, 49, 912, + /* 200 */ 912, 333, 332, 379, 123, 311, 93, 94, 84, 868, + /* 210 */ 871, 860, 860, 91, 91, 92, 92, 92, 92, 114, + /* 220 */ 90, 90, 90, 90, 89, 89, 88, 88, 88, 87, + /* 230 */ 335, 430, 408, 399, 435, 657, 856, 856, 346, 57, + /* 240 */ 232, 828, 109, 20, 912, 912, 231, 393, 937, 760, + /* 250 */ 97, 751, 752, 833, 49, 708, 708, 93, 94, 84, + /* 260 */ 868, 871, 860, 860, 91, 91, 92, 92, 92, 92, + /* 270 */ 707, 90, 90, 90, 90, 89, 89, 88, 88, 88, + /* 280 */ 87, 335, 311, 114, 22, 706, 688, 58, 408, 390, + /* 290 */ 251, 349, 240, 749, 752, 689, 689, 847, 685, 115, + /* 300 */ 21, 231, 393, 689, 689, 697, 183, 355, 430, 384, + /* 310 */ 381, 380, 192, 856, 856, 780, 123, 160, 159, 223, + /* 320 */ 379, 738, 25, 315, 362, 841, 143, 689, 689, 835, + /* 330 */ 833, 48, 339, 937, 93, 94, 84, 868, 871, 860, + /* 340 */ 860, 91, 91, 92, 92, 92, 92, 914, 90, 90, + /* 350 */ 90, 90, 89, 89, 88, 88, 88, 87, 335, 311, + /* 360 */ 840, 840, 840, 266, 430, 690, 691, 778, 114, 1300, + /* 370 */ 1300, 430, 1, 690, 691, 697, 688, 689, 689, 689, + /* 380 */ 689, 689, 689, 287, 298, 780, 833, 10, 686, 115, + /* 390 */ 856, 856, 355, 833, 10, 828, 366, 690, 691, 363, + /* 400 */ 321, 76, 123, 74, 23, 737, 807, 323, 356, 353, + /* 410 */ 847, 93, 94, 84, 868, 871, 860, 860, 91, 91, + /* 420 */ 92, 92, 92, 92, 940, 90, 90, 90, 90, 89, + /* 430 */ 89, 88, 88, 88, 87, 335, 311, 806, 841, 429, + /* 440 */ 713, 941, 835, 430, 251, 354, 250, 690, 691, 690, + /* 450 */ 691, 690, 691, 86, 83, 166, 24, 942, 151, 753, + /* 460 */ 285, 907, 403, 907, 164, 833, 10, 856, 856, 965, + /* 470 */ 306, 754, 679, 840, 840, 840, 795, 216, 794, 222, + /* 480 */ 906, 344, 906, 904, 86, 83, 166, 286, 93, 94, + /* 490 */ 84, 868, 871, 860, 860, 91, 91, 92, 92, 92, + /* 500 */ 92, 430, 90, 90, 90, 90, 89, 89, 88, 88, + /* 510 */ 88, 87, 335, 311, 430, 724, 352, 705, 427, 699, + /* 520 */ 700, 376, 210, 833, 49, 793, 397, 857, 857, 940, + /* 530 */ 213, 762, 727, 334, 699, 700, 833, 10, 86, 83, + /* 540 */ 166, 345, 396, 902, 856, 856, 941, 385, 833, 9, + /* 550 */ 406, 869, 872, 187, 890, 728, 347, 398, 404, 977, + /* 560 */ 652, 338, 942, 954, 413, 93, 94, 84, 868, 871, + /* 570 */ 860, 860, 91, 91, 92, 92, 92, 92, 861, 90, + /* 580 */ 90, 90, 90, 89, 89, 88, 88, 88, 87, 335, + /* 590 */ 311, 1219, 114, 430, 834, 430, 5, 165, 192, 688, + /* 600 */ 832, 780, 430, 723, 430, 234, 325, 189, 163, 316, + /* 610 */ 356, 955, 115, 235, 269, 833, 35, 833, 36, 747, + /* 620 */ 720, 856, 856, 793, 833, 12, 833, 27, 745, 174, + /* 630 */ 968, 1290, 968, 1291, 1290, 310, 1291, 693, 317, 245, + /* 640 */ 264, 311, 93, 94, 84, 868, 871, 860, 860, 91, + /* 650 */ 91, 92, 92, 92, 92, 832, 90, 90, 90, 90, + /* 660 */ 89, 89, 88, 88, 88, 87, 335, 430, 320, 213, + /* 670 */ 762, 780, 856, 856, 920, 920, 369, 257, 966, 220, + /* 680 */ 966, 396, 663, 664, 665, 242, 259, 244, 262, 833, + /* 690 */ 37, 650, 2, 93, 94, 84, 868, 871, 860, 860, + /* 700 */ 91, 91, 92, 92, 92, 92, 430, 90, 90, 90, + /* 710 */ 90, 89, 89, 88, 88, 88, 87, 335, 311, 430, + /* 720 */ 239, 430, 917, 368, 430, 238, 916, 793, 833, 38, + /* 730 */ 430, 825, 430, 66, 430, 392, 430, 766, 766, 430, + /* 740 */ 367, 833, 39, 833, 28, 430, 833, 29, 68, 856, + /* 750 */ 856, 900, 833, 40, 833, 41, 833, 42, 833, 11, + /* 760 */ 72, 833, 43, 243, 305, 970, 114, 833, 99, 961, + /* 770 */ 93, 94, 84, 868, 871, 860, 860, 91, 91, 92, + /* 780 */ 92, 92, 92, 430, 90, 90, 90, 90, 89, 89, + /* 790 */ 88, 88, 88, 87, 335, 311, 430, 361, 430, 165, + /* 800 */ 147, 430, 186, 185, 184, 833, 44, 430, 289, 430, + /* 810 */ 246, 430, 971, 430, 212, 163, 430, 357, 833, 45, + /* 820 */ 833, 32, 932, 833, 46, 793, 856, 856, 718, 833, + /* 830 */ 47, 833, 33, 833, 117, 833, 118, 75, 833, 119, + /* 840 */ 288, 305, 967, 214, 935, 322, 311, 93, 94, 84, + /* 850 */ 868, 871, 860, 860, 91, 91, 92, 92, 92, 92, + /* 860 */ 430, 90, 90, 90, 90, 89, 89, 88, 88, 88, + /* 870 */ 87, 335, 430, 832, 426, 317, 288, 856, 856, 114, + /* 880 */ 763, 257, 833, 53, 930, 219, 364, 257, 257, 971, + /* 890 */ 361, 396, 257, 257, 833, 34, 257, 311, 93, 94, + /* 900 */ 84, 868, 871, 860, 860, 91, 91, 92, 92, 92, + /* 910 */ 92, 430, 90, 90, 90, 90, 89, 89, 88, 88, + /* 920 */ 88, 87, 335, 430, 217, 318, 124, 253, 856, 856, + /* 930 */ 218, 943, 257, 833, 100, 898, 759, 774, 361, 755, + /* 940 */ 423, 329, 758, 1017, 289, 833, 50, 682, 311, 93, + /* 950 */ 82, 84, 868, 871, 860, 860, 91, 91, 92, 92, + /* 960 */ 92, 92, 430, 90, 90, 90, 90, 89, 89, 88, + /* 970 */ 88, 88, 87, 335, 430, 256, 419, 114, 249, 856, + /* 980 */ 856, 331, 114, 400, 833, 101, 359, 187, 1064, 726, + /* 990 */ 725, 739, 401, 416, 420, 360, 833, 102, 424, 311, + /* 1000 */ 258, 94, 84, 868, 871, 860, 860, 91, 91, 92, + /* 1010 */ 92, 92, 92, 430, 90, 90, 90, 90, 89, 89, + /* 1020 */ 88, 88, 88, 87, 335, 430, 221, 261, 114, 114, + /* 1030 */ 856, 856, 808, 114, 156, 833, 98, 772, 733, 734, + /* 1040 */ 275, 809, 771, 316, 263, 265, 960, 833, 116, 307, + /* 1050 */ 741, 274, 722, 84, 868, 871, 860, 860, 91, 91, + /* 1060 */ 92, 92, 92, 92, 430, 90, 90, 90, 90, 89, + /* 1070 */ 89, 88, 88, 88, 87, 335, 80, 425, 830, 3, + /* 1080 */ 1214, 191, 430, 721, 336, 336, 833, 113, 252, 80, + /* 1090 */ 425, 68, 3, 913, 913, 428, 270, 336, 336, 430, + /* 1100 */ 377, 784, 430, 197, 833, 106, 430, 716, 428, 430, + /* 1110 */ 267, 430, 897, 68, 414, 430, 769, 409, 430, 71, + /* 1120 */ 430, 833, 105, 123, 833, 103, 847, 414, 833, 49, + /* 1130 */ 843, 833, 104, 833, 52, 800, 123, 833, 54, 847, + /* 1140 */ 833, 51, 833, 26, 831, 802, 77, 78, 191, 389, + /* 1150 */ 430, 372, 114, 79, 432, 431, 911, 911, 835, 77, + /* 1160 */ 78, 779, 893, 408, 410, 197, 79, 432, 431, 791, + /* 1170 */ 226, 835, 833, 30, 772, 80, 425, 716, 3, 771, + /* 1180 */ 411, 412, 897, 336, 336, 290, 291, 839, 703, 840, + /* 1190 */ 840, 840, 842, 19, 428, 695, 684, 672, 111, 671, + /* 1200 */ 843, 673, 840, 840, 840, 842, 19, 207, 661, 278, + /* 1210 */ 148, 304, 280, 414, 282, 6, 822, 348, 248, 241, + /* 1220 */ 358, 934, 720, 80, 425, 847, 3, 161, 382, 273, + /* 1230 */ 284, 336, 336, 415, 296, 958, 895, 894, 157, 674, + /* 1240 */ 107, 194, 428, 948, 135, 77, 78, 777, 953, 951, + /* 1250 */ 56, 319, 79, 432, 431, 121, 66, 835, 59, 128, + /* 1260 */ 146, 414, 350, 130, 351, 819, 131, 132, 133, 375, + /* 1270 */ 173, 149, 138, 847, 936, 365, 178, 70, 425, 827, + /* 1280 */ 3, 889, 62, 371, 915, 336, 336, 792, 840, 840, + /* 1290 */ 840, 842, 19, 77, 78, 208, 428, 144, 179, 373, + /* 1300 */ 79, 432, 431, 255, 180, 835, 260, 675, 181, 308, + /* 1310 */ 388, 744, 326, 743, 742, 414, 731, 718, 712, 402, + /* 1320 */ 309, 711, 788, 65, 277, 272, 789, 847, 730, 710, + /* 1330 */ 709, 279, 193, 787, 281, 876, 840, 840, 840, 842, + /* 1340 */ 19, 786, 283, 73, 418, 330, 422, 77, 78, 227, + /* 1350 */ 96, 407, 67, 405, 79, 432, 431, 292, 228, 835, + /* 1360 */ 215, 202, 229, 293, 767, 303, 302, 301, 204, 299, + /* 1370 */ 294, 295, 676, 7, 681, 433, 669, 206, 110, 224, + /* 1380 */ 203, 205, 434, 667, 666, 658, 120, 168, 656, 237, + /* 1390 */ 840, 840, 840, 842, 19, 337, 155, 233, 236, 341, + /* 1400 */ 167, 905, 108, 313, 903, 826, 314, 125, 126, 127, + /* 1410 */ 129, 170, 247, 756, 172, 928, 134, 136, 171, 60, + /* 1420 */ 61, 123, 169, 137, 175, 933, 176, 927, 8, 13, + /* 1430 */ 177, 254, 191, 918, 139, 370, 924, 140, 678, 150, + /* 1440 */ 374, 274, 182, 378, 141, 122, 63, 14, 383, 729, + /* 1450 */ 268, 15, 64, 225, 846, 845, 874, 16, 765, 770, + /* 1460 */ 4, 162, 209, 395, 211, 142, 878, 796, 801, 312, + /* 1470 */ 190, 71, 68, 875, 873, 939, 199, 938, 17, 195, + /* 1480 */ 18, 196, 417, 975, 152, 653, 976, 198, 153, 421, + /* 1490 */ 877, 154, 200, 844, 696, 81, 343, 297, 1019, 1018, + /* 1500 */ 300, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 19, 22, 22, 23, 1, 24, 26, 15, 27, 80, + /* 0 */ 19, 144, 145, 146, 147, 24, 1, 2, 27, 80, /* 10 */ 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, - /* 20 */ 91, 92, 93, 94, 95, 108, 109, 110, 27, 28, - /* 30 */ 23, 50, 51, 80, 81, 82, 83, 122, 85, 86, - /* 40 */ 87, 88, 89, 90, 91, 92, 93, 94, 95, 22, - /* 50 */ 70, 23, 71, 72, 73, 74, 75, 76, 77, 78, - /* 60 */ 79, 80, 81, 82, 83, 122, 85, 86, 87, 88, - /* 70 */ 89, 90, 91, 92, 93, 94, 95, 19, 97, 91, - /* 80 */ 92, 93, 94, 95, 26, 85, 86, 87, 88, 89, - /* 90 */ 90, 91, 92, 93, 94, 95, 27, 28, 97, 98, - /* 100 */ 99, 122, 211, 102, 103, 104, 79, 19, 50, 51, - /* 110 */ 19, 122, 59, 55, 113, 224, 225, 226, 89, 90, - /* 120 */ 91, 92, 93, 94, 95, 23, 27, 28, 26, 71, + /* 20 */ 91, 92, 93, 94, 95, 91, 92, 93, 94, 95, + /* 30 */ 19, 50, 51, 80, 81, 82, 83, 212, 85, 86, + /* 40 */ 87, 88, 89, 90, 91, 92, 93, 94, 95, 224, + /* 50 */ 225, 226, 71, 72, 73, 74, 75, 76, 77, 78, + /* 60 */ 79, 80, 81, 82, 83, 95, 85, 86, 87, 88, + /* 70 */ 89, 90, 91, 92, 93, 94, 95, 19, 97, 85, + /* 80 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, + /* 90 */ 66, 33, 27, 28, 27, 28, 22, 201, 152, 152, + /* 100 */ 42, 27, 152, 224, 225, 226, 95, 211, 50, 51, + /* 110 */ 99, 100, 101, 102, 103, 104, 105, 27, 28, 59, + /* 120 */ 174, 175, 243, 112, 174, 175, 66, 94, 95, 71, /* 130 */ 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, - /* 140 */ 82, 83, 51, 85, 86, 87, 88, 89, 90, 91, - /* 150 */ 92, 93, 94, 95, 19, 132, 133, 58, 89, 90, - /* 160 */ 21, 108, 109, 110, 27, 28, 97, 98, 33, 100, - /* 170 */ 7, 8, 119, 120, 22, 19, 107, 42, 109, 27, - /* 180 */ 28, 27, 28, 95, 28, 50, 51, 99, 100, 101, - /* 190 */ 102, 103, 104, 105, 27, 28, 97, 98, 107, 152, - /* 200 */ 112, 132, 133, 112, 65, 69, 71, 72, 73, 74, - /* 210 */ 75, 76, 77, 78, 79, 80, 81, 82, 83, 11, + /* 140 */ 82, 83, 195, 85, 86, 87, 88, 89, 90, 91, + /* 150 */ 92, 93, 94, 95, 19, 209, 89, 90, 173, 209, + /* 160 */ 210, 26, 97, 98, 97, 98, 181, 100, 108, 109, + /* 170 */ 110, 97, 174, 152, 107, 152, 109, 89, 90, 91, + /* 180 */ 92, 93, 94, 95, 163, 50, 51, 97, 98, 99, + /* 190 */ 55, 244, 102, 103, 104, 174, 175, 174, 175, 132, + /* 200 */ 133, 89, 90, 113, 66, 19, 71, 72, 73, 74, + /* 210 */ 75, 76, 77, 78, 79, 80, 81, 82, 83, 198, /* 220 */ 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, - /* 230 */ 95, 19, 101, 97, 97, 98, 24, 101, 122, 157, - /* 240 */ 12, 99, 103, 112, 102, 103, 104, 152, 22, 97, - /* 250 */ 98, 97, 98, 27, 28, 113, 27, 29, 91, 164, - /* 260 */ 165, 124, 50, 51, 97, 98, 219, 59, 132, 133, - /* 270 */ 134, 22, 23, 45, 66, 47, 212, 213, 124, 140, - /* 280 */ 132, 133, 19, 71, 72, 73, 74, 75, 76, 77, - /* 290 */ 78, 79, 80, 81, 82, 83, 152, 85, 86, 87, - /* 300 */ 88, 89, 90, 91, 92, 93, 94, 95, 164, 165, - /* 310 */ 27, 28, 230, 50, 51, 233, 108, 109, 110, 70, - /* 320 */ 16, 59, 23, 97, 98, 26, 97, 22, 66, 185, - /* 330 */ 12, 187, 27, 28, 71, 72, 73, 74, 75, 76, - /* 340 */ 77, 78, 79, 80, 81, 82, 83, 29, 85, 86, + /* 230 */ 95, 152, 209, 210, 148, 149, 50, 51, 100, 53, + /* 240 */ 154, 59, 156, 22, 132, 133, 119, 120, 163, 163, + /* 250 */ 22, 192, 193, 174, 175, 27, 28, 71, 72, 73, + /* 260 */ 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, + /* 270 */ 174, 85, 86, 87, 88, 89, 90, 91, 92, 93, + /* 280 */ 94, 95, 19, 198, 198, 174, 152, 24, 209, 210, + /* 290 */ 108, 109, 110, 192, 193, 27, 28, 69, 164, 165, + /* 300 */ 79, 119, 120, 27, 28, 27, 99, 222, 152, 102, + /* 310 */ 103, 104, 24, 50, 51, 27, 66, 89, 90, 185, + /* 320 */ 113, 187, 22, 157, 239, 97, 58, 27, 28, 101, + /* 330 */ 174, 175, 246, 163, 71, 72, 73, 74, 75, 76, + /* 340 */ 77, 78, 79, 80, 81, 82, 83, 11, 85, 86, /* 350 */ 87, 88, 89, 90, 91, 92, 93, 94, 95, 19, - /* 360 */ 22, 148, 149, 45, 23, 47, 62, 154, 64, 156, - /* 370 */ 108, 109, 110, 37, 69, 23, 163, 59, 26, 26, - /* 380 */ 97, 98, 144, 145, 146, 147, 152, 200, 52, 23, - /* 390 */ 50, 51, 26, 22, 89, 90, 60, 210, 7, 8, - /* 400 */ 9, 138, 97, 22, 23, 26, 101, 26, 174, 175, - /* 410 */ 197, 71, 72, 73, 74, 75, 76, 77, 78, 79, - /* 420 */ 80, 81, 82, 83, 16, 85, 86, 87, 88, 89, - /* 430 */ 90, 91, 92, 93, 94, 95, 19, 132, 133, 134, - /* 440 */ 23, 152, 208, 209, 140, 152, 152, 111, 195, 196, - /* 450 */ 98, 70, 163, 160, 152, 23, 22, 164, 165, 246, - /* 460 */ 207, 27, 152, 174, 175, 171, 172, 50, 51, 137, - /* 470 */ 62, 139, 64, 171, 172, 222, 124, 27, 138, 24, - /* 480 */ 163, 89, 90, 130, 174, 175, 197, 163, 71, 72, + /* 360 */ 132, 133, 134, 23, 152, 97, 98, 91, 198, 119, + /* 370 */ 120, 152, 22, 97, 98, 97, 152, 27, 28, 27, + /* 380 */ 28, 27, 28, 227, 160, 97, 174, 175, 164, 165, + /* 390 */ 50, 51, 222, 174, 175, 59, 230, 97, 98, 233, + /* 400 */ 188, 137, 66, 139, 234, 187, 177, 188, 152, 239, + /* 410 */ 69, 71, 72, 73, 74, 75, 76, 77, 78, 79, + /* 420 */ 80, 81, 82, 83, 12, 85, 86, 87, 88, 89, + /* 430 */ 90, 91, 92, 93, 94, 95, 19, 177, 97, 152, + /* 440 */ 23, 29, 101, 152, 108, 109, 110, 97, 98, 97, + /* 450 */ 98, 97, 98, 224, 225, 226, 22, 45, 24, 47, + /* 460 */ 152, 152, 152, 152, 152, 174, 175, 50, 51, 249, + /* 470 */ 250, 59, 21, 132, 133, 134, 124, 221, 124, 188, + /* 480 */ 171, 172, 171, 172, 224, 225, 226, 152, 71, 72, /* 490 */ 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, - /* 500 */ 83, 22, 85, 86, 87, 88, 89, 90, 91, 92, - /* 510 */ 93, 94, 95, 19, 197, 181, 182, 23, 208, 209, - /* 520 */ 152, 197, 26, 189, 132, 133, 232, 224, 225, 226, - /* 530 */ 152, 97, 91, 26, 232, 116, 212, 213, 152, 222, - /* 540 */ 121, 152, 174, 175, 50, 51, 243, 97, 22, 23, - /* 550 */ 22, 234, 174, 175, 177, 23, 239, 116, 163, 177, - /* 560 */ 174, 175, 121, 174, 175, 71, 72, 73, 74, 75, - /* 570 */ 76, 77, 78, 79, 80, 81, 82, 83, 24, 85, + /* 500 */ 83, 152, 85, 86, 87, 88, 89, 90, 91, 92, + /* 510 */ 93, 94, 95, 19, 152, 183, 65, 23, 170, 171, + /* 520 */ 172, 19, 23, 174, 175, 26, 152, 50, 51, 12, + /* 530 */ 196, 197, 37, 170, 171, 172, 174, 175, 224, 225, + /* 540 */ 226, 232, 208, 232, 50, 51, 29, 52, 174, 175, + /* 550 */ 188, 74, 75, 51, 103, 60, 222, 163, 209, 0, + /* 560 */ 1, 2, 45, 152, 47, 71, 72, 73, 74, 75, + /* 570 */ 76, 77, 78, 79, 80, 81, 82, 83, 101, 85, /* 580 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, - /* 590 */ 19, 23, 197, 11, 23, 227, 70, 208, 220, 152, - /* 600 */ 31, 224, 225, 226, 35, 98, 224, 225, 226, 108, - /* 610 */ 109, 110, 115, 152, 117, 118, 27, 222, 49, 123, - /* 620 */ 24, 50, 51, 27, 0, 1, 2, 224, 225, 226, - /* 630 */ 166, 124, 168, 169, 239, 174, 175, 170, 171, 172, - /* 640 */ 22, 194, 71, 72, 73, 74, 75, 76, 77, 78, + /* 590 */ 19, 140, 198, 152, 23, 152, 22, 98, 24, 152, + /* 600 */ 152, 27, 152, 183, 152, 152, 111, 213, 214, 107, + /* 610 */ 152, 164, 165, 152, 112, 174, 175, 174, 175, 181, + /* 620 */ 182, 50, 51, 124, 174, 175, 174, 175, 190, 26, + /* 630 */ 22, 23, 22, 23, 26, 166, 26, 168, 169, 16, + /* 640 */ 16, 19, 71, 72, 73, 74, 75, 76, 77, 78, /* 650 */ 79, 80, 81, 82, 83, 152, 85, 86, 87, 88, - /* 660 */ 89, 90, 91, 92, 93, 94, 95, 19, 22, 208, - /* 670 */ 24, 23, 195, 196, 170, 171, 172, 174, 175, 152, - /* 680 */ 26, 152, 152, 152, 207, 152, 97, 152, 23, 152, - /* 690 */ 51, 244, 152, 97, 152, 247, 248, 23, 50, 51, - /* 700 */ 26, 174, 175, 174, 175, 174, 175, 174, 175, 174, - /* 710 */ 175, 174, 175, 23, 174, 175, 174, 175, 188, 71, - /* 720 */ 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, - /* 730 */ 82, 83, 152, 85, 86, 87, 88, 89, 90, 91, - /* 740 */ 92, 93, 94, 95, 19, 152, 107, 152, 33, 24, - /* 750 */ 152, 100, 101, 27, 174, 175, 152, 42, 152, 23, - /* 760 */ 152, 26, 152, 23, 152, 26, 152, 174, 175, 174, - /* 770 */ 175, 152, 174, 175, 23, 50, 51, 26, 174, 175, - /* 780 */ 174, 175, 174, 175, 174, 175, 174, 175, 174, 175, - /* 790 */ 163, 119, 120, 174, 175, 19, 71, 72, 73, 74, - /* 800 */ 75, 76, 77, 78, 79, 80, 81, 82, 83, 152, - /* 810 */ 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, - /* 820 */ 95, 66, 152, 97, 197, 23, 50, 51, 26, 53, - /* 830 */ 23, 174, 175, 26, 23, 23, 23, 26, 26, 26, - /* 840 */ 36, 106, 146, 147, 174, 175, 19, 71, 72, 73, + /* 660 */ 89, 90, 91, 92, 93, 94, 95, 152, 220, 196, + /* 670 */ 197, 97, 50, 51, 108, 109, 110, 152, 70, 221, + /* 680 */ 70, 208, 7, 8, 9, 62, 62, 64, 64, 174, + /* 690 */ 175, 146, 147, 71, 72, 73, 74, 75, 76, 77, + /* 700 */ 78, 79, 80, 81, 82, 83, 152, 85, 86, 87, + /* 710 */ 88, 89, 90, 91, 92, 93, 94, 95, 19, 152, + /* 720 */ 195, 152, 31, 220, 152, 152, 35, 26, 174, 175, + /* 730 */ 152, 163, 152, 130, 152, 115, 152, 117, 118, 152, + /* 740 */ 49, 174, 175, 174, 175, 152, 174, 175, 26, 50, + /* 750 */ 51, 152, 174, 175, 174, 175, 174, 175, 174, 175, + /* 760 */ 138, 174, 175, 140, 22, 23, 198, 174, 175, 152, + /* 770 */ 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, + /* 780 */ 81, 82, 83, 152, 85, 86, 87, 88, 89, 90, + /* 790 */ 91, 92, 93, 94, 95, 19, 152, 152, 152, 98, + /* 800 */ 24, 152, 108, 109, 110, 174, 175, 152, 152, 152, + /* 810 */ 152, 152, 70, 152, 213, 214, 152, 152, 174, 175, + /* 820 */ 174, 175, 152, 174, 175, 124, 50, 51, 106, 174, + /* 830 */ 175, 174, 175, 174, 175, 174, 175, 138, 174, 175, + /* 840 */ 152, 22, 23, 22, 163, 189, 19, 71, 72, 73, /* 850 */ 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, /* 860 */ 152, 85, 86, 87, 88, 89, 90, 91, 92, 93, - /* 870 */ 94, 95, 152, 196, 119, 120, 19, 50, 51, 168, - /* 880 */ 169, 26, 174, 175, 207, 28, 152, 249, 250, 152, - /* 890 */ 163, 163, 163, 163, 174, 175, 163, 19, 71, 72, + /* 870 */ 94, 95, 152, 152, 168, 169, 152, 50, 51, 198, + /* 880 */ 197, 152, 174, 175, 152, 240, 152, 152, 152, 70, + /* 890 */ 152, 208, 152, 152, 174, 175, 152, 19, 71, 72, /* 900 */ 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, /* 910 */ 83, 152, 85, 86, 87, 88, 89, 90, 91, 92, - /* 920 */ 93, 94, 95, 152, 197, 197, 197, 197, 50, 51, - /* 930 */ 197, 194, 36, 174, 175, 191, 192, 152, 191, 192, - /* 940 */ 163, 152, 66, 124, 152, 174, 175, 152, 19, 71, + /* 920 */ 93, 94, 95, 152, 195, 247, 248, 152, 50, 51, + /* 930 */ 195, 195, 152, 174, 175, 195, 195, 26, 152, 195, + /* 940 */ 252, 220, 163, 122, 152, 174, 175, 163, 19, 71, /* 950 */ 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, /* 960 */ 82, 83, 152, 85, 86, 87, 88, 89, 90, 91, - /* 970 */ 92, 93, 94, 95, 197, 152, 100, 188, 152, 50, - /* 980 */ 51, 152, 152, 188, 174, 175, 252, 152, 94, 95, - /* 990 */ 152, 152, 152, 1, 2, 152, 152, 174, 175, 19, + /* 970 */ 92, 93, 94, 95, 152, 195, 252, 198, 240, 50, + /* 980 */ 51, 189, 198, 19, 174, 175, 19, 51, 23, 100, + /* 990 */ 101, 26, 28, 163, 163, 28, 174, 175, 163, 19, /* 1000 */ 152, 72, 73, 74, 75, 76, 77, 78, 79, 80, /* 1010 */ 81, 82, 83, 152, 85, 86, 87, 88, 89, 90, - /* 1020 */ 91, 92, 93, 94, 95, 152, 188, 188, 22, 194, - /* 1030 */ 50, 51, 240, 173, 194, 174, 175, 252, 194, 152, - /* 1040 */ 36, 181, 28, 152, 23, 219, 122, 174, 175, 219, - /* 1050 */ 221, 152, 152, 73, 74, 75, 76, 77, 78, 79, + /* 1020 */ 91, 92, 93, 94, 95, 152, 240, 152, 198, 198, + /* 1030 */ 50, 51, 33, 198, 123, 174, 175, 116, 7, 8, + /* 1040 */ 101, 42, 121, 107, 152, 152, 23, 174, 175, 26, + /* 1050 */ 152, 112, 183, 73, 74, 75, 76, 77, 78, 79, /* 1060 */ 80, 81, 82, 83, 152, 85, 86, 87, 88, 89, - /* 1070 */ 90, 91, 92, 93, 94, 95, 19, 20, 152, 22, - /* 1080 */ 23, 194, 152, 240, 27, 28, 174, 175, 240, 19, - /* 1090 */ 20, 26, 22, 194, 194, 38, 22, 27, 28, 152, - /* 1100 */ 23, 22, 152, 116, 174, 175, 152, 23, 38, 152, - /* 1110 */ 23, 152, 221, 152, 57, 152, 23, 163, 50, 51, - /* 1120 */ 194, 174, 175, 66, 174, 175, 69, 57, 174, 175, - /* 1130 */ 40, 174, 175, 174, 175, 174, 175, 174, 175, 69, - /* 1140 */ 22, 53, 74, 75, 30, 53, 89, 90, 22, 22, - /* 1150 */ 152, 197, 23, 96, 97, 98, 22, 152, 101, 89, - /* 1160 */ 90, 91, 208, 209, 152, 53, 96, 97, 98, 101, - /* 1170 */ 22, 101, 174, 175, 152, 19, 20, 105, 22, 174, - /* 1180 */ 175, 112, 19, 27, 28, 20, 174, 175, 24, 132, - /* 1190 */ 133, 134, 135, 136, 38, 44, 174, 175, 107, 61, - /* 1200 */ 54, 26, 132, 133, 134, 135, 136, 54, 107, 22, - /* 1210 */ 5, 140, 1, 57, 36, 111, 122, 28, 79, 79, - /* 1220 */ 131, 123, 66, 19, 20, 69, 22, 1, 16, 20, - /* 1230 */ 125, 27, 28, 123, 111, 120, 23, 131, 23, 16, - /* 1240 */ 68, 142, 38, 15, 22, 89, 90, 3, 167, 4, - /* 1250 */ 248, 251, 96, 97, 98, 180, 180, 101, 251, 151, - /* 1260 */ 6, 57, 151, 13, 151, 26, 25, 151, 161, 202, - /* 1270 */ 153, 162, 153, 69, 130, 128, 203, 19, 20, 127, - /* 1280 */ 22, 126, 204, 129, 22, 27, 28, 205, 132, 133, - /* 1290 */ 134, 135, 136, 89, 90, 231, 38, 95, 137, 179, - /* 1300 */ 96, 97, 98, 206, 179, 101, 122, 107, 159, 159, - /* 1310 */ 125, 231, 216, 228, 107, 57, 184, 217, 216, 176, - /* 1320 */ 217, 176, 48, 106, 18, 184, 158, 69, 159, 158, - /* 1330 */ 46, 71, 237, 176, 176, 176, 132, 133, 134, 135, - /* 1340 */ 136, 217, 176, 137, 216, 178, 158, 89, 90, 179, - /* 1350 */ 176, 159, 179, 159, 96, 97, 98, 159, 159, 101, - /* 1360 */ 5, 158, 202, 22, 18, 10, 11, 12, 13, 14, - /* 1370 */ 190, 238, 17, 190, 158, 193, 41, 159, 202, 193, - /* 1380 */ 159, 202, 245, 193, 193, 223, 190, 32, 159, 34, - /* 1390 */ 132, 133, 134, 135, 136, 159, 39, 155, 43, 150, - /* 1400 */ 223, 177, 201, 178, 177, 186, 66, 199, 177, 152, - /* 1410 */ 253, 56, 215, 152, 182, 152, 202, 152, 63, 152, - /* 1420 */ 152, 66, 67, 242, 229, 152, 174, 152, 152, 152, - /* 1430 */ 152, 152, 152, 152, 199, 242, 202, 152, 198, 152, - /* 1440 */ 152, 152, 183, 192, 152, 215, 152, 183, 215, 183, - /* 1450 */ 152, 241, 214, 152, 211, 152, 152, 211, 211, 152, - /* 1460 */ 152, 241, 152, 152, 152, 152, 152, 152, 152, 114, - /* 1470 */ 152, 152, 235, 152, 152, 152, 174, 187, 95, 174, - /* 1480 */ 253, 253, 253, 253, 236, 253, 253, 253, 253, 253, - /* 1490 */ 253, 253, 253, 253, 253, 253, 141, + /* 1070 */ 90, 91, 92, 93, 94, 95, 19, 20, 23, 22, + /* 1080 */ 23, 26, 152, 152, 27, 28, 174, 175, 23, 19, + /* 1090 */ 20, 26, 22, 132, 133, 38, 152, 27, 28, 152, + /* 1100 */ 23, 215, 152, 26, 174, 175, 152, 27, 38, 152, + /* 1110 */ 23, 152, 27, 26, 57, 152, 23, 163, 152, 26, + /* 1120 */ 152, 174, 175, 66, 174, 175, 69, 57, 174, 175, + /* 1130 */ 27, 174, 175, 174, 175, 152, 66, 174, 175, 69, + /* 1140 */ 174, 175, 174, 175, 152, 23, 89, 90, 26, 91, + /* 1150 */ 152, 236, 198, 96, 97, 98, 132, 133, 101, 89, + /* 1160 */ 90, 152, 23, 209, 210, 26, 96, 97, 98, 152, + /* 1170 */ 212, 101, 174, 175, 116, 19, 20, 97, 22, 121, + /* 1180 */ 152, 193, 97, 27, 28, 152, 152, 152, 152, 132, + /* 1190 */ 133, 134, 135, 136, 38, 23, 152, 152, 26, 152, + /* 1200 */ 97, 152, 132, 133, 134, 135, 136, 235, 152, 212, + /* 1210 */ 199, 150, 212, 57, 212, 200, 203, 216, 241, 216, + /* 1220 */ 241, 203, 182, 19, 20, 69, 22, 186, 178, 177, + /* 1230 */ 216, 27, 28, 229, 202, 39, 177, 177, 200, 155, + /* 1240 */ 245, 122, 38, 41, 22, 89, 90, 91, 159, 159, + /* 1250 */ 242, 159, 96, 97, 98, 71, 130, 101, 242, 191, + /* 1260 */ 223, 57, 18, 194, 159, 203, 194, 194, 194, 18, + /* 1270 */ 158, 223, 191, 69, 203, 159, 158, 19, 20, 191, + /* 1280 */ 22, 203, 137, 46, 238, 27, 28, 159, 132, 133, + /* 1290 */ 134, 135, 136, 89, 90, 159, 38, 22, 158, 179, + /* 1300 */ 96, 97, 98, 237, 158, 101, 159, 159, 158, 179, + /* 1310 */ 107, 176, 48, 176, 176, 57, 184, 106, 176, 125, + /* 1320 */ 179, 178, 218, 107, 217, 176, 218, 69, 184, 176, + /* 1330 */ 176, 217, 159, 218, 217, 159, 132, 133, 134, 135, + /* 1340 */ 136, 218, 217, 137, 179, 95, 179, 89, 90, 228, + /* 1350 */ 129, 126, 128, 127, 96, 97, 98, 206, 231, 101, + /* 1360 */ 5, 25, 231, 205, 207, 10, 11, 12, 13, 14, + /* 1370 */ 204, 203, 17, 26, 162, 161, 13, 6, 180, 180, + /* 1380 */ 153, 153, 151, 151, 151, 151, 167, 32, 4, 34, + /* 1390 */ 132, 133, 134, 135, 136, 3, 22, 142, 43, 68, + /* 1400 */ 15, 23, 16, 251, 23, 120, 251, 248, 131, 111, + /* 1410 */ 123, 56, 16, 20, 125, 1, 123, 131, 63, 79, + /* 1420 */ 79, 66, 67, 111, 36, 28, 122, 1, 5, 22, + /* 1430 */ 107, 140, 26, 54, 54, 44, 61, 107, 20, 24, + /* 1440 */ 19, 112, 105, 53, 22, 40, 22, 22, 53, 30, + /* 1450 */ 23, 22, 22, 53, 23, 23, 23, 22, 116, 23, + /* 1460 */ 22, 122, 23, 26, 23, 22, 11, 124, 28, 114, + /* 1470 */ 36, 26, 26, 23, 23, 23, 122, 23, 36, 26, + /* 1480 */ 36, 22, 24, 23, 22, 1, 23, 26, 22, 24, + /* 1490 */ 23, 22, 122, 23, 23, 22, 141, 23, 122, 122, + /* 1500 */ 15, }; -#define YY_SHIFT_USE_DFLT (-86) -#define YY_SHIFT_COUNT (429) -#define YY_SHIFT_MIN (-85) -#define YY_SHIFT_MAX (1383) +#define YY_SHIFT_USE_DFLT (-72) +#define YY_SHIFT_COUNT (435) +#define YY_SHIFT_MIN (-71) +#define YY_SHIFT_MAX (1485) static const short yy_shift_ofst[] = { - /* 0 */ 992, 1057, 1355, 1156, 1204, 1204, 1, 262, -19, 135, - /* 10 */ 135, 776, 1204, 1204, 1204, 1204, 69, 69, 53, 208, - /* 20 */ 283, 755, 58, 725, 648, 571, 494, 417, 340, 263, - /* 30 */ 212, 827, 827, 827, 827, 827, 827, 827, 827, 827, - /* 40 */ 827, 827, 827, 827, 827, 827, 878, 827, 929, 980, - /* 50 */ 980, 1070, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, + /* 0 */ 5, 1057, 1355, 1070, 1204, 1204, 1204, 90, 60, -19, + /* 10 */ 58, 58, 186, 1204, 1204, 1204, 1204, 1204, 1204, 1204, + /* 20 */ 67, 67, 182, 336, 65, 250, 135, 263, 340, 417, + /* 30 */ 494, 571, 622, 699, 776, 827, 827, 827, 827, 827, + /* 40 */ 827, 827, 827, 827, 827, 827, 827, 827, 827, 827, + /* 50 */ 878, 827, 929, 980, 980, 1156, 1204, 1204, 1204, 1204, /* 60 */ 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, /* 70 */ 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, - /* 80 */ 1258, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, - /* 90 */ 1204, 1204, 1204, 1204, -71, -47, -47, -47, -47, -47, - /* 100 */ 0, 29, -12, 283, 283, 139, 91, 392, 392, 894, - /* 110 */ 672, 726, 1383, -86, -86, -86, 88, 318, 318, 99, - /* 120 */ 381, -20, 283, 283, 283, 283, 283, 283, 283, 283, - /* 130 */ 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, - /* 140 */ 283, 283, 283, 283, 624, 876, 726, 672, 1340, 1340, - /* 150 */ 1340, 1340, 1340, 1340, -86, -86, -86, 305, 136, 136, - /* 160 */ 142, 167, 226, 154, 137, 152, 283, 283, 283, 283, - /* 170 */ 283, 283, 283, 283, 283, 283, 283, 283, 283, 283, - /* 180 */ 283, 283, 283, 336, 336, 336, 283, 283, 352, 283, - /* 190 */ 283, 283, 283, 283, 228, 283, 283, 283, 283, 283, - /* 200 */ 283, 283, 283, 283, 283, 501, 569, 596, 596, 596, - /* 210 */ 507, 497, 441, 391, 353, 156, 156, 857, 353, 857, - /* 220 */ 735, 813, 639, 715, 156, 332, 715, 715, 496, 419, - /* 230 */ 646, 1357, 1184, 1184, 1335, 1335, 1184, 1341, 1260, 1144, - /* 240 */ 1346, 1346, 1346, 1346, 1184, 1306, 1144, 1341, 1260, 1260, - /* 250 */ 1144, 1184, 1306, 1206, 1284, 1184, 1184, 1306, 1184, 1306, - /* 260 */ 1184, 1306, 1262, 1207, 1207, 1207, 1274, 1262, 1207, 1217, - /* 270 */ 1207, 1274, 1207, 1207, 1185, 1200, 1185, 1200, 1185, 1200, - /* 280 */ 1184, 1184, 1161, 1262, 1202, 1202, 1262, 1154, 1155, 1147, - /* 290 */ 1152, 1144, 1241, 1239, 1250, 1250, 1254, 1254, 1254, 1254, - /* 300 */ -86, -86, -86, -86, -86, -86, 1068, 304, 526, 249, - /* 310 */ 408, -83, 434, 812, 27, 811, 807, 802, 751, 589, - /* 320 */ 651, 163, 131, 674, 366, 450, 299, 148, 23, 102, - /* 330 */ 229, -21, 1245, 1244, 1222, 1099, 1228, 1172, 1223, 1215, - /* 340 */ 1213, 1115, 1106, 1123, 1110, 1209, 1105, 1212, 1226, 1098, - /* 350 */ 1089, 1140, 1139, 1104, 1189, 1178, 1094, 1211, 1205, 1187, - /* 360 */ 1101, 1071, 1153, 1175, 1146, 1138, 1151, 1091, 1164, 1165, - /* 370 */ 1163, 1069, 1072, 1148, 1112, 1134, 1127, 1129, 1126, 1092, - /* 380 */ 1114, 1118, 1088, 1090, 1093, 1087, 1084, 987, 1079, 1077, - /* 390 */ 1074, 1065, 924, 1021, 1014, 1004, 1006, 819, 739, 896, - /* 400 */ 855, 804, 739, 740, 736, 690, 654, 665, 618, 582, - /* 410 */ 568, 528, 554, 379, 532, 479, 455, 379, 432, 371, - /* 420 */ 341, 28, 338, 116, -11, -57, -85, 7, -8, 3, + /* 80 */ 1204, 1204, 1204, 1204, 1258, 1204, 1204, 1204, 1204, 1204, + /* 90 */ 1204, 1204, 1204, 1204, 1204, 1204, 1204, 1204, -71, -47, + /* 100 */ -47, -47, -47, -47, -6, 88, -66, 65, 65, 451, + /* 110 */ 502, 112, 112, 33, 127, 278, -30, -72, -72, -72, + /* 120 */ 11, 412, 412, 268, 608, 610, 65, 65, 65, 65, + /* 130 */ 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, + /* 140 */ 65, 65, 65, 65, 65, 559, 138, 278, 127, 24, + /* 150 */ 24, 24, 24, 24, 24, -72, -72, -72, 228, 341, + /* 160 */ 341, 207, 276, 300, 352, 354, 350, 65, 65, 65, + /* 170 */ 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, + /* 180 */ 65, 65, 65, 65, 495, 495, 495, 65, 65, 499, + /* 190 */ 65, 65, 65, 574, 65, 65, 517, 65, 65, 65, + /* 200 */ 65, 65, 65, 65, 65, 65, 65, 566, 691, 288, + /* 210 */ 288, 288, 701, 620, 1058, 675, 603, 964, 964, 967, + /* 220 */ 603, 967, 722, 965, 936, 999, 964, 264, 999, 999, + /* 230 */ 911, 921, 434, 1196, 1119, 1119, 1202, 1202, 1119, 1222, + /* 240 */ 1184, 1126, 1244, 1244, 1244, 1244, 1119, 1251, 1126, 1222, + /* 250 */ 1184, 1184, 1126, 1119, 1251, 1145, 1237, 1119, 1119, 1251, + /* 260 */ 1275, 1119, 1251, 1119, 1251, 1275, 1203, 1203, 1203, 1264, + /* 270 */ 1275, 1203, 1211, 1203, 1264, 1203, 1203, 1194, 1216, 1194, + /* 280 */ 1216, 1194, 1216, 1194, 1216, 1119, 1119, 1206, 1275, 1250, + /* 290 */ 1250, 1275, 1221, 1225, 1224, 1226, 1126, 1336, 1347, 1363, + /* 300 */ 1363, 1371, 1371, 1371, 1371, -72, -72, -72, -72, -72, + /* 310 */ -72, 477, 623, 742, 819, 624, 694, 74, 1023, 221, + /* 320 */ 1055, 1065, 1077, 1087, 1080, 889, 1031, 939, 1093, 1122, + /* 330 */ 1085, 1139, 961, 1024, 1172, 1103, 821, 1384, 1392, 1374, + /* 340 */ 1255, 1385, 1331, 1386, 1378, 1381, 1285, 1277, 1298, 1287, + /* 350 */ 1393, 1289, 1396, 1414, 1293, 1286, 1340, 1341, 1312, 1397, + /* 360 */ 1388, 1304, 1426, 1423, 1407, 1323, 1291, 1379, 1406, 1380, + /* 370 */ 1375, 1391, 1330, 1415, 1418, 1421, 1329, 1337, 1422, 1390, + /* 380 */ 1424, 1425, 1427, 1429, 1395, 1419, 1430, 1400, 1405, 1431, + /* 390 */ 1432, 1433, 1342, 1435, 1436, 1438, 1437, 1339, 1439, 1441, + /* 400 */ 1440, 1434, 1443, 1343, 1445, 1442, 1446, 1444, 1445, 1450, + /* 410 */ 1451, 1452, 1453, 1454, 1459, 1455, 1460, 1462, 1458, 1461, + /* 420 */ 1463, 1466, 1465, 1461, 1467, 1469, 1470, 1471, 1473, 1354, + /* 430 */ 1370, 1376, 1377, 1474, 1485, 1484, }; -#define YY_REDUCE_USE_DFLT (-110) -#define YY_REDUCE_COUNT (305) -#define YY_REDUCE_MIN (-109) -#define YY_REDUCE_MAX (1323) +#define YY_REDUCE_USE_DFLT (-176) +#define YY_REDUCE_COUNT (310) +#define YY_REDUCE_MIN (-175) +#define YY_REDUCE_MAX (1234) static const short yy_reduce_ofst[] = { - /* 0 */ 238, 954, 213, 289, 310, 234, 144, 317, -109, 382, - /* 10 */ 377, 303, 461, 389, 378, 368, 302, 294, 253, 395, - /* 20 */ 293, 324, 403, 403, 403, 403, 403, 403, 403, 403, - /* 30 */ 403, 403, 403, 403, 403, 403, 403, 403, 403, 403, - /* 40 */ 403, 403, 403, 403, 403, 403, 403, 403, 403, 403, - /* 50 */ 403, 1022, 1012, 1005, 998, 963, 961, 959, 957, 950, - /* 60 */ 947, 930, 912, 873, 861, 823, 810, 771, 759, 720, - /* 70 */ 708, 670, 657, 619, 614, 612, 610, 608, 606, 604, - /* 80 */ 598, 595, 593, 580, 542, 540, 537, 535, 533, 531, - /* 90 */ 529, 527, 503, 386, 403, 403, 403, 403, 403, 403, - /* 100 */ 403, 403, 403, 95, 447, 82, 334, 504, 467, 403, - /* 110 */ 477, 464, 403, 403, 403, 403, 860, 747, 744, 785, - /* 120 */ 638, 638, 926, 891, 900, 899, 887, 844, 840, 835, - /* 130 */ 848, 830, 843, 829, 792, 839, 826, 737, 838, 795, - /* 140 */ 789, 47, 734, 530, 696, 777, 711, 677, 733, 730, - /* 150 */ 729, 728, 727, 627, 448, 64, 187, 1305, 1302, 1252, - /* 160 */ 1290, 1273, 1323, 1322, 1321, 1319, 1318, 1316, 1315, 1314, - /* 170 */ 1313, 1312, 1311, 1310, 1308, 1307, 1304, 1303, 1301, 1298, - /* 180 */ 1294, 1292, 1289, 1266, 1264, 1259, 1288, 1287, 1238, 1285, - /* 190 */ 1281, 1280, 1279, 1278, 1251, 1277, 1276, 1275, 1273, 1268, - /* 200 */ 1267, 1265, 1263, 1261, 1257, 1248, 1237, 1247, 1246, 1243, - /* 210 */ 1238, 1240, 1235, 1249, 1234, 1233, 1230, 1220, 1214, 1210, - /* 220 */ 1225, 1219, 1232, 1231, 1197, 1195, 1227, 1224, 1201, 1208, - /* 230 */ 1242, 1137, 1236, 1229, 1193, 1181, 1221, 1177, 1196, 1179, - /* 240 */ 1191, 1190, 1186, 1182, 1218, 1216, 1176, 1162, 1183, 1180, - /* 250 */ 1160, 1199, 1203, 1133, 1095, 1198, 1194, 1188, 1192, 1171, - /* 260 */ 1169, 1168, 1173, 1174, 1166, 1159, 1141, 1170, 1158, 1167, - /* 270 */ 1157, 1132, 1145, 1143, 1124, 1128, 1103, 1102, 1100, 1096, - /* 280 */ 1150, 1149, 1085, 1125, 1080, 1064, 1120, 1097, 1082, 1078, - /* 290 */ 1073, 1067, 1109, 1107, 1119, 1117, 1116, 1113, 1111, 1108, - /* 300 */ 1007, 1000, 1002, 1076, 1075, 1081, + /* 0 */ -143, 954, 86, 21, -50, 23, 79, 134, 170, -175, + /* 10 */ 229, 260, -121, 212, 219, 291, -54, 349, 362, 156, + /* 20 */ 309, 311, 334, 85, 224, 394, 314, 314, 314, 314, + /* 30 */ 314, 314, 314, 314, 314, 314, 314, 314, 314, 314, + /* 40 */ 314, 314, 314, 314, 314, 314, 314, 314, 314, 314, + /* 50 */ 314, 314, 314, 314, 314, 374, 441, 443, 450, 452, + /* 60 */ 515, 554, 567, 569, 572, 578, 580, 582, 584, 587, + /* 70 */ 593, 631, 644, 646, 649, 655, 657, 659, 661, 664, + /* 80 */ 708, 720, 759, 771, 810, 822, 861, 873, 912, 930, + /* 90 */ 947, 950, 957, 959, 963, 966, 968, 998, 314, 314, + /* 100 */ 314, 314, 314, 314, 314, 314, 314, 447, -53, 166, + /* 110 */ 438, 348, 363, 314, 473, 469, 314, 314, 314, 314, + /* 120 */ -15, 59, 101, 688, 220, 220, 525, 256, 729, 735, + /* 130 */ 736, 740, 741, 744, 645, 448, 738, 458, 786, 503, + /* 140 */ 780, 656, 721, 724, 792, 545, 568, 706, 683, 681, + /* 150 */ 779, 784, 830, 831, 835, 678, 601, -104, -2, 96, + /* 160 */ 111, 218, 287, 308, 310, 312, 335, 411, 453, 461, + /* 170 */ 573, 599, 617, 658, 665, 670, 732, 734, 775, 848, + /* 180 */ 875, 892, 893, 898, 332, 420, 869, 931, 944, 886, + /* 190 */ 983, 992, 1009, 958, 1017, 1028, 988, 1033, 1034, 1035, + /* 200 */ 287, 1036, 1044, 1045, 1047, 1049, 1056, 915, 972, 997, + /* 210 */ 1000, 1002, 886, 1011, 1015, 1061, 1013, 1001, 1003, 977, + /* 220 */ 1018, 979, 1050, 1041, 1040, 1052, 1014, 1004, 1059, 1060, + /* 230 */ 1032, 1038, 1084, 995, 1089, 1090, 1008, 1016, 1092, 1037, + /* 240 */ 1068, 1062, 1069, 1072, 1073, 1074, 1105, 1112, 1071, 1048, + /* 250 */ 1081, 1088, 1078, 1116, 1118, 1046, 1066, 1128, 1136, 1140, + /* 260 */ 1120, 1147, 1146, 1148, 1150, 1130, 1135, 1137, 1138, 1132, + /* 270 */ 1141, 1142, 1143, 1149, 1144, 1153, 1154, 1104, 1107, 1108, + /* 280 */ 1114, 1115, 1117, 1123, 1125, 1173, 1176, 1121, 1165, 1127, + /* 290 */ 1131, 1167, 1157, 1151, 1158, 1166, 1168, 1212, 1214, 1227, + /* 300 */ 1228, 1231, 1232, 1233, 1234, 1152, 1155, 1159, 1198, 1199, + /* 310 */ 1219, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 647, 964, 964, 964, 878, 878, 969, 964, 774, 802, - /* 10 */ 802, 938, 969, 969, 969, 876, 969, 969, 969, 964, - /* 20 */ 969, 778, 808, 969, 969, 969, 969, 969, 969, 969, - /* 30 */ 969, 937, 939, 816, 815, 918, 789, 813, 806, 810, - /* 40 */ 879, 872, 873, 871, 875, 880, 969, 809, 841, 856, - /* 50 */ 840, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 60 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 70 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 80 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 90 */ 969, 969, 969, 969, 850, 855, 862, 854, 851, 843, - /* 100 */ 842, 844, 845, 969, 969, 673, 739, 969, 969, 846, - /* 110 */ 969, 685, 847, 859, 858, 857, 680, 969, 969, 969, - /* 120 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 130 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 140 */ 969, 969, 969, 969, 647, 964, 969, 969, 964, 964, - /* 150 */ 964, 964, 964, 964, 956, 778, 768, 969, 969, 969, - /* 160 */ 969, 969, 969, 969, 969, 969, 969, 944, 942, 969, - /* 170 */ 891, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 180 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 190 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 200 */ 969, 969, 969, 969, 653, 969, 911, 774, 774, 774, - /* 210 */ 776, 754, 766, 655, 812, 791, 791, 923, 812, 923, - /* 220 */ 710, 733, 707, 802, 791, 874, 802, 802, 775, 766, - /* 230 */ 969, 949, 782, 782, 941, 941, 782, 821, 743, 812, - /* 240 */ 750, 750, 750, 750, 782, 670, 812, 821, 743, 743, - /* 250 */ 812, 782, 670, 917, 915, 782, 782, 670, 782, 670, - /* 260 */ 782, 670, 884, 741, 741, 741, 725, 884, 741, 710, - /* 270 */ 741, 725, 741, 741, 795, 790, 795, 790, 795, 790, - /* 280 */ 782, 782, 969, 884, 888, 888, 884, 807, 796, 805, - /* 290 */ 803, 812, 676, 728, 663, 663, 652, 652, 652, 652, - /* 300 */ 961, 961, 956, 712, 712, 695, 969, 969, 969, 969, - /* 310 */ 969, 969, 687, 969, 893, 969, 969, 969, 969, 969, - /* 320 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 330 */ 969, 828, 969, 648, 951, 969, 969, 948, 969, 969, - /* 340 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 350 */ 969, 969, 969, 969, 969, 969, 921, 969, 969, 969, - /* 360 */ 969, 969, 969, 914, 913, 969, 969, 969, 969, 969, - /* 370 */ 969, 969, 969, 969, 969, 969, 969, 969, 969, 969, - /* 380 */ 969, 969, 969, 969, 969, 969, 969, 757, 969, 969, - /* 390 */ 969, 761, 969, 969, 969, 969, 969, 969, 804, 969, - /* 400 */ 797, 969, 877, 969, 969, 969, 969, 969, 969, 969, - /* 410 */ 969, 969, 969, 966, 969, 969, 969, 965, 969, 969, - /* 420 */ 969, 969, 969, 830, 969, 829, 833, 969, 661, 969, - /* 430 */ 644, 649, 960, 963, 962, 959, 958, 957, 952, 950, - /* 440 */ 947, 946, 945, 943, 940, 936, 897, 895, 902, 901, - /* 450 */ 900, 899, 898, 896, 894, 892, 818, 817, 814, 811, - /* 460 */ 753, 935, 890, 752, 749, 748, 669, 953, 920, 929, - /* 470 */ 928, 927, 822, 926, 925, 924, 922, 919, 906, 820, - /* 480 */ 819, 744, 882, 881, 672, 910, 909, 908, 912, 916, - /* 490 */ 907, 784, 751, 671, 668, 675, 679, 731, 732, 740, - /* 500 */ 738, 737, 736, 735, 734, 730, 681, 686, 724, 709, - /* 510 */ 708, 717, 716, 722, 721, 720, 719, 718, 715, 714, - /* 520 */ 713, 706, 705, 711, 704, 727, 726, 723, 703, 747, - /* 530 */ 746, 745, 742, 702, 701, 700, 833, 699, 698, 838, - /* 540 */ 837, 866, 826, 755, 759, 758, 762, 763, 771, 770, - /* 550 */ 769, 780, 781, 793, 792, 824, 823, 794, 779, 773, - /* 560 */ 772, 788, 787, 786, 785, 777, 767, 799, 798, 868, - /* 570 */ 783, 867, 865, 934, 933, 932, 931, 930, 870, 967, - /* 580 */ 968, 887, 889, 886, 801, 800, 885, 869, 839, 836, - /* 590 */ 690, 691, 905, 904, 903, 693, 692, 689, 688, 863, - /* 600 */ 860, 852, 864, 861, 853, 849, 848, 834, 832, 831, - /* 610 */ 827, 835, 760, 756, 825, 765, 764, 697, 696, 694, - /* 620 */ 678, 677, 674, 667, 665, 664, 666, 662, 660, 659, - /* 630 */ 658, 657, 656, 684, 683, 682, 654, 651, 650, 646, - /* 640 */ 645, 643, + /* 0 */ 982, 1300, 1300, 1300, 1214, 1214, 1214, 1305, 1300, 1109, + /* 10 */ 1138, 1138, 1274, 1305, 1305, 1305, 1305, 1305, 1305, 1212, + /* 20 */ 1305, 1305, 1305, 1300, 1305, 1113, 1144, 1305, 1305, 1305, + /* 30 */ 1305, 1305, 1305, 1305, 1305, 1273, 1275, 1152, 1151, 1254, + /* 40 */ 1125, 1149, 1142, 1146, 1215, 1208, 1209, 1207, 1211, 1216, + /* 50 */ 1305, 1145, 1177, 1192, 1176, 1305, 1305, 1305, 1305, 1305, + /* 60 */ 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, + /* 70 */ 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, + /* 80 */ 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, + /* 90 */ 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1186, 1191, + /* 100 */ 1198, 1190, 1187, 1179, 1178, 1180, 1181, 1305, 1305, 1008, + /* 110 */ 1074, 1305, 1305, 1182, 1305, 1020, 1183, 1195, 1194, 1193, + /* 120 */ 1015, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, + /* 130 */ 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, + /* 140 */ 1305, 1305, 1305, 1305, 1305, 982, 1300, 1305, 1305, 1300, + /* 150 */ 1300, 1300, 1300, 1300, 1300, 1292, 1113, 1103, 1305, 1305, + /* 160 */ 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1280, 1278, + /* 170 */ 1305, 1227, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, + /* 180 */ 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, + /* 190 */ 1305, 1305, 1305, 1109, 1305, 1305, 1305, 1305, 1305, 1305, + /* 200 */ 1305, 1305, 1305, 1305, 1305, 1305, 988, 1305, 1247, 1109, + /* 210 */ 1109, 1109, 1111, 1089, 1101, 990, 1148, 1127, 1127, 1259, + /* 220 */ 1148, 1259, 1045, 1068, 1042, 1138, 1127, 1210, 1138, 1138, + /* 230 */ 1110, 1101, 1305, 1285, 1118, 1118, 1277, 1277, 1118, 1157, + /* 240 */ 1078, 1148, 1085, 1085, 1085, 1085, 1118, 1005, 1148, 1157, + /* 250 */ 1078, 1078, 1148, 1118, 1005, 1253, 1251, 1118, 1118, 1005, + /* 260 */ 1220, 1118, 1005, 1118, 1005, 1220, 1076, 1076, 1076, 1060, + /* 270 */ 1220, 1076, 1045, 1076, 1060, 1076, 1076, 1131, 1126, 1131, + /* 280 */ 1126, 1131, 1126, 1131, 1126, 1118, 1118, 1305, 1220, 1224, + /* 290 */ 1224, 1220, 1143, 1132, 1141, 1139, 1148, 1011, 1063, 998, + /* 300 */ 998, 987, 987, 987, 987, 1297, 1297, 1292, 1047, 1047, + /* 310 */ 1030, 1305, 1305, 1305, 1305, 1305, 1305, 1022, 1305, 1229, + /* 320 */ 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, + /* 330 */ 1305, 1305, 1305, 1305, 1305, 1305, 1164, 1305, 983, 1287, + /* 340 */ 1305, 1305, 1284, 1305, 1305, 1305, 1305, 1305, 1305, 1305, + /* 350 */ 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, + /* 360 */ 1305, 1257, 1305, 1305, 1305, 1305, 1305, 1305, 1250, 1249, + /* 370 */ 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, + /* 380 */ 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, + /* 390 */ 1305, 1305, 1092, 1305, 1305, 1305, 1096, 1305, 1305, 1305, + /* 400 */ 1305, 1305, 1305, 1305, 1140, 1305, 1133, 1305, 1213, 1305, + /* 410 */ 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1305, 1302, + /* 420 */ 1305, 1305, 1305, 1301, 1305, 1305, 1305, 1305, 1305, 1166, + /* 430 */ 1305, 1165, 1169, 1305, 996, 1305, }; /* The next table maps tokens into fallback tokens. If a construct @@ -124090,9 +126866,13 @@ static const YYCODETYPE yyFallback[] = { ** + The semantic value stored at this level of the stack. This is ** the information used by the action routines in the grammar. ** It is sometimes called the "minor" token. +** +** After the "shift" half of a SHIFTREDUCE action, the stateno field +** actually contains the reduce action for the second half of the +** SHIFTREDUCE. */ struct yyStackEntry { - YYACTIONTYPE stateno; /* The state-number */ + YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ YYCODETYPE major; /* The major token value. This is the code ** number for the token at this stack level */ YYMINORTYPE minor; /* The user-supplied minor token value. This @@ -124198,18 +126978,18 @@ static const char *const yyTokenName[] = { "column", "columnid", "type", "carglist", "typetoken", "typename", "signed", "plus_num", "minus_num", "ccons", "term", "expr", - "onconf", "sortorder", "autoinc", "idxlist_opt", + "onconf", "sortorder", "autoinc", "eidlist_opt", "refargs", "defer_subclause", "refarg", "refact", "init_deferred_pred_opt", "conslist", "tconscomma", "tcons", - "idxlist", "defer_subclause_opt", "orconf", "resolvetype", - "raisetype", "ifexists", "fullname", "selectnowith", - "oneselect", "with", "multiselect_op", "distinct", - "selcollist", "from", "where_opt", "groupby_opt", - "having_opt", "orderby_opt", "limit_opt", "values", - "nexprlist", "exprlist", "sclp", "as", - "seltablist", "stl_prefix", "joinop", "indexed_opt", - "on_opt", "using_opt", "joinop2", "idlist", - "sortlist", "setlist", "insert_cmd", "inscollist_opt", + "sortlist", "eidlist", "defer_subclause_opt", "orconf", + "resolvetype", "raisetype", "ifexists", "fullname", + "selectnowith", "oneselect", "with", "multiselect_op", + "distinct", "selcollist", "from", "where_opt", + "groupby_opt", "having_opt", "orderby_opt", "limit_opt", + "values", "nexprlist", "exprlist", "sclp", + "as", "seltablist", "stl_prefix", "joinop", + "indexed_opt", "on_opt", "using_opt", "joinop2", + "idlist", "setlist", "insert_cmd", "idlist_opt", "likeop", "between_op", "in_op", "case_operand", "case_exprlist", "case_else", "uniqueflag", "collate", "nmnum", "trigger_decl", "trigger_cmd_list", "trigger_time", @@ -124290,7 +127070,7 @@ static const char *const yyRuleName[] = { /* 62 */ "ccons ::= PRIMARY KEY sortorder onconf autoinc", /* 63 */ "ccons ::= UNIQUE onconf", /* 64 */ "ccons ::= CHECK LP expr RP", - /* 65 */ "ccons ::= REFERENCES nm idxlist_opt refargs", + /* 65 */ "ccons ::= REFERENCES nm eidlist_opt refargs", /* 66 */ "ccons ::= defer_subclause", /* 67 */ "ccons ::= COLLATE ID|STRING", /* 68 */ "autoinc ::=", @@ -124318,10 +127098,10 @@ static const char *const yyRuleName[] = { /* 90 */ "tconscomma ::= COMMA", /* 91 */ "tconscomma ::=", /* 92 */ "tcons ::= CONSTRAINT nm", - /* 93 */ "tcons ::= PRIMARY KEY LP idxlist autoinc RP onconf", - /* 94 */ "tcons ::= UNIQUE LP idxlist RP onconf", + /* 93 */ "tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf", + /* 94 */ "tcons ::= UNIQUE LP sortlist RP onconf", /* 95 */ "tcons ::= CHECK LP expr RP onconf", - /* 96 */ "tcons ::= FOREIGN KEY LP idxlist RP REFERENCES nm idxlist_opt refargs defer_subclause_opt", + /* 96 */ "tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt", /* 97 */ "defer_subclause_opt ::=", /* 98 */ "defer_subclause_opt ::= defer_subclause", /* 99 */ "onconf ::=", @@ -124334,7 +127114,7 @@ static const char *const yyRuleName[] = { /* 106 */ "cmd ::= DROP TABLE ifexists fullname", /* 107 */ "ifexists ::= IF EXISTS", /* 108 */ "ifexists ::=", - /* 109 */ "cmd ::= createkw temp VIEW ifnotexists nm dbnm AS select", + /* 109 */ "cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select", /* 110 */ "cmd ::= DROP VIEW ifexists fullname", /* 111 */ "cmd ::= select", /* 112 */ "select ::= with selectnowith", @@ -124363,195 +127143,196 @@ static const char *const yyRuleName[] = { /* 135 */ "stl_prefix ::= seltablist joinop", /* 136 */ "stl_prefix ::=", /* 137 */ "seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt", - /* 138 */ "seltablist ::= stl_prefix LP select RP as on_opt using_opt", - /* 139 */ "seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt", - /* 140 */ "dbnm ::=", - /* 141 */ "dbnm ::= DOT nm", - /* 142 */ "fullname ::= nm dbnm", - /* 143 */ "joinop ::= COMMA|JOIN", - /* 144 */ "joinop ::= JOIN_KW JOIN", - /* 145 */ "joinop ::= JOIN_KW nm JOIN", - /* 146 */ "joinop ::= JOIN_KW nm nm JOIN", - /* 147 */ "on_opt ::= ON expr", - /* 148 */ "on_opt ::=", - /* 149 */ "indexed_opt ::=", - /* 150 */ "indexed_opt ::= INDEXED BY nm", - /* 151 */ "indexed_opt ::= NOT INDEXED", - /* 152 */ "using_opt ::= USING LP idlist RP", - /* 153 */ "using_opt ::=", - /* 154 */ "orderby_opt ::=", - /* 155 */ "orderby_opt ::= ORDER BY sortlist", - /* 156 */ "sortlist ::= sortlist COMMA expr sortorder", - /* 157 */ "sortlist ::= expr sortorder", - /* 158 */ "sortorder ::= ASC", - /* 159 */ "sortorder ::= DESC", - /* 160 */ "sortorder ::=", - /* 161 */ "groupby_opt ::=", - /* 162 */ "groupby_opt ::= GROUP BY nexprlist", - /* 163 */ "having_opt ::=", - /* 164 */ "having_opt ::= HAVING expr", - /* 165 */ "limit_opt ::=", - /* 166 */ "limit_opt ::= LIMIT expr", - /* 167 */ "limit_opt ::= LIMIT expr OFFSET expr", - /* 168 */ "limit_opt ::= LIMIT expr COMMA expr", - /* 169 */ "cmd ::= with DELETE FROM fullname indexed_opt where_opt", - /* 170 */ "where_opt ::=", - /* 171 */ "where_opt ::= WHERE expr", - /* 172 */ "cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt", - /* 173 */ "setlist ::= setlist COMMA nm EQ expr", - /* 174 */ "setlist ::= nm EQ expr", - /* 175 */ "cmd ::= with insert_cmd INTO fullname inscollist_opt select", - /* 176 */ "cmd ::= with insert_cmd INTO fullname inscollist_opt DEFAULT VALUES", - /* 177 */ "insert_cmd ::= INSERT orconf", - /* 178 */ "insert_cmd ::= REPLACE", - /* 179 */ "inscollist_opt ::=", - /* 180 */ "inscollist_opt ::= LP idlist RP", - /* 181 */ "idlist ::= idlist COMMA nm", - /* 182 */ "idlist ::= nm", - /* 183 */ "expr ::= term", - /* 184 */ "expr ::= LP expr RP", - /* 185 */ "term ::= NULL", - /* 186 */ "expr ::= ID|INDEXED", - /* 187 */ "expr ::= JOIN_KW", - /* 188 */ "expr ::= nm DOT nm", - /* 189 */ "expr ::= nm DOT nm DOT nm", - /* 190 */ "term ::= INTEGER|FLOAT|BLOB", - /* 191 */ "term ::= STRING", - /* 192 */ "expr ::= VARIABLE", - /* 193 */ "expr ::= expr COLLATE ID|STRING", - /* 194 */ "expr ::= CAST LP expr AS typetoken RP", - /* 195 */ "expr ::= ID|INDEXED LP distinct exprlist RP", - /* 196 */ "expr ::= ID|INDEXED LP STAR RP", - /* 197 */ "term ::= CTIME_KW", - /* 198 */ "expr ::= expr AND expr", - /* 199 */ "expr ::= expr OR expr", - /* 200 */ "expr ::= expr LT|GT|GE|LE expr", - /* 201 */ "expr ::= expr EQ|NE expr", - /* 202 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", - /* 203 */ "expr ::= expr PLUS|MINUS expr", - /* 204 */ "expr ::= expr STAR|SLASH|REM expr", - /* 205 */ "expr ::= expr CONCAT expr", - /* 206 */ "likeop ::= LIKE_KW|MATCH", - /* 207 */ "likeop ::= NOT LIKE_KW|MATCH", - /* 208 */ "expr ::= expr likeop expr", - /* 209 */ "expr ::= expr likeop expr ESCAPE expr", - /* 210 */ "expr ::= expr ISNULL|NOTNULL", - /* 211 */ "expr ::= expr NOT NULL", - /* 212 */ "expr ::= expr IS expr", - /* 213 */ "expr ::= expr IS NOT expr", - /* 214 */ "expr ::= NOT expr", - /* 215 */ "expr ::= BITNOT expr", - /* 216 */ "expr ::= MINUS expr", - /* 217 */ "expr ::= PLUS expr", - /* 218 */ "between_op ::= BETWEEN", - /* 219 */ "between_op ::= NOT BETWEEN", - /* 220 */ "expr ::= expr between_op expr AND expr", - /* 221 */ "in_op ::= IN", - /* 222 */ "in_op ::= NOT IN", - /* 223 */ "expr ::= expr in_op LP exprlist RP", - /* 224 */ "expr ::= LP select RP", - /* 225 */ "expr ::= expr in_op LP select RP", - /* 226 */ "expr ::= expr in_op nm dbnm", - /* 227 */ "expr ::= EXISTS LP select RP", - /* 228 */ "expr ::= CASE case_operand case_exprlist case_else END", - /* 229 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", - /* 230 */ "case_exprlist ::= WHEN expr THEN expr", - /* 231 */ "case_else ::= ELSE expr", - /* 232 */ "case_else ::=", - /* 233 */ "case_operand ::= expr", - /* 234 */ "case_operand ::=", - /* 235 */ "exprlist ::= nexprlist", - /* 236 */ "exprlist ::=", - /* 237 */ "nexprlist ::= nexprlist COMMA expr", - /* 238 */ "nexprlist ::= expr", - /* 239 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP idxlist RP where_opt", - /* 240 */ "uniqueflag ::= UNIQUE", - /* 241 */ "uniqueflag ::=", - /* 242 */ "idxlist_opt ::=", - /* 243 */ "idxlist_opt ::= LP idxlist RP", - /* 244 */ "idxlist ::= idxlist COMMA nm collate sortorder", - /* 245 */ "idxlist ::= nm collate sortorder", - /* 246 */ "collate ::=", - /* 247 */ "collate ::= COLLATE ID|STRING", - /* 248 */ "cmd ::= DROP INDEX ifexists fullname", - /* 249 */ "cmd ::= VACUUM", - /* 250 */ "cmd ::= VACUUM nm", - /* 251 */ "cmd ::= PRAGMA nm dbnm", - /* 252 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", - /* 253 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", - /* 254 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", - /* 255 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", - /* 256 */ "nmnum ::= plus_num", - /* 257 */ "nmnum ::= nm", - /* 258 */ "nmnum ::= ON", - /* 259 */ "nmnum ::= DELETE", - /* 260 */ "nmnum ::= DEFAULT", - /* 261 */ "plus_num ::= PLUS INTEGER|FLOAT", - /* 262 */ "plus_num ::= INTEGER|FLOAT", - /* 263 */ "minus_num ::= MINUS INTEGER|FLOAT", - /* 264 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", - /* 265 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", - /* 266 */ "trigger_time ::= BEFORE", - /* 267 */ "trigger_time ::= AFTER", - /* 268 */ "trigger_time ::= INSTEAD OF", - /* 269 */ "trigger_time ::=", - /* 270 */ "trigger_event ::= DELETE|INSERT", - /* 271 */ "trigger_event ::= UPDATE", - /* 272 */ "trigger_event ::= UPDATE OF idlist", - /* 273 */ "foreach_clause ::=", - /* 274 */ "foreach_clause ::= FOR EACH ROW", - /* 275 */ "when_clause ::=", - /* 276 */ "when_clause ::= WHEN expr", - /* 277 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", - /* 278 */ "trigger_cmd_list ::= trigger_cmd SEMI", - /* 279 */ "trnm ::= nm", - /* 280 */ "trnm ::= nm DOT nm", - /* 281 */ "tridxby ::=", - /* 282 */ "tridxby ::= INDEXED BY nm", - /* 283 */ "tridxby ::= NOT INDEXED", - /* 284 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt", - /* 285 */ "trigger_cmd ::= insert_cmd INTO trnm inscollist_opt select", - /* 286 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt", - /* 287 */ "trigger_cmd ::= select", - /* 288 */ "expr ::= RAISE LP IGNORE RP", - /* 289 */ "expr ::= RAISE LP raisetype COMMA nm RP", - /* 290 */ "raisetype ::= ROLLBACK", - /* 291 */ "raisetype ::= ABORT", - /* 292 */ "raisetype ::= FAIL", - /* 293 */ "cmd ::= DROP TRIGGER ifexists fullname", - /* 294 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", - /* 295 */ "cmd ::= DETACH database_kw_opt expr", - /* 296 */ "key_opt ::=", - /* 297 */ "key_opt ::= KEY expr", - /* 298 */ "database_kw_opt ::= DATABASE", - /* 299 */ "database_kw_opt ::=", - /* 300 */ "cmd ::= REINDEX", - /* 301 */ "cmd ::= REINDEX nm dbnm", - /* 302 */ "cmd ::= ANALYZE", - /* 303 */ "cmd ::= ANALYZE nm dbnm", - /* 304 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", - /* 305 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column", - /* 306 */ "add_column_fullname ::= fullname", - /* 307 */ "kwcolumn_opt ::=", - /* 308 */ "kwcolumn_opt ::= COLUMNKW", - /* 309 */ "cmd ::= create_vtab", - /* 310 */ "cmd ::= create_vtab LP vtabarglist RP", - /* 311 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", - /* 312 */ "vtabarglist ::= vtabarg", - /* 313 */ "vtabarglist ::= vtabarglist COMMA vtabarg", - /* 314 */ "vtabarg ::=", - /* 315 */ "vtabarg ::= vtabarg vtabargtoken", - /* 316 */ "vtabargtoken ::= ANY", - /* 317 */ "vtabargtoken ::= lp anylist RP", - /* 318 */ "lp ::= LP", - /* 319 */ "anylist ::=", - /* 320 */ "anylist ::= anylist LP anylist RP", - /* 321 */ "anylist ::= anylist ANY", - /* 322 */ "with ::=", - /* 323 */ "with ::= WITH wqlist", - /* 324 */ "with ::= WITH RECURSIVE wqlist", - /* 325 */ "wqlist ::= nm idxlist_opt AS LP select RP", - /* 326 */ "wqlist ::= wqlist COMMA nm idxlist_opt AS LP select RP", + /* 138 */ "seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt", + /* 139 */ "seltablist ::= stl_prefix LP select RP as on_opt using_opt", + /* 140 */ "seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt", + /* 141 */ "dbnm ::=", + /* 142 */ "dbnm ::= DOT nm", + /* 143 */ "fullname ::= nm dbnm", + /* 144 */ "joinop ::= COMMA|JOIN", + /* 145 */ "joinop ::= JOIN_KW JOIN", + /* 146 */ "joinop ::= JOIN_KW nm JOIN", + /* 147 */ "joinop ::= JOIN_KW nm nm JOIN", + /* 148 */ "on_opt ::= ON expr", + /* 149 */ "on_opt ::=", + /* 150 */ "indexed_opt ::=", + /* 151 */ "indexed_opt ::= INDEXED BY nm", + /* 152 */ "indexed_opt ::= NOT INDEXED", + /* 153 */ "using_opt ::= USING LP idlist RP", + /* 154 */ "using_opt ::=", + /* 155 */ "orderby_opt ::=", + /* 156 */ "orderby_opt ::= ORDER BY sortlist", + /* 157 */ "sortlist ::= sortlist COMMA expr sortorder", + /* 158 */ "sortlist ::= expr sortorder", + /* 159 */ "sortorder ::= ASC", + /* 160 */ "sortorder ::= DESC", + /* 161 */ "sortorder ::=", + /* 162 */ "groupby_opt ::=", + /* 163 */ "groupby_opt ::= GROUP BY nexprlist", + /* 164 */ "having_opt ::=", + /* 165 */ "having_opt ::= HAVING expr", + /* 166 */ "limit_opt ::=", + /* 167 */ "limit_opt ::= LIMIT expr", + /* 168 */ "limit_opt ::= LIMIT expr OFFSET expr", + /* 169 */ "limit_opt ::= LIMIT expr COMMA expr", + /* 170 */ "cmd ::= with DELETE FROM fullname indexed_opt where_opt", + /* 171 */ "where_opt ::=", + /* 172 */ "where_opt ::= WHERE expr", + /* 173 */ "cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt", + /* 174 */ "setlist ::= setlist COMMA nm EQ expr", + /* 175 */ "setlist ::= nm EQ expr", + /* 176 */ "cmd ::= with insert_cmd INTO fullname idlist_opt select", + /* 177 */ "cmd ::= with insert_cmd INTO fullname idlist_opt DEFAULT VALUES", + /* 178 */ "insert_cmd ::= INSERT orconf", + /* 179 */ "insert_cmd ::= REPLACE", + /* 180 */ "idlist_opt ::=", + /* 181 */ "idlist_opt ::= LP idlist RP", + /* 182 */ "idlist ::= idlist COMMA nm", + /* 183 */ "idlist ::= nm", + /* 184 */ "expr ::= term", + /* 185 */ "expr ::= LP expr RP", + /* 186 */ "term ::= NULL", + /* 187 */ "expr ::= ID|INDEXED", + /* 188 */ "expr ::= JOIN_KW", + /* 189 */ "expr ::= nm DOT nm", + /* 190 */ "expr ::= nm DOT nm DOT nm", + /* 191 */ "term ::= INTEGER|FLOAT|BLOB", + /* 192 */ "term ::= STRING", + /* 193 */ "expr ::= VARIABLE", + /* 194 */ "expr ::= expr COLLATE ID|STRING", + /* 195 */ "expr ::= CAST LP expr AS typetoken RP", + /* 196 */ "expr ::= ID|INDEXED LP distinct exprlist RP", + /* 197 */ "expr ::= ID|INDEXED LP STAR RP", + /* 198 */ "term ::= CTIME_KW", + /* 199 */ "expr ::= expr AND expr", + /* 200 */ "expr ::= expr OR expr", + /* 201 */ "expr ::= expr LT|GT|GE|LE expr", + /* 202 */ "expr ::= expr EQ|NE expr", + /* 203 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", + /* 204 */ "expr ::= expr PLUS|MINUS expr", + /* 205 */ "expr ::= expr STAR|SLASH|REM expr", + /* 206 */ "expr ::= expr CONCAT expr", + /* 207 */ "likeop ::= LIKE_KW|MATCH", + /* 208 */ "likeop ::= NOT LIKE_KW|MATCH", + /* 209 */ "expr ::= expr likeop expr", + /* 210 */ "expr ::= expr likeop expr ESCAPE expr", + /* 211 */ "expr ::= expr ISNULL|NOTNULL", + /* 212 */ "expr ::= expr NOT NULL", + /* 213 */ "expr ::= expr IS expr", + /* 214 */ "expr ::= expr IS NOT expr", + /* 215 */ "expr ::= NOT expr", + /* 216 */ "expr ::= BITNOT expr", + /* 217 */ "expr ::= MINUS expr", + /* 218 */ "expr ::= PLUS expr", + /* 219 */ "between_op ::= BETWEEN", + /* 220 */ "between_op ::= NOT BETWEEN", + /* 221 */ "expr ::= expr between_op expr AND expr", + /* 222 */ "in_op ::= IN", + /* 223 */ "in_op ::= NOT IN", + /* 224 */ "expr ::= expr in_op LP exprlist RP", + /* 225 */ "expr ::= LP select RP", + /* 226 */ "expr ::= expr in_op LP select RP", + /* 227 */ "expr ::= expr in_op nm dbnm", + /* 228 */ "expr ::= EXISTS LP select RP", + /* 229 */ "expr ::= CASE case_operand case_exprlist case_else END", + /* 230 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", + /* 231 */ "case_exprlist ::= WHEN expr THEN expr", + /* 232 */ "case_else ::= ELSE expr", + /* 233 */ "case_else ::=", + /* 234 */ "case_operand ::= expr", + /* 235 */ "case_operand ::=", + /* 236 */ "exprlist ::= nexprlist", + /* 237 */ "exprlist ::=", + /* 238 */ "nexprlist ::= nexprlist COMMA expr", + /* 239 */ "nexprlist ::= expr", + /* 240 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", + /* 241 */ "uniqueflag ::= UNIQUE", + /* 242 */ "uniqueflag ::=", + /* 243 */ "eidlist_opt ::=", + /* 244 */ "eidlist_opt ::= LP eidlist RP", + /* 245 */ "eidlist ::= eidlist COMMA nm collate sortorder", + /* 246 */ "eidlist ::= nm collate sortorder", + /* 247 */ "collate ::=", + /* 248 */ "collate ::= COLLATE ID|STRING", + /* 249 */ "cmd ::= DROP INDEX ifexists fullname", + /* 250 */ "cmd ::= VACUUM", + /* 251 */ "cmd ::= VACUUM nm", + /* 252 */ "cmd ::= PRAGMA nm dbnm", + /* 253 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", + /* 254 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", + /* 255 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", + /* 256 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", + /* 257 */ "nmnum ::= plus_num", + /* 258 */ "nmnum ::= nm", + /* 259 */ "nmnum ::= ON", + /* 260 */ "nmnum ::= DELETE", + /* 261 */ "nmnum ::= DEFAULT", + /* 262 */ "plus_num ::= PLUS INTEGER|FLOAT", + /* 263 */ "plus_num ::= INTEGER|FLOAT", + /* 264 */ "minus_num ::= MINUS INTEGER|FLOAT", + /* 265 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", + /* 266 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", + /* 267 */ "trigger_time ::= BEFORE", + /* 268 */ "trigger_time ::= AFTER", + /* 269 */ "trigger_time ::= INSTEAD OF", + /* 270 */ "trigger_time ::=", + /* 271 */ "trigger_event ::= DELETE|INSERT", + /* 272 */ "trigger_event ::= UPDATE", + /* 273 */ "trigger_event ::= UPDATE OF idlist", + /* 274 */ "foreach_clause ::=", + /* 275 */ "foreach_clause ::= FOR EACH ROW", + /* 276 */ "when_clause ::=", + /* 277 */ "when_clause ::= WHEN expr", + /* 278 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", + /* 279 */ "trigger_cmd_list ::= trigger_cmd SEMI", + /* 280 */ "trnm ::= nm", + /* 281 */ "trnm ::= nm DOT nm", + /* 282 */ "tridxby ::=", + /* 283 */ "tridxby ::= INDEXED BY nm", + /* 284 */ "tridxby ::= NOT INDEXED", + /* 285 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt", + /* 286 */ "trigger_cmd ::= insert_cmd INTO trnm idlist_opt select", + /* 287 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt", + /* 288 */ "trigger_cmd ::= select", + /* 289 */ "expr ::= RAISE LP IGNORE RP", + /* 290 */ "expr ::= RAISE LP raisetype COMMA nm RP", + /* 291 */ "raisetype ::= ROLLBACK", + /* 292 */ "raisetype ::= ABORT", + /* 293 */ "raisetype ::= FAIL", + /* 294 */ "cmd ::= DROP TRIGGER ifexists fullname", + /* 295 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", + /* 296 */ "cmd ::= DETACH database_kw_opt expr", + /* 297 */ "key_opt ::=", + /* 298 */ "key_opt ::= KEY expr", + /* 299 */ "database_kw_opt ::= DATABASE", + /* 300 */ "database_kw_opt ::=", + /* 301 */ "cmd ::= REINDEX", + /* 302 */ "cmd ::= REINDEX nm dbnm", + /* 303 */ "cmd ::= ANALYZE", + /* 304 */ "cmd ::= ANALYZE nm dbnm", + /* 305 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", + /* 306 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column", + /* 307 */ "add_column_fullname ::= fullname", + /* 308 */ "kwcolumn_opt ::=", + /* 309 */ "kwcolumn_opt ::= COLUMNKW", + /* 310 */ "cmd ::= create_vtab", + /* 311 */ "cmd ::= create_vtab LP vtabarglist RP", + /* 312 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", + /* 313 */ "vtabarglist ::= vtabarg", + /* 314 */ "vtabarglist ::= vtabarglist COMMA vtabarg", + /* 315 */ "vtabarg ::=", + /* 316 */ "vtabarg ::= vtabarg vtabargtoken", + /* 317 */ "vtabargtoken ::= ANY", + /* 318 */ "vtabargtoken ::= lp anylist RP", + /* 319 */ "lp ::= LP", + /* 320 */ "anylist ::=", + /* 321 */ "anylist ::= anylist LP anylist RP", + /* 322 */ "anylist ::= anylist ANY", + /* 323 */ "with ::=", + /* 324 */ "with ::= WITH wqlist", + /* 325 */ "with ::= WITH RECURSIVE wqlist", + /* 326 */ "wqlist ::= nm eidlist_opt AS LP select RP", + /* 327 */ "wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP", }; #endif /* NDEBUG */ @@ -124631,9 +127412,9 @@ static void yy_destructor( ** inside the C code. */ case 163: /* select */ - case 195: /* selectnowith */ - case 196: /* oneselect */ - case 207: /* values */ + case 196: /* selectnowith */ + case 197: /* oneselect */ + case 208: /* values */ { sqlite3SelectDelete(pParse->db, (yypminor->yy3)); } @@ -124644,38 +127425,38 @@ sqlite3SelectDelete(pParse->db, (yypminor->yy3)); sqlite3ExprDelete(pParse->db, (yypminor->yy346).pExpr); } break; - case 179: /* idxlist_opt */ - case 188: /* idxlist */ - case 200: /* selcollist */ - case 203: /* groupby_opt */ - case 205: /* orderby_opt */ - case 208: /* nexprlist */ - case 209: /* exprlist */ - case 210: /* sclp */ - case 220: /* sortlist */ + case 179: /* eidlist_opt */ + case 188: /* sortlist */ + case 189: /* eidlist */ + case 201: /* selcollist */ + case 204: /* groupby_opt */ + case 206: /* orderby_opt */ + case 209: /* nexprlist */ + case 210: /* exprlist */ + case 211: /* sclp */ case 221: /* setlist */ case 228: /* case_exprlist */ { sqlite3ExprListDelete(pParse->db, (yypminor->yy14)); } break; - case 194: /* fullname */ - case 201: /* from */ - case 212: /* seltablist */ - case 213: /* stl_prefix */ + case 195: /* fullname */ + case 202: /* from */ + case 213: /* seltablist */ + case 214: /* stl_prefix */ { sqlite3SrcListDelete(pParse->db, (yypminor->yy65)); } break; - case 197: /* with */ + case 198: /* with */ case 252: /* wqlist */ { sqlite3WithDelete(pParse->db, (yypminor->yy59)); } break; - case 202: /* where_opt */ - case 204: /* having_opt */ - case 216: /* on_opt */ + case 203: /* where_opt */ + case 205: /* having_opt */ + case 217: /* on_opt */ case 227: /* case_operand */ case 229: /* case_else */ case 238: /* when_clause */ @@ -124684,9 +127465,9 @@ sqlite3WithDelete(pParse->db, (yypminor->yy59)); sqlite3ExprDelete(pParse->db, (yypminor->yy132)); } break; - case 217: /* using_opt */ - case 219: /* idlist */ - case 223: /* inscollist_opt */ + case 218: /* using_opt */ + case 220: /* idlist */ + case 223: /* idlist_opt */ { sqlite3IdListDelete(pParse->db, (yypminor->yy408)); } @@ -124720,7 +127501,7 @@ static int yy_pop_parser_stack(yyParser *pParser){ /* There is no mechanism by which the parser stack can be popped below ** empty in SQLite. */ - if( NEVER(pParser->yyidx<0) ) return 0; + assert( pParser->yyidx>=0 ); #ifndef NDEBUG if( yyTraceFILE && pParser->yyidx>=0 ){ fprintf(yyTraceFILE,"%sPopping %s\n", @@ -124786,10 +127567,10 @@ static int yy_find_shift_action( int i; int stateno = pParser->yystack[pParser->yyidx].stateno; - if( stateno>YY_SHIFT_COUNT - || (i = yy_shift_ofst[stateno])==YY_SHIFT_USE_DFLT ){ - return yy_default[stateno]; - } + if( stateno>=YY_MIN_REDUCE ) return stateno; + assert( stateno <= YY_SHIFT_COUNT ); + i = yy_shift_ofst[stateno]; + if( i==YY_SHIFT_USE_DFLT ) return yy_default[stateno]; assert( iLookAhead!=YYNOCODE ); i += iLookAhead; if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ @@ -124892,7 +127673,29 @@ static void yyStackOverflow(yyParser *yypParser, YYMINORTYPE *yypMinor){ } /* -** Perform a shift action. +** Print tracing information for a SHIFT action +*/ +#ifndef NDEBUG +static void yyTraceShift(yyParser *yypParser, int yyNewState){ + if( yyTraceFILE ){ + int i; + if( yyNewStateyyidx; i++) + fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); + fprintf(yyTraceFILE,"\n"); + }else{ + fprintf(yyTraceFILE,"%sShift *\n",yyTracePrompt); + } + } +} +#else +# define yyTraceShift(X,Y) +#endif + +/* +** Perform a shift action. Return the number of errors. */ static void yy_shift( yyParser *yypParser, /* The parser to be shifted */ @@ -124925,16 +127728,7 @@ static void yy_shift( yytos->stateno = (YYACTIONTYPE)yyNewState; yytos->major = (YYCODETYPE)yyMajor; yytos->minor = *yypMinor; -#ifndef NDEBUG - if( yyTraceFILE && yypParser->yyidx>0 ){ - int i; - fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyNewState); - fprintf(yyTraceFILE,"%sStack:",yyTracePrompt); - for(i=1; i<=yypParser->yyidx; i++) - fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); - fprintf(yyTraceFILE,"\n"); - } -#endif + yyTraceShift(yypParser, yyNewState); } /* The following table contains information about every rule that @@ -125041,81 +127835,82 @@ static const struct { { 187, 5 }, { 187, 5 }, { 187, 10 }, - { 189, 0 }, - { 189, 1 }, + { 190, 0 }, + { 190, 1 }, { 176, 0 }, { 176, 3 }, - { 190, 0 }, - { 190, 2 }, - { 191, 1 }, - { 191, 1 }, - { 191, 1 }, + { 191, 0 }, + { 191, 2 }, + { 192, 1 }, + { 192, 1 }, + { 192, 1 }, { 149, 4 }, - { 193, 2 }, - { 193, 0 }, - { 149, 8 }, + { 194, 2 }, + { 194, 0 }, + { 149, 9 }, { 149, 4 }, { 149, 1 }, { 163, 2 }, - { 195, 1 }, - { 195, 3 }, - { 198, 1 }, - { 198, 2 }, - { 198, 1 }, - { 196, 9 }, { 196, 1 }, - { 207, 4 }, - { 207, 5 }, + { 196, 3 }, { 199, 1 }, + { 199, 2 }, { 199, 1 }, - { 199, 0 }, - { 210, 2 }, - { 210, 0 }, - { 200, 3 }, - { 200, 2 }, - { 200, 4 }, + { 197, 9 }, + { 197, 1 }, + { 208, 4 }, + { 208, 5 }, + { 200, 1 }, + { 200, 1 }, + { 200, 0 }, { 211, 2 }, - { 211, 1 }, { 211, 0 }, - { 201, 0 }, + { 201, 3 }, { 201, 2 }, - { 213, 2 }, - { 213, 0 }, - { 212, 7 }, - { 212, 7 }, - { 212, 7 }, + { 201, 4 }, + { 212, 2 }, + { 212, 1 }, + { 212, 0 }, + { 202, 0 }, + { 202, 2 }, + { 214, 2 }, + { 214, 0 }, + { 213, 7 }, + { 213, 9 }, + { 213, 7 }, + { 213, 7 }, { 159, 0 }, { 159, 2 }, - { 194, 2 }, - { 214, 1 }, - { 214, 2 }, - { 214, 3 }, - { 214, 4 }, - { 216, 2 }, - { 216, 0 }, - { 215, 0 }, - { 215, 3 }, + { 195, 2 }, + { 215, 1 }, { 215, 2 }, - { 217, 4 }, + { 215, 3 }, + { 215, 4 }, + { 217, 2 }, { 217, 0 }, - { 205, 0 }, - { 205, 3 }, - { 220, 4 }, - { 220, 2 }, + { 216, 0 }, + { 216, 3 }, + { 216, 2 }, + { 218, 4 }, + { 218, 0 }, + { 206, 0 }, + { 206, 3 }, + { 188, 4 }, + { 188, 2 }, { 177, 1 }, { 177, 1 }, { 177, 0 }, - { 203, 0 }, - { 203, 3 }, { 204, 0 }, - { 204, 2 }, - { 206, 0 }, - { 206, 2 }, - { 206, 4 }, - { 206, 4 }, + { 204, 3 }, + { 205, 0 }, + { 205, 2 }, + { 207, 0 }, + { 207, 2 }, + { 207, 4 }, + { 207, 4 }, { 149, 6 }, - { 202, 0 }, - { 202, 2 }, + { 203, 0 }, + { 203, 2 }, { 149, 8 }, { 221, 5 }, { 221, 3 }, @@ -125125,8 +127920,8 @@ static const struct { { 222, 1 }, { 223, 0 }, { 223, 3 }, - { 219, 3 }, - { 219, 1 }, + { 220, 3 }, + { 220, 1 }, { 175, 1 }, { 175, 3 }, { 174, 1 }, @@ -125179,17 +127974,17 @@ static const struct { { 229, 0 }, { 227, 1 }, { 227, 0 }, + { 210, 1 }, + { 210, 0 }, + { 209, 3 }, { 209, 1 }, - { 209, 0 }, - { 208, 3 }, - { 208, 1 }, { 149, 12 }, { 230, 1 }, { 230, 0 }, { 179, 0 }, { 179, 3 }, - { 188, 5 }, - { 188, 3 }, + { 189, 5 }, + { 189, 3 }, { 231, 0 }, { 231, 2 }, { 149, 4 }, @@ -125234,9 +128029,9 @@ static const struct { { 239, 1 }, { 175, 4 }, { 175, 6 }, - { 192, 1 }, - { 192, 1 }, - { 192, 1 }, + { 193, 1 }, + { 193, 1 }, + { 193, 1 }, { 149, 4 }, { 149, 6 }, { 149, 3 }, @@ -125266,9 +128061,9 @@ static const struct { { 251, 0 }, { 251, 4 }, { 251, 2 }, - { 197, 0 }, - { 197, 2 }, - { 197, 3 }, + { 198, 0 }, + { 198, 2 }, + { 198, 3 }, { 252, 6 }, { 252, 8 }, }; @@ -125293,8 +128088,9 @@ static void yy_reduce( #ifndef NDEBUG if( yyTraceFILE && yyruleno>=0 && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - fprintf(yyTraceFILE, "%sReduce [%s].\n", yyTracePrompt, - yyRuleName[yyruleno]); + yysize = yyRuleInfo[yyruleno].nrhs; + fprintf(yyTraceFILE, "%sReduce [%s] -> state %d.\n", yyTracePrompt, + yyRuleName[yyruleno], yymsp[-yysize].stateno); } #endif /* NDEBUG */ @@ -125391,8 +128187,9 @@ static void yy_reduce( case 85: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ yytestcase(yyruleno==85); case 97: /* defer_subclause_opt ::= */ yytestcase(yyruleno==97); case 108: /* ifexists ::= */ yytestcase(yyruleno==108); - case 218: /* between_op ::= BETWEEN */ yytestcase(yyruleno==218); - case 221: /* in_op ::= IN */ yytestcase(yyruleno==221); + case 219: /* between_op ::= BETWEEN */ yytestcase(yyruleno==219); + case 222: /* in_op ::= IN */ yytestcase(yyruleno==222); + case 247: /* collate ::= */ yytestcase(yyruleno==247); {yygotominor.yy328 = 0;} break; case 29: /* ifnotexists ::= IF NOT EXISTS */ @@ -125400,8 +128197,9 @@ static void yy_reduce( case 69: /* autoinc ::= AUTOINCR */ yytestcase(yyruleno==69); case 84: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ yytestcase(yyruleno==84); case 107: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==107); - case 219: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==219); - case 222: /* in_op ::= NOT IN */ yytestcase(yyruleno==222); + case 220: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==220); + case 223: /* in_op ::= NOT IN */ yytestcase(yyruleno==223); + case 248: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==248); {yygotominor.yy328 = 1;} break; case 32: /* create_table_args ::= LP columnlist conslist_opt RP table_options */ @@ -125421,7 +128219,7 @@ static void yy_reduce( case 35: /* table_options ::= WITHOUT nm */ { if( yymsp[0].minor.yy0.n==5 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"rowid",5)==0 ){ - yygotominor.yy186 = TF_WithoutRowid; + yygotominor.yy186 = TF_WithoutRowid | TF_NoVisibleRowid; }else{ yygotominor.yy186 = 0; sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); @@ -125448,18 +128246,17 @@ static void yy_reduce( case 48: /* typename ::= ID|STRING */ yytestcase(yyruleno==48); case 130: /* as ::= AS nm */ yytestcase(yyruleno==130); case 131: /* as ::= ID|STRING */ yytestcase(yyruleno==131); - case 141: /* dbnm ::= DOT nm */ yytestcase(yyruleno==141); - case 150: /* indexed_opt ::= INDEXED BY nm */ yytestcase(yyruleno==150); - case 247: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==247); - case 256: /* nmnum ::= plus_num */ yytestcase(yyruleno==256); - case 257: /* nmnum ::= nm */ yytestcase(yyruleno==257); - case 258: /* nmnum ::= ON */ yytestcase(yyruleno==258); - case 259: /* nmnum ::= DELETE */ yytestcase(yyruleno==259); - case 260: /* nmnum ::= DEFAULT */ yytestcase(yyruleno==260); - case 261: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==261); - case 262: /* plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==262); - case 263: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==263); - case 279: /* trnm ::= nm */ yytestcase(yyruleno==279); + case 142: /* dbnm ::= DOT nm */ yytestcase(yyruleno==142); + case 151: /* indexed_opt ::= INDEXED BY nm */ yytestcase(yyruleno==151); + case 257: /* nmnum ::= plus_num */ yytestcase(yyruleno==257); + case 258: /* nmnum ::= nm */ yytestcase(yyruleno==258); + case 259: /* nmnum ::= ON */ yytestcase(yyruleno==259); + case 260: /* nmnum ::= DELETE */ yytestcase(yyruleno==260); + case 261: /* nmnum ::= DEFAULT */ yytestcase(yyruleno==261); + case 262: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==262); + case 263: /* plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==263); + case 264: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==264); + case 280: /* trnm ::= nm */ yytestcase(yyruleno==280); {yygotominor.yy0 = yymsp[0].minor.yy0;} break; case 44: /* type ::= typetoken */ @@ -125519,7 +128316,7 @@ static void yy_reduce( case 64: /* ccons ::= CHECK LP expr RP */ {sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy346.pExpr);} break; - case 65: /* ccons ::= REFERENCES nm idxlist_opt refargs */ + case 65: /* ccons ::= REFERENCES nm eidlist_opt refargs */ {sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy14,yymsp[0].minor.yy328);} break; case 66: /* ccons ::= defer_subclause */ @@ -125574,16 +128371,16 @@ static void yy_reduce( case 90: /* tconscomma ::= COMMA */ {pParse->constraintName.n = 0;} break; - case 93: /* tcons ::= PRIMARY KEY LP idxlist autoinc RP onconf */ + case 93: /* tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ {sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy14,yymsp[0].minor.yy328,yymsp[-2].minor.yy328,0);} break; - case 94: /* tcons ::= UNIQUE LP idxlist RP onconf */ + case 94: /* tcons ::= UNIQUE LP sortlist RP onconf */ {sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy14,yymsp[0].minor.yy328,0,0,0,0);} break; case 95: /* tcons ::= CHECK LP expr RP onconf */ {sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy346.pExpr);} break; - case 96: /* tcons ::= FOREIGN KEY LP idxlist RP REFERENCES nm idxlist_opt refargs defer_subclause_opt */ + case 96: /* tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ { sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy14, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy14, yymsp[-1].minor.yy328); sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy328); @@ -125609,9 +128406,9 @@ static void yy_reduce( sqlite3DropTable(pParse, yymsp[0].minor.yy65, 0, yymsp[-1].minor.yy328); } break; - case 109: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm AS select */ + case 109: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ { - sqlite3CreateView(pParse, &yymsp[-7].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, yymsp[0].minor.yy3, yymsp[-6].minor.yy328, yymsp[-4].minor.yy328); + sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy14, yymsp[0].minor.yy3, yymsp[-7].minor.yy328, yymsp[-5].minor.yy328); } break; case 110: /* cmd ::= DROP VIEW ifexists fullname */ @@ -125645,6 +128442,7 @@ static void yy_reduce( case 114: /* selectnowith ::= selectnowith multiselect_op oneselect */ { Select *pRhs = yymsp[0].minor.yy3; + Select *pLhs = yymsp[-2].minor.yy3; if( pRhs && pRhs->pPrior ){ SrcList *pFrom; Token x; @@ -125655,11 +128453,12 @@ static void yy_reduce( } if( pRhs ){ pRhs->op = (u8)yymsp[-1].minor.yy328; - pRhs->pPrior = yymsp[-2].minor.yy3; + pRhs->pPrior = pLhs; + if( ALWAYS(pLhs) ) pLhs->selFlags &= ~SF_MultiValue; pRhs->selFlags &= ~SF_MultiValue; if( yymsp[-1].minor.yy328!=TK_ALL ) pParse->hasCompound = 1; }else{ - sqlite3SelectDelete(pParse->db, yymsp[-2].minor.yy3); + sqlite3SelectDelete(pParse->db, pLhs); } yygotominor.yy3 = pRhs; } @@ -125720,18 +128519,20 @@ static void yy_reduce( {yygotominor.yy381 = SF_Distinct;} break; case 123: /* distinct ::= ALL */ - case 124: /* distinct ::= */ yytestcase(yyruleno==124); +{yygotominor.yy381 = SF_All;} + break; + case 124: /* distinct ::= */ {yygotominor.yy381 = 0;} break; case 125: /* sclp ::= selcollist COMMA */ - case 243: /* idxlist_opt ::= LP idxlist RP */ yytestcase(yyruleno==243); + case 244: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==244); {yygotominor.yy14 = yymsp[-1].minor.yy14;} break; case 126: /* sclp ::= */ - case 154: /* orderby_opt ::= */ yytestcase(yyruleno==154); - case 161: /* groupby_opt ::= */ yytestcase(yyruleno==161); - case 236: /* exprlist ::= */ yytestcase(yyruleno==236); - case 242: /* idxlist_opt ::= */ yytestcase(yyruleno==242); + case 155: /* orderby_opt ::= */ yytestcase(yyruleno==155); + case 162: /* groupby_opt ::= */ yytestcase(yyruleno==162); + case 237: /* exprlist ::= */ yytestcase(yyruleno==237); + case 243: /* eidlist_opt ::= */ yytestcase(yyruleno==243); {yygotominor.yy14 = 0;} break; case 127: /* selcollist ::= sclp expr as */ @@ -125770,7 +128571,7 @@ static void yy_reduce( case 135: /* stl_prefix ::= seltablist joinop */ { yygotominor.yy65 = yymsp[-1].minor.yy65; - if( ALWAYS(yygotominor.yy65 && yygotominor.yy65->nSrc>0) ) yygotominor.yy65->a[yygotominor.yy65->nSrc-1].jointype = (u8)yymsp[0].minor.yy328; + if( ALWAYS(yygotominor.yy65 && yygotominor.yy65->nSrc>0) ) yygotominor.yy65->a[yygotominor.yy65->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy328; } break; case 136: /* stl_prefix ::= */ @@ -125782,12 +128583,18 @@ static void yy_reduce( sqlite3SrcListIndexedBy(pParse, yygotominor.yy65, &yymsp[-2].minor.yy0); } break; - case 138: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */ + case 138: /* seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */ +{ + yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-8].minor.yy65,&yymsp[-7].minor.yy0,&yymsp[-6].minor.yy0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy132,yymsp[0].minor.yy408); + sqlite3SrcListFuncArgs(pParse, yygotominor.yy65, yymsp[-4].minor.yy14); +} + break; + case 139: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */ { yygotominor.yy65 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy65,0,0,&yymsp[-2].minor.yy0,yymsp[-4].minor.yy3,yymsp[-1].minor.yy132,yymsp[0].minor.yy408); } break; - case 139: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ + case 140: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ { if( yymsp[-6].minor.yy65==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy132==0 && yymsp[0].minor.yy408==0 ){ yygotominor.yy65 = yymsp[-4].minor.yy65; @@ -125811,94 +128618,96 @@ static void yy_reduce( } } break; - case 140: /* dbnm ::= */ - case 149: /* indexed_opt ::= */ yytestcase(yyruleno==149); + case 141: /* dbnm ::= */ + case 150: /* indexed_opt ::= */ yytestcase(yyruleno==150); {yygotominor.yy0.z=0; yygotominor.yy0.n=0;} break; - case 142: /* fullname ::= nm dbnm */ + case 143: /* fullname ::= nm dbnm */ {yygotominor.yy65 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0);} break; - case 143: /* joinop ::= COMMA|JOIN */ + case 144: /* joinop ::= COMMA|JOIN */ { yygotominor.yy328 = JT_INNER; } break; - case 144: /* joinop ::= JOIN_KW JOIN */ + case 145: /* joinop ::= JOIN_KW JOIN */ { yygotominor.yy328 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); } break; - case 145: /* joinop ::= JOIN_KW nm JOIN */ + case 146: /* joinop ::= JOIN_KW nm JOIN */ { yygotominor.yy328 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); } break; - case 146: /* joinop ::= JOIN_KW nm nm JOIN */ + case 147: /* joinop ::= JOIN_KW nm nm JOIN */ { yygotominor.yy328 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); } break; - case 147: /* on_opt ::= ON expr */ - case 164: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==164); - case 171: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==171); - case 231: /* case_else ::= ELSE expr */ yytestcase(yyruleno==231); - case 233: /* case_operand ::= expr */ yytestcase(yyruleno==233); + case 148: /* on_opt ::= ON expr */ + case 165: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==165); + case 172: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==172); + case 232: /* case_else ::= ELSE expr */ yytestcase(yyruleno==232); + case 234: /* case_operand ::= expr */ yytestcase(yyruleno==234); {yygotominor.yy132 = yymsp[0].minor.yy346.pExpr;} break; - case 148: /* on_opt ::= */ - case 163: /* having_opt ::= */ yytestcase(yyruleno==163); - case 170: /* where_opt ::= */ yytestcase(yyruleno==170); - case 232: /* case_else ::= */ yytestcase(yyruleno==232); - case 234: /* case_operand ::= */ yytestcase(yyruleno==234); + case 149: /* on_opt ::= */ + case 164: /* having_opt ::= */ yytestcase(yyruleno==164); + case 171: /* where_opt ::= */ yytestcase(yyruleno==171); + case 233: /* case_else ::= */ yytestcase(yyruleno==233); + case 235: /* case_operand ::= */ yytestcase(yyruleno==235); {yygotominor.yy132 = 0;} break; - case 151: /* indexed_opt ::= NOT INDEXED */ + case 152: /* indexed_opt ::= NOT INDEXED */ {yygotominor.yy0.z=0; yygotominor.yy0.n=1;} break; - case 152: /* using_opt ::= USING LP idlist RP */ - case 180: /* inscollist_opt ::= LP idlist RP */ yytestcase(yyruleno==180); + case 153: /* using_opt ::= USING LP idlist RP */ + case 181: /* idlist_opt ::= LP idlist RP */ yytestcase(yyruleno==181); {yygotominor.yy408 = yymsp[-1].minor.yy408;} break; - case 153: /* using_opt ::= */ - case 179: /* inscollist_opt ::= */ yytestcase(yyruleno==179); + case 154: /* using_opt ::= */ + case 180: /* idlist_opt ::= */ yytestcase(yyruleno==180); {yygotominor.yy408 = 0;} break; - case 155: /* orderby_opt ::= ORDER BY sortlist */ - case 162: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==162); - case 235: /* exprlist ::= nexprlist */ yytestcase(yyruleno==235); + case 156: /* orderby_opt ::= ORDER BY sortlist */ + case 163: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==163); + case 236: /* exprlist ::= nexprlist */ yytestcase(yyruleno==236); {yygotominor.yy14 = yymsp[0].minor.yy14;} break; - case 156: /* sortlist ::= sortlist COMMA expr sortorder */ + case 157: /* sortlist ::= sortlist COMMA expr sortorder */ { yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy14,yymsp[-1].minor.yy346.pExpr); - if( yygotominor.yy14 ) yygotominor.yy14->a[yygotominor.yy14->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy328; + sqlite3ExprListSetSortOrder(yygotominor.yy14,yymsp[0].minor.yy328); } break; - case 157: /* sortlist ::= expr sortorder */ + case 158: /* sortlist ::= expr sortorder */ { yygotominor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[-1].minor.yy346.pExpr); - if( yygotominor.yy14 && ALWAYS(yygotominor.yy14->a) ) yygotominor.yy14->a[0].sortOrder = (u8)yymsp[0].minor.yy328; + sqlite3ExprListSetSortOrder(yygotominor.yy14,yymsp[0].minor.yy328); } break; - case 158: /* sortorder ::= ASC */ - case 160: /* sortorder ::= */ yytestcase(yyruleno==160); + case 159: /* sortorder ::= ASC */ {yygotominor.yy328 = SQLITE_SO_ASC;} break; - case 159: /* sortorder ::= DESC */ + case 160: /* sortorder ::= DESC */ {yygotominor.yy328 = SQLITE_SO_DESC;} break; - case 165: /* limit_opt ::= */ + case 161: /* sortorder ::= */ +{yygotominor.yy328 = SQLITE_SO_UNDEFINED;} + break; + case 166: /* limit_opt ::= */ {yygotominor.yy476.pLimit = 0; yygotominor.yy476.pOffset = 0;} break; - case 166: /* limit_opt ::= LIMIT expr */ + case 167: /* limit_opt ::= LIMIT expr */ {yygotominor.yy476.pLimit = yymsp[0].minor.yy346.pExpr; yygotominor.yy476.pOffset = 0;} break; - case 167: /* limit_opt ::= LIMIT expr OFFSET expr */ + case 168: /* limit_opt ::= LIMIT expr OFFSET expr */ {yygotominor.yy476.pLimit = yymsp[-2].minor.yy346.pExpr; yygotominor.yy476.pOffset = yymsp[0].minor.yy346.pExpr;} break; - case 168: /* limit_opt ::= LIMIT expr COMMA expr */ + case 169: /* limit_opt ::= LIMIT expr COMMA expr */ {yygotominor.yy476.pOffset = yymsp[-2].minor.yy346.pExpr; yygotominor.yy476.pLimit = yymsp[0].minor.yy346.pExpr;} break; - case 169: /* cmd ::= with DELETE FROM fullname indexed_opt where_opt */ + case 170: /* cmd ::= with DELETE FROM fullname indexed_opt where_opt */ { sqlite3WithPush(pParse, yymsp[-5].minor.yy59, 1); sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy65, &yymsp[-1].minor.yy0); sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy65,yymsp[0].minor.yy132); } break; - case 172: /* cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt */ + case 173: /* cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt */ { sqlite3WithPush(pParse, yymsp[-7].minor.yy59, 1); sqlite3SrcListIndexedBy(pParse, yymsp[-4].minor.yy65, &yymsp[-3].minor.yy0); @@ -125906,58 +128715,58 @@ static void yy_reduce( sqlite3Update(pParse,yymsp[-4].minor.yy65,yymsp[-1].minor.yy14,yymsp[0].minor.yy132,yymsp[-5].minor.yy186); } break; - case 173: /* setlist ::= setlist COMMA nm EQ expr */ + case 174: /* setlist ::= setlist COMMA nm EQ expr */ { yygotominor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy14, yymsp[0].minor.yy346.pExpr); sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[-2].minor.yy0, 1); } break; - case 174: /* setlist ::= nm EQ expr */ + case 175: /* setlist ::= nm EQ expr */ { yygotominor.yy14 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy346.pExpr); sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[-2].minor.yy0, 1); } break; - case 175: /* cmd ::= with insert_cmd INTO fullname inscollist_opt select */ + case 176: /* cmd ::= with insert_cmd INTO fullname idlist_opt select */ { sqlite3WithPush(pParse, yymsp[-5].minor.yy59, 1); sqlite3Insert(pParse, yymsp[-2].minor.yy65, yymsp[0].minor.yy3, yymsp[-1].minor.yy408, yymsp[-4].minor.yy186); } break; - case 176: /* cmd ::= with insert_cmd INTO fullname inscollist_opt DEFAULT VALUES */ + case 177: /* cmd ::= with insert_cmd INTO fullname idlist_opt DEFAULT VALUES */ { sqlite3WithPush(pParse, yymsp[-6].minor.yy59, 1); sqlite3Insert(pParse, yymsp[-3].minor.yy65, 0, yymsp[-2].minor.yy408, yymsp[-5].minor.yy186); } break; - case 177: /* insert_cmd ::= INSERT orconf */ + case 178: /* insert_cmd ::= INSERT orconf */ {yygotominor.yy186 = yymsp[0].minor.yy186;} break; - case 178: /* insert_cmd ::= REPLACE */ + case 179: /* insert_cmd ::= REPLACE */ {yygotominor.yy186 = OE_Replace;} break; - case 181: /* idlist ::= idlist COMMA nm */ + case 182: /* idlist ::= idlist COMMA nm */ {yygotominor.yy408 = sqlite3IdListAppend(pParse->db,yymsp[-2].minor.yy408,&yymsp[0].minor.yy0);} break; - case 182: /* idlist ::= nm */ + case 183: /* idlist ::= nm */ {yygotominor.yy408 = sqlite3IdListAppend(pParse->db,0,&yymsp[0].minor.yy0);} break; - case 183: /* expr ::= term */ + case 184: /* expr ::= term */ {yygotominor.yy346 = yymsp[0].minor.yy346;} break; - case 184: /* expr ::= LP expr RP */ + case 185: /* expr ::= LP expr RP */ {yygotominor.yy346.pExpr = yymsp[-1].minor.yy346.pExpr; spanSet(&yygotominor.yy346,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0);} break; - case 185: /* term ::= NULL */ - case 190: /* term ::= INTEGER|FLOAT|BLOB */ yytestcase(yyruleno==190); - case 191: /* term ::= STRING */ yytestcase(yyruleno==191); + case 186: /* term ::= NULL */ + case 191: /* term ::= INTEGER|FLOAT|BLOB */ yytestcase(yyruleno==191); + case 192: /* term ::= STRING */ yytestcase(yyruleno==192); {spanExpr(&yygotominor.yy346, pParse, yymsp[0].major, &yymsp[0].minor.yy0);} break; - case 186: /* expr ::= ID|INDEXED */ - case 187: /* expr ::= JOIN_KW */ yytestcase(yyruleno==187); + case 187: /* expr ::= ID|INDEXED */ + case 188: /* expr ::= JOIN_KW */ yytestcase(yyruleno==188); {spanExpr(&yygotominor.yy346, pParse, TK_ID, &yymsp[0].minor.yy0);} break; - case 188: /* expr ::= nm DOT nm */ + case 189: /* expr ::= nm DOT nm */ { Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[0].minor.yy0); @@ -125965,7 +128774,7 @@ static void yy_reduce( spanSet(&yygotominor.yy346,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); } break; - case 189: /* expr ::= nm DOT nm DOT nm */ + case 190: /* expr ::= nm DOT nm DOT nm */ { Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-4].minor.yy0); Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); @@ -125975,7 +128784,7 @@ static void yy_reduce( spanSet(&yygotominor.yy346,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); } break; - case 192: /* expr ::= VARIABLE */ + case 193: /* expr ::= VARIABLE */ { if( yymsp[0].minor.yy0.n>=2 && yymsp[0].minor.yy0.z[0]=='#' && sqlite3Isdigit(yymsp[0].minor.yy0.z[1]) ){ /* When doing a nested parse, one can include terms in an expression @@ -125995,60 +128804,60 @@ static void yy_reduce( spanSet(&yygotominor.yy346, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); } break; - case 193: /* expr ::= expr COLLATE ID|STRING */ + case 194: /* expr ::= expr COLLATE ID|STRING */ { yygotominor.yy346.pExpr = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy346.pExpr, &yymsp[0].minor.yy0, 1); yygotominor.yy346.zStart = yymsp[-2].minor.yy346.zStart; yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 194: /* expr ::= CAST LP expr AS typetoken RP */ + case 195: /* expr ::= CAST LP expr AS typetoken RP */ { yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_CAST, yymsp[-3].minor.yy346.pExpr, 0, &yymsp[-1].minor.yy0); spanSet(&yygotominor.yy346,&yymsp[-5].minor.yy0,&yymsp[0].minor.yy0); } break; - case 195: /* expr ::= ID|INDEXED LP distinct exprlist RP */ + case 196: /* expr ::= ID|INDEXED LP distinct exprlist RP */ { if( yymsp[-1].minor.yy14 && yymsp[-1].minor.yy14->nExpr>pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){ sqlite3ErrorMsg(pParse, "too many arguments on function %T", &yymsp[-4].minor.yy0); } yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy14, &yymsp[-4].minor.yy0); spanSet(&yygotominor.yy346,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); - if( yymsp[-2].minor.yy381 && yygotominor.yy346.pExpr ){ + if( yymsp[-2].minor.yy381==SF_Distinct && yygotominor.yy346.pExpr ){ yygotominor.yy346.pExpr->flags |= EP_Distinct; } } break; - case 196: /* expr ::= ID|INDEXED LP STAR RP */ + case 197: /* expr ::= ID|INDEXED LP STAR RP */ { yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0); spanSet(&yygotominor.yy346,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); } break; - case 197: /* term ::= CTIME_KW */ + case 198: /* term ::= CTIME_KW */ { yygotominor.yy346.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0); spanSet(&yygotominor.yy346, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); } break; - case 198: /* expr ::= expr AND expr */ - case 199: /* expr ::= expr OR expr */ yytestcase(yyruleno==199); - case 200: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==200); - case 201: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==201); - case 202: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==202); - case 203: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==203); - case 204: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==204); - case 205: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==205); + case 199: /* expr ::= expr AND expr */ + case 200: /* expr ::= expr OR expr */ yytestcase(yyruleno==200); + case 201: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==201); + case 202: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==202); + case 203: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==203); + case 204: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==204); + case 205: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==205); + case 206: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==206); {spanBinaryExpr(&yygotominor.yy346,pParse,yymsp[-1].major,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy346);} break; - case 206: /* likeop ::= LIKE_KW|MATCH */ + case 207: /* likeop ::= LIKE_KW|MATCH */ {yygotominor.yy96.eOperator = yymsp[0].minor.yy0; yygotominor.yy96.bNot = 0;} break; - case 207: /* likeop ::= NOT LIKE_KW|MATCH */ + case 208: /* likeop ::= NOT LIKE_KW|MATCH */ {yygotominor.yy96.eOperator = yymsp[0].minor.yy0; yygotominor.yy96.bNot = 1;} break; - case 208: /* expr ::= expr likeop expr */ + case 209: /* expr ::= expr likeop expr */ { ExprList *pList; pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy346.pExpr); @@ -126060,7 +128869,7 @@ static void yy_reduce( if( yygotominor.yy346.pExpr ) yygotominor.yy346.pExpr->flags |= EP_InfixFunc; } break; - case 209: /* expr ::= expr likeop expr ESCAPE expr */ + case 210: /* expr ::= expr likeop expr ESCAPE expr */ { ExprList *pList; pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy346.pExpr); @@ -126073,35 +128882,35 @@ static void yy_reduce( if( yygotominor.yy346.pExpr ) yygotominor.yy346.pExpr->flags |= EP_InfixFunc; } break; - case 210: /* expr ::= expr ISNULL|NOTNULL */ + case 211: /* expr ::= expr ISNULL|NOTNULL */ {spanUnaryPostfix(&yygotominor.yy346,pParse,yymsp[0].major,&yymsp[-1].minor.yy346,&yymsp[0].minor.yy0);} break; - case 211: /* expr ::= expr NOT NULL */ + case 212: /* expr ::= expr NOT NULL */ {spanUnaryPostfix(&yygotominor.yy346,pParse,TK_NOTNULL,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy0);} break; - case 212: /* expr ::= expr IS expr */ + case 213: /* expr ::= expr IS expr */ { spanBinaryExpr(&yygotominor.yy346,pParse,TK_IS,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy346); binaryToUnaryIfNull(pParse, yymsp[0].minor.yy346.pExpr, yygotominor.yy346.pExpr, TK_ISNULL); } break; - case 213: /* expr ::= expr IS NOT expr */ + case 214: /* expr ::= expr IS NOT expr */ { spanBinaryExpr(&yygotominor.yy346,pParse,TK_ISNOT,&yymsp[-3].minor.yy346,&yymsp[0].minor.yy346); binaryToUnaryIfNull(pParse, yymsp[0].minor.yy346.pExpr, yygotominor.yy346.pExpr, TK_NOTNULL); } break; - case 214: /* expr ::= NOT expr */ - case 215: /* expr ::= BITNOT expr */ yytestcase(yyruleno==215); + case 215: /* expr ::= NOT expr */ + case 216: /* expr ::= BITNOT expr */ yytestcase(yyruleno==216); {spanUnaryPrefix(&yygotominor.yy346,pParse,yymsp[-1].major,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);} break; - case 216: /* expr ::= MINUS expr */ + case 217: /* expr ::= MINUS expr */ {spanUnaryPrefix(&yygotominor.yy346,pParse,TK_UMINUS,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);} break; - case 217: /* expr ::= PLUS expr */ + case 218: /* expr ::= PLUS expr */ {spanUnaryPrefix(&yygotominor.yy346,pParse,TK_UPLUS,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);} break; - case 220: /* expr ::= expr between_op expr AND expr */ + case 221: /* expr ::= expr between_op expr AND expr */ { ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy346.pExpr); pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy346.pExpr); @@ -126116,7 +128925,7 @@ static void yy_reduce( yygotominor.yy346.zEnd = yymsp[0].minor.yy346.zEnd; } break; - case 223: /* expr ::= expr in_op LP exprlist RP */ + case 224: /* expr ::= expr in_op LP exprlist RP */ { if( yymsp[-1].minor.yy14==0 ){ /* Expressions of the form @@ -126170,7 +128979,7 @@ static void yy_reduce( yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 224: /* expr ::= LP select RP */ + case 225: /* expr ::= LP select RP */ { yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_SELECT, 0, 0, 0); if( yygotominor.yy346.pExpr ){ @@ -126184,7 +128993,7 @@ static void yy_reduce( yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 225: /* expr ::= expr in_op LP select RP */ + case 226: /* expr ::= expr in_op LP select RP */ { yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy346.pExpr, 0, 0); if( yygotominor.yy346.pExpr ){ @@ -126199,7 +129008,7 @@ static void yy_reduce( yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 226: /* expr ::= expr in_op nm dbnm */ + case 227: /* expr ::= expr in_op nm dbnm */ { SrcList *pSrc = sqlite3SrcListAppend(pParse->db, 0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0); yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-3].minor.yy346.pExpr, 0, 0); @@ -126215,7 +129024,7 @@ static void yy_reduce( yygotominor.yy346.zEnd = yymsp[0].minor.yy0.z ? &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] : &yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]; } break; - case 227: /* expr ::= EXISTS LP select RP */ + case 228: /* expr ::= EXISTS LP select RP */ { Expr *p = yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_EXISTS, 0, 0, 0); if( p ){ @@ -126229,7 +129038,7 @@ static void yy_reduce( yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 228: /* expr ::= CASE case_operand case_exprlist case_else END */ + case 229: /* expr ::= CASE case_operand case_exprlist case_else END */ { yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy132, 0, 0); if( yygotominor.yy346.pExpr ){ @@ -126243,82 +129052,71 @@ static void yy_reduce( yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 229: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ + case 230: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ { yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, yymsp[-2].minor.yy346.pExpr); yygotominor.yy14 = sqlite3ExprListAppend(pParse,yygotominor.yy14, yymsp[0].minor.yy346.pExpr); } break; - case 230: /* case_exprlist ::= WHEN expr THEN expr */ + case 231: /* case_exprlist ::= WHEN expr THEN expr */ { yygotominor.yy14 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy346.pExpr); yygotominor.yy14 = sqlite3ExprListAppend(pParse,yygotominor.yy14, yymsp[0].minor.yy346.pExpr); } break; - case 237: /* nexprlist ::= nexprlist COMMA expr */ + case 238: /* nexprlist ::= nexprlist COMMA expr */ {yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[0].minor.yy346.pExpr);} break; - case 238: /* nexprlist ::= expr */ + case 239: /* nexprlist ::= expr */ {yygotominor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy346.pExpr);} break; - case 239: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP idxlist RP where_opt */ + case 240: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ { sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, sqlite3SrcListAppend(pParse->db,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy14, yymsp[-10].minor.yy328, &yymsp[-11].minor.yy0, yymsp[0].minor.yy132, SQLITE_SO_ASC, yymsp[-8].minor.yy328); } break; - case 240: /* uniqueflag ::= UNIQUE */ - case 291: /* raisetype ::= ABORT */ yytestcase(yyruleno==291); + case 241: /* uniqueflag ::= UNIQUE */ + case 292: /* raisetype ::= ABORT */ yytestcase(yyruleno==292); {yygotominor.yy328 = OE_Abort;} break; - case 241: /* uniqueflag ::= */ + case 242: /* uniqueflag ::= */ {yygotominor.yy328 = OE_None;} break; - case 244: /* idxlist ::= idxlist COMMA nm collate sortorder */ + case 245: /* eidlist ::= eidlist COMMA nm collate sortorder */ { - Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &yymsp[-1].minor.yy0, 1); - yygotominor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, p); - sqlite3ExprListSetName(pParse,yygotominor.yy14,&yymsp[-2].minor.yy0,1); - sqlite3ExprListCheckLength(pParse, yygotominor.yy14, "index"); - if( yygotominor.yy14 ) yygotominor.yy14->a[yygotominor.yy14->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy328; + yygotominor.yy14 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy14, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy328, yymsp[0].minor.yy328); } break; - case 245: /* idxlist ::= nm collate sortorder */ + case 246: /* eidlist ::= nm collate sortorder */ { - Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &yymsp[-1].minor.yy0, 1); - yygotominor.yy14 = sqlite3ExprListAppend(pParse,0, p); - sqlite3ExprListSetName(pParse, yygotominor.yy14, &yymsp[-2].minor.yy0, 1); - sqlite3ExprListCheckLength(pParse, yygotominor.yy14, "index"); - if( yygotominor.yy14 ) yygotominor.yy14->a[yygotominor.yy14->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy328; + yygotominor.yy14 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy328, yymsp[0].minor.yy328); } break; - case 246: /* collate ::= */ -{yygotominor.yy0.z = 0; yygotominor.yy0.n = 0;} - break; - case 248: /* cmd ::= DROP INDEX ifexists fullname */ + case 249: /* cmd ::= DROP INDEX ifexists fullname */ {sqlite3DropIndex(pParse, yymsp[0].minor.yy65, yymsp[-1].minor.yy328);} break; - case 249: /* cmd ::= VACUUM */ - case 250: /* cmd ::= VACUUM nm */ yytestcase(yyruleno==250); + case 250: /* cmd ::= VACUUM */ + case 251: /* cmd ::= VACUUM nm */ yytestcase(yyruleno==251); {sqlite3Vacuum(pParse);} break; - case 251: /* cmd ::= PRAGMA nm dbnm */ + case 252: /* cmd ::= PRAGMA nm dbnm */ {sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);} break; - case 252: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ + case 253: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);} break; - case 253: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ + case 254: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);} break; - case 254: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ + case 255: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1);} break; - case 255: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ + case 256: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,1);} break; - case 264: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + case 265: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ { Token all; all.z = yymsp[-3].minor.yy0.z; @@ -126326,38 +129124,38 @@ static void yy_reduce( sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy473, &all); } break; - case 265: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + case 266: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ { sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy328, yymsp[-4].minor.yy378.a, yymsp[-4].minor.yy378.b, yymsp[-2].minor.yy65, yymsp[0].minor.yy132, yymsp[-10].minor.yy328, yymsp[-8].minor.yy328); yygotominor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); } break; - case 266: /* trigger_time ::= BEFORE */ - case 269: /* trigger_time ::= */ yytestcase(yyruleno==269); + case 267: /* trigger_time ::= BEFORE */ + case 270: /* trigger_time ::= */ yytestcase(yyruleno==270); { yygotominor.yy328 = TK_BEFORE; } break; - case 267: /* trigger_time ::= AFTER */ + case 268: /* trigger_time ::= AFTER */ { yygotominor.yy328 = TK_AFTER; } break; - case 268: /* trigger_time ::= INSTEAD OF */ + case 269: /* trigger_time ::= INSTEAD OF */ { yygotominor.yy328 = TK_INSTEAD;} break; - case 270: /* trigger_event ::= DELETE|INSERT */ - case 271: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==271); + case 271: /* trigger_event ::= DELETE|INSERT */ + case 272: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==272); {yygotominor.yy378.a = yymsp[0].major; yygotominor.yy378.b = 0;} break; - case 272: /* trigger_event ::= UPDATE OF idlist */ + case 273: /* trigger_event ::= UPDATE OF idlist */ {yygotominor.yy378.a = TK_UPDATE; yygotominor.yy378.b = yymsp[0].minor.yy408;} break; - case 275: /* when_clause ::= */ - case 296: /* key_opt ::= */ yytestcase(yyruleno==296); + case 276: /* when_clause ::= */ + case 297: /* key_opt ::= */ yytestcase(yyruleno==297); { yygotominor.yy132 = 0; } break; - case 276: /* when_clause ::= WHEN expr */ - case 297: /* key_opt ::= KEY expr */ yytestcase(yyruleno==297); + case 277: /* when_clause ::= WHEN expr */ + case 298: /* key_opt ::= KEY expr */ yytestcase(yyruleno==298); { yygotominor.yy132 = yymsp[0].minor.yy346.pExpr; } break; - case 277: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + case 278: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ { assert( yymsp[-2].minor.yy473!=0 ); yymsp[-2].minor.yy473->pLast->pNext = yymsp[-1].minor.yy473; @@ -126365,14 +129163,14 @@ static void yy_reduce( yygotominor.yy473 = yymsp[-2].minor.yy473; } break; - case 278: /* trigger_cmd_list ::= trigger_cmd SEMI */ + case 279: /* trigger_cmd_list ::= trigger_cmd SEMI */ { assert( yymsp[-1].minor.yy473!=0 ); yymsp[-1].minor.yy473->pLast = yymsp[-1].minor.yy473; yygotominor.yy473 = yymsp[-1].minor.yy473; } break; - case 280: /* trnm ::= nm DOT nm */ + case 281: /* trnm ::= nm DOT nm */ { yygotominor.yy0 = yymsp[0].minor.yy0; sqlite3ErrorMsg(pParse, @@ -126380,33 +129178,33 @@ static void yy_reduce( "statements within triggers"); } break; - case 282: /* tridxby ::= INDEXED BY nm */ + case 283: /* tridxby ::= INDEXED BY nm */ { sqlite3ErrorMsg(pParse, "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 283: /* tridxby ::= NOT INDEXED */ + case 284: /* tridxby ::= NOT INDEXED */ { sqlite3ErrorMsg(pParse, "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 284: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt */ + case 285: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt */ { yygotominor.yy473 = sqlite3TriggerUpdateStep(pParse->db, &yymsp[-4].minor.yy0, yymsp[-1].minor.yy14, yymsp[0].minor.yy132, yymsp[-5].minor.yy186); } break; - case 285: /* trigger_cmd ::= insert_cmd INTO trnm inscollist_opt select */ + case 286: /* trigger_cmd ::= insert_cmd INTO trnm idlist_opt select */ {yygotominor.yy473 = sqlite3TriggerInsertStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy408, yymsp[0].minor.yy3, yymsp[-4].minor.yy186);} break; - case 286: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt */ + case 287: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt */ {yygotominor.yy473 = sqlite3TriggerDeleteStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[0].minor.yy132);} break; - case 287: /* trigger_cmd ::= select */ + case 288: /* trigger_cmd ::= select */ {yygotominor.yy473 = sqlite3TriggerSelectStep(pParse->db, yymsp[0].minor.yy3); } break; - case 288: /* expr ::= RAISE LP IGNORE RP */ + case 289: /* expr ::= RAISE LP IGNORE RP */ { yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, 0); if( yygotominor.yy346.pExpr ){ @@ -126416,7 +129214,7 @@ static void yy_reduce( yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 289: /* expr ::= RAISE LP raisetype COMMA nm RP */ + case 290: /* expr ::= RAISE LP raisetype COMMA nm RP */ { yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, &yymsp[-1].minor.yy0); if( yygotominor.yy346.pExpr ) { @@ -126426,87 +129224,87 @@ static void yy_reduce( yygotominor.yy346.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 290: /* raisetype ::= ROLLBACK */ + case 291: /* raisetype ::= ROLLBACK */ {yygotominor.yy328 = OE_Rollback;} break; - case 292: /* raisetype ::= FAIL */ + case 293: /* raisetype ::= FAIL */ {yygotominor.yy328 = OE_Fail;} break; - case 293: /* cmd ::= DROP TRIGGER ifexists fullname */ + case 294: /* cmd ::= DROP TRIGGER ifexists fullname */ { sqlite3DropTrigger(pParse,yymsp[0].minor.yy65,yymsp[-1].minor.yy328); } break; - case 294: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + case 295: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ { sqlite3Attach(pParse, yymsp[-3].minor.yy346.pExpr, yymsp[-1].minor.yy346.pExpr, yymsp[0].minor.yy132); } break; - case 295: /* cmd ::= DETACH database_kw_opt expr */ + case 296: /* cmd ::= DETACH database_kw_opt expr */ { sqlite3Detach(pParse, yymsp[0].minor.yy346.pExpr); } break; - case 300: /* cmd ::= REINDEX */ + case 301: /* cmd ::= REINDEX */ {sqlite3Reindex(pParse, 0, 0);} break; - case 301: /* cmd ::= REINDEX nm dbnm */ + case 302: /* cmd ::= REINDEX nm dbnm */ {sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 302: /* cmd ::= ANALYZE */ + case 303: /* cmd ::= ANALYZE */ {sqlite3Analyze(pParse, 0, 0);} break; - case 303: /* cmd ::= ANALYZE nm dbnm */ + case 304: /* cmd ::= ANALYZE nm dbnm */ {sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 304: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ + case 305: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ { sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy65,&yymsp[0].minor.yy0); } break; - case 305: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column */ + case 306: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column */ { sqlite3AlterFinishAddColumn(pParse, &yymsp[0].minor.yy0); } break; - case 306: /* add_column_fullname ::= fullname */ + case 307: /* add_column_fullname ::= fullname */ { pParse->db->lookaside.bEnabled = 0; sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy65); } break; - case 309: /* cmd ::= create_vtab */ + case 310: /* cmd ::= create_vtab */ {sqlite3VtabFinishParse(pParse,0);} break; - case 310: /* cmd ::= create_vtab LP vtabarglist RP */ + case 311: /* cmd ::= create_vtab LP vtabarglist RP */ {sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} break; - case 311: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + case 312: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ { sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy328); } break; - case 314: /* vtabarg ::= */ + case 315: /* vtabarg ::= */ {sqlite3VtabArgInit(pParse);} break; - case 316: /* vtabargtoken ::= ANY */ - case 317: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==317); - case 318: /* lp ::= LP */ yytestcase(yyruleno==318); + case 317: /* vtabargtoken ::= ANY */ + case 318: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==318); + case 319: /* lp ::= LP */ yytestcase(yyruleno==319); {sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);} break; - case 322: /* with ::= */ + case 323: /* with ::= */ {yygotominor.yy59 = 0;} break; - case 323: /* with ::= WITH wqlist */ - case 324: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==324); + case 324: /* with ::= WITH wqlist */ + case 325: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==325); { yygotominor.yy59 = yymsp[0].minor.yy59; } break; - case 325: /* wqlist ::= nm idxlist_opt AS LP select RP */ + case 326: /* wqlist ::= nm eidlist_opt AS LP select RP */ { yygotominor.yy59 = sqlite3WithAdd(pParse, 0, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy14, yymsp[-1].minor.yy3); } break; - case 326: /* wqlist ::= wqlist COMMA nm idxlist_opt AS LP select RP */ + case 327: /* wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP */ { yygotominor.yy59 = sqlite3WithAdd(pParse, yymsp[-7].minor.yy59, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy14, yymsp[-1].minor.yy3); } @@ -126534,19 +129332,19 @@ static void yy_reduce( /* (88) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==88); /* (89) conslist ::= tcons */ yytestcase(yyruleno==89); /* (91) tconscomma ::= */ yytestcase(yyruleno==91); - /* (273) foreach_clause ::= */ yytestcase(yyruleno==273); - /* (274) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==274); - /* (281) tridxby ::= */ yytestcase(yyruleno==281); - /* (298) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==298); - /* (299) database_kw_opt ::= */ yytestcase(yyruleno==299); - /* (307) kwcolumn_opt ::= */ yytestcase(yyruleno==307); - /* (308) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==308); - /* (312) vtabarglist ::= vtabarg */ yytestcase(yyruleno==312); - /* (313) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==313); - /* (315) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==315); - /* (319) anylist ::= */ yytestcase(yyruleno==319); - /* (320) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==320); - /* (321) anylist ::= anylist ANY */ yytestcase(yyruleno==321); + /* (274) foreach_clause ::= */ yytestcase(yyruleno==274); + /* (275) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==275); + /* (282) tridxby ::= */ yytestcase(yyruleno==282); + /* (299) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==299); + /* (300) database_kw_opt ::= */ yytestcase(yyruleno==300); + /* (308) kwcolumn_opt ::= */ yytestcase(yyruleno==308); + /* (309) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==309); + /* (313) vtabarglist ::= vtabarg */ yytestcase(yyruleno==313); + /* (314) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==314); + /* (316) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==316); + /* (320) anylist ::= */ yytestcase(yyruleno==320); + /* (321) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==321); + /* (322) anylist ::= anylist ANY */ yytestcase(yyruleno==322); break; }; assert( yyruleno>=0 && yyrulenoyyidx -= yysize; yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto); - if( yyact < YYNSTATE ){ -#ifdef NDEBUG - /* If we are not debugging and the reduce action popped at least + if( yyact <= YY_MAX_SHIFTREDUCE ){ + if( yyact>YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; + /* If the reduce action popped at least ** one element off the stack, then we can push the new element back ** onto the stack here, and skip the stack overflow test in yy_shift(). ** That gives a significant speed improvement. */ @@ -126566,13 +129364,12 @@ static void yy_reduce( yymsp->stateno = (YYACTIONTYPE)yyact; yymsp->major = (YYCODETYPE)yygoto; yymsp->minor = yygotominor; - }else -#endif - { + yyTraceShift(yypParser, yyact); + }else{ yy_shift(yypParser,yyact,yygoto,&yygotominor); } }else{ - assert( yyact == YYNSTATE + YYNRULE + 1 ); + assert( yyact == YY_ACCEPT_ACTION ); yy_accept(yypParser); } } @@ -126697,12 +129494,13 @@ SQLITE_PRIVATE void sqlite3Parser( do{ yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); - if( yyact YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; yy_shift(yypParser,yyact,yymajor,&yyminorunion); yypParser->yyerrcnt--; yymajor = YYNOCODE; - }else if( yyact < YYNSTATE + YYNRULE ){ - yy_reduce(yypParser,yyact-YYNSTATE); + }else if( yyact <= YY_MAX_REDUCE ){ + yy_reduce(yypParser,yyact-YY_MIN_REDUCE); }else{ assert( yyact == YY_ERROR_ACTION ); #ifdef YYERRORSYMBOL @@ -126752,7 +129550,7 @@ SQLITE_PRIVATE void sqlite3Parser( yymx != YYERRORSYMBOL && (yyact = yy_find_reduce_action( yypParser->yystack[yypParser->yyidx].stateno, - YYERRORSYMBOL)) >= YYNSTATE + YYERRORSYMBOL)) >= YY_MIN_REDUCE ){ yy_pop_parser_stack(yypParser); } @@ -126802,6 +129600,11 @@ SQLITE_PRIVATE void sqlite3Parser( #endif } }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 ); +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sReturn\n",yyTracePrompt); + } +#endif return; } @@ -126824,6 +129627,7 @@ SQLITE_PRIVATE void sqlite3Parser( ** individual tokens and sends those tokens one-by-one over to the ** parser for analysis. */ +/* #include "sqliteInt.h" */ /* #include */ /* @@ -127188,7 +129992,11 @@ SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[] = { }; #define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) #endif + +/* Make the IdChar function accessible from ctime.c */ +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS SQLITE_PRIVATE int sqlite3IsIdChar(u8 c){ return IdChar(c); } +#endif /* @@ -127455,11440 +130263,27804 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ return i; } } - *tokenType = TK_ILLEGAL; - return 1; + *tokenType = TK_ILLEGAL; + return 1; +} + +/* +** Run the parser on the given SQL string. The parser structure is +** passed in. An SQLITE_ status code is returned. If an error occurs +** then an and attempt is made to write an error message into +** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that +** error message. +*/ +SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ + int nErr = 0; /* Number of errors encountered */ + int i; /* Loop counter */ + void *pEngine; /* The LEMON-generated LALR(1) parser */ + int tokenType; /* type of the next token */ + int lastTokenParsed = -1; /* type of the previous token */ + u8 enableLookaside; /* Saved value of db->lookaside.bEnabled */ + sqlite3 *db = pParse->db; /* The database connection */ + int mxSqlLen; /* Max length of an SQL string */ + + assert( zSql!=0 ); + mxSqlLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; + if( db->nVdbeActive==0 ){ + db->u1.isInterrupted = 0; + } + pParse->rc = SQLITE_OK; + pParse->zTail = zSql; + i = 0; + assert( pzErrMsg!=0 ); + /* sqlite3ParserTrace(stdout, "parser: "); */ + pEngine = sqlite3ParserAlloc(sqlite3Malloc); + if( pEngine==0 ){ + db->mallocFailed = 1; + return SQLITE_NOMEM; + } + assert( pParse->pNewTable==0 ); + assert( pParse->pNewTrigger==0 ); + assert( pParse->nVar==0 ); + assert( pParse->nzVar==0 ); + assert( pParse->azVar==0 ); + enableLookaside = db->lookaside.bEnabled; + if( db->lookaside.pStart ) db->lookaside.bEnabled = 1; + while( !db->mallocFailed && zSql[i]!=0 ){ + assert( i>=0 ); + pParse->sLastToken.z = &zSql[i]; + pParse->sLastToken.n = sqlite3GetToken((unsigned char*)&zSql[i],&tokenType); + i += pParse->sLastToken.n; + if( i>mxSqlLen ){ + pParse->rc = SQLITE_TOOBIG; + break; + } + switch( tokenType ){ + case TK_SPACE: { + if( db->u1.isInterrupted ){ + sqlite3ErrorMsg(pParse, "interrupt"); + pParse->rc = SQLITE_INTERRUPT; + goto abort_parse; + } + break; + } + case TK_ILLEGAL: { + sqlite3ErrorMsg(pParse, "unrecognized token: \"%T\"", + &pParse->sLastToken); + goto abort_parse; + } + case TK_SEMI: { + pParse->zTail = &zSql[i]; + /* Fall thru into the default case */ + } + default: { + sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse); + lastTokenParsed = tokenType; + if( pParse->rc!=SQLITE_OK ){ + goto abort_parse; + } + break; + } + } + } +abort_parse: + assert( nErr==0 ); + if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){ + assert( zSql[i]==0 ); + if( lastTokenParsed!=TK_SEMI ){ + sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse); + pParse->zTail = &zSql[i]; + } + if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){ + sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse); + } + } +#ifdef YYTRACKMAXSTACKDEPTH + sqlite3_mutex_enter(sqlite3MallocMutex()); + sqlite3StatusSet(SQLITE_STATUS_PARSER_STACK, + sqlite3ParserStackPeak(pEngine) + ); + sqlite3_mutex_leave(sqlite3MallocMutex()); +#endif /* YYDEBUG */ + sqlite3ParserFree(pEngine, sqlite3_free); + db->lookaside.bEnabled = enableLookaside; + if( db->mallocFailed ){ + pParse->rc = SQLITE_NOMEM; + } + if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){ + pParse->zErrMsg = sqlite3MPrintf(db, "%s", sqlite3ErrStr(pParse->rc)); + } + assert( pzErrMsg!=0 ); + if( pParse->zErrMsg ){ + *pzErrMsg = pParse->zErrMsg; + sqlite3_log(pParse->rc, "%s", *pzErrMsg); + pParse->zErrMsg = 0; + nErr++; + } + if( pParse->pVdbe && pParse->nErr>0 && pParse->nested==0 ){ + sqlite3VdbeDelete(pParse->pVdbe); + pParse->pVdbe = 0; + } +#ifndef SQLITE_OMIT_SHARED_CACHE + if( pParse->nested==0 ){ + sqlite3DbFree(db, pParse->aTableLock); + pParse->aTableLock = 0; + pParse->nTableLock = 0; + } +#endif +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3_free(pParse->apVtabLock); +#endif + + if( !IN_DECLARE_VTAB ){ + /* If the pParse->declareVtab flag is set, do not delete any table + ** structure built up in pParse->pNewTable. The calling code (see vtab.c) + ** will take responsibility for freeing the Table structure. + */ + sqlite3DeleteTable(db, pParse->pNewTable); + } + + if( pParse->bFreeWith ) sqlite3WithDelete(db, pParse->pWith); + sqlite3DeleteTrigger(db, pParse->pNewTrigger); + for(i=pParse->nzVar-1; i>=0; i--) sqlite3DbFree(db, pParse->azVar[i]); + sqlite3DbFree(db, pParse->azVar); + while( pParse->pAinc ){ + AutoincInfo *p = pParse->pAinc; + pParse->pAinc = p->pNext; + sqlite3DbFree(db, p); + } + while( pParse->pZombieTab ){ + Table *p = pParse->pZombieTab; + pParse->pZombieTab = p->pNextZombie; + sqlite3DeleteTable(db, p); + } + assert( nErr==0 || pParse->rc!=SQLITE_OK ); + return nErr; +} + +/************** End of tokenize.c ********************************************/ +/************** Begin file complete.c ****************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** An tokenizer for SQL +** +** This file contains C code that implements the sqlite3_complete() API. +** This code used to be part of the tokenizer.c source file. But by +** separating it out, the code will be automatically omitted from +** static links that do not use it. +*/ +/* #include "sqliteInt.h" */ +#ifndef SQLITE_OMIT_COMPLETE + +/* +** This is defined in tokenize.c. We just have to import the definition. +*/ +#ifndef SQLITE_AMALGAMATION +#ifdef SQLITE_ASCII +#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) +#endif +#ifdef SQLITE_EBCDIC +SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[]; +#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) +#endif +#endif /* SQLITE_AMALGAMATION */ + + +/* +** Token types used by the sqlite3_complete() routine. See the header +** comments on that procedure for additional information. +*/ +#define tkSEMI 0 +#define tkWS 1 +#define tkOTHER 2 +#ifndef SQLITE_OMIT_TRIGGER +#define tkEXPLAIN 3 +#define tkCREATE 4 +#define tkTEMP 5 +#define tkTRIGGER 6 +#define tkEND 7 +#endif + +/* +** Return TRUE if the given SQL string ends in a semicolon. +** +** Special handling is require for CREATE TRIGGER statements. +** Whenever the CREATE TRIGGER keywords are seen, the statement +** must end with ";END;". +** +** This implementation uses a state machine with 8 states: +** +** (0) INVALID We have not yet seen a non-whitespace character. +** +** (1) START At the beginning or end of an SQL statement. This routine +** returns 1 if it ends in the START state and 0 if it ends +** in any other state. +** +** (2) NORMAL We are in the middle of statement which ends with a single +** semicolon. +** +** (3) EXPLAIN The keyword EXPLAIN has been seen at the beginning of +** a statement. +** +** (4) CREATE The keyword CREATE has been seen at the beginning of a +** statement, possibly preceded by EXPLAIN and/or followed by +** TEMP or TEMPORARY +** +** (5) TRIGGER We are in the middle of a trigger definition that must be +** ended by a semicolon, the keyword END, and another semicolon. +** +** (6) SEMI We've seen the first semicolon in the ";END;" that occurs at +** the end of a trigger definition. +** +** (7) END We've seen the ";END" of the ";END;" that occurs at the end +** of a trigger definition. +** +** Transitions between states above are determined by tokens extracted +** from the input. The following tokens are significant: +** +** (0) tkSEMI A semicolon. +** (1) tkWS Whitespace. +** (2) tkOTHER Any other SQL token. +** (3) tkEXPLAIN The "explain" keyword. +** (4) tkCREATE The "create" keyword. +** (5) tkTEMP The "temp" or "temporary" keyword. +** (6) tkTRIGGER The "trigger" keyword. +** (7) tkEND The "end" keyword. +** +** Whitespace never causes a state transition and is always ignored. +** This means that a SQL string of all whitespace is invalid. +** +** If we compile with SQLITE_OMIT_TRIGGER, all of the computation needed +** to recognize the end of a trigger can be omitted. All we have to do +** is look for a semicolon that is not part of an string or comment. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_complete(const char *zSql){ + u8 state = 0; /* Current state, using numbers defined in header comment */ + u8 token; /* Value of the next token */ + +#ifndef SQLITE_OMIT_TRIGGER + /* A complex statement machine used to detect the end of a CREATE TRIGGER + ** statement. This is the normal case. + */ + static const u8 trans[8][8] = { + /* Token: */ + /* State: ** SEMI WS OTHER EXPLAIN CREATE TEMP TRIGGER END */ + /* 0 INVALID: */ { 1, 0, 2, 3, 4, 2, 2, 2, }, + /* 1 START: */ { 1, 1, 2, 3, 4, 2, 2, 2, }, + /* 2 NORMAL: */ { 1, 2, 2, 2, 2, 2, 2, 2, }, + /* 3 EXPLAIN: */ { 1, 3, 3, 2, 4, 2, 2, 2, }, + /* 4 CREATE: */ { 1, 4, 2, 2, 2, 4, 5, 2, }, + /* 5 TRIGGER: */ { 6, 5, 5, 5, 5, 5, 5, 5, }, + /* 6 SEMI: */ { 6, 6, 5, 5, 5, 5, 5, 7, }, + /* 7 END: */ { 1, 7, 5, 5, 5, 5, 5, 5, }, + }; +#else + /* If triggers are not supported by this compile then the statement machine + ** used to detect the end of a statement is much simpler + */ + static const u8 trans[3][3] = { + /* Token: */ + /* State: ** SEMI WS OTHER */ + /* 0 INVALID: */ { 1, 0, 2, }, + /* 1 START: */ { 1, 1, 2, }, + /* 2 NORMAL: */ { 1, 2, 2, }, + }; +#endif /* SQLITE_OMIT_TRIGGER */ + +#ifdef SQLITE_ENABLE_API_ARMOR + if( zSql==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + + while( *zSql ){ + switch( *zSql ){ + case ';': { /* A semicolon */ + token = tkSEMI; + break; + } + case ' ': + case '\r': + case '\t': + case '\n': + case '\f': { /* White space is ignored */ + token = tkWS; + break; + } + case '/': { /* C-style comments */ + if( zSql[1]!='*' ){ + token = tkOTHER; + break; + } + zSql += 2; + while( zSql[0] && (zSql[0]!='*' || zSql[1]!='/') ){ zSql++; } + if( zSql[0]==0 ) return 0; + zSql++; + token = tkWS; + break; + } + case '-': { /* SQL-style comments from "--" to end of line */ + if( zSql[1]!='-' ){ + token = tkOTHER; + break; + } + while( *zSql && *zSql!='\n' ){ zSql++; } + if( *zSql==0 ) return state==1; + token = tkWS; + break; + } + case '[': { /* Microsoft-style identifiers in [...] */ + zSql++; + while( *zSql && *zSql!=']' ){ zSql++; } + if( *zSql==0 ) return 0; + token = tkOTHER; + break; + } + case '`': /* Grave-accent quoted symbols used by MySQL */ + case '"': /* single- and double-quoted strings */ + case '\'': { + int c = *zSql; + zSql++; + while( *zSql && *zSql!=c ){ zSql++; } + if( *zSql==0 ) return 0; + token = tkOTHER; + break; + } + default: { +#ifdef SQLITE_EBCDIC + unsigned char c; +#endif + if( IdChar((u8)*zSql) ){ + /* Keywords and unquoted identifiers */ + int nId; + for(nId=1; IdChar(zSql[nId]); nId++){} +#ifdef SQLITE_OMIT_TRIGGER + token = tkOTHER; +#else + switch( *zSql ){ + case 'c': case 'C': { + if( nId==6 && sqlite3StrNICmp(zSql, "create", 6)==0 ){ + token = tkCREATE; + }else{ + token = tkOTHER; + } + break; + } + case 't': case 'T': { + if( nId==7 && sqlite3StrNICmp(zSql, "trigger", 7)==0 ){ + token = tkTRIGGER; + }else if( nId==4 && sqlite3StrNICmp(zSql, "temp", 4)==0 ){ + token = tkTEMP; + }else if( nId==9 && sqlite3StrNICmp(zSql, "temporary", 9)==0 ){ + token = tkTEMP; + }else{ + token = tkOTHER; + } + break; + } + case 'e': case 'E': { + if( nId==3 && sqlite3StrNICmp(zSql, "end", 3)==0 ){ + token = tkEND; + }else +#ifndef SQLITE_OMIT_EXPLAIN + if( nId==7 && sqlite3StrNICmp(zSql, "explain", 7)==0 ){ + token = tkEXPLAIN; + }else +#endif + { + token = tkOTHER; + } + break; + } + default: { + token = tkOTHER; + break; + } + } +#endif /* SQLITE_OMIT_TRIGGER */ + zSql += nId-1; + }else{ + /* Operators and special symbols */ + token = tkOTHER; + } + break; + } + } + state = trans[state][token]; + zSql++; + } + return state==1; +} + +#ifndef SQLITE_OMIT_UTF16 +/* +** This routine is the same as the sqlite3_complete() routine described +** above, except that the parameter is required to be UTF-16 encoded, not +** UTF-8. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_complete16(const void *zSql){ + sqlite3_value *pVal; + char const *zSql8; + int rc; + +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + pVal = sqlite3ValueNew(0); + sqlite3ValueSetStr(pVal, -1, zSql, SQLITE_UTF16NATIVE, SQLITE_STATIC); + zSql8 = sqlite3ValueText(pVal, SQLITE_UTF8); + if( zSql8 ){ + rc = sqlite3_complete(zSql8); + }else{ + rc = SQLITE_NOMEM; + } + sqlite3ValueFree(pVal); + return rc & 0xff; +} +#endif /* SQLITE_OMIT_UTF16 */ +#endif /* SQLITE_OMIT_COMPLETE */ + +/************** End of complete.c ********************************************/ +/************** Begin file main.c ********************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Main file for the SQLite library. The routines in this file +** implement the programmer interface to the library. Routines in +** other files are for internal use by SQLite and should not be +** accessed by users of the library. +*/ +/* #include "sqliteInt.h" */ + +#ifdef SQLITE_ENABLE_FTS3 +/************** Include fts3.h in the middle of main.c ***********************/ +/************** Begin file fts3.h ********************************************/ +/* +** 2006 Oct 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file is used by programs that want to link against the +** FTS3 library. All it does is declare the sqlite3Fts3Init() interface. +*/ +/* #include "sqlite3.h" */ + +#if 0 +extern "C" { +#endif /* __cplusplus */ + +SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db); + +#if 0 +} /* extern "C" */ +#endif /* __cplusplus */ + +/************** End of fts3.h ************************************************/ +/************** Continuing where we left off in main.c ***********************/ +#endif +#ifdef SQLITE_ENABLE_RTREE +/************** Include rtree.h in the middle of main.c **********************/ +/************** Begin file rtree.h *******************************************/ +/* +** 2008 May 26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file is used by programs that want to link against the +** RTREE library. All it does is declare the sqlite3RtreeInit() interface. +*/ +/* #include "sqlite3.h" */ + +#if 0 +extern "C" { +#endif /* __cplusplus */ + +SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db); + +#if 0 +} /* extern "C" */ +#endif /* __cplusplus */ + +/************** End of rtree.h ***********************************************/ +/************** Continuing where we left off in main.c ***********************/ +#endif +#ifdef SQLITE_ENABLE_ICU +/************** Include sqliteicu.h in the middle of main.c ******************/ +/************** Begin file sqliteicu.h ***************************************/ +/* +** 2008 May 26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file is used by programs that want to link against the +** ICU extension. All it does is declare the sqlite3IcuInit() interface. +*/ +/* #include "sqlite3.h" */ + +#if 0 +extern "C" { +#endif /* __cplusplus */ + +SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db); + +#if 0 +} /* extern "C" */ +#endif /* __cplusplus */ + + +/************** End of sqliteicu.h *******************************************/ +/************** Continuing where we left off in main.c ***********************/ +#endif +#ifdef SQLITE_ENABLE_JSON1 +SQLITE_PRIVATE int sqlite3Json1Init(sqlite3*); +#endif +#ifdef SQLITE_ENABLE_FTS5 +SQLITE_PRIVATE int sqlite3Fts5Init(sqlite3*); +#endif + +#ifndef SQLITE_AMALGAMATION +/* IMPLEMENTATION-OF: R-46656-45156 The sqlite3_version[] string constant +** contains the text of SQLITE_VERSION macro. +*/ +SQLITE_API const char sqlite3_version[] = SQLITE_VERSION; +#endif + +/* IMPLEMENTATION-OF: R-53536-42575 The sqlite3_libversion() function returns +** a pointer to the to the sqlite3_version[] string constant. +*/ +SQLITE_API const char *SQLITE_STDCALL sqlite3_libversion(void){ return sqlite3_version; } + +/* IMPLEMENTATION-OF: R-63124-39300 The sqlite3_sourceid() function returns a +** pointer to a string constant whose value is the same as the +** SQLITE_SOURCE_ID C preprocessor macro. +*/ +SQLITE_API const char *SQLITE_STDCALL sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } + +/* IMPLEMENTATION-OF: R-35210-63508 The sqlite3_libversion_number() function +** returns an integer equal to SQLITE_VERSION_NUMBER. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_libversion_number(void){ return SQLITE_VERSION_NUMBER; } + +/* IMPLEMENTATION-OF: R-20790-14025 The sqlite3_threadsafe() function returns +** zero if and only if SQLite was compiled with mutexing code omitted due to +** the SQLITE_THREADSAFE compile-time option being set to 0. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; } + +/* +** When compiling the test fixture or with debugging enabled (on Win32), +** this variable being set to non-zero will cause OSTRACE macros to emit +** extra diagnostic information. +*/ +#ifdef SQLITE_HAVE_OS_TRACE +# ifndef SQLITE_DEBUG_OS_TRACE +# define SQLITE_DEBUG_OS_TRACE 0 +# endif + int sqlite3OSTrace = SQLITE_DEBUG_OS_TRACE; +#endif + +#if !defined(SQLITE_OMIT_TRACE) && defined(SQLITE_ENABLE_IOTRACE) +/* +** If the following function pointer is not NULL and if +** SQLITE_ENABLE_IOTRACE is enabled, then messages describing +** I/O active are written using this function. These messages +** are intended for debugging activity only. +*/ +SQLITE_API void (SQLITE_CDECL *sqlite3IoTrace)(const char*, ...) = 0; +#endif + +/* +** If the following global variable points to a string which is the +** name of a directory, then that directory will be used to store +** temporary files. +** +** See also the "PRAGMA temp_store_directory" SQL command. +*/ +SQLITE_API char *sqlite3_temp_directory = 0; + +/* +** If the following global variable points to a string which is the +** name of a directory, then that directory will be used to store +** all database files specified with a relative pathname. +** +** See also the "PRAGMA data_store_directory" SQL command. +*/ +SQLITE_API char *sqlite3_data_directory = 0; + +/* +** Initialize SQLite. +** +** This routine must be called to initialize the memory allocation, +** VFS, and mutex subsystems prior to doing any serious work with +** SQLite. But as long as you do not compile with SQLITE_OMIT_AUTOINIT +** this routine will be called automatically by key routines such as +** sqlite3_open(). +** +** This routine is a no-op except on its very first call for the process, +** or for the first call after a call to sqlite3_shutdown. +** +** The first thread to call this routine runs the initialization to +** completion. If subsequent threads call this routine before the first +** thread has finished the initialization process, then the subsequent +** threads must block until the first thread finishes with the initialization. +** +** The first thread might call this routine recursively. Recursive +** calls to this routine should not block, of course. Otherwise the +** initialization process would never complete. +** +** Let X be the first thread to enter this routine. Let Y be some other +** thread. Then while the initial invocation of this routine by X is +** incomplete, it is required that: +** +** * Calls to this routine from Y must block until the outer-most +** call by X completes. +** +** * Recursive calls to this routine from thread X return immediately +** without blocking. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_initialize(void){ + MUTEX_LOGIC( sqlite3_mutex *pMaster; ) /* The main static mutex */ + int rc; /* Result code */ +#ifdef SQLITE_EXTRA_INIT + int bRunExtraInit = 0; /* Extra initialization needed */ +#endif + +#ifdef SQLITE_OMIT_WSD + rc = sqlite3_wsd_init(4096, 24); + if( rc!=SQLITE_OK ){ + return rc; + } +#endif + + /* If the following assert() fails on some obscure processor/compiler + ** combination, the work-around is to set the correct pointer + ** size at compile-time using -DSQLITE_PTRSIZE=n compile-time option */ + assert( SQLITE_PTRSIZE==sizeof(char*) ); + + /* If SQLite is already completely initialized, then this call + ** to sqlite3_initialize() should be a no-op. But the initialization + ** must be complete. So isInit must not be set until the very end + ** of this routine. + */ + if( sqlite3GlobalConfig.isInit ) return SQLITE_OK; + + /* Make sure the mutex subsystem is initialized. If unable to + ** initialize the mutex subsystem, return early with the error. + ** If the system is so sick that we are unable to allocate a mutex, + ** there is not much SQLite is going to be able to do. + ** + ** The mutex subsystem must take care of serializing its own + ** initialization. + */ + rc = sqlite3MutexInit(); + if( rc ) return rc; + + /* Initialize the malloc() system and the recursive pInitMutex mutex. + ** This operation is protected by the STATIC_MASTER mutex. Note that + ** MutexAlloc() is called for a static mutex prior to initializing the + ** malloc subsystem - this implies that the allocation of a static + ** mutex must not require support from the malloc subsystem. + */ + MUTEX_LOGIC( pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) + sqlite3_mutex_enter(pMaster); + sqlite3GlobalConfig.isMutexInit = 1; + if( !sqlite3GlobalConfig.isMallocInit ){ + rc = sqlite3MallocInit(); + } + if( rc==SQLITE_OK ){ + sqlite3GlobalConfig.isMallocInit = 1; + if( !sqlite3GlobalConfig.pInitMutex ){ + sqlite3GlobalConfig.pInitMutex = + sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); + if( sqlite3GlobalConfig.bCoreMutex && !sqlite3GlobalConfig.pInitMutex ){ + rc = SQLITE_NOMEM; + } + } + } + if( rc==SQLITE_OK ){ + sqlite3GlobalConfig.nRefInitMutex++; + } + sqlite3_mutex_leave(pMaster); + + /* If rc is not SQLITE_OK at this point, then either the malloc + ** subsystem could not be initialized or the system failed to allocate + ** the pInitMutex mutex. Return an error in either case. */ + if( rc!=SQLITE_OK ){ + return rc; + } + + /* Do the rest of the initialization under the recursive mutex so + ** that we will be able to handle recursive calls into + ** sqlite3_initialize(). The recursive calls normally come through + ** sqlite3_os_init() when it invokes sqlite3_vfs_register(), but other + ** recursive calls might also be possible. + ** + ** IMPLEMENTATION-OF: R-00140-37445 SQLite automatically serializes calls + ** to the xInit method, so the xInit method need not be threadsafe. + ** + ** The following mutex is what serializes access to the appdef pcache xInit + ** methods. The sqlite3_pcache_methods.xInit() all is embedded in the + ** call to sqlite3PcacheInitialize(). + */ + sqlite3_mutex_enter(sqlite3GlobalConfig.pInitMutex); + if( sqlite3GlobalConfig.isInit==0 && sqlite3GlobalConfig.inProgress==0 ){ + FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); + sqlite3GlobalConfig.inProgress = 1; + memset(pHash, 0, sizeof(sqlite3GlobalFunctions)); + sqlite3RegisterGlobalFunctions(); + if( sqlite3GlobalConfig.isPCacheInit==0 ){ + rc = sqlite3PcacheInitialize(); + } + if( rc==SQLITE_OK ){ + sqlite3GlobalConfig.isPCacheInit = 1; + rc = sqlite3OsInit(); + } + if( rc==SQLITE_OK ){ + sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage, + sqlite3GlobalConfig.szPage, sqlite3GlobalConfig.nPage); + sqlite3GlobalConfig.isInit = 1; +#ifdef SQLITE_EXTRA_INIT + bRunExtraInit = 1; +#endif + } + sqlite3GlobalConfig.inProgress = 0; + } + sqlite3_mutex_leave(sqlite3GlobalConfig.pInitMutex); + + /* Go back under the static mutex and clean up the recursive + ** mutex to prevent a resource leak. + */ + sqlite3_mutex_enter(pMaster); + sqlite3GlobalConfig.nRefInitMutex--; + if( sqlite3GlobalConfig.nRefInitMutex<=0 ){ + assert( sqlite3GlobalConfig.nRefInitMutex==0 ); + sqlite3_mutex_free(sqlite3GlobalConfig.pInitMutex); + sqlite3GlobalConfig.pInitMutex = 0; + } + sqlite3_mutex_leave(pMaster); + + /* The following is just a sanity check to make sure SQLite has + ** been compiled correctly. It is important to run this code, but + ** we don't want to run it too often and soak up CPU cycles for no + ** reason. So we run it once during initialization. + */ +#ifndef NDEBUG +#ifndef SQLITE_OMIT_FLOATING_POINT + /* This section of code's only "output" is via assert() statements. */ + if ( rc==SQLITE_OK ){ + u64 x = (((u64)1)<<63)-1; + double y; + assert(sizeof(x)==8); + assert(sizeof(x)==sizeof(y)); + memcpy(&y, &x, 8); + assert( sqlite3IsNaN(y) ); + } +#endif +#endif + + /* Do extra initialization steps requested by the SQLITE_EXTRA_INIT + ** compile-time option. + */ +#ifdef SQLITE_EXTRA_INIT + if( bRunExtraInit ){ + int SQLITE_EXTRA_INIT(const char*); + rc = SQLITE_EXTRA_INIT(0); + } +#endif + + return rc; +} + +/* +** Undo the effects of sqlite3_initialize(). Must not be called while +** there are outstanding database connections or memory allocations or +** while any part of SQLite is otherwise in use in any thread. This +** routine is not threadsafe. But it is safe to invoke this routine +** on when SQLite is already shut down. If SQLite is already shut down +** when this routine is invoked, then this routine is a harmless no-op. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_shutdown(void){ +#ifdef SQLITE_OMIT_WSD + int rc = sqlite3_wsd_init(4096, 24); + if( rc!=SQLITE_OK ){ + return rc; + } +#endif + + if( sqlite3GlobalConfig.isInit ){ +#ifdef SQLITE_EXTRA_SHUTDOWN + void SQLITE_EXTRA_SHUTDOWN(void); + SQLITE_EXTRA_SHUTDOWN(); +#endif + sqlite3_os_end(); + sqlite3_reset_auto_extension(); + sqlite3GlobalConfig.isInit = 0; + } + if( sqlite3GlobalConfig.isPCacheInit ){ + sqlite3PcacheShutdown(); + sqlite3GlobalConfig.isPCacheInit = 0; + } + if( sqlite3GlobalConfig.isMallocInit ){ + sqlite3MallocEnd(); + sqlite3GlobalConfig.isMallocInit = 0; + +#ifndef SQLITE_OMIT_SHUTDOWN_DIRECTORIES + /* The heap subsystem has now been shutdown and these values are supposed + ** to be NULL or point to memory that was obtained from sqlite3_malloc(), + ** which would rely on that heap subsystem; therefore, make sure these + ** values cannot refer to heap memory that was just invalidated when the + ** heap subsystem was shutdown. This is only done if the current call to + ** this function resulted in the heap subsystem actually being shutdown. + */ + sqlite3_data_directory = 0; + sqlite3_temp_directory = 0; +#endif + } + if( sqlite3GlobalConfig.isMutexInit ){ + sqlite3MutexEnd(); + sqlite3GlobalConfig.isMutexInit = 0; + } + + return SQLITE_OK; +} + +/* +** This API allows applications to modify the global configuration of +** the SQLite library at run-time. +** +** This routine should only be called when there are no outstanding +** database connections or memory allocations. This routine is not +** threadsafe. Failure to heed these warnings can lead to unpredictable +** behavior. +*/ +SQLITE_API int SQLITE_CDECL sqlite3_config(int op, ...){ + va_list ap; + int rc = SQLITE_OK; + + /* sqlite3_config() shall return SQLITE_MISUSE if it is invoked while + ** the SQLite library is in use. */ + if( sqlite3GlobalConfig.isInit ) return SQLITE_MISUSE_BKPT; + + va_start(ap, op); + switch( op ){ + + /* Mutex configuration options are only available in a threadsafe + ** compile. + */ +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-54466-46756 */ + case SQLITE_CONFIG_SINGLETHREAD: { + /* EVIDENCE-OF: R-02748-19096 This option sets the threading mode to + ** Single-thread. */ + sqlite3GlobalConfig.bCoreMutex = 0; /* Disable mutex on core */ + sqlite3GlobalConfig.bFullMutex = 0; /* Disable mutex on connections */ + break; + } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-20520-54086 */ + case SQLITE_CONFIG_MULTITHREAD: { + /* EVIDENCE-OF: R-14374-42468 This option sets the threading mode to + ** Multi-thread. */ + sqlite3GlobalConfig.bCoreMutex = 1; /* Enable mutex on core */ + sqlite3GlobalConfig.bFullMutex = 0; /* Disable mutex on connections */ + break; + } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-59593-21810 */ + case SQLITE_CONFIG_SERIALIZED: { + /* EVIDENCE-OF: R-41220-51800 This option sets the threading mode to + ** Serialized. */ + sqlite3GlobalConfig.bCoreMutex = 1; /* Enable mutex on core */ + sqlite3GlobalConfig.bFullMutex = 1; /* Enable mutex on connections */ + break; + } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-63666-48755 */ + case SQLITE_CONFIG_MUTEX: { + /* Specify an alternative mutex implementation */ + sqlite3GlobalConfig.mutex = *va_arg(ap, sqlite3_mutex_methods*); + break; + } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-14450-37597 */ + case SQLITE_CONFIG_GETMUTEX: { + /* Retrieve the current mutex implementation */ + *va_arg(ap, sqlite3_mutex_methods*) = sqlite3GlobalConfig.mutex; + break; + } +#endif + + case SQLITE_CONFIG_MALLOC: { + /* EVIDENCE-OF: R-55594-21030 The SQLITE_CONFIG_MALLOC option takes a + ** single argument which is a pointer to an instance of the + ** sqlite3_mem_methods structure. The argument specifies alternative + ** low-level memory allocation routines to be used in place of the memory + ** allocation routines built into SQLite. */ + sqlite3GlobalConfig.m = *va_arg(ap, sqlite3_mem_methods*); + break; + } + case SQLITE_CONFIG_GETMALLOC: { + /* EVIDENCE-OF: R-51213-46414 The SQLITE_CONFIG_GETMALLOC option takes a + ** single argument which is a pointer to an instance of the + ** sqlite3_mem_methods structure. The sqlite3_mem_methods structure is + ** filled with the currently defined memory allocation routines. */ + if( sqlite3GlobalConfig.m.xMalloc==0 ) sqlite3MemSetDefault(); + *va_arg(ap, sqlite3_mem_methods*) = sqlite3GlobalConfig.m; + break; + } + case SQLITE_CONFIG_MEMSTATUS: { + /* EVIDENCE-OF: R-61275-35157 The SQLITE_CONFIG_MEMSTATUS option takes + ** single argument of type int, interpreted as a boolean, which enables + ** or disables the collection of memory allocation statistics. */ + sqlite3GlobalConfig.bMemstat = va_arg(ap, int); + break; + } + case SQLITE_CONFIG_SCRATCH: { + /* EVIDENCE-OF: R-08404-60887 There are three arguments to + ** SQLITE_CONFIG_SCRATCH: A pointer an 8-byte aligned memory buffer from + ** which the scratch allocations will be drawn, the size of each scratch + ** allocation (sz), and the maximum number of scratch allocations (N). */ + sqlite3GlobalConfig.pScratch = va_arg(ap, void*); + sqlite3GlobalConfig.szScratch = va_arg(ap, int); + sqlite3GlobalConfig.nScratch = va_arg(ap, int); + break; + } + case SQLITE_CONFIG_PAGECACHE: { + /* EVIDENCE-OF: R-31408-40510 There are three arguments to + ** SQLITE_CONFIG_PAGECACHE: A pointer to 8-byte aligned memory, the size + ** of each page buffer (sz), and the number of pages (N). */ + sqlite3GlobalConfig.pPage = va_arg(ap, void*); + sqlite3GlobalConfig.szPage = va_arg(ap, int); + sqlite3GlobalConfig.nPage = va_arg(ap, int); + break; + } + case SQLITE_CONFIG_PCACHE_HDRSZ: { + /* EVIDENCE-OF: R-39100-27317 The SQLITE_CONFIG_PCACHE_HDRSZ option takes + ** a single parameter which is a pointer to an integer and writes into + ** that integer the number of extra bytes per page required for each page + ** in SQLITE_CONFIG_PAGECACHE. */ + *va_arg(ap, int*) = + sqlite3HeaderSizeBtree() + + sqlite3HeaderSizePcache() + + sqlite3HeaderSizePcache1(); + break; + } + + case SQLITE_CONFIG_PCACHE: { + /* no-op */ + break; + } + case SQLITE_CONFIG_GETPCACHE: { + /* now an error */ + rc = SQLITE_ERROR; + break; + } + + case SQLITE_CONFIG_PCACHE2: { + /* EVIDENCE-OF: R-63325-48378 The SQLITE_CONFIG_PCACHE2 option takes a + ** single argument which is a pointer to an sqlite3_pcache_methods2 + ** object. This object specifies the interface to a custom page cache + ** implementation. */ + sqlite3GlobalConfig.pcache2 = *va_arg(ap, sqlite3_pcache_methods2*); + break; + } + case SQLITE_CONFIG_GETPCACHE2: { + /* EVIDENCE-OF: R-22035-46182 The SQLITE_CONFIG_GETPCACHE2 option takes a + ** single argument which is a pointer to an sqlite3_pcache_methods2 + ** object. SQLite copies of the current page cache implementation into + ** that object. */ + if( sqlite3GlobalConfig.pcache2.xInit==0 ){ + sqlite3PCacheSetDefault(); + } + *va_arg(ap, sqlite3_pcache_methods2*) = sqlite3GlobalConfig.pcache2; + break; + } + +/* EVIDENCE-OF: R-06626-12911 The SQLITE_CONFIG_HEAP option is only +** available if SQLite is compiled with either SQLITE_ENABLE_MEMSYS3 or +** SQLITE_ENABLE_MEMSYS5 and returns SQLITE_ERROR if invoked otherwise. */ +#if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) + case SQLITE_CONFIG_HEAP: { + /* EVIDENCE-OF: R-19854-42126 There are three arguments to + ** SQLITE_CONFIG_HEAP: An 8-byte aligned pointer to the memory, the + ** number of bytes in the memory buffer, and the minimum allocation size. + */ + sqlite3GlobalConfig.pHeap = va_arg(ap, void*); + sqlite3GlobalConfig.nHeap = va_arg(ap, int); + sqlite3GlobalConfig.mnReq = va_arg(ap, int); + + if( sqlite3GlobalConfig.mnReq<1 ){ + sqlite3GlobalConfig.mnReq = 1; + }else if( sqlite3GlobalConfig.mnReq>(1<<12) ){ + /* cap min request size at 2^12 */ + sqlite3GlobalConfig.mnReq = (1<<12); + } + + if( sqlite3GlobalConfig.pHeap==0 ){ + /* EVIDENCE-OF: R-49920-60189 If the first pointer (the memory pointer) + ** is NULL, then SQLite reverts to using its default memory allocator + ** (the system malloc() implementation), undoing any prior invocation of + ** SQLITE_CONFIG_MALLOC. + ** + ** Setting sqlite3GlobalConfig.m to all zeros will cause malloc to + ** revert to its default implementation when sqlite3_initialize() is run + */ + memset(&sqlite3GlobalConfig.m, 0, sizeof(sqlite3GlobalConfig.m)); + }else{ + /* EVIDENCE-OF: R-61006-08918 If the memory pointer is not NULL then the + ** alternative memory allocator is engaged to handle all of SQLites + ** memory allocation needs. */ +#ifdef SQLITE_ENABLE_MEMSYS3 + sqlite3GlobalConfig.m = *sqlite3MemGetMemsys3(); +#endif +#ifdef SQLITE_ENABLE_MEMSYS5 + sqlite3GlobalConfig.m = *sqlite3MemGetMemsys5(); +#endif + } + break; + } +#endif + + case SQLITE_CONFIG_LOOKASIDE: { + sqlite3GlobalConfig.szLookaside = va_arg(ap, int); + sqlite3GlobalConfig.nLookaside = va_arg(ap, int); + break; + } + + /* Record a pointer to the logger function and its first argument. + ** The default is NULL. Logging is disabled if the function pointer is + ** NULL. + */ + case SQLITE_CONFIG_LOG: { + /* MSVC is picky about pulling func ptrs from va lists. + ** http://support.microsoft.com/kb/47961 + ** sqlite3GlobalConfig.xLog = va_arg(ap, void(*)(void*,int,const char*)); + */ + typedef void(*LOGFUNC_t)(void*,int,const char*); + sqlite3GlobalConfig.xLog = va_arg(ap, LOGFUNC_t); + sqlite3GlobalConfig.pLogArg = va_arg(ap, void*); + break; + } + + /* EVIDENCE-OF: R-55548-33817 The compile-time setting for URI filenames + ** can be changed at start-time using the + ** sqlite3_config(SQLITE_CONFIG_URI,1) or + ** sqlite3_config(SQLITE_CONFIG_URI,0) configuration calls. + */ + case SQLITE_CONFIG_URI: { + /* EVIDENCE-OF: R-25451-61125 The SQLITE_CONFIG_URI option takes a single + ** argument of type int. If non-zero, then URI handling is globally + ** enabled. If the parameter is zero, then URI handling is globally + ** disabled. */ + sqlite3GlobalConfig.bOpenUri = va_arg(ap, int); + break; + } + + case SQLITE_CONFIG_COVERING_INDEX_SCAN: { + /* EVIDENCE-OF: R-36592-02772 The SQLITE_CONFIG_COVERING_INDEX_SCAN + ** option takes a single integer argument which is interpreted as a + ** boolean in order to enable or disable the use of covering indices for + ** full table scans in the query optimizer. */ + sqlite3GlobalConfig.bUseCis = va_arg(ap, int); + break; + } + +#ifdef SQLITE_ENABLE_SQLLOG + case SQLITE_CONFIG_SQLLOG: { + typedef void(*SQLLOGFUNC_t)(void*, sqlite3*, const char*, int); + sqlite3GlobalConfig.xSqllog = va_arg(ap, SQLLOGFUNC_t); + sqlite3GlobalConfig.pSqllogArg = va_arg(ap, void *); + break; + } +#endif + + case SQLITE_CONFIG_MMAP_SIZE: { + /* EVIDENCE-OF: R-58063-38258 SQLITE_CONFIG_MMAP_SIZE takes two 64-bit + ** integer (sqlite3_int64) values that are the default mmap size limit + ** (the default setting for PRAGMA mmap_size) and the maximum allowed + ** mmap size limit. */ + sqlite3_int64 szMmap = va_arg(ap, sqlite3_int64); + sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); + /* EVIDENCE-OF: R-53367-43190 If either argument to this option is + ** negative, then that argument is changed to its compile-time default. + ** + ** EVIDENCE-OF: R-34993-45031 The maximum allowed mmap size will be + ** silently truncated if necessary so that it does not exceed the + ** compile-time maximum mmap size set by the SQLITE_MAX_MMAP_SIZE + ** compile-time option. + */ + if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ){ + mxMmap = SQLITE_MAX_MMAP_SIZE; + } + if( szMmap<0 ) szMmap = SQLITE_DEFAULT_MMAP_SIZE; + if( szMmap>mxMmap) szMmap = mxMmap; + sqlite3GlobalConfig.mxMmap = mxMmap; + sqlite3GlobalConfig.szMmap = szMmap; + break; + } + +#if SQLITE_OS_WIN && defined(SQLITE_WIN32_MALLOC) /* IMP: R-04780-55815 */ + case SQLITE_CONFIG_WIN32_HEAPSIZE: { + /* EVIDENCE-OF: R-34926-03360 SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit + ** unsigned integer value that specifies the maximum size of the created + ** heap. */ + sqlite3GlobalConfig.nHeap = va_arg(ap, int); + break; + } +#endif + + case SQLITE_CONFIG_PMASZ: { + sqlite3GlobalConfig.szPma = va_arg(ap, unsigned int); + break; + } + + default: { + rc = SQLITE_ERROR; + break; + } + } + va_end(ap); + return rc; +} + +/* +** Set up the lookaside buffers for a database connection. +** Return SQLITE_OK on success. +** If lookaside is already active, return SQLITE_BUSY. +** +** The sz parameter is the number of bytes in each lookaside slot. +** The cnt parameter is the number of slots. If pStart is NULL the +** space for the lookaside memory is obtained from sqlite3_malloc(). +** If pStart is not NULL then it is sz*cnt bytes of memory to use for +** the lookaside memory. +*/ +static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ +#ifndef SQLITE_OMIT_LOOKASIDE + void *pStart; + if( db->lookaside.nOut ){ + return SQLITE_BUSY; + } + /* Free any existing lookaside buffer for this handle before + ** allocating a new one so we don't have to have space for + ** both at the same time. + */ + if( db->lookaside.bMalloced ){ + sqlite3_free(db->lookaside.pStart); + } + /* The size of a lookaside slot after ROUNDDOWN8 needs to be larger + ** than a pointer to be useful. + */ + sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ + if( sz<=(int)sizeof(LookasideSlot*) ) sz = 0; + if( cnt<0 ) cnt = 0; + if( sz==0 || cnt==0 ){ + sz = 0; + pStart = 0; + }else if( pBuf==0 ){ + sqlite3BeginBenignMalloc(); + pStart = sqlite3Malloc( sz*cnt ); /* IMP: R-61949-35727 */ + sqlite3EndBenignMalloc(); + if( pStart ) cnt = sqlite3MallocSize(pStart)/sz; + }else{ + pStart = pBuf; + } + db->lookaside.pStart = pStart; + db->lookaside.pFree = 0; + db->lookaside.sz = (u16)sz; + if( pStart ){ + int i; + LookasideSlot *p; + assert( sz > (int)sizeof(LookasideSlot*) ); + p = (LookasideSlot*)pStart; + for(i=cnt-1; i>=0; i--){ + p->pNext = db->lookaside.pFree; + db->lookaside.pFree = p; + p = (LookasideSlot*)&((u8*)p)[sz]; + } + db->lookaside.pEnd = p; + db->lookaside.bEnabled = 1; + db->lookaside.bMalloced = pBuf==0 ?1:0; + }else{ + db->lookaside.pStart = db; + db->lookaside.pEnd = db; + db->lookaside.bEnabled = 0; + db->lookaside.bMalloced = 0; + } +#endif /* SQLITE_OMIT_LOOKASIDE */ + return SQLITE_OK; +} + +/* +** Return the mutex associated with a database connection. +*/ +SQLITE_API sqlite3_mutex *SQLITE_STDCALL sqlite3_db_mutex(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return db->mutex; +} + +/* +** Free up as much memory as we can from the given database +** connection. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_db_release_memory(sqlite3 *db){ + int i; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + for(i=0; inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + Pager *pPager = sqlite3BtreePager(pBt); + sqlite3PagerShrink(pPager); + } + } + sqlite3BtreeLeaveAll(db); + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +/* +** Configuration settings for an individual database connection +*/ +SQLITE_API int SQLITE_CDECL sqlite3_db_config(sqlite3 *db, int op, ...){ + va_list ap; + int rc; + va_start(ap, op); + switch( op ){ + case SQLITE_DBCONFIG_LOOKASIDE: { + void *pBuf = va_arg(ap, void*); /* IMP: R-26835-10964 */ + int sz = va_arg(ap, int); /* IMP: R-47871-25994 */ + int cnt = va_arg(ap, int); /* IMP: R-04460-53386 */ + rc = setupLookaside(db, pBuf, sz, cnt); + break; + } + default: { + static const struct { + int op; /* The opcode */ + u32 mask; /* Mask of the bit in sqlite3.flags to set/clear */ + } aFlagOp[] = { + { SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_ForeignKeys }, + { SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_EnableTrigger }, + }; + unsigned int i; + rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ + for(i=0; iflags; + if( onoff>0 ){ + db->flags |= aFlagOp[i].mask; + }else if( onoff==0 ){ + db->flags &= ~aFlagOp[i].mask; + } + if( oldFlags!=db->flags ){ + sqlite3ExpirePreparedStatements(db); + } + if( pRes ){ + *pRes = (db->flags & aFlagOp[i].mask)!=0; + } + rc = SQLITE_OK; + break; + } + } + break; + } + } + va_end(ap); + return rc; +} + + +/* +** Return true if the buffer z[0..n-1] contains all spaces. +*/ +static int allSpaces(const char *z, int n){ + while( n>0 && z[n-1]==' ' ){ n--; } + return n==0; +} + +/* +** This is the default collating function named "BINARY" which is always +** available. +** +** If the padFlag argument is not NULL then space padding at the end +** of strings is ignored. This implements the RTRIM collation. +*/ +static int binCollFunc( + void *padFlag, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + int rc, n; + n = nKey1lastRowid; +} + +/* +** Return the number of changes in the most recent call to sqlite3_exec(). +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_changes(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return db->nChange; +} + +/* +** Return the number of changes since the database handle was opened. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_total_changes(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return db->nTotalChange; +} + +/* +** Close all open savepoints. This function only manipulates fields of the +** database handle object, it does not close any savepoints that may be open +** at the b-tree/pager level. +*/ +SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *db){ + while( db->pSavepoint ){ + Savepoint *pTmp = db->pSavepoint; + db->pSavepoint = pTmp->pNext; + sqlite3DbFree(db, pTmp); + } + db->nSavepoint = 0; + db->nStatement = 0; + db->isTransactionSavepoint = 0; +} + +/* +** Invoke the destructor function associated with FuncDef p, if any. Except, +** if this is not the last copy of the function, do not invoke it. Multiple +** copies of a single function are created when create_function() is called +** with SQLITE_ANY as the encoding. +*/ +static void functionDestroy(sqlite3 *db, FuncDef *p){ + FuncDestructor *pDestructor = p->pDestructor; + if( pDestructor ){ + pDestructor->nRef--; + if( pDestructor->nRef==0 ){ + pDestructor->xDestroy(pDestructor->pUserData); + sqlite3DbFree(db, pDestructor); + } + } +} + +/* +** Disconnect all sqlite3_vtab objects that belong to database connection +** db. This is called when db is being closed. +*/ +static void disconnectAllVtab(sqlite3 *db){ +#ifndef SQLITE_OMIT_VIRTUALTABLE + int i; + HashElem *p; + sqlite3BtreeEnterAll(db); + for(i=0; inDb; i++){ + Schema *pSchema = db->aDb[i].pSchema; + if( db->aDb[i].pSchema ){ + for(p=sqliteHashFirst(&pSchema->tblHash); p; p=sqliteHashNext(p)){ + Table *pTab = (Table *)sqliteHashData(p); + if( IsVirtual(pTab) ) sqlite3VtabDisconnect(db, pTab); + } + } + } + for(p=sqliteHashFirst(&db->aModule); p; p=sqliteHashNext(p)){ + Module *pMod = (Module *)sqliteHashData(p); + if( pMod->pEpoTab ){ + sqlite3VtabDisconnect(db, pMod->pEpoTab); + } + } + sqlite3VtabUnlockList(db); + sqlite3BtreeLeaveAll(db); +#else + UNUSED_PARAMETER(db); +#endif +} + +/* +** Return TRUE if database connection db has unfinalized prepared +** statements or unfinished sqlite3_backup objects. +*/ +static int connectionIsBusy(sqlite3 *db){ + int j; + assert( sqlite3_mutex_held(db->mutex) ); + if( db->pVdbe ) return 1; + for(j=0; jnDb; j++){ + Btree *pBt = db->aDb[j].pBt; + if( pBt && sqlite3BtreeIsInBackup(pBt) ) return 1; + } + return 0; +} + +/* +** Close an existing SQLite database +*/ +static int sqlite3Close(sqlite3 *db, int forceZombie){ + if( !db ){ + /* EVIDENCE-OF: R-63257-11740 Calling sqlite3_close() or + ** sqlite3_close_v2() with a NULL pointer argument is a harmless no-op. */ + return SQLITE_OK; + } + if( !sqlite3SafetyCheckSickOrOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + sqlite3_mutex_enter(db->mutex); + + /* Force xDisconnect calls on all virtual tables */ + disconnectAllVtab(db); + + /* If a transaction is open, the disconnectAllVtab() call above + ** will not have called the xDisconnect() method on any virtual + ** tables in the db->aVTrans[] array. The following sqlite3VtabRollback() + ** call will do so. We need to do this before the check for active + ** SQL statements below, as the v-table implementation may be storing + ** some prepared statements internally. + */ + sqlite3VtabRollback(db); + + /* Legacy behavior (sqlite3_close() behavior) is to return + ** SQLITE_BUSY if the connection can not be closed immediately. + */ + if( !forceZombie && connectionIsBusy(db) ){ + sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to close due to unfinalized " + "statements or unfinished backups"); + sqlite3_mutex_leave(db->mutex); + return SQLITE_BUSY; + } + +#ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ + /* Closing the handle. Fourth parameter is passed the value 2. */ + sqlite3GlobalConfig.xSqllog(sqlite3GlobalConfig.pSqllogArg, db, 0, 2); + } +#endif + + /* Convert the connection into a zombie and then close it. + */ + db->magic = SQLITE_MAGIC_ZOMBIE; + sqlite3LeaveMutexAndCloseZombie(db); + return SQLITE_OK; +} + +/* +** Two variations on the public interface for closing a database +** connection. The sqlite3_close() version returns SQLITE_BUSY and +** leaves the connection option if there are unfinalized prepared +** statements or unfinished sqlite3_backups. The sqlite3_close_v2() +** version forces the connection to become a zombie if there are +** unclosed resources, and arranges for deallocation when the last +** prepare statement or sqlite3_backup closes. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_close(sqlite3 *db){ return sqlite3Close(db,0); } +SQLITE_API int SQLITE_STDCALL sqlite3_close_v2(sqlite3 *db){ return sqlite3Close(db,1); } + + +/* +** Close the mutex on database connection db. +** +** Furthermore, if database connection db is a zombie (meaning that there +** has been a prior call to sqlite3_close(db) or sqlite3_close_v2(db)) and +** every sqlite3_stmt has now been finalized and every sqlite3_backup has +** finished, then free all resources. +*/ +SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ + HashElem *i; /* Hash table iterator */ + int j; + + /* If there are outstanding sqlite3_stmt or sqlite3_backup objects + ** or if the connection has not yet been closed by sqlite3_close_v2(), + ** then just leave the mutex and return. + */ + if( db->magic!=SQLITE_MAGIC_ZOMBIE || connectionIsBusy(db) ){ + sqlite3_mutex_leave(db->mutex); + return; + } + + /* If we reach this point, it means that the database connection has + ** closed all sqlite3_stmt and sqlite3_backup objects and has been + ** passed to sqlite3_close (meaning that it is a zombie). Therefore, + ** go ahead and free all resources. + */ + + /* If a transaction is open, roll it back. This also ensures that if + ** any database schemas have been modified by an uncommitted transaction + ** they are reset. And that the required b-tree mutex is held to make + ** the pager rollback and schema reset an atomic operation. */ + sqlite3RollbackAll(db, SQLITE_OK); + + /* Free any outstanding Savepoint structures. */ + sqlite3CloseSavepoints(db); + + /* Close all database connections */ + for(j=0; jnDb; j++){ + struct Db *pDb = &db->aDb[j]; + if( pDb->pBt ){ + sqlite3BtreeClose(pDb->pBt); + pDb->pBt = 0; + if( j!=1 ){ + pDb->pSchema = 0; + } + } + } + /* Clear the TEMP schema separately and last */ + if( db->aDb[1].pSchema ){ + sqlite3SchemaClear(db->aDb[1].pSchema); + } + sqlite3VtabUnlockList(db); + + /* Free up the array of auxiliary databases */ + sqlite3CollapseDatabaseArray(db); + assert( db->nDb<=2 ); + assert( db->aDb==db->aDbStatic ); + + /* Tell the code in notify.c that the connection no longer holds any + ** locks and does not require any further unlock-notify callbacks. + */ + sqlite3ConnectionClosed(db); + + for(j=0; jaFunc.a); j++){ + FuncDef *pNext, *pHash, *p; + for(p=db->aFunc.a[j]; p; p=pHash){ + pHash = p->pHash; + while( p ){ + functionDestroy(db, p); + pNext = p->pNext; + sqlite3DbFree(db, p); + p = pNext; + } + } + } + for(i=sqliteHashFirst(&db->aCollSeq); i; i=sqliteHashNext(i)){ + CollSeq *pColl = (CollSeq *)sqliteHashData(i); + /* Invoke any destructors registered for collation sequence user data. */ + for(j=0; j<3; j++){ + if( pColl[j].xDel ){ + pColl[j].xDel(pColl[j].pUser); + } + } + sqlite3DbFree(db, pColl); + } + sqlite3HashClear(&db->aCollSeq); +#ifndef SQLITE_OMIT_VIRTUALTABLE + for(i=sqliteHashFirst(&db->aModule); i; i=sqliteHashNext(i)){ + Module *pMod = (Module *)sqliteHashData(i); + if( pMod->xDestroy ){ + pMod->xDestroy(pMod->pAux); + } + sqlite3VtabEponymousTableClear(db, pMod); + sqlite3DbFree(db, pMod); + } + sqlite3HashClear(&db->aModule); +#endif + + sqlite3Error(db, SQLITE_OK); /* Deallocates any cached error strings. */ + sqlite3ValueFree(db->pErr); + sqlite3CloseExtensions(db); +#if SQLITE_USER_AUTHENTICATION + sqlite3_free(db->auth.zAuthUser); + sqlite3_free(db->auth.zAuthPW); +#endif + + db->magic = SQLITE_MAGIC_ERROR; + + /* The temp-database schema is allocated differently from the other schema + ** objects (using sqliteMalloc() directly, instead of sqlite3BtreeSchema()). + ** So it needs to be freed here. Todo: Why not roll the temp schema into + ** the same sqliteMalloc() as the one that allocates the database + ** structure? + */ + sqlite3DbFree(db, db->aDb[1].pSchema); + sqlite3_mutex_leave(db->mutex); + db->magic = SQLITE_MAGIC_CLOSED; + sqlite3_mutex_free(db->mutex); + assert( db->lookaside.nOut==0 ); /* Fails on a lookaside memory leak */ + if( db->lookaside.bMalloced ){ + sqlite3_free(db->lookaside.pStart); + } + sqlite3_free(db); +} + +/* +** Rollback all database files. If tripCode is not SQLITE_OK, then +** any write cursors are invalidated ("tripped" - as in "tripping a circuit +** breaker") and made to return tripCode if there are any further +** attempts to use that cursor. Read cursors remain open and valid +** but are "saved" in case the table pages are moved around. +*/ +SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ + int i; + int inTrans = 0; + int schemaChange; + assert( sqlite3_mutex_held(db->mutex) ); + sqlite3BeginBenignMalloc(); + + /* Obtain all b-tree mutexes before making any calls to BtreeRollback(). + ** This is important in case the transaction being rolled back has + ** modified the database schema. If the b-tree mutexes are not taken + ** here, then another shared-cache connection might sneak in between + ** the database rollback and schema reset, which can cause false + ** corruption reports in some cases. */ + sqlite3BtreeEnterAll(db); + schemaChange = (db->flags & SQLITE_InternChanges)!=0 && db->init.busy==0; + + for(i=0; inDb; i++){ + Btree *p = db->aDb[i].pBt; + if( p ){ + if( sqlite3BtreeIsInTrans(p) ){ + inTrans = 1; + } + sqlite3BtreeRollback(p, tripCode, !schemaChange); + } + } + sqlite3VtabRollback(db); + sqlite3EndBenignMalloc(); + + if( (db->flags&SQLITE_InternChanges)!=0 && db->init.busy==0 ){ + sqlite3ExpirePreparedStatements(db); + sqlite3ResetAllSchemasOfConnection(db); + } + sqlite3BtreeLeaveAll(db); + + /* Any deferred constraint violations have now been resolved. */ + db->nDeferredCons = 0; + db->nDeferredImmCons = 0; + db->flags &= ~SQLITE_DeferFKs; + + /* If one has been configured, invoke the rollback-hook callback */ + if( db->xRollbackCallback && (inTrans || !db->autoCommit) ){ + db->xRollbackCallback(db->pRollbackArg); + } +} + +/* +** Return a static string containing the name corresponding to the error code +** specified in the argument. +*/ +#if defined(SQLITE_NEED_ERR_NAME) +SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ + const char *zName = 0; + int i, origRc = rc; + for(i=0; i<2 && zName==0; i++, rc &= 0xff){ + switch( rc ){ + case SQLITE_OK: zName = "SQLITE_OK"; break; + case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; + case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; + case SQLITE_PERM: zName = "SQLITE_PERM"; break; + case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; + case SQLITE_ABORT_ROLLBACK: zName = "SQLITE_ABORT_ROLLBACK"; break; + case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; + case SQLITE_BUSY_RECOVERY: zName = "SQLITE_BUSY_RECOVERY"; break; + case SQLITE_BUSY_SNAPSHOT: zName = "SQLITE_BUSY_SNAPSHOT"; break; + case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; + case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; + case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; + case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; + case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; + case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; + case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; + case SQLITE_READONLY_DBMOVED: zName = "SQLITE_READONLY_DBMOVED"; break; + case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; + case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; + case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; + case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; + case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; + case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; + case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; + case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; + case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; + case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; + case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; + case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; + case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; + case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; + case SQLITE_IOERR_CHECKRESERVEDLOCK: + zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; + case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; + case SQLITE_IOERR_CLOSE: zName = "SQLITE_IOERR_CLOSE"; break; + case SQLITE_IOERR_DIR_CLOSE: zName = "SQLITE_IOERR_DIR_CLOSE"; break; + case SQLITE_IOERR_SHMOPEN: zName = "SQLITE_IOERR_SHMOPEN"; break; + case SQLITE_IOERR_SHMSIZE: zName = "SQLITE_IOERR_SHMSIZE"; break; + case SQLITE_IOERR_SHMLOCK: zName = "SQLITE_IOERR_SHMLOCK"; break; + case SQLITE_IOERR_SHMMAP: zName = "SQLITE_IOERR_SHMMAP"; break; + case SQLITE_IOERR_SEEK: zName = "SQLITE_IOERR_SEEK"; break; + case SQLITE_IOERR_DELETE_NOENT: zName = "SQLITE_IOERR_DELETE_NOENT";break; + case SQLITE_IOERR_MMAP: zName = "SQLITE_IOERR_MMAP"; break; + case SQLITE_IOERR_GETTEMPPATH: zName = "SQLITE_IOERR_GETTEMPPATH"; break; + case SQLITE_IOERR_CONVPATH: zName = "SQLITE_IOERR_CONVPATH"; break; + case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; + case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; + case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; + case SQLITE_FULL: zName = "SQLITE_FULL"; break; + case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; + case SQLITE_CANTOPEN_NOTEMPDIR: zName = "SQLITE_CANTOPEN_NOTEMPDIR";break; + case SQLITE_CANTOPEN_ISDIR: zName = "SQLITE_CANTOPEN_ISDIR"; break; + case SQLITE_CANTOPEN_FULLPATH: zName = "SQLITE_CANTOPEN_FULLPATH"; break; + case SQLITE_CANTOPEN_CONVPATH: zName = "SQLITE_CANTOPEN_CONVPATH"; break; + case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; + case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; + case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; + case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break; + case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; + case SQLITE_CONSTRAINT_UNIQUE: zName = "SQLITE_CONSTRAINT_UNIQUE"; break; + case SQLITE_CONSTRAINT_TRIGGER: zName = "SQLITE_CONSTRAINT_TRIGGER";break; + case SQLITE_CONSTRAINT_FOREIGNKEY: + zName = "SQLITE_CONSTRAINT_FOREIGNKEY"; break; + case SQLITE_CONSTRAINT_CHECK: zName = "SQLITE_CONSTRAINT_CHECK"; break; + case SQLITE_CONSTRAINT_PRIMARYKEY: + zName = "SQLITE_CONSTRAINT_PRIMARYKEY"; break; + case SQLITE_CONSTRAINT_NOTNULL: zName = "SQLITE_CONSTRAINT_NOTNULL";break; + case SQLITE_CONSTRAINT_COMMITHOOK: + zName = "SQLITE_CONSTRAINT_COMMITHOOK"; break; + case SQLITE_CONSTRAINT_VTAB: zName = "SQLITE_CONSTRAINT_VTAB"; break; + case SQLITE_CONSTRAINT_FUNCTION: + zName = "SQLITE_CONSTRAINT_FUNCTION"; break; + case SQLITE_CONSTRAINT_ROWID: zName = "SQLITE_CONSTRAINT_ROWID"; break; + case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; + case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; + case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; + case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; + case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; + case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; + case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; + case SQLITE_ROW: zName = "SQLITE_ROW"; break; + case SQLITE_NOTICE: zName = "SQLITE_NOTICE"; break; + case SQLITE_NOTICE_RECOVER_WAL: zName = "SQLITE_NOTICE_RECOVER_WAL";break; + case SQLITE_NOTICE_RECOVER_ROLLBACK: + zName = "SQLITE_NOTICE_RECOVER_ROLLBACK"; break; + case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; + case SQLITE_WARNING_AUTOINDEX: zName = "SQLITE_WARNING_AUTOINDEX"; break; + case SQLITE_DONE: zName = "SQLITE_DONE"; break; + } + } + if( zName==0 ){ + static char zBuf[50]; + sqlite3_snprintf(sizeof(zBuf), zBuf, "SQLITE_UNKNOWN(%d)", origRc); + zName = zBuf; + } + return zName; +} +#endif + +/* +** Return a static string that describes the kind of error specified in the +** argument. +*/ +SQLITE_PRIVATE const char *sqlite3ErrStr(int rc){ + static const char* const aMsg[] = { + /* SQLITE_OK */ "not an error", + /* SQLITE_ERROR */ "SQL logic error or missing database", + /* SQLITE_INTERNAL */ 0, + /* SQLITE_PERM */ "access permission denied", + /* SQLITE_ABORT */ "callback requested query abort", + /* SQLITE_BUSY */ "database is locked", + /* SQLITE_LOCKED */ "database table is locked", + /* SQLITE_NOMEM */ "out of memory", + /* SQLITE_READONLY */ "attempt to write a readonly database", + /* SQLITE_INTERRUPT */ "interrupted", + /* SQLITE_IOERR */ "disk I/O error", + /* SQLITE_CORRUPT */ "database disk image is malformed", + /* SQLITE_NOTFOUND */ "unknown operation", + /* SQLITE_FULL */ "database or disk is full", + /* SQLITE_CANTOPEN */ "unable to open database file", + /* SQLITE_PROTOCOL */ "locking protocol", + /* SQLITE_EMPTY */ "table contains no data", + /* SQLITE_SCHEMA */ "database schema has changed", + /* SQLITE_TOOBIG */ "string or blob too big", + /* SQLITE_CONSTRAINT */ "constraint failed", + /* SQLITE_MISMATCH */ "datatype mismatch", + /* SQLITE_MISUSE */ "library routine called out of sequence", + /* SQLITE_NOLFS */ "large file support is disabled", + /* SQLITE_AUTH */ "authorization denied", + /* SQLITE_FORMAT */ "auxiliary database format error", + /* SQLITE_RANGE */ "bind or column index out of range", + /* SQLITE_NOTADB */ "file is encrypted or is not a database", + }; + const char *zErr = "unknown error"; + switch( rc ){ + case SQLITE_ABORT_ROLLBACK: { + zErr = "abort due to ROLLBACK"; + break; + } + default: { + rc &= 0xff; + if( ALWAYS(rc>=0) && rcbusyTimeout; + int delay, prior; + + assert( count>=0 ); + if( count < NDELAY ){ + delay = delays[count]; + prior = totals[count]; + }else{ + delay = delays[NDELAY-1]; + prior = totals[NDELAY-1] + delay*(count-(NDELAY-1)); + } + if( prior + delay > timeout ){ + delay = timeout - prior; + if( delay<=0 ) return 0; + } + sqlite3OsSleep(db->pVfs, delay*1000); + return 1; +#else + sqlite3 *db = (sqlite3 *)ptr; + int timeout = ((sqlite3 *)ptr)->busyTimeout; + if( (count+1)*1000 > timeout ){ + return 0; + } + sqlite3OsSleep(db->pVfs, 1000000); + return 1; +#endif +} + +/* +** Invoke the given busy handler. +** +** This routine is called when an operation failed with a lock. +** If this routine returns non-zero, the lock is retried. If it +** returns 0, the operation aborts with an SQLITE_BUSY error. +*/ +SQLITE_PRIVATE int sqlite3InvokeBusyHandler(BusyHandler *p){ + int rc; + if( NEVER(p==0) || p->xFunc==0 || p->nBusy<0 ) return 0; + rc = p->xFunc(p->pArg, p->nBusy); + if( rc==0 ){ + p->nBusy = -1; + }else{ + p->nBusy++; + } + return rc; +} + +/* +** This routine sets the busy callback for an Sqlite database to the +** given callback function with the given argument. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_busy_handler( + sqlite3 *db, + int (*xBusy)(void*,int), + void *pArg +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + db->busyHandler.xFunc = xBusy; + db->busyHandler.pArg = pArg; + db->busyHandler.nBusy = 0; + db->busyTimeout = 0; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK +/* +** This routine sets the progress callback for an Sqlite database to the +** given callback function with the given argument. The progress callback will +** be invoked every nOps opcodes. +*/ +SQLITE_API void SQLITE_STDCALL sqlite3_progress_handler( + sqlite3 *db, + int nOps, + int (*xProgress)(void*), + void *pArg +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( nOps>0 ){ + db->xProgress = xProgress; + db->nProgressOps = (unsigned)nOps; + db->pProgressArg = pArg; + }else{ + db->xProgress = 0; + db->nProgressOps = 0; + db->pProgressArg = 0; + } + sqlite3_mutex_leave(db->mutex); +} +#endif + + +/* +** This routine installs a default busy handler that waits for the +** specified number of milliseconds before returning 0. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_busy_timeout(sqlite3 *db, int ms){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + if( ms>0 ){ + sqlite3_busy_handler(db, sqliteDefaultBusyCallback, (void*)db); + db->busyTimeout = ms; + }else{ + sqlite3_busy_handler(db, 0, 0); + } + return SQLITE_OK; +} + +/* +** Cause any pending operation to stop at its earliest opportunity. +*/ +SQLITE_API void SQLITE_STDCALL sqlite3_interrupt(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif + db->u1.isInterrupted = 1; +} + + +/* +** This function is exactly the same as sqlite3_create_function(), except +** that it is designed to be called by internal code. The difference is +** that if a malloc() fails in sqlite3_create_function(), an error code +** is returned and the mallocFailed flag cleared. +*/ +SQLITE_PRIVATE int sqlite3CreateFunc( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int enc, + void *pUserData, + void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*), + FuncDestructor *pDestructor +){ + FuncDef *p; + int nName; + int extraFlags; + + assert( sqlite3_mutex_held(db->mutex) ); + if( zFunctionName==0 || + (xFunc && (xFinal || xStep)) || + (!xFunc && (xFinal && !xStep)) || + (!xFunc && (!xFinal && xStep)) || + (nArg<-1 || nArg>SQLITE_MAX_FUNCTION_ARG) || + (255<(nName = sqlite3Strlen30( zFunctionName))) ){ + return SQLITE_MISUSE_BKPT; + } + + assert( SQLITE_FUNC_CONSTANT==SQLITE_DETERMINISTIC ); + extraFlags = enc & SQLITE_DETERMINISTIC; + enc &= (SQLITE_FUNC_ENCMASK|SQLITE_ANY); + +#ifndef SQLITE_OMIT_UTF16 + /* If SQLITE_UTF16 is specified as the encoding type, transform this + ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the + ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally. + ** + ** If SQLITE_ANY is specified, add three versions of the function + ** to the hash table. + */ + if( enc==SQLITE_UTF16 ){ + enc = SQLITE_UTF16NATIVE; + }else if( enc==SQLITE_ANY ){ + int rc; + rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF8|extraFlags, + pUserData, xFunc, xStep, xFinal, pDestructor); + if( rc==SQLITE_OK ){ + rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF16LE|extraFlags, + pUserData, xFunc, xStep, xFinal, pDestructor); + } + if( rc!=SQLITE_OK ){ + return rc; + } + enc = SQLITE_UTF16BE; + } +#else + enc = SQLITE_UTF8; +#endif + + /* Check if an existing function is being overridden or deleted. If so, + ** and there are active VMs, then return SQLITE_BUSY. If a function + ** is being overridden/deleted but there are no active VMs, allow the + ** operation to continue but invalidate all precompiled statements. + */ + p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 0); + if( p && (p->funcFlags & SQLITE_FUNC_ENCMASK)==enc && p->nArg==nArg ){ + if( db->nVdbeActive ){ + sqlite3ErrorWithMsg(db, SQLITE_BUSY, + "unable to delete/modify user-function due to active statements"); + assert( !db->mallocFailed ); + return SQLITE_BUSY; + }else{ + sqlite3ExpirePreparedStatements(db); + } + } + + p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 1); + assert(p || db->mallocFailed); + if( !p ){ + return SQLITE_NOMEM; + } + + /* If an older version of the function with a configured destructor is + ** being replaced invoke the destructor function here. */ + functionDestroy(db, p); + + if( pDestructor ){ + pDestructor->nRef++; + } + p->pDestructor = pDestructor; + p->funcFlags = (p->funcFlags & SQLITE_FUNC_ENCMASK) | extraFlags; + testcase( p->funcFlags & SQLITE_DETERMINISTIC ); + p->xFunc = xFunc; + p->xStep = xStep; + p->xFinalize = xFinal; + p->pUserData = pUserData; + p->nArg = (u16)nArg; + return SQLITE_OK; +} + +/* +** Create new user functions. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_create_function( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*) +){ + return sqlite3_create_function_v2(db, zFunc, nArg, enc, p, xFunc, xStep, + xFinal, 0); +} + +SQLITE_API int SQLITE_STDCALL sqlite3_create_function_v2( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*), + void (*xDestroy)(void *) +){ + int rc = SQLITE_ERROR; + FuncDestructor *pArg = 0; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( xDestroy ){ + pArg = (FuncDestructor *)sqlite3DbMallocZero(db, sizeof(FuncDestructor)); + if( !pArg ){ + xDestroy(p); + goto out; + } + pArg->xDestroy = xDestroy; + pArg->pUserData = p; + } + rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xFunc, xStep, xFinal, pArg); + if( pArg && pArg->nRef==0 ){ + assert( rc!=SQLITE_OK ); + xDestroy(p); + sqlite3DbFree(db, pArg); + } + + out: + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API int SQLITE_STDCALL sqlite3_create_function16( + sqlite3 *db, + const void *zFunctionName, + int nArg, + int eTextRep, + void *p, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*) +){ + int rc; + char *zFunc8; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zFunctionName==0 ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); + zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE); + rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xFunc, xStep, xFinal,0); + sqlite3DbFree(db, zFunc8); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} +#endif + + +/* +** Declare that a function has been overloaded by a virtual table. +** +** If the function already exists as a regular global function, then +** this routine is a no-op. If the function does not exist, then create +** a new one that always throws a run-time error. +** +** When virtual tables intend to provide an overloaded function, they +** should call this routine to make sure the global function exists. +** A global function must exist in order for name resolution to work +** properly. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_overload_function( + sqlite3 *db, + const char *zName, + int nArg +){ + int nName = sqlite3Strlen30(zName); + int rc = SQLITE_OK; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 || nArg<-2 ){ + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( sqlite3FindFunction(db, zName, nName, nArg, SQLITE_UTF8, 0)==0 ){ + rc = sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8, + 0, sqlite3InvalidFunction, 0, 0, 0); + } + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +#ifndef SQLITE_OMIT_TRACE +/* +** Register a trace function. The pArg from the previously registered trace +** is returned. +** +** A NULL trace function means that no tracing is executes. A non-NULL +** trace is a pointer to a function that is invoked at the start of each +** SQL statement. +*/ +SQLITE_API void *SQLITE_STDCALL sqlite3_trace(sqlite3 *db, void (*xTrace)(void*,const char*), void *pArg){ + void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pOld = db->pTraceArg; + db->xTrace = xTrace; + db->pTraceArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pOld; +} +/* +** Register a profile function. The pArg from the previously registered +** profile function is returned. +** +** A NULL profile function means that no profiling is executes. A non-NULL +** profile is a pointer to a function that is invoked at the conclusion of +** each SQL statement that is run. +*/ +SQLITE_API void *SQLITE_STDCALL sqlite3_profile( + sqlite3 *db, + void (*xProfile)(void*,const char*,sqlite_uint64), + void *pArg +){ + void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pOld = db->pProfileArg; + db->xProfile = xProfile; + db->pProfileArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pOld; +} +#endif /* SQLITE_OMIT_TRACE */ + +/* +** Register a function to be invoked when a transaction commits. +** If the invoked function returns non-zero, then the commit becomes a +** rollback. +*/ +SQLITE_API void *SQLITE_STDCALL sqlite3_commit_hook( + sqlite3 *db, /* Attach the hook to this database */ + int (*xCallback)(void*), /* Function to invoke on each commit */ + void *pArg /* Argument to the function */ +){ + void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pOld = db->pCommitArg; + db->xCommitCallback = xCallback; + db->pCommitArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pOld; +} + +/* +** Register a callback to be invoked each time a row is updated, +** inserted or deleted using this database connection. +*/ +SQLITE_API void *SQLITE_STDCALL sqlite3_update_hook( + sqlite3 *db, /* Attach the hook to this database */ + void (*xCallback)(void*,int,char const *,char const *,sqlite_int64), + void *pArg /* Argument to the function */ +){ + void *pRet; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pRet = db->pUpdateArg; + db->xUpdateCallback = xCallback; + db->pUpdateArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +} + +/* +** Register a callback to be invoked each time a transaction is rolled +** back by this database connection. +*/ +SQLITE_API void *SQLITE_STDCALL sqlite3_rollback_hook( + sqlite3 *db, /* Attach the hook to this database */ + void (*xCallback)(void*), /* Callback function */ + void *pArg /* Argument to the function */ +){ + void *pRet; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pRet = db->pRollbackArg; + db->xRollbackCallback = xCallback; + db->pRollbackArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +} + +#ifndef SQLITE_OMIT_WAL +/* +** The sqlite3_wal_hook() callback registered by sqlite3_wal_autocheckpoint(). +** Invoke sqlite3_wal_checkpoint if the number of frames in the log file +** is greater than sqlite3.pWalArg cast to an integer (the value configured by +** wal_autocheckpoint()). +*/ +SQLITE_PRIVATE int sqlite3WalDefaultHook( + void *pClientData, /* Argument */ + sqlite3 *db, /* Connection */ + const char *zDb, /* Database */ + int nFrame /* Size of WAL */ +){ + if( nFrame>=SQLITE_PTR_TO_INT(pClientData) ){ + sqlite3BeginBenignMalloc(); + sqlite3_wal_checkpoint(db, zDb); + sqlite3EndBenignMalloc(); + } + return SQLITE_OK; +} +#endif /* SQLITE_OMIT_WAL */ + +/* +** Configure an sqlite3_wal_hook() callback to automatically checkpoint +** a database after committing a transaction if there are nFrame or +** more frames in the log file. Passing zero or a negative value as the +** nFrame parameter disables automatic checkpoints entirely. +** +** The callback registered by this function replaces any existing callback +** registered using sqlite3_wal_hook(). Likewise, registering a callback +** using sqlite3_wal_hook() disables the automatic checkpoint mechanism +** configured by this function. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ +#ifdef SQLITE_OMIT_WAL + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(nFrame); +#else +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + if( nFrame>0 ){ + sqlite3_wal_hook(db, sqlite3WalDefaultHook, SQLITE_INT_TO_PTR(nFrame)); + }else{ + sqlite3_wal_hook(db, 0, 0); + } +#endif + return SQLITE_OK; +} + +/* +** Register a callback to be invoked each time a transaction is written +** into the write-ahead-log by this database connection. +*/ +SQLITE_API void *SQLITE_STDCALL sqlite3_wal_hook( + sqlite3 *db, /* Attach the hook to this db handle */ + int(*xCallback)(void *, sqlite3*, const char*, int), + void *pArg /* First argument passed to xCallback() */ +){ +#ifndef SQLITE_OMIT_WAL + void *pRet; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pRet = db->pWalArg; + db->xWalCallback = xCallback; + db->pWalArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +#else + return 0; +#endif +} + +/* +** Checkpoint database zDb. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint_v2( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of attached database (or NULL) */ + int eMode, /* SQLITE_CHECKPOINT_* value */ + int *pnLog, /* OUT: Size of WAL log in frames */ + int *pnCkpt /* OUT: Total number of frames checkpointed */ +){ +#ifdef SQLITE_OMIT_WAL + return SQLITE_OK; +#else + int rc; /* Return code */ + int iDb = SQLITE_MAX_ATTACHED; /* sqlite3.aDb[] index of db to checkpoint */ + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + + /* Initialize the output variables to -1 in case an error occurs. */ + if( pnLog ) *pnLog = -1; + if( pnCkpt ) *pnCkpt = -1; + + assert( SQLITE_CHECKPOINT_PASSIVE==0 ); + assert( SQLITE_CHECKPOINT_FULL==1 ); + assert( SQLITE_CHECKPOINT_RESTART==2 ); + assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); + if( eModeSQLITE_CHECKPOINT_TRUNCATE ){ + /* EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint + ** mode: */ + return SQLITE_MISUSE; + } + + sqlite3_mutex_enter(db->mutex); + if( zDb && zDb[0] ){ + iDb = sqlite3FindDbName(db, zDb); + } + if( iDb<0 ){ + rc = SQLITE_ERROR; + sqlite3ErrorWithMsg(db, SQLITE_ERROR, "unknown database: %s", zDb); + }else{ + db->busyHandler.nBusy = 0; + rc = sqlite3Checkpoint(db, iDb, eMode, pnLog, pnCkpt); + sqlite3Error(db, rc); + } + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +#endif +} + + +/* +** Checkpoint database zDb. If zDb is NULL, or if the buffer zDb points +** to contains a zero-length string, all attached databases are +** checkpointed. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ + /* EVIDENCE-OF: R-41613-20553 The sqlite3_wal_checkpoint(D,X) is equivalent to + ** sqlite3_wal_checkpoint_v2(D,X,SQLITE_CHECKPOINT_PASSIVE,0,0). */ + return sqlite3_wal_checkpoint_v2(db,zDb,SQLITE_CHECKPOINT_PASSIVE,0,0); +} + +#ifndef SQLITE_OMIT_WAL +/* +** Run a checkpoint on database iDb. This is a no-op if database iDb is +** not currently open in WAL mode. +** +** If a transaction is open on the database being checkpointed, this +** function returns SQLITE_LOCKED and a checkpoint is not attempted. If +** an error occurs while running the checkpoint, an SQLite error code is +** returned (i.e. SQLITE_IOERR). Otherwise, SQLITE_OK. +** +** The mutex on database handle db should be held by the caller. The mutex +** associated with the specific b-tree being checkpointed is taken by +** this function while the checkpoint is running. +** +** If iDb is passed SQLITE_MAX_ATTACHED, then all attached databases are +** checkpointed. If an error is encountered it is returned immediately - +** no attempt is made to checkpoint any remaining databases. +** +** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART. +*/ +SQLITE_PRIVATE int sqlite3Checkpoint(sqlite3 *db, int iDb, int eMode, int *pnLog, int *pnCkpt){ + int rc = SQLITE_OK; /* Return code */ + int i; /* Used to iterate through attached dbs */ + int bBusy = 0; /* True if SQLITE_BUSY has been encountered */ + + assert( sqlite3_mutex_held(db->mutex) ); + assert( !pnLog || *pnLog==-1 ); + assert( !pnCkpt || *pnCkpt==-1 ); + + for(i=0; inDb && rc==SQLITE_OK; i++){ + if( i==iDb || iDb==SQLITE_MAX_ATTACHED ){ + rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt, eMode, pnLog, pnCkpt); + pnLog = 0; + pnCkpt = 0; + if( rc==SQLITE_BUSY ){ + bBusy = 1; + rc = SQLITE_OK; + } + } + } + + return (rc==SQLITE_OK && bBusy) ? SQLITE_BUSY : rc; +} +#endif /* SQLITE_OMIT_WAL */ + +/* +** This function returns true if main-memory should be used instead of +** a temporary file for transient pager files and statement journals. +** The value returned depends on the value of db->temp_store (runtime +** parameter) and the compile time value of SQLITE_TEMP_STORE. The +** following table describes the relationship between these two values +** and this functions return value. +** +** SQLITE_TEMP_STORE db->temp_store Location of temporary database +** ----------------- -------------- ------------------------------ +** 0 any file (return 0) +** 1 1 file (return 0) +** 1 2 memory (return 1) +** 1 0 file (return 0) +** 2 1 file (return 0) +** 2 2 memory (return 1) +** 2 0 memory (return 1) +** 3 any memory (return 1) +*/ +SQLITE_PRIVATE int sqlite3TempInMemory(const sqlite3 *db){ +#if SQLITE_TEMP_STORE==1 + return ( db->temp_store==2 ); +#endif +#if SQLITE_TEMP_STORE==2 + return ( db->temp_store!=1 ); +#endif +#if SQLITE_TEMP_STORE==3 + UNUSED_PARAMETER(db); + return 1; +#endif +#if SQLITE_TEMP_STORE<1 || SQLITE_TEMP_STORE>3 + UNUSED_PARAMETER(db); + return 0; +#endif +} + +/* +** Return UTF-8 encoded English language explanation of the most recent +** error. +*/ +SQLITE_API const char *SQLITE_STDCALL sqlite3_errmsg(sqlite3 *db){ + const char *z; + if( !db ){ + return sqlite3ErrStr(SQLITE_NOMEM); + } + if( !sqlite3SafetyCheckSickOrOk(db) ){ + return sqlite3ErrStr(SQLITE_MISUSE_BKPT); + } + sqlite3_mutex_enter(db->mutex); + if( db->mallocFailed ){ + z = sqlite3ErrStr(SQLITE_NOMEM); + }else{ + testcase( db->pErr==0 ); + z = (char*)sqlite3_value_text(db->pErr); + assert( !db->mallocFailed ); + if( z==0 ){ + z = sqlite3ErrStr(db->errCode); + } + } + sqlite3_mutex_leave(db->mutex); + return z; +} + +#ifndef SQLITE_OMIT_UTF16 +/* +** Return UTF-16 encoded English language explanation of the most recent +** error. +*/ +SQLITE_API const void *SQLITE_STDCALL sqlite3_errmsg16(sqlite3 *db){ + static const u16 outOfMem[] = { + 'o', 'u', 't', ' ', 'o', 'f', ' ', 'm', 'e', 'm', 'o', 'r', 'y', 0 + }; + static const u16 misuse[] = { + 'l', 'i', 'b', 'r', 'a', 'r', 'y', ' ', + 'r', 'o', 'u', 't', 'i', 'n', 'e', ' ', + 'c', 'a', 'l', 'l', 'e', 'd', ' ', + 'o', 'u', 't', ' ', + 'o', 'f', ' ', + 's', 'e', 'q', 'u', 'e', 'n', 'c', 'e', 0 + }; + + const void *z; + if( !db ){ + return (void *)outOfMem; + } + if( !sqlite3SafetyCheckSickOrOk(db) ){ + return (void *)misuse; + } + sqlite3_mutex_enter(db->mutex); + if( db->mallocFailed ){ + z = (void *)outOfMem; + }else{ + z = sqlite3_value_text16(db->pErr); + if( z==0 ){ + sqlite3ErrorWithMsg(db, db->errCode, sqlite3ErrStr(db->errCode)); + z = sqlite3_value_text16(db->pErr); + } + /* A malloc() may have failed within the call to sqlite3_value_text16() + ** above. If this is the case, then the db->mallocFailed flag needs to + ** be cleared before returning. Do this directly, instead of via + ** sqlite3ApiExit(), to avoid setting the database handle error message. + */ + db->mallocFailed = 0; + } + sqlite3_mutex_leave(db->mutex); + return z; +} +#endif /* SQLITE_OMIT_UTF16 */ + +/* +** Return the most recent error code generated by an SQLite routine. If NULL is +** passed to this function, we assume a malloc() failed during sqlite3_open(). +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_errcode(sqlite3 *db){ + if( db && !sqlite3SafetyCheckSickOrOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + if( !db || db->mallocFailed ){ + return SQLITE_NOMEM; + } + return db->errCode & db->errMask; +} +SQLITE_API int SQLITE_STDCALL sqlite3_extended_errcode(sqlite3 *db){ + if( db && !sqlite3SafetyCheckSickOrOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + if( !db || db->mallocFailed ){ + return SQLITE_NOMEM; + } + return db->errCode; +} + +/* +** Return a string that describes the kind of error specified in the +** argument. For now, this simply calls the internal sqlite3ErrStr() +** function. +*/ +SQLITE_API const char *SQLITE_STDCALL sqlite3_errstr(int rc){ + return sqlite3ErrStr(rc); +} + +/* +** Create a new collating function for database "db". The name is zName +** and the encoding is enc. +*/ +static int createCollation( + sqlite3* db, + const char *zName, + u8 enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*), + void(*xDel)(void*) +){ + CollSeq *pColl; + int enc2; + + assert( sqlite3_mutex_held(db->mutex) ); + + /* If SQLITE_UTF16 is specified as the encoding type, transform this + ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the + ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally. + */ + enc2 = enc; + testcase( enc2==SQLITE_UTF16 ); + testcase( enc2==SQLITE_UTF16_ALIGNED ); + if( enc2==SQLITE_UTF16 || enc2==SQLITE_UTF16_ALIGNED ){ + enc2 = SQLITE_UTF16NATIVE; + } + if( enc2SQLITE_UTF16BE ){ + return SQLITE_MISUSE_BKPT; + } + + /* Check if this call is removing or replacing an existing collation + ** sequence. If so, and there are active VMs, return busy. If there + ** are no active VMs, invalidate any pre-compiled statements. + */ + pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 0); + if( pColl && pColl->xCmp ){ + if( db->nVdbeActive ){ + sqlite3ErrorWithMsg(db, SQLITE_BUSY, + "unable to delete/modify collation sequence due to active statements"); + return SQLITE_BUSY; + } + sqlite3ExpirePreparedStatements(db); + + /* If collation sequence pColl was created directly by a call to + ** sqlite3_create_collation, and not generated by synthCollSeq(), + ** then any copies made by synthCollSeq() need to be invalidated. + ** Also, collation destructor - CollSeq.xDel() - function may need + ** to be called. + */ + if( (pColl->enc & ~SQLITE_UTF16_ALIGNED)==enc2 ){ + CollSeq *aColl = sqlite3HashFind(&db->aCollSeq, zName); + int j; + for(j=0; j<3; j++){ + CollSeq *p = &aColl[j]; + if( p->enc==pColl->enc ){ + if( p->xDel ){ + p->xDel(p->pUser); + } + p->xCmp = 0; + } + } + } + } + + pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 1); + if( pColl==0 ) return SQLITE_NOMEM; + pColl->xCmp = xCompare; + pColl->pUser = pCtx; + pColl->xDel = xDel; + pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED)); + sqlite3Error(db, SQLITE_OK); + return SQLITE_OK; +} + + +/* +** This array defines hard upper bounds on limit values. The +** initializer must be kept in sync with the SQLITE_LIMIT_* +** #defines in sqlite3.h. +*/ +static const int aHardLimit[] = { + SQLITE_MAX_LENGTH, + SQLITE_MAX_SQL_LENGTH, + SQLITE_MAX_COLUMN, + SQLITE_MAX_EXPR_DEPTH, + SQLITE_MAX_COMPOUND_SELECT, + SQLITE_MAX_VDBE_OP, + SQLITE_MAX_FUNCTION_ARG, + SQLITE_MAX_ATTACHED, + SQLITE_MAX_LIKE_PATTERN_LENGTH, + SQLITE_MAX_VARIABLE_NUMBER, /* IMP: R-38091-32352 */ + SQLITE_MAX_TRIGGER_DEPTH, + SQLITE_MAX_WORKER_THREADS, +}; + +/* +** Make sure the hard limits are set to reasonable values +*/ +#if SQLITE_MAX_LENGTH<100 +# error SQLITE_MAX_LENGTH must be at least 100 +#endif +#if SQLITE_MAX_SQL_LENGTH<100 +# error SQLITE_MAX_SQL_LENGTH must be at least 100 +#endif +#if SQLITE_MAX_SQL_LENGTH>SQLITE_MAX_LENGTH +# error SQLITE_MAX_SQL_LENGTH must not be greater than SQLITE_MAX_LENGTH +#endif +#if SQLITE_MAX_COMPOUND_SELECT<2 +# error SQLITE_MAX_COMPOUND_SELECT must be at least 2 +#endif +#if SQLITE_MAX_VDBE_OP<40 +# error SQLITE_MAX_VDBE_OP must be at least 40 +#endif +#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>1000 +# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 1000 +#endif +#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>125 +# error SQLITE_MAX_ATTACHED must be between 0 and 125 +#endif +#if SQLITE_MAX_LIKE_PATTERN_LENGTH<1 +# error SQLITE_MAX_LIKE_PATTERN_LENGTH must be at least 1 +#endif +#if SQLITE_MAX_COLUMN>32767 +# error SQLITE_MAX_COLUMN must not exceed 32767 +#endif +#if SQLITE_MAX_TRIGGER_DEPTH<1 +# error SQLITE_MAX_TRIGGER_DEPTH must be at least 1 +#endif +#if SQLITE_MAX_WORKER_THREADS<0 || SQLITE_MAX_WORKER_THREADS>50 +# error SQLITE_MAX_WORKER_THREADS must be between 0 and 50 +#endif + + +/* +** Change the value of a limit. Report the old value. +** If an invalid limit index is supplied, report -1. +** Make no changes but still report the old value if the +** new limit is negative. +** +** A new lower limit does not shrink existing constructs. +** It merely prevents new constructs that exceed the limit +** from forming. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ + int oldLimit; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return -1; + } +#endif + + /* EVIDENCE-OF: R-30189-54097 For each limit category SQLITE_LIMIT_NAME + ** there is a hard upper bound set at compile-time by a C preprocessor + ** macro called SQLITE_MAX_NAME. (The "_LIMIT_" in the name is changed to + ** "_MAX_".) + */ + assert( aHardLimit[SQLITE_LIMIT_LENGTH]==SQLITE_MAX_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_SQL_LENGTH]==SQLITE_MAX_SQL_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_COLUMN]==SQLITE_MAX_COLUMN ); + assert( aHardLimit[SQLITE_LIMIT_EXPR_DEPTH]==SQLITE_MAX_EXPR_DEPTH ); + assert( aHardLimit[SQLITE_LIMIT_COMPOUND_SELECT]==SQLITE_MAX_COMPOUND_SELECT); + assert( aHardLimit[SQLITE_LIMIT_VDBE_OP]==SQLITE_MAX_VDBE_OP ); + assert( aHardLimit[SQLITE_LIMIT_FUNCTION_ARG]==SQLITE_MAX_FUNCTION_ARG ); + assert( aHardLimit[SQLITE_LIMIT_ATTACHED]==SQLITE_MAX_ATTACHED ); + assert( aHardLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]== + SQLITE_MAX_LIKE_PATTERN_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_VARIABLE_NUMBER]==SQLITE_MAX_VARIABLE_NUMBER); + assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH ); + assert( aHardLimit[SQLITE_LIMIT_WORKER_THREADS]==SQLITE_MAX_WORKER_THREADS ); + assert( SQLITE_LIMIT_WORKER_THREADS==(SQLITE_N_LIMIT-1) ); + + + if( limitId<0 || limitId>=SQLITE_N_LIMIT ){ + return -1; + } + oldLimit = db->aLimit[limitId]; + if( newLimit>=0 ){ /* IMP: R-52476-28732 */ + if( newLimit>aHardLimit[limitId] ){ + newLimit = aHardLimit[limitId]; /* IMP: R-51463-25634 */ + } + db->aLimit[limitId] = newLimit; + } + return oldLimit; /* IMP: R-53341-35419 */ +} + +/* +** This function is used to parse both URIs and non-URI filenames passed by the +** user to API functions sqlite3_open() or sqlite3_open_v2(), and for database +** URIs specified as part of ATTACH statements. +** +** The first argument to this function is the name of the VFS to use (or +** a NULL to signify the default VFS) if the URI does not contain a "vfs=xxx" +** query parameter. The second argument contains the URI (or non-URI filename) +** itself. When this function is called the *pFlags variable should contain +** the default flags to open the database handle with. The value stored in +** *pFlags may be updated before returning if the URI filename contains +** "cache=xxx" or "mode=xxx" query parameters. +** +** If successful, SQLITE_OK is returned. In this case *ppVfs is set to point to +** the VFS that should be used to open the database file. *pzFile is set to +** point to a buffer containing the name of the file to open. It is the +** responsibility of the caller to eventually call sqlite3_free() to release +** this buffer. +** +** If an error occurs, then an SQLite error code is returned and *pzErrMsg +** may be set to point to a buffer containing an English language error +** message. It is the responsibility of the caller to eventually release +** this buffer by calling sqlite3_free(). +*/ +SQLITE_PRIVATE int sqlite3ParseUri( + const char *zDefaultVfs, /* VFS to use if no "vfs=xxx" query option */ + const char *zUri, /* Nul-terminated URI to parse */ + unsigned int *pFlags, /* IN/OUT: SQLITE_OPEN_XXX flags */ + sqlite3_vfs **ppVfs, /* OUT: VFS to use */ + char **pzFile, /* OUT: Filename component of URI */ + char **pzErrMsg /* OUT: Error message (if rc!=SQLITE_OK) */ +){ + int rc = SQLITE_OK; + unsigned int flags = *pFlags; + const char *zVfs = zDefaultVfs; + char *zFile; + char c; + int nUri = sqlite3Strlen30(zUri); + + assert( *pzErrMsg==0 ); + + if( ((flags & SQLITE_OPEN_URI) /* IMP: R-48725-32206 */ + || sqlite3GlobalConfig.bOpenUri) /* IMP: R-51689-46548 */ + && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */ + ){ + char *zOpt; + int eState; /* Parser state when parsing URI */ + int iIn; /* Input character index */ + int iOut = 0; /* Output character index */ + u64 nByte = nUri+2; /* Bytes of space to allocate */ + + /* Make sure the SQLITE_OPEN_URI flag is set to indicate to the VFS xOpen + ** method that there may be extra parameters following the file-name. */ + flags |= SQLITE_OPEN_URI; + + for(iIn=0; iIn=0 && octet<256 ); + if( octet==0 ){ + /* This branch is taken when "%00" appears within the URI. In this + ** case we ignore all text in the remainder of the path, name or + ** value currently being parsed. So ignore the current character + ** and skip to the next "?", "=" or "&", as appropriate. */ + while( (c = zUri[iIn])!=0 && c!='#' + && (eState!=0 || c!='?') + && (eState!=1 || (c!='=' && c!='&')) + && (eState!=2 || c!='&') + ){ + iIn++; + } + continue; + } + c = octet; + }else if( eState==1 && (c=='&' || c=='=') ){ + if( zFile[iOut-1]==0 ){ + /* An empty option name. Ignore this option altogether. */ + while( zUri[iIn] && zUri[iIn]!='#' && zUri[iIn-1]!='&' ) iIn++; + continue; + } + if( c=='&' ){ + zFile[iOut++] = '\0'; + }else{ + eState = 2; + } + c = 0; + }else if( (eState==0 && c=='?') || (eState==2 && c=='&') ){ + c = 0; + eState = 1; + } + zFile[iOut++] = c; + } + if( eState==1 ) zFile[iOut++] = '\0'; + zFile[iOut++] = '\0'; + zFile[iOut++] = '\0'; + + /* Check if there were any options specified that should be interpreted + ** here. Options that are interpreted here include "vfs" and those that + ** correspond to flags that may be passed to the sqlite3_open_v2() + ** method. */ + zOpt = &zFile[sqlite3Strlen30(zFile)+1]; + while( zOpt[0] ){ + int nOpt = sqlite3Strlen30(zOpt); + char *zVal = &zOpt[nOpt+1]; + int nVal = sqlite3Strlen30(zVal); + + if( nOpt==3 && memcmp("vfs", zOpt, 3)==0 ){ + zVfs = zVal; + }else{ + struct OpenMode { + const char *z; + int mode; + } *aMode = 0; + char *zModeType = 0; + int mask = 0; + int limit = 0; + + if( nOpt==5 && memcmp("cache", zOpt, 5)==0 ){ + static struct OpenMode aCacheMode[] = { + { "shared", SQLITE_OPEN_SHAREDCACHE }, + { "private", SQLITE_OPEN_PRIVATECACHE }, + { 0, 0 } + }; + + mask = SQLITE_OPEN_SHAREDCACHE|SQLITE_OPEN_PRIVATECACHE; + aMode = aCacheMode; + limit = mask; + zModeType = "cache"; + } + if( nOpt==4 && memcmp("mode", zOpt, 4)==0 ){ + static struct OpenMode aOpenMode[] = { + { "ro", SQLITE_OPEN_READONLY }, + { "rw", SQLITE_OPEN_READWRITE }, + { "rwc", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE }, + { "memory", SQLITE_OPEN_MEMORY }, + { 0, 0 } + }; + + mask = SQLITE_OPEN_READONLY | SQLITE_OPEN_READWRITE + | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY; + aMode = aOpenMode; + limit = mask & flags; + zModeType = "access"; + } + + if( aMode ){ + int i; + int mode = 0; + for(i=0; aMode[i].z; i++){ + const char *z = aMode[i].z; + if( nVal==sqlite3Strlen30(z) && 0==memcmp(zVal, z, nVal) ){ + mode = aMode[i].mode; + break; + } + } + if( mode==0 ){ + *pzErrMsg = sqlite3_mprintf("no such %s mode: %s", zModeType, zVal); + rc = SQLITE_ERROR; + goto parse_uri_out; + } + if( (mode & ~SQLITE_OPEN_MEMORY)>limit ){ + *pzErrMsg = sqlite3_mprintf("%s mode not allowed: %s", + zModeType, zVal); + rc = SQLITE_PERM; + goto parse_uri_out; + } + flags = (flags & ~mask) | mode; + } + } + + zOpt = &zVal[nVal+1]; + } + + }else{ + zFile = sqlite3_malloc64(nUri+2); + if( !zFile ) return SQLITE_NOMEM; + memcpy(zFile, zUri, nUri); + zFile[nUri] = '\0'; + zFile[nUri+1] = '\0'; + flags &= ~SQLITE_OPEN_URI; + } + + *ppVfs = sqlite3_vfs_find(zVfs); + if( *ppVfs==0 ){ + *pzErrMsg = sqlite3_mprintf("no such vfs: %s", zVfs); + rc = SQLITE_ERROR; + } + parse_uri_out: + if( rc!=SQLITE_OK ){ + sqlite3_free(zFile); + zFile = 0; + } + *pFlags = flags; + *pzFile = zFile; + return rc; +} + + +/* +** This routine does the work of opening a database on behalf of +** sqlite3_open() and sqlite3_open16(). The database filename "zFilename" +** is UTF-8 encoded. +*/ +static int openDatabase( + const char *zFilename, /* Database filename UTF-8 encoded */ + sqlite3 **ppDb, /* OUT: Returned database handle */ + unsigned int flags, /* Operational flags */ + const char *zVfs /* Name of the VFS to use */ +){ + sqlite3 *db; /* Store allocated handle here */ + int rc; /* Return code */ + int isThreadsafe; /* True for threadsafe connections */ + char *zOpen = 0; /* Filename argument to pass to BtreeOpen() */ + char *zErrMsg = 0; /* Error message from sqlite3ParseUri() */ + +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppDb==0 ) return SQLITE_MISUSE_BKPT; +#endif + *ppDb = 0; +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + + /* Only allow sensible combinations of bits in the flags argument. + ** Throw an error if any non-sense combination is used. If we + ** do not block illegal combinations here, it could trigger + ** assert() statements in deeper layers. Sensible combinations + ** are: + ** + ** 1: SQLITE_OPEN_READONLY + ** 2: SQLITE_OPEN_READWRITE + ** 6: SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE + */ + assert( SQLITE_OPEN_READONLY == 0x01 ); + assert( SQLITE_OPEN_READWRITE == 0x02 ); + assert( SQLITE_OPEN_CREATE == 0x04 ); + testcase( (1<<(flags&7))==0x02 ); /* READONLY */ + testcase( (1<<(flags&7))==0x04 ); /* READWRITE */ + testcase( (1<<(flags&7))==0x40 ); /* READWRITE | CREATE */ + if( ((1<<(flags&7)) & 0x46)==0 ){ + return SQLITE_MISUSE_BKPT; /* IMP: R-65497-44594 */ + } + + if( sqlite3GlobalConfig.bCoreMutex==0 ){ + isThreadsafe = 0; + }else if( flags & SQLITE_OPEN_NOMUTEX ){ + isThreadsafe = 0; + }else if( flags & SQLITE_OPEN_FULLMUTEX ){ + isThreadsafe = 1; + }else{ + isThreadsafe = sqlite3GlobalConfig.bFullMutex; + } + if( flags & SQLITE_OPEN_PRIVATECACHE ){ + flags &= ~SQLITE_OPEN_SHAREDCACHE; + }else if( sqlite3GlobalConfig.sharedCacheEnabled ){ + flags |= SQLITE_OPEN_SHAREDCACHE; + } + + /* Remove harmful bits from the flags parameter + ** + ** The SQLITE_OPEN_NOMUTEX and SQLITE_OPEN_FULLMUTEX flags were + ** dealt with in the previous code block. Besides these, the only + ** valid input flags for sqlite3_open_v2() are SQLITE_OPEN_READONLY, + ** SQLITE_OPEN_READWRITE, SQLITE_OPEN_CREATE, SQLITE_OPEN_SHAREDCACHE, + ** SQLITE_OPEN_PRIVATECACHE, and some reserved bits. Silently mask + ** off all other flags. + */ + flags &= ~( SQLITE_OPEN_DELETEONCLOSE | + SQLITE_OPEN_EXCLUSIVE | + SQLITE_OPEN_MAIN_DB | + SQLITE_OPEN_TEMP_DB | + SQLITE_OPEN_TRANSIENT_DB | + SQLITE_OPEN_MAIN_JOURNAL | + SQLITE_OPEN_TEMP_JOURNAL | + SQLITE_OPEN_SUBJOURNAL | + SQLITE_OPEN_MASTER_JOURNAL | + SQLITE_OPEN_NOMUTEX | + SQLITE_OPEN_FULLMUTEX | + SQLITE_OPEN_WAL + ); + + /* Allocate the sqlite data structure */ + db = sqlite3MallocZero( sizeof(sqlite3) ); + if( db==0 ) goto opendb_out; + if( isThreadsafe ){ + db->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); + if( db->mutex==0 ){ + sqlite3_free(db); + db = 0; + goto opendb_out; + } + } + sqlite3_mutex_enter(db->mutex); + db->errMask = 0xff; + db->nDb = 2; + db->magic = SQLITE_MAGIC_BUSY; + db->aDb = db->aDbStatic; + + assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); + memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); + db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = SQLITE_DEFAULT_WORKER_THREADS; + db->autoCommit = 1; + db->nextAutovac = -1; + db->szMmap = sqlite3GlobalConfig.szMmap; + db->nextPagesize = 0; + db->nMaxSorterMmap = 0x7FFFFFFF; + db->flags |= SQLITE_ShortColNames | SQLITE_EnableTrigger | SQLITE_CacheSpill +#if !defined(SQLITE_DEFAULT_AUTOMATIC_INDEX) || SQLITE_DEFAULT_AUTOMATIC_INDEX + | SQLITE_AutoIndex +#endif +#if SQLITE_DEFAULT_CKPTFULLFSYNC + | SQLITE_CkptFullFSync +#endif +#if SQLITE_DEFAULT_FILE_FORMAT<4 + | SQLITE_LegacyFileFmt +#endif +#ifdef SQLITE_ENABLE_LOAD_EXTENSION + | SQLITE_LoadExtension +#endif +#if SQLITE_DEFAULT_RECURSIVE_TRIGGERS + | SQLITE_RecTriggers +#endif +#if defined(SQLITE_DEFAULT_FOREIGN_KEYS) && SQLITE_DEFAULT_FOREIGN_KEYS + | SQLITE_ForeignKeys +#endif +#if defined(SQLITE_REVERSE_UNORDERED_SELECTS) + | SQLITE_ReverseOrder +#endif +#if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) + | SQLITE_CellSizeCk +#endif + ; + sqlite3HashInit(&db->aCollSeq); +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3HashInit(&db->aModule); +#endif + + /* Add the default collation sequence BINARY. BINARY works for both UTF-8 + ** and UTF-16, so add a version for each to avoid any unnecessary + ** conversions. The only error that can occur here is a malloc() failure. + ** + ** EVIDENCE-OF: R-52786-44878 SQLite defines three built-in collating + ** functions: + */ + createCollation(db, "BINARY", SQLITE_UTF8, 0, binCollFunc, 0); + createCollation(db, "BINARY", SQLITE_UTF16BE, 0, binCollFunc, 0); + createCollation(db, "BINARY", SQLITE_UTF16LE, 0, binCollFunc, 0); + createCollation(db, "NOCASE", SQLITE_UTF8, 0, nocaseCollatingFunc, 0); + createCollation(db, "RTRIM", SQLITE_UTF8, (void*)1, binCollFunc, 0); + if( db->mallocFailed ){ + goto opendb_out; + } + /* EVIDENCE-OF: R-08308-17224 The default collating function for all + ** strings is BINARY. + */ + db->pDfltColl = sqlite3FindCollSeq(db, SQLITE_UTF8, "BINARY", 0); + assert( db->pDfltColl!=0 ); + + /* Parse the filename/URI argument. */ + db->openFlags = flags; + rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; + sqlite3ErrorWithMsg(db, rc, zErrMsg ? "%s" : 0, zErrMsg); + sqlite3_free(zErrMsg); + goto opendb_out; + } + + /* Open the backend database driver */ + rc = sqlite3BtreeOpen(db->pVfs, zOpen, db, &db->aDb[0].pBt, 0, + flags | SQLITE_OPEN_MAIN_DB); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_IOERR_NOMEM ){ + rc = SQLITE_NOMEM; + } + sqlite3Error(db, rc); + goto opendb_out; + } + sqlite3BtreeEnter(db->aDb[0].pBt); + db->aDb[0].pSchema = sqlite3SchemaGet(db, db->aDb[0].pBt); + if( !db->mallocFailed ) ENC(db) = SCHEMA_ENC(db); + sqlite3BtreeLeave(db->aDb[0].pBt); + db->aDb[1].pSchema = sqlite3SchemaGet(db, 0); + + /* The default safety_level for the main database is 'full'; for the temp + ** database it is 'NONE'. This matches the pager layer defaults. + */ + db->aDb[0].zName = "main"; + db->aDb[0].safety_level = 3; + db->aDb[1].zName = "temp"; + db->aDb[1].safety_level = 1; + + db->magic = SQLITE_MAGIC_OPEN; + if( db->mallocFailed ){ + goto opendb_out; + } + + /* Register all built-in functions, but do not attempt to read the + ** database schema yet. This is delayed until the first time the database + ** is accessed. + */ + sqlite3Error(db, SQLITE_OK); + sqlite3RegisterBuiltinFunctions(db); + + /* Load automatic extensions - extensions that have been registered + ** using the sqlite3_automatic_extension() API. + */ + rc = sqlite3_errcode(db); + if( rc==SQLITE_OK ){ + sqlite3AutoLoadExtensions(db); + rc = sqlite3_errcode(db); + if( rc!=SQLITE_OK ){ + goto opendb_out; + } + } + +#ifdef SQLITE_ENABLE_FTS1 + if( !db->mallocFailed ){ + extern int sqlite3Fts1Init(sqlite3*); + rc = sqlite3Fts1Init(db); + } +#endif + +#ifdef SQLITE_ENABLE_FTS2 + if( !db->mallocFailed && rc==SQLITE_OK ){ + extern int sqlite3Fts2Init(sqlite3*); + rc = sqlite3Fts2Init(db); + } +#endif + +#ifdef SQLITE_ENABLE_FTS3 /* automatically defined by SQLITE_ENABLE_FTS4 */ + if( !db->mallocFailed && rc==SQLITE_OK ){ + rc = sqlite3Fts3Init(db); + } +#endif + +#ifdef SQLITE_ENABLE_FTS5 + if( !db->mallocFailed && rc==SQLITE_OK ){ + rc = sqlite3Fts5Init(db); + } +#endif + +#ifdef SQLITE_ENABLE_ICU + if( !db->mallocFailed && rc==SQLITE_OK ){ + rc = sqlite3IcuInit(db); + } +#endif + +#ifdef SQLITE_ENABLE_RTREE + if( !db->mallocFailed && rc==SQLITE_OK){ + rc = sqlite3RtreeInit(db); + } +#endif + +#ifdef SQLITE_ENABLE_DBSTAT_VTAB + if( !db->mallocFailed && rc==SQLITE_OK){ + rc = sqlite3DbstatRegister(db); + } +#endif + +#ifdef SQLITE_ENABLE_JSON1 + if( !db->mallocFailed && rc==SQLITE_OK){ + rc = sqlite3Json1Init(db); + } +#endif + + /* -DSQLITE_DEFAULT_LOCKING_MODE=1 makes EXCLUSIVE the default locking + ** mode. -DSQLITE_DEFAULT_LOCKING_MODE=0 make NORMAL the default locking + ** mode. Doing nothing at all also makes NORMAL the default. + */ +#ifdef SQLITE_DEFAULT_LOCKING_MODE + db->dfltLockMode = SQLITE_DEFAULT_LOCKING_MODE; + sqlite3PagerLockingMode(sqlite3BtreePager(db->aDb[0].pBt), + SQLITE_DEFAULT_LOCKING_MODE); +#endif + + if( rc ) sqlite3Error(db, rc); + + /* Enable the lookaside-malloc subsystem */ + setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside, + sqlite3GlobalConfig.nLookaside); + + sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT); + +opendb_out: + sqlite3_free(zOpen); + if( db ){ + assert( db->mutex!=0 || isThreadsafe==0 + || sqlite3GlobalConfig.bFullMutex==0 ); + sqlite3_mutex_leave(db->mutex); + } + rc = sqlite3_errcode(db); + assert( db!=0 || rc==SQLITE_NOMEM ); + if( rc==SQLITE_NOMEM ){ + sqlite3_close(db); + db = 0; + }else if( rc!=SQLITE_OK ){ + db->magic = SQLITE_MAGIC_SICK; + } + *ppDb = db; +#ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ + /* Opening a db handle. Fourth parameter is passed 0. */ + void *pArg = sqlite3GlobalConfig.pSqllogArg; + sqlite3GlobalConfig.xSqllog(pArg, db, zFilename, 0); + } +#endif + return rc & 0xff; +} + +/* +** Open a new database handle. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_open( + const char *zFilename, + sqlite3 **ppDb +){ + return openDatabase(zFilename, ppDb, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); +} +SQLITE_API int SQLITE_STDCALL sqlite3_open_v2( + const char *filename, /* Database filename (UTF-8) */ + sqlite3 **ppDb, /* OUT: SQLite db handle */ + int flags, /* Flags */ + const char *zVfs /* Name of VFS module to use */ +){ + return openDatabase(filename, ppDb, (unsigned int)flags, zVfs); +} + +#ifndef SQLITE_OMIT_UTF16 +/* +** Open a new database handle. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_open16( + const void *zFilename, + sqlite3 **ppDb +){ + char const *zFilename8; /* zFilename encoded in UTF-8 instead of UTF-16 */ + sqlite3_value *pVal; + int rc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppDb==0 ) return SQLITE_MISUSE_BKPT; +#endif + *ppDb = 0; +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + if( zFilename==0 ) zFilename = "\000\000"; + pVal = sqlite3ValueNew(0); + sqlite3ValueSetStr(pVal, -1, zFilename, SQLITE_UTF16NATIVE, SQLITE_STATIC); + zFilename8 = sqlite3ValueText(pVal, SQLITE_UTF8); + if( zFilename8 ){ + rc = openDatabase(zFilename8, ppDb, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); + assert( *ppDb || rc==SQLITE_NOMEM ); + if( rc==SQLITE_OK && !DbHasProperty(*ppDb, 0, DB_SchemaLoaded) ){ + SCHEMA_ENC(*ppDb) = ENC(*ppDb) = SQLITE_UTF16NATIVE; + } + }else{ + rc = SQLITE_NOMEM; + } + sqlite3ValueFree(pVal); + + return rc & 0xff; +} +#endif /* SQLITE_OMIT_UTF16 */ + +/* +** Register a new collation sequence with the database handle db. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_create_collation( + sqlite3* db, + const char *zName, + int enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*) +){ + return sqlite3_create_collation_v2(db, zName, enc, pCtx, xCompare, 0); +} + +/* +** Register a new collation sequence with the database handle db. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_create_collation_v2( + sqlite3* db, + const char *zName, + int enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*), + void(*xDel)(void*) +){ + int rc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); + rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, xDel); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +#ifndef SQLITE_OMIT_UTF16 +/* +** Register a new collation sequence with the database handle db. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_create_collation16( + sqlite3* db, + const void *zName, + int enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*) +){ + int rc = SQLITE_OK; + char *zName8; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); + zName8 = sqlite3Utf16to8(db, zName, -1, SQLITE_UTF16NATIVE); + if( zName8 ){ + rc = createCollation(db, zName8, (u8)enc, pCtx, xCompare, 0); + sqlite3DbFree(db, zName8); + } + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} +#endif /* SQLITE_OMIT_UTF16 */ + +/* +** Register a collation sequence factory callback with the database handle +** db. Replace any previously installed collation sequence factory. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_collation_needed( + sqlite3 *db, + void *pCollNeededArg, + void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*) +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + db->xCollNeeded = xCollNeeded; + db->xCollNeeded16 = 0; + db->pCollNeededArg = pCollNeededArg; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_UTF16 +/* +** Register a collation sequence factory callback with the database handle +** db. Replace any previously installed collation sequence factory. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_collation_needed16( + sqlite3 *db, + void *pCollNeededArg, + void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*) +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + db->xCollNeeded = 0; + db->xCollNeeded16 = xCollNeeded16; + db->pCollNeededArg = pCollNeededArg; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} +#endif /* SQLITE_OMIT_UTF16 */ + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** This function is now an anachronism. It used to be used to recover from a +** malloc() failure, but SQLite now does this automatically. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_global_recover(void){ + return SQLITE_OK; +} +#endif + +/* +** Test to see whether or not the database connection is in autocommit +** mode. Return TRUE if it is and FALSE if not. Autocommit mode is on +** by default. Autocommit is disabled by a BEGIN statement and reenabled +** by the next COMMIT or ROLLBACK. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_get_autocommit(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return db->autoCommit; +} + +/* +** The following routines are substitutes for constants SQLITE_CORRUPT, +** SQLITE_MISUSE, SQLITE_CANTOPEN, SQLITE_IOERR and possibly other error +** constants. They serve two purposes: +** +** 1. Serve as a convenient place to set a breakpoint in a debugger +** to detect when version error conditions occurs. +** +** 2. Invoke sqlite3_log() to provide the source code location where +** a low-level error is first detected. +*/ +SQLITE_PRIVATE int sqlite3CorruptError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_CORRUPT, + "database corruption at line %d of [%.10s]", + lineno, 20+sqlite3_sourceid()); + return SQLITE_CORRUPT; +} +SQLITE_PRIVATE int sqlite3MisuseError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_MISUSE, + "misuse at line %d of [%.10s]", + lineno, 20+sqlite3_sourceid()); + return SQLITE_MISUSE; +} +SQLITE_PRIVATE int sqlite3CantopenError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_CANTOPEN, + "cannot open file at line %d of [%.10s]", + lineno, 20+sqlite3_sourceid()); + return SQLITE_CANTOPEN; +} + + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** This is a convenience routine that makes sure that all thread-specific +** data for this thread has been deallocated. +** +** SQLite no longer uses thread-specific data so this routine is now a +** no-op. It is retained for historical compatibility. +*/ +SQLITE_API void SQLITE_STDCALL sqlite3_thread_cleanup(void){ +} +#endif + +/* +** Return meta information about a specific column of a database table. +** See comment in sqlite3.h (sqlite.h.in) for details. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_table_column_metadata( + sqlite3 *db, /* Connection handle */ + const char *zDbName, /* Database name or NULL */ + const char *zTableName, /* Table name */ + const char *zColumnName, /* Column name */ + char const **pzDataType, /* OUTPUT: Declared data type */ + char const **pzCollSeq, /* OUTPUT: Collation sequence name */ + int *pNotNull, /* OUTPUT: True if NOT NULL constraint exists */ + int *pPrimaryKey, /* OUTPUT: True if column part of PK */ + int *pAutoinc /* OUTPUT: True if column is auto-increment */ +){ + int rc; + char *zErrMsg = 0; + Table *pTab = 0; + Column *pCol = 0; + int iCol = 0; + char const *zDataType = 0; + char const *zCollSeq = 0; + int notnull = 0; + int primarykey = 0; + int autoinc = 0; + + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zTableName==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif + + /* Ensure the database schema has been loaded */ + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + rc = sqlite3Init(db, &zErrMsg); + if( SQLITE_OK!=rc ){ + goto error_out; + } + + /* Locate the table in question */ + pTab = sqlite3FindTable(db, zTableName, zDbName); + if( !pTab || pTab->pSelect ){ + pTab = 0; + goto error_out; + } + + /* Find the column for which info is requested */ + if( zColumnName==0 ){ + /* Query for existance of table only */ + }else{ + for(iCol=0; iColnCol; iCol++){ + pCol = &pTab->aCol[iCol]; + if( 0==sqlite3StrICmp(pCol->zName, zColumnName) ){ + break; + } + } + if( iCol==pTab->nCol ){ + if( HasRowid(pTab) && sqlite3IsRowid(zColumnName) ){ + iCol = pTab->iPKey; + pCol = iCol>=0 ? &pTab->aCol[iCol] : 0; + }else{ + pTab = 0; + goto error_out; + } + } + } + + /* The following block stores the meta information that will be returned + ** to the caller in local variables zDataType, zCollSeq, notnull, primarykey + ** and autoinc. At this point there are two possibilities: + ** + ** 1. The specified column name was rowid", "oid" or "_rowid_" + ** and there is no explicitly declared IPK column. + ** + ** 2. The table is not a view and the column name identified an + ** explicitly declared column. Copy meta information from *pCol. + */ + if( pCol ){ + zDataType = pCol->zType; + zCollSeq = pCol->zColl; + notnull = pCol->notNull!=0; + primarykey = (pCol->colFlags & COLFLAG_PRIMKEY)!=0; + autoinc = pTab->iPKey==iCol && (pTab->tabFlags & TF_Autoincrement)!=0; + }else{ + zDataType = "INTEGER"; + primarykey = 1; + } + if( !zCollSeq ){ + zCollSeq = "BINARY"; + } + +error_out: + sqlite3BtreeLeaveAll(db); + + /* Whether the function call succeeded or failed, set the output parameters + ** to whatever their local counterparts contain. If an error did occur, + ** this has the effect of zeroing all output parameters. + */ + if( pzDataType ) *pzDataType = zDataType; + if( pzCollSeq ) *pzCollSeq = zCollSeq; + if( pNotNull ) *pNotNull = notnull; + if( pPrimaryKey ) *pPrimaryKey = primarykey; + if( pAutoinc ) *pAutoinc = autoinc; + + if( SQLITE_OK==rc && !pTab ){ + sqlite3DbFree(db, zErrMsg); + zErrMsg = sqlite3MPrintf(db, "no such table column: %s.%s", zTableName, + zColumnName); + rc = SQLITE_ERROR; + } + sqlite3ErrorWithMsg(db, rc, (zErrMsg?"%s":0), zErrMsg); + sqlite3DbFree(db, zErrMsg); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Sleep for a little while. Return the amount of time slept. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_sleep(int ms){ + sqlite3_vfs *pVfs; + int rc; + pVfs = sqlite3_vfs_find(0); + if( pVfs==0 ) return 0; + + /* This function works in milliseconds, but the underlying OsSleep() + ** API uses microseconds. Hence the 1000's. + */ + rc = (sqlite3OsSleep(pVfs, 1000*ms)/1000); + return rc; +} + +/* +** Enable or disable the extended result codes. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_extended_result_codes(sqlite3 *db, int onoff){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + db->errMask = onoff ? 0xffffffff : 0xff; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +/* +** Invoke the xFileControl method on a particular database. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ + int rc = SQLITE_ERROR; + Btree *pBtree; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + pBtree = sqlite3DbNameToBtree(db, zDbName); + if( pBtree ){ + Pager *pPager; + sqlite3_file *fd; + sqlite3BtreeEnter(pBtree); + pPager = sqlite3BtreePager(pBtree); + assert( pPager!=0 ); + fd = sqlite3PagerFile(pPager); + assert( fd!=0 ); + if( op==SQLITE_FCNTL_FILE_POINTER ){ + *(sqlite3_file**)pArg = fd; + rc = SQLITE_OK; + }else if( fd->pMethods ){ + rc = sqlite3OsFileControl(fd, op, pArg); + }else{ + rc = SQLITE_NOTFOUND; + } + sqlite3BtreeLeave(pBtree); + } + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Interface to the testing logic. +*/ +SQLITE_API int SQLITE_CDECL sqlite3_test_control(int op, ...){ + int rc = 0; +#ifdef SQLITE_OMIT_BUILTIN_TEST + UNUSED_PARAMETER(op); +#else + va_list ap; + va_start(ap, op); + switch( op ){ + + /* + ** Save the current state of the PRNG. + */ + case SQLITE_TESTCTRL_PRNG_SAVE: { + sqlite3PrngSaveState(); + break; + } + + /* + ** Restore the state of the PRNG to the last state saved using + ** PRNG_SAVE. If PRNG_SAVE has never before been called, then + ** this verb acts like PRNG_RESET. + */ + case SQLITE_TESTCTRL_PRNG_RESTORE: { + sqlite3PrngRestoreState(); + break; + } + + /* + ** Reset the PRNG back to its uninitialized state. The next call + ** to sqlite3_randomness() will reseed the PRNG using a single call + ** to the xRandomness method of the default VFS. + */ + case SQLITE_TESTCTRL_PRNG_RESET: { + sqlite3_randomness(0,0); + break; + } + + /* + ** sqlite3_test_control(BITVEC_TEST, size, program) + ** + ** Run a test against a Bitvec object of size. The program argument + ** is an array of integers that defines the test. Return -1 on a + ** memory allocation error, 0 on success, or non-zero for an error. + ** See the sqlite3BitvecBuiltinTest() for additional information. + */ + case SQLITE_TESTCTRL_BITVEC_TEST: { + int sz = va_arg(ap, int); + int *aProg = va_arg(ap, int*); + rc = sqlite3BitvecBuiltinTest(sz, aProg); + break; + } + + /* + ** sqlite3_test_control(FAULT_INSTALL, xCallback) + ** + ** Arrange to invoke xCallback() whenever sqlite3FaultSim() is called, + ** if xCallback is not NULL. + ** + ** As a test of the fault simulator mechanism itself, sqlite3FaultSim(0) + ** is called immediately after installing the new callback and the return + ** value from sqlite3FaultSim(0) becomes the return from + ** sqlite3_test_control(). + */ + case SQLITE_TESTCTRL_FAULT_INSTALL: { + /* MSVC is picky about pulling func ptrs from va lists. + ** http://support.microsoft.com/kb/47961 + ** sqlite3GlobalConfig.xTestCallback = va_arg(ap, int(*)(int)); + */ + typedef int(*TESTCALLBACKFUNC_t)(int); + sqlite3GlobalConfig.xTestCallback = va_arg(ap, TESTCALLBACKFUNC_t); + rc = sqlite3FaultSim(0); + break; + } + + /* + ** sqlite3_test_control(BENIGN_MALLOC_HOOKS, xBegin, xEnd) + ** + ** Register hooks to call to indicate which malloc() failures + ** are benign. + */ + case SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS: { + typedef void (*void_function)(void); + void_function xBenignBegin; + void_function xBenignEnd; + xBenignBegin = va_arg(ap, void_function); + xBenignEnd = va_arg(ap, void_function); + sqlite3BenignMallocHooks(xBenignBegin, xBenignEnd); + break; + } + + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_PENDING_BYTE, unsigned int X) + ** + ** Set the PENDING byte to the value in the argument, if X>0. + ** Make no changes if X==0. Return the value of the pending byte + ** as it existing before this routine was called. + ** + ** IMPORTANT: Changing the PENDING byte from 0x40000000 results in + ** an incompatible database file format. Changing the PENDING byte + ** while any database connection is open results in undefined and + ** deleterious behavior. + */ + case SQLITE_TESTCTRL_PENDING_BYTE: { + rc = PENDING_BYTE; +#ifndef SQLITE_OMIT_WSD + { + unsigned int newVal = va_arg(ap, unsigned int); + if( newVal ) sqlite3PendingByte = newVal; + } +#endif + break; + } + + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, int X) + ** + ** This action provides a run-time test to see whether or not + ** assert() was enabled at compile-time. If X is true and assert() + ** is enabled, then the return value is true. If X is true and + ** assert() is disabled, then the return value is zero. If X is + ** false and assert() is enabled, then the assertion fires and the + ** process aborts. If X is false and assert() is disabled, then the + ** return value is zero. + */ + case SQLITE_TESTCTRL_ASSERT: { + volatile int x = 0; + assert( (x = va_arg(ap,int))!=0 ); + rc = x; + break; + } + + + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, int X) + ** + ** This action provides a run-time test to see how the ALWAYS and + ** NEVER macros were defined at compile-time. + ** + ** The return value is ALWAYS(X). + ** + ** The recommended test is X==2. If the return value is 2, that means + ** ALWAYS() and NEVER() are both no-op pass-through macros, which is the + ** default setting. If the return value is 1, then ALWAYS() is either + ** hard-coded to true or else it asserts if its argument is false. + ** The first behavior (hard-coded to true) is the case if + ** SQLITE_TESTCTRL_ASSERT shows that assert() is disabled and the second + ** behavior (assert if the argument to ALWAYS() is false) is the case if + ** SQLITE_TESTCTRL_ASSERT shows that assert() is enabled. + ** + ** The run-time test procedure might look something like this: + ** + ** if( sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, 2)==2 ){ + ** // ALWAYS() and NEVER() are no-op pass-through macros + ** }else if( sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, 1) ){ + ** // ALWAYS(x) asserts that x is true. NEVER(x) asserts x is false. + ** }else{ + ** // ALWAYS(x) is a constant 1. NEVER(x) is a constant 0. + ** } + */ + case SQLITE_TESTCTRL_ALWAYS: { + int x = va_arg(ap,int); + rc = ALWAYS(x); + break; + } + + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_BYTEORDER); + ** + ** The integer returned reveals the byte-order of the computer on which + ** SQLite is running: + ** + ** 1 big-endian, determined at run-time + ** 10 little-endian, determined at run-time + ** 432101 big-endian, determined at compile-time + ** 123410 little-endian, determined at compile-time + */ + case SQLITE_TESTCTRL_BYTEORDER: { + rc = SQLITE_BYTEORDER*100 + SQLITE_LITTLEENDIAN*10 + SQLITE_BIGENDIAN; + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_RESERVE, sqlite3 *db, int N) + ** + ** Set the nReserve size to N for the main database on the database + ** connection db. + */ + case SQLITE_TESTCTRL_RESERVE: { + sqlite3 *db = va_arg(ap, sqlite3*); + int x = va_arg(ap,int); + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeSetPageSize(db->aDb[0].pBt, 0, x, 0); + sqlite3_mutex_leave(db->mutex); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, sqlite3 *db, int N) + ** + ** Enable or disable various optimizations for testing purposes. The + ** argument N is a bitmask of optimizations to be disabled. For normal + ** operation N should be 0. The idea is that a test program (like the + ** SQL Logic Test or SLT test module) can run the same SQL multiple times + ** with various optimizations disabled to verify that the same answer + ** is obtained in every case. + */ + case SQLITE_TESTCTRL_OPTIMIZATIONS: { + sqlite3 *db = va_arg(ap, sqlite3*); + db->dbOptFlags = (u16)(va_arg(ap, int) & 0xffff); + break; + } + +#ifdef SQLITE_N_KEYWORD + /* sqlite3_test_control(SQLITE_TESTCTRL_ISKEYWORD, const char *zWord) + ** + ** If zWord is a keyword recognized by the parser, then return the + ** number of keywords. Or if zWord is not a keyword, return 0. + ** + ** This test feature is only available in the amalgamation since + ** the SQLITE_N_KEYWORD macro is not defined in this file if SQLite + ** is built using separate source files. + */ + case SQLITE_TESTCTRL_ISKEYWORD: { + const char *zWord = va_arg(ap, const char*); + int n = sqlite3Strlen30(zWord); + rc = (sqlite3KeywordCode((u8*)zWord, n)!=TK_ID) ? SQLITE_N_KEYWORD : 0; + break; + } +#endif + + /* sqlite3_test_control(SQLITE_TESTCTRL_SCRATCHMALLOC, sz, &pNew, pFree); + ** + ** Pass pFree into sqlite3ScratchFree(). + ** If sz>0 then allocate a scratch buffer into pNew. + */ + case SQLITE_TESTCTRL_SCRATCHMALLOC: { + void *pFree, **ppNew; + int sz; + sz = va_arg(ap, int); + ppNew = va_arg(ap, void**); + pFree = va_arg(ap, void*); + if( sz ) *ppNew = sqlite3ScratchMalloc(sz); + sqlite3ScratchFree(pFree); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, int onoff); + ** + ** If parameter onoff is non-zero, configure the wrappers so that all + ** subsequent calls to localtime() and variants fail. If onoff is zero, + ** undo this setting. + */ + case SQLITE_TESTCTRL_LOCALTIME_FAULT: { + sqlite3GlobalConfig.bLocaltimeFault = va_arg(ap, int); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_NEVER_CORRUPT, int); + ** + ** Set or clear a flag that indicates that the database file is always well- + ** formed and never corrupt. This flag is clear by default, indicating that + ** database files might have arbitrary corruption. Setting the flag during + ** testing causes certain assert() statements in the code to be activated + ** that demonstrat invariants on well-formed database files. + */ + case SQLITE_TESTCTRL_NEVER_CORRUPT: { + sqlite3GlobalConfig.neverCorrupt = va_arg(ap, int); + break; + } + + + /* sqlite3_test_control(SQLITE_TESTCTRL_VDBE_COVERAGE, xCallback, ptr); + ** + ** Set the VDBE coverage callback function to xCallback with context + ** pointer ptr. + */ + case SQLITE_TESTCTRL_VDBE_COVERAGE: { +#ifdef SQLITE_VDBE_COVERAGE + typedef void (*branch_callback)(void*,int,u8,u8); + sqlite3GlobalConfig.xVdbeBranch = va_arg(ap,branch_callback); + sqlite3GlobalConfig.pVdbeBranchArg = va_arg(ap,void*); +#endif + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_SORTER_MMAP, db, nMax); */ + case SQLITE_TESTCTRL_SORTER_MMAP: { + sqlite3 *db = va_arg(ap, sqlite3*); + db->nMaxSorterMmap = va_arg(ap, int); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_ISINIT); + ** + ** Return SQLITE_OK if SQLite has been initialized and SQLITE_ERROR if + ** not. + */ + case SQLITE_TESTCTRL_ISINIT: { + if( sqlite3GlobalConfig.isInit==0 ) rc = SQLITE_ERROR; + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, dbName, onOff, tnum); + ** + ** This test control is used to create imposter tables. "db" is a pointer + ** to the database connection. dbName is the database name (ex: "main" or + ** "temp") which will receive the imposter. "onOff" turns imposter mode on + ** or off. "tnum" is the root page of the b-tree to which the imposter + ** table should connect. + ** + ** Enable imposter mode only when the schema has already been parsed. Then + ** run a single CREATE TABLE statement to construct the imposter table in + ** the parsed schema. Then turn imposter mode back off again. + ** + ** If onOff==0 and tnum>0 then reset the schema for all databases, causing + ** the schema to be reparsed the next time it is needed. This has the + ** effect of erasing all imposter tables. + */ + case SQLITE_TESTCTRL_IMPOSTER: { + sqlite3 *db = va_arg(ap, sqlite3*); + sqlite3_mutex_enter(db->mutex); + db->init.iDb = sqlite3FindDbName(db, va_arg(ap,const char*)); + db->init.busy = db->init.imposterTable = va_arg(ap,int); + db->init.newTnum = va_arg(ap,int); + if( db->init.busy==0 && db->init.newTnum>0 ){ + sqlite3ResetAllSchemasOfConnection(db); + } + sqlite3_mutex_leave(db->mutex); + break; + } + } + va_end(ap); +#endif /* SQLITE_OMIT_BUILTIN_TEST */ + return rc; +} + +/* +** This is a utility routine, useful to VFS implementations, that checks +** to see if a database file was a URI that contained a specific query +** parameter, and if so obtains the value of the query parameter. +** +** The zFilename argument is the filename pointer passed into the xOpen() +** method of a VFS implementation. The zParam argument is the name of the +** query parameter we seek. This routine returns the value of the zParam +** parameter if it exists. If the parameter does not exist, this routine +** returns a NULL pointer. +*/ +SQLITE_API const char *SQLITE_STDCALL sqlite3_uri_parameter(const char *zFilename, const char *zParam){ + if( zFilename==0 || zParam==0 ) return 0; + zFilename += sqlite3Strlen30(zFilename) + 1; + while( zFilename[0] ){ + int x = strcmp(zFilename, zParam); + zFilename += sqlite3Strlen30(zFilename) + 1; + if( x==0 ) return zFilename; + zFilename += sqlite3Strlen30(zFilename) + 1; + } + return 0; +} + +/* +** Return a boolean value for a query parameter. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_uri_boolean(const char *zFilename, const char *zParam, int bDflt){ + const char *z = sqlite3_uri_parameter(zFilename, zParam); + bDflt = bDflt!=0; + return z ? sqlite3GetBoolean(z, bDflt) : bDflt; +} + +/* +** Return a 64-bit integer value for a query parameter. +*/ +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_uri_int64( + const char *zFilename, /* Filename as passed to xOpen */ + const char *zParam, /* URI parameter sought */ + sqlite3_int64 bDflt /* return if parameter is missing */ +){ + const char *z = sqlite3_uri_parameter(zFilename, zParam); + sqlite3_int64 v; + if( z && sqlite3DecOrHexToI64(z, &v)==SQLITE_OK ){ + bDflt = v; + } + return bDflt; +} + +/* +** Return the Btree pointer identified by zDbName. Return NULL if not found. +*/ +SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){ + int i; + for(i=0; inDb; i++){ + if( db->aDb[i].pBt + && (zDbName==0 || sqlite3StrICmp(zDbName, db->aDb[i].zName)==0) + ){ + return db->aDb[i].pBt; + } + } + return 0; +} + +/* +** Return the filename of the database associated with a database +** connection. +*/ +SQLITE_API const char *SQLITE_STDCALL sqlite3_db_filename(sqlite3 *db, const char *zDbName){ + Btree *pBt; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + pBt = sqlite3DbNameToBtree(db, zDbName); + return pBt ? sqlite3BtreeGetFilename(pBt) : 0; +} + +/* +** Return 1 if database is read-only or 0 if read/write. Return -1 if +** no such database exists. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_db_readonly(sqlite3 *db, const char *zDbName){ + Btree *pBt; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return -1; + } +#endif + pBt = sqlite3DbNameToBtree(db, zDbName); + return pBt ? sqlite3BtreeIsReadonly(pBt) : -1; +} + +/************** End of main.c ************************************************/ +/************** Begin file notify.c ******************************************/ +/* +** 2009 March 3 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the implementation of the sqlite3_unlock_notify() +** API method and its associated functionality. +*/ +/* #include "sqliteInt.h" */ +/* #include "btreeInt.h" */ + +/* Omit this entire file if SQLITE_ENABLE_UNLOCK_NOTIFY is not defined. */ +#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY + +/* +** Public interfaces: +** +** sqlite3ConnectionBlocked() +** sqlite3ConnectionUnlocked() +** sqlite3ConnectionClosed() +** sqlite3_unlock_notify() +*/ + +#define assertMutexHeld() \ + assert( sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)) ) + +/* +** Head of a linked list of all sqlite3 objects created by this process +** for which either sqlite3.pBlockingConnection or sqlite3.pUnlockConnection +** is not NULL. This variable may only accessed while the STATIC_MASTER +** mutex is held. +*/ +static sqlite3 *SQLITE_WSD sqlite3BlockedList = 0; + +#ifndef NDEBUG +/* +** This function is a complex assert() that verifies the following +** properties of the blocked connections list: +** +** 1) Each entry in the list has a non-NULL value for either +** pUnlockConnection or pBlockingConnection, or both. +** +** 2) All entries in the list that share a common value for +** xUnlockNotify are grouped together. +** +** 3) If the argument db is not NULL, then none of the entries in the +** blocked connections list have pUnlockConnection or pBlockingConnection +** set to db. This is used when closing connection db. +*/ +static void checkListProperties(sqlite3 *db){ + sqlite3 *p; + for(p=sqlite3BlockedList; p; p=p->pNextBlocked){ + int seen = 0; + sqlite3 *p2; + + /* Verify property (1) */ + assert( p->pUnlockConnection || p->pBlockingConnection ); + + /* Verify property (2) */ + for(p2=sqlite3BlockedList; p2!=p; p2=p2->pNextBlocked){ + if( p2->xUnlockNotify==p->xUnlockNotify ) seen = 1; + assert( p2->xUnlockNotify==p->xUnlockNotify || !seen ); + assert( db==0 || p->pUnlockConnection!=db ); + assert( db==0 || p->pBlockingConnection!=db ); + } + } +} +#else +# define checkListProperties(x) +#endif + +/* +** Remove connection db from the blocked connections list. If connection +** db is not currently a part of the list, this function is a no-op. +*/ +static void removeFromBlockedList(sqlite3 *db){ + sqlite3 **pp; + assertMutexHeld(); + for(pp=&sqlite3BlockedList; *pp; pp = &(*pp)->pNextBlocked){ + if( *pp==db ){ + *pp = (*pp)->pNextBlocked; + break; + } + } +} + +/* +** Add connection db to the blocked connections list. It is assumed +** that it is not already a part of the list. +*/ +static void addToBlockedList(sqlite3 *db){ + sqlite3 **pp; + assertMutexHeld(); + for( + pp=&sqlite3BlockedList; + *pp && (*pp)->xUnlockNotify!=db->xUnlockNotify; + pp=&(*pp)->pNextBlocked + ); + db->pNextBlocked = *pp; + *pp = db; +} + +/* +** Obtain the STATIC_MASTER mutex. +*/ +static void enterMutex(void){ + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + checkListProperties(0); +} + +/* +** Release the STATIC_MASTER mutex. +*/ +static void leaveMutex(void){ + assertMutexHeld(); + checkListProperties(0); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); +} + +/* +** Register an unlock-notify callback. +** +** This is called after connection "db" has attempted some operation +** but has received an SQLITE_LOCKED error because another connection +** (call it pOther) in the same process was busy using the same shared +** cache. pOther is found by looking at db->pBlockingConnection. +** +** If there is no blocking connection, the callback is invoked immediately, +** before this routine returns. +** +** If pOther is already blocked on db, then report SQLITE_LOCKED, to indicate +** a deadlock. +** +** Otherwise, make arrangements to invoke xNotify when pOther drops +** its locks. +** +** Each call to this routine overrides any prior callbacks registered +** on the same "db". If xNotify==0 then any prior callbacks are immediately +** cancelled. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3_unlock_notify( + sqlite3 *db, + void (*xNotify)(void **, int), + void *pArg +){ + int rc = SQLITE_OK; + + sqlite3_mutex_enter(db->mutex); + enterMutex(); + + if( xNotify==0 ){ + removeFromBlockedList(db); + db->pBlockingConnection = 0; + db->pUnlockConnection = 0; + db->xUnlockNotify = 0; + db->pUnlockArg = 0; + }else if( 0==db->pBlockingConnection ){ + /* The blocking transaction has been concluded. Or there never was a + ** blocking transaction. In either case, invoke the notify callback + ** immediately. + */ + xNotify(&pArg, 1); + }else{ + sqlite3 *p; + + for(p=db->pBlockingConnection; p && p!=db; p=p->pUnlockConnection){} + if( p ){ + rc = SQLITE_LOCKED; /* Deadlock detected. */ + }else{ + db->pUnlockConnection = db->pBlockingConnection; + db->xUnlockNotify = xNotify; + db->pUnlockArg = pArg; + removeFromBlockedList(db); + addToBlockedList(db); + } + } + + leaveMutex(); + assert( !db->mallocFailed ); + sqlite3ErrorWithMsg(db, rc, (rc?"database is deadlocked":0)); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** This function is called while stepping or preparing a statement +** associated with connection db. The operation will return SQLITE_LOCKED +** to the user because it requires a lock that will not be available +** until connection pBlocker concludes its current transaction. +*/ +SQLITE_PRIVATE void sqlite3ConnectionBlocked(sqlite3 *db, sqlite3 *pBlocker){ + enterMutex(); + if( db->pBlockingConnection==0 && db->pUnlockConnection==0 ){ + addToBlockedList(db); + } + db->pBlockingConnection = pBlocker; + leaveMutex(); +} + +/* +** This function is called when +** the transaction opened by database db has just finished. Locks held +** by database connection db have been released. +** +** This function loops through each entry in the blocked connections +** list and does the following: +** +** 1) If the sqlite3.pBlockingConnection member of a list entry is +** set to db, then set pBlockingConnection=0. +** +** 2) If the sqlite3.pUnlockConnection member of a list entry is +** set to db, then invoke the configured unlock-notify callback and +** set pUnlockConnection=0. +** +** 3) If the two steps above mean that pBlockingConnection==0 and +** pUnlockConnection==0, remove the entry from the blocked connections +** list. +*/ +SQLITE_PRIVATE void sqlite3ConnectionUnlocked(sqlite3 *db){ + void (*xUnlockNotify)(void **, int) = 0; /* Unlock-notify cb to invoke */ + int nArg = 0; /* Number of entries in aArg[] */ + sqlite3 **pp; /* Iterator variable */ + void **aArg; /* Arguments to the unlock callback */ + void **aDyn = 0; /* Dynamically allocated space for aArg[] */ + void *aStatic[16]; /* Starter space for aArg[]. No malloc required */ + + aArg = aStatic; + enterMutex(); /* Enter STATIC_MASTER mutex */ + + /* This loop runs once for each entry in the blocked-connections list. */ + for(pp=&sqlite3BlockedList; *pp; /* no-op */ ){ + sqlite3 *p = *pp; + + /* Step 1. */ + if( p->pBlockingConnection==db ){ + p->pBlockingConnection = 0; + } + + /* Step 2. */ + if( p->pUnlockConnection==db ){ + assert( p->xUnlockNotify ); + if( p->xUnlockNotify!=xUnlockNotify && nArg!=0 ){ + xUnlockNotify(aArg, nArg); + nArg = 0; + } + + sqlite3BeginBenignMalloc(); + assert( aArg==aDyn || (aDyn==0 && aArg==aStatic) ); + assert( nArg<=(int)ArraySize(aStatic) || aArg==aDyn ); + if( (!aDyn && nArg==(int)ArraySize(aStatic)) + || (aDyn && nArg==(int)(sqlite3MallocSize(aDyn)/sizeof(void*))) + ){ + /* The aArg[] array needs to grow. */ + void **pNew = (void **)sqlite3Malloc(nArg*sizeof(void *)*2); + if( pNew ){ + memcpy(pNew, aArg, nArg*sizeof(void *)); + sqlite3_free(aDyn); + aDyn = aArg = pNew; + }else{ + /* This occurs when the array of context pointers that need to + ** be passed to the unlock-notify callback is larger than the + ** aStatic[] array allocated on the stack and the attempt to + ** allocate a larger array from the heap has failed. + ** + ** This is a difficult situation to handle. Returning an error + ** code to the caller is insufficient, as even if an error code + ** is returned the transaction on connection db will still be + ** closed and the unlock-notify callbacks on blocked connections + ** will go unissued. This might cause the application to wait + ** indefinitely for an unlock-notify callback that will never + ** arrive. + ** + ** Instead, invoke the unlock-notify callback with the context + ** array already accumulated. We can then clear the array and + ** begin accumulating any further context pointers without + ** requiring any dynamic allocation. This is sub-optimal because + ** it means that instead of one callback with a large array of + ** context pointers the application will receive two or more + ** callbacks with smaller arrays of context pointers, which will + ** reduce the applications ability to prioritize multiple + ** connections. But it is the best that can be done under the + ** circumstances. + */ + xUnlockNotify(aArg, nArg); + nArg = 0; + } + } + sqlite3EndBenignMalloc(); + + aArg[nArg++] = p->pUnlockArg; + xUnlockNotify = p->xUnlockNotify; + p->pUnlockConnection = 0; + p->xUnlockNotify = 0; + p->pUnlockArg = 0; + } + + /* Step 3. */ + if( p->pBlockingConnection==0 && p->pUnlockConnection==0 ){ + /* Remove connection p from the blocked connections list. */ + *pp = p->pNextBlocked; + p->pNextBlocked = 0; + }else{ + pp = &p->pNextBlocked; + } + } + + if( nArg!=0 ){ + xUnlockNotify(aArg, nArg); + } + sqlite3_free(aDyn); + leaveMutex(); /* Leave STATIC_MASTER mutex */ +} + +/* +** This is called when the database connection passed as an argument is +** being closed. The connection is removed from the blocked list. +*/ +SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ + sqlite3ConnectionUnlocked(db); + enterMutex(); + removeFromBlockedList(db); + checkListProperties(db); + leaveMutex(); +} +#endif + +/************** End of notify.c **********************************************/ +/************** Begin file fts3.c ********************************************/ +/* +** 2006 Oct 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is an SQLite module implementing full-text search. +*/ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ + +/* The full-text index is stored in a series of b+tree (-like) +** structures called segments which map terms to doclists. The +** structures are like b+trees in layout, but are constructed from the +** bottom up in optimal fashion and are not updatable. Since trees +** are built from the bottom up, things will be described from the +** bottom up. +** +** +**** Varints **** +** The basic unit of encoding is a variable-length integer called a +** varint. We encode variable-length integers in little-endian order +** using seven bits * per byte as follows: +** +** KEY: +** A = 0xxxxxxx 7 bits of data and one flag bit +** B = 1xxxxxxx 7 bits of data and one flag bit +** +** 7 bits - A +** 14 bits - BA +** 21 bits - BBA +** and so on. +** +** This is similar in concept to how sqlite encodes "varints" but +** the encoding is not the same. SQLite varints are big-endian +** are are limited to 9 bytes in length whereas FTS3 varints are +** little-endian and can be up to 10 bytes in length (in theory). +** +** Example encodings: +** +** 1: 0x01 +** 127: 0x7f +** 128: 0x81 0x00 +** +** +**** Document lists **** +** A doclist (document list) holds a docid-sorted list of hits for a +** given term. Doclists hold docids and associated token positions. +** A docid is the unique integer identifier for a single document. +** A position is the index of a word within the document. The first +** word of the document has a position of 0. +** +** FTS3 used to optionally store character offsets using a compile-time +** option. But that functionality is no longer supported. +** +** A doclist is stored like this: +** +** array { +** varint docid; (delta from previous doclist) +** array { (position list for column 0) +** varint position; (2 more than the delta from previous position) +** } +** array { +** varint POS_COLUMN; (marks start of position list for new column) +** varint column; (index of new column) +** array { +** varint position; (2 more than the delta from previous position) +** } +** } +** varint POS_END; (marks end of positions for this document. +** } +** +** Here, array { X } means zero or more occurrences of X, adjacent in +** memory. A "position" is an index of a token in the token stream +** generated by the tokenizer. Note that POS_END and POS_COLUMN occur +** in the same logical place as the position element, and act as sentinals +** ending a position list array. POS_END is 0. POS_COLUMN is 1. +** The positions numbers are not stored literally but rather as two more +** than the difference from the prior position, or the just the position plus +** 2 for the first position. Example: +** +** label: A B C D E F G H I J K +** value: 123 5 9 1 1 14 35 0 234 72 0 +** +** The 123 value is the first docid. For column zero in this document +** there are two matches at positions 3 and 10 (5-2 and 9-2+3). The 1 +** at D signals the start of a new column; the 1 at E indicates that the +** new column is column number 1. There are two positions at 12 and 45 +** (14-2 and 35-2+12). The 0 at H indicate the end-of-document. The +** 234 at I is the delta to next docid (357). It has one position 70 +** (72-2) and then terminates with the 0 at K. +** +** A "position-list" is the list of positions for multiple columns for +** a single docid. A "column-list" is the set of positions for a single +** column. Hence, a position-list consists of one or more column-lists, +** a document record consists of a docid followed by a position-list and +** a doclist consists of one or more document records. +** +** A bare doclist omits the position information, becoming an +** array of varint-encoded docids. +** +**** Segment leaf nodes **** +** Segment leaf nodes store terms and doclists, ordered by term. Leaf +** nodes are written using LeafWriter, and read using LeafReader (to +** iterate through a single leaf node's data) and LeavesReader (to +** iterate through a segment's entire leaf layer). Leaf nodes have +** the format: +** +** varint iHeight; (height from leaf level, always 0) +** varint nTerm; (length of first term) +** char pTerm[nTerm]; (content of first term) +** varint nDoclist; (length of term's associated doclist) +** char pDoclist[nDoclist]; (content of doclist) +** array { +** (further terms are delta-encoded) +** varint nPrefix; (length of prefix shared with previous term) +** varint nSuffix; (length of unshared suffix) +** char pTermSuffix[nSuffix];(unshared suffix of next term) +** varint nDoclist; (length of term's associated doclist) +** char pDoclist[nDoclist]; (content of doclist) +** } +** +** Here, array { X } means zero or more occurrences of X, adjacent in +** memory. +** +** Leaf nodes are broken into blocks which are stored contiguously in +** the %_segments table in sorted order. This means that when the end +** of a node is reached, the next term is in the node with the next +** greater node id. +** +** New data is spilled to a new leaf node when the current node +** exceeds LEAF_MAX bytes (default 2048). New data which itself is +** larger than STANDALONE_MIN (default 1024) is placed in a standalone +** node (a leaf node with a single term and doclist). The goal of +** these settings is to pack together groups of small doclists while +** making it efficient to directly access large doclists. The +** assumption is that large doclists represent terms which are more +** likely to be query targets. +** +** TODO(shess) It may be useful for blocking decisions to be more +** dynamic. For instance, it may make more sense to have a 2.5k leaf +** node rather than splitting into 2k and .5k nodes. My intuition is +** that this might extend through 2x or 4x the pagesize. +** +** +**** Segment interior nodes **** +** Segment interior nodes store blockids for subtree nodes and terms +** to describe what data is stored by the each subtree. Interior +** nodes are written using InteriorWriter, and read using +** InteriorReader. InteriorWriters are created as needed when +** SegmentWriter creates new leaf nodes, or when an interior node +** itself grows too big and must be split. The format of interior +** nodes: +** +** varint iHeight; (height from leaf level, always >0) +** varint iBlockid; (block id of node's leftmost subtree) +** optional { +** varint nTerm; (length of first term) +** char pTerm[nTerm]; (content of first term) +** array { +** (further terms are delta-encoded) +** varint nPrefix; (length of shared prefix with previous term) +** varint nSuffix; (length of unshared suffix) +** char pTermSuffix[nSuffix]; (unshared suffix of next term) +** } +** } +** +** Here, optional { X } means an optional element, while array { X } +** means zero or more occurrences of X, adjacent in memory. +** +** An interior node encodes n terms separating n+1 subtrees. The +** subtree blocks are contiguous, so only the first subtree's blockid +** is encoded. The subtree at iBlockid will contain all terms less +** than the first term encoded (or all terms if no term is encoded). +** Otherwise, for terms greater than or equal to pTerm[i] but less +** than pTerm[i+1], the subtree for that term will be rooted at +** iBlockid+i. Interior nodes only store enough term data to +** distinguish adjacent children (if the rightmost term of the left +** child is "something", and the leftmost term of the right child is +** "wicked", only "w" is stored). +** +** New data is spilled to a new interior node at the same height when +** the current node exceeds INTERIOR_MAX bytes (default 2048). +** INTERIOR_MIN_TERMS (default 7) keeps large terms from monopolizing +** interior nodes and making the tree too skinny. The interior nodes +** at a given height are naturally tracked by interior nodes at +** height+1, and so on. +** +** +**** Segment directory **** +** The segment directory in table %_segdir stores meta-information for +** merging and deleting segments, and also the root node of the +** segment's tree. +** +** The root node is the top node of the segment's tree after encoding +** the entire segment, restricted to ROOT_MAX bytes (default 1024). +** This could be either a leaf node or an interior node. If the top +** node requires more than ROOT_MAX bytes, it is flushed to %_segments +** and a new root interior node is generated (which should always fit +** within ROOT_MAX because it only needs space for 2 varints, the +** height and the blockid of the previous root). +** +** The meta-information in the segment directory is: +** level - segment level (see below) +** idx - index within level +** - (level,idx uniquely identify a segment) +** start_block - first leaf node +** leaves_end_block - last leaf node +** end_block - last block (including interior nodes) +** root - contents of root node +** +** If the root node is a leaf node, then start_block, +** leaves_end_block, and end_block are all 0. +** +** +**** Segment merging **** +** To amortize update costs, segments are grouped into levels and +** merged in batches. Each increase in level represents exponentially +** more documents. +** +** New documents (actually, document updates) are tokenized and +** written individually (using LeafWriter) to a level 0 segment, with +** incrementing idx. When idx reaches MERGE_COUNT (default 16), all +** level 0 segments are merged into a single level 1 segment. Level 1 +** is populated like level 0, and eventually MERGE_COUNT level 1 +** segments are merged to a single level 2 segment (representing +** MERGE_COUNT^2 updates), and so on. +** +** A segment merge traverses all segments at a given level in +** parallel, performing a straightforward sorted merge. Since segment +** leaf nodes are written in to the %_segments table in order, this +** merge traverses the underlying sqlite disk structures efficiently. +** After the merge, all segment blocks from the merged level are +** deleted. +** +** MERGE_COUNT controls how often we merge segments. 16 seems to be +** somewhat of a sweet spot for insertion performance. 32 and 64 show +** very similar performance numbers to 16 on insertion, though they're +** a tiny bit slower (perhaps due to more overhead in merge-time +** sorting). 8 is about 20% slower than 16, 4 about 50% slower than +** 16, 2 about 66% slower than 16. +** +** At query time, high MERGE_COUNT increases the number of segments +** which need to be scanned and merged. For instance, with 100k docs +** inserted: +** +** MERGE_COUNT segments +** 16 25 +** 8 12 +** 4 10 +** 2 6 +** +** This appears to have only a moderate impact on queries for very +** frequent terms (which are somewhat dominated by segment merge +** costs), and infrequent and non-existent terms still seem to be fast +** even with many segments. +** +** TODO(shess) That said, it would be nice to have a better query-side +** argument for MERGE_COUNT of 16. Also, it is possible/likely that +** optimizations to things like doclist merging will swing the sweet +** spot around. +** +** +** +**** Handling of deletions and updates **** +** Since we're using a segmented structure, with no docid-oriented +** index into the term index, we clearly cannot simply update the term +** index when a document is deleted or updated. For deletions, we +** write an empty doclist (varint(docid) varint(POS_END)), for updates +** we simply write the new doclist. Segment merges overwrite older +** data for a particular docid with newer data, so deletes or updates +** will eventually overtake the earlier data and knock it out. The +** query logic likewise merges doclists so that newer data knocks out +** older data. +*/ + +/************** Include fts3Int.h in the middle of fts3.c ********************/ +/************** Begin file fts3Int.h *****************************************/ +/* +** 2009 Nov 12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ +#ifndef _FTSINT_H +#define _FTSINT_H + +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif + +/* +** FTS4 is really an extension for FTS3. It is enabled using the +** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all +** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3. +*/ +#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) +# define SQLITE_ENABLE_FTS3 +#endif + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* If not building as part of the core, include sqlite3ext.h. */ +#ifndef SQLITE_CORE +/* # include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT3 +#endif + +/* #include "sqlite3.h" */ +/************** Include fts3_tokenizer.h in the middle of fts3Int.h **********/ +/************** Begin file fts3_tokenizer.h **********************************/ +/* +** 2006 July 10 +** +** The author disclaims copyright to this source code. +** +************************************************************************* +** Defines the interface to tokenizers used by fulltext-search. There +** are three basic components: +** +** sqlite3_tokenizer_module is a singleton defining the tokenizer +** interface functions. This is essentially the class structure for +** tokenizers. +** +** sqlite3_tokenizer is used to define a particular tokenizer, perhaps +** including customization information defined at creation time. +** +** sqlite3_tokenizer_cursor is generated by a tokenizer to generate +** tokens from a particular input. +*/ +#ifndef _FTS3_TOKENIZER_H_ +#define _FTS3_TOKENIZER_H_ + +/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. +** If tokenizers are to be allowed to call sqlite3_*() functions, then +** we will need a way to register the API consistently. +*/ +/* #include "sqlite3.h" */ + +/* +** Structures used by the tokenizer interface. When a new tokenizer +** implementation is registered, the caller provides a pointer to +** an sqlite3_tokenizer_module containing pointers to the callback +** functions that make up an implementation. +** +** When an fts3 table is created, it passes any arguments passed to +** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the +** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer +** implementation. The xCreate() function in turn returns an +** sqlite3_tokenizer structure representing the specific tokenizer to +** be used for the fts3 table (customized by the tokenizer clause arguments). +** +** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen() +** method is called. It returns an sqlite3_tokenizer_cursor object +** that may be used to tokenize a specific input buffer based on +** the tokenization rules supplied by a specific sqlite3_tokenizer +** object. +*/ +typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; +typedef struct sqlite3_tokenizer sqlite3_tokenizer; +typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; + +struct sqlite3_tokenizer_module { + + /* + ** Structure version. Should always be set to 0 or 1. + */ + int iVersion; + + /* + ** Create a new tokenizer. The values in the argv[] array are the + ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL + ** TABLE statement that created the fts3 table. For example, if + ** the following SQL is executed: + ** + ** CREATE .. USING fts3( ... , tokenizer arg1 arg2) + ** + ** then argc is set to 2, and the argv[] array contains pointers + ** to the strings "arg1" and "arg2". + ** + ** This method should return either SQLITE_OK (0), or an SQLite error + ** code. If SQLITE_OK is returned, then *ppTokenizer should be set + ** to point at the newly created tokenizer structure. The generic + ** sqlite3_tokenizer.pModule variable should not be initialized by + ** this callback. The caller will do so. + */ + int (*xCreate)( + int argc, /* Size of argv array */ + const char *const*argv, /* Tokenizer argument strings */ + sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ + ); + + /* + ** Destroy an existing tokenizer. The fts3 module calls this method + ** exactly once for each successful call to xCreate(). + */ + int (*xDestroy)(sqlite3_tokenizer *pTokenizer); + + /* + ** Create a tokenizer cursor to tokenize an input buffer. The caller + ** is responsible for ensuring that the input buffer remains valid + ** until the cursor is closed (using the xClose() method). + */ + int (*xOpen)( + sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ + const char *pInput, int nBytes, /* Input buffer */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ + ); + + /* + ** Destroy an existing tokenizer cursor. The fts3 module calls this + ** method exactly once for each successful call to xOpen(). + */ + int (*xClose)(sqlite3_tokenizer_cursor *pCursor); + + /* + ** Retrieve the next token from the tokenizer cursor pCursor. This + ** method should either return SQLITE_OK and set the values of the + ** "OUT" variables identified below, or SQLITE_DONE to indicate that + ** the end of the buffer has been reached, or an SQLite error code. + ** + ** *ppToken should be set to point at a buffer containing the + ** normalized version of the token (i.e. after any case-folding and/or + ** stemming has been performed). *pnBytes should be set to the length + ** of this buffer in bytes. The input text that generated the token is + ** identified by the byte offsets returned in *piStartOffset and + ** *piEndOffset. *piStartOffset should be set to the index of the first + ** byte of the token in the input buffer. *piEndOffset should be set + ** to the index of the first byte just past the end of the token in + ** the input buffer. + ** + ** The buffer *ppToken is set to point at is managed by the tokenizer + ** implementation. It is only required to be valid until the next call + ** to xNext() or xClose(). + */ + /* TODO(shess) current implementation requires pInput to be + ** nul-terminated. This should either be fixed, or pInput/nBytes + ** should be converted to zInput. + */ + int (*xNext)( + sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ + const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ + int *piStartOffset, /* OUT: Byte offset of token in input buffer */ + int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ + int *piPosition /* OUT: Number of tokens returned before this one */ + ); + + /*********************************************************************** + ** Methods below this point are only available if iVersion>=1. + */ + + /* + ** Configure the language id of a tokenizer cursor. + */ + int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid); +}; + +struct sqlite3_tokenizer { + const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ + /* Tokenizer implementations will typically add additional fields */ +}; + +struct sqlite3_tokenizer_cursor { + sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ + /* Tokenizer implementations will typically add additional fields */ +}; + +int fts3_global_term_cnt(int iTerm, int iCol); +int fts3_term_cnt(int iTerm, int iCol); + + +#endif /* _FTS3_TOKENIZER_H_ */ + +/************** End of fts3_tokenizer.h **************************************/ +/************** Continuing where we left off in fts3Int.h ********************/ +/************** Include fts3_hash.h in the middle of fts3Int.h ***************/ +/************** Begin file fts3_hash.h ***************************************/ +/* +** 2001 September 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This is the header file for the generic hash-table implementation +** used in SQLite. We've modified it slightly to serve as a standalone +** hash table implementation for the full-text indexing module. +** +*/ +#ifndef _FTS3_HASH_H_ +#define _FTS3_HASH_H_ + +/* Forward declarations of structures. */ +typedef struct Fts3Hash Fts3Hash; +typedef struct Fts3HashElem Fts3HashElem; + +/* A complete hash table is an instance of the following structure. +** The internals of this structure are intended to be opaque -- client +** code should not attempt to access or modify the fields of this structure +** directly. Change this structure only by using the routines below. +** However, many of the "procedures" and "functions" for modifying and +** accessing this structure are really macros, so we can't really make +** this structure opaque. +*/ +struct Fts3Hash { + char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */ + char copyKey; /* True if copy of key made on insert */ + int count; /* Number of entries in this table */ + Fts3HashElem *first; /* The first element of the array */ + int htsize; /* Number of buckets in the hash table */ + struct _fts3ht { /* the hash table */ + int count; /* Number of entries with this hash */ + Fts3HashElem *chain; /* Pointer to first entry with this hash */ + } *ht; +}; + +/* Each element in the hash table is an instance of the following +** structure. All elements are stored on a single doubly-linked list. +** +** Again, this structure is intended to be opaque, but it can't really +** be opaque because it is used by macros. +*/ +struct Fts3HashElem { + Fts3HashElem *next, *prev; /* Next and previous elements in the table */ + void *data; /* Data associated with this element */ + void *pKey; int nKey; /* Key associated with this element */ +}; + +/* +** There are 2 different modes of operation for a hash table: +** +** FTS3_HASH_STRING pKey points to a string that is nKey bytes long +** (including the null-terminator, if any). Case +** is respected in comparisons. +** +** FTS3_HASH_BINARY pKey points to binary data nKey bytes long. +** memcmp() is used to compare keys. +** +** A copy of the key is made if the copyKey parameter to fts3HashInit is 1. +*/ +#define FTS3_HASH_STRING 1 +#define FTS3_HASH_BINARY 2 + +/* +** Access routines. To delete, insert a NULL pointer. +*/ +SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey); +SQLITE_PRIVATE void *sqlite3Fts3HashInsert(Fts3Hash*, const void *pKey, int nKey, void *pData); +SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash*, const void *pKey, int nKey); +SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash*); +SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const void *, int); + +/* +** Shorthand for the functions above +*/ +#define fts3HashInit sqlite3Fts3HashInit +#define fts3HashInsert sqlite3Fts3HashInsert +#define fts3HashFind sqlite3Fts3HashFind +#define fts3HashClear sqlite3Fts3HashClear +#define fts3HashFindElem sqlite3Fts3HashFindElem + +/* +** Macros for looping over all elements of a hash table. The idiom is +** like this: +** +** Fts3Hash h; +** Fts3HashElem *p; +** ... +** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){ +** SomeStructure *pData = fts3HashData(p); +** // do something with pData +** } +*/ +#define fts3HashFirst(H) ((H)->first) +#define fts3HashNext(E) ((E)->next) +#define fts3HashData(E) ((E)->data) +#define fts3HashKey(E) ((E)->pKey) +#define fts3HashKeysize(E) ((E)->nKey) + +/* +** Number of entries in a hash table +*/ +#define fts3HashCount(H) ((H)->count) + +#endif /* _FTS3_HASH_H_ */ + +/************** End of fts3_hash.h *******************************************/ +/************** Continuing where we left off in fts3Int.h ********************/ + +/* +** This constant determines the maximum depth of an FTS expression tree +** that the library will create and use. FTS uses recursion to perform +** various operations on the query tree, so the disadvantage of a large +** limit is that it may allow very large queries to use large amounts +** of stack space (perhaps causing a stack overflow). +*/ +#ifndef SQLITE_FTS3_MAX_EXPR_DEPTH +# define SQLITE_FTS3_MAX_EXPR_DEPTH 12 +#endif + + +/* +** This constant controls how often segments are merged. Once there are +** FTS3_MERGE_COUNT segments of level N, they are merged into a single +** segment of level N+1. +*/ +#define FTS3_MERGE_COUNT 16 + +/* +** This is the maximum amount of data (in bytes) to store in the +** Fts3Table.pendingTerms hash table. Normally, the hash table is +** populated as documents are inserted/updated/deleted in a transaction +** and used to create a new segment when the transaction is committed. +** However if this limit is reached midway through a transaction, a new +** segment is created and the hash table cleared immediately. +*/ +#define FTS3_MAX_PENDING_DATA (1*1024*1024) + +/* +** Macro to return the number of elements in an array. SQLite has a +** similar macro called ArraySize(). Use a different name to avoid +** a collision when building an amalgamation with built-in FTS3. +*/ +#define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0]))) + + +#ifndef MIN +# define MIN(x,y) ((x)<(y)?(x):(y)) +#endif +#ifndef MAX +# define MAX(x,y) ((x)>(y)?(x):(y)) +#endif + +/* +** Maximum length of a varint encoded integer. The varint format is different +** from that used by SQLite, so the maximum length is 10, not 9. +*/ +#define FTS3_VARINT_MAX 10 + +/* +** FTS4 virtual tables may maintain multiple indexes - one index of all terms +** in the document set and zero or more prefix indexes. All indexes are stored +** as one or more b+-trees in the %_segments and %_segdir tables. +** +** It is possible to determine which index a b+-tree belongs to based on the +** value stored in the "%_segdir.level" column. Given this value L, the index +** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with +** level values between 0 and 1023 (inclusive) belong to index 0, all levels +** between 1024 and 2047 to index 1, and so on. +** +** It is considered impossible for an index to use more than 1024 levels. In +** theory though this may happen, but only after at least +** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables. +*/ +#define FTS3_SEGDIR_MAXLEVEL 1024 +#define FTS3_SEGDIR_MAXLEVEL_STR "1024" + +/* +** The testcase() macro is only used by the amalgamation. If undefined, +** make it a no-op. +*/ +#ifndef testcase +# define testcase(X) +#endif + +/* +** Terminator values for position-lists and column-lists. +*/ +#define POS_COLUMN (1) /* Column-list terminator */ +#define POS_END (0) /* Position-list terminator */ + +/* +** This section provides definitions to allow the +** FTS3 extension to be compiled outside of the +** amalgamation. +*/ +#ifndef SQLITE_AMALGAMATION +/* +** Macros indicating that conditional expressions are always true or +** false. +*/ +#ifdef SQLITE_COVERAGE_TEST +# define ALWAYS(x) (1) +# define NEVER(X) (0) +#elif defined(SQLITE_DEBUG) +# define ALWAYS(x) sqlite3Fts3Always((x)!=0) +# define NEVER(x) sqlite3Fts3Never((x)!=0) +SQLITE_PRIVATE int sqlite3Fts3Always(int b); +SQLITE_PRIVATE int sqlite3Fts3Never(int b); +#else +# define ALWAYS(x) (x) +# define NEVER(x) (x) +#endif + +/* +** Internal types used by SQLite. +*/ +typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ +typedef short int i16; /* 2-byte (or larger) signed integer */ +typedef unsigned int u32; /* 4-byte unsigned integer */ +typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ +typedef sqlite3_int64 i64; /* 8-byte signed integer */ + +/* +** Macro used to suppress compiler warnings for unused parameters. +*/ +#define UNUSED_PARAMETER(x) (void)(x) + +/* +** Activate assert() only if SQLITE_TEST is enabled. +*/ +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif + +/* +** The TESTONLY macro is used to enclose variable declarations or +** other bits of code that are needed to support the arguments +** within testcase() and assert() macros. +*/ +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) +# define TESTONLY(X) X +#else +# define TESTONLY(X) +#endif + +#endif /* SQLITE_AMALGAMATION */ + +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3Fts3Corrupt(void); +# define FTS_CORRUPT_VTAB sqlite3Fts3Corrupt() +#else +# define FTS_CORRUPT_VTAB SQLITE_CORRUPT_VTAB +#endif + +typedef struct Fts3Table Fts3Table; +typedef struct Fts3Cursor Fts3Cursor; +typedef struct Fts3Expr Fts3Expr; +typedef struct Fts3Phrase Fts3Phrase; +typedef struct Fts3PhraseToken Fts3PhraseToken; + +typedef struct Fts3Doclist Fts3Doclist; +typedef struct Fts3SegFilter Fts3SegFilter; +typedef struct Fts3DeferredToken Fts3DeferredToken; +typedef struct Fts3SegReader Fts3SegReader; +typedef struct Fts3MultiSegReader Fts3MultiSegReader; + +typedef struct MatchinfoBuffer MatchinfoBuffer; + +/* +** A connection to a fulltext index is an instance of the following +** structure. The xCreate and xConnect methods create an instance +** of this structure and xDestroy and xDisconnect free that instance. +** All other methods receive a pointer to the structure as one of their +** arguments. +*/ +struct Fts3Table { + sqlite3_vtab base; /* Base class used by SQLite core */ + sqlite3 *db; /* The database connection */ + const char *zDb; /* logical database name */ + const char *zName; /* virtual table name */ + int nColumn; /* number of named columns in virtual table */ + char **azColumn; /* column names. malloced */ + u8 *abNotindexed; /* True for 'notindexed' columns */ + sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ + char *zContentTbl; /* content=xxx option, or NULL */ + char *zLanguageid; /* languageid=xxx option, or NULL */ + int nAutoincrmerge; /* Value configured by 'automerge' */ + u32 nLeafAdd; /* Number of leaf blocks added this trans */ + + /* Precompiled statements used by the implementation. Each of these + ** statements is run and reset within a single virtual table API call. + */ + sqlite3_stmt *aStmt[40]; + + char *zReadExprlist; + char *zWriteExprlist; + + int nNodeSize; /* Soft limit for node size */ + u8 bFts4; /* True for FTS4, false for FTS3 */ + u8 bHasStat; /* True if %_stat table exists (2==unknown) */ + u8 bHasDocsize; /* True if %_docsize table exists */ + u8 bDescIdx; /* True if doclists are in reverse order */ + u8 bIgnoreSavepoint; /* True to ignore xSavepoint invocations */ + int nPgsz; /* Page size for host database */ + char *zSegmentsTbl; /* Name of %_segments table */ + sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ + + /* + ** The following array of hash tables is used to buffer pending index + ** updates during transactions. All pending updates buffered at any one + ** time must share a common language-id (see the FTS4 langid= feature). + ** The current language id is stored in variable iPrevLangid. + ** + ** A single FTS4 table may have multiple full-text indexes. For each index + ** there is an entry in the aIndex[] array. Index 0 is an index of all the + ** terms that appear in the document set. Each subsequent index in aIndex[] + ** is an index of prefixes of a specific length. + ** + ** Variable nPendingData contains an estimate the memory consumed by the + ** pending data structures, including hash table overhead, but not including + ** malloc overhead. When nPendingData exceeds nMaxPendingData, all hash + ** tables are flushed to disk. Variable iPrevDocid is the docid of the most + ** recently inserted record. + */ + int nIndex; /* Size of aIndex[] */ + struct Fts3Index { + int nPrefix; /* Prefix length (0 for main terms index) */ + Fts3Hash hPending; /* Pending terms table for this index */ + } *aIndex; + int nMaxPendingData; /* Max pending data before flush to disk */ + int nPendingData; /* Current bytes of pending data */ + sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ + int iPrevLangid; /* Langid of recently inserted document */ + int bPrevDelete; /* True if last operation was a delete */ + +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) + /* State variables used for validating that the transaction control + ** methods of the virtual table are called at appropriate times. These + ** values do not contribute to FTS functionality; they are used for + ** verifying the operation of the SQLite core. + */ + int inTransaction; /* True after xBegin but before xCommit/xRollback */ + int mxSavepoint; /* Largest valid xSavepoint integer */ +#endif + +#ifdef SQLITE_TEST + /* True to disable the incremental doclist optimization. This is controled + ** by special insert command 'test-no-incr-doclist'. */ + int bNoIncrDoclist; +#endif +}; + +/* +** When the core wants to read from the virtual table, it creates a +** virtual table cursor (an instance of the following structure) using +** the xOpen method. Cursors are destroyed using the xClose method. +*/ +struct Fts3Cursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + i16 eSearch; /* Search strategy (see below) */ + u8 isEof; /* True if at End Of Results */ + u8 isRequireSeek; /* True if must seek pStmt to %_content row */ + sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ + Fts3Expr *pExpr; /* Parsed MATCH query string */ + int iLangid; /* Language being queried for */ + int nPhrase; /* Number of matchable phrases in query */ + Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ + sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ + char *pNextId; /* Pointer into the body of aDoclist */ + char *aDoclist; /* List of docids for full-text queries */ + int nDoclist; /* Size of buffer at aDoclist */ + u8 bDesc; /* True to sort in descending order */ + int eEvalmode; /* An FTS3_EVAL_XX constant */ + int nRowAvg; /* Average size of database rows, in pages */ + sqlite3_int64 nDoc; /* Documents in table */ + i64 iMinDocid; /* Minimum docid to return */ + i64 iMaxDocid; /* Maximum docid to return */ + int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ + MatchinfoBuffer *pMIBuffer; /* Buffer for matchinfo data */ +}; + +#define FTS3_EVAL_FILTER 0 +#define FTS3_EVAL_NEXT 1 +#define FTS3_EVAL_MATCHINFO 2 + +/* +** The Fts3Cursor.eSearch member is always set to one of the following. +** Actualy, Fts3Cursor.eSearch can be greater than or equal to +** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index +** of the column to be searched. For example, in +** +** CREATE VIRTUAL TABLE ex1 USING fts3(a,b,c,d); +** SELECT docid FROM ex1 WHERE b MATCH 'one two three'; +** +** Because the LHS of the MATCH operator is 2nd column "b", +** Fts3Cursor.eSearch will be set to FTS3_FULLTEXT_SEARCH+1. (+0 for a, +** +1 for b, +2 for c, +3 for d.) If the LHS of MATCH were "ex1" +** indicating that all columns should be searched, +** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4. +*/ +#define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */ +#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ +#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ + +/* +** The lower 16-bits of the sqlite3_index_info.idxNum value set by +** the xBestIndex() method contains the Fts3Cursor.eSearch value described +** above. The upper 16-bits contain a combination of the following +** bits, used to describe extra constraints on full-text searches. +*/ +#define FTS3_HAVE_LANGID 0x00010000 /* languageid=? */ +#define FTS3_HAVE_DOCID_GE 0x00020000 /* docid>=? */ +#define FTS3_HAVE_DOCID_LE 0x00040000 /* docid<=? */ + +struct Fts3Doclist { + char *aAll; /* Array containing doclist (or NULL) */ + int nAll; /* Size of a[] in bytes */ + char *pNextDocid; /* Pointer to next docid */ + + sqlite3_int64 iDocid; /* Current docid (if pList!=0) */ + int bFreeList; /* True if pList should be sqlite3_free()d */ + char *pList; /* Pointer to position list following iDocid */ + int nList; /* Length of position list */ +}; + +/* +** A "phrase" is a sequence of one or more tokens that must match in +** sequence. A single token is the base case and the most common case. +** For a sequence of tokens contained in double-quotes (i.e. "one two three") +** nToken will be the number of tokens in the string. +*/ +struct Fts3PhraseToken { + char *z; /* Text of the token */ + int n; /* Number of bytes in buffer z */ + int isPrefix; /* True if token ends with a "*" character */ + int bFirst; /* True if token must appear at position 0 */ + + /* Variables above this point are populated when the expression is + ** parsed (by code in fts3_expr.c). Below this point the variables are + ** used when evaluating the expression. */ + Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ + Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */ +}; + +struct Fts3Phrase { + /* Cache of doclist for this phrase. */ + Fts3Doclist doclist; + int bIncr; /* True if doclist is loaded incrementally */ + int iDoclistToken; + + /* Used by sqlite3Fts3EvalPhrasePoslist() if this is a descendent of an + ** OR condition. */ + char *pOrPoslist; + i64 iOrDocid; + + /* Variables below this point are populated by fts3_expr.c when parsing + ** a MATCH expression. Everything above is part of the evaluation phase. + */ + int nToken; /* Number of tokens in the phrase */ + int iColumn; /* Index of column this phrase must match */ + Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ +}; + +/* +** A tree of these objects forms the RHS of a MATCH operator. +** +** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist +** points to a malloced buffer, size nDoclist bytes, containing the results +** of this phrase query in FTS3 doclist format. As usual, the initial +** "Length" field found in doclists stored on disk is omitted from this +** buffer. +** +** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global +** matchinfo data. If it is not NULL, it points to an array of size nCol*3, +** where nCol is the number of columns in the queried FTS table. The array +** is populated as follows: +** +** aMI[iCol*3 + 0] = Undefined +** aMI[iCol*3 + 1] = Number of occurrences +** aMI[iCol*3 + 2] = Number of rows containing at least one instance +** +** The aMI array is allocated using sqlite3_malloc(). It should be freed +** when the expression node is. +*/ +struct Fts3Expr { + int eType; /* One of the FTSQUERY_XXX values defined below */ + int nNear; /* Valid if eType==FTSQUERY_NEAR */ + Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */ + Fts3Expr *pLeft; /* Left operand */ + Fts3Expr *pRight; /* Right operand */ + Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ + + /* The following are used by the fts3_eval.c module. */ + sqlite3_int64 iDocid; /* Current docid */ + u8 bEof; /* True this expression is at EOF already */ + u8 bStart; /* True if iDocid is valid */ + u8 bDeferred; /* True if this expression is entirely deferred */ + + /* The following are used by the fts3_snippet.c module. */ + int iPhrase; /* Index of this phrase in matchinfo() results */ + u32 *aMI; /* See above */ +}; + +/* +** Candidate values for Fts3Query.eType. Note that the order of the first +** four values is in order of precedence when parsing expressions. For +** example, the following: +** +** "a OR b AND c NOT d NEAR e" +** +** is equivalent to: +** +** "a OR (b AND (c NOT (d NEAR e)))" +*/ +#define FTSQUERY_NEAR 1 +#define FTSQUERY_NOT 2 +#define FTSQUERY_AND 3 +#define FTSQUERY_OR 4 +#define FTSQUERY_PHRASE 5 + + +/* fts3_write.c */ +SQLITE_PRIVATE int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); +SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *); +SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *); +SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *); +SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(int, int, sqlite3_int64, + sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); +SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( + Fts3Table*,int,const char*,int,int,Fts3SegReader**); +SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *); +SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, int, sqlite3_stmt **); +SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); + +SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); +SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); + +#ifndef SQLITE_DISABLE_FTS4_DEFERRED +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); +SQLITE_PRIVATE int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); +SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); +SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); +#else +# define sqlite3Fts3FreeDeferredTokens(x) +# define sqlite3Fts3DeferToken(x,y,z) SQLITE_OK +# define sqlite3Fts3CacheDeferredDoclists(x) SQLITE_OK +# define sqlite3Fts3FreeDeferredDoclists(x) +# define sqlite3Fts3DeferredTokenList(x,y,z) SQLITE_OK +#endif + +SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *); +SQLITE_PRIVATE int sqlite3Fts3MaxLevel(Fts3Table *, int *); + +/* Special values interpreted by sqlite3SegReaderCursor() */ +#define FTS3_SEGCURSOR_PENDING -1 +#define FTS3_SEGCURSOR_ALL -2 + +SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*); +SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); +SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *); + +SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor(Fts3Table *, + int, int, int, const char *, int, int, int, Fts3MultiSegReader *); + +/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ +#define FTS3_SEGMENT_REQUIRE_POS 0x00000001 +#define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 +#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 +#define FTS3_SEGMENT_PREFIX 0x00000008 +#define FTS3_SEGMENT_SCAN 0x00000010 +#define FTS3_SEGMENT_FIRST 0x00000020 + +/* Type passed as 4th argument to SegmentReaderIterate() */ +struct Fts3SegFilter { + const char *zTerm; + int nTerm; + int iCol; + int flags; +}; + +struct Fts3MultiSegReader { + /* Used internally by sqlite3Fts3SegReaderXXX() calls */ + Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */ + int nSegment; /* Size of apSegment array */ + int nAdvance; /* How many seg-readers to advance */ + Fts3SegFilter *pFilter; /* Pointer to filter object */ + char *aBuffer; /* Buffer to merge doclists in */ + int nBuffer; /* Allocated size of aBuffer[] in bytes */ + + int iColFilter; /* If >=0, filter for this column */ + int bRestart; + + /* Used by fts3.c only. */ + int nCost; /* Cost of running iterator */ + int bLookup; /* True if a lookup of a single entry. */ + + /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ + char *zTerm; /* Pointer to term buffer */ + int nTerm; /* Size of zTerm in bytes */ + char *aDoclist; /* Pointer to doclist buffer */ + int nDoclist; /* Size of aDoclist[] in bytes */ +}; + +SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table*,int,int); + +#define fts3GetVarint32(p, piVal) ( \ + (*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \ +) + +/* fts3.c */ +SQLITE_PRIVATE void sqlite3Fts3ErrMsg(char**,const char*,...); +SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *, sqlite3_int64); +SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); +SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *); +SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64); +SQLITE_PRIVATE void sqlite3Fts3Dequote(char *); +SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); +SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); +SQLITE_PRIVATE int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); +SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int*, Fts3Table*); +SQLITE_PRIVATE int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc); + +/* fts3_tokenizer.c */ +SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *); +SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); +SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, + sqlite3_tokenizer **, char ** +); +SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char); + +/* fts3_snippet.c */ +SQLITE_PRIVATE void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*); +SQLITE_PRIVATE void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, + const char *, const char *, int, int +); +SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); +SQLITE_PRIVATE void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p); + +/* fts3_expr.c */ +SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int, + char **, int, int, int, const char *, int, Fts3Expr **, char ** +); +SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *); +#ifdef SQLITE_TEST +SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3 *db); +SQLITE_PRIVATE int sqlite3Fts3InitTerm(sqlite3 *db); +#endif + +SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int, + sqlite3_tokenizer_cursor ** +); + +/* fts3_aux.c */ +SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db); + +SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); + +SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( + Fts3Table*, Fts3MultiSegReader*, int, const char*, int); +SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( + Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); +SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **); +SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); +SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); + +/* fts3_tokenize_vtab.c */ +SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *); + +/* fts3_unicode2.c (functions generated by parsing unicode text files) */ +#ifndef SQLITE_DISABLE_FTS3_UNICODE +SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int, int); +SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int); +SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int); +#endif + +#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */ +#endif /* _FTSINT_H */ + +/************** End of fts3Int.h *********************************************/ +/************** Continuing where we left off in fts3.c ***********************/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE) +# define SQLITE_CORE 1 +#endif + +/* #include */ +/* #include */ +/* #include */ +/* #include */ +/* #include */ +/* #include */ + +/* #include "fts3.h" */ +#ifndef SQLITE_CORE +/* # include "sqlite3ext.h" */ + SQLITE_EXTENSION_INIT1 +#endif + +static int fts3EvalNext(Fts3Cursor *pCsr); +static int fts3EvalStart(Fts3Cursor *pCsr); +static int fts3TermSegReaderCursor( + Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **); + +#ifndef SQLITE_AMALGAMATION +# if defined(SQLITE_DEBUG) +SQLITE_PRIVATE int sqlite3Fts3Always(int b) { assert( b ); return b; } +SQLITE_PRIVATE int sqlite3Fts3Never(int b) { assert( !b ); return b; } +# endif +#endif + +/* +** Write a 64-bit variable-length integer to memory starting at p[0]. +** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. +** The number of bytes written is returned. +*/ +SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ + unsigned char *q = (unsigned char *) p; + sqlite_uint64 vu = v; + do{ + *q++ = (unsigned char) ((vu & 0x7f) | 0x80); + vu >>= 7; + }while( vu!=0 ); + q[-1] &= 0x7f; /* turn off high bit in final byte */ + assert( q - (unsigned char *)p <= FTS3_VARINT_MAX ); + return (int) (q - (unsigned char *)p); +} + +#define GETVARINT_STEP(v, ptr, shift, mask1, mask2, var, ret) \ + v = (v & mask1) | ( (*ptr++) << shift ); \ + if( (v & mask2)==0 ){ var = v; return ret; } +#define GETVARINT_INIT(v, ptr, shift, mask1, mask2, var, ret) \ + v = (*ptr++); \ + if( (v & mask2)==0 ){ var = v; return ret; } + +/* +** Read a 64-bit variable-length integer from memory starting at p[0]. +** Return the number of bytes read, or 0 on error. +** The value is stored in *v. +*/ +SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){ + const char *pStart = p; + u32 a; + u64 b; + int shift; + + GETVARINT_INIT(a, p, 0, 0x00, 0x80, *v, 1); + GETVARINT_STEP(a, p, 7, 0x7F, 0x4000, *v, 2); + GETVARINT_STEP(a, p, 14, 0x3FFF, 0x200000, *v, 3); + GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *v, 4); + b = (a & 0x0FFFFFFF ); + + for(shift=28; shift<=63; shift+=7){ + u64 c = *p++; + b += (c&0x7F) << shift; + if( (c & 0x80)==0 ) break; + } + *v = b; + return (int)(p - pStart); +} + +/* +** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a +** 32-bit integer before it is returned. +*/ +SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *p, int *pi){ + u32 a; + +#ifndef fts3GetVarint32 + GETVARINT_INIT(a, p, 0, 0x00, 0x80, *pi, 1); +#else + a = (*p++); + assert( a & 0x80 ); +#endif + + GETVARINT_STEP(a, p, 7, 0x7F, 0x4000, *pi, 2); + GETVARINT_STEP(a, p, 14, 0x3FFF, 0x200000, *pi, 3); + GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *pi, 4); + a = (a & 0x0FFFFFFF ); + *pi = (int)(a | ((u32)(*p & 0x0F) << 28)); + return 5; +} + +/* +** Return the number of bytes required to encode v as a varint +*/ +SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64 v){ + int i = 0; + do{ + i++; + v >>= 7; + }while( v!=0 ); + return i; +} + +/* +** Convert an SQL-style quoted string into a normal string by removing +** the quote characters. The conversion is done in-place. If the +** input does not begin with a quote character, then this routine +** is a no-op. +** +** Examples: +** +** "abc" becomes abc +** 'xyz' becomes xyz +** [pqr] becomes pqr +** `mno` becomes mno +** +*/ +SQLITE_PRIVATE void sqlite3Fts3Dequote(char *z){ + char quote; /* Quote character (if any ) */ + + quote = z[0]; + if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ + int iIn = 1; /* Index of next byte to read from input */ + int iOut = 0; /* Index of next byte to write to output */ + + /* If the first byte was a '[', then the close-quote character is a ']' */ + if( quote=='[' ) quote = ']'; + + while( z[iIn] ){ + if( z[iIn]==quote ){ + if( z[iIn+1]!=quote ) break; + z[iOut++] = quote; + iIn += 2; + }else{ + z[iOut++] = z[iIn++]; + } + } + z[iOut] = '\0'; + } +} + +/* +** Read a single varint from the doclist at *pp and advance *pp to point +** to the first byte past the end of the varint. Add the value of the varint +** to *pVal. +*/ +static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){ + sqlite3_int64 iVal; + *pp += sqlite3Fts3GetVarint(*pp, &iVal); + *pVal += iVal; +} + +/* +** When this function is called, *pp points to the first byte following a +** varint that is part of a doclist (or position-list, or any other list +** of varints). This function moves *pp to point to the start of that varint, +** and sets *pVal by the varint value. +** +** Argument pStart points to the first byte of the doclist that the +** varint is part of. +*/ +static void fts3GetReverseVarint( + char **pp, + char *pStart, + sqlite3_int64 *pVal +){ + sqlite3_int64 iVal; + char *p; + + /* Pointer p now points at the first byte past the varint we are + ** interested in. So, unless the doclist is corrupt, the 0x80 bit is + ** clear on character p[-1]. */ + for(p = (*pp)-2; p>=pStart && *p&0x80; p--); + p++; + *pp = p; + + sqlite3Fts3GetVarint(p, &iVal); + *pVal = iVal; +} + +/* +** The xDisconnect() virtual table method. +*/ +static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table *)pVtab; + int i; + + assert( p->nPendingData==0 ); + assert( p->pSegments==0 ); + + /* Free any prepared statements held */ + for(i=0; iaStmt); i++){ + sqlite3_finalize(p->aStmt[i]); + } + sqlite3_free(p->zSegmentsTbl); + sqlite3_free(p->zReadExprlist); + sqlite3_free(p->zWriteExprlist); + sqlite3_free(p->zContentTbl); + sqlite3_free(p->zLanguageid); + + /* Invoke the tokenizer destructor to free the tokenizer. */ + p->pTokenizer->pModule->xDestroy(p->pTokenizer); + + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Write an error message into *pzErr +*/ +SQLITE_PRIVATE void sqlite3Fts3ErrMsg(char **pzErr, const char *zFormat, ...){ + va_list ap; + sqlite3_free(*pzErr); + va_start(ap, zFormat); + *pzErr = sqlite3_vmprintf(zFormat, ap); + va_end(ap); +} + +/* +** Construct one or more SQL statements from the format string given +** and then evaluate those statements. The success code is written +** into *pRc. +** +** If *pRc is initially non-zero then this routine is a no-op. +*/ +static void fts3DbExec( + int *pRc, /* Success code */ + sqlite3 *db, /* Database in which to run SQL */ + const char *zFormat, /* Format string for SQL */ + ... /* Arguments to the format string */ +){ + va_list ap; + char *zSql; + if( *pRc ) return; + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( zSql==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + *pRc = sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_free(zSql); + } +} + +/* +** The xDestroy() virtual table method. +*/ +static int fts3DestroyMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table *)pVtab; + int rc = SQLITE_OK; /* Return code */ + const char *zDb = p->zDb; /* Name of database (e.g. "main", "temp") */ + sqlite3 *db = p->db; /* Database handle */ + + /* Drop the shadow tables */ + if( p->zContentTbl==0 ){ + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", zDb, p->zName); + } + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", zDb,p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", zDb, p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", zDb, p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", zDb, p->zName); + + /* If everything has worked, invoke fts3DisconnectMethod() to free the + ** memory associated with the Fts3Table structure and return SQLITE_OK. + ** Otherwise, return an SQLite error code. + */ + return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc); +} + + +/* +** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table +** passed as the first argument. This is done as part of the xConnect() +** and xCreate() methods. +** +** If *pRc is non-zero when this function is called, it is a no-op. +** Otherwise, if an error occurs, an SQLite error code is stored in *pRc +** before returning. +*/ +static void fts3DeclareVtab(int *pRc, Fts3Table *p){ + if( *pRc==SQLITE_OK ){ + int i; /* Iterator variable */ + int rc; /* Return code */ + char *zSql; /* SQL statement passed to declare_vtab() */ + char *zCols; /* List of user defined columns */ + const char *zLanguageid; + + zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid"); + sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + + /* Create a list of user columns for the virtual table */ + zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); + for(i=1; zCols && inColumn; i++){ + zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]); + } + + /* Create the whole "CREATE TABLE" statement to pass to SQLite */ + zSql = sqlite3_mprintf( + "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN, %Q HIDDEN)", + zCols, p->zName, zLanguageid + ); + if( !zCols || !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_declare_vtab(p->db, zSql); + } + + sqlite3_free(zSql); + sqlite3_free(zCols); + *pRc = rc; + } +} + +/* +** Create the %_stat table if it does not already exist. +*/ +SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int *pRc, Fts3Table *p){ + fts3DbExec(pRc, p->db, + "CREATE TABLE IF NOT EXISTS %Q.'%q_stat'" + "(id INTEGER PRIMARY KEY, value BLOB);", + p->zDb, p->zName + ); + if( (*pRc)==SQLITE_OK ) p->bHasStat = 1; +} + +/* +** Create the backing store tables (%_content, %_segments and %_segdir) +** required by the FTS3 table passed as the only argument. This is done +** as part of the vtab xCreate() method. +** +** If the p->bHasDocsize boolean is true (indicating that this is an +** FTS4 table, not an FTS3 table) then also create the %_docsize and +** %_stat tables required by FTS4. +*/ +static int fts3CreateTables(Fts3Table *p){ + int rc = SQLITE_OK; /* Return code */ + int i; /* Iterator variable */ + sqlite3 *db = p->db; /* The database connection */ + + if( p->zContentTbl==0 ){ + const char *zLanguageid = p->zLanguageid; + char *zContentCols; /* Columns of %_content table */ + + /* Create a list of user columns for the content table */ + zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); + for(i=0; zContentCols && inColumn; i++){ + char *z = p->azColumn[i]; + zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); + } + if( zLanguageid && zContentCols ){ + zContentCols = sqlite3_mprintf("%z, langid", zContentCols, zLanguageid); + } + if( zContentCols==0 ) rc = SQLITE_NOMEM; + + /* Create the content table */ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_content'(%s)", + p->zDb, p->zName, zContentCols + ); + sqlite3_free(zContentCols); + } + + /* Create other tables */ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", + p->zDb, p->zName + ); + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_segdir'(" + "level INTEGER," + "idx INTEGER," + "start_block INTEGER," + "leaves_end_block INTEGER," + "end_block INTEGER," + "root BLOB," + "PRIMARY KEY(level, idx)" + ");", + p->zDb, p->zName + ); + if( p->bHasDocsize ){ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_docsize'(docid INTEGER PRIMARY KEY, size BLOB);", + p->zDb, p->zName + ); + } + assert( p->bHasStat==p->bFts4 ); + if( p->bHasStat ){ + sqlite3Fts3CreateStatTable(&rc, p); + } + return rc; +} + +/* +** Store the current database page-size in bytes in p->nPgsz. +** +** If *pRc is non-zero when this function is called, it is a no-op. +** Otherwise, if an error occurs, an SQLite error code is stored in *pRc +** before returning. +*/ +static void fts3DatabasePageSize(int *pRc, Fts3Table *p){ + if( *pRc==SQLITE_OK ){ + int rc; /* Return code */ + char *zSql; /* SQL text "PRAGMA %Q.page_size" */ + sqlite3_stmt *pStmt; /* Compiled "PRAGMA %Q.page_size" statement */ + + zSql = sqlite3_mprintf("PRAGMA %Q.page_size", p->zDb); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_step(pStmt); + p->nPgsz = sqlite3_column_int(pStmt, 0); + rc = sqlite3_finalize(pStmt); + }else if( rc==SQLITE_AUTH ){ + p->nPgsz = 1024; + rc = SQLITE_OK; + } + } + assert( p->nPgsz>0 || rc!=SQLITE_OK ); + sqlite3_free(zSql); + *pRc = rc; + } +} + +/* +** "Special" FTS4 arguments are column specifications of the following form: +** +** = +** +** There may not be whitespace surrounding the "=" character. The +** term may be quoted, but the may not. +*/ +static int fts3IsSpecialColumn( + const char *z, + int *pnKey, + char **pzValue +){ + char *zValue; + const char *zCsr = z; + + while( *zCsr!='=' ){ + if( *zCsr=='\0' ) return 0; + zCsr++; + } + + *pnKey = (int)(zCsr-z); + zValue = sqlite3_mprintf("%s", &zCsr[1]); + if( zValue ){ + sqlite3Fts3Dequote(zValue); + } + *pzValue = zValue; + return 1; +} + +/* +** Append the output of a printf() style formatting to an existing string. +*/ +static void fts3Appendf( + int *pRc, /* IN/OUT: Error code */ + char **pz, /* IN/OUT: Pointer to string buffer */ + const char *zFormat, /* Printf format string to append */ + ... /* Arguments for printf format string */ +){ + if( *pRc==SQLITE_OK ){ + va_list ap; + char *z; + va_start(ap, zFormat); + z = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( z && *pz ){ + char *z2 = sqlite3_mprintf("%s%s", *pz, z); + sqlite3_free(z); + z = z2; + } + if( z==0 ) *pRc = SQLITE_NOMEM; + sqlite3_free(*pz); + *pz = z; + } +} + +/* +** Return a copy of input string zInput enclosed in double-quotes (") and +** with all double quote characters escaped. For example: +** +** fts3QuoteId("un \"zip\"") -> "un \"\"zip\"\"" +** +** The pointer returned points to memory obtained from sqlite3_malloc(). It +** is the callers responsibility to call sqlite3_free() to release this +** memory. +*/ +static char *fts3QuoteId(char const *zInput){ + int nRet; + char *zRet; + nRet = 2 + (int)strlen(zInput)*2 + 1; + zRet = sqlite3_malloc(nRet); + if( zRet ){ + int i; + char *z = zRet; + *(z++) = '"'; + for(i=0; zInput[i]; i++){ + if( zInput[i]=='"' ) *(z++) = '"'; + *(z++) = zInput[i]; + } + *(z++) = '"'; + *(z++) = '\0'; + } + return zRet; +} + +/* +** Return a list of comma separated SQL expressions and a FROM clause that +** could be used in a SELECT statement such as the following: +** +** SELECT FROM %_content AS x ... +** +** to return the docid, followed by each column of text data in order +** from left to write. If parameter zFunc is not NULL, then instead of +** being returned directly each column of text data is passed to an SQL +** function named zFunc first. For example, if zFunc is "unzip" and the +** table has the three user-defined columns "a", "b", and "c", the following +** string is returned: +** +** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c') FROM %_content AS x" +** +** The pointer returned points to a buffer allocated by sqlite3_malloc(). It +** is the responsibility of the caller to eventually free it. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and +** a NULL pointer is returned). Otherwise, if an OOM error is encountered +** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If +** no error occurs, *pRc is left unmodified. +*/ +static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ + char *zRet = 0; + char *zFree = 0; + char *zFunction; + int i; + + if( p->zContentTbl==0 ){ + if( !zFunc ){ + zFunction = ""; + }else{ + zFree = zFunction = fts3QuoteId(zFunc); + } + fts3Appendf(pRc, &zRet, "docid"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); + } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", x.%Q", "langid"); + } + sqlite3_free(zFree); + }else{ + fts3Appendf(pRc, &zRet, "rowid"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]); + } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", x.%Q", p->zLanguageid); + } + } + fts3Appendf(pRc, &zRet, " FROM '%q'.'%q%s' AS x", + p->zDb, + (p->zContentTbl ? p->zContentTbl : p->zName), + (p->zContentTbl ? "" : "_content") + ); + return zRet; +} + +/* +** Return a list of N comma separated question marks, where N is the number +** of columns in the %_content table (one for the docid plus one for each +** user-defined text column). +** +** If argument zFunc is not NULL, then all but the first question mark +** is preceded by zFunc and an open bracket, and followed by a closed +** bracket. For example, if zFunc is "zip" and the FTS3 table has three +** user-defined text columns, the following string is returned: +** +** "?, zip(?), zip(?), zip(?)" +** +** The pointer returned points to a buffer allocated by sqlite3_malloc(). It +** is the responsibility of the caller to eventually free it. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and +** a NULL pointer is returned). Otherwise, if an OOM error is encountered +** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If +** no error occurs, *pRc is left unmodified. +*/ +static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ + char *zRet = 0; + char *zFree = 0; + char *zFunction; + int i; + + if( !zFunc ){ + zFunction = ""; + }else{ + zFree = zFunction = fts3QuoteId(zFunc); + } + fts3Appendf(pRc, &zRet, "?"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ",%s(?)", zFunction); + } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", ?"); + } + sqlite3_free(zFree); + return zRet; +} + +/* +** This function interprets the string at (*pp) as a non-negative integer +** value. It reads the integer and sets *pnOut to the value read, then +** sets *pp to point to the byte immediately following the last byte of +** the integer value. +** +** Only decimal digits ('0'..'9') may be part of an integer value. +** +** If *pp does not being with a decimal digit SQLITE_ERROR is returned and +** the output value undefined. Otherwise SQLITE_OK is returned. +** +** This function is used when parsing the "prefix=" FTS4 parameter. +*/ +static int fts3GobbleInt(const char **pp, int *pnOut){ + const int MAX_NPREFIX = 10000000; + const char *p; /* Iterator pointer */ + int nInt = 0; /* Output value */ + + for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ + nInt = nInt * 10 + (p[0] - '0'); + if( nInt>MAX_NPREFIX ){ + nInt = 0; + break; + } + } + if( p==*pp ) return SQLITE_ERROR; + *pnOut = nInt; + *pp = p; + return SQLITE_OK; +} + +/* +** This function is called to allocate an array of Fts3Index structures +** representing the indexes maintained by the current FTS table. FTS tables +** always maintain the main "terms" index, but may also maintain one or +** more "prefix" indexes, depending on the value of the "prefix=" parameter +** (if any) specified as part of the CREATE VIRTUAL TABLE statement. +** +** Argument zParam is passed the value of the "prefix=" option if one was +** specified, or NULL otherwise. +** +** If no error occurs, SQLITE_OK is returned and *apIndex set to point to +** the allocated array. *pnIndex is set to the number of elements in the +** array. If an error does occur, an SQLite error code is returned. +** +** Regardless of whether or not an error is returned, it is the responsibility +** of the caller to call sqlite3_free() on the output array to free it. +*/ +static int fts3PrefixParameter( + const char *zParam, /* ABC in prefix=ABC parameter to parse */ + int *pnIndex, /* OUT: size of *apIndex[] array */ + struct Fts3Index **apIndex /* OUT: Array of indexes for this table */ +){ + struct Fts3Index *aIndex; /* Allocated array */ + int nIndex = 1; /* Number of entries in array */ + + if( zParam && zParam[0] ){ + const char *p; + nIndex++; + for(p=zParam; *p; p++){ + if( *p==',' ) nIndex++; + } + } + + aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex); + *apIndex = aIndex; + if( !aIndex ){ + return SQLITE_NOMEM; + } + + memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex); + if( zParam ){ + const char *p = zParam; + int i; + for(i=1; i=0 ); + if( nPrefix==0 ){ + nIndex--; + i--; + }else{ + aIndex[i].nPrefix = nPrefix; + } + p++; + } + } + + *pnIndex = nIndex; + return SQLITE_OK; +} + +/* +** This function is called when initializing an FTS4 table that uses the +** content=xxx option. It determines the number of and names of the columns +** of the new FTS4 table. +** +** The third argument passed to this function is the value passed to the +** config=xxx option (i.e. "xxx"). This function queries the database for +** a table of that name. If found, the output variables are populated +** as follows: +** +** *pnCol: Set to the number of columns table xxx has, +** +** *pnStr: Set to the total amount of space required to store a copy +** of each columns name, including the nul-terminator. +** +** *pazCol: Set to point to an array of *pnCol strings. Each string is +** the name of the corresponding column in table xxx. The array +** and its contents are allocated using a single allocation. It +** is the responsibility of the caller to free this allocation +** by eventually passing the *pazCol value to sqlite3_free(). +** +** If the table cannot be found, an error code is returned and the output +** variables are undefined. Or, if an OOM is encountered, SQLITE_NOMEM is +** returned (and the output variables are undefined). +*/ +static int fts3ContentColumns( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of db (i.e. "main", "temp" etc.) */ + const char *zTbl, /* Name of content table */ + const char ***pazCol, /* OUT: Malloc'd array of column names */ + int *pnCol, /* OUT: Size of array *pazCol */ + int *pnStr, /* OUT: Bytes of string content */ + char **pzErr /* OUT: error message */ +){ + int rc = SQLITE_OK; /* Return code */ + char *zSql; /* "SELECT *" statement on zTbl */ + sqlite3_stmt *pStmt = 0; /* Compiled version of zSql */ + + zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zTbl); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + sqlite3Fts3ErrMsg(pzErr, "%s", sqlite3_errmsg(db)); + } + } + sqlite3_free(zSql); + + if( rc==SQLITE_OK ){ + const char **azCol; /* Output array */ + int nStr = 0; /* Size of all column names (incl. 0x00) */ + int nCol; /* Number of table columns */ + int i; /* Used to iterate through columns */ + + /* Loop through the returned columns. Set nStr to the number of bytes of + ** space required to store a copy of each column name, including the + ** nul-terminator byte. */ + nCol = sqlite3_column_count(pStmt); + for(i=0; i module name ("fts3" or "fts4") +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> "column name" and other module argument fields. +*/ +static int fts3InitVtab( + int isCreate, /* True for xCreate, false for xConnect */ + sqlite3 *db, /* The SQLite database connection */ + void *pAux, /* Hash table containing tokenizers */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ + char **pzErr /* Write any error message here */ +){ + Fts3Hash *pHash = (Fts3Hash *)pAux; + Fts3Table *p = 0; /* Pointer to allocated vtab */ + int rc = SQLITE_OK; /* Return code */ + int i; /* Iterator variable */ + int nByte; /* Size of allocation used for *p */ + int iCol; /* Column index */ + int nString = 0; /* Bytes required to hold all column names */ + int nCol = 0; /* Number of columns in the FTS table */ + char *zCsr; /* Space for holding column names */ + int nDb; /* Bytes required to hold database name */ + int nName; /* Bytes required to hold table name */ + int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ + const char **aCol; /* Array of column names */ + sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ + + int nIndex = 0; /* Size of aIndex[] array */ + struct Fts3Index *aIndex = 0; /* Array of indexes for this table */ + + /* The results of parsing supported FTS4 key=value options: */ + int bNoDocsize = 0; /* True to omit %_docsize table */ + int bDescIdx = 0; /* True to store descending indexes */ + char *zPrefix = 0; /* Prefix parameter value (or NULL) */ + char *zCompress = 0; /* compress=? parameter (or NULL) */ + char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ + char *zContent = 0; /* content=? parameter (or NULL) */ + char *zLanguageid = 0; /* languageid=? parameter (or NULL) */ + char **azNotindexed = 0; /* The set of notindexed= columns */ + int nNotindexed = 0; /* Size of azNotindexed[] array */ + + assert( strlen(argv[0])==4 ); + assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) + || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) + ); + + nDb = (int)strlen(argv[1]) + 1; + nName = (int)strlen(argv[2]) + 1; + + nByte = sizeof(const char *) * (argc-2); + aCol = (const char **)sqlite3_malloc(nByte); + if( aCol ){ + memset((void*)aCol, 0, nByte); + azNotindexed = (char **)sqlite3_malloc(nByte); + } + if( azNotindexed ){ + memset(azNotindexed, 0, nByte); + } + if( !aCol || !azNotindexed ){ + rc = SQLITE_NOMEM; + goto fts3_init_out; + } + + /* Loop through all of the arguments passed by the user to the FTS3/4 + ** module (i.e. all the column names and special arguments). This loop + ** does the following: + ** + ** + Figures out the number of columns the FTSX table will have, and + ** the number of bytes of space that must be allocated to store copies + ** of the column names. + ** + ** + If there is a tokenizer specification included in the arguments, + ** initializes the tokenizer pTokenizer. + */ + for(i=3; rc==SQLITE_OK && i8 + && 0==sqlite3_strnicmp(z, "tokenize", 8) + && 0==sqlite3Fts3IsIdChar(z[8]) + ){ + rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr); + } + + /* Check if it is an FTS4 special argument. */ + else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){ + struct Fts4Option { + const char *zOpt; + int nOpt; + } aFts4Opt[] = { + { "matchinfo", 9 }, /* 0 -> MATCHINFO */ + { "prefix", 6 }, /* 1 -> PREFIX */ + { "compress", 8 }, /* 2 -> COMPRESS */ + { "uncompress", 10 }, /* 3 -> UNCOMPRESS */ + { "order", 5 }, /* 4 -> ORDER */ + { "content", 7 }, /* 5 -> CONTENT */ + { "languageid", 10 }, /* 6 -> LANGUAGEID */ + { "notindexed", 10 } /* 7 -> NOTINDEXED */ + }; + + int iOpt; + if( !zVal ){ + rc = SQLITE_NOMEM; + }else{ + for(iOpt=0; iOptnOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){ + break; + } + } + if( iOpt==SizeofArray(aFts4Opt) ){ + sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z); + rc = SQLITE_ERROR; + }else{ + switch( iOpt ){ + case 0: /* MATCHINFO */ + if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){ + sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal); + rc = SQLITE_ERROR; + } + bNoDocsize = 1; + break; + + case 1: /* PREFIX */ + sqlite3_free(zPrefix); + zPrefix = zVal; + zVal = 0; + break; + + case 2: /* COMPRESS */ + sqlite3_free(zCompress); + zCompress = zVal; + zVal = 0; + break; + + case 3: /* UNCOMPRESS */ + sqlite3_free(zUncompress); + zUncompress = zVal; + zVal = 0; + break; + + case 4: /* ORDER */ + if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) + && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) + ){ + sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal); + rc = SQLITE_ERROR; + } + bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); + break; + + case 5: /* CONTENT */ + sqlite3_free(zContent); + zContent = zVal; + zVal = 0; + break; + + case 6: /* LANGUAGEID */ + assert( iOpt==6 ); + sqlite3_free(zLanguageid); + zLanguageid = zVal; + zVal = 0; + break; + + case 7: /* NOTINDEXED */ + azNotindexed[nNotindexed++] = zVal; + zVal = 0; + break; + } + } + sqlite3_free(zVal); + } + } + + /* Otherwise, the argument is a column name. */ + else { + nString += (int)(strlen(z) + 1); + aCol[nCol++] = z; + } + } + + /* If a content=xxx option was specified, the following: + ** + ** 1. Ignore any compress= and uncompress= options. + ** + ** 2. If no column names were specified as part of the CREATE VIRTUAL + ** TABLE statement, use all columns from the content table. + */ + if( rc==SQLITE_OK && zContent ){ + sqlite3_free(zCompress); + sqlite3_free(zUncompress); + zCompress = 0; + zUncompress = 0; + if( nCol==0 ){ + sqlite3_free((void*)aCol); + aCol = 0; + rc = fts3ContentColumns(db, argv[1], zContent,&aCol,&nCol,&nString,pzErr); + + /* If a languageid= option was specified, remove the language id + ** column from the aCol[] array. */ + if( rc==SQLITE_OK && zLanguageid ){ + int j; + for(j=0; jdb = db; + p->nColumn = nCol; + p->nPendingData = 0; + p->azColumn = (char **)&p[1]; + p->pTokenizer = pTokenizer; + p->nMaxPendingData = FTS3_MAX_PENDING_DATA; + p->bHasDocsize = (isFts4 && bNoDocsize==0); + p->bHasStat = isFts4; + p->bFts4 = isFts4; + p->bDescIdx = bDescIdx; + p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */ + p->zContentTbl = zContent; + p->zLanguageid = zLanguageid; + zContent = 0; + zLanguageid = 0; + TESTONLY( p->inTransaction = -1 ); + TESTONLY( p->mxSavepoint = -1 ); + + p->aIndex = (struct Fts3Index *)&p->azColumn[nCol]; + memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex); + p->nIndex = nIndex; + for(i=0; iaIndex[i].hPending, FTS3_HASH_STRING, 1); + } + p->abNotindexed = (u8 *)&p->aIndex[nIndex]; + + /* Fill in the zName and zDb fields of the vtab structure. */ + zCsr = (char *)&p->abNotindexed[nCol]; + p->zName = zCsr; + memcpy(zCsr, argv[2], nName); + zCsr += nName; + p->zDb = zCsr; + memcpy(zCsr, argv[1], nDb); + zCsr += nDb; + + /* Fill in the azColumn array */ + for(iCol=0; iColazColumn[iCol] = zCsr; + zCsr += n+1; + assert( zCsr <= &((char *)p)[nByte] ); + } + + /* Fill in the abNotindexed array */ + for(iCol=0; iColazColumn[iCol]); + for(i=0; iazColumn[iCol], zNot, n) + ){ + p->abNotindexed[iCol] = 1; + sqlite3_free(zNot); + azNotindexed[i] = 0; + } + } + } + for(i=0; izReadExprlist = fts3ReadExprList(p, zUncompress, &rc); + p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc); + if( rc!=SQLITE_OK ) goto fts3_init_out; + + /* If this is an xCreate call, create the underlying tables in the + ** database. TODO: For xConnect(), it could verify that said tables exist. + */ + if( isCreate ){ + rc = fts3CreateTables(p); + } + + /* Check to see if a legacy fts3 table has been "upgraded" by the + ** addition of a %_stat table so that it can use incremental merge. + */ + if( !isFts4 && !isCreate ){ + p->bHasStat = 2; + } + + /* Figure out the page-size for the database. This is required in order to + ** estimate the cost of loading large doclists from the database. */ + fts3DatabasePageSize(&rc, p); + p->nNodeSize = p->nPgsz-35; + + /* Declare the table schema to SQLite. */ + fts3DeclareVtab(&rc, p); + +fts3_init_out: + sqlite3_free(zPrefix); + sqlite3_free(aIndex); + sqlite3_free(zCompress); + sqlite3_free(zUncompress); + sqlite3_free(zContent); + sqlite3_free(zLanguageid); + for(i=0; ipModule->xDestroy(pTokenizer); + } + }else{ + assert( p->pSegments==0 ); + *ppVTab = &p->base; + } + return rc; +} + +/* +** The xConnect() and xCreate() methods for the virtual table. All the +** work is done in function fts3InitVtab(). +*/ +static int fts3ConnectMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts3InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr); +} +static int fts3CreateMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr); +} + +/* +** Set the pIdxInfo->estimatedRows variable to nRow. Unless this +** extension is currently being used by a version of SQLite too old to +** support estimatedRows. In that case this function is a no-op. +*/ +static void fts3SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ +#if SQLITE_VERSION_NUMBER>=3008002 + if( sqlite3_libversion_number()>=3008002 ){ + pIdxInfo->estimatedRows = nRow; + } +#endif +} + +/* +** Set the SQLITE_INDEX_SCAN_UNIQUE flag in pIdxInfo->flags. Unless this +** extension is currently being used by a version of SQLite too old to +** support index-info flags. In that case this function is a no-op. +*/ +static void fts3SetUniqueFlag(sqlite3_index_info *pIdxInfo){ +#if SQLITE_VERSION_NUMBER>=3008012 + if( sqlite3_libversion_number()>=3008012 ){ + pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE; + } +#endif +} + +/* +** Implementation of the xBestIndex method for FTS3 tables. There +** are three possible strategies, in order of preference: +** +** 1. Direct lookup by rowid or docid. +** 2. Full-text search using a MATCH operator on a non-docid column. +** 3. Linear scan of %_content table. +*/ +static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ + Fts3Table *p = (Fts3Table *)pVTab; + int i; /* Iterator variable */ + int iCons = -1; /* Index of constraint to use */ + + int iLangidCons = -1; /* Index of langid=x constraint, if present */ + int iDocidGe = -1; /* Index of docid>=x constraint, if present */ + int iDocidLe = -1; /* Index of docid<=x constraint, if present */ + int iIdx; + + /* By default use a full table scan. This is an expensive option, + ** so search through the constraints to see if a more efficient + ** strategy is possible. + */ + pInfo->idxNum = FTS3_FULLSCAN_SEARCH; + pInfo->estimatedCost = 5000000; + for(i=0; inConstraint; i++){ + int bDocid; /* True if this constraint is on docid */ + struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i]; + if( pCons->usable==0 ){ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH ){ + /* There exists an unusable MATCH constraint. This means that if + ** the planner does elect to use the results of this call as part + ** of the overall query plan the user will see an "unable to use + ** function MATCH in the requested context" error. To discourage + ** this, return a very high cost here. */ + pInfo->idxNum = FTS3_FULLSCAN_SEARCH; + pInfo->estimatedCost = 1e50; + fts3SetEstimatedRows(pInfo, ((sqlite3_int64)1) << 50); + return SQLITE_OK; + } + continue; + } + + bDocid = (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1); + + /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */ + if( iCons<0 && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ && bDocid ){ + pInfo->idxNum = FTS3_DOCID_SEARCH; + pInfo->estimatedCost = 1.0; + iCons = i; + } + + /* A MATCH constraint. Use a full-text search. + ** + ** If there is more than one MATCH constraint available, use the first + ** one encountered. If there is both a MATCH constraint and a direct + ** rowid/docid lookup, prefer the MATCH strategy. This is done even + ** though the rowid/docid lookup is faster than a MATCH query, selecting + ** it would lead to an "unable to use function MATCH in the requested + ** context" error. + */ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH + && pCons->iColumn>=0 && pCons->iColumn<=p->nColumn + ){ + pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn; + pInfo->estimatedCost = 2.0; + iCons = i; + } + + /* Equality constraint on the langid column */ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ + && pCons->iColumn==p->nColumn + 2 + ){ + iLangidCons = i; + } + + if( bDocid ){ + switch( pCons->op ){ + case SQLITE_INDEX_CONSTRAINT_GE: + case SQLITE_INDEX_CONSTRAINT_GT: + iDocidGe = i; + break; + + case SQLITE_INDEX_CONSTRAINT_LE: + case SQLITE_INDEX_CONSTRAINT_LT: + iDocidLe = i; + break; + } + } + } + + /* If using a docid=? or rowid=? strategy, set the UNIQUE flag. */ + if( pInfo->idxNum==FTS3_DOCID_SEARCH ) fts3SetUniqueFlag(pInfo); + + iIdx = 1; + if( iCons>=0 ){ + pInfo->aConstraintUsage[iCons].argvIndex = iIdx++; + pInfo->aConstraintUsage[iCons].omit = 1; + } + if( iLangidCons>=0 ){ + pInfo->idxNum |= FTS3_HAVE_LANGID; + pInfo->aConstraintUsage[iLangidCons].argvIndex = iIdx++; + } + if( iDocidGe>=0 ){ + pInfo->idxNum |= FTS3_HAVE_DOCID_GE; + pInfo->aConstraintUsage[iDocidGe].argvIndex = iIdx++; + } + if( iDocidLe>=0 ){ + pInfo->idxNum |= FTS3_HAVE_DOCID_LE; + pInfo->aConstraintUsage[iDocidLe].argvIndex = iIdx++; + } + + /* Regardless of the strategy selected, FTS can deliver rows in rowid (or + ** docid) order. Both ascending and descending are possible. + */ + if( pInfo->nOrderBy==1 ){ + struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0]; + if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){ + if( pOrder->desc ){ + pInfo->idxStr = "DESC"; + }else{ + pInfo->idxStr = "ASC"; + } + pInfo->orderByConsumed = 1; + } + } + + assert( p->pSegments==0 ); + return SQLITE_OK; +} + +/* +** Implementation of xOpen method. +*/ +static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + sqlite3_vtab_cursor *pCsr; /* Allocated cursor */ + + UNUSED_PARAMETER(pVTab); + + /* Allocate a buffer large enough for an Fts3Cursor structure. If the + ** allocation succeeds, zero it and return SQLITE_OK. Otherwise, + ** if the allocation fails, return SQLITE_NOMEM. + */ + *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor)); + if( !pCsr ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(Fts3Cursor)); + return SQLITE_OK; +} + +/* +** Close the cursor. For additional information see the documentation +** on the xClose method of the virtual table interface. +*/ +static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + sqlite3_finalize(pCsr->pStmt); + sqlite3Fts3ExprFree(pCsr->pExpr); + sqlite3Fts3FreeDeferredTokens(pCsr); + sqlite3_free(pCsr->aDoclist); + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then +** compose and prepare an SQL statement of the form: +** +** "SELECT FROM %_content WHERE rowid = ?" +** +** (or the equivalent for a content=xxx table) and set pCsr->pStmt to +** it. If an error occurs, return an SQLite error code. +** +** Otherwise, set *ppStmt to point to pCsr->pStmt and return SQLITE_OK. +*/ +static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){ + int rc = SQLITE_OK; + if( pCsr->pStmt==0 ){ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + char *zSql; + zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); + if( !zSql ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + } + *ppStmt = pCsr->pStmt; + return rc; +} + +/* +** Position the pCsr->pStmt statement so that it is on the row +** of the %_content table that contains the last match. Return +** SQLITE_OK on success. +*/ +static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ + int rc = SQLITE_OK; + if( pCsr->isRequireSeek ){ + sqlite3_stmt *pStmt = 0; + + rc = fts3CursorSeekStmt(pCsr, &pStmt); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); + pCsr->isRequireSeek = 0; + if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ + return SQLITE_OK; + }else{ + rc = sqlite3_reset(pCsr->pStmt); + if( rc==SQLITE_OK && ((Fts3Table *)pCsr->base.pVtab)->zContentTbl==0 ){ + /* If no row was found and no error has occurred, then the %_content + ** table is missing a row that is present in the full-text index. + ** The data structures are corrupt. */ + rc = FTS_CORRUPT_VTAB; + pCsr->isEof = 1; + } + } + } + } + + if( rc!=SQLITE_OK && pContext ){ + sqlite3_result_error_code(pContext, rc); + } + return rc; +} + +/* +** This function is used to process a single interior node when searching +** a b-tree for a term or term prefix. The node data is passed to this +** function via the zNode/nNode parameters. The term to search for is +** passed in zTerm/nTerm. +** +** If piFirst is not NULL, then this function sets *piFirst to the blockid +** of the child node that heads the sub-tree that may contain the term. +** +** If piLast is not NULL, then *piLast is set to the right-most child node +** that heads a sub-tree that may contain a term for which zTerm/nTerm is +** a prefix. +** +** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. +*/ +static int fts3ScanInteriorNode( + const char *zTerm, /* Term to select leaves for */ + int nTerm, /* Size of term zTerm in bytes */ + const char *zNode, /* Buffer containing segment interior node */ + int nNode, /* Size of buffer at zNode */ + sqlite3_int64 *piFirst, /* OUT: Selected child node */ + sqlite3_int64 *piLast /* OUT: Selected child node */ +){ + int rc = SQLITE_OK; /* Return code */ + const char *zCsr = zNode; /* Cursor to iterate through node */ + const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ + char *zBuffer = 0; /* Buffer to load terms into */ + int nAlloc = 0; /* Size of allocated buffer */ + int isFirstTerm = 1; /* True when processing first term on page */ + sqlite3_int64 iChild; /* Block id of child node to descend to */ + + /* Skip over the 'height' varint that occurs at the start of every + ** interior node. Then load the blockid of the left-child of the b-tree + ** node into variable iChild. + ** + ** Even if the data structure on disk is corrupted, this (reading two + ** varints from the buffer) does not risk an overread. If zNode is a + ** root node, then the buffer comes from a SELECT statement. SQLite does + ** not make this guarantee explicitly, but in practice there are always + ** either more than 20 bytes of allocated space following the nNode bytes of + ** contents, or two zero bytes. Or, if the node is read from the %_segments + ** table, then there are always 20 bytes of zeroed padding following the + ** nNode bytes of content (see sqlite3Fts3ReadBlock() for details). + */ + zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); + zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); + if( zCsr>zEnd ){ + return FTS_CORRUPT_VTAB; + } + + while( zCsrzEnd ){ + rc = FTS_CORRUPT_VTAB; + goto finish_scan; + } + if( nPrefix+nSuffix>nAlloc ){ + char *zNew; + nAlloc = (nPrefix+nSuffix) * 2; + zNew = (char *)sqlite3_realloc(zBuffer, nAlloc); + if( !zNew ){ + rc = SQLITE_NOMEM; + goto finish_scan; + } + zBuffer = zNew; + } + assert( zBuffer ); + memcpy(&zBuffer[nPrefix], zCsr, nSuffix); + nBuffer = nPrefix + nSuffix; + zCsr += nSuffix; + + /* Compare the term we are searching for with the term just loaded from + ** the interior node. If the specified term is greater than or equal + ** to the term from the interior node, then all terms on the sub-tree + ** headed by node iChild are smaller than zTerm. No need to search + ** iChild. + ** + ** If the interior node term is larger than the specified term, then + ** the tree headed by iChild may contain the specified term. + */ + cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); + if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){ + *piFirst = iChild; + piFirst = 0; + } + + if( piLast && cmp<0 ){ + *piLast = iChild; + piLast = 0; + } + + iChild++; + }; + + if( piFirst ) *piFirst = iChild; + if( piLast ) *piLast = iChild; + + finish_scan: + sqlite3_free(zBuffer); + return rc; +} + + +/* +** The buffer pointed to by argument zNode (size nNode bytes) contains an +** interior node of a b-tree segment. The zTerm buffer (size nTerm bytes) +** contains a term. This function searches the sub-tree headed by the zNode +** node for the range of leaf nodes that may contain the specified term +** or terms for which the specified term is a prefix. +** +** If piLeaf is not NULL, then *piLeaf is set to the blockid of the +** left-most leaf node in the tree that may contain the specified term. +** If piLeaf2 is not NULL, then *piLeaf2 is set to the blockid of the +** right-most leaf node that may contain a term for which the specified +** term is a prefix. +** +** It is possible that the range of returned leaf nodes does not contain +** the specified term or any terms for which it is a prefix. However, if the +** segment does contain any such terms, they are stored within the identified +** range. Because this function only inspects interior segment nodes (and +** never loads leaf nodes into memory), it is not possible to be sure. +** +** If an error occurs, an error code other than SQLITE_OK is returned. +*/ +static int fts3SelectLeaf( + Fts3Table *p, /* Virtual table handle */ + const char *zTerm, /* Term to select leaves for */ + int nTerm, /* Size of term zTerm in bytes */ + const char *zNode, /* Buffer containing segment interior node */ + int nNode, /* Size of buffer at zNode */ + sqlite3_int64 *piLeaf, /* Selected leaf node */ + sqlite3_int64 *piLeaf2 /* Selected leaf node */ +){ + int rc = SQLITE_OK; /* Return code */ + int iHeight; /* Height of this node in tree */ + + assert( piLeaf || piLeaf2 ); + + fts3GetVarint32(zNode, &iHeight); + rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2); + assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) ); + + if( rc==SQLITE_OK && iHeight>1 ){ + char *zBlob = 0; /* Blob read from %_segments table */ + int nBlob = 0; /* Size of zBlob in bytes */ + + if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ + rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0); + if( rc==SQLITE_OK ){ + rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0); + } + sqlite3_free(zBlob); + piLeaf = 0; + zBlob = 0; + } + + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0); + } + if( rc==SQLITE_OK ){ + rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); + } + sqlite3_free(zBlob); + } + + return rc; +} + +/* +** This function is used to create delta-encoded serialized lists of FTS3 +** varints. Each call to this function appends a single varint to a list. +*/ +static void fts3PutDeltaVarint( + char **pp, /* IN/OUT: Output pointer */ + sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ + sqlite3_int64 iVal /* Write this value to the list */ +){ + assert( iVal-*piPrev > 0 || (*piPrev==0 && iVal==0) ); + *pp += sqlite3Fts3PutVarint(*pp, iVal-*piPrev); + *piPrev = iVal; +} + +/* +** When this function is called, *ppPoslist is assumed to point to the +** start of a position-list. After it returns, *ppPoslist points to the +** first byte after the position-list. +** +** A position list is list of positions (delta encoded) and columns for +** a single document record of a doclist. So, in other words, this +** routine advances *ppPoslist so that it points to the next docid in +** the doclist, or to the first byte past the end of the doclist. +** +** If pp is not NULL, then the contents of the position list are copied +** to *pp. *pp is set to point to the first byte past the last byte copied +** before this function returns. +*/ +static void fts3PoslistCopy(char **pp, char **ppPoslist){ + char *pEnd = *ppPoslist; + char c = 0; + + /* The end of a position list is marked by a zero encoded as an FTS3 + ** varint. A single POS_END (0) byte. Except, if the 0 byte is preceded by + ** a byte with the 0x80 bit set, then it is not a varint 0, but the tail + ** of some other, multi-byte, value. + ** + ** The following while-loop moves pEnd to point to the first byte that is not + ** immediately preceded by a byte with the 0x80 bit set. Then increments + ** pEnd once more so that it points to the byte immediately following the + ** last byte in the position-list. + */ + while( *pEnd | c ){ + c = *pEnd++ & 0x80; + testcase( c!=0 && (*pEnd)==0 ); + } + pEnd++; /* Advance past the POS_END terminator byte */ + + if( pp ){ + int n = (int)(pEnd - *ppPoslist); + char *p = *pp; + memcpy(p, *ppPoslist, n); + p += n; + *pp = p; + } + *ppPoslist = pEnd; +} + +/* +** When this function is called, *ppPoslist is assumed to point to the +** start of a column-list. After it returns, *ppPoslist points to the +** to the terminator (POS_COLUMN or POS_END) byte of the column-list. +** +** A column-list is list of delta-encoded positions for a single column +** within a single document within a doclist. +** +** The column-list is terminated either by a POS_COLUMN varint (1) or +** a POS_END varint (0). This routine leaves *ppPoslist pointing to +** the POS_COLUMN or POS_END that terminates the column-list. +** +** If pp is not NULL, then the contents of the column-list are copied +** to *pp. *pp is set to point to the first byte past the last byte copied +** before this function returns. The POS_COLUMN or POS_END terminator +** is not copied into *pp. +*/ +static void fts3ColumnlistCopy(char **pp, char **ppPoslist){ + char *pEnd = *ppPoslist; + char c = 0; + + /* A column-list is terminated by either a 0x01 or 0x00 byte that is + ** not part of a multi-byte varint. + */ + while( 0xFE & (*pEnd | c) ){ + c = *pEnd++ & 0x80; + testcase( c!=0 && ((*pEnd)&0xfe)==0 ); + } + if( pp ){ + int n = (int)(pEnd - *ppPoslist); + char *p = *pp; + memcpy(p, *ppPoslist, n); + p += n; + *pp = p; + } + *ppPoslist = pEnd; +} + +/* +** Value used to signify the end of an position-list. This is safe because +** it is not possible to have a document with 2^31 terms. +*/ +#define POSITION_LIST_END 0x7fffffff + +/* +** This function is used to help parse position-lists. When this function is +** called, *pp may point to the start of the next varint in the position-list +** being parsed, or it may point to 1 byte past the end of the position-list +** (in which case **pp will be a terminator bytes POS_END (0) or +** (1)). +** +** If *pp points past the end of the current position-list, set *pi to +** POSITION_LIST_END and return. Otherwise, read the next varint from *pp, +** increment the current value of *pi by the value read, and set *pp to +** point to the next value before returning. +** +** Before calling this routine *pi must be initialized to the value of +** the previous position, or zero if we are reading the first position +** in the position-list. Because positions are delta-encoded, the value +** of the previous position is needed in order to compute the value of +** the next position. +*/ +static void fts3ReadNextPos( + char **pp, /* IN/OUT: Pointer into position-list buffer */ + sqlite3_int64 *pi /* IN/OUT: Value read from position-list */ +){ + if( (**pp)&0xFE ){ + fts3GetDeltaVarint(pp, pi); + *pi -= 2; + }else{ + *pi = POSITION_LIST_END; + } +} + +/* +** If parameter iCol is not 0, write an POS_COLUMN (1) byte followed by +** the value of iCol encoded as a varint to *pp. This will start a new +** column list. +** +** Set *pp to point to the byte just after the last byte written before +** returning (do not modify it if iCol==0). Return the total number of bytes +** written (0 if iCol==0). +*/ +static int fts3PutColNumber(char **pp, int iCol){ + int n = 0; /* Number of bytes written */ + if( iCol ){ + char *p = *pp; /* Output pointer */ + n = 1 + sqlite3Fts3PutVarint(&p[1], iCol); + *p = 0x01; + *pp = &p[n]; + } + return n; +} + +/* +** Compute the union of two position lists. The output written +** into *pp contains all positions of both *pp1 and *pp2 in sorted +** order and with any duplicates removed. All pointers are +** updated appropriately. The caller is responsible for insuring +** that there is enough space in *pp to hold the complete output. +*/ +static void fts3PoslistMerge( + char **pp, /* Output buffer */ + char **pp1, /* Left input list */ + char **pp2 /* Right input list */ +){ + char *p = *pp; + char *p1 = *pp1; + char *p2 = *pp2; + + while( *p1 || *p2 ){ + int iCol1; /* The current column index in pp1 */ + int iCol2; /* The current column index in pp2 */ + + if( *p1==POS_COLUMN ) fts3GetVarint32(&p1[1], &iCol1); + else if( *p1==POS_END ) iCol1 = POSITION_LIST_END; + else iCol1 = 0; + + if( *p2==POS_COLUMN ) fts3GetVarint32(&p2[1], &iCol2); + else if( *p2==POS_END ) iCol2 = POSITION_LIST_END; + else iCol2 = 0; + + if( iCol1==iCol2 ){ + sqlite3_int64 i1 = 0; /* Last position from pp1 */ + sqlite3_int64 i2 = 0; /* Last position from pp2 */ + sqlite3_int64 iPrev = 0; + int n = fts3PutColNumber(&p, iCol1); + p1 += n; + p2 += n; + + /* At this point, both p1 and p2 point to the start of column-lists + ** for the same column (the column with index iCol1 and iCol2). + ** A column-list is a list of non-negative delta-encoded varints, each + ** incremented by 2 before being stored. Each list is terminated by a + ** POS_END (0) or POS_COLUMN (1). The following block merges the two lists + ** and writes the results to buffer p. p is left pointing to the byte + ** after the list written. No terminator (POS_END or POS_COLUMN) is + ** written to the output. + */ + fts3GetDeltaVarint(&p1, &i1); + fts3GetDeltaVarint(&p2, &i2); + do { + fts3PutDeltaVarint(&p, &iPrev, (i1pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e. +** when the *pp1 token appears before the *pp2 token, but not more than nToken +** slots before it. +** +** e.g. nToken==1 searches for adjacent positions. +*/ +static int fts3PoslistPhraseMerge( + char **pp, /* IN/OUT: Preallocated output buffer */ + int nToken, /* Maximum difference in token positions */ + int isSaveLeft, /* Save the left position */ + int isExact, /* If *pp1 is exactly nTokens before *pp2 */ + char **pp1, /* IN/OUT: Left input list */ + char **pp2 /* IN/OUT: Right input list */ +){ + char *p = *pp; + char *p1 = *pp1; + char *p2 = *pp2; + int iCol1 = 0; + int iCol2 = 0; + + /* Never set both isSaveLeft and isExact for the same invocation. */ + assert( isSaveLeft==0 || isExact==0 ); + + assert( p!=0 && *p1!=0 && *p2!=0 ); + if( *p1==POS_COLUMN ){ + p1++; + p1 += fts3GetVarint32(p1, &iCol1); + } + if( *p2==POS_COLUMN ){ + p2++; + p2 += fts3GetVarint32(p2, &iCol2); + } + + while( 1 ){ + if( iCol1==iCol2 ){ + char *pSave = p; + sqlite3_int64 iPrev = 0; + sqlite3_int64 iPos1 = 0; + sqlite3_int64 iPos2 = 0; + + if( iCol1 ){ + *p++ = POS_COLUMN; + p += sqlite3Fts3PutVarint(p, iCol1); + } + + assert( *p1!=POS_END && *p1!=POS_COLUMN ); + assert( *p2!=POS_END && *p2!=POS_COLUMN ); + fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; + fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; + + while( 1 ){ + if( iPos2==iPos1+nToken + || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken) + ){ + sqlite3_int64 iSave; + iSave = isSaveLeft ? iPos1 : iPos2; + fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2; + pSave = 0; + assert( p ); + } + if( (!isSaveLeft && iPos2<=(iPos1+nToken)) || iPos2<=iPos1 ){ + if( (*p2&0xFE)==0 ) break; + fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; + }else{ + if( (*p1&0xFE)==0 ) break; + fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; + } + } + + if( pSave ){ + assert( pp && p ); + p = pSave; + } + + fts3ColumnlistCopy(0, &p1); + fts3ColumnlistCopy(0, &p2); + assert( (*p1&0xFE)==0 && (*p2&0xFE)==0 ); + if( 0==*p1 || 0==*p2 ) break; + + p1++; + p1 += fts3GetVarint32(p1, &iCol1); + p2++; + p2 += fts3GetVarint32(p2, &iCol2); + } + + /* Advance pointer p1 or p2 (whichever corresponds to the smaller of + ** iCol1 and iCol2) so that it points to either the 0x00 that marks the + ** end of the position list, or the 0x01 that precedes the next + ** column-number in the position list. + */ + else if( iCol1=pEnd ){ + *pp = 0; + }else{ + sqlite3_int64 iVal; + *pp += sqlite3Fts3GetVarint(*pp, &iVal); + if( bDescIdx ){ + *pVal -= iVal; + }else{ + *pVal += iVal; + } + } +} + +/* +** This function is used to write a single varint to a buffer. The varint +** is written to *pp. Before returning, *pp is set to point 1 byte past the +** end of the value written. +** +** If *pbFirst is zero when this function is called, the value written to +** the buffer is that of parameter iVal. +** +** If *pbFirst is non-zero when this function is called, then the value +** written is either (iVal-*piPrev) (if bDescIdx is zero) or (*piPrev-iVal) +** (if bDescIdx is non-zero). +** +** Before returning, this function always sets *pbFirst to 1 and *piPrev +** to the value of parameter iVal. +*/ +static void fts3PutDeltaVarint3( + char **pp, /* IN/OUT: Output pointer */ + int bDescIdx, /* True for descending docids */ + sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ + int *pbFirst, /* IN/OUT: True after first int written */ + sqlite3_int64 iVal /* Write this value to the list */ +){ + sqlite3_int64 iWrite; + if( bDescIdx==0 || *pbFirst==0 ){ + iWrite = iVal - *piPrev; + }else{ + iWrite = *piPrev - iVal; + } + assert( *pbFirst || *piPrev==0 ); + assert( *pbFirst==0 || iWrite>0 ); + *pp += sqlite3Fts3PutVarint(*pp, iWrite); + *piPrev = iVal; + *pbFirst = 1; +} + + +/* +** This macro is used by various functions that merge doclists. The two +** arguments are 64-bit docid values. If the value of the stack variable +** bDescDoclist is 0 when this macro is invoked, then it returns (i1-i2). +** Otherwise, (i2-i1). +** +** Using this makes it easier to write code that can merge doclists that are +** sorted in either ascending or descending order. +*/ +#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1-i2)) + +/* +** This function does an "OR" merge of two doclists (output contains all +** positions contained in either argument doclist). If the docids in the +** input doclists are sorted in ascending order, parameter bDescDoclist +** should be false. If they are sorted in ascending order, it should be +** passed a non-zero value. +** +** If no error occurs, *paOut is set to point at an sqlite3_malloc'd buffer +** containing the output doclist and SQLITE_OK is returned. In this case +** *pnOut is set to the number of bytes in the output doclist. +** +** If an error occurs, an SQLite error code is returned. The output values +** are undefined in this case. +*/ +static int fts3DoclistOrMerge( + int bDescDoclist, /* True if arguments are desc */ + char *a1, int n1, /* First doclist */ + char *a2, int n2, /* Second doclist */ + char **paOut, int *pnOut /* OUT: Malloc'd doclist */ +){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *pEnd1 = &a1[n1]; + char *pEnd2 = &a2[n2]; + char *p1 = a1; + char *p2 = a2; + char *p; + char *aOut; + int bFirstOut = 0; + + *paOut = 0; + *pnOut = 0; + + /* Allocate space for the output. Both the input and output doclists + ** are delta encoded. If they are in ascending order (bDescDoclist==0), + ** then the first docid in each list is simply encoded as a varint. For + ** each subsequent docid, the varint stored is the difference between the + ** current and previous docid (a positive number - since the list is in + ** ascending order). + ** + ** The first docid written to the output is therefore encoded using the + ** same number of bytes as it is in whichever of the input lists it is + ** read from. And each subsequent docid read from the same input list + ** consumes either the same or less bytes as it did in the input (since + ** the difference between it and the previous value in the output must + ** be a positive value less than or equal to the delta value read from + ** the input list). The same argument applies to all but the first docid + ** read from the 'other' list. And to the contents of all position lists + ** that will be copied and merged from the input to the output. + ** + ** However, if the first docid copied to the output is a negative number, + ** then the encoding of the first docid from the 'other' input list may + ** be larger in the output than it was in the input (since the delta value + ** may be a larger positive integer than the actual docid). + ** + ** The space required to store the output is therefore the sum of the + ** sizes of the two inputs, plus enough space for exactly one of the input + ** docids to grow. + ** + ** A symetric argument may be made if the doclists are in descending + ** order. + */ + aOut = sqlite3_malloc(n1+n2+FTS3_VARINT_MAX-1); + if( !aOut ) return SQLITE_NOMEM; + + p = aOut; + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + while( p1 || p2 ){ + sqlite3_int64 iDiff = DOCID_CMP(i1, i2); + + if( p2 && p1 && iDiff==0 ){ + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); + fts3PoslistMerge(&p, &p1, &p2); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + }else if( !p2 || (p1 && iDiff<0) ){ + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); + fts3PoslistCopy(&p, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + }else{ + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i2); + fts3PoslistCopy(&p, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + } + } + + *paOut = aOut; + *pnOut = (int)(p-aOut); + assert( *pnOut<=n1+n2+FTS3_VARINT_MAX-1 ); + return SQLITE_OK; +} + +/* +** This function does a "phrase" merge of two doclists. In a phrase merge, +** the output contains a copy of each position from the right-hand input +** doclist for which there is a position in the left-hand input doclist +** exactly nDist tokens before it. +** +** If the docids in the input doclists are sorted in ascending order, +** parameter bDescDoclist should be false. If they are sorted in ascending +** order, it should be passed a non-zero value. +** +** The right-hand input doclist is overwritten by this function. +*/ +static int fts3DoclistPhraseMerge( + int bDescDoclist, /* True if arguments are desc */ + int nDist, /* Distance from left to right (1=adjacent) */ + char *aLeft, int nLeft, /* Left doclist */ + char **paRight, int *pnRight /* IN/OUT: Right/output doclist */ +){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *aRight = *paRight; + char *pEnd1 = &aLeft[nLeft]; + char *pEnd2 = &aRight[*pnRight]; + char *p1 = aLeft; + char *p2 = aRight; + char *p; + int bFirstOut = 0; + char *aOut; + + assert( nDist>0 ); + if( bDescDoclist ){ + aOut = sqlite3_malloc(*pnRight + FTS3_VARINT_MAX); + if( aOut==0 ) return SQLITE_NOMEM; + }else{ + aOut = aRight; + } + p = aOut; + + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + + while( p1 && p2 ){ + sqlite3_int64 iDiff = DOCID_CMP(i1, i2); + if( iDiff==0 ){ + char *pSave = p; + sqlite3_int64 iPrevSave = iPrev; + int bFirstOutSave = bFirstOut; + + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); + if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){ + p = pSave; + iPrev = iPrevSave; + bFirstOut = bFirstOutSave; + } + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + }else if( iDiff<0 ){ + fts3PoslistCopy(0, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + }else{ + fts3PoslistCopy(0, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + } + } + + *pnRight = (int)(p - aOut); + if( bDescDoclist ){ + sqlite3_free(aRight); + *paRight = aOut; + } + + return SQLITE_OK; +} + +/* +** Argument pList points to a position list nList bytes in size. This +** function checks to see if the position list contains any entries for +** a token in position 0 (of any column). If so, it writes argument iDelta +** to the output buffer pOut, followed by a position list consisting only +** of the entries from pList at position 0, and terminated by an 0x00 byte. +** The value returned is the number of bytes written to pOut (if any). +*/ +SQLITE_PRIVATE int sqlite3Fts3FirstFilter( + sqlite3_int64 iDelta, /* Varint that may be written to pOut */ + char *pList, /* Position list (no 0x00 term) */ + int nList, /* Size of pList in bytes */ + char *pOut /* Write output here */ +){ + int nOut = 0; + int bWritten = 0; /* True once iDelta has been written */ + char *p = pList; + char *pEnd = &pList[nList]; + + if( *p!=0x01 ){ + if( *p==0x02 ){ + nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta); + pOut[nOut++] = 0x02; + bWritten = 1; + } + fts3ColumnlistCopy(0, &p); + } + + while( paaOutput); i++){ + if( pTS->aaOutput[i] ){ + if( !aOut ){ + aOut = pTS->aaOutput[i]; + nOut = pTS->anOutput[i]; + pTS->aaOutput[i] = 0; + }else{ + int nNew; + char *aNew; + + int rc = fts3DoclistOrMerge(p->bDescIdx, + pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew + ); + if( rc!=SQLITE_OK ){ + sqlite3_free(aOut); + return rc; + } + + sqlite3_free(pTS->aaOutput[i]); + sqlite3_free(aOut); + pTS->aaOutput[i] = 0; + aOut = aNew; + nOut = nNew; + } + } + } + + pTS->aaOutput[0] = aOut; + pTS->anOutput[0] = nOut; + return SQLITE_OK; +} + +/* +** Merge the doclist aDoclist/nDoclist into the TermSelect object passed +** as the first argument. The merge is an "OR" merge (see function +** fts3DoclistOrMerge() for details). +** +** This function is called with the doclist for each term that matches +** a queried prefix. It merges all these doclists into one, the doclist +** for the specified prefix. Since there can be a very large number of +** doclists to merge, the merging is done pair-wise using the TermSelect +** object. +** +** This function returns SQLITE_OK if the merge is successful, or an +** SQLite error code (SQLITE_NOMEM) if an error occurs. +*/ +static int fts3TermSelectMerge( + Fts3Table *p, /* FTS table handle */ + TermSelect *pTS, /* TermSelect object to merge into */ + char *aDoclist, /* Pointer to doclist */ + int nDoclist /* Size of aDoclist in bytes */ +){ + if( pTS->aaOutput[0]==0 ){ + /* If this is the first term selected, copy the doclist to the output + ** buffer using memcpy(). + ** + ** Add FTS3_VARINT_MAX bytes of unused space to the end of the + ** allocation. This is so as to ensure that the buffer is big enough + ** to hold the current doclist AND'd with any other doclist. If the + ** doclists are stored in order=ASC order, this padding would not be + ** required (since the size of [doclistA AND doclistB] is always less + ** than or equal to the size of [doclistA] in that case). But this is + ** not true for order=DESC. For example, a doclist containing (1, -1) + ** may be smaller than (-1), as in the first example the -1 may be stored + ** as a single-byte delta, whereas in the second it must be stored as a + ** FTS3_VARINT_MAX byte varint. + ** + ** Similar padding is added in the fts3DoclistOrMerge() function. + */ + pTS->aaOutput[0] = sqlite3_malloc(nDoclist + FTS3_VARINT_MAX + 1); + pTS->anOutput[0] = nDoclist; + if( pTS->aaOutput[0] ){ + memcpy(pTS->aaOutput[0], aDoclist, nDoclist); + }else{ + return SQLITE_NOMEM; + } + }else{ + char *aMerge = aDoclist; + int nMerge = nDoclist; + int iOut; + + for(iOut=0; iOutaaOutput); iOut++){ + if( pTS->aaOutput[iOut]==0 ){ + assert( iOut>0 ); + pTS->aaOutput[iOut] = aMerge; + pTS->anOutput[iOut] = nMerge; + break; + }else{ + char *aNew; + int nNew; + + int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, + pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew + ); + if( rc!=SQLITE_OK ){ + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + return rc; + } + + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + sqlite3_free(pTS->aaOutput[iOut]); + pTS->aaOutput[iOut] = 0; + + aMerge = aNew; + nMerge = nNew; + if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ + pTS->aaOutput[iOut] = aMerge; + pTS->anOutput[iOut] = nMerge; + } + } + } + } + return SQLITE_OK; +} + +/* +** Append SegReader object pNew to the end of the pCsr->apSegment[] array. +*/ +static int fts3SegReaderCursorAppend( + Fts3MultiSegReader *pCsr, + Fts3SegReader *pNew +){ + if( (pCsr->nSegment%16)==0 ){ + Fts3SegReader **apNew; + int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); + apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); + if( !apNew ){ + sqlite3Fts3SegReaderFree(pNew); + return SQLITE_NOMEM; + } + pCsr->apSegment = apNew; + } + pCsr->apSegment[pCsr->nSegment++] = pNew; + return SQLITE_OK; +} + +/* +** Add seg-reader objects to the Fts3MultiSegReader object passed as the +** 8th argument. +** +** This function returns SQLITE_OK if successful, or an SQLite error code +** otherwise. +*/ +static int fts3SegReaderCursor( + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language id */ + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ + int iLevel, /* Level of segments to scan */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + int isScan, /* True to scan from zTerm to EOF */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ +){ + int rc = SQLITE_OK; /* Error code */ + sqlite3_stmt *pStmt = 0; /* Statement to iterate through segments */ + int rc2; /* Result of sqlite3_reset() */ + + /* If iLevel is less than 0 and this is not a scan, include a seg-reader + ** for the pending-terms. If this is a scan, then this call must be being + ** made by an fts4aux module, not an FTS table. In this case calling + ** Fts3SegReaderPending might segfault, as the data structures used by + ** fts4aux are not completely populated. So it's easiest to filter these + ** calls out here. */ + if( iLevel<0 && p->aIndex ){ + Fts3SegReader *pSeg = 0; + rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix||isScan, &pSeg); + if( rc==SQLITE_OK && pSeg ){ + rc = fts3SegReaderCursorAppend(pCsr, pSeg); + } + } + + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3AllSegdirs(p, iLangid, iIndex, iLevel, &pStmt); + } + + while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ + Fts3SegReader *pSeg = 0; + + /* Read the values returned by the SELECT into local variables. */ + sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1); + sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2); + sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3); + int nRoot = sqlite3_column_bytes(pStmt, 4); + char const *zRoot = sqlite3_column_blob(pStmt, 4); + + /* If zTerm is not NULL, and this segment is not stored entirely on its + ** root node, the range of leaves scanned can be reduced. Do this. */ + if( iStartBlock && zTerm ){ + sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0); + rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi); + if( rc!=SQLITE_OK ) goto finished; + if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock; + } + + rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1, + (isPrefix==0 && isScan==0), + iStartBlock, iLeavesEndBlock, + iEndBlock, zRoot, nRoot, &pSeg + ); + if( rc!=SQLITE_OK ) goto finished; + rc = fts3SegReaderCursorAppend(pCsr, pSeg); + } + } + + finished: + rc2 = sqlite3_reset(pStmt); + if( rc==SQLITE_DONE ) rc = rc2; + + return rc; +} + +/* +** Set up a cursor object for iterating through a full-text index or a +** single level therein. +*/ +SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language-id to search */ + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ + int iLevel, /* Level of segments to scan */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + int isScan, /* True to scan from zTerm to EOF */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ +){ + assert( iIndex>=0 && iIndexnIndex ); + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( iLevelbase.pVtab; + + if( isPrefix ){ + for(i=1; bFound==0 && inIndex; i++){ + if( p->aIndex[i].nPrefix==nTerm ){ + bFound = 1; + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr + ); + pSegcsr->bLookup = 1; + } + } + + for(i=1; bFound==0 && inIndex; i++){ + if( p->aIndex[i].nPrefix==nTerm+1 ){ + bFound = 1; + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr + ); + if( rc==SQLITE_OK ){ + rc = fts3SegReaderCursorAddZero( + p, pCsr->iLangid, zTerm, nTerm, pSegcsr + ); + } + } + } + } + + if( bFound==0 ){ + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr + ); + pSegcsr->bLookup = !isPrefix; + } + } + + *ppSegcsr = pSegcsr; + return rc; +} + +/* +** Free an Fts3MultiSegReader allocated by fts3TermSegReaderCursor(). +*/ +static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ + sqlite3Fts3SegReaderFinish(pSegcsr); + sqlite3_free(pSegcsr); +} + +/* +** This function retrieves the doclist for the specified term (or term +** prefix) from the database. +*/ +static int fts3TermSelect( + Fts3Table *p, /* Virtual table handle */ + Fts3PhraseToken *pTok, /* Token to query for */ + int iColumn, /* Column to query (or -ve for all columns) */ + int *pnOut, /* OUT: Size of buffer at *ppOut */ + char **ppOut /* OUT: Malloced result buffer */ +){ + int rc; /* Return code */ + Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ + TermSelect tsc; /* Object for pair-wise doclist merging */ + Fts3SegFilter filter; /* Segment term filter configuration */ + + pSegcsr = pTok->pSegcsr; + memset(&tsc, 0, sizeof(TermSelect)); + + filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | FTS3_SEGMENT_REQUIRE_POS + | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0) + | (pTok->bFirst ? FTS3_SEGMENT_FIRST : 0) + | (iColumnnColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); + filter.iCol = iColumn; + filter.zTerm = pTok->z; + filter.nTerm = pTok->n; + + rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter); + while( SQLITE_OK==rc + && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr)) + ){ + rc = fts3TermSelectMerge(p, &tsc, pSegcsr->aDoclist, pSegcsr->nDoclist); + } + + if( rc==SQLITE_OK ){ + rc = fts3TermSelectFinishMerge(p, &tsc); + } + if( rc==SQLITE_OK ){ + *ppOut = tsc.aaOutput[0]; + *pnOut = tsc.anOutput[0]; + }else{ + int i; + for(i=0; ipSegcsr = 0; + return rc; +} + +/* +** This function counts the total number of docids in the doclist stored +** in buffer aList[], size nList bytes. +** +** If the isPoslist argument is true, then it is assumed that the doclist +** contains a position-list following each docid. Otherwise, it is assumed +** that the doclist is simply a list of docids stored as delta encoded +** varints. +*/ +static int fts3DoclistCountDocids(char *aList, int nList){ + int nDoc = 0; /* Return value */ + if( aList ){ + char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */ + char *p = aList; /* Cursor */ + while( peSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){ + if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ + pCsr->isEof = 1; + rc = sqlite3_reset(pCsr->pStmt); + }else{ + pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); + rc = SQLITE_OK; + } + }else{ + rc = fts3EvalNext((Fts3Cursor *)pCursor); + } + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + return rc; +} + +/* +** The following are copied from sqliteInt.h. +** +** Constants for the largest and smallest possible 64-bit signed integers. +** These macros are designed to work correctly on both 32-bit and 64-bit +** compilers. +*/ +#ifndef SQLITE_AMALGAMATION +# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) +# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) +#endif + +/* +** If the numeric type of argument pVal is "integer", then return it +** converted to a 64-bit signed integer. Otherwise, return a copy of +** the second parameter, iDefault. +*/ +static sqlite3_int64 fts3DocidRange(sqlite3_value *pVal, i64 iDefault){ + if( pVal ){ + int eType = sqlite3_value_numeric_type(pVal); + if( eType==SQLITE_INTEGER ){ + return sqlite3_value_int64(pVal); + } + } + return iDefault; +} + +/* +** This is the xFilter interface for the virtual table. See +** the virtual table xFilter method documentation for additional +** information. +** +** If idxNum==FTS3_FULLSCAN_SEARCH then do a full table scan against +** the %_content table. +** +** If idxNum==FTS3_DOCID_SEARCH then do a docid lookup for a single entry +** in the %_content table. +** +** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index. The +** column on the left-hand side of the MATCH operator is column +** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand +** side of the MATCH operator. +*/ +static int fts3FilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + int rc = SQLITE_OK; + char *zSql; /* SQL statement used to access %_content */ + int eSearch; + Fts3Table *p = (Fts3Table *)pCursor->pVtab; + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + + sqlite3_value *pCons = 0; /* The MATCH or rowid constraint, if any */ + sqlite3_value *pLangid = 0; /* The "langid = ?" constraint, if any */ + sqlite3_value *pDocidGe = 0; /* The "docid >= ?" constraint, if any */ + sqlite3_value *pDocidLe = 0; /* The "docid <= ?" constraint, if any */ + int iIdx; + + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(nVal); + + eSearch = (idxNum & 0x0000FFFF); + assert( eSearch>=0 && eSearch<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); + assert( p->pSegments==0 ); + + /* Collect arguments into local variables */ + iIdx = 0; + if( eSearch!=FTS3_FULLSCAN_SEARCH ) pCons = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_LANGID ) pLangid = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_DOCID_GE ) pDocidGe = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_DOCID_LE ) pDocidLe = apVal[iIdx++]; + assert( iIdx==nVal ); + + /* In case the cursor has been used before, clear it now. */ + sqlite3_finalize(pCsr->pStmt); + sqlite3_free(pCsr->aDoclist); + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); + sqlite3Fts3ExprFree(pCsr->pExpr); + memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); + + /* Set the lower and upper bounds on docids to return */ + pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64); + pCsr->iMaxDocid = fts3DocidRange(pDocidLe, LARGEST_INT64); + + if( idxStr ){ + pCsr->bDesc = (idxStr[0]=='D'); + }else{ + pCsr->bDesc = p->bDescIdx; + } + pCsr->eSearch = (i16)eSearch; + + if( eSearch!=FTS3_DOCID_SEARCH && eSearch!=FTS3_FULLSCAN_SEARCH ){ + int iCol = eSearch-FTS3_FULLTEXT_SEARCH; + const char *zQuery = (const char *)sqlite3_value_text(pCons); + + if( zQuery==0 && sqlite3_value_type(pCons)!=SQLITE_NULL ){ + return SQLITE_NOMEM; + } + + pCsr->iLangid = 0; + if( pLangid ) pCsr->iLangid = sqlite3_value_int(pLangid); + + assert( p->base.zErrMsg==0 ); + rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid, + p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr, + &p->base.zErrMsg + ); + if( rc!=SQLITE_OK ){ + return rc; + } + + rc = fts3EvalStart(pCsr); + sqlite3Fts3SegmentsClose(p); + if( rc!=SQLITE_OK ) return rc; + pCsr->pNextId = pCsr->aDoclist; + pCsr->iPrevId = 0; + } + + /* Compile a SELECT statement for this cursor. For a full-table-scan, the + ** statement loops through all rows of the %_content table. For a + ** full-text query or docid lookup, the statement retrieves a single + ** row by docid. + */ + if( eSearch==FTS3_FULLSCAN_SEARCH ){ + if( pDocidGe || pDocidLe ){ + zSql = sqlite3_mprintf( + "SELECT %s WHERE rowid BETWEEN %lld AND %lld ORDER BY rowid %s", + p->zReadExprlist, pCsr->iMinDocid, pCsr->iMaxDocid, + (pCsr->bDesc ? "DESC" : "ASC") + ); + }else{ + zSql = sqlite3_mprintf("SELECT %s ORDER BY rowid %s", + p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC") + ); + } + if( zSql ){ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; + } + }else if( eSearch==FTS3_DOCID_SEARCH ){ + rc = fts3CursorSeekStmt(pCsr, &pCsr->pStmt); + if( rc==SQLITE_OK ){ + rc = sqlite3_bind_value(pCsr->pStmt, 1, pCons); + } + } + if( rc!=SQLITE_OK ) return rc; + + return fts3NextMethod(pCursor); +} + +/* +** This is the xEof method of the virtual table. SQLite calls this +** routine to find out if it has reached the end of a result set. +*/ +static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){ + return ((Fts3Cursor *)pCursor)->isEof; +} + +/* +** This is the xRowid method. The SQLite core calls this routine to +** retrieve the rowid for the current row of the result set. fts3 +** exposes %_content.docid as the rowid for the virtual table. The +** rowid should be written to *pRowid. +*/ +static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; + *pRowid = pCsr->iPrevId; + return SQLITE_OK; +} + +/* +** This is the xColumn method, called by SQLite to request a value from +** the row that the supplied cursor currently points to. +** +** If: +** +** (iCol < p->nColumn) -> The value of the iCol'th user column. +** (iCol == p->nColumn) -> Magic column with the same name as the table. +** (iCol == p->nColumn+1) -> Docid column +** (iCol == p->nColumn+2) -> Langid column +*/ +static int fts3ColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + int rc = SQLITE_OK; /* Return Code */ + Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; + Fts3Table *p = (Fts3Table *)pCursor->pVtab; + + /* The column value supplied by SQLite must be in range. */ + assert( iCol>=0 && iCol<=p->nColumn+2 ); + + if( iCol==p->nColumn+1 ){ + /* This call is a request for the "docid" column. Since "docid" is an + ** alias for "rowid", use the xRowid() method to obtain the value. + */ + sqlite3_result_int64(pCtx, pCsr->iPrevId); + }else if( iCol==p->nColumn ){ + /* The extra column whose name is the same as the table. + ** Return a blob which is a pointer to the cursor. */ + sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); + }else if( iCol==p->nColumn+2 && pCsr->pExpr ){ + sqlite3_result_int64(pCtx, pCsr->iLangid); + }else{ + /* The requested column is either a user column (one that contains + ** indexed data), or the language-id column. */ + rc = fts3CursorSeek(0, pCsr); + + if( rc==SQLITE_OK ){ + if( iCol==p->nColumn+2 ){ + int iLangid = 0; + if( p->zLanguageid ){ + iLangid = sqlite3_column_int(pCsr->pStmt, p->nColumn+1); + } + sqlite3_result_int(pCtx, iLangid); + }else if( sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){ + sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); + } + } + } + + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + return rc; +} + +/* +** This function is the implementation of the xUpdate callback used by +** FTS3 virtual tables. It is invoked by SQLite each time a row is to be +** inserted, updated or deleted. +*/ +static int fts3UpdateMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nArg, /* Size of argument array */ + sqlite3_value **apVal, /* Array of arguments */ + sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ +){ + return sqlite3Fts3UpdateMethod(pVtab, nArg, apVal, pRowid); +} + +/* +** Implementation of xSync() method. Flush the contents of the pending-terms +** hash-table to the database. +*/ +static int fts3SyncMethod(sqlite3_vtab *pVtab){ + + /* Following an incremental-merge operation, assuming that the input + ** segments are not completely consumed (the usual case), they are updated + ** in place to remove the entries that have already been merged. This + ** involves updating the leaf block that contains the smallest unmerged + ** entry and each block (if any) between the leaf and the root node. So + ** if the height of the input segment b-trees is N, and input segments + ** are merged eight at a time, updating the input segments at the end + ** of an incremental-merge requires writing (8*(1+N)) blocks. N is usually + ** small - often between 0 and 2. So the overhead of the incremental + ** merge is somewhere between 8 and 24 blocks. To avoid this overhead + ** dwarfing the actual productive work accomplished, the incremental merge + ** is only attempted if it will write at least 64 leaf blocks. Hence + ** nMinMerge. + ** + ** Of course, updating the input segments also involves deleting a bunch + ** of blocks from the segments table. But this is not considered overhead + ** as it would also be required by a crisis-merge that used the same input + ** segments. + */ + const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */ + + Fts3Table *p = (Fts3Table*)pVtab; + int rc = sqlite3Fts3PendingTermsFlush(p); + + if( rc==SQLITE_OK + && p->nLeafAdd>(nMinMerge/16) + && p->nAutoincrmerge && p->nAutoincrmerge!=0xff + ){ + int mxLevel = 0; /* Maximum relative level value in db */ + int A; /* Incr-merge parameter A */ + + rc = sqlite3Fts3MaxLevel(p, &mxLevel); + assert( rc==SQLITE_OK || mxLevel==0 ); + A = p->nLeafAdd * mxLevel; + A += (A/2); + if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge); + } + sqlite3Fts3SegmentsClose(p); + return rc; +} + +/* +** If it is currently unknown whether or not the FTS table has an %_stat +** table (if p->bHasStat==2), attempt to determine this (set p->bHasStat +** to 0 or 1). Return SQLITE_OK if successful, or an SQLite error code +** if an error occurs. +*/ +static int fts3SetHasStat(Fts3Table *p){ + int rc = SQLITE_OK; + if( p->bHasStat==2 ){ + const char *zFmt ="SELECT 1 FROM %Q.sqlite_master WHERE tbl_name='%q_stat'"; + char *zSql = sqlite3_mprintf(zFmt, p->zDb, p->zName); + if( zSql ){ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + int bHasStat = (sqlite3_step(pStmt)==SQLITE_ROW); + rc = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) p->bHasStat = bHasStat; + } + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; + } + } + return rc; +} + +/* +** Implementation of xBegin() method. +*/ +static int fts3BeginMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table*)pVtab; + UNUSED_PARAMETER(pVtab); + assert( p->pSegments==0 ); + assert( p->nPendingData==0 ); + assert( p->inTransaction!=1 ); + TESTONLY( p->inTransaction = 1 ); + TESTONLY( p->mxSavepoint = -1; ); + p->nLeafAdd = 0; + return fts3SetHasStat(p); +} + +/* +** Implementation of xCommit() method. This is a no-op. The contents of +** the pending-terms hash-table have already been flushed into the database +** by fts3SyncMethod(). +*/ +static int fts3CommitMethod(sqlite3_vtab *pVtab){ + TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); + UNUSED_PARAMETER(pVtab); + assert( p->nPendingData==0 ); + assert( p->inTransaction!=0 ); + assert( p->pSegments==0 ); + TESTONLY( p->inTransaction = 0 ); + TESTONLY( p->mxSavepoint = -1; ); + return SQLITE_OK; +} + +/* +** Implementation of xRollback(). Discard the contents of the pending-terms +** hash-table. Any changes made to the database are reverted by SQLite. +*/ +static int fts3RollbackMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table*)pVtab; + sqlite3Fts3PendingTermsClear(p); + assert( p->inTransaction!=0 ); + TESTONLY( p->inTransaction = 0 ); + TESTONLY( p->mxSavepoint = -1; ); + return SQLITE_OK; +} + +/* +** When called, *ppPoslist must point to the byte immediately following the +** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function +** moves *ppPoslist so that it instead points to the first byte of the +** same position list. +*/ +static void fts3ReversePoslist(char *pStart, char **ppPoslist){ + char *p = &(*ppPoslist)[-2]; + char c = 0; + + /* Skip backwards passed any trailing 0x00 bytes added by NearTrim() */ + while( p>pStart && (c=*p--)==0 ); + + /* Search backwards for a varint with value zero (the end of the previous + ** poslist). This is an 0x00 byte preceded by some byte that does not + ** have the 0x80 bit set. */ + while( p>pStart && (*p & 0x80) | c ){ + c = *p--; + } + assert( p==pStart || c==0 ); + + /* At this point p points to that preceding byte without the 0x80 bit + ** set. So to find the start of the poslist, skip forward 2 bytes then + ** over a varint. + ** + ** Normally. The other case is that p==pStart and the poslist to return + ** is the first in the doclist. In this case do not skip forward 2 bytes. + ** The second part of the if condition (c==0 && *ppPoslist>&p[2]) + ** is required for cases where the first byte of a doclist and the + ** doclist is empty. For example, if the first docid is 10, a doclist + ** that begins with: + ** + ** 0x0A 0x00 + */ + if( p>pStart || (c==0 && *ppPoslist>&p[2]) ){ p = &p[2]; } + while( *p++&0x80 ); + *ppPoslist = p; +} + +/* +** Helper function used by the implementation of the overloaded snippet(), +** offsets() and optimize() SQL functions. +** +** If the value passed as the third argument is a blob of size +** sizeof(Fts3Cursor*), then the blob contents are copied to the +** output variable *ppCsr and SQLITE_OK is returned. Otherwise, an error +** message is written to context pContext and SQLITE_ERROR returned. The +** string passed via zFunc is used as part of the error message. +*/ +static int fts3FunctionArg( + sqlite3_context *pContext, /* SQL function call context */ + const char *zFunc, /* Function name */ + sqlite3_value *pVal, /* argv[0] passed to function */ + Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ +){ + Fts3Cursor *pRet; + if( sqlite3_value_type(pVal)!=SQLITE_BLOB + || sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *) + ){ + char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc); + sqlite3_result_error(pContext, zErr, -1); + sqlite3_free(zErr); + return SQLITE_ERROR; + } + memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *)); + *ppCsr = pRet; + return SQLITE_OK; +} + +/* +** Implementation of the snippet() function for FTS3 +*/ +static void fts3SnippetFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of apVal[] array */ + sqlite3_value **apVal /* Array of arguments */ +){ + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + const char *zStart = ""; + const char *zEnd = ""; + const char *zEllipsis = "..."; + int iCol = -1; + int nToken = 15; /* Default number of tokens in snippet */ + + /* There must be at least one argument passed to this function (otherwise + ** the non-overloaded version would have been called instead of this one). + */ + assert( nVal>=1 ); + + if( nVal>6 ){ + sqlite3_result_error(pContext, + "wrong number of arguments to function snippet()", -1); + return; + } + if( fts3FunctionArg(pContext, "snippet", apVal[0], &pCsr) ) return; + + switch( nVal ){ + case 6: nToken = sqlite3_value_int(apVal[5]); + case 5: iCol = sqlite3_value_int(apVal[4]); + case 4: zEllipsis = (const char*)sqlite3_value_text(apVal[3]); + case 3: zEnd = (const char*)sqlite3_value_text(apVal[2]); + case 2: zStart = (const char*)sqlite3_value_text(apVal[1]); + } + if( !zEllipsis || !zEnd || !zStart ){ + sqlite3_result_error_nomem(pContext); + }else if( nToken==0 ){ + sqlite3_result_text(pContext, "", -1, SQLITE_STATIC); + }else if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ + sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis, iCol, nToken); + } +} + +/* +** Implementation of the offsets() function for FTS3 +*/ +static void fts3OffsetsFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ +){ + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + + UNUSED_PARAMETER(nVal); + + assert( nVal==1 ); + if( fts3FunctionArg(pContext, "offsets", apVal[0], &pCsr) ) return; + assert( pCsr ); + if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ + sqlite3Fts3Offsets(pContext, pCsr); + } +} + +/* +** Implementation of the special optimize() function for FTS3. This +** function merges all segments in the database to a single segment. +** Example usage is: +** +** SELECT optimize(t) FROM t LIMIT 1; +** +** where 't' is the name of an FTS3 table. +*/ +static void fts3OptimizeFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ +){ + int rc; /* Return code */ + Fts3Table *p; /* Virtual table handle */ + Fts3Cursor *pCursor; /* Cursor handle passed through apVal[0] */ + + UNUSED_PARAMETER(nVal); + + assert( nVal==1 ); + if( fts3FunctionArg(pContext, "optimize", apVal[0], &pCursor) ) return; + p = (Fts3Table *)pCursor->base.pVtab; + assert( p ); + + rc = sqlite3Fts3Optimize(p); + + switch( rc ){ + case SQLITE_OK: + sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC); + break; + case SQLITE_DONE: + sqlite3_result_text(pContext, "Index already optimal", -1, SQLITE_STATIC); + break; + default: + sqlite3_result_error_code(pContext, rc); + break; + } +} + +/* +** Implementation of the matchinfo() function for FTS3 +*/ +static void fts3MatchinfoFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ +){ + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + assert( nVal==1 || nVal==2 ); + if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){ + const char *zArg = 0; + if( nVal>1 ){ + zArg = (const char *)sqlite3_value_text(apVal[1]); + } + sqlite3Fts3Matchinfo(pContext, pCsr, zArg); + } +} + +/* +** This routine implements the xFindFunction method for the FTS3 +** virtual table. +*/ +static int fts3FindFunctionMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nArg, /* Number of SQL function arguments */ + const char *zName, /* Name of SQL function */ + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ + void **ppArg /* Unused */ +){ + struct Overloaded { + const char *zName; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aOverload[] = { + { "snippet", fts3SnippetFunc }, + { "offsets", fts3OffsetsFunc }, + { "optimize", fts3OptimizeFunc }, + { "matchinfo", fts3MatchinfoFunc }, + }; + int i; /* Iterator variable */ + + UNUSED_PARAMETER(pVtab); + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(ppArg); + + for(i=0; idb; /* Database connection */ + int rc; /* Return Code */ + + /* At this point it must be known if the %_stat table exists or not. + ** So bHasStat may not be 2. */ + rc = fts3SetHasStat(p); + + /* As it happens, the pending terms table is always empty here. This is + ** because an "ALTER TABLE RENAME TABLE" statement inside a transaction + ** always opens a savepoint transaction. And the xSavepoint() method + ** flushes the pending terms table. But leave the (no-op) call to + ** PendingTermsFlush() in in case that changes. + */ + assert( p->nPendingData==0 ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3PendingTermsFlush(p); + } + + if( p->zContentTbl==0 ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", + p->zDb, p->zName, zName + ); + } + + if( p->bHasDocsize ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';", + p->zDb, p->zName, zName + ); + } + if( p->bHasStat ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_stat' RENAME TO '%q_stat';", + p->zDb, p->zName, zName + ); + } + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';", + p->zDb, p->zName, zName + ); + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';", + p->zDb, p->zName, zName + ); + return rc; +} + +/* +** The xSavepoint() method. +** +** Flush the contents of the pending-terms table to disk. +*/ +static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ + int rc = SQLITE_OK; + UNUSED_PARAMETER(iSavepoint); + assert( ((Fts3Table *)pVtab)->inTransaction ); + assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint ); + TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); + if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){ + rc = fts3SyncMethod(pVtab); + } + return rc; +} + +/* +** The xRelease() method. +** +** This is a no-op. +*/ +static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ + TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); + UNUSED_PARAMETER(iSavepoint); + UNUSED_PARAMETER(pVtab); + assert( p->inTransaction ); + assert( p->mxSavepoint >= iSavepoint ); + TESTONLY( p->mxSavepoint = iSavepoint-1 ); + return SQLITE_OK; +} + +/* +** The xRollbackTo() method. +** +** Discard the contents of the pending terms table. +*/ +static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ + Fts3Table *p = (Fts3Table*)pVtab; + UNUSED_PARAMETER(iSavepoint); + assert( p->inTransaction ); + assert( p->mxSavepoint >= iSavepoint ); + TESTONLY( p->mxSavepoint = iSavepoint ); + sqlite3Fts3PendingTermsClear(p); + return SQLITE_OK; +} + +static const sqlite3_module fts3Module = { + /* iVersion */ 2, + /* xCreate */ fts3CreateMethod, + /* xConnect */ fts3ConnectMethod, + /* xBestIndex */ fts3BestIndexMethod, + /* xDisconnect */ fts3DisconnectMethod, + /* xDestroy */ fts3DestroyMethod, + /* xOpen */ fts3OpenMethod, + /* xClose */ fts3CloseMethod, + /* xFilter */ fts3FilterMethod, + /* xNext */ fts3NextMethod, + /* xEof */ fts3EofMethod, + /* xColumn */ fts3ColumnMethod, + /* xRowid */ fts3RowidMethod, + /* xUpdate */ fts3UpdateMethod, + /* xBegin */ fts3BeginMethod, + /* xSync */ fts3SyncMethod, + /* xCommit */ fts3CommitMethod, + /* xRollback */ fts3RollbackMethod, + /* xFindFunction */ fts3FindFunctionMethod, + /* xRename */ fts3RenameMethod, + /* xSavepoint */ fts3SavepointMethod, + /* xRelease */ fts3ReleaseMethod, + /* xRollbackTo */ fts3RollbackToMethod, +}; + +/* +** This function is registered as the module destructor (called when an +** FTS3 enabled database connection is closed). It frees the memory +** allocated for the tokenizer hash table. +*/ +static void hashDestroy(void *p){ + Fts3Hash *pHash = (Fts3Hash *)p; + sqlite3Fts3HashClear(pHash); + sqlite3_free(pHash); +} + +/* +** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are +** implemented in files fts3_tokenizer1.c, fts3_porter.c and fts3_icu.c +** respectively. The following three forward declarations are for functions +** declared in these files used to retrieve the respective implementations. +** +** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed +** to by the argument to point to the "simple" tokenizer implementation. +** And so on. +*/ +SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); +SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); +#ifndef SQLITE_DISABLE_FTS3_UNICODE +SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule); +#endif +#ifdef SQLITE_ENABLE_ICU +SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); +#endif + +/* +** Initialize the fts3 extension. If this extension is built as part +** of the sqlite library, then this function is called directly by +** SQLite. If fts3 is built as a dynamically loadable extension, this +** function is called by the sqlite3_extension_init() entry point. +*/ +SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ + int rc = SQLITE_OK; + Fts3Hash *pHash = 0; + const sqlite3_tokenizer_module *pSimple = 0; + const sqlite3_tokenizer_module *pPorter = 0; +#ifndef SQLITE_DISABLE_FTS3_UNICODE + const sqlite3_tokenizer_module *pUnicode = 0; +#endif + +#ifdef SQLITE_ENABLE_ICU + const sqlite3_tokenizer_module *pIcu = 0; + sqlite3Fts3IcuTokenizerModule(&pIcu); +#endif + +#ifndef SQLITE_DISABLE_FTS3_UNICODE + sqlite3Fts3UnicodeTokenizer(&pUnicode); +#endif + +#ifdef SQLITE_TEST + rc = sqlite3Fts3InitTerm(db); + if( rc!=SQLITE_OK ) return rc; +#endif + + rc = sqlite3Fts3InitAux(db); + if( rc!=SQLITE_OK ) return rc; + + sqlite3Fts3SimpleTokenizerModule(&pSimple); + sqlite3Fts3PorterTokenizerModule(&pPorter); + + /* Allocate and initialize the hash-table used to store tokenizers. */ + pHash = sqlite3_malloc(sizeof(Fts3Hash)); + if( !pHash ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); + } + + /* Load the built-in tokenizers into the hash table */ + if( rc==SQLITE_OK ){ + if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple) + || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) + +#ifndef SQLITE_DISABLE_FTS3_UNICODE + || sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode) +#endif +#ifdef SQLITE_ENABLE_ICU + || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu)) +#endif + ){ + rc = SQLITE_NOMEM; + } + } + +#ifdef SQLITE_TEST + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3ExprInitTestInterface(db); + } +#endif + + /* Create the virtual table wrapper around the hash-table and overload + ** the two scalar functions. If this is successful, register the + ** module with sqlite. + */ + if( SQLITE_OK==rc + && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 2)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1)) + ){ + rc = sqlite3_create_module_v2( + db, "fts3", &fts3Module, (void *)pHash, hashDestroy + ); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module_v2( + db, "fts4", &fts3Module, (void *)pHash, 0 + ); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3InitTok(db, (void *)pHash); + } + return rc; + } + + + /* An error has occurred. Delete the hash table and return the error code. */ + assert( rc!=SQLITE_OK ); + if( pHash ){ + sqlite3Fts3HashClear(pHash); + sqlite3_free(pHash); + } + return rc; +} + +/* +** Allocate an Fts3MultiSegReader for each token in the expression headed +** by pExpr. +** +** An Fts3SegReader object is a cursor that can seek or scan a range of +** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple +** Fts3SegReader objects internally to provide an interface to seek or scan +** within the union of all segments of a b-tree. Hence the name. +** +** If the allocated Fts3MultiSegReader just seeks to a single entry in a +** segment b-tree (if the term is not a prefix or it is a prefix for which +** there exists prefix b-tree of the right length) then it may be traversed +** and merged incrementally. Otherwise, it has to be merged into an in-memory +** doclist and then traversed. +*/ +static void fts3EvalAllocateReaders( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Allocate readers for this expression */ + int *pnToken, /* OUT: Total number of tokens in phrase. */ + int *pnOr, /* OUT: Total number of OR nodes in expr. */ + int *pRc /* IN/OUT: Error code */ +){ + if( pExpr && SQLITE_OK==*pRc ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + int i; + int nToken = pExpr->pPhrase->nToken; + *pnToken += nToken; + for(i=0; ipPhrase->aToken[i]; + int rc = fts3TermSegReaderCursor(pCsr, + pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr + ); + if( rc!=SQLITE_OK ){ + *pRc = rc; + return; + } + } + assert( pExpr->pPhrase->iDoclistToken==0 ); + pExpr->pPhrase->iDoclistToken = -1; + }else{ + *pnOr += (pExpr->eType==FTSQUERY_OR); + fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc); + fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc); + } + } +} + +/* +** Arguments pList/nList contain the doclist for token iToken of phrase p. +** It is merged into the main doclist stored in p->doclist.aAll/nAll. +** +** This function assumes that pList points to a buffer allocated using +** sqlite3_malloc(). This function takes responsibility for eventually +** freeing the buffer. +** +** SQLITE_OK is returned if successful, or SQLITE_NOMEM if an error occurs. +*/ +static int fts3EvalPhraseMergeToken( + Fts3Table *pTab, /* FTS Table pointer */ + Fts3Phrase *p, /* Phrase to merge pList/nList into */ + int iToken, /* Token pList/nList corresponds to */ + char *pList, /* Pointer to doclist */ + int nList /* Number of bytes in pList */ +){ + int rc = SQLITE_OK; + assert( iToken!=p->iDoclistToken ); + + if( pList==0 ){ + sqlite3_free(p->doclist.aAll); + p->doclist.aAll = 0; + p->doclist.nAll = 0; + } + + else if( p->iDoclistToken<0 ){ + p->doclist.aAll = pList; + p->doclist.nAll = nList; + } + + else if( p->doclist.aAll==0 ){ + sqlite3_free(pList); + } + + else { + char *pLeft; + char *pRight; + int nLeft; + int nRight; + int nDiff; + + if( p->iDoclistTokendoclist.aAll; + nLeft = p->doclist.nAll; + pRight = pList; + nRight = nList; + nDiff = iToken - p->iDoclistToken; + }else{ + pRight = p->doclist.aAll; + nRight = p->doclist.nAll; + pLeft = pList; + nLeft = nList; + nDiff = p->iDoclistToken - iToken; + } + + rc = fts3DoclistPhraseMerge( + pTab->bDescIdx, nDiff, pLeft, nLeft, &pRight, &nRight + ); + sqlite3_free(pLeft); + p->doclist.aAll = pRight; + p->doclist.nAll = nRight; + } + + if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken; + return rc; +} + +/* +** Load the doclist for phrase p into p->doclist.aAll/nAll. The loaded doclist +** does not take deferred tokens into account. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ +static int fts3EvalPhraseLoad( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p /* Phrase object */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int iToken; + int rc = SQLITE_OK; + + for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ + Fts3PhraseToken *pToken = &p->aToken[iToken]; + assert( pToken->pDeferred==0 || pToken->pSegcsr==0 ); + + if( pToken->pSegcsr ){ + int nThis = 0; + char *pThis = 0; + rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis); + if( rc==SQLITE_OK ){ + rc = fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); + } + } + assert( pToken->pSegcsr==0 ); + } + + return rc; +} + +/* +** This function is called on each phrase after the position lists for +** any deferred tokens have been loaded into memory. It updates the phrases +** current position list to include only those positions that are really +** instances of the phrase (after considering deferred tokens). If this +** means that the phrase does not appear in the current row, doclist.pList +** and doclist.nList are both zeroed. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ +static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ + int iToken; /* Used to iterate through phrase tokens */ + char *aPoslist = 0; /* Position list for deferred tokens */ + int nPoslist = 0; /* Number of bytes in aPoslist */ + int iPrev = -1; /* Token number of previous deferred token */ + + assert( pPhrase->doclist.bFreeList==0 ); + + for(iToken=0; iTokennToken; iToken++){ + Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; + Fts3DeferredToken *pDeferred = pToken->pDeferred; + + if( pDeferred ){ + char *pList; + int nList; + int rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); + if( rc!=SQLITE_OK ) return rc; + + if( pList==0 ){ + sqlite3_free(aPoslist); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + return SQLITE_OK; + + }else if( aPoslist==0 ){ + aPoslist = pList; + nPoslist = nList; + + }else{ + char *aOut = pList; + char *p1 = aPoslist; + char *p2 = aOut; + + assert( iPrev>=0 ); + fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2); + sqlite3_free(aPoslist); + aPoslist = pList; + nPoslist = (int)(aOut - aPoslist); + if( nPoslist==0 ){ + sqlite3_free(aPoslist); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + return SQLITE_OK; + } + } + iPrev = iToken; + } + } + + if( iPrev>=0 ){ + int nMaxUndeferred = pPhrase->iDoclistToken; + if( nMaxUndeferred<0 ){ + pPhrase->doclist.pList = aPoslist; + pPhrase->doclist.nList = nPoslist; + pPhrase->doclist.iDocid = pCsr->iPrevId; + pPhrase->doclist.bFreeList = 1; + }else{ + int nDistance; + char *p1; + char *p2; + char *aOut; + + if( nMaxUndeferred>iPrev ){ + p1 = aPoslist; + p2 = pPhrase->doclist.pList; + nDistance = nMaxUndeferred - iPrev; + }else{ + p1 = pPhrase->doclist.pList; + p2 = aPoslist; + nDistance = iPrev - nMaxUndeferred; + } + + aOut = (char *)sqlite3_malloc(nPoslist+8); + if( !aOut ){ + sqlite3_free(aPoslist); + return SQLITE_NOMEM; + } + + pPhrase->doclist.pList = aOut; + if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ + pPhrase->doclist.bFreeList = 1; + pPhrase->doclist.nList = (int)(aOut - pPhrase->doclist.pList); + }else{ + sqlite3_free(aOut); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + } + sqlite3_free(aPoslist); + } + } + + return SQLITE_OK; +} + +/* +** Maximum number of tokens a phrase may have to be considered for the +** incremental doclists strategy. +*/ +#define MAX_INCR_PHRASE_TOKENS 4 + +/* +** This function is called for each Fts3Phrase in a full-text query +** expression to initialize the mechanism for returning rows. Once this +** function has been called successfully on an Fts3Phrase, it may be +** used with fts3EvalPhraseNext() to iterate through the matching docids. +** +** If parameter bOptOk is true, then the phrase may (or may not) use the +** incremental loading strategy. Otherwise, the entire doclist is loaded into +** memory within this call. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ +static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; /* Error code */ + int i; + + /* Determine if doclists may be loaded from disk incrementally. This is + ** possible if the bOptOk argument is true, the FTS doclists will be + ** scanned in forward order, and the phrase consists of + ** MAX_INCR_PHRASE_TOKENS or fewer tokens, none of which are are "^first" + ** tokens or prefix tokens that cannot use a prefix-index. */ + int bHaveIncr = 0; + int bIncrOk = (bOptOk + && pCsr->bDesc==pTab->bDescIdx + && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0 +#ifdef SQLITE_TEST + && pTab->bNoIncrDoclist==0 +#endif + ); + for(i=0; bIncrOk==1 && inToken; i++){ + Fts3PhraseToken *pToken = &p->aToken[i]; + if( pToken->bFirst || (pToken->pSegcsr!=0 && !pToken->pSegcsr->bLookup) ){ + bIncrOk = 0; + } + if( pToken->pSegcsr ) bHaveIncr = 1; + } + + if( bIncrOk && bHaveIncr ){ + /* Use the incremental approach. */ + int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); + for(i=0; rc==SQLITE_OK && inToken; i++){ + Fts3PhraseToken *pToken = &p->aToken[i]; + Fts3MultiSegReader *pSegcsr = pToken->pSegcsr; + if( pSegcsr ){ + rc = sqlite3Fts3MsrIncrStart(pTab, pSegcsr, iCol, pToken->z, pToken->n); + } + } + p->bIncr = 1; + }else{ + /* Load the full doclist for the phrase into memory. */ + rc = fts3EvalPhraseLoad(pCsr, p); + p->bIncr = 0; + } + + assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr ); + return rc; +} + +/* +** This function is used to iterate backwards (from the end to start) +** through doclists. It is used by this module to iterate through phrase +** doclists in reverse and by the fts3_write.c module to iterate through +** pending-terms lists when writing to databases with "order=desc". +** +** The doclist may be sorted in ascending (parameter bDescIdx==0) or +** descending (parameter bDescIdx==1) order of docid. Regardless, this +** function iterates from the end of the doclist to the beginning. +*/ +SQLITE_PRIVATE void sqlite3Fts3DoclistPrev( + int bDescIdx, /* True if the doclist is desc */ + char *aDoclist, /* Pointer to entire doclist */ + int nDoclist, /* Length of aDoclist in bytes */ + char **ppIter, /* IN/OUT: Iterator pointer */ + sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ + int *pnList, /* OUT: List length pointer */ + u8 *pbEof /* OUT: End-of-file flag */ +){ + char *p = *ppIter; + + assert( nDoclist>0 ); + assert( *pbEof==0 ); + assert( p || *piDocid==0 ); + assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) ); + + if( p==0 ){ + sqlite3_int64 iDocid = 0; + char *pNext = 0; + char *pDocid = aDoclist; + char *pEnd = &aDoclist[nDoclist]; + int iMul = 1; + + while( pDocid0 ); + assert( *pbEof==0 ); + assert( p || *piDocid==0 ); + assert( !p || (p>=aDoclist && p<=&aDoclist[nDoclist]) ); + + if( p==0 ){ + p = aDoclist; + p += sqlite3Fts3GetVarint(p, piDocid); + }else{ + fts3PoslistCopy(0, &p); + while( p<&aDoclist[nDoclist] && *p==0 ) p++; + if( p>=&aDoclist[nDoclist] ){ + *pbEof = 1; + }else{ + sqlite3_int64 iVar; + p += sqlite3Fts3GetVarint(p, &iVar); + *piDocid += ((bDescIdx ? -1 : 1) * iVar); + } + } + + *ppIter = p; +} + +/* +** Advance the iterator pDL to the next entry in pDL->aAll/nAll. Set *pbEof +** to true if EOF is reached. +*/ +static void fts3EvalDlPhraseNext( + Fts3Table *pTab, + Fts3Doclist *pDL, + u8 *pbEof +){ + char *pIter; /* Used to iterate through aAll */ + char *pEnd = &pDL->aAll[pDL->nAll]; /* 1 byte past end of aAll */ + + if( pDL->pNextDocid ){ + pIter = pDL->pNextDocid; + }else{ + pIter = pDL->aAll; + } + + if( pIter>=pEnd ){ + /* We have already reached the end of this doclist. EOF. */ + *pbEof = 1; + }else{ + sqlite3_int64 iDelta; + pIter += sqlite3Fts3GetVarint(pIter, &iDelta); + if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){ + pDL->iDocid += iDelta; + }else{ + pDL->iDocid -= iDelta; + } + pDL->pList = pIter; + fts3PoslistCopy(0, &pIter); + pDL->nList = (int)(pIter - pDL->pList); + + /* pIter now points just past the 0x00 that terminates the position- + ** list for document pDL->iDocid. However, if this position-list was + ** edited in place by fts3EvalNearTrim(), then pIter may not actually + ** point to the start of the next docid value. The following line deals + ** with this case by advancing pIter past the zero-padding added by + ** fts3EvalNearTrim(). */ + while( pIterpNextDocid = pIter; + assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter ); + *pbEof = 0; + } +} + +/* +** Helper type used by fts3EvalIncrPhraseNext() and incrPhraseTokenNext(). +*/ +typedef struct TokenDoclist TokenDoclist; +struct TokenDoclist { + int bIgnore; + sqlite3_int64 iDocid; + char *pList; + int nList; +}; + +/* +** Token pToken is an incrementally loaded token that is part of a +** multi-token phrase. Advance it to the next matching document in the +** database and populate output variable *p with the details of the new +** entry. Or, if the iterator has reached EOF, set *pbEof to true. +** +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. +*/ +static int incrPhraseTokenNext( + Fts3Table *pTab, /* Virtual table handle */ + Fts3Phrase *pPhrase, /* Phrase to advance token of */ + int iToken, /* Specific token to advance */ + TokenDoclist *p, /* OUT: Docid and doclist for new entry */ + u8 *pbEof /* OUT: True if iterator is at EOF */ +){ + int rc = SQLITE_OK; + + if( pPhrase->iDoclistToken==iToken ){ + assert( p->bIgnore==0 ); + assert( pPhrase->aToken[iToken].pSegcsr==0 ); + fts3EvalDlPhraseNext(pTab, &pPhrase->doclist, pbEof); + p->pList = pPhrase->doclist.pList; + p->nList = pPhrase->doclist.nList; + p->iDocid = pPhrase->doclist.iDocid; + }else{ + Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; + assert( pToken->pDeferred==0 ); + assert( pToken->pSegcsr || pPhrase->iDoclistToken>=0 ); + if( pToken->pSegcsr ){ + assert( p->bIgnore==0 ); + rc = sqlite3Fts3MsrIncrNext( + pTab, pToken->pSegcsr, &p->iDocid, &p->pList, &p->nList + ); + if( p->pList==0 ) *pbEof = 1; + }else{ + p->bIgnore = 1; + } + } + + return rc; +} + + +/* +** The phrase iterator passed as the second argument: +** +** * features at least one token that uses an incremental doclist, and +** +** * does not contain any deferred tokens. +** +** Advance it to the next matching documnent in the database and populate +** the Fts3Doclist.pList and nList fields. +** +** If there is no "next" entry and no error occurs, then *pbEof is set to +** 1 before returning. Otherwise, if no error occurs and the iterator is +** successfully advanced, *pbEof is set to 0. +** +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. +*/ +static int fts3EvalIncrPhraseNext( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p, /* Phrase object to advance to next docid */ + u8 *pbEof /* OUT: Set to 1 if EOF */ +){ + int rc = SQLITE_OK; + Fts3Doclist *pDL = &p->doclist; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + u8 bEof = 0; + + /* This is only called if it is guaranteed that the phrase has at least + ** one incremental token. In which case the bIncr flag is set. */ + assert( p->bIncr==1 ); + + if( p->nToken==1 && p->bIncr ){ + rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, + &pDL->iDocid, &pDL->pList, &pDL->nList + ); + if( pDL->pList==0 ) bEof = 1; + }else{ + int bDescDoclist = pCsr->bDesc; + struct TokenDoclist a[MAX_INCR_PHRASE_TOKENS]; + + memset(a, 0, sizeof(a)); + assert( p->nToken<=MAX_INCR_PHRASE_TOKENS ); + assert( p->iDoclistTokennToken && bEof==0; i++){ + rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); + if( a[i].bIgnore==0 && (bMaxSet==0 || DOCID_CMP(iMax, a[i].iDocid)<0) ){ + iMax = a[i].iDocid; + bMaxSet = 1; + } + } + assert( rc!=SQLITE_OK || (p->nToken>=1 && a[p->nToken-1].bIgnore==0) ); + assert( rc!=SQLITE_OK || bMaxSet ); + + /* Keep advancing iterators until they all point to the same document */ + for(i=0; inToken; i++){ + while( rc==SQLITE_OK && bEof==0 + && a[i].bIgnore==0 && DOCID_CMP(a[i].iDocid, iMax)<0 + ){ + rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); + if( DOCID_CMP(a[i].iDocid, iMax)>0 ){ + iMax = a[i].iDocid; + i = 0; + } + } + } + + /* Check if the current entries really are a phrase match */ + if( bEof==0 ){ + int nList = 0; + int nByte = a[p->nToken-1].nList; + char *aDoclist = sqlite3_malloc(nByte+1); + if( !aDoclist ) return SQLITE_NOMEM; + memcpy(aDoclist, a[p->nToken-1].pList, nByte+1); + + for(i=0; i<(p->nToken-1); i++){ + if( a[i].bIgnore==0 ){ + char *pL = a[i].pList; + char *pR = aDoclist; + char *pOut = aDoclist; + int nDist = p->nToken-1-i; + int res = fts3PoslistPhraseMerge(&pOut, nDist, 0, 1, &pL, &pR); + if( res==0 ) break; + nList = (int)(pOut - aDoclist); + } + } + if( i==(p->nToken-1) ){ + pDL->iDocid = iMax; + pDL->pList = aDoclist; + pDL->nList = nList; + pDL->bFreeList = 1; + break; + } + sqlite3_free(aDoclist); + } + } + } + + *pbEof = bEof; + return rc; +} + +/* +** Attempt to move the phrase iterator to point to the next matching docid. +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. +** +** If there is no "next" entry and no error occurs, then *pbEof is set to +** 1 before returning. Otherwise, if no error occurs and the iterator is +** successfully advanced, *pbEof is set to 0. +*/ +static int fts3EvalPhraseNext( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p, /* Phrase object to advance to next docid */ + u8 *pbEof /* OUT: Set to 1 if EOF */ +){ + int rc = SQLITE_OK; + Fts3Doclist *pDL = &p->doclist; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + + if( p->bIncr ){ + rc = fts3EvalIncrPhraseNext(pCsr, p, pbEof); + }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){ + sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, + &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof + ); + pDL->pList = pDL->pNextDocid; + }else{ + fts3EvalDlPhraseNext(pTab, pDL, pbEof); + } + + return rc; +} + +/* +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, fts3EvalPhraseStart() is called on all phrases within the +** expression. Also the Fts3Expr.bDeferred variable is set to true for any +** expressions for which all descendent tokens are deferred. +** +** If parameter bOptOk is zero, then it is guaranteed that the +** Fts3Phrase.doclist.aAll/nAll variables contain the entire doclist for +** each phrase in the expression (subject to deferred token processing). +** Or, if bOptOk is non-zero, then one or more tokens within the expression +** may be loaded incrementally, meaning doclist.aAll/nAll is not available. +** +** If an error occurs within this function, *pRc is set to an SQLite error +** code before returning. +*/ +static void fts3EvalStartReaders( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pExpr, /* Expression to initialize phrases in */ + int *pRc /* IN/OUT: Error code */ +){ + if( pExpr && SQLITE_OK==*pRc ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + int nToken = pExpr->pPhrase->nToken; + if( nToken ){ + int i; + for(i=0; ipPhrase->aToken[i].pDeferred==0 ) break; + } + pExpr->bDeferred = (i==nToken); + } + *pRc = fts3EvalPhraseStart(pCsr, 1, pExpr->pPhrase); + }else{ + fts3EvalStartReaders(pCsr, pExpr->pLeft, pRc); + fts3EvalStartReaders(pCsr, pExpr->pRight, pRc); + pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred); + } + } +} + +/* +** An array of the following structures is assembled as part of the process +** of selecting tokens to defer before the query starts executing (as part +** of the xFilter() method). There is one element in the array for each +** token in the FTS expression. +** +** Tokens are divided into AND/NEAR clusters. All tokens in a cluster belong +** to phrases that are connected only by AND and NEAR operators (not OR or +** NOT). When determining tokens to defer, each AND/NEAR cluster is considered +** separately. The root of a tokens AND/NEAR cluster is stored in +** Fts3TokenAndCost.pRoot. +*/ +typedef struct Fts3TokenAndCost Fts3TokenAndCost; +struct Fts3TokenAndCost { + Fts3Phrase *pPhrase; /* The phrase the token belongs to */ + int iToken; /* Position of token in phrase */ + Fts3PhraseToken *pToken; /* The token itself */ + Fts3Expr *pRoot; /* Root of NEAR/AND cluster */ + int nOvfl; /* Number of overflow pages to load doclist */ + int iCol; /* The column the token must match */ +}; + +/* +** This function is used to populate an allocated Fts3TokenAndCost array. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, if an error occurs during execution, *pRc is set to an +** SQLite error code. +*/ +static void fts3EvalTokenCosts( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pRoot, /* Root of current AND/NEAR cluster */ + Fts3Expr *pExpr, /* Expression to consider */ + Fts3TokenAndCost **ppTC, /* Write new entries to *(*ppTC)++ */ + Fts3Expr ***ppOr, /* Write new OR root to *(*ppOr)++ */ + int *pRc /* IN/OUT: Error code */ +){ + if( *pRc==SQLITE_OK ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + int i; + for(i=0; *pRc==SQLITE_OK && inToken; i++){ + Fts3TokenAndCost *pTC = (*ppTC)++; + pTC->pPhrase = pPhrase; + pTC->iToken = i; + pTC->pRoot = pRoot; + pTC->pToken = &pPhrase->aToken[i]; + pTC->iCol = pPhrase->iColumn; + *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl); + } + }else if( pExpr->eType!=FTSQUERY_NOT ){ + assert( pExpr->eType==FTSQUERY_OR + || pExpr->eType==FTSQUERY_AND + || pExpr->eType==FTSQUERY_NEAR + ); + assert( pExpr->pLeft && pExpr->pRight ); + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr->pLeft; + **ppOr = pRoot; + (*ppOr)++; + } + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc); + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr->pRight; + **ppOr = pRoot; + (*ppOr)++; + } + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc); + } + } +} + +/* +** Determine the average document (row) size in pages. If successful, +** write this value to *pnPage and return SQLITE_OK. Otherwise, return +** an SQLite error code. +** +** The average document size in pages is calculated by first calculating +** determining the average size in bytes, B. If B is less than the amount +** of data that will fit on a single leaf page of an intkey table in +** this database, then the average docsize is 1. Otherwise, it is 1 plus +** the number of overflow pages consumed by a record B bytes in size. +*/ +static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ + if( pCsr->nRowAvg==0 ){ + /* The average document size, which is required to calculate the cost + ** of each doclist, has not yet been determined. Read the required + ** data from the %_stat table to calculate it. + ** + ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 + ** varints, where nCol is the number of columns in the FTS3 table. + ** The first varint is the number of documents currently stored in + ** the table. The following nCol varints contain the total amount of + ** data stored in all rows of each column of the table, from left + ** to right. + */ + int rc; + Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; + sqlite3_stmt *pStmt; + sqlite3_int64 nDoc = 0; + sqlite3_int64 nByte = 0; + const char *pEnd; + const char *a; + + rc = sqlite3Fts3SelectDoctotal(p, &pStmt); + if( rc!=SQLITE_OK ) return rc; + a = sqlite3_column_blob(pStmt, 0); + assert( a ); + + pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; + a += sqlite3Fts3GetVarint(a, &nDoc); + while( anDoc = nDoc; + pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz); + assert( pCsr->nRowAvg>0 ); + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ) return rc; + } + + *pnPage = pCsr->nRowAvg; + return SQLITE_OK; +} + +/* +** This function is called to select the tokens (if any) that will be +** deferred. The array aTC[] has already been populated when this is +** called. +** +** This function is called once for each AND/NEAR cluster in the +** expression. Each invocation determines which tokens to defer within +** the cluster with root node pRoot. See comments above the definition +** of struct Fts3TokenAndCost for more details. +** +** If no error occurs, SQLITE_OK is returned and sqlite3Fts3DeferToken() +** called on each token to defer. Otherwise, an SQLite error code is +** returned. +*/ +static int fts3EvalSelectDeferred( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pRoot, /* Consider tokens with this root node */ + Fts3TokenAndCost *aTC, /* Array of expression tokens and costs */ + int nTC /* Number of entries in aTC[] */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int nDocSize = 0; /* Number of pages per doc loaded */ + int rc = SQLITE_OK; /* Return code */ + int ii; /* Iterator variable for various purposes */ + int nOvfl = 0; /* Total overflow pages used by doclists */ + int nToken = 0; /* Total number of tokens in cluster */ + + int nMinEst = 0; /* The minimum count for any phrase so far. */ + int nLoad4 = 1; /* (Phrases that will be loaded)^4. */ + + /* Tokens are never deferred for FTS tables created using the content=xxx + ** option. The reason being that it is not guaranteed that the content + ** table actually contains the same data as the index. To prevent this from + ** causing any problems, the deferred token optimization is completely + ** disabled for content=xxx tables. */ + if( pTab->zContentTbl ){ + return SQLITE_OK; + } + + /* Count the tokens in this AND/NEAR cluster. If none of the doclists + ** associated with the tokens spill onto overflow pages, or if there is + ** only 1 token, exit early. No tokens to defer in this case. */ + for(ii=0; ii0 ); + + + /* Iterate through all tokens in this AND/NEAR cluster, in ascending order + ** of the number of overflow pages that will be loaded by the pager layer + ** to retrieve the entire doclist for the token from the full-text index. + ** Load the doclists for tokens that are either: + ** + ** a. The cheapest token in the entire query (i.e. the one visited by the + ** first iteration of this loop), or + ** + ** b. Part of a multi-token phrase. + ** + ** After each token doclist is loaded, merge it with the others from the + ** same phrase and count the number of documents that the merged doclist + ** contains. Set variable "nMinEst" to the smallest number of documents in + ** any phrase doclist for which 1 or more token doclists have been loaded. + ** Let nOther be the number of other phrases for which it is certain that + ** one or more tokens will not be deferred. + ** + ** Then, for each token, defer it if loading the doclist would result in + ** loading N or more overflow pages into memory, where N is computed as: + ** + ** (nMinEst + 4^nOther - 1) / (4^nOther) + */ + for(ii=0; iinOvfl) + ){ + pTC = &aTC[iTC]; + } + } + assert( pTC ); + + if( ii && pTC->nOvfl>=((nMinEst+(nLoad4/4)-1)/(nLoad4/4))*nDocSize ){ + /* The number of overflow pages to load for this (and therefore all + ** subsequent) tokens is greater than the estimated number of pages + ** that will be loaded if all subsequent tokens are deferred. + */ + Fts3PhraseToken *pToken = pTC->pToken; + rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); + fts3SegReaderCursorFree(pToken->pSegcsr); + pToken->pSegcsr = 0; + }else{ + /* Set nLoad4 to the value of (4^nOther) for the next iteration of the + ** for-loop. Except, limit the value to 2^24 to prevent it from + ** overflowing the 32-bit integer it is stored in. */ + if( ii<12 ) nLoad4 = nLoad4*4; + + if( ii==0 || (pTC->pPhrase->nToken>1 && ii!=nToken-1) ){ + /* Either this is the cheapest token in the entire query, or it is + ** part of a multi-token phrase. Either way, the entire doclist will + ** (eventually) be loaded into memory. It may as well be now. */ + Fts3PhraseToken *pToken = pTC->pToken; + int nList = 0; + char *pList = 0; + rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList); + assert( rc==SQLITE_OK || pList==0 ); + if( rc==SQLITE_OK ){ + rc = fts3EvalPhraseMergeToken( + pTab, pTC->pPhrase, pTC->iToken,pList,nList + ); + } + if( rc==SQLITE_OK ){ + int nCount; + nCount = fts3DoclistCountDocids( + pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll + ); + if( ii==0 || nCountpToken = 0; + } + + return rc; +} + +/* +** This function is called from within the xFilter method. It initializes +** the full-text query currently stored in pCsr->pExpr. To iterate through +** the results of a query, the caller does: +** +** fts3EvalStart(pCsr); +** while( 1 ){ +** fts3EvalNext(pCsr); +** if( pCsr->bEof ) break; +** ... return row pCsr->iPrevId to the caller ... +** } +*/ +static int fts3EvalStart(Fts3Cursor *pCsr){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int nToken = 0; + int nOr = 0; + + /* Allocate a MultiSegReader for each token in the expression. */ + fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc); + + /* Determine which, if any, tokens in the expression should be deferred. */ +#ifndef SQLITE_DISABLE_FTS4_DEFERRED + if( rc==SQLITE_OK && nToken>1 && pTab->bFts4 ){ + Fts3TokenAndCost *aTC; + Fts3Expr **apOr; + aTC = (Fts3TokenAndCost *)sqlite3_malloc( + sizeof(Fts3TokenAndCost) * nToken + + sizeof(Fts3Expr *) * nOr * 2 + ); + apOr = (Fts3Expr **)&aTC[nToken]; + + if( !aTC ){ + rc = SQLITE_NOMEM; + }else{ + int ii; + Fts3TokenAndCost *pTC = aTC; + Fts3Expr **ppOr = apOr; + + fts3EvalTokenCosts(pCsr, 0, pCsr->pExpr, &pTC, &ppOr, &rc); + nToken = (int)(pTC-aTC); + nOr = (int)(ppOr-apOr); + + if( rc==SQLITE_OK ){ + rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); + for(ii=0; rc==SQLITE_OK && iipExpr, &rc); + return rc; +} + +/* +** Invalidate the current position list for phrase pPhrase. +*/ +static void fts3EvalInvalidatePoslist(Fts3Phrase *pPhrase){ + if( pPhrase->doclist.bFreeList ){ + sqlite3_free(pPhrase->doclist.pList); + } + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + pPhrase->doclist.bFreeList = 0; +} + +/* +** This function is called to edit the position list associated with +** the phrase object passed as the fifth argument according to a NEAR +** condition. For example: +** +** abc NEAR/5 "def ghi" +** +** Parameter nNear is passed the NEAR distance of the expression (5 in +** the example above). When this function is called, *paPoslist points to +** the position list, and *pnToken is the number of phrase tokens in, the +** phrase on the other side of the NEAR operator to pPhrase. For example, +** if pPhrase refers to the "def ghi" phrase, then *paPoslist points to +** the position list associated with phrase "abc". +** +** All positions in the pPhrase position list that are not sufficiently +** close to a position in the *paPoslist position list are removed. If this +** leaves 0 positions, zero is returned. Otherwise, non-zero. +** +** Before returning, *paPoslist is set to point to the position lsit +** associated with pPhrase. And *pnToken is set to the number of tokens in +** pPhrase. +*/ +static int fts3EvalNearTrim( + int nNear, /* NEAR distance. As in "NEAR/nNear". */ + char *aTmp, /* Temporary space to use */ + char **paPoslist, /* IN/OUT: Position list */ + int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */ + Fts3Phrase *pPhrase /* The phrase object to trim the doclist of */ +){ + int nParam1 = nNear + pPhrase->nToken; + int nParam2 = nNear + *pnToken; + int nNew; + char *p2; + char *pOut; + int res; + + assert( pPhrase->doclist.pList ); + + p2 = pOut = pPhrase->doclist.pList; + res = fts3PoslistNearMerge( + &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 + ); + if( res ){ + nNew = (int)(pOut - pPhrase->doclist.pList) - 1; + assert( pPhrase->doclist.pList[nNew]=='\0' ); + assert( nNew<=pPhrase->doclist.nList && nNew>0 ); + memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew); + pPhrase->doclist.nList = nNew; + *paPoslist = pPhrase->doclist.pList; + *pnToken = pPhrase->nToken; + } + + return res; +} + +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is called. +** Otherwise, it advances the expression passed as the second argument to +** point to the next matching row in the database. Expressions iterate through +** matching rows in docid order. Ascending order if Fts3Cursor.bDesc is zero, +** or descending if it is non-zero. +** +** If an error occurs, *pRc is set to an SQLite error code. Otherwise, if +** successful, the following variables in pExpr are set: +** +** Fts3Expr.bEof (non-zero if EOF - there is no next row) +** Fts3Expr.iDocid (valid if bEof==0. The docid of the next row) +** +** If the expression is of type FTSQUERY_PHRASE, and the expression is not +** at EOF, then the following variables are populated with the position list +** for the phrase for the visited row: +** +** FTs3Expr.pPhrase->doclist.nList (length of pList in bytes) +** FTs3Expr.pPhrase->doclist.pList (pointer to position list) +** +** It says above that this function advances the expression to the next +** matching row. This is usually true, but there are the following exceptions: +** +** 1. Deferred tokens are not taken into account. If a phrase consists +** entirely of deferred tokens, it is assumed to match every row in +** the db. In this case the position-list is not populated at all. +** +** Or, if a phrase contains one or more deferred tokens and one or +** more non-deferred tokens, then the expression is advanced to the +** next possible match, considering only non-deferred tokens. In other +** words, if the phrase is "A B C", and "B" is deferred, the expression +** is advanced to the next row that contains an instance of "A * C", +** where "*" may match any single token. The position list in this case +** is populated as for "A * C" before returning. +** +** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is +** advanced to point to the next row that matches "x AND y". +** +** See sqlite3Fts3EvalTestDeferred() for details on testing if a row is +** really a match, taking into account deferred tokens and NEAR operators. +*/ +static void fts3EvalNextRow( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pExpr, /* Expr. to advance to next matching row */ + int *pRc /* IN/OUT: Error code */ +){ + if( *pRc==SQLITE_OK ){ + int bDescDoclist = pCsr->bDesc; /* Used by DOCID_CMP() macro */ + assert( pExpr->bEof==0 ); + pExpr->bStart = 1; + + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + assert( !pLeft->bDeferred || !pRight->bDeferred ); + + if( pLeft->bDeferred ){ + /* LHS is entirely deferred. So we assume it matches every row. + ** Advance the RHS iterator to find the next row visited. */ + fts3EvalNextRow(pCsr, pRight, pRc); + pExpr->iDocid = pRight->iDocid; + pExpr->bEof = pRight->bEof; + }else if( pRight->bDeferred ){ + /* RHS is entirely deferred. So we assume it matches every row. + ** Advance the LHS iterator to find the next row visited. */ + fts3EvalNextRow(pCsr, pLeft, pRc); + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + }else{ + /* Neither the RHS or LHS are deferred. */ + fts3EvalNextRow(pCsr, pLeft, pRc); + fts3EvalNextRow(pCsr, pRight, pRc); + while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ + sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( iDiff==0 ) break; + if( iDiff<0 ){ + fts3EvalNextRow(pCsr, pLeft, pRc); + }else{ + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = (pLeft->bEof || pRight->bEof); + if( pExpr->eType==FTSQUERY_NEAR && pExpr->bEof ){ + if( pRight->pPhrase && pRight->pPhrase->doclist.aAll ){ + Fts3Doclist *pDl = &pRight->pPhrase->doclist; + while( *pRc==SQLITE_OK && pRight->bEof==0 ){ + memset(pDl->pList, 0, pDl->nList); + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + if( pLeft->pPhrase && pLeft->pPhrase->doclist.aAll ){ + Fts3Doclist *pDl = &pLeft->pPhrase->doclist; + while( *pRc==SQLITE_OK && pLeft->bEof==0 ){ + memset(pDl->pList, 0, pDl->nList); + fts3EvalNextRow(pCsr, pLeft, pRc); + } + } + } + } + break; + } + + case FTSQUERY_OR: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + + assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); + assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); + + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + fts3EvalNextRow(pCsr, pLeft, pRc); + }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){ + fts3EvalNextRow(pCsr, pRight, pRc); + }else{ + fts3EvalNextRow(pCsr, pLeft, pRc); + fts3EvalNextRow(pCsr, pRight, pRc); + } + + pExpr->bEof = (pLeft->bEof && pRight->bEof); + iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + pExpr->iDocid = pLeft->iDocid; + }else{ + pExpr->iDocid = pRight->iDocid; + } + + break; + } + + case FTSQUERY_NOT: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + + if( pRight->bStart==0 ){ + fts3EvalNextRow(pCsr, pRight, pRc); + assert( *pRc!=SQLITE_OK || pRight->bStart ); + } + + fts3EvalNextRow(pCsr, pLeft, pRc); + if( pLeft->bEof==0 ){ + while( !*pRc + && !pRight->bEof + && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 + ){ + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + break; + } + + default: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + fts3EvalInvalidatePoslist(pPhrase); + *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); + pExpr->iDocid = pPhrase->doclist.iDocid; + break; + } + } + } +} + +/* +** If *pRc is not SQLITE_OK, or if pExpr is not the root node of a NEAR +** cluster, then this function returns 1 immediately. +** +** Otherwise, it checks if the current row really does match the NEAR +** expression, using the data currently stored in the position lists +** (Fts3Expr->pPhrase.doclist.pList/nList) for each phrase in the expression. +** +** If the current row is a match, the position list associated with each +** phrase in the NEAR expression is edited in place to contain only those +** phrase instances sufficiently close to their peers to satisfy all NEAR +** constraints. In this case it returns 1. If the NEAR expression does not +** match the current row, 0 is returned. The position lists may or may not +** be edited if 0 is returned. +*/ +static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ + int res = 1; + + /* The following block runs if pExpr is the root of a NEAR query. + ** For example, the query: + ** + ** "w" NEAR "x" NEAR "y" NEAR "z" + ** + ** which is represented in tree form as: + ** + ** | + ** +--NEAR--+ <-- root of NEAR query + ** | | + ** +--NEAR--+ "z" + ** | | + ** +--NEAR--+ "y" + ** | | + ** "w" "x" + ** + ** The right-hand child of a NEAR node is always a phrase. The + ** left-hand child may be either a phrase or a NEAR node. There are + ** no exceptions to this - it's the way the parser in fts3_expr.c works. + */ + if( *pRc==SQLITE_OK + && pExpr->eType==FTSQUERY_NEAR + && pExpr->bEof==0 + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ + Fts3Expr *p; + int nTmp = 0; /* Bytes of temp space */ + char *aTmp; /* Temp space for PoslistNearMerge() */ + + /* Allocate temporary working space. */ + for(p=pExpr; p->pLeft; p=p->pLeft){ + nTmp += p->pRight->pPhrase->doclist.nList; + } + nTmp += p->pPhrase->doclist.nList; + if( nTmp==0 ){ + res = 0; + }else{ + aTmp = sqlite3_malloc(nTmp*2); + if( !aTmp ){ + *pRc = SQLITE_NOMEM; + res = 0; + }else{ + char *aPoslist = p->pPhrase->doclist.pList; + int nToken = p->pPhrase->nToken; + + for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ + Fts3Phrase *pPhrase = p->pRight->pPhrase; + int nNear = p->nNear; + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } + + aPoslist = pExpr->pRight->pPhrase->doclist.pList; + nToken = pExpr->pRight->pPhrase->nToken; + for(p=pExpr->pLeft; p && res; p=p->pLeft){ + int nNear; + Fts3Phrase *pPhrase; + assert( p->pParent && p->pParent->pLeft==p ); + nNear = p->pParent->nNear; + pPhrase = ( + p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase + ); + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } + } + + sqlite3_free(aTmp); + } + } + + return res; +} + +/* +** This function is a helper function for sqlite3Fts3EvalTestDeferred(). +** Assuming no error occurs or has occurred, It returns non-zero if the +** expression passed as the second argument matches the row that pCsr +** currently points to, or zero if it does not. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** If an error occurs during execution of this function, *pRc is set to +** the appropriate SQLite error code. In this case the returned value is +** undefined. +*/ +static int fts3EvalTestExpr( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Expr to test. May or may not be root. */ + int *pRc /* IN/OUT: Error code */ +){ + int bHit = 1; /* Return value */ + if( *pRc==SQLITE_OK ){ + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: + bHit = ( + fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) + && fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) + && fts3EvalNearTest(pExpr, pRc) + ); + + /* If the NEAR expression does not match any rows, zero the doclist for + ** all phrases involved in the NEAR. This is because the snippet(), + ** offsets() and matchinfo() functions are not supposed to recognize + ** any instances of phrases that are part of unmatched NEAR queries. + ** For example if this expression: + ** + ** ... MATCH 'a OR (b NEAR c)' + ** + ** is matched against a row containing: + ** + ** 'a b d e' + ** + ** then any snippet() should ony highlight the "a" term, not the "b" + ** (as "b" is part of a non-matching NEAR clause). + */ + if( bHit==0 + && pExpr->eType==FTSQUERY_NEAR + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ + Fts3Expr *p; + for(p=pExpr; p->pPhrase==0; p=p->pLeft){ + if( p->pRight->iDocid==pCsr->iPrevId ){ + fts3EvalInvalidatePoslist(p->pRight->pPhrase); + } + } + if( p->iDocid==pCsr->iPrevId ){ + fts3EvalInvalidatePoslist(p->pPhrase); + } + } + + break; + + case FTSQUERY_OR: { + int bHit1 = fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc); + int bHit2 = fts3EvalTestExpr(pCsr, pExpr->pRight, pRc); + bHit = bHit1 || bHit2; + break; + } + + case FTSQUERY_NOT: + bHit = ( + fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) + && !fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) + ); + break; + + default: { +#ifndef SQLITE_DISABLE_FTS4_DEFERRED + if( pCsr->pDeferred + && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) + ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); + if( pExpr->bDeferred ){ + fts3EvalInvalidatePoslist(pPhrase); + } + *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); + bHit = (pPhrase->doclist.pList!=0); + pExpr->iDocid = pCsr->iPrevId; + }else +#endif + { + bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId); + } + break; + } + } + } + return bHit; +} + +/* +** This function is called as the second part of each xNext operation when +** iterating through the results of a full-text query. At this point the +** cursor points to a row that matches the query expression, with the +** following caveats: +** +** * Up until this point, "NEAR" operators in the expression have been +** treated as "AND". +** +** * Deferred tokens have not yet been considered. +** +** If *pRc is not SQLITE_OK when this function is called, it immediately +** returns 0. Otherwise, it tests whether or not after considering NEAR +** operators and deferred tokens the current row is still a match for the +** expression. It returns 1 if both of the following are true: +** +** 1. *pRc is SQLITE_OK when this function returns, and +** +** 2. After scanning the current FTS table row for the deferred tokens, +** it is determined that the row does *not* match the query. +** +** Or, if no error occurs and it seems the current row does match the FTS +** query, return 0. +*/ +SQLITE_PRIVATE int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc){ + int rc = *pRc; + int bMiss = 0; + if( rc==SQLITE_OK ){ + + /* If there are one or more deferred tokens, load the current row into + ** memory and scan it to determine the position list for each deferred + ** token. Then, see if this row is really a match, considering deferred + ** tokens and NEAR operators (neither of which were taken into account + ** earlier, by fts3EvalNextRow()). + */ + if( pCsr->pDeferred ){ + rc = fts3CursorSeek(0, pCsr); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3CacheDeferredDoclists(pCsr); + } + } + bMiss = (0==fts3EvalTestExpr(pCsr, pCsr->pExpr, &rc)); + + /* Free the position-lists accumulated for each deferred token above. */ + sqlite3Fts3FreeDeferredDoclists(pCsr); + *pRc = rc; + } + return (rc==SQLITE_OK && bMiss); +} + +/* +** Advance to the next document that matches the FTS expression in +** Fts3Cursor.pExpr. +*/ +static int fts3EvalNext(Fts3Cursor *pCsr){ + int rc = SQLITE_OK; /* Return Code */ + Fts3Expr *pExpr = pCsr->pExpr; + assert( pCsr->isEof==0 ); + if( pExpr==0 ){ + pCsr->isEof = 1; + }else{ + do { + if( pCsr->isRequireSeek==0 ){ + sqlite3_reset(pCsr->pStmt); + } + assert( sqlite3_data_count(pCsr->pStmt)==0 ); + fts3EvalNextRow(pCsr, pExpr, &rc); + pCsr->isEof = pExpr->bEof; + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = pExpr->iDocid; + }while( pCsr->isEof==0 && sqlite3Fts3EvalTestDeferred(pCsr, &rc) ); + } + + /* Check if the cursor is past the end of the docid range specified + ** by Fts3Cursor.iMinDocid/iMaxDocid. If so, set the EOF flag. */ + if( rc==SQLITE_OK && ( + (pCsr->bDesc==0 && pCsr->iPrevId>pCsr->iMaxDocid) + || (pCsr->bDesc!=0 && pCsr->iPrevIdiMinDocid) + )){ + pCsr->isEof = 1; + } + + return rc; +} + +/* +** Restart interation for expression pExpr so that the next call to +** fts3EvalNext() visits the first row. Do not allow incremental +** loading or merging of phrase doclists for this iteration. +** +** If *pRc is other than SQLITE_OK when this function is called, it is +** a no-op. If an error occurs within this function, *pRc is set to an +** SQLite error code before returning. +*/ +static void fts3EvalRestart( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int *pRc +){ + if( pExpr && *pRc==SQLITE_OK ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + + if( pPhrase ){ + fts3EvalInvalidatePoslist(pPhrase); + if( pPhrase->bIncr ){ + int i; + for(i=0; inToken; i++){ + Fts3PhraseToken *pToken = &pPhrase->aToken[i]; + assert( pToken->pDeferred==0 ); + if( pToken->pSegcsr ){ + sqlite3Fts3MsrIncrRestart(pToken->pSegcsr); + } + } + *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase); + } + pPhrase->doclist.pNextDocid = 0; + pPhrase->doclist.iDocid = 0; + pPhrase->pOrPoslist = 0; + } + + pExpr->iDocid = 0; + pExpr->bEof = 0; + pExpr->bStart = 0; + + fts3EvalRestart(pCsr, pExpr->pLeft, pRc); + fts3EvalRestart(pCsr, pExpr->pRight, pRc); + } +} + +/* +** After allocating the Fts3Expr.aMI[] array for each phrase in the +** expression rooted at pExpr, the cursor iterates through all rows matched +** by pExpr, calling this function for each row. This function increments +** the values in Fts3Expr.aMI[] according to the position-list currently +** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase +** expression nodes. +*/ +static void fts3EvalUpdateCounts(Fts3Expr *pExpr){ + if( pExpr ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + if( pPhrase && pPhrase->doclist.pList ){ + int iCol = 0; + char *p = pPhrase->doclist.pList; + + assert( *p ); + while( 1 ){ + u8 c = 0; + int iCnt = 0; + while( 0xFE & (*p | c) ){ + if( (c&0x80)==0 ) iCnt++; + c = *p++ & 0x80; + } + + /* aMI[iCol*3 + 1] = Number of occurrences + ** aMI[iCol*3 + 2] = Number of rows containing at least one instance + */ + pExpr->aMI[iCol*3 + 1] += iCnt; + pExpr->aMI[iCol*3 + 2] += (iCnt>0); + if( *p==0x00 ) break; + p++; + p += fts3GetVarint32(p, &iCol); + } + } + + fts3EvalUpdateCounts(pExpr->pLeft); + fts3EvalUpdateCounts(pExpr->pRight); + } +} + +/* +** Expression pExpr must be of type FTSQUERY_PHRASE. +** +** If it is not already allocated and populated, this function allocates and +** populates the Fts3Expr.aMI[] array for expression pExpr. If pExpr is part +** of a NEAR expression, then it also allocates and populates the same array +** for all other phrases that are part of the NEAR expression. +** +** SQLITE_OK is returned if the aMI[] array is successfully allocated and +** populated. Otherwise, if an error occurs, an SQLite error code is returned. +*/ +static int fts3EvalGatherStats( + Fts3Cursor *pCsr, /* Cursor object */ + Fts3Expr *pExpr /* FTSQUERY_PHRASE expression */ +){ + int rc = SQLITE_OK; /* Return code */ + + assert( pExpr->eType==FTSQUERY_PHRASE ); + if( pExpr->aMI==0 ){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + Fts3Expr *pRoot; /* Root of NEAR expression */ + Fts3Expr *p; /* Iterator used for several purposes */ + + sqlite3_int64 iPrevId = pCsr->iPrevId; + sqlite3_int64 iDocid; + u8 bEof; + + /* Find the root of the NEAR expression */ + pRoot = pExpr; + while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){ + pRoot = pRoot->pParent; + } + iDocid = pRoot->iDocid; + bEof = pRoot->bEof; + assert( pRoot->bStart ); + + /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */ + for(p=pRoot; p; p=p->pLeft){ + Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight); + assert( pE->aMI==0 ); + pE->aMI = (u32 *)sqlite3_malloc(pTab->nColumn * 3 * sizeof(u32)); + if( !pE->aMI ) return SQLITE_NOMEM; + memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32)); + } + + fts3EvalRestart(pCsr, pRoot, &rc); + + while( pCsr->isEof==0 && rc==SQLITE_OK ){ + + do { + /* Ensure the %_content statement is reset. */ + if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt); + assert( sqlite3_data_count(pCsr->pStmt)==0 ); + + /* Advance to the next document */ + fts3EvalNextRow(pCsr, pRoot, &rc); + pCsr->isEof = pRoot->bEof; + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = pRoot->iDocid; + }while( pCsr->isEof==0 + && pRoot->eType==FTSQUERY_NEAR + && sqlite3Fts3EvalTestDeferred(pCsr, &rc) + ); + + if( rc==SQLITE_OK && pCsr->isEof==0 ){ + fts3EvalUpdateCounts(pRoot); + } + } + + pCsr->isEof = 0; + pCsr->iPrevId = iPrevId; + + if( bEof ){ + pRoot->bEof = bEof; + }else{ + /* Caution: pRoot may iterate through docids in ascending or descending + ** order. For this reason, even though it seems more defensive, the + ** do loop can not be written: + ** + ** do {...} while( pRoot->iDocidbEof==0 ); + }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); + } + } + return rc; +} + +/* +** This function is used by the matchinfo() module to query a phrase +** expression node for the following information: +** +** 1. The total number of occurrences of the phrase in each column of +** the FTS table (considering all rows), and +** +** 2. For each column, the number of rows in the table for which the +** column contains at least one instance of the phrase. +** +** If no error occurs, SQLITE_OK is returned and the values for each column +** written into the array aiOut as follows: +** +** aiOut[iCol*3 + 1] = Number of occurrences +** aiOut[iCol*3 + 2] = Number of rows containing at least one instance +** +** Caveats: +** +** * If a phrase consists entirely of deferred tokens, then all output +** values are set to the number of documents in the table. In other +** words we assume that very common tokens occur exactly once in each +** column of each row of the table. +** +** * If a phrase contains some deferred tokens (and some non-deferred +** tokens), count the potential occurrence identified by considering +** the non-deferred tokens instead of actual phrase occurrences. +** +** * If the phrase is part of a NEAR expression, then only phrase instances +** that meet the NEAR constraint are included in the counts. +*/ +SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Phrase expression */ + u32 *aiOut /* Array to write results into (see above) */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int iCol; + + if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){ + assert( pCsr->nDoc>0 ); + for(iCol=0; iColnColumn; iCol++){ + aiOut[iCol*3 + 1] = (u32)pCsr->nDoc; + aiOut[iCol*3 + 2] = (u32)pCsr->nDoc; + } + }else{ + rc = fts3EvalGatherStats(pCsr, pExpr); + if( rc==SQLITE_OK ){ + assert( pExpr->aMI ); + for(iCol=0; iColnColumn; iCol++){ + aiOut[iCol*3 + 1] = pExpr->aMI[iCol*3 + 1]; + aiOut[iCol*3 + 2] = pExpr->aMI[iCol*3 + 2]; + } + } + } + + return rc; +} + +/* +** The expression pExpr passed as the second argument to this function +** must be of type FTSQUERY_PHRASE. +** +** The returned value is either NULL or a pointer to a buffer containing +** a position-list indicating the occurrences of the phrase in column iCol +** of the current row. +** +** More specifically, the returned buffer contains 1 varint for each +** occurrence of the phrase in the column, stored using the normal (delta+2) +** compression and is terminated by either an 0x01 or 0x00 byte. For example, +** if the requested column contains "a b X c d X X" and the position-list +** for 'X' is requested, the buffer returned may contain: +** +** 0x04 0x05 0x03 0x01 or 0x04 0x05 0x03 0x00 +** +** This function works regardless of whether or not the phrase is deferred, +** incremental, or neither. +*/ +SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist( + Fts3Cursor *pCsr, /* FTS3 cursor object */ + Fts3Expr *pExpr, /* Phrase to return doclist for */ + int iCol, /* Column to return position list for */ + char **ppOut /* OUT: Pointer to position list */ +){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + char *pIter; + int iThis; + sqlite3_int64 iDocid; + + /* If this phrase is applies specifically to some column other than + ** column iCol, return a NULL pointer. */ + *ppOut = 0; + assert( iCol>=0 && iColnColumn ); + if( (pPhrase->iColumnnColumn && pPhrase->iColumn!=iCol) ){ + return SQLITE_OK; + } + + iDocid = pExpr->iDocid; + pIter = pPhrase->doclist.pList; + if( iDocid!=pCsr->iPrevId || pExpr->bEof ){ + int rc = SQLITE_OK; + int bDescDoclist = pTab->bDescIdx; /* For DOCID_CMP macro */ + int bOr = 0; + u8 bTreeEof = 0; + Fts3Expr *p; /* Used to iterate from pExpr to root */ + Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */ + int bMatch; + + /* Check if this phrase descends from an OR expression node. If not, + ** return NULL. Otherwise, the entry that corresponds to docid + ** pCsr->iPrevId may lie earlier in the doclist buffer. Or, if the + ** tree that the node is part of has been marked as EOF, but the node + ** itself is not EOF, then it may point to an earlier entry. */ + pNear = pExpr; + for(p=pExpr->pParent; p; p=p->pParent){ + if( p->eType==FTSQUERY_OR ) bOr = 1; + if( p->eType==FTSQUERY_NEAR ) pNear = p; + if( p->bEof ) bTreeEof = 1; + } + if( bOr==0 ) return SQLITE_OK; + + /* This is the descendent of an OR node. In this case we cannot use + ** an incremental phrase. Load the entire doclist for the phrase + ** into memory in this case. */ + if( pPhrase->bIncr ){ + int bEofSave = pNear->bEof; + fts3EvalRestart(pCsr, pNear, &rc); + while( rc==SQLITE_OK && !pNear->bEof ){ + fts3EvalNextRow(pCsr, pNear, &rc); + if( bEofSave==0 && pNear->iDocid==iDocid ) break; + } + assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); + } + if( bTreeEof ){ + while( rc==SQLITE_OK && !pNear->bEof ){ + fts3EvalNextRow(pCsr, pNear, &rc); + } + } + if( rc!=SQLITE_OK ) return rc; + + bMatch = 1; + for(p=pNear; p; p=p->pLeft){ + u8 bEof = 0; + Fts3Expr *pTest = p; + Fts3Phrase *pPh; + assert( pTest->eType==FTSQUERY_NEAR || pTest->eType==FTSQUERY_PHRASE ); + if( pTest->eType==FTSQUERY_NEAR ) pTest = pTest->pRight; + assert( pTest->eType==FTSQUERY_PHRASE ); + pPh = pTest->pPhrase; + + pIter = pPh->pOrPoslist; + iDocid = pPh->iOrDocid; + if( pCsr->bDesc==bDescDoclist ){ + bEof = !pPh->doclist.nAll || + (pIter >= (pPh->doclist.aAll + pPh->doclist.nAll)); + while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){ + sqlite3Fts3DoclistNext( + bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, + &pIter, &iDocid, &bEof + ); + } + }else{ + bEof = !pPh->doclist.nAll || (pIter && pIter<=pPh->doclist.aAll); + while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){ + int dummy; + sqlite3Fts3DoclistPrev( + bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, + &pIter, &iDocid, &dummy, &bEof + ); + } + } + pPh->pOrPoslist = pIter; + pPh->iOrDocid = iDocid; + if( bEof || iDocid!=pCsr->iPrevId ) bMatch = 0; + } + + if( bMatch ){ + pIter = pPhrase->pOrPoslist; + }else{ + pIter = 0; + } + } + if( pIter==0 ) return SQLITE_OK; + + if( *pIter==0x01 ){ + pIter++; + pIter += fts3GetVarint32(pIter, &iThis); + }else{ + iThis = 0; + } + while( iThisdoclist, and +** * any Fts3MultiSegReader objects held by phrase tokens. +*/ +SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ + if( pPhrase ){ + int i; + sqlite3_free(pPhrase->doclist.aAll); + fts3EvalInvalidatePoslist(pPhrase); + memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); + for(i=0; inToken; i++){ + fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); + pPhrase->aToken[i].pSegcsr = 0; + } + } +} + + +/* +** Return SQLITE_CORRUPT_VTAB. +*/ +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3Fts3Corrupt(){ + return SQLITE_CORRUPT_VTAB; +} +#endif + +#if !SQLITE_CORE +/* +** Initialize API pointer table, if required. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +SQLITE_API int SQLITE_STDCALL sqlite3_fts3_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi) + return sqlite3Fts3Init(db); +} +#endif + +#endif + +/************** End of fts3.c ************************************************/ +/************** Begin file fts3_aux.c ****************************************/ +/* +** 2011 Jan 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ + +typedef struct Fts3auxTable Fts3auxTable; +typedef struct Fts3auxCursor Fts3auxCursor; + +struct Fts3auxTable { + sqlite3_vtab base; /* Base class used by SQLite core */ + Fts3Table *pFts3Tab; +}; + +struct Fts3auxCursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + Fts3MultiSegReader csr; /* Must be right after "base" */ + Fts3SegFilter filter; + char *zStop; + int nStop; /* Byte-length of string zStop */ + int iLangid; /* Language id to query */ + int isEof; /* True if cursor is at EOF */ + sqlite3_int64 iRowid; /* Current rowid */ + + int iCol; /* Current value of 'col' column */ + int nStat; /* Size of aStat[] array */ + struct Fts3auxColstats { + sqlite3_int64 nDoc; /* 'documents' values for current csr row */ + sqlite3_int64 nOcc; /* 'occurrences' values for current csr row */ + } *aStat; +}; + +/* +** Schema of the terms table. +*/ +#define FTS3_AUX_SCHEMA \ + "CREATE TABLE x(term, col, documents, occurrences, languageid HIDDEN)" + +/* +** This function does all the work for both the xConnect and xCreate methods. +** These tables have no persistent representation of their own, so xConnect +** and xCreate are identical operations. +*/ +static int fts3auxConnectMethod( + sqlite3 *db, /* Database connection */ + void *pUnused, /* Unused */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + char const *zDb; /* Name of database (e.g. "main") */ + char const *zFts3; /* Name of fts3 table */ + int nDb; /* Result of strlen(zDb) */ + int nFts3; /* Result of strlen(zFts3) */ + int nByte; /* Bytes of space to allocate here */ + int rc; /* value returned by declare_vtab() */ + Fts3auxTable *p; /* Virtual table object to return */ + + UNUSED_PARAMETER(pUnused); + + /* The user should invoke this in one of two forms: + ** + ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table); + ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table-db, fts4-table); + */ + if( argc!=4 && argc!=5 ) goto bad_args; + + zDb = argv[1]; + nDb = (int)strlen(zDb); + if( argc==5 ){ + if( nDb==4 && 0==sqlite3_strnicmp("temp", zDb, 4) ){ + zDb = argv[3]; + nDb = (int)strlen(zDb); + zFts3 = argv[4]; + }else{ + goto bad_args; + } + }else{ + zFts3 = argv[3]; + } + nFts3 = (int)strlen(zFts3); + + rc = sqlite3_declare_vtab(db, FTS3_AUX_SCHEMA); + if( rc!=SQLITE_OK ) return rc; + + nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2; + p = (Fts3auxTable *)sqlite3_malloc(nByte); + if( !p ) return SQLITE_NOMEM; + memset(p, 0, nByte); + + p->pFts3Tab = (Fts3Table *)&p[1]; + p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; + p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; + p->pFts3Tab->db = db; + p->pFts3Tab->nIndex = 1; + + memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); + memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); + sqlite3Fts3Dequote((char *)p->pFts3Tab->zName); + + *ppVtab = (sqlite3_vtab *)p; + return SQLITE_OK; + + bad_args: + sqlite3Fts3ErrMsg(pzErr, "invalid arguments to fts4aux constructor"); + return SQLITE_ERROR; +} + +/* +** This function does the work for both the xDisconnect and xDestroy methods. +** These tables have no persistent representation of their own, so xDisconnect +** and xDestroy are identical operations. +*/ +static int fts3auxDisconnectMethod(sqlite3_vtab *pVtab){ + Fts3auxTable *p = (Fts3auxTable *)pVtab; + Fts3Table *pFts3 = p->pFts3Tab; + int i; + + /* Free any prepared statements held */ + for(i=0; iaStmt); i++){ + sqlite3_finalize(pFts3->aStmt[i]); + } + sqlite3_free(pFts3->zSegmentsTbl); + sqlite3_free(p); + return SQLITE_OK; +} + +#define FTS4AUX_EQ_CONSTRAINT 1 +#define FTS4AUX_GE_CONSTRAINT 2 +#define FTS4AUX_LE_CONSTRAINT 4 + +/* +** xBestIndex - Analyze a WHERE and ORDER BY clause. +*/ +static int fts3auxBestIndexMethod( + sqlite3_vtab *pVTab, + sqlite3_index_info *pInfo +){ + int i; + int iEq = -1; + int iGe = -1; + int iLe = -1; + int iLangid = -1; + int iNext = 1; /* Next free argvIndex value */ + + UNUSED_PARAMETER(pVTab); + + /* This vtab delivers always results in "ORDER BY term ASC" order. */ + if( pInfo->nOrderBy==1 + && pInfo->aOrderBy[0].iColumn==0 + && pInfo->aOrderBy[0].desc==0 + ){ + pInfo->orderByConsumed = 1; + } + + /* Search for equality and range constraints on the "term" column. + ** And equality constraints on the hidden "languageid" column. */ + for(i=0; inConstraint; i++){ + if( pInfo->aConstraint[i].usable ){ + int op = pInfo->aConstraint[i].op; + int iCol = pInfo->aConstraint[i].iColumn; + + if( iCol==0 ){ + if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iEq = i; + if( op==SQLITE_INDEX_CONSTRAINT_LT ) iLe = i; + if( op==SQLITE_INDEX_CONSTRAINT_LE ) iLe = i; + if( op==SQLITE_INDEX_CONSTRAINT_GT ) iGe = i; + if( op==SQLITE_INDEX_CONSTRAINT_GE ) iGe = i; + } + if( iCol==4 ){ + if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iLangid = i; + } + } + } + + if( iEq>=0 ){ + pInfo->idxNum = FTS4AUX_EQ_CONSTRAINT; + pInfo->aConstraintUsage[iEq].argvIndex = iNext++; + pInfo->estimatedCost = 5; + }else{ + pInfo->idxNum = 0; + pInfo->estimatedCost = 20000; + if( iGe>=0 ){ + pInfo->idxNum += FTS4AUX_GE_CONSTRAINT; + pInfo->aConstraintUsage[iGe].argvIndex = iNext++; + pInfo->estimatedCost /= 2; + } + if( iLe>=0 ){ + pInfo->idxNum += FTS4AUX_LE_CONSTRAINT; + pInfo->aConstraintUsage[iLe].argvIndex = iNext++; + pInfo->estimatedCost /= 2; + } + } + if( iLangid>=0 ){ + pInfo->aConstraintUsage[iLangid].argvIndex = iNext++; + pInfo->estimatedCost--; + } + + return SQLITE_OK; +} + +/* +** xOpen - Open a cursor. +*/ +static int fts3auxOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts3auxCursor *pCsr; /* Pointer to cursor object to return */ + + UNUSED_PARAMETER(pVTab); + + pCsr = (Fts3auxCursor *)sqlite3_malloc(sizeof(Fts3auxCursor)); + if( !pCsr ) return SQLITE_NOMEM; + memset(pCsr, 0, sizeof(Fts3auxCursor)); + + *ppCsr = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** xClose - Close a cursor. +*/ +static int fts3auxCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + + sqlite3Fts3SegmentsClose(pFts3); + sqlite3Fts3SegReaderFinish(&pCsr->csr); + sqlite3_free((void *)pCsr->filter.zTerm); + sqlite3_free(pCsr->zStop); + sqlite3_free(pCsr->aStat); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +static int fts3auxGrowStatArray(Fts3auxCursor *pCsr, int nSize){ + if( nSize>pCsr->nStat ){ + struct Fts3auxColstats *aNew; + aNew = (struct Fts3auxColstats *)sqlite3_realloc(pCsr->aStat, + sizeof(struct Fts3auxColstats) * nSize + ); + if( aNew==0 ) return SQLITE_NOMEM; + memset(&aNew[pCsr->nStat], 0, + sizeof(struct Fts3auxColstats) * (nSize - pCsr->nStat) + ); + pCsr->aStat = aNew; + pCsr->nStat = nSize; + } + return SQLITE_OK; +} + +/* +** xNext - Advance the cursor to the next row, if any. +*/ +static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + int rc; + + /* Increment our pretend rowid value. */ + pCsr->iRowid++; + + for(pCsr->iCol++; pCsr->iColnStat; pCsr->iCol++){ + if( pCsr->aStat[pCsr->iCol].nDoc>0 ) return SQLITE_OK; + } + + rc = sqlite3Fts3SegReaderStep(pFts3, &pCsr->csr); + if( rc==SQLITE_ROW ){ + int i = 0; + int nDoclist = pCsr->csr.nDoclist; + char *aDoclist = pCsr->csr.aDoclist; + int iCol; + + int eState = 0; + + if( pCsr->zStop ){ + int n = (pCsr->nStopcsr.nTerm) ? pCsr->nStop : pCsr->csr.nTerm; + int mc = memcmp(pCsr->zStop, pCsr->csr.zTerm, n); + if( mc<0 || (mc==0 && pCsr->csr.nTerm>pCsr->nStop) ){ + pCsr->isEof = 1; + return SQLITE_OK; + } + } + + if( fts3auxGrowStatArray(pCsr, 2) ) return SQLITE_NOMEM; + memset(pCsr->aStat, 0, sizeof(struct Fts3auxColstats) * pCsr->nStat); + iCol = 0; + + while( iaStat[0].nDoc++; + eState = 1; + iCol = 0; + break; + + /* State 1. In this state we are expecting either a 1, indicating + ** that the following integer will be a column number, or the + ** start of a position list for column 0. + ** + ** The only difference between state 1 and state 2 is that if the + ** integer encountered in state 1 is not 0 or 1, then we need to + ** increment the column 0 "nDoc" count for this term. + */ + case 1: + assert( iCol==0 ); + if( v>1 ){ + pCsr->aStat[1].nDoc++; + } + eState = 2; + /* fall through */ + + case 2: + if( v==0 ){ /* 0x00. Next integer will be a docid. */ + eState = 0; + }else if( v==1 ){ /* 0x01. Next integer will be a column number. */ + eState = 3; + }else{ /* 2 or greater. A position. */ + pCsr->aStat[iCol+1].nOcc++; + pCsr->aStat[0].nOcc++; + } + break; + + /* State 3. The integer just read is a column number. */ + default: assert( eState==3 ); + iCol = (int)v; + if( fts3auxGrowStatArray(pCsr, iCol+2) ) return SQLITE_NOMEM; + pCsr->aStat[iCol+1].nDoc++; + eState = 2; + break; + } + } + + pCsr->iCol = 0; + rc = SQLITE_OK; + }else{ + pCsr->isEof = 1; + } + return rc; +} + +/* +** xFilter - Initialize a cursor to point at the start of its data. +*/ +static int fts3auxFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + int rc; + int isScan = 0; + int iLangVal = 0; /* Language id to query */ + + int iEq = -1; /* Index of term=? value in apVal */ + int iGe = -1; /* Index of term>=? value in apVal */ + int iLe = -1; /* Index of term<=? value in apVal */ + int iLangid = -1; /* Index of languageid=? value in apVal */ + int iNext = 0; + + UNUSED_PARAMETER(nVal); + UNUSED_PARAMETER(idxStr); + + assert( idxStr==0 ); + assert( idxNum==FTS4AUX_EQ_CONSTRAINT || idxNum==0 + || idxNum==FTS4AUX_LE_CONSTRAINT || idxNum==FTS4AUX_GE_CONSTRAINT + || idxNum==(FTS4AUX_LE_CONSTRAINT|FTS4AUX_GE_CONSTRAINT) + ); + + if( idxNum==FTS4AUX_EQ_CONSTRAINT ){ + iEq = iNext++; + }else{ + isScan = 1; + if( idxNum & FTS4AUX_GE_CONSTRAINT ){ + iGe = iNext++; + } + if( idxNum & FTS4AUX_LE_CONSTRAINT ){ + iLe = iNext++; + } + } + if( iNextfilter.zTerm); + sqlite3Fts3SegReaderFinish(&pCsr->csr); + sqlite3_free((void *)pCsr->filter.zTerm); + sqlite3_free(pCsr->aStat); + memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr); + + pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; + if( isScan ) pCsr->filter.flags |= FTS3_SEGMENT_SCAN; + + if( iEq>=0 || iGe>=0 ){ + const unsigned char *zStr = sqlite3_value_text(apVal[0]); + assert( (iEq==0 && iGe==-1) || (iEq==-1 && iGe==0) ); + if( zStr ){ + pCsr->filter.zTerm = sqlite3_mprintf("%s", zStr); + pCsr->filter.nTerm = sqlite3_value_bytes(apVal[0]); + if( pCsr->filter.zTerm==0 ) return SQLITE_NOMEM; + } + } + + if( iLe>=0 ){ + pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iLe])); + pCsr->nStop = sqlite3_value_bytes(apVal[iLe]); + if( pCsr->zStop==0 ) return SQLITE_NOMEM; + } + + if( iLangid>=0 ){ + iLangVal = sqlite3_value_int(apVal[iLangid]); + + /* If the user specified a negative value for the languageid, use zero + ** instead. This works, as the "languageid=?" constraint will also + ** be tested by the VDBE layer. The test will always be false (since + ** this module will not return a row with a negative languageid), and + ** so the overall query will return zero rows. */ + if( iLangVal<0 ) iLangVal = 0; + } + pCsr->iLangid = iLangVal; + + rc = sqlite3Fts3SegReaderCursor(pFts3, iLangVal, 0, FTS3_SEGCURSOR_ALL, + pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr + ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter); + } + + if( rc==SQLITE_OK ) rc = fts3auxNextMethod(pCursor); + return rc; +} + +/* +** xEof - Return true if the cursor is at EOF, or false otherwise. +*/ +static int fts3auxEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + return pCsr->isEof; +} + +/* +** xColumn - Return a column value. +*/ +static int fts3auxColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts3auxCursor *p = (Fts3auxCursor *)pCursor; + + assert( p->isEof==0 ); + switch( iCol ){ + case 0: /* term */ + sqlite3_result_text(pCtx, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT); + break; + + case 1: /* col */ + if( p->iCol ){ + sqlite3_result_int(pCtx, p->iCol-1); + }else{ + sqlite3_result_text(pCtx, "*", -1, SQLITE_STATIC); + } + break; + + case 2: /* documents */ + sqlite3_result_int64(pCtx, p->aStat[p->iCol].nDoc); + break; + + case 3: /* occurrences */ + sqlite3_result_int64(pCtx, p->aStat[p->iCol].nOcc); + break; + + default: /* languageid */ + assert( iCol==4 ); + sqlite3_result_int(pCtx, p->iLangid); + break; + } + + return SQLITE_OK; +} + +/* +** xRowid - Return the current rowid for the cursor. +*/ +static int fts3auxRowidMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite_int64 *pRowid /* OUT: Rowid value */ +){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + *pRowid = pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Register the fts3aux module with database connection db. Return SQLITE_OK +** if successful or an error code if sqlite3_create_module() fails. +*/ +SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db){ + static const sqlite3_module fts3aux_module = { + 0, /* iVersion */ + fts3auxConnectMethod, /* xCreate */ + fts3auxConnectMethod, /* xConnect */ + fts3auxBestIndexMethod, /* xBestIndex */ + fts3auxDisconnectMethod, /* xDisconnect */ + fts3auxDisconnectMethod, /* xDestroy */ + fts3auxOpenMethod, /* xOpen */ + fts3auxCloseMethod, /* xClose */ + fts3auxFilterMethod, /* xFilter */ + fts3auxNextMethod, /* xNext */ + fts3auxEofMethod, /* xEof */ + fts3auxColumnMethod, /* xColumn */ + fts3auxRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ + }; + int rc; /* Return code */ + + rc = sqlite3_create_module(db, "fts4aux", &fts3aux_module, 0); + return rc; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_aux.c ********************************************/ +/************** Begin file fts3_expr.c ***************************************/ +/* +** 2008 Nov 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This module contains code that implements a parser for fts3 query strings +** (the right-hand argument to the MATCH operator). Because the supported +** syntax is relatively simple, the whole tokenizer/parser system is +** hand-coded. +*/ +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* +** By default, this module parses the legacy syntax that has been +** traditionally used by fts3. Or, if SQLITE_ENABLE_FTS3_PARENTHESIS +** is defined, then it uses the new syntax. The differences between +** the new and the old syntaxes are: +** +** a) The new syntax supports parenthesis. The old does not. +** +** b) The new syntax supports the AND and NOT operators. The old does not. +** +** c) The old syntax supports the "-" token qualifier. This is not +** supported by the new syntax (it is replaced by the NOT operator). +** +** d) When using the old syntax, the OR operator has a greater precedence +** than an implicit AND. When using the new, both implicity and explicit +** AND operators have a higher precedence than OR. +** +** If compiled with SQLITE_TEST defined, then this module exports the +** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable +** to zero causes the module to use the old syntax. If it is set to +** non-zero the new syntax is activated. This is so both syntaxes can +** be tested using a single build of testfixture. +** +** The following describes the syntax supported by the fts3 MATCH +** operator in a similar format to that used by the lemon parser +** generator. This module does not use actually lemon, it uses a +** custom parser. +** +** query ::= andexpr (OR andexpr)*. +** +** andexpr ::= notexpr (AND? notexpr)*. +** +** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*. +** notexpr ::= LP query RP. +** +** nearexpr ::= phrase (NEAR distance_opt nearexpr)*. +** +** distance_opt ::= . +** distance_opt ::= / INTEGER. +** +** phrase ::= TOKEN. +** phrase ::= COLUMN:TOKEN. +** phrase ::= "TOKEN TOKEN TOKEN...". +*/ + +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_fts3_enable_parentheses = 0; +#else +# ifdef SQLITE_ENABLE_FTS3_PARENTHESIS +# define sqlite3_fts3_enable_parentheses 1 +# else +# define sqlite3_fts3_enable_parentheses 0 +# endif +#endif + +/* +** Default span for NEAR operators. +*/ +#define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10 + +/* #include */ +/* #include */ + +/* +** isNot: +** This variable is used by function getNextNode(). When getNextNode() is +** called, it sets ParseContext.isNot to true if the 'next node' is a +** FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the +** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to +** zero. +*/ +typedef struct ParseContext ParseContext; +struct ParseContext { + sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ + int iLangid; /* Language id used with tokenizer */ + const char **azCol; /* Array of column names for fts3 table */ + int bFts4; /* True to allow FTS4-only syntax */ + int nCol; /* Number of entries in azCol[] */ + int iDefaultCol; /* Default column to query */ + int isNot; /* True if getNextNode() sees a unary - */ + sqlite3_context *pCtx; /* Write error message here */ + int nNest; /* Number of nested brackets */ +}; + +/* +** This function is equivalent to the standard isspace() function. +** +** The standard isspace() can be awkward to use safely, because although it +** is defined to accept an argument of type int, its behavior when passed +** an integer that falls outside of the range of the unsigned char type +** is undefined (and sometimes, "undefined" means segfault). This wrapper +** is defined to accept an argument of type char, and always returns 0 for +** any values that fall outside of the range of the unsigned char type (i.e. +** negative values). +*/ +static int fts3isspace(char c){ + return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f'; +} + +/* +** Allocate nByte bytes of memory using sqlite3_malloc(). If successful, +** zero the memory before returning a pointer to it. If unsuccessful, +** return NULL. +*/ +static void *fts3MallocZero(int nByte){ + void *pRet = sqlite3_malloc(nByte); + if( pRet ) memset(pRet, 0, nByte); + return pRet; +} + +SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer( + sqlite3_tokenizer *pTokenizer, + int iLangid, + const char *z, + int n, + sqlite3_tokenizer_cursor **ppCsr +){ + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + sqlite3_tokenizer_cursor *pCsr = 0; + int rc; + + rc = pModule->xOpen(pTokenizer, z, n, &pCsr); + assert( rc==SQLITE_OK || pCsr==0 ); + if( rc==SQLITE_OK ){ + pCsr->pTokenizer = pTokenizer; + if( pModule->iVersion>=1 ){ + rc = pModule->xLanguageid(pCsr, iLangid); + if( rc!=SQLITE_OK ){ + pModule->xClose(pCsr); + pCsr = 0; + } + } + } + *ppCsr = pCsr; + return rc; +} + +/* +** Function getNextNode(), which is called by fts3ExprParse(), may itself +** call fts3ExprParse(). So this forward declaration is required. +*/ +static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *); + +/* +** Extract the next token from buffer z (length n) using the tokenizer +** and other information (column names etc.) in pParse. Create an Fts3Expr +** structure of type FTSQUERY_PHRASE containing a phrase consisting of this +** single token and set *ppExpr to point to it. If the end of the buffer is +** reached before a token is found, set *ppExpr to zero. It is the +** responsibility of the caller to eventually deallocate the allocated +** Fts3Expr structure (if any) by passing it to sqlite3_free(). +** +** Return SQLITE_OK if successful, or SQLITE_NOMEM if a memory allocation +** fails. +*/ +static int getNextToken( + ParseContext *pParse, /* fts3 query parse context */ + int iCol, /* Value for Fts3Phrase.iColumn */ + const char *z, int n, /* Input string */ + Fts3Expr **ppExpr, /* OUT: expression */ + int *pnConsumed /* OUT: Number of bytes consumed */ +){ + sqlite3_tokenizer *pTokenizer = pParse->pTokenizer; + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + int rc; + sqlite3_tokenizer_cursor *pCursor; + Fts3Expr *pRet = 0; + int i = 0; + + /* Set variable i to the maximum number of bytes of input to tokenize. */ + for(i=0; iiLangid, z, i, &pCursor); + if( rc==SQLITE_OK ){ + const char *zToken; + int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0; + int nByte; /* total space to allocate */ + + rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); + if( rc==SQLITE_OK ){ + nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; + pRet = (Fts3Expr *)fts3MallocZero(nByte); + if( !pRet ){ + rc = SQLITE_NOMEM; + }else{ + pRet->eType = FTSQUERY_PHRASE; + pRet->pPhrase = (Fts3Phrase *)&pRet[1]; + pRet->pPhrase->nToken = 1; + pRet->pPhrase->iColumn = iCol; + pRet->pPhrase->aToken[0].n = nToken; + pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1]; + memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken); + + if( iEndpPhrase->aToken[0].isPrefix = 1; + iEnd++; + } + + while( 1 ){ + if( !sqlite3_fts3_enable_parentheses + && iStart>0 && z[iStart-1]=='-' + ){ + pParse->isNot = 1; + iStart--; + }else if( pParse->bFts4 && iStart>0 && z[iStart-1]=='^' ){ + pRet->pPhrase->aToken[0].bFirst = 1; + iStart--; + }else{ + break; + } + } + + } + *pnConsumed = iEnd; + }else if( i && rc==SQLITE_DONE ){ + rc = SQLITE_OK; + } + + pModule->xClose(pCursor); + } + + *ppExpr = pRet; + return rc; +} + + +/* +** Enlarge a memory allocation. If an out-of-memory allocation occurs, +** then free the old allocation. +*/ +static void *fts3ReallocOrFree(void *pOrig, int nNew){ + void *pRet = sqlite3_realloc(pOrig, nNew); + if( !pRet ){ + sqlite3_free(pOrig); + } + return pRet; +} + +/* +** Buffer zInput, length nInput, contains the contents of a quoted string +** that appeared as part of an fts3 query expression. Neither quote character +** is included in the buffer. This function attempts to tokenize the entire +** input buffer and create an Fts3Expr structure of type FTSQUERY_PHRASE +** containing the results. +** +** If successful, SQLITE_OK is returned and *ppExpr set to point at the +** allocated Fts3Expr structure. Otherwise, either SQLITE_NOMEM (out of memory +** error) or SQLITE_ERROR (tokenization error) is returned and *ppExpr set +** to 0. +*/ +static int getNextString( + ParseContext *pParse, /* fts3 query parse context */ + const char *zInput, int nInput, /* Input string */ + Fts3Expr **ppExpr /* OUT: expression */ +){ + sqlite3_tokenizer *pTokenizer = pParse->pTokenizer; + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + int rc; + Fts3Expr *p = 0; + sqlite3_tokenizer_cursor *pCursor = 0; + char *zTemp = 0; + int nTemp = 0; + + const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase); + int nToken = 0; + + /* The final Fts3Expr data structure, including the Fts3Phrase, + ** Fts3PhraseToken structures token buffers are all stored as a single + ** allocation so that the expression can be freed with a single call to + ** sqlite3_free(). Setting this up requires a two pass approach. + ** + ** The first pass, in the block below, uses a tokenizer cursor to iterate + ** through the tokens in the expression. This pass uses fts3ReallocOrFree() + ** to assemble data in two dynamic buffers: + ** + ** Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase + ** structure, followed by the array of Fts3PhraseToken + ** structures. This pass only populates the Fts3PhraseToken array. + ** + ** Buffer zTemp: Contains copies of all tokens. + ** + ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below, + ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase + ** structures. + */ + rc = sqlite3Fts3OpenTokenizer( + pTokenizer, pParse->iLangid, zInput, nInput, &pCursor); + if( rc==SQLITE_OK ){ + int ii; + for(ii=0; rc==SQLITE_OK; ii++){ + const char *zByte; + int nByte = 0, iBegin = 0, iEnd = 0, iPos = 0; + rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos); + if( rc==SQLITE_OK ){ + Fts3PhraseToken *pToken; + + p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken)); + if( !p ) goto no_mem; + + zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte); + if( !zTemp ) goto no_mem; + + assert( nToken==ii ); + pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii]; + memset(pToken, 0, sizeof(Fts3PhraseToken)); + + memcpy(&zTemp[nTemp], zByte, nByte); + nTemp += nByte; + + pToken->n = nByte; + pToken->isPrefix = (iEndbFirst = (iBegin>0 && zInput[iBegin-1]=='^'); + nToken = ii+1; + } + } + + pModule->xClose(pCursor); + pCursor = 0; + } + + if( rc==SQLITE_DONE ){ + int jj; + char *zBuf = 0; + + p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp); + if( !p ) goto no_mem; + memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p); + p->eType = FTSQUERY_PHRASE; + p->pPhrase = (Fts3Phrase *)&p[1]; + p->pPhrase->iColumn = pParse->iDefaultCol; + p->pPhrase->nToken = nToken; + + zBuf = (char *)&p->pPhrase->aToken[nToken]; + if( zTemp ){ + memcpy(zBuf, zTemp, nTemp); + sqlite3_free(zTemp); + }else{ + assert( nTemp==0 ); + } + + for(jj=0; jjpPhrase->nToken; jj++){ + p->pPhrase->aToken[jj].z = zBuf; + zBuf += p->pPhrase->aToken[jj].n; + } + rc = SQLITE_OK; + } + + *ppExpr = p; + return rc; +no_mem: + + if( pCursor ){ + pModule->xClose(pCursor); + } + sqlite3_free(zTemp); + sqlite3_free(p); + *ppExpr = 0; + return SQLITE_NOMEM; +} + +/* +** The output variable *ppExpr is populated with an allocated Fts3Expr +** structure, or set to 0 if the end of the input buffer is reached. +** +** Returns an SQLite error code. SQLITE_OK if everything works, SQLITE_NOMEM +** if a malloc failure occurs, or SQLITE_ERROR if a parse error is encountered. +** If SQLITE_ERROR is returned, pContext is populated with an error message. +*/ +static int getNextNode( + ParseContext *pParse, /* fts3 query parse context */ + const char *z, int n, /* Input string */ + Fts3Expr **ppExpr, /* OUT: expression */ + int *pnConsumed /* OUT: Number of bytes consumed */ +){ + static const struct Fts3Keyword { + char *z; /* Keyword text */ + unsigned char n; /* Length of the keyword */ + unsigned char parenOnly; /* Only valid in paren mode */ + unsigned char eType; /* Keyword code */ + } aKeyword[] = { + { "OR" , 2, 0, FTSQUERY_OR }, + { "AND", 3, 1, FTSQUERY_AND }, + { "NOT", 3, 1, FTSQUERY_NOT }, + { "NEAR", 4, 0, FTSQUERY_NEAR } + }; + int ii; + int iCol; + int iColLen; + int rc; + Fts3Expr *pRet = 0; + + const char *zInput = z; + int nInput = n; + + pParse->isNot = 0; + + /* Skip over any whitespace before checking for a keyword, an open or + ** close bracket, or a quoted string. + */ + while( nInput>0 && fts3isspace(*zInput) ){ + nInput--; + zInput++; + } + if( nInput==0 ){ + return SQLITE_DONE; + } + + /* See if we are dealing with a keyword. */ + for(ii=0; ii<(int)(sizeof(aKeyword)/sizeof(struct Fts3Keyword)); ii++){ + const struct Fts3Keyword *pKey = &aKeyword[ii]; + + if( (pKey->parenOnly & ~sqlite3_fts3_enable_parentheses)!=0 ){ + continue; + } + + if( nInput>=pKey->n && 0==memcmp(zInput, pKey->z, pKey->n) ){ + int nNear = SQLITE_FTS3_DEFAULT_NEAR_PARAM; + int nKey = pKey->n; + char cNext; + + /* If this is a "NEAR" keyword, check for an explicit nearness. */ + if( pKey->eType==FTSQUERY_NEAR ){ + assert( nKey==4 ); + if( zInput[4]=='/' && zInput[5]>='0' && zInput[5]<='9' ){ + nNear = 0; + for(nKey=5; zInput[nKey]>='0' && zInput[nKey]<='9'; nKey++){ + nNear = nNear * 10 + (zInput[nKey] - '0'); + } + } + } + + /* At this point this is probably a keyword. But for that to be true, + ** the next byte must contain either whitespace, an open or close + ** parenthesis, a quote character, or EOF. + */ + cNext = zInput[nKey]; + if( fts3isspace(cNext) + || cNext=='"' || cNext=='(' || cNext==')' || cNext==0 + ){ + pRet = (Fts3Expr *)fts3MallocZero(sizeof(Fts3Expr)); + if( !pRet ){ + return SQLITE_NOMEM; + } + pRet->eType = pKey->eType; + pRet->nNear = nNear; + *ppExpr = pRet; + *pnConsumed = (int)((zInput - z) + nKey); + return SQLITE_OK; + } + + /* Turns out that wasn't a keyword after all. This happens if the + ** user has supplied a token such as "ORacle". Continue. + */ + } + } + + /* See if we are dealing with a quoted phrase. If this is the case, then + ** search for the closing quote and pass the whole string to getNextString() + ** for processing. This is easy to do, as fts3 has no syntax for escaping + ** a quote character embedded in a string. + */ + if( *zInput=='"' ){ + for(ii=1; iinNest++; + rc = fts3ExprParse(pParse, zInput+1, nInput-1, ppExpr, &nConsumed); + if( rc==SQLITE_OK && !*ppExpr ){ rc = SQLITE_DONE; } + *pnConsumed = (int)(zInput - z) + 1 + nConsumed; + return rc; + }else if( *zInput==')' ){ + pParse->nNest--; + *pnConsumed = (int)((zInput - z) + 1); + *ppExpr = 0; + return SQLITE_DONE; + } + } + + /* If control flows to this point, this must be a regular token, or + ** the end of the input. Read a regular token using the sqlite3_tokenizer + ** interface. Before doing so, figure out if there is an explicit + ** column specifier for the token. + ** + ** TODO: Strangely, it is not possible to associate a column specifier + ** with a quoted phrase, only with a single token. Not sure if this was + ** an implementation artifact or an intentional decision when fts3 was + ** first implemented. Whichever it was, this module duplicates the + ** limitation. + */ + iCol = pParse->iDefaultCol; + iColLen = 0; + for(ii=0; iinCol; ii++){ + const char *zStr = pParse->azCol[ii]; + int nStr = (int)strlen(zStr); + if( nInput>nStr && zInput[nStr]==':' + && sqlite3_strnicmp(zStr, zInput, nStr)==0 + ){ + iCol = ii; + iColLen = (int)((zInput - z) + nStr + 1); + break; + } + } + rc = getNextToken(pParse, iCol, &z[iColLen], n-iColLen, ppExpr, pnConsumed); + *pnConsumed += iColLen; + return rc; +} + +/* +** The argument is an Fts3Expr structure for a binary operator (any type +** except an FTSQUERY_PHRASE). Return an integer value representing the +** precedence of the operator. Lower values have a higher precedence (i.e. +** group more tightly). For example, in the C language, the == operator +** groups more tightly than ||, and would therefore have a higher precedence. +** +** When using the new fts3 query syntax (when SQLITE_ENABLE_FTS3_PARENTHESIS +** is defined), the order of the operators in precedence from highest to +** lowest is: +** +** NEAR +** NOT +** AND (including implicit ANDs) +** OR +** +** Note that when using the old query syntax, the OR operator has a higher +** precedence than the AND operator. +*/ +static int opPrecedence(Fts3Expr *p){ + assert( p->eType!=FTSQUERY_PHRASE ); + if( sqlite3_fts3_enable_parentheses ){ + return p->eType; + }else if( p->eType==FTSQUERY_NEAR ){ + return 1; + }else if( p->eType==FTSQUERY_OR ){ + return 2; + } + assert( p->eType==FTSQUERY_AND ); + return 3; +} + +/* +** Argument ppHead contains a pointer to the current head of a query +** expression tree being parsed. pPrev is the expression node most recently +** inserted into the tree. This function adds pNew, which is always a binary +** operator node, into the expression tree based on the relative precedence +** of pNew and the existing nodes of the tree. This may result in the head +** of the tree changing, in which case *ppHead is set to the new root node. +*/ +static void insertBinaryOperator( + Fts3Expr **ppHead, /* Pointer to the root node of a tree */ + Fts3Expr *pPrev, /* Node most recently inserted into the tree */ + Fts3Expr *pNew /* New binary node to insert into expression tree */ +){ + Fts3Expr *pSplit = pPrev; + while( pSplit->pParent && opPrecedence(pSplit->pParent)<=opPrecedence(pNew) ){ + pSplit = pSplit->pParent; + } + + if( pSplit->pParent ){ + assert( pSplit->pParent->pRight==pSplit ); + pSplit->pParent->pRight = pNew; + pNew->pParent = pSplit->pParent; + }else{ + *ppHead = pNew; + } + pNew->pLeft = pSplit; + pSplit->pParent = pNew; +} + +/* +** Parse the fts3 query expression found in buffer z, length n. This function +** returns either when the end of the buffer is reached or an unmatched +** closing bracket - ')' - is encountered. +** +** If successful, SQLITE_OK is returned, *ppExpr is set to point to the +** parsed form of the expression and *pnConsumed is set to the number of +** bytes read from buffer z. Otherwise, *ppExpr is set to 0 and SQLITE_NOMEM +** (out of memory error) or SQLITE_ERROR (parse error) is returned. +*/ +static int fts3ExprParse( + ParseContext *pParse, /* fts3 query parse context */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr, /* OUT: Parsed query structure */ + int *pnConsumed /* OUT: Number of bytes consumed */ +){ + Fts3Expr *pRet = 0; + Fts3Expr *pPrev = 0; + Fts3Expr *pNotBranch = 0; /* Only used in legacy parse mode */ + int nIn = n; + const char *zIn = z; + int rc = SQLITE_OK; + int isRequirePhrase = 1; + + while( rc==SQLITE_OK ){ + Fts3Expr *p = 0; + int nByte = 0; + + rc = getNextNode(pParse, zIn, nIn, &p, &nByte); + assert( nByte>0 || (rc!=SQLITE_OK && p==0) ); + if( rc==SQLITE_OK ){ + if( p ){ + int isPhrase; + + if( !sqlite3_fts3_enable_parentheses + && p->eType==FTSQUERY_PHRASE && pParse->isNot + ){ + /* Create an implicit NOT operator. */ + Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); + if( !pNot ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_NOMEM; + goto exprparse_out; + } + pNot->eType = FTSQUERY_NOT; + pNot->pRight = p; + p->pParent = pNot; + if( pNotBranch ){ + pNot->pLeft = pNotBranch; + pNotBranch->pParent = pNot; + } + pNotBranch = pNot; + p = pPrev; + }else{ + int eType = p->eType; + isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); + + /* The isRequirePhrase variable is set to true if a phrase or + ** an expression contained in parenthesis is required. If a + ** binary operator (AND, OR, NOT or NEAR) is encounted when + ** isRequirePhrase is set, this is a syntax error. + */ + if( !isPhrase && isRequirePhrase ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + goto exprparse_out; + } + + if( isPhrase && !isRequirePhrase ){ + /* Insert an implicit AND operator. */ + Fts3Expr *pAnd; + assert( pRet && pPrev ); + pAnd = fts3MallocZero(sizeof(Fts3Expr)); + if( !pAnd ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_NOMEM; + goto exprparse_out; + } + pAnd->eType = FTSQUERY_AND; + insertBinaryOperator(&pRet, pPrev, pAnd); + pPrev = pAnd; + } + + /* This test catches attempts to make either operand of a NEAR + ** operator something other than a phrase. For example, either of + ** the following: + ** + ** (bracketed expression) NEAR phrase + ** phrase NEAR (bracketed expression) + ** + ** Return an error in either case. + */ + if( pPrev && ( + (eType==FTSQUERY_NEAR && !isPhrase && pPrev->eType!=FTSQUERY_PHRASE) + || (eType!=FTSQUERY_PHRASE && isPhrase && pPrev->eType==FTSQUERY_NEAR) + )){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + goto exprparse_out; + } + + if( isPhrase ){ + if( pRet ){ + assert( pPrev && pPrev->pLeft && pPrev->pRight==0 ); + pPrev->pRight = p; + p->pParent = pPrev; + }else{ + pRet = p; + } + }else{ + insertBinaryOperator(&pRet, pPrev, p); + } + isRequirePhrase = !isPhrase; + } + pPrev = p; + } + assert( nByte>0 ); + } + assert( rc!=SQLITE_OK || (nByte>0 && nByte<=nIn) ); + nIn -= nByte; + zIn += nByte; + } + + if( rc==SQLITE_DONE && pRet && isRequirePhrase ){ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + if( !sqlite3_fts3_enable_parentheses && pNotBranch ){ + if( !pRet ){ + rc = SQLITE_ERROR; + }else{ + Fts3Expr *pIter = pNotBranch; + while( pIter->pLeft ){ + pIter = pIter->pLeft; + } + pIter->pLeft = pRet; + pRet->pParent = pIter; + pRet = pNotBranch; + } + } + } + *pnConsumed = n - nIn; + +exprparse_out: + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRet); + sqlite3Fts3ExprFree(pNotBranch); + pRet = 0; + } + *ppExpr = pRet; + return rc; +} + +/* +** Return SQLITE_ERROR if the maximum depth of the expression tree passed +** as the only argument is more than nMaxDepth. +*/ +static int fts3ExprCheckDepth(Fts3Expr *p, int nMaxDepth){ + int rc = SQLITE_OK; + if( p ){ + if( nMaxDepth<0 ){ + rc = SQLITE_TOOBIG; + }else{ + rc = fts3ExprCheckDepth(p->pLeft, nMaxDepth-1); + if( rc==SQLITE_OK ){ + rc = fts3ExprCheckDepth(p->pRight, nMaxDepth-1); + } + } + } + return rc; +} + +/* +** This function attempts to transform the expression tree at (*pp) to +** an equivalent but more balanced form. The tree is modified in place. +** If successful, SQLITE_OK is returned and (*pp) set to point to the +** new root expression node. +** +** nMaxDepth is the maximum allowable depth of the balanced sub-tree. +** +** Otherwise, if an error occurs, an SQLite error code is returned and +** expression (*pp) freed. +*/ +static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){ + int rc = SQLITE_OK; /* Return code */ + Fts3Expr *pRoot = *pp; /* Initial root node */ + Fts3Expr *pFree = 0; /* List of free nodes. Linked by pParent. */ + int eType = pRoot->eType; /* Type of node in this tree */ + + if( nMaxDepth==0 ){ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_OK ){ + if( (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){ + Fts3Expr **apLeaf; + apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth); + if( 0==apLeaf ){ + rc = SQLITE_NOMEM; + }else{ + memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth); + } + + if( rc==SQLITE_OK ){ + int i; + Fts3Expr *p; + + /* Set $p to point to the left-most leaf in the tree of eType nodes. */ + for(p=pRoot; p->eType==eType; p=p->pLeft){ + assert( p->pParent==0 || p->pParent->pLeft==p ); + assert( p->pLeft && p->pRight ); + } + + /* This loop runs once for each leaf in the tree of eType nodes. */ + while( 1 ){ + int iLvl; + Fts3Expr *pParent = p->pParent; /* Current parent of p */ + + assert( pParent==0 || pParent->pLeft==p ); + p->pParent = 0; + if( pParent ){ + pParent->pLeft = 0; + }else{ + pRoot = 0; + } + rc = fts3ExprBalance(&p, nMaxDepth-1); + if( rc!=SQLITE_OK ) break; + + for(iLvl=0; p && iLvlpLeft = apLeaf[iLvl]; + pFree->pRight = p; + pFree->pLeft->pParent = pFree; + pFree->pRight->pParent = pFree; + + p = pFree; + pFree = pFree->pParent; + p->pParent = 0; + apLeaf[iLvl] = 0; + } + } + if( p ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_TOOBIG; + break; + } + + /* If that was the last leaf node, break out of the loop */ + if( pParent==0 ) break; + + /* Set $p to point to the next leaf in the tree of eType nodes */ + for(p=pParent->pRight; p->eType==eType; p=p->pLeft); + + /* Remove pParent from the original tree. */ + assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent ); + pParent->pRight->pParent = pParent->pParent; + if( pParent->pParent ){ + pParent->pParent->pLeft = pParent->pRight; + }else{ + assert( pParent==pRoot ); + pRoot = pParent->pRight; + } + + /* Link pParent into the free node list. It will be used as an + ** internal node of the new tree. */ + pParent->pParent = pFree; + pFree = pParent; + } + + if( rc==SQLITE_OK ){ + p = 0; + for(i=0; ipParent = 0; + }else{ + assert( pFree!=0 ); + pFree->pRight = p; + pFree->pLeft = apLeaf[i]; + pFree->pLeft->pParent = pFree; + pFree->pRight->pParent = pFree; + + p = pFree; + pFree = pFree->pParent; + p->pParent = 0; + } + } + } + pRoot = p; + }else{ + /* An error occurred. Delete the contents of the apLeaf[] array + ** and pFree list. Everything else is cleaned up by the call to + ** sqlite3Fts3ExprFree(pRoot) below. */ + Fts3Expr *pDel; + for(i=0; ipParent; + sqlite3_free(pDel); + } + } + + assert( pFree==0 ); + sqlite3_free( apLeaf ); + } + }else if( eType==FTSQUERY_NOT ){ + Fts3Expr *pLeft = pRoot->pLeft; + Fts3Expr *pRight = pRoot->pRight; + + pRoot->pLeft = 0; + pRoot->pRight = 0; + pLeft->pParent = 0; + pRight->pParent = 0; + + rc = fts3ExprBalance(&pLeft, nMaxDepth-1); + if( rc==SQLITE_OK ){ + rc = fts3ExprBalance(&pRight, nMaxDepth-1); + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRight); + sqlite3Fts3ExprFree(pLeft); + }else{ + assert( pLeft && pRight ); + pRoot->pLeft = pLeft; + pLeft->pParent = pRoot; + pRoot->pRight = pRight; + pRight->pParent = pRoot; + } + } + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRoot); + pRoot = 0; + } + *pp = pRoot; + return rc; +} + +/* +** This function is similar to sqlite3Fts3ExprParse(), with the following +** differences: +** +** 1. It does not do expression rebalancing. +** 2. It does not check that the expression does not exceed the +** maximum allowable depth. +** 3. Even if it fails, *ppExpr may still be set to point to an +** expression tree. It should be deleted using sqlite3Fts3ExprFree() +** in this case. +*/ +static int fts3ExprParseUnbalanced( + sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ + int iLangid, /* Language id for tokenizer */ + char **azCol, /* Array of column names for fts3 table */ + int bFts4, /* True to allow FTS4-only syntax */ + int nCol, /* Number of entries in azCol[] */ + int iDefaultCol, /* Default column to query */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr /* OUT: Parsed query structure */ +){ + int nParsed; + int rc; + ParseContext sParse; + + memset(&sParse, 0, sizeof(ParseContext)); + sParse.pTokenizer = pTokenizer; + sParse.iLangid = iLangid; + sParse.azCol = (const char **)azCol; + sParse.nCol = nCol; + sParse.iDefaultCol = iDefaultCol; + sParse.bFts4 = bFts4; + if( z==0 ){ + *ppExpr = 0; + return SQLITE_OK; + } + if( n<0 ){ + n = (int)strlen(z); + } + rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed); + assert( rc==SQLITE_OK || *ppExpr==0 ); + + /* Check for mismatched parenthesis */ + if( rc==SQLITE_OK && sParse.nNest ){ + rc = SQLITE_ERROR; + } + + return rc; +} + +/* +** Parameters z and n contain a pointer to and length of a buffer containing +** an fts3 query expression, respectively. This function attempts to parse the +** query expression and create a tree of Fts3Expr structures representing the +** parsed expression. If successful, *ppExpr is set to point to the head +** of the parsed expression tree and SQLITE_OK is returned. If an error +** occurs, either SQLITE_NOMEM (out-of-memory error) or SQLITE_ERROR (parse +** error) is returned and *ppExpr is set to 0. +** +** If parameter n is a negative number, then z is assumed to point to a +** nul-terminated string and the length is determined using strlen(). +** +** The first parameter, pTokenizer, is passed the fts3 tokenizer module to +** use to normalize query tokens while parsing the expression. The azCol[] +** array, which is assumed to contain nCol entries, should contain the names +** of each column in the target fts3 table, in order from left to right. +** Column names must be nul-terminated strings. +** +** The iDefaultCol parameter should be passed the index of the table column +** that appears on the left-hand-side of the MATCH operator (the default +** column to match against for tokens for which a column name is not explicitly +** specified as part of the query string), or -1 if tokens may by default +** match any table column. +*/ +SQLITE_PRIVATE int sqlite3Fts3ExprParse( + sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ + int iLangid, /* Language id for tokenizer */ + char **azCol, /* Array of column names for fts3 table */ + int bFts4, /* True to allow FTS4-only syntax */ + int nCol, /* Number of entries in azCol[] */ + int iDefaultCol, /* Default column to query */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr, /* OUT: Parsed query structure */ + char **pzErr /* OUT: Error message (sqlite3_malloc) */ +){ + int rc = fts3ExprParseUnbalanced( + pTokenizer, iLangid, azCol, bFts4, nCol, iDefaultCol, z, n, ppExpr + ); + + /* Rebalance the expression. And check that its depth does not exceed + ** SQLITE_FTS3_MAX_EXPR_DEPTH. */ + if( rc==SQLITE_OK && *ppExpr ){ + rc = fts3ExprBalance(ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH); + if( rc==SQLITE_OK ){ + rc = fts3ExprCheckDepth(*ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH); + } + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(*ppExpr); + *ppExpr = 0; + if( rc==SQLITE_TOOBIG ){ + sqlite3Fts3ErrMsg(pzErr, + "FTS expression tree is too large (maximum depth %d)", + SQLITE_FTS3_MAX_EXPR_DEPTH + ); + rc = SQLITE_ERROR; + }else if( rc==SQLITE_ERROR ){ + sqlite3Fts3ErrMsg(pzErr, "malformed MATCH expression: [%s]", z); + } + } + + return rc; +} + +/* +** Free a single node of an expression tree. +*/ +static void fts3FreeExprNode(Fts3Expr *p){ + assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); + sqlite3Fts3EvalPhraseCleanup(p->pPhrase); + sqlite3_free(p->aMI); + sqlite3_free(p); +} + +/* +** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse(). +** +** This function would be simpler if it recursively called itself. But +** that would mean passing a sufficiently large expression to ExprParse() +** could cause a stack overflow. +*/ +SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *pDel){ + Fts3Expr *p; + assert( pDel==0 || pDel->pParent==0 ); + for(p=pDel; p && (p->pLeft||p->pRight); p=(p->pLeft ? p->pLeft : p->pRight)){ + assert( p->pParent==0 || p==p->pParent->pRight || p==p->pParent->pLeft ); + } + while( p ){ + Fts3Expr *pParent = p->pParent; + fts3FreeExprNode(p); + if( pParent && p==pParent->pLeft && pParent->pRight ){ + p = pParent->pRight; + while( p && (p->pLeft || p->pRight) ){ + assert( p==p->pParent->pRight || p==p->pParent->pLeft ); + p = (p->pLeft ? p->pLeft : p->pRight); + } + }else{ + p = pParent; + } + } +} + +/**************************************************************************** +***************************************************************************** +** Everything after this point is just test code. +*/ + +#ifdef SQLITE_TEST + +/* #include */ + +/* +** Function to query the hash-table of tokenizers (see README.tokenizers). +*/ +static int queryTestTokenizer( + sqlite3 *db, + const char *zName, + const sqlite3_tokenizer_module **pp +){ + int rc; + sqlite3_stmt *pStmt; + const char zSql[] = "SELECT fts3_tokenizer(?)"; + + *pp = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ + memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); + } + } + + return sqlite3_finalize(pStmt); +} + +/* +** Return a pointer to a buffer containing a text representation of the +** expression passed as the first argument. The buffer is obtained from +** sqlite3_malloc(). It is the responsibility of the caller to use +** sqlite3_free() to release the memory. If an OOM condition is encountered, +** NULL is returned. +** +** If the second argument is not NULL, then its contents are prepended to +** the returned expression text and then freed using sqlite3_free(). +*/ +static char *exprToString(Fts3Expr *pExpr, char *zBuf){ + if( pExpr==0 ){ + return sqlite3_mprintf(""); + } + switch( pExpr->eType ){ + case FTSQUERY_PHRASE: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + int i; + zBuf = sqlite3_mprintf( + "%zPHRASE %d 0", zBuf, pPhrase->iColumn); + for(i=0; zBuf && inToken; i++){ + zBuf = sqlite3_mprintf("%z %.*s%s", zBuf, + pPhrase->aToken[i].n, pPhrase->aToken[i].z, + (pPhrase->aToken[i].isPrefix?"+":"") + ); + } + return zBuf; + } + + case FTSQUERY_NEAR: + zBuf = sqlite3_mprintf("%zNEAR/%d ", zBuf, pExpr->nNear); + break; + case FTSQUERY_NOT: + zBuf = sqlite3_mprintf("%zNOT ", zBuf); + break; + case FTSQUERY_AND: + zBuf = sqlite3_mprintf("%zAND ", zBuf); + break; + case FTSQUERY_OR: + zBuf = sqlite3_mprintf("%zOR ", zBuf); + break; + } + + if( zBuf ) zBuf = sqlite3_mprintf("%z{", zBuf); + if( zBuf ) zBuf = exprToString(pExpr->pLeft, zBuf); + if( zBuf ) zBuf = sqlite3_mprintf("%z} {", zBuf); + + if( zBuf ) zBuf = exprToString(pExpr->pRight, zBuf); + if( zBuf ) zBuf = sqlite3_mprintf("%z}", zBuf); + + return zBuf; +} + +/* +** This is the implementation of a scalar SQL function used to test the +** expression parser. It should be called as follows: +** +** fts3_exprtest(, , , ...); +** +** The first argument, , is the name of the fts3 tokenizer used +** to parse the query expression (see README.tokenizers). The second argument +** is the query expression to parse. Each subsequent argument is the name +** of a column of the fts3 table that the query expression may refer to. +** For example: +** +** SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2'); +*/ +static void fts3ExprTest( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3_tokenizer_module const *pModule = 0; + sqlite3_tokenizer *pTokenizer = 0; + int rc; + char **azCol = 0; + const char *zExpr; + int nExpr; + int nCol; + int ii; + Fts3Expr *pExpr; + char *zBuf = 0; + sqlite3 *db = sqlite3_context_db_handle(context); + + if( argc<3 ){ + sqlite3_result_error(context, + "Usage: fts3_exprtest(tokenizer, expr, col1, ...", -1 + ); + return; + } + + rc = queryTestTokenizer(db, + (const char *)sqlite3_value_text(argv[0]), &pModule); + if( rc==SQLITE_NOMEM ){ + sqlite3_result_error_nomem(context); + goto exprtest_out; + }else if( !pModule ){ + sqlite3_result_error(context, "No such tokenizer module", -1); + goto exprtest_out; + } + + rc = pModule->xCreate(0, 0, &pTokenizer); + assert( rc==SQLITE_NOMEM || rc==SQLITE_OK ); + if( rc==SQLITE_NOMEM ){ + sqlite3_result_error_nomem(context); + goto exprtest_out; + } + pTokenizer->pModule = pModule; + + zExpr = (const char *)sqlite3_value_text(argv[1]); + nExpr = sqlite3_value_bytes(argv[1]); + nCol = argc-2; + azCol = (char **)sqlite3_malloc(nCol*sizeof(char *)); + if( !azCol ){ + sqlite3_result_error_nomem(context); + goto exprtest_out; + } + for(ii=0; iixDestroy(pTokenizer); + } + sqlite3_free(azCol); +} + +/* +** Register the query expression parser test function fts3_exprtest() +** with database connection db. +*/ +SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3* db){ + int rc = sqlite3_create_function( + db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0 + ); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "fts3_exprtest_rebalance", + -1, SQLITE_UTF8, (void *)1, fts3ExprTest, 0, 0 + ); + } + return rc; +} + +#endif +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_expr.c *******************************************/ +/************** Begin file fts3_hash.c ***************************************/ +/* +** 2001 September 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This is the implementation of generic hash-tables used in SQLite. +** We've modified it slightly to serve as a standalone hash table +** implementation for the full-text indexing module. +*/ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ +/* #include */ + +/* #include "fts3_hash.h" */ + +/* +** Malloc and Free functions +*/ +static void *fts3HashMalloc(int n){ + void *p = sqlite3_malloc(n); + if( p ){ + memset(p, 0, n); + } + return p; +} +static void fts3HashFree(void *p){ + sqlite3_free(p); +} + +/* Turn bulk memory into a hash table object by initializing the +** fields of the Hash structure. +** +** "pNew" is a pointer to the hash table that is to be initialized. +** keyClass is one of the constants +** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass +** determines what kind of key the hash table will use. "copyKey" is +** true if the hash table should make its own private copy of keys and +** false if it should just use the supplied pointer. +*/ +SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey){ + assert( pNew!=0 ); + assert( keyClass>=FTS3_HASH_STRING && keyClass<=FTS3_HASH_BINARY ); + pNew->keyClass = keyClass; + pNew->copyKey = copyKey; + pNew->first = 0; + pNew->count = 0; + pNew->htsize = 0; + pNew->ht = 0; +} + +/* Remove all entries from a hash table. Reclaim all memory. +** Call this routine to delete a hash table or to reset a hash table +** to the empty state. +*/ +SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash *pH){ + Fts3HashElem *elem; /* For looping over all elements of the table */ + + assert( pH!=0 ); + elem = pH->first; + pH->first = 0; + fts3HashFree(pH->ht); + pH->ht = 0; + pH->htsize = 0; + while( elem ){ + Fts3HashElem *next_elem = elem->next; + if( pH->copyKey && elem->pKey ){ + fts3HashFree(elem->pKey); + } + fts3HashFree(elem); + elem = next_elem; + } + pH->count = 0; +} + +/* +** Hash and comparison functions when the mode is FTS3_HASH_STRING +*/ +static int fts3StrHash(const void *pKey, int nKey){ + const char *z = (const char *)pKey; + unsigned h = 0; + if( nKey<=0 ) nKey = (int) strlen(z); + while( nKey > 0 ){ + h = (h<<3) ^ h ^ *z++; + nKey--; + } + return (int)(h & 0x7fffffff); +} +static int fts3StrCompare(const void *pKey1, int n1, const void *pKey2, int n2){ + if( n1!=n2 ) return 1; + return strncmp((const char*)pKey1,(const char*)pKey2,n1); +} + +/* +** Hash and comparison functions when the mode is FTS3_HASH_BINARY +*/ +static int fts3BinHash(const void *pKey, int nKey){ + int h = 0; + const char *z = (const char *)pKey; + while( nKey-- > 0 ){ + h = (h<<3) ^ h ^ *(z++); + } + return h & 0x7fffffff; +} +static int fts3BinCompare(const void *pKey1, int n1, const void *pKey2, int n2){ + if( n1!=n2 ) return 1; + return memcmp(pKey1,pKey2,n1); +} + +/* +** Return a pointer to the appropriate hash function given the key class. +** +** The C syntax in this function definition may be unfamilar to some +** programmers, so we provide the following additional explanation: +** +** The name of the function is "ftsHashFunction". The function takes a +** single parameter "keyClass". The return value of ftsHashFunction() +** is a pointer to another function. Specifically, the return value +** of ftsHashFunction() is a pointer to a function that takes two parameters +** with types "const void*" and "int" and returns an "int". +*/ +static int (*ftsHashFunction(int keyClass))(const void*,int){ + if( keyClass==FTS3_HASH_STRING ){ + return &fts3StrHash; + }else{ + assert( keyClass==FTS3_HASH_BINARY ); + return &fts3BinHash; + } +} + +/* +** Return a pointer to the appropriate hash function given the key class. +** +** For help in interpreted the obscure C code in the function definition, +** see the header comment on the previous function. +*/ +static int (*ftsCompareFunction(int keyClass))(const void*,int,const void*,int){ + if( keyClass==FTS3_HASH_STRING ){ + return &fts3StrCompare; + }else{ + assert( keyClass==FTS3_HASH_BINARY ); + return &fts3BinCompare; + } +} + +/* Link an element into the hash table +*/ +static void fts3HashInsertElement( + Fts3Hash *pH, /* The complete hash table */ + struct _fts3ht *pEntry, /* The entry into which pNew is inserted */ + Fts3HashElem *pNew /* The element to be inserted */ +){ + Fts3HashElem *pHead; /* First element already in pEntry */ + pHead = pEntry->chain; + if( pHead ){ + pNew->next = pHead; + pNew->prev = pHead->prev; + if( pHead->prev ){ pHead->prev->next = pNew; } + else { pH->first = pNew; } + pHead->prev = pNew; + }else{ + pNew->next = pH->first; + if( pH->first ){ pH->first->prev = pNew; } + pNew->prev = 0; + pH->first = pNew; + } + pEntry->count++; + pEntry->chain = pNew; +} + + +/* Resize the hash table so that it cantains "new_size" buckets. +** "new_size" must be a power of 2. The hash table might fail +** to resize if sqliteMalloc() fails. +** +** Return non-zero if a memory allocation error occurs. +*/ +static int fts3Rehash(Fts3Hash *pH, int new_size){ + struct _fts3ht *new_ht; /* The new hash table */ + Fts3HashElem *elem, *next_elem; /* For looping over existing elements */ + int (*xHash)(const void*,int); /* The hash function */ + + assert( (new_size & (new_size-1))==0 ); + new_ht = (struct _fts3ht *)fts3HashMalloc( new_size*sizeof(struct _fts3ht) ); + if( new_ht==0 ) return 1; + fts3HashFree(pH->ht); + pH->ht = new_ht; + pH->htsize = new_size; + xHash = ftsHashFunction(pH->keyClass); + for(elem=pH->first, pH->first=0; elem; elem = next_elem){ + int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1); + next_elem = elem->next; + fts3HashInsertElement(pH, &new_ht[h], elem); + } + return 0; +} + +/* This function (for internal use only) locates an element in an +** hash table that matches the given key. The hash for this key has +** already been computed and is passed as the 4th parameter. +*/ +static Fts3HashElem *fts3FindElementByHash( + const Fts3Hash *pH, /* The pH to be searched */ + const void *pKey, /* The key we are searching for */ + int nKey, + int h /* The hash for this key. */ +){ + Fts3HashElem *elem; /* Used to loop thru the element list */ + int count; /* Number of elements left to test */ + int (*xCompare)(const void*,int,const void*,int); /* comparison function */ + + if( pH->ht ){ + struct _fts3ht *pEntry = &pH->ht[h]; + elem = pEntry->chain; + count = pEntry->count; + xCompare = ftsCompareFunction(pH->keyClass); + while( count-- && elem ){ + if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){ + return elem; + } + elem = elem->next; + } + } + return 0; +} + +/* Remove a single entry from the hash table given a pointer to that +** element and a hash on the element's key. +*/ +static void fts3RemoveElementByHash( + Fts3Hash *pH, /* The pH containing "elem" */ + Fts3HashElem* elem, /* The element to be removed from the pH */ + int h /* Hash value for the element */ +){ + struct _fts3ht *pEntry; + if( elem->prev ){ + elem->prev->next = elem->next; + }else{ + pH->first = elem->next; + } + if( elem->next ){ + elem->next->prev = elem->prev; + } + pEntry = &pH->ht[h]; + if( pEntry->chain==elem ){ + pEntry->chain = elem->next; + } + pEntry->count--; + if( pEntry->count<=0 ){ + pEntry->chain = 0; + } + if( pH->copyKey && elem->pKey ){ + fts3HashFree(elem->pKey); + } + fts3HashFree( elem ); + pH->count--; + if( pH->count<=0 ){ + assert( pH->first==0 ); + assert( pH->count==0 ); + fts3HashClear(pH); + } +} + +SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem( + const Fts3Hash *pH, + const void *pKey, + int nKey +){ + int h; /* A hash on key */ + int (*xHash)(const void*,int); /* The hash function */ + + if( pH==0 || pH->ht==0 ) return 0; + xHash = ftsHashFunction(pH->keyClass); + assert( xHash!=0 ); + h = (*xHash)(pKey,nKey); + assert( (pH->htsize & (pH->htsize-1))==0 ); + return fts3FindElementByHash(pH,pKey,nKey, h & (pH->htsize-1)); +} + +/* +** Attempt to locate an element of the hash table pH with a key +** that matches pKey,nKey. Return the data for this element if it is +** found, or NULL if there is no match. +*/ +SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash *pH, const void *pKey, int nKey){ + Fts3HashElem *pElem; /* The element that matches key (if any) */ + + pElem = sqlite3Fts3HashFindElem(pH, pKey, nKey); + return pElem ? pElem->data : 0; +} + +/* Insert an element into the hash table pH. The key is pKey,nKey +** and the data is "data". +** +** If no element exists with a matching key, then a new +** element is created. A copy of the key is made if the copyKey +** flag is set. NULL is returned. +** +** If another element already exists with the same key, then the +** new data replaces the old data and the old data is returned. +** The key is not copied in this instance. If a malloc fails, then +** the new data is returned and the hash table is unchanged. +** +** If the "data" parameter to this function is NULL, then the +** element corresponding to "key" is removed from the hash table. +*/ +SQLITE_PRIVATE void *sqlite3Fts3HashInsert( + Fts3Hash *pH, /* The hash table to insert into */ + const void *pKey, /* The key */ + int nKey, /* Number of bytes in the key */ + void *data /* The data */ +){ + int hraw; /* Raw hash value of the key */ + int h; /* the hash of the key modulo hash table size */ + Fts3HashElem *elem; /* Used to loop thru the element list */ + Fts3HashElem *new_elem; /* New element added to the pH */ + int (*xHash)(const void*,int); /* The hash function */ + + assert( pH!=0 ); + xHash = ftsHashFunction(pH->keyClass); + assert( xHash!=0 ); + hraw = (*xHash)(pKey, nKey); + assert( (pH->htsize & (pH->htsize-1))==0 ); + h = hraw & (pH->htsize-1); + elem = fts3FindElementByHash(pH,pKey,nKey,h); + if( elem ){ + void *old_data = elem->data; + if( data==0 ){ + fts3RemoveElementByHash(pH,elem,h); + }else{ + elem->data = data; + } + return old_data; + } + if( data==0 ) return 0; + if( (pH->htsize==0 && fts3Rehash(pH,8)) + || (pH->count>=pH->htsize && fts3Rehash(pH, pH->htsize*2)) + ){ + pH->count = 0; + return data; + } + assert( pH->htsize>0 ); + new_elem = (Fts3HashElem*)fts3HashMalloc( sizeof(Fts3HashElem) ); + if( new_elem==0 ) return data; + if( pH->copyKey && pKey!=0 ){ + new_elem->pKey = fts3HashMalloc( nKey ); + if( new_elem->pKey==0 ){ + fts3HashFree(new_elem); + return data; + } + memcpy((void*)new_elem->pKey, pKey, nKey); + }else{ + new_elem->pKey = (void*)pKey; + } + new_elem->nKey = nKey; + pH->count++; + assert( pH->htsize>0 ); + assert( (pH->htsize & (pH->htsize-1))==0 ); + h = hraw & (pH->htsize-1); + fts3HashInsertElement(pH, &pH->ht[h], new_elem); + new_elem->data = data; + return 0; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_hash.c *******************************************/ +/************** Begin file fts3_porter.c *************************************/ +/* +** 2006 September 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Implementation of the full-text-search tokenizer that implements +** a Porter stemmer. +*/ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ +/* #include */ +/* #include */ + +/* #include "fts3_tokenizer.h" */ + +/* +** Class derived from sqlite3_tokenizer +*/ +typedef struct porter_tokenizer { + sqlite3_tokenizer base; /* Base class */ +} porter_tokenizer; + +/* +** Class derived from sqlite3_tokenizer_cursor +*/ +typedef struct porter_tokenizer_cursor { + sqlite3_tokenizer_cursor base; + const char *zInput; /* input we are tokenizing */ + int nInput; /* size of the input */ + int iOffset; /* current position in zInput */ + int iToken; /* index of next token to be returned */ + char *zToken; /* storage for current token */ + int nAllocated; /* space allocated to zToken buffer */ +} porter_tokenizer_cursor; + + +/* +** Create a new tokenizer instance. +*/ +static int porterCreate( + int argc, const char * const *argv, + sqlite3_tokenizer **ppTokenizer +){ + porter_tokenizer *t; + + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + + t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t)); + if( t==NULL ) return SQLITE_NOMEM; + memset(t, 0, sizeof(*t)); + *ppTokenizer = &t->base; + return SQLITE_OK; +} + +/* +** Destroy a tokenizer +*/ +static int porterDestroy(sqlite3_tokenizer *pTokenizer){ + sqlite3_free(pTokenizer); + return SQLITE_OK; +} + +/* +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is zInput[0..nInput-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. +*/ +static int porterOpen( + sqlite3_tokenizer *pTokenizer, /* The tokenizer */ + const char *zInput, int nInput, /* String to be tokenized */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ +){ + porter_tokenizer_cursor *c; + + UNUSED_PARAMETER(pTokenizer); + + c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); + if( c==NULL ) return SQLITE_NOMEM; + + c->zInput = zInput; + if( zInput==0 ){ + c->nInput = 0; + }else if( nInput<0 ){ + c->nInput = (int)strlen(zInput); + }else{ + c->nInput = nInput; + } + c->iOffset = 0; /* start tokenizing at the beginning */ + c->iToken = 0; + c->zToken = NULL; /* no space allocated, yet. */ + c->nAllocated = 0; + + *ppCursor = &c->base; + return SQLITE_OK; +} + +/* +** Close a tokenization cursor previously opened by a call to +** porterOpen() above. +*/ +static int porterClose(sqlite3_tokenizer_cursor *pCursor){ + porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor; + sqlite3_free(c->zToken); + sqlite3_free(c); + return SQLITE_OK; +} +/* +** Vowel or consonant +*/ +static const char cType[] = { + 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, + 1, 1, 1, 2, 1 +}; + +/* +** isConsonant() and isVowel() determine if their first character in +** the string they point to is a consonant or a vowel, according +** to Porter ruls. +** +** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'. +** 'Y' is a consonant unless it follows another consonant, +** in which case it is a vowel. +** +** In these routine, the letters are in reverse order. So the 'y' rule +** is that 'y' is a consonant unless it is followed by another +** consonent. +*/ +static int isVowel(const char*); +static int isConsonant(const char *z){ + int j; + char x = *z; + if( x==0 ) return 0; + assert( x>='a' && x<='z' ); + j = cType[x-'a']; + if( j<2 ) return j; + return z[1]==0 || isVowel(z + 1); +} +static int isVowel(const char *z){ + int j; + char x = *z; + if( x==0 ) return 0; + assert( x>='a' && x<='z' ); + j = cType[x-'a']; + if( j<2 ) return 1-j; + return isConsonant(z + 1); +} + +/* +** Let any sequence of one or more vowels be represented by V and let +** C be sequence of one or more consonants. Then every word can be +** represented as: +** +** [C] (VC){m} [V] +** +** In prose: A word is an optional consonant followed by zero or +** vowel-consonant pairs followed by an optional vowel. "m" is the +** number of vowel consonant pairs. This routine computes the value +** of m for the first i bytes of a word. +** +** Return true if the m-value for z is 1 or more. In other words, +** return true if z contains at least one vowel that is followed +** by a consonant. +** +** In this routine z[] is in reverse order. So we are really looking +** for an instance of a consonant followed by a vowel. +*/ +static int m_gt_0(const char *z){ + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + return *z!=0; +} + +/* Like mgt0 above except we are looking for a value of m which is +** exactly 1 +*/ +static int m_eq_1(const char *z){ + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + if( *z==0 ) return 0; + while( isVowel(z) ){ z++; } + if( *z==0 ) return 1; + while( isConsonant(z) ){ z++; } + return *z==0; +} + +/* Like mgt0 above except we are looking for a value of m>1 instead +** or m>0 +*/ +static int m_gt_1(const char *z){ + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + if( *z==0 ) return 0; + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + return *z!=0; +} + +/* +** Return TRUE if there is a vowel anywhere within z[0..n-1] +*/ +static int hasVowel(const char *z){ + while( isConsonant(z) ){ z++; } + return *z!=0; +} + +/* +** Return TRUE if the word ends in a double consonant. +** +** The text is reversed here. So we are really looking at +** the first two characters of z[]. +*/ +static int doubleConsonant(const char *z){ + return isConsonant(z) && z[0]==z[1]; +} + +/* +** Return TRUE if the word ends with three letters which +** are consonant-vowel-consonent and where the final consonant +** is not 'w', 'x', or 'y'. +** +** The word is reversed here. So we are really checking the +** first three letters and the first one cannot be in [wxy]. +*/ +static int star_oh(const char *z){ + return + isConsonant(z) && + z[0]!='w' && z[0]!='x' && z[0]!='y' && + isVowel(z+1) && + isConsonant(z+2); +} + +/* +** If the word ends with zFrom and xCond() is true for the stem +** of the word that preceeds the zFrom ending, then change the +** ending to zTo. +** +** The input word *pz and zFrom are both in reverse order. zTo +** is in normal order. +** +** Return TRUE if zFrom matches. Return FALSE if zFrom does not +** match. Not that TRUE is returned even if xCond() fails and +** no substitution occurs. +*/ +static int stem( + char **pz, /* The word being stemmed (Reversed) */ + const char *zFrom, /* If the ending matches this... (Reversed) */ + const char *zTo, /* ... change the ending to this (not reversed) */ + int (*xCond)(const char*) /* Condition that must be true */ +){ + char *z = *pz; + while( *zFrom && *zFrom==*z ){ z++; zFrom++; } + if( *zFrom!=0 ) return 0; + if( xCond && !xCond(z) ) return 1; + while( *zTo ){ + *(--z) = *(zTo++); + } + *pz = z; + return 1; +} + +/* +** This is the fallback stemmer used when the porter stemmer is +** inappropriate. The input word is copied into the output with +** US-ASCII case folding. If the input word is too long (more +** than 20 bytes if it contains no digits or more than 6 bytes if +** it contains digits) then word is truncated to 20 or 6 bytes +** by taking 10 or 3 bytes from the beginning and end. +*/ +static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ + int i, mx, j; + int hasDigit = 0; + for(i=0; i='A' && c<='Z' ){ + zOut[i] = c - 'A' + 'a'; + }else{ + if( c>='0' && c<='9' ) hasDigit = 1; + zOut[i] = c; + } + } + mx = hasDigit ? 3 : 10; + if( nIn>mx*2 ){ + for(j=mx, i=nIn-mx; i=(int)sizeof(zReverse)-7 ){ + /* The word is too big or too small for the porter stemmer. + ** Fallback to the copy stemmer */ + copy_stemmer(zIn, nIn, zOut, pnOut); + return; + } + for(i=0, j=sizeof(zReverse)-6; i='A' && c<='Z' ){ + zReverse[j] = c + 'a' - 'A'; + }else if( c>='a' && c<='z' ){ + zReverse[j] = c; + }else{ + /* The use of a character not in [a-zA-Z] means that we fallback + ** to the copy stemmer */ + copy_stemmer(zIn, nIn, zOut, pnOut); + return; + } + } + memset(&zReverse[sizeof(zReverse)-5], 0, 5); + z = &zReverse[j+1]; + + + /* Step 1a */ + if( z[0]=='s' ){ + if( + !stem(&z, "sess", "ss", 0) && + !stem(&z, "sei", "i", 0) && + !stem(&z, "ss", "ss", 0) + ){ + z++; + } + } + + /* Step 1b */ + z2 = z; + if( stem(&z, "dee", "ee", m_gt_0) ){ + /* Do nothing. The work was all in the test */ + }else if( + (stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel)) + && z!=z2 + ){ + if( stem(&z, "ta", "ate", 0) || + stem(&z, "lb", "ble", 0) || + stem(&z, "zi", "ize", 0) ){ + /* Do nothing. The work was all in the test */ + }else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){ + z++; + }else if( m_eq_1(z) && star_oh(z) ){ + *(--z) = 'e'; + } + } + + /* Step 1c */ + if( z[0]=='y' && hasVowel(z+1) ){ + z[0] = 'i'; + } + + /* Step 2 */ + switch( z[1] ){ + case 'a': + if( !stem(&z, "lanoita", "ate", m_gt_0) ){ + stem(&z, "lanoit", "tion", m_gt_0); + } + break; + case 'c': + if( !stem(&z, "icne", "ence", m_gt_0) ){ + stem(&z, "icna", "ance", m_gt_0); + } + break; + case 'e': + stem(&z, "rezi", "ize", m_gt_0); + break; + case 'g': + stem(&z, "igol", "log", m_gt_0); + break; + case 'l': + if( !stem(&z, "ilb", "ble", m_gt_0) + && !stem(&z, "illa", "al", m_gt_0) + && !stem(&z, "iltne", "ent", m_gt_0) + && !stem(&z, "ile", "e", m_gt_0) + ){ + stem(&z, "ilsuo", "ous", m_gt_0); + } + break; + case 'o': + if( !stem(&z, "noitazi", "ize", m_gt_0) + && !stem(&z, "noita", "ate", m_gt_0) + ){ + stem(&z, "rota", "ate", m_gt_0); + } + break; + case 's': + if( !stem(&z, "msila", "al", m_gt_0) + && !stem(&z, "ssenevi", "ive", m_gt_0) + && !stem(&z, "ssenluf", "ful", m_gt_0) + ){ + stem(&z, "ssensuo", "ous", m_gt_0); + } + break; + case 't': + if( !stem(&z, "itila", "al", m_gt_0) + && !stem(&z, "itivi", "ive", m_gt_0) + ){ + stem(&z, "itilib", "ble", m_gt_0); + } + break; + } + + /* Step 3 */ + switch( z[0] ){ + case 'e': + if( !stem(&z, "etaci", "ic", m_gt_0) + && !stem(&z, "evita", "", m_gt_0) + ){ + stem(&z, "ezila", "al", m_gt_0); + } + break; + case 'i': + stem(&z, "itici", "ic", m_gt_0); + break; + case 'l': + if( !stem(&z, "laci", "ic", m_gt_0) ){ + stem(&z, "luf", "", m_gt_0); + } + break; + case 's': + stem(&z, "ssen", "", m_gt_0); + break; + } + + /* Step 4 */ + switch( z[1] ){ + case 'a': + if( z[0]=='l' && m_gt_1(z+2) ){ + z += 2; + } + break; + case 'c': + if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e') && m_gt_1(z+4) ){ + z += 4; + } + break; + case 'e': + if( z[0]=='r' && m_gt_1(z+2) ){ + z += 2; + } + break; + case 'i': + if( z[0]=='c' && m_gt_1(z+2) ){ + z += 2; + } + break; + case 'l': + if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){ + z += 4; + } + break; + case 'n': + if( z[0]=='t' ){ + if( z[2]=='a' ){ + if( m_gt_1(z+3) ){ + z += 3; + } + }else if( z[2]=='e' ){ + if( !stem(&z, "tneme", "", m_gt_1) + && !stem(&z, "tnem", "", m_gt_1) + ){ + stem(&z, "tne", "", m_gt_1); + } + } + } + break; + case 'o': + if( z[0]=='u' ){ + if( m_gt_1(z+2) ){ + z += 2; + } + }else if( z[3]=='s' || z[3]=='t' ){ + stem(&z, "noi", "", m_gt_1); + } + break; + case 's': + if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){ + z += 3; + } + break; + case 't': + if( !stem(&z, "eta", "", m_gt_1) ){ + stem(&z, "iti", "", m_gt_1); + } + break; + case 'u': + if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){ + z += 3; + } + break; + case 'v': + case 'z': + if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){ + z += 3; + } + break; + } + + /* Step 5a */ + if( z[0]=='e' ){ + if( m_gt_1(z+1) ){ + z++; + }else if( m_eq_1(z+1) && !star_oh(z+1) ){ + z++; + } + } + + /* Step 5b */ + if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){ + z++; + } + + /* z[] is now the stemmed word in reverse order. Flip it back + ** around into forward order and return. + */ + *pnOut = i = (int)strlen(z); + zOut[i] = 0; + while( *z ){ + zOut[--i] = *(z++); + } +} + +/* +** Characters that can be part of a token. We assume any character +** whose value is greater than 0x80 (any UTF character) can be +** part of a token. In other words, delimiters all must have +** values of 0x7f or lower. +*/ +static const char porterIdChar[] = { +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ +}; +#define isDelim(C) (((ch=C)&0x80)==0 && (ch<0x30 || !porterIdChar[ch-0x30])) + +/* +** Extract the next token from a tokenization cursor. The cursor must +** have been opened by a prior call to porterOpen(). +*/ +static int porterNext( + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by porterOpen */ + const char **pzToken, /* OUT: *pzToken is the token text */ + int *pnBytes, /* OUT: Number of bytes in token */ + int *piStartOffset, /* OUT: Starting offset of token */ + int *piEndOffset, /* OUT: Ending offset of token */ + int *piPosition /* OUT: Position integer of token */ +){ + porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor; + const char *z = c->zInput; + + while( c->iOffsetnInput ){ + int iStartOffset, ch; + + /* Scan past delimiter characters */ + while( c->iOffsetnInput && isDelim(z[c->iOffset]) ){ + c->iOffset++; + } + + /* Count non-delimiter characters. */ + iStartOffset = c->iOffset; + while( c->iOffsetnInput && !isDelim(z[c->iOffset]) ){ + c->iOffset++; + } + + if( c->iOffset>iStartOffset ){ + int n = c->iOffset-iStartOffset; + if( n>c->nAllocated ){ + char *pNew; + c->nAllocated = n+20; + pNew = sqlite3_realloc(c->zToken, c->nAllocated); + if( !pNew ) return SQLITE_NOMEM; + c->zToken = pNew; + } + porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes); + *pzToken = c->zToken; + *piStartOffset = iStartOffset; + *piEndOffset = c->iOffset; + *piPosition = c->iToken++; + return SQLITE_OK; + } + } + return SQLITE_DONE; +} + +/* +** The set of routines that implement the porter-stemmer tokenizer +*/ +static const sqlite3_tokenizer_module porterTokenizerModule = { + 0, + porterCreate, + porterDestroy, + porterOpen, + porterClose, + porterNext, + 0 +}; + +/* +** Allocate a new porter tokenizer. Return a pointer to the new +** tokenizer in *ppModule +*/ +SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule( + sqlite3_tokenizer_module const**ppModule +){ + *ppModule = &porterTokenizerModule; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_porter.c *****************************************/ +/************** Begin file fts3_tokenizer.c **********************************/ +/* +** 2007 June 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is part of an SQLite module implementing full-text search. +** This particular file implements the generic tokenizer interface. +*/ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ + +/* +** Implementation of the SQL scalar function for accessing the underlying +** hash table. This function may be called as follows: +** +** SELECT (); +** SELECT (, ); +** +** where is the name passed as the second argument +** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer'). +** +** If the argument is specified, it must be a blob value +** containing a pointer to be stored as the hash data corresponding +** to the string . If is not specified, then +** the string must already exist in the has table. Otherwise, +** an error is returned. +** +** Whether or not the argument is specified, the value returned +** is a blob containing the pointer stored as the hash data corresponding +** to string (after the hash-table is updated, if applicable). +*/ +static void scalarFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Fts3Hash *pHash; + void *pPtr = 0; + const unsigned char *zName; + int nName; + + assert( argc==1 || argc==2 ); + + pHash = (Fts3Hash *)sqlite3_user_data(context); + + zName = sqlite3_value_text(argv[0]); + nName = sqlite3_value_bytes(argv[0])+1; + + if( argc==2 ){ + void *pOld; + int n = sqlite3_value_bytes(argv[1]); + if( zName==0 || n!=sizeof(pPtr) ){ + sqlite3_result_error(context, "argument type mismatch", -1); + return; + } + pPtr = *(void **)sqlite3_value_blob(argv[1]); + pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr); + if( pOld==pPtr ){ + sqlite3_result_error(context, "out of memory", -1); + return; + } + }else{ + if( zName ){ + pPtr = sqlite3Fts3HashFind(pHash, zName, nName); + } + if( !pPtr ){ + char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); + return; + } + } + + sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); +} + +SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char c){ + static const char isFtsIdChar[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ + 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ + }; + return (c&0x80 || isFtsIdChar[(int)(c)]); +} + +SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *zStr, int *pn){ + const char *z1; + const char *z2 = 0; + + /* Find the start of the next token. */ + z1 = zStr; + while( z2==0 ){ + char c = *z1; + switch( c ){ + case '\0': return 0; /* No more tokens here */ + case '\'': + case '"': + case '`': { + z2 = z1; + while( *++z2 && (*z2!=c || *++z2==c) ); + break; + } + case '[': + z2 = &z1[1]; + while( *z2 && z2[0]!=']' ) z2++; + if( *z2 ) z2++; + break; + + default: + if( sqlite3Fts3IsIdChar(*z1) ){ + z2 = &z1[1]; + while( sqlite3Fts3IsIdChar(*z2) ) z2++; + }else{ + z1++; + } + } + } + + *pn = (int)(z2-z1); + return z1; +} + +SQLITE_PRIVATE int sqlite3Fts3InitTokenizer( + Fts3Hash *pHash, /* Tokenizer hash table */ + const char *zArg, /* Tokenizer name */ + sqlite3_tokenizer **ppTok, /* OUT: Tokenizer (if applicable) */ + char **pzErr /* OUT: Set to malloced error message */ +){ + int rc; + char *z = (char *)zArg; + int n = 0; + char *zCopy; + char *zEnd; /* Pointer to nul-term of zCopy */ + sqlite3_tokenizer_module *m; + + zCopy = sqlite3_mprintf("%s", zArg); + if( !zCopy ) return SQLITE_NOMEM; + zEnd = &zCopy[strlen(zCopy)]; + + z = (char *)sqlite3Fts3NextToken(zCopy, &n); + if( z==0 ){ + assert( n==0 ); + z = zCopy; + } + z[n] = '\0'; + sqlite3Fts3Dequote(z); + + m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1); + if( !m ){ + sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", z); + rc = SQLITE_ERROR; + }else{ + char const **aArg = 0; + int iArg = 0; + z = &z[n+1]; + while( zxCreate(iArg, aArg, ppTok); + assert( rc!=SQLITE_OK || *ppTok ); + if( rc!=SQLITE_OK ){ + sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer"); + }else{ + (*ppTok)->pModule = m; + } + sqlite3_free((void *)aArg); + } + + sqlite3_free(zCopy); + return rc; +} + + +#ifdef SQLITE_TEST + +#include +/* #include */ + +/* +** Implementation of a special SQL scalar function for testing tokenizers +** designed to be used in concert with the Tcl testing framework. This +** function must be called with two or more arguments: +** +** SELECT (, ..., ); +** +** where is the name passed as the second argument +** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer') +** concatenated with the string '_test' (e.g. 'fts3_tokenizer_test'). +** +** The return value is a string that may be interpreted as a Tcl +** list. For each token in the , three elements are +** added to the returned list. The first is the token position, the +** second is the token text (folded, stemmed, etc.) and the third is the +** substring of associated with the token. For example, +** using the built-in "simple" tokenizer: +** +** SELECT fts_tokenizer_test('simple', 'I don't see how'); +** +** will return the string: +** +** "{0 i I 1 dont don't 2 see see 3 how how}" +** +*/ +static void testFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Fts3Hash *pHash; + sqlite3_tokenizer_module *p; + sqlite3_tokenizer *pTokenizer = 0; + sqlite3_tokenizer_cursor *pCsr = 0; + + const char *zErr = 0; + + const char *zName; + int nName; + const char *zInput; + int nInput; + + const char *azArg[64]; + + const char *zToken; + int nToken = 0; + int iStart = 0; + int iEnd = 0; + int iPos = 0; + int i; + + Tcl_Obj *pRet; + + if( argc<2 ){ + sqlite3_result_error(context, "insufficient arguments", -1); + return; + } + + nName = sqlite3_value_bytes(argv[0]); + zName = (const char *)sqlite3_value_text(argv[0]); + nInput = sqlite3_value_bytes(argv[argc-1]); + zInput = (const char *)sqlite3_value_text(argv[argc-1]); + + pHash = (Fts3Hash *)sqlite3_user_data(context); + p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); + + if( !p ){ + char *zErr2 = sqlite3_mprintf("unknown tokenizer: %s", zName); + sqlite3_result_error(context, zErr2, -1); + sqlite3_free(zErr2); + return; + } + + pRet = Tcl_NewObj(); + Tcl_IncrRefCount(pRet); + + for(i=1; ixCreate(argc-2, azArg, &pTokenizer) ){ + zErr = "error in xCreate()"; + goto finish; + } + pTokenizer->pModule = p; + if( sqlite3Fts3OpenTokenizer(pTokenizer, 0, zInput, nInput, &pCsr) ){ + zErr = "error in xOpen()"; + goto finish; + } + + while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){ + Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); + zToken = &zInput[iStart]; + nToken = iEnd-iStart; + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); + } + + if( SQLITE_OK!=p->xClose(pCsr) ){ + zErr = "error in xClose()"; + goto finish; + } + if( SQLITE_OK!=p->xDestroy(pTokenizer) ){ + zErr = "error in xDestroy()"; + goto finish; + } + +finish: + if( zErr ){ + sqlite3_result_error(context, zErr, -1); + }else{ + sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT); + } + Tcl_DecrRefCount(pRet); +} + +static +int registerTokenizer( + sqlite3 *db, + char *zName, + const sqlite3_tokenizer_module *p +){ + int rc; + sqlite3_stmt *pStmt; + const char zSql[] = "SELECT fts3_tokenizer(?, ?)"; + + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); + sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC); + sqlite3_step(pStmt); + + return sqlite3_finalize(pStmt); +} + +static +int queryTokenizer( + sqlite3 *db, + char *zName, + const sqlite3_tokenizer_module **pp +){ + int rc; + sqlite3_stmt *pStmt; + const char zSql[] = "SELECT fts3_tokenizer(?)"; + + *pp = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ + memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); + } + } + + return sqlite3_finalize(pStmt); +} + +SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); + +/* +** Implementation of the scalar function fts3_tokenizer_internal_test(). +** This function is used for testing only, it is not included in the +** build unless SQLITE_TEST is defined. +** +** The purpose of this is to test that the fts3_tokenizer() function +** can be used as designed by the C-code in the queryTokenizer and +** registerTokenizer() functions above. These two functions are repeated +** in the README.tokenizer file as an example, so it is important to +** test them. +** +** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar +** function with no arguments. An assert() will fail if a problem is +** detected. i.e.: +** +** SELECT fts3_tokenizer_internal_test(); +** +*/ +static void intTestFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int rc; + const sqlite3_tokenizer_module *p1; + const sqlite3_tokenizer_module *p2; + sqlite3 *db = (sqlite3 *)sqlite3_user_data(context); + + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + + /* Test the query function */ + sqlite3Fts3SimpleTokenizerModule(&p1); + rc = queryTokenizer(db, "simple", &p2); + assert( rc==SQLITE_OK ); + assert( p1==p2 ); + rc = queryTokenizer(db, "nosuchtokenizer", &p2); + assert( rc==SQLITE_ERROR ); + assert( p2==0 ); + assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") ); + + /* Test the storage function */ + rc = registerTokenizer(db, "nosuchtokenizer", p1); + assert( rc==SQLITE_OK ); + rc = queryTokenizer(db, "nosuchtokenizer", &p2); + assert( rc==SQLITE_OK ); + assert( p2==p1 ); + + sqlite3_result_text(context, "ok", -1, SQLITE_STATIC); +} + +#endif + +/* +** Set up SQL objects in database db used to access the contents of +** the hash table pointed to by argument pHash. The hash table must +** been initialized to use string keys, and to take a private copy +** of the key when a value is inserted. i.e. by a call similar to: +** +** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); +** +** This function adds a scalar function (see header comment above +** scalarFunc() in this file for details) and, if ENABLE_TABLE is +** defined at compilation time, a temporary virtual table (see header +** comment above struct HashTableVtab) to the database schema. Both +** provide read/write access to the contents of *pHash. +** +** The third argument to this function, zName, is used as the name +** of both the scalar and, if created, the virtual table. +*/ +SQLITE_PRIVATE int sqlite3Fts3InitHashTable( + sqlite3 *db, + Fts3Hash *pHash, + const char *zName +){ + int rc = SQLITE_OK; + void *p = (void *)pHash; + const int any = SQLITE_ANY; + +#ifdef SQLITE_TEST + char *zTest = 0; + char *zTest2 = 0; + void *pdb = (void *)db; + zTest = sqlite3_mprintf("%s_test", zName); + zTest2 = sqlite3_mprintf("%s_internal_test", zName); + if( !zTest || !zTest2 ){ + rc = SQLITE_NOMEM; + } +#endif + + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0); + } + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0); + } +#ifdef SQLITE_TEST + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zTest, -1, any, p, testFunc, 0, 0); + } + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0); + } +#endif + +#ifdef SQLITE_TEST + sqlite3_free(zTest); + sqlite3_free(zTest2); +#endif + + return rc; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_tokenizer.c **************************************/ +/************** Begin file fts3_tokenizer1.c *********************************/ +/* +** 2006 Oct 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Implementation of the "simple" full-text-search tokenizer. +*/ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ +/* #include */ +/* #include */ + +/* #include "fts3_tokenizer.h" */ + +typedef struct simple_tokenizer { + sqlite3_tokenizer base; + char delim[128]; /* flag ASCII delimiters */ +} simple_tokenizer; + +typedef struct simple_tokenizer_cursor { + sqlite3_tokenizer_cursor base; + const char *pInput; /* input we are tokenizing */ + int nBytes; /* size of the input */ + int iOffset; /* current position in pInput */ + int iToken; /* index of next token to be returned */ + char *pToken; /* storage for current token */ + int nTokenAllocated; /* space allocated to zToken buffer */ +} simple_tokenizer_cursor; + + +static int simpleDelim(simple_tokenizer *t, unsigned char c){ + return c<0x80 && t->delim[c]; +} +static int fts3_isalnum(int x){ + return (x>='0' && x<='9') || (x>='A' && x<='Z') || (x>='a' && x<='z'); +} + +/* +** Create a new tokenizer instance. +*/ +static int simpleCreate( + int argc, const char * const *argv, + sqlite3_tokenizer **ppTokenizer +){ + simple_tokenizer *t; + + t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t)); + if( t==NULL ) return SQLITE_NOMEM; + memset(t, 0, sizeof(*t)); + + /* TODO(shess) Delimiters need to remain the same from run to run, + ** else we need to reindex. One solution would be a meta-table to + ** track such information in the database, then we'd only want this + ** information on the initial create. + */ + if( argc>1 ){ + int i, n = (int)strlen(argv[1]); + for(i=0; i=0x80 ){ + sqlite3_free(t); + return SQLITE_ERROR; + } + t->delim[ch] = 1; + } + } else { + /* Mark non-alphanumeric ASCII characters as delimiters */ + int i; + for(i=1; i<0x80; i++){ + t->delim[i] = !fts3_isalnum(i) ? -1 : 0; + } + } + + *ppTokenizer = &t->base; + return SQLITE_OK; +} + +/* +** Destroy a tokenizer +*/ +static int simpleDestroy(sqlite3_tokenizer *pTokenizer){ + sqlite3_free(pTokenizer); + return SQLITE_OK; +} + +/* +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is pInput[0..nBytes-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. +*/ +static int simpleOpen( + sqlite3_tokenizer *pTokenizer, /* The tokenizer */ + const char *pInput, int nBytes, /* String to be tokenized */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ +){ + simple_tokenizer_cursor *c; + + UNUSED_PARAMETER(pTokenizer); + + c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); + if( c==NULL ) return SQLITE_NOMEM; + + c->pInput = pInput; + if( pInput==0 ){ + c->nBytes = 0; + }else if( nBytes<0 ){ + c->nBytes = (int)strlen(pInput); + }else{ + c->nBytes = nBytes; + } + c->iOffset = 0; /* start tokenizing at the beginning */ + c->iToken = 0; + c->pToken = NULL; /* no space allocated, yet. */ + c->nTokenAllocated = 0; + + *ppCursor = &c->base; + return SQLITE_OK; +} + +/* +** Close a tokenization cursor previously opened by a call to +** simpleOpen() above. +*/ +static int simpleClose(sqlite3_tokenizer_cursor *pCursor){ + simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; + sqlite3_free(c->pToken); + sqlite3_free(c); + return SQLITE_OK; +} + +/* +** Extract the next token from a tokenization cursor. The cursor must +** have been opened by a prior call to simpleOpen(). +*/ +static int simpleNext( + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */ + const char **ppToken, /* OUT: *ppToken is the token text */ + int *pnBytes, /* OUT: Number of bytes in token */ + int *piStartOffset, /* OUT: Starting offset of token */ + int *piEndOffset, /* OUT: Ending offset of token */ + int *piPosition /* OUT: Position integer of token */ +){ + simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; + simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer; + unsigned char *p = (unsigned char *)c->pInput; + + while( c->iOffsetnBytes ){ + int iStartOffset; + + /* Scan past delimiter characters */ + while( c->iOffsetnBytes && simpleDelim(t, p[c->iOffset]) ){ + c->iOffset++; + } + + /* Count non-delimiter characters. */ + iStartOffset = c->iOffset; + while( c->iOffsetnBytes && !simpleDelim(t, p[c->iOffset]) ){ + c->iOffset++; + } + + if( c->iOffset>iStartOffset ){ + int i, n = c->iOffset-iStartOffset; + if( n>c->nTokenAllocated ){ + char *pNew; + c->nTokenAllocated = n+20; + pNew = sqlite3_realloc(c->pToken, c->nTokenAllocated); + if( !pNew ) return SQLITE_NOMEM; + c->pToken = pNew; + } + for(i=0; ipToken[i] = (char)((ch>='A' && ch<='Z') ? ch-'A'+'a' : ch); + } + *ppToken = c->pToken; + *pnBytes = n; + *piStartOffset = iStartOffset; + *piEndOffset = c->iOffset; + *piPosition = c->iToken++; + + return SQLITE_OK; + } + } + return SQLITE_DONE; +} + +/* +** The set of routines that implement the simple tokenizer +*/ +static const sqlite3_tokenizer_module simpleTokenizerModule = { + 0, + simpleCreate, + simpleDestroy, + simpleOpen, + simpleClose, + simpleNext, + 0, +}; + +/* +** Allocate a new simple tokenizer. Return a pointer to the new +** tokenizer in *ppModule +*/ +SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule( + sqlite3_tokenizer_module const**ppModule +){ + *ppModule = &simpleTokenizerModule; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_tokenizer1.c *************************************/ +/************** Begin file fts3_tokenize_vtab.c ******************************/ +/* +** 2013 Apr 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code for the "fts3tokenize" virtual table module. +** An fts3tokenize virtual table is created as follows: +** +** CREATE VIRTUAL TABLE USING fts3tokenize( +** , , ... +** ); +** +** The table created has the following schema: +** +** CREATE TABLE (input, token, start, end, position) +** +** When queried, the query must include a WHERE clause of type: +** +** input = +** +** The virtual table module tokenizes this , using the FTS3 +** tokenizer specified by the arguments to the CREATE VIRTUAL TABLE +** statement and returns one row for each token in the result. With +** fields set as follows: +** +** input: Always set to a copy of +** token: A token from the input. +** start: Byte offset of the token within the input . +** end: Byte offset of the byte immediately following the end of the +** token within the input string. +** pos: Token offset of token within input. +** +*/ +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ + +typedef struct Fts3tokTable Fts3tokTable; +typedef struct Fts3tokCursor Fts3tokCursor; + +/* +** Virtual table structure. +*/ +struct Fts3tokTable { + sqlite3_vtab base; /* Base class used by SQLite core */ + const sqlite3_tokenizer_module *pMod; + sqlite3_tokenizer *pTok; +}; + +/* +** Virtual table cursor structure. +*/ +struct Fts3tokCursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + char *zInput; /* Input string */ + sqlite3_tokenizer_cursor *pCsr; /* Cursor to iterate through zInput */ + int iRowid; /* Current 'rowid' value */ + const char *zToken; /* Current 'token' value */ + int nToken; /* Size of zToken in bytes */ + int iStart; /* Current 'start' value */ + int iEnd; /* Current 'end' value */ + int iPos; /* Current 'pos' value */ +}; + +/* +** Query FTS for the tokenizer implementation named zName. +*/ +static int fts3tokQueryTokenizer( + Fts3Hash *pHash, + const char *zName, + const sqlite3_tokenizer_module **pp, + char **pzErr +){ + sqlite3_tokenizer_module *p; + int nName = (int)strlen(zName); + + p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); + if( !p ){ + sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", zName); + return SQLITE_ERROR; + } + + *pp = p; + return SQLITE_OK; +} + +/* +** The second argument, argv[], is an array of pointers to nul-terminated +** strings. This function makes a copy of the array and strings into a +** single block of memory. It then dequotes any of the strings that appear +** to be quoted. +** +** If successful, output parameter *pazDequote is set to point at the +** array of dequoted strings and SQLITE_OK is returned. The caller is +** responsible for eventually calling sqlite3_free() to free the array +** in this case. Or, if an error occurs, an SQLite error code is returned. +** The final value of *pazDequote is undefined in this case. +*/ +static int fts3tokDequoteArray( + int argc, /* Number of elements in argv[] */ + const char * const *argv, /* Input array */ + char ***pazDequote /* Output array */ +){ + int rc = SQLITE_OK; /* Return code */ + if( argc==0 ){ + *pazDequote = 0; + }else{ + int i; + int nByte = 0; + char **azDequote; + + for(i=0; ixCreate((nDequote>1 ? nDequote-1 : 0), azArg, &pTok); + } + + if( rc==SQLITE_OK ){ + pTab = (Fts3tokTable *)sqlite3_malloc(sizeof(Fts3tokTable)); + if( pTab==0 ){ + rc = SQLITE_NOMEM; + } + } + + if( rc==SQLITE_OK ){ + memset(pTab, 0, sizeof(Fts3tokTable)); + pTab->pMod = pMod; + pTab->pTok = pTok; + *ppVtab = &pTab->base; + }else{ + if( pTok ){ + pMod->xDestroy(pTok); + } + } + + sqlite3_free(azDequote); + return rc; +} + +/* +** This function does the work for both the xDisconnect and xDestroy methods. +** These tables have no persistent representation of their own, so xDisconnect +** and xDestroy are identical operations. +*/ +static int fts3tokDisconnectMethod(sqlite3_vtab *pVtab){ + Fts3tokTable *pTab = (Fts3tokTable *)pVtab; + + pTab->pMod->xDestroy(pTab->pTok); + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* +** xBestIndex - Analyze a WHERE and ORDER BY clause. +*/ +static int fts3tokBestIndexMethod( + sqlite3_vtab *pVTab, + sqlite3_index_info *pInfo +){ + int i; + UNUSED_PARAMETER(pVTab); + + for(i=0; inConstraint; i++){ + if( pInfo->aConstraint[i].usable + && pInfo->aConstraint[i].iColumn==0 + && pInfo->aConstraint[i].op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + pInfo->idxNum = 1; + pInfo->aConstraintUsage[i].argvIndex = 1; + pInfo->aConstraintUsage[i].omit = 1; + pInfo->estimatedCost = 1; + return SQLITE_OK; + } + } + + pInfo->idxNum = 0; + assert( pInfo->estimatedCost>1000000.0 ); + + return SQLITE_OK; +} + +/* +** xOpen - Open a cursor. +*/ +static int fts3tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts3tokCursor *pCsr; + UNUSED_PARAMETER(pVTab); + + pCsr = (Fts3tokCursor *)sqlite3_malloc(sizeof(Fts3tokCursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(Fts3tokCursor)); + + *ppCsr = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** Reset the tokenizer cursor passed as the only argument. As if it had +** just been returned by fts3tokOpenMethod(). +*/ +static void fts3tokResetCursor(Fts3tokCursor *pCsr){ + if( pCsr->pCsr ){ + Fts3tokTable *pTab = (Fts3tokTable *)(pCsr->base.pVtab); + pTab->pMod->xClose(pCsr->pCsr); + pCsr->pCsr = 0; + } + sqlite3_free(pCsr->zInput); + pCsr->zInput = 0; + pCsr->zToken = 0; + pCsr->nToken = 0; + pCsr->iStart = 0; + pCsr->iEnd = 0; + pCsr->iPos = 0; + pCsr->iRowid = 0; +} + +/* +** xClose - Close a cursor. +*/ +static int fts3tokCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + + fts3tokResetCursor(pCsr); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** xNext - Advance the cursor to the next row, if any. +*/ +static int fts3tokNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); + int rc; /* Return code */ + + pCsr->iRowid++; + rc = pTab->pMod->xNext(pCsr->pCsr, + &pCsr->zToken, &pCsr->nToken, + &pCsr->iStart, &pCsr->iEnd, &pCsr->iPos + ); + + if( rc!=SQLITE_OK ){ + fts3tokResetCursor(pCsr); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + + return rc; +} + +/* +** xFilter - Initialize a cursor to point at the start of its data. +*/ +static int fts3tokFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + int rc = SQLITE_ERROR; + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(nVal); + + fts3tokResetCursor(pCsr); + if( idxNum==1 ){ + const char *zByte = (const char *)sqlite3_value_text(apVal[0]); + int nByte = sqlite3_value_bytes(apVal[0]); + pCsr->zInput = sqlite3_malloc(nByte+1); + if( pCsr->zInput==0 ){ + rc = SQLITE_NOMEM; + }else{ + memcpy(pCsr->zInput, zByte, nByte); + pCsr->zInput[nByte] = 0; + rc = pTab->pMod->xOpen(pTab->pTok, pCsr->zInput, nByte, &pCsr->pCsr); + if( rc==SQLITE_OK ){ + pCsr->pCsr->pTokenizer = pTab->pTok; + } + } + } + + if( rc!=SQLITE_OK ) return rc; + return fts3tokNextMethod(pCursor); +} + +/* +** xEof - Return true if the cursor is at EOF, or false otherwise. +*/ +static int fts3tokEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + return (pCsr->zToken==0); +} + +/* +** xColumn - Return a column value. +*/ +static int fts3tokColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + + /* CREATE TABLE x(input, token, start, end, position) */ + switch( iCol ){ + case 0: + sqlite3_result_text(pCtx, pCsr->zInput, -1, SQLITE_TRANSIENT); + break; + case 1: + sqlite3_result_text(pCtx, pCsr->zToken, pCsr->nToken, SQLITE_TRANSIENT); + break; + case 2: + sqlite3_result_int(pCtx, pCsr->iStart); + break; + case 3: + sqlite3_result_int(pCtx, pCsr->iEnd); + break; + default: + assert( iCol==4 ); + sqlite3_result_int(pCtx, pCsr->iPos); + break; + } + return SQLITE_OK; +} + +/* +** xRowid - Return the current rowid for the cursor. +*/ +static int fts3tokRowidMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite_int64 *pRowid /* OUT: Rowid value */ +){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + *pRowid = (sqlite3_int64)pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Register the fts3tok module with database connection db. Return SQLITE_OK +** if successful or an error code if sqlite3_create_module() fails. +*/ +SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){ + static const sqlite3_module fts3tok_module = { + 0, /* iVersion */ + fts3tokConnectMethod, /* xCreate */ + fts3tokConnectMethod, /* xConnect */ + fts3tokBestIndexMethod, /* xBestIndex */ + fts3tokDisconnectMethod, /* xDisconnect */ + fts3tokDisconnectMethod, /* xDestroy */ + fts3tokOpenMethod, /* xOpen */ + fts3tokCloseMethod, /* xClose */ + fts3tokFilterMethod, /* xFilter */ + fts3tokNextMethod, /* xNext */ + fts3tokEofMethod, /* xEof */ + fts3tokColumnMethod, /* xColumn */ + fts3tokRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ + }; + int rc; /* Return code */ + + rc = sqlite3_create_module(db, "fts3tokenize", &fts3tok_module, (void*)pHash); + return rc; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_tokenize_vtab.c **********************************/ +/************** Begin file fts3_write.c **************************************/ +/* +** 2009 Oct 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file is part of the SQLite FTS3 extension module. Specifically, +** this file contains code to insert, update and delete rows from FTS3 +** tables. It also contains code to merge FTS3 b-tree segments. Some +** of the sub-routines used to merge segments are also used by the query +** code in fts3.c. +*/ + +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ +/* #include */ + + +#define FTS_MAX_APPENDABLE_HEIGHT 16 + +/* +** When full-text index nodes are loaded from disk, the buffer that they +** are loaded into has the following number of bytes of padding at the end +** of it. i.e. if a full-text index node is 900 bytes in size, then a buffer +** of 920 bytes is allocated for it. +** +** This means that if we have a pointer into a buffer containing node data, +** it is always safe to read up to two varints from it without risking an +** overread, even if the node data is corrupted. +*/ +#define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2) + +/* +** Under certain circumstances, b-tree nodes (doclists) can be loaded into +** memory incrementally instead of all at once. This can be a big performance +** win (reduced IO and CPU) if SQLite stops calling the virtual table xNext() +** method before retrieving all query results (as may happen, for example, +** if a query has a LIMIT clause). +** +** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD +** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes. +** The code is written so that the hard lower-limit for each of these values +** is 1. Clearly such small values would be inefficient, but can be useful +** for testing purposes. +** +** If this module is built with SQLITE_TEST defined, these constants may +** be overridden at runtime for testing purposes. File fts3_test.c contains +** a Tcl interface to read and write the values. +*/ +#ifdef SQLITE_TEST +int test_fts3_node_chunksize = (4*1024); +int test_fts3_node_chunk_threshold = (4*1024)*4; +# define FTS3_NODE_CHUNKSIZE test_fts3_node_chunksize +# define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold +#else +# define FTS3_NODE_CHUNKSIZE (4*1024) +# define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) +#endif + +/* +** The two values that may be meaningfully bound to the :1 parameter in +** statements SQL_REPLACE_STAT and SQL_SELECT_STAT. +*/ +#define FTS_STAT_DOCTOTAL 0 +#define FTS_STAT_INCRMERGEHINT 1 +#define FTS_STAT_AUTOINCRMERGE 2 + +/* +** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic +** and incremental merge operation that takes place. This is used for +** debugging FTS only, it should not usually be turned on in production +** systems. +*/ +#ifdef FTS3_LOG_MERGES +static void fts3LogMerge(int nMerge, sqlite3_int64 iAbsLevel){ + sqlite3_log(SQLITE_OK, "%d-way merge from level %d", nMerge, (int)iAbsLevel); +} +#else +#define fts3LogMerge(x, y) +#endif + + +typedef struct PendingList PendingList; +typedef struct SegmentNode SegmentNode; +typedef struct SegmentWriter SegmentWriter; + +/* +** An instance of the following data structure is used to build doclists +** incrementally. See function fts3PendingListAppend() for details. +*/ +struct PendingList { + int nData; + char *aData; + int nSpace; + sqlite3_int64 iLastDocid; + sqlite3_int64 iLastCol; + sqlite3_int64 iLastPos; +}; + + +/* +** Each cursor has a (possibly empty) linked list of the following objects. +*/ +struct Fts3DeferredToken { + Fts3PhraseToken *pToken; /* Pointer to corresponding expr token */ + int iCol; /* Column token must occur in */ + Fts3DeferredToken *pNext; /* Next in list of deferred tokens */ + PendingList *pList; /* Doclist is assembled here */ +}; + +/* +** An instance of this structure is used to iterate through the terms on +** a contiguous set of segment b-tree leaf nodes. Although the details of +** this structure are only manipulated by code in this file, opaque handles +** of type Fts3SegReader* are also used by code in fts3.c to iterate through +** terms when querying the full-text index. See functions: +** +** sqlite3Fts3SegReaderNew() +** sqlite3Fts3SegReaderFree() +** sqlite3Fts3SegReaderIterate() +** +** Methods used to manipulate Fts3SegReader structures: +** +** fts3SegReaderNext() +** fts3SegReaderFirstDocid() +** fts3SegReaderNextDocid() +*/ +struct Fts3SegReader { + int iIdx; /* Index within level, or 0x7FFFFFFF for PT */ + u8 bLookup; /* True for a lookup only */ + u8 rootOnly; /* True for a root-only reader */ + + sqlite3_int64 iStartBlock; /* Rowid of first leaf block to traverse */ + sqlite3_int64 iLeafEndBlock; /* Rowid of final leaf block to traverse */ + sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */ + sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */ + + char *aNode; /* Pointer to node data (or NULL) */ + int nNode; /* Size of buffer at aNode (or 0) */ + int nPopulate; /* If >0, bytes of buffer aNode[] loaded */ + sqlite3_blob *pBlob; /* If not NULL, blob handle to read node */ + + Fts3HashElem **ppNextElem; + + /* Variables set by fts3SegReaderNext(). These may be read directly + ** by the caller. They are valid from the time SegmentReaderNew() returns + ** until SegmentReaderNext() returns something other than SQLITE_OK + ** (i.e. SQLITE_DONE). + */ + int nTerm; /* Number of bytes in current term */ + char *zTerm; /* Pointer to current term */ + int nTermAlloc; /* Allocated size of zTerm buffer */ + char *aDoclist; /* Pointer to doclist of current entry */ + int nDoclist; /* Size of doclist in current entry */ + + /* The following variables are used by fts3SegReaderNextDocid() to iterate + ** through the current doclist (aDoclist/nDoclist). + */ + char *pOffsetList; + int nOffsetList; /* For descending pending seg-readers only */ + sqlite3_int64 iDocid; +}; + +#define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0) +#define fts3SegReaderIsRootOnly(p) ((p)->rootOnly!=0) + +/* +** An instance of this structure is used to create a segment b-tree in the +** database. The internal details of this type are only accessed by the +** following functions: +** +** fts3SegWriterAdd() +** fts3SegWriterFlush() +** fts3SegWriterFree() +*/ +struct SegmentWriter { + SegmentNode *pTree; /* Pointer to interior tree structure */ + sqlite3_int64 iFirst; /* First slot in %_segments written */ + sqlite3_int64 iFree; /* Next free slot in %_segments */ + char *zTerm; /* Pointer to previous term buffer */ + int nTerm; /* Number of bytes in zTerm */ + int nMalloc; /* Size of malloc'd buffer at zMalloc */ + char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ + int nSize; /* Size of allocation at aData */ + int nData; /* Bytes of data in aData */ + char *aData; /* Pointer to block from malloc() */ + i64 nLeafData; /* Number of bytes of leaf data written */ +}; + +/* +** Type SegmentNode is used by the following three functions to create +** the interior part of the segment b+-tree structures (everything except +** the leaf nodes). These functions and type are only ever used by code +** within the fts3SegWriterXXX() family of functions described above. +** +** fts3NodeAddTerm() +** fts3NodeWrite() +** fts3NodeFree() +** +** When a b+tree is written to the database (either as a result of a merge +** or the pending-terms table being flushed), leaves are written into the +** database file as soon as they are completely populated. The interior of +** the tree is assembled in memory and written out only once all leaves have +** been populated and stored. This is Ok, as the b+-tree fanout is usually +** very large, meaning that the interior of the tree consumes relatively +** little memory. +*/ +struct SegmentNode { + SegmentNode *pParent; /* Parent node (or NULL for root node) */ + SegmentNode *pRight; /* Pointer to right-sibling */ + SegmentNode *pLeftmost; /* Pointer to left-most node of this depth */ + int nEntry; /* Number of terms written to node so far */ + char *zTerm; /* Pointer to previous term buffer */ + int nTerm; /* Number of bytes in zTerm */ + int nMalloc; /* Size of malloc'd buffer at zMalloc */ + char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ + int nData; /* Bytes of valid data so far */ + char *aData; /* Node data */ +}; + +/* +** Valid values for the second argument to fts3SqlStmt(). +*/ +#define SQL_DELETE_CONTENT 0 +#define SQL_IS_EMPTY 1 +#define SQL_DELETE_ALL_CONTENT 2 +#define SQL_DELETE_ALL_SEGMENTS 3 +#define SQL_DELETE_ALL_SEGDIR 4 +#define SQL_DELETE_ALL_DOCSIZE 5 +#define SQL_DELETE_ALL_STAT 6 +#define SQL_SELECT_CONTENT_BY_ROWID 7 +#define SQL_NEXT_SEGMENT_INDEX 8 +#define SQL_INSERT_SEGMENTS 9 +#define SQL_NEXT_SEGMENTS_ID 10 +#define SQL_INSERT_SEGDIR 11 +#define SQL_SELECT_LEVEL 12 +#define SQL_SELECT_LEVEL_RANGE 13 +#define SQL_SELECT_LEVEL_COUNT 14 +#define SQL_SELECT_SEGDIR_MAX_LEVEL 15 +#define SQL_DELETE_SEGDIR_LEVEL 16 +#define SQL_DELETE_SEGMENTS_RANGE 17 +#define SQL_CONTENT_INSERT 18 +#define SQL_DELETE_DOCSIZE 19 +#define SQL_REPLACE_DOCSIZE 20 +#define SQL_SELECT_DOCSIZE 21 +#define SQL_SELECT_STAT 22 +#define SQL_REPLACE_STAT 23 + +#define SQL_SELECT_ALL_PREFIX_LEVEL 24 +#define SQL_DELETE_ALL_TERMS_SEGDIR 25 +#define SQL_DELETE_SEGDIR_RANGE 26 +#define SQL_SELECT_ALL_LANGID 27 +#define SQL_FIND_MERGE_LEVEL 28 +#define SQL_MAX_LEAF_NODE_ESTIMATE 29 +#define SQL_DELETE_SEGDIR_ENTRY 30 +#define SQL_SHIFT_SEGDIR_ENTRY 31 +#define SQL_SELECT_SEGDIR 32 +#define SQL_CHOMP_SEGDIR 33 +#define SQL_SEGMENT_IS_APPENDABLE 34 +#define SQL_SELECT_INDEXES 35 +#define SQL_SELECT_MXLEVEL 36 + +#define SQL_SELECT_LEVEL_RANGE2 37 +#define SQL_UPDATE_LEVEL_IDX 38 +#define SQL_UPDATE_LEVEL 39 + +/* +** This function is used to obtain an SQLite prepared statement handle +** for the statement identified by the second argument. If successful, +** *pp is set to the requested statement handle and SQLITE_OK returned. +** Otherwise, an SQLite error code is returned and *pp is set to 0. +** +** If argument apVal is not NULL, then it must point to an array with +** at least as many entries as the requested statement has bound +** parameters. The values are bound to the statements parameters before +** returning. +*/ +static int fts3SqlStmt( + Fts3Table *p, /* Virtual table handle */ + int eStmt, /* One of the SQL_XXX constants above */ + sqlite3_stmt **pp, /* OUT: Statement handle */ + sqlite3_value **apVal /* Values to bind to statement */ +){ + const char *azSql[] = { +/* 0 */ "DELETE FROM %Q.'%q_content' WHERE rowid = ?", +/* 1 */ "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)", +/* 2 */ "DELETE FROM %Q.'%q_content'", +/* 3 */ "DELETE FROM %Q.'%q_segments'", +/* 4 */ "DELETE FROM %Q.'%q_segdir'", +/* 5 */ "DELETE FROM %Q.'%q_docsize'", +/* 6 */ "DELETE FROM %Q.'%q_stat'", +/* 7 */ "SELECT %s WHERE rowid=?", +/* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1", +/* 9 */ "REPLACE INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", +/* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", +/* 11 */ "REPLACE INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", + + /* Return segments in order from oldest to newest.*/ +/* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", +/* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?" + "ORDER BY level DESC, idx ASC", + +/* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", +/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", + +/* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", +/* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", +/* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%s)", +/* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", +/* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", +/* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", +/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=?", +/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(?,?)", +/* 24 */ "", +/* 25 */ "", + +/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", +/* 27 */ "SELECT ? UNION SELECT level / (1024 * ?) FROM %Q.'%q_segdir'", + +/* This statement is used to determine which level to read the input from +** when performing an incremental merge. It returns the absolute level number +** of the oldest level in the db that contains at least ? segments. Or, +** if no level in the FTS index contains more than ? segments, the statement +** returns zero rows. */ +/* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?" + " ORDER BY (level %% 1024) ASC LIMIT 1", + +/* Estimate the upper limit on the number of leaf nodes in a new segment +** created by merging the oldest :2 segments from absolute level :1. See +** function sqlite3Fts3Incrmerge() for details. */ +/* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) " + " FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?", + +/* SQL_DELETE_SEGDIR_ENTRY +** Delete the %_segdir entry on absolute level :1 with index :2. */ +/* 30 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", + +/* SQL_SHIFT_SEGDIR_ENTRY +** Modify the idx value for the segment with idx=:3 on absolute level :2 +** to :1. */ +/* 31 */ "UPDATE %Q.'%q_segdir' SET idx = ? WHERE level=? AND idx=?", + +/* SQL_SELECT_SEGDIR +** Read a single entry from the %_segdir table. The entry from absolute +** level :1 with index value :2. */ +/* 32 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", + +/* SQL_CHOMP_SEGDIR +** Update the start_block (:1) and root (:2) fields of the %_segdir +** entry located on absolute level :3 with index :4. */ +/* 33 */ "UPDATE %Q.'%q_segdir' SET start_block = ?, root = ?" + "WHERE level = ? AND idx = ?", + +/* SQL_SEGMENT_IS_APPENDABLE +** Return a single row if the segment with end_block=? is appendable. Or +** no rows otherwise. */ +/* 34 */ "SELECT 1 FROM %Q.'%q_segments' WHERE blockid=? AND block IS NULL", + +/* SQL_SELECT_INDEXES +** Return the list of valid segment indexes for absolute level ? */ +/* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC", + +/* SQL_SELECT_MXLEVEL +** Return the largest relative level in the FTS index or indexes. */ +/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'", + + /* Return segments in order from oldest to newest.*/ +/* 37 */ "SELECT level, idx, end_block " + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? " + "ORDER BY level DESC, idx ASC", + + /* Update statements used while promoting segments */ +/* 38 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=-1,idx=? " + "WHERE level=? AND idx=?", +/* 39 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=? WHERE level=-1" + + }; + int rc = SQLITE_OK; + sqlite3_stmt *pStmt; + + assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); + assert( eStmt=0 ); + + pStmt = p->aStmt[eStmt]; + if( !pStmt ){ + char *zSql; + if( eStmt==SQL_CONTENT_INSERT ){ + zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist); + }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){ + zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist); + }else{ + zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); + } + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL); + sqlite3_free(zSql); + assert( rc==SQLITE_OK || pStmt==0 ); + p->aStmt[eStmt] = pStmt; + } + } + if( apVal ){ + int i; + int nParam = sqlite3_bind_parameter_count(pStmt); + for(i=0; rc==SQLITE_OK && inPendingData==0 ){ + sqlite3_stmt *pStmt; + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_null(pStmt, 1); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + } + } + + return rc; +} + +/* +** FTS maintains a separate indexes for each language-id (a 32-bit integer). +** Within each language id, a separate index is maintained to store the +** document terms, and each configured prefix size (configured the FTS +** "prefix=" option). And each index consists of multiple levels ("relative +** levels"). +** +** All three of these values (the language id, the specific index and the +** level within the index) are encoded in 64-bit integer values stored +** in the %_segdir table on disk. This function is used to convert three +** separate component values into the single 64-bit integer value that +** can be used to query the %_segdir table. +** +** Specifically, each language-id/index combination is allocated 1024 +** 64-bit integer level values ("absolute levels"). The main terms index +** for language-id 0 is allocate values 0-1023. The first prefix index +** (if any) for language-id 0 is allocated values 1024-2047. And so on. +** Language 1 indexes are allocated immediately following language 0. +** +** So, for a system with nPrefix prefix indexes configured, the block of +** absolute levels that corresponds to language-id iLangid and index +** iIndex starts at absolute level ((iLangid * (nPrefix+1) + iIndex) * 1024). +*/ +static sqlite3_int64 getAbsoluteLevel( + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language id */ + int iIndex, /* Index in p->aIndex[] */ + int iLevel /* Level of segments */ +){ + sqlite3_int64 iBase; /* First absolute level for iLangid/iIndex */ + assert( iLangid>=0 ); + assert( p->nIndex>0 ); + assert( iIndex>=0 && iIndexnIndex ); + + iBase = ((sqlite3_int64)iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL; + return iBase + iLevel; +} + +/* +** Set *ppStmt to a statement handle that may be used to iterate through +** all rows in the %_segdir table, from oldest to newest. If successful, +** return SQLITE_OK. If an error occurs while preparing the statement, +** return an SQLite error code. +** +** There is only ever one instance of this SQL statement compiled for +** each FTS3 table. +** +** The statement returns the following columns from the %_segdir table: +** +** 0: idx +** 1: start_block +** 2: leaves_end_block +** 3: end_block +** 4: root +*/ +SQLITE_PRIVATE int sqlite3Fts3AllSegdirs( + Fts3Table *p, /* FTS3 table */ + int iLangid, /* Language being queried */ + int iIndex, /* Index for p->aIndex[] */ + int iLevel, /* Level to select (relative level) */ + sqlite3_stmt **ppStmt /* OUT: Compiled statement */ +){ + int rc; + sqlite3_stmt *pStmt = 0; + + assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 ); + assert( iLevel=0 && iIndexnIndex ); + + if( iLevel<0 ){ + /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pStmt, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); + } + }else{ + /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel)); + } + } + *ppStmt = pStmt; + return rc; +} + + +/* +** Append a single varint to a PendingList buffer. SQLITE_OK is returned +** if successful, or an SQLite error code otherwise. +** +** This function also serves to allocate the PendingList structure itself. +** For example, to create a new PendingList structure containing two +** varints: +** +** PendingList *p = 0; +** fts3PendingListAppendVarint(&p, 1); +** fts3PendingListAppendVarint(&p, 2); +*/ +static int fts3PendingListAppendVarint( + PendingList **pp, /* IN/OUT: Pointer to PendingList struct */ + sqlite3_int64 i /* Value to append to data */ +){ + PendingList *p = *pp; + + /* Allocate or grow the PendingList as required. */ + if( !p ){ + p = sqlite3_malloc(sizeof(*p) + 100); + if( !p ){ + return SQLITE_NOMEM; + } + p->nSpace = 100; + p->aData = (char *)&p[1]; + p->nData = 0; + } + else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){ + int nNew = p->nSpace * 2; + p = sqlite3_realloc(p, sizeof(*p) + nNew); + if( !p ){ + sqlite3_free(*pp); + *pp = 0; + return SQLITE_NOMEM; + } + p->nSpace = nNew; + p->aData = (char *)&p[1]; + } + + /* Append the new serialized varint to the end of the list. */ + p->nData += sqlite3Fts3PutVarint(&p->aData[p->nData], i); + p->aData[p->nData] = '\0'; + *pp = p; + return SQLITE_OK; +} + +/* +** Add a docid/column/position entry to a PendingList structure. Non-zero +** is returned if the structure is sqlite3_realloced as part of adding +** the entry. Otherwise, zero. +** +** If an OOM error occurs, *pRc is set to SQLITE_NOMEM before returning. +** Zero is always returned in this case. Otherwise, if no OOM error occurs, +** it is set to SQLITE_OK. +*/ +static int fts3PendingListAppend( + PendingList **pp, /* IN/OUT: PendingList structure */ + sqlite3_int64 iDocid, /* Docid for entry to add */ + sqlite3_int64 iCol, /* Column for entry to add */ + sqlite3_int64 iPos, /* Position of term for entry to add */ + int *pRc /* OUT: Return code */ +){ + PendingList *p = *pp; + int rc = SQLITE_OK; + + assert( !p || p->iLastDocid<=iDocid ); + + if( !p || p->iLastDocid!=iDocid ){ + sqlite3_int64 iDelta = iDocid - (p ? p->iLastDocid : 0); + if( p ){ + assert( p->nDatanSpace ); + assert( p->aData[p->nData]==0 ); + p->nData++; + } + if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iDelta)) ){ + goto pendinglistappend_out; + } + p->iLastCol = -1; + p->iLastPos = 0; + p->iLastDocid = iDocid; + } + if( iCol>0 && p->iLastCol!=iCol ){ + if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, 1)) + || SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iCol)) + ){ + goto pendinglistappend_out; + } + p->iLastCol = iCol; + p->iLastPos = 0; + } + if( iCol>=0 ){ + assert( iPos>p->iLastPos || (iPos==0 && p->iLastPos==0) ); + rc = fts3PendingListAppendVarint(&p, 2+iPos-p->iLastPos); + if( rc==SQLITE_OK ){ + p->iLastPos = iPos; + } + } + + pendinglistappend_out: + *pRc = rc; + if( p!=*pp ){ + *pp = p; + return 1; + } + return 0; +} + +/* +** Free a PendingList object allocated by fts3PendingListAppend(). +*/ +static void fts3PendingListDelete(PendingList *pList){ + sqlite3_free(pList); +} + +/* +** Add an entry to one of the pending-terms hash tables. +*/ +static int fts3PendingTermsAddOne( + Fts3Table *p, + int iCol, + int iPos, + Fts3Hash *pHash, /* Pending terms hash table to add entry to */ + const char *zToken, + int nToken +){ + PendingList *pList; + int rc = SQLITE_OK; + + pList = (PendingList *)fts3HashFind(pHash, zToken, nToken); + if( pList ){ + p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); + } + if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ + if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){ + /* Malloc failed while inserting the new entry. This can only + ** happen if there was no previous entry for this token. + */ + assert( 0==fts3HashFind(pHash, zToken, nToken) ); + sqlite3_free(pList); + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); + } + return rc; +} + +/* +** Tokenize the nul-terminated string zText and add all tokens to the +** pending-terms hash-table. The docid used is that currently stored in +** p->iPrevDocid, and the column is specified by argument iCol. +** +** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. +*/ +static int fts3PendingTermsAdd( + Fts3Table *p, /* Table into which text will be inserted */ + int iLangid, /* Language id to use */ + const char *zText, /* Text of document to be inserted */ + int iCol, /* Column into which text is being inserted */ + u32 *pnWord /* IN/OUT: Incr. by number tokens inserted */ +){ + int rc; + int iStart = 0; + int iEnd = 0; + int iPos = 0; + int nWord = 0; + + char const *zToken; + int nToken = 0; + + sqlite3_tokenizer *pTokenizer = p->pTokenizer; + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + sqlite3_tokenizer_cursor *pCsr; + int (*xNext)(sqlite3_tokenizer_cursor *pCursor, + const char**,int*,int*,int*,int*); + + assert( pTokenizer && pModule ); + + /* If the user has inserted a NULL value, this function may be called with + ** zText==0. In this case, add zero token entries to the hash table and + ** return early. */ + if( zText==0 ){ + *pnWord = 0; + return SQLITE_OK; + } + + rc = sqlite3Fts3OpenTokenizer(pTokenizer, iLangid, zText, -1, &pCsr); + if( rc!=SQLITE_OK ){ + return rc; + } + + xNext = pModule->xNext; + while( SQLITE_OK==rc + && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) + ){ + int i; + if( iPos>=nWord ) nWord = iPos+1; + + /* Positions cannot be negative; we use -1 as a terminator internally. + ** Tokens must have a non-zero length. + */ + if( iPos<0 || !zToken || nToken<=0 ){ + rc = SQLITE_ERROR; + break; + } + + /* Add the term to the terms index */ + rc = fts3PendingTermsAddOne( + p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken + ); + + /* Add the term to each of the prefix indexes that it is not too + ** short for. */ + for(i=1; rc==SQLITE_OK && inIndex; i++){ + struct Fts3Index *pIndex = &p->aIndex[i]; + if( nTokennPrefix ) continue; + rc = fts3PendingTermsAddOne( + p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix + ); + } + } + + pModule->xClose(pCsr); + *pnWord += nWord; + return (rc==SQLITE_DONE ? SQLITE_OK : rc); +} + +/* +** Calling this function indicates that subsequent calls to +** fts3PendingTermsAdd() are to add term/position-list pairs for the +** contents of the document with docid iDocid. +*/ +static int fts3PendingTermsDocid( + Fts3Table *p, /* Full-text table handle */ + int bDelete, /* True if this op is a delete */ + int iLangid, /* Language id of row being written */ + sqlite_int64 iDocid /* Docid of row being written */ +){ + assert( iLangid>=0 ); + assert( bDelete==1 || bDelete==0 ); + + /* TODO(shess) Explore whether partially flushing the buffer on + ** forced-flush would provide better performance. I suspect that if + ** we ordered the doclists by size and flushed the largest until the + ** buffer was half empty, that would let the less frequent terms + ** generate longer doclists. + */ + if( iDocidiPrevDocid + || (iDocid==p->iPrevDocid && p->bPrevDelete==0) + || p->iPrevLangid!=iLangid + || p->nPendingData>p->nMaxPendingData + ){ + int rc = sqlite3Fts3PendingTermsFlush(p); + if( rc!=SQLITE_OK ) return rc; + } + p->iPrevDocid = iDocid; + p->iPrevLangid = iLangid; + p->bPrevDelete = bDelete; + return SQLITE_OK; +} + +/* +** Discard the contents of the pending-terms hash tables. +*/ +SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){ + int i; + for(i=0; inIndex; i++){ + Fts3HashElem *pElem; + Fts3Hash *pHash = &p->aIndex[i].hPending; + for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){ + PendingList *pList = (PendingList *)fts3HashData(pElem); + fts3PendingListDelete(pList); + } + fts3HashClear(pHash); + } + p->nPendingData = 0; +} + +/* +** This function is called by the xUpdate() method as part of an INSERT +** operation. It adds entries for each term in the new record to the +** pendingTerms hash table. +** +** Argument apVal is the same as the similarly named argument passed to +** fts3InsertData(). Parameter iDocid is the docid of the new row. +*/ +static int fts3InsertTerms( + Fts3Table *p, + int iLangid, + sqlite3_value **apVal, + u32 *aSz +){ + int i; /* Iterator variable */ + for(i=2; inColumn+2; i++){ + int iCol = i-2; + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_value_text(apVal[i]); + int rc = fts3PendingTermsAdd(p, iLangid, zText, iCol, &aSz[iCol]); + if( rc!=SQLITE_OK ){ + return rc; + } + aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]); + } + } + return SQLITE_OK; +} + +/* +** This function is called by the xUpdate() method for an INSERT operation. +** The apVal parameter is passed a copy of the apVal argument passed by +** SQLite to the xUpdate() method. i.e: +** +** apVal[0] Not used for INSERT. +** apVal[1] rowid +** apVal[2] Left-most user-defined column +** ... +** apVal[p->nColumn+1] Right-most user-defined column +** apVal[p->nColumn+2] Hidden column with same name as table +** apVal[p->nColumn+3] Hidden "docid" column (alias for rowid) +** apVal[p->nColumn+4] Hidden languageid column +*/ +static int fts3InsertData( + Fts3Table *p, /* Full-text table */ + sqlite3_value **apVal, /* Array of values to insert */ + sqlite3_int64 *piDocid /* OUT: Docid for row just inserted */ +){ + int rc; /* Return code */ + sqlite3_stmt *pContentInsert; /* INSERT INTO %_content VALUES(...) */ + + if( p->zContentTbl ){ + sqlite3_value *pRowid = apVal[p->nColumn+3]; + if( sqlite3_value_type(pRowid)==SQLITE_NULL ){ + pRowid = apVal[1]; + } + if( sqlite3_value_type(pRowid)!=SQLITE_INTEGER ){ + return SQLITE_CONSTRAINT; + } + *piDocid = sqlite3_value_int64(pRowid); + return SQLITE_OK; + } + + /* Locate the statement handle used to insert data into the %_content + ** table. The SQL for this statement is: + ** + ** INSERT INTO %_content VALUES(?, ?, ?, ...) + ** + ** The statement features N '?' variables, where N is the number of user + ** defined columns in the FTS3 table, plus one for the docid field. + */ + rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]); + if( rc==SQLITE_OK && p->zLanguageid ){ + rc = sqlite3_bind_int( + pContentInsert, p->nColumn+2, + sqlite3_value_int(apVal[p->nColumn+4]) + ); + } + if( rc!=SQLITE_OK ) return rc; + + /* There is a quirk here. The users INSERT statement may have specified + ** a value for the "rowid" field, for the "docid" field, or for both. + ** Which is a problem, since "rowid" and "docid" are aliases for the + ** same value. For example: + ** + ** INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2); + ** + ** In FTS3, this is an error. It is an error to specify non-NULL values + ** for both docid and some other rowid alias. + */ + if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){ + if( SQLITE_NULL==sqlite3_value_type(apVal[0]) + && SQLITE_NULL!=sqlite3_value_type(apVal[1]) + ){ + /* A rowid/docid conflict. */ + return SQLITE_ERROR; + } + rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]); + if( rc!=SQLITE_OK ) return rc; + } + + /* Execute the statement to insert the record. Set *piDocid to the + ** new docid value. + */ + sqlite3_step(pContentInsert); + rc = sqlite3_reset(pContentInsert); + + *piDocid = sqlite3_last_insert_rowid(p->db); + return rc; +} + + + +/* +** Remove all data from the FTS3 table. Clear the hash table containing +** pending terms. +*/ +static int fts3DeleteAll(Fts3Table *p, int bContent){ + int rc = SQLITE_OK; /* Return code */ + + /* Discard the contents of the pending-terms hash table. */ + sqlite3Fts3PendingTermsClear(p); + + /* Delete everything from the shadow tables. Except, leave %_content as + ** is if bContent is false. */ + assert( p->zContentTbl==0 || bContent==0 ); + if( bContent ) fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); + fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0); + fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); + if( p->bHasDocsize ){ + fts3SqlExec(&rc, p, SQL_DELETE_ALL_DOCSIZE, 0); + } + if( p->bHasStat ){ + fts3SqlExec(&rc, p, SQL_DELETE_ALL_STAT, 0); + } + return rc; +} + +/* +** +*/ +static int langidFromSelect(Fts3Table *p, sqlite3_stmt *pSelect){ + int iLangid = 0; + if( p->zLanguageid ) iLangid = sqlite3_column_int(pSelect, p->nColumn+1); + return iLangid; +} + +/* +** The first element in the apVal[] array is assumed to contain the docid +** (an integer) of a row about to be deleted. Remove all terms from the +** full-text index. +*/ +static void fts3DeleteTerms( + int *pRC, /* Result code */ + Fts3Table *p, /* The FTS table to delete from */ + sqlite3_value *pRowid, /* The docid to be deleted */ + u32 *aSz, /* Sizes of deleted document written here */ + int *pbFound /* OUT: Set to true if row really does exist */ +){ + int rc; + sqlite3_stmt *pSelect; + + assert( *pbFound==0 ); + if( *pRC ) return; + rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pSelect) ){ + int i; + int iLangid = langidFromSelect(p, pSelect); + i64 iDocid = sqlite3_column_int64(pSelect, 0); + rc = fts3PendingTermsDocid(p, 1, iLangid, iDocid); + for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){ + int iCol = i-1; + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pSelect, i); + rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[iCol]); + aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i); + } + } + if( rc!=SQLITE_OK ){ + sqlite3_reset(pSelect); + *pRC = rc; + return; + } + *pbFound = 1; + } + rc = sqlite3_reset(pSelect); + }else{ + sqlite3_reset(pSelect); + } + *pRC = rc; +} + +/* +** Forward declaration to account for the circular dependency between +** functions fts3SegmentMerge() and fts3AllocateSegdirIdx(). +*/ +static int fts3SegmentMerge(Fts3Table *, int, int, int); + +/* +** This function allocates a new level iLevel index in the segdir table. +** Usually, indexes are allocated within a level sequentially starting +** with 0, so the allocated index is one greater than the value returned +** by: +** +** SELECT max(idx) FROM %_segdir WHERE level = :iLevel +** +** However, if there are already FTS3_MERGE_COUNT indexes at the requested +** level, they are merged into a single level (iLevel+1) segment and the +** allocated index is 0. +** +** If successful, *piIdx is set to the allocated index slot and SQLITE_OK +** returned. Otherwise, an SQLite error code is returned. +*/ +static int fts3AllocateSegdirIdx( + Fts3Table *p, + int iLangid, /* Language id */ + int iIndex, /* Index for p->aIndex */ + int iLevel, + int *piIdx +){ + int rc; /* Return Code */ + sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */ + int iNext = 0; /* Result of query pNextIdx */ + + assert( iLangid>=0 ); + assert( p->nIndex>=1 ); + + /* Set variable iNext to the next available segdir index at level iLevel. */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64( + pNextIdx, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) + ); + if( SQLITE_ROW==sqlite3_step(pNextIdx) ){ + iNext = sqlite3_column_int(pNextIdx, 0); + } + rc = sqlite3_reset(pNextIdx); + } + + if( rc==SQLITE_OK ){ + /* If iNext is FTS3_MERGE_COUNT, indicating that level iLevel is already + ** full, merge all segments in level iLevel into a single iLevel+1 + ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise, + ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. + */ + if( iNext>=FTS3_MERGE_COUNT ){ + fts3LogMerge(16, getAbsoluteLevel(p, iLangid, iIndex, iLevel)); + rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel); + *piIdx = 0; + }else{ + *piIdx = iNext; + } + } + + return rc; +} + +/* +** The %_segments table is declared as follows: +** +** CREATE TABLE %_segments(blockid INTEGER PRIMARY KEY, block BLOB) +** +** This function reads data from a single row of the %_segments table. The +** specific row is identified by the iBlockid parameter. If paBlob is not +** NULL, then a buffer is allocated using sqlite3_malloc() and populated +** with the contents of the blob stored in the "block" column of the +** identified table row is. Whether or not paBlob is NULL, *pnBlob is set +** to the size of the blob in bytes before returning. +** +** If an error occurs, or the table does not contain the specified row, +** an SQLite error code is returned. Otherwise, SQLITE_OK is returned. If +** paBlob is non-NULL, then it is the responsibility of the caller to +** eventually free the returned buffer. +** +** This function may leave an open sqlite3_blob* handle in the +** Fts3Table.pSegments variable. This handle is reused by subsequent calls +** to this function. The handle may be closed by calling the +** sqlite3Fts3SegmentsClose() function. Reusing a blob handle is a handy +** performance improvement, but the blob handle should always be closed +** before control is returned to the user (to prevent a lock being held +** on the database file for longer than necessary). Thus, any virtual table +** method (xFilter etc.) that may directly or indirectly call this function +** must call sqlite3Fts3SegmentsClose() before returning. +*/ +SQLITE_PRIVATE int sqlite3Fts3ReadBlock( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */ + char **paBlob, /* OUT: Blob data in malloc'd buffer */ + int *pnBlob, /* OUT: Size of blob data */ + int *pnLoad /* OUT: Bytes actually loaded */ +){ + int rc; /* Return code */ + + /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */ + assert( pnBlob ); + + if( p->pSegments ){ + rc = sqlite3_blob_reopen(p->pSegments, iBlockid); + }else{ + if( 0==p->zSegmentsTbl ){ + p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName); + if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM; + } + rc = sqlite3_blob_open( + p->db, p->zDb, p->zSegmentsTbl, "block", iBlockid, 0, &p->pSegments + ); + } + + if( rc==SQLITE_OK ){ + int nByte = sqlite3_blob_bytes(p->pSegments); + *pnBlob = nByte; + if( paBlob ){ + char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING); + if( !aByte ){ + rc = SQLITE_NOMEM; + }else{ + if( pnLoad && nByte>(FTS3_NODE_CHUNK_THRESHOLD) ){ + nByte = FTS3_NODE_CHUNKSIZE; + *pnLoad = nByte; + } + rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0); + memset(&aByte[nByte], 0, FTS3_NODE_PADDING); + if( rc!=SQLITE_OK ){ + sqlite3_free(aByte); + aByte = 0; + } + } + *paBlob = aByte; + } + } + + return rc; +} + +/* +** Close the blob handle at p->pSegments, if it is open. See comments above +** the sqlite3Fts3ReadBlock() function for details. +*/ +SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *p){ + sqlite3_blob_close(p->pSegments); + p->pSegments = 0; +} + +static int fts3SegReaderIncrRead(Fts3SegReader *pReader){ + int nRead; /* Number of bytes to read */ + int rc; /* Return code */ + + nRead = MIN(pReader->nNode - pReader->nPopulate, FTS3_NODE_CHUNKSIZE); + rc = sqlite3_blob_read( + pReader->pBlob, + &pReader->aNode[pReader->nPopulate], + nRead, + pReader->nPopulate + ); + + if( rc==SQLITE_OK ){ + pReader->nPopulate += nRead; + memset(&pReader->aNode[pReader->nPopulate], 0, FTS3_NODE_PADDING); + if( pReader->nPopulate==pReader->nNode ){ + sqlite3_blob_close(pReader->pBlob); + pReader->pBlob = 0; + pReader->nPopulate = 0; + } + } + return rc; +} + +static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){ + int rc = SQLITE_OK; + assert( !pReader->pBlob + || (pFrom>=pReader->aNode && pFrom<&pReader->aNode[pReader->nNode]) + ); + while( pReader->pBlob && rc==SQLITE_OK + && (pFrom - pReader->aNode + nByte)>pReader->nPopulate + ){ + rc = fts3SegReaderIncrRead(pReader); + } + return rc; +} + +/* +** Set an Fts3SegReader cursor to point at EOF. +*/ +static void fts3SegReaderSetEof(Fts3SegReader *pSeg){ + if( !fts3SegReaderIsRootOnly(pSeg) ){ + sqlite3_free(pSeg->aNode); + sqlite3_blob_close(pSeg->pBlob); + pSeg->pBlob = 0; + } + pSeg->aNode = 0; +} + +/* +** Move the iterator passed as the first argument to the next term in the +** segment. If successful, SQLITE_OK is returned. If there is no next term, +** SQLITE_DONE. Otherwise, an SQLite error code. +*/ +static int fts3SegReaderNext( + Fts3Table *p, + Fts3SegReader *pReader, + int bIncr +){ + int rc; /* Return code of various sub-routines */ + char *pNext; /* Cursor variable */ + int nPrefix; /* Number of bytes in term prefix */ + int nSuffix; /* Number of bytes in term suffix */ + + if( !pReader->aDoclist ){ + pNext = pReader->aNode; + }else{ + pNext = &pReader->aDoclist[pReader->nDoclist]; + } + + if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ + + if( fts3SegReaderIsPending(pReader) ){ + Fts3HashElem *pElem = *(pReader->ppNextElem); + sqlite3_free(pReader->aNode); + pReader->aNode = 0; + if( pElem ){ + char *aCopy; + PendingList *pList = (PendingList *)fts3HashData(pElem); + int nCopy = pList->nData+1; + pReader->zTerm = (char *)fts3HashKey(pElem); + pReader->nTerm = fts3HashKeysize(pElem); + aCopy = (char*)sqlite3_malloc(nCopy); + if( !aCopy ) return SQLITE_NOMEM; + memcpy(aCopy, pList->aData, nCopy); + pReader->nNode = pReader->nDoclist = nCopy; + pReader->aNode = pReader->aDoclist = aCopy; + pReader->ppNextElem++; + assert( pReader->aNode ); + } + return SQLITE_OK; + } + + fts3SegReaderSetEof(pReader); + + /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf + ** blocks have already been traversed. */ + assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock ); + if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){ + return SQLITE_OK; + } + + rc = sqlite3Fts3ReadBlock( + p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode, + (bIncr ? &pReader->nPopulate : 0) + ); + if( rc!=SQLITE_OK ) return rc; + assert( pReader->pBlob==0 ); + if( bIncr && pReader->nPopulatenNode ){ + pReader->pBlob = p->pSegments; + p->pSegments = 0; + } + pNext = pReader->aNode; + } + + assert( !fts3SegReaderIsPending(pReader) ); + + rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2); + if( rc!=SQLITE_OK ) return rc; + + /* Because of the FTS3_NODE_PADDING bytes of padding, the following is + ** safe (no risk of overread) even if the node data is corrupted. */ + pNext += fts3GetVarint32(pNext, &nPrefix); + pNext += fts3GetVarint32(pNext, &nSuffix); + if( nPrefix<0 || nSuffix<=0 + || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] + ){ + return FTS_CORRUPT_VTAB; + } + + if( nPrefix+nSuffix>pReader->nTermAlloc ){ + int nNew = (nPrefix+nSuffix)*2; + char *zNew = sqlite3_realloc(pReader->zTerm, nNew); + if( !zNew ){ + return SQLITE_NOMEM; + } + pReader->zTerm = zNew; + pReader->nTermAlloc = nNew; + } + + rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX); + if( rc!=SQLITE_OK ) return rc; + + memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); + pReader->nTerm = nPrefix+nSuffix; + pNext += nSuffix; + pNext += fts3GetVarint32(pNext, &pReader->nDoclist); + pReader->aDoclist = pNext; + pReader->pOffsetList = 0; + + /* Check that the doclist does not appear to extend past the end of the + ** b-tree node. And that the final byte of the doclist is 0x00. If either + ** of these statements is untrue, then the data structure is corrupt. + */ + if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] + || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1]) + ){ + return FTS_CORRUPT_VTAB; + } + return SQLITE_OK; +} + +/* +** Set the SegReader to point to the first docid in the doclist associated +** with the current term. +*/ +static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){ + int rc = SQLITE_OK; + assert( pReader->aDoclist ); + assert( !pReader->pOffsetList ); + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + u8 bEof = 0; + pReader->iDocid = 0; + pReader->nOffsetList = 0; + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList, + &pReader->iDocid, &pReader->nOffsetList, &bEof + ); + }else{ + rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); + pReader->pOffsetList = &pReader->aDoclist[n]; + } + } + return rc; +} + +/* +** Advance the SegReader to point to the next docid in the doclist +** associated with the current term. +** +** If arguments ppOffsetList and pnOffsetList are not NULL, then +** *ppOffsetList is set to point to the first column-offset list +** in the doclist entry (i.e. immediately past the docid varint). +** *pnOffsetList is set to the length of the set of column-offset +** lists, not including the nul-terminator byte. For example: +*/ +static int fts3SegReaderNextDocid( + Fts3Table *pTab, + Fts3SegReader *pReader, /* Reader to advance to next docid */ + char **ppOffsetList, /* OUT: Pointer to current position-list */ + int *pnOffsetList /* OUT: Length of *ppOffsetList in bytes */ +){ + int rc = SQLITE_OK; + char *p = pReader->pOffsetList; + char c = 0; + + assert( p ); + + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + /* A pending-terms seg-reader for an FTS4 table that uses order=desc. + ** Pending-terms doclists are always built up in ascending order, so + ** we have to iterate through them backwards here. */ + u8 bEof = 0; + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = pReader->nOffsetList - 1; + } + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &p, &pReader->iDocid, + &pReader->nOffsetList, &bEof + ); + if( bEof ){ + pReader->pOffsetList = 0; + }else{ + pReader->pOffsetList = p; + } + }else{ + char *pEnd = &pReader->aDoclist[pReader->nDoclist]; + + /* Pointer p currently points at the first byte of an offset list. The + ** following block advances it to point one byte past the end of + ** the same offset list. */ + while( 1 ){ + + /* The following line of code (and the "p++" below the while() loop) is + ** normally all that is required to move pointer p to the desired + ** position. The exception is if this node is being loaded from disk + ** incrementally and pointer "p" now points to the first byte past + ** the populated part of pReader->aNode[]. + */ + while( *p | c ) c = *p++ & 0x80; + assert( *p==0 ); + + if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break; + rc = fts3SegReaderIncrRead(pReader); + if( rc!=SQLITE_OK ) return rc; + } + p++; + + /* If required, populate the output variables with a pointer to and the + ** size of the previous offset-list. + */ + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = (int)(p - pReader->pOffsetList - 1); + } + + /* List may have been edited in place by fts3EvalNearTrim() */ + while( p=pEnd ){ + pReader->pOffsetList = 0; + }else{ + rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + sqlite3_int64 iDelta; + pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); + if( pTab->bDescIdx ){ + pReader->iDocid -= iDelta; + }else{ + pReader->iDocid += iDelta; + } + } + } + } + + return SQLITE_OK; +} + + +SQLITE_PRIVATE int sqlite3Fts3MsrOvfl( + Fts3Cursor *pCsr, + Fts3MultiSegReader *pMsr, + int *pnOvfl +){ + Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; + int nOvfl = 0; + int ii; + int rc = SQLITE_OK; + int pgsz = p->nPgsz; + + assert( p->bFts4 ); + assert( pgsz>0 ); + + for(ii=0; rc==SQLITE_OK && iinSegment; ii++){ + Fts3SegReader *pReader = pMsr->apSegment[ii]; + if( !fts3SegReaderIsPending(pReader) + && !fts3SegReaderIsRootOnly(pReader) + ){ + sqlite3_int64 jj; + for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){ + int nBlob; + rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob, 0); + if( rc!=SQLITE_OK ) break; + if( (nBlob+35)>pgsz ){ + nOvfl += (nBlob + 34)/pgsz; + } + } + } + } + *pnOvfl = nOvfl; + return rc; +} + +/* +** Free all allocations associated with the iterator passed as the +** second argument. +*/ +SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ + if( pReader ){ + if( !fts3SegReaderIsPending(pReader) ){ + sqlite3_free(pReader->zTerm); + } + if( !fts3SegReaderIsRootOnly(pReader) ){ + sqlite3_free(pReader->aNode); + } + sqlite3_blob_close(pReader->pBlob); + } + sqlite3_free(pReader); +} + +/* +** Allocate a new SegReader object. +*/ +SQLITE_PRIVATE int sqlite3Fts3SegReaderNew( + int iAge, /* Segment "age". */ + int bLookup, /* True for a lookup only */ + sqlite3_int64 iStartLeaf, /* First leaf to traverse */ + sqlite3_int64 iEndLeaf, /* Final leaf to traverse */ + sqlite3_int64 iEndBlock, /* Final block of segment */ + const char *zRoot, /* Buffer containing root node */ + int nRoot, /* Size of buffer containing root node */ + Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */ +){ + Fts3SegReader *pReader; /* Newly allocated SegReader object */ + int nExtra = 0; /* Bytes to allocate segment root node */ + + assert( iStartLeaf<=iEndLeaf ); + if( iStartLeaf==0 ){ + nExtra = nRoot + FTS3_NODE_PADDING; + } + + pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra); + if( !pReader ){ + return SQLITE_NOMEM; + } + memset(pReader, 0, sizeof(Fts3SegReader)); + pReader->iIdx = iAge; + pReader->bLookup = bLookup!=0; + pReader->iStartBlock = iStartLeaf; + pReader->iLeafEndBlock = iEndLeaf; + pReader->iEndBlock = iEndBlock; + + if( nExtra ){ + /* The entire segment is stored in the root node. */ + pReader->aNode = (char *)&pReader[1]; + pReader->rootOnly = 1; + pReader->nNode = nRoot; + memcpy(pReader->aNode, zRoot, nRoot); + memset(&pReader->aNode[nRoot], 0, FTS3_NODE_PADDING); + }else{ + pReader->iCurrentBlock = iStartLeaf-1; + } + *ppReader = pReader; + return SQLITE_OK; +} + +/* +** This is a comparison function used as a qsort() callback when sorting +** an array of pending terms by term. This occurs as part of flushing +** the contents of the pending-terms hash table to the database. +*/ +static int SQLITE_CDECL fts3CompareElemByTerm( + const void *lhs, + const void *rhs +){ + char *z1 = fts3HashKey(*(Fts3HashElem **)lhs); + char *z2 = fts3HashKey(*(Fts3HashElem **)rhs); + int n1 = fts3HashKeysize(*(Fts3HashElem **)lhs); + int n2 = fts3HashKeysize(*(Fts3HashElem **)rhs); + + int n = (n1aIndex */ + const char *zTerm, /* Term to search for */ + int nTerm, /* Size of buffer zTerm */ + int bPrefix, /* True for a prefix iterator */ + Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */ +){ + Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */ + Fts3HashElem *pE; /* Iterator variable */ + Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */ + int nElem = 0; /* Size of array at aElem */ + int rc = SQLITE_OK; /* Return Code */ + Fts3Hash *pHash; + + pHash = &p->aIndex[iIndex].hPending; + if( bPrefix ){ + int nAlloc = 0; /* Size of allocated array at aElem */ + + for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){ + char *zKey = (char *)fts3HashKey(pE); + int nKey = fts3HashKeysize(pE); + if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){ + if( nElem==nAlloc ){ + Fts3HashElem **aElem2; + nAlloc += 16; + aElem2 = (Fts3HashElem **)sqlite3_realloc( + aElem, nAlloc*sizeof(Fts3HashElem *) + ); + if( !aElem2 ){ + rc = SQLITE_NOMEM; + nElem = 0; + break; + } + aElem = aElem2; + } + + aElem[nElem++] = pE; + } + } + + /* If more than one term matches the prefix, sort the Fts3HashElem + ** objects in term order using qsort(). This uses the same comparison + ** callback as is used when flushing terms to disk. + */ + if( nElem>1 ){ + qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm); + } + + }else{ + /* The query is a simple term lookup that matches at most one term in + ** the index. All that is required is a straight hash-lookup. + ** + ** Because the stack address of pE may be accessed via the aElem pointer + ** below, the "Fts3HashElem *pE" must be declared so that it is valid + ** within this entire function, not just this "else{...}" block. + */ + pE = fts3HashFindElem(pHash, zTerm, nTerm); + if( pE ){ + aElem = &pE; + nElem = 1; + } + } + + if( nElem>0 ){ + int nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *); + pReader = (Fts3SegReader *)sqlite3_malloc(nByte); + if( !pReader ){ + rc = SQLITE_NOMEM; + }else{ + memset(pReader, 0, nByte); + pReader->iIdx = 0x7FFFFFFF; + pReader->ppNextElem = (Fts3HashElem **)&pReader[1]; + memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *)); + } + } + + if( bPrefix ){ + sqlite3_free(aElem); + } + *ppReader = pReader; + return rc; +} + +/* +** Compare the entries pointed to by two Fts3SegReader structures. +** Comparison is as follows: +** +** 1) EOF is greater than not EOF. +** +** 2) The current terms (if any) are compared using memcmp(). If one +** term is a prefix of another, the longer term is considered the +** larger. +** +** 3) By segment age. An older segment is considered larger. +*/ +static int fts3SegReaderCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc; + if( pLhs->aNode && pRhs->aNode ){ + int rc2 = pLhs->nTerm - pRhs->nTerm; + if( rc2<0 ){ + rc = memcmp(pLhs->zTerm, pRhs->zTerm, pLhs->nTerm); + }else{ + rc = memcmp(pLhs->zTerm, pRhs->zTerm, pRhs->nTerm); + } + if( rc==0 ){ + rc = rc2; + } + }else{ + rc = (pLhs->aNode==0) - (pRhs->aNode==0); + } + if( rc==0 ){ + rc = pRhs->iIdx - pLhs->iIdx; + } + assert( rc!=0 ); + return rc; +} + +/* +** A different comparison function for SegReader structures. In this +** version, it is assumed that each SegReader points to an entry in +** a doclist for identical terms. Comparison is made as follows: +** +** 1) EOF (end of doclist in this case) is greater than not EOF. +** +** 2) By current docid. +** +** 3) By segment age. An older segment is considered larger. +*/ +static int fts3SegReaderDoclistCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); + if( rc==0 ){ + if( pLhs->iDocid==pRhs->iDocid ){ + rc = pRhs->iIdx - pLhs->iIdx; + }else{ + rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1; + } + } + assert( pLhs->aNode && pRhs->aNode ); + return rc; +} +static int fts3SegReaderDoclistCmpRev(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); + if( rc==0 ){ + if( pLhs->iDocid==pRhs->iDocid ){ + rc = pRhs->iIdx - pLhs->iIdx; + }else{ + rc = (pLhs->iDocid < pRhs->iDocid) ? 1 : -1; + } + } + assert( pLhs->aNode && pRhs->aNode ); + return rc; +} + +/* +** Compare the term that the Fts3SegReader object passed as the first argument +** points to with the term specified by arguments zTerm and nTerm. +** +** If the pSeg iterator is already at EOF, return 0. Otherwise, return +** -ve if the pSeg term is less than zTerm/nTerm, 0 if the two terms are +** equal, or +ve if the pSeg term is greater than zTerm/nTerm. +*/ +static int fts3SegReaderTermCmp( + Fts3SegReader *pSeg, /* Segment reader object */ + const char *zTerm, /* Term to compare to */ + int nTerm /* Size of term zTerm in bytes */ +){ + int res = 0; + if( pSeg->aNode ){ + if( pSeg->nTerm>nTerm ){ + res = memcmp(pSeg->zTerm, zTerm, nTerm); + }else{ + res = memcmp(pSeg->zTerm, zTerm, pSeg->nTerm); + } + if( res==0 ){ + res = pSeg->nTerm-nTerm; + } + } + return res; +} + +/* +** Argument apSegment is an array of nSegment elements. It is known that +** the final (nSegment-nSuspect) members are already in sorted order +** (according to the comparison function provided). This function shuffles +** the array around until all entries are in sorted order. +*/ +static void fts3SegReaderSort( + Fts3SegReader **apSegment, /* Array to sort entries of */ + int nSegment, /* Size of apSegment array */ + int nSuspect, /* Unsorted entry count */ + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) /* Comparison function */ +){ + int i; /* Iterator variable */ + + assert( nSuspect<=nSegment ); + + if( nSuspect==nSegment ) nSuspect--; + for(i=nSuspect-1; i>=0; i--){ + int j; + for(j=i; j<(nSegment-1); j++){ + Fts3SegReader *pTmp; + if( xCmp(apSegment[j], apSegment[j+1])<0 ) break; + pTmp = apSegment[j+1]; + apSegment[j+1] = apSegment[j]; + apSegment[j] = pTmp; + } + } + +#ifndef NDEBUG + /* Check that the list really is sorted now. */ + for(i=0; i<(nSuspect-1); i++){ + assert( xCmp(apSegment[i], apSegment[i+1])<0 ); + } +#endif +} + +/* +** Insert a record into the %_segments table. +*/ +static int fts3WriteSegment( + Fts3Table *p, /* Virtual table handle */ + sqlite3_int64 iBlock, /* Block id for new block */ + char *z, /* Pointer to buffer containing block data */ + int n /* Size of buffer z in bytes */ +){ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_INSERT_SEGMENTS, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, iBlock); + sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + } + return rc; +} + +/* +** Find the largest relative level number in the table. If successful, set +** *pnMax to this value and return SQLITE_OK. Otherwise, if an error occurs, +** set *pnMax to zero and return an SQLite error code. +*/ +SQLITE_PRIVATE int sqlite3Fts3MaxLevel(Fts3Table *p, int *pnMax){ + int rc; + int mxLevel = 0; + sqlite3_stmt *pStmt = 0; + + rc = fts3SqlStmt(p, SQL_SELECT_MXLEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + mxLevel = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_reset(pStmt); + } + *pnMax = mxLevel; + return rc; +} + +/* +** Insert a record into the %_segdir table. +*/ +static int fts3WriteSegdir( + Fts3Table *p, /* Virtual table handle */ + sqlite3_int64 iLevel, /* Value for "level" field (absolute level) */ + int iIdx, /* Value for "idx" field */ + sqlite3_int64 iStartBlock, /* Value for "start_block" field */ + sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ + sqlite3_int64 iEndBlock, /* Value for "end_block" field */ + sqlite3_int64 nLeafData, /* Bytes of leaf data in segment */ + char *zRoot, /* Blob value for "root" field */ + int nRoot /* Number of bytes in buffer zRoot */ +){ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, iLevel); + sqlite3_bind_int(pStmt, 2, iIdx); + sqlite3_bind_int64(pStmt, 3, iStartBlock); + sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); + if( nLeafData==0 ){ + sqlite3_bind_int64(pStmt, 5, iEndBlock); + }else{ + char *zEnd = sqlite3_mprintf("%lld %lld", iEndBlock, nLeafData); + if( !zEnd ) return SQLITE_NOMEM; + sqlite3_bind_text(pStmt, 5, zEnd, -1, sqlite3_free); + } + sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + } + return rc; +} + +/* +** Return the size of the common prefix (if any) shared by zPrev and +** zNext, in bytes. For example, +** +** fts3PrefixCompress("abc", 3, "abcdef", 6) // returns 3 +** fts3PrefixCompress("abX", 3, "abcdef", 6) // returns 2 +** fts3PrefixCompress("abX", 3, "Xbcdef", 6) // returns 0 +*/ +static int fts3PrefixCompress( + const char *zPrev, /* Buffer containing previous term */ + int nPrev, /* Size of buffer zPrev in bytes */ + const char *zNext, /* Buffer containing next term */ + int nNext /* Size of buffer zNext in bytes */ +){ + int n; + UNUSED_PARAMETER(nNext); + for(n=0; nnData; /* Current size of node in bytes */ + int nReq = nData; /* Required space after adding zTerm */ + int nPrefix; /* Number of bytes of prefix compression */ + int nSuffix; /* Suffix length */ + + nPrefix = fts3PrefixCompress(pTree->zTerm, pTree->nTerm, zTerm, nTerm); + nSuffix = nTerm-nPrefix; + + nReq += sqlite3Fts3VarintLen(nPrefix)+sqlite3Fts3VarintLen(nSuffix)+nSuffix; + if( nReq<=p->nNodeSize || !pTree->zTerm ){ + + if( nReq>p->nNodeSize ){ + /* An unusual case: this is the first term to be added to the node + ** and the static node buffer (p->nNodeSize bytes) is not large + ** enough. Use a separately malloced buffer instead This wastes + ** p->nNodeSize bytes, but since this scenario only comes about when + ** the database contain two terms that share a prefix of almost 2KB, + ** this is not expected to be a serious problem. + */ + assert( pTree->aData==(char *)&pTree[1] ); + pTree->aData = (char *)sqlite3_malloc(nReq); + if( !pTree->aData ){ + return SQLITE_NOMEM; + } + } + + if( pTree->zTerm ){ + /* There is no prefix-length field for first term in a node */ + nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nPrefix); + } + + nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nSuffix); + memcpy(&pTree->aData[nData], &zTerm[nPrefix], nSuffix); + pTree->nData = nData + nSuffix; + pTree->nEntry++; + + if( isCopyTerm ){ + if( pTree->nMalloczMalloc, nTerm*2); + if( !zNew ){ + return SQLITE_NOMEM; + } + pTree->nMalloc = nTerm*2; + pTree->zMalloc = zNew; + } + pTree->zTerm = pTree->zMalloc; + memcpy(pTree->zTerm, zTerm, nTerm); + pTree->nTerm = nTerm; + }else{ + pTree->zTerm = (char *)zTerm; + pTree->nTerm = nTerm; + } + return SQLITE_OK; + } + } + + /* If control flows to here, it was not possible to append zTerm to the + ** current node. Create a new node (a right-sibling of the current node). + ** If this is the first node in the tree, the term is added to it. + ** + ** Otherwise, the term is not added to the new node, it is left empty for + ** now. Instead, the term is inserted into the parent of pTree. If pTree + ** has no parent, one is created here. + */ + pNew = (SegmentNode *)sqlite3_malloc(sizeof(SegmentNode) + p->nNodeSize); + if( !pNew ){ + return SQLITE_NOMEM; + } + memset(pNew, 0, sizeof(SegmentNode)); + pNew->nData = 1 + FTS3_VARINT_MAX; + pNew->aData = (char *)&pNew[1]; + + if( pTree ){ + SegmentNode *pParent = pTree->pParent; + rc = fts3NodeAddTerm(p, &pParent, isCopyTerm, zTerm, nTerm); + if( pTree->pParent==0 ){ + pTree->pParent = pParent; + } + pTree->pRight = pNew; + pNew->pLeftmost = pTree->pLeftmost; + pNew->pParent = pParent; + pNew->zMalloc = pTree->zMalloc; + pNew->nMalloc = pTree->nMalloc; + pTree->zMalloc = 0; + }else{ + pNew->pLeftmost = pNew; + rc = fts3NodeAddTerm(p, &pNew, isCopyTerm, zTerm, nTerm); + } + + *ppTree = pNew; + return rc; +} + +/* +** Helper function for fts3NodeWrite(). +*/ +static int fts3TreeFinishNode( + SegmentNode *pTree, + int iHeight, + sqlite3_int64 iLeftChild +){ + int nStart; + assert( iHeight>=1 && iHeight<128 ); + nStart = FTS3_VARINT_MAX - sqlite3Fts3VarintLen(iLeftChild); + pTree->aData[nStart] = (char)iHeight; + sqlite3Fts3PutVarint(&pTree->aData[nStart+1], iLeftChild); + return nStart; +} + +/* +** Write the buffer for the segment node pTree and all of its peers to the +** database. Then call this function recursively to write the parent of +** pTree and its peers to the database. +** +** Except, if pTree is a root node, do not write it to the database. Instead, +** set output variables *paRoot and *pnRoot to contain the root node. +** +** If successful, SQLITE_OK is returned and output variable *piLast is +** set to the largest blockid written to the database (or zero if no +** blocks were written to the db). Otherwise, an SQLite error code is +** returned. +*/ +static int fts3NodeWrite( + Fts3Table *p, /* Virtual table handle */ + SegmentNode *pTree, /* SegmentNode handle */ + int iHeight, /* Height of this node in tree */ + sqlite3_int64 iLeaf, /* Block id of first leaf node */ + sqlite3_int64 iFree, /* Block id of next free slot in %_segments */ + sqlite3_int64 *piLast, /* OUT: Block id of last entry written */ + char **paRoot, /* OUT: Data for root node */ + int *pnRoot /* OUT: Size of root node in bytes */ +){ + int rc = SQLITE_OK; + + if( !pTree->pParent ){ + /* Root node of the tree. */ + int nStart = fts3TreeFinishNode(pTree, iHeight, iLeaf); + *piLast = iFree-1; + *pnRoot = pTree->nData - nStart; + *paRoot = &pTree->aData[nStart]; + }else{ + SegmentNode *pIter; + sqlite3_int64 iNextFree = iFree; + sqlite3_int64 iNextLeaf = iLeaf; + for(pIter=pTree->pLeftmost; pIter && rc==SQLITE_OK; pIter=pIter->pRight){ + int nStart = fts3TreeFinishNode(pIter, iHeight, iNextLeaf); + int nWrite = pIter->nData - nStart; + + rc = fts3WriteSegment(p, iNextFree, &pIter->aData[nStart], nWrite); + iNextFree++; + iNextLeaf += (pIter->nEntry+1); + } + if( rc==SQLITE_OK ){ + assert( iNextLeaf==iFree ); + rc = fts3NodeWrite( + p, pTree->pParent, iHeight+1, iFree, iNextFree, piLast, paRoot, pnRoot + ); + } + } + + return rc; +} + +/* +** Free all memory allocations associated with the tree pTree. +*/ +static void fts3NodeFree(SegmentNode *pTree){ + if( pTree ){ + SegmentNode *p = pTree->pLeftmost; + fts3NodeFree(p->pParent); + while( p ){ + SegmentNode *pRight = p->pRight; + if( p->aData!=(char *)&p[1] ){ + sqlite3_free(p->aData); + } + assert( pRight==0 || p->zMalloc==0 ); + sqlite3_free(p->zMalloc); + sqlite3_free(p); + p = pRight; + } + } +} + +/* +** Add a term to the segment being constructed by the SegmentWriter object +** *ppWriter. When adding the first term to a segment, *ppWriter should +** be passed NULL. This function will allocate a new SegmentWriter object +** and return it via the input/output variable *ppWriter in this case. +** +** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. +*/ +static int fts3SegWriterAdd( + Fts3Table *p, /* Virtual table handle */ + SegmentWriter **ppWriter, /* IN/OUT: SegmentWriter handle */ + int isCopyTerm, /* True if buffer zTerm must be copied */ + const char *zTerm, /* Pointer to buffer containing term */ + int nTerm, /* Size of term in bytes */ + const char *aDoclist, /* Pointer to buffer containing doclist */ + int nDoclist /* Size of doclist in bytes */ +){ + int nPrefix; /* Size of term prefix in bytes */ + int nSuffix; /* Size of term suffix in bytes */ + int nReq; /* Number of bytes required on leaf page */ + int nData; + SegmentWriter *pWriter = *ppWriter; + + if( !pWriter ){ + int rc; + sqlite3_stmt *pStmt; + + /* Allocate the SegmentWriter structure */ + pWriter = (SegmentWriter *)sqlite3_malloc(sizeof(SegmentWriter)); + if( !pWriter ) return SQLITE_NOMEM; + memset(pWriter, 0, sizeof(SegmentWriter)); + *ppWriter = pWriter; + + /* Allocate a buffer in which to accumulate data */ + pWriter->aData = (char *)sqlite3_malloc(p->nNodeSize); + if( !pWriter->aData ) return SQLITE_NOMEM; + pWriter->nSize = p->nNodeSize; + + /* Find the next free blockid in the %_segments table */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + pWriter->iFree = sqlite3_column_int64(pStmt, 0); + pWriter->iFirst = pWriter->iFree; + } + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ) return rc; + } + nData = pWriter->nData; + + nPrefix = fts3PrefixCompress(pWriter->zTerm, pWriter->nTerm, zTerm, nTerm); + nSuffix = nTerm-nPrefix; + + /* Figure out how many bytes are required by this new entry */ + nReq = sqlite3Fts3VarintLen(nPrefix) + /* varint containing prefix size */ + sqlite3Fts3VarintLen(nSuffix) + /* varint containing suffix size */ + nSuffix + /* Term suffix */ + sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ + nDoclist; /* Doclist data */ + + if( nData>0 && nData+nReq>p->nNodeSize ){ + int rc; + + /* The current leaf node is full. Write it out to the database. */ + rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData); + if( rc!=SQLITE_OK ) return rc; + p->nLeafAdd++; + + /* Add the current term to the interior node tree. The term added to + ** the interior tree must: + ** + ** a) be greater than the largest term on the leaf node just written + ** to the database (still available in pWriter->zTerm), and + ** + ** b) be less than or equal to the term about to be added to the new + ** leaf node (zTerm/nTerm). + ** + ** In other words, it must be the prefix of zTerm 1 byte longer than + ** the common prefix (if any) of zTerm and pWriter->zTerm. + */ + assert( nPrefixpTree, isCopyTerm, zTerm, nPrefix+1); + if( rc!=SQLITE_OK ) return rc; + + nData = 0; + pWriter->nTerm = 0; + + nPrefix = 0; + nSuffix = nTerm; + nReq = 1 + /* varint containing prefix size */ + sqlite3Fts3VarintLen(nTerm) + /* varint containing suffix size */ + nTerm + /* Term suffix */ + sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ + nDoclist; /* Doclist data */ + } + + /* Increase the total number of bytes written to account for the new entry. */ + pWriter->nLeafData += nReq; + + /* If the buffer currently allocated is too small for this entry, realloc + ** the buffer to make it large enough. + */ + if( nReq>pWriter->nSize ){ + char *aNew = sqlite3_realloc(pWriter->aData, nReq); + if( !aNew ) return SQLITE_NOMEM; + pWriter->aData = aNew; + pWriter->nSize = nReq; + } + assert( nData+nReq<=pWriter->nSize ); + + /* Append the prefix-compressed term and doclist to the buffer. */ + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nPrefix); + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nSuffix); + memcpy(&pWriter->aData[nData], &zTerm[nPrefix], nSuffix); + nData += nSuffix; + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nDoclist); + memcpy(&pWriter->aData[nData], aDoclist, nDoclist); + pWriter->nData = nData + nDoclist; + + /* Save the current term so that it can be used to prefix-compress the next. + ** If the isCopyTerm parameter is true, then the buffer pointed to by + ** zTerm is transient, so take a copy of the term data. Otherwise, just + ** store a copy of the pointer. + */ + if( isCopyTerm ){ + if( nTerm>pWriter->nMalloc ){ + char *zNew = sqlite3_realloc(pWriter->zMalloc, nTerm*2); + if( !zNew ){ + return SQLITE_NOMEM; + } + pWriter->nMalloc = nTerm*2; + pWriter->zMalloc = zNew; + pWriter->zTerm = zNew; + } + assert( pWriter->zTerm==pWriter->zMalloc ); + memcpy(pWriter->zTerm, zTerm, nTerm); + }else{ + pWriter->zTerm = (char *)zTerm; + } + pWriter->nTerm = nTerm; + + return SQLITE_OK; +} + +/* +** Flush all data associated with the SegmentWriter object pWriter to the +** database. This function must be called after all terms have been added +** to the segment using fts3SegWriterAdd(). If successful, SQLITE_OK is +** returned. Otherwise, an SQLite error code. +*/ +static int fts3SegWriterFlush( + Fts3Table *p, /* Virtual table handle */ + SegmentWriter *pWriter, /* SegmentWriter to flush to the db */ + sqlite3_int64 iLevel, /* Value for 'level' column of %_segdir */ + int iIdx /* Value for 'idx' column of %_segdir */ +){ + int rc; /* Return code */ + if( pWriter->pTree ){ + sqlite3_int64 iLast = 0; /* Largest block id written to database */ + sqlite3_int64 iLastLeaf; /* Largest leaf block id written to db */ + char *zRoot = NULL; /* Pointer to buffer containing root node */ + int nRoot = 0; /* Size of buffer zRoot */ + + iLastLeaf = pWriter->iFree; + rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, pWriter->nData); + if( rc==SQLITE_OK ){ + rc = fts3NodeWrite(p, pWriter->pTree, 1, + pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot); + } + if( rc==SQLITE_OK ){ + rc = fts3WriteSegdir(p, iLevel, iIdx, + pWriter->iFirst, iLastLeaf, iLast, pWriter->nLeafData, zRoot, nRoot); + } + }else{ + /* The entire tree fits on the root node. Write it to the segdir table. */ + rc = fts3WriteSegdir(p, iLevel, iIdx, + 0, 0, 0, pWriter->nLeafData, pWriter->aData, pWriter->nData); + } + p->nLeafAdd++; + return rc; +} + +/* +** Release all memory held by the SegmentWriter object passed as the +** first argument. +*/ +static void fts3SegWriterFree(SegmentWriter *pWriter){ + if( pWriter ){ + sqlite3_free(pWriter->aData); + sqlite3_free(pWriter->zMalloc); + fts3NodeFree(pWriter->pTree); + sqlite3_free(pWriter); + } +} + +/* +** The first value in the apVal[] array is assumed to contain an integer. +** This function tests if there exist any documents with docid values that +** are different from that integer. i.e. if deleting the document with docid +** pRowid would mean the FTS3 table were empty. +** +** If successful, *pisEmpty is set to true if the table is empty except for +** document pRowid, or false otherwise, and SQLITE_OK is returned. If an +** error occurs, an SQLite error code is returned. +*/ +static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ + sqlite3_stmt *pStmt; + int rc; + if( p->zContentTbl ){ + /* If using the content=xxx option, assume the table is never empty */ + *pisEmpty = 0; + rc = SQLITE_OK; + }else{ + rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pisEmpty = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_reset(pStmt); + } + } + return rc; +} + +/* +** Set *pnMax to the largest segment level in the database for the index +** iIndex. +** +** Segment levels are stored in the 'level' column of the %_segdir table. +** +** Return SQLITE_OK if successful, or an SQLite error code if not. +*/ +static int fts3SegmentMaxLevel( + Fts3Table *p, + int iLangid, + int iIndex, + sqlite3_int64 *pnMax +){ + sqlite3_stmt *pStmt; + int rc; + assert( iIndex>=0 && iIndexnIndex ); + + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pStmt, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pnMax = sqlite3_column_int64(pStmt, 0); + } + return sqlite3_reset(pStmt); +} + +/* +** iAbsLevel is an absolute level that may be assumed to exist within +** the database. This function checks if it is the largest level number +** within its index. Assuming no error occurs, *pbMax is set to 1 if +** iAbsLevel is indeed the largest level, or 0 otherwise, and SQLITE_OK +** is returned. If an error occurs, an error code is returned and the +** final value of *pbMax is undefined. +*/ +static int fts3SegmentIsMaxLevel(Fts3Table *p, i64 iAbsLevel, int *pbMax){ + + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pStmt, 1, iAbsLevel+1); + sqlite3_bind_int64(pStmt, 2, + ((iAbsLevel/FTS3_SEGDIR_MAXLEVEL)+1) * FTS3_SEGDIR_MAXLEVEL + ); + + *pbMax = 0; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pbMax = sqlite3_column_type(pStmt, 0)==SQLITE_NULL; + } + return sqlite3_reset(pStmt); +} + +/* +** Delete all entries in the %_segments table associated with the segment +** opened with seg-reader pSeg. This function does not affect the contents +** of the %_segdir table. +*/ +static int fts3DeleteSegment( + Fts3Table *p, /* FTS table handle */ + Fts3SegReader *pSeg /* Segment to delete */ +){ + int rc = SQLITE_OK; /* Return code */ + if( pSeg->iStartBlock ){ + sqlite3_stmt *pDelete; /* SQL statement to delete rows */ + rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, pSeg->iStartBlock); + sqlite3_bind_int64(pDelete, 2, pSeg->iEndBlock); + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + } + return rc; } /* -** Run the parser on the given SQL string. The parser structure is -** passed in. An SQLITE_ status code is returned. If an error occurs -** then an and attempt is made to write an error message into -** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that -** error message. +** This function is used after merging multiple segments into a single large +** segment to delete the old, now redundant, segment b-trees. Specifically, +** it: +** +** 1) Deletes all %_segments entries for the segments associated with +** each of the SegReader objects in the array passed as the third +** argument, and +** +** 2) deletes all %_segdir entries with level iLevel, or all %_segdir +** entries regardless of level if (iLevel<0). +** +** SQLITE_OK is returned if successful, otherwise an SQLite error code. */ -SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ - int nErr = 0; /* Number of errors encountered */ - int i; /* Loop counter */ - void *pEngine; /* The LEMON-generated LALR(1) parser */ - int tokenType; /* type of the next token */ - int lastTokenParsed = -1; /* type of the previous token */ - u8 enableLookaside; /* Saved value of db->lookaside.bEnabled */ - sqlite3 *db = pParse->db; /* The database connection */ - int mxSqlLen; /* Max length of an SQL string */ +static int fts3DeleteSegdir( + Fts3Table *p, /* Virtual table handle */ + int iLangid, /* Language id */ + int iIndex, /* Index for p->aIndex */ + int iLevel, /* Level of %_segdir entries to delete */ + Fts3SegReader **apSegment, /* Array of SegReader objects */ + int nReader /* Size of array apSegment */ +){ + int rc = SQLITE_OK; /* Return Code */ + int i; /* Iterator variable */ + sqlite3_stmt *pDelete = 0; /* SQL statement to delete rows */ - assert( zSql!=0 ); - mxSqlLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; - if( db->nVdbeActive==0 ){ - db->u1.isInterrupted = 0; - } - pParse->rc = SQLITE_OK; - pParse->zTail = zSql; - i = 0; - assert( pzErrMsg!=0 ); - pEngine = sqlite3ParserAlloc(sqlite3Malloc); - if( pEngine==0 ){ - db->mallocFailed = 1; - return SQLITE_NOMEM; + for(i=0; rc==SQLITE_OK && ipNewTable==0 ); - assert( pParse->pNewTrigger==0 ); - assert( pParse->nVar==0 ); - assert( pParse->nzVar==0 ); - assert( pParse->azVar==0 ); - enableLookaside = db->lookaside.bEnabled; - if( db->lookaside.pStart ) db->lookaside.bEnabled = 1; - while( !db->mallocFailed && zSql[i]!=0 ){ - assert( i>=0 ); - pParse->sLastToken.z = &zSql[i]; - pParse->sLastToken.n = sqlite3GetToken((unsigned char*)&zSql[i],&tokenType); - i += pParse->sLastToken.n; - if( i>mxSqlLen ){ - pParse->rc = SQLITE_TOOBIG; - break; - } - switch( tokenType ){ - case TK_SPACE: { - if( db->u1.isInterrupted ){ - sqlite3ErrorMsg(pParse, "interrupt"); - pParse->rc = SQLITE_INTERRUPT; - goto abort_parse; - } - break; - } - case TK_ILLEGAL: { - sqlite3ErrorMsg(pParse, "unrecognized token: \"%T\"", - &pParse->sLastToken); - goto abort_parse; - } - case TK_SEMI: { - pParse->zTail = &zSql[i]; - /* Fall thru into the default case */ - } - default: { - sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse); - lastTokenParsed = tokenType; - if( pParse->rc!=SQLITE_OK ){ - goto abort_parse; - } - break; - } - } + if( rc!=SQLITE_OK ){ + return rc; } -abort_parse: - assert( nErr==0 ); - if( zSql[i]==0 && pParse->rc==SQLITE_OK && db->mallocFailed==0 ){ - if( lastTokenParsed!=TK_SEMI ){ - sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse); - pParse->zTail = &zSql[i]; + + assert( iLevel>=0 || iLevel==FTS3_SEGCURSOR_ALL ); + if( iLevel==FTS3_SEGCURSOR_ALL ){ + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pDelete, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); } - if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){ - sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse); + }else{ + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64( + pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) + ); } } -#ifdef YYTRACKMAXSTACKDEPTH - sqlite3_mutex_enter(sqlite3MallocMutex()); - sqlite3StatusSet(SQLITE_STATUS_PARSER_STACK, - sqlite3ParserStackPeak(pEngine) - ); - sqlite3_mutex_leave(sqlite3MallocMutex()); -#endif /* YYDEBUG */ - sqlite3ParserFree(pEngine, sqlite3_free); - db->lookaside.bEnabled = enableLookaside; - if( db->mallocFailed ){ - pParse->rc = SQLITE_NOMEM; - } - if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){ - sqlite3SetString(&pParse->zErrMsg, db, "%s", sqlite3ErrStr(pParse->rc)); - } - assert( pzErrMsg!=0 ); - if( pParse->zErrMsg ){ - *pzErrMsg = pParse->zErrMsg; - sqlite3_log(pParse->rc, "%s", *pzErrMsg); - pParse->zErrMsg = 0; - nErr++; - } - if( pParse->pVdbe && pParse->nErr>0 && pParse->nested==0 ){ - sqlite3VdbeDelete(pParse->pVdbe); - pParse->pVdbe = 0; - } -#ifndef SQLITE_OMIT_SHARED_CACHE - if( pParse->nested==0 ){ - sqlite3DbFree(db, pParse->aTableLock); - pParse->aTableLock = 0; - pParse->nTableLock = 0; - } -#endif -#ifndef SQLITE_OMIT_VIRTUALTABLE - sqlite3_free(pParse->apVtabLock); -#endif - if( !IN_DECLARE_VTAB ){ - /* If the pParse->declareVtab flag is set, do not delete any table - ** structure built up in pParse->pNewTable. The calling code (see vtab.c) - ** will take responsibility for freeing the Table structure. - */ - sqlite3DeleteTable(db, pParse->pNewTable); + if( rc==SQLITE_OK ){ + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); } - if( pParse->bFreeWith ) sqlite3WithDelete(db, pParse->pWith); - sqlite3DeleteTrigger(db, pParse->pNewTrigger); - for(i=pParse->nzVar-1; i>=0; i--) sqlite3DbFree(db, pParse->azVar[i]); - sqlite3DbFree(db, pParse->azVar); - while( pParse->pAinc ){ - AutoincInfo *p = pParse->pAinc; - pParse->pAinc = p->pNext; - sqlite3DbFree(db, p); - } - while( pParse->pZombieTab ){ - Table *p = pParse->pZombieTab; - pParse->pZombieTab = p->pNextZombie; - sqlite3DeleteTable(db, p); - } - assert( nErr==0 || pParse->rc!=SQLITE_OK ); - return nErr; + return rc; } -/************** End of tokenize.c ********************************************/ -/************** Begin file complete.c ****************************************/ /* -** 2001 September 15 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** When this function is called, buffer *ppList (size *pnList bytes) contains +** a position list that may (or may not) feature multiple columns. This +** function adjusts the pointer *ppList and the length *pnList so that they +** identify the subset of the position list that corresponds to column iCol. ** -************************************************************************* -** An tokenizer for SQL +** If there are no entries in the input position list for column iCol, then +** *pnList is set to zero before returning. ** -** This file contains C code that implements the sqlite3_complete() API. -** This code used to be part of the tokenizer.c source file. But by -** separating it out, the code will be automatically omitted from -** static links that do not use it. +** If parameter bZero is non-zero, then any part of the input list following +** the end of the output list is zeroed before returning. */ -#ifndef SQLITE_OMIT_COMPLETE +static void fts3ColumnFilter( + int iCol, /* Column to filter on */ + int bZero, /* Zero out anything following *ppList */ + char **ppList, /* IN/OUT: Pointer to position list */ + int *pnList /* IN/OUT: Size of buffer *ppList in bytes */ +){ + char *pList = *ppList; + int nList = *pnList; + char *pEnd = &pList[nList]; + int iCurrent = 0; + char *p = pList; -/* -** This is defined in tokenize.c. We just have to import the definition. -*/ -#ifndef SQLITE_AMALGAMATION -#ifdef SQLITE_ASCII -#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) -#endif -#ifdef SQLITE_EBCDIC -SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[]; -#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) -#endif -#endif /* SQLITE_AMALGAMATION */ + assert( iCol>=0 ); + while( 1 ){ + char c = 0; + while( ppMsr->nBuffer ){ + char *pNew; + pMsr->nBuffer = nList*2; + pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer); + if( !pNew ) return SQLITE_NOMEM; + pMsr->aBuffer = pNew; + } -#ifndef SQLITE_OMIT_TRIGGER - /* A complex statement machine used to detect the end of a CREATE TRIGGER - ** statement. This is the normal case. - */ - static const u8 trans[8][8] = { - /* Token: */ - /* State: ** SEMI WS OTHER EXPLAIN CREATE TEMP TRIGGER END */ - /* 0 INVALID: */ { 1, 0, 2, 3, 4, 2, 2, 2, }, - /* 1 START: */ { 1, 1, 2, 3, 4, 2, 2, 2, }, - /* 2 NORMAL: */ { 1, 2, 2, 2, 2, 2, 2, 2, }, - /* 3 EXPLAIN: */ { 1, 3, 3, 2, 4, 2, 2, 2, }, - /* 4 CREATE: */ { 1, 4, 2, 2, 2, 4, 5, 2, }, - /* 5 TRIGGER: */ { 6, 5, 5, 5, 5, 5, 5, 5, }, - /* 6 SEMI: */ { 6, 6, 5, 5, 5, 5, 5, 7, }, - /* 7 END: */ { 1, 7, 5, 5, 5, 5, 5, 5, }, - }; -#else - /* If triggers are not supported by this compile then the statement machine - ** used to detect the end of a statement is much simpler - */ - static const u8 trans[3][3] = { - /* Token: */ - /* State: ** SEMI WS OTHER */ - /* 0 INVALID: */ { 1, 0, 2, }, - /* 1 START: */ { 1, 1, 2, }, - /* 2 NORMAL: */ { 1, 2, 2, }, - }; -#endif /* SQLITE_OMIT_TRIGGER */ + memcpy(pMsr->aBuffer, pList, nList); + return SQLITE_OK; +} -#ifdef SQLITE_ENABLE_API_ARMOR - if( zSql==0 ){ - (void)SQLITE_MISUSE_BKPT; - return 0; +SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ + sqlite3_int64 *piDocid, /* OUT: Docid value */ + char **paPoslist, /* OUT: Pointer to position list */ + int *pnPoslist /* OUT: Size of position list in bytes */ +){ + int nMerge = pMsr->nAdvance; + Fts3SegReader **apSegment = pMsr->apSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + if( nMerge==0 ){ + *paPoslist = 0; + return SQLITE_OK; } -#endif - while( *zSql ){ - switch( *zSql ){ - case ';': { /* A semicolon */ - token = tkSEMI; - break; - } - case ' ': - case '\r': - case '\t': - case '\n': - case '\f': { /* White space is ignored */ - token = tkWS; - break; - } - case '/': { /* C-style comments */ - if( zSql[1]!='*' ){ - token = tkOTHER; - break; - } - zSql += 2; - while( zSql[0] && (zSql[0]!='*' || zSql[1]!='/') ){ zSql++; } - if( zSql[0]==0 ) return 0; - zSql++; - token = tkWS; - break; - } - case '-': { /* SQL-style comments from "--" to end of line */ - if( zSql[1]!='-' ){ - token = tkOTHER; - break; - } - while( *zSql && *zSql!='\n' ){ zSql++; } - if( *zSql==0 ) return state==1; - token = tkWS; - break; + while( 1 ){ + Fts3SegReader *pSeg; + pSeg = pMsr->apSegment[0]; + + if( pSeg->pOffsetList==0 ){ + *paPoslist = 0; + break; + }else{ + int rc; + char *pList; + int nList; + int j; + sqlite3_int64 iDocid = apSegment[0]->iDocid; + + rc = fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); + j = 1; + while( rc==SQLITE_OK + && jpOffsetList + && apSegment[j]->iDocid==iDocid + ){ + rc = fts3SegReaderNextDocid(p, apSegment[j], 0, 0); + j++; } - case '[': { /* Microsoft-style identifiers in [...] */ - zSql++; - while( *zSql && *zSql!=']' ){ zSql++; } - if( *zSql==0 ) return 0; - token = tkOTHER; - break; + if( rc!=SQLITE_OK ) return rc; + fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); + + if( nList>0 && fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pMsr, pList, nList+1); + if( rc!=SQLITE_OK ) return rc; + assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); + pList = pMsr->aBuffer; } - case '`': /* Grave-accent quoted symbols used by MySQL */ - case '"': /* single- and double-quoted strings */ - case '\'': { - int c = *zSql; - zSql++; - while( *zSql && *zSql!=c ){ zSql++; } - if( *zSql==0 ) return 0; - token = tkOTHER; - break; + + if( pMsr->iColFilter>=0 ){ + fts3ColumnFilter(pMsr->iColFilter, 1, &pList, &nList); } - default: { -#ifdef SQLITE_EBCDIC - unsigned char c; -#endif - if( IdChar((u8)*zSql) ){ - /* Keywords and unquoted identifiers */ - int nId; - for(nId=1; IdChar(zSql[nId]); nId++){} -#ifdef SQLITE_OMIT_TRIGGER - token = tkOTHER; -#else - switch( *zSql ){ - case 'c': case 'C': { - if( nId==6 && sqlite3StrNICmp(zSql, "create", 6)==0 ){ - token = tkCREATE; - }else{ - token = tkOTHER; - } - break; - } - case 't': case 'T': { - if( nId==7 && sqlite3StrNICmp(zSql, "trigger", 7)==0 ){ - token = tkTRIGGER; - }else if( nId==4 && sqlite3StrNICmp(zSql, "temp", 4)==0 ){ - token = tkTEMP; - }else if( nId==9 && sqlite3StrNICmp(zSql, "temporary", 9)==0 ){ - token = tkTEMP; - }else{ - token = tkOTHER; - } - break; - } - case 'e': case 'E': { - if( nId==3 && sqlite3StrNICmp(zSql, "end", 3)==0 ){ - token = tkEND; - }else -#ifndef SQLITE_OMIT_EXPLAIN - if( nId==7 && sqlite3StrNICmp(zSql, "explain", 7)==0 ){ - token = tkEXPLAIN; - }else -#endif - { - token = tkOTHER; - } - break; - } - default: { - token = tkOTHER; - break; - } - } -#endif /* SQLITE_OMIT_TRIGGER */ - zSql += nId-1; - }else{ - /* Operators and special symbols */ - token = tkOTHER; - } + + if( nList>0 ){ + *paPoslist = pList; + *piDocid = iDocid; + *pnPoslist = nList; break; } } - state = trans[state][token]; - zSql++; } - return state==1; + + return SQLITE_OK; } -#ifndef SQLITE_OMIT_UTF16 -/* -** This routine is the same as the sqlite3_complete() routine described -** above, except that the parameter is required to be UTF-16 encoded, not -** UTF-8. -*/ -SQLITE_API int SQLITE_STDCALL sqlite3_complete16(const void *zSql){ - sqlite3_value *pVal; - char const *zSql8; - int rc; +static int fts3SegReaderStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + const char *zTerm, /* Term searched for (or NULL) */ + int nTerm /* Length of zTerm in bytes */ +){ + int i; + int nSeg = pCsr->nSegment; -#ifndef SQLITE_OMIT_AUTOINIT - rc = sqlite3_initialize(); - if( rc ) return rc; -#endif - pVal = sqlite3ValueNew(0); - sqlite3ValueSetStr(pVal, -1, zSql, SQLITE_UTF16NATIVE, SQLITE_STATIC); - zSql8 = sqlite3ValueText(pVal, SQLITE_UTF8); - if( zSql8 ){ - rc = sqlite3_complete(zSql8); - }else{ - rc = SQLITE_NOMEM; + /* If the Fts3SegFilter defines a specific term (or term prefix) to search + ** for, then advance each segment iterator until it points to a term of + ** equal or greater value than the specified term. This prevents many + ** unnecessary merge/sort operations for the case where single segment + ** b-tree leaf nodes contain more than one term. + */ + for(i=0; pCsr->bRestart==0 && inSegment; i++){ + int res = 0; + Fts3SegReader *pSeg = pCsr->apSegment[i]; + do { + int rc = fts3SegReaderNext(p, pSeg, 0); + if( rc!=SQLITE_OK ) return rc; + }while( zTerm && (res = fts3SegReaderTermCmp(pSeg, zTerm, nTerm))<0 ); + + if( pSeg->bLookup && res!=0 ){ + fts3SegReaderSetEof(pSeg); + } } - sqlite3ValueFree(pVal); - return sqlite3ApiExit(0, rc); + fts3SegReaderSort(pCsr->apSegment, nSeg, nSeg, fts3SegReaderCmp); + + return SQLITE_OK; } -#endif /* SQLITE_OMIT_UTF16 */ -#endif /* SQLITE_OMIT_COMPLETE */ -/************** End of complete.c ********************************************/ -/************** Begin file main.c ********************************************/ -/* -** 2001 September 15 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** Main file for the SQLite library. The routines in this file -** implement the programmer interface to the library. Routines in -** other files are for internal use by SQLite and should not be -** accessed by users of the library. -*/ +SQLITE_PRIVATE int sqlite3Fts3SegReaderStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + Fts3SegFilter *pFilter /* Restrictions on range of iteration */ +){ + pCsr->pFilter = pFilter; + return fts3SegReaderStart(p, pCsr, pFilter->zTerm, pFilter->nTerm); +} -#ifdef SQLITE_ENABLE_FTS3 -/************** Include fts3.h in the middle of main.c ***********************/ -/************** Begin file fts3.h ********************************************/ -/* -** 2006 Oct 10 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This header file is used by programs that want to link against the -** FTS3 library. All it does is declare the sqlite3Fts3Init() interface. -*/ +SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + int iCol, /* Column to match on. */ + const char *zTerm, /* Term to iterate through a doclist for */ + int nTerm /* Number of bytes in zTerm */ +){ + int i; + int rc; + int nSegment = pCsr->nSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); -#if 0 -extern "C" { -#endif /* __cplusplus */ + assert( pCsr->pFilter==0 ); + assert( zTerm && nTerm>0 ); -SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db); + /* Advance each segment iterator until it points to the term zTerm/nTerm. */ + rc = fts3SegReaderStart(p, pCsr, zTerm, nTerm); + if( rc!=SQLITE_OK ) return rc; -#if 0 -} /* extern "C" */ -#endif /* __cplusplus */ + /* Determine how many of the segments actually point to zTerm/nTerm. */ + for(i=0; iapSegment[i]; + if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){ + break; + } + } + pCsr->nAdvance = i; + + /* Advance each of the segments to point to the first docid. */ + for(i=0; inAdvance; i++){ + rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]); + if( rc!=SQLITE_OK ) return rc; + } + fts3SegReaderSort(pCsr->apSegment, i, i, xCmp); + + assert( iCol<0 || iColnColumn ); + pCsr->iColFilter = iCol; + + return SQLITE_OK; +} -/************** End of fts3.h ************************************************/ -/************** Continuing where we left off in main.c ***********************/ -#endif -#ifdef SQLITE_ENABLE_RTREE -/************** Include rtree.h in the middle of main.c **********************/ -/************** Begin file rtree.h *******************************************/ /* -** 2008 May 26 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** This function is called on a MultiSegReader that has been started using +** sqlite3Fts3MsrIncrStart(). One or more calls to MsrIncrNext() may also +** have been made. Calling this function puts the MultiSegReader in such +** a state that if the next two calls are: ** -****************************************************************************** +** sqlite3Fts3SegReaderStart() +** sqlite3Fts3SegReaderStep() ** -** This header file is used by programs that want to link against the -** RTREE library. All it does is declare the sqlite3RtreeInit() interface. +** then the entire doclist for the term is available in +** MultiSegReader.aDoclist/nDoclist. */ +SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){ + int i; /* Used to iterate through segment-readers */ -#if 0 -extern "C" { -#endif /* __cplusplus */ + assert( pCsr->zTerm==0 ); + assert( pCsr->nTerm==0 ); + assert( pCsr->aDoclist==0 ); + assert( pCsr->nDoclist==0 ); -SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db); + pCsr->nAdvance = 0; + pCsr->bRestart = 1; + for(i=0; inSegment; i++){ + pCsr->apSegment[i]->pOffsetList = 0; + pCsr->apSegment[i]->nOffsetList = 0; + pCsr->apSegment[i]->iDocid = 0; + } -#if 0 -} /* extern "C" */ -#endif /* __cplusplus */ + return SQLITE_OK; +} -/************** End of rtree.h ***********************************************/ -/************** Continuing where we left off in main.c ***********************/ -#endif -#ifdef SQLITE_ENABLE_ICU -/************** Include sqliteicu.h in the middle of main.c ******************/ -/************** Begin file sqliteicu.h ***************************************/ -/* -** 2008 May 26 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This header file is used by programs that want to link against the -** ICU extension. All it does is declare the sqlite3IcuInit() interface. -*/ -#if 0 -extern "C" { -#endif /* __cplusplus */ +SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr /* Cursor object */ +){ + int rc = SQLITE_OK; -SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db); + int isIgnoreEmpty = (pCsr->pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY); + int isRequirePos = (pCsr->pFilter->flags & FTS3_SEGMENT_REQUIRE_POS); + int isColFilter = (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); + int isPrefix = (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX); + int isScan = (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN); + int isFirst = (pCsr->pFilter->flags & FTS3_SEGMENT_FIRST); -#if 0 -} /* extern "C" */ -#endif /* __cplusplus */ + Fts3SegReader **apSegment = pCsr->apSegment; + int nSegment = pCsr->nSegment; + Fts3SegFilter *pFilter = pCsr->pFilter; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + if( pCsr->nSegment==0 ) return SQLITE_OK; -/************** End of sqliteicu.h *******************************************/ -/************** Continuing where we left off in main.c ***********************/ -#endif + do { + int nMerge; + int i; + + /* Advance the first pCsr->nAdvance entries in the apSegment[] array + ** forward. Then sort the list in order of current term again. + */ + for(i=0; inAdvance; i++){ + Fts3SegReader *pSeg = apSegment[i]; + if( pSeg->bLookup ){ + fts3SegReaderSetEof(pSeg); + }else{ + rc = fts3SegReaderNext(p, pSeg, 0); + } + if( rc!=SQLITE_OK ) return rc; + } + fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp); + pCsr->nAdvance = 0; -#ifndef SQLITE_AMALGAMATION -/* IMPLEMENTATION-OF: R-46656-45156 The sqlite3_version[] string constant -** contains the text of SQLITE_VERSION macro. -*/ -SQLITE_API const char sqlite3_version[] = SQLITE_VERSION; -#endif + /* If all the seg-readers are at EOF, we're finished. return SQLITE_OK. */ + assert( rc==SQLITE_OK ); + if( apSegment[0]->aNode==0 ) break; -/* IMPLEMENTATION-OF: R-53536-42575 The sqlite3_libversion() function returns -** a pointer to the to the sqlite3_version[] string constant. -*/ -SQLITE_API const char *SQLITE_STDCALL sqlite3_libversion(void){ return sqlite3_version; } + pCsr->nTerm = apSegment[0]->nTerm; + pCsr->zTerm = apSegment[0]->zTerm; -/* IMPLEMENTATION-OF: R-63124-39300 The sqlite3_sourceid() function returns a -** pointer to a string constant whose value is the same as the -** SQLITE_SOURCE_ID C preprocessor macro. -*/ -SQLITE_API const char *SQLITE_STDCALL sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } + /* If this is a prefix-search, and if the term that apSegment[0] points + ** to does not share a suffix with pFilter->zTerm/nTerm, then all + ** required callbacks have been made. In this case exit early. + ** + ** Similarly, if this is a search for an exact match, and the first term + ** of segment apSegment[0] is not a match, exit early. + */ + if( pFilter->zTerm && !isScan ){ + if( pCsr->nTermnTerm + || (!isPrefix && pCsr->nTerm>pFilter->nTerm) + || memcmp(pCsr->zTerm, pFilter->zTerm, pFilter->nTerm) + ){ + break; + } + } -/* IMPLEMENTATION-OF: R-35210-63508 The sqlite3_libversion_number() function -** returns an integer equal to SQLITE_VERSION_NUMBER. -*/ -SQLITE_API int SQLITE_STDCALL sqlite3_libversion_number(void){ return SQLITE_VERSION_NUMBER; } + nMerge = 1; + while( nMergeaNode + && apSegment[nMerge]->nTerm==pCsr->nTerm + && 0==memcmp(pCsr->zTerm, apSegment[nMerge]->zTerm, pCsr->nTerm) + ){ + nMerge++; + } -/* IMPLEMENTATION-OF: R-20790-14025 The sqlite3_threadsafe() function returns -** zero if and only if SQLite was compiled with mutexing code omitted due to -** the SQLITE_THREADSAFE compile-time option being set to 0. -*/ -SQLITE_API int SQLITE_STDCALL sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; } + assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); + if( nMerge==1 + && !isIgnoreEmpty + && !isFirst + && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) + ){ + pCsr->nDoclist = apSegment[0]->nDoclist; + if( fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist); + pCsr->aDoclist = pCsr->aBuffer; + }else{ + pCsr->aDoclist = apSegment[0]->aDoclist; + } + if( rc==SQLITE_OK ) rc = SQLITE_ROW; + }else{ + int nDoclist = 0; /* Size of doclist */ + sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */ -/* -** When compiling the test fixture or with debugging enabled (on Win32), -** this variable being set to non-zero will cause OSTRACE macros to emit -** extra diagnostic information. -*/ -#ifdef SQLITE_HAVE_OS_TRACE -# ifndef SQLITE_DEBUG_OS_TRACE -# define SQLITE_DEBUG_OS_TRACE 0 -# endif - int sqlite3OSTrace = SQLITE_DEBUG_OS_TRACE; -#endif + /* The current term of the first nMerge entries in the array + ** of Fts3SegReader objects is the same. The doclists must be merged + ** and a single term returned with the merged doclist. + */ + for(i=0; ipOffsetList ){ + int j; /* Number of segments that share a docid */ + char *pList = 0; + int nList = 0; + int nByte; + sqlite3_int64 iDocid = apSegment[0]->iDocid; + fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); + j = 1; + while( jpOffsetList + && apSegment[j]->iDocid==iDocid + ){ + fts3SegReaderNextDocid(p, apSegment[j], 0, 0); + j++; + } -#if !defined(SQLITE_OMIT_TRACE) && defined(SQLITE_ENABLE_IOTRACE) -/* -** If the following function pointer is not NULL and if -** SQLITE_ENABLE_IOTRACE is enabled, then messages describing -** I/O active are written using this function. These messages -** are intended for debugging activity only. -*/ -SQLITE_API void (SQLITE_CDECL *sqlite3IoTrace)(const char*, ...) = 0; -#endif + if( isColFilter ){ + fts3ColumnFilter(pFilter->iCol, 0, &pList, &nList); + } -/* -** If the following global variable points to a string which is the -** name of a directory, then that directory will be used to store -** temporary files. -** -** See also the "PRAGMA temp_store_directory" SQL command. -*/ -SQLITE_API char *sqlite3_temp_directory = 0; + if( !isIgnoreEmpty || nList>0 ){ -/* -** If the following global variable points to a string which is the -** name of a directory, then that directory will be used to store -** all database files specified with a relative pathname. -** -** See also the "PRAGMA data_store_directory" SQL command. -*/ -SQLITE_API char *sqlite3_data_directory = 0; + /* Calculate the 'docid' delta value to write into the merged + ** doclist. */ + sqlite3_int64 iDelta; + if( p->bDescIdx && nDoclist>0 ){ + iDelta = iPrev - iDocid; + }else{ + iDelta = iDocid - iPrev; + } + assert( iDelta>0 || (nDoclist==0 && iDelta==iDocid) ); + assert( nDoclist>0 || iDelta==iDocid ); -/* -** Initialize SQLite. -** -** This routine must be called to initialize the memory allocation, -** VFS, and mutex subsystems prior to doing any serious work with -** SQLite. But as long as you do not compile with SQLITE_OMIT_AUTOINIT -** this routine will be called automatically by key routines such as -** sqlite3_open(). -** -** This routine is a no-op except on its very first call for the process, -** or for the first call after a call to sqlite3_shutdown. -** -** The first thread to call this routine runs the initialization to -** completion. If subsequent threads call this routine before the first -** thread has finished the initialization process, then the subsequent -** threads must block until the first thread finishes with the initialization. -** -** The first thread might call this routine recursively. Recursive -** calls to this routine should not block, of course. Otherwise the -** initialization process would never complete. -** -** Let X be the first thread to enter this routine. Let Y be some other -** thread. Then while the initial invocation of this routine by X is -** incomplete, it is required that: -** -** * Calls to this routine from Y must block until the outer-most -** call by X completes. -** -** * Recursive calls to this routine from thread X return immediately -** without blocking. -*/ -SQLITE_API int SQLITE_STDCALL sqlite3_initialize(void){ - MUTEX_LOGIC( sqlite3_mutex *pMaster; ) /* The main static mutex */ - int rc; /* Result code */ -#ifdef SQLITE_EXTRA_INIT - int bRunExtraInit = 0; /* Extra initialization needed */ -#endif + nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0); + if( nDoclist+nByte>pCsr->nBuffer ){ + char *aNew; + pCsr->nBuffer = (nDoclist+nByte)*2; + aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer); + if( !aNew ){ + return SQLITE_NOMEM; + } + pCsr->aBuffer = aNew; + } -#ifdef SQLITE_OMIT_WSD - rc = sqlite3_wsd_init(4096, 24); - if( rc!=SQLITE_OK ){ - return rc; - } -#endif + if( isFirst ){ + char *a = &pCsr->aBuffer[nDoclist]; + int nWrite; + + nWrite = sqlite3Fts3FirstFilter(iDelta, pList, nList, a); + if( nWrite ){ + iPrev = iDocid; + nDoclist += nWrite; + } + }else{ + nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta); + iPrev = iDocid; + if( isRequirePos ){ + memcpy(&pCsr->aBuffer[nDoclist], pList, nList); + nDoclist += nList; + pCsr->aBuffer[nDoclist++] = '\0'; + } + } + } - /* If the following assert() fails on some obscure processor/compiler - ** combination, the work-around is to set the correct pointer - ** size at compile-time using -DSQLITE_PTRSIZE=n compile-time option */ - assert( SQLITE_PTRSIZE==sizeof(char*) ); + fts3SegReaderSort(apSegment, nMerge, j, xCmp); + } + if( nDoclist>0 ){ + pCsr->aDoclist = pCsr->aBuffer; + pCsr->nDoclist = nDoclist; + rc = SQLITE_ROW; + } + } + pCsr->nAdvance = nMerge; + }while( rc==SQLITE_OK ); - /* If SQLite is already completely initialized, then this call - ** to sqlite3_initialize() should be a no-op. But the initialization - ** must be complete. So isInit must not be set until the very end - ** of this routine. - */ - if( sqlite3GlobalConfig.isInit ) return SQLITE_OK; + return rc; +} - /* Make sure the mutex subsystem is initialized. If unable to - ** initialize the mutex subsystem, return early with the error. - ** If the system is so sick that we are unable to allocate a mutex, - ** there is not much SQLite is going to be able to do. - ** - ** The mutex subsystem must take care of serializing its own - ** initialization. - */ - rc = sqlite3MutexInit(); - if( rc ) return rc; - /* Initialize the malloc() system and the recursive pInitMutex mutex. - ** This operation is protected by the STATIC_MASTER mutex. Note that - ** MutexAlloc() is called for a static mutex prior to initializing the - ** malloc subsystem - this implies that the allocation of a static - ** mutex must not require support from the malloc subsystem. - */ - MUTEX_LOGIC( pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) - sqlite3_mutex_enter(pMaster); - sqlite3GlobalConfig.isMutexInit = 1; - if( !sqlite3GlobalConfig.isMallocInit ){ - rc = sqlite3MallocInit(); - } - if( rc==SQLITE_OK ){ - sqlite3GlobalConfig.isMallocInit = 1; - if( !sqlite3GlobalConfig.pInitMutex ){ - sqlite3GlobalConfig.pInitMutex = - sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); - if( sqlite3GlobalConfig.bCoreMutex && !sqlite3GlobalConfig.pInitMutex ){ - rc = SQLITE_NOMEM; - } +SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish( + Fts3MultiSegReader *pCsr /* Cursor object */ +){ + if( pCsr ){ + int i; + for(i=0; inSegment; i++){ + sqlite3Fts3SegReaderFree(pCsr->apSegment[i]); } - } - if( rc==SQLITE_OK ){ - sqlite3GlobalConfig.nRefInitMutex++; - } - sqlite3_mutex_leave(pMaster); + sqlite3_free(pCsr->apSegment); + sqlite3_free(pCsr->aBuffer); - /* If rc is not SQLITE_OK at this point, then either the malloc - ** subsystem could not be initialized or the system failed to allocate - ** the pInitMutex mutex. Return an error in either case. */ - if( rc!=SQLITE_OK ){ - return rc; + pCsr->nSegment = 0; + pCsr->apSegment = 0; + pCsr->aBuffer = 0; } +} - /* Do the rest of the initialization under the recursive mutex so - ** that we will be able to handle recursive calls into - ** sqlite3_initialize(). The recursive calls normally come through - ** sqlite3_os_init() when it invokes sqlite3_vfs_register(), but other - ** recursive calls might also be possible. - ** - ** IMPLEMENTATION-OF: R-00140-37445 SQLite automatically serializes calls - ** to the xInit method, so the xInit method need not be threadsafe. - ** - ** The following mutex is what serializes access to the appdef pcache xInit - ** methods. The sqlite3_pcache_methods.xInit() all is embedded in the - ** call to sqlite3PcacheInitialize(). - */ - sqlite3_mutex_enter(sqlite3GlobalConfig.pInitMutex); - if( sqlite3GlobalConfig.isInit==0 && sqlite3GlobalConfig.inProgress==0 ){ - FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); - sqlite3GlobalConfig.inProgress = 1; - memset(pHash, 0, sizeof(sqlite3GlobalFunctions)); - sqlite3RegisterGlobalFunctions(); - if( sqlite3GlobalConfig.isPCacheInit==0 ){ - rc = sqlite3PcacheInitialize(); +/* +** Decode the "end_block" field, selected by column iCol of the SELECT +** statement passed as the first argument. +** +** The "end_block" field may contain either an integer, or a text field +** containing the text representation of two non-negative integers separated +** by one or more space (0x20) characters. In the first case, set *piEndBlock +** to the integer value and *pnByte to zero before returning. In the second, +** set *piEndBlock to the first value and *pnByte to the second. +*/ +static void fts3ReadEndBlockField( + sqlite3_stmt *pStmt, + int iCol, + i64 *piEndBlock, + i64 *pnByte +){ + const unsigned char *zText = sqlite3_column_text(pStmt, iCol); + if( zText ){ + int i; + int iMul = 1; + i64 iVal = 0; + for(i=0; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); } - if( rc==SQLITE_OK ){ - sqlite3GlobalConfig.isPCacheInit = 1; - rc = sqlite3OsInit(); + *piEndBlock = iVal; + while( zText[i]==' ' ) i++; + iVal = 0; + if( zText[i]=='-' ){ + i++; + iMul = -1; } - if( rc==SQLITE_OK ){ - sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage, - sqlite3GlobalConfig.szPage, sqlite3GlobalConfig.nPage); - sqlite3GlobalConfig.isInit = 1; -#ifdef SQLITE_EXTRA_INIT - bRunExtraInit = 1; -#endif + for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); } - sqlite3GlobalConfig.inProgress = 0; + *pnByte = (iVal * (i64)iMul); } - sqlite3_mutex_leave(sqlite3GlobalConfig.pInitMutex); +} - /* Go back under the static mutex and clean up the recursive - ** mutex to prevent a resource leak. - */ - sqlite3_mutex_enter(pMaster); - sqlite3GlobalConfig.nRefInitMutex--; - if( sqlite3GlobalConfig.nRefInitMutex<=0 ){ - assert( sqlite3GlobalConfig.nRefInitMutex==0 ); - sqlite3_mutex_free(sqlite3GlobalConfig.pInitMutex); - sqlite3GlobalConfig.pInitMutex = 0; - } - sqlite3_mutex_leave(pMaster); - /* The following is just a sanity check to make sure SQLite has - ** been compiled correctly. It is important to run this code, but - ** we don't want to run it too often and soak up CPU cycles for no - ** reason. So we run it once during initialization. - */ -#ifndef NDEBUG -#ifndef SQLITE_OMIT_FLOATING_POINT - /* This section of code's only "output" is via assert() statements. */ - if ( rc==SQLITE_OK ){ - u64 x = (((u64)1)<<63)-1; - double y; - assert(sizeof(x)==8); - assert(sizeof(x)==sizeof(y)); - memcpy(&y, &x, 8); - assert( sqlite3IsNaN(y) ); - } -#endif -#endif +/* +** A segment of size nByte bytes has just been written to absolute level +** iAbsLevel. Promote any segments that should be promoted as a result. +*/ +static int fts3PromoteSegments( + Fts3Table *p, /* FTS table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level just updated */ + sqlite3_int64 nByte /* Size of new segment at iAbsLevel */ +){ + int rc = SQLITE_OK; + sqlite3_stmt *pRange; - /* Do extra initialization steps requested by the SQLITE_EXTRA_INIT - ** compile-time option. - */ -#ifdef SQLITE_EXTRA_INIT - if( bRunExtraInit ){ - int SQLITE_EXTRA_INIT(const char*); - rc = SQLITE_EXTRA_INIT(0); - } -#endif + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE2, &pRange, 0); - return rc; -} + if( rc==SQLITE_OK ){ + int bOk = 0; + i64 iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1; + i64 nLimit = (nByte*3)/2; -/* -** Undo the effects of sqlite3_initialize(). Must not be called while -** there are outstanding database connections or memory allocations or -** while any part of SQLite is otherwise in use in any thread. This -** routine is not threadsafe. But it is safe to invoke this routine -** on when SQLite is already shut down. If SQLite is already shut down -** when this routine is invoked, then this routine is a harmless no-op. -*/ -SQLITE_API int SQLITE_STDCALL sqlite3_shutdown(void){ -#ifdef SQLITE_OMIT_WSD - int rc = sqlite3_wsd_init(4096, 24); - if( rc!=SQLITE_OK ){ - return rc; - } -#endif + /* Loop through all entries in the %_segdir table corresponding to + ** segments in this index on levels greater than iAbsLevel. If there is + ** at least one such segment, and it is possible to determine that all + ** such segments are smaller than nLimit bytes in size, they will be + ** promoted to level iAbsLevel. */ + sqlite3_bind_int64(pRange, 1, iAbsLevel+1); + sqlite3_bind_int64(pRange, 2, iLast); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + i64 nSize = 0, dummy; + fts3ReadEndBlockField(pRange, 2, &dummy, &nSize); + if( nSize<=0 || nSize>nLimit ){ + /* If nSize==0, then the %_segdir.end_block field does not not + ** contain a size value. This happens if it was written by an + ** old version of FTS. In this case it is not possible to determine + ** the size of the segment, and so segment promotion does not + ** take place. */ + bOk = 0; + break; + } + bOk = 1; + } + rc = sqlite3_reset(pRange); - if( sqlite3GlobalConfig.isInit ){ -#ifdef SQLITE_EXTRA_SHUTDOWN - void SQLITE_EXTRA_SHUTDOWN(void); - SQLITE_EXTRA_SHUTDOWN(); -#endif - sqlite3_os_end(); - sqlite3_reset_auto_extension(); - sqlite3GlobalConfig.isInit = 0; - } - if( sqlite3GlobalConfig.isPCacheInit ){ - sqlite3PcacheShutdown(); - sqlite3GlobalConfig.isPCacheInit = 0; - } - if( sqlite3GlobalConfig.isMallocInit ){ - sqlite3MallocEnd(); - sqlite3GlobalConfig.isMallocInit = 0; + if( bOk ){ + int iIdx = 0; + sqlite3_stmt *pUpdate1 = 0; + sqlite3_stmt *pUpdate2 = 0; -#ifndef SQLITE_OMIT_SHUTDOWN_DIRECTORIES - /* The heap subsystem has now been shutdown and these values are supposed - ** to be NULL or point to memory that was obtained from sqlite3_malloc(), - ** which would rely on that heap subsystem; therefore, make sure these - ** values cannot refer to heap memory that was just invalidated when the - ** heap subsystem was shutdown. This is only done if the current call to - ** this function resulted in the heap subsystem actually being shutdown. - */ - sqlite3_data_directory = 0; - sqlite3_temp_directory = 0; -#endif - } - if( sqlite3GlobalConfig.isMutexInit ){ - sqlite3MutexEnd(); - sqlite3GlobalConfig.isMutexInit = 0; + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL_IDX, &pUpdate1, 0); + } + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL, &pUpdate2, 0); + } + + if( rc==SQLITE_OK ){ + + /* Loop through all %_segdir entries for segments in this index with + ** levels equal to or greater than iAbsLevel. As each entry is visited, + ** updated it to set (level = -1) and (idx = N), where N is 0 for the + ** oldest segment in the range, 1 for the next oldest, and so on. + ** + ** In other words, move all segments being promoted to level -1, + ** setting the "idx" fields as appropriate to keep them in the same + ** order. The contents of level -1 (which is never used, except + ** transiently here), will be moved back to level iAbsLevel below. */ + sqlite3_bind_int64(pRange, 1, iAbsLevel); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + sqlite3_bind_int(pUpdate1, 1, iIdx++); + sqlite3_bind_int(pUpdate1, 2, sqlite3_column_int(pRange, 0)); + sqlite3_bind_int(pUpdate1, 3, sqlite3_column_int(pRange, 1)); + sqlite3_step(pUpdate1); + rc = sqlite3_reset(pUpdate1); + if( rc!=SQLITE_OK ){ + sqlite3_reset(pRange); + break; + } + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3_reset(pRange); + } + + /* Move level -1 to level iAbsLevel */ + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pUpdate2, 1, iAbsLevel); + sqlite3_step(pUpdate2); + rc = sqlite3_reset(pUpdate2); + } + } } - return SQLITE_OK; + + return rc; } /* -** This API allows applications to modify the global configuration of -** the SQLite library at run-time. +** Merge all level iLevel segments in the database into a single +** iLevel+1 segment. Or, if iLevel<0, merge all segments into a +** single segment with a level equal to the numerically largest level +** currently present in the database. ** -** This routine should only be called when there are no outstanding -** database connections or memory allocations. This routine is not -** threadsafe. Failure to heed these warnings can lead to unpredictable -** behavior. +** If this function is called with iLevel<0, but there is only one +** segment in the database, SQLITE_DONE is returned immediately. +** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, +** an SQLite error code is returned. */ -SQLITE_API int SQLITE_CDECL sqlite3_config(int op, ...){ - va_list ap; - int rc = SQLITE_OK; +static int fts3SegmentMerge( + Fts3Table *p, + int iLangid, /* Language id to merge */ + int iIndex, /* Index in p->aIndex[] to merge */ + int iLevel /* Level to merge */ +){ + int rc; /* Return code */ + int iIdx = 0; /* Index of new segment */ + sqlite3_int64 iNewLevel = 0; /* Level/index to create new segment at */ + SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ + Fts3SegFilter filter; /* Segment term filter condition */ + Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ + int bIgnoreEmpty = 0; /* True to ignore empty segments */ + i64 iMaxLevel = 0; /* Max level number for this index/langid */ - /* sqlite3_config() shall return SQLITE_MISUSE if it is invoked while - ** the SQLite library is in use. */ - if( sqlite3GlobalConfig.isInit ) return SQLITE_MISUSE_BKPT; + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( iLevel=0 && iIndexnIndex ); - va_start(ap, op); - switch( op ){ + rc = sqlite3Fts3SegReaderCursor(p, iLangid, iIndex, iLevel, 0, 0, 1, 0, &csr); + if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; - /* Mutex configuration options are only available in a threadsafe - ** compile. - */ -#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-54466-46756 */ - case SQLITE_CONFIG_SINGLETHREAD: { - /* EVIDENCE-OF: R-02748-19096 This option sets the threading mode to - ** Single-thread. */ - sqlite3GlobalConfig.bCoreMutex = 0; /* Disable mutex on core */ - sqlite3GlobalConfig.bFullMutex = 0; /* Disable mutex on connections */ - break; - } -#endif -#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-20520-54086 */ - case SQLITE_CONFIG_MULTITHREAD: { - /* EVIDENCE-OF: R-14374-42468 This option sets the threading mode to - ** Multi-thread. */ - sqlite3GlobalConfig.bCoreMutex = 1; /* Enable mutex on core */ - sqlite3GlobalConfig.bFullMutex = 0; /* Disable mutex on connections */ - break; - } -#endif -#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-59593-21810 */ - case SQLITE_CONFIG_SERIALIZED: { - /* EVIDENCE-OF: R-41220-51800 This option sets the threading mode to - ** Serialized. */ - sqlite3GlobalConfig.bCoreMutex = 1; /* Enable mutex on core */ - sqlite3GlobalConfig.bFullMutex = 1; /* Enable mutex on connections */ - break; - } -#endif -#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-63666-48755 */ - case SQLITE_CONFIG_MUTEX: { - /* Specify an alternative mutex implementation */ - sqlite3GlobalConfig.mutex = *va_arg(ap, sqlite3_mutex_methods*); - break; - } -#endif -#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-14450-37597 */ - case SQLITE_CONFIG_GETMUTEX: { - /* Retrieve the current mutex implementation */ - *va_arg(ap, sqlite3_mutex_methods*) = sqlite3GlobalConfig.mutex; - break; - } -#endif + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iMaxLevel); + if( rc!=SQLITE_OK ) goto finished; + } - case SQLITE_CONFIG_MALLOC: { - /* EVIDENCE-OF: R-55594-21030 The SQLITE_CONFIG_MALLOC option takes a - ** single argument which is a pointer to an instance of the - ** sqlite3_mem_methods structure. The argument specifies alternative - ** low-level memory allocation routines to be used in place of the memory - ** allocation routines built into SQLite. */ - sqlite3GlobalConfig.m = *va_arg(ap, sqlite3_mem_methods*); - break; - } - case SQLITE_CONFIG_GETMALLOC: { - /* EVIDENCE-OF: R-51213-46414 The SQLITE_CONFIG_GETMALLOC option takes a - ** single argument which is a pointer to an instance of the - ** sqlite3_mem_methods structure. The sqlite3_mem_methods structure is - ** filled with the currently defined memory allocation routines. */ - if( sqlite3GlobalConfig.m.xMalloc==0 ) sqlite3MemSetDefault(); - *va_arg(ap, sqlite3_mem_methods*) = sqlite3GlobalConfig.m; - break; - } - case SQLITE_CONFIG_MEMSTATUS: { - /* EVIDENCE-OF: R-61275-35157 The SQLITE_CONFIG_MEMSTATUS option takes - ** single argument of type int, interpreted as a boolean, which enables - ** or disables the collection of memory allocation statistics. */ - sqlite3GlobalConfig.bMemstat = va_arg(ap, int); - break; - } - case SQLITE_CONFIG_SCRATCH: { - /* EVIDENCE-OF: R-08404-60887 There are three arguments to - ** SQLITE_CONFIG_SCRATCH: A pointer an 8-byte aligned memory buffer from - ** which the scratch allocations will be drawn, the size of each scratch - ** allocation (sz), and the maximum number of scratch allocations (N). */ - sqlite3GlobalConfig.pScratch = va_arg(ap, void*); - sqlite3GlobalConfig.szScratch = va_arg(ap, int); - sqlite3GlobalConfig.nScratch = va_arg(ap, int); - break; - } - case SQLITE_CONFIG_PAGECACHE: { - /* EVIDENCE-OF: R-31408-40510 There are three arguments to - ** SQLITE_CONFIG_PAGECACHE: A pointer to 8-byte aligned memory, the size - ** of each page buffer (sz), and the number of pages (N). */ - sqlite3GlobalConfig.pPage = va_arg(ap, void*); - sqlite3GlobalConfig.szPage = va_arg(ap, int); - sqlite3GlobalConfig.nPage = va_arg(ap, int); - break; - } - case SQLITE_CONFIG_PCACHE_HDRSZ: { - /* EVIDENCE-OF: R-39100-27317 The SQLITE_CONFIG_PCACHE_HDRSZ option takes - ** a single parameter which is a pointer to an integer and writes into - ** that integer the number of extra bytes per page required for each page - ** in SQLITE_CONFIG_PAGECACHE. */ - *va_arg(ap, int*) = - sqlite3HeaderSizeBtree() + - sqlite3HeaderSizePcache() + - sqlite3HeaderSizePcache1(); - break; + if( iLevel==FTS3_SEGCURSOR_ALL ){ + /* This call is to merge all segments in the database to a single + ** segment. The level of the new segment is equal to the numerically + ** greatest segment level currently present in the database for this + ** index. The idx of the new segment is always 0. */ + if( csr.nSegment==1 ){ + rc = SQLITE_DONE; + goto finished; } + iNewLevel = iMaxLevel; + bIgnoreEmpty = 1; - case SQLITE_CONFIG_PCACHE: { - /* no-op */ - break; - } - case SQLITE_CONFIG_GETPCACHE: { - /* now an error */ - rc = SQLITE_ERROR; - break; - } + }else{ + /* This call is to merge all segments at level iLevel. find the next + ** available segment index at level iLevel+1. The call to + ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to + ** a single iLevel+2 segment if necessary. */ + assert( FTS3_SEGCURSOR_PENDING==-1 ); + iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, iLevel+1); + rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx); + bIgnoreEmpty = (iLevel!=FTS3_SEGCURSOR_PENDING) && (iNewLevel>iMaxLevel); + } + if( rc!=SQLITE_OK ) goto finished; - case SQLITE_CONFIG_PCACHE2: { - /* EVIDENCE-OF: R-63325-48378 The SQLITE_CONFIG_PCACHE2 option takes a - ** single argument which is a pointer to an sqlite3_pcache_methods2 - ** object. This object specifies the interface to a custom page cache - ** implementation. */ - sqlite3GlobalConfig.pcache2 = *va_arg(ap, sqlite3_pcache_methods2*); - break; - } - case SQLITE_CONFIG_GETPCACHE2: { - /* EVIDENCE-OF: R-22035-46182 The SQLITE_CONFIG_GETPCACHE2 option takes a - ** single argument which is a pointer to an sqlite3_pcache_methods2 - ** object. SQLite copies of the current page cache implementation into - ** that object. */ - if( sqlite3GlobalConfig.pcache2.xInit==0 ){ - sqlite3PCacheSetDefault(); - } - *va_arg(ap, sqlite3_pcache_methods2*) = sqlite3GlobalConfig.pcache2; - break; - } + assert( csr.nSegment>0 ); + assert( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) ); + assert( iNewLevel(1<<12) ){ - /* cap min request size at 2^12 */ - sqlite3GlobalConfig.mnReq = (1<<12); - } + rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); + while( SQLITE_OK==rc ){ + rc = sqlite3Fts3SegReaderStep(p, &csr); + if( rc!=SQLITE_ROW ) break; + rc = fts3SegWriterAdd(p, &pWriter, 1, + csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist); + } + if( rc!=SQLITE_OK ) goto finished; + assert( pWriter || bIgnoreEmpty ); - if( sqlite3GlobalConfig.pHeap==0 ){ - /* EVIDENCE-OF: R-49920-60189 If the first pointer (the memory pointer) - ** is NULL, then SQLite reverts to using its default memory allocator - ** (the system malloc() implementation), undoing any prior invocation of - ** SQLITE_CONFIG_MALLOC. - ** - ** Setting sqlite3GlobalConfig.m to all zeros will cause malloc to - ** revert to its default implementation when sqlite3_initialize() is run - */ - memset(&sqlite3GlobalConfig.m, 0, sizeof(sqlite3GlobalConfig.m)); - }else{ - /* EVIDENCE-OF: R-61006-08918 If the memory pointer is not NULL then the - ** alternative memory allocator is engaged to handle all of SQLites - ** memory allocation needs. */ -#ifdef SQLITE_ENABLE_MEMSYS3 - sqlite3GlobalConfig.m = *sqlite3MemGetMemsys3(); -#endif -#ifdef SQLITE_ENABLE_MEMSYS5 - sqlite3GlobalConfig.m = *sqlite3MemGetMemsys5(); -#endif + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + rc = fts3DeleteSegdir( + p, iLangid, iIndex, iLevel, csr.apSegment, csr.nSegment + ); + if( rc!=SQLITE_OK ) goto finished; + } + if( pWriter ){ + rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); + if( rc==SQLITE_OK ){ + if( iLevel==FTS3_SEGCURSOR_PENDING || iNewLevelnLeafData); } - break; - } -#endif - - case SQLITE_CONFIG_LOOKASIDE: { - sqlite3GlobalConfig.szLookaside = va_arg(ap, int); - sqlite3GlobalConfig.nLookaside = va_arg(ap, int); - break; - } - - /* Record a pointer to the logger function and its first argument. - ** The default is NULL. Logging is disabled if the function pointer is - ** NULL. - */ - case SQLITE_CONFIG_LOG: { - /* MSVC is picky about pulling func ptrs from va lists. - ** http://support.microsoft.com/kb/47961 - ** sqlite3GlobalConfig.xLog = va_arg(ap, void(*)(void*,int,const char*)); - */ - typedef void(*LOGFUNC_t)(void*,int,const char*); - sqlite3GlobalConfig.xLog = va_arg(ap, LOGFUNC_t); - sqlite3GlobalConfig.pLogArg = va_arg(ap, void*); - break; } + } - /* EVIDENCE-OF: R-55548-33817 The compile-time setting for URI filenames - ** can be changed at start-time using the - ** sqlite3_config(SQLITE_CONFIG_URI,1) or - ** sqlite3_config(SQLITE_CONFIG_URI,0) configuration calls. - */ - case SQLITE_CONFIG_URI: { - /* EVIDENCE-OF: R-25451-61125 The SQLITE_CONFIG_URI option takes a single - ** argument of type int. If non-zero, then URI handling is globally - ** enabled. If the parameter is zero, then URI handling is globally - ** disabled. */ - sqlite3GlobalConfig.bOpenUri = va_arg(ap, int); - break; - } + finished: + fts3SegWriterFree(pWriter); + sqlite3Fts3SegReaderFinish(&csr); + return rc; +} - case SQLITE_CONFIG_COVERING_INDEX_SCAN: { - /* EVIDENCE-OF: R-36592-02772 The SQLITE_CONFIG_COVERING_INDEX_SCAN - ** option takes a single integer argument which is interpreted as a - ** boolean in order to enable or disable the use of covering indices for - ** full table scans in the query optimizer. */ - sqlite3GlobalConfig.bUseCis = va_arg(ap, int); - break; - } -#ifdef SQLITE_ENABLE_SQLLOG - case SQLITE_CONFIG_SQLLOG: { - typedef void(*SQLLOGFUNC_t)(void*, sqlite3*, const char*, int); - sqlite3GlobalConfig.xSqllog = va_arg(ap, SQLLOGFUNC_t); - sqlite3GlobalConfig.pSqllogArg = va_arg(ap, void *); - break; - } -#endif +/* +** Flush the contents of pendingTerms to level 0 segments. +*/ +SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ + int rc = SQLITE_OK; + int i; + + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + sqlite3Fts3PendingTermsClear(p); - case SQLITE_CONFIG_MMAP_SIZE: { - /* EVIDENCE-OF: R-58063-38258 SQLITE_CONFIG_MMAP_SIZE takes two 64-bit - ** integer (sqlite3_int64) values that are the default mmap size limit - ** (the default setting for PRAGMA mmap_size) and the maximum allowed - ** mmap size limit. */ - sqlite3_int64 szMmap = va_arg(ap, sqlite3_int64); - sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); - /* EVIDENCE-OF: R-53367-43190 If either argument to this option is - ** negative, then that argument is changed to its compile-time default. - ** - ** EVIDENCE-OF: R-34993-45031 The maximum allowed mmap size will be - ** silently truncated if necessary so that it does not exceed the - ** compile-time maximum mmap size set by the SQLITE_MAX_MMAP_SIZE - ** compile-time option. - */ - if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ){ - mxMmap = SQLITE_MAX_MMAP_SIZE; + /* Determine the auto-incr-merge setting if unknown. If enabled, + ** estimate the number of leaf blocks of content to be written + */ + if( rc==SQLITE_OK && p->bHasStat + && p->nAutoincrmerge==0xff && p->nLeafAdd>0 + ){ + sqlite3_stmt *pStmt = 0; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); + rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + p->nAutoincrmerge = sqlite3_column_int(pStmt, 0); + if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8; + }else if( rc==SQLITE_DONE ){ + p->nAutoincrmerge = 0; } - if( szMmap<0 ) szMmap = SQLITE_DEFAULT_MMAP_SIZE; - if( szMmap>mxMmap) szMmap = mxMmap; - sqlite3GlobalConfig.mxMmap = mxMmap; - sqlite3GlobalConfig.szMmap = szMmap; - break; + rc = sqlite3_reset(pStmt); } + } + return rc; +} -#if SQLITE_OS_WIN && defined(SQLITE_WIN32_MALLOC) /* IMP: R-04780-55815 */ - case SQLITE_CONFIG_WIN32_HEAPSIZE: { - /* EVIDENCE-OF: R-34926-03360 SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit - ** unsigned integer value that specifies the maximum size of the created - ** heap. */ - sqlite3GlobalConfig.nHeap = va_arg(ap, int); - break; - } -#endif +/* +** Encode N integers as varints into a blob. +*/ +static void fts3EncodeIntArray( + int N, /* The number of integers to encode */ + u32 *a, /* The integer values */ + char *zBuf, /* Write the BLOB here */ + int *pNBuf /* Write number of bytes if zBuf[] used here */ +){ + int i, j; + for(i=j=0; iiPrevDocid. The sizes are encoded as +** a blob of varints. +*/ +static void fts3InsertDocsize( + int *pRC, /* Result code */ + Fts3Table *p, /* Table into which to insert */ + u32 *aSz /* Sizes of each column, in tokens */ +){ + char *pBlob; /* The BLOB encoding of the document size */ + int nBlob; /* Number of bytes in the BLOB */ + sqlite3_stmt *pStmt; /* Statement used to insert the encoding */ + int rc; /* Result code from subfunctions */ + + if( *pRC ) return; + pBlob = sqlite3_malloc( 10*p->nColumn ); + if( pBlob==0 ){ + *pRC = SQLITE_NOMEM; + return; } - va_end(ap); - return rc; + fts3EncodeIntArray(p->nColumn, aSz, pBlob, &nBlob); + rc = fts3SqlStmt(p, SQL_REPLACE_DOCSIZE, &pStmt, 0); + if( rc ){ + sqlite3_free(pBlob); + *pRC = rc; + return; + } + sqlite3_bind_int64(pStmt, 1, p->iPrevDocid); + sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, sqlite3_free); + sqlite3_step(pStmt); + *pRC = sqlite3_reset(pStmt); } /* -** Set up the lookaside buffers for a database connection. -** Return SQLITE_OK on success. -** If lookaside is already active, return SQLITE_BUSY. +** Record 0 of the %_stat table contains a blob consisting of N varints, +** where N is the number of user defined columns in the fts3 table plus +** two. If nCol is the number of user defined columns, then values of the +** varints are set as follows: +** +** Varint 0: Total number of rows in the table. +** +** Varint 1..nCol: For each column, the total number of tokens stored in +** the column for all rows of the table. +** +** Varint 1+nCol: The total size, in bytes, of all text values in all +** columns of all rows of the table. ** -** The sz parameter is the number of bytes in each lookaside slot. -** The cnt parameter is the number of slots. If pStart is NULL the -** space for the lookaside memory is obtained from sqlite3_malloc(). -** If pStart is not NULL then it is sz*cnt bytes of memory to use for -** the lookaside memory. */ -static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ - void *pStart; - if( db->lookaside.nOut ){ - return SQLITE_BUSY; +static void fts3UpdateDocTotals( + int *pRC, /* The result code */ + Fts3Table *p, /* Table being updated */ + u32 *aSzIns, /* Size increases */ + u32 *aSzDel, /* Size decreases */ + int nChng /* Change in the number of documents */ +){ + char *pBlob; /* Storage for BLOB written into %_stat */ + int nBlob; /* Size of BLOB written into %_stat */ + u32 *a; /* Array of integers that becomes the BLOB */ + sqlite3_stmt *pStmt; /* Statement for reading and writing */ + int i; /* Loop counter */ + int rc; /* Result code from subfunctions */ + + const int nStat = p->nColumn+2; + + if( *pRC ) return; + a = sqlite3_malloc( (sizeof(u32)+10)*nStat ); + if( a==0 ){ + *pRC = SQLITE_NOMEM; + return; } - /* Free any existing lookaside buffer for this handle before - ** allocating a new one so we don't have to have space for - ** both at the same time. - */ - if( db->lookaside.bMalloced ){ - sqlite3_free(db->lookaside.pStart); + pBlob = (char*)&a[nStat]; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); + if( rc ){ + sqlite3_free(a); + *pRC = rc; + return; } - /* The size of a lookaside slot after ROUNDDOWN8 needs to be larger - ** than a pointer to be useful. - */ - sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ - if( sz<=(int)sizeof(LookasideSlot*) ) sz = 0; - if( cnt<0 ) cnt = 0; - if( sz==0 || cnt==0 ){ - sz = 0; - pStart = 0; - }else if( pBuf==0 ){ - sqlite3BeginBenignMalloc(); - pStart = sqlite3Malloc( sz*cnt ); /* IMP: R-61949-35727 */ - sqlite3EndBenignMalloc(); - if( pStart ) cnt = sqlite3MallocSize(pStart)/sz; + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + fts3DecodeIntArray(nStat, a, + sqlite3_column_blob(pStmt, 0), + sqlite3_column_bytes(pStmt, 0)); }else{ - pStart = pBuf; + memset(a, 0, sizeof(u32)*(nStat) ); } - db->lookaside.pStart = pStart; - db->lookaside.pFree = 0; - db->lookaside.sz = (u16)sz; - if( pStart ){ - int i; - LookasideSlot *p; - assert( sz > (int)sizeof(LookasideSlot*) ); - p = (LookasideSlot*)pStart; - for(i=cnt-1; i>=0; i--){ - p->pNext = db->lookaside.pFree; - db->lookaside.pFree = p; - p = (LookasideSlot*)&((u8*)p)[sz]; - } - db->lookaside.pEnd = p; - db->lookaside.bEnabled = 1; - db->lookaside.bMalloced = pBuf==0 ?1:0; + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ){ + sqlite3_free(a); + *pRC = rc; + return; + } + if( nChng<0 && a[0]<(u32)(-nChng) ){ + a[0] = 0; }else{ - db->lookaside.pStart = db; - db->lookaside.pEnd = db; - db->lookaside.bEnabled = 0; - db->lookaside.bMalloced = 0; + a[0] += nChng; } - return SQLITE_OK; + for(i=0; inColumn+1; i++){ + u32 x = a[i+1]; + if( x+aSzIns[i] < aSzDel[i] ){ + x = 0; + }else{ + x = x + aSzIns[i] - aSzDel[i]; + } + a[i+1] = x; + } + fts3EncodeIntArray(nStat, a, pBlob, &nBlob); + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); + if( rc ){ + sqlite3_free(a); + *pRC = rc; + return; + } + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); + sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC); + sqlite3_step(pStmt); + *pRC = sqlite3_reset(pStmt); + sqlite3_free(a); } /* -** Return the mutex associated with a database connection. +** Merge the entire database so that there is one segment for each +** iIndex/iLangid combination. */ -SQLITE_API sqlite3_mutex *SQLITE_STDCALL sqlite3_db_mutex(sqlite3 *db){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; +static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ + int bSeenDone = 0; + int rc; + sqlite3_stmt *pAllLangid = 0; + + rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid); + sqlite3_bind_int(pAllLangid, 2, p->nIndex); + while( sqlite3_step(pAllLangid)==SQLITE_ROW ){ + int i; + int iLangid = sqlite3_column_int(pAllLangid, 0); + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, iLangid, i, FTS3_SEGCURSOR_ALL); + if( rc==SQLITE_DONE ){ + bSeenDone = 1; + rc = SQLITE_OK; + } + } + } + rc2 = sqlite3_reset(pAllLangid); + if( rc==SQLITE_OK ) rc = rc2; } -#endif - return db->mutex; + + sqlite3Fts3SegmentsClose(p); + sqlite3Fts3PendingTermsClear(p); + + return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc; } /* -** Free up as much memory as we can from the given database -** connection. +** This function is called when the user executes the following statement: +** +** INSERT INTO () VALUES('rebuild'); +** +** The entire FTS index is discarded and rebuilt. If the table is one +** created using the content=xxx option, then the new index is based on +** the current contents of the xxx table. Otherwise, it is rebuilt based +** on the contents of the %_content table. */ -SQLITE_API int SQLITE_STDCALL sqlite3_db_release_memory(sqlite3 *db){ - int i; +static int fts3DoRebuild(Fts3Table *p){ + int rc; /* Return Code */ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; -#endif - sqlite3_mutex_enter(db->mutex); - sqlite3BtreeEnterAll(db); - for(i=0; inDb; i++){ - Btree *pBt = db->aDb[i].pBt; - if( pBt ){ - Pager *pPager = sqlite3BtreePager(pBt); - sqlite3PagerShrink(pPager); + rc = fts3DeleteAll(p, 0); + if( rc==SQLITE_OK ){ + u32 *aSz = 0; + u32 *aSzIns = 0; + u32 *aSzDel = 0; + sqlite3_stmt *pStmt = 0; + int nEntry = 0; + + /* Compose and prepare an SQL statement to loop through the content table */ + char *zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + + if( rc==SQLITE_OK ){ + int nByte = sizeof(u32) * (p->nColumn+1)*3; + aSz = (u32 *)sqlite3_malloc(nByte); + if( aSz==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(aSz, 0, nByte); + aSzIns = &aSz[p->nColumn+1]; + aSzDel = &aSzIns[p->nColumn+1]; + } + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + int iCol; + int iLangid = langidFromSelect(p, pStmt); + rc = fts3PendingTermsDocid(p, 0, iLangid, sqlite3_column_int64(pStmt, 0)); + memset(aSz, 0, sizeof(aSz[0]) * (p->nColumn+1)); + for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ + if( p->abNotindexed[iCol]==0 ){ + const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1); + rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]); + aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1); + } + } + if( p->bHasDocsize ){ + fts3InsertDocsize(&rc, p, aSz); + } + if( rc!=SQLITE_OK ){ + sqlite3_finalize(pStmt); + pStmt = 0; + }else{ + nEntry++; + for(iCol=0; iCol<=p->nColumn; iCol++){ + aSzIns[iCol] += aSz[iCol]; + } + } + } + if( p->bFts4 ){ + fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nEntry); } - } - sqlite3BtreeLeaveAll(db); - sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; -} + sqlite3_free(aSz); -/* -** Configuration settings for an individual database connection -*/ -SQLITE_API int SQLITE_CDECL sqlite3_db_config(sqlite3 *db, int op, ...){ - va_list ap; - int rc; - va_start(ap, op); - switch( op ){ - case SQLITE_DBCONFIG_LOOKASIDE: { - void *pBuf = va_arg(ap, void*); /* IMP: R-26835-10964 */ - int sz = va_arg(ap, int); /* IMP: R-47871-25994 */ - int cnt = va_arg(ap, int); /* IMP: R-04460-53386 */ - rc = setupLookaside(db, pBuf, sz, cnt); - break; - } - default: { - static const struct { - int op; /* The opcode */ - u32 mask; /* Mask of the bit in sqlite3.flags to set/clear */ - } aFlagOp[] = { - { SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_ForeignKeys }, - { SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_EnableTrigger }, - }; - unsigned int i; - rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ - for(i=0; iflags; - if( onoff>0 ){ - db->flags |= aFlagOp[i].mask; - }else if( onoff==0 ){ - db->flags &= ~aFlagOp[i].mask; - } - if( oldFlags!=db->flags ){ - sqlite3ExpirePreparedStatements(db); - } - if( pRes ){ - *pRes = (db->flags & aFlagOp[i].mask)!=0; - } - rc = SQLITE_OK; - break; - } + if( pStmt ){ + int rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ){ + rc = rc2; } - break; } } - va_end(ap); + return rc; } /* -** Return true if the buffer z[0..n-1] contains all spaces. -*/ -static int allSpaces(const char *z, int n){ - while( n>0 && z[n-1]==' ' ){ n--; } - return n==0; -} - -/* -** This is the default collating function named "BINARY" which is always -** available. -** -** If the padFlag argument is not NULL then space padding at the end -** of strings is ignored. This implements the RTRIM collation. +** This function opens a cursor used to read the input data for an +** incremental merge operation. Specifically, it opens a cursor to scan +** the oldest nSeg segments (idx=0 through idx=(nSeg-1)) in absolute +** level iAbsLevel. */ -static int binCollFunc( - void *padFlag, - int nKey1, const void *pKey1, - int nKey2, const void *pKey2 +static int fts3IncrmergeCsr( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level to open */ + int nSeg, /* Number of segments to merge */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ ){ - int rc, n; - n = nKey1apSegment[] */ + + /* Allocate space for the Fts3MultiSegReader.aCsr[] array */ + memset(pCsr, 0, sizeof(*pCsr)); + nByte = sizeof(Fts3SegReader *) * nSeg; + pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte); + + if( pCsr->apSegment==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pCsr->apSegment, 0, nByte); + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); + } + if( rc==SQLITE_OK ){ + int i; + int rc2; + sqlite3_bind_int64(pStmt, 1, iAbsLevel); + assert( pCsr->nSegment==0 ); + for(i=0; rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW && iapSegment[i] + ); + pCsr->nSegment++; } + rc2 = sqlite3_reset(pStmt); + if( rc==SQLITE_OK ) rc = rc2; } + return rc; } +typedef struct IncrmergeWriter IncrmergeWriter; +typedef struct NodeWriter NodeWriter; +typedef struct Blob Blob; +typedef struct NodeReader NodeReader; + /* -** Another built-in collating sequence: NOCASE. -** -** This collating sequence is intended to be used for "case independent -** comparison". SQLite's knowledge of upper and lower case equivalents -** extends only to the 26 characters used in the English language. +** An instance of the following structure is used as a dynamic buffer +** to build up nodes or other blobs of data in. ** -** At the moment there is only a UTF-8 implementation. +** The function blobGrowBuffer() is used to extend the allocation. */ -static int nocaseCollatingFunc( - void *NotUsed, - int nKey1, const void *pKey1, - int nKey2, const void *pKey2 -){ - int r = sqlite3StrNICmp( - (const char *)pKey1, (const char *)pKey2, (nKey1=n) */ +}; /* -** Return the ROWID of the most recent insert +** This structure is used to build up buffers containing segment b-tree +** nodes (blocks). */ -SQLITE_API sqlite_int64 SQLITE_STDCALL sqlite3_last_insert_rowid(sqlite3 *db){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif - return db->lastRowid; -} +struct NodeWriter { + sqlite3_int64 iBlock; /* Current block id */ + Blob key; /* Last key written to the current block */ + Blob block; /* Current block image */ +}; /* -** Return the number of changes in the most recent call to sqlite3_exec(). +** An object of this type contains the state required to create or append +** to an appendable b-tree segment. */ -SQLITE_API int SQLITE_STDCALL sqlite3_changes(sqlite3 *db){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif - return db->nChange; -} +struct IncrmergeWriter { + int nLeafEst; /* Space allocated for leaf blocks */ + int nWork; /* Number of leaf pages flushed */ + sqlite3_int64 iAbsLevel; /* Absolute level of input segments */ + int iIdx; /* Index of *output* segment in iAbsLevel+1 */ + sqlite3_int64 iStart; /* Block number of first allocated block */ + sqlite3_int64 iEnd; /* Block number of last allocated block */ + sqlite3_int64 nLeafData; /* Bytes of leaf page data so far */ + u8 bNoLeafData; /* If true, store 0 for segment size */ + NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT]; +}; /* -** Return the number of changes since the database handle was opened. +** An object of the following type is used to read data from a single +** FTS segment node. See the following functions: +** +** nodeReaderInit() +** nodeReaderNext() +** nodeReaderRelease() */ -SQLITE_API int SQLITE_STDCALL sqlite3_total_changes(sqlite3 *db){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif - return db->nTotalChange; -} +struct NodeReader { + const char *aNode; + int nNode; + int iOff; /* Current offset within aNode[] */ -/* -** Close all open savepoints. This function only manipulates fields of the -** database handle object, it does not close any savepoints that may be open -** at the b-tree/pager level. -*/ -SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *db){ - while( db->pSavepoint ){ - Savepoint *pTmp = db->pSavepoint; - db->pSavepoint = pTmp->pNext; - sqlite3DbFree(db, pTmp); - } - db->nSavepoint = 0; - db->nStatement = 0; - db->isTransactionSavepoint = 0; -} + /* Output variables. Containing the current node entry. */ + sqlite3_int64 iChild; /* Pointer to child node */ + Blob term; /* Current term */ + const char *aDoclist; /* Pointer to doclist */ + int nDoclist; /* Size of doclist in bytes */ +}; /* -** Invoke the destructor function associated with FuncDef p, if any. Except, -** if this is not the last copy of the function, do not invoke it. Multiple -** copies of a single function are created when create_function() is called -** with SQLITE_ANY as the encoding. +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, if the allocation at pBlob->a is not already at least nMin +** bytes in size, extend (realloc) it to be so. +** +** If an OOM error occurs, set *pRc to SQLITE_NOMEM and leave pBlob->a +** unmodified. Otherwise, if the allocation succeeds, update pBlob->nAlloc +** to reflect the new size of the pBlob->a[] buffer. */ -static void functionDestroy(sqlite3 *db, FuncDef *p){ - FuncDestructor *pDestructor = p->pDestructor; - if( pDestructor ){ - pDestructor->nRef--; - if( pDestructor->nRef==0 ){ - pDestructor->xDestroy(pDestructor->pUserData); - sqlite3DbFree(db, pDestructor); +static void blobGrowBuffer(Blob *pBlob, int nMin, int *pRc){ + if( *pRc==SQLITE_OK && nMin>pBlob->nAlloc ){ + int nAlloc = nMin; + char *a = (char *)sqlite3_realloc(pBlob->a, nAlloc); + if( a ){ + pBlob->nAlloc = nAlloc; + pBlob->a = a; + }else{ + *pRc = SQLITE_NOMEM; } } } /* -** Disconnect all sqlite3_vtab objects that belong to database connection -** db. This is called when db is being closed. +** Attempt to advance the node-reader object passed as the first argument to +** the next entry on the node. +** +** Return an error code if an error occurs (SQLITE_NOMEM is possible). +** Otherwise return SQLITE_OK. If there is no next entry on the node +** (e.g. because the current entry is the last) set NodeReader->aNode to +** NULL to indicate EOF. Otherwise, populate the NodeReader structure output +** variables for the new entry. */ -static void disconnectAllVtab(sqlite3 *db){ -#ifndef SQLITE_OMIT_VIRTUALTABLE - int i; - sqlite3BtreeEnterAll(db); - for(i=0; inDb; i++){ - Schema *pSchema = db->aDb[i].pSchema; - if( db->aDb[i].pSchema ){ - HashElem *p; - for(p=sqliteHashFirst(&pSchema->tblHash); p; p=sqliteHashNext(p)){ - Table *pTab = (Table *)sqliteHashData(p); - if( IsVirtual(pTab) ) sqlite3VtabDisconnect(db, pTab); +static int nodeReaderNext(NodeReader *p){ + int bFirst = (p->term.n==0); /* True for first term on the node */ + int nPrefix = 0; /* Bytes to copy from previous term */ + int nSuffix = 0; /* Bytes to append to the prefix */ + int rc = SQLITE_OK; /* Return code */ + + assert( p->aNode ); + if( p->iChild && bFirst==0 ) p->iChild++; + if( p->iOff>=p->nNode ){ + /* EOF */ + p->aNode = 0; + }else{ + if( bFirst==0 ){ + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nPrefix); + } + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix); + + blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc); + if( rc==SQLITE_OK ){ + memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix); + p->term.n = nPrefix+nSuffix; + p->iOff += nSuffix; + if( p->iChild==0 ){ + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist); + p->aDoclist = &p->aNode[p->iOff]; + p->iOff += p->nDoclist; } } } - sqlite3VtabUnlockList(db); - sqlite3BtreeLeaveAll(db); -#else - UNUSED_PARAMETER(db); -#endif + + assert( p->iOff<=p->nNode ); + + return rc; } /* -** Return TRUE if database connection db has unfinalized prepared -** statements or unfinished sqlite3_backup objects. +** Release all dynamic resources held by node-reader object *p. */ -static int connectionIsBusy(sqlite3 *db){ - int j; - assert( sqlite3_mutex_held(db->mutex) ); - if( db->pVdbe ) return 1; - for(j=0; jnDb; j++){ - Btree *pBt = db->aDb[j].pBt; - if( pBt && sqlite3BtreeIsInBackup(pBt) ) return 1; - } - return 0; +static void nodeReaderRelease(NodeReader *p){ + sqlite3_free(p->term.a); } /* -** Close an existing SQLite database +** Initialize a node-reader object to read the node in buffer aNode/nNode. +** +** If successful, SQLITE_OK is returned and the NodeReader object set to +** point to the first entry on the node (if any). Otherwise, an SQLite +** error code is returned. */ -static int sqlite3Close(sqlite3 *db, int forceZombie){ - if( !db ){ - /* EVIDENCE-OF: R-63257-11740 Calling sqlite3_close() or - ** sqlite3_close_v2() with a NULL pointer argument is a harmless no-op. */ - return SQLITE_OK; - } - if( !sqlite3SafetyCheckSickOrOk(db) ){ - return SQLITE_MISUSE_BKPT; - } - sqlite3_mutex_enter(db->mutex); - - /* Force xDisconnect calls on all virtual tables */ - disconnectAllVtab(db); - - /* If a transaction is open, the disconnectAllVtab() call above - ** will not have called the xDisconnect() method on any virtual - ** tables in the db->aVTrans[] array. The following sqlite3VtabRollback() - ** call will do so. We need to do this before the check for active - ** SQL statements below, as the v-table implementation may be storing - ** some prepared statements internally. - */ - sqlite3VtabRollback(db); - - /* Legacy behavior (sqlite3_close() behavior) is to return - ** SQLITE_BUSY if the connection can not be closed immediately. - */ - if( !forceZombie && connectionIsBusy(db) ){ - sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to close due to unfinalized " - "statements or unfinished backups"); - sqlite3_mutex_leave(db->mutex); - return SQLITE_BUSY; - } +static int nodeReaderInit(NodeReader *p, const char *aNode, int nNode){ + memset(p, 0, sizeof(NodeReader)); + p->aNode = aNode; + p->nNode = nNode; -#ifdef SQLITE_ENABLE_SQLLOG - if( sqlite3GlobalConfig.xSqllog ){ - /* Closing the handle. Fourth parameter is passed the value 2. */ - sqlite3GlobalConfig.xSqllog(sqlite3GlobalConfig.pSqllogArg, db, 0, 2); + /* Figure out if this is a leaf or an internal node. */ + if( p->aNode[0] ){ + /* An internal node. */ + p->iOff = 1 + sqlite3Fts3GetVarint(&p->aNode[1], &p->iChild); + }else{ + p->iOff = 1; } -#endif - /* Convert the connection into a zombie and then close it. - */ - db->magic = SQLITE_MAGIC_ZOMBIE; - sqlite3LeaveMutexAndCloseZombie(db); - return SQLITE_OK; + return nodeReaderNext(p); } /* -** Two variations on the public interface for closing a database -** connection. The sqlite3_close() version returns SQLITE_BUSY and -** leaves the connection option if there are unfinalized prepared -** statements or unfinished sqlite3_backups. The sqlite3_close_v2() -** version forces the connection to become a zombie if there are -** unclosed resources, and arranges for deallocation when the last -** prepare statement or sqlite3_backup closes. -*/ -SQLITE_API int SQLITE_STDCALL sqlite3_close(sqlite3 *db){ return sqlite3Close(db,0); } -SQLITE_API int SQLITE_STDCALL sqlite3_close_v2(sqlite3 *db){ return sqlite3Close(db,1); } - - -/* -** Close the mutex on database connection db. +** This function is called while writing an FTS segment each time a leaf o +** node is finished and written to disk. The key (zTerm/nTerm) is guaranteed +** to be greater than the largest key on the node just written, but smaller +** than or equal to the first key that will be written to the next leaf +** node. ** -** Furthermore, if database connection db is a zombie (meaning that there -** has been a prior call to sqlite3_close(db) or sqlite3_close_v2(db)) and -** every sqlite3_stmt has now been finalized and every sqlite3_backup has -** finished, then free all resources. +** The block id of the leaf node just written to disk may be found in +** (pWriter->aNodeWriter[0].iBlock) when this function is called. */ -SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ - HashElem *i; /* Hash table iterator */ - int j; - - /* If there are outstanding sqlite3_stmt or sqlite3_backup objects - ** or if the connection has not yet been closed by sqlite3_close_v2(), - ** then just leave the mutex and return. - */ - if( db->magic!=SQLITE_MAGIC_ZOMBIE || connectionIsBusy(db) ){ - sqlite3_mutex_leave(db->mutex); - return; - } +static int fts3IncrmergePush( + Fts3Table *p, /* Fts3 table handle */ + IncrmergeWriter *pWriter, /* Writer object */ + const char *zTerm, /* Term to write to internal node */ + int nTerm /* Bytes at zTerm */ +){ + sqlite3_int64 iPtr = pWriter->aNodeWriter[0].iBlock; + int iLayer; - /* If we reach this point, it means that the database connection has - ** closed all sqlite3_stmt and sqlite3_backup objects and has been - ** passed to sqlite3_close (meaning that it is a zombie). Therefore, - ** go ahead and free all resources. - */ + assert( nTerm>0 ); + for(iLayer=1; ALWAYS(iLayeraNodeWriter[iLayer]; + int rc = SQLITE_OK; + int nPrefix; + int nSuffix; + int nSpace; - /* If a transaction is open, roll it back. This also ensures that if - ** any database schemas have been modified by an uncommitted transaction - ** they are reset. And that the required b-tree mutex is held to make - ** the pager rollback and schema reset an atomic operation. */ - sqlite3RollbackAll(db, SQLITE_OK); + /* Figure out how much space the key will consume if it is written to + ** the current node of layer iLayer. Due to the prefix compression, + ** the space required changes depending on which node the key is to + ** be added to. */ + nPrefix = fts3PrefixCompress(pNode->key.a, pNode->key.n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + nSpace = sqlite3Fts3VarintLen(nPrefix); + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; - /* Free any outstanding Savepoint structures. */ - sqlite3CloseSavepoints(db); + if( pNode->key.n==0 || (pNode->block.n + nSpace)<=p->nNodeSize ){ + /* If the current node of layer iLayer contains zero keys, or if adding + ** the key to it will not cause it to grow to larger than nNodeSize + ** bytes in size, write the key here. */ - /* Close all database connections */ - for(j=0; jnDb; j++){ - struct Db *pDb = &db->aDb[j]; - if( pDb->pBt ){ - sqlite3BtreeClose(pDb->pBt); - pDb->pBt = 0; - if( j!=1 ){ - pDb->pSchema = 0; + Blob *pBlk = &pNode->block; + if( pBlk->n==0 ){ + blobGrowBuffer(pBlk, p->nNodeSize, &rc); + if( rc==SQLITE_OK ){ + pBlk->a[0] = (char)iLayer; + pBlk->n = 1 + sqlite3Fts3PutVarint(&pBlk->a[1], iPtr); + } } - } - } - /* Clear the TEMP schema separately and last */ - if( db->aDb[1].pSchema ){ - sqlite3SchemaClear(db->aDb[1].pSchema); - } - sqlite3VtabUnlockList(db); - - /* Free up the array of auxiliary databases */ - sqlite3CollapseDatabaseArray(db); - assert( db->nDb<=2 ); - assert( db->aDb==db->aDbStatic ); + blobGrowBuffer(pBlk, pBlk->n + nSpace, &rc); + blobGrowBuffer(&pNode->key, nTerm, &rc); - /* Tell the code in notify.c that the connection no longer holds any - ** locks and does not require any further unlock-notify callbacks. - */ - sqlite3ConnectionClosed(db); + if( rc==SQLITE_OK ){ + if( pNode->key.n ){ + pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nPrefix); + } + pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nSuffix); + memcpy(&pBlk->a[pBlk->n], &zTerm[nPrefix], nSuffix); + pBlk->n += nSuffix; - for(j=0; jaFunc.a); j++){ - FuncDef *pNext, *pHash, *p; - for(p=db->aFunc.a[j]; p; p=pHash){ - pHash = p->pHash; - while( p ){ - functionDestroy(db, p); - pNext = p->pNext; - sqlite3DbFree(db, p); - p = pNext; - } - } - } - for(i=sqliteHashFirst(&db->aCollSeq); i; i=sqliteHashNext(i)){ - CollSeq *pColl = (CollSeq *)sqliteHashData(i); - /* Invoke any destructors registered for collation sequence user data. */ - for(j=0; j<3; j++){ - if( pColl[j].xDel ){ - pColl[j].xDel(pColl[j].pUser); + memcpy(pNode->key.a, zTerm, nTerm); + pNode->key.n = nTerm; } - } - sqlite3DbFree(db, pColl); - } - sqlite3HashClear(&db->aCollSeq); -#ifndef SQLITE_OMIT_VIRTUALTABLE - for(i=sqliteHashFirst(&db->aModule); i; i=sqliteHashNext(i)){ - Module *pMod = (Module *)sqliteHashData(i); - if( pMod->xDestroy ){ - pMod->xDestroy(pMod->pAux); - } - sqlite3DbFree(db, pMod); - } - sqlite3HashClear(&db->aModule); -#endif + }else{ + /* Otherwise, flush the current node of layer iLayer to disk. + ** Then allocate a new, empty sibling node. The key will be written + ** into the parent of this node. */ + rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); - sqlite3Error(db, SQLITE_OK); /* Deallocates any cached error strings. */ - sqlite3ValueFree(db->pErr); - sqlite3CloseExtensions(db); -#if SQLITE_USER_AUTHENTICATION - sqlite3_free(db->auth.zAuthUser); - sqlite3_free(db->auth.zAuthPW); -#endif + assert( pNode->block.nAlloc>=p->nNodeSize ); + pNode->block.a[0] = (char)iLayer; + pNode->block.n = 1 + sqlite3Fts3PutVarint(&pNode->block.a[1], iPtr+1); - db->magic = SQLITE_MAGIC_ERROR; + iNextPtr = pNode->iBlock; + pNode->iBlock++; + pNode->key.n = 0; + } - /* The temp-database schema is allocated differently from the other schema - ** objects (using sqliteMalloc() directly, instead of sqlite3BtreeSchema()). - ** So it needs to be freed here. Todo: Why not roll the temp schema into - ** the same sqliteMalloc() as the one that allocates the database - ** structure? - */ - sqlite3DbFree(db, db->aDb[1].pSchema); - sqlite3_mutex_leave(db->mutex); - db->magic = SQLITE_MAGIC_CLOSED; - sqlite3_mutex_free(db->mutex); - assert( db->lookaside.nOut==0 ); /* Fails on a lookaside memory leak */ - if( db->lookaside.bMalloced ){ - sqlite3_free(db->lookaside.pStart); + if( rc!=SQLITE_OK || iNextPtr==0 ) return rc; + iPtr = iNextPtr; } - sqlite3_free(db); + + assert( 0 ); + return 0; } /* -** Rollback all database files. If tripCode is not SQLITE_OK, then -** any write cursors are invalidated ("tripped" - as in "tripping a circuit -** breaker") and made to return tripCode if there are any further -** attempts to use that cursor. Read cursors remain open and valid -** but are "saved" in case the table pages are moved around. +** Append a term and (optionally) doclist to the FTS segment node currently +** stored in blob *pNode. The node need not contain any terms, but the +** header must be written before this function is called. +** +** A node header is a single 0x00 byte for a leaf node, or a height varint +** followed by the left-hand-child varint for an internal node. +** +** The term to be appended is passed via arguments zTerm/nTerm. For a +** leaf node, the doclist is passed as aDoclist/nDoclist. For an internal +** node, both aDoclist and nDoclist must be passed 0. +** +** If the size of the value in blob pPrev is zero, then this is the first +** term written to the node. Otherwise, pPrev contains a copy of the +** previous term. Before this function returns, it is updated to contain a +** copy of zTerm/nTerm. +** +** It is assumed that the buffer associated with pNode is already large +** enough to accommodate the new entry. The buffer associated with pPrev +** is extended by this function if requrired. +** +** If an error (i.e. OOM condition) occurs, an SQLite error code is +** returned. Otherwise, SQLITE_OK. */ -SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ - int i; - int inTrans = 0; - int schemaChange; - assert( sqlite3_mutex_held(db->mutex) ); - sqlite3BeginBenignMalloc(); +static int fts3AppendToNode( + Blob *pNode, /* Current node image to append to */ + Blob *pPrev, /* Buffer containing previous term written */ + const char *zTerm, /* New term to write */ + int nTerm, /* Size of zTerm in bytes */ + const char *aDoclist, /* Doclist (or NULL) to write */ + int nDoclist /* Size of aDoclist in bytes */ +){ + int rc = SQLITE_OK; /* Return code */ + int bFirst = (pPrev->n==0); /* True if this is the first term written */ + int nPrefix; /* Size of term prefix in bytes */ + int nSuffix; /* Size of term suffix in bytes */ - /* Obtain all b-tree mutexes before making any calls to BtreeRollback(). - ** This is important in case the transaction being rolled back has - ** modified the database schema. If the b-tree mutexes are not taken - ** here, then another shared-cache connection might sneak in between - ** the database rollback and schema reset, which can cause false - ** corruption reports in some cases. */ - sqlite3BtreeEnterAll(db); - schemaChange = (db->flags & SQLITE_InternChanges)!=0 && db->init.busy==0; + /* Node must have already been started. There must be a doclist for a + ** leaf node, and there must not be a doclist for an internal node. */ + assert( pNode->n>0 ); + assert( (pNode->a[0]=='\0')==(aDoclist!=0) ); - for(i=0; inDb; i++){ - Btree *p = db->aDb[i].pBt; - if( p ){ - if( sqlite3BtreeIsInTrans(p) ){ - inTrans = 1; - } - sqlite3BtreeRollback(p, tripCode, !schemaChange); - } + blobGrowBuffer(pPrev, nTerm, &rc); + if( rc!=SQLITE_OK ) return rc; + + nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + memcpy(pPrev->a, zTerm, nTerm); + pPrev->n = nTerm; + + if( bFirst==0 ){ + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nPrefix); } - sqlite3VtabRollback(db); - sqlite3EndBenignMalloc(); + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nSuffix); + memcpy(&pNode->a[pNode->n], &zTerm[nPrefix], nSuffix); + pNode->n += nSuffix; - if( (db->flags&SQLITE_InternChanges)!=0 && db->init.busy==0 ){ - sqlite3ExpirePreparedStatements(db); - sqlite3ResetAllSchemasOfConnection(db); + if( aDoclist ){ + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nDoclist); + memcpy(&pNode->a[pNode->n], aDoclist, nDoclist); + pNode->n += nDoclist; } - sqlite3BtreeLeaveAll(db); - /* Any deferred constraint violations have now been resolved. */ - db->nDeferredCons = 0; - db->nDeferredImmCons = 0; - db->flags &= ~SQLITE_DeferFKs; + assert( pNode->n<=pNode->nAlloc ); - /* If one has been configured, invoke the rollback-hook callback */ - if( db->xRollbackCallback && (inTrans || !db->autoCommit) ){ - db->xRollbackCallback(db->pRollbackArg); - } + return SQLITE_OK; } /* -** Return a static string containing the name corresponding to the error code -** specified in the argument. +** Append the current term and doclist pointed to by cursor pCsr to the +** appendable b-tree segment opened for writing by pWriter. +** +** Return SQLITE_OK if successful, or an SQLite error code otherwise. */ -#if defined(SQLITE_NEED_ERR_NAME) -SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ - const char *zName = 0; - int i, origRc = rc; - for(i=0; i<2 && zName==0; i++, rc &= 0xff){ - switch( rc ){ - case SQLITE_OK: zName = "SQLITE_OK"; break; - case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; - case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; - case SQLITE_PERM: zName = "SQLITE_PERM"; break; - case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; - case SQLITE_ABORT_ROLLBACK: zName = "SQLITE_ABORT_ROLLBACK"; break; - case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; - case SQLITE_BUSY_RECOVERY: zName = "SQLITE_BUSY_RECOVERY"; break; - case SQLITE_BUSY_SNAPSHOT: zName = "SQLITE_BUSY_SNAPSHOT"; break; - case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; - case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; - case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; - case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; - case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; - case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; - case SQLITE_READONLY_DBMOVED: zName = "SQLITE_READONLY_DBMOVED"; break; - case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; - case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; - case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; - case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; - case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; - case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; - case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; - case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; - case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; - case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; - case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; - case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; - case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; - case SQLITE_IOERR_CHECKRESERVEDLOCK: - zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; - case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; - case SQLITE_IOERR_CLOSE: zName = "SQLITE_IOERR_CLOSE"; break; - case SQLITE_IOERR_DIR_CLOSE: zName = "SQLITE_IOERR_DIR_CLOSE"; break; - case SQLITE_IOERR_SHMOPEN: zName = "SQLITE_IOERR_SHMOPEN"; break; - case SQLITE_IOERR_SHMSIZE: zName = "SQLITE_IOERR_SHMSIZE"; break; - case SQLITE_IOERR_SHMLOCK: zName = "SQLITE_IOERR_SHMLOCK"; break; - case SQLITE_IOERR_SHMMAP: zName = "SQLITE_IOERR_SHMMAP"; break; - case SQLITE_IOERR_SEEK: zName = "SQLITE_IOERR_SEEK"; break; - case SQLITE_IOERR_DELETE_NOENT: zName = "SQLITE_IOERR_DELETE_NOENT";break; - case SQLITE_IOERR_MMAP: zName = "SQLITE_IOERR_MMAP"; break; - case SQLITE_IOERR_GETTEMPPATH: zName = "SQLITE_IOERR_GETTEMPPATH"; break; - case SQLITE_IOERR_CONVPATH: zName = "SQLITE_IOERR_CONVPATH"; break; - case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; - case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; - case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; - case SQLITE_FULL: zName = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; - case SQLITE_CANTOPEN_NOTEMPDIR: zName = "SQLITE_CANTOPEN_NOTEMPDIR";break; - case SQLITE_CANTOPEN_ISDIR: zName = "SQLITE_CANTOPEN_ISDIR"; break; - case SQLITE_CANTOPEN_FULLPATH: zName = "SQLITE_CANTOPEN_FULLPATH"; break; - case SQLITE_CANTOPEN_CONVPATH: zName = "SQLITE_CANTOPEN_CONVPATH"; break; - case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; - case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; - case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break; - case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; - case SQLITE_CONSTRAINT_UNIQUE: zName = "SQLITE_CONSTRAINT_UNIQUE"; break; - case SQLITE_CONSTRAINT_TRIGGER: zName = "SQLITE_CONSTRAINT_TRIGGER";break; - case SQLITE_CONSTRAINT_FOREIGNKEY: - zName = "SQLITE_CONSTRAINT_FOREIGNKEY"; break; - case SQLITE_CONSTRAINT_CHECK: zName = "SQLITE_CONSTRAINT_CHECK"; break; - case SQLITE_CONSTRAINT_PRIMARYKEY: - zName = "SQLITE_CONSTRAINT_PRIMARYKEY"; break; - case SQLITE_CONSTRAINT_NOTNULL: zName = "SQLITE_CONSTRAINT_NOTNULL";break; - case SQLITE_CONSTRAINT_COMMITHOOK: - zName = "SQLITE_CONSTRAINT_COMMITHOOK"; break; - case SQLITE_CONSTRAINT_VTAB: zName = "SQLITE_CONSTRAINT_VTAB"; break; - case SQLITE_CONSTRAINT_FUNCTION: - zName = "SQLITE_CONSTRAINT_FUNCTION"; break; - case SQLITE_CONSTRAINT_ROWID: zName = "SQLITE_CONSTRAINT_ROWID"; break; - case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; - case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; - case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; - case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; - case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; - case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; - case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; - case SQLITE_ROW: zName = "SQLITE_ROW"; break; - case SQLITE_NOTICE: zName = "SQLITE_NOTICE"; break; - case SQLITE_NOTICE_RECOVER_WAL: zName = "SQLITE_NOTICE_RECOVER_WAL";break; - case SQLITE_NOTICE_RECOVER_ROLLBACK: - zName = "SQLITE_NOTICE_RECOVER_ROLLBACK"; break; - case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; - case SQLITE_WARNING_AUTOINDEX: zName = "SQLITE_WARNING_AUTOINDEX"; break; - case SQLITE_DONE: zName = "SQLITE_DONE"; break; +static int fts3IncrmergeAppend( + Fts3Table *p, /* Fts3 table handle */ + IncrmergeWriter *pWriter, /* Writer object */ + Fts3MultiSegReader *pCsr /* Cursor containing term and doclist */ +){ + const char *zTerm = pCsr->zTerm; + int nTerm = pCsr->nTerm; + const char *aDoclist = pCsr->aDoclist; + int nDoclist = pCsr->nDoclist; + int rc = SQLITE_OK; /* Return code */ + int nSpace; /* Total space in bytes required on leaf */ + int nPrefix; /* Size of prefix shared with previous term */ + int nSuffix; /* Size of suffix (nTerm - nPrefix) */ + NodeWriter *pLeaf; /* Object used to write leaf nodes */ + + pLeaf = &pWriter->aNodeWriter[0]; + nPrefix = fts3PrefixCompress(pLeaf->key.a, pLeaf->key.n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + + nSpace = sqlite3Fts3VarintLen(nPrefix); + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; + + /* If the current block is not empty, and if adding this term/doclist + ** to the current block would make it larger than Fts3Table.nNodeSize + ** bytes, write this block out to the database. */ + if( pLeaf->block.n>0 && (pLeaf->block.n + nSpace)>p->nNodeSize ){ + rc = fts3WriteSegment(p, pLeaf->iBlock, pLeaf->block.a, pLeaf->block.n); + pWriter->nWork++; + + /* Add the current term to the parent node. The term added to the + ** parent must: + ** + ** a) be greater than the largest term on the leaf node just written + ** to the database (still available in pLeaf->key), and + ** + ** b) be less than or equal to the term about to be added to the new + ** leaf node (zTerm/nTerm). + ** + ** In other words, it must be the prefix of zTerm 1 byte longer than + ** the common prefix (if any) of zTerm and pWriter->zTerm. + */ + if( rc==SQLITE_OK ){ + rc = fts3IncrmergePush(p, pWriter, zTerm, nPrefix+1); } + + /* Advance to the next output block */ + pLeaf->iBlock++; + pLeaf->key.n = 0; + pLeaf->block.n = 0; + + nSuffix = nTerm; + nSpace = 1; + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; } - if( zName==0 ){ - static char zBuf[50]; - sqlite3_snprintf(sizeof(zBuf), zBuf, "SQLITE_UNKNOWN(%d)", origRc); - zName = zBuf; - } - return zName; -} -#endif -/* -** Return a static string that describes the kind of error specified in the -** argument. -*/ -SQLITE_PRIVATE const char *sqlite3ErrStr(int rc){ - static const char* const aMsg[] = { - /* SQLITE_OK */ "not an error", - /* SQLITE_ERROR */ "SQL logic error or missing database", - /* SQLITE_INTERNAL */ 0, - /* SQLITE_PERM */ "access permission denied", - /* SQLITE_ABORT */ "callback requested query abort", - /* SQLITE_BUSY */ "database is locked", - /* SQLITE_LOCKED */ "database table is locked", - /* SQLITE_NOMEM */ "out of memory", - /* SQLITE_READONLY */ "attempt to write a readonly database", - /* SQLITE_INTERRUPT */ "interrupted", - /* SQLITE_IOERR */ "disk I/O error", - /* SQLITE_CORRUPT */ "database disk image is malformed", - /* SQLITE_NOTFOUND */ "unknown operation", - /* SQLITE_FULL */ "database or disk is full", - /* SQLITE_CANTOPEN */ "unable to open database file", - /* SQLITE_PROTOCOL */ "locking protocol", - /* SQLITE_EMPTY */ "table contains no data", - /* SQLITE_SCHEMA */ "database schema has changed", - /* SQLITE_TOOBIG */ "string or blob too big", - /* SQLITE_CONSTRAINT */ "constraint failed", - /* SQLITE_MISMATCH */ "datatype mismatch", - /* SQLITE_MISUSE */ "library routine called out of sequence", - /* SQLITE_NOLFS */ "large file support is disabled", - /* SQLITE_AUTH */ "authorization denied", - /* SQLITE_FORMAT */ "auxiliary database format error", - /* SQLITE_RANGE */ "bind or column index out of range", - /* SQLITE_NOTADB */ "file is encrypted or is not a database", - }; - const char *zErr = "unknown error"; - switch( rc ){ - case SQLITE_ABORT_ROLLBACK: { - zErr = "abort due to ROLLBACK"; - break; - } - default: { - rc &= 0xff; - if( ALWAYS(rc>=0) && rcnLeafData += nSpace; + blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc); + if( rc==SQLITE_OK ){ + if( pLeaf->block.n==0 ){ + pLeaf->block.n = 1; + pLeaf->block.a[0] = '\0'; } + rc = fts3AppendToNode( + &pLeaf->block, &pLeaf->key, zTerm, nTerm, aDoclist, nDoclist + ); } - return zErr; + + return rc; } /* -** This routine implements a busy callback that sleeps and tries -** again until a timeout value is reached. The timeout value is -** an integer number of milliseconds passed in as the first -** argument. +** This function is called to release all dynamic resources held by the +** merge-writer object pWriter, and if no error has occurred, to flush +** all outstanding node buffers held by pWriter to disk. +** +** If *pRc is not SQLITE_OK when this function is called, then no attempt +** is made to write any data to disk. Instead, this function serves only +** to release outstanding resources. +** +** Otherwise, if *pRc is initially SQLITE_OK and an error occurs while +** flushing buffers to disk, *pRc is set to an SQLite error code before +** returning. */ -static int sqliteDefaultBusyCallback( - void *ptr, /* Database connection */ - int count /* Number of times table has been busy */ +static void fts3IncrmergeRelease( + Fts3Table *p, /* FTS3 table handle */ + IncrmergeWriter *pWriter, /* Merge-writer object */ + int *pRc /* IN/OUT: Error code */ ){ -#if SQLITE_OS_WIN || HAVE_USLEEP - static const u8 delays[] = - { 1, 2, 5, 10, 15, 20, 25, 25, 25, 50, 50, 100 }; - static const u8 totals[] = - { 0, 1, 3, 8, 18, 33, 53, 78, 103, 128, 178, 228 }; -# define NDELAY ArraySize(delays) - sqlite3 *db = (sqlite3 *)ptr; - int timeout = db->busyTimeout; - int delay, prior; + int i; /* Used to iterate through non-root layers */ + int iRoot; /* Index of root in pWriter->aNodeWriter */ + NodeWriter *pRoot; /* NodeWriter for root node */ + int rc = *pRc; /* Error code */ - assert( count>=0 ); - if( count < NDELAY ){ - delay = delays[count]; - prior = totals[count]; - }else{ - delay = delays[NDELAY-1]; - prior = totals[NDELAY-1] + delay*(count-(NDELAY-1)); + /* Set iRoot to the index in pWriter->aNodeWriter[] of the output segment + ** root node. If the segment fits entirely on a single leaf node, iRoot + ** will be set to 0. If the root node is the parent of the leaves, iRoot + ** will be 1. And so on. */ + for(iRoot=FTS_MAX_APPENDABLE_HEIGHT-1; iRoot>=0; iRoot--){ + NodeWriter *pNode = &pWriter->aNodeWriter[iRoot]; + if( pNode->block.n>0 ) break; + assert( *pRc || pNode->block.nAlloc==0 ); + assert( *pRc || pNode->key.nAlloc==0 ); + sqlite3_free(pNode->block.a); + sqlite3_free(pNode->key.a); } - if( prior + delay > timeout ){ - delay = timeout - prior; - if( delay<=0 ) return 0; + + /* Empty output segment. This is a no-op. */ + if( iRoot<0 ) return; + + /* The entire output segment fits on a single node. Normally, this means + ** the node would be stored as a blob in the "root" column of the %_segdir + ** table. However, this is not permitted in this case. The problem is that + ** space has already been reserved in the %_segments table, and so the + ** start_block and end_block fields of the %_segdir table must be populated. + ** And, by design or by accident, released versions of FTS cannot handle + ** segments that fit entirely on the root node with start_block!=0. + ** + ** Instead, create a synthetic root node that contains nothing but a + ** pointer to the single content node. So that the segment consists of a + ** single leaf and a single interior (root) node. + ** + ** Todo: Better might be to defer allocating space in the %_segments + ** table until we are sure it is needed. + */ + if( iRoot==0 ){ + Blob *pBlock = &pWriter->aNodeWriter[1].block; + blobGrowBuffer(pBlock, 1 + FTS3_VARINT_MAX, &rc); + if( rc==SQLITE_OK ){ + pBlock->a[0] = 0x01; + pBlock->n = 1 + sqlite3Fts3PutVarint( + &pBlock->a[1], pWriter->aNodeWriter[0].iBlock + ); + } + iRoot = 1; } - sqlite3OsSleep(db->pVfs, delay*1000); - return 1; -#else - sqlite3 *db = (sqlite3 *)ptr; - int timeout = ((sqlite3 *)ptr)->busyTimeout; - if( (count+1)*1000 > timeout ){ - return 0; + pRoot = &pWriter->aNodeWriter[iRoot]; + + /* Flush all currently outstanding nodes to disk. */ + for(i=0; iaNodeWriter[i]; + if( pNode->block.n>0 && rc==SQLITE_OK ){ + rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); + } + sqlite3_free(pNode->block.a); + sqlite3_free(pNode->key.a); + } + + /* Write the %_segdir record. */ + if( rc==SQLITE_OK ){ + rc = fts3WriteSegdir(p, + pWriter->iAbsLevel+1, /* level */ + pWriter->iIdx, /* idx */ + pWriter->iStart, /* start_block */ + pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */ + pWriter->iEnd, /* end_block */ + (pWriter->bNoLeafData==0 ? pWriter->nLeafData : 0), /* end_block */ + pRoot->block.a, pRoot->block.n /* root */ + ); } - sqlite3OsSleep(db->pVfs, 1000000); - return 1; -#endif -} + sqlite3_free(pRoot->block.a); + sqlite3_free(pRoot->key.a); -/* -** Invoke the given busy handler. -** -** This routine is called when an operation failed with a lock. -** If this routine returns non-zero, the lock is retried. If it -** returns 0, the operation aborts with an SQLITE_BUSY error. -*/ -SQLITE_PRIVATE int sqlite3InvokeBusyHandler(BusyHandler *p){ - int rc; - if( NEVER(p==0) || p->xFunc==0 || p->nBusy<0 ) return 0; - rc = p->xFunc(p->pArg, p->nBusy); - if( rc==0 ){ - p->nBusy = -1; - }else{ - p->nBusy++; - } - return rc; + *pRc = rc; } /* -** This routine sets the busy callback for an Sqlite database to the -** given callback function with the given argument. +** Compare the term in buffer zLhs (size in bytes nLhs) with that in +** zRhs (size in bytes nRhs) using memcmp. If one term is a prefix of +** the other, it is considered to be smaller than the other. +** +** Return -ve if zLhs is smaller than zRhs, 0 if it is equal, or +ve +** if it is greater. */ -SQLITE_API int SQLITE_STDCALL sqlite3_busy_handler( - sqlite3 *db, - int (*xBusy)(void*,int), - void *pArg +static int fts3TermCmp( + const char *zLhs, int nLhs, /* LHS of comparison */ + const char *zRhs, int nRhs /* RHS of comparison */ ){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; -#endif - sqlite3_mutex_enter(db->mutex); - db->busyHandler.xFunc = xBusy; - db->busyHandler.pArg = pArg; - db->busyHandler.nBusy = 0; - db->busyTimeout = 0; - sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; -} + int nCmp = MIN(nLhs, nRhs); + int res; -#ifndef SQLITE_OMIT_PROGRESS_CALLBACK -/* -** This routine sets the progress callback for an Sqlite database to the -** given callback function with the given argument. The progress callback will -** be invoked every nOps opcodes. -*/ -SQLITE_API void SQLITE_STDCALL sqlite3_progress_handler( - sqlite3 *db, - int nOps, - int (*xProgress)(void*), - void *pArg -){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return; - } -#endif - sqlite3_mutex_enter(db->mutex); - if( nOps>0 ){ - db->xProgress = xProgress; - db->nProgressOps = (unsigned)nOps; - db->pProgressArg = pArg; - }else{ - db->xProgress = 0; - db->nProgressOps = 0; - db->pProgressArg = 0; - } - sqlite3_mutex_leave(db->mutex); + res = memcmp(zLhs, zRhs, nCmp); + if( res==0 ) res = nLhs - nRhs; + + return res; } -#endif /* -** This routine installs a default busy handler that waits for the -** specified number of milliseconds before returning 0. +** Query to see if the entry in the %_segments table with blockid iEnd is +** NULL. If no error occurs and the entry is NULL, set *pbRes 1 before +** returning. Otherwise, set *pbRes to 0. +** +** Or, if an error occurs while querying the database, return an SQLite +** error code. The final value of *pbRes is undefined in this case. +** +** This is used to test if a segment is an "appendable" segment. If it +** is, then a NULL entry has been inserted into the %_segments table +** with blockid %_segdir.end_block. */ -SQLITE_API int SQLITE_STDCALL sqlite3_busy_timeout(sqlite3 *db, int ms){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; -#endif - if( ms>0 ){ - sqlite3_busy_handler(db, sqliteDefaultBusyCallback, (void*)db); - db->busyTimeout = ms; - }else{ - sqlite3_busy_handler(db, 0, 0); - } - return SQLITE_OK; -} +static int fts3IsAppendable(Fts3Table *p, sqlite3_int64 iEnd, int *pbRes){ + int bRes = 0; /* Result to set *pbRes to */ + sqlite3_stmt *pCheck = 0; /* Statement to query database with */ + int rc; /* Return code */ -/* -** Cause any pending operation to stop at its earliest opportunity. -*/ -SQLITE_API void SQLITE_STDCALL sqlite3_interrupt(sqlite3 *db){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return; + rc = fts3SqlStmt(p, SQL_SEGMENT_IS_APPENDABLE, &pCheck, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pCheck, 1, iEnd); + if( SQLITE_ROW==sqlite3_step(pCheck) ) bRes = 1; + rc = sqlite3_reset(pCheck); } -#endif - db->u1.isInterrupted = 1; + + *pbRes = bRes; + return rc; } - /* -** This function is exactly the same as sqlite3_create_function(), except -** that it is designed to be called by internal code. The difference is -** that if a malloc() fails in sqlite3_create_function(), an error code -** is returned and the mallocFailed flag cleared. +** This function is called when initializing an incremental-merge operation. +** It checks if the existing segment with index value iIdx at absolute level +** (iAbsLevel+1) can be appended to by the incremental merge. If it can, the +** merge-writer object *pWriter is initialized to write to it. +** +** An existing segment can be appended to by an incremental merge if: +** +** * It was initially created as an appendable segment (with all required +** space pre-allocated), and +** +** * The first key read from the input (arguments zKey and nKey) is +** greater than the largest key currently stored in the potential +** output segment. */ -SQLITE_PRIVATE int sqlite3CreateFunc( - sqlite3 *db, - const char *zFunctionName, - int nArg, - int enc, - void *pUserData, - void (*xFunc)(sqlite3_context*,int,sqlite3_value **), - void (*xStep)(sqlite3_context*,int,sqlite3_value **), - void (*xFinal)(sqlite3_context*), - FuncDestructor *pDestructor +static int fts3IncrmergeLoad( + Fts3Table *p, /* Fts3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ + int iIdx, /* Index of candidate output segment */ + const char *zKey, /* First key to write */ + int nKey, /* Number of bytes in nKey */ + IncrmergeWriter *pWriter /* Populate this object */ ){ - FuncDef *p; - int nName; - int extraFlags; + int rc; /* Return code */ + sqlite3_stmt *pSelect = 0; /* SELECT to read %_segdir entry */ - assert( sqlite3_mutex_held(db->mutex) ); - if( zFunctionName==0 || - (xFunc && (xFinal || xStep)) || - (!xFunc && (xFinal && !xStep)) || - (!xFunc && (!xFinal && xStep)) || - (nArg<-1 || nArg>SQLITE_MAX_FUNCTION_ARG) || - (255<(nName = sqlite3Strlen30( zFunctionName))) ){ - return SQLITE_MISUSE_BKPT; - } + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pSelect, 0); + if( rc==SQLITE_OK ){ + sqlite3_int64 iStart = 0; /* Value of %_segdir.start_block */ + sqlite3_int64 iLeafEnd = 0; /* Value of %_segdir.leaves_end_block */ + sqlite3_int64 iEnd = 0; /* Value of %_segdir.end_block */ + const char *aRoot = 0; /* Pointer to %_segdir.root buffer */ + int nRoot = 0; /* Size of aRoot[] in bytes */ + int rc2; /* Return code from sqlite3_reset() */ + int bAppendable = 0; /* Set to true if segment is appendable */ - assert( SQLITE_FUNC_CONSTANT==SQLITE_DETERMINISTIC ); - extraFlags = enc & SQLITE_DETERMINISTIC; - enc &= (SQLITE_FUNC_ENCMASK|SQLITE_ANY); - -#ifndef SQLITE_OMIT_UTF16 - /* If SQLITE_UTF16 is specified as the encoding type, transform this - ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the - ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally. - ** - ** If SQLITE_ANY is specified, add three versions of the function - ** to the hash table. - */ - if( enc==SQLITE_UTF16 ){ - enc = SQLITE_UTF16NATIVE; - }else if( enc==SQLITE_ANY ){ - int rc; - rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF8|extraFlags, - pUserData, xFunc, xStep, xFinal, pDestructor); - if( rc==SQLITE_OK ){ - rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF16LE|extraFlags, - pUserData, xFunc, xStep, xFinal, pDestructor); - } - if( rc!=SQLITE_OK ){ - return rc; - } - enc = SQLITE_UTF16BE; - } -#else - enc = SQLITE_UTF8; -#endif - - /* Check if an existing function is being overridden or deleted. If so, - ** and there are active VMs, then return SQLITE_BUSY. If a function - ** is being overridden/deleted but there are no active VMs, allow the - ** operation to continue but invalidate all precompiled statements. - */ - p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 0); - if( p && (p->funcFlags & SQLITE_FUNC_ENCMASK)==enc && p->nArg==nArg ){ - if( db->nVdbeActive ){ - sqlite3ErrorWithMsg(db, SQLITE_BUSY, - "unable to delete/modify user-function due to active statements"); - assert( !db->mallocFailed ); - return SQLITE_BUSY; + /* Read the %_segdir entry for index iIdx absolute level (iAbsLevel+1) */ + sqlite3_bind_int64(pSelect, 1, iAbsLevel+1); + sqlite3_bind_int(pSelect, 2, iIdx); + if( sqlite3_step(pSelect)==SQLITE_ROW ){ + iStart = sqlite3_column_int64(pSelect, 1); + iLeafEnd = sqlite3_column_int64(pSelect, 2); + fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData); + if( pWriter->nLeafData<0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } + pWriter->bNoLeafData = (pWriter->nLeafData==0); + nRoot = sqlite3_column_bytes(pSelect, 4); + aRoot = sqlite3_column_blob(pSelect, 4); }else{ - sqlite3ExpirePreparedStatements(db); + return sqlite3_reset(pSelect); } - } - p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 1); - assert(p || db->mallocFailed); - if( !p ){ - return SQLITE_NOMEM; - } + /* Check for the zero-length marker in the %_segments table */ + rc = fts3IsAppendable(p, iEnd, &bAppendable); - /* If an older version of the function with a configured destructor is - ** being replaced invoke the destructor function here. */ - functionDestroy(db, p); + /* Check that zKey/nKey is larger than the largest key the candidate */ + if( rc==SQLITE_OK && bAppendable ){ + char *aLeaf = 0; + int nLeaf = 0; - if( pDestructor ){ - pDestructor->nRef++; - } - p->pDestructor = pDestructor; - p->funcFlags = (p->funcFlags & SQLITE_FUNC_ENCMASK) | extraFlags; - testcase( p->funcFlags & SQLITE_DETERMINISTIC ); - p->xFunc = xFunc; - p->xStep = xStep; - p->xFinalize = xFinal; - p->pUserData = pUserData; - p->nArg = (u16)nArg; - return SQLITE_OK; -} + rc = sqlite3Fts3ReadBlock(p, iLeafEnd, &aLeaf, &nLeaf, 0); + if( rc==SQLITE_OK ){ + NodeReader reader; + for(rc = nodeReaderInit(&reader, aLeaf, nLeaf); + rc==SQLITE_OK && reader.aNode; + rc = nodeReaderNext(&reader) + ){ + assert( reader.aNode ); + } + if( fts3TermCmp(zKey, nKey, reader.term.a, reader.term.n)<=0 ){ + bAppendable = 0; + } + nodeReaderRelease(&reader); + } + sqlite3_free(aLeaf); + } -/* -** Create new user functions. -*/ -SQLITE_API int SQLITE_STDCALL sqlite3_create_function( - sqlite3 *db, - const char *zFunc, - int nArg, - int enc, - void *p, - void (*xFunc)(sqlite3_context*,int,sqlite3_value **), - void (*xStep)(sqlite3_context*,int,sqlite3_value **), - void (*xFinal)(sqlite3_context*) -){ - return sqlite3_create_function_v2(db, zFunc, nArg, enc, p, xFunc, xStep, - xFinal, 0); -} + if( rc==SQLITE_OK && bAppendable ){ + /* It is possible to append to this segment. Set up the IncrmergeWriter + ** object to do so. */ + int i; + int nHeight = (int)aRoot[0]; + NodeWriter *pNode; -SQLITE_API int SQLITE_STDCALL sqlite3_create_function_v2( - sqlite3 *db, - const char *zFunc, - int nArg, - int enc, - void *p, - void (*xFunc)(sqlite3_context*,int,sqlite3_value **), - void (*xStep)(sqlite3_context*,int,sqlite3_value **), - void (*xFinal)(sqlite3_context*), - void (*xDestroy)(void *) -){ - int rc = SQLITE_ERROR; - FuncDestructor *pArg = 0; + pWriter->nLeafEst = (int)((iEnd - iStart) + 1)/FTS_MAX_APPENDABLE_HEIGHT; + pWriter->iStart = iStart; + pWriter->iEnd = iEnd; + pWriter->iAbsLevel = iAbsLevel; + pWriter->iIdx = iIdx; -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - return SQLITE_MISUSE_BKPT; - } -#endif - sqlite3_mutex_enter(db->mutex); - if( xDestroy ){ - pArg = (FuncDestructor *)sqlite3DbMallocZero(db, sizeof(FuncDestructor)); - if( !pArg ){ - xDestroy(p); - goto out; + for(i=nHeight+1; iaNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; + } + + pNode = &pWriter->aNodeWriter[nHeight]; + pNode->iBlock = pWriter->iStart + pWriter->nLeafEst*nHeight; + blobGrowBuffer(&pNode->block, MAX(nRoot, p->nNodeSize), &rc); + if( rc==SQLITE_OK ){ + memcpy(pNode->block.a, aRoot, nRoot); + pNode->block.n = nRoot; + } + + for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){ + NodeReader reader; + pNode = &pWriter->aNodeWriter[i]; + + rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n); + while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader); + blobGrowBuffer(&pNode->key, reader.term.n, &rc); + if( rc==SQLITE_OK ){ + memcpy(pNode->key.a, reader.term.a, reader.term.n); + pNode->key.n = reader.term.n; + if( i>0 ){ + char *aBlock = 0; + int nBlock = 0; + pNode = &pWriter->aNodeWriter[i-1]; + pNode->iBlock = reader.iChild; + rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock, 0); + blobGrowBuffer(&pNode->block, MAX(nBlock, p->nNodeSize), &rc); + if( rc==SQLITE_OK ){ + memcpy(pNode->block.a, aBlock, nBlock); + pNode->block.n = nBlock; + } + sqlite3_free(aBlock); + } + } + nodeReaderRelease(&reader); + } } - pArg->xDestroy = xDestroy; - pArg->pUserData = p; - } - rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xFunc, xStep, xFinal, pArg); - if( pArg && pArg->nRef==0 ){ - assert( rc!=SQLITE_OK ); - xDestroy(p); - sqlite3DbFree(db, pArg); + + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; } - out: - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); return rc; } -#ifndef SQLITE_OMIT_UTF16 -SQLITE_API int SQLITE_STDCALL sqlite3_create_function16( - sqlite3 *db, - const void *zFunctionName, - int nArg, - int eTextRep, - void *p, - void (*xFunc)(sqlite3_context*,int,sqlite3_value**), - void (*xStep)(sqlite3_context*,int,sqlite3_value**), - void (*xFinal)(sqlite3_context*) +/* +** Determine the largest segment index value that exists within absolute +** level iAbsLevel+1. If no error occurs, set *piIdx to this value plus +** one before returning SQLITE_OK. Or, if there are no segments at all +** within level iAbsLevel, set *piIdx to zero. +** +** If an error occurs, return an SQLite error code. The final value of +** *piIdx is undefined in this case. +*/ +static int fts3IncrmergeOutputIdx( + Fts3Table *p, /* FTS Table handle */ + sqlite3_int64 iAbsLevel, /* Absolute index of input segments */ + int *piIdx /* OUT: Next free index at iAbsLevel+1 */ ){ int rc; - char *zFunc8; + sqlite3_stmt *pOutputIdx = 0; /* SQL used to find output index */ + + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pOutputIdx, 1, iAbsLevel+1); + sqlite3_step(pOutputIdx); + *piIdx = sqlite3_column_int(pOutputIdx, 0); + rc = sqlite3_reset(pOutputIdx); + } -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) || zFunctionName==0 ) return SQLITE_MISUSE_BKPT; -#endif - sqlite3_mutex_enter(db->mutex); - assert( !db->mallocFailed ); - zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE); - rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xFunc, xStep, xFinal,0); - sqlite3DbFree(db, zFunc8); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); return rc; } -#endif - -/* -** Declare that a function has been overloaded by a virtual table. +/* +** Allocate an appendable output segment on absolute level iAbsLevel+1 +** with idx value iIdx. ** -** If the function already exists as a regular global function, then -** this routine is a no-op. If the function does not exist, then create -** a new one that always throws a run-time error. +** In the %_segdir table, a segment is defined by the values in three +** columns: ** -** When virtual tables intend to provide an overloaded function, they -** should call this routine to make sure the global function exists. -** A global function must exist in order for name resolution to work -** properly. +** start_block +** leaves_end_block +** end_block +** +** When an appendable segment is allocated, it is estimated that the +** maximum number of leaf blocks that may be required is the sum of the +** number of leaf blocks consumed by the input segments, plus the number +** of input segments, multiplied by two. This value is stored in stack +** variable nLeafEst. +** +** A total of 16*nLeafEst blocks are allocated when an appendable segment +** is created ((1 + end_block - start_block)==16*nLeafEst). The contiguous +** array of leaf nodes starts at the first block allocated. The array +** of interior nodes that are parents of the leaf nodes start at block +** (start_block + (1 + end_block - start_block) / 16). And so on. +** +** In the actual code below, the value "16" is replaced with the +** pre-processor macro FTS_MAX_APPENDABLE_HEIGHT. */ -SQLITE_API int SQLITE_STDCALL sqlite3_overload_function( - sqlite3 *db, - const char *zName, - int nArg +static int fts3IncrmergeWriter( + Fts3Table *p, /* Fts3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ + int iIdx, /* Index of new output segment */ + Fts3MultiSegReader *pCsr, /* Cursor that data will be read from */ + IncrmergeWriter *pWriter /* Populate this object */ ){ - int nName = sqlite3Strlen30(zName); - int rc = SQLITE_OK; + int rc; /* Return Code */ + int i; /* Iterator variable */ + int nLeafEst = 0; /* Blocks allocated for leaf nodes */ + sqlite3_stmt *pLeafEst = 0; /* SQL used to determine nLeafEst */ + sqlite3_stmt *pFirstBlock = 0; /* SQL used to determine first block */ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) || zName==0 || nArg<-2 ){ - return SQLITE_MISUSE_BKPT; + /* Calculate nLeafEst. */ + rc = fts3SqlStmt(p, SQL_MAX_LEAF_NODE_ESTIMATE, &pLeafEst, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pLeafEst, 1, iAbsLevel); + sqlite3_bind_int64(pLeafEst, 2, pCsr->nSegment); + if( SQLITE_ROW==sqlite3_step(pLeafEst) ){ + nLeafEst = sqlite3_column_int(pLeafEst, 0); + } + rc = sqlite3_reset(pLeafEst); } -#endif - sqlite3_mutex_enter(db->mutex); - if( sqlite3FindFunction(db, zName, nName, nArg, SQLITE_UTF8, 0)==0 ){ - rc = sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8, - 0, sqlite3InvalidFunction, 0, 0, 0); + if( rc!=SQLITE_OK ) return rc; + + /* Calculate the first block to use in the output segment */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pFirstBlock, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pFirstBlock) ){ + pWriter->iStart = sqlite3_column_int64(pFirstBlock, 0); + pWriter->iEnd = pWriter->iStart - 1; + pWriter->iEnd += nLeafEst * FTS_MAX_APPENDABLE_HEIGHT; + } + rc = sqlite3_reset(pFirstBlock); } - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; -} + if( rc!=SQLITE_OK ) return rc; -#ifndef SQLITE_OMIT_TRACE -/* -** Register a trace function. The pArg from the previously registered trace -** is returned. -** -** A NULL trace function means that no tracing is executes. A non-NULL -** trace is a pointer to a function that is invoked at the start of each -** SQL statement. -*/ -SQLITE_API void *SQLITE_STDCALL sqlite3_trace(sqlite3 *db, void (*xTrace)(void*,const char*), void *pArg){ - void *pOld; + /* Insert the marker in the %_segments table to make sure nobody tries + ** to steal the space just allocated. This is also used to identify + ** appendable segments. */ + rc = fts3WriteSegment(p, pWriter->iEnd, 0, 0); + if( rc!=SQLITE_OK ) return rc; -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; + pWriter->iAbsLevel = iAbsLevel; + pWriter->nLeafEst = nLeafEst; + pWriter->iIdx = iIdx; + + /* Set up the array of NodeWriter objects */ + for(i=0; iaNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; } -#endif - sqlite3_mutex_enter(db->mutex); - pOld = db->pTraceArg; - db->xTrace = xTrace; - db->pTraceArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pOld; + return SQLITE_OK; } + /* -** Register a profile function. The pArg from the previously registered -** profile function is returned. +** Remove an entry from the %_segdir table. This involves running the +** following two statements: ** -** A NULL profile function means that no profiling is executes. A non-NULL -** profile is a pointer to a function that is invoked at the conclusion of -** each SQL statement that is run. +** DELETE FROM %_segdir WHERE level = :iAbsLevel AND idx = :iIdx +** UPDATE %_segdir SET idx = idx - 1 WHERE level = :iAbsLevel AND idx > :iIdx +** +** The DELETE statement removes the specific %_segdir level. The UPDATE +** statement ensures that the remaining segments have contiguously allocated +** idx values. */ -SQLITE_API void *SQLITE_STDCALL sqlite3_profile( - sqlite3 *db, - void (*xProfile)(void*,const char*,sqlite_uint64), - void *pArg +static int fts3RemoveSegdirEntry( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level to delete from */ + int iIdx /* Index of %_segdir entry to delete */ ){ - void *pOld; + int rc; /* Return code */ + sqlite3_stmt *pDelete = 0; /* DELETE statement */ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_ENTRY, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, iAbsLevel); + sqlite3_bind_int(pDelete, 2, iIdx); + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); } -#endif - sqlite3_mutex_enter(db->mutex); - pOld = db->pProfileArg; - db->xProfile = xProfile; - db->pProfileArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pOld; + + return rc; } -#endif /* SQLITE_OMIT_TRACE */ /* -** Register a function to be invoked when a transaction commits. -** If the invoked function returns non-zero, then the commit becomes a -** rollback. +** One or more segments have just been removed from absolute level iAbsLevel. +** Update the 'idx' values of the remaining segments in the level so that +** the idx values are a contiguous sequence starting from 0. */ -SQLITE_API void *SQLITE_STDCALL sqlite3_commit_hook( - sqlite3 *db, /* Attach the hook to this database */ - int (*xCallback)(void*), /* Function to invoke on each commit */ - void *pArg /* Argument to the function */ +static int fts3RepackSegdirLevel( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel /* Absolute level to repack */ ){ - void *pOld; + int rc; /* Return code */ + int *aIdx = 0; /* Array of remaining idx values */ + int nIdx = 0; /* Valid entries in aIdx[] */ + int nAlloc = 0; /* Allocated size of aIdx[] */ + int i; /* Iterator variable */ + sqlite3_stmt *pSelect = 0; /* Select statement to read idx values */ + sqlite3_stmt *pUpdate = 0; /* Update statement to modify idx values */ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; + rc = fts3SqlStmt(p, SQL_SELECT_INDEXES, &pSelect, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int64(pSelect, 1, iAbsLevel); + while( SQLITE_ROW==sqlite3_step(pSelect) ){ + if( nIdx>=nAlloc ){ + int *aNew; + nAlloc += 16; + aNew = sqlite3_realloc(aIdx, nAlloc*sizeof(int)); + if( !aNew ){ + rc = SQLITE_NOMEM; + break; + } + aIdx = aNew; + } + aIdx[nIdx++] = sqlite3_column_int(pSelect, 0); + } + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; + } + + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_SHIFT_SEGDIR_ENTRY, &pUpdate, 0); + } + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pUpdate, 2, iAbsLevel); + } + + assert( p->bIgnoreSavepoint==0 ); + p->bIgnoreSavepoint = 1; + for(i=0; rc==SQLITE_OK && ibIgnoreSavepoint = 0; + + sqlite3_free(aIdx); + return rc; +} + +static void fts3StartNode(Blob *pNode, int iHeight, sqlite3_int64 iChild){ + pNode->a[0] = (char)iHeight; + if( iChild ){ + assert( pNode->nAlloc>=1+sqlite3Fts3VarintLen(iChild) ); + pNode->n = 1 + sqlite3Fts3PutVarint(&pNode->a[1], iChild); + }else{ + assert( pNode->nAlloc>=1 ); + pNode->n = 1; } -#endif - sqlite3_mutex_enter(db->mutex); - pOld = db->pCommitArg; - db->xCommitCallback = xCallback; - db->pCommitArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pOld; } /* -** Register a callback to be invoked each time a row is updated, -** inserted or deleted using this database connection. +** The first two arguments are a pointer to and the size of a segment b-tree +** node. The node may be a leaf or an internal node. +** +** This function creates a new node image in blob object *pNew by copying +** all terms that are greater than or equal to zTerm/nTerm (for leaf nodes) +** or greater than zTerm/nTerm (for internal nodes) from aNode/nNode. */ -SQLITE_API void *SQLITE_STDCALL sqlite3_update_hook( - sqlite3 *db, /* Attach the hook to this database */ - void (*xCallback)(void*,int,char const *,char const *,sqlite_int64), - void *pArg /* Argument to the function */ +static int fts3TruncateNode( + const char *aNode, /* Current node image */ + int nNode, /* Size of aNode in bytes */ + Blob *pNew, /* OUT: Write new node image here */ + const char *zTerm, /* Omit all terms smaller than this */ + int nTerm, /* Size of zTerm in bytes */ + sqlite3_int64 *piBlock /* OUT: Block number in next layer down */ ){ - void *pRet; + NodeReader reader; /* Reader object */ + Blob prev = {0, 0, 0}; /* Previous term written to new node */ + int rc = SQLITE_OK; /* Return code */ + int bLeaf = aNode[0]=='\0'; /* True for a leaf node */ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; + /* Allocate required output space */ + blobGrowBuffer(pNew, nNode, &rc); + if( rc!=SQLITE_OK ) return rc; + pNew->n = 0; + + /* Populate new node buffer */ + for(rc = nodeReaderInit(&reader, aNode, nNode); + rc==SQLITE_OK && reader.aNode; + rc = nodeReaderNext(&reader) + ){ + if( pNew->n==0 ){ + int res = fts3TermCmp(reader.term.a, reader.term.n, zTerm, nTerm); + if( res<0 || (bLeaf==0 && res==0) ) continue; + fts3StartNode(pNew, (int)aNode[0], reader.iChild); + *piBlock = reader.iChild; + } + rc = fts3AppendToNode( + pNew, &prev, reader.term.a, reader.term.n, + reader.aDoclist, reader.nDoclist + ); + if( rc!=SQLITE_OK ) break; } -#endif - sqlite3_mutex_enter(db->mutex); - pRet = db->pUpdateArg; - db->xUpdateCallback = xCallback; - db->pUpdateArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pRet; + if( pNew->n==0 ){ + fts3StartNode(pNew, (int)aNode[0], reader.iChild); + *piBlock = reader.iChild; + } + assert( pNew->n<=pNew->nAlloc ); + + nodeReaderRelease(&reader); + sqlite3_free(prev.a); + return rc; } /* -** Register a callback to be invoked each time a transaction is rolled -** back by this database connection. +** Remove all terms smaller than zTerm/nTerm from segment iIdx in absolute +** level iAbsLevel. This may involve deleting entries from the %_segments +** table, and modifying existing entries in both the %_segments and %_segdir +** tables. +** +** SQLITE_OK is returned if the segment is updated successfully. Or an +** SQLite error code otherwise. */ -SQLITE_API void *SQLITE_STDCALL sqlite3_rollback_hook( - sqlite3 *db, /* Attach the hook to this database */ - void (*xCallback)(void*), /* Callback function */ - void *pArg /* Argument to the function */ +static int fts3TruncateSegment( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of segment to modify */ + int iIdx, /* Index within level of segment to modify */ + const char *zTerm, /* Remove terms smaller than this */ + int nTerm /* Number of bytes in buffer zTerm */ ){ - void *pRet; + int rc = SQLITE_OK; /* Return code */ + Blob root = {0,0,0}; /* New root page image */ + Blob block = {0,0,0}; /* Buffer used for any other block */ + sqlite3_int64 iBlock = 0; /* Block id */ + sqlite3_int64 iNewStart = 0; /* New value for iStartBlock */ + sqlite3_int64 iOldStart = 0; /* Old value for iStartBlock */ + sqlite3_stmt *pFetch = 0; /* Statement used to fetch segdir */ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pFetch, 0); + if( rc==SQLITE_OK ){ + int rc2; /* sqlite3_reset() return code */ + sqlite3_bind_int64(pFetch, 1, iAbsLevel); + sqlite3_bind_int(pFetch, 2, iIdx); + if( SQLITE_ROW==sqlite3_step(pFetch) ){ + const char *aRoot = sqlite3_column_blob(pFetch, 4); + int nRoot = sqlite3_column_bytes(pFetch, 4); + iOldStart = sqlite3_column_int64(pFetch, 1); + rc = fts3TruncateNode(aRoot, nRoot, &root, zTerm, nTerm, &iBlock); + } + rc2 = sqlite3_reset(pFetch); + if( rc==SQLITE_OK ) rc = rc2; } -#endif - sqlite3_mutex_enter(db->mutex); - pRet = db->pRollbackArg; - db->xRollbackCallback = xCallback; - db->pRollbackArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pRet; -} -#ifndef SQLITE_OMIT_WAL -/* -** The sqlite3_wal_hook() callback registered by sqlite3_wal_autocheckpoint(). -** Invoke sqlite3_wal_checkpoint if the number of frames in the log file -** is greater than sqlite3.pWalArg cast to an integer (the value configured by -** wal_autocheckpoint()). -*/ -SQLITE_PRIVATE int sqlite3WalDefaultHook( - void *pClientData, /* Argument */ - sqlite3 *db, /* Connection */ - const char *zDb, /* Database */ - int nFrame /* Size of WAL */ -){ - if( nFrame>=SQLITE_PTR_TO_INT(pClientData) ){ - sqlite3BeginBenignMalloc(); - sqlite3_wal_checkpoint(db, zDb); - sqlite3EndBenignMalloc(); + while( rc==SQLITE_OK && iBlock ){ + char *aBlock = 0; + int nBlock = 0; + iNewStart = iBlock; + + rc = sqlite3Fts3ReadBlock(p, iBlock, &aBlock, &nBlock, 0); + if( rc==SQLITE_OK ){ + rc = fts3TruncateNode(aBlock, nBlock, &block, zTerm, nTerm, &iBlock); + } + if( rc==SQLITE_OK ){ + rc = fts3WriteSegment(p, iNewStart, block.a, block.n); + } + sqlite3_free(aBlock); } - return SQLITE_OK; -} -#endif /* SQLITE_OMIT_WAL */ -/* -** Configure an sqlite3_wal_hook() callback to automatically checkpoint -** a database after committing a transaction if there are nFrame or -** more frames in the log file. Passing zero or a negative value as the -** nFrame parameter disables automatic checkpoints entirely. -** -** The callback registered by this function replaces any existing callback -** registered using sqlite3_wal_hook(). Likewise, registering a callback -** using sqlite3_wal_hook() disables the automatic checkpoint mechanism -** configured by this function. -*/ -SQLITE_API int SQLITE_STDCALL sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ -#ifdef SQLITE_OMIT_WAL - UNUSED_PARAMETER(db); - UNUSED_PARAMETER(nFrame); -#else -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; -#endif - if( nFrame>0 ){ - sqlite3_wal_hook(db, sqlite3WalDefaultHook, SQLITE_INT_TO_PTR(nFrame)); - }else{ - sqlite3_wal_hook(db, 0, 0); + /* Variable iNewStart now contains the first valid leaf node. */ + if( rc==SQLITE_OK && iNewStart ){ + sqlite3_stmt *pDel = 0; + rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDel, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDel, 1, iOldStart); + sqlite3_bind_int64(pDel, 2, iNewStart-1); + sqlite3_step(pDel); + rc = sqlite3_reset(pDel); + } } -#endif - return SQLITE_OK; + + if( rc==SQLITE_OK ){ + sqlite3_stmt *pChomp = 0; + rc = fts3SqlStmt(p, SQL_CHOMP_SEGDIR, &pChomp, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pChomp, 1, iNewStart); + sqlite3_bind_blob(pChomp, 2, root.a, root.n, SQLITE_STATIC); + sqlite3_bind_int64(pChomp, 3, iAbsLevel); + sqlite3_bind_int(pChomp, 4, iIdx); + sqlite3_step(pChomp); + rc = sqlite3_reset(pChomp); + } + } + + sqlite3_free(root.a); + sqlite3_free(block.a); + return rc; } /* -** Register a callback to be invoked each time a transaction is written -** into the write-ahead-log by this database connection. +** This function is called after an incrmental-merge operation has run to +** merge (or partially merge) two or more segments from absolute level +** iAbsLevel. +** +** Each input segment is either removed from the db completely (if all of +** its data was copied to the output segment by the incrmerge operation) +** or modified in place so that it no longer contains those entries that +** have been duplicated in the output segment. */ -SQLITE_API void *SQLITE_STDCALL sqlite3_wal_hook( - sqlite3 *db, /* Attach the hook to this db handle */ - int(*xCallback)(void *, sqlite3*, const char*, int), - void *pArg /* First argument passed to xCallback() */ +static int fts3IncrmergeChomp( + Fts3Table *p, /* FTS table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level containing segments */ + Fts3MultiSegReader *pCsr, /* Chomp all segments opened by this cursor */ + int *pnRem /* Number of segments not deleted */ ){ -#ifndef SQLITE_OMIT_WAL - void *pRet; -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; + int i; + int nRem = 0; + int rc = SQLITE_OK; + + for(i=pCsr->nSegment-1; i>=0 && rc==SQLITE_OK; i--){ + Fts3SegReader *pSeg = 0; + int j; + + /* Find the Fts3SegReader object with Fts3SegReader.iIdx==i. It is hiding + ** somewhere in the pCsr->apSegment[] array. */ + for(j=0; ALWAYS(jnSegment); j++){ + pSeg = pCsr->apSegment[j]; + if( pSeg->iIdx==i ) break; + } + assert( jnSegment && pSeg->iIdx==i ); + + if( pSeg->aNode==0 ){ + /* Seg-reader is at EOF. Remove the entire input segment. */ + rc = fts3DeleteSegment(p, pSeg); + if( rc==SQLITE_OK ){ + rc = fts3RemoveSegdirEntry(p, iAbsLevel, pSeg->iIdx); + } + *pnRem = 0; + }else{ + /* The incremental merge did not copy all the data from this + ** segment to the upper level. The segment is modified in place + ** so that it contains no keys smaller than zTerm/nTerm. */ + const char *zTerm = pSeg->zTerm; + int nTerm = pSeg->nTerm; + rc = fts3TruncateSegment(p, iAbsLevel, pSeg->iIdx, zTerm, nTerm); + nRem++; + } } -#endif - sqlite3_mutex_enter(db->mutex); - pRet = db->pWalArg; - db->xWalCallback = xCallback; - db->pWalArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pRet; -#else - return 0; -#endif + + if( rc==SQLITE_OK && nRem!=pCsr->nSegment ){ + rc = fts3RepackSegdirLevel(p, iAbsLevel); + } + + *pnRem = nRem; + return rc; } /* -** Checkpoint database zDb. +** Store an incr-merge hint in the database. */ -SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint_v2( - sqlite3 *db, /* Database handle */ - const char *zDb, /* Name of attached database (or NULL) */ - int eMode, /* SQLITE_CHECKPOINT_* value */ - int *pnLog, /* OUT: Size of WAL log in frames */ - int *pnCkpt /* OUT: Total number of frames checkpointed */ -){ -#ifdef SQLITE_OMIT_WAL - return SQLITE_OK; -#else +static int fts3IncrmergeHintStore(Fts3Table *p, Blob *pHint){ + sqlite3_stmt *pReplace = 0; int rc; /* Return code */ - int iDb = SQLITE_MAX_ATTACHED; /* sqlite3.aDb[] index of db to checkpoint */ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; -#endif + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pReplace, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pReplace, 1, FTS_STAT_INCRMERGEHINT); + sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC); + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); + } - /* Initialize the output variables to -1 in case an error occurs. */ - if( pnLog ) *pnLog = -1; - if( pnCkpt ) *pnCkpt = -1; + return rc; +} - assert( SQLITE_CHECKPOINT_PASSIVE==0 ); - assert( SQLITE_CHECKPOINT_FULL==1 ); - assert( SQLITE_CHECKPOINT_RESTART==2 ); - assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); - if( eModeSQLITE_CHECKPOINT_TRUNCATE ){ - /* EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint - ** mode: */ - return SQLITE_MISUSE; +/* +** Load an incr-merge hint from the database. The incr-merge hint, if one +** exists, is stored in the rowid==1 row of the %_stat table. +** +** If successful, populate blob *pHint with the value read from the %_stat +** table and return SQLITE_OK. Otherwise, if an error occurs, return an +** SQLite error code. +*/ +static int fts3IncrmergeHintLoad(Fts3Table *p, Blob *pHint){ + sqlite3_stmt *pSelect = 0; + int rc; + + pHint->n = 0; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pSelect, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int(pSelect, 1, FTS_STAT_INCRMERGEHINT); + if( SQLITE_ROW==sqlite3_step(pSelect) ){ + const char *aHint = sqlite3_column_blob(pSelect, 0); + int nHint = sqlite3_column_bytes(pSelect, 0); + if( aHint ){ + blobGrowBuffer(pHint, nHint, &rc); + if( rc==SQLITE_OK ){ + memcpy(pHint->a, aHint, nHint); + pHint->n = nHint; + } + } + } + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; } - sqlite3_mutex_enter(db->mutex); - if( zDb && zDb[0] ){ - iDb = sqlite3FindDbName(db, zDb); - } - if( iDb<0 ){ - rc = SQLITE_ERROR; - sqlite3ErrorWithMsg(db, SQLITE_ERROR, "unknown database: %s", zDb); - }else{ - db->busyHandler.nBusy = 0; - rc = sqlite3Checkpoint(db, iDb, eMode, pnLog, pnCkpt); - sqlite3Error(db, rc); - } - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); return rc; -#endif } - /* -** Checkpoint database zDb. If zDb is NULL, or if the buffer zDb points -** to contains a zero-length string, all attached databases are -** checkpointed. +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, append an entry to the hint stored in blob *pHint. Each entry +** consists of two varints, the absolute level number of the input segments +** and the number of input segments. +** +** If successful, leave *pRc set to SQLITE_OK and return. If an error occurs, +** set *pRc to an SQLite error code before returning. */ -SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ - /* EVIDENCE-OF: R-41613-20553 The sqlite3_wal_checkpoint(D,X) is equivalent to - ** sqlite3_wal_checkpoint_v2(D,X,SQLITE_CHECKPOINT_PASSIVE,0,0). */ - return sqlite3_wal_checkpoint_v2(db,zDb,SQLITE_CHECKPOINT_PASSIVE,0,0); +static void fts3IncrmergeHintPush( + Blob *pHint, /* Hint blob to append to */ + i64 iAbsLevel, /* First varint to store in hint */ + int nInput, /* Second varint to store in hint */ + int *pRc /* IN/OUT: Error code */ +){ + blobGrowBuffer(pHint, pHint->n + 2*FTS3_VARINT_MAX, pRc); + if( *pRc==SQLITE_OK ){ + pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], iAbsLevel); + pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], (i64)nInput); + } } -#ifndef SQLITE_OMIT_WAL /* -** Run a checkpoint on database iDb. This is a no-op if database iDb is -** not currently open in WAL mode. -** -** If a transaction is open on the database being checkpointed, this -** function returns SQLITE_LOCKED and a checkpoint is not attempted. If -** an error occurs while running the checkpoint, an SQLite error code is -** returned (i.e. SQLITE_IOERR). Otherwise, SQLITE_OK. -** -** The mutex on database handle db should be held by the caller. The mutex -** associated with the specific b-tree being checkpointed is taken by -** this function while the checkpoint is running. -** -** If iDb is passed SQLITE_MAX_ATTACHED, then all attached databases are -** checkpointed. If an error is encountered it is returned immediately - -** no attempt is made to checkpoint any remaining databases. +** Read the last entry (most recently pushed) from the hint blob *pHint +** and then remove the entry. Write the two values read to *piAbsLevel and +** *pnInput before returning. ** -** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART. +** If no error occurs, return SQLITE_OK. If the hint blob in *pHint does +** not contain at least two valid varints, return SQLITE_CORRUPT_VTAB. */ -SQLITE_PRIVATE int sqlite3Checkpoint(sqlite3 *db, int iDb, int eMode, int *pnLog, int *pnCkpt){ - int rc = SQLITE_OK; /* Return code */ - int i; /* Used to iterate through attached dbs */ - int bBusy = 0; /* True if SQLITE_BUSY has been encountered */ +static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){ + const int nHint = pHint->n; + int i; - assert( sqlite3_mutex_held(db->mutex) ); - assert( !pnLog || *pnLog==-1 ); - assert( !pnCkpt || *pnCkpt==-1 ); + i = pHint->n-2; + while( i>0 && (pHint->a[i-1] & 0x80) ) i--; + while( i>0 && (pHint->a[i-1] & 0x80) ) i--; - for(i=0; inDb && rc==SQLITE_OK; i++){ - if( i==iDb || iDb==SQLITE_MAX_ATTACHED ){ - rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt, eMode, pnLog, pnCkpt); - pnLog = 0; - pnCkpt = 0; - if( rc==SQLITE_BUSY ){ - bBusy = 1; - rc = SQLITE_OK; - } - } - } + pHint->n = i; + i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel); + i += fts3GetVarint32(&pHint->a[i], pnInput); + if( i!=nHint ) return FTS_CORRUPT_VTAB; - return (rc==SQLITE_OK && bBusy) ? SQLITE_BUSY : rc; + return SQLITE_OK; } -#endif /* SQLITE_OMIT_WAL */ + /* -** This function returns true if main-memory should be used instead of -** a temporary file for transient pager files and statement journals. -** The value returned depends on the value of db->temp_store (runtime -** parameter) and the compile time value of SQLITE_TEMP_STORE. The -** following table describes the relationship between these two values -** and this functions return value. +** Attempt an incremental merge that writes nMerge leaf blocks. ** -** SQLITE_TEMP_STORE db->temp_store Location of temporary database -** ----------------- -------------- ------------------------------ -** 0 any file (return 0) -** 1 1 file (return 0) -** 1 2 memory (return 1) -** 1 0 file (return 0) -** 2 1 file (return 0) -** 2 2 memory (return 1) -** 2 0 memory (return 1) -** 3 any memory (return 1) +** Incremental merges happen nMin segments at a time. The segments +** to be merged are the nMin oldest segments (the ones with the smallest +** values for the _segdir.idx field) in the highest level that contains +** at least nMin segments. Multiple merges might occur in an attempt to +** write the quota of nMerge leaf blocks. */ -SQLITE_PRIVATE int sqlite3TempInMemory(const sqlite3 *db){ -#if SQLITE_TEMP_STORE==1 - return ( db->temp_store==2 ); -#endif -#if SQLITE_TEMP_STORE==2 - return ( db->temp_store!=1 ); -#endif -#if SQLITE_TEMP_STORE==3 - return 1; -#endif -#if SQLITE_TEMP_STORE<1 || SQLITE_TEMP_STORE>3 - return 0; -#endif +SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ + int rc; /* Return code */ + int nRem = nMerge; /* Number of leaf pages yet to be written */ + Fts3MultiSegReader *pCsr; /* Cursor used to read input data */ + Fts3SegFilter *pFilter; /* Filter used with cursor pCsr */ + IncrmergeWriter *pWriter; /* Writer object */ + int nSeg = 0; /* Number of input segments */ + sqlite3_int64 iAbsLevel = 0; /* Absolute level number to work on */ + Blob hint = {0, 0, 0}; /* Hint read from %_stat table */ + int bDirtyHint = 0; /* True if blob 'hint' has been modified */ + + /* Allocate space for the cursor, filter and writer objects */ + const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter); + pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc); + if( !pWriter ) return SQLITE_NOMEM; + pFilter = (Fts3SegFilter *)&pWriter[1]; + pCsr = (Fts3MultiSegReader *)&pFilter[1]; + + rc = fts3IncrmergeHintLoad(p, &hint); + while( rc==SQLITE_OK && nRem>0 ){ + const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex; + sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ + int bUseHint = 0; /* True if attempting to append */ + int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ + + /* Search the %_segdir table for the absolute level with the smallest + ** relative level number that contains at least nMin segments, if any. + ** If one is found, set iAbsLevel to the absolute level number and + ** nSeg to nMin. If no level with at least nMin segments can be found, + ** set nSeg to -1. + */ + rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); + sqlite3_bind_int(pFindLevel, 1, nMin); + if( sqlite3_step(pFindLevel)==SQLITE_ROW ){ + iAbsLevel = sqlite3_column_int64(pFindLevel, 0); + nSeg = nMin; + }else{ + nSeg = -1; + } + rc = sqlite3_reset(pFindLevel); + + /* If the hint read from the %_stat table is not empty, check if the + ** last entry in it specifies a relative level smaller than or equal + ** to the level identified by the block above (if any). If so, this + ** iteration of the loop will work on merging at the hinted level. + */ + if( rc==SQLITE_OK && hint.n ){ + int nHint = hint.n; + sqlite3_int64 iHintAbsLevel = 0; /* Hint level */ + int nHintSeg = 0; /* Hint number of segments */ + + rc = fts3IncrmergeHintPop(&hint, &iHintAbsLevel, &nHintSeg); + if( nSeg<0 || (iAbsLevel % nMod) >= (iHintAbsLevel % nMod) ){ + iAbsLevel = iHintAbsLevel; + nSeg = nHintSeg; + bUseHint = 1; + bDirtyHint = 1; + }else{ + /* This undoes the effect of the HintPop() above - so that no entry + ** is removed from the hint blob. */ + hint.n = nHint; + } + } + + /* If nSeg is less that zero, then there is no level with at least + ** nMin segments and no hint in the %_stat table. No work to do. + ** Exit early in this case. */ + if( nSeg<0 ) break; + + /* Open a cursor to iterate through the contents of the oldest nSeg + ** indexes of absolute level iAbsLevel. If this cursor is opened using + ** the 'hint' parameters, it is possible that there are less than nSeg + ** segments available in level iAbsLevel. In this case, no work is + ** done on iAbsLevel - fall through to the next iteration of the loop + ** to start work on some other level. */ + memset(pWriter, 0, nAlloc); + pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; + + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); + assert( bUseHint==1 || bUseHint==0 ); + if( iIdx==0 || (bUseHint && iIdx==1) ){ + int bIgnore = 0; + rc = fts3SegmentIsMaxLevel(p, iAbsLevel+1, &bIgnore); + if( bIgnore ){ + pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY; + } + } + } + + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); + } + if( SQLITE_OK==rc && pCsr->nSegment==nSeg + && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) + && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) + ){ + if( bUseHint && iIdx>0 ){ + const char *zKey = pCsr->zTerm; + int nKey = pCsr->nTerm; + rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); + }else{ + rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); + } + + if( rc==SQLITE_OK && pWriter->nLeafEst ){ + fts3LogMerge(nSeg, iAbsLevel); + do { + rc = fts3IncrmergeAppend(p, pWriter, pCsr); + if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr); + if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK; + }while( rc==SQLITE_ROW ); + + /* Update or delete the input segments */ + if( rc==SQLITE_OK ){ + nRem -= (1 + pWriter->nWork); + rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr, &nSeg); + if( nSeg!=0 ){ + bDirtyHint = 1; + fts3IncrmergeHintPush(&hint, iAbsLevel, nSeg, &rc); + } + } + } + + if( nSeg!=0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } + fts3IncrmergeRelease(p, pWriter, &rc); + if( nSeg==0 && pWriter->bNoLeafData==0 ){ + fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData); + } + } + + sqlite3Fts3SegReaderFinish(pCsr); + } + + /* Write the hint values into the %_stat table for the next incr-merger */ + if( bDirtyHint && rc==SQLITE_OK ){ + rc = fts3IncrmergeHintStore(p, &hint); + } + + sqlite3_free(pWriter); + sqlite3_free(hint.a); + return rc; } /* -** Return UTF-8 encoded English language explanation of the most recent -** error. +** Convert the text beginning at *pz into an integer and return +** its value. Advance *pz to point to the first character past +** the integer. */ -SQLITE_API const char *SQLITE_STDCALL sqlite3_errmsg(sqlite3 *db){ - const char *z; - if( !db ){ - return sqlite3ErrStr(SQLITE_NOMEM); - } - if( !sqlite3SafetyCheckSickOrOk(db) ){ - return sqlite3ErrStr(SQLITE_MISUSE_BKPT); - } - sqlite3_mutex_enter(db->mutex); - if( db->mallocFailed ){ - z = sqlite3ErrStr(SQLITE_NOMEM); - }else{ - testcase( db->pErr==0 ); - z = (char*)sqlite3_value_text(db->pErr); - assert( !db->mallocFailed ); - if( z==0 ){ - z = sqlite3ErrStr(db->errCode); - } - } - sqlite3_mutex_leave(db->mutex); - return z; +static int fts3Getint(const char **pz){ + const char *z = *pz; + int i = 0; + while( (*z)>='0' && (*z)<='9' ) i = 10*i + *(z++) - '0'; + *pz = z; + return i; } -#ifndef SQLITE_OMIT_UTF16 /* -** Return UTF-16 encoded English language explanation of the most recent -** error. +** Process statements of the form: +** +** INSERT INTO table(table) VALUES('merge=A,B'); +** +** A and B are integers that decode to be the number of leaf pages +** written for the merge, and the minimum number of segments on a level +** before it will be selected for a merge, respectively. */ -SQLITE_API const void *SQLITE_STDCALL sqlite3_errmsg16(sqlite3 *db){ - static const u16 outOfMem[] = { - 'o', 'u', 't', ' ', 'o', 'f', ' ', 'm', 'e', 'm', 'o', 'r', 'y', 0 - }; - static const u16 misuse[] = { - 'l', 'i', 'b', 'r', 'a', 'r', 'y', ' ', - 'r', 'o', 'u', 't', 'i', 'n', 'e', ' ', - 'c', 'a', 'l', 'l', 'e', 'd', ' ', - 'o', 'u', 't', ' ', - 'o', 'f', ' ', - 's', 'e', 'q', 'u', 'e', 'n', 'c', 'e', 0 - }; +static int fts3DoIncrmerge( + Fts3Table *p, /* FTS3 table handle */ + const char *zParam /* Nul-terminated string containing "A,B" */ +){ + int rc; + int nMin = (FTS3_MERGE_COUNT / 2); + int nMerge = 0; + const char *z = zParam; - const void *z; - if( !db ){ - return (void *)outOfMem; - } - if( !sqlite3SafetyCheckSickOrOk(db) ){ - return (void *)misuse; + /* Read the first integer value */ + nMerge = fts3Getint(&z); + + /* If the first integer value is followed by a ',', read the second + ** integer value. */ + if( z[0]==',' && z[1]!='\0' ){ + z++; + nMin = fts3Getint(&z); } - sqlite3_mutex_enter(db->mutex); - if( db->mallocFailed ){ - z = (void *)outOfMem; + + if( z[0]!='\0' || nMin<2 ){ + rc = SQLITE_ERROR; }else{ - z = sqlite3_value_text16(db->pErr); - if( z==0 ){ - sqlite3ErrorWithMsg(db, db->errCode, sqlite3ErrStr(db->errCode)); - z = sqlite3_value_text16(db->pErr); + rc = SQLITE_OK; + if( !p->bHasStat ){ + assert( p->bFts4==0 ); + sqlite3Fts3CreateStatTable(&rc, p); } - /* A malloc() may have failed within the call to sqlite3_value_text16() - ** above. If this is the case, then the db->mallocFailed flag needs to - ** be cleared before returning. Do this directly, instead of via - ** sqlite3ApiExit(), to avoid setting the database handle error message. - */ - db->mallocFailed = 0; + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3Incrmerge(p, nMerge, nMin); + } + sqlite3Fts3SegmentsClose(p); } - sqlite3_mutex_leave(db->mutex); - return z; + return rc; } -#endif /* SQLITE_OMIT_UTF16 */ /* -** Return the most recent error code generated by an SQLite routine. If NULL is -** passed to this function, we assume a malloc() failed during sqlite3_open(). +** Process statements of the form: +** +** INSERT INTO table(table) VALUES('automerge=X'); +** +** where X is an integer. X==0 means to turn automerge off. X!=0 means +** turn it on. The setting is persistent. */ -SQLITE_API int SQLITE_STDCALL sqlite3_errcode(sqlite3 *db){ - if( db && !sqlite3SafetyCheckSickOrOk(db) ){ - return SQLITE_MISUSE_BKPT; - } - if( !db || db->mallocFailed ){ - return SQLITE_NOMEM; - } - return db->errCode & db->errMask; -} -SQLITE_API int SQLITE_STDCALL sqlite3_extended_errcode(sqlite3 *db){ - if( db && !sqlite3SafetyCheckSickOrOk(db) ){ - return SQLITE_MISUSE_BKPT; +static int fts3DoAutoincrmerge( + Fts3Table *p, /* FTS3 table handle */ + const char *zParam /* Nul-terminated string containing boolean */ +){ + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = 0; + p->nAutoincrmerge = fts3Getint(&zParam); + if( p->nAutoincrmerge==1 || p->nAutoincrmerge>FTS3_MERGE_COUNT ){ + p->nAutoincrmerge = 8; } - if( !db || db->mallocFailed ){ - return SQLITE_NOMEM; + if( !p->bHasStat ){ + assert( p->bFts4==0 ); + sqlite3Fts3CreateStatTable(&rc, p); + if( rc ) return rc; } - return db->errCode; + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); + if( rc ) return rc; + sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); + sqlite3_bind_int(pStmt, 2, p->nAutoincrmerge); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + return rc; } /* -** Return a string that describes the kind of error specified in the -** argument. For now, this simply calls the internal sqlite3ErrStr() -** function. +** Return a 64-bit checksum for the FTS index entry specified by the +** arguments to this function. */ -SQLITE_API const char *SQLITE_STDCALL sqlite3_errstr(int rc){ - return sqlite3ErrStr(rc); +static u64 fts3ChecksumEntry( + const char *zTerm, /* Pointer to buffer containing term */ + int nTerm, /* Size of zTerm in bytes */ + int iLangid, /* Language id for current row */ + int iIndex, /* Index (0..Fts3Table.nIndex-1) */ + i64 iDocid, /* Docid for current row. */ + int iCol, /* Column number */ + int iPos /* Position */ +){ + int i; + u64 ret = (u64)iDocid; + + ret += (ret<<3) + iLangid; + ret += (ret<<3) + iIndex; + ret += (ret<<3) + iCol; + ret += (ret<<3) + iPos; + for(i=0; inIndex-1) */ + int *pRc /* OUT: Return code */ ){ - CollSeq *pColl; - int enc2; - - assert( sqlite3_mutex_held(db->mutex) ); + Fts3SegFilter filter; + Fts3MultiSegReader csr; + int rc; + u64 cksum = 0; - /* If SQLITE_UTF16 is specified as the encoding type, transform this - ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the - ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally. - */ - enc2 = enc; - testcase( enc2==SQLITE_UTF16 ); - testcase( enc2==SQLITE_UTF16_ALIGNED ); - if( enc2==SQLITE_UTF16 || enc2==SQLITE_UTF16_ALIGNED ){ - enc2 = SQLITE_UTF16NATIVE; - } - if( enc2SQLITE_UTF16BE ){ - return SQLITE_MISUSE_BKPT; + assert( *pRc==SQLITE_OK ); + + memset(&filter, 0, sizeof(filter)); + memset(&csr, 0, sizeof(csr)); + filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; + filter.flags |= FTS3_SEGMENT_SCAN; + + rc = sqlite3Fts3SegReaderCursor( + p, iLangid, iIndex, FTS3_SEGCURSOR_ALL, 0, 0, 0, 1,&csr + ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); } - /* Check if this call is removing or replacing an existing collation - ** sequence. If so, and there are active VMs, return busy. If there - ** are no active VMs, invalidate any pre-compiled statements. - */ - pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 0); - if( pColl && pColl->xCmp ){ - if( db->nVdbeActive ){ - sqlite3ErrorWithMsg(db, SQLITE_BUSY, - "unable to delete/modify collation sequence due to active statements"); - return SQLITE_BUSY; - } - sqlite3ExpirePreparedStatements(db); + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, &csr)) ){ + char *pCsr = csr.aDoclist; + char *pEnd = &pCsr[csr.nDoclist]; - /* If collation sequence pColl was created directly by a call to - ** sqlite3_create_collation, and not generated by synthCollSeq(), - ** then any copies made by synthCollSeq() need to be invalidated. - ** Also, collation destructor - CollSeq.xDel() - function may need - ** to be called. - */ - if( (pColl->enc & ~SQLITE_UTF16_ALIGNED)==enc2 ){ - CollSeq *aColl = sqlite3HashFind(&db->aCollSeq, zName); - int j; - for(j=0; j<3; j++){ - CollSeq *p = &aColl[j]; - if( p->enc==pColl->enc ){ - if( p->xDel ){ - p->xDel(p->pUser); + i64 iDocid = 0; + i64 iCol = 0; + i64 iPos = 0; + + pCsr += sqlite3Fts3GetVarint(pCsr, &iDocid); + while( pCsrxCmp = 0; } } } } + sqlite3Fts3SegReaderFinish(&csr); - pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 1); - if( pColl==0 ) return SQLITE_NOMEM; - pColl->xCmp = xCompare; - pColl->pUser = pCtx; - pColl->xDel = xDel; - pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED)); - sqlite3Error(db, SQLITE_OK); - return SQLITE_OK; + *pRc = rc; + return cksum; } - /* -** This array defines hard upper bounds on limit values. The -** initializer must be kept in sync with the SQLITE_LIMIT_* -** #defines in sqlite3.h. -*/ -static const int aHardLimit[] = { - SQLITE_MAX_LENGTH, - SQLITE_MAX_SQL_LENGTH, - SQLITE_MAX_COLUMN, - SQLITE_MAX_EXPR_DEPTH, - SQLITE_MAX_COMPOUND_SELECT, - SQLITE_MAX_VDBE_OP, - SQLITE_MAX_FUNCTION_ARG, - SQLITE_MAX_ATTACHED, - SQLITE_MAX_LIKE_PATTERN_LENGTH, - SQLITE_MAX_VARIABLE_NUMBER, /* IMP: R-38091-32352 */ - SQLITE_MAX_TRIGGER_DEPTH, - SQLITE_MAX_WORKER_THREADS, -}; - -/* -** Make sure the hard limits are set to reasonable values -*/ -#if SQLITE_MAX_LENGTH<100 -# error SQLITE_MAX_LENGTH must be at least 100 -#endif -#if SQLITE_MAX_SQL_LENGTH<100 -# error SQLITE_MAX_SQL_LENGTH must be at least 100 -#endif -#if SQLITE_MAX_SQL_LENGTH>SQLITE_MAX_LENGTH -# error SQLITE_MAX_SQL_LENGTH must not be greater than SQLITE_MAX_LENGTH -#endif -#if SQLITE_MAX_COMPOUND_SELECT<2 -# error SQLITE_MAX_COMPOUND_SELECT must be at least 2 -#endif -#if SQLITE_MAX_VDBE_OP<40 -# error SQLITE_MAX_VDBE_OP must be at least 40 -#endif -#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>1000 -# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 1000 -#endif -#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>125 -# error SQLITE_MAX_ATTACHED must be between 0 and 125 -#endif -#if SQLITE_MAX_LIKE_PATTERN_LENGTH<1 -# error SQLITE_MAX_LIKE_PATTERN_LENGTH must be at least 1 -#endif -#if SQLITE_MAX_COLUMN>32767 -# error SQLITE_MAX_COLUMN must not exceed 32767 -#endif -#if SQLITE_MAX_TRIGGER_DEPTH<1 -# error SQLITE_MAX_TRIGGER_DEPTH must be at least 1 -#endif -#if SQLITE_MAX_WORKER_THREADS<0 || SQLITE_MAX_WORKER_THREADS>50 -# error SQLITE_MAX_WORKER_THREADS must be between 0 and 50 -#endif - - -/* -** Change the value of a limit. Report the old value. -** If an invalid limit index is supplied, report -1. -** Make no changes but still report the old value if the -** new limit is negative. +** Check if the contents of the FTS index match the current contents of the +** content table. If no error occurs and the contents do match, set *pbOk +** to true and return SQLITE_OK. Or if the contents do not match, set *pbOk +** to false before returning. ** -** A new lower limit does not shrink existing constructs. -** It merely prevents new constructs that exceed the limit -** from forming. +** If an error occurs (e.g. an OOM or IO error), return an SQLite error +** code. The final value of *pbOk is undefined in this case. */ -SQLITE_API int SQLITE_STDCALL sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ - int oldLimit; +static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ + int rc = SQLITE_OK; /* Return code */ + u64 cksum1 = 0; /* Checksum based on FTS index contents */ + u64 cksum2 = 0; /* Checksum based on %_content contents */ + sqlite3_stmt *pAllLangid = 0; /* Statement to return all language-ids */ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return -1; + /* This block calculates the checksum according to the FTS index. */ + rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid); + sqlite3_bind_int(pAllLangid, 2, p->nIndex); + while( rc==SQLITE_OK && sqlite3_step(pAllLangid)==SQLITE_ROW ){ + int iLangid = sqlite3_column_int(pAllLangid, 0); + int i; + for(i=0; inIndex; i++){ + cksum1 = cksum1 ^ fts3ChecksumIndex(p, iLangid, i, &rc); + } + } + rc2 = sqlite3_reset(pAllLangid); + if( rc==SQLITE_OK ) rc = rc2; } -#endif - /* EVIDENCE-OF: R-30189-54097 For each limit category SQLITE_LIMIT_NAME - ** there is a hard upper bound set at compile-time by a C preprocessor - ** macro called SQLITE_MAX_NAME. (The "_LIMIT_" in the name is changed to - ** "_MAX_".) - */ - assert( aHardLimit[SQLITE_LIMIT_LENGTH]==SQLITE_MAX_LENGTH ); - assert( aHardLimit[SQLITE_LIMIT_SQL_LENGTH]==SQLITE_MAX_SQL_LENGTH ); - assert( aHardLimit[SQLITE_LIMIT_COLUMN]==SQLITE_MAX_COLUMN ); - assert( aHardLimit[SQLITE_LIMIT_EXPR_DEPTH]==SQLITE_MAX_EXPR_DEPTH ); - assert( aHardLimit[SQLITE_LIMIT_COMPOUND_SELECT]==SQLITE_MAX_COMPOUND_SELECT); - assert( aHardLimit[SQLITE_LIMIT_VDBE_OP]==SQLITE_MAX_VDBE_OP ); - assert( aHardLimit[SQLITE_LIMIT_FUNCTION_ARG]==SQLITE_MAX_FUNCTION_ARG ); - assert( aHardLimit[SQLITE_LIMIT_ATTACHED]==SQLITE_MAX_ATTACHED ); - assert( aHardLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]== - SQLITE_MAX_LIKE_PATTERN_LENGTH ); - assert( aHardLimit[SQLITE_LIMIT_VARIABLE_NUMBER]==SQLITE_MAX_VARIABLE_NUMBER); - assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH ); - assert( aHardLimit[SQLITE_LIMIT_WORKER_THREADS]==SQLITE_MAX_WORKER_THREADS ); - assert( SQLITE_LIMIT_WORKER_THREADS==(SQLITE_N_LIMIT-1) ); + /* This block calculates the checksum according to the %_content table */ + if( rc==SQLITE_OK ){ + sqlite3_tokenizer_module const *pModule = p->pTokenizer->pModule; + sqlite3_stmt *pStmt = 0; + char *zSql; + + zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + i64 iDocid = sqlite3_column_int64(pStmt, 0); + int iLang = langidFromSelect(p, pStmt); + int iCol; - if( limitId<0 || limitId>=SQLITE_N_LIMIT ){ - return -1; - } - oldLimit = db->aLimit[limitId]; - if( newLimit>=0 ){ /* IMP: R-52476-28732 */ - if( newLimit>aHardLimit[limitId] ){ - newLimit = aHardLimit[limitId]; /* IMP: R-51463-25634 */ + for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1); + int nText = sqlite3_column_bytes(pStmt, iCol+1); + sqlite3_tokenizer_cursor *pT = 0; + + rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, nText,&pT); + while( rc==SQLITE_OK ){ + char const *zToken; /* Buffer containing token */ + int nToken = 0; /* Number of bytes in token */ + int iDum1 = 0, iDum2 = 0; /* Dummy variables */ + int iPos = 0; /* Position of token in zText */ + + rc = pModule->xNext(pT, &zToken, &nToken, &iDum1, &iDum2, &iPos); + if( rc==SQLITE_OK ){ + int i; + cksum2 = cksum2 ^ fts3ChecksumEntry( + zToken, nToken, iLang, 0, iDocid, iCol, iPos + ); + for(i=1; inIndex; i++){ + if( p->aIndex[i].nPrefix<=nToken ){ + cksum2 = cksum2 ^ fts3ChecksumEntry( + zToken, p->aIndex[i].nPrefix, iLang, i, iDocid, iCol, iPos + ); + } + } + } + } + if( pT ) pModule->xClose(pT); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + } } - db->aLimit[limitId] = newLimit; + + sqlite3_finalize(pStmt); } - return oldLimit; /* IMP: R-53341-35419 */ + + *pbOk = (cksum1==cksum2); + return rc; } /* -** This function is used to parse both URIs and non-URI filenames passed by the -** user to API functions sqlite3_open() or sqlite3_open_v2(), and for database -** URIs specified as part of ATTACH statements. +** Run the integrity-check. If no error occurs and the current contents of +** the FTS index are correct, return SQLITE_OK. Or, if the contents of the +** FTS index are incorrect, return SQLITE_CORRUPT_VTAB. ** -** The first argument to this function is the name of the VFS to use (or -** a NULL to signify the default VFS) if the URI does not contain a "vfs=xxx" -** query parameter. The second argument contains the URI (or non-URI filename) -** itself. When this function is called the *pFlags variable should contain -** the default flags to open the database handle with. The value stored in -** *pFlags may be updated before returning if the URI filename contains -** "cache=xxx" or "mode=xxx" query parameters. +** Or, if an error (e.g. an OOM or IO error) occurs, return an SQLite +** error code. ** -** If successful, SQLITE_OK is returned. In this case *ppVfs is set to point to -** the VFS that should be used to open the database file. *pzFile is set to -** point to a buffer containing the name of the file to open. It is the -** responsibility of the caller to eventually call sqlite3_free() to release -** this buffer. +** The integrity-check works as follows. For each token and indexed token +** prefix in the document set, a 64-bit checksum is calculated (by code +** in fts3ChecksumEntry()) based on the following: ** -** If an error occurs, then an SQLite error code is returned and *pzErrMsg -** may be set to point to a buffer containing an English language error -** message. It is the responsibility of the caller to eventually release -** this buffer by calling sqlite3_free(). +** + The index number (0 for the main index, 1 for the first prefix +** index etc.), +** + The token (or token prefix) text itself, +** + The language-id of the row it appears in, +** + The docid of the row it appears in, +** + The column it appears in, and +** + The tokens position within that column. +** +** The checksums for all entries in the index are XORed together to create +** a single checksum for the entire index. +** +** The integrity-check code calculates the same checksum in two ways: +** +** 1. By scanning the contents of the FTS index, and +** 2. By scanning and tokenizing the content table. +** +** If the two checksums are identical, the integrity-check is deemed to have +** passed. */ -SQLITE_PRIVATE int sqlite3ParseUri( - const char *zDefaultVfs, /* VFS to use if no "vfs=xxx" query option */ - const char *zUri, /* Nul-terminated URI to parse */ - unsigned int *pFlags, /* IN/OUT: SQLITE_OPEN_XXX flags */ - sqlite3_vfs **ppVfs, /* OUT: VFS to use */ - char **pzFile, /* OUT: Filename component of URI */ - char **pzErrMsg /* OUT: Error message (if rc!=SQLITE_OK) */ +static int fts3DoIntegrityCheck( + Fts3Table *p /* FTS3 table handle */ ){ - int rc = SQLITE_OK; - unsigned int flags = *pFlags; - const char *zVfs = zDefaultVfs; - char *zFile; - char c; - int nUri = sqlite3Strlen30(zUri); - - assert( *pzErrMsg==0 ); - - if( ((flags & SQLITE_OPEN_URI) /* IMP: R-48725-32206 */ - || sqlite3GlobalConfig.bOpenUri) /* IMP: R-51689-46548 */ - && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */ - ){ - char *zOpt; - int eState; /* Parser state when parsing URI */ - int iIn; /* Input character index */ - int iOut = 0; /* Output character index */ - u64 nByte = nUri+2; /* Bytes of space to allocate */ - - /* Make sure the SQLITE_OPEN_URI flag is set to indicate to the VFS xOpen - ** method that there may be extra parameters following the file-name. */ - flags |= SQLITE_OPEN_URI; + int rc; + int bOk = 0; + rc = fts3IntegrityCheck(p, &bOk); + if( rc==SQLITE_OK && bOk==0 ) rc = FTS_CORRUPT_VTAB; + return rc; +} - for(iIn=0; iIn)" +** +** Argument pVal contains the result of . Currently the only +** meaningful value to insert is the text 'optimize'. +*/ +static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ + int rc; /* Return Code */ + const char *zVal = (const char *)sqlite3_value_text(pVal); + int nVal = sqlite3_value_bytes(pVal); - iIn = 5; -#ifdef SQLITE_ALLOW_URI_AUTHORITY - if( strncmp(zUri+5, "///", 3)==0 ){ - iIn = 7; - /* The following condition causes URIs with five leading / characters - ** like file://///host/path to be converted into UNCs like //host/path. - ** The correct URI for that UNC has only two or four leading / characters - ** file://host/path or file:////host/path. But 5 leading slashes is a - ** common error, we are told, so we handle it as a special case. */ - if( strncmp(zUri+7, "///", 3)==0 ){ iIn++; } - }else if( strncmp(zUri+5, "//localhost/", 12)==0 ){ - iIn = 16; - } -#else - /* Discard the scheme and authority segments of the URI. */ - if( zUri[5]=='/' && zUri[6]=='/' ){ - iIn = 7; - while( zUri[iIn] && zUri[iIn]!='/' ) iIn++; - if( iIn!=7 && (iIn!=16 || memcmp("localhost", &zUri[7], 9)) ){ - *pzErrMsg = sqlite3_mprintf("invalid uri authority: %.*s", - iIn-7, &zUri[7]); - rc = SQLITE_ERROR; - goto parse_uri_out; - } - } + if( !zVal ){ + return SQLITE_NOMEM; + }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ + rc = fts3DoOptimize(p, 0); + }else if( nVal==7 && 0==sqlite3_strnicmp(zVal, "rebuild", 7) ){ + rc = fts3DoRebuild(p); + }else if( nVal==15 && 0==sqlite3_strnicmp(zVal, "integrity-check", 15) ){ + rc = fts3DoIntegrityCheck(p); + }else if( nVal>6 && 0==sqlite3_strnicmp(zVal, "merge=", 6) ){ + rc = fts3DoIncrmerge(p, &zVal[6]); + }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){ + rc = fts3DoAutoincrmerge(p, &zVal[10]); +#ifdef SQLITE_TEST + }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ + p->nNodeSize = atoi(&zVal[9]); + rc = SQLITE_OK; + }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ + p->nMaxPendingData = atoi(&zVal[11]); + rc = SQLITE_OK; + }else if( nVal>21 && 0==sqlite3_strnicmp(zVal, "test-no-incr-doclist=", 21) ){ + p->bNoIncrDoclist = atoi(&zVal[21]); + rc = SQLITE_OK; #endif + }else{ + rc = SQLITE_ERROR; + } - /* Copy the filename and any query parameters into the zFile buffer. - ** Decode %HH escape codes along the way. - ** - ** Within this loop, variable eState may be set to 0, 1 or 2, depending - ** on the parsing context. As follows: - ** - ** 0: Parsing file-name. - ** 1: Parsing name section of a name=value query parameter. - ** 2: Parsing value section of a name=value query parameter. - */ - eState = 0; - while( (c = zUri[iIn])!=0 && c!='#' ){ - iIn++; - if( c=='%' - && sqlite3Isxdigit(zUri[iIn]) - && sqlite3Isxdigit(zUri[iIn+1]) - ){ - int octet = (sqlite3HexToInt(zUri[iIn++]) << 4); - octet += sqlite3HexToInt(zUri[iIn++]); - - assert( octet>=0 && octet<256 ); - if( octet==0 ){ - /* This branch is taken when "%00" appears within the URI. In this - ** case we ignore all text in the remainder of the path, name or - ** value currently being parsed. So ignore the current character - ** and skip to the next "?", "=" or "&", as appropriate. */ - while( (c = zUri[iIn])!=0 && c!='#' - && (eState!=0 || c!='?') - && (eState!=1 || (c!='=' && c!='&')) - && (eState!=2 || c!='&') - ){ - iIn++; - } - continue; - } - c = octet; - }else if( eState==1 && (c=='&' || c=='=') ){ - if( zFile[iOut-1]==0 ){ - /* An empty option name. Ignore this option altogether. */ - while( zUri[iIn] && zUri[iIn]!='#' && zUri[iIn-1]!='&' ) iIn++; - continue; - } - if( c=='&' ){ - zFile[iOut++] = '\0'; - }else{ - eState = 2; - } - c = 0; - }else if( (eState==0 && c=='?') || (eState==2 && c=='&') ){ - c = 0; - eState = 1; - } - zFile[iOut++] = c; - } - if( eState==1 ) zFile[iOut++] = '\0'; - zFile[iOut++] = '\0'; - zFile[iOut++] = '\0'; - - /* Check if there were any options specified that should be interpreted - ** here. Options that are interpreted here include "vfs" and those that - ** correspond to flags that may be passed to the sqlite3_open_v2() - ** method. */ - zOpt = &zFile[sqlite3Strlen30(zFile)+1]; - while( zOpt[0] ){ - int nOpt = sqlite3Strlen30(zOpt); - char *zVal = &zOpt[nOpt+1]; - int nVal = sqlite3Strlen30(zVal); - - if( nOpt==3 && memcmp("vfs", zOpt, 3)==0 ){ - zVfs = zVal; - }else{ - struct OpenMode { - const char *z; - int mode; - } *aMode = 0; - char *zModeType = 0; - int mask = 0; - int limit = 0; + return rc; +} - if( nOpt==5 && memcmp("cache", zOpt, 5)==0 ){ - static struct OpenMode aCacheMode[] = { - { "shared", SQLITE_OPEN_SHAREDCACHE }, - { "private", SQLITE_OPEN_PRIVATECACHE }, - { 0, 0 } - }; +#ifndef SQLITE_DISABLE_FTS4_DEFERRED +/* +** Delete all cached deferred doclists. Deferred doclists are cached +** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. +*/ +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){ + Fts3DeferredToken *pDef; + for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){ + fts3PendingListDelete(pDef->pList); + pDef->pList = 0; + } +} - mask = SQLITE_OPEN_SHAREDCACHE|SQLITE_OPEN_PRIVATECACHE; - aMode = aCacheMode; - limit = mask; - zModeType = "cache"; - } - if( nOpt==4 && memcmp("mode", zOpt, 4)==0 ){ - static struct OpenMode aOpenMode[] = { - { "ro", SQLITE_OPEN_READONLY }, - { "rw", SQLITE_OPEN_READWRITE }, - { "rwc", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE }, - { "memory", SQLITE_OPEN_MEMORY }, - { 0, 0 } - }; +/* +** Free all entries in the pCsr->pDeffered list. Entries are added to +** this list using sqlite3Fts3DeferToken(). +*/ +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){ + Fts3DeferredToken *pDef; + Fts3DeferredToken *pNext; + for(pDef=pCsr->pDeferred; pDef; pDef=pNext){ + pNext = pDef->pNext; + fts3PendingListDelete(pDef->pList); + sqlite3_free(pDef); + } + pCsr->pDeferred = 0; +} - mask = SQLITE_OPEN_READONLY | SQLITE_OPEN_READWRITE - | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY; - aMode = aOpenMode; - limit = mask & flags; - zModeType = "access"; - } +/* +** Generate deferred-doclists for all tokens in the pCsr->pDeferred list +** based on the row that pCsr currently points to. +** +** A deferred-doclist is like any other doclist with position information +** included, except that it only contains entries for a single row of the +** table, not for all rows. +*/ +SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ + int rc = SQLITE_OK; /* Return code */ + if( pCsr->pDeferred ){ + int i; /* Used to iterate through table columns */ + sqlite3_int64 iDocid; /* Docid of the row pCsr points to */ + Fts3DeferredToken *pDef; /* Used to iterate through deferred tokens */ + + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + sqlite3_tokenizer *pT = p->pTokenizer; + sqlite3_tokenizer_module const *pModule = pT->pModule; + + assert( pCsr->isRequireSeek==0 ); + iDocid = sqlite3_column_int64(pCsr->pStmt, 0); + + for(i=0; inColumn && rc==SQLITE_OK; i++){ + if( p->abNotindexed[i]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1); + sqlite3_tokenizer_cursor *pTC = 0; - if( aMode ){ - int i; - int mode = 0; - for(i=0; aMode[i].z; i++){ - const char *z = aMode[i].z; - if( nVal==sqlite3Strlen30(z) && 0==memcmp(zVal, z, nVal) ){ - mode = aMode[i].mode; - break; + rc = sqlite3Fts3OpenTokenizer(pT, pCsr->iLangid, zText, -1, &pTC); + while( rc==SQLITE_OK ){ + char const *zToken; /* Buffer containing token */ + int nToken = 0; /* Number of bytes in token */ + int iDum1 = 0, iDum2 = 0; /* Dummy variables */ + int iPos = 0; /* Position of token in zText */ + + rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos); + for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ + Fts3PhraseToken *pPT = pDef->pToken; + if( (pDef->iCol>=p->nColumn || pDef->iCol==i) + && (pPT->bFirst==0 || iPos==0) + && (pPT->n==nToken || (pPT->isPrefix && pPT->nz, pPT->n)) + ){ + fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc); } } - if( mode==0 ){ - *pzErrMsg = sqlite3_mprintf("no such %s mode: %s", zModeType, zVal); - rc = SQLITE_ERROR; - goto parse_uri_out; - } - if( (mode & ~SQLITE_OPEN_MEMORY)>limit ){ - *pzErrMsg = sqlite3_mprintf("%s mode not allowed: %s", - zModeType, zVal); - rc = SQLITE_PERM; - goto parse_uri_out; - } - flags = (flags & ~mask) | mode; } + if( pTC ) pModule->xClose(pTC); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; } - - zOpt = &zVal[nVal+1]; } - }else{ - zFile = sqlite3_malloc64(nUri+2); - if( !zFile ) return SQLITE_NOMEM; - memcpy(zFile, zUri, nUri); - zFile[nUri] = '\0'; - zFile[nUri+1] = '\0'; - flags &= ~SQLITE_OPEN_URI; + for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ + if( pDef->pList ){ + rc = fts3PendingListAppendVarint(&pDef->pList, 0); + } + } } - *ppVfs = sqlite3_vfs_find(zVfs); - if( *ppVfs==0 ){ - *pzErrMsg = sqlite3_mprintf("no such vfs: %s", zVfs); - rc = SQLITE_ERROR; - } - parse_uri_out: - if( rc!=SQLITE_OK ){ - sqlite3_free(zFile); - zFile = 0; - } - *pFlags = flags; - *pzFile = zFile; return rc; } +SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList( + Fts3DeferredToken *p, + char **ppData, + int *pnData +){ + char *pRet; + int nSkip; + sqlite3_int64 dummy; + + *ppData = 0; + *pnData = 0; + + if( p->pList==0 ){ + return SQLITE_OK; + } + + pRet = (char *)sqlite3_malloc(p->pList->nData); + if( !pRet ) return SQLITE_NOMEM; + + nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy); + *pnData = p->pList->nData - nSkip; + *ppData = pRet; + + memcpy(pRet, &p->pList->aData[nSkip], *pnData); + return SQLITE_OK; +} /* -** This routine does the work of opening a database on behalf of -** sqlite3_open() and sqlite3_open16(). The database filename "zFilename" -** is UTF-8 encoded. +** Add an entry for token pToken to the pCsr->pDeferred list. */ -static int openDatabase( - const char *zFilename, /* Database filename UTF-8 encoded */ - sqlite3 **ppDb, /* OUT: Returned database handle */ - unsigned int flags, /* Operational flags */ - const char *zVfs /* Name of the VFS to use */ +SQLITE_PRIVATE int sqlite3Fts3DeferToken( + Fts3Cursor *pCsr, /* Fts3 table cursor */ + Fts3PhraseToken *pToken, /* Token to defer */ + int iCol /* Column that token must appear in (or -1) */ ){ - sqlite3 *db; /* Store allocated handle here */ - int rc; /* Return code */ - int isThreadsafe; /* True for threadsafe connections */ - char *zOpen = 0; /* Filename argument to pass to BtreeOpen() */ - char *zErrMsg = 0; /* Error message from sqlite3ParseUri() */ + Fts3DeferredToken *pDeferred; + pDeferred = sqlite3_malloc(sizeof(*pDeferred)); + if( !pDeferred ){ + return SQLITE_NOMEM; + } + memset(pDeferred, 0, sizeof(*pDeferred)); + pDeferred->pToken = pToken; + pDeferred->pNext = pCsr->pDeferred; + pDeferred->iCol = iCol; + pCsr->pDeferred = pDeferred; -#ifdef SQLITE_ENABLE_API_ARMOR - if( ppDb==0 ) return SQLITE_MISUSE_BKPT; -#endif - *ppDb = 0; -#ifndef SQLITE_OMIT_AUTOINIT - rc = sqlite3_initialize(); - if( rc ) return rc; + assert( pToken->pDeferred==0 ); + pToken->pDeferred = pDeferred; + + return SQLITE_OK; +} #endif - /* Only allow sensible combinations of bits in the flags argument. - ** Throw an error if any non-sense combination is used. If we - ** do not block illegal combinations here, it could trigger - ** assert() statements in deeper layers. Sensible combinations - ** are: - ** - ** 1: SQLITE_OPEN_READONLY - ** 2: SQLITE_OPEN_READWRITE - ** 6: SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE - */ - assert( SQLITE_OPEN_READONLY == 0x01 ); - assert( SQLITE_OPEN_READWRITE == 0x02 ); - assert( SQLITE_OPEN_CREATE == 0x04 ); - testcase( (1<<(flags&7))==0x02 ); /* READONLY */ - testcase( (1<<(flags&7))==0x04 ); /* READWRITE */ - testcase( (1<<(flags&7))==0x40 ); /* READWRITE | CREATE */ - if( ((1<<(flags&7)) & 0x46)==0 ){ - return SQLITE_MISUSE_BKPT; /* IMP: R-65497-44594 */ - } +/* +** SQLite value pRowid contains the rowid of a row that may or may not be +** present in the FTS3 table. If it is, delete it and adjust the contents +** of subsiduary data structures accordingly. +*/ +static int fts3DeleteByRowid( + Fts3Table *p, + sqlite3_value *pRowid, + int *pnChng, /* IN/OUT: Decrement if row is deleted */ + u32 *aSzDel +){ + int rc = SQLITE_OK; /* Return code */ + int bFound = 0; /* True if *pRowid really is in the table */ - if( sqlite3GlobalConfig.bCoreMutex==0 ){ - isThreadsafe = 0; - }else if( flags & SQLITE_OPEN_NOMUTEX ){ - isThreadsafe = 0; - }else if( flags & SQLITE_OPEN_FULLMUTEX ){ - isThreadsafe = 1; - }else{ - isThreadsafe = sqlite3GlobalConfig.bFullMutex; - } - if( flags & SQLITE_OPEN_PRIVATECACHE ){ - flags &= ~SQLITE_OPEN_SHAREDCACHE; - }else if( sqlite3GlobalConfig.sharedCacheEnabled ){ - flags |= SQLITE_OPEN_SHAREDCACHE; + fts3DeleteTerms(&rc, p, pRowid, aSzDel, &bFound); + if( bFound && rc==SQLITE_OK ){ + int isEmpty = 0; /* Deleting *pRowid leaves the table empty */ + rc = fts3IsEmpty(p, pRowid, &isEmpty); + if( rc==SQLITE_OK ){ + if( isEmpty ){ + /* Deleting this row means the whole table is empty. In this case + ** delete the contents of all three tables and throw away any + ** data in the pendingTerms hash table. */ + rc = fts3DeleteAll(p, 1); + *pnChng = 0; + memset(aSzDel, 0, sizeof(u32) * (p->nColumn+1) * 2); + }else{ + *pnChng = *pnChng - 1; + if( p->zContentTbl==0 ){ + fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); + } + if( p->bHasDocsize ){ + fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid); + } + } + } } - /* Remove harmful bits from the flags parameter - ** - ** The SQLITE_OPEN_NOMUTEX and SQLITE_OPEN_FULLMUTEX flags were - ** dealt with in the previous code block. Besides these, the only - ** valid input flags for sqlite3_open_v2() are SQLITE_OPEN_READONLY, - ** SQLITE_OPEN_READWRITE, SQLITE_OPEN_CREATE, SQLITE_OPEN_SHAREDCACHE, - ** SQLITE_OPEN_PRIVATECACHE, and some reserved bits. Silently mask - ** off all other flags. - */ - flags &= ~( SQLITE_OPEN_DELETEONCLOSE | - SQLITE_OPEN_EXCLUSIVE | - SQLITE_OPEN_MAIN_DB | - SQLITE_OPEN_TEMP_DB | - SQLITE_OPEN_TRANSIENT_DB | - SQLITE_OPEN_MAIN_JOURNAL | - SQLITE_OPEN_TEMP_JOURNAL | - SQLITE_OPEN_SUBJOURNAL | - SQLITE_OPEN_MASTER_JOURNAL | - SQLITE_OPEN_NOMUTEX | - SQLITE_OPEN_FULLMUTEX | - SQLITE_OPEN_WAL - ); + return rc; +} - /* Allocate the sqlite data structure */ - db = sqlite3MallocZero( sizeof(sqlite3) ); - if( db==0 ) goto opendb_out; - if( isThreadsafe ){ - db->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); - if( db->mutex==0 ){ - sqlite3_free(db); - db = 0; - goto opendb_out; - } - } - sqlite3_mutex_enter(db->mutex); - db->errMask = 0xff; - db->nDb = 2; - db->magic = SQLITE_MAGIC_BUSY; - db->aDb = db->aDbStatic; +/* +** This function does the work for the xUpdate method of FTS3 virtual +** tables. The schema of the virtual table being: +** +** CREATE TABLE ( +** , +**
    HIDDEN, +** docid HIDDEN, +** HIDDEN +** ); +** +** +*/ +SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( + sqlite3_vtab *pVtab, /* FTS3 vtab object */ + int nArg, /* Size of argument array */ + sqlite3_value **apVal, /* Array of arguments */ + sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ +){ + Fts3Table *p = (Fts3Table *)pVtab; + int rc = SQLITE_OK; /* Return Code */ + int isRemove = 0; /* True for an UPDATE or DELETE */ + u32 *aSzIns = 0; /* Sizes of inserted documents */ + u32 *aSzDel = 0; /* Sizes of deleted documents */ + int nChng = 0; /* Net change in number of documents */ + int bInsertDone = 0; - assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); - memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); - db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = SQLITE_DEFAULT_WORKER_THREADS; - db->autoCommit = 1; - db->nextAutovac = -1; - db->szMmap = sqlite3GlobalConfig.szMmap; - db->nextPagesize = 0; - db->nMaxSorterMmap = 0x7FFFFFFF; - db->flags |= SQLITE_ShortColNames | SQLITE_EnableTrigger | SQLITE_CacheSpill -#if !defined(SQLITE_DEFAULT_AUTOMATIC_INDEX) || SQLITE_DEFAULT_AUTOMATIC_INDEX - | SQLITE_AutoIndex -#endif -#if SQLITE_DEFAULT_CKPTFULLFSYNC - | SQLITE_CkptFullFSync -#endif -#if SQLITE_DEFAULT_FILE_FORMAT<4 - | SQLITE_LegacyFileFmt -#endif -#ifdef SQLITE_ENABLE_LOAD_EXTENSION - | SQLITE_LoadExtension -#endif -#if SQLITE_DEFAULT_RECURSIVE_TRIGGERS - | SQLITE_RecTriggers -#endif -#if defined(SQLITE_DEFAULT_FOREIGN_KEYS) && SQLITE_DEFAULT_FOREIGN_KEYS - | SQLITE_ForeignKeys -#endif -#if defined(SQLITE_REVERSE_UNORDERED_SELECTS) - | SQLITE_ReverseOrder -#endif - ; - sqlite3HashInit(&db->aCollSeq); -#ifndef SQLITE_OMIT_VIRTUALTABLE - sqlite3HashInit(&db->aModule); -#endif + /* At this point it must be known if the %_stat table exists or not. + ** So bHasStat may not be 2. */ + assert( p->bHasStat==0 || p->bHasStat==1 ); - /* Add the default collation sequence BINARY. BINARY works for both UTF-8 - ** and UTF-16, so add a version for each to avoid any unnecessary - ** conversions. The only error that can occur here is a malloc() failure. + assert( p->pSegments==0 ); + assert( + nArg==1 /* DELETE operations */ + || nArg==(2 + p->nColumn + 3) /* INSERT or UPDATE operations */ + ); + + /* Check for a "special" INSERT operation. One of the form: ** - ** EVIDENCE-OF: R-52786-44878 SQLite defines three built-in collating - ** functions: + ** INSERT INTO xyz(xyz) VALUES('command'); */ - createCollation(db, "BINARY", SQLITE_UTF8, 0, binCollFunc, 0); - createCollation(db, "BINARY", SQLITE_UTF16BE, 0, binCollFunc, 0); - createCollation(db, "BINARY", SQLITE_UTF16LE, 0, binCollFunc, 0); - createCollation(db, "NOCASE", SQLITE_UTF8, 0, nocaseCollatingFunc, 0); - createCollation(db, "RTRIM", SQLITE_UTF8, (void*)1, binCollFunc, 0); - if( db->mallocFailed ){ - goto opendb_out; + if( nArg>1 + && sqlite3_value_type(apVal[0])==SQLITE_NULL + && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL + ){ + rc = fts3SpecialInsert(p, apVal[p->nColumn+2]); + goto update_out; } - /* EVIDENCE-OF: R-08308-17224 The default collating function for all - ** strings is BINARY. - */ - db->pDfltColl = sqlite3FindCollSeq(db, SQLITE_UTF8, "BINARY", 0); - assert( db->pDfltColl!=0 ); - /* Parse the filename/URI argument. */ - db->openFlags = flags; - rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); - if( rc!=SQLITE_OK ){ - if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; - sqlite3ErrorWithMsg(db, rc, zErrMsg ? "%s" : 0, zErrMsg); - sqlite3_free(zErrMsg); - goto opendb_out; + if( nArg>1 && sqlite3_value_int(apVal[2 + p->nColumn + 2])<0 ){ + rc = SQLITE_CONSTRAINT; + goto update_out; } - /* Open the backend database driver */ - rc = sqlite3BtreeOpen(db->pVfs, zOpen, db, &db->aDb[0].pBt, 0, - flags | SQLITE_OPEN_MAIN_DB); - if( rc!=SQLITE_OK ){ - if( rc==SQLITE_IOERR_NOMEM ){ - rc = SQLITE_NOMEM; - } - sqlite3Error(db, rc); - goto opendb_out; + /* Allocate space to hold the change in document sizes */ + aSzDel = sqlite3_malloc( sizeof(aSzDel[0])*(p->nColumn+1)*2 ); + if( aSzDel==0 ){ + rc = SQLITE_NOMEM; + goto update_out; } - sqlite3BtreeEnter(db->aDb[0].pBt); - db->aDb[0].pSchema = sqlite3SchemaGet(db, db->aDb[0].pBt); - if( !db->mallocFailed ) ENC(db) = SCHEMA_ENC(db); - sqlite3BtreeLeave(db->aDb[0].pBt); - db->aDb[1].pSchema = sqlite3SchemaGet(db, 0); - - /* The default safety_level for the main database is 'full'; for the temp - ** database it is 'NONE'. This matches the pager layer defaults. - */ - db->aDb[0].zName = "main"; - db->aDb[0].safety_level = 3; - db->aDb[1].zName = "temp"; - db->aDb[1].safety_level = 1; + aSzIns = &aSzDel[p->nColumn+1]; + memset(aSzDel, 0, sizeof(aSzDel[0])*(p->nColumn+1)*2); - db->magic = SQLITE_MAGIC_OPEN; - if( db->mallocFailed ){ - goto opendb_out; - } + rc = fts3Writelock(p); + if( rc!=SQLITE_OK ) goto update_out; - /* Register all built-in functions, but do not attempt to read the - ** database schema yet. This is delayed until the first time the database - ** is accessed. + /* If this is an INSERT operation, or an UPDATE that modifies the rowid + ** value, then this operation requires constraint handling. + ** + ** If the on-conflict mode is REPLACE, this means that the existing row + ** should be deleted from the database before inserting the new row. Or, + ** if the on-conflict mode is other than REPLACE, then this method must + ** detect the conflict and return SQLITE_CONSTRAINT before beginning to + ** modify the database file. */ - sqlite3Error(db, SQLITE_OK); - sqlite3RegisterBuiltinFunctions(db); + if( nArg>1 && p->zContentTbl==0 ){ + /* Find the value object that holds the new rowid value. */ + sqlite3_value *pNewRowid = apVal[3+p->nColumn]; + if( sqlite3_value_type(pNewRowid)==SQLITE_NULL ){ + pNewRowid = apVal[1]; + } - /* Load automatic extensions - extensions that have been registered - ** using the sqlite3_automatic_extension() API. - */ - rc = sqlite3_errcode(db); - if( rc==SQLITE_OK ){ - sqlite3AutoLoadExtensions(db); - rc = sqlite3_errcode(db); - if( rc!=SQLITE_OK ){ - goto opendb_out; + if( sqlite3_value_type(pNewRowid)!=SQLITE_NULL && ( + sqlite3_value_type(apVal[0])==SQLITE_NULL + || sqlite3_value_int64(apVal[0])!=sqlite3_value_int64(pNewRowid) + )){ + /* The new rowid is not NULL (in this case the rowid will be + ** automatically assigned and there is no chance of a conflict), and + ** the statement is either an INSERT or an UPDATE that modifies the + ** rowid column. So if the conflict mode is REPLACE, then delete any + ** existing row with rowid=pNewRowid. + ** + ** Or, if the conflict mode is not REPLACE, insert the new record into + ** the %_content table. If we hit the duplicate rowid constraint (or any + ** other error) while doing so, return immediately. + ** + ** This branch may also run if pNewRowid contains a value that cannot + ** be losslessly converted to an integer. In this case, the eventual + ** call to fts3InsertData() (either just below or further on in this + ** function) will return SQLITE_MISMATCH. If fts3DeleteByRowid is + ** invoked, it will delete zero rows (since no row will have + ** docid=$pNewRowid if $pNewRowid is not an integer value). + */ + if( sqlite3_vtab_on_conflict(p->db)==SQLITE_REPLACE ){ + rc = fts3DeleteByRowid(p, pNewRowid, &nChng, aSzDel); + }else{ + rc = fts3InsertData(p, apVal, pRowid); + bInsertDone = 1; + } } } - -#ifdef SQLITE_ENABLE_FTS1 - if( !db->mallocFailed ){ - extern int sqlite3Fts1Init(sqlite3*); - rc = sqlite3Fts1Init(db); + if( rc!=SQLITE_OK ){ + goto update_out; } -#endif -#ifdef SQLITE_ENABLE_FTS2 - if( !db->mallocFailed && rc==SQLITE_OK ){ - extern int sqlite3Fts2Init(sqlite3*); - rc = sqlite3Fts2Init(db); + /* If this is a DELETE or UPDATE operation, remove the old record. */ + if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ + assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER ); + rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel); + isRemove = 1; } -#endif - -#ifdef SQLITE_ENABLE_FTS3 - if( !db->mallocFailed && rc==SQLITE_OK ){ - rc = sqlite3Fts3Init(db); + + /* If this is an INSERT or UPDATE operation, insert the new record. */ + if( nArg>1 && rc==SQLITE_OK ){ + int iLangid = sqlite3_value_int(apVal[2 + p->nColumn + 2]); + if( bInsertDone==0 ){ + rc = fts3InsertData(p, apVal, pRowid); + if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){ + rc = FTS_CORRUPT_VTAB; + } + } + if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){ + rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid); + } + if( rc==SQLITE_OK ){ + assert( p->iPrevDocid==*pRowid ); + rc = fts3InsertTerms(p, iLangid, apVal, aSzIns); + } + if( p->bHasDocsize ){ + fts3InsertDocsize(&rc, p, aSzIns); + } + nChng++; } -#endif -#ifdef SQLITE_ENABLE_ICU - if( !db->mallocFailed && rc==SQLITE_OK ){ - rc = sqlite3IcuInit(db); + if( p->bFts4 ){ + fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); } -#endif -#ifdef SQLITE_ENABLE_RTREE - if( !db->mallocFailed && rc==SQLITE_OK){ - rc = sqlite3RtreeInit(db); - } -#endif + update_out: + sqlite3_free(aSzDel); + sqlite3Fts3SegmentsClose(p); + return rc; +} -#ifdef SQLITE_ENABLE_DBSTAT_VTAB - if( !db->mallocFailed && rc==SQLITE_OK){ - int sqlite3_dbstat_register(sqlite3*); - rc = sqlite3_dbstat_register(db); +/* +** Flush any data in the pending-terms hash table to disk. If successful, +** merge all segments in the database (including the new segment, if +** there was any data to flush) into a single segment. +*/ +SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){ + int rc; + rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); + if( rc==SQLITE_OK ){ + rc = fts3DoOptimize(p, 1); + if( rc==SQLITE_OK || rc==SQLITE_DONE ){ + int rc2 = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); + if( rc2!=SQLITE_OK ) rc = rc2; + }else{ + sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); + sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); + } } -#endif + sqlite3Fts3SegmentsClose(p); + return rc; +} - /* -DSQLITE_DEFAULT_LOCKING_MODE=1 makes EXCLUSIVE the default locking - ** mode. -DSQLITE_DEFAULT_LOCKING_MODE=0 make NORMAL the default locking - ** mode. Doing nothing at all also makes NORMAL the default. - */ -#ifdef SQLITE_DEFAULT_LOCKING_MODE - db->dfltLockMode = SQLITE_DEFAULT_LOCKING_MODE; - sqlite3PagerLockingMode(sqlite3BtreePager(db->aDb[0].pBt), - SQLITE_DEFAULT_LOCKING_MODE); #endif - if( rc ) sqlite3Error(db, rc); - - /* Enable the lookaside-malloc subsystem */ - setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside, - sqlite3GlobalConfig.nLookaside); +/************** End of fts3_write.c ******************************************/ +/************** Begin file fts3_snippet.c ************************************/ +/* +** 2009 Oct 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ - sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT); +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) -opendb_out: - sqlite3_free(zOpen); - if( db ){ - assert( db->mutex!=0 || isThreadsafe==0 - || sqlite3GlobalConfig.bFullMutex==0 ); - sqlite3_mutex_leave(db->mutex); - } - rc = sqlite3_errcode(db); - assert( db!=0 || rc==SQLITE_NOMEM ); - if( rc==SQLITE_NOMEM ){ - sqlite3_close(db); - db = 0; - }else if( rc!=SQLITE_OK ){ - db->magic = SQLITE_MAGIC_SICK; - } - *ppDb = db; -#ifdef SQLITE_ENABLE_SQLLOG - if( sqlite3GlobalConfig.xSqllog ){ - /* Opening a db handle. Fourth parameter is passed 0. */ - void *pArg = sqlite3GlobalConfig.pSqllogArg; - sqlite3GlobalConfig.xSqllog(pArg, db, zFilename, 0); - } -#endif - return sqlite3ApiExit(0, rc); -} +/* #include */ +/* #include */ /* -** Open a new database handle. +** Characters that may appear in the second argument to matchinfo(). */ -SQLITE_API int SQLITE_STDCALL sqlite3_open( - const char *zFilename, - sqlite3 **ppDb -){ - return openDatabase(zFilename, ppDb, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); -} -SQLITE_API int SQLITE_STDCALL sqlite3_open_v2( - const char *filename, /* Database filename (UTF-8) */ - sqlite3 **ppDb, /* OUT: SQLite db handle */ - int flags, /* Flags */ - const char *zVfs /* Name of VFS module to use */ -){ - return openDatabase(filename, ppDb, (unsigned int)flags, zVfs); -} +#define FTS3_MATCHINFO_NPHRASE 'p' /* 1 value */ +#define FTS3_MATCHINFO_NCOL 'c' /* 1 value */ +#define FTS3_MATCHINFO_NDOC 'n' /* 1 value */ +#define FTS3_MATCHINFO_AVGLENGTH 'a' /* nCol values */ +#define FTS3_MATCHINFO_LENGTH 'l' /* nCol values */ +#define FTS3_MATCHINFO_LCS 's' /* nCol values */ +#define FTS3_MATCHINFO_HITS 'x' /* 3*nCol*nPhrase values */ +#define FTS3_MATCHINFO_LHITS 'y' /* nCol*nPhrase values */ +#define FTS3_MATCHINFO_LHITS_BM 'b' /* nCol*nPhrase values */ -#ifndef SQLITE_OMIT_UTF16 /* -** Open a new database handle. +** The default value for the second argument to matchinfo(). */ -SQLITE_API int SQLITE_STDCALL sqlite3_open16( - const void *zFilename, - sqlite3 **ppDb -){ - char const *zFilename8; /* zFilename encoded in UTF-8 instead of UTF-16 */ - sqlite3_value *pVal; - int rc; - -#ifdef SQLITE_ENABLE_API_ARMOR - if( ppDb==0 ) return SQLITE_MISUSE_BKPT; -#endif - *ppDb = 0; -#ifndef SQLITE_OMIT_AUTOINIT - rc = sqlite3_initialize(); - if( rc ) return rc; -#endif - if( zFilename==0 ) zFilename = "\000\000"; - pVal = sqlite3ValueNew(0); - sqlite3ValueSetStr(pVal, -1, zFilename, SQLITE_UTF16NATIVE, SQLITE_STATIC); - zFilename8 = sqlite3ValueText(pVal, SQLITE_UTF8); - if( zFilename8 ){ - rc = openDatabase(zFilename8, ppDb, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); - assert( *ppDb || rc==SQLITE_NOMEM ); - if( rc==SQLITE_OK && !DbHasProperty(*ppDb, 0, DB_SchemaLoaded) ){ - SCHEMA_ENC(*ppDb) = ENC(*ppDb) = SQLITE_UTF16NATIVE; - } - }else{ - rc = SQLITE_NOMEM; - } - sqlite3ValueFree(pVal); +#define FTS3_MATCHINFO_DEFAULT "pcx" - return sqlite3ApiExit(0, rc); -} -#endif /* SQLITE_OMIT_UTF16 */ /* -** Register a new collation sequence with the database handle db. +** Used as an fts3ExprIterate() context when loading phrase doclists to +** Fts3Expr.aDoclist[]/nDoclist. */ -SQLITE_API int SQLITE_STDCALL sqlite3_create_collation( - sqlite3* db, - const char *zName, - int enc, - void* pCtx, - int(*xCompare)(void*,int,const void*,int,const void*) -){ - return sqlite3_create_collation_v2(db, zName, enc, pCtx, xCompare, 0); -} +typedef struct LoadDoclistCtx LoadDoclistCtx; +struct LoadDoclistCtx { + Fts3Cursor *pCsr; /* FTS3 Cursor */ + int nPhrase; /* Number of phrases seen so far */ + int nToken; /* Number of tokens seen so far */ +}; /* -** Register a new collation sequence with the database handle db. +** The following types are used as part of the implementation of the +** fts3BestSnippet() routine. */ -SQLITE_API int SQLITE_STDCALL sqlite3_create_collation_v2( - sqlite3* db, - const char *zName, - int enc, - void* pCtx, - int(*xCompare)(void*,int,const void*,int,const void*), - void(*xDel)(void*) -){ - int rc; +typedef struct SnippetIter SnippetIter; +typedef struct SnippetPhrase SnippetPhrase; +typedef struct SnippetFragment SnippetFragment; -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; -#endif - sqlite3_mutex_enter(db->mutex); - assert( !db->mallocFailed ); - rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, xDel); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; -} +struct SnippetIter { + Fts3Cursor *pCsr; /* Cursor snippet is being generated from */ + int iCol; /* Extract snippet from this column */ + int nSnippet; /* Requested snippet length (in tokens) */ + int nPhrase; /* Number of phrases in query */ + SnippetPhrase *aPhrase; /* Array of size nPhrase */ + int iCurrent; /* First token of current snippet */ +}; -#ifndef SQLITE_OMIT_UTF16 -/* -** Register a new collation sequence with the database handle db. -*/ -SQLITE_API int SQLITE_STDCALL sqlite3_create_collation16( - sqlite3* db, - const void *zName, - int enc, - void* pCtx, - int(*xCompare)(void*,int,const void*,int,const void*) -){ - int rc = SQLITE_OK; - char *zName8; +struct SnippetPhrase { + int nToken; /* Number of tokens in phrase */ + char *pList; /* Pointer to start of phrase position list */ + int iHead; /* Next value in position list */ + char *pHead; /* Position list data following iHead */ + int iTail; /* Next value in trailing position list */ + char *pTail; /* Position list data following iTail */ +}; -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; -#endif - sqlite3_mutex_enter(db->mutex); - assert( !db->mallocFailed ); - zName8 = sqlite3Utf16to8(db, zName, -1, SQLITE_UTF16NATIVE); - if( zName8 ){ - rc = createCollation(db, zName8, (u8)enc, pCtx, xCompare, 0); - sqlite3DbFree(db, zName8); - } - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; -} -#endif /* SQLITE_OMIT_UTF16 */ +struct SnippetFragment { + int iCol; /* Column snippet is extracted from */ + int iPos; /* Index of first token in snippet */ + u64 covered; /* Mask of query phrases covered */ + u64 hlmask; /* Mask of snippet terms to highlight */ +}; /* -** Register a collation sequence factory callback with the database handle -** db. Replace any previously installed collation sequence factory. +** This type is used as an fts3ExprIterate() context object while +** accumulating the data returned by the matchinfo() function. */ -SQLITE_API int SQLITE_STDCALL sqlite3_collation_needed( - sqlite3 *db, - void *pCollNeededArg, - void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*) -){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; -#endif - sqlite3_mutex_enter(db->mutex); - db->xCollNeeded = xCollNeeded; - db->xCollNeeded16 = 0; - db->pCollNeededArg = pCollNeededArg; - sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; -} +typedef struct MatchInfo MatchInfo; +struct MatchInfo { + Fts3Cursor *pCursor; /* FTS3 Cursor */ + int nCol; /* Number of columns in table */ + int nPhrase; /* Number of matchable phrases in query */ + sqlite3_int64 nDoc; /* Number of docs in database */ + char flag; + u32 *aMatchinfo; /* Pre-allocated buffer */ +}; -#ifndef SQLITE_OMIT_UTF16 /* -** Register a collation sequence factory callback with the database handle -** db. Replace any previously installed collation sequence factory. +** An instance of this structure is used to manage a pair of buffers, each +** (nElem * sizeof(u32)) bytes in size. See the MatchinfoBuffer code below +** for details. */ -SQLITE_API int SQLITE_STDCALL sqlite3_collation_needed16( - sqlite3 *db, - void *pCollNeededArg, - void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*) -){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; -#endif - sqlite3_mutex_enter(db->mutex); - db->xCollNeeded = 0; - db->xCollNeeded16 = xCollNeeded16; - db->pCollNeededArg = pCollNeededArg; - sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; -} -#endif /* SQLITE_OMIT_UTF16 */ +struct MatchinfoBuffer { + u8 aRef[3]; + int nElem; + int bGlobal; /* Set if global data is loaded */ + char *zMatchinfo; + u32 aMatchinfo[1]; +}; + -#ifndef SQLITE_OMIT_DEPRECATED /* -** This function is now an anachronism. It used to be used to recover from a -** malloc() failure, but SQLite now does this automatically. +** The snippet() and offsets() functions both return text values. An instance +** of the following structure is used to accumulate those values while the +** functions are running. See fts3StringAppend() for details. */ -SQLITE_API int SQLITE_STDCALL sqlite3_global_recover(void){ - return SQLITE_OK; -} -#endif +typedef struct StrBuffer StrBuffer; +struct StrBuffer { + char *z; /* Pointer to buffer containing string */ + int n; /* Length of z in bytes (excl. nul-term) */ + int nAlloc; /* Allocated size of buffer z in bytes */ +}; -/* -** Test to see whether or not the database connection is in autocommit -** mode. Return TRUE if it is and FALSE if not. Autocommit mode is on -** by default. Autocommit is disabled by a BEGIN statement and reenabled -** by the next COMMIT or ROLLBACK. + +/************************************************************************* +** Start of MatchinfoBuffer code. */ -SQLITE_API int SQLITE_STDCALL sqlite3_get_autocommit(sqlite3 *db){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif - return db->autoCommit; -} /* -** The following routines are substitutes for constants SQLITE_CORRUPT, -** SQLITE_MISUSE, SQLITE_CANTOPEN, SQLITE_IOERR and possibly other error -** constants. They serve two purposes: -** -** 1. Serve as a convenient place to set a breakpoint in a debugger -** to detect when version error conditions occurs. -** -** 2. Invoke sqlite3_log() to provide the source code location where -** a low-level error is first detected. +** Allocate a two-slot MatchinfoBuffer object. */ -SQLITE_PRIVATE int sqlite3CorruptError(int lineno){ - testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(SQLITE_CORRUPT, - "database corruption at line %d of [%.10s]", - lineno, 20+sqlite3_sourceid()); - return SQLITE_CORRUPT; -} -SQLITE_PRIVATE int sqlite3MisuseError(int lineno){ - testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(SQLITE_MISUSE, - "misuse at line %d of [%.10s]", - lineno, 20+sqlite3_sourceid()); - return SQLITE_MISUSE; -} -SQLITE_PRIVATE int sqlite3CantopenError(int lineno){ - testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(SQLITE_CANTOPEN, - "cannot open file at line %d of [%.10s]", - lineno, 20+sqlite3_sourceid()); - return SQLITE_CANTOPEN; -} +static MatchinfoBuffer *fts3MIBufferNew(int nElem, const char *zMatchinfo){ + MatchinfoBuffer *pRet; + int nByte = sizeof(u32) * (2*nElem + 1) + sizeof(MatchinfoBuffer); + int nStr = (int)strlen(zMatchinfo); + pRet = sqlite3_malloc(nByte + nStr+1); + if( pRet ){ + memset(pRet, 0, nByte); + pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet; + pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + sizeof(u32)*(nElem+1); + pRet->nElem = nElem; + pRet->zMatchinfo = ((char*)pRet) + nByte; + memcpy(pRet->zMatchinfo, zMatchinfo, nStr+1); + pRet->aRef[0] = 1; + } -#ifndef SQLITE_OMIT_DEPRECATED -/* -** This is a convenience routine that makes sure that all thread-specific -** data for this thread has been deallocated. -** -** SQLite no longer uses thread-specific data so this routine is now a -** no-op. It is retained for historical compatibility. -*/ -SQLITE_API void SQLITE_STDCALL sqlite3_thread_cleanup(void){ + return pRet; } -#endif - -/* -** Return meta information about a specific column of a database table. -** See comment in sqlite3.h (sqlite.h.in) for details. -*/ -SQLITE_API int SQLITE_STDCALL sqlite3_table_column_metadata( - sqlite3 *db, /* Connection handle */ - const char *zDbName, /* Database name or NULL */ - const char *zTableName, /* Table name */ - const char *zColumnName, /* Column name */ - char const **pzDataType, /* OUTPUT: Declared data type */ - char const **pzCollSeq, /* OUTPUT: Collation sequence name */ - int *pNotNull, /* OUTPUT: True if NOT NULL constraint exists */ - int *pPrimaryKey, /* OUTPUT: True if column part of PK */ - int *pAutoinc /* OUTPUT: True if column is auto-increment */ -){ - int rc; - char *zErrMsg = 0; - Table *pTab = 0; - Column *pCol = 0; - int iCol = 0; - char const *zDataType = 0; - char const *zCollSeq = 0; - int notnull = 0; - int primarykey = 0; - int autoinc = 0; +static void fts3MIBufferFree(void *p){ + MatchinfoBuffer *pBuf = (MatchinfoBuffer*)((u8*)p - ((u32*)p)[-1]); -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) || zTableName==0 ){ - return SQLITE_MISUSE_BKPT; + assert( (u32*)p==&pBuf->aMatchinfo[1] + || (u32*)p==&pBuf->aMatchinfo[pBuf->nElem+2] + ); + if( (u32*)p==&pBuf->aMatchinfo[1] ){ + pBuf->aRef[1] = 0; + }else{ + pBuf->aRef[2] = 0; } -#endif - /* Ensure the database schema has been loaded */ - sqlite3_mutex_enter(db->mutex); - sqlite3BtreeEnterAll(db); - rc = sqlite3Init(db, &zErrMsg); - if( SQLITE_OK!=rc ){ - goto error_out; + if( pBuf->aRef[0]==0 && pBuf->aRef[1]==0 && pBuf->aRef[2]==0 ){ + sqlite3_free(pBuf); } +} - /* Locate the table in question */ - pTab = sqlite3FindTable(db, zTableName, zDbName); - if( !pTab || pTab->pSelect ){ - pTab = 0; - goto error_out; - } +static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){ + void (*xRet)(void*) = 0; + u32 *aOut = 0; - /* Find the column for which info is requested */ - if( zColumnName==0 ){ - /* Query for existance of table only */ + if( p->aRef[1]==0 ){ + p->aRef[1] = 1; + aOut = &p->aMatchinfo[1]; + xRet = fts3MIBufferFree; + } + else if( p->aRef[2]==0 ){ + p->aRef[2] = 1; + aOut = &p->aMatchinfo[p->nElem+2]; + xRet = fts3MIBufferFree; }else{ - for(iCol=0; iColnCol; iCol++){ - pCol = &pTab->aCol[iCol]; - if( 0==sqlite3StrICmp(pCol->zName, zColumnName) ){ - break; - } - } - if( iCol==pTab->nCol ){ - if( HasRowid(pTab) && sqlite3IsRowid(zColumnName) ){ - iCol = pTab->iPKey; - pCol = iCol>=0 ? &pTab->aCol[iCol] : 0; - }else{ - pTab = 0; - goto error_out; - } + aOut = (u32*)sqlite3_malloc(p->nElem * sizeof(u32)); + if( aOut ){ + xRet = sqlite3_free; + if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32)); } } - /* The following block stores the meta information that will be returned - ** to the caller in local variables zDataType, zCollSeq, notnull, primarykey - ** and autoinc. At this point there are two possibilities: - ** - ** 1. The specified column name was rowid", "oid" or "_rowid_" - ** and there is no explicitly declared IPK column. - ** - ** 2. The table is not a view and the column name identified an - ** explicitly declared column. Copy meta information from *pCol. - */ - if( pCol ){ - zDataType = pCol->zType; - zCollSeq = pCol->zColl; - notnull = pCol->notNull!=0; - primarykey = (pCol->colFlags & COLFLAG_PRIMKEY)!=0; - autoinc = pTab->iPKey==iCol && (pTab->tabFlags & TF_Autoincrement)!=0; - }else{ - zDataType = "INTEGER"; - primarykey = 1; - } - if( !zCollSeq ){ - zCollSeq = "BINARY"; + *paOut = aOut; + return xRet; +} + +static void fts3MIBufferSetGlobal(MatchinfoBuffer *p){ + p->bGlobal = 1; + memcpy(&p->aMatchinfo[2+p->nElem], &p->aMatchinfo[1], p->nElem*sizeof(u32)); +} + +/* +** Free a MatchinfoBuffer object allocated using fts3MIBufferNew() +*/ +SQLITE_PRIVATE void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p){ + if( p ){ + assert( p->aRef[0]==1 ); + p->aRef[0] = 0; + if( p->aRef[0]==0 && p->aRef[1]==0 && p->aRef[2]==0 ){ + sqlite3_free(p); + } } +} -error_out: - sqlite3BtreeLeaveAll(db); +/* +** End of MatchinfoBuffer code. +*************************************************************************/ - /* Whether the function call succeeded or failed, set the output parameters - ** to whatever their local counterparts contain. If an error did occur, - ** this has the effect of zeroing all output parameters. - */ - if( pzDataType ) *pzDataType = zDataType; - if( pzCollSeq ) *pzCollSeq = zCollSeq; - if( pNotNull ) *pNotNull = notnull; - if( pPrimaryKey ) *pPrimaryKey = primarykey; - if( pAutoinc ) *pAutoinc = autoinc; - if( SQLITE_OK==rc && !pTab ){ - sqlite3DbFree(db, zErrMsg); - zErrMsg = sqlite3MPrintf(db, "no such table column: %s.%s", zTableName, - zColumnName); - rc = SQLITE_ERROR; +/* +** This function is used to help iterate through a position-list. A position +** list is a list of unique integers, sorted from smallest to largest. Each +** element of the list is represented by an FTS3 varint that takes the value +** of the difference between the current element and the previous one plus +** two. For example, to store the position-list: +** +** 4 9 113 +** +** the three varints: +** +** 6 7 106 +** +** are encoded. +** +** When this function is called, *pp points to the start of an element of +** the list. *piPos contains the value of the previous entry in the list. +** After it returns, *piPos contains the value of the next element of the +** list and *pp is advanced to the following varint. +*/ +static void fts3GetDeltaPosition(char **pp, int *piPos){ + int iVal; + *pp += fts3GetVarint32(*pp, &iVal); + *piPos += (iVal-2); +} + +/* +** Helper function for fts3ExprIterate() (see below). +*/ +static int fts3ExprIterate2( + Fts3Expr *pExpr, /* Expression to iterate phrases of */ + int *piPhrase, /* Pointer to phrase counter */ + int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ + void *pCtx /* Second argument to pass to callback */ +){ + int rc; /* Return code */ + int eType = pExpr->eType; /* Type of expression node pExpr */ + + if( eType!=FTSQUERY_PHRASE ){ + assert( pExpr->pLeft && pExpr->pRight ); + rc = fts3ExprIterate2(pExpr->pLeft, piPhrase, x, pCtx); + if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){ + rc = fts3ExprIterate2(pExpr->pRight, piPhrase, x, pCtx); + } + }else{ + rc = x(pExpr, *piPhrase, pCtx); + (*piPhrase)++; } - sqlite3ErrorWithMsg(db, rc, (zErrMsg?"%s":0), zErrMsg); - sqlite3DbFree(db, zErrMsg); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); return rc; } /* -** Sleep for a little while. Return the amount of time slept. +** Iterate through all phrase nodes in an FTS3 query, except those that +** are part of a sub-tree that is the right-hand-side of a NOT operator. +** For each phrase node found, the supplied callback function is invoked. +** +** If the callback function returns anything other than SQLITE_OK, +** the iteration is abandoned and the error code returned immediately. +** Otherwise, SQLITE_OK is returned after a callback has been made for +** all eligible phrase nodes. */ -SQLITE_API int SQLITE_STDCALL sqlite3_sleep(int ms){ - sqlite3_vfs *pVfs; - int rc; - pVfs = sqlite3_vfs_find(0); - if( pVfs==0 ) return 0; +static int fts3ExprIterate( + Fts3Expr *pExpr, /* Expression to iterate phrases of */ + int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ + void *pCtx /* Second argument to pass to callback */ +){ + int iPhrase = 0; /* Variable used as the phrase counter */ + return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx); +} + + +/* +** This is an fts3ExprIterate() callback used while loading the doclists +** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also +** fts3ExprLoadDoclists(). +*/ +static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ + int rc = SQLITE_OK; + Fts3Phrase *pPhrase = pExpr->pPhrase; + LoadDoclistCtx *p = (LoadDoclistCtx *)ctx; + + UNUSED_PARAMETER(iPhrase); + + p->nPhrase++; + p->nToken += pPhrase->nToken; - /* This function works in milliseconds, but the underlying OsSleep() - ** API uses microseconds. Hence the 1000's. - */ - rc = (sqlite3OsSleep(pVfs, 1000*ms)/1000); return rc; } /* -** Enable or disable the extended result codes. +** Load the doclists for each phrase in the query associated with FTS3 cursor +** pCsr. +** +** If pnPhrase is not NULL, then *pnPhrase is set to the number of matchable +** phrases in the expression (all phrases except those directly or +** indirectly descended from the right-hand-side of a NOT operator). If +** pnToken is not NULL, then it is set to the number of tokens in all +** matchable phrases of the expression. */ -SQLITE_API int SQLITE_STDCALL sqlite3_extended_result_codes(sqlite3 *db, int onoff){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; -#endif - sqlite3_mutex_enter(db->mutex); - db->errMask = onoff ? 0xffffffff : 0xff; - sqlite3_mutex_leave(db->mutex); +static int fts3ExprLoadDoclists( + Fts3Cursor *pCsr, /* Fts3 cursor for current query */ + int *pnPhrase, /* OUT: Number of phrases in query */ + int *pnToken /* OUT: Number of tokens in query */ +){ + int rc; /* Return Code */ + LoadDoclistCtx sCtx = {0,0,0}; /* Context for fts3ExprIterate() */ + sCtx.pCsr = pCsr; + rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb, (void *)&sCtx); + if( pnPhrase ) *pnPhrase = sCtx.nPhrase; + if( pnToken ) *pnToken = sCtx.nToken; + return rc; +} + +static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ + (*(int *)ctx)++; + pExpr->iPhrase = iPhrase; return SQLITE_OK; } +static int fts3ExprPhraseCount(Fts3Expr *pExpr){ + int nPhrase = 0; + (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase); + return nPhrase; +} /* -** Invoke the xFileControl method on a particular database. +** Advance the position list iterator specified by the first two +** arguments so that it points to the first element with a value greater +** than or equal to parameter iNext. */ -SQLITE_API int SQLITE_STDCALL sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ - int rc = SQLITE_ERROR; - Btree *pBtree; +static void fts3SnippetAdvance(char **ppIter, int *piIter, int iNext){ + char *pIter = *ppIter; + if( pIter ){ + int iIter = *piIter; -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; -#endif - sqlite3_mutex_enter(db->mutex); - pBtree = sqlite3DbNameToBtree(db, zDbName); - if( pBtree ){ - Pager *pPager; - sqlite3_file *fd; - sqlite3BtreeEnter(pBtree); - pPager = sqlite3BtreePager(pBtree); - assert( pPager!=0 ); - fd = sqlite3PagerFile(pPager); - assert( fd!=0 ); - if( op==SQLITE_FCNTL_FILE_POINTER ){ - *(sqlite3_file**)pArg = fd; - rc = SQLITE_OK; - }else if( fd->pMethods ){ - rc = sqlite3OsFileControl(fd, op, pArg); - }else{ - rc = SQLITE_NOTFOUND; + while( iItermutex); - return rc; } /* -** Interface to the testing logic. +** Advance the snippet iterator to the next candidate snippet. */ -SQLITE_API int SQLITE_CDECL sqlite3_test_control(int op, ...){ - int rc = 0; -#ifndef SQLITE_OMIT_BUILTIN_TEST - va_list ap; - va_start(ap, op); - switch( op ){ +static int fts3SnippetNextCandidate(SnippetIter *pIter){ + int i; /* Loop counter */ - /* - ** Save the current state of the PRNG. + if( pIter->iCurrent<0 ){ + /* The SnippetIter object has just been initialized. The first snippet + ** candidate always starts at offset 0 (even if this candidate has a + ** score of 0.0). */ - case SQLITE_TESTCTRL_PRNG_SAVE: { - sqlite3PrngSaveState(); - break; - } + pIter->iCurrent = 0; - /* - ** Restore the state of the PRNG to the last state saved using - ** PRNG_SAVE. If PRNG_SAVE has never before been called, then - ** this verb acts like PRNG_RESET. + /* Advance the 'head' iterator of each phrase to the first offset that + ** is greater than or equal to (iNext+nSnippet). */ - case SQLITE_TESTCTRL_PRNG_RESTORE: { - sqlite3PrngRestoreState(); - break; + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, pIter->nSnippet); } + }else{ + int iStart; + int iEnd = 0x7FFFFFFF; - /* - ** Reset the PRNG back to its uninitialized state. The next call - ** to sqlite3_randomness() will reseed the PRNG using a single call - ** to the xRandomness method of the default VFS. - */ - case SQLITE_TESTCTRL_PRNG_RESET: { - sqlite3_randomness(0,0); - break; + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + if( pPhrase->pHead && pPhrase->iHeadiHead; + } } - - /* - ** sqlite3_test_control(BITVEC_TEST, size, program) - ** - ** Run a test against a Bitvec object of size. The program argument - ** is an array of integers that defines the test. Return -1 on a - ** memory allocation error, 0 on success, or non-zero for an error. - ** See the sqlite3BitvecBuiltinTest() for additional information. - */ - case SQLITE_TESTCTRL_BITVEC_TEST: { - int sz = va_arg(ap, int); - int *aProg = va_arg(ap, int*); - rc = sqlite3BitvecBuiltinTest(sz, aProg); - break; + if( iEnd==0x7FFFFFFF ){ + return 1; } - /* - ** sqlite3_test_control(FAULT_INSTALL, xCallback) - ** - ** Arrange to invoke xCallback() whenever sqlite3FaultSim() is called, - ** if xCallback is not NULL. - ** - ** As a test of the fault simulator mechanism itself, sqlite3FaultSim(0) - ** is called immediately after installing the new callback and the return - ** value from sqlite3FaultSim(0) becomes the return from - ** sqlite3_test_control(). - */ - case SQLITE_TESTCTRL_FAULT_INSTALL: { - /* MSVC is picky about pulling func ptrs from va lists. - ** http://support.microsoft.com/kb/47961 - ** sqlite3GlobalConfig.xTestCallback = va_arg(ap, int(*)(int)); - */ - typedef int(*TESTCALLBACKFUNC_t)(int); - sqlite3GlobalConfig.xTestCallback = va_arg(ap, TESTCALLBACKFUNC_t); - rc = sqlite3FaultSim(0); - break; + pIter->iCurrent = iStart = iEnd - pIter->nSnippet + 1; + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, iEnd+1); + fts3SnippetAdvance(&pPhrase->pTail, &pPhrase->iTail, iStart); } + } - /* - ** sqlite3_test_control(BENIGN_MALLOC_HOOKS, xBegin, xEnd) - ** - ** Register hooks to call to indicate which malloc() failures - ** are benign. - */ - case SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS: { - typedef void (*void_function)(void); - void_function xBenignBegin; - void_function xBenignEnd; - xBenignBegin = va_arg(ap, void_function); - xBenignEnd = va_arg(ap, void_function); - sqlite3BenignMallocHooks(xBenignBegin, xBenignEnd); - break; - } + return 0; +} - /* - ** sqlite3_test_control(SQLITE_TESTCTRL_PENDING_BYTE, unsigned int X) - ** - ** Set the PENDING byte to the value in the argument, if X>0. - ** Make no changes if X==0. Return the value of the pending byte - ** as it existing before this routine was called. - ** - ** IMPORTANT: Changing the PENDING byte from 0x40000000 results in - ** an incompatible database file format. Changing the PENDING byte - ** while any database connection is open results in undefined and - ** deleterious behavior. - */ - case SQLITE_TESTCTRL_PENDING_BYTE: { - rc = PENDING_BYTE; -#ifndef SQLITE_OMIT_WSD - { - unsigned int newVal = va_arg(ap, unsigned int); - if( newVal ) sqlite3PendingByte = newVal; - } -#endif - break; - } +/* +** Retrieve information about the current candidate snippet of snippet +** iterator pIter. +*/ +static void fts3SnippetDetails( + SnippetIter *pIter, /* Snippet iterator */ + u64 mCovered, /* Bitmask of phrases already covered */ + int *piToken, /* OUT: First token of proposed snippet */ + int *piScore, /* OUT: "Score" for this snippet */ + u64 *pmCover, /* OUT: Bitmask of phrases covered */ + u64 *pmHighlight /* OUT: Bitmask of terms to highlight */ +){ + int iStart = pIter->iCurrent; /* First token of snippet */ + int iScore = 0; /* Score of this snippet */ + int i; /* Loop counter */ + u64 mCover = 0; /* Mask of phrases covered by this snippet */ + u64 mHighlight = 0; /* Mask of tokens to highlight in snippet */ - /* - ** sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, int X) - ** - ** This action provides a run-time test to see whether or not - ** assert() was enabled at compile-time. If X is true and assert() - ** is enabled, then the return value is true. If X is true and - ** assert() is disabled, then the return value is zero. If X is - ** false and assert() is enabled, then the assertion fires and the - ** process aborts. If X is false and assert() is disabled, then the - ** return value is zero. - */ - case SQLITE_TESTCTRL_ASSERT: { - volatile int x = 0; - assert( (x = va_arg(ap,int))!=0 ); - rc = x; - break; - } + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + if( pPhrase->pTail ){ + char *pCsr = pPhrase->pTail; + int iCsr = pPhrase->iTail; + while( iCsr<(iStart+pIter->nSnippet) ){ + int j; + u64 mPhrase = (u64)1 << i; + u64 mPos = (u64)1 << (iCsr - iStart); + assert( iCsr>=iStart ); + if( (mCover|mCovered)&mPhrase ){ + iScore++; + }else{ + iScore += 1000; + } + mCover |= mPhrase; - /* - ** sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, int X) - ** - ** This action provides a run-time test to see how the ALWAYS and - ** NEVER macros were defined at compile-time. - ** - ** The return value is ALWAYS(X). - ** - ** The recommended test is X==2. If the return value is 2, that means - ** ALWAYS() and NEVER() are both no-op pass-through macros, which is the - ** default setting. If the return value is 1, then ALWAYS() is either - ** hard-coded to true or else it asserts if its argument is false. - ** The first behavior (hard-coded to true) is the case if - ** SQLITE_TESTCTRL_ASSERT shows that assert() is disabled and the second - ** behavior (assert if the argument to ALWAYS() is false) is the case if - ** SQLITE_TESTCTRL_ASSERT shows that assert() is enabled. - ** - ** The run-time test procedure might look something like this: - ** - ** if( sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, 2)==2 ){ - ** // ALWAYS() and NEVER() are no-op pass-through macros - ** }else if( sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, 1) ){ - ** // ALWAYS(x) asserts that x is true. NEVER(x) asserts x is false. - ** }else{ - ** // ALWAYS(x) is a constant 1. NEVER(x) is a constant 0. - ** } - */ - case SQLITE_TESTCTRL_ALWAYS: { - int x = va_arg(ap,int); - rc = ALWAYS(x); - break; - } + for(j=0; jnToken; j++){ + mHighlight |= (mPos>>j); + } - /* - ** sqlite3_test_control(SQLITE_TESTCTRL_BYTEORDER); - ** - ** The integer returned reveals the byte-order of the computer on which - ** SQLite is running: - ** - ** 1 big-endian, determined at run-time - ** 10 little-endian, determined at run-time - ** 432101 big-endian, determined at compile-time - ** 123410 little-endian, determined at compile-time - */ - case SQLITE_TESTCTRL_BYTEORDER: { - rc = SQLITE_BYTEORDER*100 + SQLITE_LITTLEENDIAN*10 + SQLITE_BIGENDIAN; - break; + if( 0==(*pCsr & 0x0FE) ) break; + fts3GetDeltaPosition(&pCsr, &iCsr); + } } + } - /* sqlite3_test_control(SQLITE_TESTCTRL_RESERVE, sqlite3 *db, int N) - ** - ** Set the nReserve size to N for the main database on the database - ** connection db. - */ - case SQLITE_TESTCTRL_RESERVE: { - sqlite3 *db = va_arg(ap, sqlite3*); - int x = va_arg(ap,int); - sqlite3_mutex_enter(db->mutex); - sqlite3BtreeSetPageSize(db->aDb[0].pBt, 0, x, 0); - sqlite3_mutex_leave(db->mutex); - break; - } + /* Set the output variables before returning. */ + *piToken = iStart; + *piScore = iScore; + *pmCover = mCover; + *pmHighlight = mHighlight; +} - /* sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, sqlite3 *db, int N) - ** - ** Enable or disable various optimizations for testing purposes. The - ** argument N is a bitmask of optimizations to be disabled. For normal - ** operation N should be 0. The idea is that a test program (like the - ** SQL Logic Test or SLT test module) can run the same SQL multiple times - ** with various optimizations disabled to verify that the same answer - ** is obtained in every case. - */ - case SQLITE_TESTCTRL_OPTIMIZATIONS: { - sqlite3 *db = va_arg(ap, sqlite3*); - db->dbOptFlags = (u16)(va_arg(ap, int) & 0xffff); - break; - } +/* +** This function is an fts3ExprIterate() callback used by fts3BestSnippet(). +** Each invocation populates an element of the SnippetIter.aPhrase[] array. +*/ +static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ + SnippetIter *p = (SnippetIter *)ctx; + SnippetPhrase *pPhrase = &p->aPhrase[iPhrase]; + char *pCsr; + int rc; -#ifdef SQLITE_N_KEYWORD - /* sqlite3_test_control(SQLITE_TESTCTRL_ISKEYWORD, const char *zWord) - ** - ** If zWord is a keyword recognized by the parser, then return the - ** number of keywords. Or if zWord is not a keyword, return 0. - ** - ** This test feature is only available in the amalgamation since - ** the SQLITE_N_KEYWORD macro is not defined in this file if SQLite - ** is built using separate source files. - */ - case SQLITE_TESTCTRL_ISKEYWORD: { - const char *zWord = va_arg(ap, const char*); - int n = sqlite3Strlen30(zWord); - rc = (sqlite3KeywordCode((u8*)zWord, n)!=TK_ID) ? SQLITE_N_KEYWORD : 0; - break; - } -#endif + pPhrase->nToken = pExpr->pPhrase->nToken; + rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pCsr); + assert( rc==SQLITE_OK || pCsr==0 ); + if( pCsr ){ + int iFirst = 0; + pPhrase->pList = pCsr; + fts3GetDeltaPosition(&pCsr, &iFirst); + assert( iFirst>=0 ); + pPhrase->pHead = pCsr; + pPhrase->pTail = pCsr; + pPhrase->iHead = iFirst; + pPhrase->iTail = iFirst; + }else{ + assert( rc!=SQLITE_OK || ( + pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 + )); + } - /* sqlite3_test_control(SQLITE_TESTCTRL_SCRATCHMALLOC, sz, &pNew, pFree); - ** - ** Pass pFree into sqlite3ScratchFree(). - ** If sz>0 then allocate a scratch buffer into pNew. - */ - case SQLITE_TESTCTRL_SCRATCHMALLOC: { - void *pFree, **ppNew; - int sz; - sz = va_arg(ap, int); - ppNew = va_arg(ap, void**); - pFree = va_arg(ap, void*); - if( sz ) *ppNew = sqlite3ScratchMalloc(sz); - sqlite3ScratchFree(pFree); - break; - } + return rc; +} - /* sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, int onoff); - ** - ** If parameter onoff is non-zero, configure the wrappers so that all - ** subsequent calls to localtime() and variants fail. If onoff is zero, - ** undo this setting. - */ - case SQLITE_TESTCTRL_LOCALTIME_FAULT: { - sqlite3GlobalConfig.bLocaltimeFault = va_arg(ap, int); - break; - } +/* +** Select the fragment of text consisting of nFragment contiguous tokens +** from column iCol that represent the "best" snippet. The best snippet +** is the snippet with the highest score, where scores are calculated +** by adding: +** +** (a) +1 point for each occurrence of a matchable phrase in the snippet. +** +** (b) +1000 points for the first occurrence of each matchable phrase in +** the snippet for which the corresponding mCovered bit is not set. +** +** The selected snippet parameters are stored in structure *pFragment before +** returning. The score of the selected snippet is stored in *piScore +** before returning. +*/ +static int fts3BestSnippet( + int nSnippet, /* Desired snippet length */ + Fts3Cursor *pCsr, /* Cursor to create snippet for */ + int iCol, /* Index of column to create snippet from */ + u64 mCovered, /* Mask of phrases already covered */ + u64 *pmSeen, /* IN/OUT: Mask of phrases seen */ + SnippetFragment *pFragment, /* OUT: Best snippet found */ + int *piScore /* OUT: Score of snippet pFragment */ +){ + int rc; /* Return Code */ + int nList; /* Number of phrases in expression */ + SnippetIter sIter; /* Iterates through snippet candidates */ + int nByte; /* Number of bytes of space to allocate */ + int iBestScore = -1; /* Best snippet score found so far */ + int i; /* Loop counter */ - /* sqlite3_test_control(SQLITE_TESTCTRL_NEVER_CORRUPT, int); - ** - ** Set or clear a flag that indicates that the database file is always well- - ** formed and never corrupt. This flag is clear by default, indicating that - ** database files might have arbitrary corruption. Setting the flag during - ** testing causes certain assert() statements in the code to be activated - ** that demonstrat invariants on well-formed database files. - */ - case SQLITE_TESTCTRL_NEVER_CORRUPT: { - sqlite3GlobalConfig.neverCorrupt = va_arg(ap, int); - break; - } + memset(&sIter, 0, sizeof(sIter)); + /* Iterate through the phrases in the expression to count them. The same + ** callback makes sure the doclists are loaded for each phrase. + */ + rc = fts3ExprLoadDoclists(pCsr, &nList, 0); + if( rc!=SQLITE_OK ){ + return rc; + } - /* sqlite3_test_control(SQLITE_TESTCTRL_VDBE_COVERAGE, xCallback, ptr); - ** - ** Set the VDBE coverage callback function to xCallback with context - ** pointer ptr. - */ - case SQLITE_TESTCTRL_VDBE_COVERAGE: { -#ifdef SQLITE_VDBE_COVERAGE - typedef void (*branch_callback)(void*,int,u8,u8); - sqlite3GlobalConfig.xVdbeBranch = va_arg(ap,branch_callback); - sqlite3GlobalConfig.pVdbeBranchArg = va_arg(ap,void*); -#endif - break; - } + /* Now that it is known how many phrases there are, allocate and zero + ** the required space using malloc(). + */ + nByte = sizeof(SnippetPhrase) * nList; + sIter.aPhrase = (SnippetPhrase *)sqlite3_malloc(nByte); + if( !sIter.aPhrase ){ + return SQLITE_NOMEM; + } + memset(sIter.aPhrase, 0, nByte); - /* sqlite3_test_control(SQLITE_TESTCTRL_SORTER_MMAP, db, nMax); */ - case SQLITE_TESTCTRL_SORTER_MMAP: { - sqlite3 *db = va_arg(ap, sqlite3*); - db->nMaxSorterMmap = va_arg(ap, int); - break; - } + /* Initialize the contents of the SnippetIter object. Then iterate through + ** the set of phrases in the expression to populate the aPhrase[] array. + */ + sIter.pCsr = pCsr; + sIter.iCol = iCol; + sIter.nSnippet = nSnippet; + sIter.nPhrase = nList; + sIter.iCurrent = -1; + rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter); + if( rc==SQLITE_OK ){ - /* sqlite3_test_control(SQLITE_TESTCTRL_ISINIT); - ** - ** Return SQLITE_OK if SQLite has been initialized and SQLITE_ERROR if - ** not. - */ - case SQLITE_TESTCTRL_ISINIT: { - if( sqlite3GlobalConfig.isInit==0 ) rc = SQLITE_ERROR; - break; + /* Set the *pmSeen output variable. */ + for(i=0; i0 then reset the schema for all databases, causing - ** the schema to be reparsed the next time it is needed. This has the - ** effect of erasing all imposter tables. - */ - case SQLITE_TESTCTRL_IMPOSTER: { - sqlite3 *db = va_arg(ap, sqlite3*); - sqlite3_mutex_enter(db->mutex); - db->init.iDb = sqlite3FindDbName(db, va_arg(ap,const char*)); - db->init.busy = db->init.imposterTable = va_arg(ap,int); - db->init.newTnum = va_arg(ap,int); - if( db->init.busy==0 && db->init.newTnum>0 ){ - sqlite3ResetAllSchemasOfConnection(db); + /* Loop through all candidate snippets. Store the best snippet in + ** *pFragment. Store its associated 'score' in iBestScore. + */ + pFragment->iCol = iCol; + while( !fts3SnippetNextCandidate(&sIter) ){ + int iPos; + int iScore; + u64 mCover; + u64 mHighlite; + fts3SnippetDetails(&sIter, mCovered, &iPos, &iScore, &mCover,&mHighlite); + assert( iScore>=0 ); + if( iScore>iBestScore ){ + pFragment->iPos = iPos; + pFragment->hlmask = mHighlite; + pFragment->covered = mCover; + iBestScore = iScore; } - sqlite3_mutex_leave(db->mutex); - break; } - } - va_end(ap); -#endif /* SQLITE_OMIT_BUILTIN_TEST */ - return rc; -} -/* -** This is a utility routine, useful to VFS implementations, that checks -** to see if a database file was a URI that contained a specific query -** parameter, and if so obtains the value of the query parameter. -** -** The zFilename argument is the filename pointer passed into the xOpen() -** method of a VFS implementation. The zParam argument is the name of the -** query parameter we seek. This routine returns the value of the zParam -** parameter if it exists. If the parameter does not exist, this routine -** returns a NULL pointer. -*/ -SQLITE_API const char *SQLITE_STDCALL sqlite3_uri_parameter(const char *zFilename, const char *zParam){ - if( zFilename==0 || zParam==0 ) return 0; - zFilename += sqlite3Strlen30(zFilename) + 1; - while( zFilename[0] ){ - int x = strcmp(zFilename, zParam); - zFilename += sqlite3Strlen30(zFilename) + 1; - if( x==0 ) return zFilename; - zFilename += sqlite3Strlen30(zFilename) + 1; + *piScore = iBestScore; } - return 0; + sqlite3_free(sIter.aPhrase); + return rc; } -/* -** Return a boolean value for a query parameter. -*/ -SQLITE_API int SQLITE_STDCALL sqlite3_uri_boolean(const char *zFilename, const char *zParam, int bDflt){ - const char *z = sqlite3_uri_parameter(zFilename, zParam); - bDflt = bDflt!=0; - return z ? sqlite3GetBoolean(z, bDflt) : bDflt; -} /* -** Return a 64-bit integer value for a query parameter. +** Append a string to the string-buffer passed as the first argument. +** +** If nAppend is negative, then the length of the string zAppend is +** determined using strlen(). */ -SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_uri_int64( - const char *zFilename, /* Filename as passed to xOpen */ - const char *zParam, /* URI parameter sought */ - sqlite3_int64 bDflt /* return if parameter is missing */ +static int fts3StringAppend( + StrBuffer *pStr, /* Buffer to append to */ + const char *zAppend, /* Pointer to data to append to buffer */ + int nAppend /* Size of zAppend in bytes (or -1) */ ){ - const char *z = sqlite3_uri_parameter(zFilename, zParam); - sqlite3_int64 v; - if( z && sqlite3DecOrHexToI64(z, &v)==SQLITE_OK ){ - bDflt = v; + if( nAppend<0 ){ + nAppend = (int)strlen(zAppend); } - return bDflt; -} -/* -** Return the Btree pointer identified by zDbName. Return NULL if not found. -*/ -SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){ - int i; - for(i=0; inDb; i++){ - if( db->aDb[i].pBt - && (zDbName==0 || sqlite3StrICmp(zDbName, db->aDb[i].zName)==0) - ){ - return db->aDb[i].pBt; + /* If there is insufficient space allocated at StrBuffer.z, use realloc() + ** to grow the buffer until so that it is big enough to accomadate the + ** appended data. + */ + if( pStr->n+nAppend+1>=pStr->nAlloc ){ + int nAlloc = pStr->nAlloc+nAppend+100; + char *zNew = sqlite3_realloc(pStr->z, nAlloc); + if( !zNew ){ + return SQLITE_NOMEM; } + pStr->z = zNew; + pStr->nAlloc = nAlloc; } - return 0; -} + assert( pStr->z!=0 && (pStr->nAlloc >= pStr->n+nAppend+1) ); -/* -** Return the filename of the database associated with a database -** connection. -*/ -SQLITE_API const char *SQLITE_STDCALL sqlite3_db_filename(sqlite3 *db, const char *zDbName){ - Btree *pBt; -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif - pBt = sqlite3DbNameToBtree(db, zDbName); - return pBt ? sqlite3BtreeGetFilename(pBt) : 0; -} + /* Append the data to the string buffer. */ + memcpy(&pStr->z[pStr->n], zAppend, nAppend); + pStr->n += nAppend; + pStr->z[pStr->n] = '\0'; -/* -** Return 1 if database is read-only or 0 if read/write. Return -1 if -** no such database exists. -*/ -SQLITE_API int SQLITE_STDCALL sqlite3_db_readonly(sqlite3 *db, const char *zDbName){ - Btree *pBt; -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return -1; - } -#endif - pBt = sqlite3DbNameToBtree(db, zDbName); - return pBt ? sqlite3BtreeIsReadonly(pBt) : -1; + return SQLITE_OK; } -/************** End of main.c ************************************************/ -/************** Begin file notify.c ******************************************/ /* -** 2009 March 3 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: +** The fts3BestSnippet() function often selects snippets that end with a +** query term. That is, the final term of the snippet is always a term +** that requires highlighting. For example, if 'X' is a highlighted term +** and '.' is a non-highlighted term, BestSnippet() may select: ** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** ........X.....X ** -************************************************************************* +** This function "shifts" the beginning of the snippet forward in the +** document so that there are approximately the same number of +** non-highlighted terms to the right of the final highlighted term as there +** are to the left of the first highlighted term. For example, to this: ** -** This file contains the implementation of the sqlite3_unlock_notify() -** API method and its associated functionality. -*/ - -/* Omit this entire file if SQLITE_ENABLE_UNLOCK_NOTIFY is not defined. */ -#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY - -/* -** Public interfaces: +** ....X.....X.... ** -** sqlite3ConnectionBlocked() -** sqlite3ConnectionUnlocked() -** sqlite3ConnectionClosed() -** sqlite3_unlock_notify() +** This is done as part of extracting the snippet text, not when selecting +** the snippet. Snippet selection is done based on doclists only, so there +** is no way for fts3BestSnippet() to know whether or not the document +** actually contains terms that follow the final highlighted term. */ +static int fts3SnippetShift( + Fts3Table *pTab, /* FTS3 table snippet comes from */ + int iLangid, /* Language id to use in tokenizing */ + int nSnippet, /* Number of tokens desired for snippet */ + const char *zDoc, /* Document text to extract snippet from */ + int nDoc, /* Size of buffer zDoc in bytes */ + int *piPos, /* IN/OUT: First token of snippet */ + u64 *pHlmask /* IN/OUT: Mask of tokens to highlight */ +){ + u64 hlmask = *pHlmask; /* Local copy of initial highlight-mask */ -#define assertMutexHeld() \ - assert( sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)) ) + if( hlmask ){ + int nLeft; /* Tokens to the left of first highlight */ + int nRight; /* Tokens to the right of last highlight */ + int nDesired; /* Ideal number of tokens to shift forward */ -/* -** Head of a linked list of all sqlite3 objects created by this process -** for which either sqlite3.pBlockingConnection or sqlite3.pUnlockConnection -** is not NULL. This variable may only accessed while the STATIC_MASTER -** mutex is held. -*/ -static sqlite3 *SQLITE_WSD sqlite3BlockedList = 0; + for(nLeft=0; !(hlmask & ((u64)1 << nLeft)); nLeft++); + for(nRight=0; !(hlmask & ((u64)1 << (nSnippet-1-nRight))); nRight++); + nDesired = (nLeft-nRight)/2; -#ifndef NDEBUG -/* -** This function is a complex assert() that verifies the following -** properties of the blocked connections list: -** -** 1) Each entry in the list has a non-NULL value for either -** pUnlockConnection or pBlockingConnection, or both. -** -** 2) All entries in the list that share a common value for -** xUnlockNotify are grouped together. -** -** 3) If the argument db is not NULL, then none of the entries in the -** blocked connections list have pUnlockConnection or pBlockingConnection -** set to db. This is used when closing connection db. -*/ -static void checkListProperties(sqlite3 *db){ - sqlite3 *p; - for(p=sqlite3BlockedList; p; p=p->pNextBlocked){ - int seen = 0; - sqlite3 *p2; + /* Ideally, the start of the snippet should be pushed forward in the + ** document nDesired tokens. This block checks if there are actually + ** nDesired tokens to the right of the snippet. If so, *piPos and + ** *pHlMask are updated to shift the snippet nDesired tokens to the + ** right. Otherwise, the snippet is shifted by the number of tokens + ** available. + */ + if( nDesired>0 ){ + int nShift; /* Number of tokens to shift snippet by */ + int iCurrent = 0; /* Token counter */ + int rc; /* Return Code */ + sqlite3_tokenizer_module *pMod; + sqlite3_tokenizer_cursor *pC; + pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule; - /* Verify property (1) */ - assert( p->pUnlockConnection || p->pBlockingConnection ); + /* Open a cursor on zDoc/nDoc. Check if there are (nSnippet+nDesired) + ** or more tokens in zDoc/nDoc. + */ + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, iLangid, zDoc, nDoc, &pC); + if( rc!=SQLITE_OK ){ + return rc; + } + while( rc==SQLITE_OK && iCurrent<(nSnippet+nDesired) ){ + const char *ZDUMMY; int DUMMY1 = 0, DUMMY2 = 0, DUMMY3 = 0; + rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &DUMMY2, &DUMMY3, &iCurrent); + } + pMod->xClose(pC); + if( rc!=SQLITE_OK && rc!=SQLITE_DONE ){ return rc; } - /* Verify property (2) */ - for(p2=sqlite3BlockedList; p2!=p; p2=p2->pNextBlocked){ - if( p2->xUnlockNotify==p->xUnlockNotify ) seen = 1; - assert( p2->xUnlockNotify==p->xUnlockNotify || !seen ); - assert( db==0 || p->pUnlockConnection!=db ); - assert( db==0 || p->pBlockingConnection!=db ); + nShift = (rc==SQLITE_DONE)+iCurrent-nSnippet; + assert( nShift<=nDesired ); + if( nShift>0 ){ + *piPos += nShift; + *pHlmask = hlmask >> nShift; + } } } + return SQLITE_OK; } -#else -# define checkListProperties(x) -#endif /* -** Remove connection db from the blocked connections list. If connection -** db is not currently a part of the list, this function is a no-op. +** Extract the snippet text for fragment pFragment from cursor pCsr and +** append it to string buffer pOut. */ -static void removeFromBlockedList(sqlite3 *db){ - sqlite3 **pp; - assertMutexHeld(); - for(pp=&sqlite3BlockedList; *pp; pp = &(*pp)->pNextBlocked){ - if( *pp==db ){ - *pp = (*pp)->pNextBlocked; - break; +static int fts3SnippetText( + Fts3Cursor *pCsr, /* FTS3 Cursor */ + SnippetFragment *pFragment, /* Snippet to extract */ + int iFragment, /* Fragment number */ + int isLast, /* True for final fragment in snippet */ + int nSnippet, /* Number of tokens in extracted snippet */ + const char *zOpen, /* String inserted before highlighted term */ + const char *zClose, /* String inserted after highlighted term */ + const char *zEllipsis, /* String inserted between snippets */ + StrBuffer *pOut /* Write output here */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc; /* Return code */ + const char *zDoc; /* Document text to extract snippet from */ + int nDoc; /* Size of zDoc in bytes */ + int iCurrent = 0; /* Current token number of document */ + int iEnd = 0; /* Byte offset of end of current token */ + int isShiftDone = 0; /* True after snippet is shifted */ + int iPos = pFragment->iPos; /* First token of snippet */ + u64 hlmask = pFragment->hlmask; /* Highlight-mask for snippet */ + int iCol = pFragment->iCol+1; /* Query column to extract text from */ + sqlite3_tokenizer_module *pMod; /* Tokenizer module methods object */ + sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor open on zDoc/nDoc */ + + zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol); + if( zDoc==0 ){ + if( sqlite3_column_type(pCsr->pStmt, iCol)!=SQLITE_NULL ){ + return SQLITE_NOMEM; } + return SQLITE_OK; } -} + nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol); -/* -** Add connection db to the blocked connections list. It is assumed -** that it is not already a part of the list. -*/ -static void addToBlockedList(sqlite3 *db){ - sqlite3 **pp; - assertMutexHeld(); - for( - pp=&sqlite3BlockedList; - *pp && (*pp)->xUnlockNotify!=db->xUnlockNotify; - pp=&(*pp)->pNextBlocked - ); - db->pNextBlocked = *pp; - *pp = db; -} + /* Open a token cursor on the document. */ + pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule; + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, zDoc,nDoc,&pC); + if( rc!=SQLITE_OK ){ + return rc; + } -/* -** Obtain the STATIC_MASTER mutex. -*/ -static void enterMutex(void){ - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); - checkListProperties(0); -} + while( rc==SQLITE_OK ){ + const char *ZDUMMY; /* Dummy argument used with tokenizer */ + int DUMMY1 = -1; /* Dummy argument used with tokenizer */ + int iBegin = 0; /* Offset in zDoc of start of token */ + int iFin = 0; /* Offset in zDoc of end of token */ + int isHighlight = 0; /* True for highlighted terms */ -/* -** Release the STATIC_MASTER mutex. -*/ -static void leaveMutex(void){ - assertMutexHeld(); - checkListProperties(0); - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); -} + /* Variable DUMMY1 is initialized to a negative value above. Elsewhere + ** in the FTS code the variable that the third argument to xNext points to + ** is initialized to zero before the first (*but not necessarily + ** subsequent*) call to xNext(). This is done for a particular application + ** that needs to know whether or not the tokenizer is being used for + ** snippet generation or for some other purpose. + ** + ** Extreme care is required when writing code to depend on this + ** initialization. It is not a documented part of the tokenizer interface. + ** If a tokenizer is used directly by any code outside of FTS, this + ** convention might not be respected. */ + rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &iBegin, &iFin, &iCurrent); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_DONE ){ + /* Special case - the last token of the snippet is also the last token + ** of the column. Append any punctuation that occurred between the end + ** of the previous token and the end of the document to the output. + ** Then break out of the loop. */ + rc = fts3StringAppend(pOut, &zDoc[iEnd], -1); + } + break; + } + if( iCurrentpBlockingConnection. -** -** If there is no blocking connection, the callback is invoked immediately, -** before this routine returns. -** -** If pOther is already blocked on db, then report SQLITE_LOCKED, to indicate -** a deadlock. -** -** Otherwise, make arrangements to invoke xNotify when pOther drops -** its locks. -** -** Each call to this routine overrides any prior callbacks registered -** on the same "db". If xNotify==0 then any prior callbacks are immediately -** cancelled. -*/ -SQLITE_API int SQLITE_STDCALL sqlite3_unlock_notify( - sqlite3 *db, - void (*xNotify)(void **, int), - void *pArg -){ - int rc = SQLITE_OK; + if( !isShiftDone ){ + int n = nDoc - iBegin; + rc = fts3SnippetShift( + pTab, pCsr->iLangid, nSnippet, &zDoc[iBegin], n, &iPos, &hlmask + ); + isShiftDone = 1; + + /* Now that the shift has been done, check if the initial "..." are + ** required. They are required if (a) this is not the first fragment, + ** or (b) this fragment does not begin at position 0 of its column. + */ + if( rc==SQLITE_OK ){ + if( iPos>0 || iFragment>0 ){ + rc = fts3StringAppend(pOut, zEllipsis, -1); + }else if( iBegin ){ + rc = fts3StringAppend(pOut, zDoc, iBegin); + } + } + if( rc!=SQLITE_OK || iCurrent=(iPos+nSnippet) ){ + if( isLast ){ + rc = fts3StringAppend(pOut, zEllipsis, -1); + } + break; + } - sqlite3_mutex_enter(db->mutex); - enterMutex(); + /* Set isHighlight to true if this term should be highlighted. */ + isHighlight = (hlmask & ((u64)1 << (iCurrent-iPos)))!=0; - if( xNotify==0 ){ - removeFromBlockedList(db); - db->pBlockingConnection = 0; - db->pUnlockConnection = 0; - db->xUnlockNotify = 0; - db->pUnlockArg = 0; - }else if( 0==db->pBlockingConnection ){ - /* The blocking transaction has been concluded. Or there never was a - ** blocking transaction. In either case, invoke the notify callback - ** immediately. - */ - xNotify(&pArg, 1); - }else{ - sqlite3 *p; + if( iCurrent>iPos ) rc = fts3StringAppend(pOut, &zDoc[iEnd], iBegin-iEnd); + if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zOpen, -1); + if( rc==SQLITE_OK ) rc = fts3StringAppend(pOut, &zDoc[iBegin], iFin-iBegin); + if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zClose, -1); - for(p=db->pBlockingConnection; p && p!=db; p=p->pUnlockConnection){} - if( p ){ - rc = SQLITE_LOCKED; /* Deadlock detected. */ - }else{ - db->pUnlockConnection = db->pBlockingConnection; - db->xUnlockNotify = xNotify; - db->pUnlockArg = pArg; - removeFromBlockedList(db); - addToBlockedList(db); - } + iEnd = iFin; } - leaveMutex(); - assert( !db->mallocFailed ); - sqlite3ErrorWithMsg(db, rc, (rc?"database is deadlocked":0)); - sqlite3_mutex_leave(db->mutex); + pMod->xClose(pC); return rc; } -/* -** This function is called while stepping or preparing a statement -** associated with connection db. The operation will return SQLITE_LOCKED -** to the user because it requires a lock that will not be available -** until connection pBlocker concludes its current transaction. -*/ -SQLITE_PRIVATE void sqlite3ConnectionBlocked(sqlite3 *db, sqlite3 *pBlocker){ - enterMutex(); - if( db->pBlockingConnection==0 && db->pUnlockConnection==0 ){ - addToBlockedList(db); - } - db->pBlockingConnection = pBlocker; - leaveMutex(); -} /* -** This function is called when -** the transaction opened by database db has just finished. Locks held -** by database connection db have been released. -** -** This function loops through each entry in the blocked connections -** list and does the following: -** -** 1) If the sqlite3.pBlockingConnection member of a list entry is -** set to db, then set pBlockingConnection=0. -** -** 2) If the sqlite3.pUnlockConnection member of a list entry is -** set to db, then invoke the configured unlock-notify callback and -** set pUnlockConnection=0. +** This function is used to count the entries in a column-list (a +** delta-encoded list of term offsets within a single column of a single +** row). When this function is called, *ppCollist should point to the +** beginning of the first varint in the column-list (the varint that +** contains the position of the first matching term in the column data). +** Before returning, *ppCollist is set to point to the first byte after +** the last varint in the column-list (either the 0x00 signifying the end +** of the position-list, or the 0x01 that precedes the column number of +** the next column in the position-list). ** -** 3) If the two steps above mean that pBlockingConnection==0 and -** pUnlockConnection==0, remove the entry from the blocked connections -** list. +** The number of elements in the column-list is returned. */ -SQLITE_PRIVATE void sqlite3ConnectionUnlocked(sqlite3 *db){ - void (*xUnlockNotify)(void **, int) = 0; /* Unlock-notify cb to invoke */ - int nArg = 0; /* Number of entries in aArg[] */ - sqlite3 **pp; /* Iterator variable */ - void **aArg; /* Arguments to the unlock callback */ - void **aDyn = 0; /* Dynamically allocated space for aArg[] */ - void *aStatic[16]; /* Starter space for aArg[]. No malloc required */ +static int fts3ColumnlistCount(char **ppCollist){ + char *pEnd = *ppCollist; + char c = 0; + int nEntry = 0; - aArg = aStatic; - enterMutex(); /* Enter STATIC_MASTER mutex */ + /* A column-list is terminated by either a 0x01 or 0x00. */ + while( 0xFE & (*pEnd | c) ){ + c = *pEnd++ & 0x80; + if( !c ) nEntry++; + } - /* This loop runs once for each entry in the blocked-connections list. */ - for(pp=&sqlite3BlockedList; *pp; /* no-op */ ){ - sqlite3 *p = *pp; + *ppCollist = pEnd; + return nEntry; +} - /* Step 1. */ - if( p->pBlockingConnection==db ){ - p->pBlockingConnection = 0; - } +/* +** This function gathers 'y' or 'b' data for a single phrase. +*/ +static void fts3ExprLHits( + Fts3Expr *pExpr, /* Phrase expression node */ + MatchInfo *p /* Matchinfo context */ +){ + Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab; + int iStart; + Fts3Phrase *pPhrase = pExpr->pPhrase; + char *pIter = pPhrase->doclist.pList; + int iCol = 0; - /* Step 2. */ - if( p->pUnlockConnection==db ){ - assert( p->xUnlockNotify ); - if( p->xUnlockNotify!=xUnlockNotify && nArg!=0 ){ - xUnlockNotify(aArg, nArg); - nArg = 0; - } + assert( p->flag==FTS3_MATCHINFO_LHITS_BM || p->flag==FTS3_MATCHINFO_LHITS ); + if( p->flag==FTS3_MATCHINFO_LHITS ){ + iStart = pExpr->iPhrase * p->nCol; + }else{ + iStart = pExpr->iPhrase * ((p->nCol + 31) / 32); + } - sqlite3BeginBenignMalloc(); - assert( aArg==aDyn || (aDyn==0 && aArg==aStatic) ); - assert( nArg<=(int)ArraySize(aStatic) || aArg==aDyn ); - if( (!aDyn && nArg==(int)ArraySize(aStatic)) - || (aDyn && nArg==(int)(sqlite3MallocSize(aDyn)/sizeof(void*))) - ){ - /* The aArg[] array needs to grow. */ - void **pNew = (void **)sqlite3Malloc(nArg*sizeof(void *)*2); - if( pNew ){ - memcpy(pNew, aArg, nArg*sizeof(void *)); - sqlite3_free(aDyn); - aDyn = aArg = pNew; - }else{ - /* This occurs when the array of context pointers that need to - ** be passed to the unlock-notify callback is larger than the - ** aStatic[] array allocated on the stack and the attempt to - ** allocate a larger array from the heap has failed. - ** - ** This is a difficult situation to handle. Returning an error - ** code to the caller is insufficient, as even if an error code - ** is returned the transaction on connection db will still be - ** closed and the unlock-notify callbacks on blocked connections - ** will go unissued. This might cause the application to wait - ** indefinitely for an unlock-notify callback that will never - ** arrive. - ** - ** Instead, invoke the unlock-notify callback with the context - ** array already accumulated. We can then clear the array and - ** begin accumulating any further context pointers without - ** requiring any dynamic allocation. This is sub-optimal because - ** it means that instead of one callback with a large array of - ** context pointers the application will receive two or more - ** callbacks with smaller arrays of context pointers, which will - ** reduce the applications ability to prioritize multiple - ** connections. But it is the best that can be done under the - ** circumstances. - */ - xUnlockNotify(aArg, nArg); - nArg = 0; - } + while( 1 ){ + int nHit = fts3ColumnlistCount(&pIter); + if( (pPhrase->iColumn>=pTab->nColumn || pPhrase->iColumn==iCol) ){ + if( p->flag==FTS3_MATCHINFO_LHITS ){ + p->aMatchinfo[iStart + iCol] = (u32)nHit; + }else if( nHit ){ + p->aMatchinfo[iStart + (iCol+1)/32] |= (1 << (iCol&0x1F)); } - sqlite3EndBenignMalloc(); - - aArg[nArg++] = p->pUnlockArg; - xUnlockNotify = p->xUnlockNotify; - p->pUnlockConnection = 0; - p->xUnlockNotify = 0; - p->pUnlockArg = 0; } - - /* Step 3. */ - if( p->pBlockingConnection==0 && p->pUnlockConnection==0 ){ - /* Remove connection p from the blocked connections list. */ - *pp = p->pNextBlocked; - p->pNextBlocked = 0; - }else{ - pp = &p->pNextBlocked; - } - } - - if( nArg!=0 ){ - xUnlockNotify(aArg, nArg); + assert( *pIter==0x00 || *pIter==0x01 ); + if( *pIter!=0x01 ) break; + pIter++; + pIter += fts3GetVarint32(pIter, &iCol); } - sqlite3_free(aDyn); - leaveMutex(); /* Leave STATIC_MASTER mutex */ } /* -** This is called when the database connection passed as an argument is -** being closed. The connection is removed from the blocked list. +** Gather the results for matchinfo directives 'y' and 'b'. */ -SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ - sqlite3ConnectionUnlocked(db); - enterMutex(); - removeFromBlockedList(db); - checkListProperties(db); - leaveMutex(); +static void fts3ExprLHitGather( + Fts3Expr *pExpr, + MatchInfo *p +){ + assert( (pExpr->pLeft==0)==(pExpr->pRight==0) ); + if( pExpr->bEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){ + if( pExpr->pLeft ){ + fts3ExprLHitGather(pExpr->pLeft, p); + fts3ExprLHitGather(pExpr->pRight, p); + }else{ + fts3ExprLHits(pExpr, p); + } + } } -#endif - -/************** End of notify.c **********************************************/ -/************** Begin file fts3.c ********************************************/ -/* -** 2006 Oct 10 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This is an SQLite module implementing full-text search. -*/ /* -** The code in this file is only compiled if: -** -** * The FTS3 module is being built as an extension -** (in which case SQLITE_CORE is not defined), or -** -** * The FTS3 module is being built into the core of -** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). -*/ - -/* The full-text index is stored in a series of b+tree (-like) -** structures called segments which map terms to doclists. The -** structures are like b+trees in layout, but are constructed from the -** bottom up in optimal fashion and are not updatable. Since trees -** are built from the bottom up, things will be described from the -** bottom up. -** -** -**** Varints **** -** The basic unit of encoding is a variable-length integer called a -** varint. We encode variable-length integers in little-endian order -** using seven bits * per byte as follows: -** -** KEY: -** A = 0xxxxxxx 7 bits of data and one flag bit -** B = 1xxxxxxx 7 bits of data and one flag bit -** -** 7 bits - A -** 14 bits - BA -** 21 bits - BBA -** and so on. -** -** This is similar in concept to how sqlite encodes "varints" but -** the encoding is not the same. SQLite varints are big-endian -** are are limited to 9 bytes in length whereas FTS3 varints are -** little-endian and can be up to 10 bytes in length (in theory). -** -** Example encodings: -** -** 1: 0x01 -** 127: 0x7f -** 128: 0x81 0x00 -** -** -**** Document lists **** -** A doclist (document list) holds a docid-sorted list of hits for a -** given term. Doclists hold docids and associated token positions. -** A docid is the unique integer identifier for a single document. -** A position is the index of a word within the document. The first -** word of the document has a position of 0. -** -** FTS3 used to optionally store character offsets using a compile-time -** option. But that functionality is no longer supported. -** -** A doclist is stored like this: -** -** array { -** varint docid; (delta from previous doclist) -** array { (position list for column 0) -** varint position; (2 more than the delta from previous position) -** } -** array { -** varint POS_COLUMN; (marks start of position list for new column) -** varint column; (index of new column) -** array { -** varint position; (2 more than the delta from previous position) -** } -** } -** varint POS_END; (marks end of positions for this document. -** } -** -** Here, array { X } means zero or more occurrences of X, adjacent in -** memory. A "position" is an index of a token in the token stream -** generated by the tokenizer. Note that POS_END and POS_COLUMN occur -** in the same logical place as the position element, and act as sentinals -** ending a position list array. POS_END is 0. POS_COLUMN is 1. -** The positions numbers are not stored literally but rather as two more -** than the difference from the prior position, or the just the position plus -** 2 for the first position. Example: -** -** label: A B C D E F G H I J K -** value: 123 5 9 1 1 14 35 0 234 72 0 -** -** The 123 value is the first docid. For column zero in this document -** there are two matches at positions 3 and 10 (5-2 and 9-2+3). The 1 -** at D signals the start of a new column; the 1 at E indicates that the -** new column is column number 1. There are two positions at 12 and 45 -** (14-2 and 35-2+12). The 0 at H indicate the end-of-document. The -** 234 at I is the delta to next docid (357). It has one position 70 -** (72-2) and then terminates with the 0 at K. -** -** A "position-list" is the list of positions for multiple columns for -** a single docid. A "column-list" is the set of positions for a single -** column. Hence, a position-list consists of one or more column-lists, -** a document record consists of a docid followed by a position-list and -** a doclist consists of one or more document records. -** -** A bare doclist omits the position information, becoming an -** array of varint-encoded docids. -** -**** Segment leaf nodes **** -** Segment leaf nodes store terms and doclists, ordered by term. Leaf -** nodes are written using LeafWriter, and read using LeafReader (to -** iterate through a single leaf node's data) and LeavesReader (to -** iterate through a segment's entire leaf layer). Leaf nodes have -** the format: -** -** varint iHeight; (height from leaf level, always 0) -** varint nTerm; (length of first term) -** char pTerm[nTerm]; (content of first term) -** varint nDoclist; (length of term's associated doclist) -** char pDoclist[nDoclist]; (content of doclist) -** array { -** (further terms are delta-encoded) -** varint nPrefix; (length of prefix shared with previous term) -** varint nSuffix; (length of unshared suffix) -** char pTermSuffix[nSuffix];(unshared suffix of next term) -** varint nDoclist; (length of term's associated doclist) -** char pDoclist[nDoclist]; (content of doclist) -** } -** -** Here, array { X } means zero or more occurrences of X, adjacent in -** memory. -** -** Leaf nodes are broken into blocks which are stored contiguously in -** the %_segments table in sorted order. This means that when the end -** of a node is reached, the next term is in the node with the next -** greater node id. -** -** New data is spilled to a new leaf node when the current node -** exceeds LEAF_MAX bytes (default 2048). New data which itself is -** larger than STANDALONE_MIN (default 1024) is placed in a standalone -** node (a leaf node with a single term and doclist). The goal of -** these settings is to pack together groups of small doclists while -** making it efficient to directly access large doclists. The -** assumption is that large doclists represent terms which are more -** likely to be query targets. -** -** TODO(shess) It may be useful for blocking decisions to be more -** dynamic. For instance, it may make more sense to have a 2.5k leaf -** node rather than splitting into 2k and .5k nodes. My intuition is -** that this might extend through 2x or 4x the pagesize. +** fts3ExprIterate() callback used to collect the "global" matchinfo stats +** for a single query. ** +** fts3ExprIterate() callback to load the 'global' elements of a +** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements +** of the matchinfo array that are constant for all rows returned by the +** current query. ** -**** Segment interior nodes **** -** Segment interior nodes store blockids for subtree nodes and terms -** to describe what data is stored by the each subtree. Interior -** nodes are written using InteriorWriter, and read using -** InteriorReader. InteriorWriters are created as needed when -** SegmentWriter creates new leaf nodes, or when an interior node -** itself grows too big and must be split. The format of interior -** nodes: +** Argument pCtx is actually a pointer to a struct of type MatchInfo. This +** function populates Matchinfo.aMatchinfo[] as follows: ** -** varint iHeight; (height from leaf level, always >0) -** varint iBlockid; (block id of node's leftmost subtree) -** optional { -** varint nTerm; (length of first term) -** char pTerm[nTerm]; (content of first term) -** array { -** (further terms are delta-encoded) -** varint nPrefix; (length of shared prefix with previous term) -** varint nSuffix; (length of unshared suffix) -** char pTermSuffix[nSuffix]; (unshared suffix of next term) +** for(iCol=0; iColpCursor, pExpr, &p->aMatchinfo[3*iPhrase*p->nCol] + ); +} /* -** FTS4 is really an extension for FTS3. It is enabled using the -** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all -** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3. +** fts3ExprIterate() callback used to collect the "local" part of the +** FTS3_MATCHINFO_HITS array. The local stats are those elements of the +** array that are different for each row returned by the query. */ -#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) -# define SQLITE_ENABLE_FTS3 -#endif +static int fts3ExprLocalHitsCb( + Fts3Expr *pExpr, /* Phrase expression node */ + int iPhrase, /* Phrase number */ + void *pCtx /* Pointer to MatchInfo structure */ +){ + int rc = SQLITE_OK; + MatchInfo *p = (MatchInfo *)pCtx; + int iStart = iPhrase * p->nCol * 3; + int i; -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + for(i=0; inCol && rc==SQLITE_OK; i++){ + char *pCsr; + rc = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i, &pCsr); + if( pCsr ){ + p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr); + }else{ + p->aMatchinfo[iStart+i*3] = 0; + } + } -/* If not building as part of the core, include sqlite3ext.h. */ -#ifndef SQLITE_CORE -SQLITE_EXTENSION_INIT3 -#endif + return rc; +} -/************** Include fts3_tokenizer.h in the middle of fts3Int.h **********/ -/************** Begin file fts3_tokenizer.h **********************************/ -/* -** 2006 July 10 -** -** The author disclaims copyright to this source code. -** -************************************************************************* -** Defines the interface to tokenizers used by fulltext-search. There -** are three basic components: -** -** sqlite3_tokenizer_module is a singleton defining the tokenizer -** interface functions. This is essentially the class structure for -** tokenizers. -** -** sqlite3_tokenizer is used to define a particular tokenizer, perhaps -** including customization information defined at creation time. -** -** sqlite3_tokenizer_cursor is generated by a tokenizer to generate -** tokens from a particular input. -*/ -#ifndef _FTS3_TOKENIZER_H_ -#define _FTS3_TOKENIZER_H_ +static int fts3MatchinfoCheck( + Fts3Table *pTab, + char cArg, + char **pzErr +){ + if( (cArg==FTS3_MATCHINFO_NPHRASE) + || (cArg==FTS3_MATCHINFO_NCOL) + || (cArg==FTS3_MATCHINFO_NDOC && pTab->bFts4) + || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bFts4) + || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize) + || (cArg==FTS3_MATCHINFO_LCS) + || (cArg==FTS3_MATCHINFO_HITS) + || (cArg==FTS3_MATCHINFO_LHITS) + || (cArg==FTS3_MATCHINFO_LHITS_BM) + ){ + return SQLITE_OK; + } + sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo request: %c", cArg); + return SQLITE_ERROR; +} -/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. -** If tokenizers are to be allowed to call sqlite3_*() functions, then -** we will need a way to register the API consistently. -*/ +static int fts3MatchinfoSize(MatchInfo *pInfo, char cArg){ + int nVal; /* Number of integers output by cArg */ -/* -** Structures used by the tokenizer interface. When a new tokenizer -** implementation is registered, the caller provides a pointer to -** an sqlite3_tokenizer_module containing pointers to the callback -** functions that make up an implementation. -** -** When an fts3 table is created, it passes any arguments passed to -** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the -** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer -** implementation. The xCreate() function in turn returns an -** sqlite3_tokenizer structure representing the specific tokenizer to -** be used for the fts3 table (customized by the tokenizer clause arguments). -** -** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen() -** method is called. It returns an sqlite3_tokenizer_cursor object -** that may be used to tokenize a specific input buffer based on -** the tokenization rules supplied by a specific sqlite3_tokenizer -** object. -*/ -typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; -typedef struct sqlite3_tokenizer sqlite3_tokenizer; -typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; + switch( cArg ){ + case FTS3_MATCHINFO_NDOC: + case FTS3_MATCHINFO_NPHRASE: + case FTS3_MATCHINFO_NCOL: + nVal = 1; + break; -struct sqlite3_tokenizer_module { + case FTS3_MATCHINFO_AVGLENGTH: + case FTS3_MATCHINFO_LENGTH: + case FTS3_MATCHINFO_LCS: + nVal = pInfo->nCol; + break; - /* - ** Structure version. Should always be set to 0 or 1. - */ - int iVersion; + case FTS3_MATCHINFO_LHITS: + nVal = pInfo->nCol * pInfo->nPhrase; + break; - /* - ** Create a new tokenizer. The values in the argv[] array are the - ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL - ** TABLE statement that created the fts3 table. For example, if - ** the following SQL is executed: - ** - ** CREATE .. USING fts3( ... , tokenizer arg1 arg2) - ** - ** then argc is set to 2, and the argv[] array contains pointers - ** to the strings "arg1" and "arg2". - ** - ** This method should return either SQLITE_OK (0), or an SQLite error - ** code. If SQLITE_OK is returned, then *ppTokenizer should be set - ** to point at the newly created tokenizer structure. The generic - ** sqlite3_tokenizer.pModule variable should not be initialized by - ** this callback. The caller will do so. - */ - int (*xCreate)( - int argc, /* Size of argv array */ - const char *const*argv, /* Tokenizer argument strings */ - sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ - ); + case FTS3_MATCHINFO_LHITS_BM: + nVal = pInfo->nPhrase * ((pInfo->nCol + 31) / 32); + break; - /* - ** Destroy an existing tokenizer. The fts3 module calls this method - ** exactly once for each successful call to xCreate(). - */ - int (*xDestroy)(sqlite3_tokenizer *pTokenizer); + default: + assert( cArg==FTS3_MATCHINFO_HITS ); + nVal = pInfo->nCol * pInfo->nPhrase * 3; + break; + } - /* - ** Create a tokenizer cursor to tokenize an input buffer. The caller - ** is responsible for ensuring that the input buffer remains valid - ** until the cursor is closed (using the xClose() method). - */ - int (*xOpen)( - sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ - const char *pInput, int nBytes, /* Input buffer */ - sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ - ); + return nVal; +} - /* - ** Destroy an existing tokenizer cursor. The fts3 module calls this - ** method exactly once for each successful call to xOpen(). - */ - int (*xClose)(sqlite3_tokenizer_cursor *pCursor); +static int fts3MatchinfoSelectDoctotal( + Fts3Table *pTab, + sqlite3_stmt **ppStmt, + sqlite3_int64 *pnDoc, + const char **paLen +){ + sqlite3_stmt *pStmt; + const char *a; + sqlite3_int64 nDoc; - /* - ** Retrieve the next token from the tokenizer cursor pCursor. This - ** method should either return SQLITE_OK and set the values of the - ** "OUT" variables identified below, or SQLITE_DONE to indicate that - ** the end of the buffer has been reached, or an SQLite error code. - ** - ** *ppToken should be set to point at a buffer containing the - ** normalized version of the token (i.e. after any case-folding and/or - ** stemming has been performed). *pnBytes should be set to the length - ** of this buffer in bytes. The input text that generated the token is - ** identified by the byte offsets returned in *piStartOffset and - ** *piEndOffset. *piStartOffset should be set to the index of the first - ** byte of the token in the input buffer. *piEndOffset should be set - ** to the index of the first byte just past the end of the token in - ** the input buffer. - ** - ** The buffer *ppToken is set to point at is managed by the tokenizer - ** implementation. It is only required to be valid until the next call - ** to xNext() or xClose(). - */ - /* TODO(shess) current implementation requires pInput to be - ** nul-terminated. This should either be fixed, or pInput/nBytes - ** should be converted to zInput. - */ - int (*xNext)( - sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ - const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ - int *piStartOffset, /* OUT: Byte offset of token in input buffer */ - int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ - int *piPosition /* OUT: Number of tokens returned before this one */ - ); + if( !*ppStmt ){ + int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt); + if( rc!=SQLITE_OK ) return rc; + } + pStmt = *ppStmt; + assert( sqlite3_data_count(pStmt)==1 ); - /*********************************************************************** - ** Methods below this point are only available if iVersion>=1. - */ + a = sqlite3_column_blob(pStmt, 0); + a += sqlite3Fts3GetVarint(a, &nDoc); + if( nDoc==0 ) return FTS_CORRUPT_VTAB; + *pnDoc = (u32)nDoc; - /* - ** Configure the language id of a tokenizer cursor. - */ - int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid); -}; + if( paLen ) *paLen = a; + return SQLITE_OK; +} -struct sqlite3_tokenizer { - const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ - /* Tokenizer implementations will typically add additional fields */ +/* +** An instance of the following structure is used to store state while +** iterating through a multi-column position-list corresponding to the +** hits for a single phrase on a single row in order to calculate the +** values for a matchinfo() FTS3_MATCHINFO_LCS request. +*/ +typedef struct LcsIterator LcsIterator; +struct LcsIterator { + Fts3Expr *pExpr; /* Pointer to phrase expression */ + int iPosOffset; /* Tokens count up to end of this phrase */ + char *pRead; /* Cursor used to iterate through aDoclist */ + int iPos; /* Current position */ }; -struct sqlite3_tokenizer_cursor { - sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ - /* Tokenizer implementations will typically add additional fields */ -}; +/* +** If LcsIterator.iCol is set to the following value, the iterator has +** finished iterating through all offsets for all columns. +*/ +#define LCS_ITERATOR_FINISHED 0x7FFFFFFF; -int fts3_global_term_cnt(int iTerm, int iCol); -int fts3_term_cnt(int iTerm, int iCol); +static int fts3MatchinfoLcsCb( + Fts3Expr *pExpr, /* Phrase expression node */ + int iPhrase, /* Phrase number (numbered from zero) */ + void *pCtx /* Pointer to MatchInfo structure */ +){ + LcsIterator *aIter = (LcsIterator *)pCtx; + aIter[iPhrase].pExpr = pExpr; + return SQLITE_OK; +} +/* +** Advance the iterator passed as an argument to the next position. Return +** 1 if the iterator is at EOF or if it now points to the start of the +** position list for the next column. +*/ +static int fts3LcsIteratorAdvance(LcsIterator *pIter){ + char *pRead = pIter->pRead; + sqlite3_int64 iRead; + int rc = 0; -#endif /* _FTS3_TOKENIZER_H_ */ + pRead += sqlite3Fts3GetVarint(pRead, &iRead); + if( iRead==0 || iRead==1 ){ + pRead = 0; + rc = 1; + }else{ + pIter->iPos += (int)(iRead-2); + } -/************** End of fts3_tokenizer.h **************************************/ -/************** Continuing where we left off in fts3Int.h ********************/ -/************** Include fts3_hash.h in the middle of fts3Int.h ***************/ -/************** Begin file fts3_hash.h ***************************************/ + pIter->pRead = pRead; + return rc; +} + /* -** 2001 September 22 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** This function implements the FTS3_MATCHINFO_LCS matchinfo() flag. ** -************************************************************************* -** This is the header file for the generic hash-table implementation -** used in SQLite. We've modified it slightly to serve as a standalone -** hash table implementation for the full-text indexing module. +** If the call is successful, the longest-common-substring lengths for each +** column are written into the first nCol elements of the pInfo->aMatchinfo[] +** array before returning. SQLITE_OK is returned in this case. ** +** Otherwise, if an error occurs, an SQLite error code is returned and the +** data written to the first nCol elements of pInfo->aMatchinfo[] is +** undefined. */ -#ifndef _FTS3_HASH_H_ -#define _FTS3_HASH_H_ +static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ + LcsIterator *aIter; + int i; + int iCol; + int nToken = 0; -/* Forward declarations of structures. */ -typedef struct Fts3Hash Fts3Hash; -typedef struct Fts3HashElem Fts3HashElem; + /* Allocate and populate the array of LcsIterator objects. The array + ** contains one element for each matchable phrase in the query. + **/ + aIter = sqlite3_malloc(sizeof(LcsIterator) * pCsr->nPhrase); + if( !aIter ) return SQLITE_NOMEM; + memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase); + (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); -/* A complete hash table is an instance of the following structure. -** The internals of this structure are intended to be opaque -- client -** code should not attempt to access or modify the fields of this structure -** directly. Change this structure only by using the routines below. -** However, many of the "procedures" and "functions" for modifying and -** accessing this structure are really macros, so we can't really make -** this structure opaque. -*/ -struct Fts3Hash { - char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */ - char copyKey; /* True if copy of key made on insert */ - int count; /* Number of entries in this table */ - Fts3HashElem *first; /* The first element of the array */ - int htsize; /* Number of buckets in the hash table */ - struct _fts3ht { /* the hash table */ - int count; /* Number of entries with this hash */ - Fts3HashElem *chain; /* Pointer to first entry with this hash */ - } *ht; -}; + for(i=0; inPhrase; i++){ + LcsIterator *pIter = &aIter[i]; + nToken -= pIter->pExpr->pPhrase->nToken; + pIter->iPosOffset = nToken; + } -/* Each element in the hash table is an instance of the following -** structure. All elements are stored on a single doubly-linked list. -** -** Again, this structure is intended to be opaque, but it can't really -** be opaque because it is used by macros. -*/ -struct Fts3HashElem { - Fts3HashElem *next, *prev; /* Next and previous elements in the table */ - void *data; /* Data associated with this element */ - void *pKey; int nKey; /* Key associated with this element */ -}; + for(iCol=0; iColnCol; iCol++){ + int nLcs = 0; /* LCS value for this column */ + int nLive = 0; /* Number of iterators in aIter not at EOF */ -/* -** There are 2 different modes of operation for a hash table: -** -** FTS3_HASH_STRING pKey points to a string that is nKey bytes long -** (including the null-terminator, if any). Case -** is respected in comparisons. -** -** FTS3_HASH_BINARY pKey points to binary data nKey bytes long. -** memcmp() is used to compare keys. -** -** A copy of the key is made if the copyKey parameter to fts3HashInit is 1. -*/ -#define FTS3_HASH_STRING 1 -#define FTS3_HASH_BINARY 2 + for(i=0; inPhrase; i++){ + int rc; + LcsIterator *pIt = &aIter[i]; + rc = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol, &pIt->pRead); + if( rc!=SQLITE_OK ) return rc; + if( pIt->pRead ){ + pIt->iPos = pIt->iPosOffset; + fts3LcsIteratorAdvance(&aIter[i]); + nLive++; + } + } -/* -** Access routines. To delete, insert a NULL pointer. -*/ -SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey); -SQLITE_PRIVATE void *sqlite3Fts3HashInsert(Fts3Hash*, const void *pKey, int nKey, void *pData); -SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash*, const void *pKey, int nKey); -SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash*); -SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const void *, int); + while( nLive>0 ){ + LcsIterator *pAdv = 0; /* The iterator to advance by one position */ + int nThisLcs = 0; /* LCS for the current iterator positions */ -/* -** Shorthand for the functions above -*/ -#define fts3HashInit sqlite3Fts3HashInit -#define fts3HashInsert sqlite3Fts3HashInsert -#define fts3HashFind sqlite3Fts3HashFind -#define fts3HashClear sqlite3Fts3HashClear -#define fts3HashFindElem sqlite3Fts3HashFindElem + for(i=0; inPhrase; i++){ + LcsIterator *pIter = &aIter[i]; + if( pIter->pRead==0 ){ + /* This iterator is already at EOF for this column. */ + nThisLcs = 0; + }else{ + if( pAdv==0 || pIter->iPosiPos ){ + pAdv = pIter; + } + if( nThisLcs==0 || pIter->iPos==pIter[-1].iPos ){ + nThisLcs++; + }else{ + nThisLcs = 1; + } + if( nThisLcs>nLcs ) nLcs = nThisLcs; + } + } + if( fts3LcsIteratorAdvance(pAdv) ) nLive--; + } + + pInfo->aMatchinfo[iCol] = nLcs; + } + + sqlite3_free(aIter); + return SQLITE_OK; +} /* -** Macros for looping over all elements of a hash table. The idiom is -** like this: +** Populate the buffer pInfo->aMatchinfo[] with an array of integers to +** be returned by the matchinfo() function. Argument zArg contains the +** format string passed as the second argument to matchinfo (or the +** default value "pcx" if no second argument was specified). The format +** string has already been validated and the pInfo->aMatchinfo[] array +** is guaranteed to be large enough for the output. ** -** Fts3Hash h; -** Fts3HashElem *p; -** ... -** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){ -** SomeStructure *pData = fts3HashData(p); -** // do something with pData -** } +** If bGlobal is true, then populate all fields of the matchinfo() output. +** If it is false, then assume that those fields that do not change between +** rows (i.e. FTS3_MATCHINFO_NPHRASE, NCOL, NDOC, AVGLENGTH and part of HITS) +** have already been populated. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. If a value other than SQLITE_OK is returned, the state the +** pInfo->aMatchinfo[] buffer is left in is undefined. */ -#define fts3HashFirst(H) ((H)->first) -#define fts3HashNext(E) ((E)->next) -#define fts3HashData(E) ((E)->data) -#define fts3HashKey(E) ((E)->pKey) -#define fts3HashKeysize(E) ((E)->nKey) +static int fts3MatchinfoValues( + Fts3Cursor *pCsr, /* FTS3 cursor object */ + int bGlobal, /* True to grab the global stats */ + MatchInfo *pInfo, /* Matchinfo context object */ + const char *zArg /* Matchinfo format string */ +){ + int rc = SQLITE_OK; + int i; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + sqlite3_stmt *pSelect = 0; -/* -** Number of entries in a hash table -*/ -#define fts3HashCount(H) ((H)->count) + for(i=0; rc==SQLITE_OK && zArg[i]; i++){ + pInfo->flag = zArg[i]; + switch( zArg[i] ){ + case FTS3_MATCHINFO_NPHRASE: + if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase; + break; -#endif /* _FTS3_HASH_H_ */ + case FTS3_MATCHINFO_NCOL: + if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nCol; + break; + + case FTS3_MATCHINFO_NDOC: + if( bGlobal ){ + sqlite3_int64 nDoc = 0; + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0); + pInfo->aMatchinfo[0] = (u32)nDoc; + } + break; -/************** End of fts3_hash.h *******************************************/ -/************** Continuing where we left off in fts3Int.h ********************/ + case FTS3_MATCHINFO_AVGLENGTH: + if( bGlobal ){ + sqlite3_int64 nDoc; /* Number of rows in table */ + const char *a; /* Aggregate column length array */ -/* -** This constant determines the maximum depth of an FTS expression tree -** that the library will create and use. FTS uses recursion to perform -** various operations on the query tree, so the disadvantage of a large -** limit is that it may allow very large queries to use large amounts -** of stack space (perhaps causing a stack overflow). -*/ -#ifndef SQLITE_FTS3_MAX_EXPR_DEPTH -# define SQLITE_FTS3_MAX_EXPR_DEPTH 12 -#endif + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a); + if( rc==SQLITE_OK ){ + int iCol; + for(iCol=0; iColnCol; iCol++){ + u32 iVal; + sqlite3_int64 nToken; + a += sqlite3Fts3GetVarint(a, &nToken); + iVal = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc); + pInfo->aMatchinfo[iCol] = iVal; + } + } + } + break; + case FTS3_MATCHINFO_LENGTH: { + sqlite3_stmt *pSelectDocsize = 0; + rc = sqlite3Fts3SelectDocsize(pTab, pCsr->iPrevId, &pSelectDocsize); + if( rc==SQLITE_OK ){ + int iCol; + const char *a = sqlite3_column_blob(pSelectDocsize, 0); + for(iCol=0; iColnCol; iCol++){ + sqlite3_int64 nToken; + a += sqlite3Fts3GetVarint(a, &nToken); + pInfo->aMatchinfo[iCol] = (u32)nToken; + } + } + sqlite3_reset(pSelectDocsize); + break; + } -/* -** This constant controls how often segments are merged. Once there are -** FTS3_MERGE_COUNT segments of level N, they are merged into a single -** segment of level N+1. -*/ -#define FTS3_MERGE_COUNT 16 + case FTS3_MATCHINFO_LCS: + rc = fts3ExprLoadDoclists(pCsr, 0, 0); + if( rc==SQLITE_OK ){ + rc = fts3MatchinfoLcs(pCsr, pInfo); + } + break; -/* -** This is the maximum amount of data (in bytes) to store in the -** Fts3Table.pendingTerms hash table. Normally, the hash table is -** populated as documents are inserted/updated/deleted in a transaction -** and used to create a new segment when the transaction is committed. -** However if this limit is reached midway through a transaction, a new -** segment is created and the hash table cleared immediately. -*/ -#define FTS3_MAX_PENDING_DATA (1*1024*1024) + case FTS3_MATCHINFO_LHITS_BM: + case FTS3_MATCHINFO_LHITS: { + int nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32); + memset(pInfo->aMatchinfo, 0, nZero); + fts3ExprLHitGather(pCsr->pExpr, pInfo); + break; + } -/* -** Macro to return the number of elements in an array. SQLite has a -** similar macro called ArraySize(). Use a different name to avoid -** a collision when building an amalgamation with built-in FTS3. -*/ -#define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0]))) + default: { + Fts3Expr *pExpr; + assert( zArg[i]==FTS3_MATCHINFO_HITS ); + pExpr = pCsr->pExpr; + rc = fts3ExprLoadDoclists(pCsr, 0, 0); + if( rc!=SQLITE_OK ) break; + if( bGlobal ){ + if( pCsr->pDeferred ){ + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc, 0); + if( rc!=SQLITE_OK ) break; + } + rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo); + sqlite3Fts3EvalTestDeferred(pCsr, &rc); + if( rc!=SQLITE_OK ) break; + } + (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo); + break; + } + } + pInfo->aMatchinfo += fts3MatchinfoSize(pInfo, zArg[i]); + } -#ifndef MIN -# define MIN(x,y) ((x)<(y)?(x):(y)) -#endif -#ifndef MAX -# define MAX(x,y) ((x)>(y)?(x):(y)) -#endif + sqlite3_reset(pSelect); + return rc; +} -/* -** Maximum length of a varint encoded integer. The varint format is different -** from that used by SQLite, so the maximum length is 10, not 9. -*/ -#define FTS3_VARINT_MAX 10 /* -** FTS4 virtual tables may maintain multiple indexes - one index of all terms -** in the document set and zero or more prefix indexes. All indexes are stored -** as one or more b+-trees in the %_segments and %_segdir tables. -** -** It is possible to determine which index a b+-tree belongs to based on the -** value stored in the "%_segdir.level" column. Given this value L, the index -** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with -** level values between 0 and 1023 (inclusive) belong to index 0, all levels -** between 1024 and 2047 to index 1, and so on. -** -** It is considered impossible for an index to use more than 1024 levels. In -** theory though this may happen, but only after at least -** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables. +** Populate pCsr->aMatchinfo[] with data for the current row. The +** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32). */ -#define FTS3_SEGDIR_MAXLEVEL 1024 -#define FTS3_SEGDIR_MAXLEVEL_STR "1024" +static void fts3GetMatchinfo( + sqlite3_context *pCtx, /* Return results here */ + Fts3Cursor *pCsr, /* FTS3 Cursor object */ + const char *zArg /* Second argument to matchinfo() function */ +){ + MatchInfo sInfo; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int bGlobal = 0; /* Collect 'global' stats as well as local */ -/* -** The testcase() macro is only used by the amalgamation. If undefined, -** make it a no-op. -*/ -#ifndef testcase -# define testcase(X) -#endif + u32 *aOut = 0; + void (*xDestroyOut)(void*) = 0; -/* -** Terminator values for position-lists and column-lists. -*/ -#define POS_COLUMN (1) /* Column-list terminator */ -#define POS_END (0) /* Position-list terminator */ + memset(&sInfo, 0, sizeof(MatchInfo)); + sInfo.pCursor = pCsr; + sInfo.nCol = pTab->nColumn; -/* -** This section provides definitions to allow the -** FTS3 extension to be compiled outside of the -** amalgamation. -*/ -#ifndef SQLITE_AMALGAMATION -/* -** Macros indicating that conditional expressions are always true or -** false. -*/ -#ifdef SQLITE_COVERAGE_TEST -# define ALWAYS(x) (1) -# define NEVER(X) (0) -#elif defined(SQLITE_DEBUG) -# define ALWAYS(x) sqlite3Fts3Always((x)!=0) -# define NEVER(x) sqlite3Fts3Never((x)!=0) -SQLITE_PRIVATE int sqlite3Fts3Always(int b); -SQLITE_PRIVATE int sqlite3Fts3Never(int b); -#else -# define ALWAYS(x) (x) -# define NEVER(x) (x) -#endif + /* If there is cached matchinfo() data, but the format string for the + ** cache does not match the format string for this request, discard + ** the cached data. */ + if( pCsr->pMIBuffer && strcmp(pCsr->pMIBuffer->zMatchinfo, zArg) ){ + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); + pCsr->pMIBuffer = 0; + } -/* -** Internal types used by SQLite. -*/ -typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ -typedef short int i16; /* 2-byte (or larger) signed integer */ -typedef unsigned int u32; /* 4-byte unsigned integer */ -typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ -typedef sqlite3_int64 i64; /* 8-byte signed integer */ + /* If Fts3Cursor.pMIBuffer is NULL, then this is the first time the + ** matchinfo function has been called for this query. In this case + ** allocate the array used to accumulate the matchinfo data and + ** initialize those elements that are constant for every row. + */ + if( pCsr->pMIBuffer==0 ){ + int nMatchinfo = 0; /* Number of u32 elements in match-info */ + int i; /* Used to iterate through zArg */ -/* -** Macro used to suppress compiler warnings for unused parameters. -*/ -#define UNUSED_PARAMETER(x) (void)(x) + /* Determine the number of phrases in the query */ + pCsr->nPhrase = fts3ExprPhraseCount(pCsr->pExpr); + sInfo.nPhrase = pCsr->nPhrase; -/* -** Activate assert() only if SQLITE_TEST is enabled. -*/ -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) -# define NDEBUG 1 -#endif + /* Determine the number of integers in the buffer returned by this call. */ + for(i=0; zArg[i]; i++){ + char *zErr = 0; + if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){ + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); + return; + } + nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]); + } -/* -** The TESTONLY macro is used to enclose variable declarations or -** other bits of code that are needed to support the arguments -** within testcase() and assert() macros. -*/ -#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) -# define TESTONLY(X) X -#else -# define TESTONLY(X) -#endif + /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */ + pCsr->pMIBuffer = fts3MIBufferNew(nMatchinfo, zArg); + if( !pCsr->pMIBuffer ) rc = SQLITE_NOMEM; -#endif /* SQLITE_AMALGAMATION */ + pCsr->isMatchinfoNeeded = 1; + bGlobal = 1; + } -#ifdef SQLITE_DEBUG -SQLITE_PRIVATE int sqlite3Fts3Corrupt(void); -# define FTS_CORRUPT_VTAB sqlite3Fts3Corrupt() -#else -# define FTS_CORRUPT_VTAB SQLITE_CORRUPT_VTAB -#endif + if( rc==SQLITE_OK ){ + xDestroyOut = fts3MIBufferAlloc(pCsr->pMIBuffer, &aOut); + if( xDestroyOut==0 ){ + rc = SQLITE_NOMEM; + } + } -typedef struct Fts3Table Fts3Table; -typedef struct Fts3Cursor Fts3Cursor; -typedef struct Fts3Expr Fts3Expr; -typedef struct Fts3Phrase Fts3Phrase; -typedef struct Fts3PhraseToken Fts3PhraseToken; + if( rc==SQLITE_OK ){ + sInfo.aMatchinfo = aOut; + sInfo.nPhrase = pCsr->nPhrase; + rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg); + if( bGlobal ){ + fts3MIBufferSetGlobal(pCsr->pMIBuffer); + } + } -typedef struct Fts3Doclist Fts3Doclist; -typedef struct Fts3SegFilter Fts3SegFilter; -typedef struct Fts3DeferredToken Fts3DeferredToken; -typedef struct Fts3SegReader Fts3SegReader; -typedef struct Fts3MultiSegReader Fts3MultiSegReader; + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + if( xDestroyOut ) xDestroyOut(aOut); + }else{ + int n = pCsr->pMIBuffer->nElem * sizeof(u32); + sqlite3_result_blob(pCtx, aOut, n, xDestroyOut); + } +} /* -** A connection to a fulltext index is an instance of the following -** structure. The xCreate and xConnect methods create an instance -** of this structure and xDestroy and xDisconnect free that instance. -** All other methods receive a pointer to the structure as one of their -** arguments. +** Implementation of snippet() function. */ -struct Fts3Table { - sqlite3_vtab base; /* Base class used by SQLite core */ - sqlite3 *db; /* The database connection */ - const char *zDb; /* logical database name */ - const char *zName; /* virtual table name */ - int nColumn; /* number of named columns in virtual table */ - char **azColumn; /* column names. malloced */ - u8 *abNotindexed; /* True for 'notindexed' columns */ - sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ - char *zContentTbl; /* content=xxx option, or NULL */ - char *zLanguageid; /* languageid=xxx option, or NULL */ - int nAutoincrmerge; /* Value configured by 'automerge' */ - u32 nLeafAdd; /* Number of leaf blocks added this trans */ +SQLITE_PRIVATE void sqlite3Fts3Snippet( + sqlite3_context *pCtx, /* SQLite function call context */ + Fts3Cursor *pCsr, /* Cursor object */ + const char *zStart, /* Snippet start text - "" */ + const char *zEnd, /* Snippet end text - "" */ + const char *zEllipsis, /* Snippet ellipsis text - "..." */ + int iCol, /* Extract snippet from this column */ + int nToken /* Approximate number of tokens in snippet */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int i; + StrBuffer res = {0, 0, 0}; - /* Precompiled statements used by the implementation. Each of these - ** statements is run and reset within a single virtual table API call. + /* The returned text includes up to four fragments of text extracted from + ** the data in the current row. The first iteration of the for(...) loop + ** below attempts to locate a single fragment of text nToken tokens in + ** size that contains at least one instance of all phrases in the query + ** expression that appear in the current row. If such a fragment of text + ** cannot be found, the second iteration of the loop attempts to locate + ** a pair of fragments, and so on. */ - sqlite3_stmt *aStmt[40]; + int nSnippet = 0; /* Number of fragments in this snippet */ + SnippetFragment aSnippet[4]; /* Maximum of 4 fragments per snippet */ + int nFToken = -1; /* Number of tokens in each fragment */ - char *zReadExprlist; - char *zWriteExprlist; + if( !pCsr->pExpr ){ + sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); + return; + } - int nNodeSize; /* Soft limit for node size */ - u8 bFts4; /* True for FTS4, false for FTS3 */ - u8 bHasStat; /* True if %_stat table exists (2==unknown) */ - u8 bHasDocsize; /* True if %_docsize table exists */ - u8 bDescIdx; /* True if doclists are in reverse order */ - u8 bIgnoreSavepoint; /* True to ignore xSavepoint invocations */ - int nPgsz; /* Page size for host database */ - char *zSegmentsTbl; /* Name of %_segments table */ - sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ + for(nSnippet=1; 1; nSnippet++){ - /* - ** The following array of hash tables is used to buffer pending index - ** updates during transactions. All pending updates buffered at any one - ** time must share a common language-id (see the FTS4 langid= feature). - ** The current language id is stored in variable iPrevLangid. - ** - ** A single FTS4 table may have multiple full-text indexes. For each index - ** there is an entry in the aIndex[] array. Index 0 is an index of all the - ** terms that appear in the document set. Each subsequent index in aIndex[] - ** is an index of prefixes of a specific length. - ** - ** Variable nPendingData contains an estimate the memory consumed by the - ** pending data structures, including hash table overhead, but not including - ** malloc overhead. When nPendingData exceeds nMaxPendingData, all hash - ** tables are flushed to disk. Variable iPrevDocid is the docid of the most - ** recently inserted record. - */ - int nIndex; /* Size of aIndex[] */ - struct Fts3Index { - int nPrefix; /* Prefix length (0 for main terms index) */ - Fts3Hash hPending; /* Pending terms table for this index */ - } *aIndex; - int nMaxPendingData; /* Max pending data before flush to disk */ - int nPendingData; /* Current bytes of pending data */ - sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ - int iPrevLangid; /* Langid of recently inserted document */ + int iSnip; /* Loop counter 0..nSnippet-1 */ + u64 mCovered = 0; /* Bitmask of phrases covered by snippet */ + u64 mSeen = 0; /* Bitmask of phrases seen by BestSnippet() */ -#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) - /* State variables used for validating that the transaction control - ** methods of the virtual table are called at appropriate times. These - ** values do not contribute to FTS functionality; they are used for - ** verifying the operation of the SQLite core. - */ - int inTransaction; /* True after xBegin but before xCommit/xRollback */ - int mxSavepoint; /* Largest valid xSavepoint integer */ -#endif + if( nToken>=0 ){ + nFToken = (nToken+nSnippet-1) / nSnippet; + }else{ + nFToken = -1 * nToken; + } -#ifdef SQLITE_TEST - /* True to disable the incremental doclist optimization. This is controled - ** by special insert command 'test-no-incr-doclist'. */ - int bNoIncrDoclist; -#endif -}; + for(iSnip=0; iSnipnColumn; iRead++){ + SnippetFragment sF = {0, 0, 0, 0}; + int iS = 0; + if( iCol>=0 && iRead!=iCol ) continue; -/* -** The Fts3Cursor.eSearch member is always set to one of the following. -** Actualy, Fts3Cursor.eSearch can be greater than or equal to -** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index -** of the column to be searched. For example, in -** -** CREATE VIRTUAL TABLE ex1 USING fts3(a,b,c,d); -** SELECT docid FROM ex1 WHERE b MATCH 'one two three'; -** -** Because the LHS of the MATCH operator is 2nd column "b", -** Fts3Cursor.eSearch will be set to FTS3_FULLTEXT_SEARCH+1. (+0 for a, -** +1 for b, +2 for c, +3 for d.) If the LHS of MATCH were "ex1" -** indicating that all columns should be searched, -** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4. -*/ -#define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */ -#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ -#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ + /* Find the best snippet of nFToken tokens in column iRead. */ + rc = fts3BestSnippet(nFToken, pCsr, iRead, mCovered, &mSeen, &sF, &iS); + if( rc!=SQLITE_OK ){ + goto snippet_out; + } + if( iS>iBestScore ){ + *pFragment = sF; + iBestScore = iS; + } + } -/* -** The lower 16-bits of the sqlite3_index_info.idxNum value set by -** the xBestIndex() method contains the Fts3Cursor.eSearch value described -** above. The upper 16-bits contain a combination of the following -** bits, used to describe extra constraints on full-text searches. -*/ -#define FTS3_HAVE_LANGID 0x00010000 /* languageid=? */ -#define FTS3_HAVE_DOCID_GE 0x00020000 /* docid>=? */ -#define FTS3_HAVE_DOCID_LE 0x00040000 /* docid<=? */ + mCovered |= pFragment->covered; + } -struct Fts3Doclist { - char *aAll; /* Array containing doclist (or NULL) */ - int nAll; /* Size of a[] in bytes */ - char *pNextDocid; /* Pointer to next docid */ + /* If all query phrases seen by fts3BestSnippet() are present in at least + ** one of the nSnippet snippet fragments, break out of the loop. + */ + assert( (mCovered&mSeen)==mCovered ); + if( mSeen==mCovered || nSnippet==SizeofArray(aSnippet) ) break; + } - sqlite3_int64 iDocid; /* Current docid (if pList!=0) */ - int bFreeList; /* True if pList should be sqlite3_free()d */ - char *pList; /* Pointer to position list following iDocid */ - int nList; /* Length of position list */ -}; + assert( nFToken>0 ); -/* -** A "phrase" is a sequence of one or more tokens that must match in -** sequence. A single token is the base case and the most common case. -** For a sequence of tokens contained in double-quotes (i.e. "one two three") -** nToken will be the number of tokens in the string. -*/ -struct Fts3PhraseToken { - char *z; /* Text of the token */ - int n; /* Number of bytes in buffer z */ - int isPrefix; /* True if token ends with a "*" character */ - int bFirst; /* True if token must appear at position 0 */ + for(i=0; ipLeft==this or pParent->pRight==this */ - Fts3Expr *pLeft; /* Left operand */ - Fts3Expr *pRight; /* Right operand */ - Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ - - /* The following are used by the fts3_eval.c module. */ - sqlite3_int64 iDocid; /* Current docid */ - u8 bEof; /* True this expression is at EOF already */ - u8 bStart; /* True if iDocid is valid */ - u8 bDeferred; /* True if this expression is entirely deferred */ - - u32 *aMI; +struct TermOffsetCtx { + Fts3Cursor *pCsr; + int iCol; /* Column of table to populate aTerm for */ + int iTerm; + sqlite3_int64 iDocid; + TermOffset *aTerm; }; /* -** Candidate values for Fts3Query.eType. Note that the order of the first -** four values is in order of precedence when parsing expressions. For -** example, the following: -** -** "a OR b AND c NOT d NEAR e" -** -** is equivalent to: -** -** "a OR (b AND (c NOT (d NEAR e)))" +** This function is an fts3ExprIterate() callback used by sqlite3Fts3Offsets(). */ -#define FTSQUERY_NEAR 1 -#define FTSQUERY_NOT 2 -#define FTSQUERY_AND 3 -#define FTSQUERY_OR 4 -#define FTSQUERY_PHRASE 5 - - -/* fts3_write.c */ -SQLITE_PRIVATE int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); -SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *); -SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *); -SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *); -SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(int, int, sqlite3_int64, - sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); -SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( - Fts3Table*,int,const char*,int,int,Fts3SegReader**); -SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *); -SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, int, sqlite3_stmt **); -SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); - -SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); -SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); +static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ + TermOffsetCtx *p = (TermOffsetCtx *)ctx; + int nTerm; /* Number of tokens in phrase */ + int iTerm; /* For looping through nTerm phrase terms */ + char *pList; /* Pointer to position list for phrase */ + int iPos = 0; /* First position in position-list */ + int rc; -#ifndef SQLITE_DISABLE_FTS4_DEFERRED -SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); -SQLITE_PRIVATE int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); -SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); -SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); -SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); -#else -# define sqlite3Fts3FreeDeferredTokens(x) -# define sqlite3Fts3DeferToken(x,y,z) SQLITE_OK -# define sqlite3Fts3CacheDeferredDoclists(x) SQLITE_OK -# define sqlite3Fts3FreeDeferredDoclists(x) -# define sqlite3Fts3DeferredTokenList(x,y,z) SQLITE_OK -#endif + UNUSED_PARAMETER(iPhrase); + rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pList); + nTerm = pExpr->pPhrase->nToken; + if( pList ){ + fts3GetDeltaPosition(&pList, &iPos); + assert( iPos>=0 ); + } -SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *); -SQLITE_PRIVATE int sqlite3Fts3MaxLevel(Fts3Table *, int *); + for(iTerm=0; iTermaTerm[p->iTerm++]; + pT->iOff = nTerm-iTerm-1; + pT->pList = pList; + pT->iPos = iPos; + } -/* Special values interpreted by sqlite3SegReaderCursor() */ -#define FTS3_SEGCURSOR_PENDING -1 -#define FTS3_SEGCURSOR_ALL -2 + return rc; +} -SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*); -SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); -SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *); +/* +** Implementation of offsets() function. +*/ +SQLITE_PRIVATE void sqlite3Fts3Offsets( + sqlite3_context *pCtx, /* SQLite function call context */ + Fts3Cursor *pCsr /* Cursor object */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + sqlite3_tokenizer_module const *pMod = pTab->pTokenizer->pModule; + int rc; /* Return Code */ + int nToken; /* Number of tokens in query */ + int iCol; /* Column currently being processed */ + StrBuffer res = {0, 0, 0}; /* Result string */ + TermOffsetCtx sCtx; /* Context for fts3ExprTermOffsetInit() */ -SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor(Fts3Table *, - int, int, int, const char *, int, int, int, Fts3MultiSegReader *); + if( !pCsr->pExpr ){ + sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); + return; + } -/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ -#define FTS3_SEGMENT_REQUIRE_POS 0x00000001 -#define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 -#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 -#define FTS3_SEGMENT_PREFIX 0x00000008 -#define FTS3_SEGMENT_SCAN 0x00000010 -#define FTS3_SEGMENT_FIRST 0x00000020 + memset(&sCtx, 0, sizeof(sCtx)); + assert( pCsr->isRequireSeek==0 ); -/* Type passed as 4th argument to SegmentReaderIterate() */ -struct Fts3SegFilter { - const char *zTerm; - int nTerm; - int iCol; - int flags; -}; + /* Count the number of terms in the query */ + rc = fts3ExprLoadDoclists(pCsr, 0, &nToken); + if( rc!=SQLITE_OK ) goto offsets_out; -struct Fts3MultiSegReader { - /* Used internally by sqlite3Fts3SegReaderXXX() calls */ - Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */ - int nSegment; /* Size of apSegment array */ - int nAdvance; /* How many seg-readers to advance */ - Fts3SegFilter *pFilter; /* Pointer to filter object */ - char *aBuffer; /* Buffer to merge doclists in */ - int nBuffer; /* Allocated size of aBuffer[] in bytes */ + /* Allocate the array of TermOffset iterators. */ + sCtx.aTerm = (TermOffset *)sqlite3_malloc(sizeof(TermOffset)*nToken); + if( 0==sCtx.aTerm ){ + rc = SQLITE_NOMEM; + goto offsets_out; + } + sCtx.iDocid = pCsr->iPrevId; + sCtx.pCsr = pCsr; - int iColFilter; /* If >=0, filter for this column */ - int bRestart; + /* Loop through the table columns, appending offset information to + ** string-buffer res for each column. + */ + for(iCol=0; iColnColumn; iCol++){ + sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor */ + const char *ZDUMMY; /* Dummy argument used with xNext() */ + int NDUMMY = 0; /* Dummy argument used with xNext() */ + int iStart = 0; + int iEnd = 0; + int iCurrent = 0; + const char *zDoc; + int nDoc; - /* Used by fts3.c only. */ - int nCost; /* Cost of running iterator */ - int bLookup; /* True if a lookup of a single entry. */ + /* Initialize the contents of sCtx.aTerm[] for column iCol. There is + ** no way that this operation can fail, so the return code from + ** fts3ExprIterate() can be discarded. + */ + sCtx.iCol = iCol; + sCtx.iTerm = 0; + (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx); - /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ - char *zTerm; /* Pointer to term buffer */ - int nTerm; /* Size of zTerm in bytes */ - char *aDoclist; /* Pointer to doclist buffer */ - int nDoclist; /* Size of aDoclist[] in bytes */ -}; + /* Retreive the text stored in column iCol. If an SQL NULL is stored + ** in column iCol, jump immediately to the next iteration of the loop. + ** If an OOM occurs while retrieving the data (this can happen if SQLite + ** needs to transform the data from utf-16 to utf-8), return SQLITE_NOMEM + ** to the caller. + */ + zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol+1); + nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol+1); + if( zDoc==0 ){ + if( sqlite3_column_type(pCsr->pStmt, iCol+1)==SQLITE_NULL ){ + continue; + } + rc = SQLITE_NOMEM; + goto offsets_out; + } -SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table*,int,int); + /* Initialize a tokenizer iterator to iterate through column iCol. */ + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, + zDoc, nDoc, &pC + ); + if( rc!=SQLITE_OK ) goto offsets_out; -#define fts3GetVarint32(p, piVal) ( \ - (*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \ -) + rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent); + while( rc==SQLITE_OK ){ + int i; /* Used to loop through terms */ + int iMinPos = 0x7FFFFFFF; /* Position of next token */ + TermOffset *pTerm = 0; /* TermOffset associated with next token */ -/* fts3.c */ -SQLITE_PRIVATE void sqlite3Fts3ErrMsg(char**,const char*,...); -SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *, sqlite3_int64); -SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); -SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *); -SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64); -SQLITE_PRIVATE void sqlite3Fts3Dequote(char *); -SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); -SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); -SQLITE_PRIVATE int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); -SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int*, Fts3Table*); + for(i=0; ipList && (pT->iPos-pT->iOff)iPos-pT->iOff; + pTerm = pT; + } + } -/* fts3_tokenizer.c */ -SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *); -SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); -SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, - sqlite3_tokenizer **, char ** -); -SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char); + if( !pTerm ){ + /* All offsets for this column have been gathered. */ + rc = SQLITE_DONE; + }else{ + assert( iCurrent<=iMinPos ); + if( 0==(0xFE&*pTerm->pList) ){ + pTerm->pList = 0; + }else{ + fts3GetDeltaPosition(&pTerm->pList, &pTerm->iPos); + } + while( rc==SQLITE_OK && iCurrentxNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent); + } + if( rc==SQLITE_OK ){ + char aBuffer[64]; + sqlite3_snprintf(sizeof(aBuffer), aBuffer, + "%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart + ); + rc = fts3StringAppend(&res, aBuffer, -1); + }else if( rc==SQLITE_DONE && pTab->zContentTbl==0 ){ + rc = FTS_CORRUPT_VTAB; + } + } + } + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + } -/* fts3_snippet.c */ -SQLITE_PRIVATE void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*); -SQLITE_PRIVATE void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, - const char *, const char *, int, int -); -SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); + pMod->xClose(pC); + if( rc!=SQLITE_OK ) goto offsets_out; + } -/* fts3_expr.c */ -SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int, - char **, int, int, int, const char *, int, Fts3Expr **, char ** -); -SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *); -#ifdef SQLITE_TEST -SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3 *db); -SQLITE_PRIVATE int sqlite3Fts3InitTerm(sqlite3 *db); -#endif + offsets_out: + sqlite3_free(sCtx.aTerm); + assert( rc!=SQLITE_DONE ); + sqlite3Fts3SegmentsClose(pTab); + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + sqlite3_free(res.z); + }else{ + sqlite3_result_text(pCtx, res.z, res.n-1, sqlite3_free); + } + return; +} -SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int, - sqlite3_tokenizer_cursor ** -); +/* +** Implementation of matchinfo() function. +*/ +SQLITE_PRIVATE void sqlite3Fts3Matchinfo( + sqlite3_context *pContext, /* Function call context */ + Fts3Cursor *pCsr, /* FTS3 table cursor */ + const char *zArg /* Second arg to matchinfo() function */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + const char *zFormat; -/* fts3_aux.c */ -SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db); + if( zArg ){ + zFormat = zArg; + }else{ + zFormat = FTS3_MATCHINFO_DEFAULT; + } -SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); + if( !pCsr->pExpr ){ + sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC); + return; + }else{ + /* Retrieve matchinfo() data. */ + fts3GetMatchinfo(pContext, pCsr, zFormat); + sqlite3Fts3SegmentsClose(pTab); + } +} -SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( - Fts3Table*, Fts3MultiSegReader*, int, const char*, int); -SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( - Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); -SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **); -SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); -SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); +#endif -/* fts3_tokenize_vtab.c */ -SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *); +/************** End of fts3_snippet.c ****************************************/ +/************** Begin file fts3_unicode.c ************************************/ +/* +** 2012 May 24 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Implementation of the "unicode" full-text-search tokenizer. +*/ -/* fts3_unicode2.c (functions generated by parsing unicode text files) */ #ifndef SQLITE_DISABLE_FTS3_UNICODE -SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int, int); -SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int); -SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int); -#endif - -#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */ -#endif /* _FTSINT_H */ -/************** End of fts3Int.h *********************************************/ -/************** Continuing where we left off in fts3.c ***********************/ +/* #include "fts3Int.h" */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) -#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE) -# define SQLITE_CORE 1 -#endif - /* #include */ /* #include */ -/* #include */ /* #include */ /* #include */ -/* #include */ - -#ifndef SQLITE_CORE - SQLITE_EXTENSION_INIT1 -#endif -static int fts3EvalNext(Fts3Cursor *pCsr); -static int fts3EvalStart(Fts3Cursor *pCsr); -static int fts3TermSegReaderCursor( - Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **); +/* #include "fts3_tokenizer.h" */ +/* +** The following two macros - READ_UTF8 and WRITE_UTF8 - have been copied +** from the sqlite3 source file utf.c. If this file is compiled as part +** of the amalgamation, they are not required. +*/ #ifndef SQLITE_AMALGAMATION -# if defined(SQLITE_DEBUG) -SQLITE_PRIVATE int sqlite3Fts3Always(int b) { assert( b ); return b; } -SQLITE_PRIVATE int sqlite3Fts3Never(int b) { assert( !b ); return b; } -# endif -#endif -/* -** Write a 64-bit variable-length integer to memory starting at p[0]. -** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. -** The number of bytes written is returned. -*/ -SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ - unsigned char *q = (unsigned char *) p; - sqlite_uint64 vu = v; - do{ - *q++ = (unsigned char) ((vu & 0x7f) | 0x80); - vu >>= 7; - }while( vu!=0 ); - q[-1] &= 0x7f; /* turn off high bit in final byte */ - assert( q - (unsigned char *)p <= FTS3_VARINT_MAX ); - return (int) (q - (unsigned char *)p); -} +static const unsigned char sqlite3Utf8Trans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +}; -#define GETVARINT_STEP(v, ptr, shift, mask1, mask2, var, ret) \ - v = (v & mask1) | ( (*ptr++) << shift ); \ - if( (v & mask2)==0 ){ var = v; return ret; } -#define GETVARINT_INIT(v, ptr, shift, mask1, mask2, var, ret) \ - v = (*ptr++); \ - if( (v & mask2)==0 ){ var = v; return ret; } +#define READ_UTF8(zIn, zTerm, c) \ + c = *(zIn++); \ + if( c>=0xc0 ){ \ + c = sqlite3Utf8Trans1[c-0xc0]; \ + while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \ + c = (c<<6) + (0x3f & *(zIn++)); \ + } \ + if( c<0x80 \ + || (c&0xFFFFF800)==0xD800 \ + || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ + } -/* -** Read a 64-bit variable-length integer from memory starting at p[0]. -** Return the number of bytes read, or 0 on error. -** The value is stored in *v. -*/ -SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){ - const char *pStart = p; - u32 a; - u64 b; - int shift; +#define WRITE_UTF8(zOut, c) { \ + if( c<0x00080 ){ \ + *zOut++ = (u8)(c&0xFF); \ + } \ + else if( c<0x00800 ){ \ + *zOut++ = 0xC0 + (u8)((c>>6)&0x1F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } \ + else if( c<0x10000 ){ \ + *zOut++ = 0xE0 + (u8)((c>>12)&0x0F); \ + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + }else{ \ + *zOut++ = 0xF0 + (u8)((c>>18) & 0x07); \ + *zOut++ = 0x80 + (u8)((c>>12) & 0x3F); \ + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } \ +} - GETVARINT_INIT(a, p, 0, 0x00, 0x80, *v, 1); - GETVARINT_STEP(a, p, 7, 0x7F, 0x4000, *v, 2); - GETVARINT_STEP(a, p, 14, 0x3FFF, 0x200000, *v, 3); - GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *v, 4); - b = (a & 0x0FFFFFFF ); +#endif /* ifndef SQLITE_AMALGAMATION */ - for(shift=28; shift<=63; shift+=7){ - u64 c = *p++; - b += (c&0x7F) << shift; - if( (c & 0x80)==0 ) break; - } - *v = b; - return (int)(p - pStart); -} +typedef struct unicode_tokenizer unicode_tokenizer; +typedef struct unicode_cursor unicode_cursor; -/* -** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a -** 32-bit integer before it is returned. -*/ -SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *p, int *pi){ - u32 a; +struct unicode_tokenizer { + sqlite3_tokenizer base; + int bRemoveDiacritic; + int nException; + int *aiException; +}; -#ifndef fts3GetVarint32 - GETVARINT_INIT(a, p, 0, 0x00, 0x80, *pi, 1); -#else - a = (*p++); - assert( a & 0x80 ); -#endif +struct unicode_cursor { + sqlite3_tokenizer_cursor base; + const unsigned char *aInput; /* Input text being tokenized */ + int nInput; /* Size of aInput[] in bytes */ + int iOff; /* Current offset within aInput[] */ + int iToken; /* Index of next token to be returned */ + char *zToken; /* storage for current token */ + int nAlloc; /* space allocated at zToken */ +}; - GETVARINT_STEP(a, p, 7, 0x7F, 0x4000, *pi, 2); - GETVARINT_STEP(a, p, 14, 0x3FFF, 0x200000, *pi, 3); - GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *pi, 4); - a = (a & 0x0FFFFFFF ); - *pi = (int)(a | ((u32)(*p & 0x0F) << 28)); - return 5; -} /* -** Return the number of bytes required to encode v as a varint +** Destroy a tokenizer allocated by unicodeCreate(). */ -SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64 v){ - int i = 0; - do{ - i++; - v >>= 7; - }while( v!=0 ); - return i; +static int unicodeDestroy(sqlite3_tokenizer *pTokenizer){ + if( pTokenizer ){ + unicode_tokenizer *p = (unicode_tokenizer *)pTokenizer; + sqlite3_free(p->aiException); + sqlite3_free(p); + } + return SQLITE_OK; } /* -** Convert an SQL-style quoted string into a normal string by removing -** the quote characters. The conversion is done in-place. If the -** input does not begin with a quote character, then this routine -** is a no-op. -** -** Examples: +** As part of a tokenchars= or separators= option, the CREATE VIRTUAL TABLE +** statement has specified that the tokenizer for this table shall consider +** all characters in string zIn/nIn to be separators (if bAlnum==0) or +** token characters (if bAlnum==1). ** -** "abc" becomes abc -** 'xyz' becomes xyz -** [pqr] becomes pqr -** `mno` becomes mno +** For each codepoint in the zIn/nIn string, this function checks if the +** sqlite3FtsUnicodeIsalnum() function already returns the desired result. +** If so, no action is taken. Otherwise, the codepoint is added to the +** unicode_tokenizer.aiException[] array. For the purposes of tokenization, +** the return value of sqlite3FtsUnicodeIsalnum() is inverted for all +** codepoints in the aiException[] array. ** +** If a standalone diacritic mark (one that sqlite3FtsUnicodeIsdiacritic() +** identifies as a diacritic) occurs in the zIn/nIn string it is ignored. +** It is not possible to change the behavior of the tokenizer with respect +** to these codepoints. */ -SQLITE_PRIVATE void sqlite3Fts3Dequote(char *z){ - char quote; /* Quote character (if any ) */ - - quote = z[0]; - if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ - int iIn = 1; /* Index of next byte to read from input */ - int iOut = 0; /* Index of next byte to write to output */ +static int unicodeAddExceptions( + unicode_tokenizer *p, /* Tokenizer to add exceptions to */ + int bAlnum, /* Replace Isalnum() return value with this */ + const char *zIn, /* Array of characters to make exceptions */ + int nIn /* Length of z in bytes */ +){ + const unsigned char *z = (const unsigned char *)zIn; + const unsigned char *zTerm = &z[nIn]; + int iCode; + int nEntry = 0; - /* If the first byte was a '[', then the close-quote character is a ']' */ - if( quote=='[' ) quote = ']'; + assert( bAlnum==0 || bAlnum==1 ); - while( z[iIn] ){ - if( z[iIn]==quote ){ - if( z[iIn+1]!=quote ) break; - z[iOut++] = quote; - iIn += 2; - }else{ - z[iOut++] = z[iIn++]; - } + while( zaiException, (p->nException+nEntry)*sizeof(int)); + if( aNew==0 ) return SQLITE_NOMEM; + nNew = p->nException; - /* Pointer p now points at the first byte past the varint we are - ** interested in. So, unless the doclist is corrupt, the 0x80 bit is - ** clear on character p[-1]. */ - for(p = (*pp)-2; p>=pStart && *p&0x80; p--); - p++; - *pp = p; + z = (const unsigned char *)zIn; + while( zi; j--) aNew[j] = aNew[j-1]; + aNew[i] = iCode; + nNew++; + } + } + p->aiException = aNew; + p->nException = nNew; + } - sqlite3Fts3GetVarint(p, &iVal); - *pVal = iVal; + return SQLITE_OK; } /* -** The xDisconnect() virtual table method. +** Return true if the p->aiException[] array contains the value iCode. */ -static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ - Fts3Table *p = (Fts3Table *)pVtab; - int i; - - assert( p->nPendingData==0 ); - assert( p->pSegments==0 ); +static int unicodeIsException(unicode_tokenizer *p, int iCode){ + if( p->nException>0 ){ + int *a = p->aiException; + int iLo = 0; + int iHi = p->nException-1; - /* Free any prepared statements held */ - for(i=0; iaStmt); i++){ - sqlite3_finalize(p->aStmt[i]); + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( iCode==a[iTest] ){ + return 1; + }else if( iCode>a[iTest] ){ + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } } - sqlite3_free(p->zSegmentsTbl); - sqlite3_free(p->zReadExprlist); - sqlite3_free(p->zWriteExprlist); - sqlite3_free(p->zContentTbl); - sqlite3_free(p->zLanguageid); - - /* Invoke the tokenizer destructor to free the tokenizer. */ - p->pTokenizer->pModule->xDestroy(p->pTokenizer); - sqlite3_free(p); - return SQLITE_OK; + return 0; } /* -** Write an error message into *pzErr +** Return true if, for the purposes of tokenization, codepoint iCode is +** considered a token character (not a separator). */ -SQLITE_PRIVATE void sqlite3Fts3ErrMsg(char **pzErr, const char *zFormat, ...){ - va_list ap; - sqlite3_free(*pzErr); - va_start(ap, zFormat); - *pzErr = sqlite3_vmprintf(zFormat, ap); - va_end(ap); +static int unicodeIsAlnum(unicode_tokenizer *p, int iCode){ + assert( (sqlite3FtsUnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 ); + return sqlite3FtsUnicodeIsalnum(iCode) ^ unicodeIsException(p, iCode); } /* -** Construct one or more SQL statements from the format string given -** and then evaluate those statements. The success code is written -** into *pRc. -** -** If *pRc is initially non-zero then this routine is a no-op. +** Create a new tokenizer instance. */ -static void fts3DbExec( - int *pRc, /* Success code */ - sqlite3 *db, /* Database in which to run SQL */ - const char *zFormat, /* Format string for SQL */ - ... /* Arguments to the format string */ +static int unicodeCreate( + int nArg, /* Size of array argv[] */ + const char * const *azArg, /* Tokenizer creation arguments */ + sqlite3_tokenizer **pp /* OUT: New tokenizer handle */ ){ - va_list ap; - char *zSql; - if( *pRc ) return; - va_start(ap, zFormat); - zSql = sqlite3_vmprintf(zFormat, ap); - va_end(ap); - if( zSql==0 ){ - *pRc = SQLITE_NOMEM; - }else{ - *pRc = sqlite3_exec(db, zSql, 0, 0, 0); - sqlite3_free(zSql); - } -} + unicode_tokenizer *pNew; /* New tokenizer object */ + int i; + int rc = SQLITE_OK; -/* -** The xDestroy() virtual table method. -*/ -static int fts3DestroyMethod(sqlite3_vtab *pVtab){ - Fts3Table *p = (Fts3Table *)pVtab; - int rc = SQLITE_OK; /* Return code */ - const char *zDb = p->zDb; /* Name of database (e.g. "main", "temp") */ - sqlite3 *db = p->db; /* Database handle */ + pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer)); + if( pNew==NULL ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(unicode_tokenizer)); + pNew->bRemoveDiacritic = 1; - /* Drop the shadow tables */ - if( p->zContentTbl==0 ){ - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", zDb, p->zName); + for(i=0; rc==SQLITE_OK && ibRemoveDiacritic = 1; + } + else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){ + pNew->bRemoveDiacritic = 0; + } + else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){ + rc = unicodeAddExceptions(pNew, 1, &z[11], n-11); + } + else if( n>=11 && memcmp("separators=", z, 11)==0 ){ + rc = unicodeAddExceptions(pNew, 0, &z[11], n-11); + } + else{ + /* Unrecognized argument */ + rc = SQLITE_ERROR; + } } - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", zDb,p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", zDb, p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", zDb, p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", zDb, p->zName); - /* If everything has worked, invoke fts3DisconnectMethod() to free the - ** memory associated with the Fts3Table structure and return SQLITE_OK. - ** Otherwise, return an SQLite error code. - */ - return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc); + if( rc!=SQLITE_OK ){ + unicodeDestroy((sqlite3_tokenizer *)pNew); + pNew = 0; + } + *pp = (sqlite3_tokenizer *)pNew; + return rc; } - /* -** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table -** passed as the first argument. This is done as part of the xConnect() -** and xCreate() methods. -** -** If *pRc is non-zero when this function is called, it is a no-op. -** Otherwise, if an error occurs, an SQLite error code is stored in *pRc -** before returning. +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is pInput[0..nBytes-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. */ -static void fts3DeclareVtab(int *pRc, Fts3Table *p){ - if( *pRc==SQLITE_OK ){ - int i; /* Iterator variable */ - int rc; /* Return code */ - char *zSql; /* SQL statement passed to declare_vtab() */ - char *zCols; /* List of user defined columns */ - const char *zLanguageid; - - zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid"); - sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); - - /* Create a list of user columns for the virtual table */ - zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); - for(i=1; zCols && inColumn; i++){ - zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]); - } +static int unicodeOpen( + sqlite3_tokenizer *p, /* The tokenizer */ + const char *aInput, /* Input string */ + int nInput, /* Size of string aInput in bytes */ + sqlite3_tokenizer_cursor **pp /* OUT: New cursor object */ +){ + unicode_cursor *pCsr; - /* Create the whole "CREATE TABLE" statement to pass to SQLite */ - zSql = sqlite3_mprintf( - "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN, %Q HIDDEN)", - zCols, p->zName, zLanguageid - ); - if( !zCols || !zSql ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_declare_vtab(p->db, zSql); - } + pCsr = (unicode_cursor *)sqlite3_malloc(sizeof(unicode_cursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(unicode_cursor)); - sqlite3_free(zSql); - sqlite3_free(zCols); - *pRc = rc; + pCsr->aInput = (const unsigned char *)aInput; + if( aInput==0 ){ + pCsr->nInput = 0; + }else if( nInput<0 ){ + pCsr->nInput = (int)strlen(aInput); + }else{ + pCsr->nInput = nInput; } + + *pp = &pCsr->base; + UNUSED_PARAMETER(p); + return SQLITE_OK; } /* -** Create the %_stat table if it does not already exist. +** Close a tokenization cursor previously opened by a call to +** simpleOpen() above. */ -SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int *pRc, Fts3Table *p){ - fts3DbExec(pRc, p->db, - "CREATE TABLE IF NOT EXISTS %Q.'%q_stat'" - "(id INTEGER PRIMARY KEY, value BLOB);", - p->zDb, p->zName - ); - if( (*pRc)==SQLITE_OK ) p->bHasStat = 1; +static int unicodeClose(sqlite3_tokenizer_cursor *pCursor){ + unicode_cursor *pCsr = (unicode_cursor *) pCursor; + sqlite3_free(pCsr->zToken); + sqlite3_free(pCsr); + return SQLITE_OK; } /* -** Create the backing store tables (%_content, %_segments and %_segdir) -** required by the FTS3 table passed as the only argument. This is done -** as part of the vtab xCreate() method. -** -** If the p->bHasDocsize boolean is true (indicating that this is an -** FTS4 table, not an FTS3 table) then also create the %_docsize and -** %_stat tables required by FTS4. +** Extract the next token from a tokenization cursor. The cursor must +** have been opened by a prior call to simpleOpen(). */ -static int fts3CreateTables(Fts3Table *p){ - int rc = SQLITE_OK; /* Return code */ - int i; /* Iterator variable */ - sqlite3 *db = p->db; /* The database connection */ +static int unicodeNext( + sqlite3_tokenizer_cursor *pC, /* Cursor returned by simpleOpen */ + const char **paToken, /* OUT: Token text */ + int *pnToken, /* OUT: Number of bytes at *paToken */ + int *piStart, /* OUT: Starting offset of token */ + int *piEnd, /* OUT: Ending offset of token */ + int *piPos /* OUT: Position integer of token */ +){ + unicode_cursor *pCsr = (unicode_cursor *)pC; + unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer); + int iCode = 0; + char *zOut; + const unsigned char *z = &pCsr->aInput[pCsr->iOff]; + const unsigned char *zStart = z; + const unsigned char *zEnd; + const unsigned char *zTerm = &pCsr->aInput[pCsr->nInput]; - if( p->zContentTbl==0 ){ - const char *zLanguageid = p->zLanguageid; - char *zContentCols; /* Columns of %_content table */ + /* Scan past any delimiter characters before the start of the next token. + ** Return SQLITE_DONE early if this takes us all the way to the end of + ** the input. */ + while( z=zTerm ) return SQLITE_DONE; - /* Create a list of user columns for the content table */ - zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); - for(i=0; zContentCols && inColumn; i++){ - char *z = p->azColumn[i]; - zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); + zOut = pCsr->zToken; + do { + int iOut; + + /* Grow the output buffer if required. */ + if( (zOut-pCsr->zToken)>=(pCsr->nAlloc-4) ){ + char *zNew = sqlite3_realloc(pCsr->zToken, pCsr->nAlloc+64); + if( !zNew ) return SQLITE_NOMEM; + zOut = &zNew[zOut - pCsr->zToken]; + pCsr->zToken = zNew; + pCsr->nAlloc += 64; } - if( zLanguageid && zContentCols ){ - zContentCols = sqlite3_mprintf("%z, langid", zContentCols, zLanguageid); + + /* Write the folded case of the last character read to the output */ + zEnd = z; + iOut = sqlite3FtsUnicodeFold(iCode, p->bRemoveDiacritic); + if( iOut ){ + WRITE_UTF8(zOut, iOut); } - if( zContentCols==0 ) rc = SQLITE_NOMEM; - - /* Create the content table */ - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_content'(%s)", - p->zDb, p->zName, zContentCols - ); - sqlite3_free(zContentCols); - } - /* Create other tables */ - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", - p->zDb, p->zName - ); - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_segdir'(" - "level INTEGER," - "idx INTEGER," - "start_block INTEGER," - "leaves_end_block INTEGER," - "end_block INTEGER," - "root BLOB," - "PRIMARY KEY(level, idx)" - ");", - p->zDb, p->zName + /* If the cursor is not at EOF, read the next character */ + if( z>=zTerm ) break; + READ_UTF8(z, zTerm, iCode); + }while( unicodeIsAlnum(p, iCode) + || sqlite3FtsUnicodeIsdiacritic(iCode) ); - if( p->bHasDocsize ){ - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_docsize'(docid INTEGER PRIMARY KEY, size BLOB);", - p->zDb, p->zName - ); - } - assert( p->bHasStat==p->bFts4 ); - if( p->bHasStat ){ - sqlite3Fts3CreateStatTable(&rc, p); - } - return rc; + + /* Set the output variables and return. */ + pCsr->iOff = (int)(z - pCsr->aInput); + *paToken = pCsr->zToken; + *pnToken = (int)(zOut - pCsr->zToken); + *piStart = (int)(zStart - pCsr->aInput); + *piEnd = (int)(zEnd - pCsr->aInput); + *piPos = pCsr->iToken++; + return SQLITE_OK; } /* -** Store the current database page-size in bytes in p->nPgsz. -** -** If *pRc is non-zero when this function is called, it is a no-op. -** Otherwise, if an error occurs, an SQLite error code is stored in *pRc -** before returning. +** Set *ppModule to a pointer to the sqlite3_tokenizer_module +** structure for the unicode tokenizer. */ -static void fts3DatabasePageSize(int *pRc, Fts3Table *p){ - if( *pRc==SQLITE_OK ){ - int rc; /* Return code */ - char *zSql; /* SQL text "PRAGMA %Q.page_size" */ - sqlite3_stmt *pStmt; /* Compiled "PRAGMA %Q.page_size" statement */ - - zSql = sqlite3_mprintf("PRAGMA %Q.page_size", p->zDb); - if( !zSql ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0); - if( rc==SQLITE_OK ){ - sqlite3_step(pStmt); - p->nPgsz = sqlite3_column_int(pStmt, 0); - rc = sqlite3_finalize(pStmt); - }else if( rc==SQLITE_AUTH ){ - p->nPgsz = 1024; - rc = SQLITE_OK; - } - } - assert( p->nPgsz>0 || rc!=SQLITE_OK ); - sqlite3_free(zSql); - *pRc = rc; - } +SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const **ppModule){ + static const sqlite3_tokenizer_module module = { + 0, + unicodeCreate, + unicodeDestroy, + unicodeOpen, + unicodeClose, + unicodeNext, + 0, + }; + *ppModule = &module; } +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ +#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */ + +/************** End of fts3_unicode.c ****************************************/ +/************** Begin file fts3_unicode2.c ***********************************/ /* -** "Special" FTS4 arguments are column specifications of the following form: +** 2012 May 25 ** -** = +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** There may not be whitespace surrounding the "=" character. The -** term may be quoted, but the may not. +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** */ -static int fts3IsSpecialColumn( - const char *z, - int *pnKey, - char **pzValue -){ - char *zValue; - const char *zCsr = z; - - while( *zCsr!='=' ){ - if( *zCsr=='\0' ) return 0; - zCsr++; - } - - *pnKey = (int)(zCsr-z); - zValue = sqlite3_mprintf("%s", &zCsr[1]); - if( zValue ){ - sqlite3Fts3Dequote(zValue); - } - *pzValue = zValue; - return 1; -} /* -** Append the output of a printf() style formatting to an existing string. +** DO NOT EDIT THIS MACHINE GENERATED FILE. */ -static void fts3Appendf( - int *pRc, /* IN/OUT: Error code */ - char **pz, /* IN/OUT: Pointer to string buffer */ - const char *zFormat, /* Printf format string to append */ - ... /* Arguments for printf format string */ -){ - if( *pRc==SQLITE_OK ){ - va_list ap; - char *z; - va_start(ap, zFormat); - z = sqlite3_vmprintf(zFormat, ap); - va_end(ap); - if( z && *pz ){ - char *z2 = sqlite3_mprintf("%s%s", *pz, z); - sqlite3_free(z); - z = z2; - } - if( z==0 ) *pRc = SQLITE_NOMEM; - sqlite3_free(*pz); - *pz = z; - } -} + +#ifndef SQLITE_DISABLE_FTS3_UNICODE +#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) + +/* #include */ /* -** Return a copy of input string zInput enclosed in double-quotes (") and -** with all double quote characters escaped. For example: -** -** fts3QuoteId("un \"zip\"") -> "un \"\"zip\"\"" +** Return true if the argument corresponds to a unicode codepoint +** classified as either a letter or a number. Otherwise false. ** -** The pointer returned points to memory obtained from sqlite3_malloc(). It -** is the callers responsibility to call sqlite3_free() to release this -** memory. +** The results are undefined if the value passed to this function +** is less than zero. */ -static char *fts3QuoteId(char const *zInput){ - int nRet; - char *zRet; - nRet = 2 + (int)strlen(zInput)*2 + 1; - zRet = sqlite3_malloc(nRet); - if( zRet ){ - int i; - char *z = zRet; - *(z++) = '"'; - for(i=0; zInput[i]; i++){ - if( zInput[i]=='"' ) *(z++) = '"'; - *(z++) = zInput[i]; +SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int c){ + /* Each unsigned integer in the following array corresponds to a contiguous + ** range of unicode codepoints that are not either letters or numbers (i.e. + ** codepoints for which this function should return 0). + ** + ** The most significant 22 bits in each 32-bit value contain the first + ** codepoint in the range. The least significant 10 bits are used to store + ** the size of the range (always at least 1). In other words, the value + ** ((C<<22) + N) represents a range of N codepoints starting with codepoint + ** C. It is not possible to represent a range larger than 1023 codepoints + ** using this format. + */ + static const unsigned int aEntry[] = { + 0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07, + 0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01, + 0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401, + 0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01, + 0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01, + 0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802, + 0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F, + 0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401, + 0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804, + 0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403, + 0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812, + 0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001, + 0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802, + 0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805, + 0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401, + 0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03, + 0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807, + 0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001, + 0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01, + 0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804, + 0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001, + 0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802, + 0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01, + 0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06, + 0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007, + 0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006, + 0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417, + 0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14, + 0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07, + 0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01, + 0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001, + 0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802, + 0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F, + 0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002, + 0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802, + 0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006, + 0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D, + 0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802, + 0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027, + 0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403, + 0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805, + 0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04, + 0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401, + 0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005, + 0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B, + 0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A, + 0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001, + 0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59, + 0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807, + 0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01, + 0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E, + 0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100, + 0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10, + 0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402, + 0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804, + 0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012, + 0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004, + 0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002, + 0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803, + 0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07, + 0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02, + 0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802, + 0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013, + 0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06, + 0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003, + 0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01, + 0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403, + 0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009, + 0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003, + 0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003, + 0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E, + 0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046, + 0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401, + 0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401, + 0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F, + 0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C, + 0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002, + 0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025, + 0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6, + 0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46, + 0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060, + 0x380400F0, + }; + static const unsigned int aAscii[4] = { + 0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001, + }; + + if( c<128 ){ + return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); + }else if( c<(1<<22) ){ + unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; + int iRes = 0; + int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; + int iLo = 0; + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( key >= aEntry[iTest] ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } } - *(z++) = '"'; - *(z++) = '\0'; + assert( aEntry[0]=aEntry[iRes] ); + return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF))); } - return zRet; + return 1; } + /* -** Return a list of comma separated SQL expressions and a FROM clause that -** could be used in a SELECT statement such as the following: -** -** SELECT FROM %_content AS x ... -** -** to return the docid, followed by each column of text data in order -** from left to write. If parameter zFunc is not NULL, then instead of -** being returned directly each column of text data is passed to an SQL -** function named zFunc first. For example, if zFunc is "unzip" and the -** table has the three user-defined columns "a", "b", and "c", the following -** string is returned: -** -** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c') FROM %_content AS x" -** -** The pointer returned points to a buffer allocated by sqlite3_malloc(). It -** is the responsibility of the caller to eventually free it. -** -** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and -** a NULL pointer is returned). Otherwise, if an OOM error is encountered -** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If -** no error occurs, *pRc is left unmodified. +** If the argument is a codepoint corresponding to a lowercase letter +** in the ASCII range with a diacritic added, return the codepoint +** of the ASCII letter only. For example, if passed 235 - "LATIN +** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER +** E"). The resuls of passing a codepoint that corresponds to an +** uppercase letter are undefined. */ -static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ - char *zRet = 0; - char *zFree = 0; - char *zFunction; - int i; +static int remove_diacritic(int c){ + unsigned short aDia[] = { + 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995, + 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286, + 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732, + 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336, + 3456, 3696, 3712, 3728, 3744, 3896, 3912, 3928, + 3968, 4008, 4040, 4106, 4138, 4170, 4202, 4234, + 4266, 4296, 4312, 4344, 4408, 4424, 4472, 4504, + 6148, 6198, 6264, 6280, 6360, 6429, 6505, 6529, + 61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, + 61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, + 62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, + 62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, + 62924, 63050, 63082, 63274, 63390, + }; + char aChar[] = { + '\0', 'a', 'c', 'e', 'i', 'n', 'o', 'u', 'y', 'y', 'a', 'c', + 'd', 'e', 'e', 'g', 'h', 'i', 'j', 'k', 'l', 'n', 'o', 'r', + 's', 't', 'u', 'u', 'w', 'y', 'z', 'o', 'u', 'a', 'i', 'o', + 'u', 'g', 'k', 'o', 'j', 'g', 'n', 'a', 'e', 'i', 'o', 'r', + 'u', 's', 't', 'h', 'a', 'e', 'o', 'y', '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', 'a', 'b', 'd', 'd', 'e', 'f', 'g', 'h', + 'h', 'i', 'k', 'l', 'l', 'm', 'n', 'p', 'r', 'r', 's', 't', + 'u', 'v', 'w', 'w', 'x', 'y', 'z', 'h', 't', 'w', 'y', 'a', + 'e', 'i', 'o', 'u', 'y', + }; - if( p->zContentTbl==0 ){ - if( !zFunc ){ - zFunction = ""; + unsigned int key = (((unsigned int)c)<<3) | 0x00000007; + int iRes = 0; + int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1; + int iLo = 0; + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( key >= aDia[iTest] ){ + iRes = iTest; + iLo = iTest+1; }else{ - zFree = zFunction = fts3QuoteId(zFunc); - } - fts3Appendf(pRc, &zRet, "docid"); - for(i=0; inColumn; i++){ - fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); - } - if( p->zLanguageid ){ - fts3Appendf(pRc, &zRet, ", x.%Q", "langid"); - } - sqlite3_free(zFree); - }else{ - fts3Appendf(pRc, &zRet, "rowid"); - for(i=0; inColumn; i++){ - fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]); - } - if( p->zLanguageid ){ - fts3Appendf(pRc, &zRet, ", x.%Q", p->zLanguageid); + iHi = iTest-1; } } - fts3Appendf(pRc, &zRet, " FROM '%q'.'%q%s' AS x", - p->zDb, - (p->zContentTbl ? p->zContentTbl : p->zName), - (p->zContentTbl ? "" : "_content") - ); - return zRet; + assert( key>=aDia[iRes] ); + return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]); } -/* -** Return a list of N comma separated question marks, where N is the number -** of columns in the %_content table (one for the docid plus one for each -** user-defined text column). -** -** If argument zFunc is not NULL, then all but the first question mark -** is preceded by zFunc and an open bracket, and followed by a closed -** bracket. For example, if zFunc is "zip" and the FTS3 table has three -** user-defined text columns, the following string is returned: -** -** "?, zip(?), zip(?), zip(?)" -** -** The pointer returned points to a buffer allocated by sqlite3_malloc(). It -** is the responsibility of the caller to eventually free it. -** -** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and -** a NULL pointer is returned). Otherwise, if an OOM error is encountered -** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If -** no error occurs, *pRc is left unmodified. -*/ -static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ - char *zRet = 0; - char *zFree = 0; - char *zFunction; - int i; - - if( !zFunc ){ - zFunction = ""; - }else{ - zFree = zFunction = fts3QuoteId(zFunc); - } - fts3Appendf(pRc, &zRet, "?"); - for(i=0; inColumn; i++){ - fts3Appendf(pRc, &zRet, ",%s(?)", zFunction); - } - if( p->zLanguageid ){ - fts3Appendf(pRc, &zRet, ", ?"); - } - sqlite3_free(zFree); - return zRet; -} /* -** This function interprets the string at (*pp) as a non-negative integer -** value. It reads the integer and sets *pnOut to the value read, then -** sets *pp to point to the byte immediately following the last byte of -** the integer value. -** -** Only decimal digits ('0'..'9') may be part of an integer value. -** -** If *pp does not being with a decimal digit SQLITE_ERROR is returned and -** the output value undefined. Otherwise SQLITE_OK is returned. -** -** This function is used when parsing the "prefix=" FTS4 parameter. +** Return true if the argument interpreted as a unicode codepoint +** is a diacritical modifier character. */ -static int fts3GobbleInt(const char **pp, int *pnOut){ - const int MAX_NPREFIX = 10000000; - const char *p; /* Iterator pointer */ - int nInt = 0; /* Output value */ - - for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ - nInt = nInt * 10 + (p[0] - '0'); - if( nInt>MAX_NPREFIX ){ - nInt = 0; - break; - } - } - if( p==*pp ) return SQLITE_ERROR; - *pnOut = nInt; - *pp = p; - return SQLITE_OK; +SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int c){ + unsigned int mask0 = 0x08029FDF; + unsigned int mask1 = 0x000361F8; + if( c<768 || c>817 ) return 0; + return (c < 768+32) ? + (mask0 & (1 << (c-768))) : + (mask1 & (1 << (c-768-32))); } + /* -** This function is called to allocate an array of Fts3Index structures -** representing the indexes maintained by the current FTS table. FTS tables -** always maintain the main "terms" index, but may also maintain one or -** more "prefix" indexes, depending on the value of the "prefix=" parameter -** (if any) specified as part of the CREATE VIRTUAL TABLE statement. -** -** Argument zParam is passed the value of the "prefix=" option if one was -** specified, or NULL otherwise. -** -** If no error occurs, SQLITE_OK is returned and *apIndex set to point to -** the allocated array. *pnIndex is set to the number of elements in the -** array. If an error does occur, an SQLite error code is returned. +** Interpret the argument as a unicode codepoint. If the codepoint +** is an upper case character that has a lower case equivalent, +** return the codepoint corresponding to the lower case version. +** Otherwise, return a copy of the argument. ** -** Regardless of whether or not an error is returned, it is the responsibility -** of the caller to call sqlite3_free() on the output array to free it. +** The results are undefined if the value passed to this function +** is less than zero. */ -static int fts3PrefixParameter( - const char *zParam, /* ABC in prefix=ABC parameter to parse */ - int *pnIndex, /* OUT: size of *apIndex[] array */ - struct Fts3Index **apIndex /* OUT: Array of indexes for this table */ -){ - struct Fts3Index *aIndex; /* Allocated array */ - int nIndex = 1; /* Number of entries in array */ +SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ + /* Each entry in the following array defines a rule for folding a range + ** of codepoints to lower case. The rule applies to a range of nRange + ** codepoints starting at codepoint iCode. + ** + ** If the least significant bit in flags is clear, then the rule applies + ** to all nRange codepoints (i.e. all nRange codepoints are upper case and + ** need to be folded). Or, if it is set, then the rule only applies to + ** every second codepoint in the range, starting with codepoint C. + ** + ** The 7 most significant bits in flags are an index into the aiOff[] + ** array. If a specific codepoint C does require folding, then its lower + ** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF). + ** + ** The contents of this array are generated by parsing the CaseFolding.txt + ** file distributed as part of the "Unicode Character Database". See + ** http://www.unicode.org for details. + */ + static const struct TableEntry { + unsigned short iCode; + unsigned char flags; + unsigned char nRange; + } aEntry[] = { + {65, 14, 26}, {181, 64, 1}, {192, 14, 23}, + {216, 14, 7}, {256, 1, 48}, {306, 1, 6}, + {313, 1, 16}, {330, 1, 46}, {376, 116, 1}, + {377, 1, 6}, {383, 104, 1}, {385, 50, 1}, + {386, 1, 4}, {390, 44, 1}, {391, 0, 1}, + {393, 42, 2}, {395, 0, 1}, {398, 32, 1}, + {399, 38, 1}, {400, 40, 1}, {401, 0, 1}, + {403, 42, 1}, {404, 46, 1}, {406, 52, 1}, + {407, 48, 1}, {408, 0, 1}, {412, 52, 1}, + {413, 54, 1}, {415, 56, 1}, {416, 1, 6}, + {422, 60, 1}, {423, 0, 1}, {425, 60, 1}, + {428, 0, 1}, {430, 60, 1}, {431, 0, 1}, + {433, 58, 2}, {435, 1, 4}, {439, 62, 1}, + {440, 0, 1}, {444, 0, 1}, {452, 2, 1}, + {453, 0, 1}, {455, 2, 1}, {456, 0, 1}, + {458, 2, 1}, {459, 1, 18}, {478, 1, 18}, + {497, 2, 1}, {498, 1, 4}, {502, 122, 1}, + {503, 134, 1}, {504, 1, 40}, {544, 110, 1}, + {546, 1, 18}, {570, 70, 1}, {571, 0, 1}, + {573, 108, 1}, {574, 68, 1}, {577, 0, 1}, + {579, 106, 1}, {580, 28, 1}, {581, 30, 1}, + {582, 1, 10}, {837, 36, 1}, {880, 1, 4}, + {886, 0, 1}, {902, 18, 1}, {904, 16, 3}, + {908, 26, 1}, {910, 24, 2}, {913, 14, 17}, + {931, 14, 9}, {962, 0, 1}, {975, 4, 1}, + {976, 140, 1}, {977, 142, 1}, {981, 146, 1}, + {982, 144, 1}, {984, 1, 24}, {1008, 136, 1}, + {1009, 138, 1}, {1012, 130, 1}, {1013, 128, 1}, + {1015, 0, 1}, {1017, 152, 1}, {1018, 0, 1}, + {1021, 110, 3}, {1024, 34, 16}, {1040, 14, 32}, + {1120, 1, 34}, {1162, 1, 54}, {1216, 6, 1}, + {1217, 1, 14}, {1232, 1, 88}, {1329, 22, 38}, + {4256, 66, 38}, {4295, 66, 1}, {4301, 66, 1}, + {7680, 1, 150}, {7835, 132, 1}, {7838, 96, 1}, + {7840, 1, 96}, {7944, 150, 8}, {7960, 150, 6}, + {7976, 150, 8}, {7992, 150, 8}, {8008, 150, 6}, + {8025, 151, 8}, {8040, 150, 8}, {8072, 150, 8}, + {8088, 150, 8}, {8104, 150, 8}, {8120, 150, 2}, + {8122, 126, 2}, {8124, 148, 1}, {8126, 100, 1}, + {8136, 124, 4}, {8140, 148, 1}, {8152, 150, 2}, + {8154, 120, 2}, {8168, 150, 2}, {8170, 118, 2}, + {8172, 152, 1}, {8184, 112, 2}, {8186, 114, 2}, + {8188, 148, 1}, {8486, 98, 1}, {8490, 92, 1}, + {8491, 94, 1}, {8498, 12, 1}, {8544, 8, 16}, + {8579, 0, 1}, {9398, 10, 26}, {11264, 22, 47}, + {11360, 0, 1}, {11362, 88, 1}, {11363, 102, 1}, + {11364, 90, 1}, {11367, 1, 6}, {11373, 84, 1}, + {11374, 86, 1}, {11375, 80, 1}, {11376, 82, 1}, + {11378, 0, 1}, {11381, 0, 1}, {11390, 78, 2}, + {11392, 1, 100}, {11499, 1, 4}, {11506, 0, 1}, + {42560, 1, 46}, {42624, 1, 24}, {42786, 1, 14}, + {42802, 1, 62}, {42873, 1, 4}, {42877, 76, 1}, + {42878, 1, 10}, {42891, 0, 1}, {42893, 74, 1}, + {42896, 1, 4}, {42912, 1, 10}, {42922, 72, 1}, + {65313, 14, 26}, + }; + static const unsigned short aiOff[] = { + 1, 2, 8, 15, 16, 26, 28, 32, + 37, 38, 40, 48, 63, 64, 69, 71, + 79, 80, 116, 202, 203, 205, 206, 207, + 209, 210, 211, 213, 214, 217, 218, 219, + 775, 7264, 10792, 10795, 23228, 23256, 30204, 54721, + 54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274, + 57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406, + 65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462, + 65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511, + 65514, 65521, 65527, 65528, 65529, + }; - if( zParam && zParam[0] ){ - const char *p; - nIndex++; - for(p=zParam; *p; p++){ - if( *p==',' ) nIndex++; - } - } + int ret = c; - aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex); - *apIndex = aIndex; - if( !aIndex ){ - return SQLITE_NOMEM; - } + assert( c>=0 ); + assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 ); - memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex); - if( zParam ){ - const char *p = zParam; - int i; - for(i=1; i=0 ); - if( nPrefix==0 ){ - nIndex--; - i--; + if( c<128 ){ + if( c>='A' && c<='Z' ) ret = c + ('a' - 'A'); + }else if( c<65536 ){ + int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; + int iLo = 0; + int iRes = -1; + + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + int cmp = (c - aEntry[iTest].iCode); + if( cmp>=0 ){ + iRes = iTest; + iLo = iTest+1; }else{ - aIndex[i].nPrefix = nPrefix; + iHi = iTest-1; + } + } + assert( iRes<0 || c>=aEntry[iRes].iCode ); + + if( iRes>=0 ){ + const struct TableEntry *p = &aEntry[iRes]; + if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ + ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; + assert( ret>0 ); } - p++; } + + if( bRemoveDiacritic ) ret = remove_diacritic(ret); + } + + else if( c>=66560 && c<66600 ){ + ret = c + 40; } - *pnIndex = nIndex; - return SQLITE_OK; + return ret; } +#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */ +#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */ +/************** End of fts3_unicode2.c ***************************************/ +/************** Begin file rtree.c *******************************************/ /* -** This function is called when initializing an FTS4 table that uses the -** content=xxx option. It determines the number of and names of the columns -** of the new FTS4 table. +** 2001 September 15 ** -** The third argument passed to this function is the value passed to the -** config=xxx option (i.e. "xxx"). This function queries the database for -** a table of that name. If found, the output variables are populated -** as follows: +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** *pnCol: Set to the number of columns table xxx has, +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** -** *pnStr: Set to the total amount of space required to store a copy -** of each columns name, including the nul-terminator. +************************************************************************* +** This file contains code for implementations of the r-tree and r*-tree +** algorithms packaged as an SQLite virtual table module. +*/ + +/* +** Database Format of R-Tree Tables +** -------------------------------- ** -** *pazCol: Set to point to an array of *pnCol strings. Each string is -** the name of the corresponding column in table xxx. The array -** and its contents are allocated using a single allocation. It -** is the responsibility of the caller to free this allocation -** by eventually passing the *pazCol value to sqlite3_free(). +** The data structure for a single virtual r-tree table is stored in three +** native SQLite tables declared as follows. In each case, the '%' character +** in the table name is replaced with the user-supplied name of the r-tree +** table. ** -** If the table cannot be found, an error code is returned and the output -** variables are undefined. Or, if an OOM is encountered, SQLITE_NOMEM is -** returned (and the output variables are undefined). +** CREATE TABLE %_node(nodeno INTEGER PRIMARY KEY, data BLOB) +** CREATE TABLE %_parent(nodeno INTEGER PRIMARY KEY, parentnode INTEGER) +** CREATE TABLE %_rowid(rowid INTEGER PRIMARY KEY, nodeno INTEGER) +** +** The data for each node of the r-tree structure is stored in the %_node +** table. For each node that is not the root node of the r-tree, there is +** an entry in the %_parent table associating the node with its parent. +** And for each row of data in the table, there is an entry in the %_rowid +** table that maps from the entries rowid to the id of the node that it +** is stored on. +** +** The root node of an r-tree always exists, even if the r-tree table is +** empty. The nodeno of the root node is always 1. All other nodes in the +** table must be the same size as the root node. The content of each node +** is formatted as follows: +** +** 1. If the node is the root node (node 1), then the first 2 bytes +** of the node contain the tree depth as a big-endian integer. +** For non-root nodes, the first 2 bytes are left unused. +** +** 2. The next 2 bytes contain the number of entries currently +** stored in the node. +** +** 3. The remainder of the node contains the node entries. Each entry +** consists of a single 8-byte integer followed by an even number +** of 4-byte coordinates. For leaf nodes the integer is the rowid +** of a record. For internal nodes it is the node number of a +** child page. */ -static int fts3ContentColumns( - sqlite3 *db, /* Database handle */ - const char *zDb, /* Name of db (i.e. "main", "temp" etc.) */ - const char *zTbl, /* Name of content table */ - const char ***pazCol, /* OUT: Malloc'd array of column names */ - int *pnCol, /* OUT: Size of array *pazCol */ - int *pnStr, /* OUT: Bytes of string content */ - char **pzErr /* OUT: error message */ -){ - int rc = SQLITE_OK; /* Return code */ - char *zSql; /* "SELECT *" statement on zTbl */ - sqlite3_stmt *pStmt = 0; /* Compiled version of zSql */ - - zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zTbl); - if( !zSql ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); - if( rc!=SQLITE_OK ){ - sqlite3Fts3ErrMsg(pzErr, "%s", sqlite3_errmsg(db)); - } - } - sqlite3_free(zSql); - - if( rc==SQLITE_OK ){ - const char **azCol; /* Output array */ - int nStr = 0; /* Size of all column names (incl. 0x00) */ - int nCol; /* Number of table columns */ - int i; /* Used to iterate through columns */ - /* Loop through the returned columns. Set nStr to the number of bytes of - ** space required to store a copy of each column name, including the - ** nul-terminator byte. */ - nCol = sqlite3_column_count(pStmt); - for(i=0; i */ +/* #include */ +/* #include */ - return rc; -} +#ifndef SQLITE_AMALGAMATION +#include "sqlite3rtree.h" +typedef sqlite3_int64 i64; +typedef unsigned char u8; +typedef unsigned short u16; +typedef unsigned int u32; +#endif -/* -** This function is the implementation of both the xConnect and xCreate -** methods of the FTS3 virtual table. -** -** The argv[] array contains the following: -** -** argv[0] -> module name ("fts3" or "fts4") -** argv[1] -> database name -** argv[2] -> table name -** argv[...] -> "column name" and other module argument fields. +/* The following macro is used to suppress compiler warnings. */ -static int fts3InitVtab( - int isCreate, /* True for xCreate, false for xConnect */ - sqlite3 *db, /* The SQLite database connection */ - void *pAux, /* Hash table containing tokenizers */ - int argc, /* Number of elements in argv array */ - const char * const *argv, /* xCreate/xConnect argument array */ - sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ - char **pzErr /* Write any error message here */ -){ - Fts3Hash *pHash = (Fts3Hash *)pAux; - Fts3Table *p = 0; /* Pointer to allocated vtab */ - int rc = SQLITE_OK; /* Return code */ - int i; /* Iterator variable */ - int nByte; /* Size of allocation used for *p */ - int iCol; /* Column index */ - int nString = 0; /* Bytes required to hold all column names */ - int nCol = 0; /* Number of columns in the FTS table */ - char *zCsr; /* Space for holding column names */ - int nDb; /* Bytes required to hold database name */ - int nName; /* Bytes required to hold table name */ - int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ - const char **aCol; /* Array of column names */ - sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ +#ifndef UNUSED_PARAMETER +# define UNUSED_PARAMETER(x) (void)(x) +#endif - int nIndex = 0; /* Size of aIndex[] array */ - struct Fts3Index *aIndex = 0; /* Array of indexes for this table */ +typedef struct Rtree Rtree; +typedef struct RtreeCursor RtreeCursor; +typedef struct RtreeNode RtreeNode; +typedef struct RtreeCell RtreeCell; +typedef struct RtreeConstraint RtreeConstraint; +typedef struct RtreeMatchArg RtreeMatchArg; +typedef struct RtreeGeomCallback RtreeGeomCallback; +typedef union RtreeCoord RtreeCoord; +typedef struct RtreeSearchPoint RtreeSearchPoint; - /* The results of parsing supported FTS4 key=value options: */ - int bNoDocsize = 0; /* True to omit %_docsize table */ - int bDescIdx = 0; /* True to store descending indexes */ - char *zPrefix = 0; /* Prefix parameter value (or NULL) */ - char *zCompress = 0; /* compress=? parameter (or NULL) */ - char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ - char *zContent = 0; /* content=? parameter (or NULL) */ - char *zLanguageid = 0; /* languageid=? parameter (or NULL) */ - char **azNotindexed = 0; /* The set of notindexed= columns */ - int nNotindexed = 0; /* Size of azNotindexed[] array */ +/* The rtree may have between 1 and RTREE_MAX_DIMENSIONS dimensions. */ +#define RTREE_MAX_DIMENSIONS 5 - assert( strlen(argv[0])==4 ); - assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) - || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) - ); +/* Size of hash table Rtree.aHash. This hash table is not expected to +** ever contain very many entries, so a fixed number of buckets is +** used. +*/ +#define HASHSIZE 97 - nDb = (int)strlen(argv[1]) + 1; - nName = (int)strlen(argv[2]) + 1; +/* The xBestIndex method of this virtual table requires an estimate of +** the number of rows in the virtual table to calculate the costs of +** various strategies. If possible, this estimate is loaded from the +** sqlite_stat1 table (with RTREE_MIN_ROWEST as a hard-coded minimum). +** Otherwise, if no sqlite_stat1 entry is available, use +** RTREE_DEFAULT_ROWEST. +*/ +#define RTREE_DEFAULT_ROWEST 1048576 +#define RTREE_MIN_ROWEST 100 - nByte = sizeof(const char *) * (argc-2); - aCol = (const char **)sqlite3_malloc(nByte); - if( aCol ){ - memset((void*)aCol, 0, nByte); - azNotindexed = (char **)sqlite3_malloc(nByte); - } - if( azNotindexed ){ - memset(azNotindexed, 0, nByte); - } - if( !aCol || !azNotindexed ){ - rc = SQLITE_NOMEM; - goto fts3_init_out; - } +/* +** An rtree virtual-table object. +*/ +struct Rtree { + sqlite3_vtab base; /* Base class. Must be first */ + sqlite3 *db; /* Host database connection */ + int iNodeSize; /* Size in bytes of each node in the node table */ + u8 nDim; /* Number of dimensions */ + u8 eCoordType; /* RTREE_COORD_REAL32 or RTREE_COORD_INT32 */ + u8 nBytesPerCell; /* Bytes consumed per cell */ + int iDepth; /* Current depth of the r-tree structure */ + char *zDb; /* Name of database containing r-tree table */ + char *zName; /* Name of r-tree table */ + int nBusy; /* Current number of users of this structure */ + i64 nRowEst; /* Estimated number of rows in this table */ - /* Loop through all of the arguments passed by the user to the FTS3/4 - ** module (i.e. all the column names and special arguments). This loop - ** does the following: - ** - ** + Figures out the number of columns the FTSX table will have, and - ** the number of bytes of space that must be allocated to store copies - ** of the column names. - ** - ** + If there is a tokenizer specification included in the arguments, - ** initializes the tokenizer pTokenizer. + /* List of nodes removed during a CondenseTree operation. List is + ** linked together via the pointer normally used for hash chains - + ** RtreeNode.pNext. RtreeNode.iNode stores the depth of the sub-tree + ** headed by the node (leaf nodes have RtreeNode.iNode==0). */ - for(i=3; rc==SQLITE_OK && i8 - && 0==sqlite3_strnicmp(z, "tokenize", 8) - && 0==sqlite3Fts3IsIdChar(z[8]) - ){ - rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr); - } + /* Statements to read/write/delete a record from xxx_node */ + sqlite3_stmt *pReadNode; + sqlite3_stmt *pWriteNode; + sqlite3_stmt *pDeleteNode; - /* Check if it is an FTS4 special argument. */ - else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){ - struct Fts4Option { - const char *zOpt; - int nOpt; - } aFts4Opt[] = { - { "matchinfo", 9 }, /* 0 -> MATCHINFO */ - { "prefix", 6 }, /* 1 -> PREFIX */ - { "compress", 8 }, /* 2 -> COMPRESS */ - { "uncompress", 10 }, /* 3 -> UNCOMPRESS */ - { "order", 5 }, /* 4 -> ORDER */ - { "content", 7 }, /* 5 -> CONTENT */ - { "languageid", 10 }, /* 6 -> LANGUAGEID */ - { "notindexed", 10 } /* 7 -> NOTINDEXED */ - }; + /* Statements to read/write/delete a record from xxx_rowid */ + sqlite3_stmt *pReadRowid; + sqlite3_stmt *pWriteRowid; + sqlite3_stmt *pDeleteRowid; - int iOpt; - if( !zVal ){ - rc = SQLITE_NOMEM; - }else{ - for(iOpt=0; iOptnOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){ - break; - } - } - if( iOpt==SizeofArray(aFts4Opt) ){ - sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z); - rc = SQLITE_ERROR; - }else{ - switch( iOpt ){ - case 0: /* MATCHINFO */ - if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){ - sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal); - rc = SQLITE_ERROR; - } - bNoDocsize = 1; - break; + /* Statements to read/write/delete a record from xxx_parent */ + sqlite3_stmt *pReadParent; + sqlite3_stmt *pWriteParent; + sqlite3_stmt *pDeleteParent; - case 1: /* PREFIX */ - sqlite3_free(zPrefix); - zPrefix = zVal; - zVal = 0; - break; + RtreeNode *aHash[HASHSIZE]; /* Hash table of in-memory nodes. */ +}; - case 2: /* COMPRESS */ - sqlite3_free(zCompress); - zCompress = zVal; - zVal = 0; - break; +/* Possible values for Rtree.eCoordType: */ +#define RTREE_COORD_REAL32 0 +#define RTREE_COORD_INT32 1 - case 3: /* UNCOMPRESS */ - sqlite3_free(zUncompress); - zUncompress = zVal; - zVal = 0; - break; +/* +** If SQLITE_RTREE_INT_ONLY is defined, then this virtual table will +** only deal with integer coordinates. No floating point operations +** will be done. +*/ +#ifdef SQLITE_RTREE_INT_ONLY + typedef sqlite3_int64 RtreeDValue; /* High accuracy coordinate */ + typedef int RtreeValue; /* Low accuracy coordinate */ +# define RTREE_ZERO 0 +#else + typedef double RtreeDValue; /* High accuracy coordinate */ + typedef float RtreeValue; /* Low accuracy coordinate */ +# define RTREE_ZERO 0.0 +#endif - case 4: /* ORDER */ - if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) - && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) - ){ - sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal); - rc = SQLITE_ERROR; - } - bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); - break; +/* +** When doing a search of an r-tree, instances of the following structure +** record intermediate results from the tree walk. +** +** The id is always a node-id. For iLevel>=1 the id is the node-id of +** the node that the RtreeSearchPoint represents. When iLevel==0, however, +** the id is of the parent node and the cell that RtreeSearchPoint +** represents is the iCell-th entry in the parent node. +*/ +struct RtreeSearchPoint { + RtreeDValue rScore; /* The score for this node. Smallest goes first. */ + sqlite3_int64 id; /* Node ID */ + u8 iLevel; /* 0=entries. 1=leaf node. 2+ for higher */ + u8 eWithin; /* PARTLY_WITHIN or FULLY_WITHIN */ + u8 iCell; /* Cell index within the node */ +}; - case 5: /* CONTENT */ - sqlite3_free(zContent); - zContent = zVal; - zVal = 0; - break; +/* +** The minimum number of cells allowed for a node is a third of the +** maximum. In Gutman's notation: +** +** m = M/3 +** +** If an R*-tree "Reinsert" operation is required, the same number of +** cells are removed from the overfull node and reinserted into the tree. +*/ +#define RTREE_MINCELLS(p) ((((p)->iNodeSize-4)/(p)->nBytesPerCell)/3) +#define RTREE_REINSERT(p) RTREE_MINCELLS(p) +#define RTREE_MAXCELLS 51 - case 6: /* LANGUAGEID */ - assert( iOpt==6 ); - sqlite3_free(zLanguageid); - zLanguageid = zVal; - zVal = 0; - break; +/* +** The smallest possible node-size is (512-64)==448 bytes. And the largest +** supported cell size is 48 bytes (8 byte rowid + ten 4 byte coordinates). +** Therefore all non-root nodes must contain at least 3 entries. Since +** 2^40 is greater than 2^64, an r-tree structure always has a depth of +** 40 or less. +*/ +#define RTREE_MAX_DEPTH 40 - case 7: /* NOTINDEXED */ - azNotindexed[nNotindexed++] = zVal; - zVal = 0; - break; - } - } - sqlite3_free(zVal); - } - } - /* Otherwise, the argument is a column name. */ - else { - nString += (int)(strlen(z) + 1); - aCol[nCol++] = z; - } - } +/* +** Number of entries in the cursor RtreeNode cache. The first entry is +** used to cache the RtreeNode for RtreeCursor.sPoint. The remaining +** entries cache the RtreeNode for the first elements of the priority queue. +*/ +#define RTREE_CACHE_SZ 5 - /* If a content=xxx option was specified, the following: - ** - ** 1. Ignore any compress= and uncompress= options. - ** - ** 2. If no column names were specified as part of the CREATE VIRTUAL - ** TABLE statement, use all columns from the content table. - */ - if( rc==SQLITE_OK && zContent ){ - sqlite3_free(zCompress); - sqlite3_free(zUncompress); - zCompress = 0; - zUncompress = 0; - if( nCol==0 ){ - sqlite3_free((void*)aCol); - aCol = 0; - rc = fts3ContentColumns(db, argv[1], zContent,&aCol,&nCol,&nString,pzErr); +/* +** An rtree cursor object. +*/ +struct RtreeCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + u8 atEOF; /* True if at end of search */ + u8 bPoint; /* True if sPoint is valid */ + int iStrategy; /* Copy of idxNum search parameter */ + int nConstraint; /* Number of entries in aConstraint */ + RtreeConstraint *aConstraint; /* Search constraints. */ + int nPointAlloc; /* Number of slots allocated for aPoint[] */ + int nPoint; /* Number of slots used in aPoint[] */ + int mxLevel; /* iLevel value for root of the tree */ + RtreeSearchPoint *aPoint; /* Priority queue for search points */ + RtreeSearchPoint sPoint; /* Cached next search point */ + RtreeNode *aNode[RTREE_CACHE_SZ]; /* Rtree node cache */ + u32 anQueue[RTREE_MAX_DEPTH+1]; /* Number of queued entries by iLevel */ +}; - /* If a languageid= option was specified, remove the language id - ** column from the aCol[] array. */ - if( rc==SQLITE_OK && zLanguageid ){ - int j; - for(j=0; jbase.pVtab)) - if( nCol==0 ){ - assert( nString==0 ); - aCol[0] = "content"; - nString = 8; - nCol = 1; - } +/* +** A coordinate can be either a floating point number or a integer. All +** coordinates within a single R-Tree are always of the same time. +*/ +union RtreeCoord { + RtreeValue f; /* Floating point value */ + int i; /* Integer value */ + u32 u; /* Unsigned for byte-order conversions */ +}; - if( pTokenizer==0 ){ - rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr); - if( rc!=SQLITE_OK ) goto fts3_init_out; - } - assert( pTokenizer ); +/* +** The argument is an RtreeCoord. Return the value stored within the RtreeCoord +** formatted as a RtreeDValue (double or int64). This macro assumes that local +** variable pRtree points to the Rtree structure associated with the +** RtreeCoord. +*/ +#ifdef SQLITE_RTREE_INT_ONLY +# define DCOORD(coord) ((RtreeDValue)coord.i) +#else +# define DCOORD(coord) ( \ + (pRtree->eCoordType==RTREE_COORD_REAL32) ? \ + ((double)coord.f) : \ + ((double)coord.i) \ + ) +#endif - rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex); - if( rc==SQLITE_ERROR ){ - assert( zPrefix ); - sqlite3Fts3ErrMsg(pzErr, "error parsing prefix parameter: %s", zPrefix); - } - if( rc!=SQLITE_OK ) goto fts3_init_out; +/* +** A search constraint. +*/ +struct RtreeConstraint { + int iCoord; /* Index of constrained coordinate */ + int op; /* Constraining operation */ + union { + RtreeDValue rValue; /* Constraint value. */ + int (*xGeom)(sqlite3_rtree_geometry*,int,RtreeDValue*,int*); + int (*xQueryFunc)(sqlite3_rtree_query_info*); + } u; + sqlite3_rtree_query_info *pInfo; /* xGeom and xQueryFunc argument */ +}; - /* Allocate and populate the Fts3Table structure. */ - nByte = sizeof(Fts3Table) + /* Fts3Table */ - nCol * sizeof(char *) + /* azColumn */ - nIndex * sizeof(struct Fts3Index) + /* aIndex */ - nCol * sizeof(u8) + /* abNotindexed */ - nName + /* zName */ - nDb + /* zDb */ - nString; /* Space for azColumn strings */ - p = (Fts3Table*)sqlite3_malloc(nByte); - if( p==0 ){ - rc = SQLITE_NOMEM; - goto fts3_init_out; - } - memset(p, 0, nByte); - p->db = db; - p->nColumn = nCol; - p->nPendingData = 0; - p->azColumn = (char **)&p[1]; - p->pTokenizer = pTokenizer; - p->nMaxPendingData = FTS3_MAX_PENDING_DATA; - p->bHasDocsize = (isFts4 && bNoDocsize==0); - p->bHasStat = isFts4; - p->bFts4 = isFts4; - p->bDescIdx = bDescIdx; - p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */ - p->zContentTbl = zContent; - p->zLanguageid = zLanguageid; - zContent = 0; - zLanguageid = 0; - TESTONLY( p->inTransaction = -1 ); - TESTONLY( p->mxSavepoint = -1 ); +/* Possible values for RtreeConstraint.op */ +#define RTREE_EQ 0x41 /* A */ +#define RTREE_LE 0x42 /* B */ +#define RTREE_LT 0x43 /* C */ +#define RTREE_GE 0x44 /* D */ +#define RTREE_GT 0x45 /* E */ +#define RTREE_MATCH 0x46 /* F: Old-style sqlite3_rtree_geometry_callback() */ +#define RTREE_QUERY 0x47 /* G: New-style sqlite3_rtree_query_callback() */ - p->aIndex = (struct Fts3Index *)&p->azColumn[nCol]; - memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex); - p->nIndex = nIndex; - for(i=0; iaIndex[i].hPending, FTS3_HASH_STRING, 1); - } - p->abNotindexed = (u8 *)&p->aIndex[nIndex]; - /* Fill in the zName and zDb fields of the vtab structure. */ - zCsr = (char *)&p->abNotindexed[nCol]; - p->zName = zCsr; - memcpy(zCsr, argv[2], nName); - zCsr += nName; - p->zDb = zCsr; - memcpy(zCsr, argv[1], nDb); - zCsr += nDb; +/* +** An rtree structure node. +*/ +struct RtreeNode { + RtreeNode *pParent; /* Parent node */ + i64 iNode; /* The node number */ + int nRef; /* Number of references to this node */ + int isDirty; /* True if the node needs to be written to disk */ + u8 *zData; /* Content of the node, as should be on disk */ + RtreeNode *pNext; /* Next node in this hash collision chain */ +}; - /* Fill in the azColumn array */ - for(iCol=0; iColazColumn[iCol] = zCsr; - zCsr += n+1; - assert( zCsr <= &((char *)p)[nByte] ); - } +/* Return the number of cells in a node */ +#define NCELL(pNode) readInt16(&(pNode)->zData[2]) - /* Fill in the abNotindexed array */ - for(iCol=0; iColazColumn[iCol]); - for(i=0; iazColumn[iCol], zNot, n) - ){ - p->abNotindexed[iCol] = 1; - sqlite3_free(zNot); - azNotindexed[i] = 0; - } - } - } - for(i=0; izReadExprlist = fts3ReadExprList(p, zUncompress, &rc); - p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc); - if( rc!=SQLITE_OK ) goto fts3_init_out; - /* If this is an xCreate call, create the underlying tables in the - ** database. TODO: For xConnect(), it could verify that said tables exist. - */ - if( isCreate ){ - rc = fts3CreateTables(p); - } +/* +** This object becomes the sqlite3_user_data() for the SQL functions +** that are created by sqlite3_rtree_geometry_callback() and +** sqlite3_rtree_query_callback() and which appear on the right of MATCH +** operators in order to constrain a search. +** +** xGeom and xQueryFunc are the callback functions. Exactly one of +** xGeom and xQueryFunc fields is non-NULL, depending on whether the +** SQL function was created using sqlite3_rtree_geometry_callback() or +** sqlite3_rtree_query_callback(). +** +** This object is deleted automatically by the destructor mechanism in +** sqlite3_create_function_v2(). +*/ +struct RtreeGeomCallback { + int (*xGeom)(sqlite3_rtree_geometry*, int, RtreeDValue*, int*); + int (*xQueryFunc)(sqlite3_rtree_query_info*); + void (*xDestructor)(void*); + void *pContext; +}; - /* Check to see if a legacy fts3 table has been "upgraded" by the - ** addition of a %_stat table so that it can use incremental merge. - */ - if( !isFts4 && !isCreate ){ - p->bHasStat = 2; - } - /* Figure out the page-size for the database. This is required in order to - ** estimate the cost of loading large doclists from the database. */ - fts3DatabasePageSize(&rc, p); - p->nNodeSize = p->nPgsz-35; +/* +** Value for the first field of every RtreeMatchArg object. The MATCH +** operator tests that the first field of a blob operand matches this +** value to avoid operating on invalid blobs (which could cause a segfault). +*/ +#define RTREE_GEOMETRY_MAGIC 0x891245AB - /* Declare the table schema to SQLite. */ - fts3DeclareVtab(&rc, p); +/* +** An instance of this structure (in the form of a BLOB) is returned by +** the SQL functions that sqlite3_rtree_geometry_callback() and +** sqlite3_rtree_query_callback() create, and is read as the right-hand +** operand to the MATCH operator of an R-Tree. +*/ +struct RtreeMatchArg { + u32 magic; /* Always RTREE_GEOMETRY_MAGIC */ + RtreeGeomCallback cb; /* Info about the callback functions */ + int nParam; /* Number of parameters to the SQL function */ + sqlite3_value **apSqlParam; /* Original SQL parameter values */ + RtreeDValue aParam[1]; /* Values for parameters to the SQL function */ +}; -fts3_init_out: - sqlite3_free(zPrefix); - sqlite3_free(aIndex); - sqlite3_free(zCompress); - sqlite3_free(zUncompress); - sqlite3_free(zContent); - sqlite3_free(zLanguageid); - for(i=0; ipModule->xDestroy(pTokenizer); - } - }else{ - assert( p->pSegments==0 ); - *ppVTab = &p->base; - } - return rc; -} +#ifndef MAX +# define MAX(x,y) ((x) < (y) ? (y) : (x)) +#endif +#ifndef MIN +# define MIN(x,y) ((x) > (y) ? (y) : (x)) +#endif /* -** The xConnect() and xCreate() methods for the virtual table. All the -** work is done in function fts3InitVtab(). +** Functions to deserialize a 16 bit integer, 32 bit real number and +** 64 bit integer. The deserialized value is returned. */ -static int fts3ConnectMethod( - sqlite3 *db, /* Database connection */ - void *pAux, /* Pointer to tokenizer hash table */ - int argc, /* Number of elements in argv array */ - const char * const *argv, /* xCreate/xConnect argument array */ - sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ - char **pzErr /* OUT: sqlite3_malloc'd error message */ -){ - return fts3InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr); +static int readInt16(u8 *p){ + return (p[0]<<8) + p[1]; } -static int fts3CreateMethod( - sqlite3 *db, /* Database connection */ - void *pAux, /* Pointer to tokenizer hash table */ - int argc, /* Number of elements in argv array */ - const char * const *argv, /* xCreate/xConnect argument array */ - sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ - char **pzErr /* OUT: sqlite3_malloc'd error message */ -){ - return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr); +static void readCoord(u8 *p, RtreeCoord *pCoord){ + pCoord->u = ( + (((u32)p[0]) << 24) + + (((u32)p[1]) << 16) + + (((u32)p[2]) << 8) + + (((u32)p[3]) << 0) + ); +} +static i64 readInt64(u8 *p){ + return ( + (((i64)p[0]) << 56) + + (((i64)p[1]) << 48) + + (((i64)p[2]) << 40) + + (((i64)p[3]) << 32) + + (((i64)p[4]) << 24) + + (((i64)p[5]) << 16) + + (((i64)p[6]) << 8) + + (((i64)p[7]) << 0) + ); } /* -** Set the pIdxInfo->estimatedRows variable to nRow. Unless this -** extension is currently being used by a version of SQLite too old to -** support estimatedRows. In that case this function is a no-op. +** Functions to serialize a 16 bit integer, 32 bit real number and +** 64 bit integer. The value returned is the number of bytes written +** to the argument buffer (always 2, 4 and 8 respectively). */ -static void fts3SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ -#if SQLITE_VERSION_NUMBER>=3008002 - if( sqlite3_libversion_number()>=3008002 ){ - pIdxInfo->estimatedRows = nRow; - } -#endif +static int writeInt16(u8 *p, int i){ + p[0] = (i>> 8)&0xFF; + p[1] = (i>> 0)&0xFF; + return 2; +} +static int writeCoord(u8 *p, RtreeCoord *pCoord){ + u32 i; + assert( sizeof(RtreeCoord)==4 ); + assert( sizeof(u32)==4 ); + i = pCoord->u; + p[0] = (i>>24)&0xFF; + p[1] = (i>>16)&0xFF; + p[2] = (i>> 8)&0xFF; + p[3] = (i>> 0)&0xFF; + return 4; +} +static int writeInt64(u8 *p, i64 i){ + p[0] = (i>>56)&0xFF; + p[1] = (i>>48)&0xFF; + p[2] = (i>>40)&0xFF; + p[3] = (i>>32)&0xFF; + p[4] = (i>>24)&0xFF; + p[5] = (i>>16)&0xFF; + p[6] = (i>> 8)&0xFF; + p[7] = (i>> 0)&0xFF; + return 8; } -/* -** Implementation of the xBestIndex method for FTS3 tables. There -** are three possible strategies, in order of preference: -** -** 1. Direct lookup by rowid or docid. -** 2. Full-text search using a MATCH operator on a non-docid column. -** 3. Linear scan of %_content table. +/* +** Increment the reference count of node p. */ -static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ - Fts3Table *p = (Fts3Table *)pVTab; - int i; /* Iterator variable */ - int iCons = -1; /* Index of constraint to use */ - - int iLangidCons = -1; /* Index of langid=x constraint, if present */ - int iDocidGe = -1; /* Index of docid>=x constraint, if present */ - int iDocidLe = -1; /* Index of docid<=x constraint, if present */ - int iIdx; - - /* By default use a full table scan. This is an expensive option, - ** so search through the constraints to see if a more efficient - ** strategy is possible. - */ - pInfo->idxNum = FTS3_FULLSCAN_SEARCH; - pInfo->estimatedCost = 5000000; - for(i=0; inConstraint; i++){ - int bDocid; /* True if this constraint is on docid */ - struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i]; - if( pCons->usable==0 ){ - if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH ){ - /* There exists an unusable MATCH constraint. This means that if - ** the planner does elect to use the results of this call as part - ** of the overall query plan the user will see an "unable to use - ** function MATCH in the requested context" error. To discourage - ** this, return a very high cost here. */ - pInfo->idxNum = FTS3_FULLSCAN_SEARCH; - pInfo->estimatedCost = 1e50; - fts3SetEstimatedRows(pInfo, ((sqlite3_int64)1) << 50); - return SQLITE_OK; - } - continue; - } - - bDocid = (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1); - - /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */ - if( iCons<0 && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ && bDocid ){ - pInfo->idxNum = FTS3_DOCID_SEARCH; - pInfo->estimatedCost = 1.0; - iCons = i; - } - - /* A MATCH constraint. Use a full-text search. - ** - ** If there is more than one MATCH constraint available, use the first - ** one encountered. If there is both a MATCH constraint and a direct - ** rowid/docid lookup, prefer the MATCH strategy. This is done even - ** though the rowid/docid lookup is faster than a MATCH query, selecting - ** it would lead to an "unable to use function MATCH in the requested - ** context" error. - */ - if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH - && pCons->iColumn>=0 && pCons->iColumn<=p->nColumn - ){ - pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn; - pInfo->estimatedCost = 2.0; - iCons = i; - } - - /* Equality constraint on the langid column */ - if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ - && pCons->iColumn==p->nColumn + 2 - ){ - iLangidCons = i; - } - - if( bDocid ){ - switch( pCons->op ){ - case SQLITE_INDEX_CONSTRAINT_GE: - case SQLITE_INDEX_CONSTRAINT_GT: - iDocidGe = i; - break; - - case SQLITE_INDEX_CONSTRAINT_LE: - case SQLITE_INDEX_CONSTRAINT_LT: - iDocidLe = i; - break; - } - } +static void nodeReference(RtreeNode *p){ + if( p ){ + p->nRef++; } +} - iIdx = 1; - if( iCons>=0 ){ - pInfo->aConstraintUsage[iCons].argvIndex = iIdx++; - pInfo->aConstraintUsage[iCons].omit = 1; - } - if( iLangidCons>=0 ){ - pInfo->idxNum |= FTS3_HAVE_LANGID; - pInfo->aConstraintUsage[iLangidCons].argvIndex = iIdx++; - } - if( iDocidGe>=0 ){ - pInfo->idxNum |= FTS3_HAVE_DOCID_GE; - pInfo->aConstraintUsage[iDocidGe].argvIndex = iIdx++; - } - if( iDocidLe>=0 ){ - pInfo->idxNum |= FTS3_HAVE_DOCID_LE; - pInfo->aConstraintUsage[iDocidLe].argvIndex = iIdx++; - } - - /* Regardless of the strategy selected, FTS can deliver rows in rowid (or - ** docid) order. Both ascending and descending are possible. - */ - if( pInfo->nOrderBy==1 ){ - struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0]; - if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){ - if( pOrder->desc ){ - pInfo->idxStr = "DESC"; - }else{ - pInfo->idxStr = "ASC"; - } - pInfo->orderByConsumed = 1; - } - } +/* +** Clear the content of node p (set all bytes to 0x00). +*/ +static void nodeZero(Rtree *pRtree, RtreeNode *p){ + memset(&p->zData[2], 0, pRtree->iNodeSize-2); + p->isDirty = 1; +} - assert( p->pSegments==0 ); - return SQLITE_OK; +/* +** Given a node number iNode, return the corresponding key to use +** in the Rtree.aHash table. +*/ +static int nodeHash(i64 iNode){ + return iNode % HASHSIZE; } /* -** Implementation of xOpen method. +** Search the node hash table for node iNode. If found, return a pointer +** to it. Otherwise, return 0. */ -static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ - sqlite3_vtab_cursor *pCsr; /* Allocated cursor */ - - UNUSED_PARAMETER(pVTab); - - /* Allocate a buffer large enough for an Fts3Cursor structure. If the - ** allocation succeeds, zero it and return SQLITE_OK. Otherwise, - ** if the allocation fails, return SQLITE_NOMEM. - */ - *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor)); - if( !pCsr ){ - return SQLITE_NOMEM; - } - memset(pCsr, 0, sizeof(Fts3Cursor)); - return SQLITE_OK; +static RtreeNode *nodeHashLookup(Rtree *pRtree, i64 iNode){ + RtreeNode *p; + for(p=pRtree->aHash[nodeHash(iNode)]; p && p->iNode!=iNode; p=p->pNext); + return p; } /* -** Close the cursor. For additional information see the documentation -** on the xClose method of the virtual table interface. +** Add node pNode to the node hash table. */ -static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ - Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; - assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); - sqlite3_finalize(pCsr->pStmt); - sqlite3Fts3ExprFree(pCsr->pExpr); - sqlite3Fts3FreeDeferredTokens(pCsr); - sqlite3_free(pCsr->aDoclist); - sqlite3_free(pCsr->aMatchinfo); - assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); - sqlite3_free(pCsr); - return SQLITE_OK; +static void nodeHashInsert(Rtree *pRtree, RtreeNode *pNode){ + int iHash; + assert( pNode->pNext==0 ); + iHash = nodeHash(pNode->iNode); + pNode->pNext = pRtree->aHash[iHash]; + pRtree->aHash[iHash] = pNode; } /* -** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then -** compose and prepare an SQL statement of the form: -** -** "SELECT FROM %_content WHERE rowid = ?" -** -** (or the equivalent for a content=xxx table) and set pCsr->pStmt to -** it. If an error occurs, return an SQLite error code. -** -** Otherwise, set *ppStmt to point to pCsr->pStmt and return SQLITE_OK. +** Remove node pNode from the node hash table. */ -static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){ - int rc = SQLITE_OK; - if( pCsr->pStmt==0 ){ - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; - char *zSql; - zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); - if( !zSql ) return SQLITE_NOMEM; - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); - sqlite3_free(zSql); +static void nodeHashDelete(Rtree *pRtree, RtreeNode *pNode){ + RtreeNode **pp; + if( pNode->iNode!=0 ){ + pp = &pRtree->aHash[nodeHash(pNode->iNode)]; + for( ; (*pp)!=pNode; pp = &(*pp)->pNext){ assert(*pp); } + *pp = pNode->pNext; + pNode->pNext = 0; } - *ppStmt = pCsr->pStmt; - return rc; } /* -** Position the pCsr->pStmt statement so that it is on the row -** of the %_content table that contains the last match. Return -** SQLITE_OK on success. +** Allocate and return new r-tree node. Initially, (RtreeNode.iNode==0), +** indicating that node has not yet been assigned a node number. It is +** assigned a node number when nodeWrite() is called to write the +** node contents out to the database. */ -static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ - int rc = SQLITE_OK; - if( pCsr->isRequireSeek ){ - sqlite3_stmt *pStmt = 0; - - rc = fts3CursorSeekStmt(pCsr, &pStmt); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); - pCsr->isRequireSeek = 0; - if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ - return SQLITE_OK; - }else{ - rc = sqlite3_reset(pCsr->pStmt); - if( rc==SQLITE_OK && ((Fts3Table *)pCsr->base.pVtab)->zContentTbl==0 ){ - /* If no row was found and no error has occurred, then the %_content - ** table is missing a row that is present in the full-text index. - ** The data structures are corrupt. */ - rc = FTS_CORRUPT_VTAB; - pCsr->isEof = 1; - } - } - } - } - - if( rc!=SQLITE_OK && pContext ){ - sqlite3_result_error_code(pContext, rc); +static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ + RtreeNode *pNode; + pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode) + pRtree->iNodeSize); + if( pNode ){ + memset(pNode, 0, sizeof(RtreeNode) + pRtree->iNodeSize); + pNode->zData = (u8 *)&pNode[1]; + pNode->nRef = 1; + pNode->pParent = pParent; + pNode->isDirty = 1; + nodeReference(pParent); } - return rc; + return pNode; } /* -** This function is used to process a single interior node when searching -** a b-tree for a term or term prefix. The node data is passed to this -** function via the zNode/nNode parameters. The term to search for is -** passed in zTerm/nTerm. -** -** If piFirst is not NULL, then this function sets *piFirst to the blockid -** of the child node that heads the sub-tree that may contain the term. -** -** If piLast is not NULL, then *piLast is set to the right-most child node -** that heads a sub-tree that may contain a term for which zTerm/nTerm is -** a prefix. -** -** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. +** Obtain a reference to an r-tree node. */ -static int fts3ScanInteriorNode( - const char *zTerm, /* Term to select leaves for */ - int nTerm, /* Size of term zTerm in bytes */ - const char *zNode, /* Buffer containing segment interior node */ - int nNode, /* Size of buffer at zNode */ - sqlite3_int64 *piFirst, /* OUT: Selected child node */ - sqlite3_int64 *piLast /* OUT: Selected child node */ +static int nodeAcquire( + Rtree *pRtree, /* R-tree structure */ + i64 iNode, /* Node number to load */ + RtreeNode *pParent, /* Either the parent node or NULL */ + RtreeNode **ppNode /* OUT: Acquired node */ ){ - int rc = SQLITE_OK; /* Return code */ - const char *zCsr = zNode; /* Cursor to iterate through node */ - const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ - char *zBuffer = 0; /* Buffer to load terms into */ - int nAlloc = 0; /* Size of allocated buffer */ - int isFirstTerm = 1; /* True when processing first term on page */ - sqlite3_int64 iChild; /* Block id of child node to descend to */ + int rc; + int rc2 = SQLITE_OK; + RtreeNode *pNode; - /* Skip over the 'height' varint that occurs at the start of every - ** interior node. Then load the blockid of the left-child of the b-tree - ** node into variable iChild. - ** - ** Even if the data structure on disk is corrupted, this (reading two - ** varints from the buffer) does not risk an overread. If zNode is a - ** root node, then the buffer comes from a SELECT statement. SQLite does - ** not make this guarantee explicitly, but in practice there are always - ** either more than 20 bytes of allocated space following the nNode bytes of - ** contents, or two zero bytes. Or, if the node is read from the %_segments - ** table, then there are always 20 bytes of zeroed padding following the - ** nNode bytes of content (see sqlite3Fts3ReadBlock() for details). + /* Check if the requested node is already in the hash table. If so, + ** increase its reference count and return it. */ - zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); - zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); - if( zCsr>zEnd ){ - return FTS_CORRUPT_VTAB; - } - - while( zCsrzEnd ){ - rc = FTS_CORRUPT_VTAB; - goto finish_scan; + if( (pNode = nodeHashLookup(pRtree, iNode)) ){ + assert( !pParent || !pNode->pParent || pNode->pParent==pParent ); + if( pParent && !pNode->pParent ){ + nodeReference(pParent); + pNode->pParent = pParent; } - if( nPrefix+nSuffix>nAlloc ){ - char *zNew; - nAlloc = (nPrefix+nSuffix) * 2; - zNew = (char *)sqlite3_realloc(zBuffer, nAlloc); - if( !zNew ){ - rc = SQLITE_NOMEM; - goto finish_scan; + pNode->nRef++; + *ppNode = pNode; + return SQLITE_OK; + } + + sqlite3_bind_int64(pRtree->pReadNode, 1, iNode); + rc = sqlite3_step(pRtree->pReadNode); + if( rc==SQLITE_ROW ){ + const u8 *zBlob = sqlite3_column_blob(pRtree->pReadNode, 0); + if( pRtree->iNodeSize==sqlite3_column_bytes(pRtree->pReadNode, 0) ){ + pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode)+pRtree->iNodeSize); + if( !pNode ){ + rc2 = SQLITE_NOMEM; + }else{ + pNode->pParent = pParent; + pNode->zData = (u8 *)&pNode[1]; + pNode->nRef = 1; + pNode->iNode = iNode; + pNode->isDirty = 0; + pNode->pNext = 0; + memcpy(pNode->zData, zBlob, pRtree->iNodeSize); + nodeReference(pParent); } - zBuffer = zNew; } - assert( zBuffer ); - memcpy(&zBuffer[nPrefix], zCsr, nSuffix); - nBuffer = nPrefix + nSuffix; - zCsr += nSuffix; + } + rc = sqlite3_reset(pRtree->pReadNode); + if( rc==SQLITE_OK ) rc = rc2; - /* Compare the term we are searching for with the term just loaded from - ** the interior node. If the specified term is greater than or equal - ** to the term from the interior node, then all terms on the sub-tree - ** headed by node iChild are smaller than zTerm. No need to search - ** iChild. - ** - ** If the interior node term is larger than the specified term, then - ** the tree headed by iChild may contain the specified term. - */ - cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); - if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){ - *piFirst = iChild; - piFirst = 0; + /* If the root node was just loaded, set pRtree->iDepth to the height + ** of the r-tree structure. A height of zero means all data is stored on + ** the root node. A height of one means the children of the root node + ** are the leaves, and so on. If the depth as specified on the root node + ** is greater than RTREE_MAX_DEPTH, the r-tree structure must be corrupt. + */ + if( pNode && iNode==1 ){ + pRtree->iDepth = readInt16(pNode->zData); + if( pRtree->iDepth>RTREE_MAX_DEPTH ){ + rc = SQLITE_CORRUPT_VTAB; } + } - if( piLast && cmp<0 ){ - *piLast = iChild; - piLast = 0; + /* If no error has occurred so far, check if the "number of entries" + ** field on the node is too large. If so, set the return code to + ** SQLITE_CORRUPT_VTAB. + */ + if( pNode && rc==SQLITE_OK ){ + if( NCELL(pNode)>((pRtree->iNodeSize-4)/pRtree->nBytesPerCell) ){ + rc = SQLITE_CORRUPT_VTAB; } + } - iChild++; - }; - - if( piFirst ) *piFirst = iChild; - if( piLast ) *piLast = iChild; + if( rc==SQLITE_OK ){ + if( pNode!=0 ){ + nodeHashInsert(pRtree, pNode); + }else{ + rc = SQLITE_CORRUPT_VTAB; + } + *ppNode = pNode; + }else{ + sqlite3_free(pNode); + *ppNode = 0; + } - finish_scan: - sqlite3_free(zBuffer); return rc; } +/* +** Overwrite cell iCell of node pNode with the contents of pCell. +*/ +static void nodeOverwriteCell( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node into which the cell is to be written */ + RtreeCell *pCell, /* The cell to write */ + int iCell /* Index into pNode into which pCell is written */ +){ + int ii; + u8 *p = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; + p += writeInt64(p, pCell->iRowid); + for(ii=0; ii<(pRtree->nDim*2); ii++){ + p += writeCoord(p, &pCell->aCoord[ii]); + } + pNode->isDirty = 1; +} /* -** The buffer pointed to by argument zNode (size nNode bytes) contains an -** interior node of a b-tree segment. The zTerm buffer (size nTerm bytes) -** contains a term. This function searches the sub-tree headed by the zNode -** node for the range of leaf nodes that may contain the specified term -** or terms for which the specified term is a prefix. -** -** If piLeaf is not NULL, then *piLeaf is set to the blockid of the -** left-most leaf node in the tree that may contain the specified term. -** If piLeaf2 is not NULL, then *piLeaf2 is set to the blockid of the -** right-most leaf node that may contain a term for which the specified -** term is a prefix. -** -** It is possible that the range of returned leaf nodes does not contain -** the specified term or any terms for which it is a prefix. However, if the -** segment does contain any such terms, they are stored within the identified -** range. Because this function only inspects interior segment nodes (and -** never loads leaf nodes into memory), it is not possible to be sure. +** Remove the cell with index iCell from node pNode. +*/ +static void nodeDeleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell){ + u8 *pDst = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; + u8 *pSrc = &pDst[pRtree->nBytesPerCell]; + int nByte = (NCELL(pNode) - iCell - 1) * pRtree->nBytesPerCell; + memmove(pDst, pSrc, nByte); + writeInt16(&pNode->zData[2], NCELL(pNode)-1); + pNode->isDirty = 1; +} + +/* +** Insert the contents of cell pCell into node pNode. If the insert +** is successful, return SQLITE_OK. ** -** If an error occurs, an error code other than SQLITE_OK is returned. -*/ -static int fts3SelectLeaf( - Fts3Table *p, /* Virtual table handle */ - const char *zTerm, /* Term to select leaves for */ - int nTerm, /* Size of term zTerm in bytes */ - const char *zNode, /* Buffer containing segment interior node */ - int nNode, /* Size of buffer at zNode */ - sqlite3_int64 *piLeaf, /* Selected leaf node */ - sqlite3_int64 *piLeaf2 /* Selected leaf node */ +** If there is not enough free space in pNode, return SQLITE_FULL. +*/ +static int nodeInsertCell( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* Write new cell into this node */ + RtreeCell *pCell /* The cell to be inserted */ ){ - int rc = SQLITE_OK; /* Return code */ - int iHeight; /* Height of this node in tree */ - - assert( piLeaf || piLeaf2 ); + int nCell; /* Current number of cells in pNode */ + int nMaxCell; /* Maximum number of cells for pNode */ - fts3GetVarint32(zNode, &iHeight); - rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2); - assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) ); + nMaxCell = (pRtree->iNodeSize-4)/pRtree->nBytesPerCell; + nCell = NCELL(pNode); - if( rc==SQLITE_OK && iHeight>1 ){ - char *zBlob = 0; /* Blob read from %_segments table */ - int nBlob = 0; /* Size of zBlob in bytes */ + assert( nCell<=nMaxCell ); + if( nCellzData[2], nCell+1); + pNode->isDirty = 1; + } - if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ - rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0); - if( rc==SQLITE_OK ){ - rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0); - } - sqlite3_free(zBlob); - piLeaf = 0; - zBlob = 0; - } + return (nCell==nMaxCell); +} - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0); +/* +** If the node is dirty, write it out to the database. +*/ +static int nodeWrite(Rtree *pRtree, RtreeNode *pNode){ + int rc = SQLITE_OK; + if( pNode->isDirty ){ + sqlite3_stmt *p = pRtree->pWriteNode; + if( pNode->iNode ){ + sqlite3_bind_int64(p, 1, pNode->iNode); + }else{ + sqlite3_bind_null(p, 1); } - if( rc==SQLITE_OK ){ - rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); + sqlite3_bind_blob(p, 2, pNode->zData, pRtree->iNodeSize, SQLITE_STATIC); + sqlite3_step(p); + pNode->isDirty = 0; + rc = sqlite3_reset(p); + if( pNode->iNode==0 && rc==SQLITE_OK ){ + pNode->iNode = sqlite3_last_insert_rowid(pRtree->db); + nodeHashInsert(pRtree, pNode); } - sqlite3_free(zBlob); } + return rc; +} +/* +** Release a reference to a node. If the node is dirty and the reference +** count drops to zero, the node data is written to the database. +*/ +static int nodeRelease(Rtree *pRtree, RtreeNode *pNode){ + int rc = SQLITE_OK; + if( pNode ){ + assert( pNode->nRef>0 ); + pNode->nRef--; + if( pNode->nRef==0 ){ + if( pNode->iNode==1 ){ + pRtree->iDepth = -1; + } + if( pNode->pParent ){ + rc = nodeRelease(pRtree, pNode->pParent); + } + if( rc==SQLITE_OK ){ + rc = nodeWrite(pRtree, pNode); + } + nodeHashDelete(pRtree, pNode); + sqlite3_free(pNode); + } + } return rc; } /* -** This function is used to create delta-encoded serialized lists of FTS3 -** varints. Each call to this function appends a single varint to a list. +** Return the 64-bit integer value associated with cell iCell of +** node pNode. If pNode is a leaf node, this is a rowid. If it is +** an internal node, then the 64-bit integer is a child page number. */ -static void fts3PutDeltaVarint( - char **pp, /* IN/OUT: Output pointer */ - sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ - sqlite3_int64 iVal /* Write this value to the list */ +static i64 nodeGetRowid( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node from which to extract the ID */ + int iCell /* The cell index from which to extract the ID */ ){ - assert( iVal-*piPrev > 0 || (*piPrev==0 && iVal==0) ); - *pp += sqlite3Fts3PutVarint(*pp, iVal-*piPrev); - *piPrev = iVal; + assert( iCellzData[4 + pRtree->nBytesPerCell*iCell]); } /* -** When this function is called, *ppPoslist is assumed to point to the -** start of a position-list. After it returns, *ppPoslist points to the -** first byte after the position-list. -** -** A position list is list of positions (delta encoded) and columns for -** a single document record of a doclist. So, in other words, this -** routine advances *ppPoslist so that it points to the next docid in -** the doclist, or to the first byte past the end of the doclist. -** -** If pp is not NULL, then the contents of the position list are copied -** to *pp. *pp is set to point to the first byte past the last byte copied -** before this function returns. +** Return coordinate iCoord from cell iCell in node pNode. */ -static void fts3PoslistCopy(char **pp, char **ppPoslist){ - char *pEnd = *ppPoslist; - char c = 0; - - /* The end of a position list is marked by a zero encoded as an FTS3 - ** varint. A single POS_END (0) byte. Except, if the 0 byte is preceded by - ** a byte with the 0x80 bit set, then it is not a varint 0, but the tail - ** of some other, multi-byte, value. - ** - ** The following while-loop moves pEnd to point to the first byte that is not - ** immediately preceded by a byte with the 0x80 bit set. Then increments - ** pEnd once more so that it points to the byte immediately following the - ** last byte in the position-list. - */ - while( *pEnd | c ){ - c = *pEnd++ & 0x80; - testcase( c!=0 && (*pEnd)==0 ); - } - pEnd++; /* Advance past the POS_END terminator byte */ +static void nodeGetCoord( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node from which to extract a coordinate */ + int iCell, /* The index of the cell within the node */ + int iCoord, /* Which coordinate to extract */ + RtreeCoord *pCoord /* OUT: Space to write result to */ +){ + readCoord(&pNode->zData[12 + pRtree->nBytesPerCell*iCell + 4*iCoord], pCoord); +} - if( pp ){ - int n = (int)(pEnd - *ppPoslist); - char *p = *pp; - memcpy(p, *ppPoslist, n); - p += n; - *pp = p; +/* +** Deserialize cell iCell of node pNode. Populate the structure pointed +** to by pCell with the results. +*/ +static void nodeGetCell( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node containing the cell to be read */ + int iCell, /* Index of the cell within the node */ + RtreeCell *pCell /* OUT: Write the cell contents here */ +){ + u8 *pData; + RtreeCoord *pCoord; + int ii; + pCell->iRowid = nodeGetRowid(pRtree, pNode, iCell); + pData = pNode->zData + (12 + pRtree->nBytesPerCell*iCell); + pCoord = pCell->aCoord; + for(ii=0; iinDim*2; ii++){ + readCoord(&pData[ii*4], &pCoord[ii]); } - *ppPoslist = pEnd; } -/* -** When this function is called, *ppPoslist is assumed to point to the -** start of a column-list. After it returns, *ppPoslist points to the -** to the terminator (POS_COLUMN or POS_END) byte of the column-list. -** -** A column-list is list of delta-encoded positions for a single column -** within a single document within a doclist. -** -** The column-list is terminated either by a POS_COLUMN varint (1) or -** a POS_END varint (0). This routine leaves *ppPoslist pointing to -** the POS_COLUMN or POS_END that terminates the column-list. -** -** If pp is not NULL, then the contents of the column-list are copied -** to *pp. *pp is set to point to the first byte past the last byte copied -** before this function returns. The POS_COLUMN or POS_END terminator -** is not copied into *pp. + +/* Forward declaration for the function that does the work of +** the virtual table module xCreate() and xConnect() methods. */ -static void fts3ColumnlistCopy(char **pp, char **ppPoslist){ - char *pEnd = *ppPoslist; - char c = 0; +static int rtreeInit( + sqlite3 *, void *, int, const char *const*, sqlite3_vtab **, char **, int +); - /* A column-list is terminated by either a 0x01 or 0x00 byte that is - ** not part of a multi-byte varint. - */ - while( 0xFE & (*pEnd | c) ){ - c = *pEnd++ & 0x80; - testcase( c!=0 && ((*pEnd)&0xfe)==0 ); - } - if( pp ){ - int n = (int)(pEnd - *ppPoslist); - char *p = *pp; - memcpy(p, *ppPoslist, n); - p += n; - *pp = p; - } - *ppPoslist = pEnd; +/* +** Rtree virtual table module xCreate method. +*/ +static int rtreeCreate( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + return rtreeInit(db, pAux, argc, argv, ppVtab, pzErr, 1); } -/* -** Value used to signify the end of an position-list. This is safe because -** it is not possible to have a document with 2^31 terms. +/* +** Rtree virtual table module xConnect method. */ -#define POSITION_LIST_END 0x7fffffff +static int rtreeConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + return rtreeInit(db, pAux, argc, argv, ppVtab, pzErr, 0); +} /* -** This function is used to help parse position-lists. When this function is -** called, *pp may point to the start of the next varint in the position-list -** being parsed, or it may point to 1 byte past the end of the position-list -** (in which case **pp will be a terminator bytes POS_END (0) or -** (1)). -** -** If *pp points past the end of the current position-list, set *pi to -** POSITION_LIST_END and return. Otherwise, read the next varint from *pp, -** increment the current value of *pi by the value read, and set *pp to -** point to the next value before returning. -** -** Before calling this routine *pi must be initialized to the value of -** the previous position, or zero if we are reading the first position -** in the position-list. Because positions are delta-encoded, the value -** of the previous position is needed in order to compute the value of -** the next position. +** Increment the r-tree reference count. */ -static void fts3ReadNextPos( - char **pp, /* IN/OUT: Pointer into position-list buffer */ - sqlite3_int64 *pi /* IN/OUT: Value read from position-list */ -){ - if( (**pp)&0xFE ){ - fts3GetDeltaVarint(pp, pi); - *pi -= 2; - }else{ - *pi = POSITION_LIST_END; - } +static void rtreeReference(Rtree *pRtree){ + pRtree->nBusy++; } /* -** If parameter iCol is not 0, write an POS_COLUMN (1) byte followed by -** the value of iCol encoded as a varint to *pp. This will start a new -** column list. -** -** Set *pp to point to the byte just after the last byte written before -** returning (do not modify it if iCol==0). Return the total number of bytes -** written (0 if iCol==0). +** Decrement the r-tree reference count. When the reference count reaches +** zero the structure is deleted. */ -static int fts3PutColNumber(char **pp, int iCol){ - int n = 0; /* Number of bytes written */ - if( iCol ){ - char *p = *pp; /* Output pointer */ - n = 1 + sqlite3Fts3PutVarint(&p[1], iCol); - *p = 0x01; - *pp = &p[n]; +static void rtreeRelease(Rtree *pRtree){ + pRtree->nBusy--; + if( pRtree->nBusy==0 ){ + sqlite3_finalize(pRtree->pReadNode); + sqlite3_finalize(pRtree->pWriteNode); + sqlite3_finalize(pRtree->pDeleteNode); + sqlite3_finalize(pRtree->pReadRowid); + sqlite3_finalize(pRtree->pWriteRowid); + sqlite3_finalize(pRtree->pDeleteRowid); + sqlite3_finalize(pRtree->pReadParent); + sqlite3_finalize(pRtree->pWriteParent); + sqlite3_finalize(pRtree->pDeleteParent); + sqlite3_free(pRtree); } - return n; } -/* -** Compute the union of two position lists. The output written -** into *pp contains all positions of both *pp1 and *pp2 in sorted -** order and with any duplicates removed. All pointers are -** updated appropriately. The caller is responsible for insuring -** that there is enough space in *pp to hold the complete output. +/* +** Rtree virtual table module xDisconnect method. */ -static void fts3PoslistMerge( - char **pp, /* Output buffer */ - char **pp1, /* Left input list */ - char **pp2 /* Right input list */ -){ - char *p = *pp; - char *p1 = *pp1; - char *p2 = *pp2; +static int rtreeDisconnect(sqlite3_vtab *pVtab){ + rtreeRelease((Rtree *)pVtab); + return SQLITE_OK; +} - while( *p1 || *p2 ){ - int iCol1; /* The current column index in pp1 */ - int iCol2; /* The current column index in pp2 */ +/* +** Rtree virtual table module xDestroy method. +*/ +static int rtreeDestroy(sqlite3_vtab *pVtab){ + Rtree *pRtree = (Rtree *)pVtab; + int rc; + char *zCreate = sqlite3_mprintf( + "DROP TABLE '%q'.'%q_node';" + "DROP TABLE '%q'.'%q_rowid';" + "DROP TABLE '%q'.'%q_parent';", + pRtree->zDb, pRtree->zName, + pRtree->zDb, pRtree->zName, + pRtree->zDb, pRtree->zName + ); + if( !zCreate ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_exec(pRtree->db, zCreate, 0, 0, 0); + sqlite3_free(zCreate); + } + if( rc==SQLITE_OK ){ + rtreeRelease(pRtree); + } - if( *p1==POS_COLUMN ) fts3GetVarint32(&p1[1], &iCol1); - else if( *p1==POS_END ) iCol1 = POSITION_LIST_END; - else iCol1 = 0; + return rc; +} - if( *p2==POS_COLUMN ) fts3GetVarint32(&p2[1], &iCol2); - else if( *p2==POS_END ) iCol2 = POSITION_LIST_END; - else iCol2 = 0; +/* +** Rtree virtual table module xOpen method. +*/ +static int rtreeOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + int rc = SQLITE_NOMEM; + RtreeCursor *pCsr; - if( iCol1==iCol2 ){ - sqlite3_int64 i1 = 0; /* Last position from pp1 */ - sqlite3_int64 i2 = 0; /* Last position from pp2 */ - sqlite3_int64 iPrev = 0; - int n = fts3PutColNumber(&p, iCol1); - p1 += n; - p2 += n; + pCsr = (RtreeCursor *)sqlite3_malloc(sizeof(RtreeCursor)); + if( pCsr ){ + memset(pCsr, 0, sizeof(RtreeCursor)); + pCsr->base.pVtab = pVTab; + rc = SQLITE_OK; + } + *ppCursor = (sqlite3_vtab_cursor *)pCsr; - /* At this point, both p1 and p2 point to the start of column-lists - ** for the same column (the column with index iCol1 and iCol2). - ** A column-list is a list of non-negative delta-encoded varints, each - ** incremented by 2 before being stored. Each list is terminated by a - ** POS_END (0) or POS_COLUMN (1). The following block merges the two lists - ** and writes the results to buffer p. p is left pointing to the byte - ** after the list written. No terminator (POS_END or POS_COLUMN) is - ** written to the output. - */ - fts3GetDeltaVarint(&p1, &i1); - fts3GetDeltaVarint(&p2, &i2); - do { - fts3PutDeltaVarint(&p, &iPrev, (i1aConstraint ){ + int i; /* Used to iterate through constraint array */ + for(i=0; inConstraint; i++){ + sqlite3_rtree_query_info *pInfo = pCsr->aConstraint[i].pInfo; + if( pInfo ){ + if( pInfo->xDelUser ) pInfo->xDelUser(pInfo->pUser); + sqlite3_free(pInfo); + } } + sqlite3_free(pCsr->aConstraint); + pCsr->aConstraint = 0; } +} - *p++ = POS_END; - *pp = p; - *pp1 = p1 + 1; - *pp2 = p2 + 1; +/* +** Rtree virtual table module xClose method. +*/ +static int rtreeClose(sqlite3_vtab_cursor *cur){ + Rtree *pRtree = (Rtree *)(cur->pVtab); + int ii; + RtreeCursor *pCsr = (RtreeCursor *)cur; + freeCursorConstraints(pCsr); + sqlite3_free(pCsr->aPoint); + for(ii=0; iiaNode[ii]); + sqlite3_free(pCsr); + return SQLITE_OK; } /* -** This function is used to merge two position lists into one. When it is -** called, *pp1 and *pp2 must both point to position lists. A position-list is -** the part of a doclist that follows each document id. For example, if a row -** contains: -** -** 'a b c'|'x y z'|'a b b a' -** -** Then the position list for this row for token 'b' would consist of: -** -** 0x02 0x01 0x02 0x03 0x03 0x00 -** -** When this function returns, both *pp1 and *pp2 are left pointing to the -** byte following the 0x00 terminator of their respective position lists. +** Rtree virtual table module xEof method. ** -** If isSaveLeft is 0, an entry is added to the output position list for -** each position in *pp2 for which there exists one or more positions in -** *pp1 so that (pos(*pp2)>pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e. -** when the *pp1 token appears before the *pp2 token, but not more than nToken -** slots before it. +** Return non-zero if the cursor does not currently point to a valid +** record (i.e if the scan has finished), or zero otherwise. +*/ +static int rtreeEof(sqlite3_vtab_cursor *cur){ + RtreeCursor *pCsr = (RtreeCursor *)cur; + return pCsr->atEOF; +} + +/* +** Convert raw bits from the on-disk RTree record into a coordinate value. +** The on-disk format is big-endian and needs to be converted for little- +** endian platforms. The on-disk record stores integer coordinates if +** eInt is true and it stores 32-bit floating point records if eInt is +** false. a[] is the four bytes of the on-disk record to be decoded. +** Store the results in "r". ** -** e.g. nToken==1 searches for adjacent positions. +** There are three versions of this macro, one each for little-endian and +** big-endian processors and a third generic implementation. The endian- +** specific implementations are much faster and are preferred if the +** processor endianness is known at compile-time. The SQLITE_BYTEORDER +** macro is part of sqliteInt.h and hence the endian-specific +** implementation will only be used if this module is compiled as part +** of the amalgamation. */ -static int fts3PoslistPhraseMerge( - char **pp, /* IN/OUT: Preallocated output buffer */ - int nToken, /* Maximum difference in token positions */ - int isSaveLeft, /* Save the left position */ - int isExact, /* If *pp1 is exactly nTokens before *pp2 */ - char **pp1, /* IN/OUT: Left input list */ - char **pp2 /* IN/OUT: Right input list */ +#if defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==1234 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + memcpy(&c.u,a,4); \ + c.u = ((c.u>>24)&0xff)|((c.u>>8)&0xff00)| \ + ((c.u&0xff)<<24)|((c.u&0xff00)<<8); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#elif defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==4321 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + memcpy(&c.u,a,4); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#else +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + c.u = ((u32)a[0]<<24) + ((u32)a[1]<<16) \ + +((u32)a[2]<<8) + a[3]; \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#endif + +/* +** Check the RTree node or entry given by pCellData and p against the MATCH +** constraint pConstraint. +*/ +static int rtreeCallbackConstraint( + RtreeConstraint *pConstraint, /* The constraint to test */ + int eInt, /* True if RTree holding integer coordinates */ + u8 *pCellData, /* Raw cell content */ + RtreeSearchPoint *pSearch, /* Container of this cell */ + sqlite3_rtree_dbl *prScore, /* OUT: score for the cell */ + int *peWithin /* OUT: visibility of the cell */ ){ - char *p = *pp; - char *p1 = *pp1; - char *p2 = *pp2; - int iCol1 = 0; - int iCol2 = 0; + int i; /* Loop counter */ + sqlite3_rtree_query_info *pInfo = pConstraint->pInfo; /* Callback info */ + int nCoord = pInfo->nCoord; /* No. of coordinates */ + int rc; /* Callback return code */ + sqlite3_rtree_dbl aCoord[RTREE_MAX_DIMENSIONS*2]; /* Decoded coordinates */ - /* Never set both isSaveLeft and isExact for the same invocation. */ - assert( isSaveLeft==0 || isExact==0 ); + assert( pConstraint->op==RTREE_MATCH || pConstraint->op==RTREE_QUERY ); + assert( nCoord==2 || nCoord==4 || nCoord==6 || nCoord==8 || nCoord==10 ); - assert( p!=0 && *p1!=0 && *p2!=0 ); - if( *p1==POS_COLUMN ){ - p1++; - p1 += fts3GetVarint32(p1, &iCol1); + if( pConstraint->op==RTREE_QUERY && pSearch->iLevel==1 ){ + pInfo->iRowid = readInt64(pCellData); } - if( *p2==POS_COLUMN ){ - p2++; - p2 += fts3GetVarint32(p2, &iCol2); + pCellData += 8; + for(i=0; iiPos1 && iPos2<=iPos1+nToken) - ){ - sqlite3_int64 iSave; - iSave = isSaveLeft ? iPos1 : iPos2; - fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2; - pSave = 0; - assert( p ); - } - if( (!isSaveLeft && iPos2<=(iPos1+nToken)) || iPos2<=iPos1 ){ - if( (*p2&0xFE)==0 ) break; - fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; - }else{ - if( (*p1&0xFE)==0 ) break; - fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; - } - } - - if( pSave ){ - assert( pp && p ); - p = pSave; - } - - fts3ColumnlistCopy(0, &p1); - fts3ColumnlistCopy(0, &p2); - assert( (*p1&0xFE)==0 && (*p2&0xFE)==0 ); - if( 0==*p1 || 0==*p2 ) break; - - p1++; - p1 += fts3GetVarint32(p1, &iCol1); - p2++; - p2 += fts3GetVarint32(p2, &iCol2); - } - - /* Advance pointer p1 or p2 (whichever corresponds to the smaller of - ** iCol1 and iCol2) so that it points to either the 0x00 that marks the - ** end of the position list, or the 0x01 that precedes the next - ** column-number in the position list. - */ - else if( iCol1op==RTREE_MATCH ){ + rc = pConstraint->u.xGeom((sqlite3_rtree_geometry*)pInfo, + nCoord, aCoord, &i); + if( i==0 ) *peWithin = NOT_WITHIN; + *prScore = RTREE_ZERO; + }else{ + pInfo->aCoord = aCoord; + pInfo->iLevel = pSearch->iLevel - 1; + pInfo->rScore = pInfo->rParentScore = pSearch->rScore; + pInfo->eWithin = pInfo->eParentWithin = pSearch->eWithin; + rc = pConstraint->u.xQueryFunc(pInfo); + if( pInfo->eWithin<*peWithin ) *peWithin = pInfo->eWithin; + if( pInfo->rScore<*prScore || *prScorerScore; } } - - fts3PoslistCopy(0, &p2); - fts3PoslistCopy(0, &p1); - *pp1 = p1; - *pp2 = p2; - if( *pp==p ){ - return 0; - } - *p++ = 0x00; - *pp = p; - return 1; + return rc; } -/* -** Merge two position-lists as required by the NEAR operator. The argument -** position lists correspond to the left and right phrases of an expression -** like: -** -** "phrase 1" NEAR "phrase number 2" -** -** Position list *pp1 corresponds to the left-hand side of the NEAR -** expression and *pp2 to the right. As usual, the indexes in the position -** lists are the offsets of the last token in each phrase (tokens "1" and "2" -** in the example above). -** -** The output position list - written to *pp - is a copy of *pp2 with those -** entries that are not sufficiently NEAR entries in *pp1 removed. +/* +** Check the internal RTree node given by pCellData against constraint p. +** If this constraint cannot be satisfied by any child within the node, +** set *peWithin to NOT_WITHIN. */ -static int fts3PoslistNearMerge( - char **pp, /* Output buffer */ - char *aTmp, /* Temporary buffer space */ - int nRight, /* Maximum difference in token positions */ - int nLeft, /* Maximum difference in token positions */ - char **pp1, /* IN/OUT: Left input list */ - char **pp2 /* IN/OUT: Right input list */ +static void rtreeNonleafConstraint( + RtreeConstraint *p, /* The constraint to test */ + int eInt, /* True if RTree holds integer coordinates */ + u8 *pCellData, /* Raw cell content as appears on disk */ + int *peWithin /* Adjust downward, as appropriate */ ){ - char *p1 = *pp1; - char *p2 = *pp2; + sqlite3_rtree_dbl val; /* Coordinate value convert to a double */ - char *pTmp1 = aTmp; - char *pTmp2; - char *aTmp2; - int res = 1; + /* p->iCoord might point to either a lower or upper bound coordinate + ** in a coordinate pair. But make pCellData point to the lower bound. + */ + pCellData += 8 + 4*(p->iCoord&0xfe); + + assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE + || p->op==RTREE_GT || p->op==RTREE_EQ ); + switch( p->op ){ + case RTREE_LE: + case RTREE_LT: + case RTREE_EQ: + RTREE_DECODE_COORD(eInt, pCellData, val); + /* val now holds the lower bound of the coordinate pair */ + if( p->u.rValue>=val ) return; + if( p->op!=RTREE_EQ ) break; /* RTREE_LE and RTREE_LT end here */ + /* Fall through for the RTREE_EQ case */ - fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2); - aTmp2 = pTmp2 = pTmp1; - *pp1 = p1; - *pp2 = p2; - fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1); - if( pTmp1!=aTmp && pTmp2!=aTmp2 ){ - fts3PoslistMerge(pp, &aTmp, &aTmp2); - }else if( pTmp1!=aTmp ){ - fts3PoslistCopy(pp, &aTmp); - }else if( pTmp2!=aTmp2 ){ - fts3PoslistCopy(pp, &aTmp2); - }else{ - res = 0; + default: /* RTREE_GT or RTREE_GE, or fallthrough of RTREE_EQ */ + pCellData += 4; + RTREE_DECODE_COORD(eInt, pCellData, val); + /* val now holds the upper bound of the coordinate pair */ + if( p->u.rValue<=val ) return; } - - return res; + *peWithin = NOT_WITHIN; } -/* -** An instance of this function is used to merge together the (potentially -** large number of) doclists for each term that matches a prefix query. -** See function fts3TermSelectMerge() for details. -*/ -typedef struct TermSelect TermSelect; -struct TermSelect { - char *aaOutput[16]; /* Malloc'd output buffers */ - int anOutput[16]; /* Size each output buffer in bytes */ -}; - /* -** This function is used to read a single varint from a buffer. Parameter -** pEnd points 1 byte past the end of the buffer. When this function is -** called, if *pp points to pEnd or greater, then the end of the buffer -** has been reached. In this case *pp is set to 0 and the function returns. +** Check the leaf RTree cell given by pCellData against constraint p. +** If this constraint is not satisfied, set *peWithin to NOT_WITHIN. +** If the constraint is satisfied, leave *peWithin unchanged. ** -** If *pp does not point to or past pEnd, then a single varint is read -** from *pp. *pp is then set to point 1 byte past the end of the read varint. +** The constraint is of the form: xN op $val ** -** If bDescIdx is false, the value read is added to *pVal before returning. -** If it is true, the value read is subtracted from *pVal before this -** function returns. +** The op is given by p->op. The xN is p->iCoord-th coordinate in +** pCellData. $val is given by p->u.rValue. */ -static void fts3GetDeltaVarint3( - char **pp, /* IN/OUT: Point to read varint from */ - char *pEnd, /* End of buffer */ - int bDescIdx, /* True if docids are descending */ - sqlite3_int64 *pVal /* IN/OUT: Integer value */ +static void rtreeLeafConstraint( + RtreeConstraint *p, /* The constraint to test */ + int eInt, /* True if RTree holds integer coordinates */ + u8 *pCellData, /* Raw cell content as appears on disk */ + int *peWithin /* Adjust downward, as appropriate */ ){ - if( *pp>=pEnd ){ - *pp = 0; - }else{ - sqlite3_int64 iVal; - *pp += sqlite3Fts3GetVarint(*pp, &iVal); - if( bDescIdx ){ - *pVal -= iVal; - }else{ - *pVal += iVal; - } + RtreeDValue xN; /* Coordinate value converted to a double */ + + assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE + || p->op==RTREE_GT || p->op==RTREE_EQ ); + pCellData += 8 + p->iCoord*4; + RTREE_DECODE_COORD(eInt, pCellData, xN); + switch( p->op ){ + case RTREE_LE: if( xN <= p->u.rValue ) return; break; + case RTREE_LT: if( xN < p->u.rValue ) return; break; + case RTREE_GE: if( xN >= p->u.rValue ) return; break; + case RTREE_GT: if( xN > p->u.rValue ) return; break; + default: if( xN == p->u.rValue ) return; break; } + *peWithin = NOT_WITHIN; } /* -** This function is used to write a single varint to a buffer. The varint -** is written to *pp. Before returning, *pp is set to point 1 byte past the -** end of the value written. -** -** If *pbFirst is zero when this function is called, the value written to -** the buffer is that of parameter iVal. -** -** If *pbFirst is non-zero when this function is called, then the value -** written is either (iVal-*piPrev) (if bDescIdx is zero) or (*piPrev-iVal) -** (if bDescIdx is non-zero). -** -** Before returning, this function always sets *pbFirst to 1 and *piPrev -** to the value of parameter iVal. +** One of the cells in node pNode is guaranteed to have a 64-bit +** integer value equal to iRowid. Return the index of this cell. */ -static void fts3PutDeltaVarint3( - char **pp, /* IN/OUT: Output pointer */ - int bDescIdx, /* True for descending docids */ - sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ - int *pbFirst, /* IN/OUT: True after first int written */ - sqlite3_int64 iVal /* Write this value to the list */ +static int nodeRowidIndex( + Rtree *pRtree, + RtreeNode *pNode, + i64 iRowid, + int *piIndex ){ - sqlite3_int64 iWrite; - if( bDescIdx==0 || *pbFirst==0 ){ - iWrite = iVal - *piPrev; - }else{ - iWrite = *piPrev - iVal; + int ii; + int nCell = NCELL(pNode); + assert( nCell<200 ); + for(ii=0; ii0 ); - *pp += sqlite3Fts3PutVarint(*pp, iWrite); - *piPrev = iVal; - *pbFirst = 1; + return SQLITE_CORRUPT_VTAB; } - /* -** This macro is used by various functions that merge doclists. The two -** arguments are 64-bit docid values. If the value of the stack variable -** bDescDoclist is 0 when this macro is invoked, then it returns (i1-i2). -** Otherwise, (i2-i1). -** -** Using this makes it easier to write code that can merge doclists that are -** sorted in either ascending or descending order. +** Return the index of the cell containing a pointer to node pNode +** in its parent. If pNode is the root node, return -1. */ -#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1-i2)) +static int nodeParentIndex(Rtree *pRtree, RtreeNode *pNode, int *piIndex){ + RtreeNode *pParent = pNode->pParent; + if( pParent ){ + return nodeRowidIndex(pRtree, pParent, pNode->iNode, piIndex); + } + *piIndex = -1; + return SQLITE_OK; +} /* -** This function does an "OR" merge of two doclists (output contains all -** positions contained in either argument doclist). If the docids in the -** input doclists are sorted in ascending order, parameter bDescDoclist -** should be false. If they are sorted in ascending order, it should be -** passed a non-zero value. -** -** If no error occurs, *paOut is set to point at an sqlite3_malloc'd buffer -** containing the output doclist and SQLITE_OK is returned. In this case -** *pnOut is set to the number of bytes in the output doclist. +** Compare two search points. Return negative, zero, or positive if the first +** is less than, equal to, or greater than the second. ** -** If an error occurs, an SQLite error code is returned. The output values -** are undefined in this case. +** The rScore is the primary key. Smaller rScore values come first. +** If the rScore is a tie, then use iLevel as the tie breaker with smaller +** iLevel values coming first. In this way, if rScore is the same for all +** SearchPoints, then iLevel becomes the deciding factor and the result +** is a depth-first search, which is the desired default behavior. */ -static int fts3DoclistOrMerge( - int bDescDoclist, /* True if arguments are desc */ - char *a1, int n1, /* First doclist */ - char *a2, int n2, /* Second doclist */ - char **paOut, int *pnOut /* OUT: Malloc'd doclist */ +static int rtreeSearchPointCompare( + const RtreeSearchPoint *pA, + const RtreeSearchPoint *pB ){ - sqlite3_int64 i1 = 0; - sqlite3_int64 i2 = 0; - sqlite3_int64 iPrev = 0; - char *pEnd1 = &a1[n1]; - char *pEnd2 = &a2[n2]; - char *p1 = a1; - char *p2 = a2; - char *p; - char *aOut; - int bFirstOut = 0; - - *paOut = 0; - *pnOut = 0; - - /* Allocate space for the output. Both the input and output doclists - ** are delta encoded. If they are in ascending order (bDescDoclist==0), - ** then the first docid in each list is simply encoded as a varint. For - ** each subsequent docid, the varint stored is the difference between the - ** current and previous docid (a positive number - since the list is in - ** ascending order). - ** - ** The first docid written to the output is therefore encoded using the - ** same number of bytes as it is in whichever of the input lists it is - ** read from. And each subsequent docid read from the same input list - ** consumes either the same or less bytes as it did in the input (since - ** the difference between it and the previous value in the output must - ** be a positive value less than or equal to the delta value read from - ** the input list). The same argument applies to all but the first docid - ** read from the 'other' list. And to the contents of all position lists - ** that will be copied and merged from the input to the output. - ** - ** However, if the first docid copied to the output is a negative number, - ** then the encoding of the first docid from the 'other' input list may - ** be larger in the output than it was in the input (since the delta value - ** may be a larger positive integer than the actual docid). - ** - ** The space required to store the output is therefore the sum of the - ** sizes of the two inputs, plus enough space for exactly one of the input - ** docids to grow. - ** - ** A symetric argument may be made if the doclists are in descending - ** order. - */ - aOut = sqlite3_malloc(n1+n2+FTS3_VARINT_MAX-1); - if( !aOut ) return SQLITE_NOMEM; - - p = aOut; - fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); - while( p1 || p2 ){ - sqlite3_int64 iDiff = DOCID_CMP(i1, i2); + if( pA->rScorerScore ) return -1; + if( pA->rScore>pB->rScore ) return +1; + if( pA->iLeveliLevel ) return -1; + if( pA->iLevel>pB->iLevel ) return +1; + return 0; +} - if( p2 && p1 && iDiff==0 ){ - fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); - fts3PoslistMerge(&p, &p1, &p2); - fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); - }else if( !p2 || (p1 && iDiff<0) ){ - fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); - fts3PoslistCopy(&p, &p1); - fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); +/* +** Interchange to search points in a cursor. +*/ +static void rtreeSearchPointSwap(RtreeCursor *p, int i, int j){ + RtreeSearchPoint t = p->aPoint[i]; + assert( iaPoint[i] = p->aPoint[j]; + p->aPoint[j] = t; + i++; j++; + if( i=RTREE_CACHE_SZ ){ + nodeRelease(RTREE_OF_CURSOR(p), p->aNode[i]); + p->aNode[i] = 0; }else{ - fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i2); - fts3PoslistCopy(&p, &p2); - fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + RtreeNode *pTemp = p->aNode[i]; + p->aNode[i] = p->aNode[j]; + p->aNode[j] = pTemp; } } - - *paOut = aOut; - *pnOut = (int)(p-aOut); - assert( *pnOut<=n1+n2+FTS3_VARINT_MAX-1 ); - return SQLITE_OK; } /* -** This function does a "phrase" merge of two doclists. In a phrase merge, -** the output contains a copy of each position from the right-hand input -** doclist for which there is a position in the left-hand input doclist -** exactly nDist tokens before it. -** -** If the docids in the input doclists are sorted in ascending order, -** parameter bDescDoclist should be false. If they are sorted in ascending -** order, it should be passed a non-zero value. -** -** The right-hand input doclist is overwritten by this function. +** Return the search point with the lowest current score. */ -static int fts3DoclistPhraseMerge( - int bDescDoclist, /* True if arguments are desc */ - int nDist, /* Distance from left to right (1=adjacent) */ - char *aLeft, int nLeft, /* Left doclist */ - char **paRight, int *pnRight /* IN/OUT: Right/output doclist */ -){ - sqlite3_int64 i1 = 0; - sqlite3_int64 i2 = 0; - sqlite3_int64 iPrev = 0; - char *aRight = *paRight; - char *pEnd1 = &aLeft[nLeft]; - char *pEnd2 = &aRight[*pnRight]; - char *p1 = aLeft; - char *p2 = aRight; - char *p; - int bFirstOut = 0; - char *aOut; +static RtreeSearchPoint *rtreeSearchPointFirst(RtreeCursor *pCur){ + return pCur->bPoint ? &pCur->sPoint : pCur->nPoint ? pCur->aPoint : 0; +} - assert( nDist>0 ); - if( bDescDoclist ){ - aOut = sqlite3_malloc(*pnRight + FTS3_VARINT_MAX); - if( aOut==0 ) return SQLITE_NOMEM; - }else{ - aOut = aRight; +/* +** Get the RtreeNode for the search point with the lowest score. +*/ +static RtreeNode *rtreeNodeOfFirstSearchPoint(RtreeCursor *pCur, int *pRC){ + sqlite3_int64 id; + int ii = 1 - pCur->bPoint; + assert( ii==0 || ii==1 ); + assert( pCur->bPoint || pCur->nPoint ); + if( pCur->aNode[ii]==0 ){ + assert( pRC!=0 ); + id = ii ? pCur->aPoint[0].id : pCur->sPoint.id; + *pRC = nodeAcquire(RTREE_OF_CURSOR(pCur), id, 0, &pCur->aNode[ii]); } - p = aOut; - - fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); - - while( p1 && p2 ){ - sqlite3_int64 iDiff = DOCID_CMP(i1, i2); - if( iDiff==0 ){ - char *pSave = p; - sqlite3_int64 iPrevSave = iPrev; - int bFirstOutSave = bFirstOut; + return pCur->aNode[ii]; +} - fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); - if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){ - p = pSave; - iPrev = iPrevSave; - bFirstOut = bFirstOutSave; - } - fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); - }else if( iDiff<0 ){ - fts3PoslistCopy(0, &p1); - fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); - }else{ - fts3PoslistCopy(0, &p2); - fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); - } +/* +** Push a new element onto the priority queue +*/ +static RtreeSearchPoint *rtreeEnqueue( + RtreeCursor *pCur, /* The cursor */ + RtreeDValue rScore, /* Score for the new search point */ + u8 iLevel /* Level for the new search point */ +){ + int i, j; + RtreeSearchPoint *pNew; + if( pCur->nPoint>=pCur->nPointAlloc ){ + int nNew = pCur->nPointAlloc*2 + 8; + pNew = sqlite3_realloc(pCur->aPoint, nNew*sizeof(pCur->aPoint[0])); + if( pNew==0 ) return 0; + pCur->aPoint = pNew; + pCur->nPointAlloc = nNew; } - - *pnRight = (int)(p - aOut); - if( bDescDoclist ){ - sqlite3_free(aRight); - *paRight = aOut; + i = pCur->nPoint++; + pNew = pCur->aPoint + i; + pNew->rScore = rScore; + pNew->iLevel = iLevel; + assert( iLevel<=RTREE_MAX_DEPTH ); + while( i>0 ){ + RtreeSearchPoint *pParent; + j = (i-1)/2; + pParent = pCur->aPoint + j; + if( rtreeSearchPointCompare(pNew, pParent)>=0 ) break; + rtreeSearchPointSwap(pCur, j, i); + i = j; + pNew = pParent; } - - return SQLITE_OK; + return pNew; } /* -** Argument pList points to a position list nList bytes in size. This -** function checks to see if the position list contains any entries for -** a token in position 0 (of any column). If so, it writes argument iDelta -** to the output buffer pOut, followed by a position list consisting only -** of the entries from pList at position 0, and terminated by an 0x00 byte. -** The value returned is the number of bytes written to pOut (if any). +** Allocate a new RtreeSearchPoint and return a pointer to it. Return +** NULL if malloc fails. */ -SQLITE_PRIVATE int sqlite3Fts3FirstFilter( - sqlite3_int64 iDelta, /* Varint that may be written to pOut */ - char *pList, /* Position list (no 0x00 term) */ - int nList, /* Size of pList in bytes */ - char *pOut /* Write output here */ +static RtreeSearchPoint *rtreeSearchPointNew( + RtreeCursor *pCur, /* The cursor */ + RtreeDValue rScore, /* Score for the new search point */ + u8 iLevel /* Level for the new search point */ ){ - int nOut = 0; - int bWritten = 0; /* True once iDelta has been written */ - char *p = pList; - char *pEnd = &pList[nList]; - - if( *p!=0x01 ){ - if( *p==0x02 ){ - nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta); - pOut[nOut++] = 0x02; - bWritten = 1; + RtreeSearchPoint *pNew, *pFirst; + pFirst = rtreeSearchPointFirst(pCur); + pCur->anQueue[iLevel]++; + if( pFirst==0 + || pFirst->rScore>rScore + || (pFirst->rScore==rScore && pFirst->iLevel>iLevel) + ){ + if( pCur->bPoint ){ + int ii; + pNew = rtreeEnqueue(pCur, rScore, iLevel); + if( pNew==0 ) return 0; + ii = (int)(pNew - pCur->aPoint) + 1; + if( iiaNode[ii]==0 ); + pCur->aNode[ii] = pCur->aNode[0]; + }else{ + nodeRelease(RTREE_OF_CURSOR(pCur), pCur->aNode[0]); + } + pCur->aNode[0] = 0; + *pNew = pCur->sPoint; } - fts3ColumnlistCopy(0, &p); + pCur->sPoint.rScore = rScore; + pCur->sPoint.iLevel = iLevel; + pCur->bPoint = 1; + return &pCur->sPoint; + }else{ + return rtreeEnqueue(pCur, rScore, iLevel); } +} - while( piLevel, p->id, p->iCell, p->rScore, p->eWithin + ); + idx++; + if( idxaNode[idx]); + }else{ + printf("\n"); } - if( bWritten ){ - pOut[nOut++] = 0x00; +} +static void traceQueue(RtreeCursor *pCur, const char *zPrefix){ + int ii; + printf("=== %9s ", zPrefix); + if( pCur->bPoint ){ + tracePoint(&pCur->sPoint, -1, pCur); + } + for(ii=0; iinPoint; ii++){ + if( ii>0 || pCur->bPoint ) printf(" "); + tracePoint(&pCur->aPoint[ii], ii, pCur); } - - return nOut; } +# define RTREE_QUEUE_TRACE(A,B) traceQueue(A,B) +#else +# define RTREE_QUEUE_TRACE(A,B) /* no-op */ +#endif - -/* -** Merge all doclists in the TermSelect.aaOutput[] array into a single -** doclist stored in TermSelect.aaOutput[0]. If successful, delete all -** other doclists (except the aaOutput[0] one) and return SQLITE_OK. -** -** If an OOM error occurs, return SQLITE_NOMEM. In this case it is -** the responsibility of the caller to free any doclists left in the -** TermSelect.aaOutput[] array. +/* Remove the search point with the lowest current score. */ -static int fts3TermSelectFinishMerge(Fts3Table *p, TermSelect *pTS){ - char *aOut = 0; - int nOut = 0; - int i; - - /* Loop through the doclists in the aaOutput[] array. Merge them all - ** into a single doclist. - */ - for(i=0; iaaOutput); i++){ - if( pTS->aaOutput[i] ){ - if( !aOut ){ - aOut = pTS->aaOutput[i]; - nOut = pTS->anOutput[i]; - pTS->aaOutput[i] = 0; +static void rtreeSearchPointPop(RtreeCursor *p){ + int i, j, k, n; + i = 1 - p->bPoint; + assert( i==0 || i==1 ); + if( p->aNode[i] ){ + nodeRelease(RTREE_OF_CURSOR(p), p->aNode[i]); + p->aNode[i] = 0; + } + if( p->bPoint ){ + p->anQueue[p->sPoint.iLevel]--; + p->bPoint = 0; + }else if( p->nPoint ){ + p->anQueue[p->aPoint[0].iLevel]--; + n = --p->nPoint; + p->aPoint[0] = p->aPoint[n]; + if( naNode[1] = p->aNode[n+1]; + p->aNode[n+1] = 0; + } + i = 0; + while( (j = i*2+1)aPoint[k], &p->aPoint[j])<0 ){ + if( rtreeSearchPointCompare(&p->aPoint[k], &p->aPoint[i])<0 ){ + rtreeSearchPointSwap(p, i, k); + i = k; + }else{ + break; + } }else{ - int nNew; - char *aNew; - - int rc = fts3DoclistOrMerge(p->bDescIdx, - pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew - ); - if( rc!=SQLITE_OK ){ - sqlite3_free(aOut); - return rc; + if( rtreeSearchPointCompare(&p->aPoint[j], &p->aPoint[i])<0 ){ + rtreeSearchPointSwap(p, i, j); + i = j; + }else{ + break; } - - sqlite3_free(pTS->aaOutput[i]); - sqlite3_free(aOut); - pTS->aaOutput[i] = 0; - aOut = aNew; - nOut = nNew; } } } - - pTS->aaOutput[0] = aOut; - pTS->anOutput[0] = nOut; - return SQLITE_OK; } + /* -** Merge the doclist aDoclist/nDoclist into the TermSelect object passed -** as the first argument. The merge is an "OR" merge (see function -** fts3DoclistOrMerge() for details). -** -** This function is called with the doclist for each term that matches -** a queried prefix. It merges all these doclists into one, the doclist -** for the specified prefix. Since there can be a very large number of -** doclists to merge, the merging is done pair-wise using the TermSelect -** object. -** -** This function returns SQLITE_OK if the merge is successful, or an -** SQLite error code (SQLITE_NOMEM) if an error occurs. +** Continue the search on cursor pCur until the front of the queue +** contains an entry suitable for returning as a result-set row, +** or until the RtreeSearchPoint queue is empty, indicating that the +** query has completed. */ -static int fts3TermSelectMerge( - Fts3Table *p, /* FTS table handle */ - TermSelect *pTS, /* TermSelect object to merge into */ - char *aDoclist, /* Pointer to doclist */ - int nDoclist /* Size of aDoclist in bytes */ -){ - if( pTS->aaOutput[0]==0 ){ - /* If this is the first term selected, copy the doclist to the output - ** buffer using memcpy(). - ** - ** Add FTS3_VARINT_MAX bytes of unused space to the end of the - ** allocation. This is so as to ensure that the buffer is big enough - ** to hold the current doclist AND'd with any other doclist. If the - ** doclists are stored in order=ASC order, this padding would not be - ** required (since the size of [doclistA AND doclistB] is always less - ** than or equal to the size of [doclistA] in that case). But this is - ** not true for order=DESC. For example, a doclist containing (1, -1) - ** may be smaller than (-1), as in the first example the -1 may be stored - ** as a single-byte delta, whereas in the second it must be stored as a - ** FTS3_VARINT_MAX byte varint. - ** - ** Similar padding is added in the fts3DoclistOrMerge() function. - */ - pTS->aaOutput[0] = sqlite3_malloc(nDoclist + FTS3_VARINT_MAX + 1); - pTS->anOutput[0] = nDoclist; - if( pTS->aaOutput[0] ){ - memcpy(pTS->aaOutput[0], aDoclist, nDoclist); - }else{ - return SQLITE_NOMEM; - } - }else{ - char *aMerge = aDoclist; - int nMerge = nDoclist; - int iOut; - - for(iOut=0; iOutaaOutput); iOut++){ - if( pTS->aaOutput[iOut]==0 ){ - assert( iOut>0 ); - pTS->aaOutput[iOut] = aMerge; - pTS->anOutput[iOut] = nMerge; - break; - }else{ - char *aNew; - int nNew; - - int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, - pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew - ); - if( rc!=SQLITE_OK ){ - if( aMerge!=aDoclist ) sqlite3_free(aMerge); - return rc; - } +static int rtreeStepToLeaf(RtreeCursor *pCur){ + RtreeSearchPoint *p; + Rtree *pRtree = RTREE_OF_CURSOR(pCur); + RtreeNode *pNode; + int eWithin; + int rc = SQLITE_OK; + int nCell; + int nConstraint = pCur->nConstraint; + int ii; + int eInt; + RtreeSearchPoint x; - if( aMerge!=aDoclist ) sqlite3_free(aMerge); - sqlite3_free(pTS->aaOutput[iOut]); - pTS->aaOutput[iOut] = 0; - - aMerge = aNew; - nMerge = nNew; - if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ - pTS->aaOutput[iOut] = aMerge; - pTS->anOutput[iOut] = nMerge; + eInt = pRtree->eCoordType==RTREE_COORD_INT32; + while( (p = rtreeSearchPointFirst(pCur))!=0 && p->iLevel>0 ){ + pNode = rtreeNodeOfFirstSearchPoint(pCur, &rc); + if( rc ) return rc; + nCell = NCELL(pNode); + assert( nCell<200 ); + while( p->iCellzData + (4+pRtree->nBytesPerCell*p->iCell); + eWithin = FULLY_WITHIN; + for(ii=0; iiaConstraint + ii; + if( pConstraint->op>=RTREE_MATCH ){ + rc = rtreeCallbackConstraint(pConstraint, eInt, pCellData, p, + &rScore, &eWithin); + if( rc ) return rc; + }else if( p->iLevel==1 ){ + rtreeLeafConstraint(pConstraint, eInt, pCellData, &eWithin); + }else{ + rtreeNonleafConstraint(pConstraint, eInt, pCellData, &eWithin); } + if( eWithin==NOT_WITHIN ) break; + } + p->iCell++; + if( eWithin==NOT_WITHIN ) continue; + x.iLevel = p->iLevel - 1; + if( x.iLevel ){ + x.id = readInt64(pCellData); + x.iCell = 0; + }else{ + x.id = p->id; + x.iCell = p->iCell - 1; + } + if( p->iCell>=nCell ){ + RTREE_QUEUE_TRACE(pCur, "POP-S:"); + rtreeSearchPointPop(pCur); } + if( rScoreeWithin = eWithin; + p->id = x.id; + p->iCell = x.iCell; + RTREE_QUEUE_TRACE(pCur, "PUSH-S:"); + break; + } + if( p->iCell>=nCell ){ + RTREE_QUEUE_TRACE(pCur, "POP-Se:"); + rtreeSearchPointPop(pCur); } } + pCur->atEOF = p==0; return SQLITE_OK; } -/* -** Append SegReader object pNew to the end of the pCsr->apSegment[] array. +/* +** Rtree virtual table module xNext method. */ -static int fts3SegReaderCursorAppend( - Fts3MultiSegReader *pCsr, - Fts3SegReader *pNew -){ - if( (pCsr->nSegment%16)==0 ){ - Fts3SegReader **apNew; - int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); - apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); - if( !apNew ){ - sqlite3Fts3SegReaderFree(pNew); - return SQLITE_NOMEM; - } - pCsr->apSegment = apNew; - } - pCsr->apSegment[pCsr->nSegment++] = pNew; - return SQLITE_OK; +static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){ + RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; + int rc = SQLITE_OK; + + /* Move to the next entry that matches the configured constraints. */ + RTREE_QUEUE_TRACE(pCsr, "POP-Nx:"); + rtreeSearchPointPop(pCsr); + rc = rtreeStepToLeaf(pCsr); + return rc; } -/* -** Add seg-reader objects to the Fts3MultiSegReader object passed as the -** 8th argument. -** -** This function returns SQLITE_OK if successful, or an SQLite error code -** otherwise. +/* +** Rtree virtual table module xRowid method. */ -static int fts3SegReaderCursor( - Fts3Table *p, /* FTS3 table handle */ - int iLangid, /* Language id */ - int iIndex, /* Index to search (from 0 to p->nIndex-1) */ - int iLevel, /* Level of segments to scan */ - const char *zTerm, /* Term to query for */ - int nTerm, /* Size of zTerm in bytes */ - int isPrefix, /* True for a prefix search */ - int isScan, /* True to scan from zTerm to EOF */ - Fts3MultiSegReader *pCsr /* Cursor object to populate */ -){ - int rc = SQLITE_OK; /* Error code */ - sqlite3_stmt *pStmt = 0; /* Statement to iterate through segments */ - int rc2; /* Result of sqlite3_reset() */ - - /* If iLevel is less than 0 and this is not a scan, include a seg-reader - ** for the pending-terms. If this is a scan, then this call must be being - ** made by an fts4aux module, not an FTS table. In this case calling - ** Fts3SegReaderPending might segfault, as the data structures used by - ** fts4aux are not completely populated. So it's easiest to filter these - ** calls out here. */ - if( iLevel<0 && p->aIndex ){ - Fts3SegReader *pSeg = 0; - rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix||isScan, &pSeg); - if( rc==SQLITE_OK && pSeg ){ - rc = fts3SegReaderCursorAppend(pCsr, pSeg); - } +static int rtreeRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *pRowid){ + RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; + RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); + int rc = SQLITE_OK; + RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); + if( rc==SQLITE_OK && p ){ + *pRowid = nodeGetRowid(RTREE_OF_CURSOR(pCsr), pNode, p->iCell); } + return rc; +} - if( iLevel!=FTS3_SEGCURSOR_PENDING ){ - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3AllSegdirs(p, iLangid, iIndex, iLevel, &pStmt); - } - - while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ - Fts3SegReader *pSeg = 0; - - /* Read the values returned by the SELECT into local variables. */ - sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1); - sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2); - sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3); - int nRoot = sqlite3_column_bytes(pStmt, 4); - char const *zRoot = sqlite3_column_blob(pStmt, 4); +/* +** Rtree virtual table module xColumn method. +*/ +static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + Rtree *pRtree = (Rtree *)cur->pVtab; + RtreeCursor *pCsr = (RtreeCursor *)cur; + RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); + RtreeCoord c; + int rc = SQLITE_OK; + RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); - /* If zTerm is not NULL, and this segment is not stored entirely on its - ** root node, the range of leaves scanned can be reduced. Do this. */ - if( iStartBlock && zTerm ){ - sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0); - rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi); - if( rc!=SQLITE_OK ) goto finished; - if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock; - } - - rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1, - (isPrefix==0 && isScan==0), - iStartBlock, iLeavesEndBlock, - iEndBlock, zRoot, nRoot, &pSeg - ); - if( rc!=SQLITE_OK ) goto finished; - rc = fts3SegReaderCursorAppend(pCsr, pSeg); + if( rc ) return rc; + if( p==0 ) return SQLITE_OK; + if( i==0 ){ + sqlite3_result_int64(ctx, nodeGetRowid(pRtree, pNode, p->iCell)); + }else{ + if( rc ) return rc; + nodeGetCoord(pRtree, pNode, p->iCell, i-1, &c); +#ifndef SQLITE_RTREE_INT_ONLY + if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ + sqlite3_result_double(ctx, c.f); + }else +#endif + { + assert( pRtree->eCoordType==RTREE_COORD_INT32 ); + sqlite3_result_int(ctx, c.i); } } - - finished: - rc2 = sqlite3_reset(pStmt); - if( rc==SQLITE_DONE ) rc = rc2; - - return rc; + return SQLITE_OK; } -/* -** Set up a cursor object for iterating through a full-text index or a -** single level therein. +/* +** Use nodeAcquire() to obtain the leaf node containing the record with +** rowid iRowid. If successful, set *ppLeaf to point to the node and +** return SQLITE_OK. If there is no such record in the table, set +** *ppLeaf to 0 and return SQLITE_OK. If an error occurs, set *ppLeaf +** to zero and return an SQLite error code. */ -SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( - Fts3Table *p, /* FTS3 table handle */ - int iLangid, /* Language-id to search */ - int iIndex, /* Index to search (from 0 to p->nIndex-1) */ - int iLevel, /* Level of segments to scan */ - const char *zTerm, /* Term to query for */ - int nTerm, /* Size of zTerm in bytes */ - int isPrefix, /* True for a prefix search */ - int isScan, /* True to scan from zTerm to EOF */ - Fts3MultiSegReader *pCsr /* Cursor object to populate */ +static int findLeafNode( + Rtree *pRtree, /* RTree to search */ + i64 iRowid, /* The rowid searching for */ + RtreeNode **ppLeaf, /* Write the node here */ + sqlite3_int64 *piNode /* Write the node-id here */ ){ - assert( iIndex>=0 && iIndexnIndex ); - assert( iLevel==FTS3_SEGCURSOR_ALL - || iLevel==FTS3_SEGCURSOR_PENDING - || iLevel>=0 - ); - assert( iLevelpReadRowid, 1, iRowid); + if( sqlite3_step(pRtree->pReadRowid)==SQLITE_ROW ){ + i64 iNode = sqlite3_column_int64(pRtree->pReadRowid, 0); + if( piNode ) *piNode = iNode; + rc = nodeAcquire(pRtree, iNode, 0, ppLeaf); + sqlite3_reset(pRtree->pReadRowid); + }else{ + rc = sqlite3_reset(pRtree->pReadRowid); + } + return rc; } /* -** In addition to its current configuration, have the Fts3MultiSegReader -** passed as the 4th argument also scan the doclist for term zTerm/nTerm. -** -** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +** This function is called to configure the RtreeConstraint object passed +** as the second argument for a MATCH constraint. The value passed as the +** first argument to this function is the right-hand operand to the MATCH +** operator. */ -static int fts3SegReaderCursorAddZero( - Fts3Table *p, /* FTS virtual table handle */ - int iLangid, - const char *zTerm, /* Term to scan doclist of */ - int nTerm, /* Number of bytes in zTerm */ - Fts3MultiSegReader *pCsr /* Fts3MultiSegReader to modify */ -){ - return fts3SegReaderCursor(p, - iLangid, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr - ); +static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ + RtreeMatchArg *pBlob; /* BLOB returned by geometry function */ + sqlite3_rtree_query_info *pInfo; /* Callback information */ + int nBlob; /* Size of the geometry function blob */ + int nExpected; /* Expected size of the BLOB */ + + /* Check that value is actually a blob. */ + if( sqlite3_value_type(pValue)!=SQLITE_BLOB ) return SQLITE_ERROR; + + /* Check that the blob is roughly the right size. */ + nBlob = sqlite3_value_bytes(pValue); + if( nBlob<(int)sizeof(RtreeMatchArg) ){ + return SQLITE_ERROR; + } + + pInfo = (sqlite3_rtree_query_info*)sqlite3_malloc( sizeof(*pInfo)+nBlob ); + if( !pInfo ) return SQLITE_NOMEM; + memset(pInfo, 0, sizeof(*pInfo)); + pBlob = (RtreeMatchArg*)&pInfo[1]; + + memcpy(pBlob, sqlite3_value_blob(pValue), nBlob); + nExpected = (int)(sizeof(RtreeMatchArg) + + pBlob->nParam*sizeof(sqlite3_value*) + + (pBlob->nParam-1)*sizeof(RtreeDValue)); + if( pBlob->magic!=RTREE_GEOMETRY_MAGIC || nBlob!=nExpected ){ + sqlite3_free(pInfo); + return SQLITE_ERROR; + } + pInfo->pContext = pBlob->cb.pContext; + pInfo->nParam = pBlob->nParam; + pInfo->aParam = pBlob->aParam; + pInfo->apSqlParam = pBlob->apSqlParam; + + if( pBlob->cb.xGeom ){ + pCons->u.xGeom = pBlob->cb.xGeom; + }else{ + pCons->op = RTREE_QUERY; + pCons->u.xQueryFunc = pBlob->cb.xQueryFunc; + } + pCons->pInfo = pInfo; + return SQLITE_OK; } -/* -** Open an Fts3MultiSegReader to scan the doclist for term zTerm/nTerm. Or, -** if isPrefix is true, to scan the doclist for all terms for which -** zTerm/nTerm is a prefix. If successful, return SQLITE_OK and write -** a pointer to the new Fts3MultiSegReader to *ppSegcsr. Otherwise, return -** an SQLite error code. -** -** It is the responsibility of the caller to free this object by eventually -** passing it to fts3SegReaderCursorFree() -** -** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. -** Output parameter *ppSegcsr is set to 0 if an error occurs. +/* +** Rtree virtual table module xFilter method. */ -static int fts3TermSegReaderCursor( - Fts3Cursor *pCsr, /* Virtual table cursor handle */ - const char *zTerm, /* Term to query for */ - int nTerm, /* Size of zTerm in bytes */ - int isPrefix, /* True for a prefix search */ - Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ +static int rtreeFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv ){ - Fts3MultiSegReader *pSegcsr; /* Object to allocate and return */ - int rc = SQLITE_NOMEM; /* Return code */ + Rtree *pRtree = (Rtree *)pVtabCursor->pVtab; + RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; + RtreeNode *pRoot = 0; + int ii; + int rc = SQLITE_OK; + int iCell = 0; - pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader)); - if( pSegcsr ){ - int i; - int bFound = 0; /* True once an index has been found */ - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + rtreeReference(pRtree); - if( isPrefix ){ - for(i=1; bFound==0 && inIndex; i++){ - if( p->aIndex[i].nPrefix==nTerm ){ - bFound = 1; - rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, - i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr - ); - pSegcsr->bLookup = 1; - } - } + /* Reset the cursor to the same state as rtreeOpen() leaves it in. */ + freeCursorConstraints(pCsr); + sqlite3_free(pCsr->aPoint); + memset(pCsr, 0, sizeof(RtreeCursor)); + pCsr->base.pVtab = (sqlite3_vtab*)pRtree; - for(i=1; bFound==0 && inIndex; i++){ - if( p->aIndex[i].nPrefix==nTerm+1 ){ - bFound = 1; - rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, - i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr - ); - if( rc==SQLITE_OK ){ - rc = fts3SegReaderCursorAddZero( - p, pCsr->iLangid, zTerm, nTerm, pSegcsr - ); + pCsr->iStrategy = idxNum; + if( idxNum==1 ){ + /* Special case - lookup by rowid. */ + RtreeNode *pLeaf; /* Leaf on which the required cell resides */ + RtreeSearchPoint *p; /* Search point for the the leaf */ + i64 iRowid = sqlite3_value_int64(argv[0]); + i64 iNode = 0; + rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); + if( rc==SQLITE_OK && pLeaf!=0 ){ + p = rtreeSearchPointNew(pCsr, RTREE_ZERO, 0); + assert( p!=0 ); /* Always returns pCsr->sPoint */ + pCsr->aNode[0] = pLeaf; + p->id = iNode; + p->eWithin = PARTLY_WITHIN; + rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &iCell); + p->iCell = iCell; + RTREE_QUEUE_TRACE(pCsr, "PUSH-F1:"); + }else{ + pCsr->atEOF = 1; + } + }else{ + /* Normal case - r-tree scan. Set up the RtreeCursor.aConstraint array + ** with the configured constraints. + */ + rc = nodeAcquire(pRtree, 1, 0, &pRoot); + if( rc==SQLITE_OK && argc>0 ){ + pCsr->aConstraint = sqlite3_malloc(sizeof(RtreeConstraint)*argc); + pCsr->nConstraint = argc; + if( !pCsr->aConstraint ){ + rc = SQLITE_NOMEM; + }else{ + memset(pCsr->aConstraint, 0, sizeof(RtreeConstraint)*argc); + memset(pCsr->anQueue, 0, sizeof(u32)*(pRtree->iDepth + 1)); + assert( (idxStr==0 && argc==0) + || (idxStr && (int)strlen(idxStr)==argc*2) ); + for(ii=0; iiaConstraint[ii]; + p->op = idxStr[ii*2]; + p->iCoord = idxStr[ii*2+1]-'0'; + if( p->op>=RTREE_MATCH ){ + /* A MATCH operator. The right-hand-side must be a blob that + ** can be cast into an RtreeMatchArg object. One created using + ** an sqlite3_rtree_geometry_callback() SQL user function. + */ + rc = deserializeGeometry(argv[ii], p); + if( rc!=SQLITE_OK ){ + break; + } + p->pInfo->nCoord = pRtree->nDim*2; + p->pInfo->anQueue = pCsr->anQueue; + p->pInfo->mxLevel = pRtree->iDepth + 1; + }else{ +#ifdef SQLITE_RTREE_INT_ONLY + p->u.rValue = sqlite3_value_int64(argv[ii]); +#else + p->u.rValue = sqlite3_value_double(argv[ii]); +#endif } } } } - - if( bFound==0 ){ - rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, - 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr - ); - pSegcsr->bLookup = !isPrefix; + if( rc==SQLITE_OK ){ + RtreeSearchPoint *pNew; + pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, pRtree->iDepth+1); + if( pNew==0 ) return SQLITE_NOMEM; + pNew->id = 1; + pNew->iCell = 0; + pNew->eWithin = PARTLY_WITHIN; + assert( pCsr->bPoint==1 ); + pCsr->aNode[0] = pRoot; + pRoot = 0; + RTREE_QUEUE_TRACE(pCsr, "PUSH-Fm:"); + rc = rtreeStepToLeaf(pCsr); } } - *ppSegcsr = pSegcsr; + nodeRelease(pRtree, pRoot); + rtreeRelease(pRtree); return rc; } /* -** Free an Fts3MultiSegReader allocated by fts3TermSegReaderCursor(). +** Set the pIdxInfo->estimatedRows variable to nRow. Unless this +** extension is currently being used by a version of SQLite too old to +** support estimatedRows. In that case this function is a no-op. */ -static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ - sqlite3Fts3SegReaderFinish(pSegcsr); - sqlite3_free(pSegcsr); +static void setEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ +#if SQLITE_VERSION_NUMBER>=3008002 + if( sqlite3_libversion_number()>=3008002 ){ + pIdxInfo->estimatedRows = nRow; + } +#endif } /* -** This function retrieves the doclist for the specified term (or term -** prefix) from the database. +** Rtree virtual table module xBestIndex method. There are three +** table scan strategies to choose from (in order from most to +** least desirable): +** +** idxNum idxStr Strategy +** ------------------------------------------------ +** 1 Unused Direct lookup by rowid. +** 2 See below R-tree query or full-table scan. +** ------------------------------------------------ +** +** If strategy 1 is used, then idxStr is not meaningful. If strategy +** 2 is used, idxStr is formatted to contain 2 bytes for each +** constraint used. The first two bytes of idxStr correspond to +** the constraint in sqlite3_index_info.aConstraintUsage[] with +** (argvIndex==1) etc. +** +** The first of each pair of bytes in idxStr identifies the constraint +** operator as follows: +** +** Operator Byte Value +** ---------------------- +** = 0x41 ('A') +** <= 0x42 ('B') +** < 0x43 ('C') +** >= 0x44 ('D') +** > 0x45 ('E') +** MATCH 0x46 ('F') +** ---------------------- +** +** The second of each pair of bytes identifies the coordinate column +** to which the constraint applies. The leftmost coordinate column +** is 'a', the second from the left 'b' etc. */ -static int fts3TermSelect( - Fts3Table *p, /* Virtual table handle */ - Fts3PhraseToken *pTok, /* Token to query for */ - int iColumn, /* Column to query (or -ve for all columns) */ - int *pnOut, /* OUT: Size of buffer at *ppOut */ - char **ppOut /* OUT: Malloced result buffer */ -){ - int rc; /* Return code */ - Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ - TermSelect tsc; /* Object for pair-wise doclist merging */ - Fts3SegFilter filter; /* Segment term filter configuration */ - - pSegcsr = pTok->pSegcsr; - memset(&tsc, 0, sizeof(TermSelect)); +static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + Rtree *pRtree = (Rtree*)tab; + int rc = SQLITE_OK; + int ii; + int bMatch = 0; /* True if there exists a MATCH constraint */ + i64 nRow; /* Estimated rows returned by this scan */ - filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | FTS3_SEGMENT_REQUIRE_POS - | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0) - | (pTok->bFirst ? FTS3_SEGMENT_FIRST : 0) - | (iColumnnColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); - filter.iCol = iColumn; - filter.zTerm = pTok->z; - filter.nTerm = pTok->n; + int iIdx = 0; + char zIdxStr[RTREE_MAX_DIMENSIONS*8+1]; + memset(zIdxStr, 0, sizeof(zIdxStr)); - rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter); - while( SQLITE_OK==rc - && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr)) - ){ - rc = fts3TermSelectMerge(p, &tsc, pSegcsr->aDoclist, pSegcsr->nDoclist); + /* Check if there exists a MATCH constraint - even an unusable one. If there + ** is, do not consider the lookup-by-rowid plan as using such a plan would + ** require the VDBE to evaluate the MATCH constraint, which is not currently + ** possible. */ + for(ii=0; iinConstraint; ii++){ + if( pIdxInfo->aConstraint[ii].op==SQLITE_INDEX_CONSTRAINT_MATCH ){ + bMatch = 1; + } } - if( rc==SQLITE_OK ){ - rc = fts3TermSelectFinishMerge(p, &tsc); - } - if( rc==SQLITE_OK ){ - *ppOut = tsc.aaOutput[0]; - *pnOut = tsc.anOutput[0]; - }else{ - int i; - for(i=0; iidxStr==0 ); + for(ii=0; iinConstraint && iIdx<(int)(sizeof(zIdxStr)-1); ii++){ + struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[ii]; + + if( bMatch==0 && p->usable + && p->iColumn==0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + /* We have an equality constraint on the rowid. Use strategy 1. */ + int jj; + for(jj=0; jjaConstraintUsage[jj].argvIndex = 0; + pIdxInfo->aConstraintUsage[jj].omit = 0; + } + pIdxInfo->idxNum = 1; + pIdxInfo->aConstraintUsage[ii].argvIndex = 1; + pIdxInfo->aConstraintUsage[jj].omit = 1; + + /* This strategy involves a two rowid lookups on an B-Tree structures + ** and then a linear search of an R-Tree node. This should be + ** considered almost as quick as a direct rowid lookup (for which + ** sqlite uses an internal cost of 0.0). It is expected to return + ** a single row. + */ + pIdxInfo->estimatedCost = 30.0; + setEstimatedRows(pIdxInfo, 1); + return SQLITE_OK; + } + + if( p->usable && (p->iColumn>0 || p->op==SQLITE_INDEX_CONSTRAINT_MATCH) ){ + u8 op; + switch( p->op ){ + case SQLITE_INDEX_CONSTRAINT_EQ: op = RTREE_EQ; break; + case SQLITE_INDEX_CONSTRAINT_GT: op = RTREE_GT; break; + case SQLITE_INDEX_CONSTRAINT_LE: op = RTREE_LE; break; + case SQLITE_INDEX_CONSTRAINT_LT: op = RTREE_LT; break; + case SQLITE_INDEX_CONSTRAINT_GE: op = RTREE_GE; break; + default: + assert( p->op==SQLITE_INDEX_CONSTRAINT_MATCH ); + op = RTREE_MATCH; + break; + } + zIdxStr[iIdx++] = op; + zIdxStr[iIdx++] = p->iColumn - 1 + '0'; + pIdxInfo->aConstraintUsage[ii].argvIndex = (iIdx/2); + pIdxInfo->aConstraintUsage[ii].omit = 1; } } - fts3SegReaderCursorFree(pSegcsr); - pTok->pSegcsr = 0; + pIdxInfo->idxNum = 2; + pIdxInfo->needToFreeIdxStr = 1; + if( iIdx>0 && 0==(pIdxInfo->idxStr = sqlite3_mprintf("%s", zIdxStr)) ){ + return SQLITE_NOMEM; + } + + nRow = pRtree->nRowEst / (iIdx + 1); + pIdxInfo->estimatedCost = (double)6.0 * (double)nRow; + setEstimatedRows(pIdxInfo, nRow); + return rc; } /* -** This function counts the total number of docids in the doclist stored -** in buffer aList[], size nList bytes. -** -** If the isPoslist argument is true, then it is assumed that the doclist -** contains a position-list following each docid. Otherwise, it is assumed -** that the doclist is simply a list of docids stored as delta encoded -** varints. +** Return the N-dimensional volumn of the cell stored in *p. */ -static int fts3DoclistCountDocids(char *aList, int nList){ - int nDoc = 0; /* Return value */ - if( aList ){ - char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */ - char *p = aList; /* Cursor */ - while( pnDim*2); ii+=2){ + area = (area * (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii]))); } + return area; +} - return nDoc; +/* +** Return the margin length of cell p. The margin length is the sum +** of the objects size in each dimension. +*/ +static RtreeDValue cellMargin(Rtree *pRtree, RtreeCell *p){ + RtreeDValue margin = (RtreeDValue)0; + int ii; + for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + margin += (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])); + } + return margin; } /* -** Advance the cursor to the next row in the %_content table that -** matches the search criteria. For a MATCH search, this will be -** the next row that matches. For a full-table scan, this will be -** simply the next row in the %_content table. For a docid lookup, -** this routine simply sets the EOF flag. -** -** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned -** even if we reach end-of-file. The fts3EofMethod() will be called -** subsequently to determine whether or not an EOF was hit. +** Store the union of cells p1 and p2 in p1. */ -static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ - int rc; - Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; - if( pCsr->eSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){ - if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ - pCsr->isEof = 1; - rc = sqlite3_reset(pCsr->pStmt); - }else{ - pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); - rc = SQLITE_OK; +static void cellUnion(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ + int ii; + if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ + for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + p1->aCoord[ii].f = MIN(p1->aCoord[ii].f, p2->aCoord[ii].f); + p1->aCoord[ii+1].f = MAX(p1->aCoord[ii+1].f, p2->aCoord[ii+1].f); } }else{ - rc = fts3EvalNext((Fts3Cursor *)pCursor); + for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + p1->aCoord[ii].i = MIN(p1->aCoord[ii].i, p2->aCoord[ii].i); + p1->aCoord[ii+1].i = MAX(p1->aCoord[ii+1].i, p2->aCoord[ii+1].i); + } } - assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); - return rc; } /* -** The following are copied from sqliteInt.h. -** -** Constants for the largest and smallest possible 64-bit signed integers. -** These macros are designed to work correctly on both 32-bit and 64-bit -** compilers. +** Return true if the area covered by p2 is a subset of the area covered +** by p1. False otherwise. */ -#ifndef SQLITE_AMALGAMATION -# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) -# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) -#endif +static int cellContains(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ + int ii; + int isInt = (pRtree->eCoordType==RTREE_COORD_INT32); + for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + RtreeCoord *a1 = &p1->aCoord[ii]; + RtreeCoord *a2 = &p2->aCoord[ii]; + if( (!isInt && (a2[0].fa1[1].f)) + || ( isInt && (a2[0].ia1[1].i)) + ){ + return 0; + } + } + return 1; +} /* -** If the numeric type of argument pVal is "integer", then return it -** converted to a 64-bit signed integer. Otherwise, return a copy of -** the second parameter, iDefault. +** Return the amount cell p would grow by if it were unioned with pCell. */ -static sqlite3_int64 fts3DocidRange(sqlite3_value *pVal, i64 iDefault){ - if( pVal ){ - int eType = sqlite3_value_numeric_type(pVal); - if( eType==SQLITE_INTEGER ){ - return sqlite3_value_int64(pVal); +static RtreeDValue cellGrowth(Rtree *pRtree, RtreeCell *p, RtreeCell *pCell){ + RtreeDValue area; + RtreeCell cell; + memcpy(&cell, p, sizeof(RtreeCell)); + area = cellArea(pRtree, &cell); + cellUnion(pRtree, &cell, pCell); + return (cellArea(pRtree, &cell)-area); +} + +static RtreeDValue cellOverlap( + Rtree *pRtree, + RtreeCell *p, + RtreeCell *aCell, + int nCell +){ + int ii; + RtreeDValue overlap = RTREE_ZERO; + for(ii=0; iinDim*2); jj+=2){ + RtreeDValue x1, x2; + x1 = MAX(DCOORD(p->aCoord[jj]), DCOORD(aCell[ii].aCoord[jj])); + x2 = MIN(DCOORD(p->aCoord[jj+1]), DCOORD(aCell[ii].aCoord[jj+1])); + if( x2=FTS3_FULLTEXT_SEARCH then use the full text index. The -** column on the left-hand side of the MATCH operator is column -** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand -** side of the MATCH operator. +** This function implements the ChooseLeaf algorithm from Gutman[84]. +** ChooseSubTree in r*tree terminology. */ -static int fts3FilterMethod( - sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ - int idxNum, /* Strategy index */ - const char *idxStr, /* Unused */ - int nVal, /* Number of elements in apVal */ - sqlite3_value **apVal /* Arguments for the indexing scheme */ +static int ChooseLeaf( + Rtree *pRtree, /* Rtree table */ + RtreeCell *pCell, /* Cell to insert into rtree */ + int iHeight, /* Height of sub-tree rooted at pCell */ + RtreeNode **ppLeaf /* OUT: Selected leaf page */ ){ - int rc = SQLITE_OK; - char *zSql; /* SQL statement used to access %_content */ - int eSearch; - Fts3Table *p = (Fts3Table *)pCursor->pVtab; - Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; - - sqlite3_value *pCons = 0; /* The MATCH or rowid constraint, if any */ - sqlite3_value *pLangid = 0; /* The "langid = ?" constraint, if any */ - sqlite3_value *pDocidGe = 0; /* The "docid >= ?" constraint, if any */ - sqlite3_value *pDocidLe = 0; /* The "docid <= ?" constraint, if any */ - int iIdx; + int rc; + int ii; + RtreeNode *pNode; + rc = nodeAcquire(pRtree, 1, 0, &pNode); - UNUSED_PARAMETER(idxStr); - UNUSED_PARAMETER(nVal); + for(ii=0; rc==SQLITE_OK && ii<(pRtree->iDepth-iHeight); ii++){ + int iCell; + sqlite3_int64 iBest = 0; - eSearch = (idxNum & 0x0000FFFF); - assert( eSearch>=0 && eSearch<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); - assert( p->pSegments==0 ); + RtreeDValue fMinGrowth = RTREE_ZERO; + RtreeDValue fMinArea = RTREE_ZERO; - /* Collect arguments into local variables */ - iIdx = 0; - if( eSearch!=FTS3_FULLSCAN_SEARCH ) pCons = apVal[iIdx++]; - if( idxNum & FTS3_HAVE_LANGID ) pLangid = apVal[iIdx++]; - if( idxNum & FTS3_HAVE_DOCID_GE ) pDocidGe = apVal[iIdx++]; - if( idxNum & FTS3_HAVE_DOCID_LE ) pDocidLe = apVal[iIdx++]; - assert( iIdx==nVal ); + int nCell = NCELL(pNode); + RtreeCell cell; + RtreeNode *pChild; - /* In case the cursor has been used before, clear it now. */ - sqlite3_finalize(pCsr->pStmt); - sqlite3_free(pCsr->aDoclist); - sqlite3_free(pCsr->aMatchinfo); - sqlite3Fts3ExprFree(pCsr->pExpr); - memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); + RtreeCell *aCell = 0; - /* Set the lower and upper bounds on docids to return */ - pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64); - pCsr->iMaxDocid = fts3DocidRange(pDocidLe, LARGEST_INT64); + /* Select the child node which will be enlarged the least if pCell + ** is inserted into it. Resolve ties by choosing the entry with + ** the smallest area. + */ + for(iCell=0; iCellbDesc = (idxStr[0]=='D'); - }else{ - pCsr->bDesc = p->bDescIdx; + sqlite3_free(aCell); + rc = nodeAcquire(pRtree, iBest, pNode, &pChild); + nodeRelease(pRtree, pNode); + pNode = pChild; } - pCsr->eSearch = (i16)eSearch; - - if( eSearch!=FTS3_DOCID_SEARCH && eSearch!=FTS3_FULLSCAN_SEARCH ){ - int iCol = eSearch-FTS3_FULLTEXT_SEARCH; - const char *zQuery = (const char *)sqlite3_value_text(pCons); - if( zQuery==0 && sqlite3_value_type(pCons)!=SQLITE_NULL ){ - return SQLITE_NOMEM; - } + *ppLeaf = pNode; + return rc; +} - pCsr->iLangid = 0; - if( pLangid ) pCsr->iLangid = sqlite3_value_int(pLangid); +/* +** A cell with the same content as pCell has just been inserted into +** the node pNode. This function updates the bounding box cells in +** all ancestor elements. +*/ +static int AdjustTree( + Rtree *pRtree, /* Rtree table */ + RtreeNode *pNode, /* Adjust ancestry of this node. */ + RtreeCell *pCell /* This cell was just inserted */ +){ + RtreeNode *p = pNode; + while( p->pParent ){ + RtreeNode *pParent = p->pParent; + RtreeCell cell; + int iCell; - assert( p->base.zErrMsg==0 ); - rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid, - p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr, - &p->base.zErrMsg - ); - if( rc!=SQLITE_OK ){ - return rc; + if( nodeParentIndex(pRtree, p, &iCell) ){ + return SQLITE_CORRUPT_VTAB; } - rc = fts3EvalStart(pCsr); - sqlite3Fts3SegmentsClose(p); - if( rc!=SQLITE_OK ) return rc; - pCsr->pNextId = pCsr->aDoclist; - pCsr->iPrevId = 0; - } - - /* Compile a SELECT statement for this cursor. For a full-table-scan, the - ** statement loops through all rows of the %_content table. For a - ** full-text query or docid lookup, the statement retrieves a single - ** row by docid. - */ - if( eSearch==FTS3_FULLSCAN_SEARCH ){ - if( pDocidGe || pDocidLe ){ - zSql = sqlite3_mprintf( - "SELECT %s WHERE rowid BETWEEN %lld AND %lld ORDER BY rowid %s", - p->zReadExprlist, pCsr->iMinDocid, pCsr->iMaxDocid, - (pCsr->bDesc ? "DESC" : "ASC") - ); - }else{ - zSql = sqlite3_mprintf("SELECT %s ORDER BY rowid %s", - p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC") - ); - } - if( zSql ){ - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); - sqlite3_free(zSql); - }else{ - rc = SQLITE_NOMEM; - } - }else if( eSearch==FTS3_DOCID_SEARCH ){ - rc = fts3CursorSeekStmt(pCsr, &pCsr->pStmt); - if( rc==SQLITE_OK ){ - rc = sqlite3_bind_value(pCsr->pStmt, 1, pCons); + nodeGetCell(pRtree, pParent, iCell, &cell); + if( !cellContains(pRtree, &cell, pCell) ){ + cellUnion(pRtree, &cell, pCell); + nodeOverwriteCell(pRtree, pParent, &cell, iCell); } + + p = pParent; } - if( rc!=SQLITE_OK ) return rc; - - return fts3NextMethod(pCursor); + return SQLITE_OK; } -/* -** This is the xEof method of the virtual table. SQLite calls this -** routine to find out if it has reached the end of a result set. +/* +** Write mapping (iRowid->iNode) to the _rowid table. */ -static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){ - return ((Fts3Cursor *)pCursor)->isEof; +static int rowidWrite(Rtree *pRtree, sqlite3_int64 iRowid, sqlite3_int64 iNode){ + sqlite3_bind_int64(pRtree->pWriteRowid, 1, iRowid); + sqlite3_bind_int64(pRtree->pWriteRowid, 2, iNode); + sqlite3_step(pRtree->pWriteRowid); + return sqlite3_reset(pRtree->pWriteRowid); } -/* -** This is the xRowid method. The SQLite core calls this routine to -** retrieve the rowid for the current row of the result set. fts3 -** exposes %_content.docid as the rowid for the virtual table. The -** rowid should be written to *pRowid. +/* +** Write mapping (iNode->iPar) to the _parent table. */ -static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ - Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; - *pRowid = pCsr->iPrevId; - return SQLITE_OK; +static int parentWrite(Rtree *pRtree, sqlite3_int64 iNode, sqlite3_int64 iPar){ + sqlite3_bind_int64(pRtree->pWriteParent, 1, iNode); + sqlite3_bind_int64(pRtree->pWriteParent, 2, iPar); + sqlite3_step(pRtree->pWriteParent); + return sqlite3_reset(pRtree->pWriteParent); } -/* -** This is the xColumn method, called by SQLite to request a value from -** the row that the supplied cursor currently points to. +static int rtreeInsertCell(Rtree *, RtreeNode *, RtreeCell *, int); + + +/* +** Arguments aIdx, aDistance and aSpare all point to arrays of size +** nIdx. The aIdx array contains the set of integers from 0 to +** (nIdx-1) in no particular order. This function sorts the values +** in aIdx according to the indexed values in aDistance. For +** example, assuming the inputs: ** -** If: +** aIdx = { 0, 1, 2, 3 } +** aDistance = { 5.0, 2.0, 7.0, 6.0 } ** -** (iCol < p->nColumn) -> The value of the iCol'th user column. -** (iCol == p->nColumn) -> Magic column with the same name as the table. -** (iCol == p->nColumn+1) -> Docid column -** (iCol == p->nColumn+2) -> Langid column +** this function sets the aIdx array to contain: +** +** aIdx = { 0, 1, 2, 3 } +** +** The aSpare array is used as temporary working space by the +** sorting algorithm. */ -static int fts3ColumnMethod( - sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ - sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ - int iCol /* Index of column to read value from */ +static void SortByDistance( + int *aIdx, + int nIdx, + RtreeDValue *aDistance, + int *aSpare ){ - int rc = SQLITE_OK; /* Return Code */ - Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; - Fts3Table *p = (Fts3Table *)pCursor->pVtab; + if( nIdx>1 ){ + int iLeft = 0; + int iRight = 0; - /* The column value supplied by SQLite must be in range. */ - assert( iCol>=0 && iCol<=p->nColumn+2 ); + int nLeft = nIdx/2; + int nRight = nIdx-nLeft; + int *aLeft = aIdx; + int *aRight = &aIdx[nLeft]; - if( iCol==p->nColumn+1 ){ - /* This call is a request for the "docid" column. Since "docid" is an - ** alias for "rowid", use the xRowid() method to obtain the value. - */ - sqlite3_result_int64(pCtx, pCsr->iPrevId); - }else if( iCol==p->nColumn ){ - /* The extra column whose name is the same as the table. - ** Return a blob which is a pointer to the cursor. */ - sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); - }else if( iCol==p->nColumn+2 && pCsr->pExpr ){ - sqlite3_result_int64(pCtx, pCsr->iLangid); - }else{ - /* The requested column is either a user column (one that contains - ** indexed data), or the language-id column. */ - rc = fts3CursorSeek(0, pCsr); + SortByDistance(aLeft, nLeft, aDistance, aSpare); + SortByDistance(aRight, nRight, aDistance, aSpare); - if( rc==SQLITE_OK ){ - if( iCol==p->nColumn+2 ){ - int iLangid = 0; - if( p->zLanguageid ){ - iLangid = sqlite3_column_int(pCsr->pStmt, p->nColumn+1); + memcpy(aSpare, aLeft, sizeof(int)*nLeft); + aLeft = aSpare; + + while( iLeftpStmt)>(iCol+1) ){ - sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); } } - } - assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); - return rc; +#if 0 + /* Check that the sort worked */ + { + int jj; + for(jj=1; jj1 ){ -/* -** Implementation of xSync() method. Flush the contents of the pending-terms -** hash-table to the database. -*/ -static int fts3SyncMethod(sqlite3_vtab *pVtab){ + int iLeft = 0; + int iRight = 0; - /* Following an incremental-merge operation, assuming that the input - ** segments are not completely consumed (the usual case), they are updated - ** in place to remove the entries that have already been merged. This - ** involves updating the leaf block that contains the smallest unmerged - ** entry and each block (if any) between the leaf and the root node. So - ** if the height of the input segment b-trees is N, and input segments - ** are merged eight at a time, updating the input segments at the end - ** of an incremental-merge requires writing (8*(1+N)) blocks. N is usually - ** small - often between 0 and 2. So the overhead of the incremental - ** merge is somewhere between 8 and 24 blocks. To avoid this overhead - ** dwarfing the actual productive work accomplished, the incremental merge - ** is only attempted if it will write at least 64 leaf blocks. Hence - ** nMinMerge. - ** - ** Of course, updating the input segments also involves deleting a bunch - ** of blocks from the segments table. But this is not considered overhead - ** as it would also be required by a crisis-merge that used the same input - ** segments. - */ - const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */ + int nLeft = nIdx/2; + int nRight = nIdx-nLeft; + int *aLeft = aIdx; + int *aRight = &aIdx[nLeft]; - Fts3Table *p = (Fts3Table*)pVtab; - int rc = sqlite3Fts3PendingTermsFlush(p); + SortByDimension(pRtree, aLeft, nLeft, iDim, aCell, aSpare); + SortByDimension(pRtree, aRight, nRight, iDim, aCell, aSpare); - if( rc==SQLITE_OK - && p->nLeafAdd>(nMinMerge/16) - && p->nAutoincrmerge && p->nAutoincrmerge!=0xff - ){ - int mxLevel = 0; /* Maximum relative level value in db */ - int A; /* Incr-merge parameter A */ + memcpy(aSpare, aLeft, sizeof(int)*nLeft); + aLeft = aSpare; + while( iLeftnLeafAdd * mxLevel; - A += (A/2); - if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge); +#if 0 + /* Check that the sort worked */ + { + int jj; + for(jj=1; jjbHasStat==2), attempt to determine this (set p->bHasStat -** to 0 or 1). Return SQLITE_OK if successful, or an SQLite error code -** if an error occurs. +** Implementation of the R*-tree variant of SplitNode from Beckman[1990]. */ -static int fts3SetHasStat(Fts3Table *p){ - int rc = SQLITE_OK; - if( p->bHasStat==2 ){ - const char *zFmt ="SELECT 1 FROM %Q.sqlite_master WHERE tbl_name='%q_stat'"; - char *zSql = sqlite3_mprintf(zFmt, p->zDb, p->zName); - if( zSql ){ - sqlite3_stmt *pStmt = 0; - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); - if( rc==SQLITE_OK ){ - int bHasStat = (sqlite3_step(pStmt)==SQLITE_ROW); - rc = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK ) p->bHasStat = bHasStat; +static int splitNodeStartree( + Rtree *pRtree, + RtreeCell *aCell, + int nCell, + RtreeNode *pLeft, + RtreeNode *pRight, + RtreeCell *pBboxLeft, + RtreeCell *pBboxRight +){ + int **aaSorted; + int *aSpare; + int ii; + + int iBestDim = 0; + int iBestSplit = 0; + RtreeDValue fBestMargin = RTREE_ZERO; + + int nByte = (pRtree->nDim+1)*(sizeof(int*)+nCell*sizeof(int)); + + aaSorted = (int **)sqlite3_malloc(nByte); + if( !aaSorted ){ + return SQLITE_NOMEM; + } + + aSpare = &((int *)&aaSorted[pRtree->nDim])[pRtree->nDim*nCell]; + memset(aaSorted, 0, nByte); + for(ii=0; iinDim; ii++){ + int jj; + aaSorted[ii] = &((int *)&aaSorted[pRtree->nDim])[ii*nCell]; + for(jj=0; jjnDim; ii++){ + RtreeDValue margin = RTREE_ZERO; + RtreeDValue fBestOverlap = RTREE_ZERO; + RtreeDValue fBestArea = RTREE_ZERO; + int iBestLeft = 0; + int nLeft; + + for( + nLeft=RTREE_MINCELLS(pRtree); + nLeft<=(nCell-RTREE_MINCELLS(pRtree)); + nLeft++ + ){ + RtreeCell left; + RtreeCell right; + int kk; + RtreeDValue overlap; + RtreeDValue area; + + memcpy(&left, &aCell[aaSorted[ii][0]], sizeof(RtreeCell)); + memcpy(&right, &aCell[aaSorted[ii][nCell-1]], sizeof(RtreeCell)); + for(kk=1; kk<(nCell-1); kk++){ + if( kkpSegments==0 ); - assert( p->nPendingData==0 ); - assert( p->inTransaction!=1 ); - TESTONLY( p->inTransaction = 1 ); - TESTONLY( p->mxSavepoint = -1; ); - p->nLeafAdd = 0; - return fts3SetHasStat(p); -} + memcpy(pBboxLeft, &aCell[aaSorted[iBestDim][0]], sizeof(RtreeCell)); + memcpy(pBboxRight, &aCell[aaSorted[iBestDim][iBestSplit]], sizeof(RtreeCell)); + for(ii=0; iinPendingData==0 ); - assert( p->inTransaction!=0 ); - assert( p->pSegments==0 ); - TESTONLY( p->inTransaction = 0 ); - TESTONLY( p->mxSavepoint = -1; ); + sqlite3_free(aaSorted); return SQLITE_OK; } -/* -** Implementation of xRollback(). Discard the contents of the pending-terms -** hash-table. Any changes made to the database are reverted by SQLite. -*/ -static int fts3RollbackMethod(sqlite3_vtab *pVtab){ - Fts3Table *p = (Fts3Table*)pVtab; - sqlite3Fts3PendingTermsClear(p); - assert( p->inTransaction!=0 ); - TESTONLY( p->inTransaction = 0 ); - TESTONLY( p->mxSavepoint = -1; ); - return SQLITE_OK; + +static int updateMapping( + Rtree *pRtree, + i64 iRowid, + RtreeNode *pNode, + int iHeight +){ + int (*xSetMapping)(Rtree *, sqlite3_int64, sqlite3_int64); + xSetMapping = ((iHeight==0)?rowidWrite:parentWrite); + if( iHeight>0 ){ + RtreeNode *pChild = nodeHashLookup(pRtree, iRowid); + if( pChild ){ + nodeRelease(pRtree, pChild->pParent); + nodeReference(pNode); + pChild->pParent = pNode; + } + } + return xSetMapping(pRtree, iRowid, pNode->iNode); } -/* -** When called, *ppPoslist must point to the byte immediately following the -** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function -** moves *ppPoslist so that it instead points to the first byte of the -** same position list. -*/ -static void fts3ReversePoslist(char *pStart, char **ppPoslist){ - char *p = &(*ppPoslist)[-2]; - char c = 0; +static int SplitNode( + Rtree *pRtree, + RtreeNode *pNode, + RtreeCell *pCell, + int iHeight +){ + int i; + int newCellIsRight = 0; - /* Skip backwards passed any trailing 0x00 bytes added by NearTrim() */ - while( p>pStart && (c=*p--)==0 ); + int rc = SQLITE_OK; + int nCell = NCELL(pNode); + RtreeCell *aCell; + int *aiUsed; - /* Search backwards for a varint with value zero (the end of the previous - ** poslist). This is an 0x00 byte preceded by some byte that does not - ** have the 0x80 bit set. */ - while( p>pStart && (*p & 0x80) | c ){ - c = *p--; - } - assert( p==pStart || c==0 ); + RtreeNode *pLeft = 0; + RtreeNode *pRight = 0; - /* At this point p points to that preceding byte without the 0x80 bit - ** set. So to find the start of the poslist, skip forward 2 bytes then - ** over a varint. - ** - ** Normally. The other case is that p==pStart and the poslist to return - ** is the first in the doclist. In this case do not skip forward 2 bytes. - ** The second part of the if condition (c==0 && *ppPoslist>&p[2]) - ** is required for cases where the first byte of a doclist and the - ** doclist is empty. For example, if the first docid is 10, a doclist - ** that begins with: - ** - ** 0x0A 0x00 + RtreeCell leftbbox; + RtreeCell rightbbox; + + /* Allocate an array and populate it with a copy of pCell and + ** all cells from node pLeft. Then zero the original node. */ - if( p>pStart || (c==0 && *ppPoslist>&p[2]) ){ p = &p[2]; } - while( *p++&0x80 ); - *ppPoslist = p; -} + aCell = sqlite3_malloc((sizeof(RtreeCell)+sizeof(int))*(nCell+1)); + if( !aCell ){ + rc = SQLITE_NOMEM; + goto splitnode_out; + } + aiUsed = (int *)&aCell[nCell+1]; + memset(aiUsed, 0, sizeof(int)*(nCell+1)); + for(i=0; iiNode==1 ){ + pRight = nodeNew(pRtree, pNode); + pLeft = nodeNew(pRtree, pNode); + pRtree->iDepth++; + pNode->isDirty = 1; + writeInt16(pNode->zData, pRtree->iDepth); + }else{ + pLeft = pNode; + pRight = nodeNew(pRtree, pLeft->pParent); + nodeReference(pLeft); + } + + if( !pLeft || !pRight ){ + rc = SQLITE_NOMEM; + goto splitnode_out; } - memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *)); - *ppCsr = pRet; - return SQLITE_OK; -} -/* -** Implementation of the snippet() function for FTS3 -*/ -static void fts3SnippetFunc( - sqlite3_context *pContext, /* SQLite function call context */ - int nVal, /* Size of apVal[] array */ - sqlite3_value **apVal /* Array of arguments */ -){ - Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ - const char *zStart = ""; - const char *zEnd = ""; - const char *zEllipsis = "..."; - int iCol = -1; - int nToken = 15; /* Default number of tokens in snippet */ + memset(pLeft->zData, 0, pRtree->iNodeSize); + memset(pRight->zData, 0, pRtree->iNodeSize); - /* There must be at least one argument passed to this function (otherwise - ** the non-overloaded version would have been called instead of this one). + rc = splitNodeStartree(pRtree, aCell, nCell, pLeft, pRight, + &leftbbox, &rightbbox); + if( rc!=SQLITE_OK ){ + goto splitnode_out; + } + + /* Ensure both child nodes have node numbers assigned to them by calling + ** nodeWrite(). Node pRight always needs a node number, as it was created + ** by nodeNew() above. But node pLeft sometimes already has a node number. + ** In this case avoid the all to nodeWrite(). */ - assert( nVal>=1 ); + if( SQLITE_OK!=(rc = nodeWrite(pRtree, pRight)) + || (0==pLeft->iNode && SQLITE_OK!=(rc = nodeWrite(pRtree, pLeft))) + ){ + goto splitnode_out; + } - if( nVal>6 ){ - sqlite3_result_error(pContext, - "wrong number of arguments to function snippet()", -1); - return; + rightbbox.iRowid = pRight->iNode; + leftbbox.iRowid = pLeft->iNode; + + if( pNode->iNode==1 ){ + rc = rtreeInsertCell(pRtree, pLeft->pParent, &leftbbox, iHeight+1); + if( rc!=SQLITE_OK ){ + goto splitnode_out; + } + }else{ + RtreeNode *pParent = pLeft->pParent; + int iCell; + rc = nodeParentIndex(pRtree, pLeft, &iCell); + if( rc==SQLITE_OK ){ + nodeOverwriteCell(pRtree, pParent, &leftbbox, iCell); + rc = AdjustTree(pRtree, pParent, &leftbbox); + } + if( rc!=SQLITE_OK ){ + goto splitnode_out; + } + } + if( (rc = rtreeInsertCell(pRtree, pRight->pParent, &rightbbox, iHeight+1)) ){ + goto splitnode_out; } - if( fts3FunctionArg(pContext, "snippet", apVal[0], &pCsr) ) return; - switch( nVal ){ - case 6: nToken = sqlite3_value_int(apVal[5]); - case 5: iCol = sqlite3_value_int(apVal[4]); - case 4: zEllipsis = (const char*)sqlite3_value_text(apVal[3]); - case 3: zEnd = (const char*)sqlite3_value_text(apVal[2]); - case 2: zStart = (const char*)sqlite3_value_text(apVal[1]); + for(i=0; iiRowid ){ + newCellIsRight = 1; + } + if( rc!=SQLITE_OK ){ + goto splitnode_out; + } } - if( !zEllipsis || !zEnd || !zStart ){ - sqlite3_result_error_nomem(pContext); - }else if( nToken==0 ){ - sqlite3_result_text(pContext, "", -1, SQLITE_STATIC); - }else if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ - sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis, iCol, nToken); + if( pNode->iNode==1 ){ + for(i=0; iiRowid, pLeft, iHeight); + } + + if( rc==SQLITE_OK ){ + rc = nodeRelease(pRtree, pRight); + pRight = 0; + } + if( rc==SQLITE_OK ){ + rc = nodeRelease(pRtree, pLeft); + pLeft = 0; } + +splitnode_out: + nodeRelease(pRtree, pRight); + nodeRelease(pRtree, pLeft); + sqlite3_free(aCell); + return rc; } /* -** Implementation of the offsets() function for FTS3 +** If node pLeaf is not the root of the r-tree and its pParent pointer is +** still NULL, load all ancestor nodes of pLeaf into memory and populate +** the pLeaf->pParent chain all the way up to the root node. +** +** This operation is required when a row is deleted (or updated - an update +** is implemented as a delete followed by an insert). SQLite provides the +** rowid of the row to delete, which can be used to find the leaf on which +** the entry resides (argument pLeaf). Once the leaf is located, this +** function is called to determine its ancestry. */ -static void fts3OffsetsFunc( - sqlite3_context *pContext, /* SQLite function call context */ - int nVal, /* Size of argument array */ - sqlite3_value **apVal /* Array of arguments */ -){ - Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ - - UNUSED_PARAMETER(nVal); +static int fixLeafParent(Rtree *pRtree, RtreeNode *pLeaf){ + int rc = SQLITE_OK; + RtreeNode *pChild = pLeaf; + while( rc==SQLITE_OK && pChild->iNode!=1 && pChild->pParent==0 ){ + int rc2 = SQLITE_OK; /* sqlite3_reset() return code */ + sqlite3_bind_int64(pRtree->pReadParent, 1, pChild->iNode); + rc = sqlite3_step(pRtree->pReadParent); + if( rc==SQLITE_ROW ){ + RtreeNode *pTest; /* Used to test for reference loops */ + i64 iNode; /* Node number of parent node */ - assert( nVal==1 ); - if( fts3FunctionArg(pContext, "offsets", apVal[0], &pCsr) ) return; - assert( pCsr ); - if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ - sqlite3Fts3Offsets(pContext, pCsr); + /* Before setting pChild->pParent, test that we are not creating a + ** loop of references (as we would if, say, pChild==pParent). We don't + ** want to do this as it leads to a memory leak when trying to delete + ** the referenced counted node structures. + */ + iNode = sqlite3_column_int64(pRtree->pReadParent, 0); + for(pTest=pLeaf; pTest && pTest->iNode!=iNode; pTest=pTest->pParent); + if( !pTest ){ + rc2 = nodeAcquire(pRtree, iNode, 0, &pChild->pParent); + } + } + rc = sqlite3_reset(pRtree->pReadParent); + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK && !pChild->pParent ) rc = SQLITE_CORRUPT_VTAB; + pChild = pChild->pParent; } + return rc; } -/* -** Implementation of the special optimize() function for FTS3. This -** function merges all segments in the database to a single segment. -** Example usage is: -** -** SELECT optimize(t) FROM t LIMIT 1; -** -** where 't' is the name of an FTS3 table. -*/ -static void fts3OptimizeFunc( - sqlite3_context *pContext, /* SQLite function call context */ - int nVal, /* Size of argument array */ - sqlite3_value **apVal /* Array of arguments */ -){ - int rc; /* Return code */ - Fts3Table *p; /* Virtual table handle */ - Fts3Cursor *pCursor; /* Cursor handle passed through apVal[0] */ +static int deleteCell(Rtree *, RtreeNode *, int, int); - UNUSED_PARAMETER(nVal); +static int removeNode(Rtree *pRtree, RtreeNode *pNode, int iHeight){ + int rc; + int rc2; + RtreeNode *pParent = 0; + int iCell; - assert( nVal==1 ); - if( fts3FunctionArg(pContext, "optimize", apVal[0], &pCursor) ) return; - p = (Fts3Table *)pCursor->base.pVtab; - assert( p ); + assert( pNode->nRef==1 ); - rc = sqlite3Fts3Optimize(p); + /* Remove the entry in the parent cell. */ + rc = nodeParentIndex(pRtree, pNode, &iCell); + if( rc==SQLITE_OK ){ + pParent = pNode->pParent; + pNode->pParent = 0; + rc = deleteCell(pRtree, pParent, iCell, iHeight+1); + } + rc2 = nodeRelease(pRtree, pParent); + if( rc==SQLITE_OK ){ + rc = rc2; + } + if( rc!=SQLITE_OK ){ + return rc; + } - switch( rc ){ - case SQLITE_OK: - sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC); - break; - case SQLITE_DONE: - sqlite3_result_text(pContext, "Index already optimal", -1, SQLITE_STATIC); - break; - default: - sqlite3_result_error_code(pContext, rc); - break; + /* Remove the xxx_node entry. */ + sqlite3_bind_int64(pRtree->pDeleteNode, 1, pNode->iNode); + sqlite3_step(pRtree->pDeleteNode); + if( SQLITE_OK!=(rc = sqlite3_reset(pRtree->pDeleteNode)) ){ + return rc; + } + + /* Remove the xxx_parent entry. */ + sqlite3_bind_int64(pRtree->pDeleteParent, 1, pNode->iNode); + sqlite3_step(pRtree->pDeleteParent); + if( SQLITE_OK!=(rc = sqlite3_reset(pRtree->pDeleteParent)) ){ + return rc; } + + /* Remove the node from the in-memory hash table and link it into + ** the Rtree.pDeleted list. Its contents will be re-inserted later on. + */ + nodeHashDelete(pRtree, pNode); + pNode->iNode = iHeight; + pNode->pNext = pRtree->pDeleted; + pNode->nRef++; + pRtree->pDeleted = pNode; + + return SQLITE_OK; } -/* -** Implementation of the matchinfo() function for FTS3 -*/ -static void fts3MatchinfoFunc( - sqlite3_context *pContext, /* SQLite function call context */ - int nVal, /* Size of argument array */ - sqlite3_value **apVal /* Array of arguments */ -){ - Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ - assert( nVal==1 || nVal==2 ); - if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){ - const char *zArg = 0; - if( nVal>1 ){ - zArg = (const char *)sqlite3_value_text(apVal[1]); +static int fixBoundingBox(Rtree *pRtree, RtreeNode *pNode){ + RtreeNode *pParent = pNode->pParent; + int rc = SQLITE_OK; + if( pParent ){ + int ii; + int nCell = NCELL(pNode); + RtreeCell box; /* Bounding box for pNode */ + nodeGetCell(pRtree, pNode, 0, &box); + for(ii=1; iiiNode; + rc = nodeParentIndex(pRtree, pNode, &ii); + if( rc==SQLITE_OK ){ + nodeOverwriteCell(pRtree, pParent, &box, ii); + rc = fixBoundingBox(pRtree, pParent); } - sqlite3Fts3Matchinfo(pContext, pCsr, zArg); } + return rc; } /* -** This routine implements the xFindFunction method for the FTS3 -** virtual table. +** Delete the cell at index iCell of node pNode. After removing the +** cell, adjust the r-tree data structure if required. */ -static int fts3FindFunctionMethod( - sqlite3_vtab *pVtab, /* Virtual table handle */ - int nArg, /* Number of SQL function arguments */ - const char *zName, /* Name of SQL function */ - void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ - void **ppArg /* Unused */ -){ - struct Overloaded { - const char *zName; - void (*xFunc)(sqlite3_context*,int,sqlite3_value**); - } aOverload[] = { - { "snippet", fts3SnippetFunc }, - { "offsets", fts3OffsetsFunc }, - { "optimize", fts3OptimizeFunc }, - { "matchinfo", fts3MatchinfoFunc }, - }; - int i; /* Iterator variable */ +static int deleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell, int iHeight){ + RtreeNode *pParent; + int rc; - UNUSED_PARAMETER(pVtab); - UNUSED_PARAMETER(nArg); - UNUSED_PARAMETER(ppArg); + if( SQLITE_OK!=(rc = fixLeafParent(pRtree, pNode)) ){ + return rc; + } - for(i=0; ipParent; + assert( pParent || pNode->iNode==1 ); + if( pParent ){ + if( NCELL(pNode)db; /* Database connection */ - int rc; /* Return Code */ + int *aOrder; + int *aSpare; + RtreeCell *aCell; + RtreeDValue *aDistance; + int nCell; + RtreeDValue aCenterCoord[RTREE_MAX_DIMENSIONS]; + int iDim; + int ii; + int rc = SQLITE_OK; + int n; - /* At this point it must be known if the %_stat table exists or not. - ** So bHasStat may not be 2. */ - rc = fts3SetHasStat(p); - - /* As it happens, the pending terms table is always empty here. This is - ** because an "ALTER TABLE RENAME TABLE" statement inside a transaction - ** always opens a savepoint transaction. And the xSavepoint() method - ** flushes the pending terms table. But leave the (no-op) call to - ** PendingTermsFlush() in in case that changes. + memset(aCenterCoord, 0, sizeof(RtreeDValue)*RTREE_MAX_DIMENSIONS); + + nCell = NCELL(pNode)+1; + n = (nCell+1)&(~1); + + /* Allocate the buffers used by this operation. The allocation is + ** relinquished before this function returns. */ - assert( p->nPendingData==0 ); - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3PendingTermsFlush(p); + aCell = (RtreeCell *)sqlite3_malloc(n * ( + sizeof(RtreeCell) + /* aCell array */ + sizeof(int) + /* aOrder array */ + sizeof(int) + /* aSpare array */ + sizeof(RtreeDValue) /* aDistance array */ + )); + if( !aCell ){ + return SQLITE_NOMEM; } + aOrder = (int *)&aCell[n]; + aSpare = (int *)&aOrder[n]; + aDistance = (RtreeDValue *)&aSpare[n]; - if( p->zContentTbl==0 ){ - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", - p->zDb, p->zName, zName - ); + for(ii=0; iinDim; iDim++){ + aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2]); + aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2+1]); + } + } + for(iDim=0; iDimnDim; iDim++){ + aCenterCoord[iDim] = (aCenterCoord[iDim]/(nCell*(RtreeDValue)2)); } - if( p->bHasDocsize ){ - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';", - p->zDb, p->zName, zName - ); + for(ii=0; iinDim; iDim++){ + RtreeDValue coord = (DCOORD(aCell[ii].aCoord[iDim*2+1]) - + DCOORD(aCell[ii].aCoord[iDim*2])); + aDistance[ii] += (coord-aCenterCoord[iDim])*(coord-aCenterCoord[iDim]); + } } - if( p->bHasStat ){ - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_stat' RENAME TO '%q_stat';", - p->zDb, p->zName, zName - ); + + SortByDistance(aOrder, nCell, aDistance, aSpare); + nodeZero(pRtree, pNode); + + for(ii=0; rc==SQLITE_OK && ii<(nCell-(RTREE_MINCELLS(pRtree)+1)); ii++){ + RtreeCell *p = &aCell[aOrder[ii]]; + nodeInsertCell(pRtree, pNode, p); + if( p->iRowid==pCell->iRowid ){ + if( iHeight==0 ){ + rc = rowidWrite(pRtree, p->iRowid, pNode->iNode); + }else{ + rc = parentWrite(pRtree, p->iRowid, pNode->iNode); + } + } } - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';", - p->zDb, p->zName, zName - ); - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';", - p->zDb, p->zName, zName - ); + if( rc==SQLITE_OK ){ + rc = fixBoundingBox(pRtree, pNode); + } + for(; rc==SQLITE_OK && iiiNode currently contains + ** the height of the sub-tree headed by the cell. + */ + RtreeNode *pInsert; + RtreeCell *p = &aCell[aOrder[ii]]; + rc = ChooseLeaf(pRtree, p, iHeight, &pInsert); + if( rc==SQLITE_OK ){ + int rc2; + rc = rtreeInsertCell(pRtree, pInsert, p, iHeight); + rc2 = nodeRelease(pRtree, pInsert); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + } + + sqlite3_free(aCell); return rc; } /* -** The xSavepoint() method. -** -** Flush the contents of the pending-terms table to disk. +** Insert cell pCell into node pNode. Node pNode is the head of a +** subtree iHeight high (leaf nodes have iHeight==0). */ -static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ +static int rtreeInsertCell( + Rtree *pRtree, + RtreeNode *pNode, + RtreeCell *pCell, + int iHeight +){ int rc = SQLITE_OK; - UNUSED_PARAMETER(iSavepoint); - assert( ((Fts3Table *)pVtab)->inTransaction ); - assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint ); - TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); - if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){ - rc = fts3SyncMethod(pVtab); + if( iHeight>0 ){ + RtreeNode *pChild = nodeHashLookup(pRtree, pCell->iRowid); + if( pChild ){ + nodeRelease(pRtree, pChild->pParent); + nodeReference(pNode); + pChild->pParent = pNode; + } + } + if( nodeInsertCell(pRtree, pNode, pCell) ){ + if( iHeight<=pRtree->iReinsertHeight || pNode->iNode==1){ + rc = SplitNode(pRtree, pNode, pCell, iHeight); + }else{ + pRtree->iReinsertHeight = iHeight; + rc = Reinsert(pRtree, pNode, pCell, iHeight); + } + }else{ + rc = AdjustTree(pRtree, pNode, pCell); + if( rc==SQLITE_OK ){ + if( iHeight==0 ){ + rc = rowidWrite(pRtree, pCell->iRowid, pNode->iNode); + }else{ + rc = parentWrite(pRtree, pCell->iRowid, pNode->iNode); + } + } } return rc; } -/* -** The xRelease() method. -** -** This is a no-op. -*/ -static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ - TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); - UNUSED_PARAMETER(iSavepoint); - UNUSED_PARAMETER(pVtab); - assert( p->inTransaction ); - assert( p->mxSavepoint >= iSavepoint ); - TESTONLY( p->mxSavepoint = iSavepoint-1 ); - return SQLITE_OK; -} - -/* -** The xRollbackTo() method. -** -** Discard the contents of the pending terms table. -*/ -static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ - Fts3Table *p = (Fts3Table*)pVtab; - UNUSED_PARAMETER(iSavepoint); - assert( p->inTransaction ); - assert( p->mxSavepoint >= iSavepoint ); - TESTONLY( p->mxSavepoint = iSavepoint ); - sqlite3Fts3PendingTermsClear(p); - return SQLITE_OK; -} +static int reinsertNodeContent(Rtree *pRtree, RtreeNode *pNode){ + int ii; + int rc = SQLITE_OK; + int nCell = NCELL(pNode); -static const sqlite3_module fts3Module = { - /* iVersion */ 2, - /* xCreate */ fts3CreateMethod, - /* xConnect */ fts3ConnectMethod, - /* xBestIndex */ fts3BestIndexMethod, - /* xDisconnect */ fts3DisconnectMethod, - /* xDestroy */ fts3DestroyMethod, - /* xOpen */ fts3OpenMethod, - /* xClose */ fts3CloseMethod, - /* xFilter */ fts3FilterMethod, - /* xNext */ fts3NextMethod, - /* xEof */ fts3EofMethod, - /* xColumn */ fts3ColumnMethod, - /* xRowid */ fts3RowidMethod, - /* xUpdate */ fts3UpdateMethod, - /* xBegin */ fts3BeginMethod, - /* xSync */ fts3SyncMethod, - /* xCommit */ fts3CommitMethod, - /* xRollback */ fts3RollbackMethod, - /* xFindFunction */ fts3FindFunctionMethod, - /* xRename */ fts3RenameMethod, - /* xSavepoint */ fts3SavepointMethod, - /* xRelease */ fts3ReleaseMethod, - /* xRollbackTo */ fts3RollbackToMethod, -}; + for(ii=0; rc==SQLITE_OK && iiiNode currently contains + ** the height of the sub-tree headed by the cell. + */ + rc = ChooseLeaf(pRtree, &cell, (int)pNode->iNode, &pInsert); + if( rc==SQLITE_OK ){ + int rc2; + rc = rtreeInsertCell(pRtree, pInsert, &cell, (int)pNode->iNode); + rc2 = nodeRelease(pRtree, pInsert); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + } + return rc; } /* -** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are -** implemented in files fts3_tokenizer1.c, fts3_porter.c and fts3_icu.c -** respectively. The following three forward declarations are for functions -** declared in these files used to retrieve the respective implementations. -** -** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed -** to by the argument to point to the "simple" tokenizer implementation. -** And so on. +** Select a currently unused rowid for a new r-tree record. */ -SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); -SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); -#ifndef SQLITE_DISABLE_FTS3_UNICODE -SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule); -#endif -#ifdef SQLITE_ENABLE_ICU -SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); -#endif +static int newRowid(Rtree *pRtree, i64 *piRowid){ + int rc; + sqlite3_bind_null(pRtree->pWriteRowid, 1); + sqlite3_bind_null(pRtree->pWriteRowid, 2); + sqlite3_step(pRtree->pWriteRowid); + rc = sqlite3_reset(pRtree->pWriteRowid); + *piRowid = sqlite3_last_insert_rowid(pRtree->db); + return rc; +} /* -** Initialize the fts3 extension. If this extension is built as part -** of the sqlite library, then this function is called directly by -** SQLite. If fts3 is built as a dynamically loadable extension, this -** function is called by the sqlite3_extension_init() entry point. +** Remove the entry with rowid=iDelete from the r-tree structure. */ -SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ - int rc = SQLITE_OK; - Fts3Hash *pHash = 0; - const sqlite3_tokenizer_module *pSimple = 0; - const sqlite3_tokenizer_module *pPorter = 0; -#ifndef SQLITE_DISABLE_FTS3_UNICODE - const sqlite3_tokenizer_module *pUnicode = 0; -#endif - -#ifdef SQLITE_ENABLE_ICU - const sqlite3_tokenizer_module *pIcu = 0; - sqlite3Fts3IcuTokenizerModule(&pIcu); -#endif - -#ifndef SQLITE_DISABLE_FTS3_UNICODE - sqlite3Fts3UnicodeTokenizer(&pUnicode); -#endif - -#ifdef SQLITE_TEST - rc = sqlite3Fts3InitTerm(db); - if( rc!=SQLITE_OK ) return rc; -#endif +static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){ + int rc; /* Return code */ + RtreeNode *pLeaf = 0; /* Leaf node containing record iDelete */ + int iCell; /* Index of iDelete cell in pLeaf */ + RtreeNode *pRoot; /* Root node of rtree structure */ - rc = sqlite3Fts3InitAux(db); - if( rc!=SQLITE_OK ) return rc; - sqlite3Fts3SimpleTokenizerModule(&pSimple); - sqlite3Fts3PorterTokenizerModule(&pPorter); + /* Obtain a reference to the root node to initialize Rtree.iDepth */ + rc = nodeAcquire(pRtree, 1, 0, &pRoot); - /* Allocate and initialize the hash-table used to store tokenizers. */ - pHash = sqlite3_malloc(sizeof(Fts3Hash)); - if( !pHash ){ - rc = SQLITE_NOMEM; - }else{ - sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); + /* Obtain a reference to the leaf node that contains the entry + ** about to be deleted. + */ + if( rc==SQLITE_OK ){ + rc = findLeafNode(pRtree, iDelete, &pLeaf, 0); } - /* Load the built-in tokenizers into the hash table */ + /* Delete the cell in question from the leaf node. */ if( rc==SQLITE_OK ){ - if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple) - || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) - -#ifndef SQLITE_DISABLE_FTS3_UNICODE - || sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode) -#endif -#ifdef SQLITE_ENABLE_ICU - || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu)) -#endif - ){ - rc = SQLITE_NOMEM; + int rc2; + rc = nodeRowidIndex(pRtree, pLeaf, iDelete, &iCell); + if( rc==SQLITE_OK ){ + rc = deleteCell(pRtree, pLeaf, iCell, 0); + } + rc2 = nodeRelease(pRtree, pLeaf); + if( rc==SQLITE_OK ){ + rc = rc2; } } -#ifdef SQLITE_TEST + /* Delete the corresponding entry in the _rowid table. */ if( rc==SQLITE_OK ){ - rc = sqlite3Fts3ExprInitTestInterface(db); + sqlite3_bind_int64(pRtree->pDeleteRowid, 1, iDelete); + sqlite3_step(pRtree->pDeleteRowid); + rc = sqlite3_reset(pRtree->pDeleteRowid); } -#endif - /* Create the virtual table wrapper around the hash-table and overload - ** the two scalar functions. If this is successful, register the - ** module with sqlite. + /* Check if the root node now has exactly one child. If so, remove + ** it, schedule the contents of the child for reinsertion and + ** reduce the tree height by one. + ** + ** This is equivalent to copying the contents of the child into + ** the root node (the operation that Gutman's paper says to perform + ** in this scenario). */ - if( SQLITE_OK==rc - && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1)) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1)) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 2)) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1)) - ){ - rc = sqlite3_create_module_v2( - db, "fts3", &fts3Module, (void *)pHash, hashDestroy - ); + if( rc==SQLITE_OK && pRtree->iDepth>0 && NCELL(pRoot)==1 ){ + int rc2; + RtreeNode *pChild; + i64 iChild = nodeGetRowid(pRtree, pRoot, 0); + rc = nodeAcquire(pRtree, iChild, pRoot, &pChild); if( rc==SQLITE_OK ){ - rc = sqlite3_create_module_v2( - db, "fts4", &fts3Module, (void *)pHash, 0 - ); + rc = removeNode(pRtree, pChild, pRtree->iDepth-1); } + rc2 = nodeRelease(pRtree, pChild); + if( rc==SQLITE_OK ) rc = rc2; if( rc==SQLITE_OK ){ - rc = sqlite3Fts3InitTok(db, (void *)pHash); + pRtree->iDepth--; + writeInt16(pRoot->zData, pRtree->iDepth); + pRoot->isDirty = 1; } - return rc; } + /* Re-insert the contents of any underfull nodes removed from the tree. */ + for(pLeaf=pRtree->pDeleted; pLeaf; pLeaf=pRtree->pDeleted){ + if( rc==SQLITE_OK ){ + rc = reinsertNodeContent(pRtree, pLeaf); + } + pRtree->pDeleted = pLeaf->pNext; + sqlite3_free(pLeaf); + } - /* An error has occurred. Delete the hash table and return the error code. */ - assert( rc!=SQLITE_OK ); - if( pHash ){ - sqlite3Fts3HashClear(pHash); - sqlite3_free(pHash); + /* Release the reference to the root node. */ + if( rc==SQLITE_OK ){ + rc = nodeRelease(pRtree, pRoot); + }else{ + nodeRelease(pRtree, pRoot); } + return rc; } /* -** Allocate an Fts3MultiSegReader for each token in the expression headed -** by pExpr. -** -** An Fts3SegReader object is a cursor that can seek or scan a range of -** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple -** Fts3SegReader objects internally to provide an interface to seek or scan -** within the union of all segments of a b-tree. Hence the name. -** -** If the allocated Fts3MultiSegReader just seeks to a single entry in a -** segment b-tree (if the term is not a prefix or it is a prefix for which -** there exists prefix b-tree of the right length) then it may be traversed -** and merged incrementally. Otherwise, it has to be merged into an in-memory -** doclist and then traversed. +** Rounding constants for float->double conversion. */ -static void fts3EvalAllocateReaders( - Fts3Cursor *pCsr, /* FTS cursor handle */ - Fts3Expr *pExpr, /* Allocate readers for this expression */ - int *pnToken, /* OUT: Total number of tokens in phrase. */ - int *pnOr, /* OUT: Total number of OR nodes in expr. */ - int *pRc /* IN/OUT: Error code */ -){ - if( pExpr && SQLITE_OK==*pRc ){ - if( pExpr->eType==FTSQUERY_PHRASE ){ - int i; - int nToken = pExpr->pPhrase->nToken; - *pnToken += nToken; - for(i=0; ipPhrase->aToken[i]; - int rc = fts3TermSegReaderCursor(pCsr, - pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr - ); - if( rc!=SQLITE_OK ){ - *pRc = rc; - return; - } - } - assert( pExpr->pPhrase->iDoclistToken==0 ); - pExpr->pPhrase->iDoclistToken = -1; - }else{ - *pnOr += (pExpr->eType==FTSQUERY_OR); - fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc); - fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc); - } - } -} +#define RNDTOWARDS (1.0 - 1.0/8388608.0) /* Round towards zero */ +#define RNDAWAY (1.0 + 1.0/8388608.0) /* Round away from zero */ +#if !defined(SQLITE_RTREE_INT_ONLY) /* -** Arguments pList/nList contain the doclist for token iToken of phrase p. -** It is merged into the main doclist stored in p->doclist.aAll/nAll. -** -** This function assumes that pList points to a buffer allocated using -** sqlite3_malloc(). This function takes responsibility for eventually -** freeing the buffer. -** -** SQLITE_OK is returned if successful, or SQLITE_NOMEM if an error occurs. +** Convert an sqlite3_value into an RtreeValue (presumably a float) +** while taking care to round toward negative or positive, respectively. */ -static int fts3EvalPhraseMergeToken( - Fts3Table *pTab, /* FTS Table pointer */ - Fts3Phrase *p, /* Phrase to merge pList/nList into */ - int iToken, /* Token pList/nList corresponds to */ - char *pList, /* Pointer to doclist */ - int nList /* Number of bytes in pList */ -){ - int rc = SQLITE_OK; - assert( iToken!=p->iDoclistToken ); - - if( pList==0 ){ - sqlite3_free(p->doclist.aAll); - p->doclist.aAll = 0; - p->doclist.nAll = 0; - } - - else if( p->iDoclistToken<0 ){ - p->doclist.aAll = pList; - p->doclist.nAll = nList; - } - - else if( p->doclist.aAll==0 ){ - sqlite3_free(pList); +static RtreeValue rtreeValueDown(sqlite3_value *v){ + double d = sqlite3_value_double(v); + float f = (float)d; + if( f>d ){ + f = (float)(d*(d<0 ? RNDAWAY : RNDTOWARDS)); } - - else { - char *pLeft; - char *pRight; - int nLeft; - int nRight; - int nDiff; - - if( p->iDoclistTokendoclist.aAll; - nLeft = p->doclist.nAll; - pRight = pList; - nRight = nList; - nDiff = iToken - p->iDoclistToken; - }else{ - pRight = p->doclist.aAll; - nRight = p->doclist.nAll; - pLeft = pList; - nLeft = nList; - nDiff = p->iDoclistToken - iToken; - } - - rc = fts3DoclistPhraseMerge( - pTab->bDescIdx, nDiff, pLeft, nLeft, &pRight, &nRight - ); - sqlite3_free(pLeft); - p->doclist.aAll = pRight; - p->doclist.nAll = nRight; + return f; +} +static RtreeValue rtreeValueUp(sqlite3_value *v){ + double d = sqlite3_value_double(v); + float f = (float)d; + if( fp->iDoclistToken ) p->iDoclistToken = iToken; - return rc; + return f; } +#endif /* !defined(SQLITE_RTREE_INT_ONLY) */ + /* -** Load the doclist for phrase p into p->doclist.aAll/nAll. The loaded doclist -** does not take deferred tokens into account. -** -** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +** The xUpdate method for rtree module virtual tables. */ -static int fts3EvalPhraseLoad( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Phrase *p /* Phrase object */ +static int rtreeUpdate( + sqlite3_vtab *pVtab, + int nData, + sqlite3_value **azData, + sqlite_int64 *pRowid ){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int iToken; + Rtree *pRtree = (Rtree *)pVtab; int rc = SQLITE_OK; + RtreeCell cell; /* New cell to insert if nData>1 */ + int bHaveRowid = 0; /* Set to 1 after new rowid is determined */ - for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ - Fts3PhraseToken *pToken = &p->aToken[iToken]; - assert( pToken->pDeferred==0 || pToken->pSegcsr==0 ); - - if( pToken->pSegcsr ){ - int nThis = 0; - char *pThis = 0; - rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis); - if( rc==SQLITE_OK ){ - rc = fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); - } - } - assert( pToken->pSegcsr==0 ); - } - - return rc; -} - -/* -** This function is called on each phrase after the position lists for -** any deferred tokens have been loaded into memory. It updates the phrases -** current position list to include only those positions that are really -** instances of the phrase (after considering deferred tokens). If this -** means that the phrase does not appear in the current row, doclist.pList -** and doclist.nList are both zeroed. -** -** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. -*/ -static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ - int iToken; /* Used to iterate through phrase tokens */ - char *aPoslist = 0; /* Position list for deferred tokens */ - int nPoslist = 0; /* Number of bytes in aPoslist */ - int iPrev = -1; /* Token number of previous deferred token */ - - assert( pPhrase->doclist.bFreeList==0 ); - - for(iToken=0; iTokennToken; iToken++){ - Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; - Fts3DeferredToken *pDeferred = pToken->pDeferred; + rtreeReference(pRtree); + assert(nData>=1); - if( pDeferred ){ - char *pList; - int nList; - int rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); - if( rc!=SQLITE_OK ) return rc; + cell.iRowid = 0; /* Used only to suppress a compiler warning */ - if( pList==0 ){ - sqlite3_free(aPoslist); - pPhrase->doclist.pList = 0; - pPhrase->doclist.nList = 0; - return SQLITE_OK; + /* Constraint handling. A write operation on an r-tree table may return + ** SQLITE_CONSTRAINT for two reasons: + ** + ** 1. A duplicate rowid value, or + ** 2. The supplied data violates the "x2>=x1" constraint. + ** + ** In the first case, if the conflict-handling mode is REPLACE, then + ** the conflicting row can be removed before proceeding. In the second + ** case, SQLITE_CONSTRAINT must be returned regardless of the + ** conflict-handling mode specified by the user. + */ + if( nData>1 ){ + int ii; - }else if( aPoslist==0 ){ - aPoslist = pList; - nPoslist = nList; + /* Populate the cell.aCoord[] array. The first coordinate is azData[3]. + ** + ** NB: nData can only be less than nDim*2+3 if the rtree is mis-declared + ** with "column" that are interpreted as table constraints. + ** Example: CREATE VIRTUAL TABLE bad USING rtree(x,y,CHECK(y>5)); + ** This problem was discovered after years of use, so we silently ignore + ** these kinds of misdeclared tables to avoid breaking any legacy. + */ + assert( nData<=(pRtree->nDim*2 + 3) ); - }else{ - char *aOut = pList; - char *p1 = aPoslist; - char *p2 = aOut; +#ifndef SQLITE_RTREE_INT_ONLY + if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ + for(ii=0; iicell.aCoord[ii+1].f ){ + rc = SQLITE_CONSTRAINT; + goto constraint; + } + } + }else +#endif + { + for(ii=0; iicell.aCoord[ii+1].i ){ + rc = SQLITE_CONSTRAINT; + goto constraint; + } + } + } - assert( iPrev>=0 ); - fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2); - sqlite3_free(aPoslist); - aPoslist = pList; - nPoslist = (int)(aOut - aPoslist); - if( nPoslist==0 ){ - sqlite3_free(aPoslist); - pPhrase->doclist.pList = 0; - pPhrase->doclist.nList = 0; - return SQLITE_OK; + /* If a rowid value was supplied, check if it is already present in + ** the table. If so, the constraint has failed. */ + if( sqlite3_value_type(azData[2])!=SQLITE_NULL ){ + cell.iRowid = sqlite3_value_int64(azData[2]); + if( sqlite3_value_type(azData[0])==SQLITE_NULL + || sqlite3_value_int64(azData[0])!=cell.iRowid + ){ + int steprc; + sqlite3_bind_int64(pRtree->pReadRowid, 1, cell.iRowid); + steprc = sqlite3_step(pRtree->pReadRowid); + rc = sqlite3_reset(pRtree->pReadRowid); + if( SQLITE_ROW==steprc ){ + if( sqlite3_vtab_on_conflict(pRtree->db)==SQLITE_REPLACE ){ + rc = rtreeDeleteRowid(pRtree, cell.iRowid); + }else{ + rc = SQLITE_CONSTRAINT; + goto constraint; + } } } - iPrev = iToken; + bHaveRowid = 1; } } - if( iPrev>=0 ){ - int nMaxUndeferred = pPhrase->iDoclistToken; - if( nMaxUndeferred<0 ){ - pPhrase->doclist.pList = aPoslist; - pPhrase->doclist.nList = nPoslist; - pPhrase->doclist.iDocid = pCsr->iPrevId; - pPhrase->doclist.bFreeList = 1; - }else{ - int nDistance; - char *p1; - char *p2; - char *aOut; + /* If azData[0] is not an SQL NULL value, it is the rowid of a + ** record to delete from the r-tree table. The following block does + ** just that. + */ + if( sqlite3_value_type(azData[0])!=SQLITE_NULL ){ + rc = rtreeDeleteRowid(pRtree, sqlite3_value_int64(azData[0])); + } - if( nMaxUndeferred>iPrev ){ - p1 = aPoslist; - p2 = pPhrase->doclist.pList; - nDistance = nMaxUndeferred - iPrev; - }else{ - p1 = pPhrase->doclist.pList; - p2 = aPoslist; - nDistance = iPrev - nMaxUndeferred; - } + /* If the azData[] array contains more than one element, elements + ** (azData[2]..azData[argc-1]) contain a new record to insert into + ** the r-tree structure. + */ + if( rc==SQLITE_OK && nData>1 ){ + /* Insert the new record into the r-tree */ + RtreeNode *pLeaf = 0; - aOut = (char *)sqlite3_malloc(nPoslist+8); - if( !aOut ){ - sqlite3_free(aPoslist); - return SQLITE_NOMEM; - } - - pPhrase->doclist.pList = aOut; - if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ - pPhrase->doclist.bFreeList = 1; - pPhrase->doclist.nList = (int)(aOut - pPhrase->doclist.pList); - }else{ - sqlite3_free(aOut); - pPhrase->doclist.pList = 0; - pPhrase->doclist.nList = 0; + /* Figure out the rowid of the new row. */ + if( bHaveRowid==0 ){ + rc = newRowid(pRtree, &cell.iRowid); + } + *pRowid = cell.iRowid; + + if( rc==SQLITE_OK ){ + rc = ChooseLeaf(pRtree, &cell, 0, &pLeaf); + } + if( rc==SQLITE_OK ){ + int rc2; + pRtree->iReinsertHeight = -1; + rc = rtreeInsertCell(pRtree, pLeaf, &cell, 0); + rc2 = nodeRelease(pRtree, pLeaf); + if( rc==SQLITE_OK ){ + rc = rc2; } - sqlite3_free(aPoslist); } } - return SQLITE_OK; +constraint: + rtreeRelease(pRtree); + return rc; } /* -** Maximum number of tokens a phrase may have to be considered for the -** incremental doclists strategy. +** The xRename method for rtree module virtual tables. */ -#define MAX_INCR_PHRASE_TOKENS 4 +static int rtreeRename(sqlite3_vtab *pVtab, const char *zNewName){ + Rtree *pRtree = (Rtree *)pVtab; + int rc = SQLITE_NOMEM; + char *zSql = sqlite3_mprintf( + "ALTER TABLE %Q.'%q_node' RENAME TO \"%w_node\";" + "ALTER TABLE %Q.'%q_parent' RENAME TO \"%w_parent\";" + "ALTER TABLE %Q.'%q_rowid' RENAME TO \"%w_rowid\";" + , pRtree->zDb, pRtree->zName, zNewName + , pRtree->zDb, pRtree->zName, zNewName + , pRtree->zDb, pRtree->zName, zNewName + ); + if( zSql ){ + rc = sqlite3_exec(pRtree->db, zSql, 0, 0, 0); + sqlite3_free(zSql); + } + return rc; +} /* -** This function is called for each Fts3Phrase in a full-text query -** expression to initialize the mechanism for returning rows. Once this -** function has been called successfully on an Fts3Phrase, it may be -** used with fts3EvalPhraseNext() to iterate through the matching docids. -** -** If parameter bOptOk is true, then the phrase may (or may not) use the -** incremental loading strategy. Otherwise, the entire doclist is loaded into -** memory within this call. -** -** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +** This function populates the pRtree->nRowEst variable with an estimate +** of the number of rows in the virtual table. If possible, this is based +** on sqlite_stat1 data. Otherwise, use RTREE_DEFAULT_ROWEST. */ -static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int rc = SQLITE_OK; /* Error code */ - int i; +static int rtreeQueryStat1(sqlite3 *db, Rtree *pRtree){ + const char *zFmt = "SELECT stat FROM %Q.sqlite_stat1 WHERE tbl = '%q_rowid'"; + char *zSql; + sqlite3_stmt *p; + int rc; + i64 nRow = 0; - /* Determine if doclists may be loaded from disk incrementally. This is - ** possible if the bOptOk argument is true, the FTS doclists will be - ** scanned in forward order, and the phrase consists of - ** MAX_INCR_PHRASE_TOKENS or fewer tokens, none of which are are "^first" - ** tokens or prefix tokens that cannot use a prefix-index. */ - int bHaveIncr = 0; - int bIncrOk = (bOptOk - && pCsr->bDesc==pTab->bDescIdx - && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0 - && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0 -#ifdef SQLITE_TEST - && pTab->bNoIncrDoclist==0 -#endif - ); - for(i=0; bIncrOk==1 && inToken; i++){ - Fts3PhraseToken *pToken = &p->aToken[i]; - if( pToken->bFirst || (pToken->pSegcsr!=0 && !pToken->pSegcsr->bLookup) ){ - bIncrOk = 0; + zSql = sqlite3_mprintf(zFmt, pRtree->zDb, pRtree->zName); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(db, zSql, -1, &p, 0); + if( rc==SQLITE_OK ){ + if( sqlite3_step(p)==SQLITE_ROW ) nRow = sqlite3_column_int64(p, 0); + rc = sqlite3_finalize(p); + }else if( rc!=SQLITE_NOMEM ){ + rc = SQLITE_OK; } - if( pToken->pSegcsr ) bHaveIncr = 1; - } - if( bIncrOk && bHaveIncr ){ - /* Use the incremental approach. */ - int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); - for(i=0; rc==SQLITE_OK && inToken; i++){ - Fts3PhraseToken *pToken = &p->aToken[i]; - Fts3MultiSegReader *pSegcsr = pToken->pSegcsr; - if( pSegcsr ){ - rc = sqlite3Fts3MsrIncrStart(pTab, pSegcsr, iCol, pToken->z, pToken->n); + if( rc==SQLITE_OK ){ + if( nRow==0 ){ + pRtree->nRowEst = RTREE_DEFAULT_ROWEST; + }else{ + pRtree->nRowEst = MAX(nRow, RTREE_MIN_ROWEST); } } - p->bIncr = 1; - }else{ - /* Load the full doclist for the phrase into memory. */ - rc = fts3EvalPhraseLoad(pCsr, p); - p->bIncr = 0; + sqlite3_free(zSql); } - assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr ); return rc; } -/* -** This function is used to iterate backwards (from the end to start) -** through doclists. It is used by this module to iterate through phrase -** doclists in reverse and by the fts3_write.c module to iterate through -** pending-terms lists when writing to databases with "order=desc". -** -** The doclist may be sorted in ascending (parameter bDescIdx==0) or -** descending (parameter bDescIdx==1) order of docid. Regardless, this -** function iterates from the end of the doclist to the beginning. -*/ -SQLITE_PRIVATE void sqlite3Fts3DoclistPrev( - int bDescIdx, /* True if the doclist is desc */ - char *aDoclist, /* Pointer to entire doclist */ - int nDoclist, /* Length of aDoclist in bytes */ - char **ppIter, /* IN/OUT: Iterator pointer */ - sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ - int *pnList, /* OUT: List length pointer */ - u8 *pbEof /* OUT: End-of-file flag */ +static sqlite3_module rtreeModule = { + 0, /* iVersion */ + rtreeCreate, /* xCreate - create a table */ + rtreeConnect, /* xConnect - connect to an existing table */ + rtreeBestIndex, /* xBestIndex - Determine search strategy */ + rtreeDisconnect, /* xDisconnect - Disconnect from a table */ + rtreeDestroy, /* xDestroy - Drop a table */ + rtreeOpen, /* xOpen - open a cursor */ + rtreeClose, /* xClose - close a cursor */ + rtreeFilter, /* xFilter - configure scan constraints */ + rtreeNext, /* xNext - advance a cursor */ + rtreeEof, /* xEof */ + rtreeColumn, /* xColumn - read data */ + rtreeRowid, /* xRowid - read data */ + rtreeUpdate, /* xUpdate - write data */ + 0, /* xBegin - begin transaction */ + 0, /* xSync - sync transaction */ + 0, /* xCommit - commit transaction */ + 0, /* xRollback - rollback transaction */ + 0, /* xFindFunction - function overloading */ + rtreeRename, /* xRename - rename the table */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ +}; + +static int rtreeSqlInit( + Rtree *pRtree, + sqlite3 *db, + const char *zDb, + const char *zPrefix, + int isCreate ){ - char *p = *ppIter; + int rc = SQLITE_OK; - assert( nDoclist>0 ); - assert( *pbEof==0 ); - assert( p || *piDocid==0 ); - assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) ); + #define N_STATEMENT 9 + static const char *azSql[N_STATEMENT] = { + /* Read and write the xxx_node table */ + "SELECT data FROM '%q'.'%q_node' WHERE nodeno = :1", + "INSERT OR REPLACE INTO '%q'.'%q_node' VALUES(:1, :2)", + "DELETE FROM '%q'.'%q_node' WHERE nodeno = :1", - if( p==0 ){ - sqlite3_int64 iDocid = 0; - char *pNext = 0; - char *pDocid = aDoclist; - char *pEnd = &aDoclist[nDoclist]; - int iMul = 1; + /* Read and write the xxx_rowid table */ + "SELECT nodeno FROM '%q'.'%q_rowid' WHERE rowid = :1", + "INSERT OR REPLACE INTO '%q'.'%q_rowid' VALUES(:1, :2)", + "DELETE FROM '%q'.'%q_rowid' WHERE rowid = :1", - while( pDociddb = db; - if( p==aDoclist ){ - *pbEof = 1; - }else{ - char *pSave = p; - fts3ReversePoslist(aDoclist, &p); - *pnList = (int)(pSave - p); + if( isCreate ){ + char *zCreate = sqlite3_mprintf( +"CREATE TABLE \"%w\".\"%w_node\"(nodeno INTEGER PRIMARY KEY, data BLOB);" +"CREATE TABLE \"%w\".\"%w_rowid\"(rowid INTEGER PRIMARY KEY, nodeno INTEGER);" +"CREATE TABLE \"%w\".\"%w_parent\"(nodeno INTEGER PRIMARY KEY," + " parentnode INTEGER);" +"INSERT INTO '%q'.'%q_node' VALUES(1, zeroblob(%d))", + zDb, zPrefix, zDb, zPrefix, zDb, zPrefix, zDb, zPrefix, pRtree->iNodeSize + ); + if( !zCreate ){ + return SQLITE_NOMEM; + } + rc = sqlite3_exec(db, zCreate, 0, 0, 0); + sqlite3_free(zCreate); + if( rc!=SQLITE_OK ){ + return rc; } - *ppIter = p; } -} - -/* -** Iterate forwards through a doclist. -*/ -SQLITE_PRIVATE void sqlite3Fts3DoclistNext( - int bDescIdx, /* True if the doclist is desc */ - char *aDoclist, /* Pointer to entire doclist */ - int nDoclist, /* Length of aDoclist in bytes */ - char **ppIter, /* IN/OUT: Iterator pointer */ - sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ - u8 *pbEof /* OUT: End-of-file flag */ -){ - char *p = *ppIter; - assert( nDoclist>0 ); - assert( *pbEof==0 ); - assert( p || *piDocid==0 ); - assert( !p || (p>=aDoclist && p<=&aDoclist[nDoclist]) ); + appStmt[0] = &pRtree->pReadNode; + appStmt[1] = &pRtree->pWriteNode; + appStmt[2] = &pRtree->pDeleteNode; + appStmt[3] = &pRtree->pReadRowid; + appStmt[4] = &pRtree->pWriteRowid; + appStmt[5] = &pRtree->pDeleteRowid; + appStmt[6] = &pRtree->pReadParent; + appStmt[7] = &pRtree->pWriteParent; + appStmt[8] = &pRtree->pDeleteParent; - if( p==0 ){ - p = aDoclist; - p += sqlite3Fts3GetVarint(p, piDocid); - }else{ - fts3PoslistCopy(0, &p); - if( p>=&aDoclist[nDoclist] ){ - *pbEof = 1; + rc = rtreeQueryStat1(db, pRtree); + for(i=0; iaAll/nAll. Set *pbEof -** to true if EOF is reached. +** The second argument to this function contains the text of an SQL statement +** that returns a single integer value. The statement is compiled and executed +** using database connection db. If successful, the integer value returned +** is written to *piVal and SQLITE_OK returned. Otherwise, an SQLite error +** code is returned and the value of *piVal after returning is not defined. */ -static void fts3EvalDlPhraseNext( - Fts3Table *pTab, - Fts3Doclist *pDL, - u8 *pbEof -){ - char *pIter; /* Used to iterate through aAll */ - char *pEnd = &pDL->aAll[pDL->nAll]; /* 1 byte past end of aAll */ - - if( pDL->pNextDocid ){ - pIter = pDL->pNextDocid; - }else{ - pIter = pDL->aAll; - } - - if( pIter>=pEnd ){ - /* We have already reached the end of this doclist. EOF. */ - *pbEof = 1; - }else{ - sqlite3_int64 iDelta; - pIter += sqlite3Fts3GetVarint(pIter, &iDelta); - if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){ - pDL->iDocid += iDelta; - }else{ - pDL->iDocid -= iDelta; +static int getIntFromStmt(sqlite3 *db, const char *zSql, int *piVal){ + int rc = SQLITE_NOMEM; + if( zSql ){ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *piVal = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_finalize(pStmt); } - pDL->pList = pIter; - fts3PoslistCopy(0, &pIter); - pDL->nList = (int)(pIter - pDL->pList); - - /* pIter now points just past the 0x00 that terminates the position- - ** list for document pDL->iDocid. However, if this position-list was - ** edited in place by fts3EvalNearTrim(), then pIter may not actually - ** point to the start of the next docid value. The following line deals - ** with this case by advancing pIter past the zero-padding added by - ** fts3EvalNearTrim(). */ - while( pIterpNextDocid = pIter; - assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter ); - *pbEof = 0; } + return rc; } /* -** Helper type used by fts3EvalIncrPhraseNext() and incrPhraseTokenNext(). -*/ -typedef struct TokenDoclist TokenDoclist; -struct TokenDoclist { - int bIgnore; - sqlite3_int64 iDocid; - char *pList; - int nList; -}; - -/* -** Token pToken is an incrementally loaded token that is part of a -** multi-token phrase. Advance it to the next matching document in the -** database and populate output variable *p with the details of the new -** entry. Or, if the iterator has reached EOF, set *pbEof to true. +** This function is called from within the xConnect() or xCreate() method to +** determine the node-size used by the rtree table being created or connected +** to. If successful, pRtree->iNodeSize is populated and SQLITE_OK returned. +** Otherwise, an SQLite error code is returned. ** -** If an error occurs, return an SQLite error code. Otherwise, return -** SQLITE_OK. +** If this function is being called as part of an xConnect(), then the rtree +** table already exists. In this case the node-size is determined by inspecting +** the root node of the tree. +** +** Otherwise, for an xCreate(), use 64 bytes less than the database page-size. +** This ensures that each node is stored on a single database page. If the +** database page-size is so large that more than RTREE_MAXCELLS entries +** would fit in a single node, use a smaller node-size. */ -static int incrPhraseTokenNext( - Fts3Table *pTab, /* Virtual table handle */ - Fts3Phrase *pPhrase, /* Phrase to advance token of */ - int iToken, /* Specific token to advance */ - TokenDoclist *p, /* OUT: Docid and doclist for new entry */ - u8 *pbEof /* OUT: True if iterator is at EOF */ +static int getNodeSize( + sqlite3 *db, /* Database handle */ + Rtree *pRtree, /* Rtree handle */ + int isCreate, /* True for xCreate, false for xConnect */ + char **pzErr /* OUT: Error message, if any */ ){ - int rc = SQLITE_OK; - - if( pPhrase->iDoclistToken==iToken ){ - assert( p->bIgnore==0 ); - assert( pPhrase->aToken[iToken].pSegcsr==0 ); - fts3EvalDlPhraseNext(pTab, &pPhrase->doclist, pbEof); - p->pList = pPhrase->doclist.pList; - p->nList = pPhrase->doclist.nList; - p->iDocid = pPhrase->doclist.iDocid; - }else{ - Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; - assert( pToken->pDeferred==0 ); - assert( pToken->pSegcsr || pPhrase->iDoclistToken>=0 ); - if( pToken->pSegcsr ){ - assert( p->bIgnore==0 ); - rc = sqlite3Fts3MsrIncrNext( - pTab, pToken->pSegcsr, &p->iDocid, &p->pList, &p->nList - ); - if( p->pList==0 ) *pbEof = 1; + int rc; + char *zSql; + if( isCreate ){ + int iPageSize = 0; + zSql = sqlite3_mprintf("PRAGMA %Q.page_size", pRtree->zDb); + rc = getIntFromStmt(db, zSql, &iPageSize); + if( rc==SQLITE_OK ){ + pRtree->iNodeSize = iPageSize-64; + if( (4+pRtree->nBytesPerCell*RTREE_MAXCELLS)iNodeSize ){ + pRtree->iNodeSize = 4+pRtree->nBytesPerCell*RTREE_MAXCELLS; + } }else{ - p->bIgnore = 1; + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } + }else{ + zSql = sqlite3_mprintf( + "SELECT length(data) FROM '%q'.'%q_node' WHERE nodeno = 1", + pRtree->zDb, pRtree->zName + ); + rc = getIntFromStmt(db, zSql, &pRtree->iNodeSize); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); } } + sqlite3_free(zSql); return rc; } - -/* -** The phrase iterator passed as the second argument: -** -** * features at least one token that uses an incremental doclist, and -** -** * does not contain any deferred tokens. -** -** Advance it to the next matching documnent in the database and populate -** the Fts3Doclist.pList and nList fields. -** -** If there is no "next" entry and no error occurs, then *pbEof is set to -** 1 before returning. Otherwise, if no error occurs and the iterator is -** successfully advanced, *pbEof is set to 0. +/* +** This function is the implementation of both the xConnect and xCreate +** methods of the r-tree virtual table. ** -** If an error occurs, return an SQLite error code. Otherwise, return -** SQLITE_OK. +** argv[0] -> module name +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> column names... */ -static int fts3EvalIncrPhraseNext( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Phrase *p, /* Phrase object to advance to next docid */ - u8 *pbEof /* OUT: Set to 1 if EOF */ +static int rtreeInit( + sqlite3 *db, /* Database connection */ + void *pAux, /* One of the RTREE_COORD_* constants */ + int argc, const char *const*argv, /* Parameters to CREATE TABLE statement */ + sqlite3_vtab **ppVtab, /* OUT: New virtual table */ + char **pzErr, /* OUT: Error message, if any */ + int isCreate /* True for xCreate, false for xConnect */ ){ int rc = SQLITE_OK; - Fts3Doclist *pDL = &p->doclist; - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - u8 bEof = 0; + Rtree *pRtree; + int nDb; /* Length of string argv[1] */ + int nName; /* Length of string argv[2] */ + int eCoordType = (pAux ? RTREE_COORD_INT32 : RTREE_COORD_REAL32); - /* This is only called if it is guaranteed that the phrase has at least - ** one incremental token. In which case the bIncr flag is set. */ - assert( p->bIncr==1 ); + const char *aErrMsg[] = { + 0, /* 0 */ + "Wrong number of columns for an rtree table", /* 1 */ + "Too few columns for an rtree table", /* 2 */ + "Too many columns for an rtree table" /* 3 */ + }; - if( p->nToken==1 && p->bIncr ){ - rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, - &pDL->iDocid, &pDL->pList, &pDL->nList - ); - if( pDL->pList==0 ) bEof = 1; - }else{ - int bDescDoclist = pCsr->bDesc; - struct TokenDoclist a[MAX_INCR_PHRASE_TOKENS]; + int iErr = (argc<6) ? 2 : argc>(RTREE_MAX_DIMENSIONS*2+4) ? 3 : argc%2; + if( aErrMsg[iErr] ){ + *pzErr = sqlite3_mprintf("%s", aErrMsg[iErr]); + return SQLITE_ERROR; + } - memset(a, 0, sizeof(a)); - assert( p->nToken<=MAX_INCR_PHRASE_TOKENS ); - assert( p->iDoclistTokennBusy = 1; + pRtree->base.pModule = &rtreeModule; + pRtree->zDb = (char *)&pRtree[1]; + pRtree->zName = &pRtree->zDb[nDb+1]; + pRtree->nDim = (argc-4)/2; + pRtree->nBytesPerCell = 8 + pRtree->nDim*4*2; + pRtree->eCoordType = eCoordType; + memcpy(pRtree->zDb, argv[1], nDb); + memcpy(pRtree->zName, argv[2], nName); - /* Advance the iterator for each token in the phrase once. */ - for(i=0; rc==SQLITE_OK && inToken && bEof==0; i++){ - rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); - if( a[i].bIgnore==0 && (bMaxSet==0 || DOCID_CMP(iMax, a[i].iDocid)<0) ){ - iMax = a[i].iDocid; - bMaxSet = 1; - } - } - assert( rc!=SQLITE_OK || (p->nToken>=1 && a[p->nToken-1].bIgnore==0) ); - assert( rc!=SQLITE_OK || bMaxSet ); + /* Figure out the node size to use. */ + rc = getNodeSize(db, pRtree, isCreate, pzErr); - /* Keep advancing iterators until they all point to the same document */ - for(i=0; inToken; i++){ - while( rc==SQLITE_OK && bEof==0 - && a[i].bIgnore==0 && DOCID_CMP(a[i].iDocid, iMax)<0 - ){ - rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); - if( DOCID_CMP(a[i].iDocid, iMax)>0 ){ - iMax = a[i].iDocid; - i = 0; - } - } + /* Create/Connect to the underlying relational database schema. If + ** that is successful, call sqlite3_declare_vtab() to configure + ** the r-tree table schema. + */ + if( rc==SQLITE_OK ){ + if( (rc = rtreeSqlInit(pRtree, db, argv[1], argv[2], isCreate)) ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + }else{ + char *zSql = sqlite3_mprintf("CREATE TABLE x(%s", argv[3]); + char *zTmp; + int ii; + for(ii=4; zSql && iinToken-1].nList; - char *aDoclist = sqlite3_malloc(nByte+1); - if( !aDoclist ) return SQLITE_NOMEM; - memcpy(aDoclist, a[p->nToken-1].pList, nByte+1); - - for(i=0; i<(p->nToken-1); i++){ - if( a[i].bIgnore==0 ){ - char *pL = a[i].pList; - char *pR = aDoclist; - char *pOut = aDoclist; - int nDist = p->nToken-1-i; - int res = fts3PoslistPhraseMerge(&pOut, nDist, 0, 1, &pL, &pR); - if( res==0 ) break; - nList = (int)(pOut - aDoclist); - } - } - if( i==(p->nToken-1) ){ - pDL->iDocid = iMax; - pDL->pList = aDoclist; - pDL->nList = nList; - pDL->bFreeList = 1; - break; - } - sqlite3_free(aDoclist); + if( zSql ){ + zTmp = zSql; + zSql = sqlite3_mprintf("%s);", zTmp); + sqlite3_free(zTmp); + } + if( !zSql ){ + rc = SQLITE_NOMEM; + }else if( SQLITE_OK!=(rc = sqlite3_declare_vtab(db, zSql)) ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); } + sqlite3_free(zSql); } } - *pbEof = bEof; - return rc; -} - -/* -** Attempt to move the phrase iterator to point to the next matching docid. -** If an error occurs, return an SQLite error code. Otherwise, return -** SQLITE_OK. -** -** If there is no "next" entry and no error occurs, then *pbEof is set to -** 1 before returning. Otherwise, if no error occurs and the iterator is -** successfully advanced, *pbEof is set to 0. -*/ -static int fts3EvalPhraseNext( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Phrase *p, /* Phrase object to advance to next docid */ - u8 *pbEof /* OUT: Set to 1 if EOF */ -){ - int rc = SQLITE_OK; - Fts3Doclist *pDL = &p->doclist; - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - - if( p->bIncr ){ - rc = fts3EvalIncrPhraseNext(pCsr, p, pbEof); - }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){ - sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, - &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof - ); - pDL->pList = pDL->pNextDocid; + if( rc==SQLITE_OK ){ + *ppVtab = (sqlite3_vtab *)pRtree; }else{ - fts3EvalDlPhraseNext(pTab, pDL, pbEof); + assert( *ppVtab==0 ); + assert( pRtree->nBusy==1 ); + rtreeRelease(pRtree); } - return rc; } -/* -** -** If *pRc is not SQLITE_OK when this function is called, it is a no-op. -** Otherwise, fts3EvalPhraseStart() is called on all phrases within the -** expression. Also the Fts3Expr.bDeferred variable is set to true for any -** expressions for which all descendent tokens are deferred. -** -** If parameter bOptOk is zero, then it is guaranteed that the -** Fts3Phrase.doclist.aAll/nAll variables contain the entire doclist for -** each phrase in the expression (subject to deferred token processing). -** Or, if bOptOk is non-zero, then one or more tokens within the expression -** may be loaded incrementally, meaning doclist.aAll/nAll is not available. -** -** If an error occurs within this function, *pRc is set to an SQLite error -** code before returning. -*/ -static void fts3EvalStartReaders( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Expr *pExpr, /* Expression to initialize phrases in */ - int *pRc /* IN/OUT: Error code */ -){ - if( pExpr && SQLITE_OK==*pRc ){ - if( pExpr->eType==FTSQUERY_PHRASE ){ - int nToken = pExpr->pPhrase->nToken; - if( nToken ){ - int i; - for(i=0; ipPhrase->aToken[i].pDeferred==0 ) break; - } - pExpr->bDeferred = (i==nToken); - } - *pRc = fts3EvalPhraseStart(pCsr, 1, pExpr->pPhrase); - }else{ - fts3EvalStartReaders(pCsr, pExpr->pLeft, pRc); - fts3EvalStartReaders(pCsr, pExpr->pRight, pRc); - pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred); - } - } -} /* -** An array of the following structures is assembled as part of the process -** of selecting tokens to defer before the query starts executing (as part -** of the xFilter() method). There is one element in the array for each -** token in the FTS expression. +** Implementation of a scalar function that decodes r-tree nodes to +** human readable strings. This can be used for debugging and analysis. ** -** Tokens are divided into AND/NEAR clusters. All tokens in a cluster belong -** to phrases that are connected only by AND and NEAR operators (not OR or -** NOT). When determining tokens to defer, each AND/NEAR cluster is considered -** separately. The root of a tokens AND/NEAR cluster is stored in -** Fts3TokenAndCost.pRoot. -*/ -typedef struct Fts3TokenAndCost Fts3TokenAndCost; -struct Fts3TokenAndCost { - Fts3Phrase *pPhrase; /* The phrase the token belongs to */ - int iToken; /* Position of token in phrase */ - Fts3PhraseToken *pToken; /* The token itself */ - Fts3Expr *pRoot; /* Root of NEAR/AND cluster */ - int nOvfl; /* Number of overflow pages to load doclist */ - int iCol; /* The column the token must match */ -}; - -/* -** This function is used to populate an allocated Fts3TokenAndCost array. +** The scalar function takes two arguments: (1) the number of dimensions +** to the rtree (between 1 and 5, inclusive) and (2) a blob of data containing +** an r-tree node. For a two-dimensional r-tree structure called "rt", to +** deserialize all nodes, a statement like: ** -** If *pRc is not SQLITE_OK when this function is called, it is a no-op. -** Otherwise, if an error occurs during execution, *pRc is set to an -** SQLite error code. -*/ -static void fts3EvalTokenCosts( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Expr *pRoot, /* Root of current AND/NEAR cluster */ - Fts3Expr *pExpr, /* Expression to consider */ - Fts3TokenAndCost **ppTC, /* Write new entries to *(*ppTC)++ */ - Fts3Expr ***ppOr, /* Write new OR root to *(*ppOr)++ */ - int *pRc /* IN/OUT: Error code */ -){ - if( *pRc==SQLITE_OK ){ - if( pExpr->eType==FTSQUERY_PHRASE ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - int i; - for(i=0; *pRc==SQLITE_OK && inToken; i++){ - Fts3TokenAndCost *pTC = (*ppTC)++; - pTC->pPhrase = pPhrase; - pTC->iToken = i; - pTC->pRoot = pRoot; - pTC->pToken = &pPhrase->aToken[i]; - pTC->iCol = pPhrase->iColumn; - *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl); - } - }else if( pExpr->eType!=FTSQUERY_NOT ){ - assert( pExpr->eType==FTSQUERY_OR - || pExpr->eType==FTSQUERY_AND - || pExpr->eType==FTSQUERY_NEAR - ); - assert( pExpr->pLeft && pExpr->pRight ); - if( pExpr->eType==FTSQUERY_OR ){ - pRoot = pExpr->pLeft; - **ppOr = pRoot; - (*ppOr)++; - } - fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc); - if( pExpr->eType==FTSQUERY_OR ){ - pRoot = pExpr->pRight; - **ppOr = pRoot; - (*ppOr)++; - } - fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc); - } - } -} - -/* -** Determine the average document (row) size in pages. If successful, -** write this value to *pnPage and return SQLITE_OK. Otherwise, return -** an SQLite error code. +** SELECT rtreenode(2, data) FROM rt_node; ** -** The average document size in pages is calculated by first calculating -** determining the average size in bytes, B. If B is less than the amount -** of data that will fit on a single leaf page of an intkey table in -** this database, then the average docsize is 1. Otherwise, it is 1 plus -** the number of overflow pages consumed by a record B bytes in size. +** The human readable string takes the form of a Tcl list with one +** entry for each cell in the r-tree node. Each entry is itself a +** list, containing the 8-byte rowid/pageno followed by the +** *2 coordinates. */ -static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ - if( pCsr->nRowAvg==0 ){ - /* The average document size, which is required to calculate the cost - ** of each doclist, has not yet been determined. Read the required - ** data from the %_stat table to calculate it. - ** - ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 - ** varints, where nCol is the number of columns in the FTS3 table. - ** The first varint is the number of documents currently stored in - ** the table. The following nCol varints contain the total amount of - ** data stored in all rows of each column of the table, from left - ** to right. - */ - int rc; - Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; - sqlite3_stmt *pStmt; - sqlite3_int64 nDoc = 0; - sqlite3_int64 nByte = 0; - const char *pEnd; - const char *a; +static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ + char *zText = 0; + RtreeNode node; + Rtree tree; + int ii; - rc = sqlite3Fts3SelectDoctotal(p, &pStmt); - if( rc!=SQLITE_OK ) return rc; - a = sqlite3_column_blob(pStmt, 0); - assert( a ); + UNUSED_PARAMETER(nArg); + memset(&node, 0, sizeof(RtreeNode)); + memset(&tree, 0, sizeof(Rtree)); + tree.nDim = sqlite3_value_int(apArg[0]); + tree.nBytesPerCell = 8 + 8 * tree.nDim; + node.zData = (u8 *)sqlite3_value_blob(apArg[1]); - pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; - a += sqlite3Fts3GetVarint(a, &nDoc); - while( anDoc = nDoc; - pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz); - assert( pCsr->nRowAvg>0 ); - rc = sqlite3_reset(pStmt); - if( rc!=SQLITE_OK ) return rc; + if( zText ){ + char *zTextNew = sqlite3_mprintf("%s {%s}", zText, zCell); + sqlite3_free(zText); + zText = zTextNew; + }else{ + zText = sqlite3_mprintf("{%s}", zCell); + } } - - *pnPage = pCsr->nRowAvg; - return SQLITE_OK; + + sqlite3_result_text(ctx, zText, -1, sqlite3_free); } -/* -** This function is called to select the tokens (if any) that will be -** deferred. The array aTC[] has already been populated when this is -** called. +/* This routine implements an SQL function that returns the "depth" parameter +** from the front of a blob that is an r-tree node. For example: ** -** This function is called once for each AND/NEAR cluster in the -** expression. Each invocation determines which tokens to defer within -** the cluster with root node pRoot. See comments above the definition -** of struct Fts3TokenAndCost for more details. +** SELECT rtreedepth(data) FROM rt_node WHERE nodeno=1; ** -** If no error occurs, SQLITE_OK is returned and sqlite3Fts3DeferToken() -** called on each token to defer. Otherwise, an SQLite error code is -** returned. +** The depth value is 0 for all nodes other than the root node, and the root +** node always has nodeno=1, so the example above is the primary use for this +** routine. This routine is intended for testing and analysis only. */ -static int fts3EvalSelectDeferred( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Expr *pRoot, /* Consider tokens with this root node */ - Fts3TokenAndCost *aTC, /* Array of expression tokens and costs */ - int nTC /* Number of entries in aTC[] */ -){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int nDocSize = 0; /* Number of pages per doc loaded */ - int rc = SQLITE_OK; /* Return code */ - int ii; /* Iterator variable for various purposes */ - int nOvfl = 0; /* Total overflow pages used by doclists */ - int nToken = 0; /* Total number of tokens in cluster */ - - int nMinEst = 0; /* The minimum count for any phrase so far. */ - int nLoad4 = 1; /* (Phrases that will be loaded)^4. */ - - /* Tokens are never deferred for FTS tables created using the content=xxx - ** option. The reason being that it is not guaranteed that the content - ** table actually contains the same data as the index. To prevent this from - ** causing any problems, the deferred token optimization is completely - ** disabled for content=xxx tables. */ - if( pTab->zContentTbl ){ - return SQLITE_OK; - } - - /* Count the tokens in this AND/NEAR cluster. If none of the doclists - ** associated with the tokens spill onto overflow pages, or if there is - ** only 1 token, exit early. No tokens to defer in this case. */ - for(ii=0; ii0 ); - - - /* Iterate through all tokens in this AND/NEAR cluster, in ascending order - ** of the number of overflow pages that will be loaded by the pager layer - ** to retrieve the entire doclist for the token from the full-text index. - ** Load the doclists for tokens that are either: - ** - ** a. The cheapest token in the entire query (i.e. the one visited by the - ** first iteration of this loop), or - ** - ** b. Part of a multi-token phrase. - ** - ** After each token doclist is loaded, merge it with the others from the - ** same phrase and count the number of documents that the merged doclist - ** contains. Set variable "nMinEst" to the smallest number of documents in - ** any phrase doclist for which 1 or more token doclists have been loaded. - ** Let nOther be the number of other phrases for which it is certain that - ** one or more tokens will not be deferred. - ** - ** Then, for each token, defer it if loading the doclist would result in - ** loading N or more overflow pages into memory, where N is computed as: - ** - ** (nMinEst + 4^nOther - 1) / (4^nOther) - */ - for(ii=0; iinOvfl) - ){ - pTC = &aTC[iTC]; - } - } - assert( pTC ); - - if( ii && pTC->nOvfl>=((nMinEst+(nLoad4/4)-1)/(nLoad4/4))*nDocSize ){ - /* The number of overflow pages to load for this (and therefore all - ** subsequent) tokens is greater than the estimated number of pages - ** that will be loaded if all subsequent tokens are deferred. - */ - Fts3PhraseToken *pToken = pTC->pToken; - rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); - fts3SegReaderCursorFree(pToken->pSegcsr); - pToken->pSegcsr = 0; - }else{ - /* Set nLoad4 to the value of (4^nOther) for the next iteration of the - ** for-loop. Except, limit the value to 2^24 to prevent it from - ** overflowing the 32-bit integer it is stored in. */ - if( ii<12 ) nLoad4 = nLoad4*4; +} - if( ii==0 || (pTC->pPhrase->nToken>1 && ii!=nToken-1) ){ - /* Either this is the cheapest token in the entire query, or it is - ** part of a multi-token phrase. Either way, the entire doclist will - ** (eventually) be loaded into memory. It may as well be now. */ - Fts3PhraseToken *pToken = pTC->pToken; - int nList = 0; - char *pList = 0; - rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList); - assert( rc==SQLITE_OK || pList==0 ); - if( rc==SQLITE_OK ){ - rc = fts3EvalPhraseMergeToken( - pTab, pTC->pPhrase, pTC->iToken,pList,nList - ); - } - if( rc==SQLITE_OK ){ - int nCount; - nCount = fts3DoclistCountDocids( - pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll - ); - if( ii==0 || nCountpToken = 0; +/* +** Register the r-tree module with database handle db. This creates the +** virtual table module "rtree" and the debugging/analysis scalar +** function "rtreenode". +*/ +SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db){ + const int utf8 = SQLITE_UTF8; + int rc; + + rc = sqlite3_create_function(db, "rtreenode", 2, utf8, 0, rtreenode, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "rtreedepth", 1, utf8, 0,rtreedepth, 0, 0); + } + if( rc==SQLITE_OK ){ +#ifdef SQLITE_RTREE_INT_ONLY + void *c = (void *)RTREE_COORD_INT32; +#else + void *c = (void *)RTREE_COORD_REAL32; +#endif + rc = sqlite3_create_module_v2(db, "rtree", &rtreeModule, c, 0); + } + if( rc==SQLITE_OK ){ + void *c = (void *)RTREE_COORD_INT32; + rc = sqlite3_create_module_v2(db, "rtree_i32", &rtreeModule, c, 0); } return rc; } /* -** This function is called from within the xFilter method. It initializes -** the full-text query currently stored in pCsr->pExpr. To iterate through -** the results of a query, the caller does: -** -** fts3EvalStart(pCsr); -** while( 1 ){ -** fts3EvalNext(pCsr); -** if( pCsr->bEof ) break; -** ... return row pCsr->iPrevId to the caller ... -** } +** This routine deletes the RtreeGeomCallback object that was attached +** one of the SQL functions create by sqlite3_rtree_geometry_callback() +** or sqlite3_rtree_query_callback(). In other words, this routine is the +** destructor for an RtreeGeomCallback objecct. This routine is called when +** the corresponding SQL function is deleted. */ -static int fts3EvalStart(Fts3Cursor *pCsr){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int rc = SQLITE_OK; - int nToken = 0; - int nOr = 0; +static void rtreeFreeCallback(void *p){ + RtreeGeomCallback *pInfo = (RtreeGeomCallback*)p; + if( pInfo->xDestructor ) pInfo->xDestructor(pInfo->pContext); + sqlite3_free(p); +} - /* Allocate a MultiSegReader for each token in the expression. */ - fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc); +/* +** This routine frees the BLOB that is returned by geomCallback(). +*/ +static void rtreeMatchArgFree(void *pArg){ + int i; + RtreeMatchArg *p = (RtreeMatchArg*)pArg; + for(i=0; inParam; i++){ + sqlite3_value_free(p->apSqlParam[i]); + } + sqlite3_free(p); +} - /* Determine which, if any, tokens in the expression should be deferred. */ -#ifndef SQLITE_DISABLE_FTS4_DEFERRED - if( rc==SQLITE_OK && nToken>1 && pTab->bFts4 ){ - Fts3TokenAndCost *aTC; - Fts3Expr **apOr; - aTC = (Fts3TokenAndCost *)sqlite3_malloc( - sizeof(Fts3TokenAndCost) * nToken - + sizeof(Fts3Expr *) * nOr * 2 - ); - apOr = (Fts3Expr **)&aTC[nToken]; +/* +** Each call to sqlite3_rtree_geometry_callback() or +** sqlite3_rtree_query_callback() creates an ordinary SQLite +** scalar function that is implemented by this routine. +** +** All this function does is construct an RtreeMatchArg object that +** contains the geometry-checking callback routines and a list of +** parameters to this function, then return that RtreeMatchArg object +** as a BLOB. +** +** The R-Tree MATCH operator will read the returned BLOB, deserialize +** the RtreeMatchArg object, and use the RtreeMatchArg object to figure +** out which elements of the R-Tree should be returned by the query. +*/ +static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ + RtreeGeomCallback *pGeomCtx = (RtreeGeomCallback *)sqlite3_user_data(ctx); + RtreeMatchArg *pBlob; + int nBlob; + int memErr = 0; - if( !aTC ){ - rc = SQLITE_NOMEM; + nBlob = sizeof(RtreeMatchArg) + (nArg-1)*sizeof(RtreeDValue) + + nArg*sizeof(sqlite3_value*); + pBlob = (RtreeMatchArg *)sqlite3_malloc(nBlob); + if( !pBlob ){ + sqlite3_result_error_nomem(ctx); + }else{ + int i; + pBlob->magic = RTREE_GEOMETRY_MAGIC; + pBlob->cb = pGeomCtx[0]; + pBlob->apSqlParam = (sqlite3_value**)&pBlob->aParam[nArg]; + pBlob->nParam = nArg; + for(i=0; iapSqlParam[i] = sqlite3_value_dup(aArg[i]); + if( pBlob->apSqlParam[i]==0 ) memErr = 1; +#ifdef SQLITE_RTREE_INT_ONLY + pBlob->aParam[i] = sqlite3_value_int64(aArg[i]); +#else + pBlob->aParam[i] = sqlite3_value_double(aArg[i]); +#endif + } + if( memErr ){ + sqlite3_result_error_nomem(ctx); + rtreeMatchArgFree(pBlob); }else{ - int ii; - Fts3TokenAndCost *pTC = aTC; - Fts3Expr **ppOr = apOr; - - fts3EvalTokenCosts(pCsr, 0, pCsr->pExpr, &pTC, &ppOr, &rc); - nToken = (int)(pTC-aTC); - nOr = (int)(ppOr-apOr); - - if( rc==SQLITE_OK ){ - rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); - for(ii=0; rc==SQLITE_OK && iipExpr, &rc); - return rc; } /* -** Invalidate the current position list for phrase pPhrase. +** Register a new geometry function for use with the r-tree MATCH operator. */ -static void fts3EvalInvalidatePoslist(Fts3Phrase *pPhrase){ - if( pPhrase->doclist.bFreeList ){ - sqlite3_free(pPhrase->doclist.pList); - } - pPhrase->doclist.pList = 0; - pPhrase->doclist.nList = 0; - pPhrase->doclist.bFreeList = 0; +SQLITE_API int SQLITE_STDCALL sqlite3_rtree_geometry_callback( + sqlite3 *db, /* Register SQL function on this connection */ + const char *zGeom, /* Name of the new SQL function */ + int (*xGeom)(sqlite3_rtree_geometry*,int,RtreeDValue*,int*), /* Callback */ + void *pContext /* Extra data associated with the callback */ +){ + RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ + + /* Allocate and populate the context object. */ + pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); + if( !pGeomCtx ) return SQLITE_NOMEM; + pGeomCtx->xGeom = xGeom; + pGeomCtx->xQueryFunc = 0; + pGeomCtx->xDestructor = 0; + pGeomCtx->pContext = pContext; + return sqlite3_create_function_v2(db, zGeom, -1, SQLITE_ANY, + (void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback + ); } /* -** This function is called to edit the position list associated with -** the phrase object passed as the fifth argument according to a NEAR -** condition. For example: -** -** abc NEAR/5 "def ghi" -** -** Parameter nNear is passed the NEAR distance of the expression (5 in -** the example above). When this function is called, *paPoslist points to -** the position list, and *pnToken is the number of phrase tokens in, the -** phrase on the other side of the NEAR operator to pPhrase. For example, -** if pPhrase refers to the "def ghi" phrase, then *paPoslist points to -** the position list associated with phrase "abc". -** -** All positions in the pPhrase position list that are not sufficiently -** close to a position in the *paPoslist position list are removed. If this -** leaves 0 positions, zero is returned. Otherwise, non-zero. -** -** Before returning, *paPoslist is set to point to the position lsit -** associated with pPhrase. And *pnToken is set to the number of tokens in -** pPhrase. +** Register a new 2nd-generation geometry function for use with the +** r-tree MATCH operator. */ -static int fts3EvalNearTrim( - int nNear, /* NEAR distance. As in "NEAR/nNear". */ - char *aTmp, /* Temporary space to use */ - char **paPoslist, /* IN/OUT: Position list */ - int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */ - Fts3Phrase *pPhrase /* The phrase object to trim the doclist of */ +SQLITE_API int SQLITE_STDCALL sqlite3_rtree_query_callback( + sqlite3 *db, /* Register SQL function on this connection */ + const char *zQueryFunc, /* Name of new SQL function */ + int (*xQueryFunc)(sqlite3_rtree_query_info*), /* Callback */ + void *pContext, /* Extra data passed into the callback */ + void (*xDestructor)(void*) /* Destructor for the extra data */ ){ - int nParam1 = nNear + pPhrase->nToken; - int nParam2 = nNear + *pnToken; - int nNew; - char *p2; - char *pOut; - int res; - - assert( pPhrase->doclist.pList ); + RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ - p2 = pOut = pPhrase->doclist.pList; - res = fts3PoslistNearMerge( - &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 + /* Allocate and populate the context object. */ + pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); + if( !pGeomCtx ) return SQLITE_NOMEM; + pGeomCtx->xGeom = 0; + pGeomCtx->xQueryFunc = xQueryFunc; + pGeomCtx->xDestructor = xDestructor; + pGeomCtx->pContext = pContext; + return sqlite3_create_function_v2(db, zQueryFunc, -1, SQLITE_ANY, + (void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback ); - if( res ){ - nNew = (int)(pOut - pPhrase->doclist.pList) - 1; - assert( pPhrase->doclist.pList[nNew]=='\0' ); - assert( nNew<=pPhrase->doclist.nList && nNew>0 ); - memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew); - pPhrase->doclist.nList = nNew; - *paPoslist = pPhrase->doclist.pList; - *pnToken = pPhrase->nToken; - } +} - return res; +#if !SQLITE_CORE +#ifdef _WIN32 +__declspec(dllexport) +#endif +SQLITE_API int SQLITE_STDCALL sqlite3_rtree_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi) + return sqlite3RtreeInit(db); } +#endif + +#endif +/************** End of rtree.c ***********************************************/ +/************** Begin file icu.c *********************************************/ /* -** This function is a no-op if *pRc is other than SQLITE_OK when it is called. -** Otherwise, it advances the expression passed as the second argument to -** point to the next matching row in the database. Expressions iterate through -** matching rows in docid order. Ascending order if Fts3Cursor.bDesc is zero, -** or descending if it is non-zero. +** 2007 May 6 ** -** If an error occurs, *pRc is set to an SQLite error code. Otherwise, if -** successful, the following variables in pExpr are set: +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** Fts3Expr.bEof (non-zero if EOF - there is no next row) -** Fts3Expr.iDocid (valid if bEof==0. The docid of the next row) +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** -** If the expression is of type FTSQUERY_PHRASE, and the expression is not -** at EOF, then the following variables are populated with the position list -** for the phrase for the visited row: +************************************************************************* +** $Id: icu.c,v 1.7 2007/12/13 21:54:11 drh Exp $ ** -** FTs3Expr.pPhrase->doclist.nList (length of pList in bytes) -** FTs3Expr.pPhrase->doclist.pList (pointer to position list) +** This file implements an integration between the ICU library +** ("International Components for Unicode", an open-source library +** for handling unicode data) and SQLite. The integration uses +** ICU to provide the following to SQLite: ** -** It says above that this function advances the expression to the next -** matching row. This is usually true, but there are the following exceptions: +** * An implementation of the SQL regexp() function (and hence REGEXP +** operator) using the ICU uregex_XX() APIs. ** -** 1. Deferred tokens are not taken into account. If a phrase consists -** entirely of deferred tokens, it is assumed to match every row in -** the db. In this case the position-list is not populated at all. +** * Implementations of the SQL scalar upper() and lower() functions +** for case mapping. ** -** Or, if a phrase contains one or more deferred tokens and one or -** more non-deferred tokens, then the expression is advanced to the -** next possible match, considering only non-deferred tokens. In other -** words, if the phrase is "A B C", and "B" is deferred, the expression -** is advanced to the next row that contains an instance of "A * C", -** where "*" may match any single token. The position list in this case -** is populated as for "A * C" before returning. +** * Integration of ICU and SQLite collation sequences. ** -** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is -** advanced to point to the next row that matches "x AND y". -** -** See fts3EvalTestDeferredAndNear() for details on testing if a row is -** really a match, taking into account deferred tokens and NEAR operators. +** * An implementation of the LIKE operator that uses ICU to +** provide case-independent matching. */ -static void fts3EvalNextRow( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Expr *pExpr, /* Expr. to advance to next matching row */ - int *pRc /* IN/OUT: Error code */ -){ - if( *pRc==SQLITE_OK ){ - int bDescDoclist = pCsr->bDesc; /* Used by DOCID_CMP() macro */ - assert( pExpr->bEof==0 ); - pExpr->bStart = 1; - - switch( pExpr->eType ){ - case FTSQUERY_NEAR: - case FTSQUERY_AND: { - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; - assert( !pLeft->bDeferred || !pRight->bDeferred ); - - if( pLeft->bDeferred ){ - /* LHS is entirely deferred. So we assume it matches every row. - ** Advance the RHS iterator to find the next row visited. */ - fts3EvalNextRow(pCsr, pRight, pRc); - pExpr->iDocid = pRight->iDocid; - pExpr->bEof = pRight->bEof; - }else if( pRight->bDeferred ){ - /* RHS is entirely deferred. So we assume it matches every row. - ** Advance the LHS iterator to find the next row visited. */ - fts3EvalNextRow(pCsr, pLeft, pRc); - pExpr->iDocid = pLeft->iDocid; - pExpr->bEof = pLeft->bEof; - }else{ - /* Neither the RHS or LHS are deferred. */ - fts3EvalNextRow(pCsr, pLeft, pRc); - fts3EvalNextRow(pCsr, pRight, pRc); - while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ - sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); - if( iDiff==0 ) break; - if( iDiff<0 ){ - fts3EvalNextRow(pCsr, pLeft, pRc); - }else{ - fts3EvalNextRow(pCsr, pRight, pRc); - } - } - pExpr->iDocid = pLeft->iDocid; - pExpr->bEof = (pLeft->bEof || pRight->bEof); - if( pExpr->eType==FTSQUERY_NEAR && pExpr->bEof ){ - if( pRight->pPhrase && pRight->pPhrase->doclist.aAll ){ - Fts3Doclist *pDl = &pRight->pPhrase->doclist; - while( *pRc==SQLITE_OK && pRight->bEof==0 ){ - memset(pDl->pList, 0, pDl->nList); - fts3EvalNextRow(pCsr, pRight, pRc); - } - } - if( pLeft->pPhrase && pLeft->pPhrase->doclist.aAll ){ - Fts3Doclist *pDl = &pLeft->pPhrase->doclist; - while( *pRc==SQLITE_OK && pLeft->bEof==0 ){ - memset(pDl->pList, 0, pDl->nList); - fts3EvalNextRow(pCsr, pLeft, pRc); - } - } - } - } - break; - } - - case FTSQUERY_OR: { - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; - sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); - - assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); - assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); - - if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ - fts3EvalNextRow(pCsr, pLeft, pRc); - }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){ - fts3EvalNextRow(pCsr, pRight, pRc); - }else{ - fts3EvalNextRow(pCsr, pLeft, pRc); - fts3EvalNextRow(pCsr, pRight, pRc); - } - pExpr->bEof = (pLeft->bEof && pRight->bEof); - iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); - if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ - pExpr->iDocid = pLeft->iDocid; - }else{ - pExpr->iDocid = pRight->iDocid; - } +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) - break; - } +/* Include ICU headers */ +#include +#include +#include +#include - case FTSQUERY_NOT: { - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; +/* #include */ - if( pRight->bStart==0 ){ - fts3EvalNextRow(pCsr, pRight, pRc); - assert( *pRc!=SQLITE_OK || pRight->bStart ); - } +#ifndef SQLITE_CORE +/* #include "sqlite3ext.h" */ + SQLITE_EXTENSION_INIT1 +#else +/* #include "sqlite3.h" */ +#endif - fts3EvalNextRow(pCsr, pLeft, pRc); - if( pLeft->bEof==0 ){ - while( !*pRc - && !pRight->bEof - && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 - ){ - fts3EvalNextRow(pCsr, pRight, pRc); - } - } - pExpr->iDocid = pLeft->iDocid; - pExpr->bEof = pLeft->bEof; - break; - } +/* +** Maximum length (in bytes) of the pattern in a LIKE or GLOB +** operator. +*/ +#ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH +# define SQLITE_MAX_LIKE_PATTERN_LENGTH 50000 +#endif - default: { - Fts3Phrase *pPhrase = pExpr->pPhrase; - fts3EvalInvalidatePoslist(pPhrase); - *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); - pExpr->iDocid = pPhrase->doclist.iDocid; - break; - } - } - } +/* +** Version of sqlite3_free() that is always a function, never a macro. +*/ +static void xFree(void *p){ + sqlite3_free(p); } /* -** If *pRc is not SQLITE_OK, or if pExpr is not the root node of a NEAR -** cluster, then this function returns 1 immediately. -** -** Otherwise, it checks if the current row really does match the NEAR -** expression, using the data currently stored in the position lists -** (Fts3Expr->pPhrase.doclist.pList/nList) for each phrase in the expression. -** -** If the current row is a match, the position list associated with each -** phrase in the NEAR expression is edited in place to contain only those -** phrase instances sufficiently close to their peers to satisfy all NEAR -** constraints. In this case it returns 1. If the NEAR expression does not -** match the current row, 0 is returned. The position lists may or may not -** be edited if 0 is returned. +** Compare two UTF-8 strings for equality where the first string is +** a "LIKE" expression. Return true (1) if they are the same and +** false (0) if they are different. */ -static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ - int res = 1; - - /* The following block runs if pExpr is the root of a NEAR query. - ** For example, the query: - ** - ** "w" NEAR "x" NEAR "y" NEAR "z" - ** - ** which is represented in tree form as: - ** - ** | - ** +--NEAR--+ <-- root of NEAR query - ** | | - ** +--NEAR--+ "z" - ** | | - ** +--NEAR--+ "y" - ** | | - ** "w" "x" - ** - ** The right-hand child of a NEAR node is always a phrase. The - ** left-hand child may be either a phrase or a NEAR node. There are - ** no exceptions to this - it's the way the parser in fts3_expr.c works. - */ - if( *pRc==SQLITE_OK - && pExpr->eType==FTSQUERY_NEAR - && pExpr->bEof==0 - && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) - ){ - Fts3Expr *p; - int nTmp = 0; /* Bytes of temp space */ - char *aTmp; /* Temp space for PoslistNearMerge() */ - - /* Allocate temporary working space. */ - for(p=pExpr; p->pLeft; p=p->pLeft){ - nTmp += p->pRight->pPhrase->doclist.nList; - } - nTmp += p->pPhrase->doclist.nList; - if( nTmp==0 ){ - res = 0; - }else{ - aTmp = sqlite3_malloc(nTmp*2); - if( !aTmp ){ - *pRc = SQLITE_NOMEM; - res = 0; - }else{ - char *aPoslist = p->pPhrase->doclist.pList; - int nToken = p->pPhrase->nToken; +static int icuLikeCompare( + const uint8_t *zPattern, /* LIKE pattern */ + const uint8_t *zString, /* The UTF-8 string to compare against */ + const UChar32 uEsc /* The escape character */ +){ + static const int MATCH_ONE = (UChar32)'_'; + static const int MATCH_ALL = (UChar32)'%'; - for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ - Fts3Phrase *pPhrase = p->pRight->pPhrase; - int nNear = p->nNear; - res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); - } + int iPattern = 0; /* Current byte index in zPattern */ + int iString = 0; /* Current byte index in zString */ - aPoslist = pExpr->pRight->pPhrase->doclist.pList; - nToken = pExpr->pRight->pPhrase->nToken; - for(p=pExpr->pLeft; p && res; p=p->pLeft){ - int nNear; - Fts3Phrase *pPhrase; - assert( p->pParent && p->pParent->pLeft==p ); - nNear = p->pParent->nNear; - pPhrase = ( - p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase - ); - res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); - } - } + int prevEscape = 0; /* True if the previous character was uEsc */ - sqlite3_free(aTmp); - } - } + while( zPattern[iPattern]!=0 ){ - return res; -} + /* Read (and consume) the next character from the input pattern. */ + UChar32 uPattern; + U8_NEXT_UNSAFE(zPattern, iPattern, uPattern); -/* -** This function is a helper function for fts3EvalTestDeferredAndNear(). -** Assuming no error occurs or has occurred, It returns non-zero if the -** expression passed as the second argument matches the row that pCsr -** currently points to, or zero if it does not. -** -** If *pRc is not SQLITE_OK when this function is called, it is a no-op. -** If an error occurs during execution of this function, *pRc is set to -** the appropriate SQLite error code. In this case the returned value is -** undefined. -*/ -static int fts3EvalTestExpr( - Fts3Cursor *pCsr, /* FTS cursor handle */ - Fts3Expr *pExpr, /* Expr to test. May or may not be root. */ - int *pRc /* IN/OUT: Error code */ -){ - int bHit = 1; /* Return value */ - if( *pRc==SQLITE_OK ){ - switch( pExpr->eType ){ - case FTSQUERY_NEAR: - case FTSQUERY_AND: - bHit = ( - fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) - && fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) - && fts3EvalNearTest(pExpr, pRc) - ); + /* There are now 4 possibilities: + ** + ** 1. uPattern is an unescaped match-all character "%", + ** 2. uPattern is an unescaped match-one character "_", + ** 3. uPattern is an unescaped escape character, or + ** 4. uPattern is to be handled as an ordinary character + */ + if( !prevEscape && uPattern==MATCH_ALL ){ + /* Case 1. */ + uint8_t c; - /* If the NEAR expression does not match any rows, zero the doclist for - ** all phrases involved in the NEAR. This is because the snippet(), - ** offsets() and matchinfo() functions are not supposed to recognize - ** any instances of phrases that are part of unmatched NEAR queries. - ** For example if this expression: - ** - ** ... MATCH 'a OR (b NEAR c)' - ** - ** is matched against a row containing: - ** - ** 'a b d e' - ** - ** then any snippet() should ony highlight the "a" term, not the "b" - ** (as "b" is part of a non-matching NEAR clause). - */ - if( bHit==0 - && pExpr->eType==FTSQUERY_NEAR - && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) - ){ - Fts3Expr *p; - for(p=pExpr; p->pPhrase==0; p=p->pLeft){ - if( p->pRight->iDocid==pCsr->iPrevId ){ - fts3EvalInvalidatePoslist(p->pRight->pPhrase); - } - } - if( p->iDocid==pCsr->iPrevId ){ - fts3EvalInvalidatePoslist(p->pPhrase); - } + /* Skip any MATCH_ALL or MATCH_ONE characters that follow a + ** MATCH_ALL. For each MATCH_ONE, skip one character in the + ** test string. + */ + while( (c=zPattern[iPattern]) == MATCH_ALL || c == MATCH_ONE ){ + if( c==MATCH_ONE ){ + if( zString[iString]==0 ) return 0; + U8_FWD_1_UNSAFE(zString, iString); } + iPattern++; + } - break; + if( zPattern[iPattern]==0 ) return 1; - case FTSQUERY_OR: { - int bHit1 = fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc); - int bHit2 = fts3EvalTestExpr(pCsr, pExpr->pRight, pRc); - bHit = bHit1 || bHit2; - break; + while( zString[iString] ){ + if( icuLikeCompare(&zPattern[iPattern], &zString[iString], uEsc) ){ + return 1; + } + U8_FWD_1_UNSAFE(zString, iString); } + return 0; - case FTSQUERY_NOT: - bHit = ( - fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) - && !fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) - ); - break; + }else if( !prevEscape && uPattern==MATCH_ONE ){ + /* Case 2. */ + if( zString[iString]==0 ) return 0; + U8_FWD_1_UNSAFE(zString, iString); - default: { -#ifndef SQLITE_DISABLE_FTS4_DEFERRED - if( pCsr->pDeferred - && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) - ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); - if( pExpr->bDeferred ){ - fts3EvalInvalidatePoslist(pPhrase); - } - *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); - bHit = (pPhrase->doclist.pList!=0); - pExpr->iDocid = pCsr->iPrevId; - }else -#endif - { - bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId); - } - break; + }else if( !prevEscape && uPattern==uEsc){ + /* Case 3. */ + prevEscape = 1; + + }else{ + /* Case 4. */ + UChar32 uString; + U8_NEXT_UNSAFE(zString, iString, uString); + uString = u_foldCase(uString, U_FOLD_CASE_DEFAULT); + uPattern = u_foldCase(uPattern, U_FOLD_CASE_DEFAULT); + if( uString!=uPattern ){ + return 0; } + prevEscape = 0; } } - return bHit; + + return zString[iString]==0; } /* -** This function is called as the second part of each xNext operation when -** iterating through the results of a full-text query. At this point the -** cursor points to a row that matches the query expression, with the -** following caveats: -** -** * Up until this point, "NEAR" operators in the expression have been -** treated as "AND". -** -** * Deferred tokens have not yet been considered. +** Implementation of the like() SQL function. This function implements +** the build-in LIKE operator. The first argument to the function is the +** pattern and the second argument is the string. So, the SQL statements: ** -** If *pRc is not SQLITE_OK when this function is called, it immediately -** returns 0. Otherwise, it tests whether or not after considering NEAR -** operators and deferred tokens the current row is still a match for the -** expression. It returns 1 if both of the following are true: +** A LIKE B ** -** 1. *pRc is SQLITE_OK when this function returns, and +** is implemented as like(B, A). If there is an escape character E, ** -** 2. After scanning the current FTS table row for the deferred tokens, -** it is determined that the row does *not* match the query. +** A LIKE B ESCAPE E ** -** Or, if no error occurs and it seems the current row does match the FTS -** query, return 0. +** is mapped to like(B, A, E). */ -static int fts3EvalTestDeferredAndNear(Fts3Cursor *pCsr, int *pRc){ - int rc = *pRc; - int bMiss = 0; - if( rc==SQLITE_OK ){ - - /* If there are one or more deferred tokens, load the current row into - ** memory and scan it to determine the position list for each deferred - ** token. Then, see if this row is really a match, considering deferred - ** tokens and NEAR operators (neither of which were taken into account - ** earlier, by fts3EvalNextRow()). - */ - if( pCsr->pDeferred ){ - rc = fts3CursorSeek(0, pCsr); - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3CacheDeferredDoclists(pCsr); - } - } - bMiss = (0==fts3EvalTestExpr(pCsr, pCsr->pExpr, &rc)); +static void icuLikeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zA = sqlite3_value_text(argv[0]); + const unsigned char *zB = sqlite3_value_text(argv[1]); + UChar32 uEsc = 0; - /* Free the position-lists accumulated for each deferred token above. */ - sqlite3Fts3FreeDeferredDoclists(pCsr); - *pRc = rc; + /* Limit the length of the LIKE or GLOB pattern to avoid problems + ** of deep recursion and N*N behavior in patternCompare(). + */ + if( sqlite3_value_bytes(argv[0])>SQLITE_MAX_LIKE_PATTERN_LENGTH ){ + sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1); + return; } - return (rc==SQLITE_OK && bMiss); -} -/* -** Advance to the next document that matches the FTS expression in -** Fts3Cursor.pExpr. -*/ -static int fts3EvalNext(Fts3Cursor *pCsr){ - int rc = SQLITE_OK; /* Return Code */ - Fts3Expr *pExpr = pCsr->pExpr; - assert( pCsr->isEof==0 ); - if( pExpr==0 ){ - pCsr->isEof = 1; - }else{ - do { - if( pCsr->isRequireSeek==0 ){ - sqlite3_reset(pCsr->pStmt); - } - assert( sqlite3_data_count(pCsr->pStmt)==0 ); - fts3EvalNextRow(pCsr, pExpr, &rc); - pCsr->isEof = pExpr->bEof; - pCsr->isRequireSeek = 1; - pCsr->isMatchinfoNeeded = 1; - pCsr->iPrevId = pExpr->iDocid; - }while( pCsr->isEof==0 && fts3EvalTestDeferredAndNear(pCsr, &rc) ); - } - /* Check if the cursor is past the end of the docid range specified - ** by Fts3Cursor.iMinDocid/iMaxDocid. If so, set the EOF flag. */ - if( rc==SQLITE_OK && ( - (pCsr->bDesc==0 && pCsr->iPrevId>pCsr->iMaxDocid) - || (pCsr->bDesc!=0 && pCsr->iPrevIdiMinDocid) - )){ - pCsr->isEof = 1; + if( argc==3 ){ + /* The escape character string must consist of a single UTF-8 character. + ** Otherwise, return an error. + */ + int nE= sqlite3_value_bytes(argv[2]); + const unsigned char *zE = sqlite3_value_text(argv[2]); + int i = 0; + if( zE==0 ) return; + U8_NEXT(zE, i, nE, uEsc); + if( i!=nE){ + sqlite3_result_error(context, + "ESCAPE expression must be a single character", -1); + return; + } } - return rc; + if( zA && zB ){ + sqlite3_result_int(context, icuLikeCompare(zA, zB, uEsc)); + } } /* -** Restart interation for expression pExpr so that the next call to -** fts3EvalNext() visits the first row. Do not allow incremental -** loading or merging of phrase doclists for this iteration. +** This function is called when an ICU function called from within +** the implementation of an SQL scalar function returns an error. ** -** If *pRc is other than SQLITE_OK when this function is called, it is -** a no-op. If an error occurs within this function, *pRc is set to an -** SQLite error code before returning. +** The scalar function context passed as the first argument is +** loaded with an error message based on the following two args. */ -static void fts3EvalRestart( - Fts3Cursor *pCsr, - Fts3Expr *pExpr, - int *pRc +static void icuFunctionError( + sqlite3_context *pCtx, /* SQLite scalar function context */ + const char *zName, /* Name of ICU function that failed */ + UErrorCode e /* Error code returned by ICU function */ ){ - if( pExpr && *pRc==SQLITE_OK ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - - if( pPhrase ){ - fts3EvalInvalidatePoslist(pPhrase); - if( pPhrase->bIncr ){ - int i; - for(i=0; inToken; i++){ - Fts3PhraseToken *pToken = &pPhrase->aToken[i]; - assert( pToken->pDeferred==0 ); - if( pToken->pSegcsr ){ - sqlite3Fts3MsrIncrRestart(pToken->pSegcsr); - } - } - *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase); - } - pPhrase->doclist.pNextDocid = 0; - pPhrase->doclist.iDocid = 0; - pPhrase->pOrPoslist = 0; - } - - pExpr->iDocid = 0; - pExpr->bEof = 0; - pExpr->bStart = 0; - - fts3EvalRestart(pCsr, pExpr->pLeft, pRc); - fts3EvalRestart(pCsr, pExpr->pRight, pRc); - } + char zBuf[128]; + sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e)); + zBuf[127] = '\0'; + sqlite3_result_error(pCtx, zBuf, -1); } /* -** After allocating the Fts3Expr.aMI[] array for each phrase in the -** expression rooted at pExpr, the cursor iterates through all rows matched -** by pExpr, calling this function for each row. This function increments -** the values in Fts3Expr.aMI[] according to the position-list currently -** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase -** expression nodes. +** Function to delete compiled regexp objects. Registered as +** a destructor function with sqlite3_set_auxdata(). */ -static void fts3EvalUpdateCounts(Fts3Expr *pExpr){ - if( pExpr ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - if( pPhrase && pPhrase->doclist.pList ){ - int iCol = 0; - char *p = pPhrase->doclist.pList; - - assert( *p ); - while( 1 ){ - u8 c = 0; - int iCnt = 0; - while( 0xFE & (*p | c) ){ - if( (c&0x80)==0 ) iCnt++; - c = *p++ & 0x80; - } - - /* aMI[iCol*3 + 1] = Number of occurrences - ** aMI[iCol*3 + 2] = Number of rows containing at least one instance - */ - pExpr->aMI[iCol*3 + 1] += iCnt; - pExpr->aMI[iCol*3 + 2] += (iCnt>0); - if( *p==0x00 ) break; - p++; - p += fts3GetVarint32(p, &iCol); - } - } - - fts3EvalUpdateCounts(pExpr->pLeft); - fts3EvalUpdateCounts(pExpr->pRight); - } +static void icuRegexpDelete(void *p){ + URegularExpression *pExpr = (URegularExpression *)p; + uregex_close(pExpr); } /* -** Expression pExpr must be of type FTSQUERY_PHRASE. +** Implementation of SQLite REGEXP operator. This scalar function takes +** two arguments. The first is a regular expression pattern to compile +** the second is a string to match against that pattern. If either +** argument is an SQL NULL, then NULL Is returned. Otherwise, the result +** is 1 if the string matches the pattern, or 0 otherwise. ** -** If it is not already allocated and populated, this function allocates and -** populates the Fts3Expr.aMI[] array for expression pExpr. If pExpr is part -** of a NEAR expression, then it also allocates and populates the same array -** for all other phrases that are part of the NEAR expression. +** SQLite maps the regexp() function to the regexp() operator such +** that the following two are equivalent: ** -** SQLITE_OK is returned if the aMI[] array is successfully allocated and -** populated. Otherwise, if an error occurs, an SQLite error code is returned. +** zString REGEXP zPattern +** regexp(zPattern, zString) +** +** Uses the following ICU regexp APIs: +** +** uregex_open() +** uregex_matches() +** uregex_close() */ -static int fts3EvalGatherStats( - Fts3Cursor *pCsr, /* Cursor object */ - Fts3Expr *pExpr /* FTSQUERY_PHRASE expression */ -){ - int rc = SQLITE_OK; /* Return code */ - - assert( pExpr->eType==FTSQUERY_PHRASE ); - if( pExpr->aMI==0 ){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - Fts3Expr *pRoot; /* Root of NEAR expression */ - Fts3Expr *p; /* Iterator used for several purposes */ - - sqlite3_int64 iPrevId = pCsr->iPrevId; - sqlite3_int64 iDocid; - u8 bEof; - - /* Find the root of the NEAR expression */ - pRoot = pExpr; - while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){ - pRoot = pRoot->pParent; - } - iDocid = pRoot->iDocid; - bEof = pRoot->bEof; - assert( pRoot->bStart ); - - /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */ - for(p=pRoot; p; p=p->pLeft){ - Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight); - assert( pE->aMI==0 ); - pE->aMI = (u32 *)sqlite3_malloc(pTab->nColumn * 3 * sizeof(u32)); - if( !pE->aMI ) return SQLITE_NOMEM; - memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32)); - } - - fts3EvalRestart(pCsr, pRoot, &rc); - - while( pCsr->isEof==0 && rc==SQLITE_OK ){ +static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){ + UErrorCode status = U_ZERO_ERROR; + URegularExpression *pExpr; + UBool res; + const UChar *zString = sqlite3_value_text16(apArg[1]); - do { - /* Ensure the %_content statement is reset. */ - if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt); - assert( sqlite3_data_count(pCsr->pStmt)==0 ); + (void)nArg; /* Unused parameter */ - /* Advance to the next document */ - fts3EvalNextRow(pCsr, pRoot, &rc); - pCsr->isEof = pRoot->bEof; - pCsr->isRequireSeek = 1; - pCsr->isMatchinfoNeeded = 1; - pCsr->iPrevId = pRoot->iDocid; - }while( pCsr->isEof==0 - && pRoot->eType==FTSQUERY_NEAR - && fts3EvalTestDeferredAndNear(pCsr, &rc) - ); + /* If the left hand side of the regexp operator is NULL, + ** then the result is also NULL. + */ + if( !zString ){ + return; + } - if( rc==SQLITE_OK && pCsr->isEof==0 ){ - fts3EvalUpdateCounts(pRoot); - } + pExpr = sqlite3_get_auxdata(p, 0); + if( !pExpr ){ + const UChar *zPattern = sqlite3_value_text16(apArg[0]); + if( !zPattern ){ + return; } + pExpr = uregex_open(zPattern, -1, 0, 0, &status); - pCsr->isEof = 0; - pCsr->iPrevId = iPrevId; - - if( bEof ){ - pRoot->bEof = bEof; + if( U_SUCCESS(status) ){ + sqlite3_set_auxdata(p, 0, pExpr, icuRegexpDelete); }else{ - /* Caution: pRoot may iterate through docids in ascending or descending - ** order. For this reason, even though it seems more defensive, the - ** do loop can not be written: - ** - ** do {...} while( pRoot->iDocidbEof==0 ); - }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); - fts3EvalTestDeferredAndNear(pCsr, &rc); + assert(!pExpr); + icuFunctionError(p, "uregex_open", status); + return; } } - return rc; -} -/* -** This function is used by the matchinfo() module to query a phrase -** expression node for the following information: -** -** 1. The total number of occurrences of the phrase in each column of -** the FTS table (considering all rows), and -** -** 2. For each column, the number of rows in the table for which the -** column contains at least one instance of the phrase. -** -** If no error occurs, SQLITE_OK is returned and the values for each column -** written into the array aiOut as follows: -** -** aiOut[iCol*3 + 1] = Number of occurrences -** aiOut[iCol*3 + 2] = Number of rows containing at least one instance -** -** Caveats: -** -** * If a phrase consists entirely of deferred tokens, then all output -** values are set to the number of documents in the table. In other -** words we assume that very common tokens occur exactly once in each -** column of each row of the table. -** -** * If a phrase contains some deferred tokens (and some non-deferred -** tokens), count the potential occurrence identified by considering -** the non-deferred tokens instead of actual phrase occurrences. -** -** * If the phrase is part of a NEAR expression, then only phrase instances -** that meet the NEAR constraint are included in the counts. -*/ -SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats( - Fts3Cursor *pCsr, /* FTS cursor handle */ - Fts3Expr *pExpr, /* Phrase expression */ - u32 *aiOut /* Array to write results into (see above) */ -){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int rc = SQLITE_OK; - int iCol; + /* Configure the text that the regular expression operates on. */ + uregex_setText(pExpr, zString, -1, &status); + if( !U_SUCCESS(status) ){ + icuFunctionError(p, "uregex_setText", status); + return; + } - if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){ - assert( pCsr->nDoc>0 ); - for(iCol=0; iColnColumn; iCol++){ - aiOut[iCol*3 + 1] = (u32)pCsr->nDoc; - aiOut[iCol*3 + 2] = (u32)pCsr->nDoc; - } - }else{ - rc = fts3EvalGatherStats(pCsr, pExpr); - if( rc==SQLITE_OK ){ - assert( pExpr->aMI ); - for(iCol=0; iColnColumn; iCol++){ - aiOut[iCol*3 + 1] = pExpr->aMI[iCol*3 + 1]; - aiOut[iCol*3 + 2] = pExpr->aMI[iCol*3 + 2]; - } - } + /* Attempt the match */ + res = uregex_matches(pExpr, 0, &status); + if( !U_SUCCESS(status) ){ + icuFunctionError(p, "uregex_matches", status); + return; } - return rc; + /* Set the text that the regular expression operates on to a NULL + ** pointer. This is not really necessary, but it is tidier than + ** leaving the regular expression object configured with an invalid + ** pointer after this function returns. + */ + uregex_setText(pExpr, 0, 0, &status); + + /* Return 1 or 0. */ + sqlite3_result_int(p, res ? 1 : 0); } /* -** The expression pExpr passed as the second argument to this function -** must be of type FTSQUERY_PHRASE. +** Implementations of scalar functions for case mapping - upper() and +** lower(). Function upper() converts its input to upper-case (ABC). +** Function lower() converts to lower-case (abc). ** -** The returned value is either NULL or a pointer to a buffer containing -** a position-list indicating the occurrences of the phrase in column iCol -** of the current row. +** ICU provides two types of case mapping, "general" case mapping and +** "language specific". Refer to ICU documentation for the differences +** between the two. ** -** More specifically, the returned buffer contains 1 varint for each -** occurrence of the phrase in the column, stored using the normal (delta+2) -** compression and is terminated by either an 0x01 or 0x00 byte. For example, -** if the requested column contains "a b X c d X X" and the position-list -** for 'X' is requested, the buffer returned may contain: +** To utilise "general" case mapping, the upper() or lower() scalar +** functions are invoked with one argument: ** -** 0x04 0x05 0x03 0x01 or 0x04 0x05 0x03 0x00 +** upper('ABC') -> 'abc' +** lower('abc') -> 'ABC' ** -** This function works regardless of whether or not the phrase is deferred, -** incremental, or neither. +** To access ICU "language specific" case mapping, upper() or lower() +** should be invoked with two arguments. The second argument is the name +** of the locale to use. Passing an empty string ("") or SQL NULL value +** as the second argument is the same as invoking the 1 argument version +** of upper() or lower(). +** +** lower('I', 'en_us') -> 'i' +** lower('I', 'tr_tr') -> 'ı' (small dotless i) +** +** http://www.icu-project.org/userguide/posix.html#case_mappings */ -SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist( - Fts3Cursor *pCsr, /* FTS3 cursor object */ - Fts3Expr *pExpr, /* Phrase to return doclist for */ - int iCol, /* Column to return position list for */ - char **ppOut /* OUT: Pointer to position list */ -){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - char *pIter; - int iThis; - sqlite3_int64 iDocid; - - /* If this phrase is applies specifically to some column other than - ** column iCol, return a NULL pointer. */ - *ppOut = 0; - assert( iCol>=0 && iColnColumn ); - if( (pPhrase->iColumnnColumn && pPhrase->iColumn!=iCol) ){ - return SQLITE_OK; - } - - iDocid = pExpr->iDocid; - pIter = pPhrase->doclist.pList; - if( iDocid!=pCsr->iPrevId || pExpr->bEof ){ - int rc = SQLITE_OK; - int bDescDoclist = pTab->bDescIdx; /* For DOCID_CMP macro */ - int bOr = 0; - u8 bEof = 0; - u8 bTreeEof = 0; - Fts3Expr *p; /* Used to iterate from pExpr to root */ - Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */ +static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){ + const UChar *zInput; + UChar *zOutput; + int nInput; + int nOutput; - /* Check if this phrase descends from an OR expression node. If not, - ** return NULL. Otherwise, the entry that corresponds to docid - ** pCsr->iPrevId may lie earlier in the doclist buffer. Or, if the - ** tree that the node is part of has been marked as EOF, but the node - ** itself is not EOF, then it may point to an earlier entry. */ - pNear = pExpr; - for(p=pExpr->pParent; p; p=p->pParent){ - if( p->eType==FTSQUERY_OR ) bOr = 1; - if( p->eType==FTSQUERY_NEAR ) pNear = p; - if( p->bEof ) bTreeEof = 1; - } - if( bOr==0 ) return SQLITE_OK; + UErrorCode status = U_ZERO_ERROR; + const char *zLocale = 0; - /* This is the descendent of an OR node. In this case we cannot use - ** an incremental phrase. Load the entire doclist for the phrase - ** into memory in this case. */ - if( pPhrase->bIncr ){ - int bEofSave = pNear->bEof; - fts3EvalRestart(pCsr, pNear, &rc); - while( rc==SQLITE_OK && !pNear->bEof ){ - fts3EvalNextRow(pCsr, pNear, &rc); - if( bEofSave==0 && pNear->iDocid==iDocid ) break; - } - assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); - } - if( bTreeEof ){ - while( rc==SQLITE_OK && !pNear->bEof ){ - fts3EvalNextRow(pCsr, pNear, &rc); - } - } - if( rc!=SQLITE_OK ) return rc; + assert(nArg==1 || nArg==2); + if( nArg==2 ){ + zLocale = (const char *)sqlite3_value_text(apArg[1]); + } - pIter = pPhrase->pOrPoslist; - iDocid = pPhrase->iOrDocid; - if( pCsr->bDesc==bDescDoclist ){ - bEof = !pPhrase->doclist.nAll || - (pIter >= (pPhrase->doclist.aAll + pPhrase->doclist.nAll)); - while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){ - sqlite3Fts3DoclistNext( - bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll, - &pIter, &iDocid, &bEof - ); - } - }else{ - bEof = !pPhrase->doclist.nAll || (pIter && pIter<=pPhrase->doclist.aAll); - while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){ - int dummy; - sqlite3Fts3DoclistPrev( - bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll, - &pIter, &iDocid, &dummy, &bEof - ); - } - } - pPhrase->pOrPoslist = pIter; - pPhrase->iOrDocid = iDocid; + zInput = sqlite3_value_text16(apArg[0]); + if( !zInput ){ + return; + } + nInput = sqlite3_value_bytes16(apArg[0]); - if( bEof || iDocid!=pCsr->iPrevId ) pIter = 0; + nOutput = nInput * 2 + 2; + zOutput = sqlite3_malloc(nOutput); + if( !zOutput ){ + return; } - if( pIter==0 ) return SQLITE_OK; - if( *pIter==0x01 ){ - pIter++; - pIter += fts3GetVarint32(pIter, &iThis); + if( sqlite3_user_data(p) ){ + u_strToUpper(zOutput, nOutput/2, zInput, nInput/2, zLocale, &status); }else{ - iThis = 0; - } - while( iThisdoclist, and -** * any Fts3MultiSegReader objects held by phrase tokens. +** Collation sequence destructor function. The pCtx argument points to +** a UCollator structure previously allocated using ucol_open(). */ -SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ - if( pPhrase ){ - int i; - sqlite3_free(pPhrase->doclist.aAll); - fts3EvalInvalidatePoslist(pPhrase); - memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); - for(i=0; inToken; i++){ - fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); - pPhrase->aToken[i].pSegcsr = 0; - } - } +static void icuCollationDel(void *pCtx){ + UCollator *p = (UCollator *)pCtx; + ucol_close(p); } +/* +** Collation sequence comparison function. The pCtx argument points to +** a UCollator structure previously allocated using ucol_open(). +*/ +static int icuCollationColl( + void *pCtx, + int nLeft, + const void *zLeft, + int nRight, + const void *zRight +){ + UCollationResult res; + UCollator *p = (UCollator *)pCtx; + res = ucol_strcoll(p, (UChar *)zLeft, nLeft/2, (UChar *)zRight, nRight/2); + switch( res ){ + case UCOL_LESS: return -1; + case UCOL_GREATER: return +1; + case UCOL_EQUAL: return 0; + } + assert(!"Unexpected return value from ucol_strcoll()"); + return 0; +} /* -** Return SQLITE_CORRUPT_VTAB. +** Implementation of the scalar function icu_load_collation(). +** +** This scalar function is used to add ICU collation based collation +** types to an SQLite database connection. It is intended to be called +** as follows: +** +** SELECT icu_load_collation(, ); +** +** Where is a string containing an ICU locale identifier (i.e. +** "en_AU", "tr_TR" etc.) and is the name of the +** collation sequence to create. */ -#ifdef SQLITE_DEBUG -SQLITE_PRIVATE int sqlite3Fts3Corrupt(){ - return SQLITE_CORRUPT_VTAB; +static void icuLoadCollation( + sqlite3_context *p, + int nArg, + sqlite3_value **apArg +){ + sqlite3 *db = (sqlite3 *)sqlite3_user_data(p); + UErrorCode status = U_ZERO_ERROR; + const char *zLocale; /* Locale identifier - (eg. "jp_JP") */ + const char *zName; /* SQL Collation sequence name (eg. "japanese") */ + UCollator *pUCollator; /* ICU library collation object */ + int rc; /* Return code from sqlite3_create_collation_x() */ + + assert(nArg==2); + (void)nArg; /* Unused parameter */ + zLocale = (const char *)sqlite3_value_text(apArg[0]); + zName = (const char *)sqlite3_value_text(apArg[1]); + + if( !zLocale || !zName ){ + return; + } + + pUCollator = ucol_open(zLocale, &status); + if( !U_SUCCESS(status) ){ + icuFunctionError(p, "ucol_open", status); + return; + } + assert(p); + + rc = sqlite3_create_collation_v2(db, zName, SQLITE_UTF16, (void *)pUCollator, + icuCollationColl, icuCollationDel + ); + if( rc!=SQLITE_OK ){ + ucol_close(pUCollator); + sqlite3_result_error(p, "Error registering collation function", -1); + } } -#endif -#if !SQLITE_CORE /* -** Initialize API pointer table, if required. +** Register the ICU extension functions with database db. */ +SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db){ + struct IcuScalar { + const char *zName; /* Function name */ + int nArg; /* Number of arguments */ + int enc; /* Optimal text encoding */ + void *pContext; /* sqlite3_user_data() context */ + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } scalars[] = { + {"regexp", 2, SQLITE_ANY, 0, icuRegexpFunc}, + + {"lower", 1, SQLITE_UTF16, 0, icuCaseFunc16}, + {"lower", 2, SQLITE_UTF16, 0, icuCaseFunc16}, + {"upper", 1, SQLITE_UTF16, (void*)1, icuCaseFunc16}, + {"upper", 2, SQLITE_UTF16, (void*)1, icuCaseFunc16}, + + {"lower", 1, SQLITE_UTF8, 0, icuCaseFunc16}, + {"lower", 2, SQLITE_UTF8, 0, icuCaseFunc16}, + {"upper", 1, SQLITE_UTF8, (void*)1, icuCaseFunc16}, + {"upper", 2, SQLITE_UTF8, (void*)1, icuCaseFunc16}, + + {"like", 2, SQLITE_UTF8, 0, icuLikeFunc}, + {"like", 3, SQLITE_UTF8, 0, icuLikeFunc}, + + {"icu_load_collation", 2, SQLITE_UTF8, (void*)db, icuLoadCollation}, + }; + + int rc = SQLITE_OK; + int i; + + for(i=0; rc==SQLITE_OK && i<(int)(sizeof(scalars)/sizeof(scalars[0])); i++){ + struct IcuScalar *p = &scalars[i]; + rc = sqlite3_create_function( + db, p->zName, p->nArg, p->enc, p->pContext, p->xFunc, 0, 0 + ); + } + + return rc; +} + +#if !SQLITE_CORE #ifdef _WIN32 __declspec(dllexport) #endif -SQLITE_API int SQLITE_STDCALL sqlite3_fts3_init( +SQLITE_API int SQLITE_STDCALL sqlite3_icu_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi ){ SQLITE_EXTENSION_INIT2(pApi) - return sqlite3Fts3Init(db); + return sqlite3IcuInit(db); } #endif #endif -/************** End of fts3.c ************************************************/ -/************** Begin file fts3_aux.c ****************************************/ +/************** End of icu.c *************************************************/ +/************** Begin file fts3_icu.c ****************************************/ /* -** 2011 Jan 27 +** 2007 June 22 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -138897,550 +158069,354 @@ SQLITE_API int SQLITE_STDCALL sqlite3_fts3_init( ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** -****************************************************************************** -** +************************************************************************* +** This file implements a tokenizer for fts3 based on the ICU library. */ +/* #include "fts3Int.h" */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +#ifdef SQLITE_ENABLE_ICU -/* #include */ /* #include */ +/* #include */ +/* #include "fts3_tokenizer.h" */ -typedef struct Fts3auxTable Fts3auxTable; -typedef struct Fts3auxCursor Fts3auxCursor; +#include +/* #include */ +/* #include */ +#include -struct Fts3auxTable { - sqlite3_vtab base; /* Base class used by SQLite core */ - Fts3Table *pFts3Tab; +typedef struct IcuTokenizer IcuTokenizer; +typedef struct IcuCursor IcuCursor; + +struct IcuTokenizer { + sqlite3_tokenizer base; + char *zLocale; }; -struct Fts3auxCursor { - sqlite3_vtab_cursor base; /* Base class used by SQLite core */ - Fts3MultiSegReader csr; /* Must be right after "base" */ - Fts3SegFilter filter; - char *zStop; - int nStop; /* Byte-length of string zStop */ - int iLangid; /* Language id to query */ - int isEof; /* True if cursor is at EOF */ - sqlite3_int64 iRowid; /* Current rowid */ +struct IcuCursor { + sqlite3_tokenizer_cursor base; - int iCol; /* Current value of 'col' column */ - int nStat; /* Size of aStat[] array */ - struct Fts3auxColstats { - sqlite3_int64 nDoc; /* 'documents' values for current csr row */ - sqlite3_int64 nOcc; /* 'occurrences' values for current csr row */ - } *aStat; -}; + UBreakIterator *pIter; /* ICU break-iterator object */ + int nChar; /* Number of UChar elements in pInput */ + UChar *aChar; /* Copy of input using utf-16 encoding */ + int *aOffset; /* Offsets of each character in utf-8 input */ -/* -** Schema of the terms table. -*/ -#define FTS3_AUX_SCHEMA \ - "CREATE TABLE x(term, col, documents, occurrences, languageid HIDDEN)" + int nBuffer; + char *zBuffer; + + int iToken; +}; /* -** This function does all the work for both the xConnect and xCreate methods. -** These tables have no persistent representation of their own, so xConnect -** and xCreate are identical operations. +** Create a new tokenizer instance. */ -static int fts3auxConnectMethod( - sqlite3 *db, /* Database connection */ - void *pUnused, /* Unused */ - int argc, /* Number of elements in argv array */ - const char * const *argv, /* xCreate/xConnect argument array */ - sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ - char **pzErr /* OUT: sqlite3_malloc'd error message */ +static int icuCreate( + int argc, /* Number of entries in argv[] */ + const char * const *argv, /* Tokenizer creation arguments */ + sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ ){ - char const *zDb; /* Name of database (e.g. "main") */ - char const *zFts3; /* Name of fts3 table */ - int nDb; /* Result of strlen(zDb) */ - int nFts3; /* Result of strlen(zFts3) */ - int nByte; /* Bytes of space to allocate here */ - int rc; /* value returned by declare_vtab() */ - Fts3auxTable *p; /* Virtual table object to return */ - - UNUSED_PARAMETER(pUnused); - - /* The user should invoke this in one of two forms: - ** - ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table); - ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table-db, fts4-table); - */ - if( argc!=4 && argc!=5 ) goto bad_args; + IcuTokenizer *p; + int n = 0; - zDb = argv[1]; - nDb = (int)strlen(zDb); - if( argc==5 ){ - if( nDb==4 && 0==sqlite3_strnicmp("temp", zDb, 4) ){ - zDb = argv[3]; - nDb = (int)strlen(zDb); - zFts3 = argv[4]; - }else{ - goto bad_args; - } - }else{ - zFts3 = argv[3]; + if( argc>0 ){ + n = strlen(argv[0])+1; } - nFts3 = (int)strlen(zFts3); - - rc = sqlite3_declare_vtab(db, FTS3_AUX_SCHEMA); - if( rc!=SQLITE_OK ) return rc; - - nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2; - p = (Fts3auxTable *)sqlite3_malloc(nByte); - if( !p ) return SQLITE_NOMEM; - memset(p, 0, nByte); + p = (IcuTokenizer *)sqlite3_malloc(sizeof(IcuTokenizer)+n); + if( !p ){ + return SQLITE_NOMEM; + } + memset(p, 0, sizeof(IcuTokenizer)); - p->pFts3Tab = (Fts3Table *)&p[1]; - p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; - p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; - p->pFts3Tab->db = db; - p->pFts3Tab->nIndex = 1; + if( n ){ + p->zLocale = (char *)&p[1]; + memcpy(p->zLocale, argv[0], n); + } - memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); - memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); - sqlite3Fts3Dequote((char *)p->pFts3Tab->zName); + *ppTokenizer = (sqlite3_tokenizer *)p; - *ppVtab = (sqlite3_vtab *)p; return SQLITE_OK; - - bad_args: - sqlite3Fts3ErrMsg(pzErr, "invalid arguments to fts4aux constructor"); - return SQLITE_ERROR; } /* -** This function does the work for both the xDisconnect and xDestroy methods. -** These tables have no persistent representation of their own, so xDisconnect -** and xDestroy are identical operations. +** Destroy a tokenizer */ -static int fts3auxDisconnectMethod(sqlite3_vtab *pVtab){ - Fts3auxTable *p = (Fts3auxTable *)pVtab; - Fts3Table *pFts3 = p->pFts3Tab; - int i; - - /* Free any prepared statements held */ - for(i=0; iaStmt); i++){ - sqlite3_finalize(pFts3->aStmt[i]); - } - sqlite3_free(pFts3->zSegmentsTbl); +static int icuDestroy(sqlite3_tokenizer *pTokenizer){ + IcuTokenizer *p = (IcuTokenizer *)pTokenizer; sqlite3_free(p); return SQLITE_OK; } -#define FTS4AUX_EQ_CONSTRAINT 1 -#define FTS4AUX_GE_CONSTRAINT 2 -#define FTS4AUX_LE_CONSTRAINT 4 - /* -** xBestIndex - Analyze a WHERE and ORDER BY clause. +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is pInput[0..nBytes-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. */ -static int fts3auxBestIndexMethod( - sqlite3_vtab *pVTab, - sqlite3_index_info *pInfo +static int icuOpen( + sqlite3_tokenizer *pTokenizer, /* The tokenizer */ + const char *zInput, /* Input string */ + int nInput, /* Length of zInput in bytes */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ ){ - int i; - int iEq = -1; - int iGe = -1; - int iLe = -1; - int iLangid = -1; - int iNext = 1; /* Next free argvIndex value */ + IcuTokenizer *p = (IcuTokenizer *)pTokenizer; + IcuCursor *pCsr; - UNUSED_PARAMETER(pVTab); + const int32_t opt = U_FOLD_CASE_DEFAULT; + UErrorCode status = U_ZERO_ERROR; + int nChar; - /* This vtab delivers always results in "ORDER BY term ASC" order. */ - if( pInfo->nOrderBy==1 - && pInfo->aOrderBy[0].iColumn==0 - && pInfo->aOrderBy[0].desc==0 - ){ - pInfo->orderByConsumed = 1; - } + UChar32 c; + int iInput = 0; + int iOut = 0; - /* Search for equality and range constraints on the "term" column. - ** And equality constraints on the hidden "languageid" column. */ - for(i=0; inConstraint; i++){ - if( pInfo->aConstraint[i].usable ){ - int op = pInfo->aConstraint[i].op; - int iCol = pInfo->aConstraint[i].iColumn; + *ppCursor = 0; - if( iCol==0 ){ - if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iEq = i; - if( op==SQLITE_INDEX_CONSTRAINT_LT ) iLe = i; - if( op==SQLITE_INDEX_CONSTRAINT_LE ) iLe = i; - if( op==SQLITE_INDEX_CONSTRAINT_GT ) iGe = i; - if( op==SQLITE_INDEX_CONSTRAINT_GE ) iGe = i; - } - if( iCol==4 ){ - if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iLangid = i; - } - } + if( zInput==0 ){ + nInput = 0; + zInput = ""; + }else if( nInput<0 ){ + nInput = strlen(zInput); + } + nChar = nInput+1; + pCsr = (IcuCursor *)sqlite3_malloc( + sizeof(IcuCursor) + /* IcuCursor */ + ((nChar+3)&~3) * sizeof(UChar) + /* IcuCursor.aChar[] */ + (nChar+1) * sizeof(int) /* IcuCursor.aOffset[] */ + ); + if( !pCsr ){ + return SQLITE_NOMEM; } + memset(pCsr, 0, sizeof(IcuCursor)); + pCsr->aChar = (UChar *)&pCsr[1]; + pCsr->aOffset = (int *)&pCsr->aChar[(nChar+3)&~3]; - if( iEq>=0 ){ - pInfo->idxNum = FTS4AUX_EQ_CONSTRAINT; - pInfo->aConstraintUsage[iEq].argvIndex = iNext++; - pInfo->estimatedCost = 5; - }else{ - pInfo->idxNum = 0; - pInfo->estimatedCost = 20000; - if( iGe>=0 ){ - pInfo->idxNum += FTS4AUX_GE_CONSTRAINT; - pInfo->aConstraintUsage[iGe].argvIndex = iNext++; - pInfo->estimatedCost /= 2; + pCsr->aOffset[iOut] = iInput; + U8_NEXT(zInput, iInput, nInput, c); + while( c>0 ){ + int isError = 0; + c = u_foldCase(c, opt); + U16_APPEND(pCsr->aChar, iOut, nChar, c, isError); + if( isError ){ + sqlite3_free(pCsr); + return SQLITE_ERROR; } - if( iLe>=0 ){ - pInfo->idxNum += FTS4AUX_LE_CONSTRAINT; - pInfo->aConstraintUsage[iLe].argvIndex = iNext++; - pInfo->estimatedCost /= 2; + pCsr->aOffset[iOut] = iInput; + + if( iInput=0 ){ - pInfo->aConstraintUsage[iLangid].argvIndex = iNext++; - pInfo->estimatedCost--; - } - - return SQLITE_OK; -} - -/* -** xOpen - Open a cursor. -*/ -static int fts3auxOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ - Fts3auxCursor *pCsr; /* Pointer to cursor object to return */ - - UNUSED_PARAMETER(pVTab); - pCsr = (Fts3auxCursor *)sqlite3_malloc(sizeof(Fts3auxCursor)); - if( !pCsr ) return SQLITE_NOMEM; - memset(pCsr, 0, sizeof(Fts3auxCursor)); + pCsr->pIter = ubrk_open(UBRK_WORD, p->zLocale, pCsr->aChar, iOut, &status); + if( !U_SUCCESS(status) ){ + sqlite3_free(pCsr); + return SQLITE_ERROR; + } + pCsr->nChar = iOut; - *ppCsr = (sqlite3_vtab_cursor *)pCsr; + ubrk_first(pCsr->pIter); + *ppCursor = (sqlite3_tokenizer_cursor *)pCsr; return SQLITE_OK; } /* -** xClose - Close a cursor. +** Close a tokenization cursor previously opened by a call to icuOpen(). */ -static int fts3auxCloseMethod(sqlite3_vtab_cursor *pCursor){ - Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; - Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; - - sqlite3Fts3SegmentsClose(pFts3); - sqlite3Fts3SegReaderFinish(&pCsr->csr); - sqlite3_free((void *)pCsr->filter.zTerm); - sqlite3_free(pCsr->zStop); - sqlite3_free(pCsr->aStat); +static int icuClose(sqlite3_tokenizer_cursor *pCursor){ + IcuCursor *pCsr = (IcuCursor *)pCursor; + ubrk_close(pCsr->pIter); + sqlite3_free(pCsr->zBuffer); sqlite3_free(pCsr); return SQLITE_OK; } -static int fts3auxGrowStatArray(Fts3auxCursor *pCsr, int nSize){ - if( nSize>pCsr->nStat ){ - struct Fts3auxColstats *aNew; - aNew = (struct Fts3auxColstats *)sqlite3_realloc(pCsr->aStat, - sizeof(struct Fts3auxColstats) * nSize - ); - if( aNew==0 ) return SQLITE_NOMEM; - memset(&aNew[pCsr->nStat], 0, - sizeof(struct Fts3auxColstats) * (nSize - pCsr->nStat) - ); - pCsr->aStat = aNew; - pCsr->nStat = nSize; - } - return SQLITE_OK; -} - /* -** xNext - Advance the cursor to the next row, if any. +** Extract the next token from a tokenization cursor. */ -static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){ - Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; - Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; - int rc; - - /* Increment our pretend rowid value. */ - pCsr->iRowid++; - - for(pCsr->iCol++; pCsr->iColnStat; pCsr->iCol++){ - if( pCsr->aStat[pCsr->iCol].nDoc>0 ) return SQLITE_OK; - } +static int icuNext( + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */ + const char **ppToken, /* OUT: *ppToken is the token text */ + int *pnBytes, /* OUT: Number of bytes in token */ + int *piStartOffset, /* OUT: Starting offset of token */ + int *piEndOffset, /* OUT: Ending offset of token */ + int *piPosition /* OUT: Position integer of token */ +){ + IcuCursor *pCsr = (IcuCursor *)pCursor; - rc = sqlite3Fts3SegReaderStep(pFts3, &pCsr->csr); - if( rc==SQLITE_ROW ){ - int i = 0; - int nDoclist = pCsr->csr.nDoclist; - char *aDoclist = pCsr->csr.aDoclist; - int iCol; + int iStart = 0; + int iEnd = 0; + int nByte = 0; - int eState = 0; + while( iStart==iEnd ){ + UChar32 c; - if( pCsr->zStop ){ - int n = (pCsr->nStopcsr.nTerm) ? pCsr->nStop : pCsr->csr.nTerm; - int mc = memcmp(pCsr->zStop, pCsr->csr.zTerm, n); - if( mc<0 || (mc==0 && pCsr->csr.nTerm>pCsr->nStop) ){ - pCsr->isEof = 1; - return SQLITE_OK; - } + iStart = ubrk_current(pCsr->pIter); + iEnd = ubrk_next(pCsr->pIter); + if( iEnd==UBRK_DONE ){ + return SQLITE_DONE; } - if( fts3auxGrowStatArray(pCsr, 2) ) return SQLITE_NOMEM; - memset(pCsr->aStat, 0, sizeof(struct Fts3auxColstats) * pCsr->nStat); - iCol = 0; - - while( iaStat[0].nDoc++; - eState = 1; - iCol = 0; - break; - - /* State 1. In this state we are expecting either a 1, indicating - ** that the following integer will be a column number, or the - ** start of a position list for column 0. - ** - ** The only difference between state 1 and state 2 is that if the - ** integer encountered in state 1 is not 0 or 1, then we need to - ** increment the column 0 "nDoc" count for this term. - */ - case 1: - assert( iCol==0 ); - if( v>1 ){ - pCsr->aStat[1].nDoc++; - } - eState = 2; - /* fall through */ - - case 2: - if( v==0 ){ /* 0x00. Next integer will be a docid. */ - eState = 0; - }else if( v==1 ){ /* 0x01. Next integer will be a column number. */ - eState = 3; - }else{ /* 2 or greater. A position. */ - pCsr->aStat[iCol+1].nOcc++; - pCsr->aStat[0].nOcc++; - } - break; - - /* State 3. The integer just read is a column number. */ - default: assert( eState==3 ); - iCol = (int)v; - if( fts3auxGrowStatArray(pCsr, iCol+2) ) return SQLITE_NOMEM; - pCsr->aStat[iCol+1].nDoc++; - eState = 2; - break; + while( iStartaChar, iWhite, pCsr->nChar, c); + if( u_isspace(c) ){ + iStart = iWhite; + }else{ + break; } } - - pCsr->iCol = 0; - rc = SQLITE_OK; - }else{ - pCsr->isEof = 1; - } - return rc; -} - -/* -** xFilter - Initialize a cursor to point at the start of its data. -*/ -static int fts3auxFilterMethod( - sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ - int idxNum, /* Strategy index */ - const char *idxStr, /* Unused */ - int nVal, /* Number of elements in apVal */ - sqlite3_value **apVal /* Arguments for the indexing scheme */ -){ - Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; - Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; - int rc; - int isScan = 0; - int iLangVal = 0; /* Language id to query */ - - int iEq = -1; /* Index of term=? value in apVal */ - int iGe = -1; /* Index of term>=? value in apVal */ - int iLe = -1; /* Index of term<=? value in apVal */ - int iLangid = -1; /* Index of languageid=? value in apVal */ - int iNext = 0; - - UNUSED_PARAMETER(nVal); - UNUSED_PARAMETER(idxStr); - - assert( idxStr==0 ); - assert( idxNum==FTS4AUX_EQ_CONSTRAINT || idxNum==0 - || idxNum==FTS4AUX_LE_CONSTRAINT || idxNum==FTS4AUX_GE_CONSTRAINT - || idxNum==(FTS4AUX_LE_CONSTRAINT|FTS4AUX_GE_CONSTRAINT) - ); - - if( idxNum==FTS4AUX_EQ_CONSTRAINT ){ - iEq = iNext++; - }else{ - isScan = 1; - if( idxNum & FTS4AUX_GE_CONSTRAINT ){ - iGe = iNext++; - } - if( idxNum & FTS4AUX_LE_CONSTRAINT ){ - iLe = iNext++; - } - } - if( iNextfilter.zTerm); - sqlite3Fts3SegReaderFinish(&pCsr->csr); - sqlite3_free((void *)pCsr->filter.zTerm); - sqlite3_free(pCsr->aStat); - memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr); - - pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; - if( isScan ) pCsr->filter.flags |= FTS3_SEGMENT_SCAN; - - if( iEq>=0 || iGe>=0 ){ - const unsigned char *zStr = sqlite3_value_text(apVal[0]); - assert( (iEq==0 && iGe==-1) || (iEq==-1 && iGe==0) ); - if( zStr ){ - pCsr->filter.zTerm = sqlite3_mprintf("%s", zStr); - pCsr->filter.nTerm = sqlite3_value_bytes(apVal[0]); - if( pCsr->filter.zTerm==0 ) return SQLITE_NOMEM; + do { + UErrorCode status = U_ZERO_ERROR; + if( nByte ){ + char *zNew = sqlite3_realloc(pCsr->zBuffer, nByte); + if( !zNew ){ + return SQLITE_NOMEM; + } + pCsr->zBuffer = zNew; + pCsr->nBuffer = nByte; } - } - - if( iLe>=0 ){ - pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iLe])); - pCsr->nStop = sqlite3_value_bytes(apVal[iLe]); - if( pCsr->zStop==0 ) return SQLITE_NOMEM; - } - - if( iLangid>=0 ){ - iLangVal = sqlite3_value_int(apVal[iLangid]); - /* If the user specified a negative value for the languageid, use zero - ** instead. This works, as the "languageid=?" constraint will also - ** be tested by the VDBE layer. The test will always be false (since - ** this module will not return a row with a negative languageid), and - ** so the overall query will return zero rows. */ - if( iLangVal<0 ) iLangVal = 0; - } - pCsr->iLangid = iLangVal; + u_strToUTF8( + pCsr->zBuffer, pCsr->nBuffer, &nByte, /* Output vars */ + &pCsr->aChar[iStart], iEnd-iStart, /* Input vars */ + &status /* Output success/failure */ + ); + } while( nByte>pCsr->nBuffer ); - rc = sqlite3Fts3SegReaderCursor(pFts3, iLangVal, 0, FTS3_SEGCURSOR_ALL, - pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr - ); - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter); - } + *ppToken = pCsr->zBuffer; + *pnBytes = nByte; + *piStartOffset = pCsr->aOffset[iStart]; + *piEndOffset = pCsr->aOffset[iEnd]; + *piPosition = pCsr->iToken++; - if( rc==SQLITE_OK ) rc = fts3auxNextMethod(pCursor); - return rc; + return SQLITE_OK; } /* -** xEof - Return true if the cursor is at EOF, or false otherwise. +** The set of routines that implement the simple tokenizer */ -static int fts3auxEofMethod(sqlite3_vtab_cursor *pCursor){ - Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; - return pCsr->isEof; -} +static const sqlite3_tokenizer_module icuTokenizerModule = { + 0, /* iVersion */ + icuCreate, /* xCreate */ + icuDestroy, /* xCreate */ + icuOpen, /* xOpen */ + icuClose, /* xClose */ + icuNext, /* xNext */ + 0, /* xLanguageid */ +}; /* -** xColumn - Return a column value. +** Set *ppModule to point at the implementation of the ICU tokenizer. */ -static int fts3auxColumnMethod( - sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ - sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ - int iCol /* Index of column to read value from */ +SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( + sqlite3_tokenizer_module const**ppModule ){ - Fts3auxCursor *p = (Fts3auxCursor *)pCursor; - - assert( p->isEof==0 ); - switch( iCol ){ - case 0: /* term */ - sqlite3_result_text(pCtx, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT); - break; - - case 1: /* col */ - if( p->iCol ){ - sqlite3_result_int(pCtx, p->iCol-1); - }else{ - sqlite3_result_text(pCtx, "*", -1, SQLITE_STATIC); - } - break; - - case 2: /* documents */ - sqlite3_result_int64(pCtx, p->aStat[p->iCol].nDoc); - break; - - case 3: /* occurrences */ - sqlite3_result_int64(pCtx, p->aStat[p->iCol].nOcc); - break; - - default: /* languageid */ - assert( iCol==4 ); - sqlite3_result_int(pCtx, p->iLangid); - break; - } - - return SQLITE_OK; + *ppModule = &icuTokenizerModule; } -/* -** xRowid - Return the current rowid for the cursor. -*/ -static int fts3auxRowidMethod( - sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ - sqlite_int64 *pRowid /* OUT: Rowid value */ -){ - Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; - *pRowid = pCsr->iRowid; - return SQLITE_OK; -} +#endif /* defined(SQLITE_ENABLE_ICU) */ +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ +/************** End of fts3_icu.c ********************************************/ +/************** Begin file sqlite3rbu.c **************************************/ /* -** Register the fts3aux module with database connection db. Return SQLITE_OK -** if successful or an error code if sqlite3_create_module() fails. +** 2014 August 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** +** OVERVIEW +** +** The RBU extension requires that the RBU update be packaged as an +** SQLite database. The tables it expects to find are described in +** sqlite3rbu.h. Essentially, for each table xyz in the target database +** that the user wishes to write to, a corresponding data_xyz table is +** created in the RBU database and populated with one row for each row to +** update, insert or delete from the target table. +** +** The update proceeds in three stages: +** +** 1) The database is updated. The modified database pages are written +** to a *-oal file. A *-oal file is just like a *-wal file, except +** that it is named "-oal" instead of "-wal". +** Because regular SQLite clients do not look for file named +** "-oal", they go on using the original database in +** rollback mode while the *-oal file is being generated. +** +** During this stage RBU does not update the database by writing +** directly to the target tables. Instead it creates "imposter" +** tables using the SQLITE_TESTCTRL_IMPOSTER interface that it uses +** to update each b-tree individually. All updates required by each +** b-tree are completed before moving on to the next, and all +** updates are done in sorted key order. +** +** 2) The "-oal" file is moved to the equivalent "-wal" +** location using a call to rename(2). Before doing this the RBU +** module takes an EXCLUSIVE lock on the database file, ensuring +** that there are no other active readers. +** +** Once the EXCLUSIVE lock is released, any other database readers +** detect the new *-wal file and read the database in wal mode. At +** this point they see the new version of the database - including +** the updates made as part of the RBU update. +** +** 3) The new *-wal file is checkpointed. This proceeds in the same way +** as a regular database checkpoint, except that a single frame is +** checkpointed each time sqlite3rbu_step() is called. If the RBU +** handle is closed before the entire *-wal file is checkpointed, +** the checkpoint progress is saved in the RBU database and the +** checkpoint can be resumed by another RBU client at some point in +** the future. +** +** POTENTIAL PROBLEMS +** +** The rename() call might not be portable. And RBU is not currently +** syncing the directory after renaming the file. +** +** When state is saved, any commit to the *-oal file and the commit to +** the RBU update database are not atomic. So if the power fails at the +** wrong moment they might get out of sync. As the main database will be +** committed before the RBU update database this will likely either just +** pass unnoticed, or result in SQLITE_CONSTRAINT errors (due to UNIQUE +** constraint violations). +** +** If some client does modify the target database mid RBU update, or some +** other error occurs, the RBU extension will keep throwing errors. It's +** not really clear how to get out of this state. The system could just +** by delete the RBU update database and *-oal file and have the device +** download the update again and start over. +** +** At present, for an UPDATE, both the new.* and old.* records are +** collected in the rbu_xyz table. And for both UPDATEs and DELETEs all +** fields are collected. This means we're probably writing a lot more +** data to disk when saving the state of an ongoing update to the RBU +** update database than is strictly necessary. +** */ -SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db){ - static const sqlite3_module fts3aux_module = { - 0, /* iVersion */ - fts3auxConnectMethod, /* xCreate */ - fts3auxConnectMethod, /* xConnect */ - fts3auxBestIndexMethod, /* xBestIndex */ - fts3auxDisconnectMethod, /* xDisconnect */ - fts3auxDisconnectMethod, /* xDestroy */ - fts3auxOpenMethod, /* xOpen */ - fts3auxCloseMethod, /* xClose */ - fts3auxFilterMethod, /* xFilter */ - fts3auxNextMethod, /* xNext */ - fts3auxEofMethod, /* xEof */ - fts3auxColumnMethod, /* xColumn */ - fts3auxRowidMethod, /* xRowid */ - 0, /* xUpdate */ - 0, /* xBegin */ - 0, /* xSync */ - 0, /* xCommit */ - 0, /* xRollback */ - 0, /* xFindFunction */ - 0, /* xRename */ - 0, /* xSavepoint */ - 0, /* xRelease */ - 0 /* xRollbackTo */ - }; - int rc; /* Return code */ - rc = sqlite3_create_module(db, "fts4aux", &fts3aux_module, 0); - return rc; -} +/* #include */ +/* #include */ +/* #include */ -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ +/* #include "sqlite3.h" */ -/************** End of fts3_aux.c ********************************************/ -/************** Begin file fts3_expr.c ***************************************/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU) +/************** Include sqlite3rbu.h in the middle of sqlite3rbu.c ***********/ +/************** Begin file sqlite3rbu.h **************************************/ /* -** 2008 Nov 28 +** 2014 August 30 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -139449,3061 +158425,4431 @@ SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db){ ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** -****************************************************************************** +************************************************************************* ** -** This module contains code that implements a parser for fts3 query strings -** (the right-hand argument to the MATCH operator). Because the supported -** syntax is relatively simple, the whole tokenizer/parser system is -** hand-coded. +** This file contains the public interface for the RBU extension. */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) /* -** By default, this module parses the legacy syntax that has been -** traditionally used by fts3. Or, if SQLITE_ENABLE_FTS3_PARENTHESIS -** is defined, then it uses the new syntax. The differences between -** the new and the old syntaxes are: +** SUMMARY ** -** a) The new syntax supports parenthesis. The old does not. +** Writing a transaction containing a large number of operations on +** b-tree indexes that are collectively larger than the available cache +** memory can be very inefficient. ** -** b) The new syntax supports the AND and NOT operators. The old does not. +** The problem is that in order to update a b-tree, the leaf page (at least) +** containing the entry being inserted or deleted must be modified. If the +** working set of leaves is larger than the available cache memory, then a +** single leaf that is modified more than once as part of the transaction +** may be loaded from or written to the persistent media multiple times. +** Additionally, because the index updates are likely to be applied in +** random order, access to pages within the database is also likely to be in +** random order, which is itself quite inefficient. ** -** c) The old syntax supports the "-" token qualifier. This is not -** supported by the new syntax (it is replaced by the NOT operator). +** One way to improve the situation is to sort the operations on each index +** by index key before applying them to the b-tree. This leads to an IO +** pattern that resembles a single linear scan through the index b-tree, +** and all but guarantees each modified leaf page is loaded and stored +** exactly once. SQLite uses this trick to improve the performance of +** CREATE INDEX commands. This extension allows it to be used to improve +** the performance of large transactions on existing databases. ** -** d) When using the old syntax, the OR operator has a greater precedence -** than an implicit AND. When using the new, both implicity and explicit -** AND operators have a higher precedence than OR. +** Additionally, this extension allows the work involved in writing the +** large transaction to be broken down into sub-transactions performed +** sequentially by separate processes. This is useful if the system cannot +** guarantee that a single update process will run for long enough to apply +** the entire update, for example because the update is being applied on a +** mobile device that is frequently rebooted. Even after the writer process +** has committed one or more sub-transactions, other database clients continue +** to read from the original database snapshot. In other words, partially +** applied transactions are not visible to other clients. ** -** If compiled with SQLITE_TEST defined, then this module exports the -** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable -** to zero causes the module to use the old syntax. If it is set to -** non-zero the new syntax is activated. This is so both syntaxes can -** be tested using a single build of testfixture. +** "RBU" stands for "Resumable Bulk Update". As in a large database update +** transmitted via a wireless network to a mobile device. A transaction +** applied using this extension is hence refered to as an "RBU update". +** +** +** LIMITATIONS +** +** An "RBU update" transaction is subject to the following limitations: +** +** * The transaction must consist of INSERT, UPDATE and DELETE operations +** only. +** +** * INSERT statements may not use any default values. +** +** * UPDATE and DELETE statements must identify their target rows by +** non-NULL PRIMARY KEY values. Rows with NULL values stored in PRIMARY +** KEY fields may not be updated or deleted. If the table being written +** has no PRIMARY KEY, affected rows must be identified by rowid. +** +** * UPDATE statements may not modify PRIMARY KEY columns. +** +** * No triggers will be fired. +** +** * No foreign key violations are detected or reported. +** +** * CHECK constraints are not enforced. +** +** * No constraint handling mode except for "OR ROLLBACK" is supported. +** +** +** PREPARATION +** +** An "RBU update" is stored as a separate SQLite database. A database +** containing an RBU update is an "RBU database". For each table in the +** target database to be updated, the RBU database should contain a table +** named "data_" containing the same set of columns as the +** target table, and one more - "rbu_control". The data_% table should +** have no PRIMARY KEY or UNIQUE constraints, but each column should have +** the same type as the corresponding column in the target database. +** The "rbu_control" column should have no type at all. For example, if +** the target database contains: +** +** CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT, c UNIQUE); +** +** Then the RBU database should contain: +** +** CREATE TABLE data_t1(a INTEGER, b TEXT, c, rbu_control); +** +** The order of the columns in the data_% table does not matter. +** +** Instead of a regular table, the RBU database may also contain virtual +** tables or view named using the data_ naming scheme. +** +** Instead of the plain data_ naming scheme, RBU database tables +** may also be named data_, where is any sequence +** of zero or more numeric characters (0-9). This can be significant because +** tables within the RBU database are always processed in order sorted by +** name. By judicious selection of the the portion of the names +** of the RBU tables the user can therefore control the order in which they +** are processed. This can be useful, for example, to ensure that "external +** content" FTS4 tables are updated before their underlying content tables. +** +** If the target database table is a virtual table or a table that has no +** PRIMARY KEY declaration, the data_% table must also contain a column +** named "rbu_rowid". This column is mapped to the tables implicit primary +** key column - "rowid". Virtual tables for which the "rowid" column does +** not function like a primary key value cannot be updated using RBU. For +** example, if the target db contains either of the following: +** +** CREATE VIRTUAL TABLE x1 USING fts3(a, b); +** CREATE TABLE x1(a, b) +** +** then the RBU database should contain: +** +** CREATE TABLE data_x1(a, b, rbu_rowid, rbu_control); +** +** All non-hidden columns (i.e. all columns matched by "SELECT *") of the +** target table must be present in the input table. For virtual tables, +** hidden columns are optional - they are updated by RBU if present in +** the input table, or not otherwise. For example, to write to an fts4 +** table with a hidden languageid column such as: +** +** CREATE VIRTUAL TABLE ft1 USING fts4(a, b, languageid='langid'); +** +** Either of the following input table schemas may be used: +** +** CREATE TABLE data_ft1(a, b, langid, rbu_rowid, rbu_control); +** CREATE TABLE data_ft1(a, b, rbu_rowid, rbu_control); +** +** For each row to INSERT into the target database as part of the RBU +** update, the corresponding data_% table should contain a single record +** with the "rbu_control" column set to contain integer value 0. The +** other columns should be set to the values that make up the new record +** to insert. +** +** If the target database table has an INTEGER PRIMARY KEY, it is not +** possible to insert a NULL value into the IPK column. Attempting to +** do so results in an SQLITE_MISMATCH error. +** +** For each row to DELETE from the target database as part of the RBU +** update, the corresponding data_% table should contain a single record +** with the "rbu_control" column set to contain integer value 1. The +** real primary key values of the row to delete should be stored in the +** corresponding columns of the data_% table. The values stored in the +** other columns are not used. +** +** For each row to UPDATE from the target database as part of the RBU +** update, the corresponding data_% table should contain a single record +** with the "rbu_control" column set to contain a value of type text. +** The real primary key values identifying the row to update should be +** stored in the corresponding columns of the data_% table row, as should +** the new values of all columns being update. The text value in the +** "rbu_control" column must contain the same number of characters as +** there are columns in the target database table, and must consist entirely +** of 'x' and '.' characters (or in some special cases 'd' - see below). For +** each column that is being updated, the corresponding character is set to +** 'x'. For those that remain as they are, the corresponding character of the +** rbu_control value should be set to '.'. For example, given the tables +** above, the update statement: +** +** UPDATE t1 SET c = 'usa' WHERE a = 4; +** +** is represented by the data_t1 row created by: +** +** INSERT INTO data_t1(a, b, c, rbu_control) VALUES(4, NULL, 'usa', '..x'); +** +** Instead of an 'x' character, characters of the rbu_control value specified +** for UPDATEs may also be set to 'd'. In this case, instead of updating the +** target table with the value stored in the corresponding data_% column, the +** user-defined SQL function "rbu_delta()" is invoked and the result stored in +** the target table column. rbu_delta() is invoked with two arguments - the +** original value currently stored in the target table column and the +** value specified in the data_xxx table. +** +** For example, this row: +** +** INSERT INTO data_t1(a, b, c, rbu_control) VALUES(4, NULL, 'usa', '..d'); +** +** is similar to an UPDATE statement such as: +** +** UPDATE t1 SET c = rbu_delta(c, 'usa') WHERE a = 4; +** +** Finally, if an 'f' character appears in place of a 'd' or 's' in an +** ota_control string, the contents of the data_xxx table column is assumed +** to be a "fossil delta" - a patch to be applied to a blob value in the +** format used by the fossil source-code management system. In this case +** the existing value within the target database table must be of type BLOB. +** It is replaced by the result of applying the specified fossil delta to +** itself. +** +** If the target database table is a virtual table or a table with no PRIMARY +** KEY, the rbu_control value should not include a character corresponding +** to the rbu_rowid value. For example, this: +** +** INSERT INTO data_ft1(a, b, rbu_rowid, rbu_control) +** VALUES(NULL, 'usa', 12, '.x'); +** +** causes a result similar to: +** +** UPDATE ft1 SET b = 'usa' WHERE rowid = 12; +** +** The data_xxx tables themselves should have no PRIMARY KEY declarations. +** However, RBU is more efficient if reading the rows in from each data_xxx +** table in "rowid" order is roughly the same as reading them sorted by +** the PRIMARY KEY of the corresponding target database table. In other +** words, rows should be sorted using the destination table PRIMARY KEY +** fields before they are inserted into the data_xxx tables. +** +** USAGE +** +** The API declared below allows an application to apply an RBU update +** stored on disk to an existing target database. Essentially, the +** application: +** +** 1) Opens an RBU handle using the sqlite3rbu_open() function. +** +** 2) Registers any required virtual table modules with the database +** handle returned by sqlite3rbu_db(). Also, if required, register +** the rbu_delta() implementation. +** +** 3) Calls the sqlite3rbu_step() function one or more times on +** the new handle. Each call to sqlite3rbu_step() performs a single +** b-tree operation, so thousands of calls may be required to apply +** a complete update. ** -** The following describes the syntax supported by the fts3 MATCH -** operator in a similar format to that used by the lemon parser -** generator. This module does not use actually lemon, it uses a -** custom parser. +** 4) Calls sqlite3rbu_close() to close the RBU update handle. If +** sqlite3rbu_step() has been called enough times to completely +** apply the update to the target database, then the RBU database +** is marked as fully applied. Otherwise, the state of the RBU +** update application is saved in the RBU database for later +** resumption. ** -** query ::= andexpr (OR andexpr)*. +** See comments below for more detail on APIs. ** -** andexpr ::= notexpr (AND? notexpr)*. +** If an update is only partially applied to the target database by the +** time sqlite3rbu_close() is called, various state information is saved +** within the RBU database. This allows subsequent processes to automatically +** resume the RBU update from where it left off. ** -** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*. -** notexpr ::= LP query RP. +** To remove all RBU extension state information, returning an RBU database +** to its original contents, it is sufficient to drop all tables that begin +** with the prefix "rbu_" ** -** nearexpr ::= phrase (NEAR distance_opt nearexpr)*. +** DATABASE LOCKING ** -** distance_opt ::= . -** distance_opt ::= / INTEGER. +** An RBU update may not be applied to a database in WAL mode. Attempting +** to do so is an error (SQLITE_ERROR). ** -** phrase ::= TOKEN. -** phrase ::= COLUMN:TOKEN. -** phrase ::= "TOKEN TOKEN TOKEN...". +** While an RBU handle is open, a SHARED lock may be held on the target +** database file. This means it is possible for other clients to read the +** database, but not to write it. +** +** If an RBU update is started and then suspended before it is completed, +** then an external client writes to the database, then attempting to resume +** the suspended RBU update is also an error (SQLITE_BUSY). */ -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_fts3_enable_parentheses = 0; -#else -# ifdef SQLITE_ENABLE_FTS3_PARENTHESIS -# define sqlite3_fts3_enable_parentheses 1 -# else -# define sqlite3_fts3_enable_parentheses 0 -# endif +#ifndef _SQLITE3RBU_H +#define _SQLITE3RBU_H + +/* #include "sqlite3.h" ** Required for error code definitions ** */ + +#if 0 +extern "C" { #endif +typedef struct sqlite3rbu sqlite3rbu; + /* -** Default span for NEAR operators. +** Open an RBU handle. +** +** Argument zTarget is the path to the target database. Argument zRbu is +** the path to the RBU database. Each call to this function must be matched +** by a call to sqlite3rbu_close(). When opening the databases, RBU passes +** the SQLITE_CONFIG_URI flag to sqlite3_open_v2(). So if either zTarget +** or zRbu begin with "file:", it will be interpreted as an SQLite +** database URI, not a regular file name. +** +** If the zState argument is passed a NULL value, the RBU extension stores +** the current state of the update (how many rows have been updated, which +** indexes are yet to be updated etc.) within the RBU database itself. This +** can be convenient, as it means that the RBU application does not need to +** organize removing a separate state file after the update is concluded. +** Or, if zState is non-NULL, it must be a path to a database file in which +** the RBU extension can store the state of the update. +** +** When resuming an RBU update, the zState argument must be passed the same +** value as when the RBU update was started. +** +** Once the RBU update is finished, the RBU extension does not +** automatically remove any zState database file, even if it created it. +** +** By default, RBU uses the default VFS to access the files on disk. To +** use a VFS other than the default, an SQLite "file:" URI containing a +** "vfs=..." option may be passed as the zTarget option. +** +** IMPORTANT NOTE FOR ZIPVFS USERS: The RBU extension works with all of +** SQLite's built-in VFSs, including the multiplexor VFS. However it does +** not work out of the box with zipvfs. Refer to the comment describing +** the zipvfs_create_vfs() API below for details on using RBU with zipvfs. */ -#define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10 - -/* #include */ -/* #include */ +SQLITE_API sqlite3rbu *SQLITE_STDCALL sqlite3rbu_open( + const char *zTarget, + const char *zRbu, + const char *zState +); /* -** isNot: -** This variable is used by function getNextNode(). When getNextNode() is -** called, it sets ParseContext.isNot to true if the 'next node' is a -** FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the -** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to -** zero. +** Internally, each RBU connection uses a separate SQLite database +** connection to access the target and rbu update databases. This +** API allows the application direct access to these database handles. +** +** The first argument passed to this function must be a valid, open, RBU +** handle. The second argument should be passed zero to access the target +** database handle, or non-zero to access the rbu update database handle. +** Accessing the underlying database handles may be useful in the +** following scenarios: +** +** * If any target tables are virtual tables, it may be necessary to +** call sqlite3_create_module() on the target database handle to +** register the required virtual table implementations. +** +** * If the data_xxx tables in the RBU source database are virtual +** tables, the application may need to call sqlite3_create_module() on +** the rbu update db handle to any required virtual table +** implementations. +** +** * If the application uses the "rbu_delta()" feature described above, +** it must use sqlite3_create_function() or similar to register the +** rbu_delta() implementation with the target database handle. +** +** If an error has occurred, either while opening or stepping the RBU object, +** this function may return NULL. The error code and message may be collected +** when sqlite3rbu_close() is called. */ -typedef struct ParseContext ParseContext; -struct ParseContext { - sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ - int iLangid; /* Language id used with tokenizer */ - const char **azCol; /* Array of column names for fts3 table */ - int bFts4; /* True to allow FTS4-only syntax */ - int nCol; /* Number of entries in azCol[] */ - int iDefaultCol; /* Default column to query */ - int isNot; /* True if getNextNode() sees a unary - */ - sqlite3_context *pCtx; /* Write error message here */ - int nNest; /* Number of nested brackets */ -}; +SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3rbu_db(sqlite3rbu*, int bRbu); /* -** This function is equivalent to the standard isspace() function. +** Do some work towards applying the RBU update to the target db. ** -** The standard isspace() can be awkward to use safely, because although it -** is defined to accept an argument of type int, its behavior when passed -** an integer that falls outside of the range of the unsigned char type -** is undefined (and sometimes, "undefined" means segfault). This wrapper -** is defined to accept an argument of type char, and always returns 0 for -** any values that fall outside of the range of the unsigned char type (i.e. -** negative values). +** Return SQLITE_DONE if the update has been completely applied, or +** SQLITE_OK if no error occurs but there remains work to do to apply +** the RBU update. If an error does occur, some other error code is +** returned. +** +** Once a call to sqlite3rbu_step() has returned a value other than +** SQLITE_OK, all subsequent calls on the same RBU handle are no-ops +** that immediately return the same value. */ -static int fts3isspace(char c){ - return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f'; -} +SQLITE_API int SQLITE_STDCALL sqlite3rbu_step(sqlite3rbu *pRbu); /* -** Allocate nByte bytes of memory using sqlite3_malloc(). If successful, -** zero the memory before returning a pointer to it. If unsuccessful, -** return NULL. +** Force RBU to save its state to disk. +** +** If a power failure or application crash occurs during an update, following +** system recovery RBU may resume the update from the point at which the state +** was last saved. In other words, from the most recent successful call to +** sqlite3rbu_close() or this function. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ -static void *fts3MallocZero(int nByte){ - void *pRet = sqlite3_malloc(nByte); - if( pRet ) memset(pRet, 0, nByte); - return pRet; -} +SQLITE_API int SQLITE_STDCALL sqlite3rbu_savestate(sqlite3rbu *pRbu); -SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer( - sqlite3_tokenizer *pTokenizer, - int iLangid, - const char *z, - int n, - sqlite3_tokenizer_cursor **ppCsr -){ - sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; - sqlite3_tokenizer_cursor *pCsr = 0; - int rc; +/* +** Close an RBU handle. +** +** If the RBU update has been completely applied, mark the RBU database +** as fully applied. Otherwise, assuming no error has occurred, save the +** current state of the RBU update appliation to the RBU database. +** +** If an error has already occurred as part of an sqlite3rbu_step() +** or sqlite3rbu_open() call, or if one occurs within this function, an +** SQLite error code is returned. Additionally, *pzErrmsg may be set to +** point to a buffer containing a utf-8 formatted English language error +** message. It is the responsibility of the caller to eventually free any +** such buffer using sqlite3_free(). +** +** Otherwise, if no error occurs, this function returns SQLITE_OK if the +** update has been partially applied, or SQLITE_DONE if it has been +** completely applied. +*/ +SQLITE_API int SQLITE_STDCALL sqlite3rbu_close(sqlite3rbu *pRbu, char **pzErrmsg); - rc = pModule->xOpen(pTokenizer, z, n, &pCsr); - assert( rc==SQLITE_OK || pCsr==0 ); - if( rc==SQLITE_OK ){ - pCsr->pTokenizer = pTokenizer; - if( pModule->iVersion>=1 ){ - rc = pModule->xLanguageid(pCsr, iLangid); - if( rc!=SQLITE_OK ){ - pModule->xClose(pCsr); - pCsr = 0; - } - } - } - *ppCsr = pCsr; - return rc; -} +/* +** Return the total number of key-value operations (inserts, deletes or +** updates) that have been performed on the target database since the +** current RBU update was started. +*/ +SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3rbu_progress(sqlite3rbu *pRbu); /* -** Function getNextNode(), which is called by fts3ExprParse(), may itself -** call fts3ExprParse(). So this forward declaration is required. +** Create an RBU VFS named zName that accesses the underlying file-system +** via existing VFS zParent. Or, if the zParent parameter is passed NULL, +** then the new RBU VFS uses the default system VFS to access the file-system. +** The new object is registered as a non-default VFS with SQLite before +** returning. +** +** Part of the RBU implementation uses a custom VFS object. Usually, this +** object is created and deleted automatically by RBU. +** +** The exception is for applications that also use zipvfs. In this case, +** the custom VFS must be explicitly created by the user before the RBU +** handle is opened. The RBU VFS should be installed so that the zipvfs +** VFS uses the RBU VFS, which in turn uses any other VFS layers in use +** (for example multiplexor) to access the file-system. For example, +** to assemble an RBU enabled VFS stack that uses both zipvfs and +** multiplexor (error checking omitted): +** +** // Create a VFS named "multiplex" (not the default). +** sqlite3_multiplex_initialize(0, 0); +** +** // Create an rbu VFS named "rbu" that uses multiplexor. If the +** // second argument were replaced with NULL, the "rbu" VFS would +** // access the file-system via the system default VFS, bypassing the +** // multiplexor. +** sqlite3rbu_create_vfs("rbu", "multiplex"); +** +** // Create a zipvfs VFS named "zipvfs" that uses rbu. +** zipvfs_create_vfs_v3("zipvfs", "rbu", 0, xCompressorAlgorithmDetector); +** +** // Make zipvfs the default VFS. +** sqlite3_vfs_register(sqlite3_vfs_find("zipvfs"), 1); +** +** Because the default VFS created above includes a RBU functionality, it +** may be used by RBU clients. Attempting to use RBU with a zipvfs VFS stack +** that does not include the RBU layer results in an error. +** +** The overhead of adding the "rbu" VFS to the system is negligible for +** non-RBU users. There is no harm in an application accessing the +** file-system via "rbu" all the time, even if it only uses RBU functionality +** occasionally. */ -static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *); +SQLITE_API int SQLITE_STDCALL sqlite3rbu_create_vfs(const char *zName, const char *zParent); /* -** Extract the next token from buffer z (length n) using the tokenizer -** and other information (column names etc.) in pParse. Create an Fts3Expr -** structure of type FTSQUERY_PHRASE containing a phrase consisting of this -** single token and set *ppExpr to point to it. If the end of the buffer is -** reached before a token is found, set *ppExpr to zero. It is the -** responsibility of the caller to eventually deallocate the allocated -** Fts3Expr structure (if any) by passing it to sqlite3_free(). +** Deregister and destroy an RBU vfs created by an earlier call to +** sqlite3rbu_create_vfs(). ** -** Return SQLITE_OK if successful, or SQLITE_NOMEM if a memory allocation -** fails. +** VFS objects are not reference counted. If a VFS object is destroyed +** before all database handles that use it have been closed, the results +** are undefined. */ -static int getNextToken( - ParseContext *pParse, /* fts3 query parse context */ - int iCol, /* Value for Fts3Phrase.iColumn */ - const char *z, int n, /* Input string */ - Fts3Expr **ppExpr, /* OUT: expression */ - int *pnConsumed /* OUT: Number of bytes consumed */ -){ - sqlite3_tokenizer *pTokenizer = pParse->pTokenizer; - sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; - int rc; - sqlite3_tokenizer_cursor *pCursor; - Fts3Expr *pRet = 0; - int i = 0; +SQLITE_API void SQLITE_STDCALL sqlite3rbu_destroy_vfs(const char *zName); - /* Set variable i to the maximum number of bytes of input to tokenize. */ - for(i=0; iiLangid, z, i, &pCursor); - if( rc==SQLITE_OK ){ - const char *zToken; - int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0; - int nByte; /* total space to allocate */ +#endif /* _SQLITE3RBU_H */ - rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); - if( rc==SQLITE_OK ){ - nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; - pRet = (Fts3Expr *)fts3MallocZero(nByte); - if( !pRet ){ - rc = SQLITE_NOMEM; - }else{ - pRet->eType = FTSQUERY_PHRASE; - pRet->pPhrase = (Fts3Phrase *)&pRet[1]; - pRet->pPhrase->nToken = 1; - pRet->pPhrase->iColumn = iCol; - pRet->pPhrase->aToken[0].n = nToken; - pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1]; - memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken); +/************** End of sqlite3rbu.h ******************************************/ +/************** Continuing where we left off in sqlite3rbu.c *****************/ - if( iEndpPhrase->aToken[0].isPrefix = 1; - iEnd++; - } +#if defined(_WIN32_WCE) +/* #include "windows.h" */ +#endif - while( 1 ){ - if( !sqlite3_fts3_enable_parentheses - && iStart>0 && z[iStart-1]=='-' - ){ - pParse->isNot = 1; - iStart--; - }else if( pParse->bFts4 && iStart>0 && z[iStart-1]=='^' ){ - pRet->pPhrase->aToken[0].bFirst = 1; - iStart--; - }else{ - break; - } - } +/* Maximum number of prepared UPDATE statements held by this module */ +#define SQLITE_RBU_UPDATE_CACHESIZE 16 - } - *pnConsumed = iEnd; - }else if( i && rc==SQLITE_DONE ){ - rc = SQLITE_OK; - } +/* +** Swap two objects of type TYPE. +*/ +#if !defined(SQLITE_AMALGAMATION) +# define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;} +#endif + +/* +** The rbu_state table is used to save the state of a partially applied +** update so that it can be resumed later. The table consists of integer +** keys mapped to values as follows: +** +** RBU_STATE_STAGE: +** May be set to integer values 1, 2, 4 or 5. As follows: +** 1: the *-rbu file is currently under construction. +** 2: the *-rbu file has been constructed, but not yet moved +** to the *-wal path. +** 4: the checkpoint is underway. +** 5: the rbu update has been checkpointed. +** +** RBU_STATE_TBL: +** Only valid if STAGE==1. The target database name of the table +** currently being written. +** +** RBU_STATE_IDX: +** Only valid if STAGE==1. The target database name of the index +** currently being written, or NULL if the main table is currently being +** updated. +** +** RBU_STATE_ROW: +** Only valid if STAGE==1. Number of rows already processed for the current +** table/index. +** +** RBU_STATE_PROGRESS: +** Trbul number of sqlite3rbu_step() calls made so far as part of this +** rbu update. +** +** RBU_STATE_CKPT: +** Valid if STAGE==4. The 64-bit checksum associated with the wal-index +** header created by recovering the *-wal file. This is used to detect +** cases when another client appends frames to the *-wal file in the +** middle of an incremental checkpoint (an incremental checkpoint cannot +** be continued if this happens). +** +** RBU_STATE_COOKIE: +** Valid if STAGE==1. The current change-counter cookie value in the +** target db file. +** +** RBU_STATE_OALSZ: +** Valid if STAGE==1. The size in bytes of the *-oal file. +*/ +#define RBU_STATE_STAGE 1 +#define RBU_STATE_TBL 2 +#define RBU_STATE_IDX 3 +#define RBU_STATE_ROW 4 +#define RBU_STATE_PROGRESS 5 +#define RBU_STATE_CKPT 6 +#define RBU_STATE_COOKIE 7 +#define RBU_STATE_OALSZ 8 + +#define RBU_STAGE_OAL 1 +#define RBU_STAGE_MOVE 2 +#define RBU_STAGE_CAPTURE 3 +#define RBU_STAGE_CKPT 4 +#define RBU_STAGE_DONE 5 + + +#define RBU_CREATE_STATE \ + "CREATE TABLE IF NOT EXISTS %s.rbu_state(k INTEGER PRIMARY KEY, v)" + +typedef struct RbuFrame RbuFrame; +typedef struct RbuObjIter RbuObjIter; +typedef struct RbuState RbuState; +typedef struct rbu_vfs rbu_vfs; +typedef struct rbu_file rbu_file; +typedef struct RbuUpdateStmt RbuUpdateStmt; + +#if !defined(SQLITE_AMALGAMATION) +typedef unsigned int u32; +typedef unsigned char u8; +typedef sqlite3_int64 i64; +#endif - pModule->xClose(pCursor); - } - - *ppExpr = pRet; - return rc; -} +/* +** These values must match the values defined in wal.c for the equivalent +** locks. These are not magic numbers as they are part of the SQLite file +** format. +*/ +#define WAL_LOCK_WRITE 0 +#define WAL_LOCK_CKPT 1 +#define WAL_LOCK_READ0 3 + +/* +** A structure to store values read from the rbu_state table in memory. +*/ +struct RbuState { + int eStage; + char *zTbl; + char *zIdx; + i64 iWalCksum; + int nRow; + i64 nProgress; + u32 iCookie; + i64 iOalSz; +}; +struct RbuUpdateStmt { + char *zMask; /* Copy of update mask used with pUpdate */ + sqlite3_stmt *pUpdate; /* Last update statement (or NULL) */ + RbuUpdateStmt *pNext; +}; /* -** Enlarge a memory allocation. If an out-of-memory allocation occurs, -** then free the old allocation. +** An iterator of this type is used to iterate through all objects in +** the target database that require updating. For each such table, the +** iterator visits, in order: +** +** * the table itself, +** * each index of the table (zero or more points to visit), and +** * a special "cleanup table" state. +** +** abIndexed: +** If the table has no indexes on it, abIndexed is set to NULL. Otherwise, +** it points to an array of flags nTblCol elements in size. The flag is +** set for each column that is either a part of the PK or a part of an +** index. Or clear otherwise. +** */ -static void *fts3ReallocOrFree(void *pOrig, int nNew){ - void *pRet = sqlite3_realloc(pOrig, nNew); - if( !pRet ){ - sqlite3_free(pOrig); - } - return pRet; -} +struct RbuObjIter { + sqlite3_stmt *pTblIter; /* Iterate through tables */ + sqlite3_stmt *pIdxIter; /* Index iterator */ + int nTblCol; /* Size of azTblCol[] array */ + char **azTblCol; /* Array of unquoted target column names */ + char **azTblType; /* Array of target column types */ + int *aiSrcOrder; /* src table col -> target table col */ + u8 *abTblPk; /* Array of flags, set on target PK columns */ + u8 *abNotNull; /* Array of flags, set on NOT NULL columns */ + u8 *abIndexed; /* Array of flags, set on indexed & PK cols */ + int eType; /* Table type - an RBU_PK_XXX value */ + + /* Output variables. zTbl==0 implies EOF. */ + int bCleanup; /* True in "cleanup" state */ + const char *zTbl; /* Name of target db table */ + const char *zDataTbl; /* Name of rbu db table (or null) */ + const char *zIdx; /* Name of target db index (or null) */ + int iTnum; /* Root page of current object */ + int iPkTnum; /* If eType==EXTERNAL, root of PK index */ + int bUnique; /* Current index is unique */ + + /* Statements created by rbuObjIterPrepareAll() */ + int nCol; /* Number of columns in current object */ + sqlite3_stmt *pSelect; /* Source data */ + sqlite3_stmt *pInsert; /* Statement for INSERT operations */ + sqlite3_stmt *pDelete; /* Statement for DELETE ops */ + sqlite3_stmt *pTmpInsert; /* Insert into rbu_tmp_$zDataTbl */ + + /* Last UPDATE used (for PK b-tree updates only), or NULL. */ + RbuUpdateStmt *pRbuUpdate; +}; /* -** Buffer zInput, length nInput, contains the contents of a quoted string -** that appeared as part of an fts3 query expression. Neither quote character -** is included in the buffer. This function attempts to tokenize the entire -** input buffer and create an Fts3Expr structure of type FTSQUERY_PHRASE -** containing the results. +** Values for RbuObjIter.eType ** -** If successful, SQLITE_OK is returned and *ppExpr set to point at the -** allocated Fts3Expr structure. Otherwise, either SQLITE_NOMEM (out of memory -** error) or SQLITE_ERROR (tokenization error) is returned and *ppExpr set -** to 0. +** 0: Table does not exist (error) +** 1: Table has an implicit rowid. +** 2: Table has an explicit IPK column. +** 3: Table has an external PK index. +** 4: Table is WITHOUT ROWID. +** 5: Table is a virtual table. */ -static int getNextString( - ParseContext *pParse, /* fts3 query parse context */ - const char *zInput, int nInput, /* Input string */ - Fts3Expr **ppExpr /* OUT: expression */ -){ - sqlite3_tokenizer *pTokenizer = pParse->pTokenizer; - sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; - int rc; - Fts3Expr *p = 0; - sqlite3_tokenizer_cursor *pCursor = 0; - char *zTemp = 0; - int nTemp = 0; +#define RBU_PK_NOTABLE 0 +#define RBU_PK_NONE 1 +#define RBU_PK_IPK 2 +#define RBU_PK_EXTERNAL 3 +#define RBU_PK_WITHOUT_ROWID 4 +#define RBU_PK_VTAB 5 - const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase); - int nToken = 0; - /* The final Fts3Expr data structure, including the Fts3Phrase, - ** Fts3PhraseToken structures token buffers are all stored as a single - ** allocation so that the expression can be freed with a single call to - ** sqlite3_free(). Setting this up requires a two pass approach. - ** - ** The first pass, in the block below, uses a tokenizer cursor to iterate - ** through the tokens in the expression. This pass uses fts3ReallocOrFree() - ** to assemble data in two dynamic buffers: - ** - ** Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase - ** structure, followed by the array of Fts3PhraseToken - ** structures. This pass only populates the Fts3PhraseToken array. - ** - ** Buffer zTemp: Contains copies of all tokens. - ** - ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below, - ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase - ** structures. - */ - rc = sqlite3Fts3OpenTokenizer( - pTokenizer, pParse->iLangid, zInput, nInput, &pCursor); - if( rc==SQLITE_OK ){ - int ii; - for(ii=0; rc==SQLITE_OK; ii++){ - const char *zByte; - int nByte = 0, iBegin = 0, iEnd = 0, iPos = 0; - rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos); - if( rc==SQLITE_OK ){ - Fts3PhraseToken *pToken; +/* +** Within the RBU_STAGE_OAL stage, each call to sqlite3rbu_step() performs +** one of the following operations. +*/ +#define RBU_INSERT 1 /* Insert on a main table b-tree */ +#define RBU_DELETE 2 /* Delete a row from a main table b-tree */ +#define RBU_IDX_DELETE 3 /* Delete a row from an aux. index b-tree */ +#define RBU_IDX_INSERT 4 /* Insert on an aux. index b-tree */ +#define RBU_UPDATE 5 /* Update a row in a main table b-tree */ - p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken)); - if( !p ) goto no_mem; - zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte); - if( !zTemp ) goto no_mem; +/* +** A single step of an incremental checkpoint - frame iWalFrame of the wal +** file should be copied to page iDbPage of the database file. +*/ +struct RbuFrame { + u32 iDbPage; + u32 iWalFrame; +}; - assert( nToken==ii ); - pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii]; - memset(pToken, 0, sizeof(Fts3PhraseToken)); +/* +** RBU handle. +*/ +struct sqlite3rbu { + int eStage; /* Value of RBU_STATE_STAGE field */ + sqlite3 *dbMain; /* target database handle */ + sqlite3 *dbRbu; /* rbu database handle */ + char *zTarget; /* Path to target db */ + char *zRbu; /* Path to rbu db */ + char *zState; /* Path to state db (or NULL if zRbu) */ + char zStateDb[5]; /* Db name for state ("stat" or "main") */ + int rc; /* Value returned by last rbu_step() call */ + char *zErrmsg; /* Error message if rc!=SQLITE_OK */ + int nStep; /* Rows processed for current object */ + int nProgress; /* Rows processed for all objects */ + RbuObjIter objiter; /* Iterator for skipping through tbl/idx */ + const char *zVfsName; /* Name of automatically created rbu vfs */ + rbu_file *pTargetFd; /* File handle open on target db */ + i64 iOalSz; + + /* The following state variables are used as part of the incremental + ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding + ** function rbuSetupCheckpoint() for details. */ + u32 iMaxFrame; /* Largest iWalFrame value in aFrame[] */ + u32 mLock; + int nFrame; /* Entries in aFrame[] array */ + int nFrameAlloc; /* Allocated size of aFrame[] array */ + RbuFrame *aFrame; + int pgsz; + u8 *aBuf; + i64 iWalCksum; +}; - memcpy(&zTemp[nTemp], zByte, nByte); - nTemp += nByte; +/* +** An rbu VFS is implemented using an instance of this structure. +*/ +struct rbu_vfs { + sqlite3_vfs base; /* rbu VFS shim methods */ + sqlite3_vfs *pRealVfs; /* Underlying VFS */ + sqlite3_mutex *mutex; /* Mutex to protect pMain */ + rbu_file *pMain; /* Linked list of main db files */ +}; - pToken->n = nByte; - pToken->isPrefix = (iEndbFirst = (iBegin>0 && zInput[iBegin-1]=='^'); - nToken = ii+1; - } - } +/* +** Each file opened by an rbu VFS is represented by an instance of +** the following structure. +*/ +struct rbu_file { + sqlite3_file base; /* sqlite3_file methods */ + sqlite3_file *pReal; /* Underlying file handle */ + rbu_vfs *pRbuVfs; /* Pointer to the rbu_vfs object */ + sqlite3rbu *pRbu; /* Pointer to rbu object (rbu target only) */ - pModule->xClose(pCursor); - pCursor = 0; - } + int openFlags; /* Flags this file was opened with */ + u32 iCookie; /* Cookie value for main db files */ + u8 iWriteVer; /* "write-version" value for main db files */ - if( rc==SQLITE_DONE ){ - int jj; - char *zBuf = 0; + int nShm; /* Number of entries in apShm[] array */ + char **apShm; /* Array of mmap'd *-shm regions */ + char *zDel; /* Delete this when closing file */ - p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp); - if( !p ) goto no_mem; - memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p); - p->eType = FTSQUERY_PHRASE; - p->pPhrase = (Fts3Phrase *)&p[1]; - p->pPhrase->iColumn = pParse->iDefaultCol; - p->pPhrase->nToken = nToken; + const char *zWal; /* Wal filename for this main db file */ + rbu_file *pWalFd; /* Wal file descriptor for this main db */ + rbu_file *pMainNext; /* Next MAIN_DB file */ +}; - zBuf = (char *)&p->pPhrase->aToken[nToken]; - if( zTemp ){ - memcpy(zBuf, zTemp, nTemp); - sqlite3_free(zTemp); - }else{ - assert( nTemp==0 ); - } - for(jj=0; jjpPhrase->nToken; jj++){ - p->pPhrase->aToken[jj].z = zBuf; - zBuf += p->pPhrase->aToken[jj].n; +/************************************************************************* +** The following three functions, found below: +** +** rbuDeltaGetInt() +** rbuDeltaChecksum() +** rbuDeltaApply() +** +** are lifted from the fossil source code (http://fossil-scm.org). They +** are used to implement the scalar SQL function rbu_fossil_delta(). +*/ + +/* +** Read bytes from *pz and convert them into a positive integer. When +** finished, leave *pz pointing to the first character past the end of +** the integer. The *pLen parameter holds the length of the string +** in *pz and is decremented once for each character in the integer. +*/ +static unsigned int rbuDeltaGetInt(const char **pz, int *pLen){ + static const signed char zValue[] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, -1, -1, -1, -1, 36, + -1, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, -1, -1, -1, 63, -1, + }; + unsigned int v = 0; + int c; + unsigned char *z = (unsigned char*)*pz; + unsigned char *zStart = z; + while( (c = zValue[0x7f&*(z++)])>=0 ){ + v = (v<<6) + c; + } + z--; + *pLen -= z - zStart; + *pz = (char*)z; + return v; +} + +/* +** Compute a 32-bit checksum on the N-byte buffer. Return the result. +*/ +static unsigned int rbuDeltaChecksum(const char *zIn, size_t N){ + const unsigned char *z = (const unsigned char *)zIn; + unsigned sum0 = 0; + unsigned sum1 = 0; + unsigned sum2 = 0; + unsigned sum3 = 0; + while(N >= 16){ + sum0 += ((unsigned)z[0] + z[4] + z[8] + z[12]); + sum1 += ((unsigned)z[1] + z[5] + z[9] + z[13]); + sum2 += ((unsigned)z[2] + z[6] + z[10]+ z[14]); + sum3 += ((unsigned)z[3] + z[7] + z[11]+ z[15]); + z += 16; + N -= 16; + } + while(N >= 4){ + sum0 += z[0]; + sum1 += z[1]; + sum2 += z[2]; + sum3 += z[3]; + z += 4; + N -= 4; + } + sum3 += (sum2 << 8) + (sum1 << 16) + (sum0 << 24); + switch(N){ + case 3: sum3 += (z[2] << 8); + case 2: sum3 += (z[1] << 16); + case 1: sum3 += (z[0] << 24); + default: ; + } + return sum3; +} + +/* +** Apply a delta. +** +** The output buffer should be big enough to hold the whole output +** file and a NUL terminator at the end. The delta_output_size() +** routine will determine this size for you. +** +** The delta string should be null-terminated. But the delta string +** may contain embedded NUL characters (if the input and output are +** binary files) so we also have to pass in the length of the delta in +** the lenDelta parameter. +** +** This function returns the size of the output file in bytes (excluding +** the final NUL terminator character). Except, if the delta string is +** malformed or intended for use with a source file other than zSrc, +** then this routine returns -1. +** +** Refer to the delta_create() documentation above for a description +** of the delta file format. +*/ +static int rbuDeltaApply( + const char *zSrc, /* The source or pattern file */ + int lenSrc, /* Length of the source file */ + const char *zDelta, /* Delta to apply to the pattern */ + int lenDelta, /* Length of the delta */ + char *zOut /* Write the output into this preallocated buffer */ +){ + unsigned int limit; + unsigned int total = 0; +#ifndef FOSSIL_OMIT_DELTA_CKSUM_TEST + char *zOrigOut = zOut; +#endif + + limit = rbuDeltaGetInt(&zDelta, &lenDelta); + if( *zDelta!='\n' ){ + /* ERROR: size integer not terminated by "\n" */ + return -1; + } + zDelta++; lenDelta--; + while( *zDelta && lenDelta>0 ){ + unsigned int cnt, ofst; + cnt = rbuDeltaGetInt(&zDelta, &lenDelta); + switch( zDelta[0] ){ + case '@': { + zDelta++; lenDelta--; + ofst = rbuDeltaGetInt(&zDelta, &lenDelta); + if( lenDelta>0 && zDelta[0]!=',' ){ + /* ERROR: copy command not terminated by ',' */ + return -1; + } + zDelta++; lenDelta--; + total += cnt; + if( total>limit ){ + /* ERROR: copy exceeds output file size */ + return -1; + } + if( (int)(ofst+cnt) > lenSrc ){ + /* ERROR: copy extends past end of input */ + return -1; + } + memcpy(zOut, &zSrc[ofst], cnt); + zOut += cnt; + break; + } + case ':': { + zDelta++; lenDelta--; + total += cnt; + if( total>limit ){ + /* ERROR: insert command gives an output larger than predicted */ + return -1; + } + if( (int)cnt>lenDelta ){ + /* ERROR: insert count exceeds size of delta */ + return -1; + } + memcpy(zOut, zDelta, cnt); + zOut += cnt; + zDelta += cnt; + lenDelta -= cnt; + break; + } + case ';': { + zDelta++; lenDelta--; + zOut[0] = 0; +#ifndef FOSSIL_OMIT_DELTA_CKSUM_TEST + if( cnt!=rbuDeltaChecksum(zOrigOut, total) ){ + /* ERROR: bad checksum */ + return -1; + } +#endif + if( total!=limit ){ + /* ERROR: generated size does not match predicted size */ + return -1; + } + return total; + } + default: { + /* ERROR: unknown delta operator */ + return -1; + } } - rc = SQLITE_OK; } + /* ERROR: unterminated delta */ + return -1; +} - *ppExpr = p; - return rc; -no_mem: - - if( pCursor ){ - pModule->xClose(pCursor); +static int rbuDeltaOutputSize(const char *zDelta, int lenDelta){ + int size; + size = rbuDeltaGetInt(&zDelta, &lenDelta); + if( *zDelta!='\n' ){ + /* ERROR: size integer not terminated by "\n" */ + return -1; } - sqlite3_free(zTemp); - sqlite3_free(p); - *ppExpr = 0; - return SQLITE_NOMEM; + return size; } /* -** The output variable *ppExpr is populated with an allocated Fts3Expr -** structure, or set to 0 if the end of the input buffer is reached. +** End of code taken from fossil. +*************************************************************************/ + +/* +** Implementation of SQL scalar function rbu_fossil_delta(). ** -** Returns an SQLite error code. SQLITE_OK if everything works, SQLITE_NOMEM -** if a malloc failure occurs, or SQLITE_ERROR if a parse error is encountered. -** If SQLITE_ERROR is returned, pContext is populated with an error message. +** This function applies a fossil delta patch to a blob. Exactly two +** arguments must be passed to this function. The first is the blob to +** patch and the second the patch to apply. If no error occurs, this +** function returns the patched blob. */ -static int getNextNode( - ParseContext *pParse, /* fts3 query parse context */ - const char *z, int n, /* Input string */ - Fts3Expr **ppExpr, /* OUT: expression */ - int *pnConsumed /* OUT: Number of bytes consumed */ +static void rbuFossilDeltaFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv ){ - static const struct Fts3Keyword { - char *z; /* Keyword text */ - unsigned char n; /* Length of the keyword */ - unsigned char parenOnly; /* Only valid in paren mode */ - unsigned char eType; /* Keyword code */ - } aKeyword[] = { - { "OR" , 2, 0, FTSQUERY_OR }, - { "AND", 3, 1, FTSQUERY_AND }, - { "NOT", 3, 1, FTSQUERY_NOT }, - { "NEAR", 4, 0, FTSQUERY_NEAR } - }; - int ii; - int iCol; - int iColLen; - int rc; - Fts3Expr *pRet = 0; - - const char *zInput = z; - int nInput = n; - - pParse->isNot = 0; - - /* Skip over any whitespace before checking for a keyword, an open or - ** close bracket, or a quoted string. - */ - while( nInput>0 && fts3isspace(*zInput) ){ - nInput--; - zInput++; - } - if( nInput==0 ){ - return SQLITE_DONE; - } - - /* See if we are dealing with a keyword. */ - for(ii=0; ii<(int)(sizeof(aKeyword)/sizeof(struct Fts3Keyword)); ii++){ - const struct Fts3Keyword *pKey = &aKeyword[ii]; - - if( (pKey->parenOnly & ~sqlite3_fts3_enable_parentheses)!=0 ){ - continue; - } + const char *aDelta; + int nDelta; + const char *aOrig; + int nOrig; - if( nInput>=pKey->n && 0==memcmp(zInput, pKey->z, pKey->n) ){ - int nNear = SQLITE_FTS3_DEFAULT_NEAR_PARAM; - int nKey = pKey->n; - char cNext; + int nOut; + int nOut2; + char *aOut; - /* If this is a "NEAR" keyword, check for an explicit nearness. */ - if( pKey->eType==FTSQUERY_NEAR ){ - assert( nKey==4 ); - if( zInput[4]=='/' && zInput[5]>='0' && zInput[5]<='9' ){ - nNear = 0; - for(nKey=5; zInput[nKey]>='0' && zInput[nKey]<='9'; nKey++){ - nNear = nNear * 10 + (zInput[nKey] - '0'); - } - } - } + assert( argc==2 ); - /* At this point this is probably a keyword. But for that to be true, - ** the next byte must contain either whitespace, an open or close - ** parenthesis, a quote character, or EOF. - */ - cNext = zInput[nKey]; - if( fts3isspace(cNext) - || cNext=='"' || cNext=='(' || cNext==')' || cNext==0 - ){ - pRet = (Fts3Expr *)fts3MallocZero(sizeof(Fts3Expr)); - if( !pRet ){ - return SQLITE_NOMEM; - } - pRet->eType = pKey->eType; - pRet->nNear = nNear; - *ppExpr = pRet; - *pnConsumed = (int)((zInput - z) + nKey); - return SQLITE_OK; - } + nOrig = sqlite3_value_bytes(argv[0]); + aOrig = (const char*)sqlite3_value_blob(argv[0]); + nDelta = sqlite3_value_bytes(argv[1]); + aDelta = (const char*)sqlite3_value_blob(argv[1]); - /* Turns out that wasn't a keyword after all. This happens if the - ** user has supplied a token such as "ORacle". Continue. - */ - } + /* Figure out the size of the output */ + nOut = rbuDeltaOutputSize(aDelta, nDelta); + if( nOut<0 ){ + sqlite3_result_error(context, "corrupt fossil delta", -1); + return; } - /* See if we are dealing with a quoted phrase. If this is the case, then - ** search for the closing quote and pass the whole string to getNextString() - ** for processing. This is easy to do, as fts3 has no syntax for escaping - ** a quote character embedded in a string. - */ - if( *zInput=='"' ){ - for(ii=1; iinNest++; - rc = fts3ExprParse(pParse, zInput+1, nInput-1, ppExpr, &nConsumed); - if( rc==SQLITE_OK && !*ppExpr ){ rc = SQLITE_DONE; } - *pnConsumed = (int)(zInput - z) + 1 + nConsumed; - return rc; - }else if( *zInput==')' ){ - pParse->nNest--; - *pnConsumed = (int)((zInput - z) + 1); - *ppExpr = 0; - return SQLITE_DONE; - } - } - /* If control flows to this point, this must be a regular token, or - ** the end of the input. Read a regular token using the sqlite3_tokenizer - ** interface. Before doing so, figure out if there is an explicit - ** column specifier for the token. - ** - ** TODO: Strangely, it is not possible to associate a column specifier - ** with a quoted phrase, only with a single token. Not sure if this was - ** an implementation artifact or an intentional decision when fts3 was - ** first implemented. Whichever it was, this module duplicates the - ** limitation. - */ - iCol = pParse->iDefaultCol; - iColLen = 0; - for(ii=0; iinCol; ii++){ - const char *zStr = pParse->azCol[ii]; - int nStr = (int)strlen(zStr); - if( nInput>nStr && zInput[nStr]==':' - && sqlite3_strnicmp(zStr, zInput, nStr)==0 - ){ - iCol = ii; - iColLen = (int)((zInput - z) + nStr + 1); - break; - } +/* +** Prepare the SQL statement in buffer zSql against database handle db. +** If successful, set *ppStmt to point to the new statement and return +** SQLITE_OK. +** +** Otherwise, if an error does occur, set *ppStmt to NULL and return +** an SQLite error code. Additionally, set output variable *pzErrmsg to +** point to a buffer containing an error message. It is the responsibility +** of the caller to (eventually) free this buffer using sqlite3_free(). +*/ +static int prepareAndCollectError( + sqlite3 *db, + sqlite3_stmt **ppStmt, + char **pzErrmsg, + const char *zSql +){ + int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0); + if( rc!=SQLITE_OK ){ + *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + *ppStmt = 0; } - rc = getNextToken(pParse, iCol, &z[iColLen], n-iColLen, ppExpr, pnConsumed); - *pnConsumed += iColLen; return rc; } /* -** The argument is an Fts3Expr structure for a binary operator (any type -** except an FTSQUERY_PHRASE). Return an integer value representing the -** precedence of the operator. Lower values have a higher precedence (i.e. -** group more tightly). For example, in the C language, the == operator -** groups more tightly than ||, and would therefore have a higher precedence. +** Reset the SQL statement passed as the first argument. Return a copy +** of the value returned by sqlite3_reset(). ** -** When using the new fts3 query syntax (when SQLITE_ENABLE_FTS3_PARENTHESIS -** is defined), the order of the operators in precedence from highest to -** lowest is: -** -** NEAR -** NOT -** AND (including implicit ANDs) -** OR -** -** Note that when using the old query syntax, the OR operator has a higher -** precedence than the AND operator. +** If an error has occurred, then set *pzErrmsg to point to a buffer +** containing an error message. It is the responsibility of the caller +** to eventually free this buffer using sqlite3_free(). */ -static int opPrecedence(Fts3Expr *p){ - assert( p->eType!=FTSQUERY_PHRASE ); - if( sqlite3_fts3_enable_parentheses ){ - return p->eType; - }else if( p->eType==FTSQUERY_NEAR ){ - return 1; - }else if( p->eType==FTSQUERY_OR ){ - return 2; +static int resetAndCollectError(sqlite3_stmt *pStmt, char **pzErrmsg){ + int rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ){ + *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(sqlite3_db_handle(pStmt))); } - assert( p->eType==FTSQUERY_AND ); - return 3; + return rc; } /* -** Argument ppHead contains a pointer to the current head of a query -** expression tree being parsed. pPrev is the expression node most recently -** inserted into the tree. This function adds pNew, which is always a binary -** operator node, into the expression tree based on the relative precedence -** of pNew and the existing nodes of the tree. This may result in the head -** of the tree changing, in which case *ppHead is set to the new root node. +** Unless it is NULL, argument zSql points to a buffer allocated using +** sqlite3_malloc containing an SQL statement. This function prepares the SQL +** statement against database db and frees the buffer. If statement +** compilation is successful, *ppStmt is set to point to the new statement +** handle and SQLITE_OK is returned. +** +** Otherwise, if an error occurs, *ppStmt is set to NULL and an error code +** returned. In this case, *pzErrmsg may also be set to point to an error +** message. It is the responsibility of the caller to free this error message +** buffer using sqlite3_free(). +** +** If argument zSql is NULL, this function assumes that an OOM has occurred. +** In this case SQLITE_NOMEM is returned and *ppStmt set to NULL. */ -static void insertBinaryOperator( - Fts3Expr **ppHead, /* Pointer to the root node of a tree */ - Fts3Expr *pPrev, /* Node most recently inserted into the tree */ - Fts3Expr *pNew /* New binary node to insert into expression tree */ +static int prepareFreeAndCollectError( + sqlite3 *db, + sqlite3_stmt **ppStmt, + char **pzErrmsg, + char *zSql ){ - Fts3Expr *pSplit = pPrev; - while( pSplit->pParent && opPrecedence(pSplit->pParent)<=opPrecedence(pNew) ){ - pSplit = pSplit->pParent; - } - - if( pSplit->pParent ){ - assert( pSplit->pParent->pRight==pSplit ); - pSplit->pParent->pRight = pNew; - pNew->pParent = pSplit->pParent; + int rc; + assert( *pzErrmsg==0 ); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + *ppStmt = 0; }else{ - *ppHead = pNew; + rc = prepareAndCollectError(db, ppStmt, pzErrmsg, zSql); + sqlite3_free(zSql); } - pNew->pLeft = pSplit; - pSplit->pParent = pNew; + return rc; } /* -** Parse the fts3 query expression found in buffer z, length n. This function -** returns either when the end of the buffer is reached or an unmatched -** closing bracket - ')' - is encountered. -** -** If successful, SQLITE_OK is returned, *ppExpr is set to point to the -** parsed form of the expression and *pnConsumed is set to the number of -** bytes read from buffer z. Otherwise, *ppExpr is set to 0 and SQLITE_NOMEM -** (out of memory error) or SQLITE_ERROR (parse error) is returned. +** Free the RbuObjIter.azTblCol[] and RbuObjIter.abTblPk[] arrays allocated +** by an earlier call to rbuObjIterCacheTableInfo(). */ -static int fts3ExprParse( - ParseContext *pParse, /* fts3 query parse context */ - const char *z, int n, /* Text of MATCH query */ - Fts3Expr **ppExpr, /* OUT: Parsed query structure */ - int *pnConsumed /* OUT: Number of bytes consumed */ -){ - Fts3Expr *pRet = 0; - Fts3Expr *pPrev = 0; - Fts3Expr *pNotBranch = 0; /* Only used in legacy parse mode */ - int nIn = n; - const char *zIn = z; - int rc = SQLITE_OK; - int isRequirePhrase = 1; - - while( rc==SQLITE_OK ){ - Fts3Expr *p = 0; - int nByte = 0; - - rc = getNextNode(pParse, zIn, nIn, &p, &nByte); - assert( nByte>0 || (rc!=SQLITE_OK && p==0) ); - if( rc==SQLITE_OK ){ - if( p ){ - int isPhrase; - - if( !sqlite3_fts3_enable_parentheses - && p->eType==FTSQUERY_PHRASE && pParse->isNot - ){ - /* Create an implicit NOT operator. */ - Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); - if( !pNot ){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_NOMEM; - goto exprparse_out; - } - pNot->eType = FTSQUERY_NOT; - pNot->pRight = p; - p->pParent = pNot; - if( pNotBranch ){ - pNot->pLeft = pNotBranch; - pNotBranch->pParent = pNot; - } - pNotBranch = pNot; - p = pPrev; - }else{ - int eType = p->eType; - isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); - - /* The isRequirePhrase variable is set to true if a phrase or - ** an expression contained in parenthesis is required. If a - ** binary operator (AND, OR, NOT or NEAR) is encounted when - ** isRequirePhrase is set, this is a syntax error. - */ - if( !isPhrase && isRequirePhrase ){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_ERROR; - goto exprparse_out; - } - - if( isPhrase && !isRequirePhrase ){ - /* Insert an implicit AND operator. */ - Fts3Expr *pAnd; - assert( pRet && pPrev ); - pAnd = fts3MallocZero(sizeof(Fts3Expr)); - if( !pAnd ){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_NOMEM; - goto exprparse_out; - } - pAnd->eType = FTSQUERY_AND; - insertBinaryOperator(&pRet, pPrev, pAnd); - pPrev = pAnd; - } - - /* This test catches attempts to make either operand of a NEAR - ** operator something other than a phrase. For example, either of - ** the following: - ** - ** (bracketed expression) NEAR phrase - ** phrase NEAR (bracketed expression) - ** - ** Return an error in either case. - */ - if( pPrev && ( - (eType==FTSQUERY_NEAR && !isPhrase && pPrev->eType!=FTSQUERY_PHRASE) - || (eType!=FTSQUERY_PHRASE && isPhrase && pPrev->eType==FTSQUERY_NEAR) - )){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_ERROR; - goto exprparse_out; - } - - if( isPhrase ){ - if( pRet ){ - assert( pPrev && pPrev->pLeft && pPrev->pRight==0 ); - pPrev->pRight = p; - p->pParent = pPrev; - }else{ - pRet = p; - } - }else{ - insertBinaryOperator(&pRet, pPrev, p); - } - isRequirePhrase = !isPhrase; - } - pPrev = p; - } - assert( nByte>0 ); - } - assert( rc!=SQLITE_OK || (nByte>0 && nByte<=nIn) ); - nIn -= nByte; - zIn += nByte; - } - - if( rc==SQLITE_DONE && pRet && isRequirePhrase ){ - rc = SQLITE_ERROR; - } - - if( rc==SQLITE_DONE ){ - rc = SQLITE_OK; - if( !sqlite3_fts3_enable_parentheses && pNotBranch ){ - if( !pRet ){ - rc = SQLITE_ERROR; - }else{ - Fts3Expr *pIter = pNotBranch; - while( pIter->pLeft ){ - pIter = pIter->pLeft; - } - pIter->pLeft = pRet; - pRet->pParent = pIter; - pRet = pNotBranch; - } - } - } - *pnConsumed = n - nIn; - -exprparse_out: - if( rc!=SQLITE_OK ){ - sqlite3Fts3ExprFree(pRet); - sqlite3Fts3ExprFree(pNotBranch); - pRet = 0; +static void rbuObjIterFreeCols(RbuObjIter *pIter){ + int i; + for(i=0; inTblCol; i++){ + sqlite3_free(pIter->azTblCol[i]); + sqlite3_free(pIter->azTblType[i]); + } + sqlite3_free(pIter->azTblCol); + pIter->azTblCol = 0; + pIter->azTblType = 0; + pIter->aiSrcOrder = 0; + pIter->abTblPk = 0; + pIter->abNotNull = 0; + pIter->nTblCol = 0; + pIter->eType = 0; /* Invalid value */ +} + +/* +** Finalize all statements and free all allocations that are specific to +** the current object (table/index pair). +*/ +static void rbuObjIterClearStatements(RbuObjIter *pIter){ + RbuUpdateStmt *pUp; + + sqlite3_finalize(pIter->pSelect); + sqlite3_finalize(pIter->pInsert); + sqlite3_finalize(pIter->pDelete); + sqlite3_finalize(pIter->pTmpInsert); + pUp = pIter->pRbuUpdate; + while( pUp ){ + RbuUpdateStmt *pTmp = pUp->pNext; + sqlite3_finalize(pUp->pUpdate); + sqlite3_free(pUp); + pUp = pTmp; } - *ppExpr = pRet; - return rc; + + pIter->pSelect = 0; + pIter->pInsert = 0; + pIter->pDelete = 0; + pIter->pRbuUpdate = 0; + pIter->pTmpInsert = 0; + pIter->nCol = 0; } /* -** Return SQLITE_ERROR if the maximum depth of the expression tree passed -** as the only argument is more than nMaxDepth. +** Clean up any resources allocated as part of the iterator object passed +** as the only argument. */ -static int fts3ExprCheckDepth(Fts3Expr *p, int nMaxDepth){ - int rc = SQLITE_OK; - if( p ){ - if( nMaxDepth<0 ){ - rc = SQLITE_TOOBIG; - }else{ - rc = fts3ExprCheckDepth(p->pLeft, nMaxDepth-1); - if( rc==SQLITE_OK ){ - rc = fts3ExprCheckDepth(p->pRight, nMaxDepth-1); - } - } - } - return rc; +static void rbuObjIterFinalize(RbuObjIter *pIter){ + rbuObjIterClearStatements(pIter); + sqlite3_finalize(pIter->pTblIter); + sqlite3_finalize(pIter->pIdxIter); + rbuObjIterFreeCols(pIter); + memset(pIter, 0, sizeof(RbuObjIter)); } /* -** This function attempts to transform the expression tree at (*pp) to -** an equivalent but more balanced form. The tree is modified in place. -** If successful, SQLITE_OK is returned and (*pp) set to point to the -** new root expression node. -** -** nMaxDepth is the maximum allowable depth of the balanced sub-tree. +** Advance the iterator to the next position. ** -** Otherwise, if an error occurs, an SQLite error code is returned and -** expression (*pp) freed. +** If no error occurs, SQLITE_OK is returned and the iterator is left +** pointing to the next entry. Otherwise, an error code and message is +** left in the RBU handle passed as the first argument. A copy of the +** error code is returned. */ -static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){ - int rc = SQLITE_OK; /* Return code */ - Fts3Expr *pRoot = *pp; /* Initial root node */ - Fts3Expr *pFree = 0; /* List of free nodes. Linked by pParent. */ - int eType = pRoot->eType; /* Type of node in this tree */ - - if( nMaxDepth==0 ){ - rc = SQLITE_ERROR; - } +static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ + int rc = p->rc; + if( rc==SQLITE_OK ){ - if( rc==SQLITE_OK && (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){ - Fts3Expr **apLeaf; - apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth); - if( 0==apLeaf ){ - rc = SQLITE_NOMEM; - }else{ - memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth); + /* Free any SQLite statements used while processing the previous object */ + rbuObjIterClearStatements(pIter); + if( pIter->zIdx==0 ){ + rc = sqlite3_exec(p->dbMain, + "DROP TRIGGER IF EXISTS temp.rbu_insert_tr;" + "DROP TRIGGER IF EXISTS temp.rbu_update1_tr;" + "DROP TRIGGER IF EXISTS temp.rbu_update2_tr;" + "DROP TRIGGER IF EXISTS temp.rbu_delete_tr;" + , 0, 0, &p->zErrmsg + ); } if( rc==SQLITE_OK ){ - int i; - Fts3Expr *p; - - /* Set $p to point to the left-most leaf in the tree of eType nodes. */ - for(p=pRoot; p->eType==eType; p=p->pLeft){ - assert( p->pParent==0 || p->pParent->pLeft==p ); - assert( p->pLeft && p->pRight ); - } - - /* This loop runs once for each leaf in the tree of eType nodes. */ - while( 1 ){ - int iLvl; - Fts3Expr *pParent = p->pParent; /* Current parent of p */ - - assert( pParent==0 || pParent->pLeft==p ); - p->pParent = 0; - if( pParent ){ - pParent->pLeft = 0; - }else{ - pRoot = 0; - } - rc = fts3ExprBalance(&p, nMaxDepth-1); - if( rc!=SQLITE_OK ) break; - - for(iLvl=0; p && iLvlpLeft = apLeaf[iLvl]; - pFree->pRight = p; - pFree->pLeft->pParent = pFree; - pFree->pRight->pParent = pFree; - - p = pFree; - pFree = pFree->pParent; - p->pParent = 0; - apLeaf[iLvl] = 0; - } - } - if( p ){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_TOOBIG; - break; - } - - /* If that was the last leaf node, break out of the loop */ - if( pParent==0 ) break; - - /* Set $p to point to the next leaf in the tree of eType nodes */ - for(p=pParent->pRight; p->eType==eType; p=p->pLeft); - - /* Remove pParent from the original tree. */ - assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent ); - pParent->pRight->pParent = pParent->pParent; - if( pParent->pParent ){ - pParent->pParent->pLeft = pParent->pRight; + if( pIter->bCleanup ){ + rbuObjIterFreeCols(pIter); + pIter->bCleanup = 0; + rc = sqlite3_step(pIter->pTblIter); + if( rc!=SQLITE_ROW ){ + rc = resetAndCollectError(pIter->pTblIter, &p->zErrmsg); + pIter->zTbl = 0; }else{ - assert( pParent==pRoot ); - pRoot = pParent->pRight; - } - - /* Link pParent into the free node list. It will be used as an - ** internal node of the new tree. */ - pParent->pParent = pFree; - pFree = pParent; - } - - if( rc==SQLITE_OK ){ - p = 0; - for(i=0; ipParent = 0; - }else{ - assert( pFree!=0 ); - pFree->pRight = p; - pFree->pLeft = apLeaf[i]; - pFree->pLeft->pParent = pFree; - pFree->pRight->pParent = pFree; - - p = pFree; - pFree = pFree->pParent; - p->pParent = 0; - } - } + pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0); + pIter->zDataTbl = (const char*)sqlite3_column_text(pIter->pTblIter,1); + rc = (pIter->zDataTbl && pIter->zTbl) ? SQLITE_OK : SQLITE_NOMEM; } - pRoot = p; }else{ - /* An error occurred. Delete the contents of the apLeaf[] array - ** and pFree list. Everything else is cleaned up by the call to - ** sqlite3Fts3ExprFree(pRoot) below. */ - Fts3Expr *pDel; - for(i=0; izIdx==0 ){ + sqlite3_stmt *pIdx = pIter->pIdxIter; + rc = sqlite3_bind_text(pIdx, 1, pIter->zTbl, -1, SQLITE_STATIC); } - while( (pDel=pFree)!=0 ){ - pFree = pDel->pParent; - sqlite3_free(pDel); + if( rc==SQLITE_OK ){ + rc = sqlite3_step(pIter->pIdxIter); + if( rc!=SQLITE_ROW ){ + rc = resetAndCollectError(pIter->pIdxIter, &p->zErrmsg); + pIter->bCleanup = 1; + pIter->zIdx = 0; + }else{ + pIter->zIdx = (const char*)sqlite3_column_text(pIter->pIdxIter, 0); + pIter->iTnum = sqlite3_column_int(pIter->pIdxIter, 1); + pIter->bUnique = sqlite3_column_int(pIter->pIdxIter, 2); + rc = pIter->zIdx ? SQLITE_OK : SQLITE_NOMEM; + } } } - - assert( pFree==0 ); - sqlite3_free( apLeaf ); } } if( rc!=SQLITE_OK ){ - sqlite3Fts3ExprFree(pRoot); - pRoot = 0; + rbuObjIterFinalize(pIter); + p->rc = rc; } - *pp = pRoot; return rc; } + /* -** This function is similar to sqlite3Fts3ExprParse(), with the following -** differences: +** The implementation of the rbu_target_name() SQL function. This function +** accepts one argument - the name of a table in the RBU database. If the +** table name matches the pattern: ** -** 1. It does not do expression rebalancing. -** 2. It does not check that the expression does not exceed the -** maximum allowable depth. -** 3. Even if it fails, *ppExpr may still be set to point to an -** expression tree. It should be deleted using sqlite3Fts3ExprFree() -** in this case. +** data[0-9]_ +** +** where is any sequence of 1 or more characters, is returned. +** Otherwise, if the only argument does not match the above pattern, an SQL +** NULL is returned. +** +** "data_t1" -> "t1" +** "data0123_t2" -> "t2" +** "dataAB_t3" -> NULL */ -static int fts3ExprParseUnbalanced( - sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ - int iLangid, /* Language id for tokenizer */ - char **azCol, /* Array of column names for fts3 table */ - int bFts4, /* True to allow FTS4-only syntax */ - int nCol, /* Number of entries in azCol[] */ - int iDefaultCol, /* Default column to query */ - const char *z, int n, /* Text of MATCH query */ - Fts3Expr **ppExpr /* OUT: Parsed query structure */ +static void rbuTargetNameFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv ){ - int nParsed; - int rc; - ParseContext sParse; + const char *zIn; + assert( argc==1 ); - memset(&sParse, 0, sizeof(ParseContext)); - sParse.pTokenizer = pTokenizer; - sParse.iLangid = iLangid; - sParse.azCol = (const char **)azCol; - sParse.nCol = nCol; - sParse.iDefaultCol = iDefaultCol; - sParse.bFts4 = bFts4; - if( z==0 ){ - *ppExpr = 0; - return SQLITE_OK; - } - if( n<0 ){ - n = (int)strlen(z); + zIn = (const char*)sqlite3_value_text(argv[0]); + if( zIn && strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){ + int i; + for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++); + if( zIn[i]=='_' && zIn[i+1] ){ + sqlite3_result_text(context, &zIn[i+1], -1, SQLITE_STATIC); + } } - rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed); - assert( rc==SQLITE_OK || *ppExpr==0 ); +} - /* Check for mismatched parenthesis */ - if( rc==SQLITE_OK && sParse.nNest ){ - rc = SQLITE_ERROR; +/* +** Initialize the iterator structure passed as the second argument. +** +** If no error occurs, SQLITE_OK is returned and the iterator is left +** pointing to the first entry. Otherwise, an error code and message is +** left in the RBU handle passed as the first argument. A copy of the +** error code is returned. +*/ +static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){ + int rc; + memset(pIter, 0, sizeof(RbuObjIter)); + + rc = prepareAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, + "SELECT rbu_target_name(name) AS target, name FROM sqlite_master " + "WHERE type IN ('table', 'view') AND target IS NOT NULL " + "ORDER BY name" + ); + + if( rc==SQLITE_OK ){ + rc = prepareAndCollectError(p->dbMain, &pIter->pIdxIter, &p->zErrmsg, + "SELECT name, rootpage, sql IS NULL OR substr(8, 6)=='UNIQUE' " + " FROM main.sqlite_master " + " WHERE type='index' AND tbl_name = ?" + ); } - - return rc; + + pIter->bCleanup = 1; + p->rc = rc; + return rbuObjIterNext(p, pIter); } /* -** Parameters z and n contain a pointer to and length of a buffer containing -** an fts3 query expression, respectively. This function attempts to parse the -** query expression and create a tree of Fts3Expr structures representing the -** parsed expression. If successful, *ppExpr is set to point to the head -** of the parsed expression tree and SQLITE_OK is returned. If an error -** occurs, either SQLITE_NOMEM (out-of-memory error) or SQLITE_ERROR (parse -** error) is returned and *ppExpr is set to 0. -** -** If parameter n is a negative number, then z is assumed to point to a -** nul-terminated string and the length is determined using strlen(). +** This is a wrapper around "sqlite3_mprintf(zFmt, ...)". If an OOM occurs, +** an error code is stored in the RBU handle passed as the first argument. ** -** The first parameter, pTokenizer, is passed the fts3 tokenizer module to -** use to normalize query tokens while parsing the expression. The azCol[] -** array, which is assumed to contain nCol entries, should contain the names -** of each column in the target fts3 table, in order from left to right. -** Column names must be nul-terminated strings. +** If an error has already occurred (p->rc is already set to something other +** than SQLITE_OK), then this function returns NULL without modifying the +** stored error code. In this case it still calls sqlite3_free() on any +** printf() parameters associated with %z conversions. +*/ +static char *rbuMPrintf(sqlite3rbu *p, const char *zFmt, ...){ + char *zSql = 0; + va_list ap; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK ){ + if( zSql==0 ) p->rc = SQLITE_NOMEM; + }else{ + sqlite3_free(zSql); + zSql = 0; + } + va_end(ap); + return zSql; +} + +/* +** Argument zFmt is a sqlite3_mprintf() style format string. The trailing +** arguments are the usual subsitution values. This function performs +** the printf() style substitutions and executes the result as an SQL +** statement on the RBU handles database. ** -** The iDefaultCol parameter should be passed the index of the table column -** that appears on the left-hand-side of the MATCH operator (the default -** column to match against for tokens for which a column name is not explicitly -** specified as part of the query string), or -1 if tokens may by default -** match any table column. +** If an error occurs, an error code and error message is stored in the +** RBU handle. If an error has already occurred when this function is +** called, it is a no-op. */ -SQLITE_PRIVATE int sqlite3Fts3ExprParse( - sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ - int iLangid, /* Language id for tokenizer */ - char **azCol, /* Array of column names for fts3 table */ - int bFts4, /* True to allow FTS4-only syntax */ - int nCol, /* Number of entries in azCol[] */ - int iDefaultCol, /* Default column to query */ - const char *z, int n, /* Text of MATCH query */ - Fts3Expr **ppExpr, /* OUT: Parsed query structure */ - char **pzErr /* OUT: Error message (sqlite3_malloc) */ -){ - int rc = fts3ExprParseUnbalanced( - pTokenizer, iLangid, azCol, bFts4, nCol, iDefaultCol, z, n, ppExpr - ); - - /* Rebalance the expression. And check that its depth does not exceed - ** SQLITE_FTS3_MAX_EXPR_DEPTH. */ - if( rc==SQLITE_OK && *ppExpr ){ - rc = fts3ExprBalance(ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH); - if( rc==SQLITE_OK ){ - rc = fts3ExprCheckDepth(*ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH); +static int rbuMPrintfExec(sqlite3rbu *p, sqlite3 *db, const char *zFmt, ...){ + va_list ap; + char *zSql; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK ){ + if( zSql==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + p->rc = sqlite3_exec(db, zSql, 0, 0, &p->zErrmsg); } } + sqlite3_free(zSql); + va_end(ap); + return p->rc; +} - if( rc!=SQLITE_OK ){ - sqlite3Fts3ExprFree(*ppExpr); - *ppExpr = 0; - if( rc==SQLITE_TOOBIG ){ - sqlite3Fts3ErrMsg(pzErr, - "FTS expression tree is too large (maximum depth %d)", - SQLITE_FTS3_MAX_EXPR_DEPTH - ); - rc = SQLITE_ERROR; - }else if( rc==SQLITE_ERROR ){ - sqlite3Fts3ErrMsg(pzErr, "malformed MATCH expression: [%s]", z); +/* +** Attempt to allocate and return a pointer to a zeroed block of nByte +** bytes. +** +** If an error (i.e. an OOM condition) occurs, return NULL and leave an +** error code in the rbu handle passed as the first argument. Or, if an +** error has already occurred when this function is called, return NULL +** immediately without attempting the allocation or modifying the stored +** error code. +*/ +static void *rbuMalloc(sqlite3rbu *p, int nByte){ + void *pRet = 0; + if( p->rc==SQLITE_OK ){ + assert( nByte>0 ); + pRet = sqlite3_malloc(nByte); + if( pRet==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + memset(pRet, 0, nByte); } } + return pRet; +} - return rc; + +/* +** Allocate and zero the pIter->azTblCol[] and abTblPk[] arrays so that +** there is room for at least nCol elements. If an OOM occurs, store an +** error code in the RBU handle passed as the first argument. +*/ +static void rbuAllocateIterArrays(sqlite3rbu *p, RbuObjIter *pIter, int nCol){ + int nByte = (2*sizeof(char*) + sizeof(int) + 3*sizeof(u8)) * nCol; + char **azNew; + + azNew = (char**)rbuMalloc(p, nByte); + if( azNew ){ + pIter->azTblCol = azNew; + pIter->azTblType = &azNew[nCol]; + pIter->aiSrcOrder = (int*)&pIter->azTblType[nCol]; + pIter->abTblPk = (u8*)&pIter->aiSrcOrder[nCol]; + pIter->abNotNull = (u8*)&pIter->abTblPk[nCol]; + pIter->abIndexed = (u8*)&pIter->abNotNull[nCol]; + } } /* -** Free a single node of an expression tree. +** The first argument must be a nul-terminated string. This function +** returns a copy of the string in memory obtained from sqlite3_malloc(). +** It is the responsibility of the caller to eventually free this memory +** using sqlite3_free(). +** +** If an OOM condition is encountered when attempting to allocate memory, +** output variable (*pRc) is set to SQLITE_NOMEM before returning. Otherwise, +** if the allocation succeeds, (*pRc) is left unchanged. */ -static void fts3FreeExprNode(Fts3Expr *p){ - assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); - sqlite3Fts3EvalPhraseCleanup(p->pPhrase); - sqlite3_free(p->aMI); - sqlite3_free(p); +static char *rbuStrndup(const char *zStr, int *pRc){ + char *zRet = 0; + + assert( *pRc==SQLITE_OK ); + if( zStr ){ + int nCopy = strlen(zStr) + 1; + zRet = (char*)sqlite3_malloc(nCopy); + if( zRet ){ + memcpy(zRet, zStr, nCopy); + }else{ + *pRc = SQLITE_NOMEM; + } + } + + return zRet; } /* -** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse(). +** Finalize the statement passed as the second argument. ** -** This function would be simpler if it recursively called itself. But -** that would mean passing a sufficiently large expression to ExprParse() -** could cause a stack overflow. +** If the sqlite3_finalize() call indicates that an error occurs, and the +** rbu handle error code is not already set, set the error code and error +** message accordingly. */ -SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *pDel){ - Fts3Expr *p; - assert( pDel==0 || pDel->pParent==0 ); - for(p=pDel; p && (p->pLeft||p->pRight); p=(p->pLeft ? p->pLeft : p->pRight)){ - assert( p->pParent==0 || p==p->pParent->pRight || p==p->pParent->pLeft ); +static void rbuFinalize(sqlite3rbu *p, sqlite3_stmt *pStmt){ + sqlite3 *db = sqlite3_db_handle(pStmt); + int rc = sqlite3_finalize(pStmt); + if( p->rc==SQLITE_OK && rc!=SQLITE_OK ){ + p->rc = rc; + p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } +} + +/* Determine the type of a table. +** +** peType is of type (int*), a pointer to an output parameter of type +** (int). This call sets the output parameter as follows, depending +** on the type of the table specified by parameters dbName and zTbl. +** +** RBU_PK_NOTABLE: No such table. +** RBU_PK_NONE: Table has an implicit rowid. +** RBU_PK_IPK: Table has an explicit IPK column. +** RBU_PK_EXTERNAL: Table has an external PK index. +** RBU_PK_WITHOUT_ROWID: Table is WITHOUT ROWID. +** RBU_PK_VTAB: Table is a virtual table. +** +** Argument *piPk is also of type (int*), and also points to an output +** parameter. Unless the table has an external primary key index +** (i.e. unless *peType is set to 3), then *piPk is set to zero. Or, +** if the table does have an external primary key index, then *piPk +** is set to the root page number of the primary key index before +** returning. +** +** ALGORITHM: +** +** if( no entry exists in sqlite_master ){ +** return RBU_PK_NOTABLE +** }else if( sql for the entry starts with "CREATE VIRTUAL" ){ +** return RBU_PK_VTAB +** }else if( "PRAGMA index_list()" for the table contains a "pk" index ){ +** if( the index that is the pk exists in sqlite_master ){ +** *piPK = rootpage of that index. +** return RBU_PK_EXTERNAL +** }else{ +** return RBU_PK_WITHOUT_ROWID +** } +** }else if( "PRAGMA table_info()" lists one or more "pk" columns ){ +** return RBU_PK_IPK +** }else{ +** return RBU_PK_NONE +** } +*/ +static void rbuTableType( + sqlite3rbu *p, + const char *zTab, + int *peType, + int *piTnum, + int *piPk +){ + /* + ** 0) SELECT count(*) FROM sqlite_master where name=%Q AND IsVirtual(%Q) + ** 1) PRAGMA index_list = ? + ** 2) SELECT count(*) FROM sqlite_master where name=%Q + ** 3) PRAGMA table_info = ? + */ + sqlite3_stmt *aStmt[4] = {0, 0, 0, 0}; + + *peType = RBU_PK_NOTABLE; + *piPk = 0; + + assert( p->rc==SQLITE_OK ); + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[0], &p->zErrmsg, + sqlite3_mprintf( + "SELECT (sql LIKE 'create virtual%%'), rootpage" + " FROM sqlite_master" + " WHERE name=%Q", zTab + )); + if( p->rc!=SQLITE_OK || sqlite3_step(aStmt[0])!=SQLITE_ROW ){ + /* Either an error, or no such table. */ + goto rbuTableType_end; } - while( p ){ - Fts3Expr *pParent = p->pParent; - fts3FreeExprNode(p); - if( pParent && p==pParent->pLeft && pParent->pRight ){ - p = pParent->pRight; - while( p && (p->pLeft || p->pRight) ){ - assert( p==p->pParent->pRight || p==p->pParent->pLeft ); - p = (p->pLeft ? p->pLeft : p->pRight); + if( sqlite3_column_int(aStmt[0], 0) ){ + *peType = RBU_PK_VTAB; /* virtual table */ + goto rbuTableType_end; + } + *piTnum = sqlite3_column_int(aStmt[0], 1); + + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[1], &p->zErrmsg, + sqlite3_mprintf("PRAGMA index_list=%Q",zTab) + ); + if( p->rc ) goto rbuTableType_end; + while( sqlite3_step(aStmt[1])==SQLITE_ROW ){ + const u8 *zOrig = sqlite3_column_text(aStmt[1], 3); + const u8 *zIdx = sqlite3_column_text(aStmt[1], 1); + if( zOrig && zIdx && zOrig[0]=='p' ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[2], &p->zErrmsg, + sqlite3_mprintf( + "SELECT rootpage FROM sqlite_master WHERE name = %Q", zIdx + )); + if( p->rc==SQLITE_OK ){ + if( sqlite3_step(aStmt[2])==SQLITE_ROW ){ + *piPk = sqlite3_column_int(aStmt[2], 0); + *peType = RBU_PK_EXTERNAL; + }else{ + *peType = RBU_PK_WITHOUT_ROWID; + } } - }else{ - p = pParent; + goto rbuTableType_end; } } -} - -/**************************************************************************** -***************************************************************************** -** Everything after this point is just test code. -*/ -#ifdef SQLITE_TEST + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[3], &p->zErrmsg, + sqlite3_mprintf("PRAGMA table_info=%Q",zTab) + ); + if( p->rc==SQLITE_OK ){ + while( sqlite3_step(aStmt[3])==SQLITE_ROW ){ + if( sqlite3_column_int(aStmt[3],5)>0 ){ + *peType = RBU_PK_IPK; /* explicit IPK column */ + goto rbuTableType_end; + } + } + *peType = RBU_PK_NONE; + } -/* #include */ +rbuTableType_end: { + unsigned int i; + for(i=0; iabIndexed[] array. */ -static int queryTestTokenizer( - sqlite3 *db, - const char *zName, - const sqlite3_tokenizer_module **pp -){ - int rc; - sqlite3_stmt *pStmt; - const char zSql[] = "SELECT fts3_tokenizer(?)"; +static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){ + sqlite3_stmt *pList = 0; + int bIndex = 0; - *pp = 0; - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - if( rc!=SQLITE_OK ){ - return rc; + if( p->rc==SQLITE_OK ){ + memcpy(pIter->abIndexed, pIter->abTblPk, sizeof(u8)*pIter->nTblCol); + p->rc = prepareFreeAndCollectError(p->dbMain, &pList, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl) + ); } - sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ - memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pList) ){ + const char *zIdx = (const char*)sqlite3_column_text(pList, 1); + sqlite3_stmt *pXInfo = 0; + if( zIdx==0 ) break; + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) + ); + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + int iCid = sqlite3_column_int(pXInfo, 1); + if( iCid>=0 ) pIter->abIndexed[iCid] = 1; } + rbuFinalize(p, pXInfo); + bIndex = 1; } - return sqlite3_finalize(pStmt); + rbuFinalize(p, pList); + if( bIndex==0 ) pIter->abIndexed = 0; } + /* -** Return a pointer to a buffer containing a text representation of the -** expression passed as the first argument. The buffer is obtained from -** sqlite3_malloc(). It is the responsibility of the caller to use -** sqlite3_free() to release the memory. If an OOM condition is encountered, -** NULL is returned. +** If they are not already populated, populate the pIter->azTblCol[], +** pIter->abTblPk[], pIter->nTblCol and pIter->bRowid variables according to +** the table (not index) that the iterator currently points to. ** -** If the second argument is not NULL, then its contents are prepended to -** the returned expression text and then freed using sqlite3_free(). +** Return SQLITE_OK if successful, or an SQLite error code otherwise. If +** an error does occur, an error code and error message are also left in +** the RBU handle. */ -static char *exprToString(Fts3Expr *pExpr, char *zBuf){ - if( pExpr==0 ){ - return sqlite3_mprintf(""); - } - switch( pExpr->eType ){ - case FTSQUERY_PHRASE: { - Fts3Phrase *pPhrase = pExpr->pPhrase; - int i; - zBuf = sqlite3_mprintf( - "%zPHRASE %d 0", zBuf, pPhrase->iColumn); - for(i=0; zBuf && inToken; i++){ - zBuf = sqlite3_mprintf("%z %.*s%s", zBuf, - pPhrase->aToken[i].n, pPhrase->aToken[i].z, - (pPhrase->aToken[i].isPrefix?"+":"") +static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ + if( pIter->azTblCol==0 ){ + sqlite3_stmt *pStmt = 0; + int nCol = 0; + int i; /* for() loop iterator variable */ + int bRbuRowid = 0; /* If input table has column "rbu_rowid" */ + int iOrder = 0; + int iTnum = 0; + + /* Figure out the type of table this step will deal with. */ + assert( pIter->eType==0 ); + rbuTableType(p, pIter->zTbl, &pIter->eType, &iTnum, &pIter->iPkTnum); + if( p->rc==SQLITE_OK && pIter->eType==RBU_PK_NOTABLE ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("no such table: %s", pIter->zTbl); + } + if( p->rc ) return p->rc; + if( pIter->zIdx==0 ) pIter->iTnum = iTnum; + + assert( pIter->eType==RBU_PK_NONE || pIter->eType==RBU_PK_IPK + || pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_WITHOUT_ROWID + || pIter->eType==RBU_PK_VTAB + ); + + /* Populate the azTblCol[] and nTblCol variables based on the columns + ** of the input table. Ignore any input table columns that begin with + ** "rbu_". */ + p->rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + sqlite3_mprintf("SELECT * FROM '%q'", pIter->zDataTbl) + ); + if( p->rc==SQLITE_OK ){ + nCol = sqlite3_column_count(pStmt); + rbuAllocateIterArrays(p, pIter, nCol); + } + for(i=0; p->rc==SQLITE_OK && irc); + pIter->aiSrcOrder[pIter->nTblCol] = pIter->nTblCol; + pIter->azTblCol[pIter->nTblCol++] = zCopy; + } + else if( 0==sqlite3_stricmp("rbu_rowid", zName) ){ + bRbuRowid = 1; + } + } + sqlite3_finalize(pStmt); + pStmt = 0; + + if( p->rc==SQLITE_OK + && bRbuRowid!=(pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE) + ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf( + "table %q %s rbu_rowid column", pIter->zDataTbl, + (bRbuRowid ? "may not have" : "requires") + ); + } + + /* Check that all non-HIDDEN columns in the destination table are also + ** present in the input table. Populate the abTblPk[], azTblType[] and + ** aiTblOrder[] arrays at the same time. */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &p->zErrmsg, + sqlite3_mprintf("PRAGMA table_info(%Q)", pIter->zTbl) + ); + } + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zName = (const char*)sqlite3_column_text(pStmt, 1); + if( zName==0 ) break; /* An OOM - finalize() below returns S_NOMEM */ + for(i=iOrder; inTblCol; i++){ + if( 0==strcmp(zName, pIter->azTblCol[i]) ) break; + } + if( i==pIter->nTblCol ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("column missing from %q: %s", + pIter->zDataTbl, zName ); + }else{ + int iPk = sqlite3_column_int(pStmt, 5); + int bNotNull = sqlite3_column_int(pStmt, 3); + const char *zType = (const char*)sqlite3_column_text(pStmt, 2); + + if( i!=iOrder ){ + SWAP(int, pIter->aiSrcOrder[i], pIter->aiSrcOrder[iOrder]); + SWAP(char*, pIter->azTblCol[i], pIter->azTblCol[iOrder]); + } + + pIter->azTblType[iOrder] = rbuStrndup(zType, &p->rc); + pIter->abTblPk[iOrder] = (iPk!=0); + pIter->abNotNull[iOrder] = (u8)bNotNull || (iPk!=0); + iOrder++; } - return zBuf; } - case FTSQUERY_NEAR: - zBuf = sqlite3_mprintf("%zNEAR/%d ", zBuf, pExpr->nNear); - break; - case FTSQUERY_NOT: - zBuf = sqlite3_mprintf("%zNOT ", zBuf); - break; - case FTSQUERY_AND: - zBuf = sqlite3_mprintf("%zAND ", zBuf); - break; - case FTSQUERY_OR: - zBuf = sqlite3_mprintf("%zOR ", zBuf); - break; + rbuFinalize(p, pStmt); + rbuObjIterCacheIndexedCols(p, pIter); + assert( pIter->eType!=RBU_PK_VTAB || pIter->abIndexed==0 ); } - if( zBuf ) zBuf = sqlite3_mprintf("%z{", zBuf); - if( zBuf ) zBuf = exprToString(pExpr->pLeft, zBuf); - if( zBuf ) zBuf = sqlite3_mprintf("%z} {", zBuf); - - if( zBuf ) zBuf = exprToString(pExpr->pRight, zBuf); - if( zBuf ) zBuf = sqlite3_mprintf("%z}", zBuf); + return p->rc; +} - return zBuf; +/* +** This function constructs and returns a pointer to a nul-terminated +** string containing some SQL clause or list based on one or more of the +** column names currently stored in the pIter->azTblCol[] array. +*/ +static char *rbuObjIterGetCollist( + sqlite3rbu *p, /* RBU object */ + RbuObjIter *pIter /* Object iterator for column names */ +){ + char *zList = 0; + const char *zSep = ""; + int i; + for(i=0; inTblCol; i++){ + const char *z = pIter->azTblCol[i]; + zList = rbuMPrintf(p, "%z%s\"%w\"", zList, zSep, z); + zSep = ", "; + } + return zList; } /* -** This is the implementation of a scalar SQL function used to test the -** expression parser. It should be called as follows: +** This function is used to create a SELECT list (the list of SQL +** expressions that follows a SELECT keyword) for a SELECT statement +** used to read from an data_xxx or rbu_tmp_xxx table while updating the +** index object currently indicated by the iterator object passed as the +** second argument. A "PRAGMA index_xinfo = " statement is used +** to obtain the required information. ** -** fts3_exprtest(, , , ...); +** If the index is of the following form: ** -** The first argument, , is the name of the fts3 tokenizer used -** to parse the query expression (see README.tokenizers). The second argument -** is the query expression to parse. Each subsequent argument is the name -** of a column of the fts3 table that the query expression may refer to. -** For example: +** CREATE INDEX i1 ON t1(c, b COLLATE nocase); ** -** SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2'); +** and "t1" is a table with an explicit INTEGER PRIMARY KEY column +** "ipk", the returned string is: +** +** "`c` COLLATE 'BINARY', `b` COLLATE 'NOCASE', `ipk` COLLATE 'BINARY'" +** +** As well as the returned string, three other malloc'd strings are +** returned via output parameters. As follows: +** +** pzImposterCols: ... +** pzImposterPk: ... +** pzWhere: ... */ -static void fts3ExprTest( - sqlite3_context *context, - int argc, - sqlite3_value **argv +static char *rbuObjIterGetIndexCols( + sqlite3rbu *p, /* RBU object */ + RbuObjIter *pIter, /* Object iterator for column names */ + char **pzImposterCols, /* OUT: Columns for imposter table */ + char **pzImposterPk, /* OUT: Imposter PK clause */ + char **pzWhere, /* OUT: WHERE clause */ + int *pnBind /* OUT: Trbul number of columns */ ){ - sqlite3_tokenizer_module const *pModule = 0; - sqlite3_tokenizer *pTokenizer = 0; - int rc; - char **azCol = 0; - const char *zExpr; - int nExpr; - int nCol; - int ii; - Fts3Expr *pExpr; - char *zBuf = 0; - sqlite3 *db = sqlite3_context_db_handle(context); + int rc = p->rc; /* Error code */ + int rc2; /* sqlite3_finalize() return code */ + char *zRet = 0; /* String to return */ + char *zImpCols = 0; /* String to return via *pzImposterCols */ + char *zImpPK = 0; /* String to return via *pzImposterPK */ + char *zWhere = 0; /* String to return via *pzWhere */ + int nBind = 0; /* Value to return via *pnBind */ + const char *zCom = ""; /* Set to ", " later on */ + const char *zAnd = ""; /* Set to " AND " later on */ + sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = ? */ - if( argc<3 ){ - sqlite3_result_error(context, - "Usage: fts3_exprtest(tokenizer, expr, col1, ...", -1 + if( rc==SQLITE_OK ){ + assert( p->zErrmsg==0 ); + rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", pIter->zIdx) ); - return; - } - - rc = queryTestTokenizer(db, - (const char *)sqlite3_value_text(argv[0]), &pModule); - if( rc==SQLITE_NOMEM ){ - sqlite3_result_error_nomem(context); - goto exprtest_out; - }else if( !pModule ){ - sqlite3_result_error(context, "No such tokenizer module", -1); - goto exprtest_out; } - rc = pModule->xCreate(0, 0, &pTokenizer); - assert( rc==SQLITE_NOMEM || rc==SQLITE_OK ); - if( rc==SQLITE_NOMEM ){ - sqlite3_result_error_nomem(context); - goto exprtest_out; - } - pTokenizer->pModule = pModule; + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + int iCid = sqlite3_column_int(pXInfo, 1); + int bDesc = sqlite3_column_int(pXInfo, 3); + const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); + const char *zCol; + const char *zType; - zExpr = (const char *)sqlite3_value_text(argv[1]); - nExpr = sqlite3_value_bytes(argv[1]); - nCol = argc-2; - azCol = (char **)sqlite3_malloc(nCol*sizeof(char *)); - if( !azCol ){ - sqlite3_result_error_nomem(context); - goto exprtest_out; - } - for(ii=0; iieType==RBU_PK_IPK ){ + int i; + for(i=0; pIter->abTblPk[i]==0; i++); + assert( inTblCol ); + zCol = pIter->azTblCol[i]; + }else{ + zCol = "rbu_rowid"; + } + zType = "INTEGER"; + }else{ + zCol = pIter->azTblCol[iCid]; + zType = pIter->azTblType[iCid]; + } - if( sqlite3_user_data(context) ){ - char *zDummy = 0; - rc = sqlite3Fts3ExprParse( - pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr, &zDummy + zRet = sqlite3_mprintf("%z%s\"%w\" COLLATE %Q", zRet, zCom, zCol, zCollate); + if( pIter->bUnique==0 || sqlite3_column_int(pXInfo, 5) ){ + const char *zOrder = (bDesc ? " DESC" : ""); + zImpPK = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\"%s", + zImpPK, zCom, nBind, zCol, zOrder + ); + } + zImpCols = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\" %s COLLATE %Q", + zImpCols, zCom, nBind, zCol, zType, zCollate ); - assert( rc==SQLITE_OK || pExpr==0 ); - sqlite3_free(zDummy); - }else{ - rc = fts3ExprParseUnbalanced( - pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr + zWhere = sqlite3_mprintf( + "%z%s\"rbu_imp_%d%w\" IS ?", zWhere, zAnd, nBind, zCol ); + if( zRet==0 || zImpPK==0 || zImpCols==0 || zWhere==0 ) rc = SQLITE_NOMEM; + zCom = ", "; + zAnd = " AND "; + nBind++; } - if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){ - sqlite3Fts3ExprFree(pExpr); - sqlite3_result_error(context, "Error parsing expression", -1); - }else if( rc==SQLITE_NOMEM || !(zBuf = exprToString(pExpr, 0)) ){ - sqlite3_result_error_nomem(context); - }else{ - sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); - sqlite3_free(zBuf); - } - - sqlite3Fts3ExprFree(pExpr); + rc2 = sqlite3_finalize(pXInfo); + if( rc==SQLITE_OK ) rc = rc2; -exprtest_out: - if( pModule && pTokenizer ){ - rc = pModule->xDestroy(pTokenizer); + if( rc!=SQLITE_OK ){ + sqlite3_free(zRet); + sqlite3_free(zImpCols); + sqlite3_free(zImpPK); + sqlite3_free(zWhere); + zRet = 0; + zImpCols = 0; + zImpPK = 0; + zWhere = 0; + p->rc = rc; } - sqlite3_free(azCol); -} -/* -** Register the query expression parser test function fts3_exprtest() -** with database connection db. -*/ -SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3* db){ - int rc = sqlite3_create_function( - db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0 - ); - if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "fts3_exprtest_rebalance", - -1, SQLITE_UTF8, (void *)1, fts3ExprTest, 0, 0 - ); - } - return rc; + *pzImposterCols = zImpCols; + *pzImposterPk = zImpPK; + *pzWhere = zWhere; + *pnBind = nBind; + return zRet; } -#endif -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ - -/************** End of fts3_expr.c *******************************************/ -/************** Begin file fts3_hash.c ***************************************/ /* -** 2001 September 22 +** Assuming the current table columns are "a", "b" and "c", and the zObj +** paramter is passed "old", return a string of the form: ** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: +** "old.a, old.b, old.b" ** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** With the column names escaped. ** -************************************************************************* -** This is the implementation of generic hash-tables used in SQLite. -** We've modified it slightly to serve as a standalone hash table -** implementation for the full-text indexing module. +** For tables with implicit rowids - RBU_PK_EXTERNAL and RBU_PK_NONE, append +** the text ", old._rowid_" to the returned value. */ +static char *rbuObjIterGetOldlist( + sqlite3rbu *p, + RbuObjIter *pIter, + const char *zObj +){ + char *zList = 0; + if( p->rc==SQLITE_OK && pIter->abIndexed ){ + const char *zS = ""; + int i; + for(i=0; inTblCol; i++){ + if( pIter->abIndexed[i] ){ + const char *zCol = pIter->azTblCol[i]; + zList = sqlite3_mprintf("%z%s%s.\"%w\"", zList, zS, zObj, zCol); + }else{ + zList = sqlite3_mprintf("%z%sNULL", zList, zS); + } + zS = ", "; + if( zList==0 ){ + p->rc = SQLITE_NOMEM; + break; + } + } + + /* For a table with implicit rowids, append "old._rowid_" to the list. */ + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ + zList = rbuMPrintf(p, "%z, %s._rowid_", zList, zObj); + } + } + return zList; +} /* -** The code in this file is only compiled if: +** Return an expression that can be used in a WHERE clause to match the +** primary key of the current table. For example, if the table is: ** -** * The FTS3 module is being built as an extension -** (in which case SQLITE_CORE is not defined), or +** CREATE TABLE t1(a, b, c, PRIMARY KEY(b, c)); ** -** * The FTS3 module is being built into the core of -** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +** Return the string: +** +** "b = ?1 AND c = ?2" */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - -/* #include */ -/* #include */ -/* #include */ - +static char *rbuObjIterGetWhere( + sqlite3rbu *p, + RbuObjIter *pIter +){ + char *zList = 0; + if( pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE ){ + zList = rbuMPrintf(p, "_rowid_ = ?%d", pIter->nTblCol+1); + }else if( pIter->eType==RBU_PK_EXTERNAL ){ + const char *zSep = ""; + int i; + for(i=0; inTblCol; i++){ + if( pIter->abTblPk[i] ){ + zList = rbuMPrintf(p, "%z%sc%d=?%d", zList, zSep, i, i+1); + zSep = " AND "; + } + } + zList = rbuMPrintf(p, + "_rowid_ = (SELECT id FROM rbu_imposter2 WHERE %z)", zList + ); -/* -** Malloc and Free functions -*/ -static void *fts3HashMalloc(int n){ - void *p = sqlite3_malloc(n); - if( p ){ - memset(p, 0, n); + }else{ + const char *zSep = ""; + int i; + for(i=0; inTblCol; i++){ + if( pIter->abTblPk[i] ){ + const char *zCol = pIter->azTblCol[i]; + zList = rbuMPrintf(p, "%z%s\"%w\"=?%d", zList, zSep, zCol, i+1); + zSep = " AND "; + } + } } - return p; -} -static void fts3HashFree(void *p){ - sqlite3_free(p); + return zList; } -/* Turn bulk memory into a hash table object by initializing the -** fields of the Hash structure. -** -** "pNew" is a pointer to the hash table that is to be initialized. -** keyClass is one of the constants -** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass -** determines what kind of key the hash table will use. "copyKey" is -** true if the hash table should make its own private copy of keys and -** false if it should just use the supplied pointer. +/* +** The SELECT statement iterating through the keys for the current object +** (p->objiter.pSelect) currently points to a valid row. However, there +** is something wrong with the rbu_control value in the rbu_control value +** stored in the (p->nCol+1)'th column. Set the error code and error message +** of the RBU handle to something reflecting this. */ -SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey){ - assert( pNew!=0 ); - assert( keyClass>=FTS3_HASH_STRING && keyClass<=FTS3_HASH_BINARY ); - pNew->keyClass = keyClass; - pNew->copyKey = copyKey; - pNew->first = 0; - pNew->count = 0; - pNew->htsize = 0; - pNew->ht = 0; +static void rbuBadControlError(sqlite3rbu *p){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("invalid rbu_control value"); } -/* Remove all entries from a hash table. Reclaim all memory. -** Call this routine to delete a hash table or to reset a hash table -** to the empty state. + +/* +** Return a nul-terminated string containing the comma separated list of +** assignments that should be included following the "SET" keyword of +** an UPDATE statement used to update the table object that the iterator +** passed as the second argument currently points to if the rbu_control +** column of the data_xxx table entry is set to zMask. +** +** The memory for the returned string is obtained from sqlite3_malloc(). +** It is the responsibility of the caller to eventually free it using +** sqlite3_free(). +** +** If an OOM error is encountered when allocating space for the new +** string, an error code is left in the rbu handle passed as the first +** argument and NULL is returned. Or, if an error has already occurred +** when this function is called, NULL is returned immediately, without +** attempting the allocation or modifying the stored error code. */ -SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash *pH){ - Fts3HashElem *elem; /* For looping over all elements of the table */ +static char *rbuObjIterGetSetlist( + sqlite3rbu *p, + RbuObjIter *pIter, + const char *zMask +){ + char *zList = 0; + if( p->rc==SQLITE_OK ){ + int i; - assert( pH!=0 ); - elem = pH->first; - pH->first = 0; - fts3HashFree(pH->ht); - pH->ht = 0; - pH->htsize = 0; - while( elem ){ - Fts3HashElem *next_elem = elem->next; - if( pH->copyKey && elem->pKey ){ - fts3HashFree(elem->pKey); + if( (int)strlen(zMask)!=pIter->nTblCol ){ + rbuBadControlError(p); + }else{ + const char *zSep = ""; + for(i=0; inTblCol; i++){ + char c = zMask[pIter->aiSrcOrder[i]]; + if( c=='x' ){ + zList = rbuMPrintf(p, "%z%s\"%w\"=?%d", + zList, zSep, pIter->azTblCol[i], i+1 + ); + zSep = ", "; + } + else if( c=='d' ){ + zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_delta(\"%w\", ?%d)", + zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1 + ); + zSep = ", "; + } + else if( c=='f' ){ + zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_fossil_delta(\"%w\", ?%d)", + zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1 + ); + zSep = ", "; + } + } } - fts3HashFree(elem); - elem = next_elem; } - pH->count = 0; + return zList; } /* -** Hash and comparison functions when the mode is FTS3_HASH_STRING +** Return a nul-terminated string consisting of nByte comma separated +** "?" expressions. For example, if nByte is 3, return a pointer to +** a buffer containing the string "?,?,?". +** +** The memory for the returned string is obtained from sqlite3_malloc(). +** It is the responsibility of the caller to eventually free it using +** sqlite3_free(). +** +** If an OOM error is encountered when allocating space for the new +** string, an error code is left in the rbu handle passed as the first +** argument and NULL is returned. Or, if an error has already occurred +** when this function is called, NULL is returned immediately, without +** attempting the allocation or modifying the stored error code. */ -static int fts3StrHash(const void *pKey, int nKey){ - const char *z = (const char *)pKey; - unsigned h = 0; - if( nKey<=0 ) nKey = (int) strlen(z); - while( nKey > 0 ){ - h = (h<<3) ^ h ^ *z++; - nKey--; +static char *rbuObjIterGetBindlist(sqlite3rbu *p, int nBind){ + char *zRet = 0; + int nByte = nBind*2 + 1; + + zRet = (char*)rbuMalloc(p, nByte); + if( zRet ){ + int i; + for(i=0; i 0 ){ - h = (h<<3) ^ h ^ *(z++); +static char *rbuWithoutRowidPK(sqlite3rbu *p, RbuObjIter *pIter){ + char *z = 0; + assert( pIter->zIdx==0 ); + if( p->rc==SQLITE_OK ){ + const char *zSep = "PRIMARY KEY("; + sqlite3_stmt *pXList = 0; /* PRAGMA index_list = (pIter->zTbl) */ + sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = */ + + p->rc = prepareFreeAndCollectError(p->dbMain, &pXList, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl) + ); + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXList) ){ + const char *zOrig = (const char*)sqlite3_column_text(pXList,3); + if( zOrig && strcmp(zOrig, "pk")==0 ){ + const char *zIdx = (const char*)sqlite3_column_text(pXList,1); + if( zIdx ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) + ); + } + break; + } + } + rbuFinalize(p, pXList); + + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + if( sqlite3_column_int(pXInfo, 5) ){ + /* int iCid = sqlite3_column_int(pXInfo, 0); */ + const char *zCol = (const char*)sqlite3_column_text(pXInfo, 2); + const char *zDesc = sqlite3_column_int(pXInfo, 3) ? " DESC" : ""; + z = rbuMPrintf(p, "%z%s\"%w\"%s", z, zSep, zCol, zDesc); + zSep = ", "; + } + } + z = rbuMPrintf(p, "%z)", z); + rbuFinalize(p, pXInfo); } - return h & 0x7fffffff; -} -static int fts3BinCompare(const void *pKey1, int n1, const void *pKey2, int n2){ - if( n1!=n2 ) return 1; - return memcmp(pKey1,pKey2,n1); + return z; } /* -** Return a pointer to the appropriate hash function given the key class. +** This function creates the second imposter table used when writing to +** a table b-tree where the table has an external primary key. If the +** iterator passed as the second argument does not currently point to +** a table (not index) with an external primary key, this function is a +** no-op. ** -** The C syntax in this function definition may be unfamilar to some -** programmers, so we provide the following additional explanation: +** Assuming the iterator does point to a table with an external PK, this +** function creates a WITHOUT ROWID imposter table named "rbu_imposter2" +** used to access that PK index. For example, if the target table is +** declared as follows: +** +** CREATE TABLE t1(a, b TEXT, c REAL, PRIMARY KEY(b, c)); +** +** then the imposter table schema is: +** +** CREATE TABLE rbu_imposter2(c1 TEXT, c2 REAL, id INTEGER) WITHOUT ROWID; ** -** The name of the function is "ftsHashFunction". The function takes a -** single parameter "keyClass". The return value of ftsHashFunction() -** is a pointer to another function. Specifically, the return value -** of ftsHashFunction() is a pointer to a function that takes two parameters -** with types "const void*" and "int" and returns an "int". */ -static int (*ftsHashFunction(int keyClass))(const void*,int){ - if( keyClass==FTS3_HASH_STRING ){ - return &fts3StrHash; - }else{ - assert( keyClass==FTS3_HASH_BINARY ); - return &fts3BinHash; +static void rbuCreateImposterTable2(sqlite3rbu *p, RbuObjIter *pIter){ + if( p->rc==SQLITE_OK && pIter->eType==RBU_PK_EXTERNAL ){ + int tnum = pIter->iPkTnum; /* Root page of PK index */ + sqlite3_stmt *pQuery = 0; /* SELECT name ... WHERE rootpage = $tnum */ + const char *zIdx = 0; /* Name of PK index */ + sqlite3_stmt *pXInfo = 0; /* PRAGMA main.index_xinfo = $zIdx */ + const char *zComma = ""; + char *zCols = 0; /* Used to build up list of table cols */ + char *zPk = 0; /* Used to build up table PK declaration */ + + /* Figure out the name of the primary key index for the current table. + ** This is needed for the argument to "PRAGMA index_xinfo". Set + ** zIdx to point to a nul-terminated string containing this name. */ + p->rc = prepareAndCollectError(p->dbMain, &pQuery, &p->zErrmsg, + "SELECT name FROM sqlite_master WHERE rootpage = ?" + ); + if( p->rc==SQLITE_OK ){ + sqlite3_bind_int(pQuery, 1, tnum); + if( SQLITE_ROW==sqlite3_step(pQuery) ){ + zIdx = (const char*)sqlite3_column_text(pQuery, 0); + } + } + if( zIdx ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) + ); + } + rbuFinalize(p, pQuery); + + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + int bKey = sqlite3_column_int(pXInfo, 5); + if( bKey ){ + int iCid = sqlite3_column_int(pXInfo, 1); + int bDesc = sqlite3_column_int(pXInfo, 3); + const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); + zCols = rbuMPrintf(p, "%z%sc%d %s COLLATE %s", zCols, zComma, + iCid, pIter->azTblType[iCid], zCollate + ); + zPk = rbuMPrintf(p, "%z%sc%d%s", zPk, zComma, iCid, bDesc?" DESC":""); + zComma = ", "; + } + } + zCols = rbuMPrintf(p, "%z, id INTEGER", zCols); + rbuFinalize(p, pXInfo); + + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1, tnum); + rbuMPrintfExec(p, p->dbMain, + "CREATE TABLE rbu_imposter2(%z, PRIMARY KEY(%z)) WITHOUT ROWID", + zCols, zPk + ); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0); } } /* -** Return a pointer to the appropriate hash function given the key class. +** If an error has already occurred when this function is called, it +** immediately returns zero (without doing any work). Or, if an error +** occurs during the execution of this function, it sets the error code +** in the sqlite3rbu object indicated by the first argument and returns +** zero. ** -** For help in interpreted the obscure C code in the function definition, -** see the header comment on the previous function. -*/ -static int (*ftsCompareFunction(int keyClass))(const void*,int,const void*,int){ - if( keyClass==FTS3_HASH_STRING ){ - return &fts3StrCompare; - }else{ - assert( keyClass==FTS3_HASH_BINARY ); - return &fts3BinCompare; +** The iterator passed as the second argument is guaranteed to point to +** a table (not an index) when this function is called. This function +** attempts to create any imposter table required to write to the main +** table b-tree of the table before returning. Non-zero is returned if +** an imposter table are created, or zero otherwise. +** +** An imposter table is required in all cases except RBU_PK_VTAB. Only +** virtual tables are written to directly. The imposter table has the +** same schema as the actual target table (less any UNIQUE constraints). +** More precisely, the "same schema" means the same columns, types, +** collation sequences. For tables that do not have an external PRIMARY +** KEY, it also means the same PRIMARY KEY declaration. +*/ +static void rbuCreateImposterTable(sqlite3rbu *p, RbuObjIter *pIter){ + if( p->rc==SQLITE_OK && pIter->eType!=RBU_PK_VTAB ){ + int tnum = pIter->iTnum; + const char *zComma = ""; + char *zSql = 0; + int iCol; + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 1); + + for(iCol=0; p->rc==SQLITE_OK && iColnTblCol; iCol++){ + const char *zPk = ""; + const char *zCol = pIter->azTblCol[iCol]; + const char *zColl = 0; + + p->rc = sqlite3_table_column_metadata( + p->dbMain, "main", pIter->zTbl, zCol, 0, &zColl, 0, 0, 0 + ); + + if( pIter->eType==RBU_PK_IPK && pIter->abTblPk[iCol] ){ + /* If the target table column is an "INTEGER PRIMARY KEY", add + ** "PRIMARY KEY" to the imposter table column declaration. */ + zPk = "PRIMARY KEY "; + } + zSql = rbuMPrintf(p, "%z%s\"%w\" %s %sCOLLATE %s%s", + zSql, zComma, zCol, pIter->azTblType[iCol], zPk, zColl, + (pIter->abNotNull[iCol] ? " NOT NULL" : "") + ); + zComma = ", "; + } + + if( pIter->eType==RBU_PK_WITHOUT_ROWID ){ + char *zPk = rbuWithoutRowidPK(p, pIter); + if( zPk ){ + zSql = rbuMPrintf(p, "%z, %z", zSql, zPk); + } + } + + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1, tnum); + rbuMPrintfExec(p, p->dbMain, "CREATE TABLE \"rbu_imp_%w\"(%z)%s", + pIter->zTbl, zSql, + (pIter->eType==RBU_PK_WITHOUT_ROWID ? " WITHOUT ROWID" : "") + ); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0); } } -/* Link an element into the hash table +/* +** Prepare a statement used to insert rows into the "rbu_tmp_xxx" table. +** Specifically a statement of the form: +** +** INSERT INTO rbu_tmp_xxx VALUES(?, ?, ? ...); +** +** The number of bound variables is equal to the number of columns in +** the target table, plus one (for the rbu_control column), plus one more +** (for the rbu_rowid column) if the target table is an implicit IPK or +** virtual table. */ -static void fts3HashInsertElement( - Fts3Hash *pH, /* The complete hash table */ - struct _fts3ht *pEntry, /* The entry into which pNew is inserted */ - Fts3HashElem *pNew /* The element to be inserted */ -){ - Fts3HashElem *pHead; /* First element already in pEntry */ - pHead = pEntry->chain; - if( pHead ){ - pNew->next = pHead; - pNew->prev = pHead->prev; - if( pHead->prev ){ pHead->prev->next = pNew; } - else { pH->first = pNew; } - pHead->prev = pNew; - }else{ - pNew->next = pH->first; - if( pH->first ){ pH->first->prev = pNew; } - pNew->prev = 0; - pH->first = pNew; +static void rbuObjIterPrepareTmpInsert( + sqlite3rbu *p, + RbuObjIter *pIter, + const char *zCollist, + const char *zRbuRowid +){ + int bRbuRowid = (pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE); + char *zBind = rbuObjIterGetBindlist(p, pIter->nTblCol + 1 + bRbuRowid); + if( zBind ){ + assert( pIter->pTmpInsert==0 ); + p->rc = prepareFreeAndCollectError( + p->dbRbu, &pIter->pTmpInsert, &p->zErrmsg, sqlite3_mprintf( + "INSERT INTO %s.'rbu_tmp_%q'(rbu_control,%s%s) VALUES(%z)", + p->zStateDb, pIter->zDataTbl, zCollist, zRbuRowid, zBind + )); } - pEntry->count++; - pEntry->chain = pNew; } +static void rbuTmpInsertFunc( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + sqlite3rbu *p = sqlite3_user_data(pCtx); + int rc = SQLITE_OK; + int i; -/* Resize the hash table so that it cantains "new_size" buckets. -** "new_size" must be a power of 2. The hash table might fail -** to resize if sqliteMalloc() fails. -** -** Return non-zero if a memory allocation error occurs. -*/ -static int fts3Rehash(Fts3Hash *pH, int new_size){ - struct _fts3ht *new_ht; /* The new hash table */ - Fts3HashElem *elem, *next_elem; /* For looping over existing elements */ - int (*xHash)(const void*,int); /* The hash function */ + for(i=0; rc==SQLITE_OK && iobjiter.pTmpInsert, i+1, apVal[i]); + } + if( rc==SQLITE_OK ){ + sqlite3_step(p->objiter.pTmpInsert); + rc = sqlite3_reset(p->objiter.pTmpInsert); + } - assert( (new_size & (new_size-1))==0 ); - new_ht = (struct _fts3ht *)fts3HashMalloc( new_size*sizeof(struct _fts3ht) ); - if( new_ht==0 ) return 1; - fts3HashFree(pH->ht); - pH->ht = new_ht; - pH->htsize = new_size; - xHash = ftsHashFunction(pH->keyClass); - for(elem=pH->first, pH->first=0; elem; elem = next_elem){ - int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1); - next_elem = elem->next; - fts3HashInsertElement(pH, &new_ht[h], elem); + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); } - return 0; } -/* This function (for internal use only) locates an element in an -** hash table that matches the given key. The hash for this key has -** already been computed and is passed as the 4th parameter. +/* +** Ensure that the SQLite statement handles required to update the +** target database object currently indicated by the iterator passed +** as the second argument are available. */ -static Fts3HashElem *fts3FindElementByHash( - const Fts3Hash *pH, /* The pH to be searched */ - const void *pKey, /* The key we are searching for */ - int nKey, - int h /* The hash for this key. */ +static int rbuObjIterPrepareAll( + sqlite3rbu *p, + RbuObjIter *pIter, + int nOffset /* Add "LIMIT -1 OFFSET $nOffset" to SELECT */ ){ - Fts3HashElem *elem; /* Used to loop thru the element list */ - int count; /* Number of elements left to test */ - int (*xCompare)(const void*,int,const void*,int); /* comparison function */ + assert( pIter->bCleanup==0 ); + if( pIter->pSelect==0 && rbuObjIterCacheTableInfo(p, pIter)==SQLITE_OK ){ + const int tnum = pIter->iTnum; + char *zCollist = 0; /* List of indexed columns */ + char **pz = &p->zErrmsg; + const char *zIdx = pIter->zIdx; + char *zLimit = 0; - if( pH->ht ){ - struct _fts3ht *pEntry = &pH->ht[h]; - elem = pEntry->chain; - count = pEntry->count; - xCompare = ftsCompareFunction(pH->keyClass); - while( count-- && elem ){ - if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){ - return elem; + if( nOffset ){ + zLimit = sqlite3_mprintf(" LIMIT -1 OFFSET %d", nOffset); + if( !zLimit ) p->rc = SQLITE_NOMEM; + } + + if( zIdx ){ + const char *zTbl = pIter->zTbl; + char *zImposterCols = 0; /* Columns for imposter table */ + char *zImposterPK = 0; /* Primary key declaration for imposter */ + char *zWhere = 0; /* WHERE clause on PK columns */ + char *zBind = 0; + int nBind = 0; + + assert( pIter->eType!=RBU_PK_VTAB ); + zCollist = rbuObjIterGetIndexCols( + p, pIter, &zImposterCols, &zImposterPK, &zWhere, &nBind + ); + zBind = rbuObjIterGetBindlist(p, nBind); + + /* Create the imposter table used to write to this index. */ + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 1); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1,tnum); + rbuMPrintfExec(p, p->dbMain, + "CREATE TABLE \"rbu_imp_%w\"( %s, PRIMARY KEY( %s ) ) WITHOUT ROWID", + zTbl, zImposterCols, zImposterPK + ); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0); + + /* Create the statement to insert index entries */ + pIter->nCol = nBind; + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError( + p->dbMain, &pIter->pInsert, &p->zErrmsg, + sqlite3_mprintf("INSERT INTO \"rbu_imp_%w\" VALUES(%s)", zTbl, zBind) + ); } - elem = elem->next; + + /* And to delete index entries */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError( + p->dbMain, &pIter->pDelete, &p->zErrmsg, + sqlite3_mprintf("DELETE FROM \"rbu_imp_%w\" WHERE %s", zTbl, zWhere) + ); + } + + /* Create the SELECT statement to read keys in sorted order */ + if( p->rc==SQLITE_OK ){ + char *zSql; + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ + zSql = sqlite3_mprintf( + "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' ORDER BY %s%s", + zCollist, p->zStateDb, pIter->zDataTbl, + zCollist, zLimit + ); + }else{ + zSql = sqlite3_mprintf( + "SELECT %s, rbu_control FROM '%q' " + "WHERE typeof(rbu_control)='integer' AND rbu_control!=1 " + "UNION ALL " + "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' " + "ORDER BY %s%s", + zCollist, pIter->zDataTbl, + zCollist, p->zStateDb, pIter->zDataTbl, + zCollist, zLimit + ); + } + p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, zSql); + } + + sqlite3_free(zImposterCols); + sqlite3_free(zImposterPK); + sqlite3_free(zWhere); + sqlite3_free(zBind); + }else{ + int bRbuRowid = (pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE); + const char *zTbl = pIter->zTbl; /* Table this step applies to */ + const char *zWrite; /* Imposter table name */ + + char *zBindings = rbuObjIterGetBindlist(p, pIter->nTblCol + bRbuRowid); + char *zWhere = rbuObjIterGetWhere(p, pIter); + char *zOldlist = rbuObjIterGetOldlist(p, pIter, "old"); + char *zNewlist = rbuObjIterGetOldlist(p, pIter, "new"); + + zCollist = rbuObjIterGetCollist(p, pIter); + pIter->nCol = pIter->nTblCol; + + /* Create the imposter table or tables (if required). */ + rbuCreateImposterTable(p, pIter); + rbuCreateImposterTable2(p, pIter); + zWrite = (pIter->eType==RBU_PK_VTAB ? "" : "rbu_imp_"); + + /* Create the INSERT statement to write to the target PK b-tree */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pInsert, pz, + sqlite3_mprintf( + "INSERT INTO \"%s%w\"(%s%s) VALUES(%s)", + zWrite, zTbl, zCollist, (bRbuRowid ? ", _rowid_" : ""), zBindings + ) + ); + } + + /* Create the DELETE statement to write to the target PK b-tree */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pDelete, pz, + sqlite3_mprintf( + "DELETE FROM \"%s%w\" WHERE %s", zWrite, zTbl, zWhere + ) + ); + } + + if( pIter->abIndexed ){ + const char *zRbuRowid = ""; + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ + zRbuRowid = ", rbu_rowid"; + } + + /* Create the rbu_tmp_xxx table and the triggers to populate it. */ + rbuMPrintfExec(p, p->dbRbu, + "CREATE TABLE IF NOT EXISTS %s.'rbu_tmp_%q' AS " + "SELECT *%s FROM '%q' WHERE 0;" + , p->zStateDb, pIter->zDataTbl + , (pIter->eType==RBU_PK_EXTERNAL ? ", 0 AS rbu_rowid" : "") + , pIter->zDataTbl + ); + + rbuMPrintfExec(p, p->dbMain, + "CREATE TEMP TRIGGER rbu_delete_tr BEFORE DELETE ON \"%s%w\" " + "BEGIN " + " SELECT rbu_tmp_insert(2, %s);" + "END;" + + "CREATE TEMP TRIGGER rbu_update1_tr BEFORE UPDATE ON \"%s%w\" " + "BEGIN " + " SELECT rbu_tmp_insert(2, %s);" + "END;" + + "CREATE TEMP TRIGGER rbu_update2_tr AFTER UPDATE ON \"%s%w\" " + "BEGIN " + " SELECT rbu_tmp_insert(3, %s);" + "END;", + zWrite, zTbl, zOldlist, + zWrite, zTbl, zOldlist, + zWrite, zTbl, zNewlist + ); + + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ + rbuMPrintfExec(p, p->dbMain, + "CREATE TEMP TRIGGER rbu_insert_tr AFTER INSERT ON \"%s%w\" " + "BEGIN " + " SELECT rbu_tmp_insert(0, %s);" + "END;", + zWrite, zTbl, zNewlist + ); + } + + rbuObjIterPrepareTmpInsert(p, pIter, zCollist, zRbuRowid); + } + + /* Create the SELECT statement to read keys from data_xxx */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, + sqlite3_mprintf( + "SELECT %s, rbu_control%s FROM '%q'%s", + zCollist, (bRbuRowid ? ", rbu_rowid" : ""), + pIter->zDataTbl, zLimit + ) + ); + } + + sqlite3_free(zWhere); + sqlite3_free(zOldlist); + sqlite3_free(zNewlist); + sqlite3_free(zBindings); } + sqlite3_free(zCollist); + sqlite3_free(zLimit); } - return 0; + + return p->rc; } -/* Remove a single entry from the hash table given a pointer to that -** element and a hash on the element's key. +/* +** Set output variable *ppStmt to point to an UPDATE statement that may +** be used to update the imposter table for the main table b-tree of the +** table object that pIter currently points to, assuming that the +** rbu_control column of the data_xyz table contains zMask. +** +** If the zMask string does not specify any columns to update, then this +** is not an error. Output variable *ppStmt is set to NULL in this case. */ -static void fts3RemoveElementByHash( - Fts3Hash *pH, /* The pH containing "elem" */ - Fts3HashElem* elem, /* The element to be removed from the pH */ - int h /* Hash value for the element */ +static int rbuGetUpdateStmt( + sqlite3rbu *p, /* RBU handle */ + RbuObjIter *pIter, /* Object iterator */ + const char *zMask, /* rbu_control value ('x.x.') */ + sqlite3_stmt **ppStmt /* OUT: UPDATE statement handle */ ){ - struct _fts3ht *pEntry; - if( elem->prev ){ - elem->prev->next = elem->next; + RbuUpdateStmt **pp; + RbuUpdateStmt *pUp = 0; + int nUp = 0; + + /* In case an error occurs */ + *ppStmt = 0; + + /* Search for an existing statement. If one is found, shift it to the front + ** of the LRU queue and return immediately. Otherwise, leave nUp pointing + ** to the number of statements currently in the cache and pUp to the + ** last object in the list. */ + for(pp=&pIter->pRbuUpdate; *pp; pp=&((*pp)->pNext)){ + pUp = *pp; + if( strcmp(pUp->zMask, zMask)==0 ){ + *pp = pUp->pNext; + pUp->pNext = pIter->pRbuUpdate; + pIter->pRbuUpdate = pUp; + *ppStmt = pUp->pUpdate; + return SQLITE_OK; + } + nUp++; + } + assert( pUp==0 || pUp->pNext==0 ); + + if( nUp>=SQLITE_RBU_UPDATE_CACHESIZE ){ + for(pp=&pIter->pRbuUpdate; *pp!=pUp; pp=&((*pp)->pNext)); + *pp = 0; + sqlite3_finalize(pUp->pUpdate); + pUp->pUpdate = 0; }else{ - pH->first = elem->next; + pUp = (RbuUpdateStmt*)rbuMalloc(p, sizeof(RbuUpdateStmt)+pIter->nTblCol+1); } - if( elem->next ){ - elem->next->prev = elem->prev; + + if( pUp ){ + char *zWhere = rbuObjIterGetWhere(p, pIter); + char *zSet = rbuObjIterGetSetlist(p, pIter, zMask); + char *zUpdate = 0; + + pUp->zMask = (char*)&pUp[1]; + memcpy(pUp->zMask, zMask, pIter->nTblCol); + pUp->pNext = pIter->pRbuUpdate; + pIter->pRbuUpdate = pUp; + + if( zSet ){ + const char *zPrefix = ""; + + if( pIter->eType!=RBU_PK_VTAB ) zPrefix = "rbu_imp_"; + zUpdate = sqlite3_mprintf("UPDATE \"%s%w\" SET %s WHERE %s", + zPrefix, pIter->zTbl, zSet, zWhere + ); + p->rc = prepareFreeAndCollectError( + p->dbMain, &pUp->pUpdate, &p->zErrmsg, zUpdate + ); + *ppStmt = pUp->pUpdate; + } + sqlite3_free(zWhere); + sqlite3_free(zSet); } - pEntry = &pH->ht[h]; - if( pEntry->chain==elem ){ - pEntry->chain = elem->next; + + return p->rc; +} + +static sqlite3 *rbuOpenDbhandle(sqlite3rbu *p, const char *zName){ + sqlite3 *db = 0; + if( p->rc==SQLITE_OK ){ + const int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_URI; + p->rc = sqlite3_open_v2(zName, &db, flags, p->zVfsName); + if( p->rc ){ + p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + sqlite3_close(db); + db = 0; + } } - pEntry->count--; - if( pEntry->count<=0 ){ - pEntry->chain = 0; + return db; +} + +/* +** Open the database handle and attach the RBU database as "rbu". If an +** error occurs, leave an error code and message in the RBU handle. +*/ +static void rbuOpenDatabase(sqlite3rbu *p){ + assert( p->rc==SQLITE_OK ); + assert( p->dbMain==0 && p->dbRbu==0 ); + + p->eStage = 0; + p->dbMain = rbuOpenDbhandle(p, p->zTarget); + p->dbRbu = rbuOpenDbhandle(p, p->zRbu); + + /* If using separate RBU and state databases, attach the state database to + ** the RBU db handle now. */ + if( p->zState ){ + rbuMPrintfExec(p, p->dbRbu, "ATTACH %Q AS stat", p->zState); + memcpy(p->zStateDb, "stat", 4); + }else{ + memcpy(p->zStateDb, "main", 4); } - if( pH->copyKey && elem->pKey ){ - fts3HashFree(elem->pKey); + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_create_function(p->dbMain, + "rbu_tmp_insert", -1, SQLITE_UTF8, (void*)p, rbuTmpInsertFunc, 0, 0 + ); } - fts3HashFree( elem ); - pH->count--; - if( pH->count<=0 ){ - assert( pH->first==0 ); - assert( pH->count==0 ); - fts3HashClear(pH); + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_create_function(p->dbMain, + "rbu_fossil_delta", 2, SQLITE_UTF8, 0, rbuFossilDeltaFunc, 0, 0 + ); } -} -SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem( - const Fts3Hash *pH, - const void *pKey, - int nKey -){ - int h; /* A hash on key */ - int (*xHash)(const void*,int); /* The hash function */ + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_create_function(p->dbRbu, + "rbu_target_name", 1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0 + ); + } - if( pH==0 || pH->ht==0 ) return 0; - xHash = ftsHashFunction(pH->keyClass); - assert( xHash!=0 ); - h = (*xHash)(pKey,nKey); - assert( (pH->htsize & (pH->htsize-1))==0 ); - return fts3FindElementByHash(pH,pKey,nKey, h & (pH->htsize-1)); -} + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p); + } + rbuMPrintfExec(p, p->dbMain, "SELECT * FROM sqlite_master"); -/* -** Attempt to locate an element of the hash table pH with a key -** that matches pKey,nKey. Return the data for this element if it is -** found, or NULL if there is no match. -*/ -SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash *pH, const void *pKey, int nKey){ - Fts3HashElem *pElem; /* The element that matches key (if any) */ + /* Mark the database file just opened as an RBU target database. If + ** this call returns SQLITE_NOTFOUND, then the RBU vfs is not in use. + ** This is an error. */ + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p); + } - pElem = sqlite3Fts3HashFindElem(pH, pKey, nKey); - return pElem ? pElem->data : 0; + if( p->rc==SQLITE_NOTFOUND ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("rbu vfs not found"); + } } -/* Insert an element into the hash table pH. The key is pKey,nKey -** and the data is "data". +/* +** This routine is a copy of the sqlite3FileSuffix3() routine from the core. +** It is a no-op unless SQLITE_ENABLE_8_3_NAMES is defined. ** -** If no element exists with a matching key, then a new -** element is created. A copy of the key is made if the copyKey -** flag is set. NULL is returned. +** If SQLITE_ENABLE_8_3_NAMES is set at compile-time and if the database +** filename in zBaseFilename is a URI with the "8_3_names=1" parameter and +** if filename in z[] has a suffix (a.k.a. "extension") that is longer than +** three characters, then shorten the suffix on z[] to be the last three +** characters of the original suffix. ** -** If another element already exists with the same key, then the -** new data replaces the old data and the old data is returned. -** The key is not copied in this instance. If a malloc fails, then -** the new data is returned and the hash table is unchanged. +** If SQLITE_ENABLE_8_3_NAMES is set to 2 at compile-time, then always +** do the suffix shortening regardless of URI parameter. ** -** If the "data" parameter to this function is NULL, then the -** element corresponding to "key" is removed from the hash table. +** Examples: +** +** test.db-journal => test.nal +** test.db-wal => test.wal +** test.db-shm => test.shm +** test.db-mj7f3319fa => test.9fa */ -SQLITE_PRIVATE void *sqlite3Fts3HashInsert( - Fts3Hash *pH, /* The hash table to insert into */ - const void *pKey, /* The key */ - int nKey, /* Number of bytes in the key */ - void *data /* The data */ -){ - int hraw; /* Raw hash value of the key */ - int h; /* the hash of the key modulo hash table size */ - Fts3HashElem *elem; /* Used to loop thru the element list */ - Fts3HashElem *new_elem; /* New element added to the pH */ - int (*xHash)(const void*,int); /* The hash function */ - - assert( pH!=0 ); - xHash = ftsHashFunction(pH->keyClass); - assert( xHash!=0 ); - hraw = (*xHash)(pKey, nKey); - assert( (pH->htsize & (pH->htsize-1))==0 ); - h = hraw & (pH->htsize-1); - elem = fts3FindElementByHash(pH,pKey,nKey,h); - if( elem ){ - void *old_data = elem->data; - if( data==0 ){ - fts3RemoveElementByHash(pH,elem,h); - }else{ - elem->data = data; - } - return old_data; - } - if( data==0 ) return 0; - if( (pH->htsize==0 && fts3Rehash(pH,8)) - || (pH->count>=pH->htsize && fts3Rehash(pH, pH->htsize*2)) - ){ - pH->count = 0; - return data; - } - assert( pH->htsize>0 ); - new_elem = (Fts3HashElem*)fts3HashMalloc( sizeof(Fts3HashElem) ); - if( new_elem==0 ) return data; - if( pH->copyKey && pKey!=0 ){ - new_elem->pKey = fts3HashMalloc( nKey ); - if( new_elem->pKey==0 ){ - fts3HashFree(new_elem); - return data; - } - memcpy((void*)new_elem->pKey, pKey, nKey); - }else{ - new_elem->pKey = (void*)pKey; +static void rbuFileSuffix3(const char *zBase, char *z){ +#ifdef SQLITE_ENABLE_8_3_NAMES +#if SQLITE_ENABLE_8_3_NAMES<2 + if( sqlite3_uri_boolean(zBase, "8_3_names", 0) ) +#endif + { + int i, sz; + sz = sqlite3Strlen30(z); + for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} + if( z[i]=='.' && ALWAYS(sz>i+4) ) memmove(&z[i+1], &z[sz-3], 4); } - new_elem->nKey = nKey; - pH->count++; - assert( pH->htsize>0 ); - assert( (pH->htsize & (pH->htsize-1))==0 ); - h = hraw & (pH->htsize-1); - fts3HashInsertElement(pH, &pH->ht[h], new_elem); - new_elem->data = data; - return 0; +#endif } -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ - -/************** End of fts3_hash.c *******************************************/ -/************** Begin file fts3_porter.c *************************************/ /* -** 2006 September 30 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** Return the current wal-index header checksum for the target database +** as a 64-bit integer. ** -************************************************************************* -** Implementation of the full-text-search tokenizer that implements -** a Porter stemmer. +** The checksum is store in the first page of xShmMap memory as an 8-byte +** blob starting at byte offset 40. */ +static i64 rbuShmChecksum(sqlite3rbu *p){ + i64 iRet = 0; + if( p->rc==SQLITE_OK ){ + sqlite3_file *pDb = p->pTargetFd->pReal; + u32 volatile *ptr; + p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, (void volatile**)&ptr); + if( p->rc==SQLITE_OK ){ + iRet = ((i64)ptr[10] << 32) + ptr[11]; + } + } + return iRet; +} /* -** The code in this file is only compiled if: +** This function is called as part of initializing or reinitializing an +** incremental checkpoint. ** -** * The FTS3 module is being built as an extension -** (in which case SQLITE_CORE is not defined), or +** It populates the sqlite3rbu.aFrame[] array with the set of +** (wal frame -> db page) copy operations required to checkpoint the +** current wal file, and obtains the set of shm locks required to safely +** perform the copy operations directly on the file-system. ** -** * The FTS3 module is being built into the core of -** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +** If argument pState is not NULL, then the incremental checkpoint is +** being resumed. In this case, if the checksum of the wal-index-header +** following recovery is not the same as the checksum saved in the RbuState +** object, then the rbu handle is set to DONE state. This occurs if some +** other client appends a transaction to the wal file in the middle of +** an incremental checkpoint. */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - -/* #include */ -/* #include */ -/* #include */ -/* #include */ +static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){ + /* If pState is NULL, then the wal file may not have been opened and + ** recovered. Running a read-statement here to ensure that doing so + ** does not interfere with the "capture" process below. */ + if( pState==0 ){ + p->eStage = 0; + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->dbMain, "SELECT * FROM sqlite_master", 0, 0, 0); + } + } -/* -** Class derived from sqlite3_tokenizer -*/ -typedef struct porter_tokenizer { - sqlite3_tokenizer base; /* Base class */ -} porter_tokenizer; + /* Assuming no error has occurred, run a "restart" checkpoint with the + ** sqlite3rbu.eStage variable set to CAPTURE. This turns on the following + ** special behaviour in the rbu VFS: + ** + ** * If the exclusive shm WRITER or READ0 lock cannot be obtained, + ** the checkpoint fails with SQLITE_BUSY (normally SQLite would + ** proceed with running a passive checkpoint instead of failing). + ** + ** * Attempts to read from the *-wal file or write to the database file + ** do not perform any IO. Instead, the frame/page combinations that + ** would be read/written are recorded in the sqlite3rbu.aFrame[] + ** array. + ** + ** * Calls to xShmLock(UNLOCK) to release the exclusive shm WRITER, + ** READ0 and CHECKPOINT locks taken as part of the checkpoint are + ** no-ops. These locks will not be released until the connection + ** is closed. + ** + ** * Attempting to xSync() the database file causes an SQLITE_INTERNAL + ** error. + ** + ** As a result, unless an error (i.e. OOM or SQLITE_BUSY) occurs, the + ** checkpoint below fails with SQLITE_INTERNAL, and leaves the aFrame[] + ** array populated with a set of (frame -> page) mappings. Because the + ** WRITER, CHECKPOINT and READ0 locks are still held, it is safe to copy + ** data from the wal file into the database file according to the + ** contents of aFrame[]. + */ + if( p->rc==SQLITE_OK ){ + int rc2; + p->eStage = RBU_STAGE_CAPTURE; + rc2 = sqlite3_exec(p->dbMain, "PRAGMA main.wal_checkpoint=restart", 0, 0,0); + if( rc2!=SQLITE_INTERNAL ) p->rc = rc2; + } -/* -** Class derived from sqlite3_tokenizer_cursor -*/ -typedef struct porter_tokenizer_cursor { - sqlite3_tokenizer_cursor base; - const char *zInput; /* input we are tokenizing */ - int nInput; /* size of the input */ - int iOffset; /* current position in zInput */ - int iToken; /* index of next token to be returned */ - char *zToken; /* storage for current token */ - int nAllocated; /* space allocated to zToken buffer */ -} porter_tokenizer_cursor; + if( p->rc==SQLITE_OK ){ + p->eStage = RBU_STAGE_CKPT; + p->nStep = (pState ? pState->nRow : 0); + p->aBuf = rbuMalloc(p, p->pgsz); + p->iWalCksum = rbuShmChecksum(p); + } + if( p->rc==SQLITE_OK && pState && pState->iWalCksum!=p->iWalCksum ){ + p->rc = SQLITE_DONE; + p->eStage = RBU_STAGE_DONE; + } +} /* -** Create a new tokenizer instance. +** Called when iAmt bytes are read from offset iOff of the wal file while +** the rbu object is in capture mode. Record the frame number of the frame +** being read in the aFrame[] array. */ -static int porterCreate( - int argc, const char * const *argv, - sqlite3_tokenizer **ppTokenizer -){ - porter_tokenizer *t; +static int rbuCaptureWalRead(sqlite3rbu *pRbu, i64 iOff, int iAmt){ + const u32 mReq = (1<mLock!=mReq ){ + pRbu->rc = SQLITE_BUSY; + return SQLITE_INTERNAL; + } - t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t)); - if( t==NULL ) return SQLITE_NOMEM; - memset(t, 0, sizeof(*t)); - *ppTokenizer = &t->base; + pRbu->pgsz = iAmt; + if( pRbu->nFrame==pRbu->nFrameAlloc ){ + int nNew = (pRbu->nFrameAlloc ? pRbu->nFrameAlloc : 64) * 2; + RbuFrame *aNew; + aNew = (RbuFrame*)sqlite3_realloc(pRbu->aFrame, nNew * sizeof(RbuFrame)); + if( aNew==0 ) return SQLITE_NOMEM; + pRbu->aFrame = aNew; + pRbu->nFrameAlloc = nNew; + } + + iFrame = (u32)((iOff-32) / (i64)(iAmt+24)) + 1; + if( pRbu->iMaxFrame