latest and stable version of sqlite
this most probably will be sqlite for HC - unless some serious bugs come up
with this version.
Change-Id: Ib7d9023c861025fd28492ae133d21fc479c0c891
diff --git a/dist/sqlite3.c b/dist/sqlite3.c
index f274f6e..18060a8 100644
--- a/dist/sqlite3.c
+++ b/dist/sqlite3.c
@@ -1,10 +1,10 @@
/******************************************************************************
** This file is an amalgamation of many separate C source files from SQLite
-** version 3.7.3. By combining all the individual C code files into this
+** version 3.7.4. By combining all the individual C code files into this
** single large file, the entire code can be compiled as a one translation
** unit. This allows many compilers to do optimizations that would not be
** possible if the files were compiled separately. Performance improvements
-** of 5% are more are commonly seen when SQLite is compiled as a single
+** of 5% or more are commonly seen when SQLite is compiled as a single
** translation unit.
**
** This file is all you need to compile SQLite. To use SQLite in other
@@ -654,9 +654,9 @@
** [sqlite3_libversion_number()], [sqlite3_sourceid()],
** [sqlite_version()] and [sqlite_source_id()].
*/
-#define SQLITE_VERSION "3.7.3"
-#define SQLITE_VERSION_NUMBER 3007003
-#define SQLITE_SOURCE_ID "2010-09-28 07:24:50 88108fff672b6012ffe87ba4b1efe6d5a07037d1"
+#define SQLITE_VERSION "3.7.4"
+#define SQLITE_VERSION_NUMBER 3007004
+#define SQLITE_SOURCE_ID "2010-12-07 20:14:09 a586a4deeb25330037a49df295b36aaf624d0f45"
/*
** CAPI3REF: Run-Time Library Version Numbers
@@ -1092,6 +1092,18 @@
** equal SQLITE_SYNC_NORMAL, that means to use normal fsync() semantics.
** If the lower four bits equal SQLITE_SYNC_FULL, that means
** to use Mac OS X style fullsync instead of fsync().
+**
+** Do not confuse the SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL flags
+** with the [PRAGMA synchronous]=NORMAL and [PRAGMA synchronous]=FULL
+** settings. The [synchronous pragma] determines when calls to the
+** xSync VFS method occur and applies uniformly across all platforms.
+** The SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL flags determine how
+** energetic or rigorous or forceful the sync operations are and
+** only make a difference on Mac OSX for the default SQLite code.
+** (Third-party VFS implementations might also make the distinction
+** between SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL, but among the
+** operating systems natively supported by SQLite, only Mac OSX
+** cares about the difference.)
*/
#define SQLITE_SYNC_NORMAL 0x00002
#define SQLITE_SYNC_FULL 0x00003
@@ -1260,6 +1272,8 @@
#define SQLITE_LAST_ERRNO 4
#define SQLITE_FCNTL_SIZE_HINT 5
#define SQLITE_FCNTL_CHUNK_SIZE 6
+#define SQLITE_FCNTL_FILE_POINTER 7
+
/*
** CAPI3REF: Mutex Handle
@@ -2286,6 +2300,9 @@
/*
** CAPI3REF: Convenience Routines For Running Queries
**
+** This is a legacy interface that is preserved for backwards compatibility.
+** Use of this interface is not recommended.
+**
** Definition: A <b>result table</b> is memory data structure created by the
** [sqlite3_get_table()] interface. A result table records the
** complete query results from one or more queries.
@@ -2306,7 +2323,7 @@
** It is not safe to pass a result table directly to [sqlite3_free()].
** A result table should be deallocated using [sqlite3_free_table()].
**
-** As an example of the result table format, suppose a query result
+** ^(As an example of the result table format, suppose a query result
** is as follows:
**
** <blockquote><pre>
@@ -2330,7 +2347,7 @@
** azResult[5] = "28";
** azResult[6] = "Cindy";
** azResult[7] = "21";
-** </pre></blockquote>
+** </pre></blockquote>)^
**
** ^The sqlite3_get_table() function evaluates one or more
** semicolon-separated SQL statements in the zero-terminated UTF-8
@@ -2338,19 +2355,19 @@
** pointer given in its 3rd parameter.
**
** After the application has finished with the result from sqlite3_get_table(),
-** it should pass the result table pointer to sqlite3_free_table() in order to
+** it must pass the result table pointer to sqlite3_free_table() in order to
** release the memory that was malloced. Because of the way the
** [sqlite3_malloc()] happens within sqlite3_get_table(), the calling
** function must not try to call [sqlite3_free()] directly. Only
** [sqlite3_free_table()] is able to release the memory properly and safely.
**
-** ^(The sqlite3_get_table() interface is implemented as a wrapper around
+** The sqlite3_get_table() interface is implemented as a wrapper around
** [sqlite3_exec()]. The sqlite3_get_table() routine does not have access
** to any internal data structures of SQLite. It uses only the public
** interface defined here. As a consequence, errors that occur in the
** wrapper layer outside of the internal [sqlite3_exec()] call are not
** reflected in subsequent calls to [sqlite3_errcode()] or
-** [sqlite3_errmsg()].)^
+** [sqlite3_errmsg()].
*/
SQLITE_API int sqlite3_get_table(
sqlite3 *db, /* An open database */
@@ -3181,6 +3198,20 @@
SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt);
/*
+** CAPI3REF: Determine If An SQL Statement Writes The Database
+**
+** ^The sqlite3_stmt_readonly(X) interface returns true (non-zero) if
+** the [prepared statement] X is [SELECT] statement and false (zero) if
+** X is an [INSERT], [UPDATE], [DELETE], CREATE, DROP, [ANALYZE],
+** [ALTER], or [REINDEX] statement.
+** If X is a NULL pointer or any other kind of statement, including but
+** not limited to [ATTACH], [DETACH], [COMMIT], [ROLLBACK], [RELEASE],
+** [SAVEPOINT], [PRAGMA], or [VACUUM] the result of sqlite3_stmt_readonly(X) is
+** undefined.
+*/
+SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt);
+
+/*
** CAPI3REF: Dynamically Typed Value Object
** KEYWORDS: {protected sqlite3_value} {unprotected sqlite3_value}
**
@@ -3279,7 +3310,10 @@
**
** ^The fifth argument to sqlite3_bind_blob(), sqlite3_bind_text(), and
** sqlite3_bind_text16() is a destructor used to dispose of the BLOB or
-** string after SQLite has finished with it. ^If the fifth argument is
+** string after SQLite has finished with it. ^The destructor is called
+** to dispose of the BLOB or string even if the call to sqlite3_bind_blob(),
+** sqlite3_bind_text(), or sqlite3_bind_text16() fails.
+** ^If the fifth argument is
** the special value [SQLITE_STATIC], then SQLite assumes that the
** information is in static, unmanaged space and does not need to be freed.
** ^If the fifth argument has the value [SQLITE_TRANSIENT], then
@@ -3812,17 +3846,26 @@
** CAPI3REF: Destroy A Prepared Statement Object
**
** ^The sqlite3_finalize() function is called to delete a [prepared statement].
-** ^If the statement was executed successfully or not executed at all, then
-** SQLITE_OK is returned. ^If execution of the statement failed then an
-** [error code] or [extended error code] is returned.
+** ^If the most recent evaluation of the statement encountered no errors or
+** or if the statement is never been evaluated, then sqlite3_finalize() returns
+** SQLITE_OK. ^If the most recent evaluation of statement S failed, then
+** sqlite3_finalize(S) returns the appropriate [error code] or
+** [extended error code].
**
-** ^This routine can be called at any point during the execution of the
-** [prepared statement]. ^If the virtual machine has not
-** completed execution when this routine is called, that is like
-** encountering an error or an [sqlite3_interrupt | interrupt].
-** ^Incomplete updates may be rolled back and transactions canceled,
-** depending on the circumstances, and the
-** [error code] returned will be [SQLITE_ABORT].
+** ^The sqlite3_finalize(S) routine can be called at any point during
+** the life cycle of [prepared statement] S:
+** before statement S is ever evaluated, after
+** one or more calls to [sqlite3_reset()], or after any call
+** to [sqlite3_step()] regardless of whether or not the statement has
+** completed execution.
+**
+** ^Invoking sqlite3_finalize() on a NULL pointer is a harmless no-op.
+**
+** The application must finalize every [prepared statement] in order to avoid
+** resource leaks. It is a grievous error for the application to try to use
+** a prepared statement after it has been finalized. Any use of a prepared
+** statement after it has been finalized can result in undefined and
+** undesirable behavior such as segfaults and heap corruption.
*/
SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt);
@@ -3910,12 +3953,15 @@
** SQL function or aggregate, pass NULL poiners for all three function
** callbacks.
**
-** ^If the tenth parameter to sqlite3_create_function_v2() is not NULL,
-** then it is invoked when the function is deleted, either by being
-** overloaded or when the database connection closes.
-** ^When the destructure callback of the tenth parameter is invoked, it
-** is passed a single argument which is a copy of the pointer which was
-** the fifth parameter to sqlite3_create_function_v2().
+** ^(If the tenth parameter to sqlite3_create_function_v2() is not NULL,
+** then it is destructor for the application data pointer.
+** The destructor is invoked when the function is deleted, either by being
+** overloaded or when the database connection closes.)^
+** ^The destructor is also invoked if the call to
+** sqlite3_create_function_v2() fails.
+** ^When the destructor callback of the tenth parameter is invoked, it
+** is passed a single argument which is a copy of the application data
+** pointer which was the fifth parameter to sqlite3_create_function_v2().
**
** ^It is permitted to register multiple implementations of the same
** functions with the same name but with either differing numbers of
@@ -4379,6 +4425,15 @@
** calls to the collation creation functions or when the
** [database connection] is closed using [sqlite3_close()].
**
+** ^The xDestroy callback is <u>not</u> called if the
+** sqlite3_create_collation_v2() function fails. Applications that invoke
+** sqlite3_create_collation_v2() with a non-NULL xDestroy argument should
+** check the return code and dispose of the application data pointer
+** themselves rather than expecting SQLite to deal with it for them.
+** This is different from every other SQLite interface. The inconsistency
+** is unfortunate but cannot be changed without breaking backwards
+** compatibility.
+**
** See also: [sqlite3_collation_needed()] and [sqlite3_collation_needed16()].
*/
SQLITE_API int sqlite3_create_collation(
@@ -5133,7 +5188,9 @@
** ^The sqlite3_create_module_v2() interface has a fifth parameter which
** is a pointer to a destructor for the pClientData. ^SQLite will
** invoke the destructor function (if it is not NULL) when SQLite
-** no longer needs the pClientData pointer. ^The sqlite3_create_module()
+** no longer needs the pClientData pointer. ^The destructor will also
+** be invoked if the call to sqlite3_create_module_v2() fails.
+** ^The sqlite3_create_module()
** interface is equivalent to sqlite3_create_module_v2() with a NULL
** destructor.
*/
@@ -5317,6 +5374,30 @@
);
/*
+** CAPI3REF: Move a BLOB Handle to a New Row
+**
+** ^This function is used to move an existing blob handle so that it points
+** to a different row of the same database table. ^The new row is identified
+** by the rowid value passed as the second argument. Only the row can be
+** changed. ^The database, table and column on which the blob handle is open
+** remain the same. Moving an existing blob handle to a new row can be
+** faster than closing the existing handle and opening a new one.
+**
+** ^(The new row must meet the same criteria as for [sqlite3_blob_open()] -
+** it must exist and there must be either a blob or text value stored in
+** the nominated column.)^ ^If the new row is not present in the table, or if
+** it does not contain a blob or text value, or if another error occurs, an
+** SQLite error code is returned and the blob handle is considered aborted.
+** ^All subsequent calls to [sqlite3_blob_read()], [sqlite3_blob_write()] or
+** [sqlite3_blob_reopen()] on an aborted blob handle immediately return
+** SQLITE_ABORT. ^Calling [sqlite3_blob_bytes()] on an aborted blob handle
+** always returns zero.
+**
+** ^This function sets the database handle error code and message.
+*/
+SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64);
+
+/*
** CAPI3REF: Close A BLOB Handle
**
** ^Closes an open [BLOB handle].
@@ -5723,7 +5804,7 @@
** ^The [sqlite3_file_control()] interface makes a direct call to the
** xFileControl method for the [sqlite3_io_methods] object associated
** with a particular database identified by the second argument. ^The
-** name of the database "main" for the main database or "temp" for the
+** name of the database is "main" for the main database or "temp" for the
** TEMP database, or the name that appears after the AS keyword for
** databases that are added using the [ATTACH] SQL command.
** ^A NULL pointer can be used in place of "main" to refer to the
@@ -5733,6 +5814,12 @@
** the xFileControl method. ^The return value of the xFileControl
** method becomes the return value of this routine.
**
+** ^The SQLITE_FCNTL_FILE_POINTER value for the op parameter causes
+** a pointer to the underlying [sqlite3_file] object to be written into
+** the space pointed to by the 4th parameter. ^The SQLITE_FCNTL_FILE_POINTER
+** case is a short-circuit path which does not actually invoke the
+** underlying sqlite3_io_methods.xFileControl method.
+**
** ^If the second parameter (zDbName) does not match the name of any
** open database file, then SQLITE_ERROR is returned. ^This error
** code is not remembered and will not be recalled by [sqlite3_errcode()]
@@ -7402,7 +7489,7 @@
SQLITE_PRIVATE int sqlite3BtreeClose(Btree*);
SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree*,int);
-SQLITE_PRIVATE int sqlite3BtreeSetSafetyLevel(Btree*,int,int);
+SQLITE_PRIVATE int sqlite3BtreeSetSafetyLevel(Btree*,int,int,int);
SQLITE_PRIVATE int sqlite3BtreeSyncDisabled(Btree*);
SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix);
SQLITE_PRIVATE int sqlite3BtreeGetPageSize(Btree*);
@@ -7886,12 +7973,12 @@
#define OP_VRename 132
#define OP_VUpdate 133
#define OP_Pagecount 134
-#define OP_Trace 135
-#define OP_Noop 136
-#define OP_Explain 137
+#define OP_MaxPgcnt 135
+#define OP_Trace 136
+#define OP_Noop 137
+#define OP_Explain 138
/* The following opcode values are never used */
-#define OP_NotUsed_138 138
#define OP_NotUsed_139 139
#define OP_NotUsed_140 140
@@ -7924,7 +8011,7 @@
/* 104 */ 0x00, 0x0c, 0x45, 0x15, 0x01, 0x02, 0x00, 0x01,\
/* 112 */ 0x08, 0x05, 0x05, 0x05, 0x00, 0x00, 0x00, 0x02,\
/* 120 */ 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\
-/* 128 */ 0x01, 0x00, 0x02, 0x01, 0x00, 0x00, 0x02, 0x00,\
+/* 128 */ 0x01, 0x00, 0x02, 0x01, 0x00, 0x00, 0x02, 0x02,\
/* 136 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04,\
/* 144 */ 0x04, 0x04,}
@@ -8109,7 +8196,7 @@
SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager*, u32*, int);
SQLITE_PRIVATE int sqlite3PagerMaxPageCount(Pager*, int);
SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager*, int);
-SQLITE_PRIVATE void sqlite3PagerSetSafetyLevel(Pager*,int,int);
+SQLITE_PRIVATE void sqlite3PagerSetSafetyLevel(Pager*,int,int,int);
SQLITE_PRIVATE int sqlite3PagerLockingMode(Pager *, int);
SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *, int);
SQLITE_PRIVATE int sqlite3PagerGetJournalMode(Pager*);
@@ -8974,13 +9061,14 @@
#define SQLITE_ReadUncommitted 0x0080000 /* For shared-cache mode */
#define SQLITE_LegacyFileFmt 0x00100000 /* Create new databases in format 1 */
#define SQLITE_FullFSync 0x00200000 /* Use full fsync on the backend */
-#define SQLITE_LoadExtension 0x00400000 /* Enable load_extension */
+#define SQLITE_CkptFullFSync 0x00400000 /* Use full fsync for checkpoint */
#define SQLITE_RecoveryMode 0x00800000 /* Ignore schema errors */
#define SQLITE_ReverseOrder 0x01000000 /* Reverse unordered SELECTs */
#define SQLITE_RecTriggers 0x02000000 /* Enable recursive triggers */
#define SQLITE_ForeignKeys 0x04000000 /* Enforce foreign key constraints */
#define SQLITE_AutoIndex 0x08000000 /* Enable automatic indexes */
#define SQLITE_PreferBuiltin 0x10000000 /* Preference to built-in funcs */
+#define SQLITE_LoadExtension 0x20000000 /* Enable load_extension */
/*
** Bits of the sqlite3.flags field that are used by the
@@ -8993,6 +9081,7 @@
#define SQLITE_IndexSearch 0x08 /* Disable indexes for searching */
#define SQLITE_IndexCover 0x10 /* Disable index covering table */
#define SQLITE_GroupByOrder 0x20 /* Disable GROUPBY cover of ORDERBY */
+#define SQLITE_FactorOutConst 0x40 /* Disable factoring out constants */
#define SQLITE_OptMask 0xff /* Mask of all disablable opts */
/*
@@ -9084,15 +9173,15 @@
*/
#define FUNCTION(zName, nArg, iArg, bNC, xFunc) \
{nArg, SQLITE_UTF8, bNC*SQLITE_FUNC_NEEDCOLL, \
- SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0}
+ SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0}
#define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \
{nArg, SQLITE_UTF8, bNC*SQLITE_FUNC_NEEDCOLL, \
- pArg, 0, xFunc, 0, 0, #zName, 0}
+ pArg, 0, xFunc, 0, 0, #zName, 0, 0}
#define LIKEFUNC(zName, nArg, arg, flags) \
- {nArg, SQLITE_UTF8, flags, (void *)arg, 0, likeFunc, 0, 0, #zName, 0}
+ {nArg, SQLITE_UTF8, flags, (void *)arg, 0, likeFunc, 0, 0, #zName, 0, 0}
#define AGGREGATE(zName, nArg, arg, nc, xStep, xFinal) \
{nArg, SQLITE_UTF8, nc*SQLITE_FUNC_NEEDCOLL, \
- SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0}
+ SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0,0}
/*
** All current savepoints are stored in a linked list starting at
@@ -9881,6 +9970,9 @@
u8 isPopulated; /* Temporary table associated with SELECT is populated */
u8 jointype; /* Type of join between this able and the previous */
u8 notIndexed; /* True if there is a NOT INDEXED clause */
+#ifndef SQLITE_OMIT_EXPLAIN
+ u8 iSelectId; /* If pSelect!=0, the id of the sub-select in EQP */
+#endif
int iCursor; /* The VDBE cursor number used to access this table */
Expr *pOn; /* The ON clause of a join */
IdList *pUsing; /* The USING clause of a join */
@@ -9919,6 +10011,7 @@
struct WherePlan {
u32 wsFlags; /* WHERE_* flags that describe the strategy */
u32 nEq; /* Number of == constraints */
+ double nRow; /* Estimated number of rows (for EQP) */
union {
Index *pIdx; /* Index when WHERE_INDEXED is true */
struct WhereTerm *pTerm; /* WHERE clause term for OR-search */
@@ -10003,6 +10096,7 @@
int nLevel; /* Number of nested loop */
struct WhereClause *pWC; /* Decomposition of the WHERE clause */
double savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */
+ double nRowOut; /* Estimated number of output rows */
WhereLevel a[1]; /* Information about each nest loop in WHERE */
};
@@ -10078,6 +10172,7 @@
Expr *pOffset; /* OFFSET expression. NULL means not used. */
int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */
int addrOpenEphm[3]; /* OP_OpenEphem opcodes related to this select */
+ double nSelectRow; /* Estimated number of result rows */
};
/*
@@ -10273,6 +10368,11 @@
int nHeight; /* Expression tree height of current sub-select */
Table *pZombieTab; /* List of Table objects to delete after code gen */
TriggerPrg *pTriggerPrg; /* Linked list of coded triggers */
+
+#ifndef SQLITE_OMIT_EXPLAIN
+ int iSelectId;
+ int iNextSelectId;
+#endif
};
#ifdef SQLITE_OMIT_VIRTUALTABLE
@@ -10567,7 +10667,6 @@
** Internal function prototypes
*/
SQLITE_PRIVATE int sqlite3StrICmp(const char *, const char *);
-SQLITE_PRIVATE int sqlite3IsNumber(const char*, int*, u8);
SQLITE_PRIVATE int sqlite3Strlen30(const char*);
#define sqlite3StrNICmp sqlite3_strnicmp
@@ -10887,9 +10986,9 @@
SQLITE_PRIVATE int sqlite3FixExpr(DbFixer*, Expr*);
SQLITE_PRIVATE int sqlite3FixExprList(DbFixer*, ExprList*);
SQLITE_PRIVATE int sqlite3FixTriggerStep(DbFixer*, TriggerStep*);
-SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*);
+SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*, int, u8);
SQLITE_PRIVATE int sqlite3GetInt32(const char *, int*);
-SQLITE_PRIVATE int sqlite3FitsIn64Bits(const char *, int);
+SQLITE_PRIVATE int sqlite3Atoi(const char*);
SQLITE_PRIVATE int sqlite3Utf16ByteLen(const void *pData, int nChar);
SQLITE_PRIVATE int sqlite3Utf8CharLen(const char *pData, int nByte);
SQLITE_PRIVATE int sqlite3Utf8Read(const u8*, const u8**);
@@ -10935,7 +11034,7 @@
SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2);
SQLITE_PRIVATE int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity);
SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr);
-SQLITE_PRIVATE int sqlite3Atoi64(const char*, i64*);
+SQLITE_PRIVATE int sqlite3Atoi64(const char*, i64*, int, u8);
SQLITE_PRIVATE void sqlite3Error(sqlite3*, int, const char*,...);
SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3*, const char *z, int n);
SQLITE_PRIVATE int sqlite3TwoPartName(Parse *, Token *, Token *, Token **);
@@ -11968,26 +12067,34 @@
** restoring the state of the VM to as it was before the sub-program
** began executing.
**
-** Frames are stored in a linked list headed at Vdbe.pParent. Vdbe.pParent
-** is the parent of the current frame, or zero if the current frame
-** is the main Vdbe program.
+** The memory for a VdbeFrame object is allocated and managed by a memory
+** cell in the parent (calling) frame. When the memory cell is deleted or
+** overwritten, the VdbeFrame object is not freed immediately. Instead, it
+** is linked into the Vdbe.pDelFrame list. The contents of the Vdbe.pDelFrame
+** list is deleted when the VM is reset in VdbeHalt(). The reason for doing
+** this instead of deleting the VdbeFrame immediately is to avoid recursive
+** calls to sqlite3VdbeMemRelease() when the memory cells belonging to the
+** child frame are released.
+**
+** The currently executing frame is stored in Vdbe.pFrame. Vdbe.pFrame is
+** set to NULL if the currently executing frame is the main program.
*/
typedef struct VdbeFrame VdbeFrame;
struct VdbeFrame {
Vdbe *v; /* VM this frame belongs to */
- int pc; /* Program Counter */
- Op *aOp; /* Program instructions */
+ int pc; /* Program Counter in parent (calling) frame */
+ Op *aOp; /* Program instructions for parent frame */
int nOp; /* Size of aOp array */
- Mem *aMem; /* Array of memory cells */
+ Mem *aMem; /* Array of memory cells for parent frame */
int nMem; /* Number of entries in aMem */
- VdbeCursor **apCsr; /* Element of Vdbe cursors */
+ VdbeCursor **apCsr; /* Array of Vdbe cursors for parent frame */
u16 nCursor; /* Number of entries in apCsr */
void *token; /* Copy of SubProgram.token */
int nChildMem; /* Number of memory cells for child frame */
int nChildCsr; /* Number of cursors for child frame */
i64 lastRowid; /* Last insert rowid (sqlite3.lastRowid) */
int nChange; /* Statement changes (Vdbe.nChanges) */
- VdbeFrame *pParent; /* Parent of this frame */
+ VdbeFrame *pParent; /* Parent of this frame, or NULL if parent is main */
};
#define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))])
@@ -12204,6 +12311,7 @@
FILE *trace; /* Write an execution trace here, if not NULL */
#endif
VdbeFrame *pFrame; /* Parent frame */
+ VdbeFrame *pDelFrame; /* List of frame objects to free on VM reset */
int nFrame; /* Number of frames in pFrame list */
u32 expmask; /* Binding to these vars invalidates VM */
SubProgram *pProgram; /* Linked list of all sub-programs used by VM */
@@ -12635,12 +12743,6 @@
}
/*
-** Read text from z[] and convert into a floating point number. Return
-** the number of digits converted.
-*/
-#define getValue sqlite3AtoF
-
-/*
** Parse a timezone extension on the end of a date-time.
** The extension is of the form:
**
@@ -12841,7 +12943,7 @@
const char *zDate,
DateTime *p
){
- int isRealNum; /* Return from sqlite3IsNumber(). Not used */
+ double r;
if( parseYyyyMmDd(zDate,p)==0 ){
return 0;
}else if( parseHhMmSs(zDate, p)==0 ){
@@ -12849,9 +12951,7 @@
}else if( sqlite3StrICmp(zDate,"now")==0){
setDateTimeToCurrent(context, p);
return 0;
- }else if( sqlite3IsNumber(zDate, &isRealNum, SQLITE_UTF8) ){
- double r;
- getValue(zDate, &r);
+ }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8) ){
p->iJD = (sqlite3_int64)(r*86400000.0 + 0.5);
p->validJD = 1;
return 0;
@@ -13072,8 +13172,9 @@
** weekday N where 0==Sunday, 1==Monday, and so forth. If the
** date is already on the appropriate weekday, this is a no-op.
*/
- if( strncmp(z, "weekday ", 8)==0 && getValue(&z[8],&r)>0
- && (n=(int)r)==r && n>=0 && r<7 ){
+ if( strncmp(z, "weekday ", 8)==0
+ && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8)
+ && (n=(int)r)==r && n>=0 && r<7 ){
sqlite3_int64 Z;
computeYMD_HMS(p);
p->validTZ = 0;
@@ -13128,8 +13229,11 @@
case '8':
case '9': {
double rRounder;
- n = getValue(z, &r);
- assert( n>=1 );
+ for(n=1; z[n] && z[n]!=':' && !sqlite3Isspace(z[n]); n++){}
+ if( !sqlite3AtoF(z, &r, n, SQLITE_UTF8) ){
+ rc = 1;
+ break;
+ }
if( z[n]==':' ){
/* A modifier of the form (+|-)HH:MM:SS.FFF adds (or subtracts) the
** specified number of hours, minutes, seconds, and fractional seconds
@@ -17428,7 +17532,7 @@
** sqlite3_soft_heap_limit() setting.
*/
int nearlyFull;
-} mem0 = { 0, 0, 0, 0, 0, 0, 0 };
+} mem0 = { 0, 0, 0, 0, 0, 0, 0, 0 };
#define mem0 GLOBAL(struct Mem0Global, mem0)
@@ -17499,7 +17603,7 @@
sqlite3MemoryAlarm(0, 0, 0);
}
excess = sqlite3_memory_used() - n;
- if( excess>0 ) sqlite3_release_memory(excess & 0x7fffffff);
+ if( excess>0 ) sqlite3_release_memory((int)(excess & 0x7fffffff));
return priorLimit;
}
SQLITE_API void sqlite3_soft_heap_limit(int n){
@@ -20093,121 +20197,111 @@
}
/*
-** Return TRUE if z is a pure numeric string. Return FALSE and leave
-** *realnum unchanged if the string contains any character which is not
-** part of a number.
+** The string z[] is an text representation of a real number.
+** Convert this string to a double and write it into *pResult.
**
-** If the string is pure numeric, set *realnum to TRUE if the string
-** contains the '.' character or an "E+000" style exponentiation suffix.
-** Otherwise set *realnum to FALSE. Note that just becaue *realnum is
-** false does not mean that the number can be successfully converted into
-** an integer - it might be too big.
+** The string z[] is length bytes in length (bytes, not characters) and
+** uses the encoding enc. The string is not necessarily zero-terminated.
**
-** An empty string is considered non-numeric.
+** Return TRUE if the result is a valid real number (or integer) and FALSE
+** if the string is empty or contains extraneous text. Valid numbers
+** are in one of these formats:
+**
+** [+-]digits[E[+-]digits]
+** [+-]digits.[digits][E[+-]digits]
+** [+-].digits[E[+-]digits]
+**
+** Leading and trailing whitespace is ignored for the purpose of determining
+** validity.
+**
+** If some prefix of the input string is a valid number, this routine
+** returns FALSE but it still converts the prefix and writes the result
+** into *pResult.
*/
-SQLITE_PRIVATE int sqlite3IsNumber(const char *z, int *realnum, u8 enc){
+SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){
+#ifndef SQLITE_OMIT_FLOATING_POINT
int incr = (enc==SQLITE_UTF8?1:2);
- if( enc==SQLITE_UTF16BE ) z++;
- if( *z=='-' || *z=='+' ) z += incr;
- if( !sqlite3Isdigit(*z) ){
- return 0;
- }
- z += incr;
- *realnum = 0;
- while( sqlite3Isdigit(*z) ){ z += incr; }
-#ifndef SQLITE_OMIT_FLOATING_POINT
- if( *z=='.' ){
- z += incr;
- if( !sqlite3Isdigit(*z) ) return 0;
- while( sqlite3Isdigit(*z) ){ z += incr; }
- *realnum = 1;
- }
- if( *z=='e' || *z=='E' ){
- z += incr;
- if( *z=='+' || *z=='-' ) z += incr;
- if( !sqlite3Isdigit(*z) ) return 0;
- while( sqlite3Isdigit(*z) ){ z += incr; }
- *realnum = 1;
- }
-#endif
- return *z==0;
-}
-
-/*
-** The string z[] is an ASCII representation of a real number.
-** Convert this string to a double.
-**
-** This routine assumes that z[] really is a valid number. If it
-** is not, the result is undefined.
-**
-** This routine is used instead of the library atof() function because
-** the library atof() might want to use "," as the decimal point instead
-** of "." depending on how locale is set. But that would cause problems
-** for SQL. So this routine always uses "." regardless of locale.
-*/
-SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult){
-#ifndef SQLITE_OMIT_FLOATING_POINT
- const char *zBegin = z;
+ const char *zEnd = z + length;
/* sign * significand * (10 ^ (esign * exponent)) */
- int sign = 1; /* sign of significand */
- i64 s = 0; /* significand */
- int d = 0; /* adjust exponent for shifting decimal point */
- int esign = 1; /* sign of exponent */
- int e = 0; /* exponent */
+ int sign = 1; /* sign of significand */
+ i64 s = 0; /* significand */
+ int d = 0; /* adjust exponent for shifting decimal point */
+ int esign = 1; /* sign of exponent */
+ int e = 0; /* exponent */
+ int eValid = 1; /* True exponent is either not used or is well-formed */
double result;
int nDigits = 0;
+ *pResult = 0.0; /* Default return value, in case of an error */
+
+ if( enc==SQLITE_UTF16BE ) z++;
+
/* skip leading spaces */
- while( sqlite3Isspace(*z) ) z++;
+ while( z<zEnd && sqlite3Isspace(*z) ) z+=incr;
+ if( z>=zEnd ) return 0;
+
/* get sign of significand */
if( *z=='-' ){
sign = -1;
- z++;
+ z+=incr;
}else if( *z=='+' ){
- z++;
+ z+=incr;
}
+
/* skip leading zeroes */
- while( z[0]=='0' ) z++, nDigits++;
+ while( z<zEnd && z[0]=='0' ) z+=incr, nDigits++;
/* copy max significant digits to significand */
- while( sqlite3Isdigit(*z) && s<((LARGEST_INT64-9)/10) ){
+ while( z<zEnd && sqlite3Isdigit(*z) && s<((LARGEST_INT64-9)/10) ){
s = s*10 + (*z - '0');
- z++, nDigits++;
+ z+=incr, nDigits++;
}
+
/* skip non-significant significand digits
** (increase exponent by d to shift decimal left) */
- while( sqlite3Isdigit(*z) ) z++, nDigits++, d++;
+ while( z<zEnd && sqlite3Isdigit(*z) ) z+=incr, nDigits++, d++;
+ if( z>=zEnd ) goto do_atof_calc;
/* if decimal point is present */
if( *z=='.' ){
- z++;
+ z+=incr;
/* copy digits from after decimal to significand
** (decrease exponent by d to shift decimal right) */
- while( sqlite3Isdigit(*z) && s<((LARGEST_INT64-9)/10) ){
+ while( z<zEnd && sqlite3Isdigit(*z) && s<((LARGEST_INT64-9)/10) ){
s = s*10 + (*z - '0');
- z++, nDigits++, d--;
+ z+=incr, nDigits++, d--;
}
/* skip non-significant digits */
- while( sqlite3Isdigit(*z) ) z++, nDigits++;
+ while( z<zEnd && sqlite3Isdigit(*z) ) z+=incr, nDigits++;
}
+ if( z>=zEnd ) goto do_atof_calc;
/* if exponent is present */
if( *z=='e' || *z=='E' ){
- z++;
+ z+=incr;
+ eValid = 0;
+ if( z>=zEnd ) goto do_atof_calc;
/* get sign of exponent */
if( *z=='-' ){
esign = -1;
- z++;
+ z+=incr;
}else if( *z=='+' ){
- z++;
+ z+=incr;
}
/* copy digits to exponent */
- while( sqlite3Isdigit(*z) ){
+ while( z<zEnd && sqlite3Isdigit(*z) ){
e = e*10 + (*z - '0');
- z++;
+ z+=incr;
+ eValid = 1;
}
}
+ /* skip trailing spaces */
+ if( nDigits && eValid ){
+ while( z<zEnd && sqlite3Isspace(*z) ) z+=incr;
+ }
+
+do_atof_calc:
/* adjust exponent by d, and update sign */
e = (e*esign) + d;
if( e<0 ) {
@@ -20266,10 +20360,10 @@
/* store the result */
*pResult = result;
- /* return number of characters used */
- return (int)(z - zBegin);
+ /* return true if number and no extra non-whitespace chracters after */
+ return z>=zEnd && nDigits>0 && eValid;
#else
- return sqlite3Atoi64(z, pResult);
+ return !sqlite3Atoi64(z, pResult, length, enc);
#endif /* SQLITE_OMIT_FLOATING_POINT */
}
@@ -20277,20 +20371,26 @@
** Compare the 19-character string zNum against the text representation
** value 2^63: 9223372036854775808. Return negative, zero, or positive
** if zNum is less than, equal to, or greater than the string.
+** Note that zNum must contain exactly 19 characters.
**
** Unlike memcmp() this routine is guaranteed to return the difference
** in the values of the last digit if the only difference is in the
** last digit. So, for example,
**
-** compare2pow63("9223372036854775800")
+** compare2pow63("9223372036854775800", 1)
**
** will return -8.
*/
-static int compare2pow63(const char *zNum){
- int c;
- c = memcmp(zNum,"922337203685477580",18)*10;
+static int compare2pow63(const char *zNum, int incr){
+ int c = 0;
+ int i;
+ /* 012345678901234567 */
+ const char *pow63 = "922337203685477580";
+ for(i=0; c==0 && i<18; i++){
+ c = (zNum[i*incr]-pow63[i])*10;
+ }
if( c==0 ){
- c = zNum[18] - '8';
+ c = zNum[18*incr] - '8';
testcase( c==(-1) );
testcase( c==0 );
testcase( c==(+1) );
@@ -20300,94 +20400,60 @@
/*
-** Return TRUE if zNum is a 64-bit signed integer and write
-** the value of the integer into *pNum. If zNum is not an integer
-** or is an integer that is too large to be expressed with 64 bits,
-** then return false.
+** Convert zNum to a 64-bit signed integer and write
+** the value of the integer into *pNum.
+** If zNum is exactly 9223372036854665808, return 2.
+** This is a special case as the context will determine
+** if it is too big (used as a negative).
+** If zNum is not an integer or is an integer that
+** is too large to be expressed with 64 bits,
+** then return 1. Otherwise return 0.
**
-** When this routine was originally written it dealt with only
-** 32-bit numbers. At that time, it was much faster than the
-** atoi() library routine in RedHat 7.2.
+** length is the number of bytes in the string (bytes, not characters).
+** The string is not necessarily zero-terminated. The encoding is
+** given by enc.
*/
-SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum){
+SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){
+ int incr = (enc==SQLITE_UTF8?1:2);
i64 v = 0;
- int neg;
- int i, c;
+ int neg = 0; /* assume positive */
+ int i;
+ int c = 0;
const char *zStart;
- while( sqlite3Isspace(*zNum) ) zNum++;
+ const char *zEnd = zNum + length;
+ if( enc==SQLITE_UTF16BE ) zNum++;
+ while( zNum<zEnd && sqlite3Isspace(*zNum) ) zNum+=incr;
+ if( zNum>=zEnd ) goto do_atoi_calc;
if( *zNum=='-' ){
neg = 1;
- zNum++;
+ zNum+=incr;
}else if( *zNum=='+' ){
- neg = 0;
- zNum++;
- }else{
- neg = 0;
+ zNum+=incr;
}
+do_atoi_calc:
zStart = zNum;
- while( zNum[0]=='0' ){ zNum++; } /* Skip over leading zeros. Ticket #2454 */
- for(i=0; (c=zNum[i])>='0' && c<='9'; i++){
+ while( zNum<zEnd && zNum[0]=='0' ){ zNum+=incr; } /* Skip leading zeros. */
+ for(i=0; &zNum[i]<zEnd && (c=zNum[i])>='0' && c<='9'; i+=incr){
v = v*10 + c - '0';
}
*pNum = neg ? -v : v;
testcase( i==18 );
testcase( i==19 );
testcase( i==20 );
- if( c!=0 || (i==0 && zStart==zNum) || i>19 ){
+ if( (c!=0 && &zNum[i]<zEnd) || (i==0 && zStart==zNum) || i>19*incr ){
/* zNum is empty or contains non-numeric text or is longer
- ** than 19 digits (thus guaranting that it is too large) */
- return 0;
- }else if( i<19 ){
- /* Less than 19 digits, so we know that it fits in 64 bits */
+ ** than 19 digits (thus guaranteeing that it is too large) */
return 1;
+ }else if( i<19*incr ){
+ /* Less than 19 digits, so we know that it fits in 64 bits */
+ return 0;
}else{
/* 19-digit numbers must be no larger than 9223372036854775807 if positive
** or 9223372036854775808 if negative. Note that 9223372036854665808
- ** is 2^63. */
- return compare2pow63(zNum)<neg;
- }
-}
-
-/*
-** The string zNum represents an unsigned integer. The zNum string
-** consists of one or more digit characters and is terminated by
-** a zero character. Any stray characters in zNum result in undefined
-** behavior.
-**
-** If the unsigned integer that zNum represents will fit in a
-** 64-bit signed integer, return TRUE. Otherwise return FALSE.
-**
-** If the negFlag parameter is true, that means that zNum really represents
-** a negative number. (The leading "-" is omitted from zNum.) This
-** parameter is needed to determine a boundary case. A string
-** of "9223373036854775808" returns false if negFlag is false or true
-** if negFlag is true.
-**
-** Leading zeros are ignored.
-*/
-SQLITE_PRIVATE int sqlite3FitsIn64Bits(const char *zNum, int negFlag){
- int i;
- int neg = 0;
-
- assert( zNum[0]>='0' && zNum[0]<='9' ); /* zNum is an unsigned number */
-
- if( negFlag ) neg = 1-neg;
- while( *zNum=='0' ){
- zNum++; /* Skip leading zeros. Ticket #2454 */
- }
- for(i=0; zNum[i]; i++){ assert( zNum[i]>='0' && zNum[i]<='9' ); }
- testcase( i==18 );
- testcase( i==19 );
- testcase( i==20 );
- if( i<19 ){
- /* Guaranteed to fit if less than 19 digits */
- return 1;
- }else if( i>19 ){
- /* Guaranteed to be too big if greater than 19 digits */
- return 0;
- }else{
- /* Compare against 2^63. */
- return compare2pow63(zNum)<neg;
+ ** is 2^63. Return 1 if to large */
+ c=compare2pow63(zNum, incr);
+ if( c==0 && neg==0 ) return 2; /* too big, exactly 9223372036854665808 */
+ return c<neg ? 0 : 1;
}
}
@@ -20435,6 +20501,16 @@
}
/*
+** Return a 32-bit integer value extracted from a string. If the
+** string is not an integer, just return 0.
+*/
+SQLITE_PRIVATE int sqlite3Atoi(const char *z){
+ int x = 0;
+ if( z ) sqlite3GetInt32(z, &x);
+ return x;
+}
+
+/*
** The variable-length integer encoding is as follows:
**
** KEY:
@@ -21363,10 +21439,10 @@
/* 132 */ "VRename",
/* 133 */ "VUpdate",
/* 134 */ "Pagecount",
- /* 135 */ "Trace",
- /* 136 */ "Noop",
- /* 137 */ "Explain",
- /* 138 */ "NotUsed_138",
+ /* 135 */ "MaxPgcnt",
+ /* 136 */ "Trace",
+ /* 137 */ "Noop",
+ /* 138 */ "Explain",
/* 139 */ "NotUsed_139",
/* 140 */ "NotUsed_140",
/* 141 */ "ToText",
@@ -26998,11 +27074,21 @@
*/
UNUSED_PARAMETER(isDelete);
+ /* Usually the path zFilename should not be a relative pathname. The
+ ** exception is when opening the proxy "conch" file in builds that
+ ** include the special Apple locking styles.
+ */
+#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE
+ assert( zFilename==0 || zFilename[0]=='/'
+ || pVfs->pAppData==(void*)&autolockIoFinder );
+#else
+ assert( zFilename==0 || zFilename[0]=='/' );
+#endif
+
OSTRACE(("OPEN %-3d %s\n", h, zFilename));
pNew->h = h;
pNew->dirfd = dirfd;
pNew->fileFlags = 0;
- assert( zFilename==0 || zFilename[0]=='/' ); /* Never a relative pathname */
pNew->zPath = zFilename;
#if OS_VXWORKS
@@ -27342,9 +27428,24 @@
int nDb; /* Number of valid bytes in zDb */
struct stat sStat; /* Output of stat() on database file */
- nDb = sqlite3Strlen30(zPath) - ((flags & SQLITE_OPEN_WAL) ? 4 : 8);
+ /* zPath is a path to a WAL or journal file. The following block derives
+ ** the path to the associated database file from zPath. This block handles
+ ** the following naming conventions:
+ **
+ ** "<path to db>-journal"
+ ** "<path to db>-wal"
+ ** "<path to db>-journal-NNNN"
+ ** "<path to db>-wal-NNNN"
+ **
+ ** where NNNN is a 4 digit decimal number. The NNNN naming schemes are
+ ** used by the test_multiplex.c module.
+ */
+ nDb = sqlite3Strlen30(zPath) - 1;
+ while( nDb>0 && zPath[nDb]!='l' ) nDb--;
+ nDb -= ((flags & SQLITE_OPEN_WAL) ? 3 : 7);
memcpy(zDb, zPath, nDb);
zDb[nDb] = '\0';
+
if( 0==stat(zDb, &sStat) ){
*pMode = sStat.st_mode & 0777;
}else{
@@ -27759,7 +27860,7 @@
** error message.
*/
static void unixDlError(sqlite3_vfs *NotUsed, int nBuf, char *zBufOut){
- char *zErr;
+ const char *zErr;
UNUSED_PARAMETER(NotUsed);
unixEnterMutex();
zErr = dlerror();
@@ -27896,7 +27997,7 @@
#if defined(NO_GETTOD)
time_t t;
time(&t);
- *piNow = ((sqlite3_int64)i)*1000 + unixEpoch;
+ *piNow = ((sqlite3_int64)t)*1000 + unixEpoch;
#elif OS_VXWORKS
struct timespec sNow;
clock_gettime(CLOCK_REALTIME, &sNow);
@@ -28353,27 +28454,27 @@
pathLen = strlcpy(tPath, cPath, MAXPATHLEN);
if( pathLen>MAXPATHLEN || pathLen<6 ||
(strlcpy(&tPath[pathLen-5], "break", 6) != 5) ){
- sprintf(errmsg, "path error (len %d)", (int)pathLen);
+ sqlite3_snprintf(sizeof(errmsg),errmsg,"path error (len %d)",(int)pathLen);
goto end_breaklock;
}
/* read the conch content */
readLen = pread(conchFile->h, buf, PROXY_MAXCONCHLEN, 0);
if( readLen<PROXY_PATHINDEX ){
- sprintf(errmsg, "read error (len %d)", (int)readLen);
+ sqlite3_snprintf(sizeof(errmsg),errmsg,"read error (len %d)",(int)readLen);
goto end_breaklock;
}
/* write it out to the temporary break file */
fd = open(tPath, (O_RDWR|O_CREAT|O_EXCL), SQLITE_DEFAULT_FILE_PERMISSIONS);
if( fd<0 ){
- sprintf(errmsg, "create failed (%d)", errno);
+ sqlite3_snprintf(sizeof(errmsg), errmsg, "create failed (%d)", errno);
goto end_breaklock;
}
if( pwrite(fd, buf, readLen, 0) != (ssize_t)readLen ){
- sprintf(errmsg, "write failed (%d)", errno);
+ sqlite3_snprintf(sizeof(errmsg), errmsg, "write failed (%d)", errno);
goto end_breaklock;
}
if( rename(tPath, cPath) ){
- sprintf(errmsg, "rename failed (%d)", errno);
+ sqlite3_snprintf(sizeof(errmsg), errmsg, "rename failed (%d)", errno);
goto end_breaklock;
}
rc = 0;
@@ -34550,6 +34651,7 @@
# define sqlite3WalCheckpoint(u,v,w,x) 0
# define sqlite3WalCallback(z) 0
# define sqlite3WalExclusiveMode(y,z) 0
+# define sqlite3WalHeapMemory(z) 0
#else
#define WAL_SAVEPOINT_NDATA 4
@@ -34560,7 +34662,7 @@
typedef struct Wal Wal;
/* Open and close a connection to a write-ahead log. */
-SQLITE_PRIVATE int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *zName, Wal**);
+SQLITE_PRIVATE int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *zName, int, Wal**);
SQLITE_PRIVATE int sqlite3WalClose(Wal *pWal, int sync_flags, int, u8 *);
/* Used by readers to open (lock) and close (unlock) a snapshot. A
@@ -34617,6 +34719,12 @@
*/
SQLITE_PRIVATE int sqlite3WalExclusiveMode(Wal *pWal, int op);
+/* Return true if the argument is non-NULL and the WAL module is using
+** heap-memory for the wal-index. Otherwise, if the argument is NULL or the
+** WAL module is using shared-memory, return false.
+*/
+SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal);
+
#endif /* ifndef SQLITE_OMIT_WAL */
#endif /* _WAL_H_ */
@@ -35216,7 +35324,8 @@
u8 noReadlock; /* Do not bother to obtain readlocks */
u8 noSync; /* Do not sync the journal if true */
u8 fullSync; /* Do extra syncs of the journal for robustness */
- u8 sync_flags; /* One of SYNC_NORMAL or SYNC_FULL */
+ u8 ckptSyncFlags; /* SYNC_NORMAL or SYNC_FULL for checkpoint */
+ u8 syncFlags; /* SYNC_NORMAL or SYNC_FULL otherwise */
u8 tempFile; /* zFilename is a temporary file */
u8 readOnly; /* True for a read-only database */
u8 memDb; /* True to inhibit all file I/O */
@@ -35527,7 +35636,9 @@
return 1;
}
+#endif /* ifndef NDEBUG */
+#ifdef SQLITE_DEBUG
/*
** Return a pointer to a human readable string in a static buffer
** containing the state of the Pager object passed as an argument. This
@@ -35651,7 +35762,7 @@
static int pagerUnlockDb(Pager *pPager, int eLock){
int rc = SQLITE_OK;
- assert( !pPager->exclusiveMode );
+ assert( !pPager->exclusiveMode || pPager->eLock==eLock );
assert( eLock==NO_LOCK || eLock==SHARED_LOCK );
assert( eLock!=NO_LOCK || pagerUseWal(pPager)==0 );
if( isOpen(pPager->fd) ){
@@ -35898,7 +36009,7 @@
rc = sqlite3OsWrite(pPager->jfd, zeroHdr, sizeof(zeroHdr), 0);
}
if( rc==SQLITE_OK && !pPager->noSync ){
- rc = sqlite3OsSync(pPager->jfd, SQLITE_SYNC_DATAONLY|pPager->sync_flags);
+ rc = sqlite3OsSync(pPager->jfd, SQLITE_SYNC_DATAONLY|pPager->syncFlags);
}
/* At this point the transaction is committed but the write lock
@@ -37350,7 +37461,7 @@
if( rc==SQLITE_OK && !pPager->noSync
&& (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN)
){
- rc = sqlite3OsSync(pPager->fd, pPager->sync_flags);
+ rc = sqlite3OsSync(pPager->fd, pPager->syncFlags);
}
if( rc==SQLITE_OK ){
rc = pager_end_transaction(pPager, zMaster[0]!='\0');
@@ -37524,13 +37635,13 @@
PgHdr *pList, /* List of frames to log */
Pgno nTruncate, /* Database size after this commit */
int isCommit, /* True if this is a commit */
- int sync_flags /* Flags to pass to OsSync() (or 0) */
+ int syncFlags /* Flags to pass to OsSync() (or 0) */
){
int rc; /* Return code */
assert( pPager->pWal );
rc = sqlite3WalFrames(pPager->pWal,
- pPager->pageSize, pList, nTruncate, isCommit, sync_flags
+ pPager->pageSize, pList, nTruncate, isCommit, syncFlags
);
if( rc==SQLITE_OK && pPager->pBackup ){
PgHdr *p;
@@ -37860,14 +37971,49 @@
** assurance that the journal will not be corrupted to the
** point of causing damage to the database during rollback.
**
+** The above is for a rollback-journal mode. For WAL mode, OFF continues
+** to mean that no syncs ever occur. NORMAL means that the WAL is synced
+** prior to the start of checkpoint and that the database file is synced
+** at the conclusion of the checkpoint if the entire content of the WAL
+** was written back into the database. But no sync operations occur for
+** an ordinary commit in NORMAL mode with WAL. FULL means that the WAL
+** file is synced following each commit operation, in addition to the
+** syncs associated with NORMAL.
+**
+** Do not confuse synchronous=FULL with SQLITE_SYNC_FULL. The
+** SQLITE_SYNC_FULL macro means to use the MacOSX-style full-fsync
+** using fcntl(F_FULLFSYNC). SQLITE_SYNC_NORMAL means to do an
+** ordinary fsync() call. There is no difference between SQLITE_SYNC_FULL
+** and SQLITE_SYNC_NORMAL on platforms other than MacOSX. But the
+** synchronous=FULL versus synchronous=NORMAL setting determines when
+** the xSync primitive is called and is relevant to all platforms.
+**
** Numeric values associated with these states are OFF==1, NORMAL=2,
** and FULL=3.
*/
#ifndef SQLITE_OMIT_PAGER_PRAGMAS
-SQLITE_PRIVATE void sqlite3PagerSetSafetyLevel(Pager *pPager, int level, int bFullFsync){
+SQLITE_PRIVATE void sqlite3PagerSetSafetyLevel(
+ Pager *pPager, /* The pager to set safety level for */
+ int level, /* PRAGMA synchronous. 1=OFF, 2=NORMAL, 3=FULL */
+ int bFullFsync, /* PRAGMA fullfsync */
+ int bCkptFullFsync /* PRAGMA checkpoint_fullfsync */
+){
+ assert( level>=1 && level<=3 );
pPager->noSync = (level==1 || pPager->tempFile) ?1:0;
pPager->fullSync = (level==3 && !pPager->tempFile) ?1:0;
- pPager->sync_flags = (bFullFsync?SQLITE_SYNC_FULL:SQLITE_SYNC_NORMAL);
+ if( pPager->noSync ){
+ pPager->syncFlags = 0;
+ pPager->ckptSyncFlags = 0;
+ }else if( bFullFsync ){
+ pPager->syncFlags = SQLITE_SYNC_FULL;
+ pPager->ckptSyncFlags = SQLITE_SYNC_FULL;
+ }else if( bCkptFullFsync ){
+ pPager->syncFlags = SQLITE_SYNC_NORMAL;
+ pPager->ckptSyncFlags = SQLITE_SYNC_FULL;
+ }else{
+ pPager->syncFlags = SQLITE_SYNC_NORMAL;
+ pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL;
+ }
}
#endif
@@ -38046,9 +38192,8 @@
if( mxPage>0 ){
pPager->mxPgno = mxPage;
}
- if( pPager->eState!=PAGER_OPEN && pPager->mxPgno<pPager->dbSize ){
- pPager->mxPgno = pPager->dbSize;
- }
+ assert( pPager->eState!=PAGER_OPEN ); /* Called only by OP_MaxPgcnt */
+ assert( pPager->mxPgno>=pPager->dbSize ); /* OP_MaxPgcnt enforces this */
return pPager->mxPgno;
}
@@ -38253,10 +38398,7 @@
/* pPager->errCode = 0; */
pPager->exclusiveMode = 0;
#ifndef SQLITE_OMIT_WAL
- sqlite3WalClose(pPager->pWal,
- (pPager->noSync ? 0 : pPager->sync_flags),
- pPager->pageSize, pTmp
- );
+ sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, pPager->pageSize, pTmp);
pPager->pWal = 0;
#endif
pager_reset(pPager);
@@ -38422,7 +38564,7 @@
if( pPager->fullSync && 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){
PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager)));
IOTRACE(("JSYNC %p\n", pPager))
- rc = sqlite3OsSync(pPager->jfd, pPager->sync_flags);
+ rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags);
if( rc!=SQLITE_OK ) return rc;
}
IOTRACE(("JHDR %p %lld\n", pPager, pPager->journalHdr));
@@ -38434,8 +38576,8 @@
if( 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){
PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager)));
IOTRACE(("JSYNC %p\n", pPager))
- rc = sqlite3OsSync(pPager->jfd, pPager->sync_flags|
- (pPager->sync_flags==SQLITE_SYNC_FULL?SQLITE_SYNC_DATAONLY:0)
+ rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags|
+ (pPager->syncFlags==SQLITE_SYNC_FULL?SQLITE_SYNC_DATAONLY:0)
);
if( rc!=SQLITE_OK ) return rc;
}
@@ -39025,7 +39167,8 @@
assert( useJournal || pPager->tempFile );
pPager->noSync = pPager->tempFile;
pPager->fullSync = pPager->noSync ?0:1;
- pPager->sync_flags = SQLITE_SYNC_NORMAL;
+ pPager->syncFlags = pPager->noSync ? 0 : SQLITE_SYNC_NORMAL;
+ pPager->ckptSyncFlags = pPager->syncFlags;
/* pPager->pFirst = 0; */
/* pPager->pFirstSynced = 0; */
/* pPager->pLast = 0; */
@@ -39125,7 +39268,7 @@
sqlite3BeginBenignMalloc();
if( pagerLockDb(pPager, RESERVED_LOCK)==SQLITE_OK ){
sqlite3OsDelete(pVfs, pPager->zJournal, 0);
- pagerUnlockDb(pPager, SHARED_LOCK);
+ if( !pPager->exclusiveMode ) pagerUnlockDb(pPager, SHARED_LOCK);
}
sqlite3EndBenignMalloc();
}else{
@@ -39806,29 +39949,29 @@
CHECK_PAGE(pPg);
+ /* The journal file needs to be opened. Higher level routines have already
+ ** obtained the necessary locks to begin the write-transaction, but the
+ ** rollback journal might not yet be open. Open it now if this is the case.
+ **
+ ** This is done before calling sqlite3PcacheMakeDirty() on the page.
+ ** Otherwise, if it were done after calling sqlite3PcacheMakeDirty(), then
+ ** an error might occur and the pager would end up in WRITER_LOCKED state
+ ** with pages marked as dirty in the cache.
+ */
+ if( pPager->eState==PAGER_WRITER_LOCKED ){
+ rc = pager_open_journal(pPager);
+ if( rc!=SQLITE_OK ) return rc;
+ }
+ assert( pPager->eState>=PAGER_WRITER_CACHEMOD );
+ assert( assert_pager_state(pPager) );
+
/* Mark the page as dirty. If the page has already been written
** to the journal then we can return right away.
*/
sqlite3PcacheMakeDirty(pPg);
if( pageInJournal(pPg) && !subjRequiresPage(pPg) ){
assert( !pagerUseWal(pPager) );
- assert( pPager->eState>=PAGER_WRITER_CACHEMOD );
}else{
-
- /* If we get this far, it means that the page needs to be
- ** written to the transaction journal or the checkpoint journal
- ** or both.
- **
- ** Higher level routines have already obtained the necessary locks
- ** to begin the write-transaction, but the rollback journal might not
- ** yet be open. Open it now if this is the case.
- */
- if( pPager->eState==PAGER_WRITER_LOCKED ){
- rc = pager_open_journal(pPager);
- if( rc!=SQLITE_OK ) return rc;
- }
- assert( pPager->eState>=PAGER_WRITER_CACHEMOD );
- assert( assert_pager_state(pPager) );
/* The transaction journal now exists and we have a RESERVED or an
** EXCLUSIVE lock on the main database file. Write the current page to
@@ -40148,10 +40291,10 @@
}
/*
-** Sync the pager file to disk. This is a no-op for in-memory files
+** Sync the database file to disk. This is a no-op for in-memory databases
** or pages with the Pager.noSync flag set.
**
-** If successful, or called on a pager for which it is a no-op, this
+** If successful, or if called on a pager for which it is a no-op, this
** function returns SQLITE_OK. Otherwise, an IO error code is returned.
*/
SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager){
@@ -40160,7 +40303,7 @@
if( pPager->noSync ){
rc = SQLITE_OK;
}else{
- rc = sqlite3OsSync(pPager->fd, pPager->sync_flags);
+ rc = sqlite3OsSync(pPager->fd, pPager->syncFlags);
}
return rc;
}
@@ -40249,7 +40392,7 @@
PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache);
if( pList ){
rc = pagerWalFrames(pPager, pList, pPager->dbSize, 1,
- (pPager->fullSync ? pPager->sync_flags : 0)
+ (pPager->fullSync ? pPager->syncFlags : 0)
);
}
if( rc==SQLITE_OK ){
@@ -40380,7 +40523,7 @@
/* Finally, sync the database file. */
if( !pPager->noSync && !noSync ){
- rc = sqlite3OsSync(pPager->fd, pPager->sync_flags);
+ rc = sqlite3OsSync(pPager->fd, pPager->syncFlags);
}
IOTRACE(("DBSYNC %p\n", pPager))
}
@@ -40951,7 +41094,8 @@
|| eMode==PAGER_LOCKINGMODE_EXCLUSIVE );
assert( PAGER_LOCKINGMODE_QUERY<0 );
assert( PAGER_LOCKINGMODE_NORMAL>=0 && PAGER_LOCKINGMODE_EXCLUSIVE>=0 );
- if( eMode>=0 && !pPager->tempFile ){
+ assert( pPager->exclusiveMode || 0==sqlite3WalHeapMemory(pPager->pWal) );
+ if( eMode>=0 && !pPager->tempFile && !sqlite3WalHeapMemory(pPager->pWal) ){
pPager->exclusiveMode = (u8)eMode;
}
return (int)pPager->exclusiveMode;
@@ -41120,10 +41264,8 @@
int rc = SQLITE_OK;
if( pPager->pWal ){
u8 *zBuf = (u8 *)pPager->pTmpSpace;
- rc = sqlite3WalCheckpoint(pPager->pWal,
- (pPager->noSync ? 0 : pPager->sync_flags),
- pPager->pageSize, zBuf
- );
+ rc = sqlite3WalCheckpoint(pPager->pWal, pPager->ckptSyncFlags,
+ pPager->pageSize, zBuf);
}
return rc;
}
@@ -41138,10 +41280,62 @@
*/
SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager){
const sqlite3_io_methods *pMethods = pPager->fd->pMethods;
- return pMethods->iVersion>=2 && pMethods->xShmMap!=0;
+ return pPager->exclusiveMode || (pMethods->iVersion>=2 && pMethods->xShmMap);
}
/*
+** Attempt to take an exclusive lock on the database file. If a PENDING lock
+** is obtained instead, immediately release it.
+*/
+static int pagerExclusiveLock(Pager *pPager){
+ int rc; /* Return code */
+
+ assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK );
+ rc = pagerLockDb(pPager, EXCLUSIVE_LOCK);
+ if( rc!=SQLITE_OK ){
+ /* If the attempt to grab the pending lock failed, release the
+ ** exclusive lock that may have been obtained instead. */
+ pagerUnlockDb(pPager, SHARED_LOCK);
+ }
+
+ return rc;
+}
+
+/*
+** Call sqlite3WalOpen() to open the WAL handle. If the pager is in
+** exclusive-locking mode when this function is called, take an EXCLUSIVE
+** lock on the database file and use heap-memory to store the wal-index
+** in. Otherwise, use the normal shared-memory.
+*/
+static int pagerOpenWal(Pager *pPager){
+ int rc = SQLITE_OK;
+
+ assert( pPager->pWal==0 && pPager->tempFile==0 );
+ assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK || pPager->noReadlock);
+
+ /* If the pager is already in exclusive-mode, the WAL module will use
+ ** heap-memory for the wal-index instead of the VFS shared-memory
+ ** implementation. Take the exclusive lock now, before opening the WAL
+ ** file, to make sure this is safe.
+ */
+ if( pPager->exclusiveMode ){
+ rc = pagerExclusiveLock(pPager);
+ }
+
+ /* Open the connection to the log file. If this operation fails,
+ ** (e.g. due to malloc() failure), return an error code.
+ */
+ if( rc==SQLITE_OK ){
+ rc = sqlite3WalOpen(pPager->pVfs,
+ pPager->fd, pPager->zWal, pPager->exclusiveMode, &pPager->pWal
+ );
+ }
+
+ return rc;
+}
+
+
+/*
** The caller must be holding a SHARED lock on the database file to call
** this function.
**
@@ -41174,11 +41368,7 @@
/* Close any rollback journal previously open */
sqlite3OsClose(pPager->jfd);
- /* Open the connection to the log file. If this operation fails,
- ** (e.g. due to malloc() failure), unlock the database file and
- ** return an error code.
- */
- rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, pPager->zWal, &pPager->pWal);
+ rc = pagerOpenWal(pPager);
if( rc==SQLITE_OK ){
pPager->journalMode = PAGER_JOURNALMODE_WAL;
pPager->eState = PAGER_OPEN;
@@ -41217,8 +41407,7 @@
);
}
if( rc==SQLITE_OK && logexists ){
- rc = sqlite3WalOpen(pPager->pVfs, pPager->fd,
- pPager->zWal, &pPager->pWal);
+ rc = pagerOpenWal(pPager);
}
}
@@ -41226,17 +41415,11 @@
** the database file, the log and log-summary files will be deleted.
*/
if( rc==SQLITE_OK && pPager->pWal ){
- rc = pagerLockDb(pPager, EXCLUSIVE_LOCK);
+ rc = pagerExclusiveLock(pPager);
if( rc==SQLITE_OK ){
- rc = sqlite3WalClose(pPager->pWal,
- (pPager->noSync ? 0 : pPager->sync_flags),
- pPager->pageSize, (u8*)pPager->pTmpSpace
- );
+ rc = sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags,
+ pPager->pageSize, (u8*)pPager->pTmpSpace);
pPager->pWal = 0;
- }else{
- /* If we cannot get an EXCLUSIVE lock, downgrade the PENDING lock
- ** that we did get back to SHARED. */
- pagerUnlockDb(pPager, SQLITE_LOCK_SHARED);
}
}
return rc;
@@ -41693,6 +41876,13 @@
};
/*
+** Candidate values for Wal.exclusiveMode.
+*/
+#define WAL_NORMAL_MODE 0
+#define WAL_EXCLUSIVE_MODE 1
+#define WAL_HEAPMEMORY_MODE 2
+
+/*
** Each page of the wal-index mapping contains a hash-table made up of
** an array of HASHTABLE_NSLOT elements of the following type.
*/
@@ -41778,9 +41968,14 @@
/* Request a pointer to the required page from the VFS */
if( pWal->apWiData[iPage]==0 ){
- rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ,
- pWal->writeLock, (void volatile **)&pWal->apWiData[iPage]
- );
+ if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){
+ pWal->apWiData[iPage] = (u32 volatile *)sqlite3MallocZero(WALINDEX_PGSZ);
+ if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ,
+ pWal->writeLock, (void volatile **)&pWal->apWiData[iPage]
+ );
+ }
}
*ppPage = pWal->apWiData[iPage];
@@ -41863,6 +42058,12 @@
aOut[1] = s2;
}
+static void walShmBarrier(Wal *pWal){
+ if( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE ){
+ sqlite3OsShmBarrier(pWal->pDbFd);
+ }
+}
+
/*
** Write the header information in pWal->hdr into the wal-index.
**
@@ -41877,7 +42078,7 @@
pWal->hdr.iVersion = WALINDEX_MAX_VERSION;
walChecksumBytes(1, (u8*)&pWal->hdr, nCksum, 0, pWal->hdr.aCksum);
memcpy((void *)&aHdr[1], (void *)&pWal->hdr, sizeof(WalIndexHdr));
- sqlite3OsShmBarrier(pWal->pDbFd);
+ walShmBarrier(pWal);
memcpy((void *)&aHdr[0], (void *)&pWal->hdr, sizeof(WalIndexHdr));
}
@@ -42449,7 +42650,15 @@
** Close an open wal-index.
*/
static void walIndexClose(Wal *pWal, int isDelete){
- sqlite3OsShmUnmap(pWal->pDbFd, isDelete);
+ if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){
+ int i;
+ for(i=0; i<pWal->nWiData; i++){
+ sqlite3_free((void *)pWal->apWiData[i]);
+ pWal->apWiData[i] = 0;
+ }
+ }else{
+ sqlite3OsShmUnmap(pWal->pDbFd, isDelete);
+ }
}
/*
@@ -42471,6 +42680,7 @@
sqlite3_vfs *pVfs, /* vfs module to open wal and wal-index */
sqlite3_file *pDbFd, /* The open database file */
const char *zWalName, /* Name of the WAL file */
+ int bNoShm, /* True to run in heap-memory mode */
Wal **ppWal /* OUT: Allocated Wal handle */
){
int rc; /* Return Code */
@@ -42504,6 +42714,7 @@
pRet->pDbFd = pDbFd;
pRet->readLock = -1;
pRet->zWalName = zWalName;
+ pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE);
/* Open file handle on the write-ahead log file. */
flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL);
@@ -42937,7 +43148,9 @@
*/
rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE);
if( rc==SQLITE_OK ){
- pWal->exclusiveMode = 1;
+ if( pWal->exclusiveMode==WAL_NORMAL_MODE ){
+ pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
+ }
rc = sqlite3WalCheckpoint(pWal, sync_flags, nBuf, zBuf);
if( rc==SQLITE_OK ){
isDelete = 1;
@@ -42993,7 +43206,7 @@
*/
aHdr = walIndexHdr(pWal);
memcpy(&h1, (void *)&aHdr[0], sizeof(h1));
- sqlite3OsShmBarrier(pWal->pDbFd);
+ walShmBarrier(pWal);
memcpy(&h2, (void *)&aHdr[1], sizeof(h2));
if( memcmp(&h1, &h2, sizeof(h1))!=0 ){
@@ -43194,7 +43407,7 @@
** and can be safely ignored.
*/
rc = walLockShared(pWal, WAL_READ_LOCK(0));
- sqlite3OsShmBarrier(pWal->pDbFd);
+ walShmBarrier(pWal);
if( rc==SQLITE_OK ){
if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){
/* It is not safe to allow the reader to continue here if frames
@@ -43288,7 +43501,7 @@
** log-wrap (either of which would require an exclusive lock on
** WAL_READ_LOCK(mxI)) has not occurred since the snapshot was valid.
*/
- sqlite3OsShmBarrier(pWal->pDbFd);
+ walShmBarrier(pWal);
if( pInfo->aReadMark[mxI]!=mxReadMark
|| memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr))
){
@@ -43630,7 +43843,7 @@
**
** SQLITE_OK is returned if no error is encountered (regardless of whether
** or not pWal->hdr.mxFrame is modified). An SQLite error code is returned
-** if some error
+** if an error occurs.
*/
static int walRestartLog(Wal *pWal){
int rc = SQLITE_OK;
@@ -43663,6 +43876,8 @@
for(i=1; i<WAL_NREADER; i++) pInfo->aReadMark[i] = READMARK_NOT_USED;
assert( pInfo->aReadMark[0]==0 );
walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1);
+ }else if( rc!=SQLITE_BUSY ){
+ return rc;
}
}
walUnlockShared(pWal, WAL_READ_LOCK(0));
@@ -43929,13 +44144,14 @@
** on the main database file before invoking this operation.
**
** If op is negative, then do a dry-run of the op==1 case but do
-** not actually change anything. The pager uses this to see if it
+** not actually change anything. The pager uses this to see if it
** should acquire the database exclusive lock prior to invoking
** the op==1 case.
*/
SQLITE_PRIVATE int sqlite3WalExclusiveMode(Wal *pWal, int op){
int rc;
assert( pWal->writeLock==0 );
+ assert( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE || op==-1 );
/* pWal->readLock is usually set, but might be -1 if there was a
** prior error while attempting to acquire are read-lock. This cannot
@@ -43969,6 +44185,15 @@
return rc;
}
+/*
+** Return true if the argument is non-NULL and the WAL module is using
+** heap-memory for the wal-index. Otherwise, if the argument is NULL or the
+** WAL module is using shared-memory, return false.
+*/
+SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal){
+ return (pWal && pWal->exclusiveMode==WAL_HEAPMEMORY_MODE );
+}
+
#endif /* #ifndef SQLITE_OMIT_WAL */
/************** End of wal.c *************************************************/
@@ -46692,17 +46917,13 @@
const int isTempDb = zFilename==0 || zFilename[0]==0;
/* Set the variable isMemdb to true for an in-memory database, or
- ** false for a file-based database. This symbol is only required if
- ** either of the shared-data or autovacuum features are compiled
- ** into the library.
+ ** false for a file-based database.
*/
-#if !defined(SQLITE_OMIT_SHARED_CACHE) || !defined(SQLITE_OMIT_AUTOVACUUM)
- #ifdef SQLITE_OMIT_MEMORYDB
- const int isMemdb = 0;
- #else
- const int isMemdb = (zFilename && strcmp(zFilename, ":memory:")==0)
- || (isTempDb && sqlite3TempInMemory(db));
- #endif
+#ifdef SQLITE_OMIT_MEMORYDB
+ const int isMemdb = 0;
+#else
+ const int isMemdb = (zFilename && strcmp(zFilename, ":memory:")==0)
+ || (isTempDb && sqlite3TempInMemory(db));
#endif
assert( db!=0 );
@@ -47086,11 +47307,17 @@
** probability of damage to near zero but with a write performance reduction.
*/
#ifndef SQLITE_OMIT_PAGER_PRAGMAS
-SQLITE_PRIVATE int sqlite3BtreeSetSafetyLevel(Btree *p, int level, int fullSync){
+SQLITE_PRIVATE int sqlite3BtreeSetSafetyLevel(
+ Btree *p, /* The btree to set the safety level on */
+ int level, /* PRAGMA synchronous. 1=OFF, 2=NORMAL, 3=FULL */
+ int fullSync, /* PRAGMA fullfsync. */
+ int ckptFullSync /* PRAGMA checkpoint_fullfync */
+){
BtShared *pBt = p->pBt;
assert( sqlite3_mutex_held(p->db->mutex) );
+ assert( level>=1 && level<=3 );
sqlite3BtreeEnter(p);
- sqlite3PagerSetSafetyLevel(pBt->pPager, level, fullSync);
+ sqlite3PagerSetSafetyLevel(pBt->pPager, level, fullSync, ckptFullSync);
sqlite3BtreeLeave(p);
return SQLITE_OK;
}
@@ -48148,8 +48375,8 @@
** are no active cursors, it also releases the read lock.
*/
SQLITE_PRIVATE int sqlite3BtreeCommitPhaseTwo(Btree *p){
- BtShared *pBt = p->pBt;
+ if( p->inTrans==TRANS_NONE ) return SQLITE_OK;
sqlite3BtreeEnter(p);
btreeIntegrity(p);
@@ -48158,6 +48385,7 @@
*/
if( p->inTrans==TRANS_WRITE ){
int rc;
+ BtShared *pBt = p->pBt;
assert( pBt->inTransaction==TRANS_WRITE );
assert( pBt->nTransaction>0 );
rc = sqlite3PagerCommitPhaseTwo(pBt->pPager);
@@ -53076,8 +53304,7 @@
SQLITE_PRIVATE void sqlite3BtreeCacheOverflow(BtCursor *pCur){
assert( cursorHoldsMutex(pCur) );
assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) );
- assert(!pCur->isIncrblobHandle);
- assert(!pCur->aOverflow);
+ invalidateOverflowCache(pCur);
pCur->isIncrblobHandle = 1;
}
#endif
@@ -54146,13 +54373,9 @@
return doubleToInt64(pMem->r);
}else if( flags & (MEM_Str|MEM_Blob) ){
i64 value;
- pMem->flags |= MEM_Str;
- if( sqlite3VdbeChangeEncoding(pMem, SQLITE_UTF8)
- || sqlite3VdbeMemNulTerminate(pMem) ){
- return 0;
- }
- assert( pMem->z );
- sqlite3Atoi64(pMem->z, &value);
+ assert( pMem->z || pMem->n==0 );
+ testcase( pMem->z==0 );
+ sqlite3Atoi64(pMem->z, &value, pMem->n, pMem->enc);
return value;
}else{
return 0;
@@ -54175,14 +54398,7 @@
}else if( pMem->flags & (MEM_Str|MEM_Blob) ){
/* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */
double val = (double)0;
- pMem->flags |= MEM_Str;
- if( sqlite3VdbeChangeEncoding(pMem, SQLITE_UTF8)
- || sqlite3VdbeMemNulTerminate(pMem) ){
- /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */
- return (double)0;
- }
- assert( pMem->z );
- sqlite3AtoF(pMem->z, &val);
+ sqlite3AtoF(pMem->z, &val, pMem->n, pMem->enc);
return val;
}else{
/* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */
@@ -54255,21 +54471,19 @@
** as much of the string as we can and ignore the rest.
*/
SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){
- int rc;
- assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))==0 );
- assert( (pMem->flags & (MEM_Blob|MEM_Str))!=0 );
- assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) );
- rc = sqlite3VdbeChangeEncoding(pMem, SQLITE_UTF8);
- if( rc ) return rc;
- rc = sqlite3VdbeMemNulTerminate(pMem);
- if( rc ) return rc;
- if( sqlite3Atoi64(pMem->z, &pMem->u.i) ){
- MemSetTypeFlag(pMem, MEM_Int);
- }else{
- pMem->r = sqlite3VdbeRealValue(pMem);
- MemSetTypeFlag(pMem, MEM_Real);
- sqlite3VdbeIntegerAffinity(pMem);
+ if( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))==0 ){
+ assert( (pMem->flags & (MEM_Blob|MEM_Str))!=0 );
+ assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) );
+ if( 0==sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc) ){
+ MemSetTypeFlag(pMem, MEM_Int);
+ }else{
+ pMem->r = sqlite3VdbeRealValue(pMem);
+ MemSetTypeFlag(pMem, MEM_Real);
+ sqlite3VdbeIntegerAffinity(pMem);
+ }
}
+ assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))!=0 );
+ pMem->flags &= ~(MEM_Str|MEM_Blob);
return SQLITE_OK;
}
@@ -54278,7 +54492,9 @@
*/
SQLITE_PRIVATE void sqlite3VdbeMemSetNull(Mem *pMem){
if( pMem->flags & MEM_Frame ){
- sqlite3VdbeFrameDelete(pMem->u.pFrame);
+ VdbeFrame *pFrame = pMem->u.pFrame;
+ pFrame->pParent = pFrame->v->pDelFrame;
+ pFrame->v->pDelFrame = pFrame;
}
if( pMem->flags & MEM_RowSet ){
sqlite3RowSetClear(pMem->u.pRowSet);
@@ -54812,6 +55028,8 @@
int op;
char *zVal = 0;
sqlite3_value *pVal = 0;
+ int negInt = 1;
+ const char *zNeg = "";
if( !pExpr ){
*ppVal = 0;
@@ -54829,13 +55047,24 @@
if( NEVER(op==TK_REGISTER) ) op = pExpr->op2;
#endif
+ /* Handle negative integers in a single step. This is needed in the
+ ** case when the value is -9223372036854775808.
+ */
+ if( op==TK_UMINUS
+ && (pExpr->pLeft->op==TK_INTEGER || pExpr->pLeft->op==TK_FLOAT) ){
+ pExpr = pExpr->pLeft;
+ op = pExpr->op;
+ negInt = -1;
+ zNeg = "-";
+ }
+
if( op==TK_STRING || op==TK_FLOAT || op==TK_INTEGER ){
pVal = sqlite3ValueNew(db);
if( pVal==0 ) goto no_mem;
if( ExprHasProperty(pExpr, EP_IntValue) ){
- sqlite3VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue);
+ sqlite3VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue*negInt);
}else{
- zVal = sqlite3DbStrDup(db, pExpr->u.zToken);
+ zVal = sqlite3MPrintf(db, "%s%s", zNeg, pExpr->u.zToken);
if( zVal==0 ) goto no_mem;
sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC);
if( op==TK_FLOAT ) pVal->type = SQLITE_FLOAT;
@@ -54845,14 +55074,18 @@
}else{
sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8);
}
+ if( pVal->flags & (MEM_Int|MEM_Real) ) pVal->flags &= ~MEM_Str;
if( enc!=SQLITE_UTF8 ){
sqlite3VdbeChangeEncoding(pVal, enc);
}
}else if( op==TK_UMINUS ) {
+ /* This branch happens for multiple negative signs. Ex: -(-5) */
if( SQLITE_OK==sqlite3ValueFromExpr(db,pExpr->pLeft,enc,affinity,&pVal) ){
+ sqlite3VdbeMemNumerify(pVal);
pVal->u.i = -1 * pVal->u.i;
/* (double)-1 In case of SQLITE_OMIT_FLOATING_POINT... */
pVal->r = (double)-1 * pVal->r;
+ sqlite3ValueApplyAffinity(pVal, affinity, enc);
}
}
#ifndef SQLITE_OMIT_BLOB_LITERAL
@@ -56106,12 +56339,10 @@
pMem->type = SQLITE_INTEGER;
pMem++;
- if( p->explain==1 ){
- pMem->flags = MEM_Int;
- pMem->u.i = pOp->p3; /* P3 */
- pMem->type = SQLITE_INTEGER;
- pMem++;
- }
+ pMem->flags = MEM_Int;
+ pMem->u.i = pOp->p3; /* P3 */
+ pMem->type = SQLITE_INTEGER;
+ pMem++;
if( sqlite3VdbeMemGrow(pMem, 32, 0) ){ /* P4 */
assert( p->db->mallocFailed );
@@ -56156,7 +56387,7 @@
}
}
- p->nResColumn = 8 - 5*(p->explain-1);
+ p->nResColumn = 8 - 4*(p->explain-1);
p->rc = SQLITE_OK;
rc = SQLITE_ROW;
}
@@ -56463,6 +56694,11 @@
if( p->aMem ){
releaseMemArray(&p->aMem[1], p->nMem);
}
+ while( p->pDelFrame ){
+ VdbeFrame *pDel = p->pDelFrame;
+ p->pDelFrame = pDel->pParent;
+ sqlite3VdbeFrameDelete(pDel);
+ }
}
/*
@@ -58146,6 +58382,8 @@
SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt){
int rc;
if( pStmt==0 ){
+ /* IMPLEMENTATION-OF: R-57228-12904 Invoking sqlite3_finalize() on a NULL
+ ** pointer is a harmless no-op. */
rc = SQLITE_OK;
}else{
Vdbe *v = (Vdbe*)pStmt;
@@ -59101,6 +59339,8 @@
rc = sqlite3ApiExit(p->db, rc);
}
sqlite3_mutex_leave(p->db->mutex);
+ }else if( xDel!=SQLITE_STATIC && xDel!=SQLITE_TRANSIENT ){
+ xDel((void*)zData);
}
return rc;
}
@@ -59344,6 +59584,14 @@
}
/*
+** Return true if the prepared statement is guaranteed to not modify the
+** database.
+*/
+SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt){
+ return pStmt ? ((Vdbe*)pStmt)->readOnly : 1;
+}
+
+/*
** Return a pointer to the next prepared statement after pStmt associated
** with database connection pDb. If pStmt is NULL, return the first
** prepared statement for the database connection. Return NULL if there
@@ -59765,31 +60013,17 @@
*/
static void applyNumericAffinity(Mem *pRec){
if( (pRec->flags & (MEM_Real|MEM_Int))==0 ){
- int realnum;
+ double rValue;
+ i64 iValue;
u8 enc = pRec->enc;
- sqlite3VdbeMemNulTerminate(pRec);
- if( (pRec->flags&MEM_Str) && sqlite3IsNumber(pRec->z, &realnum, enc) ){
- i64 value;
- char *zUtf8 = pRec->z;
-#ifndef SQLITE_OMIT_UTF16
- if( enc!=SQLITE_UTF8 ){
- assert( pRec->db );
- zUtf8 = sqlite3Utf16to8(pRec->db, pRec->z, pRec->n, enc);
- if( !zUtf8 ) return;
- }
-#endif
- if( !realnum && sqlite3Atoi64(zUtf8, &value) ){
- pRec->u.i = value;
- MemSetTypeFlag(pRec, MEM_Int);
- }else{
- sqlite3AtoF(zUtf8, &pRec->r);
- MemSetTypeFlag(pRec, MEM_Real);
- }
-#ifndef SQLITE_OMIT_UTF16
- if( enc!=SQLITE_UTF8 ){
- sqlite3DbFree(pRec->db, zUtf8);
- }
-#endif
+ if( (pRec->flags&MEM_Str)==0 ) return;
+ if( sqlite3AtoF(pRec->z, &rValue, pRec->n, enc)==0 ) return;
+ if( 0==sqlite3Atoi64(pRec->z, &iValue, pRec->n, enc) ){
+ pRec->u.i = iValue;
+ pRec->flags |= MEM_Int;
+ }else{
+ pRec->r = rValue;
+ pRec->flags |= MEM_Real;
}
}
}
@@ -59841,13 +60075,13 @@
** into a numeric representation. Use either INTEGER or REAL whichever
** is appropriate. But only do the conversion if it is possible without
** loss of information and return the revised type of the argument.
-**
-** This is an EXPERIMENTAL api and is subject to change or removal.
*/
SQLITE_API int sqlite3_value_numeric_type(sqlite3_value *pVal){
Mem *pMem = (Mem*)pVal;
- applyNumericAffinity(pMem);
- sqlite3VdbeMemStoreType(pMem);
+ if( pMem->type==SQLITE_TEXT ){
+ applyNumericAffinity(pMem);
+ sqlite3VdbeMemStoreType(pMem);
+ }
return pMem->type;
}
@@ -61548,7 +61782,6 @@
*/
case OP_MustBeInt: { /* jump, in1 */
pIn1 = &aMem[pOp->p1];
- memAboutToChange(p, pIn1);
applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding);
if( (pIn1->flags & MEM_Int)==0 ){
if( pOp->p2==0 ){
@@ -61575,7 +61808,6 @@
*/
case OP_RealAffinity: { /* in1 */
pIn1 = &aMem[pOp->p1];
- memAboutToChange(p, pIn1);
if( pIn1->flags & MEM_Int ){
sqlite3VdbeMemRealify(pIn1);
}
@@ -61618,7 +61850,6 @@
*/
case OP_ToBlob: { /* same as TK_TO_BLOB, in1 */
pIn1 = &aMem[pOp->p1];
- memAboutToChange(p, pIn1);
if( pIn1->flags & MEM_Null ) break;
if( (pIn1->flags & MEM_Blob)==0 ){
applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding);
@@ -61643,10 +61874,7 @@
*/
case OP_ToNumeric: { /* same as TK_TO_NUMERIC, in1 */
pIn1 = &aMem[pOp->p1];
- memAboutToChange(p, pIn1);
- if( (pIn1->flags & (MEM_Null|MEM_Int|MEM_Real))==0 ){
- sqlite3VdbeMemNumerify(pIn1);
- }
+ sqlite3VdbeMemNumerify(pIn1);
break;
}
#endif /* SQLITE_OMIT_CAST */
@@ -61662,7 +61890,6 @@
*/
case OP_ToInt: { /* same as TK_TO_INT, in1 */
pIn1 = &aMem[pOp->p1];
- memAboutToChange(p, pIn1);
if( (pIn1->flags & MEM_Null)==0 ){
sqlite3VdbeMemIntegerify(pIn1);
}
@@ -61776,8 +62003,6 @@
pIn1 = &aMem[pOp->p1];
pIn3 = &aMem[pOp->p3];
- memAboutToChange(p, pIn1);
- memAboutToChange(p, pIn3);
u.ai.flags1 = pIn1->flags;
u.ai.flags3 = pIn3->flags;
if( (pIn1->flags | pIn3->flags)&MEM_Null ){
@@ -62410,7 +62635,6 @@
while( (u.an.cAff = *(u.an.zAffinity++))!=0 ){
assert( pIn1 <= &p->aMem[p->nMem] );
assert( memIsValid(pIn1) );
- memAboutToChange(p, pIn1);
ExpandBlob(pIn1);
applyAffinity(pIn1, u.an.cAff, encoding);
pIn1++;
@@ -62490,7 +62714,6 @@
for(u.ao.pRec=u.ao.pData0; u.ao.pRec<=u.ao.pLast; u.ao.pRec++){
assert( memIsValid(u.ao.pRec) );
if( u.ao.zAffinity ){
- memAboutToChange(p, u.ao.pRec);
applyAffinity(u.ao.pRec, u.ao.zAffinity[u.ao.pRec-u.ao.pData0], encoding);
}
if( u.ao.pRec->flags&MEM_Zero && u.ao.pRec->n>0 ){
@@ -65944,6 +66167,32 @@
}
#endif
+
+#ifndef SQLITE_OMIT_PAGER_PRAGMAS
+/* Opcode: MaxPgcnt P1 P2 P3 * *
+**
+** Try to set the maximum page count for database P1 to the value in P3.
+** Do not let the maximum page count fall below the current page count and
+** do not change the maximum page count value if P3==0.
+**
+** Store the maximum page count after the change in register P2.
+*/
+case OP_MaxPgcnt: { /* out2-prerelease */
+ unsigned int newMax;
+ Btree *pBt;
+
+ pBt = db->aDb[pOp->p1].pBt;
+ newMax = 0;
+ if( pOp->p3 ){
+ newMax = sqlite3BtreeLastPage(pBt);
+ if( newMax < (unsigned)pOp->p3 ) newMax = (unsigned)pOp->p3;
+ }
+ pOut->u.i = sqlite3BtreeMaxPageCount(pBt, newMax);
+ break;
+}
+#endif
+
+
#ifndef SQLITE_OMIT_TRACE
/* Opcode: Trace * * * P4 *
**
@@ -66118,11 +66367,82 @@
int flags; /* Copy of "flags" passed to sqlite3_blob_open() */
int nByte; /* Size of open blob, in bytes */
int iOffset; /* Byte offset of blob in cursor data */
+ int iCol; /* Table column this handle is open on */
BtCursor *pCsr; /* Cursor pointing at blob row */
sqlite3_stmt *pStmt; /* Statement holding cursor open */
sqlite3 *db; /* The associated database */
};
+
+/*
+** This function is used by both blob_open() and blob_reopen(). It seeks
+** the b-tree cursor associated with blob handle p to point to row iRow.
+** If successful, SQLITE_OK is returned and subsequent calls to
+** sqlite3_blob_read() or sqlite3_blob_write() access the specified row.
+**
+** If an error occurs, or if the specified row does not exist or does not
+** contain a value of type TEXT or BLOB in the column nominated when the
+** blob handle was opened, then an error code is returned and *pzErr may
+** be set to point to a buffer containing an error message. It is the
+** responsibility of the caller to free the error message buffer using
+** sqlite3DbFree().
+**
+** If an error does occur, then the b-tree cursor is closed. All subsequent
+** calls to sqlite3_blob_read(), blob_write() or blob_reopen() will
+** immediately return SQLITE_ABORT.
+*/
+static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){
+ int rc; /* Error code */
+ char *zErr = 0; /* Error message */
+ Vdbe *v = (Vdbe *)p->pStmt;
+
+ /* Set the value of the SQL statements only variable to integer iRow.
+ ** This is done directly instead of using sqlite3_bind_int64() to avoid
+ ** triggering asserts related to mutexes.
+ */
+ assert( v->aVar[0].flags&MEM_Int );
+ v->aVar[0].u.i = iRow;
+
+ rc = sqlite3_step(p->pStmt);
+ if( rc==SQLITE_ROW ){
+ u32 type = v->apCsr[0]->aType[p->iCol];
+ if( type<12 ){
+ zErr = sqlite3MPrintf(p->db, "cannot open value of type %s",
+ type==0?"null": type==7?"real": "integer"
+ );
+ rc = SQLITE_ERROR;
+ sqlite3_finalize(p->pStmt);
+ p->pStmt = 0;
+ }else{
+ p->iOffset = v->apCsr[0]->aOffset[p->iCol];
+ p->nByte = sqlite3VdbeSerialTypeLen(type);
+ p->pCsr = v->apCsr[0]->pCursor;
+ sqlite3BtreeEnterCursor(p->pCsr);
+ sqlite3BtreeCacheOverflow(p->pCsr);
+ sqlite3BtreeLeaveCursor(p->pCsr);
+ }
+ }
+
+ if( rc==SQLITE_ROW ){
+ rc = SQLITE_OK;
+ }else if( p->pStmt ){
+ rc = sqlite3_finalize(p->pStmt);
+ p->pStmt = 0;
+ if( rc==SQLITE_OK ){
+ zErr = sqlite3MPrintf(p->db, "no such rowid: %lld", iRow);
+ rc = SQLITE_ERROR;
+ }else{
+ zErr = sqlite3MPrintf(p->db, "%s", sqlite3_errmsg(p->db));
+ }
+ }
+
+ assert( rc!=SQLITE_OK || zErr==0 );
+ assert( rc!=SQLITE_ROW && rc!=SQLITE_DONE );
+
+ *pzErr = zErr;
+ return rc;
+}
+
/*
** Open a blob handle.
*/
@@ -66163,29 +66483,35 @@
{OP_OpenWrite, 0, 0, 0}, /* 4: Open cursor 0 for read/write */
{OP_Variable, 1, 1, 1}, /* 5: Push the rowid to the stack */
- {OP_NotExists, 0, 9, 1}, /* 6: Seek the cursor */
+ {OP_NotExists, 0, 10, 1}, /* 6: Seek the cursor */
{OP_Column, 0, 0, 1}, /* 7 */
{OP_ResultRow, 1, 0, 0}, /* 8 */
- {OP_Close, 0, 0, 0}, /* 9 */
- {OP_Halt, 0, 0, 0}, /* 10 */
+ {OP_Goto, 0, 5, 0}, /* 9 */
+ {OP_Close, 0, 0, 0}, /* 10 */
+ {OP_Halt, 0, 0, 0}, /* 11 */
};
- Vdbe *v = 0;
int rc = SQLITE_OK;
char *zErr = 0;
Table *pTab;
- Parse *pParse;
+ Parse *pParse = 0;
+ Incrblob *pBlob = 0;
+ flags = !!flags; /* flags = (flags ? 1 : 0); */
*ppBlob = 0;
+
sqlite3_mutex_enter(db->mutex);
+
+ pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob));
+ if( !pBlob ) goto blob_open_out;
pParse = sqlite3StackAllocRaw(db, sizeof(*pParse));
- if( pParse==0 ){
- rc = SQLITE_NOMEM;
- goto blob_open_out;
- }
+ if( !pParse ) goto blob_open_out;
+
do {
memset(pParse, 0, sizeof(Parse));
pParse->db = db;
+ sqlite3DbFree(db, zErr);
+ zErr = 0;
sqlite3BtreeEnterAll(db);
pTab = sqlite3LocateTable(pParse, 0, zTable, zDb);
@@ -66211,7 +66537,7 @@
}
/* Now search pTab for the exact column. */
- for(iCol=0; iCol < pTab->nCol; iCol++) {
+ for(iCol=0; iCol<pTab->nCol; iCol++) {
if( sqlite3StrICmp(pTab->aCol[iCol].zName, zColumn)==0 ){
break;
}
@@ -66265,11 +66591,14 @@
}
}
- v = sqlite3VdbeCreate(db);
- if( v ){
+ pBlob->pStmt = (sqlite3_stmt *)sqlite3VdbeCreate(db);
+ assert( pBlob->pStmt || db->mallocFailed );
+ if( pBlob->pStmt ){
+ Vdbe *v = (Vdbe *)pBlob->pStmt;
int iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
+
sqlite3VdbeAddOpList(v, sizeof(openBlob)/sizeof(VdbeOpList), openBlob);
- flags = !!flags; /* flags = (flags ? 1 : 0); */
+
/* Configure the OP_Transaction */
sqlite3VdbeChangeP1(v, 0, iDb);
@@ -66312,65 +66641,25 @@
}
}
+ pBlob->flags = flags;
+ pBlob->iCol = iCol;
+ pBlob->db = db;
sqlite3BtreeLeaveAll(db);
if( db->mallocFailed ){
goto blob_open_out;
}
-
- sqlite3_bind_int64((sqlite3_stmt *)v, 1, iRow);
- rc = sqlite3_step((sqlite3_stmt *)v);
- if( rc!=SQLITE_ROW ){
- nAttempt++;
- rc = sqlite3_finalize((sqlite3_stmt *)v);
- sqlite3DbFree(db, zErr);
- zErr = sqlite3MPrintf(db, sqlite3_errmsg(db));
- v = 0;
- }
- } while( nAttempt<5 && rc==SQLITE_SCHEMA );
-
- if( rc==SQLITE_ROW ){
- /* The row-record has been opened successfully. Check that the
- ** column in question contains text or a blob. If it contains
- ** text, it is up to the caller to get the encoding right.
- */
- Incrblob *pBlob;
- u32 type = v->apCsr[0]->aType[iCol];
-
- if( type<12 ){
- sqlite3DbFree(db, zErr);
- zErr = sqlite3MPrintf(db, "cannot open value of type %s",
- type==0?"null": type==7?"real": "integer"
- );
- rc = SQLITE_ERROR;
- goto blob_open_out;
- }
- pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob));
- if( db->mallocFailed ){
- sqlite3DbFree(db, pBlob);
- goto blob_open_out;
- }
- pBlob->flags = flags;
- pBlob->pCsr = v->apCsr[0]->pCursor;
- sqlite3BtreeEnterCursor(pBlob->pCsr);
- sqlite3BtreeCacheOverflow(pBlob->pCsr);
- sqlite3BtreeLeaveCursor(pBlob->pCsr);
- pBlob->pStmt = (sqlite3_stmt *)v;
- pBlob->iOffset = v->apCsr[0]->aOffset[iCol];
- pBlob->nByte = sqlite3VdbeSerialTypeLen(type);
- pBlob->db = db;
- *ppBlob = (sqlite3_blob *)pBlob;
- rc = SQLITE_OK;
- }else if( rc==SQLITE_OK ){
- sqlite3DbFree(db, zErr);
- zErr = sqlite3MPrintf(db, "no such rowid: %lld", iRow);
- rc = SQLITE_ERROR;
- }
+ sqlite3_bind_int64(pBlob->pStmt, 1, iRow);
+ rc = blobSeekToRow(pBlob, iRow, &zErr);
+ } while( (++nAttempt)<5 && rc==SQLITE_SCHEMA );
blob_open_out:
- if( v && (rc!=SQLITE_OK || db->mallocFailed) ){
- sqlite3VdbeFinalize(v);
+ if( rc==SQLITE_OK && db->mallocFailed==0 ){
+ *ppBlob = (sqlite3_blob *)pBlob;
+ }else{
+ if( pBlob && pBlob->pStmt ) sqlite3VdbeFinalize((Vdbe *)pBlob->pStmt);
+ sqlite3DbFree(db, pBlob);
}
- sqlite3Error(db, rc, zErr);
+ sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr);
sqlite3DbFree(db, zErr);
sqlite3StackFree(db, pParse);
rc = sqlite3ApiExit(db, rc);
@@ -66423,7 +66712,7 @@
/* Request is out of range. Return a transient error. */
rc = SQLITE_ERROR;
sqlite3Error(db, SQLITE_ERROR, 0);
- } else if( v==0 ){
+ }else if( v==0 ){
/* If there is no statement handle, then the blob-handle has
** already been invalidated. Return SQLITE_ABORT in this case.
*/
@@ -66471,7 +66760,47 @@
*/
SQLITE_API int sqlite3_blob_bytes(sqlite3_blob *pBlob){
Incrblob *p = (Incrblob *)pBlob;
- return p ? p->nByte : 0;
+ return (p && p->pStmt) ? p->nByte : 0;
+}
+
+/*
+** Move an existing blob handle to point to a different row of the same
+** database table.
+**
+** If an error occurs, or if the specified row does not exist or does not
+** contain a blob or text value, then an error code is returned and the
+** database handle error code and message set. If this happens, then all
+** subsequent calls to sqlite3_blob_xxx() functions (except blob_close())
+** immediately return SQLITE_ABORT.
+*/
+SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){
+ int rc;
+ Incrblob *p = (Incrblob *)pBlob;
+ sqlite3 *db;
+
+ if( p==0 ) return SQLITE_MISUSE_BKPT;
+ db = p->db;
+ sqlite3_mutex_enter(db->mutex);
+
+ if( p->pStmt==0 ){
+ /* If there is no statement handle, then the blob-handle has
+ ** already been invalidated. Return SQLITE_ABORT in this case.
+ */
+ rc = SQLITE_ABORT;
+ }else{
+ char *zErr;
+ rc = blobSeekToRow(p, iRow, &zErr);
+ if( rc!=SQLITE_OK ){
+ sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr);
+ sqlite3DbFree(db, zErr);
+ }
+ assert( rc!=SQLITE_SCHEMA );
+ }
+
+ rc = sqlite3ApiExit(db, rc);
+ assert( rc==SQLITE_OK || p->pStmt==0 );
+ sqlite3_mutex_leave(db->mutex);
+ return rc;
}
#endif /* #ifndef SQLITE_OMIT_INCRBLOB */
@@ -68804,6 +69133,9 @@
){
Expr *p = sqlite3ExprAlloc(pParse->db, op, pToken, 1);
sqlite3ExprAttachSubtrees(pParse->db, p, pLeft, pRight);
+ if( p ) {
+ sqlite3ExprCheckHeight(pParse, p->nHeight);
+ }
return p;
}
@@ -68875,7 +69207,7 @@
/* Wildcard of the form "?nnn". Convert "nnn" to an integer and
** use it as the variable number */
i64 i;
- int bOk = sqlite3Atoi64(&z[1], &i);
+ int bOk = 0==sqlite3Atoi64(&z[1], &i, sqlite3Strlen30(&z[1]), SQLITE_UTF8);
pExpr->iColumn = (ynVar)i;
testcase( i==0 );
testcase( i==1 );
@@ -69917,6 +70249,16 @@
assert( testAddr>0 || pParse->db->mallocFailed );
}
+#ifndef SQLITE_OMIT_EXPLAIN
+ if( pParse->explain==2 ){
+ char *zMsg = sqlite3MPrintf(
+ pParse->db, "EXECUTE %s%s SUBQUERY %d", testAddr?"":"CORRELATED ",
+ pExpr->op==TK_IN?"LIST":"SCALAR", pParse->iNextSelectId
+ );
+ sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC);
+ }
+#endif
+
switch( pExpr->op ){
case TK_IN: {
char affinity; /* Affinity of the LHS of the IN */
@@ -69962,6 +70304,7 @@
sqlite3SelectDestInit(&dest, SRT_Set, pExpr->iTable);
dest.affinity = (u8)affinity;
assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable );
+ pExpr->x.pSelect->iLimit = 0;
if( sqlite3Select(pParse, pExpr->x.pSelect, &dest) ){
return 0;
}
@@ -70062,6 +70405,7 @@
sqlite3ExprDelete(pParse->db, pSel->pLimit);
pSel->pLimit = sqlite3PExpr(pParse, TK_INTEGER, 0, 0,
&sqlite3IntTokens[1]);
+ pSel->iLimit = 0;
if( sqlite3Select(pParse, pSel, &dest) ){
return 0;
}
@@ -70238,7 +70582,7 @@
if( ALWAYS(z!=0) ){
double value;
char *zV;
- sqlite3AtoF(z, &value);
+ sqlite3AtoF(z, &value, sqlite3Strlen30(z), SQLITE_UTF8);
assert( !sqlite3IsNaN(value) ); /* The new AtoF never returns NaN */
if( negateFlag ) value = -value;
zV = dup8bytes(v, (char*)&value);
@@ -70252,9 +70596,7 @@
** Generate an instruction that will put the integer describe by
** text z[0..n-1] into register iMem.
**
-** The z[] string will probably not be zero-terminated. But the
-** z[n] character is guaranteed to be something that does not look
-** like the continuation of the number.
+** Expr.u.zToken is always UTF8 and zero-terminated.
*/
static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){
Vdbe *v = pParse->pVdbe;
@@ -70263,13 +70605,14 @@
if( negFlag ) i = -i;
sqlite3VdbeAddOp2(v, OP_Integer, i, iMem);
}else{
+ int c;
+ i64 value;
const char *z = pExpr->u.zToken;
assert( z!=0 );
- if( sqlite3FitsIn64Bits(z, negFlag) ){
- i64 value;
+ c = sqlite3Atoi64(z, &value, sqlite3Strlen30(z), SQLITE_UTF8);
+ if( c==0 || (c==2 && negFlag) ){
char *zV;
- sqlite3Atoi64(z, &value);
- if( negFlag ) value = -value;
+ if( negFlag ){ value = -value; }
zV = dup8bytes(v, (char*)&value);
sqlite3VdbeAddOp4(v, OP_Int64, 0, iMem, 0, zV, P4_INT64);
}else{
@@ -71346,9 +71689,22 @@
** Preevaluate constant subexpressions within pExpr and store the
** results in registers. Modify pExpr so that the constant subexpresions
** are TK_REGISTER opcodes that refer to the precomputed values.
+**
+** This routine is a no-op if the jump to the cookie-check code has
+** already occur. Since the cookie-check jump is generated prior to
+** any other serious processing, this check ensures that there is no
+** way to accidently bypass the constant initializations.
+**
+** This routine is also a no-op if the SQLITE_FactorOutConst optimization
+** is disabled via the sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS)
+** interface. This allows test logic to verify that the same answer is
+** obtained for queries regardless of whether or not constants are
+** precomputed into registers or if they are inserted in-line.
*/
SQLITE_PRIVATE void sqlite3ExprCodeConstants(Parse *pParse, Expr *pExpr){
Walker w;
+ if( pParse->cookieGoto ) return;
+ if( (pParse->db->flags & SQLITE_FactorOutConst)!=0 ) return;
w.xExprCallback = evalConstExpr;
w.xSelectCallback = 0;
w.pParse = pParse;
@@ -72361,6 +72717,11 @@
}
}
}
+ if( zWhere ){
+ char *zNew = sqlite3MPrintf(pParse->db, "type='trigger' AND (%s)", zWhere);
+ sqlite3DbFree(pParse->db, zWhere);
+ zWhere = zNew;
+ }
return zWhere;
}
@@ -72968,7 +73329,7 @@
int i; /* Loop counter */
int topOfLoop; /* The top of the loop */
int endOfLoop; /* The end of the loop */
- int addr; /* The address of an instruction */
+ int addr = 0; /* The address of an instruction */
int jZeroRows = 0; /* Jump from here if number of rows is zero */
int iDb; /* Index of database containing pTab */
int regTabname = iMem++; /* Register containing table name */
@@ -72991,8 +73352,8 @@
if( v==0 || NEVER(pTab==0) ){
return;
}
- if( pTab->pSelect ){
- /* Do not gather statistics on views */
+ if( pTab->tnum==0 ){
+ /* Do not gather statistics on views or virtual tables */
return;
}
if( memcmp(pTab->zName, "sqlite_", 7)==0 ){
@@ -73912,7 +74273,8 @@
0, /* xStep */
0, /* xFinalize */
"sqlite_detach", /* zName */
- 0 /* pHash */
+ 0, /* pHash */
+ 0 /* pDestructor */
};
codeAttach(pParse, SQLITE_DETACH, &detach_func, pDbname, 0, 0, pDbname);
}
@@ -73933,7 +74295,8 @@
0, /* xStep */
0, /* xFinalize */
"sqlite_attach", /* zName */
- 0 /* pHash */
+ 0, /* pHash */
+ 0 /* pDestructor */
};
codeAttach(pParse, SQLITE_ATTACH, &attach_func, p, p, pDbname, pKey);
}
@@ -75062,8 +75425,9 @@
*/
iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName);
if( iDb<0 ) return;
- if( !OMIT_TEMPDB && isTemp && iDb>1 ){
- /* If creating a temp table, the name may not be qualified */
+ if( !OMIT_TEMPDB && isTemp && pName2->n>0 && iDb!=1 ){
+ /* If creating a temp table, the name may not be qualified. Unless
+ ** the database name is "temp" anyway. */
sqlite3ErrorMsg(pParse, "temporary table name must be unqualified");
return;
}
@@ -75111,17 +75475,18 @@
** collisions.
*/
if( !IN_DECLARE_VTAB ){
+ char *zDb = db->aDb[iDb].zName;
if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){
goto begin_table_error;
}
- pTable = sqlite3FindTable(db, zName, db->aDb[iDb].zName);
+ pTable = sqlite3FindTable(db, zName, zDb);
if( pTable ){
if( !noErr ){
sqlite3ErrorMsg(pParse, "table %T already exists", pName);
}
goto begin_table_error;
}
- if( sqlite3FindIndex(db, zName, 0)!=0 && (iDb==0 || !db->init.busy) ){
+ if( sqlite3FindIndex(db, zName, zDb)!=0 ){
sqlite3ErrorMsg(pParse, "there is already an index named %s", zName);
goto begin_table_error;
}
@@ -77106,7 +77471,8 @@
sqlite3RefillIndex(pParse, pIndex, iMem);
sqlite3ChangeCookie(pParse, iDb);
sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0,
- sqlite3MPrintf(db, "name='%q'", pIndex->zName), P4_DYNAMIC);
+ sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName),
+ P4_DYNAMIC);
sqlite3VdbeAddOp1(v, OP_Expire, 0);
}
}
@@ -77234,7 +77600,7 @@
if( v ){
sqlite3BeginWriteOperation(pParse, 1, iDb);
sqlite3NestedParse(pParse,
- "DELETE FROM %Q.%s WHERE name=%Q",
+ "DELETE FROM %Q.%s WHERE name=%Q AND type='index'",
db->aDb[iDb].zName, SCHEMA_TABLE(iDb),
pIndex->zName
);
@@ -79399,7 +79765,7 @@
sqlite3_result_error_nomem(context);
return;
}
- sqlite3AtoF(zBuf, &r);
+ sqlite3AtoF(zBuf, &r, sqlite3Strlen30(zBuf), SQLITE_UTF8);
sqlite3_free(zBuf);
}
sqlite3_result_double(context, r);
@@ -80651,10 +81017,10 @@
FUNCTION(coalesce, 1, 0, 0, 0 ),
FUNCTION(coalesce, 0, 0, 0, 0 ),
/* FUNCTION(coalesce, -1, 0, 0, ifnullFunc ), */
- {-1,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"coalesce",0},
+ {-1,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"coalesce",0,0},
FUNCTION(hex, 1, 0, 0, hexFunc ),
/* FUNCTION(ifnull, 2, 0, 0, ifnullFunc ), */
- {2,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"ifnull",0},
+ {2,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"ifnull",0,0},
FUNCTION(random, 0, 0, 0, randomFunc ),
FUNCTION(randomblob, 1, 0, 0, randomBlob ),
FUNCTION(nullif, 2, 0, 1, nullifFunc ),
@@ -80681,7 +81047,7 @@
AGGREGATE(total, 1, 0, 0, sumStep, totalFinalize ),
AGGREGATE(avg, 1, 0, 0, sumStep, avgFinalize ),
/* AGGREGATE(count, 0, 0, 0, countStep, countFinalize ), */
- {0,SQLITE_UTF8,SQLITE_FUNC_COUNT,0,0,0,countStep,countFinalize,"count",0},
+ {0,SQLITE_UTF8,SQLITE_FUNC_COUNT,0,0,0,countStep,countFinalize,"count",0,0},
AGGREGATE(count, 1, 0, 0, countStep, countFinalize ),
AGGREGATE(group_concat, 1, 0, 0, groupConcatStep, groupConcatFinalize),
AGGREGATE(group_concat, 2, 0, 0, groupConcatStep, groupConcatFinalize),
@@ -84094,6 +84460,27 @@
sqlite3_stmt *(*next_stmt)(sqlite3*,sqlite3_stmt*);
const char *(*sql)(sqlite3_stmt*);
int (*status)(int,int*,int*,int);
+ int (*backup_finish)(sqlite3_backup*);
+ sqlite3_backup *(*backup_init)(sqlite3*,const char*,sqlite3*,const char*);
+ int (*backup_pagecount)(sqlite3_backup*);
+ int (*backup_remaining)(sqlite3_backup*);
+ int (*backup_step)(sqlite3_backup*,int);
+ const char *(*compileoption_get)(int);
+ int (*compileoption_used)(const char*);
+ int (*create_function_v2)(sqlite3*,const char*,int,int,void*,void (*xFunc)(sqlite3_context*,int,sqlite3_value**),void (*xStep)(sqlite3_context*,int,sqlite3_value**),void (*xFinal)(sqlite3_context*),void(*xDestroy)(void*));
+ int (*db_config)(sqlite3*,int,...);
+ sqlite3_mutex *(*db_mutex)(sqlite3*);
+ int (*db_status)(sqlite3*,int,int*,int*,int);
+ int (*extended_errcode)(sqlite3*);
+ void (*log)(int,const char*,...);
+ sqlite3_int64 (*soft_heap_limit64)(sqlite3_int64);
+ const char *(*sourceid)(void);
+ int (*stmt_status)(sqlite3_stmt*,int,int);
+ int (*strnicmp)(const char*,const char*,int);
+ int (*unlock_notify)(sqlite3*,void(*)(void**,int),void*);
+ int (*wal_autocheckpoint)(sqlite3*,int);
+ int (*wal_checkpoint)(sqlite3*,const char*);
+ void *(*wal_hook)(sqlite3*,int(*)(void*,sqlite3*,const char*,int),void*);
};
/*
@@ -84273,6 +84660,27 @@
#define sqlite3_next_stmt sqlite3_api->next_stmt
#define sqlite3_sql sqlite3_api->sql
#define sqlite3_status sqlite3_api->status
+#define sqlite3_backup_finish sqlite3_api->backup_finish
+#define sqlite3_backup_init sqlite3_api->backup_init
+#define sqlite3_backup_pagecount sqlite3_api->backup_pagecount
+#define sqlite3_backup_remaining sqlite3_api->backup_remaining
+#define sqlite3_backup_step sqlite3_api->backup_step
+#define sqlite3_compileoption_get sqlite3_api->compileoption_get
+#define sqlite3_compileoption_used sqlite3_api->compileoption_used
+#define sqlite3_create_function_v2 sqlite3_api->create_function_v2
+#define sqlite3_db_config sqlite3_api->db_config
+#define sqlite3_db_mutex sqlite3_api->db_mutex
+#define sqlite3_db_status sqlite3_api->db_status
+#define sqlite3_extended_errcode sqlite3_api->extended_errcode
+#define sqlite3_log sqlite3_api->log
+#define sqlite3_soft_heap_limit64 sqlite3_api->soft_heap_limit64
+#define sqlite3_sourceid sqlite3_api->sourceid
+#define sqlite3_stmt_status sqlite3_api->stmt_status
+#define sqlite3_strnicmp sqlite3_api->strnicmp
+#define sqlite3_unlock_notify sqlite3_api->unlock_notify
+#define sqlite3_wal_autocheckpoint sqlite3_api->wal_autocheckpoint
+#define sqlite3_wal_checkpoint sqlite3_api->wal_checkpoint
+#define sqlite3_wal_hook sqlite3_api->wal_hook
#endif /* SQLITE_CORE */
#define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api = 0;
@@ -84590,6 +84998,46 @@
sqlite3_next_stmt,
sqlite3_sql,
sqlite3_status,
+
+ /*
+ ** Added for 3.7.4
+ */
+ sqlite3_backup_finish,
+ sqlite3_backup_init,
+ sqlite3_backup_pagecount,
+ sqlite3_backup_remaining,
+ sqlite3_backup_step,
+#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS
+ sqlite3_compileoption_get,
+ sqlite3_compileoption_used,
+#else
+ 0,
+ 0,
+#endif
+ sqlite3_create_function_v2,
+ sqlite3_db_config,
+ sqlite3_db_mutex,
+ sqlite3_db_status,
+ sqlite3_extended_errcode,
+ sqlite3_log,
+ sqlite3_soft_heap_limit64,
+ sqlite3_sourceid,
+ sqlite3_stmt_status,
+ sqlite3_strnicmp,
+#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY
+ sqlite3_unlock_notify,
+#else
+ 0,
+#endif
+#ifndef SQLITE_OMIT_WAL
+ sqlite3_wal_autocheckpoint,
+ sqlite3_wal_checkpoint,
+ sqlite3_wal_hook,
+#else
+ 0,
+ 0,
+ 0,
+#endif
};
/*
@@ -84905,7 +85353,7 @@
static const u8 iValue[] = {1, 0, 0, 0, 1, 1, 2};
int i, n;
if( sqlite3Isdigit(*z) ){
- return (u8)atoi(z);
+ return (u8)sqlite3Atoi(z);
}
n = sqlite3Strlen30(z);
for(i=0; i<ArraySize(iLength); i++){
@@ -84946,7 +85394,7 @@
if( 0==sqlite3StrICmp(z, "none") ) return BTREE_AUTOVACUUM_NONE;
if( 0==sqlite3StrICmp(z, "full") ) return BTREE_AUTOVACUUM_FULL;
if( 0==sqlite3StrICmp(z, "incremental") ) return BTREE_AUTOVACUUM_INCR;
- i = atoi(z);
+ i = sqlite3Atoi(z);
return (u8)((i>=0&&i<=2)?i:0);
}
#endif /* ifndef SQLITE_OMIT_AUTOVACUUM */
@@ -85042,6 +85490,7 @@
{ "empty_result_callbacks", SQLITE_NullCallback },
{ "legacy_file_format", SQLITE_LegacyFileFmt },
{ "fullfsync", SQLITE_FullFSync },
+ { "checkpoint_fullfsync", SQLITE_CkptFullFSync },
{ "reverse_unordered_selects", SQLITE_ReverseOrder },
#ifndef SQLITE_OMIT_AUTOMATIC_INDEX
{ "automatic_index", SQLITE_AutoIndex },
@@ -85253,7 +85702,7 @@
sqlite3VdbeChangeP1(v, addr+1, iDb);
sqlite3VdbeChangeP1(v, addr+6, SQLITE_DEFAULT_CACHE_SIZE);
}else{
- int size = atoi(zRight);
+ int size = sqlite3Atoi(zRight);
if( size<0 ) size = -size;
sqlite3BeginWriteOperation(pParse, 0, iDb);
sqlite3VdbeAddOp2(v, OP_Integer, size, 1);
@@ -85282,7 +85731,7 @@
/* Malloc may fail when setting the page-size, as there is an internal
** buffer that the pager module resizes using sqlite3_realloc().
*/
- db->nextPagesize = atoi(zRight);
+ db->nextPagesize = sqlite3Atoi(zRight);
if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize, -1, 0) ){
db->mallocFailed = 1;
}
@@ -85290,28 +85739,6 @@
}else
/*
- ** PRAGMA [database.]max_page_count
- ** PRAGMA [database.]max_page_count=N
- **
- ** The first form reports the current setting for the
- ** maximum number of pages in the database file. The
- ** second form attempts to change this setting. Both
- ** forms return the current setting.
- */
- if( sqlite3StrICmp(zLeft,"max_page_count")==0 ){
- Btree *pBt = pDb->pBt;
- int newMax = 0;
- assert( pBt!=0 );
- if( zRight ){
- newMax = atoi(zRight);
- }
- if( ALWAYS(pBt) ){
- newMax = sqlite3BtreeMaxPageCount(pBt, newMax);
- }
- returnSingleInt(pParse, "max_page_count", newMax);
- }else
-
- /*
** PRAGMA [database.]secure_delete
** PRAGMA [database.]secure_delete=ON/OFF
**
@@ -85337,19 +85764,33 @@
}else
/*
+ ** PRAGMA [database.]max_page_count
+ ** PRAGMA [database.]max_page_count=N
+ **
+ ** The first form reports the current setting for the
+ ** maximum number of pages in the database file. The
+ ** second form attempts to change this setting. Both
+ ** forms return the current setting.
+ **
** PRAGMA [database.]page_count
**
** Return the number of pages in the specified database.
*/
- if( sqlite3StrICmp(zLeft,"page_count")==0 ){
+ if( sqlite3StrICmp(zLeft,"page_count")==0
+ || sqlite3StrICmp(zLeft,"max_page_count")==0
+ ){
int iReg;
if( sqlite3ReadSchema(pParse) ) goto pragma_out;
sqlite3CodeVerifySchema(pParse, iDb);
iReg = ++pParse->nMem;
- sqlite3VdbeAddOp2(v, OP_Pagecount, iDb, iReg);
+ if( zLeft[0]=='p' ){
+ sqlite3VdbeAddOp2(v, OP_Pagecount, iDb, iReg);
+ }else{
+ sqlite3VdbeAddOp3(v, OP_MaxPgcnt, iDb, iReg, sqlite3Atoi(zRight));
+ }
sqlite3VdbeAddOp2(v, OP_ResultRow, iReg, 1);
sqlite3VdbeSetNumCols(v, 1);
- sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "page_count", SQLITE_STATIC);
+ sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLeft, SQLITE_TRANSIENT);
}else
/*
@@ -85457,7 +85898,7 @@
Pager *pPager = sqlite3BtreePager(pDb->pBt);
i64 iLimit = -2;
if( zRight ){
- sqlite3Atoi64(zRight, &iLimit);
+ sqlite3Atoi64(zRight, &iLimit, 1000000, SQLITE_UTF8);
if( iLimit<-1 ) iLimit = -1;
}
iLimit = sqlite3PagerJournalSizeLimit(pPager, iLimit);
@@ -85571,7 +86012,7 @@
if( !zRight ){
returnSingleInt(pParse, "cache_size", pDb->pSchema->cache_size);
}else{
- int size = atoi(zRight);
+ int size = sqlite3Atoi(zRight);
if( size<0 ) size = -size;
pDb->pSchema->cache_size = size;
sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size);
@@ -85964,7 +86405,7 @@
/* Set the maximum error count */
mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX;
if( zRight ){
- mxErr = atoi(zRight);
+ sqlite3GetInt32(zRight, &mxErr);
if( mxErr<=0 ){
mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX;
}
@@ -86221,7 +86662,7 @@
};
int addr = sqlite3VdbeAddOpList(v, ArraySize(setCookie), setCookie);
sqlite3VdbeChangeP1(v, addr, iDb);
- sqlite3VdbeChangeP1(v, addr+1, atoi(zRight));
+ sqlite3VdbeChangeP1(v, addr+1, sqlite3Atoi(zRight));
sqlite3VdbeChangeP1(v, addr+2, iDb);
sqlite3VdbeChangeP2(v, addr+2, iCookie);
}else{
@@ -86282,8 +86723,7 @@
*/
if( sqlite3StrICmp(zLeft, "wal_autocheckpoint")==0 ){
if( zRight ){
- int nAuto = atoi(zRight);
- sqlite3_wal_autocheckpoint(db, nAuto);
+ sqlite3_wal_autocheckpoint(db, sqlite3Atoi(zRight));
}
returnSingleInt(pParse, "wal_autocheckpoint",
db->xWalCallback==sqlite3WalDefaultHook ?
@@ -86373,7 +86813,8 @@
#ifndef SQLITE_OMIT_PAGER_PRAGMAS
if( db->autoCommit ){
sqlite3BtreeSetSafetyLevel(pDb->pBt, pDb->safety_level,
- (db->flags&SQLITE_FullFSync)!=0);
+ (db->flags&SQLITE_FullFSync)!=0,
+ (db->flags&SQLITE_CkptFullFSync)!=0);
}
#endif
pragma_out:
@@ -86465,7 +86906,7 @@
assert( db->init.busy );
db->init.iDb = iDb;
- db->init.newTnum = atoi(argv[1]);
+ db->init.newTnum = sqlite3Atoi(argv[1]);
db->init.orphanTrigger = 0;
TESTONLY(rcp = ) sqlite3_prepare(db, argv[2], -1, &pStmt, 0);
rc = db->errCode;
@@ -87014,13 +87455,13 @@
if( rc==SQLITE_OK && pParse->pVdbe && pParse->explain ){
static const char * const azColName[] = {
"addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment",
- "order", "from", "detail"
+ "selectid", "order", "from", "detail"
};
int iFirst, mx;
if( pParse->explain==2 ){
- sqlite3VdbeSetNumCols(pParse->pVdbe, 3);
+ sqlite3VdbeSetNumCols(pParse->pVdbe, 4);
iFirst = 8;
- mx = 11;
+ mx = 12;
}else{
sqlite3VdbeSetNumCols(pParse->pVdbe, 8);
iFirst = 0;
@@ -87690,7 +88131,6 @@
sqlite3VdbeAddOp1(v, OP_Last, pOrderBy->iECursor);
sqlite3VdbeAddOp1(v, OP_Delete, pOrderBy->iECursor);
sqlite3VdbeJumpHere(v, addr2);
- pSelect->iLimit = 0;
}
}
@@ -87969,11 +88409,11 @@
#endif
}
- /* Jump to the end of the loop if the LIMIT is reached.
+ /* Jump to the end of the loop if the LIMIT is reached. Except, if
+ ** there is a sorter, in which case the sorter has already limited
+ ** the output for us.
*/
- if( p->iLimit ){
- assert( pOrderBy==0 ); /* If there is an ORDER BY, the call to
- ** pushOntoSorter() would have cleared p->iLimit */
+ if( pOrderBy==0 && p->iLimit ){
sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1);
}
}
@@ -88020,6 +88460,88 @@
return pInfo;
}
+#ifndef SQLITE_OMIT_COMPOUND_SELECT
+/*
+** Name of the connection operator, used for error messages.
+*/
+static const char *selectOpName(int id){
+ char *z;
+ switch( id ){
+ case TK_ALL: z = "UNION ALL"; break;
+ case TK_INTERSECT: z = "INTERSECT"; break;
+ case TK_EXCEPT: z = "EXCEPT"; break;
+ default: z = "UNION"; break;
+ }
+ return z;
+}
+#endif /* SQLITE_OMIT_COMPOUND_SELECT */
+
+#ifndef SQLITE_OMIT_EXPLAIN
+/*
+** Unless an "EXPLAIN QUERY PLAN" command is being processed, this function
+** is a no-op. Otherwise, it adds a single row of output to the EQP result,
+** where the caption is of the form:
+**
+** "USE TEMP B-TREE FOR xxx"
+**
+** where xxx is one of "DISTINCT", "ORDER BY" or "GROUP BY". Exactly which
+** is determined by the zUsage argument.
+*/
+static void explainTempTable(Parse *pParse, const char *zUsage){
+ if( pParse->explain==2 ){
+ Vdbe *v = pParse->pVdbe;
+ char *zMsg = sqlite3MPrintf(pParse->db, "USE TEMP B-TREE FOR %s", zUsage);
+ sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC);
+ }
+}
+
+/*
+** Unless an "EXPLAIN QUERY PLAN" command is being processed, this function
+** is a no-op. Otherwise, it adds a single row of output to the EQP result,
+** where the caption is of one of the two forms:
+**
+** "COMPOSITE SUBQUERIES iSub1 and iSub2 (op)"
+** "COMPOSITE SUBQUERIES iSub1 and iSub2 USING TEMP B-TREE (op)"
+**
+** where iSub1 and iSub2 are the integers passed as the corresponding
+** function parameters, and op is the text representation of the parameter
+** of the same name. The parameter "op" must be one of TK_UNION, TK_EXCEPT,
+** TK_INTERSECT or TK_ALL. The first form is used if argument bUseTmp is
+** false, or the second form if it is true.
+*/
+static void explainComposite(
+ Parse *pParse, /* Parse context */
+ int op, /* One of TK_UNION, TK_EXCEPT etc. */
+ int iSub1, /* Subquery id 1 */
+ int iSub2, /* Subquery id 2 */
+ int bUseTmp /* True if a temp table was used */
+){
+ assert( op==TK_UNION || op==TK_EXCEPT || op==TK_INTERSECT || op==TK_ALL );
+ if( pParse->explain==2 ){
+ Vdbe *v = pParse->pVdbe;
+ char *zMsg = sqlite3MPrintf(
+ pParse->db, "COMPOUND SUBQUERIES %d AND %d %s(%s)", iSub1, iSub2,
+ bUseTmp?"USING TEMP B-TREE ":"", selectOpName(op)
+ );
+ sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC);
+ }
+}
+
+/*
+** Assign expression b to lvalue a. A second, no-op, version of this macro
+** is provided when SQLITE_OMIT_EXPLAIN is defined. This allows the code
+** in sqlite3Select() to assign values to structure member variables that
+** only exist if SQLITE_OMIT_EXPLAIN is not defined without polluting the
+** code with #ifndef directives.
+*/
+# define explainSetInteger(a, b) a = b
+
+#else
+/* No-op versions of the explainXXX() functions and macros. */
+# define explainTempTable(y,z)
+# define explainComposite(v,w,x,y,z)
+# define explainSetInteger(y,z)
+#endif
/*
** If the inner loop was generated using a non-null pOrderBy argument,
@@ -88108,10 +88630,6 @@
sqlite3ReleaseTempReg(pParse, regRow);
sqlite3ReleaseTempReg(pParse, regRowid);
- /* LIMIT has been implemented by the pushOntoSorter() routine.
- */
- assert( p->iLimit==0 );
-
/* The bottom of the loop
*/
sqlite3VdbeResolveLabel(v, addrContinue);
@@ -88371,22 +88889,6 @@
generateColumnTypes(pParse, pTabList, pEList);
}
-#ifndef SQLITE_OMIT_COMPOUND_SELECT
-/*
-** Name of the connection operator, used for error messages.
-*/
-static const char *selectOpName(int id){
- char *z;
- switch( id ){
- case TK_ALL: z = "UNION ALL"; break;
- case TK_INTERSECT: z = "INTERSECT"; break;
- case TK_EXCEPT: z = "EXCEPT"; break;
- default: z = "UNION"; break;
- }
- return z;
-}
-#endif /* SQLITE_OMIT_COMPOUND_SELECT */
-
/*
** Given a an expression list (which is really the list of expressions
** that form the result set of a SELECT statement) compute appropriate
@@ -88549,6 +89051,7 @@
assert( db->lookaside.bEnabled==0 );
pTab->nRef = 1;
pTab->zName = 0;
+ pTab->nRowEst = 1000000;
selectColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol);
selectAddColumnTypeAndCollation(pParse, pTab->nCol, pTab->aCol, pSelect);
pTab->iPKey = -1;
@@ -88619,6 +89122,8 @@
VdbeComment((v, "LIMIT counter"));
if( n==0 ){
sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak);
+ }else{
+ if( p->nSelectRow > (double)n ) p->nSelectRow = (double)n;
}
}else{
sqlite3ExprCode(pParse, p->pLimit, iLimit);
@@ -88719,6 +89224,10 @@
SelectDest dest; /* Alternative data destination */
Select *pDelete = 0; /* Chain of simple selects to delete */
sqlite3 *db; /* Database connection */
+#ifndef SQLITE_OMIT_EXPLAIN
+ int iSub1; /* EQP id of left-hand query */
+ int iSub2; /* EQP id of right-hand query */
+#endif
/* Make sure there is no ORDER BY or LIMIT clause on prior SELECTs. Only
** the last (right-most) SELECT in the series may have an ORDER BY or LIMIT.
@@ -88776,9 +89285,11 @@
switch( p->op ){
case TK_ALL: {
int addr = 0;
+ int nLimit;
assert( !pPrior->pLimit );
pPrior->pLimit = p->pLimit;
pPrior->pOffset = p->pOffset;
+ explainSetInteger(iSub1, pParse->iNextSelectId);
rc = sqlite3Select(pParse, pPrior, &dest);
p->pLimit = 0;
p->pOffset = 0;
@@ -88792,10 +89303,18 @@
addr = sqlite3VdbeAddOp1(v, OP_IfZero, p->iLimit);
VdbeComment((v, "Jump ahead if LIMIT reached"));
}
+ explainSetInteger(iSub2, pParse->iNextSelectId);
rc = sqlite3Select(pParse, p, &dest);
testcase( rc!=SQLITE_OK );
pDelete = p->pPrior;
p->pPrior = pPrior;
+ p->nSelectRow += pPrior->nSelectRow;
+ if( pPrior->pLimit
+ && sqlite3ExprIsInteger(pPrior->pLimit, &nLimit)
+ && p->nSelectRow > (double)nLimit
+ ){
+ p->nSelectRow = (double)nLimit;
+ }
if( addr ){
sqlite3VdbeJumpHere(v, addr);
}
@@ -88839,6 +89358,7 @@
*/
assert( !pPrior->pOrderBy );
sqlite3SelectDestInit(&uniondest, priorOp, unionTab);
+ explainSetInteger(iSub1, pParse->iNextSelectId);
rc = sqlite3Select(pParse, pPrior, &uniondest);
if( rc ){
goto multi_select_end;
@@ -88858,6 +89378,7 @@
pOffset = p->pOffset;
p->pOffset = 0;
uniondest.eDest = op;
+ explainSetInteger(iSub2, pParse->iNextSelectId);
rc = sqlite3Select(pParse, p, &uniondest);
testcase( rc!=SQLITE_OK );
/* Query flattening in sqlite3Select() might refill p->pOrderBy.
@@ -88866,6 +89387,7 @@
pDelete = p->pPrior;
p->pPrior = pPrior;
p->pOrderBy = 0;
+ if( p->op==TK_UNION ) p->nSelectRow += pPrior->nSelectRow;
sqlite3ExprDelete(db, p->pLimit);
p->pLimit = pLimit;
p->pOffset = pOffset;
@@ -88923,6 +89445,7 @@
/* Code the SELECTs to our left into temporary table "tab1".
*/
sqlite3SelectDestInit(&intersectdest, SRT_Union, tab1);
+ explainSetInteger(iSub1, pParse->iNextSelectId);
rc = sqlite3Select(pParse, pPrior, &intersectdest);
if( rc ){
goto multi_select_end;
@@ -88939,10 +89462,12 @@
pOffset = p->pOffset;
p->pOffset = 0;
intersectdest.iParm = tab2;
+ explainSetInteger(iSub2, pParse->iNextSelectId);
rc = sqlite3Select(pParse, p, &intersectdest);
testcase( rc!=SQLITE_OK );
pDelete = p->pPrior;
p->pPrior = pPrior;
+ if( p->nSelectRow>pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow;
sqlite3ExprDelete(db, p->pLimit);
p->pLimit = pLimit;
p->pOffset = pOffset;
@@ -88975,6 +89500,8 @@
}
}
+ explainComposite(pParse, p->op, iSub1, iSub2, p->op!=TK_ALL);
+
/* Compute collating sequences used by
** temporary tables needed to implement the compound select.
** Attach the KeyInfo structure to all temporary tables.
@@ -89318,6 +89845,10 @@
ExprList *pOrderBy; /* The ORDER BY clause */
int nOrderBy; /* Number of terms in the ORDER BY clause */
int *aPermute; /* Mapping from ORDER BY terms to result set columns */
+#ifndef SQLITE_OMIT_EXPLAIN
+ int iSub1; /* EQP id of left-hand query */
+ int iSub2; /* EQP id of right-hand query */
+#endif
assert( p->pOrderBy!=0 );
assert( pKeyDup==0 ); /* "Managed" code needs this. Ticket #3382. */
@@ -89429,7 +89960,6 @@
/* Separate the left and the right query from one another
*/
p->pPrior = 0;
- pPrior->pRightmost = 0;
sqlite3ResolveOrderGroupBy(pParse, p, p->pOrderBy, "ORDER");
if( pPrior->pPrior==0 ){
sqlite3ResolveOrderGroupBy(pParse, pPrior, pPrior->pOrderBy, "ORDER");
@@ -89472,6 +90002,7 @@
*/
VdbeNoopComment((v, "Begin coroutine for left SELECT"));
pPrior->iLimit = regLimitA;
+ explainSetInteger(iSub1, pParse->iNextSelectId);
sqlite3Select(pParse, pPrior, &destA);
sqlite3VdbeAddOp2(v, OP_Integer, 1, regEofA);
sqlite3VdbeAddOp1(v, OP_Yield, regAddrA);
@@ -89486,6 +90017,7 @@
savedOffset = p->iOffset;
p->iLimit = regLimitB;
p->iOffset = 0;
+ explainSetInteger(iSub2, pParse->iNextSelectId);
sqlite3Select(pParse, p, &destB);
p->iLimit = savedLimit;
p->iOffset = savedOffset;
@@ -89522,6 +90054,7 @@
sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB);
sqlite3VdbeAddOp1(v, OP_Yield, regAddrB);
sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEofA);
+ p->nSelectRow += pPrior->nSelectRow;
}
/* Generate a subroutine to run when the results from select B
@@ -89529,6 +90062,7 @@
*/
if( op==TK_INTERSECT ){
addrEofB = addrEofA;
+ if( p->nSelectRow > pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow;
}else{
VdbeNoopComment((v, "eof-B subroutine"));
addrEofB = sqlite3VdbeAddOp2(v, OP_If, regEofA, labelEnd);
@@ -89616,6 +90150,7 @@
/*** TBD: Insert subroutine calls to close cursors on incomplete
**** subqueries ****/
+ explainComposite(pParse, p->op, iSub1, iSub2, 0);
return SQLITE_OK;
}
#endif
@@ -90349,6 +90884,7 @@
while( pSel->pPrior ){ pSel = pSel->pPrior; }
selectColumnsFromExprList(pParse, pSel->pEList, &pTab->nCol, &pTab->aCol);
pTab->iPKey = -1;
+ pTab->nRowEst = 1000000;
pTab->tabFlags |= TF_Ephemeral;
#endif
}else{
@@ -90842,6 +91378,11 @@
int iEnd; /* Address of the end of the query */
sqlite3 *db; /* The database connection */
+#ifndef SQLITE_OMIT_EXPLAIN
+ int iRestoreSelectId = pParse->iSelectId;
+ pParse->iSelectId = pParse->iNextSelectId++;
+#endif
+
db = pParse->db;
if( p==0 || db->mallocFailed || pParse->nErr ){
return 1;
@@ -90913,8 +91454,10 @@
}else{
sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor);
assert( pItem->isPopulated==0 );
+ explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId);
sqlite3Select(pParse, pSub, &dest);
pItem->isPopulated = 1;
+ pItem->pTab->nRowEst = (unsigned)pSub->nSelectRow;
}
if( /*pParse->nErr ||*/ db->mallocFailed ){
goto select_end;
@@ -90948,10 +91491,12 @@
mxSelect = db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT];
if( mxSelect && cnt>mxSelect ){
sqlite3ErrorMsg(pParse, "too many terms in compound SELECT");
- return 1;
+ goto select_end;
}
}
- return multiSelect(pParse, p, pDest);
+ rc = multiSelect(pParse, p, pDest);
+ explainSetInteger(pParse->iSelectId, iRestoreSelectId);
+ return rc;
}
#endif
@@ -90963,7 +91508,6 @@
p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0);
pGroupBy = p->pGroupBy;
p->selFlags &= ~SF_Distinct;
- isDistinct = 0;
}
/* If there is both a GROUP BY and an ORDER BY clause and they are
@@ -91006,11 +91550,12 @@
/* Set the limiter.
*/
iEnd = sqlite3VdbeMakeLabel(v);
+ p->nSelectRow = (double)LARGEST_INT64;
computeLimitRegisters(pParse, p, iEnd);
/* Open a virtual index to use for the distinct set.
*/
- if( isDistinct ){
+ if( p->selFlags & SF_Distinct ){
KeyInfo *pKeyInfo;
assert( isAgg || pGroupBy );
distinct = pParse->nTab++;
@@ -91029,6 +91574,7 @@
*/
pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, 0);
if( pWInfo==0 ) goto select_end;
+ if( pWInfo->nRowOut < p->nSelectRow ) p->nSelectRow = pWInfo->nRowOut;
/* If sorting index that was created by a prior OP_OpenEphemeral
** instruction ended up not being needed, then change the OP_OpenEphemeral
@@ -91073,6 +91619,9 @@
for(k=pGroupBy->nExpr, pItem=pGroupBy->a; k>0; k--, pItem++){
pItem->iAlias = 0;
}
+ if( p->nSelectRow>(double)100 ) p->nSelectRow = (double)100;
+ }else{
+ p->nSelectRow = (double)1;
}
@@ -91169,6 +91718,9 @@
int nCol;
int nGroupBy;
+ explainTempTable(pParse,
+ isDistinct && !(p->selFlags&SF_Distinct)?"DISTINCT":"GROUP BY");
+
groupBySort = 1;
nGroupBy = pGroupBy->nExpr;
nCol = nGroupBy + 1;
@@ -91430,10 +91982,15 @@
} /* endif aggregate query */
+ if( distinct>=0 ){
+ explainTempTable(pParse, "DISTINCT");
+ }
+
/* If there is an ORDER BY clause, then we need to sort the results
** and send them to the callback one by one.
*/
if( pOrderBy ){
+ explainTempTable(pParse, "ORDER BY");
generateSortTail(pParse, p, v, pEList->nExpr, pDest);
}
@@ -91450,6 +92007,7 @@
** successful coding of the SELECT.
*/
select_end:
+ explainSetInteger(pParse->iSelectId, iRestoreSelectId);
/* Identify column names if results of the SELECT are to be output.
*/
@@ -94238,7 +94796,7 @@
sqlite3ChangeCookie(pParse, iDb);
sqlite3VdbeAddOp2(v, OP_Expire, 0, 0);
- zWhere = sqlite3MPrintf(db, "name='%q'", pTab->zName);
+ zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName);
sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 1, 0, zWhere, P4_DYNAMIC);
sqlite3VdbeAddOp4(v, OP_VCreate, iDb, 0, 0,
pTab->zName, sqlite3Strlen30(pTab->zName) + 1);
@@ -94539,7 +95097,7 @@
}
db->pVTab = 0;
}else{
- sqlite3Error(db, SQLITE_ERROR, zErr);
+ sqlite3Error(db, SQLITE_ERROR, (zErr ? "%s" : 0), zErr);
sqlite3DbFree(db, zErr);
rc = SQLITE_ERROR;
}
@@ -95001,7 +95559,6 @@
struct WhereCost {
WherePlan plan; /* The lookup strategy */
double rCost; /* Overall cost of pursuing this search strategy */
- double nRow; /* Estimated number of output rows */
Bitmask used; /* Bitmask of cursors used by this plan */
};
@@ -95044,10 +95601,11 @@
#define WHERE_COLUMN_IN 0x00040000 /* x IN (...) */
#define WHERE_COLUMN_NULL 0x00080000 /* x IS NULL */
#define WHERE_INDEXED 0x000f0000 /* Anything that uses an index */
-#define WHERE_NOT_FULLSCAN 0x000f3000 /* Does not do a full table scan */
+#define WHERE_NOT_FULLSCAN 0x100f3000 /* Does not do a full table scan */
#define WHERE_IN_ABLE 0x000f1000 /* Able to support an IN operator */
#define WHERE_TOP_LIMIT 0x00100000 /* x<EXPR or x<=EXPR constraint */
#define WHERE_BTM_LIMIT 0x00200000 /* x>EXPR or x>=EXPR constraint */
+#define WHERE_BOTH_LIMIT 0x00300000 /* Both x>EXPR and x<EXPR */
#define WHERE_IDX_ONLY 0x00800000 /* Use index only - omit table */
#define WHERE_ORDERBY 0x01000000 /* Output will appear in correct order */
#define WHERE_REVERSE 0x02000000 /* Scan in reverse order */
@@ -96365,7 +96923,8 @@
** Required because bestIndex() is called by bestOrClauseIndex()
*/
static void bestIndex(
- Parse*, WhereClause*, struct SrcList_item*, Bitmask, ExprList*, WhereCost*);
+ Parse*, WhereClause*, struct SrcList_item*,
+ Bitmask, Bitmask, ExprList*, WhereCost*);
/*
** This routine attempts to find an scanning strategy that can be used
@@ -96378,7 +96937,8 @@
Parse *pParse, /* The parsing context */
WhereClause *pWC, /* The WHERE clause */
struct SrcList_item *pSrc, /* The FROM clause term to search */
- Bitmask notReady, /* Mask of cursors that are not available */
+ Bitmask notReady, /* Mask of cursors not available for indexing */
+ Bitmask notValid, /* Cursors not available for any purpose */
ExprList *pOrderBy, /* The ORDER BY clause */
WhereCost *pCost /* Lowest cost query plan */
){
@@ -96388,8 +96948,9 @@
WhereTerm * const pWCEnd = &pWC->a[pWC->nTerm]; /* End of pWC->a[] */
WhereTerm *pTerm; /* A single term of the WHERE clause */
- /* No OR-clause optimization allowed if the NOT INDEXED clause is used */
- if( pSrc->notIndexed ){
+ /* No OR-clause optimization allowed if the INDEXED BY or NOT INDEXED clauses
+ ** are used */
+ if( pSrc->notIndexed || pSrc->pIndex!=0 ){
return;
}
@@ -96414,7 +96975,7 @@
));
if( pOrTerm->eOperator==WO_AND ){
WhereClause *pAndWC = &pOrTerm->u.pAndInfo->wc;
- bestIndex(pParse, pAndWC, pSrc, notReady, 0, &sTermCost);
+ bestIndex(pParse, pAndWC, pSrc, notReady, notValid, 0, &sTermCost);
}else if( pOrTerm->leftCursor==iCur ){
WhereClause tempWC;
tempWC.pParse = pWC->pParse;
@@ -96422,12 +96983,12 @@
tempWC.op = TK_AND;
tempWC.a = pOrTerm;
tempWC.nTerm = 1;
- bestIndex(pParse, &tempWC, pSrc, notReady, 0, &sTermCost);
+ bestIndex(pParse, &tempWC, pSrc, notReady, notValid, 0, &sTermCost);
}else{
continue;
}
rTotal += sTermCost.rCost;
- nRow += sTermCost.nRow;
+ nRow += sTermCost.plan.nRow;
used |= sTermCost.used;
if( rTotal>=pCost->rCost ) break;
}
@@ -96446,8 +97007,8 @@
WHERETRACE(("... multi-index OR cost=%.9g nrow=%.9g\n", rTotal, nRow));
if( rTotal<pCost->rCost ){
pCost->rCost = rTotal;
- pCost->nRow = nRow;
pCost->used = used;
+ pCost->plan.nRow = nRow;
pCost->plan.wsFlags = flags;
pCost->plan.u.pTerm = pTerm;
}
@@ -96531,7 +97092,7 @@
WHERETRACE(("auto-index reduces cost from %.2f to %.2f\n",
pCost->rCost, costTempIdx));
pCost->rCost = costTempIdx;
- pCost->nRow = logN + 1;
+ pCost->plan.nRow = logN + 1;
pCost->plan.wsFlags = WHERE_TEMP_INDEX;
pCost->used = pTerm->prereqRight;
break;
@@ -96869,7 +97430,8 @@
Parse *pParse, /* The parsing context */
WhereClause *pWC, /* The WHERE clause */
struct SrcList_item *pSrc, /* The FROM clause term to search */
- Bitmask notReady, /* Mask of cursors that are not available */
+ Bitmask notReady, /* Mask of cursors not available for index */
+ Bitmask notValid, /* Cursors not valid for any purpose */
ExprList *pOrderBy, /* The order by clause */
WhereCost *pCost, /* Lowest cost query plan */
sqlite3_index_info **ppIdxInfo /* Index information passed to xBestIndex */
@@ -96999,7 +97561,7 @@
/* Try to find a more efficient access pattern by using multiple indexes
** to optimize an OR expression within the WHERE clause.
*/
- bestOrClauseIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost);
+ bestOrClauseIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost);
}
#endif /* SQLITE_OMIT_VIRTUALTABLE */
@@ -97280,7 +97842,8 @@
Parse *pParse, /* The parsing context */
WhereClause *pWC, /* The WHERE clause */
struct SrcList_item *pSrc, /* The FROM clause term to search */
- Bitmask notReady, /* Mask of cursors that are not available */
+ Bitmask notReady, /* Mask of cursors not available for indexing */
+ Bitmask notValid, /* Cursors not available for any purpose */
ExprList *pOrderBy, /* The ORDER BY clause */
WhereCost *pCost /* Lowest cost query plan */
){
@@ -97542,16 +98105,16 @@
** with this step if we already know this index will not be chosen.
** Also, never reduce the output row count below 2 using this step.
**
- ** Do not reduce the output row count if pSrc is the only table that
- ** is notReady; if notReady is a power of two. This will be the case
- ** when the main sqlite3WhereBegin() loop is scanning for a table with
- ** and "optimal" index, and on such a scan the output row count
- ** reduction is not valid because it does not update the "pCost->used"
- ** bitmap. The notReady bitmap will also be a power of two when we
- ** are scanning for the last table in a 64-way join. We are willing
- ** to bypass this optimization in that corner case.
+ ** It is critical that the notValid mask be used here instead of
+ ** the notReady mask. When computing an "optimal" index, the notReady
+ ** mask will only have one bit set - the bit for the current table.
+ ** The notValid mask, on the other hand, always has all bits set for
+ ** tables that are not in outer loops. If notReady is used here instead
+ ** of notValid, then a optimal index that depends on inner joins loops
+ ** might be selected even when there exists an optimal index that has
+ ** no such dependency.
*/
- if( nRow>2 && cost<=pCost->rCost && (notReady & (notReady-1))!=0 ){
+ if( nRow>2 && cost<=pCost->rCost ){
int k; /* Loop counter */
int nSkipEq = nEq; /* Number of == constraints to skip */
int nSkipRange = nBound; /* Number of < constraints to skip */
@@ -97560,7 +98123,7 @@
thisTab = getMask(pWC->pMaskSet, iCur);
for(pTerm=pWC->a, k=pWC->nTerm; nRow>2 && k; k--, pTerm++){
if( pTerm->wtFlags & TERM_VIRTUAL ) continue;
- if( (pTerm->prereqAll & notReady)!=thisTab ) continue;
+ if( (pTerm->prereqAll & notValid)!=thisTab ) continue;
if( pTerm->eOperator & (WO_EQ|WO_IN|WO_ISNULL) ){
if( nSkipEq ){
/* Ignore the first nEq equality matches since the index
@@ -97602,11 +98165,11 @@
** index and its cost in the pCost structure.
*/
if( (!pIdx || wsFlags)
- && (cost<pCost->rCost || (cost<=pCost->rCost && nRow<pCost->nRow))
+ && (cost<pCost->rCost || (cost<=pCost->rCost && nRow<pCost->plan.nRow))
){
pCost->rCost = cost;
- pCost->nRow = nRow;
pCost->used = used;
+ pCost->plan.nRow = nRow;
pCost->plan.wsFlags = (wsFlags&wsFlagMask);
pCost->plan.nEq = nEq;
pCost->plan.u.pIdx = pIdx;
@@ -97642,7 +98205,7 @@
pCost->plan.u.pIdx ? pCost->plan.u.pIdx->zName : "ipk")
));
- bestOrClauseIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost);
+ bestOrClauseIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost);
bestAutomaticIndex(pParse, pWC, pSrc, notReady, pCost);
pCost->plan.wsFlags |= eqTermMask;
}
@@ -97657,14 +98220,15 @@
Parse *pParse, /* The parsing context */
WhereClause *pWC, /* The WHERE clause */
struct SrcList_item *pSrc, /* The FROM clause term to search */
- Bitmask notReady, /* Mask of cursors that are not available */
+ Bitmask notReady, /* Mask of cursors not available for indexing */
+ Bitmask notValid, /* Cursors not available for any purpose */
ExprList *pOrderBy, /* The ORDER BY clause */
WhereCost *pCost /* Lowest cost query plan */
){
#ifndef SQLITE_OMIT_VIRTUALTABLE
if( IsVirtual(pSrc->pTab) ){
sqlite3_index_info *p = 0;
- bestVirtualIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost, &p);
+ bestVirtualIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost,&p);
if( p->needToFreeIdxStr ){
sqlite3_free(p->idxStr);
}
@@ -97672,7 +98236,7 @@
}else
#endif
{
- bestBtreeIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost);
+ bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost);
}
}
@@ -97934,6 +98498,161 @@
return regBase;
}
+#ifndef SQLITE_OMIT_EXPLAIN
+/*
+** This routine is a helper for explainIndexRange() below
+**
+** pStr holds the text of an expression that we are building up one term
+** at a time. This routine adds a new term to the end of the expression.
+** Terms are separated by AND so add the "AND" text for second and subsequent
+** terms only.
+*/
+static void explainAppendTerm(
+ StrAccum *pStr, /* The text expression being built */
+ int iTerm, /* Index of this term. First is zero */
+ const char *zColumn, /* Name of the column */
+ const char *zOp /* Name of the operator */
+){
+ if( iTerm ) sqlite3StrAccumAppend(pStr, " AND ", 5);
+ sqlite3StrAccumAppend(pStr, zColumn, -1);
+ sqlite3StrAccumAppend(pStr, zOp, 1);
+ sqlite3StrAccumAppend(pStr, "?", 1);
+}
+
+/*
+** Argument pLevel describes a strategy for scanning table pTab. This
+** function returns a pointer to a string buffer containing a description
+** of the subset of table rows scanned by the strategy in the form of an
+** SQL expression. Or, if all rows are scanned, NULL is returned.
+**
+** For example, if the query:
+**
+** SELECT * FROM t1 WHERE a=1 AND b>2;
+**
+** is run and there is an index on (a, b), then this function returns a
+** string similar to:
+**
+** "a=? AND b>?"
+**
+** The returned pointer points to memory obtained from sqlite3DbMalloc().
+** It is the responsibility of the caller to free the buffer when it is
+** no longer required.
+*/
+static char *explainIndexRange(sqlite3 *db, WhereLevel *pLevel, Table *pTab){
+ WherePlan *pPlan = &pLevel->plan;
+ Index *pIndex = pPlan->u.pIdx;
+ int nEq = pPlan->nEq;
+ int i, j;
+ Column *aCol = pTab->aCol;
+ int *aiColumn = pIndex->aiColumn;
+ StrAccum txt;
+
+ if( nEq==0 && (pPlan->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ){
+ return 0;
+ }
+ sqlite3StrAccumInit(&txt, 0, 0, SQLITE_MAX_LENGTH);
+ txt.db = db;
+ sqlite3StrAccumAppend(&txt, " (", 2);
+ for(i=0; i<nEq; i++){
+ explainAppendTerm(&txt, i, aCol[aiColumn[i]].zName, "=");
+ }
+
+ j = i;
+ if( pPlan->wsFlags&WHERE_BTM_LIMIT ){
+ explainAppendTerm(&txt, i++, aCol[aiColumn[j]].zName, ">");
+ }
+ if( pPlan->wsFlags&WHERE_TOP_LIMIT ){
+ explainAppendTerm(&txt, i, aCol[aiColumn[j]].zName, "<");
+ }
+ sqlite3StrAccumAppend(&txt, ")", 1);
+ return sqlite3StrAccumFinish(&txt);
+}
+
+/*
+** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN
+** command. If the query being compiled is an EXPLAIN QUERY PLAN, a single
+** record is added to the output to describe the table scan strategy in
+** pLevel.
+*/
+static void explainOneScan(
+ Parse *pParse, /* Parse context */
+ SrcList *pTabList, /* Table list this loop refers to */
+ WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */
+ int iLevel, /* Value for "level" column of output */
+ int iFrom, /* Value for "from" column of output */
+ u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */
+){
+ if( pParse->explain==2 ){
+ u32 flags = pLevel->plan.wsFlags;
+ struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom];
+ Vdbe *v = pParse->pVdbe; /* VM being constructed */
+ sqlite3 *db = pParse->db; /* Database handle */
+ char *zMsg; /* Text to add to EQP output */
+ sqlite3_int64 nRow; /* Expected number of rows visited by scan */
+ int iId = pParse->iSelectId; /* Select id (left-most output column) */
+ int isSearch; /* True for a SEARCH. False for SCAN. */
+
+ if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return;
+
+ isSearch = (pLevel->plan.nEq>0)
+ || (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0
+ || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX));
+
+ zMsg = sqlite3MPrintf(db, "%s", isSearch?"SEARCH":"SCAN");
+ if( pItem->pSelect ){
+ zMsg = sqlite3MAppendf(db, zMsg, "%s SUBQUERY %d", zMsg,pItem->iSelectId);
+ }else{
+ zMsg = sqlite3MAppendf(db, zMsg, "%s TABLE %s", zMsg, pItem->zName);
+ }
+
+ if( pItem->zAlias ){
+ zMsg = sqlite3MAppendf(db, zMsg, "%s AS %s", zMsg, pItem->zAlias);
+ }
+ if( (flags & WHERE_INDEXED)!=0 ){
+ char *zWhere = explainIndexRange(db, pLevel, pItem->pTab);
+ zMsg = sqlite3MAppendf(db, zMsg, "%s USING %s%sINDEX%s%s%s", zMsg,
+ ((flags & WHERE_TEMP_INDEX)?"AUTOMATIC ":""),
+ ((flags & WHERE_IDX_ONLY)?"COVERING ":""),
+ ((flags & WHERE_TEMP_INDEX)?"":" "),
+ ((flags & WHERE_TEMP_INDEX)?"": pLevel->plan.u.pIdx->zName),
+ zWhere
+ );
+ sqlite3DbFree(db, zWhere);
+ }else if( flags & (WHERE_ROWID_EQ|WHERE_ROWID_RANGE) ){
+ zMsg = sqlite3MAppendf(db, zMsg, "%s USING INTEGER PRIMARY KEY", zMsg);
+
+ if( flags&WHERE_ROWID_EQ ){
+ zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid=?)", zMsg);
+ }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){
+ zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid>? AND rowid<?)", zMsg);
+ }else if( flags&WHERE_BTM_LIMIT ){
+ zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid>?)", zMsg);
+ }else if( flags&WHERE_TOP_LIMIT ){
+ zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid<?)", zMsg);
+ }
+ }
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+ else if( (flags & WHERE_VIRTUALTABLE)!=0 ){
+ sqlite3_index_info *pVtabIdx = pLevel->plan.u.pVtabIdx;
+ zMsg = sqlite3MAppendf(db, zMsg, "%s VIRTUAL TABLE INDEX %d:%s", zMsg,
+ pVtabIdx->idxNum, pVtabIdx->idxStr);
+ }
+#endif
+ if( wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX) ){
+ testcase( wctrlFlags & WHERE_ORDERBY_MIN );
+ nRow = 1;
+ }else{
+ nRow = (sqlite3_int64)pLevel->plan.nRow;
+ }
+ zMsg = sqlite3MAppendf(db, zMsg, "%s (~%lld rows)", zMsg, nRow);
+ sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg, P4_DYNAMIC);
+ }
+}
+#else
+# define explainOneScan(u,v,w,x,y,z)
+#endif /* SQLITE_OMIT_EXPLAIN */
+
+
/*
** Generate code for the start of the iLevel-th loop in the WHERE clause
** implementation described by pWInfo.
@@ -98341,7 +99060,7 @@
r1 = sqlite3GetTempReg(pParse);
testcase( pLevel->plan.wsFlags & WHERE_BTM_LIMIT );
testcase( pLevel->plan.wsFlags & WHERE_TOP_LIMIT );
- if( pLevel->plan.wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT) ){
+ if( (pLevel->plan.wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 ){
sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, nEq, r1);
sqlite3VdbeAddOp2(v, OP_IsNull, r1, addrCont);
}
@@ -98475,6 +99194,9 @@
WHERE_OMIT_OPEN | WHERE_OMIT_CLOSE |
WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY);
if( pSubWInfo ){
+ explainOneScan(
+ pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0
+ );
if( (wctrlFlags & WHERE_DUPLICATES_OK)==0 ){
int iSet = ((ii==pOrWc->nTerm-1)?-1:ii);
int r;
@@ -98870,6 +99592,7 @@
memset(&bestPlan, 0, sizeof(bestPlan));
bestPlan.rCost = SQLITE_BIG_DBL;
+ WHERETRACE(("*** Begin search for loop %d ***\n", i));
/* Loop through the remaining entries in the FROM clause to find the
** next nested loop. The loop tests all FROM clause entries
@@ -98888,9 +99611,15 @@
** other FROM clause terms that are notReady. If no notReady terms are
** used then the "optimal" query plan works.
**
+ ** Note that the WhereCost.nRow parameter for an optimal scan might
+ ** not be as small as it would be if the table really were the innermost
+ ** join. The nRow value can be reduced by WHERE clause constraints
+ ** that do not use indices. But this nRow reduction only happens if the
+ ** table really is the innermost join.
+ **
** The second loop iteration is only performed if no optimal scan
- ** strategies were found by the first loop. This 2nd iteration is used to
- ** search for the lowest cost scan overall.
+ ** strategies were found by the first iteration. This second iteration
+ ** is used to search for the lowest cost scan overall.
**
** Previous versions of SQLite performed only the second iteration -
** the next outermost loop was always that with the lowest overall
@@ -98910,7 +99639,7 @@
*/
nUnconstrained = 0;
notIndexed = 0;
- for(isOptimal=(iFrom<nTabList-1); isOptimal>=0; isOptimal--){
+ for(isOptimal=(iFrom<nTabList-1); isOptimal>=0 && bestJ<0; isOptimal--){
Bitmask mask; /* Mask of tables not yet ready */
for(j=iFrom, pTabItem=&pTabList->a[j]; j<nTabList; j++, pTabItem++){
int doNotReorder; /* True if this table should not be reordered */
@@ -98928,15 +99657,19 @@
pOrderBy = ((i==0 && ppOrderBy )?*ppOrderBy:0);
if( pTabItem->pIndex==0 ) nUnconstrained++;
+ WHERETRACE(("=== trying table %d with isOptimal=%d ===\n",
+ j, isOptimal));
assert( pTabItem->pTab );
#ifndef SQLITE_OMIT_VIRTUALTABLE
if( IsVirtual(pTabItem->pTab) ){
sqlite3_index_info **pp = &pWInfo->a[j].pIdxInfo;
- bestVirtualIndex(pParse, pWC, pTabItem, mask, pOrderBy, &sCost, pp);
+ bestVirtualIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy,
+ &sCost, pp);
}else
#endif
{
- bestBtreeIndex(pParse, pWC, pTabItem, mask, pOrderBy, &sCost);
+ bestBtreeIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy,
+ &sCost);
}
assert( isOptimal || (sCost.used¬Ready)==0 );
@@ -98976,10 +99709,12 @@
&& (nUnconstrained==0 || pTabItem->pIndex==0 /* (3) */
|| NEVER((sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0))
&& (bestJ<0 || sCost.rCost<bestPlan.rCost /* (4) */
- || (sCost.rCost<=bestPlan.rCost && sCost.nRow<bestPlan.nRow))
+ || (sCost.rCost<=bestPlan.rCost
+ && sCost.plan.nRow<bestPlan.plan.nRow))
){
- WHERETRACE(("... best so far with cost=%g and nRow=%g\n",
- sCost.rCost, sCost.nRow));
+ WHERETRACE(("=== table %d is best so far"
+ " with cost=%g and nRow=%g\n",
+ j, sCost.rCost, sCost.plan.nRow));
bestPlan = sCost;
bestJ = j;
}
@@ -98988,8 +99723,9 @@
}
assert( bestJ>=0 );
assert( notReady & getMask(pMaskSet, pTabList->a[bestJ].iCursor) );
- WHERETRACE(("*** Optimizer selects table %d for loop %d\n", bestJ,
- pLevel-pWInfo->a));
+ WHERETRACE(("*** Optimizer selects table %d for loop %d"
+ " with cost=%g and nRow=%g\n",
+ bestJ, pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow));
if( (bestPlan.plan.wsFlags & WHERE_ORDERBY)!=0 ){
*ppOrderBy = 0;
}
@@ -99004,7 +99740,9 @@
}
notReady &= ~getMask(pMaskSet, pTabList->a[bestJ].iCursor);
pLevel->iFrom = (u8)bestJ;
- if( bestPlan.nRow>=(double)1 ) pParse->nQueryLoop *= bestPlan.nRow;
+ if( bestPlan.plan.nRow>=(double)1 ){
+ pParse->nQueryLoop *= bestPlan.plan.nRow;
+ }
/* Check that if the table scanned by this loop iteration had an
** INDEXED BY clause attached to it, that the named index is being
@@ -99052,44 +99790,15 @@
*/
sqlite3CodeVerifySchema(pParse, -1); /* Insert the cookie verifier Goto */
notReady = ~(Bitmask)0;
+ pWInfo->nRowOut = (double)1;
for(i=0, pLevel=pWInfo->a; i<nTabList; i++, pLevel++){
Table *pTab; /* Table to open */
int iDb; /* Index of database containing table/index */
-#ifndef SQLITE_OMIT_EXPLAIN
- if( pParse->explain==2 ){
- char *zMsg;
- struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom];
- zMsg = sqlite3MPrintf(db, "TABLE %s", pItem->zName);
- if( pItem->zAlias ){
- zMsg = sqlite3MAppendf(db, zMsg, "%s AS %s", zMsg, pItem->zAlias);
- }
- if( (pLevel->plan.wsFlags & WHERE_TEMP_INDEX)!=0 ){
- zMsg = sqlite3MAppendf(db, zMsg, "%s WITH AUTOMATIC INDEX", zMsg);
- }else if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){
- zMsg = sqlite3MAppendf(db, zMsg, "%s WITH INDEX %s",
- zMsg, pLevel->plan.u.pIdx->zName);
- }else if( pLevel->plan.wsFlags & WHERE_MULTI_OR ){
- zMsg = sqlite3MAppendf(db, zMsg, "%s VIA MULTI-INDEX UNION", zMsg);
- }else if( pLevel->plan.wsFlags & (WHERE_ROWID_EQ|WHERE_ROWID_RANGE) ){
- zMsg = sqlite3MAppendf(db, zMsg, "%s USING PRIMARY KEY", zMsg);
- }
-#ifndef SQLITE_OMIT_VIRTUALTABLE
- else if( (pLevel->plan.wsFlags & WHERE_VIRTUALTABLE)!=0 ){
- sqlite3_index_info *pVtabIdx = pLevel->plan.u.pVtabIdx;
- zMsg = sqlite3MAppendf(db, zMsg, "%s VIRTUAL TABLE INDEX %d:%s", zMsg,
- pVtabIdx->idxNum, pVtabIdx->idxStr);
- }
-#endif
- if( pLevel->plan.wsFlags & WHERE_ORDERBY ){
- zMsg = sqlite3MAppendf(db, zMsg, "%s ORDER BY", zMsg);
- }
- sqlite3VdbeAddOp4(v, OP_Explain, i, pLevel->iFrom, 0, zMsg, P4_DYNAMIC);
- }
-#endif /* SQLITE_OMIT_EXPLAIN */
pTabItem = &pTabList->a[pLevel->iFrom];
pTab = pTabItem->pTab;
pLevel->iTabCur = pTabItem->iCursor;
+ pWInfo->nRowOut *= pLevel->plan.nRow;
iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
if( (pTab->tabFlags & TF_Ephemeral)!=0 || pTab->pSelect ){
/* Do nothing */
@@ -99145,8 +99854,10 @@
*/
notReady = ~(Bitmask)0;
for(i=0; i<nTabList; i++){
+ pLevel = &pWInfo->a[i];
+ explainOneScan(pParse, pTabList, pLevel, i, pLevel->iFrom, wctrlFlags);
notReady = codeOneLoopStart(pWInfo, i, wctrlFlags, notReady);
- pWInfo->iContinue = pWInfo->a[i].addrCont;
+ pWInfo->iContinue = pLevel->addrCont;
}
#ifdef SQLITE_TEST /* For testing and debugging use only */
@@ -104611,17 +105322,17 @@
/* If there are any outstanding VMs, return SQLITE_BUSY. */
if( db->pVdbe ){
- // Begin Android Change
+ // Begin Android Change
// print the first unfinalized statement in the error message, to help the developer
// figure out what the unfinalized statement is.
char buff[120];
snprintf(buff, sizeof(buff), "%d,%s", (int)db->pVdbe, db->pVdbe->zSql);
- sqlite3Error(db, SQLITE_UNCLOSED, buff);
- // End Android Change
+ sqlite3Error(db, SQLITE_UNCLOSED, buff);
+ // End Android Change
sqlite3_mutex_leave(db->mutex);
return SQLITE_UNCLOSED;
- }
+ }
assert( sqlite3SafetyCheckSickOrOk(db) );
for(j=0; j<db->nDb; j++){
@@ -105554,13 +106265,12 @@
}
pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 1);
- if( pColl ){
- pColl->xCmp = xCompare;
- pColl->pUser = pCtx;
- pColl->xDel = xDel;
- pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED));
- pColl->type = collType;
- }
+ if( pColl==0 ) return SQLITE_NOMEM;
+ pColl->xCmp = xCompare;
+ pColl->pUser = pCtx;
+ pColl->xDel = xDel;
+ pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED));
+ pColl->type = collType;
sqlite3Error(db, SQLITE_OK, 0);
return SQLITE_OK;
}
@@ -106324,7 +107034,10 @@
assert( pPager!=0 );
fd = sqlite3PagerFile(pPager);
assert( fd!=0 );
- if( fd->pMethods ){
+ if( op==SQLITE_FCNTL_FILE_POINTER ){
+ *(sqlite3_file**)pArg = fd;
+ rc = SQLITE_OK;
+ }else if( fd->pMethods ){
rc = sqlite3OsFileControl(fd, op, pArg);
}
sqlite3BtreeLeave(pBtree);
@@ -107543,8 +108256,14 @@
** Macros indicating that conditional expressions are always true or
** false.
*/
+#ifdef SQLITE_COVERAGE_TEST
+# define ALWAYS(x) (1)
+# define NEVER(X) (0)
+#else
# define ALWAYS(x) (x)
# define NEVER(X) (x)
+#endif
+
/*
** Internal types used by SQLite.
*/
@@ -107562,8 +108281,12 @@
typedef struct Fts3Cursor Fts3Cursor;
typedef struct Fts3Expr Fts3Expr;
typedef struct Fts3Phrase Fts3Phrase;
-typedef struct Fts3SegReader Fts3SegReader;
+typedef struct Fts3PhraseToken Fts3PhraseToken;
+
typedef struct Fts3SegFilter Fts3SegFilter;
+typedef struct Fts3DeferredToken Fts3DeferredToken;
+typedef struct Fts3SegReader Fts3SegReader;
+typedef struct Fts3SegReaderArray Fts3SegReaderArray;
/*
** A connection to a fulltext index is an instance of the following
@@ -107584,22 +108307,14 @@
/* Precompiled statements used by the implementation. Each of these
** statements is run and reset within a single virtual table API call.
*/
- sqlite3_stmt *aStmt[25];
-
- /* Pointer to string containing the SQL:
- **
- ** "SELECT block FROM %_segments WHERE blockid BETWEEN ? AND ?
- ** ORDER BY blockid"
- */
- char *zSelectLeaves;
- int nLeavesStmt; /* Valid statements in aLeavesStmt */
- int nLeavesTotal; /* Total number of prepared leaves stmts */
- int nLeavesAlloc; /* Allocated size of aLeavesStmt */
- sqlite3_stmt **aLeavesStmt; /* Array of prepared zSelectLeaves stmts */
+ sqlite3_stmt *aStmt[24];
int nNodeSize; /* Soft limit for node size */
- u8 bHasContent; /* True if %_content table exists */
+ u8 bHasStat; /* True if %_stat table exists */
u8 bHasDocsize; /* True if %_docsize table exists */
+ int nPgsz; /* Page size for host database */
+ char *zSegmentsTbl; /* Name of %_segments table */
+ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */
/* The following hash table is used to buffer pending index updates during
** transactions. Variable nPendingData estimates the memory size of the
@@ -107626,14 +108341,25 @@
u8 isRequireSeek; /* True if must seek pStmt to %_content row */
sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */
Fts3Expr *pExpr; /* Parsed MATCH query string */
+ int nPhrase; /* Number of matchable phrases in query */
+ Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */
sqlite3_int64 iPrevId; /* Previous id read from aDoclist */
char *pNextId; /* Pointer into the body of aDoclist */
char *aDoclist; /* List of docids for full-text queries */
int nDoclist; /* Size of buffer at aDoclist */
+ int eEvalmode; /* An FTS3_EVAL_XX constant */
+ int nRowAvg; /* Average size of database rows, in pages */
+
int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */
u32 *aMatchinfo; /* Information about most recent match */
+ int nMatchinfo; /* Number of elements in aMatchinfo[] */
+ char *zMatchinfo; /* Matchinfo specification */
};
+#define FTS3_EVAL_FILTER 0
+#define FTS3_EVAL_NEXT 1
+#define FTS3_EVAL_MATCHINFO 2
+
/*
** The Fts3Cursor.eSearch member is always set to one of the following.
** Actualy, Fts3Cursor.eSearch can be greater than or equal to
@@ -107656,18 +108382,30 @@
/*
** A "phrase" is a sequence of one or more tokens that must match in
** sequence. A single token is the base case and the most common case.
-** For a sequence of tokens contained in "...", nToken will be the number
-** of tokens in the string.
+** For a sequence of tokens contained in double-quotes (i.e. "one two three")
+** nToken will be the number of tokens in the string.
+**
+** The nDocMatch and nMatch variables contain data that may be used by the
+** matchinfo() function. They are populated when the full-text index is
+** queried for hits on the phrase. If one or more tokens in the phrase
+** are deferred, the nDocMatch and nMatch variables are populated based
+** on the assumption that the
*/
+struct Fts3PhraseToken {
+ char *z; /* Text of the token */
+ int n; /* Number of bytes in buffer z */
+ int isPrefix; /* True if token ends with a "*" character */
+ int bFulltext; /* True if full-text index was used */
+ Fts3SegReaderArray *pArray; /* Segment-reader for this token */
+ Fts3DeferredToken *pDeferred; /* Deferred token object for this token */
+};
+
struct Fts3Phrase {
+ /* Variables populated by fts3_expr.c when parsing a MATCH expression */
int nToken; /* Number of tokens in the phrase */
int iColumn; /* Index of column this phrase must match */
int isNot; /* Phrase prefixed by unary not (-) operator */
- struct PhraseToken {
- char *z; /* Text of the token */
- int n; /* Number of bytes in buffer pointed to by z */
- int isPrefix; /* True if token ends in with a "*" character */
- } aToken[1]; /* One entry for each token in the phrase */
+ Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */
};
/*
@@ -107717,29 +108455,34 @@
#define FTSQUERY_PHRASE 5
-/* fts3_init.c */
-SQLITE_PRIVATE int sqlite3Fts3DeleteVtab(int, sqlite3_vtab *);
-SQLITE_PRIVATE int sqlite3Fts3InitVtab(int, sqlite3*, void*, int, const char*const*,
- sqlite3_vtab **, char **);
-
/* fts3_write.c */
SQLITE_PRIVATE int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*);
SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *);
SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *);
SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *);
-SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(Fts3Table *,int, sqlite3_int64,
+SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(int, sqlite3_int64,
sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
SQLITE_PRIVATE int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**);
-SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3Table *, Fts3SegReader *);
+SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *);
SQLITE_PRIVATE int sqlite3Fts3SegReaderIterate(
Fts3Table *, Fts3SegReader **, int, Fts3SegFilter *,
int (*)(Fts3Table *, void *, char *, int, char *, int), void *
);
-SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char const**, int*);
+SQLITE_PRIVATE int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *);
SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **);
-SQLITE_PRIVATE int sqlite3Fts3MatchinfoDocsizeLocal(Fts3Cursor*, u32*);
-SQLITE_PRIVATE int sqlite3Fts3MatchinfoDocsizeGlobal(Fts3Cursor*, u32*);
SQLITE_PRIVATE int sqlite3Fts3ReadLock(Fts3Table *);
+SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*);
+
+SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **);
+SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **);
+
+SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *);
+SQLITE_PRIVATE int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int);
+SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *);
+SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *);
+SQLITE_PRIVATE char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *);
+
+SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *);
/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
#define FTS3_SEGMENT_REQUIRE_POS 0x00000001
@@ -107763,22 +108506,24 @@
SQLITE_PRIVATE void sqlite3Fts3Dequote(char *);
SQLITE_PRIVATE char *sqlite3Fts3FindPositions(Fts3Expr *, sqlite3_int64, int);
-SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Table *, Fts3Expr *);
+SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *);
+SQLITE_PRIVATE int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *);
SQLITE_PRIVATE int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int);
/* fts3_tokenizer.c */
SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *);
SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *);
-SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash,
- const char *, sqlite3_tokenizer **, const char **, char **
+SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *,
+ sqlite3_tokenizer **, char **
);
+SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char);
/* fts3_snippet.c */
SQLITE_PRIVATE void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*);
SQLITE_PRIVATE void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *,
const char *, const char *, int, int
);
-SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *);
+SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *);
/* fts3_expr.c */
SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *,
@@ -107927,16 +108672,13 @@
int i;
assert( p->nPendingData==0 );
+ assert( p->pSegments==0 );
/* Free any prepared statements held */
for(i=0; i<SizeofArray(p->aStmt); i++){
sqlite3_finalize(p->aStmt[i]);
}
- for(i=0; i<p->nLeavesStmt; i++){
- sqlite3_finalize(p->aLeavesStmt[i]);
- }
- sqlite3_free(p->zSelectLeaves);
- sqlite3_free(p->aLeavesStmt);
+ sqlite3_free(p->zSegmentsTbl);
/* Invoke the tokenizer destructor to free the tokenizer. */
p->pTokenizer->pModule->xDestroy(p->pTokenizer);
@@ -107947,7 +108689,7 @@
/*
** Construct one or more SQL statements from the format string given
-** and then evaluate those statements. The success code is writting
+** and then evaluate those statements. The success code is written
** into *pRc.
**
** If *pRc is initially non-zero then this routine is a no-op.
@@ -107999,33 +108741,38 @@
** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table
** passed as the first argument. This is done as part of the xConnect()
** and xCreate() methods.
+**
+** If *pRc is non-zero when this function is called, it is a no-op.
+** Otherwise, if an error occurs, an SQLite error code is stored in *pRc
+** before returning.
*/
-static int fts3DeclareVtab(Fts3Table *p){
- int i; /* Iterator variable */
- int rc; /* Return code */
- char *zSql; /* SQL statement passed to declare_vtab() */
- char *zCols; /* List of user defined columns */
+static void fts3DeclareVtab(int *pRc, Fts3Table *p){
+ if( *pRc==SQLITE_OK ){
+ int i; /* Iterator variable */
+ int rc; /* Return code */
+ char *zSql; /* SQL statement passed to declare_vtab() */
+ char *zCols; /* List of user defined columns */
- /* Create a list of user columns for the virtual table */
- zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]);
- for(i=1; zCols && i<p->nColumn; i++){
- zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]);
+ /* Create a list of user columns for the virtual table */
+ zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]);
+ for(i=1; zCols && i<p->nColumn; i++){
+ zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]);
+ }
+
+ /* Create the whole "CREATE TABLE" statement to pass to SQLite */
+ zSql = sqlite3_mprintf(
+ "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName
+ );
+ if( !zCols || !zSql ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_declare_vtab(p->db, zSql);
+ }
+
+ sqlite3_free(zSql);
+ sqlite3_free(zCols);
+ *pRc = rc;
}
-
- /* Create the whole "CREATE TABLE" statement to pass to SQLite */
- zSql = sqlite3_mprintf(
- "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName
- );
-
- if( !zCols || !zSql ){
- rc = SQLITE_NOMEM;
- }else{
- rc = sqlite3_declare_vtab(p->db, zSql);
- }
-
- sqlite3_free(zSql);
- sqlite3_free(zCols);
- return rc;
}
/*
@@ -108044,21 +108791,19 @@
sqlite3 *db = p->db; /* The database connection */
/* Create a list of user columns for the content table */
- if( p->bHasContent ){
- zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY");
- for(i=0; zContentCols && i<p->nColumn; i++){
- char *z = p->azColumn[i];
- zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z);
- }
- if( zContentCols==0 ) rc = SQLITE_NOMEM;
-
- /* Create the content table */
- fts3DbExec(&rc, db,
- "CREATE TABLE %Q.'%q_content'(%s)",
- p->zDb, p->zName, zContentCols
- );
- sqlite3_free(zContentCols);
+ zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY");
+ for(i=0; zContentCols && i<p->nColumn; i++){
+ char *z = p->azColumn[i];
+ zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z);
}
+ if( zContentCols==0 ) rc = SQLITE_NOMEM;
+
+ /* Create the content table */
+ fts3DbExec(&rc, db,
+ "CREATE TABLE %Q.'%q_content'(%s)",
+ p->zDb, p->zName, zContentCols
+ );
+ sqlite3_free(zContentCols);
/* Create other tables */
fts3DbExec(&rc, db,
"CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);",
@@ -108081,6 +108826,8 @@
"CREATE TABLE %Q.'%q_docsize'(docid INTEGER PRIMARY KEY, size BLOB);",
p->zDb, p->zName
);
+ }
+ if( p->bHasStat ){
fts3DbExec(&rc, db,
"CREATE TABLE %Q.'%q_stat'(id INTEGER PRIMARY KEY, value BLOB);",
p->zDb, p->zName
@@ -108090,39 +108837,63 @@
}
/*
-** An sqlite3_exec() callback for fts3TableExists.
+** Store the current database page-size in bytes in p->nPgsz.
+**
+** If *pRc is non-zero when this function is called, it is a no-op.
+** Otherwise, if an error occurs, an SQLite error code is stored in *pRc
+** before returning.
*/
-static int fts3TableExistsCallback(void *pArg, int n, char **pp1, char **pp2){
- UNUSED_PARAMETER(n);
- UNUSED_PARAMETER(pp1);
- UNUSED_PARAMETER(pp2);
- *(int*)pArg = 1;
- return 1;
+static void fts3DatabasePageSize(int *pRc, Fts3Table *p){
+ if( *pRc==SQLITE_OK ){
+ int rc; /* Return code */
+ char *zSql; /* SQL text "PRAGMA %Q.page_size" */
+ sqlite3_stmt *pStmt; /* Compiled "PRAGMA %Q.page_size" statement */
+
+ zSql = sqlite3_mprintf("PRAGMA %Q.page_size", p->zDb);
+ if( !zSql ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_step(pStmt);
+ p->nPgsz = sqlite3_column_int(pStmt, 0);
+ rc = sqlite3_finalize(pStmt);
+ }
+ }
+ assert( p->nPgsz>0 || rc!=SQLITE_OK );
+ sqlite3_free(zSql);
+ *pRc = rc;
+ }
}
/*
-** Determine if a table currently exists in the database.
+** "Special" FTS4 arguments are column specifications of the following form:
+**
+** <key> = <value>
+**
+** There may not be whitespace surrounding the "=" character. The <value>
+** term may be quoted, but the <key> may not.
*/
-static void fts3TableExists(
- int *pRc, /* Success code */
- sqlite3 *db, /* The database connection to test */
- const char *zDb, /* ATTACHed database within the connection */
- const char *zName, /* Name of the FTS3 table */
- const char *zSuffix, /* Shadow table extension */
- u8 *pResult /* Write results here */
+static int fts3IsSpecialColumn(
+ const char *z,
+ int *pnKey,
+ char **pzValue
){
- int rc = SQLITE_OK;
- int res = 0;
- char *zSql;
- if( *pRc ) return;
- zSql = sqlite3_mprintf(
- "SELECT 1 FROM %Q.sqlite_master WHERE name='%q%s'",
- zDb, zName, zSuffix
- );
- rc = sqlite3_exec(db, zSql, fts3TableExistsCallback, &res, 0);
- sqlite3_free(zSql);
- *pResult = (u8)(res & 0xff);
- if( rc!=SQLITE_ABORT ) *pRc = rc;
+ char *zValue;
+ const char *zCsr = z;
+
+ while( *zCsr!='=' ){
+ if( *zCsr=='\0' ) return 0;
+ zCsr++;
+ }
+
+ *pnKey = (int)(zCsr-z);
+ zValue = sqlite3_mprintf("%s", &zCsr[1]);
+ if( zValue ){
+ sqlite3Fts3Dequote(zValue);
+ }
+ *pzValue = zValue;
+ return 1;
}
/*
@@ -108146,8 +108917,8 @@
char **pzErr /* Write any error message here */
){
Fts3Hash *pHash = (Fts3Hash *)pAux;
- Fts3Table *p; /* Pointer to allocated vtab */
- int rc; /* Return code */
+ Fts3Table *p = 0; /* Pointer to allocated vtab */
+ int rc = SQLITE_OK; /* Return code */
int i; /* Iterator variable */
int nByte; /* Size of allocation used for *p */
int iCol; /* Column index */
@@ -108156,35 +108927,90 @@
char *zCsr; /* Space for holding column names */
int nDb; /* Bytes required to hold database name */
int nName; /* Bytes required to hold table name */
-
- const char *zTokenizer = 0; /* Name of tokenizer to use */
+ int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */
+ int bNoDocsize = 0; /* True to omit %_docsize table */
+ const char **aCol; /* Array of column names */
sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */
+ assert( strlen(argv[0])==4 );
+ assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4)
+ || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4)
+ );
+
nDb = (int)strlen(argv[1]) + 1;
nName = (int)strlen(argv[2]) + 1;
- for(i=3; i<argc; i++){
+
+ aCol = (const char **)sqlite3_malloc(sizeof(const char *) * (argc-2) );
+ if( !aCol ) return SQLITE_NOMEM;
+ memset((void *)aCol, 0, sizeof(const char *) * (argc-2));
+
+ /* Loop through all of the arguments passed by the user to the FTS3/4
+ ** module (i.e. all the column names and special arguments). This loop
+ ** does the following:
+ **
+ ** + Figures out the number of columns the FTSX table will have, and
+ ** the number of bytes of space that must be allocated to store copies
+ ** of the column names.
+ **
+ ** + If there is a tokenizer specification included in the arguments,
+ ** initializes the tokenizer pTokenizer.
+ */
+ for(i=3; rc==SQLITE_OK && i<argc; i++){
char const *z = argv[i];
- rc = sqlite3Fts3InitTokenizer(pHash, z, &pTokenizer, &zTokenizer, pzErr);
- if( rc!=SQLITE_OK ){
- return rc;
+ int nKey;
+ char *zVal;
+
+ /* Check if this is a tokenizer specification */
+ if( !pTokenizer
+ && strlen(z)>8
+ && 0==sqlite3_strnicmp(z, "tokenize", 8)
+ && 0==sqlite3Fts3IsIdChar(z[8])
+ ){
+ rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr);
}
- if( z!=zTokenizer ){
+
+ /* Check if it is an FTS4 special argument. */
+ else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){
+ if( !zVal ){
+ rc = SQLITE_NOMEM;
+ goto fts3_init_out;
+ }
+ if( nKey==9 && 0==sqlite3_strnicmp(z, "matchinfo", 9) ){
+ if( strlen(zVal)==4 && 0==sqlite3_strnicmp(zVal, "fts3", 4) ){
+ bNoDocsize = 1;
+ }else{
+ *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal);
+ rc = SQLITE_ERROR;
+ }
+ }else{
+ *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z);
+ rc = SQLITE_ERROR;
+ }
+ sqlite3_free(zVal);
+ }
+
+ /* Otherwise, the argument is a column name. */
+ else {
nString += (int)(strlen(z) + 1);
+ aCol[nCol++] = z;
}
}
- nCol = argc - 3 - (zTokenizer!=0);
- if( zTokenizer==0 ){
- rc = sqlite3Fts3InitTokenizer(pHash, 0, &pTokenizer, 0, pzErr);
- if( rc!=SQLITE_OK ){
- return rc;
- }
- assert( pTokenizer );
- }
+ if( rc!=SQLITE_OK ) goto fts3_init_out;
if( nCol==0 ){
+ assert( nString==0 );
+ aCol[0] = "content";
+ nString = 8;
nCol = 1;
}
+ if( pTokenizer==0 ){
+ rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr);
+ if( rc!=SQLITE_OK ) goto fts3_init_out;
+ }
+ assert( pTokenizer );
+
+
/* Allocate and populate the Fts3Table structure. */
nByte = sizeof(Fts3Table) + /* Fts3Table */
nCol * sizeof(char *) + /* azColumn */
@@ -108197,7 +109023,6 @@
goto fts3_init_out;
}
memset(p, 0, nByte);
-
p->db = db;
p->nColumn = nCol;
p->nPendingData = 0;
@@ -108205,11 +109030,12 @@
p->pTokenizer = pTokenizer;
p->nNodeSize = 1000;
p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
- zCsr = (char *)&p->azColumn[nCol];
-
+ p->bHasDocsize = (isFts4 && bNoDocsize==0);
+ p->bHasStat = isFts4;
fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1);
/* Fill in the zName and zDb fields of the vtab structure. */
+ zCsr = (char *)&p->azColumn[nCol];
p->zName = zCsr;
memcpy(zCsr, argv[2], nName);
zCsr += nName;
@@ -108218,52 +109044,45 @@
zCsr += nDb;
/* Fill in the azColumn array */
- iCol = 0;
- for(i=3; i<argc; i++){
- if( argv[i]!=zTokenizer ){
- char *z;
- int n;
- z = (char *)sqlite3Fts3NextToken(argv[i], &n);
- memcpy(zCsr, z, n);
- zCsr[n] = '\0';
- sqlite3Fts3Dequote(zCsr);
- p->azColumn[iCol++] = zCsr;
- zCsr += n+1;
- assert( zCsr <= &((char *)p)[nByte] );
- }
- }
- if( iCol==0 ){
- assert( nCol==1 );
- p->azColumn[0] = "content";
+ for(iCol=0; iCol<nCol; iCol++){
+ char *z;
+ int n;
+ z = (char *)sqlite3Fts3NextToken(aCol[iCol], &n);
+ memcpy(zCsr, z, n);
+ zCsr[n] = '\0';
+ sqlite3Fts3Dequote(zCsr);
+ p->azColumn[iCol] = zCsr;
+ zCsr += n+1;
+ assert( zCsr <= &((char *)p)[nByte] );
}
/* If this is an xCreate call, create the underlying tables in the
** database. TODO: For xConnect(), it could verify that said tables exist.
*/
if( isCreate ){
- p->bHasContent = 1;
- p->bHasDocsize = argv[0][3]=='4';
rc = fts3CreateTables(p);
- }else{
- rc = SQLITE_OK;
- fts3TableExists(&rc, db, argv[1], argv[2], "_content", &p->bHasContent);
- fts3TableExists(&rc, db, argv[1], argv[2], "_docsize", &p->bHasDocsize);
}
- if( rc!=SQLITE_OK ) goto fts3_init_out;
- rc = fts3DeclareVtab(p);
- if( rc!=SQLITE_OK ) goto fts3_init_out;
+ /* Figure out the page-size for the database. This is required in order to
+ ** estimate the cost of loading large doclists from the database (see
+ ** function sqlite3Fts3SegReaderCost() for details).
+ */
+ fts3DatabasePageSize(&rc, p);
- *ppVTab = &p->base;
+ /* Declare the table schema to SQLite. */
+ fts3DeclareVtab(&rc, p);
fts3_init_out:
- assert( p || (pTokenizer && rc!=SQLITE_OK) );
+
+ sqlite3_free((void *)aCol);
if( rc!=SQLITE_OK ){
if( p ){
fts3DisconnectMethod((sqlite3_vtab *)p);
- }else{
+ }else if( pTokenizer ){
pTokenizer->pModule->xDestroy(pTokenizer);
}
+ }else{
+ *ppVTab = &p->base;
}
return rc;
}
@@ -108375,10 +109194,12 @@
** Close the cursor. For additional information see the documentation
** on the xClose method of the virtual table interface.
*/
-static int fulltextClose(sqlite3_vtab_cursor *pCursor){
+static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
+ assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
sqlite3_finalize(pCsr->pStmt);
sqlite3Fts3ExprFree(pCsr->pExpr);
+ sqlite3Fts3FreeDeferredTokens(pCsr);
sqlite3_free(pCsr->aDoclist);
sqlite3_free(pCsr->aMatchinfo);
sqlite3_free(pCsr);
@@ -108417,50 +109238,137 @@
}
/*
-** Advance the cursor to the next row in the %_content table that
-** matches the search criteria. For a MATCH search, this will be
-** the next row that matches. For a full-table scan, this will be
-** simply the next row in the %_content table. For a docid lookup,
-** this routine simply sets the EOF flag.
+** This function is used to process a single interior node when searching
+** a b-tree for a term or term prefix. The node data is passed to this
+** function via the zNode/nNode parameters. The term to search for is
+** passed in zTerm/nTerm.
**
-** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned
-** even if we reach end-of-file. The fts3EofMethod() will be called
-** subsequently to determine whether or not an EOF was hit.
+** If piFirst is not NULL, then this function sets *piFirst to the blockid
+** of the child node that heads the sub-tree that may contain the term.
+**
+** If piLast is not NULL, then *piLast is set to the right-most child node
+** that heads a sub-tree that may contain a term for which zTerm/nTerm is
+** a prefix.
+**
+** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK.
*/
-static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
+static int fts3ScanInteriorNode(
+ const char *zTerm, /* Term to select leaves for */
+ int nTerm, /* Size of term zTerm in bytes */
+ const char *zNode, /* Buffer containing segment interior node */
+ int nNode, /* Size of buffer at zNode */
+ sqlite3_int64 *piFirst, /* OUT: Selected child node */
+ sqlite3_int64 *piLast /* OUT: Selected child node */
+){
int rc = SQLITE_OK; /* Return code */
- Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
+ const char *zCsr = zNode; /* Cursor to iterate through node */
+ const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
+ char *zBuffer = 0; /* Buffer to load terms into */
+ int nAlloc = 0; /* Size of allocated buffer */
+ int isFirstTerm = 1; /* True when processing first term on page */
+ sqlite3_int64 iChild; /* Block id of child node to descend to */
- if( pCsr->aDoclist==0 ){
- if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
- pCsr->isEof = 1;
- rc = sqlite3_reset(pCsr->pStmt);
- }
- }else if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){
- pCsr->isEof = 1;
- }else{
- sqlite3_reset(pCsr->pStmt);
- fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId);
- pCsr->isRequireSeek = 1;
- pCsr->isMatchinfoNeeded = 1;
+ /* Skip over the 'height' varint that occurs at the start of every
+ ** interior node. Then load the blockid of the left-child of the b-tree
+ ** node into variable iChild.
+ **
+ ** Even if the data structure on disk is corrupted, this (reading two
+ ** varints from the buffer) does not risk an overread. If zNode is a
+ ** root node, then the buffer comes from a SELECT statement. SQLite does
+ ** not make this guarantee explicitly, but in practice there are always
+ ** either more than 20 bytes of allocated space following the nNode bytes of
+ ** contents, or two zero bytes. Or, if the node is read from the %_segments
+ ** table, then there are always 20 bytes of zeroed padding following the
+ ** nNode bytes of content (see sqlite3Fts3ReadBlock() for details).
+ */
+ zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
+ zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
+ if( zCsr>zEnd ){
+ return SQLITE_CORRUPT;
}
+
+ while( zCsr<zEnd && (piFirst || piLast) ){
+ int cmp; /* memcmp() result */
+ int nSuffix; /* Size of term suffix */
+ int nPrefix = 0; /* Size of term prefix */
+ int nBuffer; /* Total term size */
+
+ /* Load the next term on the node into zBuffer. Use realloc() to expand
+ ** the size of zBuffer if required. */
+ if( !isFirstTerm ){
+ zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix);
+ }
+ isFirstTerm = 0;
+ zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix);
+
+ if( nPrefix<0 || nSuffix<0 || &zCsr[nSuffix]>zEnd ){
+ rc = SQLITE_CORRUPT;
+ goto finish_scan;
+ }
+ if( nPrefix+nSuffix>nAlloc ){
+ char *zNew;
+ nAlloc = (nPrefix+nSuffix) * 2;
+ zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
+ if( !zNew ){
+ rc = SQLITE_NOMEM;
+ goto finish_scan;
+ }
+ zBuffer = zNew;
+ }
+ memcpy(&zBuffer[nPrefix], zCsr, nSuffix);
+ nBuffer = nPrefix + nSuffix;
+ zCsr += nSuffix;
+
+ /* Compare the term we are searching for with the term just loaded from
+ ** the interior node. If the specified term is greater than or equal
+ ** to the term from the interior node, then all terms on the sub-tree
+ ** headed by node iChild are smaller than zTerm. No need to search
+ ** iChild.
+ **
+ ** If the interior node term is larger than the specified term, then
+ ** the tree headed by iChild may contain the specified term.
+ */
+ cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer));
+ if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){
+ *piFirst = iChild;
+ piFirst = 0;
+ }
+
+ if( piLast && cmp<0 ){
+ *piLast = iChild;
+ piLast = 0;
+ }
+
+ iChild++;
+ };
+
+ if( piFirst ) *piFirst = iChild;
+ if( piLast ) *piLast = iChild;
+
+ finish_scan:
+ sqlite3_free(zBuffer);
return rc;
}
/*
-** The buffer pointed to by argument zNode (size nNode bytes) contains the
-** root node of a b-tree segment. The segment is guaranteed to be at least
-** one level high (i.e. the root node is not also a leaf). If successful,
-** this function locates the leaf node of the segment that may contain the
-** term specified by arguments zTerm and nTerm and writes its block number
-** to *piLeaf.
+** The buffer pointed to by argument zNode (size nNode bytes) contains an
+** interior node of a b-tree segment. The zTerm buffer (size nTerm bytes)
+** contains a term. This function searches the sub-tree headed by the zNode
+** node for the range of leaf nodes that may contain the specified term
+** or terms for which the specified term is a prefix.
**
-** It is possible that the returned leaf node does not contain the specified
-** term. However, if the segment does contain said term, it is stored on
-** the identified leaf node. Because this function only inspects interior
-** segment nodes (and never loads leaf nodes into memory), it is not possible
-** to be sure.
+** If piLeaf is not NULL, then *piLeaf is set to the blockid of the
+** left-most leaf node in the tree that may contain the specified term.
+** If piLeaf2 is not NULL, then *piLeaf2 is set to the blockid of the
+** right-most leaf node that may contain a term for which the specified
+** term is a prefix.
+**
+** It is possible that the range of returned leaf nodes does not contain
+** the specified term or any terms for which it is a prefix. However, if the
+** segment does contain any such terms, they are stored within the identified
+** range. Because this function only inspects interior segment nodes (and
+** never loads leaf nodes into memory), it is not possible to be sure.
**
** If an error occurs, an error code other than SQLITE_OK is returned.
*/
@@ -108470,77 +109378,41 @@
int nTerm, /* Size of term zTerm in bytes */
const char *zNode, /* Buffer containing segment interior node */
int nNode, /* Size of buffer at zNode */
- sqlite3_int64 *piLeaf /* Selected leaf node */
+ sqlite3_int64 *piLeaf, /* Selected leaf node */
+ sqlite3_int64 *piLeaf2 /* Selected leaf node */
){
- int rc = SQLITE_OK; /* Return code */
- const char *zCsr = zNode; /* Cursor to iterate through node */
- const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
- char *zBuffer = 0; /* Buffer to load terms into */
- int nAlloc = 0; /* Size of allocated buffer */
+ int rc; /* Return code */
+ int iHeight; /* Height of this node in tree */
- while( 1 ){
- int isFirstTerm = 1; /* True when processing first term on page */
- int iHeight; /* Height of this node in tree */
- sqlite3_int64 iChild; /* Block id of child node to descend to */
- int nBlock; /* Size of child node in bytes */
+ assert( piLeaf || piLeaf2 );
- zCsr += sqlite3Fts3GetVarint32(zCsr, &iHeight);
- zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
-
- while( zCsr<zEnd ){
- int cmp; /* memcmp() result */
- int nSuffix; /* Size of term suffix */
- int nPrefix = 0; /* Size of term prefix */
- int nBuffer; /* Total term size */
-
- /* Load the next term on the node into zBuffer */
- if( !isFirstTerm ){
- zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix);
+ sqlite3Fts3GetVarint32(zNode, &iHeight);
+ rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2);
+ assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) );
+
+ if( rc==SQLITE_OK && iHeight>1 ){
+ char *zBlob = 0; /* Blob read from %_segments table */
+ int nBlob; /* Size of zBlob in bytes */
+
+ if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){
+ rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob);
+ if( rc==SQLITE_OK ){
+ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0);
}
- isFirstTerm = 0;
- zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix);
- if( nPrefix+nSuffix>nAlloc ){
- char *zNew;
- nAlloc = (nPrefix+nSuffix) * 2;
- zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
- if( !zNew ){
- sqlite3_free(zBuffer);
- return SQLITE_NOMEM;
- }
- zBuffer = zNew;
- }
- memcpy(&zBuffer[nPrefix], zCsr, nSuffix);
- nBuffer = nPrefix + nSuffix;
- zCsr += nSuffix;
-
- /* Compare the term we are searching for with the term just loaded from
- ** the interior node. If the specified term is greater than or equal
- ** to the term from the interior node, then all terms on the sub-tree
- ** headed by node iChild are smaller than zTerm. No need to search
- ** iChild.
- **
- ** If the interior node term is larger than the specified term, then
- ** the tree headed by iChild may contain the specified term.
- */
- cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer));
- if( cmp<0 || (cmp==0 && nBuffer>nTerm) ) break;
- iChild++;
- };
-
- /* If (iHeight==1), the children of this interior node are leaves. The
- ** specified term may be present on leaf node iChild.
- */
- if( iHeight==1 ){
- *piLeaf = iChild;
- break;
+ sqlite3_free(zBlob);
+ piLeaf = 0;
+ zBlob = 0;
}
- /* Descend to interior node iChild. */
- rc = sqlite3Fts3ReadBlock(p, iChild, &zCsr, &nBlock);
- if( rc!=SQLITE_OK ) break;
- zEnd = &zCsr[nBlock];
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts3ReadBlock(p, piLeaf ? *piLeaf : *piLeaf2, &zBlob, &nBlob);
+ }
+ if( rc==SQLITE_OK ){
+ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2);
+ }
+ sqlite3_free(zBlob);
}
- sqlite3_free(zBuffer);
+
return rc;
}
@@ -108772,20 +109644,44 @@
/*
** nToken==1 searches for adjacent positions.
+**
+** This function is used to merge two position lists into one. When it is
+** called, *pp1 and *pp2 must both point to position lists. A position-list is
+** the part of a doclist that follows each document id. For example, if a row
+** contains:
+**
+** 'a b c'|'x y z'|'a b b a'
+**
+** Then the position list for this row for token 'b' would consist of:
+**
+** 0x02 0x01 0x02 0x03 0x03 0x00
+**
+** When this function returns, both *pp1 and *pp2 are left pointing to the
+** byte following the 0x00 terminator of their respective position lists.
+**
+** If isSaveLeft is 0, an entry is added to the output position list for
+** each position in *pp2 for which there exists one or more positions in
+** *pp1 so that (pos(*pp2)>pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e.
+** when the *pp1 token appears before the *pp2 token, but not more than nToken
+** slots before it.
*/
static int fts3PoslistPhraseMerge(
- char **pp, /* Output buffer */
+ char **pp, /* IN/OUT: Preallocated output buffer */
int nToken, /* Maximum difference in token positions */
int isSaveLeft, /* Save the left position */
- char **pp1, /* Left input list */
- char **pp2 /* Right input list */
+ int isExact, /* If *pp1 is exactly nTokens before *pp2 */
+ char **pp1, /* IN/OUT: Left input list */
+ char **pp2 /* IN/OUT: Right input list */
){
char *p = (pp ? *pp : 0);
char *p1 = *pp1;
char *p2 = *pp2;
-
int iCol1 = 0;
int iCol2 = 0;
+
+ /* Never set both isSaveLeft and isExact for the same invocation. */
+ assert( isSaveLeft==0 || isExact==0 );
+
assert( *p1!=0 && *p2!=0 );
if( *p1==POS_COLUMN ){
p1++;
@@ -108814,7 +109710,9 @@
fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2;
while( 1 ){
- if( iPos2>iPos1 && iPos2<=iPos1+nToken ){
+ if( iPos2==iPos1+nToken
+ || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken)
+ ){
sqlite3_int64 iSave;
if( !pp ){
fts3PoslistCopy(0, &p2);
@@ -108897,21 +109795,21 @@
char *p2 = *pp2;
if( !pp ){
- if( fts3PoslistPhraseMerge(0, nRight, 0, pp1, pp2) ) return 1;
+ if( fts3PoslistPhraseMerge(0, nRight, 0, 0, pp1, pp2) ) return 1;
*pp1 = p1;
*pp2 = p2;
- return fts3PoslistPhraseMerge(0, nLeft, 0, pp2, pp1);
+ return fts3PoslistPhraseMerge(0, nLeft, 0, 0, pp2, pp1);
}else{
char *pTmp1 = aTmp;
char *pTmp2;
char *aTmp2;
int res = 1;
- fts3PoslistPhraseMerge(&pTmp1, nRight, 0, pp1, pp2);
+ fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2);
aTmp2 = pTmp2 = pTmp1;
*pp1 = p1;
*pp2 = p2;
- fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, pp2, pp1);
+ fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1);
if( pTmp1!=aTmp && pTmp2!=aTmp2 ){
fts3PoslistMerge(pp, &aTmp, &aTmp2);
}else if( pTmp1!=aTmp ){
@@ -108957,7 +109855,8 @@
char *a1, /* Buffer containing first doclist */
int n1, /* Size of buffer a1 */
char *a2, /* Buffer containing second doclist */
- int n2 /* Size of buffer a2 */
+ int n2, /* Size of buffer a2 */
+ int *pnDoc /* OUT: Number of docids in output */
){
sqlite3_int64 i1 = 0;
sqlite3_int64 i2 = 0;
@@ -108968,6 +109867,7 @@
char *p2 = a2;
char *pEnd1 = &a1[n1];
char *pEnd2 = &a2[n2];
+ int nDoc = 0;
assert( mergetype==MERGE_OR || mergetype==MERGE_POS_OR
|| mergetype==MERGE_AND || mergetype==MERGE_NOT
@@ -109011,6 +109911,7 @@
fts3PutDeltaVarint(&p, &iPrev, i1);
fts3GetDeltaVarint2(&p1, pEnd1, &i1);
fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+ nDoc++;
}else if( i1<i2 ){
fts3GetDeltaVarint2(&p1, pEnd1, &i1);
}else{
@@ -109041,9 +109942,11 @@
char *pSave = p;
sqlite3_int64 iPrevSave = iPrev;
fts3PutDeltaVarint(&p, &iPrev, i1);
- if( 0==fts3PoslistPhraseMerge(ppPos, 1, 0, &p1, &p2) ){
+ if( 0==fts3PoslistPhraseMerge(ppPos, nParam1, 0, 1, &p1, &p2) ){
p = pSave;
iPrev = iPrevSave;
+ }else{
+ nDoc++;
}
fts3GetDeltaVarint2(&p1, pEnd1, &i1);
fts3GetDeltaVarint2(&p2, pEnd2, &i2);
@@ -109096,6 +109999,7 @@
}
}
+ if( pnDoc ) *pnDoc = nDoc;
*pnBuffer = (int)(p-aBuffer);
return SQLITE_OK;
}
@@ -109134,7 +110038,7 @@
if( !aOut ){
aOut = pTS->aaOutput[i];
nOut = pTS->anOutput[i];
- pTS->aaOutput[0] = 0;
+ pTS->aaOutput[i] = 0;
}else{
int nNew = nOut + pTS->anOutput[i];
char *aNew = sqlite3_malloc(nNew);
@@ -109143,7 +110047,7 @@
return SQLITE_NOMEM;
}
fts3DoclistMerge(mergetype, 0, 0,
- aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut
+ aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, 0
);
sqlite3_free(pTS->aaOutput[i]);
sqlite3_free(aOut);
@@ -109214,8 +110118,8 @@
}
return SQLITE_NOMEM;
}
- fts3DoclistMerge(mergetype, 0, 0,
- aNew, &nNew, pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge
+ fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew,
+ pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge, 0
);
if( iOut>0 ) sqlite3_free(aMerge);
@@ -109233,6 +110137,161 @@
return SQLITE_OK;
}
+static int fts3DeferredTermSelect(
+ Fts3DeferredToken *pToken, /* Phrase token */
+ int isTermPos, /* True to include positions */
+ int *pnOut, /* OUT: Size of list */
+ char **ppOut /* OUT: Body of list */
+){
+ char *aSource;
+ int nSource;
+
+ aSource = sqlite3Fts3DeferredDoclist(pToken, &nSource);
+ if( !aSource ){
+ *pnOut = 0;
+ *ppOut = 0;
+ }else if( isTermPos ){
+ *ppOut = sqlite3_malloc(nSource);
+ if( !*ppOut ) return SQLITE_NOMEM;
+ memcpy(*ppOut, aSource, nSource);
+ *pnOut = nSource;
+ }else{
+ sqlite3_int64 docid;
+ *pnOut = sqlite3Fts3GetVarint(aSource, &docid);
+ *ppOut = sqlite3_malloc(*pnOut);
+ if( !*ppOut ) return SQLITE_NOMEM;
+ sqlite3Fts3PutVarint(*ppOut, docid);
+ }
+
+ return SQLITE_OK;
+}
+
+/*
+** An Fts3SegReaderArray is used to store an array of Fts3SegReader objects.
+** Elements are added to the array using fts3SegReaderArrayAdd().
+*/
+struct Fts3SegReaderArray {
+ int nSegment; /* Number of valid entries in apSegment[] */
+ int nAlloc; /* Allocated size of apSegment[] */
+ int nCost; /* The cost of executing SegReaderIterate() */
+ Fts3SegReader *apSegment[1]; /* Array of seg-reader objects */
+};
+
+
+/*
+** Free an Fts3SegReaderArray object. Also free all seg-readers in the
+** array (using sqlite3Fts3SegReaderFree()).
+*/
+static void fts3SegReaderArrayFree(Fts3SegReaderArray *pArray){
+ if( pArray ){
+ int i;
+ for(i=0; i<pArray->nSegment; i++){
+ sqlite3Fts3SegReaderFree(pArray->apSegment[i]);
+ }
+ sqlite3_free(pArray);
+ }
+}
+
+static int fts3SegReaderArrayAdd(
+ Fts3SegReaderArray **ppArray,
+ Fts3SegReader *pNew
+){
+ Fts3SegReaderArray *pArray = *ppArray;
+
+ if( !pArray || pArray->nAlloc==pArray->nSegment ){
+ int nNew = (pArray ? pArray->nAlloc+16 : 16);
+ pArray = (Fts3SegReaderArray *)sqlite3_realloc(pArray,
+ sizeof(Fts3SegReaderArray) + (nNew-1) * sizeof(Fts3SegReader*)
+ );
+ if( !pArray ){
+ sqlite3Fts3SegReaderFree(pNew);
+ return SQLITE_NOMEM;
+ }
+ if( nNew==16 ){
+ pArray->nSegment = 0;
+ pArray->nCost = 0;
+ }
+ pArray->nAlloc = nNew;
+ *ppArray = pArray;
+ }
+
+ pArray->apSegment[pArray->nSegment++] = pNew;
+ return SQLITE_OK;
+}
+
+static int fts3TermSegReaderArray(
+ Fts3Cursor *pCsr, /* Virtual table cursor handle */
+ const char *zTerm, /* Term to query for */
+ int nTerm, /* Size of zTerm in bytes */
+ int isPrefix, /* True for a prefix search */
+ Fts3SegReaderArray **ppArray /* OUT: Allocated seg-reader array */
+){
+ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
+ int rc; /* Return code */
+ Fts3SegReaderArray *pArray = 0; /* Array object to build */
+ Fts3SegReader *pReader = 0; /* Seg-reader to add to pArray */
+ sqlite3_stmt *pStmt = 0; /* SQL statement to scan %_segdir table */
+ int iAge = 0; /* Used to assign ages to segments */
+
+ /* Allocate a seg-reader to scan the pending terms, if any. */
+ rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &pReader);
+ if( rc==SQLITE_OK && pReader ) {
+ rc = fts3SegReaderArrayAdd(&pArray, pReader);
+ }
+
+ /* Loop through the entire %_segdir table. For each segment, create a
+ ** Fts3SegReader to iterate through the subset of the segment leaves
+ ** that may contain a term that matches zTerm/nTerm. For non-prefix
+ ** searches, this is always a single leaf. For prefix searches, this
+ ** may be a contiguous block of leaves.
+ */
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts3AllSegdirs(p, &pStmt);
+ }
+ while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
+ Fts3SegReader *pNew = 0;
+ int nRoot = sqlite3_column_bytes(pStmt, 4);
+ char const *zRoot = sqlite3_column_blob(pStmt, 4);
+ if( sqlite3_column_int64(pStmt, 1)==0 ){
+ /* The entire segment is stored on the root node (which must be a
+ ** leaf). Do not bother inspecting any data in this case, just
+ ** create a Fts3SegReader to scan the single leaf.
+ */
+ rc = sqlite3Fts3SegReaderNew(iAge, 0, 0, 0, zRoot, nRoot, &pNew);
+ }else{
+ sqlite3_int64 i1; /* First leaf that may contain zTerm */
+ sqlite3_int64 i2; /* Final leaf that may contain zTerm */
+ rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1, (isPrefix?&i2:0));
+ if( isPrefix==0 ) i2 = i1;
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts3SegReaderNew(iAge, i1, i2, 0, 0, 0, &pNew);
+ }
+ }
+ assert( (pNew==0)==(rc!=SQLITE_OK) );
+
+ /* If a new Fts3SegReader was allocated, add it to the array. */
+ if( rc==SQLITE_OK ){
+ rc = fts3SegReaderArrayAdd(&pArray, pNew);
+ }
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts3SegReaderCost(pCsr, pNew, &pArray->nCost);
+ }
+ iAge++;
+ }
+
+ if( rc==SQLITE_DONE ){
+ rc = sqlite3_reset(pStmt);
+ }else{
+ sqlite3_reset(pStmt);
+ }
+ if( rc!=SQLITE_OK ){
+ fts3SegReaderArrayFree(pArray);
+ pArray = 0;
+ }
+ *ppArray = pArray;
+ return rc;
+}
+
/*
** This function retreives the doclist for the specified term (or term
** prefix) from the database.
@@ -109246,112 +110305,31 @@
*/
static int fts3TermSelect(
Fts3Table *p, /* Virtual table handle */
+ Fts3PhraseToken *pTok, /* Token to query for */
int iColumn, /* Column to query (or -ve for all columns) */
- const char *zTerm, /* Term to query for */
- int nTerm, /* Size of zTerm in bytes */
- int isPrefix, /* True for a prefix search */
int isReqPos, /* True to include position lists in output */
int *pnOut, /* OUT: Size of buffer at *ppOut */
char **ppOut /* OUT: Malloced result buffer */
){
- int i;
- TermSelect tsc;
- Fts3SegFilter filter; /* Segment term filter configuration */
- Fts3SegReader **apSegment; /* Array of segments to read data from */
- int nSegment = 0; /* Size of apSegment array */
- int nAlloc = 16; /* Allocated size of segment array */
int rc; /* Return code */
- sqlite3_stmt *pStmt = 0; /* SQL statement to scan %_segdir table */
- int iAge = 0; /* Used to assign ages to segments */
+ Fts3SegReaderArray *pArray; /* Seg-reader array for this term */
+ TermSelect tsc; /* Context object for fts3TermSelectCb() */
+ Fts3SegFilter filter; /* Segment term filter configuration */
- apSegment = (Fts3SegReader **)sqlite3_malloc(sizeof(Fts3SegReader*)*nAlloc);
- if( !apSegment ) return SQLITE_NOMEM;
- rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &apSegment[0]);
- if( rc!=SQLITE_OK ) goto finished;
- if( apSegment[0] ){
- nSegment = 1;
- }
-
- /* Loop through the entire %_segdir table. For each segment, create a
- ** Fts3SegReader to iterate through the subset of the segment leaves
- ** that may contain a term that matches zTerm/nTerm. For non-prefix
- ** searches, this is always a single leaf. For prefix searches, this
- ** may be a contiguous block of leaves.
- **
- ** The code in this loop does not actually load any leaves into memory
- ** (unless the root node happens to be a leaf). It simply examines the
- ** b-tree structure to determine which leaves need to be inspected.
- */
- rc = sqlite3Fts3AllSegdirs(p, &pStmt);
- while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
- Fts3SegReader *pNew = 0;
- int nRoot = sqlite3_column_bytes(pStmt, 4);
- char const *zRoot = sqlite3_column_blob(pStmt, 4);
- if( sqlite3_column_int64(pStmt, 1)==0 ){
- /* The entire segment is stored on the root node (which must be a
- ** leaf). Do not bother inspecting any data in this case, just
- ** create a Fts3SegReader to scan the single leaf.
- */
- rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew);
- }else{
- int rc2; /* Return value of sqlite3Fts3ReadBlock() */
- sqlite3_int64 i1; /* Blockid of leaf that may contain zTerm */
- rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1);
- if( rc==SQLITE_OK ){
- sqlite3_int64 i2 = sqlite3_column_int64(pStmt, 2);
- rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew);
- }
-
- /* The following call to ReadBlock() serves to reset the SQL statement
- ** used to retrieve blocks of data from the %_segments table. If it is
- ** not reset here, then it may remain classified as an active statement
- ** by SQLite, which may lead to "DROP TABLE" or "DETACH" commands
- ** failing.
- */
- rc2 = sqlite3Fts3ReadBlock(p, 0, 0, 0);
- if( rc==SQLITE_OK ){
- rc = rc2;
- }
- }
- iAge++;
-
- /* If a new Fts3SegReader was allocated, add it to the apSegment array. */
- assert( pNew!=0 || rc!=SQLITE_OK );
- if( pNew ){
- if( nSegment==nAlloc ){
- Fts3SegReader **pArray;
- nAlloc += 16;
- pArray = (Fts3SegReader **)sqlite3_realloc(
- apSegment, nAlloc*sizeof(Fts3SegReader *)
- );
- if( !pArray ){
- sqlite3Fts3SegReaderFree(p, pNew);
- rc = SQLITE_NOMEM;
- goto finished;
- }
- apSegment = pArray;
- }
- apSegment[nSegment++] = pNew;
- }
- }
- if( rc!=SQLITE_DONE ){
- assert( rc!=SQLITE_OK );
- goto finished;
- }
-
+ pArray = pTok->pArray;
memset(&tsc, 0, sizeof(TermSelect));
tsc.isReqPos = isReqPos;
filter.flags = FTS3_SEGMENT_IGNORE_EMPTY
- | (isPrefix ? FTS3_SEGMENT_PREFIX : 0)
+ | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0)
| (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0)
| (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0);
filter.iCol = iColumn;
- filter.zTerm = zTerm;
- filter.nTerm = nTerm;
+ filter.zTerm = pTok->z;
+ filter.nTerm = pTok->n;
- rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment, &filter,
- fts3TermSelectCb, (void *)&tsc
+ rc = sqlite3Fts3SegReaderIterate(p, pArray->apSegment, pArray->nSegment,
+ &filter, fts3TermSelectCb, (void *)&tsc
);
if( rc==SQLITE_OK ){
rc = fts3TermSelectMerge(&tsc);
@@ -109361,26 +110339,112 @@
*ppOut = tsc.aaOutput[0];
*pnOut = tsc.anOutput[0];
}else{
+ int i;
for(i=0; i<SizeofArray(tsc.aaOutput); i++){
sqlite3_free(tsc.aaOutput[i]);
}
}
-finished:
- sqlite3_reset(pStmt);
- for(i=0; i<nSegment; i++){
- sqlite3Fts3SegReaderFree(p, apSegment[i]);
- }
- sqlite3_free(apSegment);
+ fts3SegReaderArrayFree(pArray);
+ pTok->pArray = 0;
return rc;
}
+/*
+** This function counts the total number of docids in the doclist stored
+** in buffer aList[], size nList bytes.
+**
+** If the isPoslist argument is true, then it is assumed that the doclist
+** contains a position-list following each docid. Otherwise, it is assumed
+** that the doclist is simply a list of docids stored as delta encoded
+** varints.
+*/
+static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){
+ int nDoc = 0; /* Return value */
+ if( aList ){
+ char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */
+ char *p = aList; /* Cursor */
+ if( !isPoslist ){
+ /* The number of docids in the list is the same as the number of
+ ** varints. In FTS3 a varint consists of a single byte with the 0x80
+ ** bit cleared and zero or more bytes with the 0x80 bit set. So to
+ ** count the varints in the buffer, just count the number of bytes
+ ** with the 0x80 bit clear. */
+ while( p<aEnd ) nDoc += (((*p++)&0x80)==0);
+ }else{
+ while( p<aEnd ){
+ nDoc++;
+ while( (*p++)&0x80 ); /* Skip docid varint */
+ fts3PoslistCopy(0, &p); /* Skip over position list */
+ }
+ }
+ }
+
+ return nDoc;
+}
+
+/*
+** Call sqlite3Fts3DeferToken() for each token in the expression pExpr.
+*/
+static int fts3DeferExpression(Fts3Cursor *pCsr, Fts3Expr *pExpr){
+ int rc = SQLITE_OK;
+ if( pExpr ){
+ rc = fts3DeferExpression(pCsr, pExpr->pLeft);
+ if( rc==SQLITE_OK ){
+ rc = fts3DeferExpression(pCsr, pExpr->pRight);
+ }
+ if( pExpr->eType==FTSQUERY_PHRASE ){
+ int iCol = pExpr->pPhrase->iColumn;
+ int i;
+ for(i=0; rc==SQLITE_OK && i<pExpr->pPhrase->nToken; i++){
+ Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i];
+ if( pToken->pDeferred==0 ){
+ rc = sqlite3Fts3DeferToken(pCsr, pToken, iCol);
+ }
+ }
+ }
+ }
+ return rc;
+}
+
+/*
+** This function removes the position information from a doclist. When
+** called, buffer aList (size *pnList bytes) contains a doclist that includes
+** position information. This function removes the position information so
+** that aList contains only docids, and adjusts *pnList to reflect the new
+** (possibly reduced) size of the doclist.
+*/
+static void fts3DoclistStripPositions(
+ char *aList, /* IN/OUT: Buffer containing doclist */
+ int *pnList /* IN/OUT: Size of doclist in bytes */
+){
+ if( aList ){
+ char *aEnd = &aList[*pnList]; /* Pointer to one byte after EOF */
+ char *p = aList; /* Input cursor */
+ char *pOut = aList; /* Output cursor */
+
+ while( p<aEnd ){
+ sqlite3_int64 delta;
+ p += sqlite3Fts3GetVarint(p, &delta);
+ fts3PoslistCopy(0, &p);
+ pOut += sqlite3Fts3PutVarint(pOut, delta);
+ }
+
+ *pnList = (int)(pOut - aList);
+ }
+}
/*
** Return a DocList corresponding to the phrase *pPhrase.
+**
+** If this function returns SQLITE_OK, but *pnOut is set to a negative value,
+** then no tokens in the phrase were looked up in the full-text index. This
+** is only possible when this function is called from within xFilter(). The
+** caller should assume that all documents match the phrase. The actual
+** filtering will take place in xNext().
*/
static int fts3PhraseSelect(
- Fts3Table *p, /* Virtual table handle */
+ Fts3Cursor *pCsr, /* Virtual table cursor handle */
Fts3Phrase *pPhrase, /* Phrase to return a doclist for */
int isReqPos, /* True if output should contain positions */
char **paOut, /* OUT: Pointer to malloc'd result buffer */
@@ -109392,42 +110456,137 @@
int ii;
int iCol = pPhrase->iColumn;
int isTermPos = (pPhrase->nToken>1 || isReqPos);
+ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
+ int isFirst = 1;
+
+ int iPrevTok = 0;
+ int nDoc = 0;
+
+ /* If this is an xFilter() evaluation, create a segment-reader for each
+ ** phrase token. Or, if this is an xNext() or snippet/offsets/matchinfo
+ ** evaluation, only create segment-readers if there are no Fts3DeferredToken
+ ** objects attached to the phrase-tokens.
+ */
+ for(ii=0; ii<pPhrase->nToken; ii++){
+ Fts3PhraseToken *pTok = &pPhrase->aToken[ii];
+ if( pTok->pArray==0 ){
+ if( (pCsr->eEvalmode==FTS3_EVAL_FILTER)
+ || (pCsr->eEvalmode==FTS3_EVAL_NEXT && pCsr->pDeferred==0)
+ || (pCsr->eEvalmode==FTS3_EVAL_MATCHINFO && pTok->bFulltext)
+ ){
+ rc = fts3TermSegReaderArray(
+ pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pArray
+ );
+ if( rc!=SQLITE_OK ) return rc;
+ }
+ }
+ }
for(ii=0; ii<pPhrase->nToken; ii++){
- struct PhraseToken *pTok = &pPhrase->aToken[ii];
- char *z = pTok->z; /* Next token of the phrase */
- int n = pTok->n; /* Size of z in bytes */
- int isPrefix = pTok->isPrefix;/* True if token is a prefix */
- char *pList; /* Pointer to token doclist */
- int nList; /* Size of buffer at pList */
+ Fts3PhraseToken *pTok; /* Token to find doclist for */
+ int iTok = 0; /* The token being queried this iteration */
+ char *pList = 0; /* Pointer to token doclist */
+ int nList = 0; /* Size of buffer at pList */
- rc = fts3TermSelect(p, iCol, z, n, isPrefix, isTermPos, &nList, &pList);
+ /* Select a token to process. If this is an xFilter() call, then tokens
+ ** are processed in order from least to most costly. Otherwise, tokens
+ ** are processed in the order in which they occur in the phrase.
+ */
+ if( pCsr->eEvalmode==FTS3_EVAL_MATCHINFO ){
+ assert( isReqPos );
+ iTok = ii;
+ pTok = &pPhrase->aToken[iTok];
+ if( pTok->bFulltext==0 ) continue;
+ }else if( pCsr->eEvalmode==FTS3_EVAL_NEXT || isReqPos ){
+ iTok = ii;
+ pTok = &pPhrase->aToken[iTok];
+ }else{
+ int nMinCost = 0x7FFFFFFF;
+ int jj;
+
+ /* Find the remaining token with the lowest cost. */
+ for(jj=0; jj<pPhrase->nToken; jj++){
+ Fts3SegReaderArray *pArray = pPhrase->aToken[jj].pArray;
+ if( pArray && pArray->nCost<nMinCost ){
+ iTok = jj;
+ nMinCost = pArray->nCost;
+ }
+ }
+ pTok = &pPhrase->aToken[iTok];
+
+ /* This branch is taken if it is determined that loading the doclist
+ ** for the next token would require more IO than loading all documents
+ ** currently identified by doclist pOut/nOut. No further doclists will
+ ** be loaded from the full-text index for this phrase.
+ */
+ if( nMinCost>nDoc && ii>0 ){
+ rc = fts3DeferExpression(pCsr, pCsr->pExpr);
+ break;
+ }
+ }
+
+ if( pCsr->eEvalmode==FTS3_EVAL_NEXT && pTok->pDeferred ){
+ rc = fts3DeferredTermSelect(pTok->pDeferred, isTermPos, &nList, &pList);
+ }else{
+ if( pTok->pArray ){
+ rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList);
+ }
+ pTok->bFulltext = 1;
+ }
+ assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pArray==0 );
if( rc!=SQLITE_OK ) break;
- if( ii==0 ){
+ if( isFirst ){
pOut = pList;
nOut = nList;
- }else{
- /* Merge the new term list and the current output. If this is the
- ** last term in the phrase, and positions are not required in the
- ** output of this function, the positions can be dropped as part
- ** of this merge. Either way, the result of this merge will be
- ** smaller than nList bytes. The code in fts3DoclistMerge() is written
- ** so that it is safe to use pList as the output as well as an input
- ** in this case.
- */
- int mergetype = MERGE_POS_PHRASE;
- if( ii==pPhrase->nToken-1 && !isReqPos ){
- mergetype = MERGE_PHRASE;
+ if( pCsr->eEvalmode==FTS3_EVAL_FILTER && pPhrase->nToken>1 ){
+ nDoc = fts3DoclistCountDocids(1, pOut, nOut);
}
- fts3DoclistMerge(mergetype, 0, 0, pList, &nOut, pOut, nOut, pList, nList);
- sqlite3_free(pOut);
- pOut = pList;
+ isFirst = 0;
+ iPrevTok = iTok;
+ }else{
+ /* Merge the new term list and the current output. */
+ char *aLeft, *aRight;
+ int nLeft, nRight;
+ int nDist;
+ int mt;
+
+ /* If this is the final token of the phrase, and positions were not
+ ** requested by the caller, use MERGE_PHRASE instead of POS_PHRASE.
+ ** This drops the position information from the output list.
+ */
+ mt = MERGE_POS_PHRASE;
+ if( ii==pPhrase->nToken-1 && !isReqPos ) mt = MERGE_PHRASE;
+
+ assert( iPrevTok!=iTok );
+ if( iPrevTok<iTok ){
+ aLeft = pOut;
+ nLeft = nOut;
+ aRight = pList;
+ nRight = nList;
+ nDist = iTok-iPrevTok;
+ iPrevTok = iTok;
+ }else{
+ aRight = pOut;
+ nRight = nOut;
+ aLeft = pList;
+ nLeft = nList;
+ nDist = iPrevTok-iTok;
+ }
+ pOut = aRight;
+ fts3DoclistMerge(
+ mt, nDist, 0, pOut, &nOut, aLeft, nLeft, aRight, nRight, &nDoc
+ );
+ sqlite3_free(aLeft);
}
assert( nOut==0 || pOut!=0 );
}
if( rc==SQLITE_OK ){
+ if( ii!=pPhrase->nToken ){
+ assert( pCsr->eEvalmode==FTS3_EVAL_FILTER && isReqPos==0 );
+ fts3DoclistStripPositions(pOut, &nOut);
+ }
*paOut = pOut;
*pnOut = nOut;
}else{
@@ -109436,6 +110595,14 @@
return rc;
}
+/*
+** This function merges two doclists according to the requirements of a
+** NEAR operator.
+**
+** Both input doclists must include position information. The output doclist
+** includes position information if the first argument to this function
+** is MERGE_POS_NEAR, or does not if it is MERGE_NEAR.
+*/
static int fts3NearMerge(
int mergetype, /* MERGE_POS_NEAR or MERGE_NEAR */
int nNear, /* Parameter to NEAR operator */
@@ -109448,8 +110615,8 @@
char **paOut, /* OUT: Results of merge (malloced) */
int *pnOut /* OUT: Sized of output buffer */
){
- char *aOut;
- int rc;
+ char *aOut; /* Buffer to write output doclist to */
+ int rc; /* Return code */
assert( mergetype==MERGE_POS_NEAR || MERGE_NEAR );
@@ -109458,7 +110625,7 @@
rc = SQLITE_NOMEM;
}else{
rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft,
- aOut, pnOut, aLeft, nLeft, aRight, nRight
+ aOut, pnOut, aLeft, nLeft, aRight, nRight, 0
);
if( rc!=SQLITE_OK ){
sqlite3_free(aOut);
@@ -109470,8 +110637,23 @@
return rc;
}
+/*
+** This function is used as part of the processing for the snippet() and
+** offsets() functions.
+**
+** Both pLeft and pRight are expression nodes of type FTSQUERY_PHRASE. Both
+** have their respective doclists (including position information) loaded
+** in Fts3Expr.aDoclist/nDoclist. This function removes all entries from
+** each doclist that are not within nNear tokens of a corresponding entry
+** in the other doclist.
+*/
SQLITE_PRIVATE int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){
- int rc;
+ int rc; /* Return code */
+
+ assert( pLeft->eType==FTSQUERY_PHRASE );
+ assert( pRight->eType==FTSQUERY_PHRASE );
+ assert( pLeft->isLoaded && pRight->isLoaded );
+
if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){
sqlite3_free(pLeft->aDoclist);
sqlite3_free(pRight->aDoclist);
@@ -109479,8 +110661,8 @@
pLeft->aDoclist = 0;
rc = SQLITE_OK;
}else{
- char *aOut;
- int nOut;
+ char *aOut; /* Buffer in which to assemble new doclist */
+ int nOut; /* Size of buffer aOut in bytes */
rc = fts3NearMerge(MERGE_POS_NEAR, nNear,
pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist,
@@ -109504,14 +110686,156 @@
return rc;
}
+
/*
-** Evaluate the full-text expression pExpr against fts3 table pTab. Store
-** the resulting doclist in *paOut and *pnOut. This routine mallocs for
-** the space needed to store the output. The caller is responsible for
-** freeing the space when it has finished.
+** Allocate an Fts3SegReaderArray for each token in the expression pExpr.
+** The allocated objects are stored in the Fts3PhraseToken.pArray member
+** variables of each token structure.
*/
-static int evalFts3Expr(
- Fts3Table *p, /* Virtual table handle */
+static int fts3ExprAllocateSegReaders(
+ Fts3Cursor *pCsr, /* FTS3 table */
+ Fts3Expr *pExpr, /* Expression to create seg-readers for */
+ int *pnExpr /* OUT: Number of AND'd expressions */
+){
+ int rc = SQLITE_OK; /* Return code */
+
+ assert( pCsr->eEvalmode==FTS3_EVAL_FILTER );
+ if( pnExpr && pExpr->eType!=FTSQUERY_AND ){
+ (*pnExpr)++;
+ pnExpr = 0;
+ }
+
+ if( pExpr->eType==FTSQUERY_PHRASE ){
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
+ int ii;
+
+ for(ii=0; rc==SQLITE_OK && ii<pPhrase->nToken; ii++){
+ Fts3PhraseToken *pTok = &pPhrase->aToken[ii];
+ if( pTok->pArray==0 ){
+ rc = fts3TermSegReaderArray(
+ pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pArray
+ );
+ }
+ }
+ }else{
+ rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pLeft, pnExpr);
+ if( rc==SQLITE_OK ){
+ rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pRight, pnExpr);
+ }
+ }
+ return rc;
+}
+
+/*
+** Free the Fts3SegReaderArray objects associated with each token in the
+** expression pExpr. In other words, this function frees the resources
+** allocated by fts3ExprAllocateSegReaders().
+*/
+static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){
+ if( pExpr ){
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
+ if( pPhrase ){
+ int kk;
+ for(kk=0; kk<pPhrase->nToken; kk++){
+ fts3SegReaderArrayFree(pPhrase->aToken[kk].pArray);
+ pPhrase->aToken[kk].pArray = 0;
+ }
+ }
+ fts3ExprFreeSegReaders(pExpr->pLeft);
+ fts3ExprFreeSegReaders(pExpr->pRight);
+ }
+}
+
+/*
+** Return the sum of the costs of all tokens in the expression pExpr. This
+** function must be called after Fts3SegReaderArrays have been allocated
+** for all tokens using fts3ExprAllocateSegReaders().
+*/
+static int fts3ExprCost(Fts3Expr *pExpr){
+ int nCost; /* Return value */
+ if( pExpr->eType==FTSQUERY_PHRASE ){
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
+ int ii;
+ nCost = 0;
+ for(ii=0; ii<pPhrase->nToken; ii++){
+ Fts3SegReaderArray *pArray = pPhrase->aToken[ii].pArray;
+ if( pArray ){
+ nCost += pPhrase->aToken[ii].pArray->nCost;
+ }
+ }
+ }else{
+ nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight);
+ }
+ return nCost;
+}
+
+/*
+** The following is a helper function (and type) for fts3EvalExpr(). It
+** must be called after Fts3SegReaders have been allocated for every token
+** in the expression. See the context it is called from in fts3EvalExpr()
+** for further explanation.
+*/
+typedef struct ExprAndCost ExprAndCost;
+struct ExprAndCost {
+ Fts3Expr *pExpr;
+ int nCost;
+};
+static void fts3ExprAssignCosts(
+ Fts3Expr *pExpr, /* Expression to create seg-readers for */
+ ExprAndCost **ppExprCost /* OUT: Write to *ppExprCost */
+){
+ if( pExpr->eType==FTSQUERY_AND ){
+ fts3ExprAssignCosts(pExpr->pLeft, ppExprCost);
+ fts3ExprAssignCosts(pExpr->pRight, ppExprCost);
+ }else{
+ (*ppExprCost)->pExpr = pExpr;
+ (*ppExprCost)->nCost = fts3ExprCost(pExpr);
+ (*ppExprCost)++;
+ }
+}
+
+/*
+** Evaluate the full-text expression pExpr against FTS3 table pTab. Store
+** the resulting doclist in *paOut and *pnOut. This routine mallocs for
+** the space needed to store the output. The caller is responsible for
+** freeing the space when it has finished.
+**
+** This function is called in two distinct contexts:
+**
+** * From within the virtual table xFilter() method. In this case, the
+** output doclist contains entries for all rows in the table, based on
+** data read from the full-text index.
+**
+** In this case, if the query expression contains one or more tokens that
+** are very common, then the returned doclist may contain a superset of
+** the documents that actually match the expression.
+**
+** * From within the virtual table xNext() method. This call is only made
+** if the call from within xFilter() found that there were very common
+** tokens in the query expression and did return a superset of the
+** matching documents. In this case the returned doclist contains only
+** entries that correspond to the current row of the table. Instead of
+** reading the data for each token from the full-text index, the data is
+** already available in-memory in the Fts3PhraseToken.pDeferred structures.
+** See fts3EvalDeferred() for how it gets there.
+**
+** In the first case above, Fts3Cursor.doDeferred==0. In the second (if it is
+** required) Fts3Cursor.doDeferred==1.
+**
+** If the SQLite invokes the snippet(), offsets() or matchinfo() function
+** as part of a SELECT on an FTS3 table, this function is called on each
+** individual phrase expression in the query. If there were very common tokens
+** found in the xFilter() call, then this function is called once for phrase
+** for each row visited, and the returned doclist contains entries for the
+** current row only. Otherwise, if there were no very common tokens, then this
+** function is called once only for each phrase in the query and the returned
+** doclist contains entries for all rows of the table.
+**
+** Fts3Cursor.doDeferred==1 when this function is called on phrases as a
+** result of a snippet(), offsets() or matchinfo() invocation.
+*/
+static int fts3EvalExpr(
+ Fts3Cursor *p, /* Virtual table cursor handle */
Fts3Expr *pExpr, /* Parsed fts3 expression */
char **paOut, /* OUT: Pointer to malloc'd result buffer */
int *pnOut, /* OUT: Size of buffer at *paOut */
@@ -109524,33 +110848,102 @@
*pnOut = 0;
if( pExpr ){
- assert( pExpr->eType==FTSQUERY_PHRASE
- || pExpr->eType==FTSQUERY_NEAR
- || isReqPos==0
+ assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR
+ || pExpr->eType==FTSQUERY_AND || pExpr->eType==FTSQUERY_NOT
+ || pExpr->eType==FTSQUERY_PHRASE
);
+ assert( pExpr->eType==FTSQUERY_PHRASE || isReqPos==0 );
+
if( pExpr->eType==FTSQUERY_PHRASE ){
- rc = fts3PhraseSelect(p, pExpr->pPhrase,
+ rc = fts3PhraseSelect(p, pExpr->pPhrase,
isReqPos || (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR),
paOut, pnOut
);
+ fts3ExprFreeSegReaders(pExpr);
+ }else if( p->eEvalmode==FTS3_EVAL_FILTER && pExpr->eType==FTSQUERY_AND ){
+ ExprAndCost *aExpr = 0; /* Array of AND'd expressions and costs */
+ int nExpr = 0; /* Size of aExpr[] */
+ char *aRet = 0; /* Doclist to return to caller */
+ int nRet = 0; /* Length of aRet[] in bytes */
+ int nDoc = 0x7FFFFFFF;
+
+ assert( !isReqPos );
+
+ rc = fts3ExprAllocateSegReaders(p, pExpr, &nExpr);
+ if( rc==SQLITE_OK ){
+ assert( nExpr>1 );
+ aExpr = sqlite3_malloc(sizeof(ExprAndCost) * nExpr);
+ if( !aExpr ) rc = SQLITE_NOMEM;
+ }
+ if( rc==SQLITE_OK ){
+ int ii; /* Used to iterate through expressions */
+
+ fts3ExprAssignCosts(pExpr, &aExpr);
+ aExpr -= nExpr;
+ for(ii=0; ii<nExpr; ii++){
+ char *aNew;
+ int nNew;
+ int jj;
+ ExprAndCost *pBest = 0;
+
+ for(jj=0; jj<nExpr; jj++){
+ ExprAndCost *pCand = &aExpr[jj];
+ if( pCand->pExpr && (pBest==0 || pCand->nCost<pBest->nCost) ){
+ pBest = pCand;
+ }
+ }
+
+ if( pBest->nCost>nDoc ){
+ rc = fts3DeferExpression(p, p->pExpr);
+ break;
+ }else{
+ rc = fts3EvalExpr(p, pBest->pExpr, &aNew, &nNew, 0);
+ if( rc!=SQLITE_OK ) break;
+ pBest->pExpr = 0;
+ if( ii==0 ){
+ aRet = aNew;
+ nRet = nNew;
+ nDoc = fts3DoclistCountDocids(0, aRet, nRet);
+ }else{
+ fts3DoclistMerge(
+ MERGE_AND, 0, 0, aRet, &nRet, aRet, nRet, aNew, nNew, &nDoc
+ );
+ sqlite3_free(aNew);
+ }
+ }
+ }
+ }
+
+ if( rc==SQLITE_OK ){
+ *paOut = aRet;
+ *pnOut = nRet;
+ }else{
+ assert( *paOut==0 );
+ sqlite3_free(aRet);
+ }
+ sqlite3_free(aExpr);
+ fts3ExprFreeSegReaders(pExpr);
+
}else{
char *aLeft;
char *aRight;
int nLeft;
int nRight;
- if( 0==(rc = evalFts3Expr(p, pExpr->pRight, &aRight, &nRight, isReqPos))
- && 0==(rc = evalFts3Expr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos))
+ assert( pExpr->eType==FTSQUERY_NEAR
+ || pExpr->eType==FTSQUERY_OR
+ || pExpr->eType==FTSQUERY_NOT
+ || (pExpr->eType==FTSQUERY_AND && p->eEvalmode==FTS3_EVAL_NEXT)
+ );
+
+ if( 0==(rc = fts3EvalExpr(p, pExpr->pRight, &aRight, &nRight, isReqPos))
+ && 0==(rc = fts3EvalExpr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos))
){
- assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR
- || pExpr->eType==FTSQUERY_AND || pExpr->eType==FTSQUERY_NOT
- );
switch( pExpr->eType ){
case FTSQUERY_NEAR: {
Fts3Expr *pLeft;
Fts3Expr *pRight;
- int mergetype = isReqPos ? MERGE_POS_NEAR : MERGE_NEAR;
-
+ int mergetype = MERGE_NEAR;
if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){
mergetype = MERGE_POS_NEAR;
}
@@ -109579,7 +110972,7 @@
*/
char *aBuffer = sqlite3_malloc(nRight+nLeft+1);
rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut,
- aLeft, nLeft, aRight, nRight
+ aLeft, nLeft, aRight, nRight, 0
);
*paOut = aBuffer;
sqlite3_free(aLeft);
@@ -109589,7 +110982,7 @@
default: {
assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND );
fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut,
- aLeft, nLeft, aRight, nRight
+ aLeft, nLeft, aRight, nRight, 0
);
*paOut = aLeft;
break;
@@ -109600,6 +110993,89 @@
}
}
+ assert( rc==SQLITE_OK || *paOut==0 );
+ return rc;
+}
+
+/*
+** This function is called from within xNext() for each row visited by
+** an FTS3 query. If evaluating the FTS3 query expression within xFilter()
+** was able to determine the exact set of matching rows, this function sets
+** *pbRes to true and returns SQLITE_IO immediately.
+**
+** Otherwise, if evaluating the query expression within xFilter() returned a
+** superset of the matching documents instead of an exact set (this happens
+** when the query includes very common tokens and it is deemed too expensive to
+** load their doclists from disk), this function tests if the current row
+** really does match the FTS3 query.
+**
+** If an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK
+** is returned and *pbRes is set to true if the current row matches the
+** FTS3 query (and should be included in the results returned to SQLite), or
+** false otherwise.
+*/
+static int fts3EvalDeferred(
+ Fts3Cursor *pCsr, /* FTS3 cursor pointing at row to test */
+ int *pbRes /* OUT: Set to true if row is a match */
+){
+ int rc = SQLITE_OK;
+ if( pCsr->pDeferred==0 ){
+ *pbRes = 1;
+ }else{
+ rc = fts3CursorSeek(0, pCsr);
+ if( rc==SQLITE_OK ){
+ sqlite3Fts3FreeDeferredDoclists(pCsr);
+ rc = sqlite3Fts3CacheDeferredDoclists(pCsr);
+ }
+ if( rc==SQLITE_OK ){
+ char *a = 0;
+ int n = 0;
+ rc = fts3EvalExpr(pCsr, pCsr->pExpr, &a, &n, 0);
+ assert( n>=0 );
+ *pbRes = (n>0);
+ sqlite3_free(a);
+ }
+ }
+ return rc;
+}
+
+/*
+** Advance the cursor to the next row in the %_content table that
+** matches the search criteria. For a MATCH search, this will be
+** the next row that matches. For a full-table scan, this will be
+** simply the next row in the %_content table. For a docid lookup,
+** this routine simply sets the EOF flag.
+**
+** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned
+** even if we reach end-of-file. The fts3EofMethod() will be called
+** subsequently to determine whether or not an EOF was hit.
+*/
+static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
+ int res;
+ int rc = SQLITE_OK; /* Return code */
+ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
+
+ pCsr->eEvalmode = FTS3_EVAL_NEXT;
+ do {
+ if( pCsr->aDoclist==0 ){
+ if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
+ pCsr->isEof = 1;
+ rc = sqlite3_reset(pCsr->pStmt);
+ break;
+ }
+ pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0);
+ }else{
+ if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){
+ pCsr->isEof = 1;
+ break;
+ }
+ sqlite3_reset(pCsr->pStmt);
+ fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId);
+ pCsr->isRequireSeek = 1;
+ pCsr->isMatchinfoNeeded = 1;
+ }
+ }while( SQLITE_OK==(rc = fts3EvalDeferred(pCsr, &res)) && res==0 );
+
return rc;
}
@@ -109619,11 +111095,6 @@
** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand
** side of the MATCH operator.
*/
-/* TODO(shess) Upgrade the cursor initialization and destruction to
-** account for fts3FilterMethod() being called multiple times on the
-** same cursor. The current solution is very fragile. Apply fix to
-** fts3 as appropriate.
-*/
static int fts3FilterMethod(
sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
int idxNum, /* Strategy index */
@@ -109646,6 +111117,7 @@
assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) );
assert( nVal==0 || nVal==1 );
assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) );
+ assert( p->pSegments==0 );
/* In case the cursor has been used before, clear it now. */
sqlite3_finalize(pCsr->pStmt);
@@ -109653,24 +111125,7 @@
sqlite3Fts3ExprFree(pCsr->pExpr);
memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
- /* Compile a SELECT statement for this cursor. For a full-table-scan, the
- ** statement loops through all rows of the %_content table. For a
- ** full-text query or docid lookup, the statement retrieves a single
- ** row by docid.
- */
- zSql = sqlite3_mprintf(azSql[idxNum==FTS3_FULLSCAN_SEARCH], p->zDb, p->zName);
- if( !zSql ){
- rc = SQLITE_NOMEM;
- }else{
- rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
- sqlite3_free(zSql);
- }
- if( rc!=SQLITE_OK ) return rc;
- pCsr->eSearch = (i16)idxNum;
-
- if( idxNum==FTS3_DOCID_SEARCH ){
- rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
- }else if( idxNum!=FTS3_FULLSCAN_SEARCH ){
+ if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){
int iCol = idxNum-FTS3_FULLTEXT_SEARCH;
const char *zQuery = (const char *)sqlite3_value_text(apVal[0]);
@@ -109692,11 +111147,30 @@
rc = sqlite3Fts3ReadLock(p);
if( rc!=SQLITE_OK ) return rc;
- rc = evalFts3Expr(p, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0);
+ rc = fts3EvalExpr(pCsr, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0);
+ sqlite3Fts3SegmentsClose(p);
+ if( rc!=SQLITE_OK ) return rc;
pCsr->pNextId = pCsr->aDoclist;
pCsr->iPrevId = 0;
}
+ /* Compile a SELECT statement for this cursor. For a full-table-scan, the
+ ** statement loops through all rows of the %_content table. For a
+ ** full-text query or docid lookup, the statement retrieves a single
+ ** row by docid.
+ */
+ zSql = sqlite3_mprintf(azSql[idxNum==FTS3_FULLSCAN_SEARCH], p->zDb, p->zName);
+ if( !zSql ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
+ sqlite3_free(zSql);
+ }
+ if( rc==SQLITE_OK && idxNum==FTS3_DOCID_SEARCH ){
+ rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
+ }
+ pCsr->eSearch = (i16)idxNum;
+
if( rc!=SQLITE_OK ) return rc;
return fts3NextMethod(pCursor);
}
@@ -109720,6 +111194,11 @@
if( pCsr->aDoclist ){
*pRowid = pCsr->iPrevId;
}else{
+ /* This branch runs if the query is implemented using a full-table scan
+ ** (not using the full-text index). In this case grab the rowid from the
+ ** SELECT statement.
+ */
+ assert( pCsr->isRequireSeek==0 );
*pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
}
return SQLITE_OK;
@@ -109782,7 +111261,9 @@
** hash-table to the database.
*/
static int fts3SyncMethod(sqlite3_vtab *pVtab){
- return sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab);
+ int rc = sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab);
+ sqlite3Fts3SegmentsClose((Fts3Table *)pVtab);
+ return rc;
}
/*
@@ -109820,8 +111301,27 @@
** This is used by the matchinfo(), snippet() and offsets() auxillary
** functions.
*/
-SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Table *pTab, Fts3Expr *pExpr){
- return evalFts3Expr(pTab, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1);
+SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){
+ int rc;
+ assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase );
+ assert( pCsr->eEvalmode==FTS3_EVAL_NEXT );
+ rc = fts3EvalExpr(pCsr, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1);
+ return rc;
+}
+
+SQLITE_PRIVATE int sqlite3Fts3ExprLoadFtDoclist(
+ Fts3Cursor *pCsr,
+ Fts3Expr *pExpr,
+ char **paDoclist,
+ int *pnDoclist
+){
+ int rc;
+ assert( pCsr->eEvalmode==FTS3_EVAL_NEXT );
+ assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase );
+ pCsr->eEvalmode = FTS3_EVAL_MATCHINFO;
+ rc = fts3EvalExpr(pCsr, pExpr, paDoclist, pnDoclist, 1);
+ pCsr->eEvalmode = FTS3_EVAL_NEXT;
+ return rc;
}
/*
@@ -109887,7 +111387,7 @@
sqlite3_context *pContext, /* SQL function call context */
const char *zFunc, /* Function name */
sqlite3_value *pVal, /* argv[0] passed to function */
- Fts3Cursor **ppCsr /* OUT: Store cursor handle here */
+ Fts3Cursor **ppCsr /* OUT: Store cursor handle here */
){
Fts3Cursor *pRet;
if( sqlite3_value_type(pVal)!=SQLITE_BLOB
@@ -110013,15 +111513,13 @@
sqlite3_value **apVal /* Array of arguments */
){
Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */
-
- if( nVal!=1 ){
- sqlite3_result_error(pContext,
- "wrong number of arguments to function matchinfo()", -1);
- return;
- }
-
+ assert( nVal==1 || nVal==2 );
if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){
- sqlite3Fts3Matchinfo(pContext, pCsr);
+ const char *zArg = 0;
+ if( nVal>1 ){
+ zArg = (const char *)sqlite3_value_text(apVal[1]);
+ }
+ sqlite3Fts3Matchinfo(pContext, pCsr, zArg);
}
}
@@ -110082,12 +111580,13 @@
"ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';",
p->zDb, p->zName, zName
);
- if( rc==SQLITE_ERROR ) rc = SQLITE_OK;
if( p->bHasDocsize ){
fts3DbExec(&rc, db,
"ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';",
p->zDb, p->zName, zName
);
+ }
+ if( p->bHasStat ){
fts3DbExec(&rc, db,
"ALTER TABLE %Q.'%q_stat' RENAME TO '%q_stat';",
p->zDb, p->zName, zName
@@ -110112,7 +111611,7 @@
/* xDisconnect */ fts3DisconnectMethod,
/* xDestroy */ fts3DestroyMethod,
/* xOpen */ fts3OpenMethod,
- /* xClose */ fulltextClose,
+ /* xClose */ fts3CloseMethod,
/* xFilter */ fts3FilterMethod,
/* xNext */ fts3NextMethod,
/* xEof */ fts3EofMethod,
@@ -110139,19 +111638,20 @@
}
/*
-** The fts3 built-in tokenizers - "simple" and "porter" - are implemented
-** in files fts3_tokenizer1.c and fts3_porter.c respectively. The following
-** two forward declarations are for functions declared in these files
-** used to retrieve the respective implementations.
+** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are
+** implemented in files fts3_tokenizer1.c, fts3_porter.c and fts3_icu.c
+** respectively. The following three forward declarations are for functions
+** declared in these files used to retrieve the respective implementations.
**
** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed
** to by the argument to point to the "simple" tokenizer implementation.
-** Function ...PorterTokenizerModule() sets *pModule to point to the
-** porter tokenizer/stemmer implementation.
+** And so on.
*/
SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+#ifdef SQLITE_ENABLE_ICU
SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+#endif
/*
** Initialise the fts3 extension. If this extension is built as part
@@ -110207,7 +111707,8 @@
&& SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer"))
&& SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
&& SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1))
- && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", -1))
+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1))
+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 2))
&& SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1))
){
rc = sqlite3_create_module_v2(
@@ -110354,6 +111855,18 @@
}
/*
+** Allocate nByte bytes of memory using sqlite3_malloc(). If successful,
+** zero the memory before returning a pointer to it. If unsuccessful,
+** return NULL.
+*/
+static void *fts3MallocZero(int nByte){
+ void *pRet = sqlite3_malloc(nByte);
+ if( pRet ) memset(pRet, 0, nByte);
+ return pRet;
+}
+
+
+/*
** Extract the next token from buffer z (length n) using the tokenizer
** and other information (column names etc.) in pParse. Create an Fts3Expr
** structure of type FTSQUERY_PHRASE containing a phrase consisting of this
@@ -110390,11 +111903,10 @@
if( rc==SQLITE_OK ){
nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken;
- pRet = (Fts3Expr *)sqlite3_malloc(nByte);
+ pRet = (Fts3Expr *)fts3MallocZero(nByte);
if( !pRet ){
rc = SQLITE_NOMEM;
}else{
- memset(pRet, 0, nByte);
pRet->eType = FTSQUERY_PHRASE;
pRet->pPhrase = (Fts3Phrase *)&pRet[1];
pRet->pPhrase->nToken = 1;
@@ -110470,7 +111982,7 @@
rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos);
if( rc==SQLITE_OK ){
int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
- p = fts3ReallocOrFree(p, nByte+ii*sizeof(struct PhraseToken));
+ p = fts3ReallocOrFree(p, nByte+ii*sizeof(Fts3PhraseToken));
zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken);
if( !p || !zTemp ){
goto no_mem;
@@ -110480,6 +111992,7 @@
p->pPhrase = (Fts3Phrase *)&p[1];
}
p->pPhrase = (Fts3Phrase *)&p[1];
+ memset(&p->pPhrase->aToken[ii], 0, sizeof(Fts3PhraseToken));
p->pPhrase->nToken = ii+1;
p->pPhrase->aToken[ii].n = nToken;
memcpy(&zTemp[nTemp], zToken, nToken);
@@ -110501,7 +112014,7 @@
char *zNew = NULL;
int nNew = 0;
int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
- nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(struct PhraseToken);
+ nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(Fts3PhraseToken);
p = fts3ReallocOrFree(p, nByte + nTemp);
if( !p ){
goto no_mem;
@@ -110619,11 +112132,10 @@
if( fts3isspace(cNext)
|| cNext=='"' || cNext=='(' || cNext==')' || cNext==0
){
- pRet = (Fts3Expr *)sqlite3_malloc(sizeof(Fts3Expr));
+ pRet = (Fts3Expr *)fts3MallocZero(sizeof(Fts3Expr));
if( !pRet ){
return SQLITE_NOMEM;
}
- memset(pRet, 0, sizeof(Fts3Expr));
pRet->eType = pKey->eType;
pRet->nNear = nNear;
*ppExpr = pRet;
@@ -110641,7 +112153,6 @@
if( sqlite3_fts3_enable_parentheses ){
if( *zInput=='(' ){
int nConsumed;
- int rc;
pParse->nNest++;
rc = fts3ExprParse(pParse, &zInput[1], nInput-1, ppExpr, &nConsumed);
if( rc==SQLITE_OK && !*ppExpr ){
@@ -110799,13 +112310,12 @@
&& p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot
){
/* Create an implicit NOT operator. */
- Fts3Expr *pNot = sqlite3_malloc(sizeof(Fts3Expr));
+ Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr));
if( !pNot ){
sqlite3Fts3ExprFree(p);
rc = SQLITE_NOMEM;
goto exprparse_out;
}
- memset(pNot, 0, sizeof(Fts3Expr));
pNot->eType = FTSQUERY_NOT;
pNot->pRight = p;
if( pNotBranch ){
@@ -110833,13 +112343,12 @@
/* Insert an implicit AND operator. */
Fts3Expr *pAnd;
assert( pRet && pPrev );
- pAnd = sqlite3_malloc(sizeof(Fts3Expr));
+ pAnd = fts3MallocZero(sizeof(Fts3Expr));
if( !pAnd ){
sqlite3Fts3ExprFree(p);
rc = SQLITE_NOMEM;
goto exprparse_out;
}
- memset(pAnd, 0, sizeof(Fts3Expr));
pAnd->eType = FTSQUERY_AND;
insertBinaryOperator(&pRet, pPrev, pAnd);
pPrev = pAnd;
@@ -111023,47 +112532,53 @@
}
/*
-** This function is part of the test interface for the query parser. It
-** writes a text representation of the query expression pExpr into the
-** buffer pointed to by argument zBuf. It is assumed that zBuf is large
-** enough to store the required text representation.
+** Return a pointer to a buffer containing a text representation of the
+** expression passed as the first argument. The buffer is obtained from
+** sqlite3_malloc(). It is the responsibility of the caller to use
+** sqlite3_free() to release the memory. If an OOM condition is encountered,
+** NULL is returned.
+**
+** If the second argument is not NULL, then its contents are prepended to
+** the returned expression text and then freed using sqlite3_free().
*/
-static void exprToString(Fts3Expr *pExpr, char *zBuf){
+static char *exprToString(Fts3Expr *pExpr, char *zBuf){
switch( pExpr->eType ){
case FTSQUERY_PHRASE: {
Fts3Phrase *pPhrase = pExpr->pPhrase;
int i;
- zBuf += sprintf(zBuf, "PHRASE %d %d", pPhrase->iColumn, pPhrase->isNot);
- for(i=0; i<pPhrase->nToken; i++){
- zBuf += sprintf(zBuf," %.*s",pPhrase->aToken[i].n,pPhrase->aToken[i].z);
- zBuf += sprintf(zBuf,"%s", (pPhrase->aToken[i].isPrefix?"+":""));
+ zBuf = sqlite3_mprintf(
+ "%zPHRASE %d %d", zBuf, pPhrase->iColumn, pPhrase->isNot);
+ for(i=0; zBuf && i<pPhrase->nToken; i++){
+ zBuf = sqlite3_mprintf("%z %.*s%s", zBuf,
+ pPhrase->aToken[i].n, pPhrase->aToken[i].z,
+ (pPhrase->aToken[i].isPrefix?"+":"")
+ );
}
- return;
+ return zBuf;
}
case FTSQUERY_NEAR:
- zBuf += sprintf(zBuf, "NEAR/%d ", pExpr->nNear);
+ zBuf = sqlite3_mprintf("%zNEAR/%d ", zBuf, pExpr->nNear);
break;
case FTSQUERY_NOT:
- zBuf += sprintf(zBuf, "NOT ");
+ zBuf = sqlite3_mprintf("%zNOT ", zBuf);
break;
case FTSQUERY_AND:
- zBuf += sprintf(zBuf, "AND ");
+ zBuf = sqlite3_mprintf("%zAND ", zBuf);
break;
case FTSQUERY_OR:
- zBuf += sprintf(zBuf, "OR ");
+ zBuf = sqlite3_mprintf("%zOR ", zBuf);
break;
}
- zBuf += sprintf(zBuf, "{");
- exprToString(pExpr->pLeft, zBuf);
- zBuf += strlen(zBuf);
- zBuf += sprintf(zBuf, "} ");
+ if( zBuf ) zBuf = sqlite3_mprintf("%z{", zBuf);
+ if( zBuf ) zBuf = exprToString(pExpr->pLeft, zBuf);
+ if( zBuf ) zBuf = sqlite3_mprintf("%z} {", zBuf);
- zBuf += sprintf(zBuf, "{");
- exprToString(pExpr->pRight, zBuf);
- zBuf += strlen(zBuf);
- zBuf += sprintf(zBuf, "}");
+ if( zBuf ) zBuf = exprToString(pExpr->pRight, zBuf);
+ if( zBuf ) zBuf = sqlite3_mprintf("%z}", zBuf);
+
+ return zBuf;
}
/*
@@ -111094,6 +112609,7 @@
int nCol;
int ii;
Fts3Expr *pExpr;
+ char *zBuf = 0;
sqlite3 *db = sqlite3_context_db_handle(context);
if( argc<3 ){
@@ -111136,18 +112652,17 @@
rc = sqlite3Fts3ExprParse(
pTokenizer, azCol, nCol, nCol, zExpr, nExpr, &pExpr
);
- if( rc==SQLITE_NOMEM ){
- sqlite3_result_error_nomem(context);
- goto exprtest_out;
- }else if( rc==SQLITE_OK ){
- char zBuf[4096];
- exprToString(pExpr, zBuf);
- sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
- sqlite3Fts3ExprFree(pExpr);
- }else{
+ if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){
sqlite3_result_error(context, "Error parsing expression", -1);
+ }else if( rc==SQLITE_NOMEM || !(zBuf = exprToString(pExpr, 0)) ){
+ sqlite3_result_error_nomem(context);
+ }else{
+ sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
+ sqlite3_free(zBuf);
}
+ sqlite3Fts3ExprFree(pExpr);
+
exprtest_out:
if( pModule && pTokenizer ){
rc = pModule->xDestroy(pTokenizer);
@@ -111890,7 +113405,7 @@
int i, j;
char zReverse[28];
char *z, *z2;
- if( nIn<3 || nIn>=sizeof(zReverse)-7 ){
+ if( nIn<3 || nIn>=(int)sizeof(zReverse)-7 ){
/* The word is too big or too small for the porter stemmer.
** Fallback to the copy stemmer */
copy_stemmer(zIn, nIn, zOut, pnOut);
@@ -112289,7 +113804,7 @@
sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
}
-static int fts3IsIdChar(char c){
+SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char c){
static const char isFtsIdChar[] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
@@ -112327,9 +113842,9 @@
break;
default:
- if( fts3IsIdChar(*z1) ){
+ if( sqlite3Fts3IsIdChar(*z1) ){
z2 = &z1[1];
- while( fts3IsIdChar(*z2) ) z2++;
+ while( sqlite3Fts3IsIdChar(*z2) ) z2++;
}else{
z1++;
}
@@ -112342,9 +113857,8 @@
SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(
Fts3Hash *pHash, /* Tokenizer hash table */
- const char *zArg, /* Possible tokenizer specification */
+ const char *zArg, /* Tokenizer name */
sqlite3_tokenizer **ppTok, /* OUT: Tokenizer (if applicable) */
- const char **pzTokenizer, /* OUT: Set to zArg if is tokenizer */
char **pzErr /* OUT: Set to malloced error message */
){
int rc;
@@ -112354,26 +113868,15 @@
char *zEnd; /* Pointer to nul-term of zCopy */
sqlite3_tokenizer_module *m;
- if( !z ){
- zCopy = sqlite3_mprintf("simple");
- }else{
- if( sqlite3_strnicmp(z, "tokenize", 8) || fts3IsIdChar(z[8])){
- return SQLITE_OK;
- }
- zCopy = sqlite3_mprintf("%s", &z[8]);
- *pzTokenizer = zArg;
- }
- if( !zCopy ){
- return SQLITE_NOMEM;
- }
-
+ zCopy = sqlite3_mprintf("%s", zArg);
+ if( !zCopy ) return SQLITE_NOMEM;
zEnd = &zCopy[strlen(zCopy)];
z = (char *)sqlite3Fts3NextToken(zCopy, &n);
z[n] = '\0';
sqlite3Fts3Dequote(z);
- m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, z, (int)strlen(z)+1);
+ m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1);
if( !m ){
*pzErr = sqlite3_mprintf("unknown tokenizer: %s", z);
rc = SQLITE_ERROR;
@@ -112667,15 +114170,23 @@
}
#endif
- if( SQLITE_OK!=rc
- || SQLITE_OK!=(rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0))
- || SQLITE_OK!=(rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0))
+ if( SQLITE_OK==rc ){
+ rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0);
+ }
+ if( SQLITE_OK==rc ){
+ rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0);
+ }
#ifdef SQLITE_TEST
- || SQLITE_OK!=(rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0))
- || SQLITE_OK!=(rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0))
- || SQLITE_OK!=(rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0))
+ if( SQLITE_OK==rc ){
+ rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0);
+ }
+ if( SQLITE_OK==rc ){
+ rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0);
+ }
+ if( SQLITE_OK==rc ){
+ rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0);
+ }
#endif
- );
#ifdef SQLITE_TEST
sqlite3_free(zTest);
@@ -112942,6 +114453,18 @@
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+/*
+** When full-text index nodes are loaded from disk, the buffer that they
+** are loaded into has the following number of bytes of padding at the end
+** of it. i.e. if a full-text index node is 900 bytes in size, then a buffer
+** of 920 bytes is allocated for it.
+**
+** This means that if we have a pointer into a buffer containing node data,
+** it is always safe to read up to two varints from it without risking an
+** overread, even if the node data is corrupted.
+*/
+#define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2)
+
typedef struct PendingList PendingList;
typedef struct SegmentNode SegmentNode;
typedef struct SegmentWriter SegmentWriter;
@@ -112960,6 +114483,17 @@
sqlite3_int64 iLastPos;
};
+
+/*
+** Each cursor has a (possibly empty) linked list of the following objects.
+*/
+struct Fts3DeferredToken {
+ Fts3PhraseToken *pToken; /* Pointer to corresponding expr token */
+ int iCol; /* Column token must occur in */
+ Fts3DeferredToken *pNext; /* Next in list of deferred tokens */
+ PendingList *pList; /* Doclist is assembled here */
+};
+
/*
** An instance of this structure is used to iterate through the terms on
** a contiguous set of segment b-tree leaf nodes. Although the details of
@@ -112969,6 +114503,7 @@
**
** sqlite3Fts3SegReaderNew()
** sqlite3Fts3SegReaderFree()
+** sqlite3Fts3SegReaderCost()
** sqlite3Fts3SegReaderIterate()
**
** Methods used to manipulate Fts3SegReader structures:
@@ -112979,12 +114514,14 @@
*/
struct Fts3SegReader {
int iIdx; /* Index within level, or 0x7FFFFFFF for PT */
- sqlite3_int64 iStartBlock;
- sqlite3_int64 iEndBlock;
- sqlite3_stmt *pStmt; /* SQL Statement to access leaf nodes */
+
+ sqlite3_int64 iStartBlock; /* Rowid of first leaf block to traverse */
+ sqlite3_int64 iLeafEndBlock; /* Rowid of final leaf block to traverse */
+ sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */
+ sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */
+
char *aNode; /* Pointer to node data (or NULL) */
int nNode; /* Size of buffer at aNode (or 0) */
- int nTermAlloc; /* Allocated size of zTerm buffer */
Fts3HashElem **ppNextElem;
/* Variables set by fts3SegReaderNext(). These may be read directly
@@ -112994,6 +114531,7 @@
*/
int nTerm; /* Number of bytes in current term */
char *zTerm; /* Pointer to current term */
+ int nTermAlloc; /* Allocated size of zTerm buffer */
char *aDoclist; /* Pointer to doclist of current entry */
int nDoclist; /* Size of doclist in current entry */
@@ -113003,6 +114541,7 @@
};
#define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0)
+#define fts3SegReaderIsRootOnly(p) ((p)->aNode==(char *)&(p)[1])
/*
** An instance of this structure is used to create a segment b-tree in the
@@ -113071,12 +114610,11 @@
#define SQL_DELETE_SEGDIR_BY_LEVEL 16
#define SQL_DELETE_SEGMENTS_RANGE 17
#define SQL_CONTENT_INSERT 18
-#define SQL_GET_BLOCK 19
-#define SQL_DELETE_DOCSIZE 20
-#define SQL_REPLACE_DOCSIZE 21
-#define SQL_SELECT_DOCSIZE 22
-#define SQL_SELECT_DOCTOTAL 23
-#define SQL_REPLACE_DOCTOTAL 24
+#define SQL_DELETE_DOCSIZE 19
+#define SQL_REPLACE_DOCSIZE 20
+#define SQL_SELECT_DOCSIZE 21
+#define SQL_SELECT_DOCTOTAL 22
+#define SQL_REPLACE_DOCTOTAL 23
/*
** This function is used to obtain an SQLite prepared statement handle
@@ -113121,12 +114659,11 @@
/* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?",
/* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?",
/* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%z)",
-/* 19 */ "SELECT block FROM %Q.'%q_segments' WHERE blockid = ?",
-/* 20 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?",
-/* 21 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",
-/* 22 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
-/* 23 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0",
-/* 24 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)",
+/* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?",
+/* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",
+/* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
+/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0",
+/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)",
};
int rc = SQLITE_OK;
sqlite3_stmt *pStmt;
@@ -113175,6 +114712,51 @@
return rc;
}
+static int fts3SelectDocsize(
+ Fts3Table *pTab, /* FTS3 table handle */
+ int eStmt, /* Either SQL_SELECT_DOCSIZE or DOCTOTAL */
+ sqlite3_int64 iDocid, /* Docid to bind for SQL_SELECT_DOCSIZE */
+ sqlite3_stmt **ppStmt /* OUT: Statement handle */
+){
+ sqlite3_stmt *pStmt = 0; /* Statement requested from fts3SqlStmt() */
+ int rc; /* Return code */
+
+ assert( eStmt==SQL_SELECT_DOCSIZE || eStmt==SQL_SELECT_DOCTOTAL );
+
+ rc = fts3SqlStmt(pTab, eStmt, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ if( eStmt==SQL_SELECT_DOCSIZE ){
+ sqlite3_bind_int64(pStmt, 1, iDocid);
+ }
+ rc = sqlite3_step(pStmt);
+ if( rc!=SQLITE_ROW ){
+ rc = sqlite3_reset(pStmt);
+ if( rc==SQLITE_OK ) rc = SQLITE_CORRUPT;
+ pStmt = 0;
+ }else{
+ rc = SQLITE_OK;
+ }
+ }
+
+ *ppStmt = pStmt;
+ return rc;
+}
+
+SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(
+ Fts3Table *pTab, /* Fts3 table handle */
+ sqlite3_stmt **ppStmt /* OUT: Statement handle */
+){
+ return fts3SelectDocsize(pTab, SQL_SELECT_DOCTOTAL, 0, ppStmt);
+}
+
+SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(
+ Fts3Table *pTab, /* Fts3 table handle */
+ sqlite3_int64 iDocid, /* Docid to read size data for */
+ sqlite3_stmt **ppStmt /* OUT: Statement handle */
+){
+ return fts3SelectDocsize(pTab, SQL_SELECT_DOCSIZE, iDocid, ppStmt);
+}
+
/*
** Similar to fts3SqlStmt(). Except, after binding the parameters in
** array apVal[] to the SQL statement identified by eStmt, the statement
@@ -113202,45 +114784,6 @@
/*
-** Read a single block from the %_segments table. If the specified block
-** does not exist, return SQLITE_CORRUPT. If some other error (malloc, IO
-** etc.) occurs, return the appropriate SQLite error code.
-**
-** Otherwise, if successful, set *pzBlock to point to a buffer containing
-** the block read from the database, and *pnBlock to the size of the read
-** block in bytes.
-**
-** WARNING: The returned buffer is only valid until the next call to
-** sqlite3Fts3ReadBlock().
-*/
-SQLITE_PRIVATE int sqlite3Fts3ReadBlock(
- Fts3Table *p,
- sqlite3_int64 iBlock,
- char const **pzBlock,
- int *pnBlock
-){
- sqlite3_stmt *pStmt;
- int rc = fts3SqlStmt(p, SQL_GET_BLOCK, &pStmt, 0);
- if( rc!=SQLITE_OK ) return rc;
- sqlite3_reset(pStmt);
-
- if( pzBlock ){
- sqlite3_bind_int64(pStmt, 1, iBlock);
- rc = sqlite3_step(pStmt);
- if( rc!=SQLITE_ROW ){
- return (rc==SQLITE_DONE ? SQLITE_CORRUPT : rc);
- }
-
- *pnBlock = sqlite3_column_bytes(pStmt, 0);
- *pzBlock = (char *)sqlite3_column_blob(pStmt, 0);
- if( sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){
- return SQLITE_CORRUPT;
- }
- }
- return SQLITE_OK;
-}
-
-/*
** This function ensures that the caller has obtained a shared-cache
** table-lock on the %_content table. This is required before reading
** data from the fts3 table. If this lock is not acquired first, then
@@ -113408,10 +114951,10 @@
** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code.
*/
static int fts3PendingTermsAdd(
- Fts3Table *p, /* FTS table into which text will be inserted */
- const char *zText, /* Text of document to be inseted */
- int iCol, /* Column number into which text is inserted */
- u32 *pnWord /* OUT: Number of tokens inserted */
+ Fts3Table *p, /* Table into which text will be inserted */
+ const char *zText, /* Text of document to be inserted */
+ int iCol, /* Column into which text is being inserted */
+ u32 *pnWord /* OUT: Number of tokens inserted */
){
int rc;
int iStart;
@@ -113496,6 +115039,9 @@
return SQLITE_OK;
}
+/*
+** Discard the contents of the pending-terms hash table.
+*/
SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){
Fts3HashElem *pElem;
for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){
@@ -113523,6 +115069,7 @@
return rc;
}
}
+ aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]);
}
return SQLITE_OK;
}
@@ -113610,6 +115157,8 @@
fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0);
if( p->bHasDocsize ){
fts3SqlExec(&rc, p, SQL_DELETE_ALL_DOCSIZE, 0);
+ }
+ if( p->bHasStat ){
fts3SqlExec(&rc, p, SQL_DELETE_ALL_STAT, 0);
}
return rc;
@@ -113620,7 +115169,7 @@
** (an integer) of a row about to be deleted. Remove all terms from the
** full-text index.
*/
-static void fts3DeleteTerms(
+static void fts3DeleteTerms(
int *pRC, /* Result code */
Fts3Table *p, /* The FTS table to delete from */
sqlite3_value **apVal, /* apVal[] contains the docid to be deleted */
@@ -113642,6 +115191,7 @@
*pRC = rc;
return;
}
+ aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i);
}
}
rc = sqlite3_reset(pSelect);
@@ -113705,11 +115255,92 @@
}
/*
+** The %_segments table is declared as follows:
+**
+** CREATE TABLE %_segments(blockid INTEGER PRIMARY KEY, block BLOB)
+**
+** This function reads data from a single row of the %_segments table. The
+** specific row is identified by the iBlockid parameter. If paBlob is not
+** NULL, then a buffer is allocated using sqlite3_malloc() and populated
+** with the contents of the blob stored in the "block" column of the
+** identified table row is. Whether or not paBlob is NULL, *pnBlob is set
+** to the size of the blob in bytes before returning.
+**
+** If an error occurs, or the table does not contain the specified row,
+** an SQLite error code is returned. Otherwise, SQLITE_OK is returned. If
+** paBlob is non-NULL, then it is the responsibility of the caller to
+** eventually free the returned buffer.
+**
+** This function may leave an open sqlite3_blob* handle in the
+** Fts3Table.pSegments variable. This handle is reused by subsequent calls
+** to this function. The handle may be closed by calling the
+** sqlite3Fts3SegmentsClose() function. Reusing a blob handle is a handy
+** performance improvement, but the blob handle should always be closed
+** before control is returned to the user (to prevent a lock being held
+** on the database file for longer than necessary). Thus, any virtual table
+** method (xFilter etc.) that may directly or indirectly call this function
+** must call sqlite3Fts3SegmentsClose() before returning.
+*/
+SQLITE_PRIVATE int sqlite3Fts3ReadBlock(
+ Fts3Table *p, /* FTS3 table handle */
+ sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */
+ char **paBlob, /* OUT: Blob data in malloc'd buffer */
+ int *pnBlob /* OUT: Size of blob data */
+){
+ int rc; /* Return code */
+
+ /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */
+ assert( pnBlob);
+
+ if( p->pSegments ){
+ rc = sqlite3_blob_reopen(p->pSegments, iBlockid);
+ }else{
+ if( 0==p->zSegmentsTbl ){
+ p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName);
+ if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM;
+ }
+ rc = sqlite3_blob_open(
+ p->db, p->zDb, p->zSegmentsTbl, "block", iBlockid, 0, &p->pSegments
+ );
+ }
+
+ if( rc==SQLITE_OK ){
+ int nByte = sqlite3_blob_bytes(p->pSegments);
+ if( paBlob ){
+ char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING);
+ if( !aByte ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0);
+ memset(&aByte[nByte], 0, FTS3_NODE_PADDING);
+ if( rc!=SQLITE_OK ){
+ sqlite3_free(aByte);
+ aByte = 0;
+ }
+ }
+ *paBlob = aByte;
+ }
+ *pnBlob = nByte;
+ }
+
+ return rc;
+}
+
+/*
+** Close the blob handle at p->pSegments, if it is open. See comments above
+** the sqlite3Fts3ReadBlock() function for details.
+*/
+SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *p){
+ sqlite3_blob_close(p->pSegments);
+ p->pSegments = 0;
+}
+
+/*
** Move the iterator passed as the first argument to the next term in the
** segment. If successful, SQLITE_OK is returned. If there is no next term,
** SQLITE_DONE. Otherwise, an SQLite error code.
*/
-static int fts3SegReaderNext(Fts3SegReader *pReader){
+static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){
char *pNext; /* Cursor variable */
int nPrefix; /* Number of bytes in term prefix */
int nSuffix; /* Number of bytes in term suffix */
@@ -113721,7 +115352,8 @@
}
if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){
- int rc;
+ int rc; /* Return code from Fts3ReadBlock() */
+
if( fts3SegReaderIsPending(pReader) ){
Fts3HashElem *pElem = *(pReader->ppNextElem);
if( pElem==0 ){
@@ -113737,22 +115369,36 @@
}
return SQLITE_OK;
}
- if( !pReader->pStmt ){
- pReader->aNode = 0;
+
+ if( !fts3SegReaderIsRootOnly(pReader) ){
+ sqlite3_free(pReader->aNode);
+ }
+ pReader->aNode = 0;
+
+ /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf
+ ** blocks have already been traversed. */
+ assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock );
+ if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){
return SQLITE_OK;
}
- rc = sqlite3_step(pReader->pStmt);
- if( rc!=SQLITE_ROW ){
- pReader->aNode = 0;
- return (rc==SQLITE_DONE ? SQLITE_OK : rc);
- }
- pReader->nNode = sqlite3_column_bytes(pReader->pStmt, 0);
- pReader->aNode = (char *)sqlite3_column_blob(pReader->pStmt, 0);
+
+ rc = sqlite3Fts3ReadBlock(
+ p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode
+ );
+ if( rc!=SQLITE_OK ) return rc;
pNext = pReader->aNode;
}
+ /* Because of the FTS3_NODE_PADDING bytes of padding, the following is
+ ** safe (no risk of overread) even if the node data is corrupted.
+ */
pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix);
pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix);
+ if( nPrefix<0 || nSuffix<=0
+ || &pNext[nSuffix]>&pReader->aNode[pReader->nNode]
+ ){
+ return SQLITE_CORRUPT;
+ }
if( nPrefix+nSuffix>pReader->nTermAlloc ){
int nNew = (nPrefix+nSuffix)*2;
@@ -113767,9 +115413,18 @@
pReader->nTerm = nPrefix+nSuffix;
pNext += nSuffix;
pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist);
- assert( pNext<&pReader->aNode[pReader->nNode] );
pReader->aDoclist = pNext;
pReader->pOffsetList = 0;
+
+ /* Check that the doclist does not appear to extend past the end of the
+ ** b-tree node. And that the final byte of the doclist is 0x00. If either
+ ** of these statements is untrue, then the data structure is corrupt.
+ */
+ if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode]
+ || pReader->aDoclist[pReader->nDoclist-1]
+ ){
+ return SQLITE_CORRUPT;
+ }
return SQLITE_OK;
}
@@ -113833,31 +115488,105 @@
}
/*
+** This function is called to estimate the amount of data that will be
+** loaded from the disk If SegReaderIterate() is called on this seg-reader,
+** in units of average document size.
+**
+** This can be used as follows: If the caller has a small doclist that
+** contains references to N documents, and is considering merging it with
+** a large doclist (size X "average documents"), it may opt not to load
+** the large doclist if X>N.
+*/
+SQLITE_PRIVATE int sqlite3Fts3SegReaderCost(
+ Fts3Cursor *pCsr, /* FTS3 cursor handle */
+ Fts3SegReader *pReader, /* Segment-reader handle */
+ int *pnCost /* IN/OUT: Number of bytes read */
+){
+ Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
+ int rc = SQLITE_OK; /* Return code */
+ int nCost = 0; /* Cost in bytes to return */
+ int pgsz = p->nPgsz; /* Database page size */
+
+ /* If this seg-reader is reading the pending-terms table, or if all data
+ ** for the segment is stored on the root page of the b-tree, then the cost
+ ** is zero. In this case all required data is already in main memory.
+ */
+ if( p->bHasStat
+ && !fts3SegReaderIsPending(pReader)
+ && !fts3SegReaderIsRootOnly(pReader)
+ ){
+ int nBlob = 0;
+ sqlite3_int64 iBlock;
+
+ if( pCsr->nRowAvg==0 ){
+ /* The average document size, which is required to calculate the cost
+ ** of each doclist, has not yet been determined. Read the required
+ ** data from the %_stat table to calculate it.
+ **
+ ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3
+ ** varints, where nCol is the number of columns in the FTS3 table.
+ ** The first varint is the number of documents currently stored in
+ ** the table. The following nCol varints contain the total amount of
+ ** data stored in all rows of each column of the table, from left
+ ** to right.
+ */
+ sqlite3_stmt *pStmt;
+ rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0);
+ if( rc ) return rc;
+ if( sqlite3_step(pStmt)==SQLITE_ROW ){
+ sqlite3_int64 nDoc = 0;
+ sqlite3_int64 nByte = 0;
+ const char *a = sqlite3_column_blob(pStmt, 0);
+ if( a ){
+ const char *pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
+ a += sqlite3Fts3GetVarint(a, &nDoc);
+ while( a<pEnd ){
+ a += sqlite3Fts3GetVarint(a, &nByte);
+ }
+ }
+
+ pCsr->nRowAvg = (int)(((nByte / nDoc) + pgsz - 1) / pgsz);
+ }
+ rc = sqlite3_reset(pStmt);
+ if( rc!=SQLITE_OK || pCsr->nRowAvg==0 ) return rc;
+ }
+
+ /* Assume that a blob flows over onto overflow pages if it is larger
+ ** than (pgsz-35) bytes in size (the file-format documentation
+ ** confirms this).
+ */
+ for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){
+ rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob);
+ if( rc!=SQLITE_OK ) break;
+ if( (nBlob+35)>pgsz ){
+ int nOvfl = (nBlob + 34)/pgsz;
+ nCost += ((nOvfl + pCsr->nRowAvg - 1)/pCsr->nRowAvg);
+ }
+ }
+ }
+
+ *pnCost += nCost;
+ return rc;
+}
+
+/*
** Free all allocations associated with the iterator passed as the
** second argument.
*/
-SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3Table *p, Fts3SegReader *pReader){
- if( pReader ){
- if( pReader->pStmt ){
- /* Move the leaf-range SELECT statement to the aLeavesStmt[] array,
- ** so that it can be reused when required by another query.
- */
- assert( p->nLeavesStmt<p->nLeavesTotal );
- sqlite3_reset(pReader->pStmt);
- p->aLeavesStmt[p->nLeavesStmt++] = pReader->pStmt;
+SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){
+ if( pReader && !fts3SegReaderIsPending(pReader) ){
+ sqlite3_free(pReader->zTerm);
+ if( !fts3SegReaderIsRootOnly(pReader) ){
+ sqlite3_free(pReader->aNode);
}
- if( !fts3SegReaderIsPending(pReader) ){
- sqlite3_free(pReader->zTerm);
- }
- sqlite3_free(pReader);
}
+ sqlite3_free(pReader);
}
/*
** Allocate a new SegReader object.
*/
SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(
- Fts3Table *p, /* Virtual table handle */
int iAge, /* Segment "age". */
sqlite3_int64 iStartLeaf, /* First leaf to traverse */
sqlite3_int64 iEndLeaf, /* Final leaf to traverse */
@@ -113870,8 +115599,9 @@
Fts3SegReader *pReader; /* Newly allocated SegReader object */
int nExtra = 0; /* Bytes to allocate segment root node */
+ assert( iStartLeaf<=iEndLeaf );
if( iStartLeaf==0 ){
- nExtra = nRoot;
+ nExtra = nRoot + FTS3_NODE_PADDING;
}
pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra);
@@ -113879,8 +115609,9 @@
return SQLITE_NOMEM;
}
memset(pReader, 0, sizeof(Fts3SegReader));
- pReader->iStartBlock = iStartLeaf;
pReader->iIdx = iAge;
+ pReader->iStartBlock = iStartLeaf;
+ pReader->iLeafEndBlock = iEndLeaf;
pReader->iEndBlock = iEndBlock;
if( nExtra ){
@@ -113888,59 +115619,15 @@
pReader->aNode = (char *)&pReader[1];
pReader->nNode = nRoot;
memcpy(pReader->aNode, zRoot, nRoot);
+ memset(&pReader->aNode[nRoot], 0, FTS3_NODE_PADDING);
}else{
- /* If the text of the SQL statement to iterate through a contiguous
- ** set of entries in the %_segments table has not yet been composed,
- ** compose it now.
- */
- if( !p->zSelectLeaves ){
- p->zSelectLeaves = sqlite3_mprintf(
- "SELECT block FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ? "
- "ORDER BY blockid", p->zDb, p->zName
- );
- if( !p->zSelectLeaves ){
- rc = SQLITE_NOMEM;
- goto finished;
- }
- }
-
- /* If there are no free statements in the aLeavesStmt[] array, prepare
- ** a new statement now. Otherwise, reuse a prepared statement from
- ** aLeavesStmt[].
- */
- if( p->nLeavesStmt==0 ){
- if( p->nLeavesTotal==p->nLeavesAlloc ){
- int nNew = p->nLeavesAlloc + 16;
- sqlite3_stmt **aNew = (sqlite3_stmt **)sqlite3_realloc(
- p->aLeavesStmt, nNew*sizeof(sqlite3_stmt *)
- );
- if( !aNew ){
- rc = SQLITE_NOMEM;
- goto finished;
- }
- p->nLeavesAlloc = nNew;
- p->aLeavesStmt = aNew;
- }
- rc = sqlite3_prepare_v2(p->db, p->zSelectLeaves, -1, &pReader->pStmt, 0);
- if( rc!=SQLITE_OK ){
- goto finished;
- }
- p->nLeavesTotal++;
- }else{
- pReader->pStmt = p->aLeavesStmt[--p->nLeavesStmt];
- }
-
- /* Bind the start and end leaf blockids to the prepared SQL statement. */
- sqlite3_bind_int64(pReader->pStmt, 1, iStartLeaf);
- sqlite3_bind_int64(pReader->pStmt, 2, iEndLeaf);
+ pReader->iCurrentBlock = iStartLeaf-1;
}
- rc = fts3SegReaderNext(pReader);
- finished:
if( rc==SQLITE_OK ){
*ppReader = pReader;
}else{
- sqlite3Fts3SegReaderFree(p, pReader);
+ sqlite3Fts3SegReaderFree(pReader);
}
return rc;
}
@@ -114031,7 +115718,6 @@
pReader->iIdx = 0x7FFFFFFF;
pReader->ppNextElem = (Fts3HashElem **)&pReader[1];
memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *));
- fts3SegReaderNext(pReader);
}
}
@@ -114064,12 +115750,11 @@
** code is returned.
*/
static int fts3SegReaderNew(
- Fts3Table *p, /* Virtual table handle */
sqlite3_stmt *pStmt, /* See above */
int iAge, /* Segment "age". */
Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */
){
- return sqlite3Fts3SegReaderNew(p, iAge,
+ return sqlite3Fts3SegReaderNew(iAge,
sqlite3_column_int64(pStmt, 1),
sqlite3_column_int64(pStmt, 2),
sqlite3_column_int64(pStmt, 3),
@@ -114273,7 +115958,7 @@
** (according to memcmp) than the previous term.
*/
static int fts3NodeAddTerm(
- Fts3Table *p, /* Virtual table handle */
+ Fts3Table *p, /* Virtual table handle */
SegmentNode **ppTree, /* IN/OUT: SegmentNode handle */
int isCopyTerm, /* True if zTerm/nTerm is transient */
const char *zTerm, /* Pointer to buffer containing term */
@@ -114903,15 +116588,14 @@
** unnecessary merge/sort operations for the case where single segment
** b-tree leaf nodes contain more than one term.
*/
- if( pFilter->zTerm ){
+ for(i=0; i<nSegment; i++){
int nTerm = pFilter->nTerm;
const char *zTerm = pFilter->zTerm;
- for(i=0; i<nSegment; i++){
- Fts3SegReader *pSeg = apSegment[i];
- while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ){
- rc = fts3SegReaderNext(pSeg);
- if( rc!=SQLITE_OK ) goto finished; }
- }
+ Fts3SegReader *pSeg = apSegment[i];
+ do {
+ rc = fts3SegReaderNext(p, pSeg);
+ if( rc!=SQLITE_OK ) goto finished;
+ }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 );
}
fts3SegReaderSort(apSegment, nSegment, nSegment, fts3SegReaderCmp);
@@ -115020,7 +116704,7 @@
}
for(i=0; i<nMerge; i++){
- rc = fts3SegReaderNext(apSegment[i]);
+ rc = fts3SegReaderNext(p, apSegment[i]);
if( rc!=SQLITE_OK ) goto finished;
}
fts3SegReaderSort(apSegment, nSegment, nMerge, fts3SegReaderCmp);
@@ -115046,7 +116730,7 @@
int i; /* Iterator variable */
int rc; /* Return code */
int iIdx; /* Index of new segment */
- int iNewLevel; /* Level to create new segment at */
+ int iNewLevel = 0; /* Level to create new segment at */
sqlite3_stmt *pStmt = 0;
SegmentWriter *pWriter = 0;
int nSegment = 0; /* Number of segments being merged */
@@ -115101,7 +116785,7 @@
if( rc!=SQLITE_OK ) goto finished;
sqlite3_bind_int(pStmt, 1, iLevel);
for(i=0; SQLITE_ROW==(sqlite3_step(pStmt)); i++){
- rc = fts3SegReaderNew(p, pStmt, i, &apSegment[i]);
+ rc = fts3SegReaderNew(pStmt, i, &apSegment[i]);
if( rc!=SQLITE_OK ){
goto finished;
}
@@ -115131,11 +116815,11 @@
fts3SegWriterFree(pWriter);
if( apSegment ){
for(i=0; i<nSegment; i++){
- sqlite3Fts3SegReaderFree(p, apSegment[i]);
+ sqlite3Fts3SegReaderFree(apSegment[i]);
}
sqlite3_free(apSegment);
}
- sqlite3Fts3SegReaderFree(p, pPending);
+ sqlite3Fts3SegReaderFree(pPending);
sqlite3_reset(pStmt);
return rc;
}
@@ -115188,7 +116872,7 @@
rc = fts3SegWriterFlush(p, pWriter, 0, idx);
}
fts3SegWriterFree(pWriter);
- sqlite3Fts3SegReaderFree(p, pReader);
+ sqlite3Fts3SegReaderFree(pReader);
if( rc==SQLITE_OK ){
sqlite3Fts3PendingTermsClear(p);
@@ -115232,75 +116916,6 @@
}
/*
-** Fill in the document size auxiliary information for the matchinfo
-** structure. The auxiliary information is:
-**
-** N Total number of documents in the full-text index
-** a0 Average length of column 0 over the whole index
-** n0 Length of column 0 on the matching row
-** ...
-** aM Average length of column M over the whole index
-** nM Length of column M on the matching row
-**
-** The fts3MatchinfoDocsizeLocal() routine fills in the nX values.
-** The fts3MatchinfoDocsizeGlobal() routine fills in N and the aX values.
-*/
-SQLITE_PRIVATE int sqlite3Fts3MatchinfoDocsizeLocal(Fts3Cursor *pCur, u32 *a){
- const char *pBlob; /* The BLOB holding %_docsize info */
- int nBlob; /* Size of the BLOB */
- sqlite3_stmt *pStmt; /* Statement for reading and writing */
- int i, j; /* Loop counters */
- sqlite3_int64 x; /* Varint value */
- int rc; /* Result code from subfunctions */
- Fts3Table *p; /* The FTS table */
-
- p = (Fts3Table*)pCur->base.pVtab;
- rc = fts3SqlStmt(p, SQL_SELECT_DOCSIZE, &pStmt, 0);
- if( rc ){
- return rc;
- }
- sqlite3_bind_int64(pStmt, 1, pCur->iPrevId);
- if( sqlite3_step(pStmt)==SQLITE_ROW ){
- nBlob = sqlite3_column_bytes(pStmt, 0);
- pBlob = (const char*)sqlite3_column_blob(pStmt, 0);
- for(i=j=0; i<p->nColumn && j<nBlob; i++){
- j = sqlite3Fts3GetVarint(&pBlob[j], &x);
- a[2+i*2] = (u32)(x & 0xffffffff);
- }
- }
- sqlite3_reset(pStmt);
- return SQLITE_OK;
-}
-SQLITE_PRIVATE int sqlite3Fts3MatchinfoDocsizeGlobal(Fts3Cursor *pCur, u32 *a){
- const char *pBlob; /* The BLOB holding %_stat info */
- int nBlob; /* Size of the BLOB */
- sqlite3_stmt *pStmt; /* Statement for reading and writing */
- int i, j; /* Loop counters */
- sqlite3_int64 x; /* Varint value */
- int nDoc; /* Number of documents */
- int rc; /* Result code from subfunctions */
- Fts3Table *p; /* The FTS table */
-
- p = (Fts3Table*)pCur->base.pVtab;
- rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0);
- if( rc ){
- return rc;
- }
- if( sqlite3_step(pStmt)==SQLITE_ROW ){
- nBlob = sqlite3_column_bytes(pStmt, 0);
- pBlob = (const char*)sqlite3_column_blob(pStmt, 0);
- j = sqlite3Fts3GetVarint(pBlob, &x);
- a[0] = nDoc = (u32)(x & 0xffffffff);
- for(i=0; i<p->nColumn && j<nBlob; i++){
- j = sqlite3Fts3GetVarint(&pBlob[j], &x);
- a[1+i*2] = ((u32)(x & 0xffffffff) + nDoc/2)/nDoc;
- }
- }
- sqlite3_reset(pStmt);
- return SQLITE_OK;
-}
-
-/*
** Insert the sizes (in tokens) for each column of the document
** with docid equal to p->iPrevDocid. The sizes are encoded as
** a blob of varints.
@@ -115335,16 +116950,26 @@
}
/*
-** Update the 0 record of the %_stat table so that it holds a blob
-** which contains the document count followed by the cumulative
-** document sizes for all columns.
+** Record 0 of the %_stat table contains a blob consisting of N varints,
+** where N is the number of user defined columns in the fts3 table plus
+** two. If nCol is the number of user defined columns, then values of the
+** varints are set as follows:
+**
+** Varint 0: Total number of rows in the table.
+**
+** Varint 1..nCol: For each column, the total number of tokens stored in
+** the column for all rows of the table.
+**
+** Varint 1+nCol: The total size, in bytes, of all text values in all
+** columns of all rows of the table.
+**
*/
static void fts3UpdateDocTotals(
- int *pRC, /* The result code */
- Fts3Table *p, /* Table being updated */
- u32 *aSzIns, /* Size increases */
- u32 *aSzDel, /* Size decreases */
- int nChng /* Change in the number of documents */
+ int *pRC, /* The result code */
+ Fts3Table *p, /* Table being updated */
+ u32 *aSzIns, /* Size increases */
+ u32 *aSzDel, /* Size decreases */
+ int nChng /* Change in the number of documents */
){
char *pBlob; /* Storage for BLOB written into %_stat */
int nBlob; /* Size of BLOB written into %_stat */
@@ -115353,13 +116978,15 @@
int i; /* Loop counter */
int rc; /* Result code from subfunctions */
+ const int nStat = p->nColumn+2;
+
if( *pRC ) return;
- a = sqlite3_malloc( (sizeof(u32)+10)*(p->nColumn+1) );
+ a = sqlite3_malloc( (sizeof(u32)+10)*nStat );
if( a==0 ){
*pRC = SQLITE_NOMEM;
return;
}
- pBlob = (char*)&a[p->nColumn+1];
+ pBlob = (char*)&a[nStat];
rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0);
if( rc ){
sqlite3_free(a);
@@ -115367,11 +116994,11 @@
return;
}
if( sqlite3_step(pStmt)==SQLITE_ROW ){
- fts3DecodeIntArray(p->nColumn+1, a,
+ fts3DecodeIntArray(nStat, a,
sqlite3_column_blob(pStmt, 0),
sqlite3_column_bytes(pStmt, 0));
}else{
- memset(a, 0, sizeof(u32)*(p->nColumn+1) );
+ memset(a, 0, sizeof(u32)*(nStat) );
}
sqlite3_reset(pStmt);
if( nChng<0 && a[0]<(u32)(-nChng) ){
@@ -115379,7 +117006,7 @@
}else{
a[0] += nChng;
}
- for(i=0; i<p->nColumn; i++){
+ for(i=0; i<p->nColumn+1; i++){
u32 x = a[i+1];
if( x+aSzIns[i] < aSzDel[i] ){
x = 0;
@@ -115388,7 +117015,7 @@
}
a[i+1] = x;
}
- fts3EncodeIntArray(p->nColumn+1, a, pBlob, &nBlob);
+ fts3EncodeIntArray(nStat, a, pBlob, &nBlob);
rc = fts3SqlStmt(p, SQL_REPLACE_DOCTOTAL, &pStmt, 0);
if( rc ){
sqlite3_free(a);
@@ -115435,10 +117062,160 @@
rc = SQLITE_ERROR;
}
+ sqlite3Fts3SegmentsClose(p);
return rc;
}
/*
+** Return the deferred doclist associated with deferred token pDeferred.
+** This function assumes that sqlite3Fts3CacheDeferredDoclists() has already
+** been called to allocate and populate the doclist.
+*/
+SQLITE_PRIVATE char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *pDeferred, int *pnByte){
+ if( pDeferred->pList ){
+ *pnByte = pDeferred->pList->nData;
+ return pDeferred->pList->aData;
+ }
+ *pnByte = 0;
+ return 0;
+}
+
+/*
+** Helper fucntion for FreeDeferredDoclists(). This function removes all
+** references to deferred doclists from within the tree of Fts3Expr
+** structures headed by
+*/
+static void fts3DeferredDoclistClear(Fts3Expr *pExpr){
+ if( pExpr ){
+ fts3DeferredDoclistClear(pExpr->pLeft);
+ fts3DeferredDoclistClear(pExpr->pRight);
+ if( pExpr->isLoaded ){
+ sqlite3_free(pExpr->aDoclist);
+ pExpr->isLoaded = 0;
+ pExpr->aDoclist = 0;
+ pExpr->nDoclist = 0;
+ pExpr->pCurrent = 0;
+ pExpr->iCurrent = 0;
+ }
+ }
+}
+
+/*
+** Delete all cached deferred doclists. Deferred doclists are cached
+** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function.
+*/
+SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){
+ Fts3DeferredToken *pDef;
+ for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){
+ sqlite3_free(pDef->pList);
+ pDef->pList = 0;
+ }
+ if( pCsr->pDeferred ){
+ fts3DeferredDoclistClear(pCsr->pExpr);
+ }
+}
+
+/*
+** Free all entries in the pCsr->pDeffered list. Entries are added to
+** this list using sqlite3Fts3DeferToken().
+*/
+SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){
+ Fts3DeferredToken *pDef;
+ Fts3DeferredToken *pNext;
+ for(pDef=pCsr->pDeferred; pDef; pDef=pNext){
+ pNext = pDef->pNext;
+ sqlite3_free(pDef->pList);
+ sqlite3_free(pDef);
+ }
+ pCsr->pDeferred = 0;
+}
+
+/*
+** Generate deferred-doclists for all tokens in the pCsr->pDeferred list
+** based on the row that pCsr currently points to.
+**
+** A deferred-doclist is like any other doclist with position information
+** included, except that it only contains entries for a single row of the
+** table, not for all rows.
+*/
+SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){
+ int rc = SQLITE_OK; /* Return code */
+ if( pCsr->pDeferred ){
+ int i; /* Used to iterate through table columns */
+ sqlite3_int64 iDocid; /* Docid of the row pCsr points to */
+ Fts3DeferredToken *pDef; /* Used to iterate through deferred tokens */
+
+ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
+ sqlite3_tokenizer *pT = p->pTokenizer;
+ sqlite3_tokenizer_module const *pModule = pT->pModule;
+
+ assert( pCsr->isRequireSeek==0 );
+ iDocid = sqlite3_column_int64(pCsr->pStmt, 0);
+
+ for(i=0; i<p->nColumn && rc==SQLITE_OK; i++){
+ const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1);
+ sqlite3_tokenizer_cursor *pTC = 0;
+
+ rc = pModule->xOpen(pT, zText, -1, &pTC);
+ while( rc==SQLITE_OK ){
+ char const *zToken; /* Buffer containing token */
+ int nToken; /* Number of bytes in token */
+ int iDum1, iDum2; /* Dummy variables */
+ int iPos; /* Position of token in zText */
+
+ pTC->pTokenizer = pT;
+ rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos);
+ for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){
+ Fts3PhraseToken *pPT = pDef->pToken;
+ if( (pDef->iCol>=p->nColumn || pDef->iCol==i)
+ && (pPT->n==nToken || (pPT->isPrefix && pPT->n<nToken))
+ && (0==memcmp(zToken, pPT->z, pPT->n))
+ ){
+ fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc);
+ }
+ }
+ }
+ if( pTC ) pModule->xClose(pTC);
+ if( rc==SQLITE_DONE ) rc = SQLITE_OK;
+ }
+
+ for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){
+ if( pDef->pList ){
+ rc = fts3PendingListAppendVarint(&pDef->pList, 0);
+ }
+ }
+ }
+
+ return rc;
+}
+
+/*
+** Add an entry for token pToken to the pCsr->pDeferred list.
+*/
+SQLITE_PRIVATE int sqlite3Fts3DeferToken(
+ Fts3Cursor *pCsr, /* Fts3 table cursor */
+ Fts3PhraseToken *pToken, /* Token to defer */
+ int iCol /* Column that token must appear in (or -1) */
+){
+ Fts3DeferredToken *pDeferred;
+ pDeferred = sqlite3_malloc(sizeof(*pDeferred));
+ if( !pDeferred ){
+ return SQLITE_NOMEM;
+ }
+ memset(pDeferred, 0, sizeof(*pDeferred));
+ pDeferred->pToken = pToken;
+ pDeferred->pNext = pCsr->pDeferred;
+ pDeferred->iCol = iCol;
+ pCsr->pDeferred = pDeferred;
+
+ assert( pToken->pDeferred==0 );
+ pToken->pDeferred = pDeferred;
+
+ return SQLITE_OK;
+}
+
+
+/*
** This function does the work for the xUpdate method of FTS3 virtual
** tables.
*/
@@ -115456,16 +117233,17 @@
u32 *aSzDel; /* Sizes of deleted documents */
int nChng = 0; /* Net change in number of documents */
+ assert( p->pSegments==0 );
/* Allocate space to hold the change in document sizes */
- aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*p->nColumn*2 );
+ aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 );
if( aSzIns==0 ) return SQLITE_NOMEM;
- aSzDel = &aSzIns[p->nColumn];
- memset(aSzIns, 0, sizeof(aSzIns[0])*p->nColumn*2);
+ aSzDel = &aSzIns[p->nColumn+1];
+ memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2);
/* If this is a DELETE or UPDATE operation, remove the old record. */
if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
- int isEmpty;
+ int isEmpty = 0;
rc = fts3IsEmpty(p, apVal, &isEmpty);
if( rc==SQLITE_OK ){
if( isEmpty ){
@@ -115482,8 +117260,8 @@
fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, apVal);
if( p->bHasDocsize ){
fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, apVal);
- nChng--;
}
+ nChng--;
}
}
}else if( sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL ){
@@ -115501,16 +117279,17 @@
rc = fts3InsertTerms(p, apVal, aSzIns);
}
if( p->bHasDocsize ){
- nChng++;
fts3InsertDocsize(&rc, p, aSzIns);
}
+ nChng++;
}
- if( p->bHasDocsize ){
+ if( p->bHasStat ){
fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng);
}
sqlite3_free(aSzIns);
+ sqlite3Fts3SegmentsClose(p);
return rc;
}
@@ -115534,6 +117313,7 @@
sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
}
}
+ sqlite3Fts3SegmentsClose(p);
return rc;
}
@@ -115557,6 +117337,22 @@
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+/*
+** Characters that may appear in the second argument to matchinfo().
+*/
+#define FTS3_MATCHINFO_NPHRASE 'p' /* 1 value */
+#define FTS3_MATCHINFO_NCOL 'c' /* 1 value */
+#define FTS3_MATCHINFO_NDOC 'n' /* 1 value */
+#define FTS3_MATCHINFO_AVGLENGTH 'a' /* nCol values */
+#define FTS3_MATCHINFO_LENGTH 'l' /* nCol values */
+#define FTS3_MATCHINFO_LCS 's' /* nCol values */
+#define FTS3_MATCHINFO_HITS 'x' /* 3*nCol*nPhrase values */
+
+/*
+** The default value for the second argument to matchinfo().
+*/
+#define FTS3_MATCHINFO_DEFAULT "pcx"
+
/*
** Used as an fts3ExprIterate() context when loading phrase doclists to
@@ -115564,7 +117360,7 @@
*/
typedef struct LoadDoclistCtx LoadDoclistCtx;
struct LoadDoclistCtx {
- Fts3Table *pTab; /* FTS3 Table */
+ Fts3Cursor *pCsr; /* FTS3 Cursor */
int nPhrase; /* Number of phrases seen so far */
int nToken; /* Number of tokens seen so far */
};
@@ -115610,6 +117406,8 @@
struct MatchInfo {
Fts3Cursor *pCursor; /* FTS3 Cursor */
int nCol; /* Number of columns in table */
+ int nPhrase; /* Number of matchable phrases in query */
+ sqlite3_int64 nDoc; /* Number of docs in database */
u32 *aMatchinfo; /* Pre-allocated buffer */
};
@@ -115758,7 +117556,7 @@
p->nToken += pExpr->pPhrase->nToken;
if( pExpr->isLoaded==0 ){
- rc = sqlite3Fts3ExprLoadDoclist(p->pTab, pExpr);
+ rc = sqlite3Fts3ExprLoadDoclist(p->pCsr, pExpr);
pExpr->isLoaded = 1;
if( rc==SQLITE_OK ){
rc = fts3ExprNearTrim(pExpr);
@@ -115801,7 +117599,7 @@
){
int rc; /* Return Code */
LoadDoclistCtx sCtx = {0,0,0}; /* Context for fts3ExprIterate() */
- sCtx.pTab = (Fts3Table *)pCsr->base.pVtab;
+ sCtx.pCsr = pCsr;
rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb1, (void *)&sCtx);
if( rc==SQLITE_OK ){
(void)fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb2, 0);
@@ -115811,6 +117609,18 @@
return rc;
}
+static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
+ (*(int *)ctx)++;
+ UNUSED_PARAMETER(pExpr);
+ UNUSED_PARAMETER(iPhrase);
+ return SQLITE_OK;
+}
+static int fts3ExprPhraseCount(Fts3Expr *pExpr){
+ int nPhrase = 0;
+ (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase);
+ return nPhrase;
+}
+
/*
** Advance the position list iterator specified by the first two
** arguments so that it points to the first element with a value greater
@@ -116323,38 +118133,87 @@
/*
** fts3ExprIterate() callback used to collect the "global" matchinfo stats
-** for a single query. The "global" stats are those elements of the matchinfo
-** array that are constant for all rows returned by the current query.
+** for a single query.
+**
+** fts3ExprIterate() callback to load the 'global' elements of a
+** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements
+** of the matchinfo array that are constant for all rows returned by the
+** current query.
+**
+** Argument pCtx is actually a pointer to a struct of type MatchInfo. This
+** function populates Matchinfo.aMatchinfo[] as follows:
+**
+** for(iCol=0; iCol<nCol; iCol++){
+** aMatchinfo[3*iPhrase*nCol + 3*iCol + 1] = X;
+** aMatchinfo[3*iPhrase*nCol + 3*iCol + 2] = Y;
+** }
+**
+** where X is the number of matches for phrase iPhrase is column iCol of all
+** rows of the table. Y is the number of rows for which column iCol contains
+** at least one instance of phrase iPhrase.
+**
+** If the phrase pExpr consists entirely of deferred tokens, then all X and
+** Y values are set to nDoc, where nDoc is the number of documents in the
+** file system. This is done because the full-text index doclist is required
+** to calculate these values properly, and the full-text index doclist is
+** not available for deferred tokens.
*/
-static int fts3ExprGlobalMatchinfoCb(
+static int fts3ExprGlobalHitsCb(
Fts3Expr *pExpr, /* Phrase expression node */
int iPhrase, /* Phrase number (numbered from zero) */
void *pCtx /* Pointer to MatchInfo structure */
){
MatchInfo *p = (MatchInfo *)pCtx;
- char *pCsr;
+ Fts3Cursor *pCsr = p->pCursor;
+ char *pIter;
char *pEnd;
- const int iStart = 2 + (iPhrase * p->nCol * 3) + 1;
+ char *pFree = 0;
+ u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol];
assert( pExpr->isLoaded );
+ assert( pExpr->eType==FTSQUERY_PHRASE );
- /* Fill in the global hit count matrix row for this phrase. */
- pCsr = pExpr->aDoclist;
- pEnd = &pExpr->aDoclist[pExpr->nDoclist];
- while( pCsr<pEnd ){
- while( *pCsr++ & 0x80 ); /* Skip past docid. */
- fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 1);
+ if( pCsr->pDeferred ){
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
+ int ii;
+ for(ii=0; ii<pPhrase->nToken; ii++){
+ if( pPhrase->aToken[ii].bFulltext ) break;
+ }
+ if( ii<pPhrase->nToken ){
+ int nFree = 0;
+ int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree);
+ if( rc!=SQLITE_OK ) return rc;
+ pIter = pFree;
+ pEnd = &pFree[nFree];
+ }else{
+ int iCol; /* Column index */
+ for(iCol=0; iCol<p->nCol; iCol++){
+ aOut[iCol*3 + 1] = (u32)p->nDoc;
+ aOut[iCol*3 + 2] = (u32)p->nDoc;
+ }
+ return SQLITE_OK;
+ }
+ }else{
+ pIter = pExpr->aDoclist;
+ pEnd = &pExpr->aDoclist[pExpr->nDoclist];
}
+ /* Fill in the global hit count matrix row for this phrase. */
+ while( pIter<pEnd ){
+ while( *pIter++ & 0x80 ); /* Skip past docid. */
+ fts3LoadColumnlistCounts(&pIter, &aOut[1], 1);
+ }
+
+ sqlite3_free(pFree);
return SQLITE_OK;
}
/*
-** fts3ExprIterate() callback used to collect the "local" matchinfo stats
-** for a single query. The "local" stats are those elements of the matchinfo
+** fts3ExprIterate() callback used to collect the "local" part of the
+** FTS3_MATCHINFO_HITS array. The local stats are those elements of the
** array that are different for each row returned by the query.
*/
-static int fts3ExprLocalMatchinfoCb(
+static int fts3ExprLocalHitsCb(
Fts3Expr *pExpr, /* Phrase expression node */
int iPhrase, /* Phrase number */
void *pCtx /* Pointer to MatchInfo structure */
@@ -116363,7 +118222,7 @@
if( pExpr->aDoclist ){
char *pCsr;
- int iStart = 2 + (iPhrase * p->nCol * 3);
+ int iStart = iPhrase * p->nCol * 3;
int i;
for(i=0; i<p->nCol; i++) p->aMatchinfo[iStart+i*3] = 0;
@@ -116377,67 +118236,399 @@
return SQLITE_OK;
}
+static int fts3MatchinfoCheck(
+ Fts3Table *pTab,
+ char cArg,
+ char **pzErr
+){
+ if( (cArg==FTS3_MATCHINFO_NPHRASE)
+ || (cArg==FTS3_MATCHINFO_NCOL)
+ || (cArg==FTS3_MATCHINFO_NDOC && pTab->bHasStat)
+ || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bHasStat)
+ || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize)
+ || (cArg==FTS3_MATCHINFO_LCS)
+ || (cArg==FTS3_MATCHINFO_HITS)
+ ){
+ return SQLITE_OK;
+ }
+ *pzErr = sqlite3_mprintf("unrecognized matchinfo request: %c", cArg);
+ return SQLITE_ERROR;
+}
+
+static int fts3MatchinfoSize(MatchInfo *pInfo, char cArg){
+ int nVal; /* Number of integers output by cArg */
+
+ switch( cArg ){
+ case FTS3_MATCHINFO_NDOC:
+ case FTS3_MATCHINFO_NPHRASE:
+ case FTS3_MATCHINFO_NCOL:
+ nVal = 1;
+ break;
+
+ case FTS3_MATCHINFO_AVGLENGTH:
+ case FTS3_MATCHINFO_LENGTH:
+ case FTS3_MATCHINFO_LCS:
+ nVal = pInfo->nCol;
+ break;
+
+ default:
+ assert( cArg==FTS3_MATCHINFO_HITS );
+ nVal = pInfo->nCol * pInfo->nPhrase * 3;
+ break;
+ }
+
+ return nVal;
+}
+
+static int fts3MatchinfoSelectDoctotal(
+ Fts3Table *pTab,
+ sqlite3_stmt **ppStmt,
+ sqlite3_int64 *pnDoc,
+ const char **paLen
+){
+ sqlite3_stmt *pStmt;
+ const char *a;
+ sqlite3_int64 nDoc;
+
+ if( !*ppStmt ){
+ int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt);
+ if( rc!=SQLITE_OK ) return rc;
+ }
+ pStmt = *ppStmt;
+
+ a = sqlite3_column_blob(pStmt, 0);
+ a += sqlite3Fts3GetVarint(a, &nDoc);
+ *pnDoc = (u32)nDoc;
+
+ if( paLen ) *paLen = a;
+ return SQLITE_OK;
+}
+
+/*
+** An instance of the following structure is used to store state while
+** iterating through a multi-column position-list corresponding to the
+** hits for a single phrase on a single row in order to calculate the
+** values for a matchinfo() FTS3_MATCHINFO_LCS request.
+*/
+typedef struct LcsIterator LcsIterator;
+struct LcsIterator {
+ Fts3Expr *pExpr; /* Pointer to phrase expression */
+ char *pRead; /* Cursor used to iterate through aDoclist */
+ int iPosOffset; /* Tokens count up to end of this phrase */
+ int iCol; /* Current column number */
+ int iPos; /* Current position */
+};
+
+/*
+** If LcsIterator.iCol is set to the following value, the iterator has
+** finished iterating through all offsets for all columns.
+*/
+#define LCS_ITERATOR_FINISHED 0x7FFFFFFF;
+
+static int fts3MatchinfoLcsCb(
+ Fts3Expr *pExpr, /* Phrase expression node */
+ int iPhrase, /* Phrase number (numbered from zero) */
+ void *pCtx /* Pointer to MatchInfo structure */
+){
+ LcsIterator *aIter = (LcsIterator *)pCtx;
+ aIter[iPhrase].pExpr = pExpr;
+ return SQLITE_OK;
+}
+
+/*
+** Advance the iterator passed as an argument to the next position. Return
+** 1 if the iterator is at EOF or if it now points to the start of the
+** position list for the next column.
+*/
+static int fts3LcsIteratorAdvance(LcsIterator *pIter){
+ char *pRead = pIter->pRead;
+ sqlite3_int64 iRead;
+ int rc = 0;
+
+ pRead += sqlite3Fts3GetVarint(pRead, &iRead);
+ if( iRead==0 ){
+ pIter->iCol = LCS_ITERATOR_FINISHED;
+ rc = 1;
+ }else{
+ if( iRead==1 ){
+ pRead += sqlite3Fts3GetVarint(pRead, &iRead);
+ pIter->iCol = (int)iRead;
+ pIter->iPos = pIter->iPosOffset;
+ pRead += sqlite3Fts3GetVarint(pRead, &iRead);
+ rc = 1;
+ }
+ pIter->iPos += (int)(iRead-2);
+ }
+
+ pIter->pRead = pRead;
+ return rc;
+}
+
+/*
+** This function implements the FTS3_MATCHINFO_LCS matchinfo() flag.
+**
+** If the call is successful, the longest-common-substring lengths for each
+** column are written into the first nCol elements of the pInfo->aMatchinfo[]
+** array before returning. SQLITE_OK is returned in this case.
+**
+** Otherwise, if an error occurs, an SQLite error code is returned and the
+** data written to the first nCol elements of pInfo->aMatchinfo[] is
+** undefined.
+*/
+static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){
+ LcsIterator *aIter;
+ int i;
+ int iCol;
+ int nToken = 0;
+
+ /* Allocate and populate the array of LcsIterator objects. The array
+ ** contains one element for each matchable phrase in the query.
+ **/
+ aIter = sqlite3_malloc(sizeof(LcsIterator) * pCsr->nPhrase);
+ if( !aIter ) return SQLITE_NOMEM;
+ memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase);
+ (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);
+ for(i=0; i<pInfo->nPhrase; i++){
+ LcsIterator *pIter = &aIter[i];
+ nToken -= pIter->pExpr->pPhrase->nToken;
+ pIter->iPosOffset = nToken;
+ pIter->pRead = sqlite3Fts3FindPositions(pIter->pExpr, pCsr->iPrevId, -1);
+ if( pIter->pRead ){
+ pIter->iPos = pIter->iPosOffset;
+ fts3LcsIteratorAdvance(&aIter[i]);
+ }else{
+ pIter->iCol = LCS_ITERATOR_FINISHED;
+ }
+ }
+
+ for(iCol=0; iCol<pInfo->nCol; iCol++){
+ int nLcs = 0; /* LCS value for this column */
+ int nLive = 0; /* Number of iterators in aIter not at EOF */
+
+ /* Loop through the iterators in aIter[]. Set nLive to the number of
+ ** iterators that point to a position-list corresponding to column iCol.
+ */
+ for(i=0; i<pInfo->nPhrase; i++){
+ assert( aIter[i].iCol>=iCol );
+ if( aIter[i].iCol==iCol ) nLive++;
+ }
+
+ /* The following loop runs until all iterators in aIter[] have finished
+ ** iterating through positions in column iCol. Exactly one of the
+ ** iterators is advanced each time the body of the loop is run.
+ */
+ while( nLive>0 ){
+ LcsIterator *pAdv = 0; /* The iterator to advance by one position */
+ int nThisLcs = 0; /* LCS for the current iterator positions */
+
+ for(i=0; i<pInfo->nPhrase; i++){
+ LcsIterator *pIter = &aIter[i];
+ if( iCol!=pIter->iCol ){
+ /* This iterator is already at EOF for this column. */
+ nThisLcs = 0;
+ }else{
+ if( pAdv==0 || pIter->iPos<pAdv->iPos ){
+ pAdv = pIter;
+ }
+ if( nThisLcs==0 || pIter->iPos==pIter[-1].iPos ){
+ nThisLcs++;
+ }else{
+ nThisLcs = 1;
+ }
+ if( nThisLcs>nLcs ) nLcs = nThisLcs;
+ }
+ }
+ if( fts3LcsIteratorAdvance(pAdv) ) nLive--;
+ }
+
+ pInfo->aMatchinfo[iCol] = nLcs;
+ }
+
+ sqlite3_free(aIter);
+ return SQLITE_OK;
+}
+
+/*
+** Populate the buffer pInfo->aMatchinfo[] with an array of integers to
+** be returned by the matchinfo() function. Argument zArg contains the
+** format string passed as the second argument to matchinfo (or the
+** default value "pcx" if no second argument was specified). The format
+** string has already been validated and the pInfo->aMatchinfo[] array
+** is guaranteed to be large enough for the output.
+**
+** If bGlobal is true, then populate all fields of the matchinfo() output.
+** If it is false, then assume that those fields that do not change between
+** rows (i.e. FTS3_MATCHINFO_NPHRASE, NCOL, NDOC, AVGLENGTH and part of HITS)
+** have already been populated.
+**
+** Return SQLITE_OK if successful, or an SQLite error code if an error
+** occurs. If a value other than SQLITE_OK is returned, the state the
+** pInfo->aMatchinfo[] buffer is left in is undefined.
+*/
+static int fts3MatchinfoValues(
+ Fts3Cursor *pCsr, /* FTS3 cursor object */
+ int bGlobal, /* True to grab the global stats */
+ MatchInfo *pInfo, /* Matchinfo context object */
+ const char *zArg /* Matchinfo format string */
+){
+ int rc = SQLITE_OK;
+ int i;
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+ sqlite3_stmt *pSelect = 0;
+
+ for(i=0; rc==SQLITE_OK && zArg[i]; i++){
+
+ switch( zArg[i] ){
+ case FTS3_MATCHINFO_NPHRASE:
+ if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase;
+ break;
+
+ case FTS3_MATCHINFO_NCOL:
+ if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nCol;
+ break;
+
+ case FTS3_MATCHINFO_NDOC:
+ if( bGlobal ){
+ sqlite3_int64 nDoc;
+ rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0);
+ pInfo->aMatchinfo[0] = (u32)nDoc;
+ }
+ break;
+
+ case FTS3_MATCHINFO_AVGLENGTH:
+ if( bGlobal ){
+ sqlite3_int64 nDoc; /* Number of rows in table */
+ const char *a; /* Aggregate column length array */
+
+ rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a);
+ if( rc==SQLITE_OK ){
+ int iCol;
+ for(iCol=0; iCol<pInfo->nCol; iCol++){
+ sqlite3_int64 nToken;
+ a += sqlite3Fts3GetVarint(a, &nToken);
+ pInfo->aMatchinfo[iCol] = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc);
+ }
+ }
+ }
+ break;
+
+ case FTS3_MATCHINFO_LENGTH: {
+ sqlite3_stmt *pSelectDocsize = 0;
+ rc = sqlite3Fts3SelectDocsize(pTab, pCsr->iPrevId, &pSelectDocsize);
+ if( rc==SQLITE_OK ){
+ int iCol;
+ const char *a = sqlite3_column_blob(pSelectDocsize, 0);
+ for(iCol=0; iCol<pInfo->nCol; iCol++){
+ sqlite3_int64 nToken;
+ a += sqlite3Fts3GetVarint(a, &nToken);
+ pInfo->aMatchinfo[iCol] = (u32)nToken;
+ }
+ }
+ sqlite3_reset(pSelectDocsize);
+ break;
+ }
+
+ case FTS3_MATCHINFO_LCS:
+ rc = fts3ExprLoadDoclists(pCsr, 0, 0);
+ if( rc==SQLITE_OK ){
+ rc = fts3MatchinfoLcs(pCsr, pInfo);
+ }
+ break;
+
+ default: {
+ Fts3Expr *pExpr;
+ assert( zArg[i]==FTS3_MATCHINFO_HITS );
+ pExpr = pCsr->pExpr;
+ rc = fts3ExprLoadDoclists(pCsr, 0, 0);
+ if( rc!=SQLITE_OK ) break;
+ if( bGlobal ){
+ if( pCsr->pDeferred ){
+ rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc, 0);
+ if( rc!=SQLITE_OK ) break;
+ }
+ rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo);
+ if( rc!=SQLITE_OK ) break;
+ }
+ (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo);
+ break;
+ }
+ }
+
+ pInfo->aMatchinfo += fts3MatchinfoSize(pInfo, zArg[i]);
+ }
+
+ sqlite3_reset(pSelect);
+ return rc;
+}
+
+
/*
** Populate pCsr->aMatchinfo[] with data for the current row. The
** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32).
*/
-static int fts3GetMatchinfo(Fts3Cursor *pCsr){
+static int fts3GetMatchinfo(
+ Fts3Cursor *pCsr, /* FTS3 Cursor object */
+ const char *zArg /* Second argument to matchinfo() function */
+){
MatchInfo sInfo;
Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
int rc = SQLITE_OK;
+ int bGlobal = 0; /* Collect 'global' stats as well as local */
+ memset(&sInfo, 0, sizeof(MatchInfo));
sInfo.pCursor = pCsr;
sInfo.nCol = pTab->nColumn;
+ /* If there is cached matchinfo() data, but the format string for the
+ ** cache does not match the format string for this request, discard
+ ** the cached data. */
+ if( pCsr->zMatchinfo && strcmp(pCsr->zMatchinfo, zArg) ){
+ assert( pCsr->aMatchinfo );
+ sqlite3_free(pCsr->aMatchinfo);
+ pCsr->zMatchinfo = 0;
+ pCsr->aMatchinfo = 0;
+ }
+
+ /* If Fts3Cursor.aMatchinfo[] is NULL, then this is the first time the
+ ** matchinfo function has been called for this query. In this case
+ ** allocate the array used to accumulate the matchinfo data and
+ ** initialize those elements that are constant for every row.
+ */
if( pCsr->aMatchinfo==0 ){
- /* If Fts3Cursor.aMatchinfo[] is NULL, then this is the first time the
- ** matchinfo function has been called for this query. In this case
- ** allocate the array used to accumulate the matchinfo data and
- ** initialize those elements that are constant for every row.
- */
- int nPhrase; /* Number of phrases */
- int nMatchinfo; /* Number of u32 elements in match-info */
+ int nMatchinfo = 0; /* Number of u32 elements in match-info */
+ int nArg; /* Bytes in zArg */
+ int i; /* Used to iterate through zArg */
- /* Load doclists for each phrase in the query. */
- rc = fts3ExprLoadDoclists(pCsr, &nPhrase, 0);
- if( rc!=SQLITE_OK ){
- return rc;
- }
- nMatchinfo = 2 + 3*sInfo.nCol*nPhrase;
- if( pTab->bHasDocsize ){
- nMatchinfo += 1 + 2*pTab->nColumn;
+ /* Determine the number of phrases in the query */
+ pCsr->nPhrase = fts3ExprPhraseCount(pCsr->pExpr);
+ sInfo.nPhrase = pCsr->nPhrase;
+
+ /* Determine the number of integers in the buffer returned by this call. */
+ for(i=0; zArg[i]; i++){
+ nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]);
}
- sInfo.aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo);
- if( !sInfo.aMatchinfo ){
- return SQLITE_NOMEM;
- }
- memset(sInfo.aMatchinfo, 0, sizeof(u32)*nMatchinfo);
+ /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */
+ nArg = (int)strlen(zArg);
+ pCsr->aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo + nArg + 1);
+ if( !pCsr->aMatchinfo ) return SQLITE_NOMEM;
-
- /* First element of match-info is the number of phrases in the query */
- sInfo.aMatchinfo[0] = nPhrase;
- sInfo.aMatchinfo[1] = sInfo.nCol;
- (void)fts3ExprIterate(pCsr->pExpr, fts3ExprGlobalMatchinfoCb,(void*)&sInfo);
- if( pTab->bHasDocsize ){
- int ofst = 2 + 3*sInfo.aMatchinfo[0]*sInfo.aMatchinfo[1];
- rc = sqlite3Fts3MatchinfoDocsizeGlobal(pCsr, &sInfo.aMatchinfo[ofst]);
- }
- pCsr->aMatchinfo = sInfo.aMatchinfo;
+ pCsr->zMatchinfo = (char *)&pCsr->aMatchinfo[nMatchinfo];
+ pCsr->nMatchinfo = nMatchinfo;
+ memcpy(pCsr->zMatchinfo, zArg, nArg+1);
+ memset(pCsr->aMatchinfo, 0, sizeof(u32)*nMatchinfo);
pCsr->isMatchinfoNeeded = 1;
+ bGlobal = 1;
}
sInfo.aMatchinfo = pCsr->aMatchinfo;
- if( rc==SQLITE_OK && pCsr->isMatchinfoNeeded ){
- (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLocalMatchinfoCb, (void*)&sInfo);
- if( pTab->bHasDocsize ){
- int ofst = 2 + 3*sInfo.aMatchinfo[0]*sInfo.aMatchinfo[1];
- rc = sqlite3Fts3MatchinfoDocsizeLocal(pCsr, &sInfo.aMatchinfo[ofst]);
- }
+ sInfo.nPhrase = pCsr->nPhrase;
+ if( pCsr->isMatchinfoNeeded ){
+ rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg);
pCsr->isMatchinfoNeeded = 0;
}
- return SQLITE_OK;
+ return rc;
}
/*
@@ -116498,7 +118689,7 @@
** columns of the FTS3 table. Otherwise, only column iCol is considered.
*/
for(iRead=0; iRead<pTab->nColumn; iRead++){
- SnippetFragment sF;
+ SnippetFragment sF = {0, 0, 0, 0};
int iS;
if( iCol>=0 && iRead!=iCol ) continue;
@@ -116532,6 +118723,7 @@
}
snippet_out:
+ sqlite3Fts3SegmentsClose(pTab);
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pCtx, rc);
sqlite3_free(res.z);
@@ -116711,6 +118903,7 @@
offsets_out:
sqlite3_free(sCtx.aTerm);
assert( rc!=SQLITE_DONE );
+ sqlite3Fts3SegmentsClose(pTab);
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pCtx, rc);
sqlite3_free(res.z);
@@ -116723,21 +118916,43 @@
/*
** Implementation of matchinfo() function.
*/
-SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *pContext, Fts3Cursor *pCsr){
+SQLITE_PRIVATE void sqlite3Fts3Matchinfo(
+ sqlite3_context *pContext, /* Function call context */
+ Fts3Cursor *pCsr, /* FTS3 table cursor */
+ const char *zArg /* Second arg to matchinfo() function */
+){
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
int rc;
+ int i;
+ const char *zFormat;
+
+ if( zArg ){
+ for(i=0; zArg[i]; i++){
+ char *zErr = 0;
+ if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){
+ sqlite3_result_error(pContext, zErr, -1);
+ sqlite3_free(zErr);
+ return;
+ }
+ }
+ zFormat = zArg;
+ }else{
+ zFormat = FTS3_MATCHINFO_DEFAULT;
+ }
+
if( !pCsr->pExpr ){
sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC);
return;
}
- rc = fts3GetMatchinfo(pCsr);
+
+ /* Retrieve matchinfo() data. */
+ rc = fts3GetMatchinfo(pCsr, zFormat);
+ sqlite3Fts3SegmentsClose(pTab);
+
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pContext, rc);
}else{
- Fts3Table *pTab = (Fts3Table*)pCsr->base.pVtab;
- int n = sizeof(u32)*(2+pCsr->aMatchinfo[0]*pCsr->aMatchinfo[1]*3);
- if( pTab->bHasDocsize ){
- n += sizeof(u32)*(1 + 2*pTab->nColumn);
- }
+ int n = pCsr->nMatchinfo * sizeof(u32);
sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT);
}
}
@@ -116857,6 +119072,12 @@
typedef unsigned int u32;
#endif
+/* The following macro is used to suppress compiler warnings.
+*/
+#ifndef UNUSED_PARAMETER
+# define UNUSED_PARAMETER(x) (void)(x)
+#endif
+
typedef struct Rtree Rtree;
typedef struct RtreeCursor RtreeCursor;
typedef struct RtreeNode RtreeNode;
@@ -117631,6 +119852,7 @@
RtreeCell cell;
int ii;
int bRes = 0;
+ int rc = SQLITE_OK;
nodeGetCell(pRtree, pCursor->pNode, pCursor->iCell, &cell);
for(ii=0; bRes==0 && ii<pCursor->nConstraint; ii++){
@@ -117656,12 +119878,8 @@
break;
default: {
- int rc;
assert( p->op==RTREE_MATCH );
rc = testRtreeGeom(pRtree, p, &cell, &bRes);
- if( rc!=SQLITE_OK ){
- return rc;
- }
bRes = !bRes;
break;
}
@@ -117669,7 +119887,7 @@
}
*pbEof = bRes;
- return SQLITE_OK;
+ return rc;
}
/*
@@ -117752,14 +119970,13 @@
rc = testRtreeCell(pRtree, pCursor, &isEof);
}
if( rc!=SQLITE_OK || isEof || iHeight==0 ){
- *pEof = isEof;
- return rc;
+ goto descend_to_cell_out;
}
iRowid = nodeGetRowid(pRtree, pCursor->pNode, pCursor->iCell);
rc = nodeAcquire(pRtree, iRowid, pCursor->pNode, &pChild);
if( rc!=SQLITE_OK ){
- return rc;
+ goto descend_to_cell_out;
}
nodeRelease(pRtree, pCursor->pNode);
@@ -117769,7 +119986,7 @@
pCursor->iCell = ii;
rc = descendToCell(pRtree, pCursor, iHeight-1, &isEof);
if( rc!=SQLITE_OK ){
- return rc;
+ goto descend_to_cell_out;
}
}
@@ -117781,8 +119998,9 @@
pCursor->iCell = iSavedCell;
}
+descend_to_cell_out:
*pEof = isEof;
- return SQLITE_OK;
+ return rc;
}
/*
@@ -117938,7 +120156,7 @@
/* Check that the blob is roughly the right size. */
nBlob = sqlite3_value_bytes(pValue);
- if( nBlob<sizeof(RtreeMatchArg)
+ if( nBlob<(int)sizeof(RtreeMatchArg)
|| ((nBlob-sizeof(RtreeMatchArg))%sizeof(double))!=0
){
return SQLITE_ERROR;
@@ -117953,7 +120171,7 @@
memcpy(p, sqlite3_value_blob(pValue), nBlob);
if( p->magic!=RTREE_GEOMETRY_MAGIC
- || nBlob!=(sizeof(RtreeMatchArg) + (p->nParam-1)*sizeof(double))
+ || nBlob!=(int)(sizeof(RtreeMatchArg) + (p->nParam-1)*sizeof(double))
){
sqlite3_free(pGeom);
return SQLITE_ERROR;
@@ -118099,6 +120317,7 @@
int iIdx = 0;
char zIdxStr[RTREE_MAX_DIMENSIONS*8+1];
memset(zIdxStr, 0, sizeof(zIdxStr));
+ UNUSED_PARAMETER(tab);
assert( pIdxInfo->idxStr==0 );
for(ii=0; ii<pIdxInfo->nConstraint; ii++){
@@ -118272,6 +120491,7 @@
if( ii!=iExclude )
#else
assert( iExclude==-1 );
+ UNUSED_PARAMETER(iExclude);
#endif
{
int jj;
@@ -119393,16 +121613,6 @@
return rc;
}
-#ifndef NDEBUG
-static int hashIsEmpty(Rtree *pRtree){
- int ii;
- for(ii=0; ii<HASHSIZE; ii++){
- assert( !pRtree->aHash[ii] );
- }
- return 1;
-}
-#endif
-
/*
** The xUpdate method for rtree module virtual tables.
*/
@@ -119868,6 +122078,7 @@
Rtree tree;
int ii;
+ UNUSED_PARAMETER(nArg);
memset(&node, 0, sizeof(RtreeNode));
memset(&tree, 0, sizeof(Rtree));
tree.nDim = sqlite3_value_int(apArg[0]);
@@ -119901,6 +122112,7 @@
}
static void rtreedepth(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){
+ UNUSED_PARAMETER(nArg);
if( sqlite3_value_type(apArg[0])!=SQLITE_BLOB
|| sqlite3_value_bytes(apArg[0])<2
){
@@ -119922,7 +122134,6 @@
rc = sqlite3_create_function(db, "rtreenode", 2, utf8, 0, rtreenode, 0, 0);
if( rc==SQLITE_OK ){
- int utf8 = SQLITE_UTF8;
rc = sqlite3_create_function(db, "rtreedepth", 1, utf8, 0,rtreedepth, 0, 0);
}
if( rc==SQLITE_OK ){
diff --git a/dist/sqlite3.c.orig b/dist/sqlite3.c.orig
index b15a389..a5b93c4 100644
--- a/dist/sqlite3.c.orig
+++ b/dist/sqlite3.c.orig
@@ -1,10 +1,10 @@
/******************************************************************************
** This file is an amalgamation of many separate C source files from SQLite
-** version 3.7.3. By combining all the individual C code files into this
+** version 3.7.4. By combining all the individual C code files into this
** single large file, the entire code can be compiled as a one translation
** unit. This allows many compilers to do optimizations that would not be
** possible if the files were compiled separately. Performance improvements
-** of 5% are more are commonly seen when SQLite is compiled as a single
+** of 5% or more are commonly seen when SQLite is compiled as a single
** translation unit.
**
** This file is all you need to compile SQLite. To use SQLite in other
@@ -650,9 +650,9 @@
** [sqlite3_libversion_number()], [sqlite3_sourceid()],
** [sqlite_version()] and [sqlite_source_id()].
*/
-#define SQLITE_VERSION "3.7.3"
-#define SQLITE_VERSION_NUMBER 3007003
-#define SQLITE_SOURCE_ID "2010-09-28 07:24:50 88108fff672b6012ffe87ba4b1efe6d5a07037d1"
+#define SQLITE_VERSION "3.7.4"
+#define SQLITE_VERSION_NUMBER 3007004
+#define SQLITE_SOURCE_ID "2010-12-07 20:14:09 a586a4deeb25330037a49df295b36aaf624d0f45"
/*
** CAPI3REF: Run-Time Library Version Numbers
@@ -1085,6 +1085,18 @@
** equal SQLITE_SYNC_NORMAL, that means to use normal fsync() semantics.
** If the lower four bits equal SQLITE_SYNC_FULL, that means
** to use Mac OS X style fullsync instead of fsync().
+**
+** Do not confuse the SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL flags
+** with the [PRAGMA synchronous]=NORMAL and [PRAGMA synchronous]=FULL
+** settings. The [synchronous pragma] determines when calls to the
+** xSync VFS method occur and applies uniformly across all platforms.
+** The SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL flags determine how
+** energetic or rigorous or forceful the sync operations are and
+** only make a difference on Mac OSX for the default SQLite code.
+** (Third-party VFS implementations might also make the distinction
+** between SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL, but among the
+** operating systems natively supported by SQLite, only Mac OSX
+** cares about the difference.)
*/
#define SQLITE_SYNC_NORMAL 0x00002
#define SQLITE_SYNC_FULL 0x00003
@@ -1253,6 +1265,8 @@
#define SQLITE_LAST_ERRNO 4
#define SQLITE_FCNTL_SIZE_HINT 5
#define SQLITE_FCNTL_CHUNK_SIZE 6
+#define SQLITE_FCNTL_FILE_POINTER 7
+
/*
** CAPI3REF: Mutex Handle
@@ -2279,6 +2293,9 @@
/*
** CAPI3REF: Convenience Routines For Running Queries
**
+** This is a legacy interface that is preserved for backwards compatibility.
+** Use of this interface is not recommended.
+**
** Definition: A <b>result table</b> is memory data structure created by the
** [sqlite3_get_table()] interface. A result table records the
** complete query results from one or more queries.
@@ -2299,7 +2316,7 @@
** It is not safe to pass a result table directly to [sqlite3_free()].
** A result table should be deallocated using [sqlite3_free_table()].
**
-** As an example of the result table format, suppose a query result
+** ^(As an example of the result table format, suppose a query result
** is as follows:
**
** <blockquote><pre>
@@ -2323,7 +2340,7 @@
** azResult[5] = "28";
** azResult[6] = "Cindy";
** azResult[7] = "21";
-** </pre></blockquote>
+** </pre></blockquote>)^
**
** ^The sqlite3_get_table() function evaluates one or more
** semicolon-separated SQL statements in the zero-terminated UTF-8
@@ -2331,19 +2348,19 @@
** pointer given in its 3rd parameter.
**
** After the application has finished with the result from sqlite3_get_table(),
-** it should pass the result table pointer to sqlite3_free_table() in order to
+** it must pass the result table pointer to sqlite3_free_table() in order to
** release the memory that was malloced. Because of the way the
** [sqlite3_malloc()] happens within sqlite3_get_table(), the calling
** function must not try to call [sqlite3_free()] directly. Only
** [sqlite3_free_table()] is able to release the memory properly and safely.
**
-** ^(The sqlite3_get_table() interface is implemented as a wrapper around
+** The sqlite3_get_table() interface is implemented as a wrapper around
** [sqlite3_exec()]. The sqlite3_get_table() routine does not have access
** to any internal data structures of SQLite. It uses only the public
** interface defined here. As a consequence, errors that occur in the
** wrapper layer outside of the internal [sqlite3_exec()] call are not
** reflected in subsequent calls to [sqlite3_errcode()] or
-** [sqlite3_errmsg()].)^
+** [sqlite3_errmsg()].
*/
SQLITE_API int sqlite3_get_table(
sqlite3 *db, /* An open database */
@@ -3174,6 +3191,20 @@
SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt);
/*
+** CAPI3REF: Determine If An SQL Statement Writes The Database
+**
+** ^The sqlite3_stmt_readonly(X) interface returns true (non-zero) if
+** the [prepared statement] X is [SELECT] statement and false (zero) if
+** X is an [INSERT], [UPDATE], [DELETE], CREATE, DROP, [ANALYZE],
+** [ALTER], or [REINDEX] statement.
+** If X is a NULL pointer or any other kind of statement, including but
+** not limited to [ATTACH], [DETACH], [COMMIT], [ROLLBACK], [RELEASE],
+** [SAVEPOINT], [PRAGMA], or [VACUUM] the result of sqlite3_stmt_readonly(X) is
+** undefined.
+*/
+SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt);
+
+/*
** CAPI3REF: Dynamically Typed Value Object
** KEYWORDS: {protected sqlite3_value} {unprotected sqlite3_value}
**
@@ -3272,7 +3303,10 @@
**
** ^The fifth argument to sqlite3_bind_blob(), sqlite3_bind_text(), and
** sqlite3_bind_text16() is a destructor used to dispose of the BLOB or
-** string after SQLite has finished with it. ^If the fifth argument is
+** string after SQLite has finished with it. ^The destructor is called
+** to dispose of the BLOB or string even if the call to sqlite3_bind_blob(),
+** sqlite3_bind_text(), or sqlite3_bind_text16() fails.
+** ^If the fifth argument is
** the special value [SQLITE_STATIC], then SQLite assumes that the
** information is in static, unmanaged space and does not need to be freed.
** ^If the fifth argument has the value [SQLITE_TRANSIENT], then
@@ -3805,17 +3839,26 @@
** CAPI3REF: Destroy A Prepared Statement Object
**
** ^The sqlite3_finalize() function is called to delete a [prepared statement].
-** ^If the statement was executed successfully or not executed at all, then
-** SQLITE_OK is returned. ^If execution of the statement failed then an
-** [error code] or [extended error code] is returned.
+** ^If the most recent evaluation of the statement encountered no errors or
+** or if the statement is never been evaluated, then sqlite3_finalize() returns
+** SQLITE_OK. ^If the most recent evaluation of statement S failed, then
+** sqlite3_finalize(S) returns the appropriate [error code] or
+** [extended error code].
**
-** ^This routine can be called at any point during the execution of the
-** [prepared statement]. ^If the virtual machine has not
-** completed execution when this routine is called, that is like
-** encountering an error or an [sqlite3_interrupt | interrupt].
-** ^Incomplete updates may be rolled back and transactions canceled,
-** depending on the circumstances, and the
-** [error code] returned will be [SQLITE_ABORT].
+** ^The sqlite3_finalize(S) routine can be called at any point during
+** the life cycle of [prepared statement] S:
+** before statement S is ever evaluated, after
+** one or more calls to [sqlite3_reset()], or after any call
+** to [sqlite3_step()] regardless of whether or not the statement has
+** completed execution.
+**
+** ^Invoking sqlite3_finalize() on a NULL pointer is a harmless no-op.
+**
+** The application must finalize every [prepared statement] in order to avoid
+** resource leaks. It is a grievous error for the application to try to use
+** a prepared statement after it has been finalized. Any use of a prepared
+** statement after it has been finalized can result in undefined and
+** undesirable behavior such as segfaults and heap corruption.
*/
SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt);
@@ -3903,12 +3946,15 @@
** SQL function or aggregate, pass NULL poiners for all three function
** callbacks.
**
-** ^If the tenth parameter to sqlite3_create_function_v2() is not NULL,
-** then it is invoked when the function is deleted, either by being
-** overloaded or when the database connection closes.
-** ^When the destructure callback of the tenth parameter is invoked, it
-** is passed a single argument which is a copy of the pointer which was
-** the fifth parameter to sqlite3_create_function_v2().
+** ^(If the tenth parameter to sqlite3_create_function_v2() is not NULL,
+** then it is destructor for the application data pointer.
+** The destructor is invoked when the function is deleted, either by being
+** overloaded or when the database connection closes.)^
+** ^The destructor is also invoked if the call to
+** sqlite3_create_function_v2() fails.
+** ^When the destructor callback of the tenth parameter is invoked, it
+** is passed a single argument which is a copy of the application data
+** pointer which was the fifth parameter to sqlite3_create_function_v2().
**
** ^It is permitted to register multiple implementations of the same
** functions with the same name but with either differing numbers of
@@ -4372,6 +4418,15 @@
** calls to the collation creation functions or when the
** [database connection] is closed using [sqlite3_close()].
**
+** ^The xDestroy callback is <u>not</u> called if the
+** sqlite3_create_collation_v2() function fails. Applications that invoke
+** sqlite3_create_collation_v2() with a non-NULL xDestroy argument should
+** check the return code and dispose of the application data pointer
+** themselves rather than expecting SQLite to deal with it for them.
+** This is different from every other SQLite interface. The inconsistency
+** is unfortunate but cannot be changed without breaking backwards
+** compatibility.
+**
** See also: [sqlite3_collation_needed()] and [sqlite3_collation_needed16()].
*/
SQLITE_API int sqlite3_create_collation(
@@ -5126,7 +5181,9 @@
** ^The sqlite3_create_module_v2() interface has a fifth parameter which
** is a pointer to a destructor for the pClientData. ^SQLite will
** invoke the destructor function (if it is not NULL) when SQLite
-** no longer needs the pClientData pointer. ^The sqlite3_create_module()
+** no longer needs the pClientData pointer. ^The destructor will also
+** be invoked if the call to sqlite3_create_module_v2() fails.
+** ^The sqlite3_create_module()
** interface is equivalent to sqlite3_create_module_v2() with a NULL
** destructor.
*/
@@ -5310,6 +5367,30 @@
);
/*
+** CAPI3REF: Move a BLOB Handle to a New Row
+**
+** ^This function is used to move an existing blob handle so that it points
+** to a different row of the same database table. ^The new row is identified
+** by the rowid value passed as the second argument. Only the row can be
+** changed. ^The database, table and column on which the blob handle is open
+** remain the same. Moving an existing blob handle to a new row can be
+** faster than closing the existing handle and opening a new one.
+**
+** ^(The new row must meet the same criteria as for [sqlite3_blob_open()] -
+** it must exist and there must be either a blob or text value stored in
+** the nominated column.)^ ^If the new row is not present in the table, or if
+** it does not contain a blob or text value, or if another error occurs, an
+** SQLite error code is returned and the blob handle is considered aborted.
+** ^All subsequent calls to [sqlite3_blob_read()], [sqlite3_blob_write()] or
+** [sqlite3_blob_reopen()] on an aborted blob handle immediately return
+** SQLITE_ABORT. ^Calling [sqlite3_blob_bytes()] on an aborted blob handle
+** always returns zero.
+**
+** ^This function sets the database handle error code and message.
+*/
+SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64);
+
+/*
** CAPI3REF: Close A BLOB Handle
**
** ^Closes an open [BLOB handle].
@@ -5716,7 +5797,7 @@
** ^The [sqlite3_file_control()] interface makes a direct call to the
** xFileControl method for the [sqlite3_io_methods] object associated
** with a particular database identified by the second argument. ^The
-** name of the database "main" for the main database or "temp" for the
+** name of the database is "main" for the main database or "temp" for the
** TEMP database, or the name that appears after the AS keyword for
** databases that are added using the [ATTACH] SQL command.
** ^A NULL pointer can be used in place of "main" to refer to the
@@ -5726,6 +5807,12 @@
** the xFileControl method. ^The return value of the xFileControl
** method becomes the return value of this routine.
**
+** ^The SQLITE_FCNTL_FILE_POINTER value for the op parameter causes
+** a pointer to the underlying [sqlite3_file] object to be written into
+** the space pointed to by the 4th parameter. ^The SQLITE_FCNTL_FILE_POINTER
+** case is a short-circuit path which does not actually invoke the
+** underlying sqlite3_io_methods.xFileControl method.
+**
** ^If the second parameter (zDbName) does not match the name of any
** open database file, then SQLITE_ERROR is returned. ^This error
** code is not remembered and will not be recalled by [sqlite3_errcode()]
@@ -7395,7 +7482,7 @@
SQLITE_PRIVATE int sqlite3BtreeClose(Btree*);
SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree*,int);
-SQLITE_PRIVATE int sqlite3BtreeSetSafetyLevel(Btree*,int,int);
+SQLITE_PRIVATE int sqlite3BtreeSetSafetyLevel(Btree*,int,int,int);
SQLITE_PRIVATE int sqlite3BtreeSyncDisabled(Btree*);
SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix);
SQLITE_PRIVATE int sqlite3BtreeGetPageSize(Btree*);
@@ -7879,12 +7966,12 @@
#define OP_VRename 132
#define OP_VUpdate 133
#define OP_Pagecount 134
-#define OP_Trace 135
-#define OP_Noop 136
-#define OP_Explain 137
+#define OP_MaxPgcnt 135
+#define OP_Trace 136
+#define OP_Noop 137
+#define OP_Explain 138
/* The following opcode values are never used */
-#define OP_NotUsed_138 138
#define OP_NotUsed_139 139
#define OP_NotUsed_140 140
@@ -7917,7 +8004,7 @@
/* 104 */ 0x00, 0x0c, 0x45, 0x15, 0x01, 0x02, 0x00, 0x01,\
/* 112 */ 0x08, 0x05, 0x05, 0x05, 0x00, 0x00, 0x00, 0x02,\
/* 120 */ 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\
-/* 128 */ 0x01, 0x00, 0x02, 0x01, 0x00, 0x00, 0x02, 0x00,\
+/* 128 */ 0x01, 0x00, 0x02, 0x01, 0x00, 0x00, 0x02, 0x02,\
/* 136 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04,\
/* 144 */ 0x04, 0x04,}
@@ -8102,7 +8189,7 @@
SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager*, u32*, int);
SQLITE_PRIVATE int sqlite3PagerMaxPageCount(Pager*, int);
SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager*, int);
-SQLITE_PRIVATE void sqlite3PagerSetSafetyLevel(Pager*,int,int);
+SQLITE_PRIVATE void sqlite3PagerSetSafetyLevel(Pager*,int,int,int);
SQLITE_PRIVATE int sqlite3PagerLockingMode(Pager *, int);
SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *, int);
SQLITE_PRIVATE int sqlite3PagerGetJournalMode(Pager*);
@@ -8967,13 +9054,14 @@
#define SQLITE_ReadUncommitted 0x0080000 /* For shared-cache mode */
#define SQLITE_LegacyFileFmt 0x00100000 /* Create new databases in format 1 */
#define SQLITE_FullFSync 0x00200000 /* Use full fsync on the backend */
-#define SQLITE_LoadExtension 0x00400000 /* Enable load_extension */
+#define SQLITE_CkptFullFSync 0x00400000 /* Use full fsync for checkpoint */
#define SQLITE_RecoveryMode 0x00800000 /* Ignore schema errors */
#define SQLITE_ReverseOrder 0x01000000 /* Reverse unordered SELECTs */
#define SQLITE_RecTriggers 0x02000000 /* Enable recursive triggers */
#define SQLITE_ForeignKeys 0x04000000 /* Enforce foreign key constraints */
#define SQLITE_AutoIndex 0x08000000 /* Enable automatic indexes */
#define SQLITE_PreferBuiltin 0x10000000 /* Preference to built-in funcs */
+#define SQLITE_LoadExtension 0x20000000 /* Enable load_extension */
/*
** Bits of the sqlite3.flags field that are used by the
@@ -8986,6 +9074,7 @@
#define SQLITE_IndexSearch 0x08 /* Disable indexes for searching */
#define SQLITE_IndexCover 0x10 /* Disable index covering table */
#define SQLITE_GroupByOrder 0x20 /* Disable GROUPBY cover of ORDERBY */
+#define SQLITE_FactorOutConst 0x40 /* Disable factoring out constants */
#define SQLITE_OptMask 0xff /* Mask of all disablable opts */
/*
@@ -9077,15 +9166,15 @@
*/
#define FUNCTION(zName, nArg, iArg, bNC, xFunc) \
{nArg, SQLITE_UTF8, bNC*SQLITE_FUNC_NEEDCOLL, \
- SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0}
+ SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0}
#define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \
{nArg, SQLITE_UTF8, bNC*SQLITE_FUNC_NEEDCOLL, \
- pArg, 0, xFunc, 0, 0, #zName, 0}
+ pArg, 0, xFunc, 0, 0, #zName, 0, 0}
#define LIKEFUNC(zName, nArg, arg, flags) \
- {nArg, SQLITE_UTF8, flags, (void *)arg, 0, likeFunc, 0, 0, #zName, 0}
+ {nArg, SQLITE_UTF8, flags, (void *)arg, 0, likeFunc, 0, 0, #zName, 0, 0}
#define AGGREGATE(zName, nArg, arg, nc, xStep, xFinal) \
{nArg, SQLITE_UTF8, nc*SQLITE_FUNC_NEEDCOLL, \
- SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0}
+ SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0,0}
/*
** All current savepoints are stored in a linked list starting at
@@ -9874,6 +9963,9 @@
u8 isPopulated; /* Temporary table associated with SELECT is populated */
u8 jointype; /* Type of join between this able and the previous */
u8 notIndexed; /* True if there is a NOT INDEXED clause */
+#ifndef SQLITE_OMIT_EXPLAIN
+ u8 iSelectId; /* If pSelect!=0, the id of the sub-select in EQP */
+#endif
int iCursor; /* The VDBE cursor number used to access this table */
Expr *pOn; /* The ON clause of a join */
IdList *pUsing; /* The USING clause of a join */
@@ -9912,6 +10004,7 @@
struct WherePlan {
u32 wsFlags; /* WHERE_* flags that describe the strategy */
u32 nEq; /* Number of == constraints */
+ double nRow; /* Estimated number of rows (for EQP) */
union {
Index *pIdx; /* Index when WHERE_INDEXED is true */
struct WhereTerm *pTerm; /* WHERE clause term for OR-search */
@@ -9996,6 +10089,7 @@
int nLevel; /* Number of nested loop */
struct WhereClause *pWC; /* Decomposition of the WHERE clause */
double savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */
+ double nRowOut; /* Estimated number of output rows */
WhereLevel a[1]; /* Information about each nest loop in WHERE */
};
@@ -10071,6 +10165,7 @@
Expr *pOffset; /* OFFSET expression. NULL means not used. */
int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */
int addrOpenEphm[3]; /* OP_OpenEphem opcodes related to this select */
+ double nSelectRow; /* Estimated number of result rows */
};
/*
@@ -10266,6 +10361,11 @@
int nHeight; /* Expression tree height of current sub-select */
Table *pZombieTab; /* List of Table objects to delete after code gen */
TriggerPrg *pTriggerPrg; /* Linked list of coded triggers */
+
+#ifndef SQLITE_OMIT_EXPLAIN
+ int iSelectId;
+ int iNextSelectId;
+#endif
};
#ifdef SQLITE_OMIT_VIRTUALTABLE
@@ -10560,7 +10660,6 @@
** Internal function prototypes
*/
SQLITE_PRIVATE int sqlite3StrICmp(const char *, const char *);
-SQLITE_PRIVATE int sqlite3IsNumber(const char*, int*, u8);
SQLITE_PRIVATE int sqlite3Strlen30(const char*);
#define sqlite3StrNICmp sqlite3_strnicmp
@@ -10880,9 +10979,9 @@
SQLITE_PRIVATE int sqlite3FixExpr(DbFixer*, Expr*);
SQLITE_PRIVATE int sqlite3FixExprList(DbFixer*, ExprList*);
SQLITE_PRIVATE int sqlite3FixTriggerStep(DbFixer*, TriggerStep*);
-SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*);
+SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*, int, u8);
SQLITE_PRIVATE int sqlite3GetInt32(const char *, int*);
-SQLITE_PRIVATE int sqlite3FitsIn64Bits(const char *, int);
+SQLITE_PRIVATE int sqlite3Atoi(const char*);
SQLITE_PRIVATE int sqlite3Utf16ByteLen(const void *pData, int nChar);
SQLITE_PRIVATE int sqlite3Utf8CharLen(const char *pData, int nByte);
SQLITE_PRIVATE int sqlite3Utf8Read(const u8*, const u8**);
@@ -10928,7 +11027,7 @@
SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2);
SQLITE_PRIVATE int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity);
SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr);
-SQLITE_PRIVATE int sqlite3Atoi64(const char*, i64*);
+SQLITE_PRIVATE int sqlite3Atoi64(const char*, i64*, int, u8);
SQLITE_PRIVATE void sqlite3Error(sqlite3*, int, const char*,...);
SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3*, const char *z, int n);
SQLITE_PRIVATE int sqlite3TwoPartName(Parse *, Token *, Token *, Token **);
@@ -11961,26 +12060,34 @@
** restoring the state of the VM to as it was before the sub-program
** began executing.
**
-** Frames are stored in a linked list headed at Vdbe.pParent. Vdbe.pParent
-** is the parent of the current frame, or zero if the current frame
-** is the main Vdbe program.
+** The memory for a VdbeFrame object is allocated and managed by a memory
+** cell in the parent (calling) frame. When the memory cell is deleted or
+** overwritten, the VdbeFrame object is not freed immediately. Instead, it
+** is linked into the Vdbe.pDelFrame list. The contents of the Vdbe.pDelFrame
+** list is deleted when the VM is reset in VdbeHalt(). The reason for doing
+** this instead of deleting the VdbeFrame immediately is to avoid recursive
+** calls to sqlite3VdbeMemRelease() when the memory cells belonging to the
+** child frame are released.
+**
+** The currently executing frame is stored in Vdbe.pFrame. Vdbe.pFrame is
+** set to NULL if the currently executing frame is the main program.
*/
typedef struct VdbeFrame VdbeFrame;
struct VdbeFrame {
Vdbe *v; /* VM this frame belongs to */
- int pc; /* Program Counter */
- Op *aOp; /* Program instructions */
+ int pc; /* Program Counter in parent (calling) frame */
+ Op *aOp; /* Program instructions for parent frame */
int nOp; /* Size of aOp array */
- Mem *aMem; /* Array of memory cells */
+ Mem *aMem; /* Array of memory cells for parent frame */
int nMem; /* Number of entries in aMem */
- VdbeCursor **apCsr; /* Element of Vdbe cursors */
+ VdbeCursor **apCsr; /* Array of Vdbe cursors for parent frame */
u16 nCursor; /* Number of entries in apCsr */
void *token; /* Copy of SubProgram.token */
int nChildMem; /* Number of memory cells for child frame */
int nChildCsr; /* Number of cursors for child frame */
i64 lastRowid; /* Last insert rowid (sqlite3.lastRowid) */
int nChange; /* Statement changes (Vdbe.nChanges) */
- VdbeFrame *pParent; /* Parent of this frame */
+ VdbeFrame *pParent; /* Parent of this frame, or NULL if parent is main */
};
#define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))])
@@ -12197,6 +12304,7 @@
FILE *trace; /* Write an execution trace here, if not NULL */
#endif
VdbeFrame *pFrame; /* Parent frame */
+ VdbeFrame *pDelFrame; /* List of frame objects to free on VM reset */
int nFrame; /* Number of frames in pFrame list */
u32 expmask; /* Binding to these vars invalidates VM */
SubProgram *pProgram; /* Linked list of all sub-programs used by VM */
@@ -12628,12 +12736,6 @@
}
/*
-** Read text from z[] and convert into a floating point number. Return
-** the number of digits converted.
-*/
-#define getValue sqlite3AtoF
-
-/*
** Parse a timezone extension on the end of a date-time.
** The extension is of the form:
**
@@ -12834,7 +12936,7 @@
const char *zDate,
DateTime *p
){
- int isRealNum; /* Return from sqlite3IsNumber(). Not used */
+ double r;
if( parseYyyyMmDd(zDate,p)==0 ){
return 0;
}else if( parseHhMmSs(zDate, p)==0 ){
@@ -12842,9 +12944,7 @@
}else if( sqlite3StrICmp(zDate,"now")==0){
setDateTimeToCurrent(context, p);
return 0;
- }else if( sqlite3IsNumber(zDate, &isRealNum, SQLITE_UTF8) ){
- double r;
- getValue(zDate, &r);
+ }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8) ){
p->iJD = (sqlite3_int64)(r*86400000.0 + 0.5);
p->validJD = 1;
return 0;
@@ -13065,8 +13165,9 @@
** weekday N where 0==Sunday, 1==Monday, and so forth. If the
** date is already on the appropriate weekday, this is a no-op.
*/
- if( strncmp(z, "weekday ", 8)==0 && getValue(&z[8],&r)>0
- && (n=(int)r)==r && n>=0 && r<7 ){
+ if( strncmp(z, "weekday ", 8)==0
+ && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8)
+ && (n=(int)r)==r && n>=0 && r<7 ){
sqlite3_int64 Z;
computeYMD_HMS(p);
p->validTZ = 0;
@@ -13121,8 +13222,11 @@
case '8':
case '9': {
double rRounder;
- n = getValue(z, &r);
- assert( n>=1 );
+ for(n=1; z[n] && z[n]!=':' && !sqlite3Isspace(z[n]); n++){}
+ if( !sqlite3AtoF(z, &r, n, SQLITE_UTF8) ){
+ rc = 1;
+ break;
+ }
if( z[n]==':' ){
/* A modifier of the form (+|-)HH:MM:SS.FFF adds (or subtracts) the
** specified number of hours, minutes, seconds, and fractional seconds
@@ -17421,7 +17525,7 @@
** sqlite3_soft_heap_limit() setting.
*/
int nearlyFull;
-} mem0 = { 0, 0, 0, 0, 0, 0, 0 };
+} mem0 = { 0, 0, 0, 0, 0, 0, 0, 0 };
#define mem0 GLOBAL(struct Mem0Global, mem0)
@@ -17492,7 +17596,7 @@
sqlite3MemoryAlarm(0, 0, 0);
}
excess = sqlite3_memory_used() - n;
- if( excess>0 ) sqlite3_release_memory(excess & 0x7fffffff);
+ if( excess>0 ) sqlite3_release_memory((int)(excess & 0x7fffffff));
return priorLimit;
}
SQLITE_API void sqlite3_soft_heap_limit(int n){
@@ -20086,121 +20190,111 @@
}
/*
-** Return TRUE if z is a pure numeric string. Return FALSE and leave
-** *realnum unchanged if the string contains any character which is not
-** part of a number.
+** The string z[] is an text representation of a real number.
+** Convert this string to a double and write it into *pResult.
**
-** If the string is pure numeric, set *realnum to TRUE if the string
-** contains the '.' character or an "E+000" style exponentiation suffix.
-** Otherwise set *realnum to FALSE. Note that just becaue *realnum is
-** false does not mean that the number can be successfully converted into
-** an integer - it might be too big.
+** The string z[] is length bytes in length (bytes, not characters) and
+** uses the encoding enc. The string is not necessarily zero-terminated.
**
-** An empty string is considered non-numeric.
+** Return TRUE if the result is a valid real number (or integer) and FALSE
+** if the string is empty or contains extraneous text. Valid numbers
+** are in one of these formats:
+**
+** [+-]digits[E[+-]digits]
+** [+-]digits.[digits][E[+-]digits]
+** [+-].digits[E[+-]digits]
+**
+** Leading and trailing whitespace is ignored for the purpose of determining
+** validity.
+**
+** If some prefix of the input string is a valid number, this routine
+** returns FALSE but it still converts the prefix and writes the result
+** into *pResult.
*/
-SQLITE_PRIVATE int sqlite3IsNumber(const char *z, int *realnum, u8 enc){
+SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){
+#ifndef SQLITE_OMIT_FLOATING_POINT
int incr = (enc==SQLITE_UTF8?1:2);
- if( enc==SQLITE_UTF16BE ) z++;
- if( *z=='-' || *z=='+' ) z += incr;
- if( !sqlite3Isdigit(*z) ){
- return 0;
- }
- z += incr;
- *realnum = 0;
- while( sqlite3Isdigit(*z) ){ z += incr; }
-#ifndef SQLITE_OMIT_FLOATING_POINT
- if( *z=='.' ){
- z += incr;
- if( !sqlite3Isdigit(*z) ) return 0;
- while( sqlite3Isdigit(*z) ){ z += incr; }
- *realnum = 1;
- }
- if( *z=='e' || *z=='E' ){
- z += incr;
- if( *z=='+' || *z=='-' ) z += incr;
- if( !sqlite3Isdigit(*z) ) return 0;
- while( sqlite3Isdigit(*z) ){ z += incr; }
- *realnum = 1;
- }
-#endif
- return *z==0;
-}
-
-/*
-** The string z[] is an ASCII representation of a real number.
-** Convert this string to a double.
-**
-** This routine assumes that z[] really is a valid number. If it
-** is not, the result is undefined.
-**
-** This routine is used instead of the library atof() function because
-** the library atof() might want to use "," as the decimal point instead
-** of "." depending on how locale is set. But that would cause problems
-** for SQL. So this routine always uses "." regardless of locale.
-*/
-SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult){
-#ifndef SQLITE_OMIT_FLOATING_POINT
- const char *zBegin = z;
+ const char *zEnd = z + length;
/* sign * significand * (10 ^ (esign * exponent)) */
- int sign = 1; /* sign of significand */
- i64 s = 0; /* significand */
- int d = 0; /* adjust exponent for shifting decimal point */
- int esign = 1; /* sign of exponent */
- int e = 0; /* exponent */
+ int sign = 1; /* sign of significand */
+ i64 s = 0; /* significand */
+ int d = 0; /* adjust exponent for shifting decimal point */
+ int esign = 1; /* sign of exponent */
+ int e = 0; /* exponent */
+ int eValid = 1; /* True exponent is either not used or is well-formed */
double result;
int nDigits = 0;
+ *pResult = 0.0; /* Default return value, in case of an error */
+
+ if( enc==SQLITE_UTF16BE ) z++;
+
/* skip leading spaces */
- while( sqlite3Isspace(*z) ) z++;
+ while( z<zEnd && sqlite3Isspace(*z) ) z+=incr;
+ if( z>=zEnd ) return 0;
+
/* get sign of significand */
if( *z=='-' ){
sign = -1;
- z++;
+ z+=incr;
}else if( *z=='+' ){
- z++;
+ z+=incr;
}
+
/* skip leading zeroes */
- while( z[0]=='0' ) z++, nDigits++;
+ while( z<zEnd && z[0]=='0' ) z+=incr, nDigits++;
/* copy max significant digits to significand */
- while( sqlite3Isdigit(*z) && s<((LARGEST_INT64-9)/10) ){
+ while( z<zEnd && sqlite3Isdigit(*z) && s<((LARGEST_INT64-9)/10) ){
s = s*10 + (*z - '0');
- z++, nDigits++;
+ z+=incr, nDigits++;
}
+
/* skip non-significant significand digits
** (increase exponent by d to shift decimal left) */
- while( sqlite3Isdigit(*z) ) z++, nDigits++, d++;
+ while( z<zEnd && sqlite3Isdigit(*z) ) z+=incr, nDigits++, d++;
+ if( z>=zEnd ) goto do_atof_calc;
/* if decimal point is present */
if( *z=='.' ){
- z++;
+ z+=incr;
/* copy digits from after decimal to significand
** (decrease exponent by d to shift decimal right) */
- while( sqlite3Isdigit(*z) && s<((LARGEST_INT64-9)/10) ){
+ while( z<zEnd && sqlite3Isdigit(*z) && s<((LARGEST_INT64-9)/10) ){
s = s*10 + (*z - '0');
- z++, nDigits++, d--;
+ z+=incr, nDigits++, d--;
}
/* skip non-significant digits */
- while( sqlite3Isdigit(*z) ) z++, nDigits++;
+ while( z<zEnd && sqlite3Isdigit(*z) ) z+=incr, nDigits++;
}
+ if( z>=zEnd ) goto do_atof_calc;
/* if exponent is present */
if( *z=='e' || *z=='E' ){
- z++;
+ z+=incr;
+ eValid = 0;
+ if( z>=zEnd ) goto do_atof_calc;
/* get sign of exponent */
if( *z=='-' ){
esign = -1;
- z++;
+ z+=incr;
}else if( *z=='+' ){
- z++;
+ z+=incr;
}
/* copy digits to exponent */
- while( sqlite3Isdigit(*z) ){
+ while( z<zEnd && sqlite3Isdigit(*z) ){
e = e*10 + (*z - '0');
- z++;
+ z+=incr;
+ eValid = 1;
}
}
+ /* skip trailing spaces */
+ if( nDigits && eValid ){
+ while( z<zEnd && sqlite3Isspace(*z) ) z+=incr;
+ }
+
+do_atof_calc:
/* adjust exponent by d, and update sign */
e = (e*esign) + d;
if( e<0 ) {
@@ -20259,10 +20353,10 @@
/* store the result */
*pResult = result;
- /* return number of characters used */
- return (int)(z - zBegin);
+ /* return true if number and no extra non-whitespace chracters after */
+ return z>=zEnd && nDigits>0 && eValid;
#else
- return sqlite3Atoi64(z, pResult);
+ return !sqlite3Atoi64(z, pResult, length, enc);
#endif /* SQLITE_OMIT_FLOATING_POINT */
}
@@ -20270,20 +20364,26 @@
** Compare the 19-character string zNum against the text representation
** value 2^63: 9223372036854775808. Return negative, zero, or positive
** if zNum is less than, equal to, or greater than the string.
+** Note that zNum must contain exactly 19 characters.
**
** Unlike memcmp() this routine is guaranteed to return the difference
** in the values of the last digit if the only difference is in the
** last digit. So, for example,
**
-** compare2pow63("9223372036854775800")
+** compare2pow63("9223372036854775800", 1)
**
** will return -8.
*/
-static int compare2pow63(const char *zNum){
- int c;
- c = memcmp(zNum,"922337203685477580",18)*10;
+static int compare2pow63(const char *zNum, int incr){
+ int c = 0;
+ int i;
+ /* 012345678901234567 */
+ const char *pow63 = "922337203685477580";
+ for(i=0; c==0 && i<18; i++){
+ c = (zNum[i*incr]-pow63[i])*10;
+ }
if( c==0 ){
- c = zNum[18] - '8';
+ c = zNum[18*incr] - '8';
testcase( c==(-1) );
testcase( c==0 );
testcase( c==(+1) );
@@ -20293,94 +20393,60 @@
/*
-** Return TRUE if zNum is a 64-bit signed integer and write
-** the value of the integer into *pNum. If zNum is not an integer
-** or is an integer that is too large to be expressed with 64 bits,
-** then return false.
+** Convert zNum to a 64-bit signed integer and write
+** the value of the integer into *pNum.
+** If zNum is exactly 9223372036854665808, return 2.
+** This is a special case as the context will determine
+** if it is too big (used as a negative).
+** If zNum is not an integer or is an integer that
+** is too large to be expressed with 64 bits,
+** then return 1. Otherwise return 0.
**
-** When this routine was originally written it dealt with only
-** 32-bit numbers. At that time, it was much faster than the
-** atoi() library routine in RedHat 7.2.
+** length is the number of bytes in the string (bytes, not characters).
+** The string is not necessarily zero-terminated. The encoding is
+** given by enc.
*/
-SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum){
+SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){
+ int incr = (enc==SQLITE_UTF8?1:2);
i64 v = 0;
- int neg;
- int i, c;
+ int neg = 0; /* assume positive */
+ int i;
+ int c = 0;
const char *zStart;
- while( sqlite3Isspace(*zNum) ) zNum++;
+ const char *zEnd = zNum + length;
+ if( enc==SQLITE_UTF16BE ) zNum++;
+ while( zNum<zEnd && sqlite3Isspace(*zNum) ) zNum+=incr;
+ if( zNum>=zEnd ) goto do_atoi_calc;
if( *zNum=='-' ){
neg = 1;
- zNum++;
+ zNum+=incr;
}else if( *zNum=='+' ){
- neg = 0;
- zNum++;
- }else{
- neg = 0;
+ zNum+=incr;
}
+do_atoi_calc:
zStart = zNum;
- while( zNum[0]=='0' ){ zNum++; } /* Skip over leading zeros. Ticket #2454 */
- for(i=0; (c=zNum[i])>='0' && c<='9'; i++){
+ while( zNum<zEnd && zNum[0]=='0' ){ zNum+=incr; } /* Skip leading zeros. */
+ for(i=0; &zNum[i]<zEnd && (c=zNum[i])>='0' && c<='9'; i+=incr){
v = v*10 + c - '0';
}
*pNum = neg ? -v : v;
testcase( i==18 );
testcase( i==19 );
testcase( i==20 );
- if( c!=0 || (i==0 && zStart==zNum) || i>19 ){
+ if( (c!=0 && &zNum[i]<zEnd) || (i==0 && zStart==zNum) || i>19*incr ){
/* zNum is empty or contains non-numeric text or is longer
- ** than 19 digits (thus guaranting that it is too large) */
- return 0;
- }else if( i<19 ){
- /* Less than 19 digits, so we know that it fits in 64 bits */
+ ** than 19 digits (thus guaranteeing that it is too large) */
return 1;
+ }else if( i<19*incr ){
+ /* Less than 19 digits, so we know that it fits in 64 bits */
+ return 0;
}else{
/* 19-digit numbers must be no larger than 9223372036854775807 if positive
** or 9223372036854775808 if negative. Note that 9223372036854665808
- ** is 2^63. */
- return compare2pow63(zNum)<neg;
- }
-}
-
-/*
-** The string zNum represents an unsigned integer. The zNum string
-** consists of one or more digit characters and is terminated by
-** a zero character. Any stray characters in zNum result in undefined
-** behavior.
-**
-** If the unsigned integer that zNum represents will fit in a
-** 64-bit signed integer, return TRUE. Otherwise return FALSE.
-**
-** If the negFlag parameter is true, that means that zNum really represents
-** a negative number. (The leading "-" is omitted from zNum.) This
-** parameter is needed to determine a boundary case. A string
-** of "9223373036854775808" returns false if negFlag is false or true
-** if negFlag is true.
-**
-** Leading zeros are ignored.
-*/
-SQLITE_PRIVATE int sqlite3FitsIn64Bits(const char *zNum, int negFlag){
- int i;
- int neg = 0;
-
- assert( zNum[0]>='0' && zNum[0]<='9' ); /* zNum is an unsigned number */
-
- if( negFlag ) neg = 1-neg;
- while( *zNum=='0' ){
- zNum++; /* Skip leading zeros. Ticket #2454 */
- }
- for(i=0; zNum[i]; i++){ assert( zNum[i]>='0' && zNum[i]<='9' ); }
- testcase( i==18 );
- testcase( i==19 );
- testcase( i==20 );
- if( i<19 ){
- /* Guaranteed to fit if less than 19 digits */
- return 1;
- }else if( i>19 ){
- /* Guaranteed to be too big if greater than 19 digits */
- return 0;
- }else{
- /* Compare against 2^63. */
- return compare2pow63(zNum)<neg;
+ ** is 2^63. Return 1 if to large */
+ c=compare2pow63(zNum, incr);
+ if( c==0 && neg==0 ) return 2; /* too big, exactly 9223372036854665808 */
+ return c<neg ? 0 : 1;
}
}
@@ -20428,6 +20494,16 @@
}
/*
+** Return a 32-bit integer value extracted from a string. If the
+** string is not an integer, just return 0.
+*/
+SQLITE_PRIVATE int sqlite3Atoi(const char *z){
+ int x = 0;
+ if( z ) sqlite3GetInt32(z, &x);
+ return x;
+}
+
+/*
** The variable-length integer encoding is as follows:
**
** KEY:
@@ -21356,10 +21432,10 @@
/* 132 */ "VRename",
/* 133 */ "VUpdate",
/* 134 */ "Pagecount",
- /* 135 */ "Trace",
- /* 136 */ "Noop",
- /* 137 */ "Explain",
- /* 138 */ "NotUsed_138",
+ /* 135 */ "MaxPgcnt",
+ /* 136 */ "Trace",
+ /* 137 */ "Noop",
+ /* 138 */ "Explain",
/* 139 */ "NotUsed_139",
/* 140 */ "NotUsed_140",
/* 141 */ "ToText",
@@ -26991,11 +27067,21 @@
*/
UNUSED_PARAMETER(isDelete);
+ /* Usually the path zFilename should not be a relative pathname. The
+ ** exception is when opening the proxy "conch" file in builds that
+ ** include the special Apple locking styles.
+ */
+#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE
+ assert( zFilename==0 || zFilename[0]=='/'
+ || pVfs->pAppData==(void*)&autolockIoFinder );
+#else
+ assert( zFilename==0 || zFilename[0]=='/' );
+#endif
+
OSTRACE(("OPEN %-3d %s\n", h, zFilename));
pNew->h = h;
pNew->dirfd = dirfd;
pNew->fileFlags = 0;
- assert( zFilename==0 || zFilename[0]=='/' ); /* Never a relative pathname */
pNew->zPath = zFilename;
#if OS_VXWORKS
@@ -27335,9 +27421,24 @@
int nDb; /* Number of valid bytes in zDb */
struct stat sStat; /* Output of stat() on database file */
- nDb = sqlite3Strlen30(zPath) - ((flags & SQLITE_OPEN_WAL) ? 4 : 8);
+ /* zPath is a path to a WAL or journal file. The following block derives
+ ** the path to the associated database file from zPath. This block handles
+ ** the following naming conventions:
+ **
+ ** "<path to db>-journal"
+ ** "<path to db>-wal"
+ ** "<path to db>-journal-NNNN"
+ ** "<path to db>-wal-NNNN"
+ **
+ ** where NNNN is a 4 digit decimal number. The NNNN naming schemes are
+ ** used by the test_multiplex.c module.
+ */
+ nDb = sqlite3Strlen30(zPath) - 1;
+ while( nDb>0 && zPath[nDb]!='l' ) nDb--;
+ nDb -= ((flags & SQLITE_OPEN_WAL) ? 3 : 7);
memcpy(zDb, zPath, nDb);
zDb[nDb] = '\0';
+
if( 0==stat(zDb, &sStat) ){
*pMode = sStat.st_mode & 0777;
}else{
@@ -27752,7 +27853,7 @@
** error message.
*/
static void unixDlError(sqlite3_vfs *NotUsed, int nBuf, char *zBufOut){
- char *zErr;
+ const char *zErr;
UNUSED_PARAMETER(NotUsed);
unixEnterMutex();
zErr = dlerror();
@@ -27889,7 +27990,7 @@
#if defined(NO_GETTOD)
time_t t;
time(&t);
- *piNow = ((sqlite3_int64)i)*1000 + unixEpoch;
+ *piNow = ((sqlite3_int64)t)*1000 + unixEpoch;
#elif OS_VXWORKS
struct timespec sNow;
clock_gettime(CLOCK_REALTIME, &sNow);
@@ -28346,27 +28447,27 @@
pathLen = strlcpy(tPath, cPath, MAXPATHLEN);
if( pathLen>MAXPATHLEN || pathLen<6 ||
(strlcpy(&tPath[pathLen-5], "break", 6) != 5) ){
- sprintf(errmsg, "path error (len %d)", (int)pathLen);
+ sqlite3_snprintf(sizeof(errmsg),errmsg,"path error (len %d)",(int)pathLen);
goto end_breaklock;
}
/* read the conch content */
readLen = pread(conchFile->h, buf, PROXY_MAXCONCHLEN, 0);
if( readLen<PROXY_PATHINDEX ){
- sprintf(errmsg, "read error (len %d)", (int)readLen);
+ sqlite3_snprintf(sizeof(errmsg),errmsg,"read error (len %d)",(int)readLen);
goto end_breaklock;
}
/* write it out to the temporary break file */
fd = open(tPath, (O_RDWR|O_CREAT|O_EXCL), SQLITE_DEFAULT_FILE_PERMISSIONS);
if( fd<0 ){
- sprintf(errmsg, "create failed (%d)", errno);
+ sqlite3_snprintf(sizeof(errmsg), errmsg, "create failed (%d)", errno);
goto end_breaklock;
}
if( pwrite(fd, buf, readLen, 0) != (ssize_t)readLen ){
- sprintf(errmsg, "write failed (%d)", errno);
+ sqlite3_snprintf(sizeof(errmsg), errmsg, "write failed (%d)", errno);
goto end_breaklock;
}
if( rename(tPath, cPath) ){
- sprintf(errmsg, "rename failed (%d)", errno);
+ sqlite3_snprintf(sizeof(errmsg), errmsg, "rename failed (%d)", errno);
goto end_breaklock;
}
rc = 0;
@@ -34543,6 +34644,7 @@
# define sqlite3WalCheckpoint(u,v,w,x) 0
# define sqlite3WalCallback(z) 0
# define sqlite3WalExclusiveMode(y,z) 0
+# define sqlite3WalHeapMemory(z) 0
#else
#define WAL_SAVEPOINT_NDATA 4
@@ -34553,7 +34655,7 @@
typedef struct Wal Wal;
/* Open and close a connection to a write-ahead log. */
-SQLITE_PRIVATE int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *zName, Wal**);
+SQLITE_PRIVATE int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *zName, int, Wal**);
SQLITE_PRIVATE int sqlite3WalClose(Wal *pWal, int sync_flags, int, u8 *);
/* Used by readers to open (lock) and close (unlock) a snapshot. A
@@ -34610,6 +34712,12 @@
*/
SQLITE_PRIVATE int sqlite3WalExclusiveMode(Wal *pWal, int op);
+/* Return true if the argument is non-NULL and the WAL module is using
+** heap-memory for the wal-index. Otherwise, if the argument is NULL or the
+** WAL module is using shared-memory, return false.
+*/
+SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal);
+
#endif /* ifndef SQLITE_OMIT_WAL */
#endif /* _WAL_H_ */
@@ -35209,7 +35317,8 @@
u8 noReadlock; /* Do not bother to obtain readlocks */
u8 noSync; /* Do not sync the journal if true */
u8 fullSync; /* Do extra syncs of the journal for robustness */
- u8 sync_flags; /* One of SYNC_NORMAL or SYNC_FULL */
+ u8 ckptSyncFlags; /* SYNC_NORMAL or SYNC_FULL for checkpoint */
+ u8 syncFlags; /* SYNC_NORMAL or SYNC_FULL otherwise */
u8 tempFile; /* zFilename is a temporary file */
u8 readOnly; /* True for a read-only database */
u8 memDb; /* True to inhibit all file I/O */
@@ -35520,7 +35629,9 @@
return 1;
}
+#endif /* ifndef NDEBUG */
+#ifdef SQLITE_DEBUG
/*
** Return a pointer to a human readable string in a static buffer
** containing the state of the Pager object passed as an argument. This
@@ -35644,7 +35755,7 @@
static int pagerUnlockDb(Pager *pPager, int eLock){
int rc = SQLITE_OK;
- assert( !pPager->exclusiveMode );
+ assert( !pPager->exclusiveMode || pPager->eLock==eLock );
assert( eLock==NO_LOCK || eLock==SHARED_LOCK );
assert( eLock!=NO_LOCK || pagerUseWal(pPager)==0 );
if( isOpen(pPager->fd) ){
@@ -35891,7 +36002,7 @@
rc = sqlite3OsWrite(pPager->jfd, zeroHdr, sizeof(zeroHdr), 0);
}
if( rc==SQLITE_OK && !pPager->noSync ){
- rc = sqlite3OsSync(pPager->jfd, SQLITE_SYNC_DATAONLY|pPager->sync_flags);
+ rc = sqlite3OsSync(pPager->jfd, SQLITE_SYNC_DATAONLY|pPager->syncFlags);
}
/* At this point the transaction is committed but the write lock
@@ -37343,7 +37454,7 @@
if( rc==SQLITE_OK && !pPager->noSync
&& (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN)
){
- rc = sqlite3OsSync(pPager->fd, pPager->sync_flags);
+ rc = sqlite3OsSync(pPager->fd, pPager->syncFlags);
}
if( rc==SQLITE_OK ){
rc = pager_end_transaction(pPager, zMaster[0]!='\0');
@@ -37517,13 +37628,13 @@
PgHdr *pList, /* List of frames to log */
Pgno nTruncate, /* Database size after this commit */
int isCommit, /* True if this is a commit */
- int sync_flags /* Flags to pass to OsSync() (or 0) */
+ int syncFlags /* Flags to pass to OsSync() (or 0) */
){
int rc; /* Return code */
assert( pPager->pWal );
rc = sqlite3WalFrames(pPager->pWal,
- pPager->pageSize, pList, nTruncate, isCommit, sync_flags
+ pPager->pageSize, pList, nTruncate, isCommit, syncFlags
);
if( rc==SQLITE_OK && pPager->pBackup ){
PgHdr *p;
@@ -37853,14 +37964,49 @@
** assurance that the journal will not be corrupted to the
** point of causing damage to the database during rollback.
**
+** The above is for a rollback-journal mode. For WAL mode, OFF continues
+** to mean that no syncs ever occur. NORMAL means that the WAL is synced
+** prior to the start of checkpoint and that the database file is synced
+** at the conclusion of the checkpoint if the entire content of the WAL
+** was written back into the database. But no sync operations occur for
+** an ordinary commit in NORMAL mode with WAL. FULL means that the WAL
+** file is synced following each commit operation, in addition to the
+** syncs associated with NORMAL.
+**
+** Do not confuse synchronous=FULL with SQLITE_SYNC_FULL. The
+** SQLITE_SYNC_FULL macro means to use the MacOSX-style full-fsync
+** using fcntl(F_FULLFSYNC). SQLITE_SYNC_NORMAL means to do an
+** ordinary fsync() call. There is no difference between SQLITE_SYNC_FULL
+** and SQLITE_SYNC_NORMAL on platforms other than MacOSX. But the
+** synchronous=FULL versus synchronous=NORMAL setting determines when
+** the xSync primitive is called and is relevant to all platforms.
+**
** Numeric values associated with these states are OFF==1, NORMAL=2,
** and FULL=3.
*/
#ifndef SQLITE_OMIT_PAGER_PRAGMAS
-SQLITE_PRIVATE void sqlite3PagerSetSafetyLevel(Pager *pPager, int level, int bFullFsync){
+SQLITE_PRIVATE void sqlite3PagerSetSafetyLevel(
+ Pager *pPager, /* The pager to set safety level for */
+ int level, /* PRAGMA synchronous. 1=OFF, 2=NORMAL, 3=FULL */
+ int bFullFsync, /* PRAGMA fullfsync */
+ int bCkptFullFsync /* PRAGMA checkpoint_fullfsync */
+){
+ assert( level>=1 && level<=3 );
pPager->noSync = (level==1 || pPager->tempFile) ?1:0;
pPager->fullSync = (level==3 && !pPager->tempFile) ?1:0;
- pPager->sync_flags = (bFullFsync?SQLITE_SYNC_FULL:SQLITE_SYNC_NORMAL);
+ if( pPager->noSync ){
+ pPager->syncFlags = 0;
+ pPager->ckptSyncFlags = 0;
+ }else if( bFullFsync ){
+ pPager->syncFlags = SQLITE_SYNC_FULL;
+ pPager->ckptSyncFlags = SQLITE_SYNC_FULL;
+ }else if( bCkptFullFsync ){
+ pPager->syncFlags = SQLITE_SYNC_NORMAL;
+ pPager->ckptSyncFlags = SQLITE_SYNC_FULL;
+ }else{
+ pPager->syncFlags = SQLITE_SYNC_NORMAL;
+ pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL;
+ }
}
#endif
@@ -38039,9 +38185,8 @@
if( mxPage>0 ){
pPager->mxPgno = mxPage;
}
- if( pPager->eState!=PAGER_OPEN && pPager->mxPgno<pPager->dbSize ){
- pPager->mxPgno = pPager->dbSize;
- }
+ assert( pPager->eState!=PAGER_OPEN ); /* Called only by OP_MaxPgcnt */
+ assert( pPager->mxPgno>=pPager->dbSize ); /* OP_MaxPgcnt enforces this */
return pPager->mxPgno;
}
@@ -38246,10 +38391,7 @@
/* pPager->errCode = 0; */
pPager->exclusiveMode = 0;
#ifndef SQLITE_OMIT_WAL
- sqlite3WalClose(pPager->pWal,
- (pPager->noSync ? 0 : pPager->sync_flags),
- pPager->pageSize, pTmp
- );
+ sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, pPager->pageSize, pTmp);
pPager->pWal = 0;
#endif
pager_reset(pPager);
@@ -38415,7 +38557,7 @@
if( pPager->fullSync && 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){
PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager)));
IOTRACE(("JSYNC %p\n", pPager))
- rc = sqlite3OsSync(pPager->jfd, pPager->sync_flags);
+ rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags);
if( rc!=SQLITE_OK ) return rc;
}
IOTRACE(("JHDR %p %lld\n", pPager, pPager->journalHdr));
@@ -38427,8 +38569,8 @@
if( 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){
PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager)));
IOTRACE(("JSYNC %p\n", pPager))
- rc = sqlite3OsSync(pPager->jfd, pPager->sync_flags|
- (pPager->sync_flags==SQLITE_SYNC_FULL?SQLITE_SYNC_DATAONLY:0)
+ rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags|
+ (pPager->syncFlags==SQLITE_SYNC_FULL?SQLITE_SYNC_DATAONLY:0)
);
if( rc!=SQLITE_OK ) return rc;
}
@@ -39018,7 +39160,8 @@
assert( useJournal || pPager->tempFile );
pPager->noSync = pPager->tempFile;
pPager->fullSync = pPager->noSync ?0:1;
- pPager->sync_flags = SQLITE_SYNC_NORMAL;
+ pPager->syncFlags = pPager->noSync ? 0 : SQLITE_SYNC_NORMAL;
+ pPager->ckptSyncFlags = pPager->syncFlags;
/* pPager->pFirst = 0; */
/* pPager->pFirstSynced = 0; */
/* pPager->pLast = 0; */
@@ -39118,7 +39261,7 @@
sqlite3BeginBenignMalloc();
if( pagerLockDb(pPager, RESERVED_LOCK)==SQLITE_OK ){
sqlite3OsDelete(pVfs, pPager->zJournal, 0);
- pagerUnlockDb(pPager, SHARED_LOCK);
+ if( !pPager->exclusiveMode ) pagerUnlockDb(pPager, SHARED_LOCK);
}
sqlite3EndBenignMalloc();
}else{
@@ -39799,29 +39942,29 @@
CHECK_PAGE(pPg);
+ /* The journal file needs to be opened. Higher level routines have already
+ ** obtained the necessary locks to begin the write-transaction, but the
+ ** rollback journal might not yet be open. Open it now if this is the case.
+ **
+ ** This is done before calling sqlite3PcacheMakeDirty() on the page.
+ ** Otherwise, if it were done after calling sqlite3PcacheMakeDirty(), then
+ ** an error might occur and the pager would end up in WRITER_LOCKED state
+ ** with pages marked as dirty in the cache.
+ */
+ if( pPager->eState==PAGER_WRITER_LOCKED ){
+ rc = pager_open_journal(pPager);
+ if( rc!=SQLITE_OK ) return rc;
+ }
+ assert( pPager->eState>=PAGER_WRITER_CACHEMOD );
+ assert( assert_pager_state(pPager) );
+
/* Mark the page as dirty. If the page has already been written
** to the journal then we can return right away.
*/
sqlite3PcacheMakeDirty(pPg);
if( pageInJournal(pPg) && !subjRequiresPage(pPg) ){
assert( !pagerUseWal(pPager) );
- assert( pPager->eState>=PAGER_WRITER_CACHEMOD );
}else{
-
- /* If we get this far, it means that the page needs to be
- ** written to the transaction journal or the checkpoint journal
- ** or both.
- **
- ** Higher level routines have already obtained the necessary locks
- ** to begin the write-transaction, but the rollback journal might not
- ** yet be open. Open it now if this is the case.
- */
- if( pPager->eState==PAGER_WRITER_LOCKED ){
- rc = pager_open_journal(pPager);
- if( rc!=SQLITE_OK ) return rc;
- }
- assert( pPager->eState>=PAGER_WRITER_CACHEMOD );
- assert( assert_pager_state(pPager) );
/* The transaction journal now exists and we have a RESERVED or an
** EXCLUSIVE lock on the main database file. Write the current page to
@@ -40141,10 +40284,10 @@
}
/*
-** Sync the pager file to disk. This is a no-op for in-memory files
+** Sync the database file to disk. This is a no-op for in-memory databases
** or pages with the Pager.noSync flag set.
**
-** If successful, or called on a pager for which it is a no-op, this
+** If successful, or if called on a pager for which it is a no-op, this
** function returns SQLITE_OK. Otherwise, an IO error code is returned.
*/
SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager){
@@ -40153,7 +40296,7 @@
if( pPager->noSync ){
rc = SQLITE_OK;
}else{
- rc = sqlite3OsSync(pPager->fd, pPager->sync_flags);
+ rc = sqlite3OsSync(pPager->fd, pPager->syncFlags);
}
return rc;
}
@@ -40242,7 +40385,7 @@
PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache);
if( pList ){
rc = pagerWalFrames(pPager, pList, pPager->dbSize, 1,
- (pPager->fullSync ? pPager->sync_flags : 0)
+ (pPager->fullSync ? pPager->syncFlags : 0)
);
}
if( rc==SQLITE_OK ){
@@ -40373,7 +40516,7 @@
/* Finally, sync the database file. */
if( !pPager->noSync && !noSync ){
- rc = sqlite3OsSync(pPager->fd, pPager->sync_flags);
+ rc = sqlite3OsSync(pPager->fd, pPager->syncFlags);
}
IOTRACE(("DBSYNC %p\n", pPager))
}
@@ -40944,7 +41087,8 @@
|| eMode==PAGER_LOCKINGMODE_EXCLUSIVE );
assert( PAGER_LOCKINGMODE_QUERY<0 );
assert( PAGER_LOCKINGMODE_NORMAL>=0 && PAGER_LOCKINGMODE_EXCLUSIVE>=0 );
- if( eMode>=0 && !pPager->tempFile ){
+ assert( pPager->exclusiveMode || 0==sqlite3WalHeapMemory(pPager->pWal) );
+ if( eMode>=0 && !pPager->tempFile && !sqlite3WalHeapMemory(pPager->pWal) ){
pPager->exclusiveMode = (u8)eMode;
}
return (int)pPager->exclusiveMode;
@@ -41113,10 +41257,8 @@
int rc = SQLITE_OK;
if( pPager->pWal ){
u8 *zBuf = (u8 *)pPager->pTmpSpace;
- rc = sqlite3WalCheckpoint(pPager->pWal,
- (pPager->noSync ? 0 : pPager->sync_flags),
- pPager->pageSize, zBuf
- );
+ rc = sqlite3WalCheckpoint(pPager->pWal, pPager->ckptSyncFlags,
+ pPager->pageSize, zBuf);
}
return rc;
}
@@ -41131,10 +41273,62 @@
*/
SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager){
const sqlite3_io_methods *pMethods = pPager->fd->pMethods;
- return pMethods->iVersion>=2 && pMethods->xShmMap!=0;
+ return pPager->exclusiveMode || (pMethods->iVersion>=2 && pMethods->xShmMap);
}
/*
+** Attempt to take an exclusive lock on the database file. If a PENDING lock
+** is obtained instead, immediately release it.
+*/
+static int pagerExclusiveLock(Pager *pPager){
+ int rc; /* Return code */
+
+ assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK );
+ rc = pagerLockDb(pPager, EXCLUSIVE_LOCK);
+ if( rc!=SQLITE_OK ){
+ /* If the attempt to grab the pending lock failed, release the
+ ** exclusive lock that may have been obtained instead. */
+ pagerUnlockDb(pPager, SHARED_LOCK);
+ }
+
+ return rc;
+}
+
+/*
+** Call sqlite3WalOpen() to open the WAL handle. If the pager is in
+** exclusive-locking mode when this function is called, take an EXCLUSIVE
+** lock on the database file and use heap-memory to store the wal-index
+** in. Otherwise, use the normal shared-memory.
+*/
+static int pagerOpenWal(Pager *pPager){
+ int rc = SQLITE_OK;
+
+ assert( pPager->pWal==0 && pPager->tempFile==0 );
+ assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK || pPager->noReadlock);
+
+ /* If the pager is already in exclusive-mode, the WAL module will use
+ ** heap-memory for the wal-index instead of the VFS shared-memory
+ ** implementation. Take the exclusive lock now, before opening the WAL
+ ** file, to make sure this is safe.
+ */
+ if( pPager->exclusiveMode ){
+ rc = pagerExclusiveLock(pPager);
+ }
+
+ /* Open the connection to the log file. If this operation fails,
+ ** (e.g. due to malloc() failure), return an error code.
+ */
+ if( rc==SQLITE_OK ){
+ rc = sqlite3WalOpen(pPager->pVfs,
+ pPager->fd, pPager->zWal, pPager->exclusiveMode, &pPager->pWal
+ );
+ }
+
+ return rc;
+}
+
+
+/*
** The caller must be holding a SHARED lock on the database file to call
** this function.
**
@@ -41167,11 +41361,7 @@
/* Close any rollback journal previously open */
sqlite3OsClose(pPager->jfd);
- /* Open the connection to the log file. If this operation fails,
- ** (e.g. due to malloc() failure), unlock the database file and
- ** return an error code.
- */
- rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, pPager->zWal, &pPager->pWal);
+ rc = pagerOpenWal(pPager);
if( rc==SQLITE_OK ){
pPager->journalMode = PAGER_JOURNALMODE_WAL;
pPager->eState = PAGER_OPEN;
@@ -41210,8 +41400,7 @@
);
}
if( rc==SQLITE_OK && logexists ){
- rc = sqlite3WalOpen(pPager->pVfs, pPager->fd,
- pPager->zWal, &pPager->pWal);
+ rc = pagerOpenWal(pPager);
}
}
@@ -41219,17 +41408,11 @@
** the database file, the log and log-summary files will be deleted.
*/
if( rc==SQLITE_OK && pPager->pWal ){
- rc = pagerLockDb(pPager, EXCLUSIVE_LOCK);
+ rc = pagerExclusiveLock(pPager);
if( rc==SQLITE_OK ){
- rc = sqlite3WalClose(pPager->pWal,
- (pPager->noSync ? 0 : pPager->sync_flags),
- pPager->pageSize, (u8*)pPager->pTmpSpace
- );
+ rc = sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags,
+ pPager->pageSize, (u8*)pPager->pTmpSpace);
pPager->pWal = 0;
- }else{
- /* If we cannot get an EXCLUSIVE lock, downgrade the PENDING lock
- ** that we did get back to SHARED. */
- pagerUnlockDb(pPager, SQLITE_LOCK_SHARED);
}
}
return rc;
@@ -41686,6 +41869,13 @@
};
/*
+** Candidate values for Wal.exclusiveMode.
+*/
+#define WAL_NORMAL_MODE 0
+#define WAL_EXCLUSIVE_MODE 1
+#define WAL_HEAPMEMORY_MODE 2
+
+/*
** Each page of the wal-index mapping contains a hash-table made up of
** an array of HASHTABLE_NSLOT elements of the following type.
*/
@@ -41771,9 +41961,14 @@
/* Request a pointer to the required page from the VFS */
if( pWal->apWiData[iPage]==0 ){
- rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ,
- pWal->writeLock, (void volatile **)&pWal->apWiData[iPage]
- );
+ if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){
+ pWal->apWiData[iPage] = (u32 volatile *)sqlite3MallocZero(WALINDEX_PGSZ);
+ if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ,
+ pWal->writeLock, (void volatile **)&pWal->apWiData[iPage]
+ );
+ }
}
*ppPage = pWal->apWiData[iPage];
@@ -41856,6 +42051,12 @@
aOut[1] = s2;
}
+static void walShmBarrier(Wal *pWal){
+ if( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE ){
+ sqlite3OsShmBarrier(pWal->pDbFd);
+ }
+}
+
/*
** Write the header information in pWal->hdr into the wal-index.
**
@@ -41870,7 +42071,7 @@
pWal->hdr.iVersion = WALINDEX_MAX_VERSION;
walChecksumBytes(1, (u8*)&pWal->hdr, nCksum, 0, pWal->hdr.aCksum);
memcpy((void *)&aHdr[1], (void *)&pWal->hdr, sizeof(WalIndexHdr));
- sqlite3OsShmBarrier(pWal->pDbFd);
+ walShmBarrier(pWal);
memcpy((void *)&aHdr[0], (void *)&pWal->hdr, sizeof(WalIndexHdr));
}
@@ -42442,7 +42643,15 @@
** Close an open wal-index.
*/
static void walIndexClose(Wal *pWal, int isDelete){
- sqlite3OsShmUnmap(pWal->pDbFd, isDelete);
+ if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){
+ int i;
+ for(i=0; i<pWal->nWiData; i++){
+ sqlite3_free((void *)pWal->apWiData[i]);
+ pWal->apWiData[i] = 0;
+ }
+ }else{
+ sqlite3OsShmUnmap(pWal->pDbFd, isDelete);
+ }
}
/*
@@ -42464,6 +42673,7 @@
sqlite3_vfs *pVfs, /* vfs module to open wal and wal-index */
sqlite3_file *pDbFd, /* The open database file */
const char *zWalName, /* Name of the WAL file */
+ int bNoShm, /* True to run in heap-memory mode */
Wal **ppWal /* OUT: Allocated Wal handle */
){
int rc; /* Return Code */
@@ -42497,6 +42707,7 @@
pRet->pDbFd = pDbFd;
pRet->readLock = -1;
pRet->zWalName = zWalName;
+ pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE);
/* Open file handle on the write-ahead log file. */
flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL);
@@ -42930,7 +43141,9 @@
*/
rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE);
if( rc==SQLITE_OK ){
- pWal->exclusiveMode = 1;
+ if( pWal->exclusiveMode==WAL_NORMAL_MODE ){
+ pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
+ }
rc = sqlite3WalCheckpoint(pWal, sync_flags, nBuf, zBuf);
if( rc==SQLITE_OK ){
isDelete = 1;
@@ -42986,7 +43199,7 @@
*/
aHdr = walIndexHdr(pWal);
memcpy(&h1, (void *)&aHdr[0], sizeof(h1));
- sqlite3OsShmBarrier(pWal->pDbFd);
+ walShmBarrier(pWal);
memcpy(&h2, (void *)&aHdr[1], sizeof(h2));
if( memcmp(&h1, &h2, sizeof(h1))!=0 ){
@@ -43187,7 +43400,7 @@
** and can be safely ignored.
*/
rc = walLockShared(pWal, WAL_READ_LOCK(0));
- sqlite3OsShmBarrier(pWal->pDbFd);
+ walShmBarrier(pWal);
if( rc==SQLITE_OK ){
if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){
/* It is not safe to allow the reader to continue here if frames
@@ -43281,7 +43494,7 @@
** log-wrap (either of which would require an exclusive lock on
** WAL_READ_LOCK(mxI)) has not occurred since the snapshot was valid.
*/
- sqlite3OsShmBarrier(pWal->pDbFd);
+ walShmBarrier(pWal);
if( pInfo->aReadMark[mxI]!=mxReadMark
|| memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr))
){
@@ -43623,7 +43836,7 @@
**
** SQLITE_OK is returned if no error is encountered (regardless of whether
** or not pWal->hdr.mxFrame is modified). An SQLite error code is returned
-** if some error
+** if an error occurs.
*/
static int walRestartLog(Wal *pWal){
int rc = SQLITE_OK;
@@ -43656,6 +43869,8 @@
for(i=1; i<WAL_NREADER; i++) pInfo->aReadMark[i] = READMARK_NOT_USED;
assert( pInfo->aReadMark[0]==0 );
walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1);
+ }else if( rc!=SQLITE_BUSY ){
+ return rc;
}
}
walUnlockShared(pWal, WAL_READ_LOCK(0));
@@ -43922,13 +44137,14 @@
** on the main database file before invoking this operation.
**
** If op is negative, then do a dry-run of the op==1 case but do
-** not actually change anything. The pager uses this to see if it
+** not actually change anything. The pager uses this to see if it
** should acquire the database exclusive lock prior to invoking
** the op==1 case.
*/
SQLITE_PRIVATE int sqlite3WalExclusiveMode(Wal *pWal, int op){
int rc;
assert( pWal->writeLock==0 );
+ assert( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE || op==-1 );
/* pWal->readLock is usually set, but might be -1 if there was a
** prior error while attempting to acquire are read-lock. This cannot
@@ -43962,6 +44178,15 @@
return rc;
}
+/*
+** Return true if the argument is non-NULL and the WAL module is using
+** heap-memory for the wal-index. Otherwise, if the argument is NULL or the
+** WAL module is using shared-memory, return false.
+*/
+SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal){
+ return (pWal && pWal->exclusiveMode==WAL_HEAPMEMORY_MODE );
+}
+
#endif /* #ifndef SQLITE_OMIT_WAL */
/************** End of wal.c *************************************************/
@@ -46685,17 +46910,13 @@
const int isTempDb = zFilename==0 || zFilename[0]==0;
/* Set the variable isMemdb to true for an in-memory database, or
- ** false for a file-based database. This symbol is only required if
- ** either of the shared-data or autovacuum features are compiled
- ** into the library.
+ ** false for a file-based database.
*/
-#if !defined(SQLITE_OMIT_SHARED_CACHE) || !defined(SQLITE_OMIT_AUTOVACUUM)
- #ifdef SQLITE_OMIT_MEMORYDB
- const int isMemdb = 0;
- #else
- const int isMemdb = (zFilename && strcmp(zFilename, ":memory:")==0)
- || (isTempDb && sqlite3TempInMemory(db));
- #endif
+#ifdef SQLITE_OMIT_MEMORYDB
+ const int isMemdb = 0;
+#else
+ const int isMemdb = (zFilename && strcmp(zFilename, ":memory:")==0)
+ || (isTempDb && sqlite3TempInMemory(db));
#endif
assert( db!=0 );
@@ -47079,11 +47300,17 @@
** probability of damage to near zero but with a write performance reduction.
*/
#ifndef SQLITE_OMIT_PAGER_PRAGMAS
-SQLITE_PRIVATE int sqlite3BtreeSetSafetyLevel(Btree *p, int level, int fullSync){
+SQLITE_PRIVATE int sqlite3BtreeSetSafetyLevel(
+ Btree *p, /* The btree to set the safety level on */
+ int level, /* PRAGMA synchronous. 1=OFF, 2=NORMAL, 3=FULL */
+ int fullSync, /* PRAGMA fullfsync. */
+ int ckptFullSync /* PRAGMA checkpoint_fullfync */
+){
BtShared *pBt = p->pBt;
assert( sqlite3_mutex_held(p->db->mutex) );
+ assert( level>=1 && level<=3 );
sqlite3BtreeEnter(p);
- sqlite3PagerSetSafetyLevel(pBt->pPager, level, fullSync);
+ sqlite3PagerSetSafetyLevel(pBt->pPager, level, fullSync, ckptFullSync);
sqlite3BtreeLeave(p);
return SQLITE_OK;
}
@@ -48141,8 +48368,8 @@
** are no active cursors, it also releases the read lock.
*/
SQLITE_PRIVATE int sqlite3BtreeCommitPhaseTwo(Btree *p){
- BtShared *pBt = p->pBt;
+ if( p->inTrans==TRANS_NONE ) return SQLITE_OK;
sqlite3BtreeEnter(p);
btreeIntegrity(p);
@@ -48151,6 +48378,7 @@
*/
if( p->inTrans==TRANS_WRITE ){
int rc;
+ BtShared *pBt = p->pBt;
assert( pBt->inTransaction==TRANS_WRITE );
assert( pBt->nTransaction>0 );
rc = sqlite3PagerCommitPhaseTwo(pBt->pPager);
@@ -53069,8 +53297,7 @@
SQLITE_PRIVATE void sqlite3BtreeCacheOverflow(BtCursor *pCur){
assert( cursorHoldsMutex(pCur) );
assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) );
- assert(!pCur->isIncrblobHandle);
- assert(!pCur->aOverflow);
+ invalidateOverflowCache(pCur);
pCur->isIncrblobHandle = 1;
}
#endif
@@ -54139,13 +54366,9 @@
return doubleToInt64(pMem->r);
}else if( flags & (MEM_Str|MEM_Blob) ){
i64 value;
- pMem->flags |= MEM_Str;
- if( sqlite3VdbeChangeEncoding(pMem, SQLITE_UTF8)
- || sqlite3VdbeMemNulTerminate(pMem) ){
- return 0;
- }
- assert( pMem->z );
- sqlite3Atoi64(pMem->z, &value);
+ assert( pMem->z || pMem->n==0 );
+ testcase( pMem->z==0 );
+ sqlite3Atoi64(pMem->z, &value, pMem->n, pMem->enc);
return value;
}else{
return 0;
@@ -54168,14 +54391,7 @@
}else if( pMem->flags & (MEM_Str|MEM_Blob) ){
/* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */
double val = (double)0;
- pMem->flags |= MEM_Str;
- if( sqlite3VdbeChangeEncoding(pMem, SQLITE_UTF8)
- || sqlite3VdbeMemNulTerminate(pMem) ){
- /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */
- return (double)0;
- }
- assert( pMem->z );
- sqlite3AtoF(pMem->z, &val);
+ sqlite3AtoF(pMem->z, &val, pMem->n, pMem->enc);
return val;
}else{
/* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */
@@ -54248,21 +54464,19 @@
** as much of the string as we can and ignore the rest.
*/
SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){
- int rc;
- assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))==0 );
- assert( (pMem->flags & (MEM_Blob|MEM_Str))!=0 );
- assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) );
- rc = sqlite3VdbeChangeEncoding(pMem, SQLITE_UTF8);
- if( rc ) return rc;
- rc = sqlite3VdbeMemNulTerminate(pMem);
- if( rc ) return rc;
- if( sqlite3Atoi64(pMem->z, &pMem->u.i) ){
- MemSetTypeFlag(pMem, MEM_Int);
- }else{
- pMem->r = sqlite3VdbeRealValue(pMem);
- MemSetTypeFlag(pMem, MEM_Real);
- sqlite3VdbeIntegerAffinity(pMem);
+ if( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))==0 ){
+ assert( (pMem->flags & (MEM_Blob|MEM_Str))!=0 );
+ assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) );
+ if( 0==sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc) ){
+ MemSetTypeFlag(pMem, MEM_Int);
+ }else{
+ pMem->r = sqlite3VdbeRealValue(pMem);
+ MemSetTypeFlag(pMem, MEM_Real);
+ sqlite3VdbeIntegerAffinity(pMem);
+ }
}
+ assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))!=0 );
+ pMem->flags &= ~(MEM_Str|MEM_Blob);
return SQLITE_OK;
}
@@ -54271,7 +54485,9 @@
*/
SQLITE_PRIVATE void sqlite3VdbeMemSetNull(Mem *pMem){
if( pMem->flags & MEM_Frame ){
- sqlite3VdbeFrameDelete(pMem->u.pFrame);
+ VdbeFrame *pFrame = pMem->u.pFrame;
+ pFrame->pParent = pFrame->v->pDelFrame;
+ pFrame->v->pDelFrame = pFrame;
}
if( pMem->flags & MEM_RowSet ){
sqlite3RowSetClear(pMem->u.pRowSet);
@@ -54805,6 +55021,8 @@
int op;
char *zVal = 0;
sqlite3_value *pVal = 0;
+ int negInt = 1;
+ const char *zNeg = "";
if( !pExpr ){
*ppVal = 0;
@@ -54822,13 +55040,24 @@
if( NEVER(op==TK_REGISTER) ) op = pExpr->op2;
#endif
+ /* Handle negative integers in a single step. This is needed in the
+ ** case when the value is -9223372036854775808.
+ */
+ if( op==TK_UMINUS
+ && (pExpr->pLeft->op==TK_INTEGER || pExpr->pLeft->op==TK_FLOAT) ){
+ pExpr = pExpr->pLeft;
+ op = pExpr->op;
+ negInt = -1;
+ zNeg = "-";
+ }
+
if( op==TK_STRING || op==TK_FLOAT || op==TK_INTEGER ){
pVal = sqlite3ValueNew(db);
if( pVal==0 ) goto no_mem;
if( ExprHasProperty(pExpr, EP_IntValue) ){
- sqlite3VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue);
+ sqlite3VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue*negInt);
}else{
- zVal = sqlite3DbStrDup(db, pExpr->u.zToken);
+ zVal = sqlite3MPrintf(db, "%s%s", zNeg, pExpr->u.zToken);
if( zVal==0 ) goto no_mem;
sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC);
if( op==TK_FLOAT ) pVal->type = SQLITE_FLOAT;
@@ -54838,14 +55067,18 @@
}else{
sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8);
}
+ if( pVal->flags & (MEM_Int|MEM_Real) ) pVal->flags &= ~MEM_Str;
if( enc!=SQLITE_UTF8 ){
sqlite3VdbeChangeEncoding(pVal, enc);
}
}else if( op==TK_UMINUS ) {
+ /* This branch happens for multiple negative signs. Ex: -(-5) */
if( SQLITE_OK==sqlite3ValueFromExpr(db,pExpr->pLeft,enc,affinity,&pVal) ){
+ sqlite3VdbeMemNumerify(pVal);
pVal->u.i = -1 * pVal->u.i;
/* (double)-1 In case of SQLITE_OMIT_FLOATING_POINT... */
pVal->r = (double)-1 * pVal->r;
+ sqlite3ValueApplyAffinity(pVal, affinity, enc);
}
}
#ifndef SQLITE_OMIT_BLOB_LITERAL
@@ -56099,12 +56332,10 @@
pMem->type = SQLITE_INTEGER;
pMem++;
- if( p->explain==1 ){
- pMem->flags = MEM_Int;
- pMem->u.i = pOp->p3; /* P3 */
- pMem->type = SQLITE_INTEGER;
- pMem++;
- }
+ pMem->flags = MEM_Int;
+ pMem->u.i = pOp->p3; /* P3 */
+ pMem->type = SQLITE_INTEGER;
+ pMem++;
if( sqlite3VdbeMemGrow(pMem, 32, 0) ){ /* P4 */
assert( p->db->mallocFailed );
@@ -56149,7 +56380,7 @@
}
}
- p->nResColumn = 8 - 5*(p->explain-1);
+ p->nResColumn = 8 - 4*(p->explain-1);
p->rc = SQLITE_OK;
rc = SQLITE_ROW;
}
@@ -56456,6 +56687,11 @@
if( p->aMem ){
releaseMemArray(&p->aMem[1], p->nMem);
}
+ while( p->pDelFrame ){
+ VdbeFrame *pDel = p->pDelFrame;
+ p->pDelFrame = pDel->pParent;
+ sqlite3VdbeFrameDelete(pDel);
+ }
}
/*
@@ -58139,6 +58375,8 @@
SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt){
int rc;
if( pStmt==0 ){
+ /* IMPLEMENTATION-OF: R-57228-12904 Invoking sqlite3_finalize() on a NULL
+ ** pointer is a harmless no-op. */
rc = SQLITE_OK;
}else{
Vdbe *v = (Vdbe*)pStmt;
@@ -59094,6 +59332,8 @@
rc = sqlite3ApiExit(p->db, rc);
}
sqlite3_mutex_leave(p->db->mutex);
+ }else if( xDel!=SQLITE_STATIC && xDel!=SQLITE_TRANSIENT ){
+ xDel((void*)zData);
}
return rc;
}
@@ -59337,6 +59577,14 @@
}
/*
+** Return true if the prepared statement is guaranteed to not modify the
+** database.
+*/
+SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt){
+ return pStmt ? ((Vdbe*)pStmt)->readOnly : 1;
+}
+
+/*
** Return a pointer to the next prepared statement after pStmt associated
** with database connection pDb. If pStmt is NULL, return the first
** prepared statement for the database connection. Return NULL if there
@@ -59758,31 +60006,17 @@
*/
static void applyNumericAffinity(Mem *pRec){
if( (pRec->flags & (MEM_Real|MEM_Int))==0 ){
- int realnum;
+ double rValue;
+ i64 iValue;
u8 enc = pRec->enc;
- sqlite3VdbeMemNulTerminate(pRec);
- if( (pRec->flags&MEM_Str) && sqlite3IsNumber(pRec->z, &realnum, enc) ){
- i64 value;
- char *zUtf8 = pRec->z;
-#ifndef SQLITE_OMIT_UTF16
- if( enc!=SQLITE_UTF8 ){
- assert( pRec->db );
- zUtf8 = sqlite3Utf16to8(pRec->db, pRec->z, pRec->n, enc);
- if( !zUtf8 ) return;
- }
-#endif
- if( !realnum && sqlite3Atoi64(zUtf8, &value) ){
- pRec->u.i = value;
- MemSetTypeFlag(pRec, MEM_Int);
- }else{
- sqlite3AtoF(zUtf8, &pRec->r);
- MemSetTypeFlag(pRec, MEM_Real);
- }
-#ifndef SQLITE_OMIT_UTF16
- if( enc!=SQLITE_UTF8 ){
- sqlite3DbFree(pRec->db, zUtf8);
- }
-#endif
+ if( (pRec->flags&MEM_Str)==0 ) return;
+ if( sqlite3AtoF(pRec->z, &rValue, pRec->n, enc)==0 ) return;
+ if( 0==sqlite3Atoi64(pRec->z, &iValue, pRec->n, enc) ){
+ pRec->u.i = iValue;
+ pRec->flags |= MEM_Int;
+ }else{
+ pRec->r = rValue;
+ pRec->flags |= MEM_Real;
}
}
}
@@ -59834,13 +60068,13 @@
** into a numeric representation. Use either INTEGER or REAL whichever
** is appropriate. But only do the conversion if it is possible without
** loss of information and return the revised type of the argument.
-**
-** This is an EXPERIMENTAL api and is subject to change or removal.
*/
SQLITE_API int sqlite3_value_numeric_type(sqlite3_value *pVal){
Mem *pMem = (Mem*)pVal;
- applyNumericAffinity(pMem);
- sqlite3VdbeMemStoreType(pMem);
+ if( pMem->type==SQLITE_TEXT ){
+ applyNumericAffinity(pMem);
+ sqlite3VdbeMemStoreType(pMem);
+ }
return pMem->type;
}
@@ -61541,7 +61775,6 @@
*/
case OP_MustBeInt: { /* jump, in1 */
pIn1 = &aMem[pOp->p1];
- memAboutToChange(p, pIn1);
applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding);
if( (pIn1->flags & MEM_Int)==0 ){
if( pOp->p2==0 ){
@@ -61568,7 +61801,6 @@
*/
case OP_RealAffinity: { /* in1 */
pIn1 = &aMem[pOp->p1];
- memAboutToChange(p, pIn1);
if( pIn1->flags & MEM_Int ){
sqlite3VdbeMemRealify(pIn1);
}
@@ -61611,7 +61843,6 @@
*/
case OP_ToBlob: { /* same as TK_TO_BLOB, in1 */
pIn1 = &aMem[pOp->p1];
- memAboutToChange(p, pIn1);
if( pIn1->flags & MEM_Null ) break;
if( (pIn1->flags & MEM_Blob)==0 ){
applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding);
@@ -61636,10 +61867,7 @@
*/
case OP_ToNumeric: { /* same as TK_TO_NUMERIC, in1 */
pIn1 = &aMem[pOp->p1];
- memAboutToChange(p, pIn1);
- if( (pIn1->flags & (MEM_Null|MEM_Int|MEM_Real))==0 ){
- sqlite3VdbeMemNumerify(pIn1);
- }
+ sqlite3VdbeMemNumerify(pIn1);
break;
}
#endif /* SQLITE_OMIT_CAST */
@@ -61655,7 +61883,6 @@
*/
case OP_ToInt: { /* same as TK_TO_INT, in1 */
pIn1 = &aMem[pOp->p1];
- memAboutToChange(p, pIn1);
if( (pIn1->flags & MEM_Null)==0 ){
sqlite3VdbeMemIntegerify(pIn1);
}
@@ -61769,8 +61996,6 @@
pIn1 = &aMem[pOp->p1];
pIn3 = &aMem[pOp->p3];
- memAboutToChange(p, pIn1);
- memAboutToChange(p, pIn3);
u.ai.flags1 = pIn1->flags;
u.ai.flags3 = pIn3->flags;
if( (pIn1->flags | pIn3->flags)&MEM_Null ){
@@ -62403,7 +62628,6 @@
while( (u.an.cAff = *(u.an.zAffinity++))!=0 ){
assert( pIn1 <= &p->aMem[p->nMem] );
assert( memIsValid(pIn1) );
- memAboutToChange(p, pIn1);
ExpandBlob(pIn1);
applyAffinity(pIn1, u.an.cAff, encoding);
pIn1++;
@@ -62483,7 +62707,6 @@
for(u.ao.pRec=u.ao.pData0; u.ao.pRec<=u.ao.pLast; u.ao.pRec++){
assert( memIsValid(u.ao.pRec) );
if( u.ao.zAffinity ){
- memAboutToChange(p, u.ao.pRec);
applyAffinity(u.ao.pRec, u.ao.zAffinity[u.ao.pRec-u.ao.pData0], encoding);
}
if( u.ao.pRec->flags&MEM_Zero && u.ao.pRec->n>0 ){
@@ -65937,6 +66160,32 @@
}
#endif
+
+#ifndef SQLITE_OMIT_PAGER_PRAGMAS
+/* Opcode: MaxPgcnt P1 P2 P3 * *
+**
+** Try to set the maximum page count for database P1 to the value in P3.
+** Do not let the maximum page count fall below the current page count and
+** do not change the maximum page count value if P3==0.
+**
+** Store the maximum page count after the change in register P2.
+*/
+case OP_MaxPgcnt: { /* out2-prerelease */
+ unsigned int newMax;
+ Btree *pBt;
+
+ pBt = db->aDb[pOp->p1].pBt;
+ newMax = 0;
+ if( pOp->p3 ){
+ newMax = sqlite3BtreeLastPage(pBt);
+ if( newMax < (unsigned)pOp->p3 ) newMax = (unsigned)pOp->p3;
+ }
+ pOut->u.i = sqlite3BtreeMaxPageCount(pBt, newMax);
+ break;
+}
+#endif
+
+
#ifndef SQLITE_OMIT_TRACE
/* Opcode: Trace * * * P4 *
**
@@ -66111,11 +66360,82 @@
int flags; /* Copy of "flags" passed to sqlite3_blob_open() */
int nByte; /* Size of open blob, in bytes */
int iOffset; /* Byte offset of blob in cursor data */
+ int iCol; /* Table column this handle is open on */
BtCursor *pCsr; /* Cursor pointing at blob row */
sqlite3_stmt *pStmt; /* Statement holding cursor open */
sqlite3 *db; /* The associated database */
};
+
+/*
+** This function is used by both blob_open() and blob_reopen(). It seeks
+** the b-tree cursor associated with blob handle p to point to row iRow.
+** If successful, SQLITE_OK is returned and subsequent calls to
+** sqlite3_blob_read() or sqlite3_blob_write() access the specified row.
+**
+** If an error occurs, or if the specified row does not exist or does not
+** contain a value of type TEXT or BLOB in the column nominated when the
+** blob handle was opened, then an error code is returned and *pzErr may
+** be set to point to a buffer containing an error message. It is the
+** responsibility of the caller to free the error message buffer using
+** sqlite3DbFree().
+**
+** If an error does occur, then the b-tree cursor is closed. All subsequent
+** calls to sqlite3_blob_read(), blob_write() or blob_reopen() will
+** immediately return SQLITE_ABORT.
+*/
+static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){
+ int rc; /* Error code */
+ char *zErr = 0; /* Error message */
+ Vdbe *v = (Vdbe *)p->pStmt;
+
+ /* Set the value of the SQL statements only variable to integer iRow.
+ ** This is done directly instead of using sqlite3_bind_int64() to avoid
+ ** triggering asserts related to mutexes.
+ */
+ assert( v->aVar[0].flags&MEM_Int );
+ v->aVar[0].u.i = iRow;
+
+ rc = sqlite3_step(p->pStmt);
+ if( rc==SQLITE_ROW ){
+ u32 type = v->apCsr[0]->aType[p->iCol];
+ if( type<12 ){
+ zErr = sqlite3MPrintf(p->db, "cannot open value of type %s",
+ type==0?"null": type==7?"real": "integer"
+ );
+ rc = SQLITE_ERROR;
+ sqlite3_finalize(p->pStmt);
+ p->pStmt = 0;
+ }else{
+ p->iOffset = v->apCsr[0]->aOffset[p->iCol];
+ p->nByte = sqlite3VdbeSerialTypeLen(type);
+ p->pCsr = v->apCsr[0]->pCursor;
+ sqlite3BtreeEnterCursor(p->pCsr);
+ sqlite3BtreeCacheOverflow(p->pCsr);
+ sqlite3BtreeLeaveCursor(p->pCsr);
+ }
+ }
+
+ if( rc==SQLITE_ROW ){
+ rc = SQLITE_OK;
+ }else if( p->pStmt ){
+ rc = sqlite3_finalize(p->pStmt);
+ p->pStmt = 0;
+ if( rc==SQLITE_OK ){
+ zErr = sqlite3MPrintf(p->db, "no such rowid: %lld", iRow);
+ rc = SQLITE_ERROR;
+ }else{
+ zErr = sqlite3MPrintf(p->db, "%s", sqlite3_errmsg(p->db));
+ }
+ }
+
+ assert( rc!=SQLITE_OK || zErr==0 );
+ assert( rc!=SQLITE_ROW && rc!=SQLITE_DONE );
+
+ *pzErr = zErr;
+ return rc;
+}
+
/*
** Open a blob handle.
*/
@@ -66156,29 +66476,35 @@
{OP_OpenWrite, 0, 0, 0}, /* 4: Open cursor 0 for read/write */
{OP_Variable, 1, 1, 1}, /* 5: Push the rowid to the stack */
- {OP_NotExists, 0, 9, 1}, /* 6: Seek the cursor */
+ {OP_NotExists, 0, 10, 1}, /* 6: Seek the cursor */
{OP_Column, 0, 0, 1}, /* 7 */
{OP_ResultRow, 1, 0, 0}, /* 8 */
- {OP_Close, 0, 0, 0}, /* 9 */
- {OP_Halt, 0, 0, 0}, /* 10 */
+ {OP_Goto, 0, 5, 0}, /* 9 */
+ {OP_Close, 0, 0, 0}, /* 10 */
+ {OP_Halt, 0, 0, 0}, /* 11 */
};
- Vdbe *v = 0;
int rc = SQLITE_OK;
char *zErr = 0;
Table *pTab;
- Parse *pParse;
+ Parse *pParse = 0;
+ Incrblob *pBlob = 0;
+ flags = !!flags; /* flags = (flags ? 1 : 0); */
*ppBlob = 0;
+
sqlite3_mutex_enter(db->mutex);
+
+ pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob));
+ if( !pBlob ) goto blob_open_out;
pParse = sqlite3StackAllocRaw(db, sizeof(*pParse));
- if( pParse==0 ){
- rc = SQLITE_NOMEM;
- goto blob_open_out;
- }
+ if( !pParse ) goto blob_open_out;
+
do {
memset(pParse, 0, sizeof(Parse));
pParse->db = db;
+ sqlite3DbFree(db, zErr);
+ zErr = 0;
sqlite3BtreeEnterAll(db);
pTab = sqlite3LocateTable(pParse, 0, zTable, zDb);
@@ -66204,7 +66530,7 @@
}
/* Now search pTab for the exact column. */
- for(iCol=0; iCol < pTab->nCol; iCol++) {
+ for(iCol=0; iCol<pTab->nCol; iCol++) {
if( sqlite3StrICmp(pTab->aCol[iCol].zName, zColumn)==0 ){
break;
}
@@ -66258,11 +66584,14 @@
}
}
- v = sqlite3VdbeCreate(db);
- if( v ){
+ pBlob->pStmt = (sqlite3_stmt *)sqlite3VdbeCreate(db);
+ assert( pBlob->pStmt || db->mallocFailed );
+ if( pBlob->pStmt ){
+ Vdbe *v = (Vdbe *)pBlob->pStmt;
int iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
+
sqlite3VdbeAddOpList(v, sizeof(openBlob)/sizeof(VdbeOpList), openBlob);
- flags = !!flags; /* flags = (flags ? 1 : 0); */
+
/* Configure the OP_Transaction */
sqlite3VdbeChangeP1(v, 0, iDb);
@@ -66305,65 +66634,25 @@
}
}
+ pBlob->flags = flags;
+ pBlob->iCol = iCol;
+ pBlob->db = db;
sqlite3BtreeLeaveAll(db);
if( db->mallocFailed ){
goto blob_open_out;
}
-
- sqlite3_bind_int64((sqlite3_stmt *)v, 1, iRow);
- rc = sqlite3_step((sqlite3_stmt *)v);
- if( rc!=SQLITE_ROW ){
- nAttempt++;
- rc = sqlite3_finalize((sqlite3_stmt *)v);
- sqlite3DbFree(db, zErr);
- zErr = sqlite3MPrintf(db, sqlite3_errmsg(db));
- v = 0;
- }
- } while( nAttempt<5 && rc==SQLITE_SCHEMA );
-
- if( rc==SQLITE_ROW ){
- /* The row-record has been opened successfully. Check that the
- ** column in question contains text or a blob. If it contains
- ** text, it is up to the caller to get the encoding right.
- */
- Incrblob *pBlob;
- u32 type = v->apCsr[0]->aType[iCol];
-
- if( type<12 ){
- sqlite3DbFree(db, zErr);
- zErr = sqlite3MPrintf(db, "cannot open value of type %s",
- type==0?"null": type==7?"real": "integer"
- );
- rc = SQLITE_ERROR;
- goto blob_open_out;
- }
- pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob));
- if( db->mallocFailed ){
- sqlite3DbFree(db, pBlob);
- goto blob_open_out;
- }
- pBlob->flags = flags;
- pBlob->pCsr = v->apCsr[0]->pCursor;
- sqlite3BtreeEnterCursor(pBlob->pCsr);
- sqlite3BtreeCacheOverflow(pBlob->pCsr);
- sqlite3BtreeLeaveCursor(pBlob->pCsr);
- pBlob->pStmt = (sqlite3_stmt *)v;
- pBlob->iOffset = v->apCsr[0]->aOffset[iCol];
- pBlob->nByte = sqlite3VdbeSerialTypeLen(type);
- pBlob->db = db;
- *ppBlob = (sqlite3_blob *)pBlob;
- rc = SQLITE_OK;
- }else if( rc==SQLITE_OK ){
- sqlite3DbFree(db, zErr);
- zErr = sqlite3MPrintf(db, "no such rowid: %lld", iRow);
- rc = SQLITE_ERROR;
- }
+ sqlite3_bind_int64(pBlob->pStmt, 1, iRow);
+ rc = blobSeekToRow(pBlob, iRow, &zErr);
+ } while( (++nAttempt)<5 && rc==SQLITE_SCHEMA );
blob_open_out:
- if( v && (rc!=SQLITE_OK || db->mallocFailed) ){
- sqlite3VdbeFinalize(v);
+ if( rc==SQLITE_OK && db->mallocFailed==0 ){
+ *ppBlob = (sqlite3_blob *)pBlob;
+ }else{
+ if( pBlob && pBlob->pStmt ) sqlite3VdbeFinalize((Vdbe *)pBlob->pStmt);
+ sqlite3DbFree(db, pBlob);
}
- sqlite3Error(db, rc, zErr);
+ sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr);
sqlite3DbFree(db, zErr);
sqlite3StackFree(db, pParse);
rc = sqlite3ApiExit(db, rc);
@@ -66416,7 +66705,7 @@
/* Request is out of range. Return a transient error. */
rc = SQLITE_ERROR;
sqlite3Error(db, SQLITE_ERROR, 0);
- } else if( v==0 ){
+ }else if( v==0 ){
/* If there is no statement handle, then the blob-handle has
** already been invalidated. Return SQLITE_ABORT in this case.
*/
@@ -66464,7 +66753,47 @@
*/
SQLITE_API int sqlite3_blob_bytes(sqlite3_blob *pBlob){
Incrblob *p = (Incrblob *)pBlob;
- return p ? p->nByte : 0;
+ return (p && p->pStmt) ? p->nByte : 0;
+}
+
+/*
+** Move an existing blob handle to point to a different row of the same
+** database table.
+**
+** If an error occurs, or if the specified row does not exist or does not
+** contain a blob or text value, then an error code is returned and the
+** database handle error code and message set. If this happens, then all
+** subsequent calls to sqlite3_blob_xxx() functions (except blob_close())
+** immediately return SQLITE_ABORT.
+*/
+SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){
+ int rc;
+ Incrblob *p = (Incrblob *)pBlob;
+ sqlite3 *db;
+
+ if( p==0 ) return SQLITE_MISUSE_BKPT;
+ db = p->db;
+ sqlite3_mutex_enter(db->mutex);
+
+ if( p->pStmt==0 ){
+ /* If there is no statement handle, then the blob-handle has
+ ** already been invalidated. Return SQLITE_ABORT in this case.
+ */
+ rc = SQLITE_ABORT;
+ }else{
+ char *zErr;
+ rc = blobSeekToRow(p, iRow, &zErr);
+ if( rc!=SQLITE_OK ){
+ sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr);
+ sqlite3DbFree(db, zErr);
+ }
+ assert( rc!=SQLITE_SCHEMA );
+ }
+
+ rc = sqlite3ApiExit(db, rc);
+ assert( rc==SQLITE_OK || p->pStmt==0 );
+ sqlite3_mutex_leave(db->mutex);
+ return rc;
}
#endif /* #ifndef SQLITE_OMIT_INCRBLOB */
@@ -68797,6 +69126,9 @@
){
Expr *p = sqlite3ExprAlloc(pParse->db, op, pToken, 1);
sqlite3ExprAttachSubtrees(pParse->db, p, pLeft, pRight);
+ if( p ) {
+ sqlite3ExprCheckHeight(pParse, p->nHeight);
+ }
return p;
}
@@ -68868,7 +69200,7 @@
/* Wildcard of the form "?nnn". Convert "nnn" to an integer and
** use it as the variable number */
i64 i;
- int bOk = sqlite3Atoi64(&z[1], &i);
+ int bOk = 0==sqlite3Atoi64(&z[1], &i, sqlite3Strlen30(&z[1]), SQLITE_UTF8);
pExpr->iColumn = (ynVar)i;
testcase( i==0 );
testcase( i==1 );
@@ -69910,6 +70242,16 @@
assert( testAddr>0 || pParse->db->mallocFailed );
}
+#ifndef SQLITE_OMIT_EXPLAIN
+ if( pParse->explain==2 ){
+ char *zMsg = sqlite3MPrintf(
+ pParse->db, "EXECUTE %s%s SUBQUERY %d", testAddr?"":"CORRELATED ",
+ pExpr->op==TK_IN?"LIST":"SCALAR", pParse->iNextSelectId
+ );
+ sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC);
+ }
+#endif
+
switch( pExpr->op ){
case TK_IN: {
char affinity; /* Affinity of the LHS of the IN */
@@ -69955,6 +70297,7 @@
sqlite3SelectDestInit(&dest, SRT_Set, pExpr->iTable);
dest.affinity = (u8)affinity;
assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable );
+ pExpr->x.pSelect->iLimit = 0;
if( sqlite3Select(pParse, pExpr->x.pSelect, &dest) ){
return 0;
}
@@ -70055,6 +70398,7 @@
sqlite3ExprDelete(pParse->db, pSel->pLimit);
pSel->pLimit = sqlite3PExpr(pParse, TK_INTEGER, 0, 0,
&sqlite3IntTokens[1]);
+ pSel->iLimit = 0;
if( sqlite3Select(pParse, pSel, &dest) ){
return 0;
}
@@ -70231,7 +70575,7 @@
if( ALWAYS(z!=0) ){
double value;
char *zV;
- sqlite3AtoF(z, &value);
+ sqlite3AtoF(z, &value, sqlite3Strlen30(z), SQLITE_UTF8);
assert( !sqlite3IsNaN(value) ); /* The new AtoF never returns NaN */
if( negateFlag ) value = -value;
zV = dup8bytes(v, (char*)&value);
@@ -70245,9 +70589,7 @@
** Generate an instruction that will put the integer describe by
** text z[0..n-1] into register iMem.
**
-** The z[] string will probably not be zero-terminated. But the
-** z[n] character is guaranteed to be something that does not look
-** like the continuation of the number.
+** Expr.u.zToken is always UTF8 and zero-terminated.
*/
static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){
Vdbe *v = pParse->pVdbe;
@@ -70256,13 +70598,14 @@
if( negFlag ) i = -i;
sqlite3VdbeAddOp2(v, OP_Integer, i, iMem);
}else{
+ int c;
+ i64 value;
const char *z = pExpr->u.zToken;
assert( z!=0 );
- if( sqlite3FitsIn64Bits(z, negFlag) ){
- i64 value;
+ c = sqlite3Atoi64(z, &value, sqlite3Strlen30(z), SQLITE_UTF8);
+ if( c==0 || (c==2 && negFlag) ){
char *zV;
- sqlite3Atoi64(z, &value);
- if( negFlag ) value = -value;
+ if( negFlag ){ value = -value; }
zV = dup8bytes(v, (char*)&value);
sqlite3VdbeAddOp4(v, OP_Int64, 0, iMem, 0, zV, P4_INT64);
}else{
@@ -71339,9 +71682,22 @@
** Preevaluate constant subexpressions within pExpr and store the
** results in registers. Modify pExpr so that the constant subexpresions
** are TK_REGISTER opcodes that refer to the precomputed values.
+**
+** This routine is a no-op if the jump to the cookie-check code has
+** already occur. Since the cookie-check jump is generated prior to
+** any other serious processing, this check ensures that there is no
+** way to accidently bypass the constant initializations.
+**
+** This routine is also a no-op if the SQLITE_FactorOutConst optimization
+** is disabled via the sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS)
+** interface. This allows test logic to verify that the same answer is
+** obtained for queries regardless of whether or not constants are
+** precomputed into registers or if they are inserted in-line.
*/
SQLITE_PRIVATE void sqlite3ExprCodeConstants(Parse *pParse, Expr *pExpr){
Walker w;
+ if( pParse->cookieGoto ) return;
+ if( (pParse->db->flags & SQLITE_FactorOutConst)!=0 ) return;
w.xExprCallback = evalConstExpr;
w.xSelectCallback = 0;
w.pParse = pParse;
@@ -72354,6 +72710,11 @@
}
}
}
+ if( zWhere ){
+ char *zNew = sqlite3MPrintf(pParse->db, "type='trigger' AND (%s)", zWhere);
+ sqlite3DbFree(pParse->db, zWhere);
+ zWhere = zNew;
+ }
return zWhere;
}
@@ -72961,7 +73322,7 @@
int i; /* Loop counter */
int topOfLoop; /* The top of the loop */
int endOfLoop; /* The end of the loop */
- int addr; /* The address of an instruction */
+ int addr = 0; /* The address of an instruction */
int jZeroRows = 0; /* Jump from here if number of rows is zero */
int iDb; /* Index of database containing pTab */
int regTabname = iMem++; /* Register containing table name */
@@ -72984,8 +73345,8 @@
if( v==0 || NEVER(pTab==0) ){
return;
}
- if( pTab->pSelect ){
- /* Do not gather statistics on views */
+ if( pTab->tnum==0 ){
+ /* Do not gather statistics on views or virtual tables */
return;
}
if( memcmp(pTab->zName, "sqlite_", 7)==0 ){
@@ -73905,7 +74266,8 @@
0, /* xStep */
0, /* xFinalize */
"sqlite_detach", /* zName */
- 0 /* pHash */
+ 0, /* pHash */
+ 0 /* pDestructor */
};
codeAttach(pParse, SQLITE_DETACH, &detach_func, pDbname, 0, 0, pDbname);
}
@@ -73926,7 +74288,8 @@
0, /* xStep */
0, /* xFinalize */
"sqlite_attach", /* zName */
- 0 /* pHash */
+ 0, /* pHash */
+ 0 /* pDestructor */
};
codeAttach(pParse, SQLITE_ATTACH, &attach_func, p, p, pDbname, pKey);
}
@@ -75055,8 +75418,9 @@
*/
iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName);
if( iDb<0 ) return;
- if( !OMIT_TEMPDB && isTemp && iDb>1 ){
- /* If creating a temp table, the name may not be qualified */
+ if( !OMIT_TEMPDB && isTemp && pName2->n>0 && iDb!=1 ){
+ /* If creating a temp table, the name may not be qualified. Unless
+ ** the database name is "temp" anyway. */
sqlite3ErrorMsg(pParse, "temporary table name must be unqualified");
return;
}
@@ -75104,17 +75468,18 @@
** collisions.
*/
if( !IN_DECLARE_VTAB ){
+ char *zDb = db->aDb[iDb].zName;
if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){
goto begin_table_error;
}
- pTable = sqlite3FindTable(db, zName, db->aDb[iDb].zName);
+ pTable = sqlite3FindTable(db, zName, zDb);
if( pTable ){
if( !noErr ){
sqlite3ErrorMsg(pParse, "table %T already exists", pName);
}
goto begin_table_error;
}
- if( sqlite3FindIndex(db, zName, 0)!=0 && (iDb==0 || !db->init.busy) ){
+ if( sqlite3FindIndex(db, zName, zDb)!=0 ){
sqlite3ErrorMsg(pParse, "there is already an index named %s", zName);
goto begin_table_error;
}
@@ -77099,7 +77464,8 @@
sqlite3RefillIndex(pParse, pIndex, iMem);
sqlite3ChangeCookie(pParse, iDb);
sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0,
- sqlite3MPrintf(db, "name='%q'", pIndex->zName), P4_DYNAMIC);
+ sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName),
+ P4_DYNAMIC);
sqlite3VdbeAddOp1(v, OP_Expire, 0);
}
}
@@ -77227,7 +77593,7 @@
if( v ){
sqlite3BeginWriteOperation(pParse, 1, iDb);
sqlite3NestedParse(pParse,
- "DELETE FROM %Q.%s WHERE name=%Q",
+ "DELETE FROM %Q.%s WHERE name=%Q AND type='index'",
db->aDb[iDb].zName, SCHEMA_TABLE(iDb),
pIndex->zName
);
@@ -79392,7 +79758,7 @@
sqlite3_result_error_nomem(context);
return;
}
- sqlite3AtoF(zBuf, &r);
+ sqlite3AtoF(zBuf, &r, sqlite3Strlen30(zBuf), SQLITE_UTF8);
sqlite3_free(zBuf);
}
sqlite3_result_double(context, r);
@@ -80644,10 +81010,10 @@
FUNCTION(coalesce, 1, 0, 0, 0 ),
FUNCTION(coalesce, 0, 0, 0, 0 ),
/* FUNCTION(coalesce, -1, 0, 0, ifnullFunc ), */
- {-1,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"coalesce",0},
+ {-1,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"coalesce",0,0},
FUNCTION(hex, 1, 0, 0, hexFunc ),
/* FUNCTION(ifnull, 2, 0, 0, ifnullFunc ), */
- {2,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"ifnull",0},
+ {2,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"ifnull",0,0},
FUNCTION(random, 0, 0, 0, randomFunc ),
FUNCTION(randomblob, 1, 0, 0, randomBlob ),
FUNCTION(nullif, 2, 0, 1, nullifFunc ),
@@ -80674,7 +81040,7 @@
AGGREGATE(total, 1, 0, 0, sumStep, totalFinalize ),
AGGREGATE(avg, 1, 0, 0, sumStep, avgFinalize ),
/* AGGREGATE(count, 0, 0, 0, countStep, countFinalize ), */
- {0,SQLITE_UTF8,SQLITE_FUNC_COUNT,0,0,0,countStep,countFinalize,"count",0},
+ {0,SQLITE_UTF8,SQLITE_FUNC_COUNT,0,0,0,countStep,countFinalize,"count",0,0},
AGGREGATE(count, 1, 0, 0, countStep, countFinalize ),
AGGREGATE(group_concat, 1, 0, 0, groupConcatStep, groupConcatFinalize),
AGGREGATE(group_concat, 2, 0, 0, groupConcatStep, groupConcatFinalize),
@@ -84087,6 +84453,27 @@
sqlite3_stmt *(*next_stmt)(sqlite3*,sqlite3_stmt*);
const char *(*sql)(sqlite3_stmt*);
int (*status)(int,int*,int*,int);
+ int (*backup_finish)(sqlite3_backup*);
+ sqlite3_backup *(*backup_init)(sqlite3*,const char*,sqlite3*,const char*);
+ int (*backup_pagecount)(sqlite3_backup*);
+ int (*backup_remaining)(sqlite3_backup*);
+ int (*backup_step)(sqlite3_backup*,int);
+ const char *(*compileoption_get)(int);
+ int (*compileoption_used)(const char*);
+ int (*create_function_v2)(sqlite3*,const char*,int,int,void*,void (*xFunc)(sqlite3_context*,int,sqlite3_value**),void (*xStep)(sqlite3_context*,int,sqlite3_value**),void (*xFinal)(sqlite3_context*),void(*xDestroy)(void*));
+ int (*db_config)(sqlite3*,int,...);
+ sqlite3_mutex *(*db_mutex)(sqlite3*);
+ int (*db_status)(sqlite3*,int,int*,int*,int);
+ int (*extended_errcode)(sqlite3*);
+ void (*log)(int,const char*,...);
+ sqlite3_int64 (*soft_heap_limit64)(sqlite3_int64);
+ const char *(*sourceid)(void);
+ int (*stmt_status)(sqlite3_stmt*,int,int);
+ int (*strnicmp)(const char*,const char*,int);
+ int (*unlock_notify)(sqlite3*,void(*)(void**,int),void*);
+ int (*wal_autocheckpoint)(sqlite3*,int);
+ int (*wal_checkpoint)(sqlite3*,const char*);
+ void *(*wal_hook)(sqlite3*,int(*)(void*,sqlite3*,const char*,int),void*);
};
/*
@@ -84266,6 +84653,27 @@
#define sqlite3_next_stmt sqlite3_api->next_stmt
#define sqlite3_sql sqlite3_api->sql
#define sqlite3_status sqlite3_api->status
+#define sqlite3_backup_finish sqlite3_api->backup_finish
+#define sqlite3_backup_init sqlite3_api->backup_init
+#define sqlite3_backup_pagecount sqlite3_api->backup_pagecount
+#define sqlite3_backup_remaining sqlite3_api->backup_remaining
+#define sqlite3_backup_step sqlite3_api->backup_step
+#define sqlite3_compileoption_get sqlite3_api->compileoption_get
+#define sqlite3_compileoption_used sqlite3_api->compileoption_used
+#define sqlite3_create_function_v2 sqlite3_api->create_function_v2
+#define sqlite3_db_config sqlite3_api->db_config
+#define sqlite3_db_mutex sqlite3_api->db_mutex
+#define sqlite3_db_status sqlite3_api->db_status
+#define sqlite3_extended_errcode sqlite3_api->extended_errcode
+#define sqlite3_log sqlite3_api->log
+#define sqlite3_soft_heap_limit64 sqlite3_api->soft_heap_limit64
+#define sqlite3_sourceid sqlite3_api->sourceid
+#define sqlite3_stmt_status sqlite3_api->stmt_status
+#define sqlite3_strnicmp sqlite3_api->strnicmp
+#define sqlite3_unlock_notify sqlite3_api->unlock_notify
+#define sqlite3_wal_autocheckpoint sqlite3_api->wal_autocheckpoint
+#define sqlite3_wal_checkpoint sqlite3_api->wal_checkpoint
+#define sqlite3_wal_hook sqlite3_api->wal_hook
#endif /* SQLITE_CORE */
#define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api = 0;
@@ -84583,6 +84991,46 @@
sqlite3_next_stmt,
sqlite3_sql,
sqlite3_status,
+
+ /*
+ ** Added for 3.7.4
+ */
+ sqlite3_backup_finish,
+ sqlite3_backup_init,
+ sqlite3_backup_pagecount,
+ sqlite3_backup_remaining,
+ sqlite3_backup_step,
+#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS
+ sqlite3_compileoption_get,
+ sqlite3_compileoption_used,
+#else
+ 0,
+ 0,
+#endif
+ sqlite3_create_function_v2,
+ sqlite3_db_config,
+ sqlite3_db_mutex,
+ sqlite3_db_status,
+ sqlite3_extended_errcode,
+ sqlite3_log,
+ sqlite3_soft_heap_limit64,
+ sqlite3_sourceid,
+ sqlite3_stmt_status,
+ sqlite3_strnicmp,
+#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY
+ sqlite3_unlock_notify,
+#else
+ 0,
+#endif
+#ifndef SQLITE_OMIT_WAL
+ sqlite3_wal_autocheckpoint,
+ sqlite3_wal_checkpoint,
+ sqlite3_wal_hook,
+#else
+ 0,
+ 0,
+ 0,
+#endif
};
/*
@@ -84898,7 +85346,7 @@
static const u8 iValue[] = {1, 0, 0, 0, 1, 1, 2};
int i, n;
if( sqlite3Isdigit(*z) ){
- return (u8)atoi(z);
+ return (u8)sqlite3Atoi(z);
}
n = sqlite3Strlen30(z);
for(i=0; i<ArraySize(iLength); i++){
@@ -84939,7 +85387,7 @@
if( 0==sqlite3StrICmp(z, "none") ) return BTREE_AUTOVACUUM_NONE;
if( 0==sqlite3StrICmp(z, "full") ) return BTREE_AUTOVACUUM_FULL;
if( 0==sqlite3StrICmp(z, "incremental") ) return BTREE_AUTOVACUUM_INCR;
- i = atoi(z);
+ i = sqlite3Atoi(z);
return (u8)((i>=0&&i<=2)?i:0);
}
#endif /* ifndef SQLITE_OMIT_AUTOVACUUM */
@@ -85035,6 +85483,7 @@
{ "empty_result_callbacks", SQLITE_NullCallback },
{ "legacy_file_format", SQLITE_LegacyFileFmt },
{ "fullfsync", SQLITE_FullFSync },
+ { "checkpoint_fullfsync", SQLITE_CkptFullFSync },
{ "reverse_unordered_selects", SQLITE_ReverseOrder },
#ifndef SQLITE_OMIT_AUTOMATIC_INDEX
{ "automatic_index", SQLITE_AutoIndex },
@@ -85246,7 +85695,7 @@
sqlite3VdbeChangeP1(v, addr+1, iDb);
sqlite3VdbeChangeP1(v, addr+6, SQLITE_DEFAULT_CACHE_SIZE);
}else{
- int size = atoi(zRight);
+ int size = sqlite3Atoi(zRight);
if( size<0 ) size = -size;
sqlite3BeginWriteOperation(pParse, 0, iDb);
sqlite3VdbeAddOp2(v, OP_Integer, size, 1);
@@ -85275,7 +85724,7 @@
/* Malloc may fail when setting the page-size, as there is an internal
** buffer that the pager module resizes using sqlite3_realloc().
*/
- db->nextPagesize = atoi(zRight);
+ db->nextPagesize = sqlite3Atoi(zRight);
if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize, -1, 0) ){
db->mallocFailed = 1;
}
@@ -85283,28 +85732,6 @@
}else
/*
- ** PRAGMA [database.]max_page_count
- ** PRAGMA [database.]max_page_count=N
- **
- ** The first form reports the current setting for the
- ** maximum number of pages in the database file. The
- ** second form attempts to change this setting. Both
- ** forms return the current setting.
- */
- if( sqlite3StrICmp(zLeft,"max_page_count")==0 ){
- Btree *pBt = pDb->pBt;
- int newMax = 0;
- assert( pBt!=0 );
- if( zRight ){
- newMax = atoi(zRight);
- }
- if( ALWAYS(pBt) ){
- newMax = sqlite3BtreeMaxPageCount(pBt, newMax);
- }
- returnSingleInt(pParse, "max_page_count", newMax);
- }else
-
- /*
** PRAGMA [database.]secure_delete
** PRAGMA [database.]secure_delete=ON/OFF
**
@@ -85330,19 +85757,33 @@
}else
/*
+ ** PRAGMA [database.]max_page_count
+ ** PRAGMA [database.]max_page_count=N
+ **
+ ** The first form reports the current setting for the
+ ** maximum number of pages in the database file. The
+ ** second form attempts to change this setting. Both
+ ** forms return the current setting.
+ **
** PRAGMA [database.]page_count
**
** Return the number of pages in the specified database.
*/
- if( sqlite3StrICmp(zLeft,"page_count")==0 ){
+ if( sqlite3StrICmp(zLeft,"page_count")==0
+ || sqlite3StrICmp(zLeft,"max_page_count")==0
+ ){
int iReg;
if( sqlite3ReadSchema(pParse) ) goto pragma_out;
sqlite3CodeVerifySchema(pParse, iDb);
iReg = ++pParse->nMem;
- sqlite3VdbeAddOp2(v, OP_Pagecount, iDb, iReg);
+ if( zLeft[0]=='p' ){
+ sqlite3VdbeAddOp2(v, OP_Pagecount, iDb, iReg);
+ }else{
+ sqlite3VdbeAddOp3(v, OP_MaxPgcnt, iDb, iReg, sqlite3Atoi(zRight));
+ }
sqlite3VdbeAddOp2(v, OP_ResultRow, iReg, 1);
sqlite3VdbeSetNumCols(v, 1);
- sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "page_count", SQLITE_STATIC);
+ sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLeft, SQLITE_TRANSIENT);
}else
/*
@@ -85450,7 +85891,7 @@
Pager *pPager = sqlite3BtreePager(pDb->pBt);
i64 iLimit = -2;
if( zRight ){
- sqlite3Atoi64(zRight, &iLimit);
+ sqlite3Atoi64(zRight, &iLimit, 1000000, SQLITE_UTF8);
if( iLimit<-1 ) iLimit = -1;
}
iLimit = sqlite3PagerJournalSizeLimit(pPager, iLimit);
@@ -85564,7 +86005,7 @@
if( !zRight ){
returnSingleInt(pParse, "cache_size", pDb->pSchema->cache_size);
}else{
- int size = atoi(zRight);
+ int size = sqlite3Atoi(zRight);
if( size<0 ) size = -size;
pDb->pSchema->cache_size = size;
sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size);
@@ -85957,7 +86398,7 @@
/* Set the maximum error count */
mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX;
if( zRight ){
- mxErr = atoi(zRight);
+ sqlite3GetInt32(zRight, &mxErr);
if( mxErr<=0 ){
mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX;
}
@@ -86214,7 +86655,7 @@
};
int addr = sqlite3VdbeAddOpList(v, ArraySize(setCookie), setCookie);
sqlite3VdbeChangeP1(v, addr, iDb);
- sqlite3VdbeChangeP1(v, addr+1, atoi(zRight));
+ sqlite3VdbeChangeP1(v, addr+1, sqlite3Atoi(zRight));
sqlite3VdbeChangeP1(v, addr+2, iDb);
sqlite3VdbeChangeP2(v, addr+2, iCookie);
}else{
@@ -86275,8 +86716,7 @@
*/
if( sqlite3StrICmp(zLeft, "wal_autocheckpoint")==0 ){
if( zRight ){
- int nAuto = atoi(zRight);
- sqlite3_wal_autocheckpoint(db, nAuto);
+ sqlite3_wal_autocheckpoint(db, sqlite3Atoi(zRight));
}
returnSingleInt(pParse, "wal_autocheckpoint",
db->xWalCallback==sqlite3WalDefaultHook ?
@@ -86366,7 +86806,8 @@
#ifndef SQLITE_OMIT_PAGER_PRAGMAS
if( db->autoCommit ){
sqlite3BtreeSetSafetyLevel(pDb->pBt, pDb->safety_level,
- (db->flags&SQLITE_FullFSync)!=0);
+ (db->flags&SQLITE_FullFSync)!=0,
+ (db->flags&SQLITE_CkptFullFSync)!=0);
}
#endif
pragma_out:
@@ -86458,7 +86899,7 @@
assert( db->init.busy );
db->init.iDb = iDb;
- db->init.newTnum = atoi(argv[1]);
+ db->init.newTnum = sqlite3Atoi(argv[1]);
db->init.orphanTrigger = 0;
TESTONLY(rcp = ) sqlite3_prepare(db, argv[2], -1, &pStmt, 0);
rc = db->errCode;
@@ -87007,13 +87448,13 @@
if( rc==SQLITE_OK && pParse->pVdbe && pParse->explain ){
static const char * const azColName[] = {
"addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment",
- "order", "from", "detail"
+ "selectid", "order", "from", "detail"
};
int iFirst, mx;
if( pParse->explain==2 ){
- sqlite3VdbeSetNumCols(pParse->pVdbe, 3);
+ sqlite3VdbeSetNumCols(pParse->pVdbe, 4);
iFirst = 8;
- mx = 11;
+ mx = 12;
}else{
sqlite3VdbeSetNumCols(pParse->pVdbe, 8);
iFirst = 0;
@@ -87683,7 +88124,6 @@
sqlite3VdbeAddOp1(v, OP_Last, pOrderBy->iECursor);
sqlite3VdbeAddOp1(v, OP_Delete, pOrderBy->iECursor);
sqlite3VdbeJumpHere(v, addr2);
- pSelect->iLimit = 0;
}
}
@@ -87962,11 +88402,11 @@
#endif
}
- /* Jump to the end of the loop if the LIMIT is reached.
+ /* Jump to the end of the loop if the LIMIT is reached. Except, if
+ ** there is a sorter, in which case the sorter has already limited
+ ** the output for us.
*/
- if( p->iLimit ){
- assert( pOrderBy==0 ); /* If there is an ORDER BY, the call to
- ** pushOntoSorter() would have cleared p->iLimit */
+ if( pOrderBy==0 && p->iLimit ){
sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1);
}
}
@@ -88013,6 +88453,88 @@
return pInfo;
}
+#ifndef SQLITE_OMIT_COMPOUND_SELECT
+/*
+** Name of the connection operator, used for error messages.
+*/
+static const char *selectOpName(int id){
+ char *z;
+ switch( id ){
+ case TK_ALL: z = "UNION ALL"; break;
+ case TK_INTERSECT: z = "INTERSECT"; break;
+ case TK_EXCEPT: z = "EXCEPT"; break;
+ default: z = "UNION"; break;
+ }
+ return z;
+}
+#endif /* SQLITE_OMIT_COMPOUND_SELECT */
+
+#ifndef SQLITE_OMIT_EXPLAIN
+/*
+** Unless an "EXPLAIN QUERY PLAN" command is being processed, this function
+** is a no-op. Otherwise, it adds a single row of output to the EQP result,
+** where the caption is of the form:
+**
+** "USE TEMP B-TREE FOR xxx"
+**
+** where xxx is one of "DISTINCT", "ORDER BY" or "GROUP BY". Exactly which
+** is determined by the zUsage argument.
+*/
+static void explainTempTable(Parse *pParse, const char *zUsage){
+ if( pParse->explain==2 ){
+ Vdbe *v = pParse->pVdbe;
+ char *zMsg = sqlite3MPrintf(pParse->db, "USE TEMP B-TREE FOR %s", zUsage);
+ sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC);
+ }
+}
+
+/*
+** Unless an "EXPLAIN QUERY PLAN" command is being processed, this function
+** is a no-op. Otherwise, it adds a single row of output to the EQP result,
+** where the caption is of one of the two forms:
+**
+** "COMPOSITE SUBQUERIES iSub1 and iSub2 (op)"
+** "COMPOSITE SUBQUERIES iSub1 and iSub2 USING TEMP B-TREE (op)"
+**
+** where iSub1 and iSub2 are the integers passed as the corresponding
+** function parameters, and op is the text representation of the parameter
+** of the same name. The parameter "op" must be one of TK_UNION, TK_EXCEPT,
+** TK_INTERSECT or TK_ALL. The first form is used if argument bUseTmp is
+** false, or the second form if it is true.
+*/
+static void explainComposite(
+ Parse *pParse, /* Parse context */
+ int op, /* One of TK_UNION, TK_EXCEPT etc. */
+ int iSub1, /* Subquery id 1 */
+ int iSub2, /* Subquery id 2 */
+ int bUseTmp /* True if a temp table was used */
+){
+ assert( op==TK_UNION || op==TK_EXCEPT || op==TK_INTERSECT || op==TK_ALL );
+ if( pParse->explain==2 ){
+ Vdbe *v = pParse->pVdbe;
+ char *zMsg = sqlite3MPrintf(
+ pParse->db, "COMPOUND SUBQUERIES %d AND %d %s(%s)", iSub1, iSub2,
+ bUseTmp?"USING TEMP B-TREE ":"", selectOpName(op)
+ );
+ sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC);
+ }
+}
+
+/*
+** Assign expression b to lvalue a. A second, no-op, version of this macro
+** is provided when SQLITE_OMIT_EXPLAIN is defined. This allows the code
+** in sqlite3Select() to assign values to structure member variables that
+** only exist if SQLITE_OMIT_EXPLAIN is not defined without polluting the
+** code with #ifndef directives.
+*/
+# define explainSetInteger(a, b) a = b
+
+#else
+/* No-op versions of the explainXXX() functions and macros. */
+# define explainTempTable(y,z)
+# define explainComposite(v,w,x,y,z)
+# define explainSetInteger(y,z)
+#endif
/*
** If the inner loop was generated using a non-null pOrderBy argument,
@@ -88101,10 +88623,6 @@
sqlite3ReleaseTempReg(pParse, regRow);
sqlite3ReleaseTempReg(pParse, regRowid);
- /* LIMIT has been implemented by the pushOntoSorter() routine.
- */
- assert( p->iLimit==0 );
-
/* The bottom of the loop
*/
sqlite3VdbeResolveLabel(v, addrContinue);
@@ -88364,22 +88882,6 @@
generateColumnTypes(pParse, pTabList, pEList);
}
-#ifndef SQLITE_OMIT_COMPOUND_SELECT
-/*
-** Name of the connection operator, used for error messages.
-*/
-static const char *selectOpName(int id){
- char *z;
- switch( id ){
- case TK_ALL: z = "UNION ALL"; break;
- case TK_INTERSECT: z = "INTERSECT"; break;
- case TK_EXCEPT: z = "EXCEPT"; break;
- default: z = "UNION"; break;
- }
- return z;
-}
-#endif /* SQLITE_OMIT_COMPOUND_SELECT */
-
/*
** Given a an expression list (which is really the list of expressions
** that form the result set of a SELECT statement) compute appropriate
@@ -88542,6 +89044,7 @@
assert( db->lookaside.bEnabled==0 );
pTab->nRef = 1;
pTab->zName = 0;
+ pTab->nRowEst = 1000000;
selectColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol);
selectAddColumnTypeAndCollation(pParse, pTab->nCol, pTab->aCol, pSelect);
pTab->iPKey = -1;
@@ -88612,6 +89115,8 @@
VdbeComment((v, "LIMIT counter"));
if( n==0 ){
sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak);
+ }else{
+ if( p->nSelectRow > (double)n ) p->nSelectRow = (double)n;
}
}else{
sqlite3ExprCode(pParse, p->pLimit, iLimit);
@@ -88712,6 +89217,10 @@
SelectDest dest; /* Alternative data destination */
Select *pDelete = 0; /* Chain of simple selects to delete */
sqlite3 *db; /* Database connection */
+#ifndef SQLITE_OMIT_EXPLAIN
+ int iSub1; /* EQP id of left-hand query */
+ int iSub2; /* EQP id of right-hand query */
+#endif
/* Make sure there is no ORDER BY or LIMIT clause on prior SELECTs. Only
** the last (right-most) SELECT in the series may have an ORDER BY or LIMIT.
@@ -88769,9 +89278,11 @@
switch( p->op ){
case TK_ALL: {
int addr = 0;
+ int nLimit;
assert( !pPrior->pLimit );
pPrior->pLimit = p->pLimit;
pPrior->pOffset = p->pOffset;
+ explainSetInteger(iSub1, pParse->iNextSelectId);
rc = sqlite3Select(pParse, pPrior, &dest);
p->pLimit = 0;
p->pOffset = 0;
@@ -88785,10 +89296,18 @@
addr = sqlite3VdbeAddOp1(v, OP_IfZero, p->iLimit);
VdbeComment((v, "Jump ahead if LIMIT reached"));
}
+ explainSetInteger(iSub2, pParse->iNextSelectId);
rc = sqlite3Select(pParse, p, &dest);
testcase( rc!=SQLITE_OK );
pDelete = p->pPrior;
p->pPrior = pPrior;
+ p->nSelectRow += pPrior->nSelectRow;
+ if( pPrior->pLimit
+ && sqlite3ExprIsInteger(pPrior->pLimit, &nLimit)
+ && p->nSelectRow > (double)nLimit
+ ){
+ p->nSelectRow = (double)nLimit;
+ }
if( addr ){
sqlite3VdbeJumpHere(v, addr);
}
@@ -88832,6 +89351,7 @@
*/
assert( !pPrior->pOrderBy );
sqlite3SelectDestInit(&uniondest, priorOp, unionTab);
+ explainSetInteger(iSub1, pParse->iNextSelectId);
rc = sqlite3Select(pParse, pPrior, &uniondest);
if( rc ){
goto multi_select_end;
@@ -88851,6 +89371,7 @@
pOffset = p->pOffset;
p->pOffset = 0;
uniondest.eDest = op;
+ explainSetInteger(iSub2, pParse->iNextSelectId);
rc = sqlite3Select(pParse, p, &uniondest);
testcase( rc!=SQLITE_OK );
/* Query flattening in sqlite3Select() might refill p->pOrderBy.
@@ -88859,6 +89380,7 @@
pDelete = p->pPrior;
p->pPrior = pPrior;
p->pOrderBy = 0;
+ if( p->op==TK_UNION ) p->nSelectRow += pPrior->nSelectRow;
sqlite3ExprDelete(db, p->pLimit);
p->pLimit = pLimit;
p->pOffset = pOffset;
@@ -88916,6 +89438,7 @@
/* Code the SELECTs to our left into temporary table "tab1".
*/
sqlite3SelectDestInit(&intersectdest, SRT_Union, tab1);
+ explainSetInteger(iSub1, pParse->iNextSelectId);
rc = sqlite3Select(pParse, pPrior, &intersectdest);
if( rc ){
goto multi_select_end;
@@ -88932,10 +89455,12 @@
pOffset = p->pOffset;
p->pOffset = 0;
intersectdest.iParm = tab2;
+ explainSetInteger(iSub2, pParse->iNextSelectId);
rc = sqlite3Select(pParse, p, &intersectdest);
testcase( rc!=SQLITE_OK );
pDelete = p->pPrior;
p->pPrior = pPrior;
+ if( p->nSelectRow>pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow;
sqlite3ExprDelete(db, p->pLimit);
p->pLimit = pLimit;
p->pOffset = pOffset;
@@ -88968,6 +89493,8 @@
}
}
+ explainComposite(pParse, p->op, iSub1, iSub2, p->op!=TK_ALL);
+
/* Compute collating sequences used by
** temporary tables needed to implement the compound select.
** Attach the KeyInfo structure to all temporary tables.
@@ -89311,6 +89838,10 @@
ExprList *pOrderBy; /* The ORDER BY clause */
int nOrderBy; /* Number of terms in the ORDER BY clause */
int *aPermute; /* Mapping from ORDER BY terms to result set columns */
+#ifndef SQLITE_OMIT_EXPLAIN
+ int iSub1; /* EQP id of left-hand query */
+ int iSub2; /* EQP id of right-hand query */
+#endif
assert( p->pOrderBy!=0 );
assert( pKeyDup==0 ); /* "Managed" code needs this. Ticket #3382. */
@@ -89422,7 +89953,6 @@
/* Separate the left and the right query from one another
*/
p->pPrior = 0;
- pPrior->pRightmost = 0;
sqlite3ResolveOrderGroupBy(pParse, p, p->pOrderBy, "ORDER");
if( pPrior->pPrior==0 ){
sqlite3ResolveOrderGroupBy(pParse, pPrior, pPrior->pOrderBy, "ORDER");
@@ -89465,6 +89995,7 @@
*/
VdbeNoopComment((v, "Begin coroutine for left SELECT"));
pPrior->iLimit = regLimitA;
+ explainSetInteger(iSub1, pParse->iNextSelectId);
sqlite3Select(pParse, pPrior, &destA);
sqlite3VdbeAddOp2(v, OP_Integer, 1, regEofA);
sqlite3VdbeAddOp1(v, OP_Yield, regAddrA);
@@ -89479,6 +90010,7 @@
savedOffset = p->iOffset;
p->iLimit = regLimitB;
p->iOffset = 0;
+ explainSetInteger(iSub2, pParse->iNextSelectId);
sqlite3Select(pParse, p, &destB);
p->iLimit = savedLimit;
p->iOffset = savedOffset;
@@ -89515,6 +90047,7 @@
sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB);
sqlite3VdbeAddOp1(v, OP_Yield, regAddrB);
sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEofA);
+ p->nSelectRow += pPrior->nSelectRow;
}
/* Generate a subroutine to run when the results from select B
@@ -89522,6 +90055,7 @@
*/
if( op==TK_INTERSECT ){
addrEofB = addrEofA;
+ if( p->nSelectRow > pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow;
}else{
VdbeNoopComment((v, "eof-B subroutine"));
addrEofB = sqlite3VdbeAddOp2(v, OP_If, regEofA, labelEnd);
@@ -89609,6 +90143,7 @@
/*** TBD: Insert subroutine calls to close cursors on incomplete
**** subqueries ****/
+ explainComposite(pParse, p->op, iSub1, iSub2, 0);
return SQLITE_OK;
}
#endif
@@ -90342,6 +90877,7 @@
while( pSel->pPrior ){ pSel = pSel->pPrior; }
selectColumnsFromExprList(pParse, pSel->pEList, &pTab->nCol, &pTab->aCol);
pTab->iPKey = -1;
+ pTab->nRowEst = 1000000;
pTab->tabFlags |= TF_Ephemeral;
#endif
}else{
@@ -90835,6 +91371,11 @@
int iEnd; /* Address of the end of the query */
sqlite3 *db; /* The database connection */
+#ifndef SQLITE_OMIT_EXPLAIN
+ int iRestoreSelectId = pParse->iSelectId;
+ pParse->iSelectId = pParse->iNextSelectId++;
+#endif
+
db = pParse->db;
if( p==0 || db->mallocFailed || pParse->nErr ){
return 1;
@@ -90906,8 +91447,10 @@
}else{
sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor);
assert( pItem->isPopulated==0 );
+ explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId);
sqlite3Select(pParse, pSub, &dest);
pItem->isPopulated = 1;
+ pItem->pTab->nRowEst = (unsigned)pSub->nSelectRow;
}
if( /*pParse->nErr ||*/ db->mallocFailed ){
goto select_end;
@@ -90941,10 +91484,12 @@
mxSelect = db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT];
if( mxSelect && cnt>mxSelect ){
sqlite3ErrorMsg(pParse, "too many terms in compound SELECT");
- return 1;
+ goto select_end;
}
}
- return multiSelect(pParse, p, pDest);
+ rc = multiSelect(pParse, p, pDest);
+ explainSetInteger(pParse->iSelectId, iRestoreSelectId);
+ return rc;
}
#endif
@@ -90956,7 +91501,6 @@
p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0);
pGroupBy = p->pGroupBy;
p->selFlags &= ~SF_Distinct;
- isDistinct = 0;
}
/* If there is both a GROUP BY and an ORDER BY clause and they are
@@ -90999,11 +91543,12 @@
/* Set the limiter.
*/
iEnd = sqlite3VdbeMakeLabel(v);
+ p->nSelectRow = (double)LARGEST_INT64;
computeLimitRegisters(pParse, p, iEnd);
/* Open a virtual index to use for the distinct set.
*/
- if( isDistinct ){
+ if( p->selFlags & SF_Distinct ){
KeyInfo *pKeyInfo;
assert( isAgg || pGroupBy );
distinct = pParse->nTab++;
@@ -91022,6 +91567,7 @@
*/
pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, 0);
if( pWInfo==0 ) goto select_end;
+ if( pWInfo->nRowOut < p->nSelectRow ) p->nSelectRow = pWInfo->nRowOut;
/* If sorting index that was created by a prior OP_OpenEphemeral
** instruction ended up not being needed, then change the OP_OpenEphemeral
@@ -91066,6 +91612,9 @@
for(k=pGroupBy->nExpr, pItem=pGroupBy->a; k>0; k--, pItem++){
pItem->iAlias = 0;
}
+ if( p->nSelectRow>(double)100 ) p->nSelectRow = (double)100;
+ }else{
+ p->nSelectRow = (double)1;
}
@@ -91162,6 +91711,9 @@
int nCol;
int nGroupBy;
+ explainTempTable(pParse,
+ isDistinct && !(p->selFlags&SF_Distinct)?"DISTINCT":"GROUP BY");
+
groupBySort = 1;
nGroupBy = pGroupBy->nExpr;
nCol = nGroupBy + 1;
@@ -91423,10 +91975,15 @@
} /* endif aggregate query */
+ if( distinct>=0 ){
+ explainTempTable(pParse, "DISTINCT");
+ }
+
/* If there is an ORDER BY clause, then we need to sort the results
** and send them to the callback one by one.
*/
if( pOrderBy ){
+ explainTempTable(pParse, "ORDER BY");
generateSortTail(pParse, p, v, pEList->nExpr, pDest);
}
@@ -91443,6 +92000,7 @@
** successful coding of the SELECT.
*/
select_end:
+ explainSetInteger(pParse->iSelectId, iRestoreSelectId);
/* Identify column names if results of the SELECT are to be output.
*/
@@ -94231,7 +94789,7 @@
sqlite3ChangeCookie(pParse, iDb);
sqlite3VdbeAddOp2(v, OP_Expire, 0, 0);
- zWhere = sqlite3MPrintf(db, "name='%q'", pTab->zName);
+ zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName);
sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 1, 0, zWhere, P4_DYNAMIC);
sqlite3VdbeAddOp4(v, OP_VCreate, iDb, 0, 0,
pTab->zName, sqlite3Strlen30(pTab->zName) + 1);
@@ -94532,7 +95090,7 @@
}
db->pVTab = 0;
}else{
- sqlite3Error(db, SQLITE_ERROR, zErr);
+ sqlite3Error(db, SQLITE_ERROR, (zErr ? "%s" : 0), zErr);
sqlite3DbFree(db, zErr);
rc = SQLITE_ERROR;
}
@@ -94994,7 +95552,6 @@
struct WhereCost {
WherePlan plan; /* The lookup strategy */
double rCost; /* Overall cost of pursuing this search strategy */
- double nRow; /* Estimated number of output rows */
Bitmask used; /* Bitmask of cursors used by this plan */
};
@@ -95037,10 +95594,11 @@
#define WHERE_COLUMN_IN 0x00040000 /* x IN (...) */
#define WHERE_COLUMN_NULL 0x00080000 /* x IS NULL */
#define WHERE_INDEXED 0x000f0000 /* Anything that uses an index */
-#define WHERE_NOT_FULLSCAN 0x000f3000 /* Does not do a full table scan */
+#define WHERE_NOT_FULLSCAN 0x100f3000 /* Does not do a full table scan */
#define WHERE_IN_ABLE 0x000f1000 /* Able to support an IN operator */
#define WHERE_TOP_LIMIT 0x00100000 /* x<EXPR or x<=EXPR constraint */
#define WHERE_BTM_LIMIT 0x00200000 /* x>EXPR or x>=EXPR constraint */
+#define WHERE_BOTH_LIMIT 0x00300000 /* Both x>EXPR and x<EXPR */
#define WHERE_IDX_ONLY 0x00800000 /* Use index only - omit table */
#define WHERE_ORDERBY 0x01000000 /* Output will appear in correct order */
#define WHERE_REVERSE 0x02000000 /* Scan in reverse order */
@@ -96358,7 +96916,8 @@
** Required because bestIndex() is called by bestOrClauseIndex()
*/
static void bestIndex(
- Parse*, WhereClause*, struct SrcList_item*, Bitmask, ExprList*, WhereCost*);
+ Parse*, WhereClause*, struct SrcList_item*,
+ Bitmask, Bitmask, ExprList*, WhereCost*);
/*
** This routine attempts to find an scanning strategy that can be used
@@ -96371,7 +96930,8 @@
Parse *pParse, /* The parsing context */
WhereClause *pWC, /* The WHERE clause */
struct SrcList_item *pSrc, /* The FROM clause term to search */
- Bitmask notReady, /* Mask of cursors that are not available */
+ Bitmask notReady, /* Mask of cursors not available for indexing */
+ Bitmask notValid, /* Cursors not available for any purpose */
ExprList *pOrderBy, /* The ORDER BY clause */
WhereCost *pCost /* Lowest cost query plan */
){
@@ -96381,8 +96941,9 @@
WhereTerm * const pWCEnd = &pWC->a[pWC->nTerm]; /* End of pWC->a[] */
WhereTerm *pTerm; /* A single term of the WHERE clause */
- /* No OR-clause optimization allowed if the NOT INDEXED clause is used */
- if( pSrc->notIndexed ){
+ /* No OR-clause optimization allowed if the INDEXED BY or NOT INDEXED clauses
+ ** are used */
+ if( pSrc->notIndexed || pSrc->pIndex!=0 ){
return;
}
@@ -96407,7 +96968,7 @@
));
if( pOrTerm->eOperator==WO_AND ){
WhereClause *pAndWC = &pOrTerm->u.pAndInfo->wc;
- bestIndex(pParse, pAndWC, pSrc, notReady, 0, &sTermCost);
+ bestIndex(pParse, pAndWC, pSrc, notReady, notValid, 0, &sTermCost);
}else if( pOrTerm->leftCursor==iCur ){
WhereClause tempWC;
tempWC.pParse = pWC->pParse;
@@ -96415,12 +96976,12 @@
tempWC.op = TK_AND;
tempWC.a = pOrTerm;
tempWC.nTerm = 1;
- bestIndex(pParse, &tempWC, pSrc, notReady, 0, &sTermCost);
+ bestIndex(pParse, &tempWC, pSrc, notReady, notValid, 0, &sTermCost);
}else{
continue;
}
rTotal += sTermCost.rCost;
- nRow += sTermCost.nRow;
+ nRow += sTermCost.plan.nRow;
used |= sTermCost.used;
if( rTotal>=pCost->rCost ) break;
}
@@ -96439,8 +97000,8 @@
WHERETRACE(("... multi-index OR cost=%.9g nrow=%.9g\n", rTotal, nRow));
if( rTotal<pCost->rCost ){
pCost->rCost = rTotal;
- pCost->nRow = nRow;
pCost->used = used;
+ pCost->plan.nRow = nRow;
pCost->plan.wsFlags = flags;
pCost->plan.u.pTerm = pTerm;
}
@@ -96524,7 +97085,7 @@
WHERETRACE(("auto-index reduces cost from %.2f to %.2f\n",
pCost->rCost, costTempIdx));
pCost->rCost = costTempIdx;
- pCost->nRow = logN + 1;
+ pCost->plan.nRow = logN + 1;
pCost->plan.wsFlags = WHERE_TEMP_INDEX;
pCost->used = pTerm->prereqRight;
break;
@@ -96862,7 +97423,8 @@
Parse *pParse, /* The parsing context */
WhereClause *pWC, /* The WHERE clause */
struct SrcList_item *pSrc, /* The FROM clause term to search */
- Bitmask notReady, /* Mask of cursors that are not available */
+ Bitmask notReady, /* Mask of cursors not available for index */
+ Bitmask notValid, /* Cursors not valid for any purpose */
ExprList *pOrderBy, /* The order by clause */
WhereCost *pCost, /* Lowest cost query plan */
sqlite3_index_info **ppIdxInfo /* Index information passed to xBestIndex */
@@ -96992,7 +97554,7 @@
/* Try to find a more efficient access pattern by using multiple indexes
** to optimize an OR expression within the WHERE clause.
*/
- bestOrClauseIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost);
+ bestOrClauseIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost);
}
#endif /* SQLITE_OMIT_VIRTUALTABLE */
@@ -97273,7 +97835,8 @@
Parse *pParse, /* The parsing context */
WhereClause *pWC, /* The WHERE clause */
struct SrcList_item *pSrc, /* The FROM clause term to search */
- Bitmask notReady, /* Mask of cursors that are not available */
+ Bitmask notReady, /* Mask of cursors not available for indexing */
+ Bitmask notValid, /* Cursors not available for any purpose */
ExprList *pOrderBy, /* The ORDER BY clause */
WhereCost *pCost /* Lowest cost query plan */
){
@@ -97535,16 +98098,16 @@
** with this step if we already know this index will not be chosen.
** Also, never reduce the output row count below 2 using this step.
**
- ** Do not reduce the output row count if pSrc is the only table that
- ** is notReady; if notReady is a power of two. This will be the case
- ** when the main sqlite3WhereBegin() loop is scanning for a table with
- ** and "optimal" index, and on such a scan the output row count
- ** reduction is not valid because it does not update the "pCost->used"
- ** bitmap. The notReady bitmap will also be a power of two when we
- ** are scanning for the last table in a 64-way join. We are willing
- ** to bypass this optimization in that corner case.
+ ** It is critical that the notValid mask be used here instead of
+ ** the notReady mask. When computing an "optimal" index, the notReady
+ ** mask will only have one bit set - the bit for the current table.
+ ** The notValid mask, on the other hand, always has all bits set for
+ ** tables that are not in outer loops. If notReady is used here instead
+ ** of notValid, then a optimal index that depends on inner joins loops
+ ** might be selected even when there exists an optimal index that has
+ ** no such dependency.
*/
- if( nRow>2 && cost<=pCost->rCost && (notReady & (notReady-1))!=0 ){
+ if( nRow>2 && cost<=pCost->rCost ){
int k; /* Loop counter */
int nSkipEq = nEq; /* Number of == constraints to skip */
int nSkipRange = nBound; /* Number of < constraints to skip */
@@ -97553,7 +98116,7 @@
thisTab = getMask(pWC->pMaskSet, iCur);
for(pTerm=pWC->a, k=pWC->nTerm; nRow>2 && k; k--, pTerm++){
if( pTerm->wtFlags & TERM_VIRTUAL ) continue;
- if( (pTerm->prereqAll & notReady)!=thisTab ) continue;
+ if( (pTerm->prereqAll & notValid)!=thisTab ) continue;
if( pTerm->eOperator & (WO_EQ|WO_IN|WO_ISNULL) ){
if( nSkipEq ){
/* Ignore the first nEq equality matches since the index
@@ -97595,11 +98158,11 @@
** index and its cost in the pCost structure.
*/
if( (!pIdx || wsFlags)
- && (cost<pCost->rCost || (cost<=pCost->rCost && nRow<pCost->nRow))
+ && (cost<pCost->rCost || (cost<=pCost->rCost && nRow<pCost->plan.nRow))
){
pCost->rCost = cost;
- pCost->nRow = nRow;
pCost->used = used;
+ pCost->plan.nRow = nRow;
pCost->plan.wsFlags = (wsFlags&wsFlagMask);
pCost->plan.nEq = nEq;
pCost->plan.u.pIdx = pIdx;
@@ -97635,7 +98198,7 @@
pCost->plan.u.pIdx ? pCost->plan.u.pIdx->zName : "ipk")
));
- bestOrClauseIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost);
+ bestOrClauseIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost);
bestAutomaticIndex(pParse, pWC, pSrc, notReady, pCost);
pCost->plan.wsFlags |= eqTermMask;
}
@@ -97650,14 +98213,15 @@
Parse *pParse, /* The parsing context */
WhereClause *pWC, /* The WHERE clause */
struct SrcList_item *pSrc, /* The FROM clause term to search */
- Bitmask notReady, /* Mask of cursors that are not available */
+ Bitmask notReady, /* Mask of cursors not available for indexing */
+ Bitmask notValid, /* Cursors not available for any purpose */
ExprList *pOrderBy, /* The ORDER BY clause */
WhereCost *pCost /* Lowest cost query plan */
){
#ifndef SQLITE_OMIT_VIRTUALTABLE
if( IsVirtual(pSrc->pTab) ){
sqlite3_index_info *p = 0;
- bestVirtualIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost, &p);
+ bestVirtualIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost,&p);
if( p->needToFreeIdxStr ){
sqlite3_free(p->idxStr);
}
@@ -97665,7 +98229,7 @@
}else
#endif
{
- bestBtreeIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost);
+ bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost);
}
}
@@ -97927,6 +98491,161 @@
return regBase;
}
+#ifndef SQLITE_OMIT_EXPLAIN
+/*
+** This routine is a helper for explainIndexRange() below
+**
+** pStr holds the text of an expression that we are building up one term
+** at a time. This routine adds a new term to the end of the expression.
+** Terms are separated by AND so add the "AND" text for second and subsequent
+** terms only.
+*/
+static void explainAppendTerm(
+ StrAccum *pStr, /* The text expression being built */
+ int iTerm, /* Index of this term. First is zero */
+ const char *zColumn, /* Name of the column */
+ const char *zOp /* Name of the operator */
+){
+ if( iTerm ) sqlite3StrAccumAppend(pStr, " AND ", 5);
+ sqlite3StrAccumAppend(pStr, zColumn, -1);
+ sqlite3StrAccumAppend(pStr, zOp, 1);
+ sqlite3StrAccumAppend(pStr, "?", 1);
+}
+
+/*
+** Argument pLevel describes a strategy for scanning table pTab. This
+** function returns a pointer to a string buffer containing a description
+** of the subset of table rows scanned by the strategy in the form of an
+** SQL expression. Or, if all rows are scanned, NULL is returned.
+**
+** For example, if the query:
+**
+** SELECT * FROM t1 WHERE a=1 AND b>2;
+**
+** is run and there is an index on (a, b), then this function returns a
+** string similar to:
+**
+** "a=? AND b>?"
+**
+** The returned pointer points to memory obtained from sqlite3DbMalloc().
+** It is the responsibility of the caller to free the buffer when it is
+** no longer required.
+*/
+static char *explainIndexRange(sqlite3 *db, WhereLevel *pLevel, Table *pTab){
+ WherePlan *pPlan = &pLevel->plan;
+ Index *pIndex = pPlan->u.pIdx;
+ int nEq = pPlan->nEq;
+ int i, j;
+ Column *aCol = pTab->aCol;
+ int *aiColumn = pIndex->aiColumn;
+ StrAccum txt;
+
+ if( nEq==0 && (pPlan->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ){
+ return 0;
+ }
+ sqlite3StrAccumInit(&txt, 0, 0, SQLITE_MAX_LENGTH);
+ txt.db = db;
+ sqlite3StrAccumAppend(&txt, " (", 2);
+ for(i=0; i<nEq; i++){
+ explainAppendTerm(&txt, i, aCol[aiColumn[i]].zName, "=");
+ }
+
+ j = i;
+ if( pPlan->wsFlags&WHERE_BTM_LIMIT ){
+ explainAppendTerm(&txt, i++, aCol[aiColumn[j]].zName, ">");
+ }
+ if( pPlan->wsFlags&WHERE_TOP_LIMIT ){
+ explainAppendTerm(&txt, i, aCol[aiColumn[j]].zName, "<");
+ }
+ sqlite3StrAccumAppend(&txt, ")", 1);
+ return sqlite3StrAccumFinish(&txt);
+}
+
+/*
+** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN
+** command. If the query being compiled is an EXPLAIN QUERY PLAN, a single
+** record is added to the output to describe the table scan strategy in
+** pLevel.
+*/
+static void explainOneScan(
+ Parse *pParse, /* Parse context */
+ SrcList *pTabList, /* Table list this loop refers to */
+ WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */
+ int iLevel, /* Value for "level" column of output */
+ int iFrom, /* Value for "from" column of output */
+ u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */
+){
+ if( pParse->explain==2 ){
+ u32 flags = pLevel->plan.wsFlags;
+ struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom];
+ Vdbe *v = pParse->pVdbe; /* VM being constructed */
+ sqlite3 *db = pParse->db; /* Database handle */
+ char *zMsg; /* Text to add to EQP output */
+ sqlite3_int64 nRow; /* Expected number of rows visited by scan */
+ int iId = pParse->iSelectId; /* Select id (left-most output column) */
+ int isSearch; /* True for a SEARCH. False for SCAN. */
+
+ if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return;
+
+ isSearch = (pLevel->plan.nEq>0)
+ || (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0
+ || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX));
+
+ zMsg = sqlite3MPrintf(db, "%s", isSearch?"SEARCH":"SCAN");
+ if( pItem->pSelect ){
+ zMsg = sqlite3MAppendf(db, zMsg, "%s SUBQUERY %d", zMsg,pItem->iSelectId);
+ }else{
+ zMsg = sqlite3MAppendf(db, zMsg, "%s TABLE %s", zMsg, pItem->zName);
+ }
+
+ if( pItem->zAlias ){
+ zMsg = sqlite3MAppendf(db, zMsg, "%s AS %s", zMsg, pItem->zAlias);
+ }
+ if( (flags & WHERE_INDEXED)!=0 ){
+ char *zWhere = explainIndexRange(db, pLevel, pItem->pTab);
+ zMsg = sqlite3MAppendf(db, zMsg, "%s USING %s%sINDEX%s%s%s", zMsg,
+ ((flags & WHERE_TEMP_INDEX)?"AUTOMATIC ":""),
+ ((flags & WHERE_IDX_ONLY)?"COVERING ":""),
+ ((flags & WHERE_TEMP_INDEX)?"":" "),
+ ((flags & WHERE_TEMP_INDEX)?"": pLevel->plan.u.pIdx->zName),
+ zWhere
+ );
+ sqlite3DbFree(db, zWhere);
+ }else if( flags & (WHERE_ROWID_EQ|WHERE_ROWID_RANGE) ){
+ zMsg = sqlite3MAppendf(db, zMsg, "%s USING INTEGER PRIMARY KEY", zMsg);
+
+ if( flags&WHERE_ROWID_EQ ){
+ zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid=?)", zMsg);
+ }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){
+ zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid>? AND rowid<?)", zMsg);
+ }else if( flags&WHERE_BTM_LIMIT ){
+ zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid>?)", zMsg);
+ }else if( flags&WHERE_TOP_LIMIT ){
+ zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid<?)", zMsg);
+ }
+ }
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+ else if( (flags & WHERE_VIRTUALTABLE)!=0 ){
+ sqlite3_index_info *pVtabIdx = pLevel->plan.u.pVtabIdx;
+ zMsg = sqlite3MAppendf(db, zMsg, "%s VIRTUAL TABLE INDEX %d:%s", zMsg,
+ pVtabIdx->idxNum, pVtabIdx->idxStr);
+ }
+#endif
+ if( wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX) ){
+ testcase( wctrlFlags & WHERE_ORDERBY_MIN );
+ nRow = 1;
+ }else{
+ nRow = (sqlite3_int64)pLevel->plan.nRow;
+ }
+ zMsg = sqlite3MAppendf(db, zMsg, "%s (~%lld rows)", zMsg, nRow);
+ sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg, P4_DYNAMIC);
+ }
+}
+#else
+# define explainOneScan(u,v,w,x,y,z)
+#endif /* SQLITE_OMIT_EXPLAIN */
+
+
/*
** Generate code for the start of the iLevel-th loop in the WHERE clause
** implementation described by pWInfo.
@@ -98334,7 +99053,7 @@
r1 = sqlite3GetTempReg(pParse);
testcase( pLevel->plan.wsFlags & WHERE_BTM_LIMIT );
testcase( pLevel->plan.wsFlags & WHERE_TOP_LIMIT );
- if( pLevel->plan.wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT) ){
+ if( (pLevel->plan.wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 ){
sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, nEq, r1);
sqlite3VdbeAddOp2(v, OP_IsNull, r1, addrCont);
}
@@ -98468,6 +99187,9 @@
WHERE_OMIT_OPEN | WHERE_OMIT_CLOSE |
WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY);
if( pSubWInfo ){
+ explainOneScan(
+ pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0
+ );
if( (wctrlFlags & WHERE_DUPLICATES_OK)==0 ){
int iSet = ((ii==pOrWc->nTerm-1)?-1:ii);
int r;
@@ -98863,6 +99585,7 @@
memset(&bestPlan, 0, sizeof(bestPlan));
bestPlan.rCost = SQLITE_BIG_DBL;
+ WHERETRACE(("*** Begin search for loop %d ***\n", i));
/* Loop through the remaining entries in the FROM clause to find the
** next nested loop. The loop tests all FROM clause entries
@@ -98881,9 +99604,15 @@
** other FROM clause terms that are notReady. If no notReady terms are
** used then the "optimal" query plan works.
**
+ ** Note that the WhereCost.nRow parameter for an optimal scan might
+ ** not be as small as it would be if the table really were the innermost
+ ** join. The nRow value can be reduced by WHERE clause constraints
+ ** that do not use indices. But this nRow reduction only happens if the
+ ** table really is the innermost join.
+ **
** The second loop iteration is only performed if no optimal scan
- ** strategies were found by the first loop. This 2nd iteration is used to
- ** search for the lowest cost scan overall.
+ ** strategies were found by the first iteration. This second iteration
+ ** is used to search for the lowest cost scan overall.
**
** Previous versions of SQLite performed only the second iteration -
** the next outermost loop was always that with the lowest overall
@@ -98903,7 +99632,7 @@
*/
nUnconstrained = 0;
notIndexed = 0;
- for(isOptimal=(iFrom<nTabList-1); isOptimal>=0; isOptimal--){
+ for(isOptimal=(iFrom<nTabList-1); isOptimal>=0 && bestJ<0; isOptimal--){
Bitmask mask; /* Mask of tables not yet ready */
for(j=iFrom, pTabItem=&pTabList->a[j]; j<nTabList; j++, pTabItem++){
int doNotReorder; /* True if this table should not be reordered */
@@ -98921,15 +99650,19 @@
pOrderBy = ((i==0 && ppOrderBy )?*ppOrderBy:0);
if( pTabItem->pIndex==0 ) nUnconstrained++;
+ WHERETRACE(("=== trying table %d with isOptimal=%d ===\n",
+ j, isOptimal));
assert( pTabItem->pTab );
#ifndef SQLITE_OMIT_VIRTUALTABLE
if( IsVirtual(pTabItem->pTab) ){
sqlite3_index_info **pp = &pWInfo->a[j].pIdxInfo;
- bestVirtualIndex(pParse, pWC, pTabItem, mask, pOrderBy, &sCost, pp);
+ bestVirtualIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy,
+ &sCost, pp);
}else
#endif
{
- bestBtreeIndex(pParse, pWC, pTabItem, mask, pOrderBy, &sCost);
+ bestBtreeIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy,
+ &sCost);
}
assert( isOptimal || (sCost.used¬Ready)==0 );
@@ -98969,10 +99702,12 @@
&& (nUnconstrained==0 || pTabItem->pIndex==0 /* (3) */
|| NEVER((sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0))
&& (bestJ<0 || sCost.rCost<bestPlan.rCost /* (4) */
- || (sCost.rCost<=bestPlan.rCost && sCost.nRow<bestPlan.nRow))
+ || (sCost.rCost<=bestPlan.rCost
+ && sCost.plan.nRow<bestPlan.plan.nRow))
){
- WHERETRACE(("... best so far with cost=%g and nRow=%g\n",
- sCost.rCost, sCost.nRow));
+ WHERETRACE(("=== table %d is best so far"
+ " with cost=%g and nRow=%g\n",
+ j, sCost.rCost, sCost.plan.nRow));
bestPlan = sCost;
bestJ = j;
}
@@ -98981,8 +99716,9 @@
}
assert( bestJ>=0 );
assert( notReady & getMask(pMaskSet, pTabList->a[bestJ].iCursor) );
- WHERETRACE(("*** Optimizer selects table %d for loop %d\n", bestJ,
- pLevel-pWInfo->a));
+ WHERETRACE(("*** Optimizer selects table %d for loop %d"
+ " with cost=%g and nRow=%g\n",
+ bestJ, pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow));
if( (bestPlan.plan.wsFlags & WHERE_ORDERBY)!=0 ){
*ppOrderBy = 0;
}
@@ -98997,7 +99733,9 @@
}
notReady &= ~getMask(pMaskSet, pTabList->a[bestJ].iCursor);
pLevel->iFrom = (u8)bestJ;
- if( bestPlan.nRow>=(double)1 ) pParse->nQueryLoop *= bestPlan.nRow;
+ if( bestPlan.plan.nRow>=(double)1 ){
+ pParse->nQueryLoop *= bestPlan.plan.nRow;
+ }
/* Check that if the table scanned by this loop iteration had an
** INDEXED BY clause attached to it, that the named index is being
@@ -99045,44 +99783,15 @@
*/
sqlite3CodeVerifySchema(pParse, -1); /* Insert the cookie verifier Goto */
notReady = ~(Bitmask)0;
+ pWInfo->nRowOut = (double)1;
for(i=0, pLevel=pWInfo->a; i<nTabList; i++, pLevel++){
Table *pTab; /* Table to open */
int iDb; /* Index of database containing table/index */
-#ifndef SQLITE_OMIT_EXPLAIN
- if( pParse->explain==2 ){
- char *zMsg;
- struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom];
- zMsg = sqlite3MPrintf(db, "TABLE %s", pItem->zName);
- if( pItem->zAlias ){
- zMsg = sqlite3MAppendf(db, zMsg, "%s AS %s", zMsg, pItem->zAlias);
- }
- if( (pLevel->plan.wsFlags & WHERE_TEMP_INDEX)!=0 ){
- zMsg = sqlite3MAppendf(db, zMsg, "%s WITH AUTOMATIC INDEX", zMsg);
- }else if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){
- zMsg = sqlite3MAppendf(db, zMsg, "%s WITH INDEX %s",
- zMsg, pLevel->plan.u.pIdx->zName);
- }else if( pLevel->plan.wsFlags & WHERE_MULTI_OR ){
- zMsg = sqlite3MAppendf(db, zMsg, "%s VIA MULTI-INDEX UNION", zMsg);
- }else if( pLevel->plan.wsFlags & (WHERE_ROWID_EQ|WHERE_ROWID_RANGE) ){
- zMsg = sqlite3MAppendf(db, zMsg, "%s USING PRIMARY KEY", zMsg);
- }
-#ifndef SQLITE_OMIT_VIRTUALTABLE
- else if( (pLevel->plan.wsFlags & WHERE_VIRTUALTABLE)!=0 ){
- sqlite3_index_info *pVtabIdx = pLevel->plan.u.pVtabIdx;
- zMsg = sqlite3MAppendf(db, zMsg, "%s VIRTUAL TABLE INDEX %d:%s", zMsg,
- pVtabIdx->idxNum, pVtabIdx->idxStr);
- }
-#endif
- if( pLevel->plan.wsFlags & WHERE_ORDERBY ){
- zMsg = sqlite3MAppendf(db, zMsg, "%s ORDER BY", zMsg);
- }
- sqlite3VdbeAddOp4(v, OP_Explain, i, pLevel->iFrom, 0, zMsg, P4_DYNAMIC);
- }
-#endif /* SQLITE_OMIT_EXPLAIN */
pTabItem = &pTabList->a[pLevel->iFrom];
pTab = pTabItem->pTab;
pLevel->iTabCur = pTabItem->iCursor;
+ pWInfo->nRowOut *= pLevel->plan.nRow;
iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
if( (pTab->tabFlags & TF_Ephemeral)!=0 || pTab->pSelect ){
/* Do nothing */
@@ -99138,8 +99847,10 @@
*/
notReady = ~(Bitmask)0;
for(i=0; i<nTabList; i++){
+ pLevel = &pWInfo->a[i];
+ explainOneScan(pParse, pTabList, pLevel, i, pLevel->iFrom, wctrlFlags);
notReady = codeOneLoopStart(pWInfo, i, wctrlFlags, notReady);
- pWInfo->iContinue = pWInfo->a[i].addrCont;
+ pWInfo->iContinue = pLevel->addrCont;
}
#ifdef SQLITE_TEST /* For testing and debugging use only */
@@ -105541,13 +106252,12 @@
}
pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 1);
- if( pColl ){
- pColl->xCmp = xCompare;
- pColl->pUser = pCtx;
- pColl->xDel = xDel;
- pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED));
- pColl->type = collType;
- }
+ if( pColl==0 ) return SQLITE_NOMEM;
+ pColl->xCmp = xCompare;
+ pColl->pUser = pCtx;
+ pColl->xDel = xDel;
+ pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED));
+ pColl->type = collType;
sqlite3Error(db, SQLITE_OK, 0);
return SQLITE_OK;
}
@@ -106296,7 +107006,10 @@
assert( pPager!=0 );
fd = sqlite3PagerFile(pPager);
assert( fd!=0 );
- if( fd->pMethods ){
+ if( op==SQLITE_FCNTL_FILE_POINTER ){
+ *(sqlite3_file**)pArg = fd;
+ rc = SQLITE_OK;
+ }else if( fd->pMethods ){
rc = sqlite3OsFileControl(fd, op, pArg);
}
sqlite3BtreeLeave(pBtree);
@@ -107515,8 +108228,14 @@
** Macros indicating that conditional expressions are always true or
** false.
*/
+#ifdef SQLITE_COVERAGE_TEST
+# define ALWAYS(x) (1)
+# define NEVER(X) (0)
+#else
# define ALWAYS(x) (x)
# define NEVER(X) (x)
+#endif
+
/*
** Internal types used by SQLite.
*/
@@ -107534,8 +108253,12 @@
typedef struct Fts3Cursor Fts3Cursor;
typedef struct Fts3Expr Fts3Expr;
typedef struct Fts3Phrase Fts3Phrase;
-typedef struct Fts3SegReader Fts3SegReader;
+typedef struct Fts3PhraseToken Fts3PhraseToken;
+
typedef struct Fts3SegFilter Fts3SegFilter;
+typedef struct Fts3DeferredToken Fts3DeferredToken;
+typedef struct Fts3SegReader Fts3SegReader;
+typedef struct Fts3SegReaderArray Fts3SegReaderArray;
/*
** A connection to a fulltext index is an instance of the following
@@ -107556,22 +108279,14 @@
/* Precompiled statements used by the implementation. Each of these
** statements is run and reset within a single virtual table API call.
*/
- sqlite3_stmt *aStmt[25];
-
- /* Pointer to string containing the SQL:
- **
- ** "SELECT block FROM %_segments WHERE blockid BETWEEN ? AND ?
- ** ORDER BY blockid"
- */
- char *zSelectLeaves;
- int nLeavesStmt; /* Valid statements in aLeavesStmt */
- int nLeavesTotal; /* Total number of prepared leaves stmts */
- int nLeavesAlloc; /* Allocated size of aLeavesStmt */
- sqlite3_stmt **aLeavesStmt; /* Array of prepared zSelectLeaves stmts */
+ sqlite3_stmt *aStmt[24];
int nNodeSize; /* Soft limit for node size */
- u8 bHasContent; /* True if %_content table exists */
+ u8 bHasStat; /* True if %_stat table exists */
u8 bHasDocsize; /* True if %_docsize table exists */
+ int nPgsz; /* Page size for host database */
+ char *zSegmentsTbl; /* Name of %_segments table */
+ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */
/* The following hash table is used to buffer pending index updates during
** transactions. Variable nPendingData estimates the memory size of the
@@ -107598,14 +108313,25 @@
u8 isRequireSeek; /* True if must seek pStmt to %_content row */
sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */
Fts3Expr *pExpr; /* Parsed MATCH query string */
+ int nPhrase; /* Number of matchable phrases in query */
+ Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */
sqlite3_int64 iPrevId; /* Previous id read from aDoclist */
char *pNextId; /* Pointer into the body of aDoclist */
char *aDoclist; /* List of docids for full-text queries */
int nDoclist; /* Size of buffer at aDoclist */
+ int eEvalmode; /* An FTS3_EVAL_XX constant */
+ int nRowAvg; /* Average size of database rows, in pages */
+
int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */
u32 *aMatchinfo; /* Information about most recent match */
+ int nMatchinfo; /* Number of elements in aMatchinfo[] */
+ char *zMatchinfo; /* Matchinfo specification */
};
+#define FTS3_EVAL_FILTER 0
+#define FTS3_EVAL_NEXT 1
+#define FTS3_EVAL_MATCHINFO 2
+
/*
** The Fts3Cursor.eSearch member is always set to one of the following.
** Actualy, Fts3Cursor.eSearch can be greater than or equal to
@@ -107628,18 +108354,30 @@
/*
** A "phrase" is a sequence of one or more tokens that must match in
** sequence. A single token is the base case and the most common case.
-** For a sequence of tokens contained in "...", nToken will be the number
-** of tokens in the string.
+** For a sequence of tokens contained in double-quotes (i.e. "one two three")
+** nToken will be the number of tokens in the string.
+**
+** The nDocMatch and nMatch variables contain data that may be used by the
+** matchinfo() function. They are populated when the full-text index is
+** queried for hits on the phrase. If one or more tokens in the phrase
+** are deferred, the nDocMatch and nMatch variables are populated based
+** on the assumption that the
*/
+struct Fts3PhraseToken {
+ char *z; /* Text of the token */
+ int n; /* Number of bytes in buffer z */
+ int isPrefix; /* True if token ends with a "*" character */
+ int bFulltext; /* True if full-text index was used */
+ Fts3SegReaderArray *pArray; /* Segment-reader for this token */
+ Fts3DeferredToken *pDeferred; /* Deferred token object for this token */
+};
+
struct Fts3Phrase {
+ /* Variables populated by fts3_expr.c when parsing a MATCH expression */
int nToken; /* Number of tokens in the phrase */
int iColumn; /* Index of column this phrase must match */
int isNot; /* Phrase prefixed by unary not (-) operator */
- struct PhraseToken {
- char *z; /* Text of the token */
- int n; /* Number of bytes in buffer pointed to by z */
- int isPrefix; /* True if token ends in with a "*" character */
- } aToken[1]; /* One entry for each token in the phrase */
+ Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */
};
/*
@@ -107689,29 +108427,34 @@
#define FTSQUERY_PHRASE 5
-/* fts3_init.c */
-SQLITE_PRIVATE int sqlite3Fts3DeleteVtab(int, sqlite3_vtab *);
-SQLITE_PRIVATE int sqlite3Fts3InitVtab(int, sqlite3*, void*, int, const char*const*,
- sqlite3_vtab **, char **);
-
/* fts3_write.c */
SQLITE_PRIVATE int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*);
SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *);
SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *);
SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *);
-SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(Fts3Table *,int, sqlite3_int64,
+SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(int, sqlite3_int64,
sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
SQLITE_PRIVATE int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**);
-SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3Table *, Fts3SegReader *);
+SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *);
SQLITE_PRIVATE int sqlite3Fts3SegReaderIterate(
Fts3Table *, Fts3SegReader **, int, Fts3SegFilter *,
int (*)(Fts3Table *, void *, char *, int, char *, int), void *
);
-SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char const**, int*);
+SQLITE_PRIVATE int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *);
SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **);
-SQLITE_PRIVATE int sqlite3Fts3MatchinfoDocsizeLocal(Fts3Cursor*, u32*);
-SQLITE_PRIVATE int sqlite3Fts3MatchinfoDocsizeGlobal(Fts3Cursor*, u32*);
SQLITE_PRIVATE int sqlite3Fts3ReadLock(Fts3Table *);
+SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*);
+
+SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **);
+SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **);
+
+SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *);
+SQLITE_PRIVATE int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int);
+SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *);
+SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *);
+SQLITE_PRIVATE char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *);
+
+SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *);
/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
#define FTS3_SEGMENT_REQUIRE_POS 0x00000001
@@ -107735,22 +108478,24 @@
SQLITE_PRIVATE void sqlite3Fts3Dequote(char *);
SQLITE_PRIVATE char *sqlite3Fts3FindPositions(Fts3Expr *, sqlite3_int64, int);
-SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Table *, Fts3Expr *);
+SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *);
+SQLITE_PRIVATE int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *);
SQLITE_PRIVATE int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int);
/* fts3_tokenizer.c */
SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *);
SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *);
-SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash,
- const char *, sqlite3_tokenizer **, const char **, char **
+SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *,
+ sqlite3_tokenizer **, char **
);
+SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char);
/* fts3_snippet.c */
SQLITE_PRIVATE void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*);
SQLITE_PRIVATE void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *,
const char *, const char *, int, int
);
-SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *);
+SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *);
/* fts3_expr.c */
SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *,
@@ -107899,16 +108644,13 @@
int i;
assert( p->nPendingData==0 );
+ assert( p->pSegments==0 );
/* Free any prepared statements held */
for(i=0; i<SizeofArray(p->aStmt); i++){
sqlite3_finalize(p->aStmt[i]);
}
- for(i=0; i<p->nLeavesStmt; i++){
- sqlite3_finalize(p->aLeavesStmt[i]);
- }
- sqlite3_free(p->zSelectLeaves);
- sqlite3_free(p->aLeavesStmt);
+ sqlite3_free(p->zSegmentsTbl);
/* Invoke the tokenizer destructor to free the tokenizer. */
p->pTokenizer->pModule->xDestroy(p->pTokenizer);
@@ -107919,7 +108661,7 @@
/*
** Construct one or more SQL statements from the format string given
-** and then evaluate those statements. The success code is writting
+** and then evaluate those statements. The success code is written
** into *pRc.
**
** If *pRc is initially non-zero then this routine is a no-op.
@@ -107971,33 +108713,38 @@
** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table
** passed as the first argument. This is done as part of the xConnect()
** and xCreate() methods.
+**
+** If *pRc is non-zero when this function is called, it is a no-op.
+** Otherwise, if an error occurs, an SQLite error code is stored in *pRc
+** before returning.
*/
-static int fts3DeclareVtab(Fts3Table *p){
- int i; /* Iterator variable */
- int rc; /* Return code */
- char *zSql; /* SQL statement passed to declare_vtab() */
- char *zCols; /* List of user defined columns */
+static void fts3DeclareVtab(int *pRc, Fts3Table *p){
+ if( *pRc==SQLITE_OK ){
+ int i; /* Iterator variable */
+ int rc; /* Return code */
+ char *zSql; /* SQL statement passed to declare_vtab() */
+ char *zCols; /* List of user defined columns */
- /* Create a list of user columns for the virtual table */
- zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]);
- for(i=1; zCols && i<p->nColumn; i++){
- zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]);
+ /* Create a list of user columns for the virtual table */
+ zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]);
+ for(i=1; zCols && i<p->nColumn; i++){
+ zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]);
+ }
+
+ /* Create the whole "CREATE TABLE" statement to pass to SQLite */
+ zSql = sqlite3_mprintf(
+ "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName
+ );
+ if( !zCols || !zSql ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_declare_vtab(p->db, zSql);
+ }
+
+ sqlite3_free(zSql);
+ sqlite3_free(zCols);
+ *pRc = rc;
}
-
- /* Create the whole "CREATE TABLE" statement to pass to SQLite */
- zSql = sqlite3_mprintf(
- "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName
- );
-
- if( !zCols || !zSql ){
- rc = SQLITE_NOMEM;
- }else{
- rc = sqlite3_declare_vtab(p->db, zSql);
- }
-
- sqlite3_free(zSql);
- sqlite3_free(zCols);
- return rc;
}
/*
@@ -108016,21 +108763,19 @@
sqlite3 *db = p->db; /* The database connection */
/* Create a list of user columns for the content table */
- if( p->bHasContent ){
- zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY");
- for(i=0; zContentCols && i<p->nColumn; i++){
- char *z = p->azColumn[i];
- zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z);
- }
- if( zContentCols==0 ) rc = SQLITE_NOMEM;
-
- /* Create the content table */
- fts3DbExec(&rc, db,
- "CREATE TABLE %Q.'%q_content'(%s)",
- p->zDb, p->zName, zContentCols
- );
- sqlite3_free(zContentCols);
+ zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY");
+ for(i=0; zContentCols && i<p->nColumn; i++){
+ char *z = p->azColumn[i];
+ zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z);
}
+ if( zContentCols==0 ) rc = SQLITE_NOMEM;
+
+ /* Create the content table */
+ fts3DbExec(&rc, db,
+ "CREATE TABLE %Q.'%q_content'(%s)",
+ p->zDb, p->zName, zContentCols
+ );
+ sqlite3_free(zContentCols);
/* Create other tables */
fts3DbExec(&rc, db,
"CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);",
@@ -108053,6 +108798,8 @@
"CREATE TABLE %Q.'%q_docsize'(docid INTEGER PRIMARY KEY, size BLOB);",
p->zDb, p->zName
);
+ }
+ if( p->bHasStat ){
fts3DbExec(&rc, db,
"CREATE TABLE %Q.'%q_stat'(id INTEGER PRIMARY KEY, value BLOB);",
p->zDb, p->zName
@@ -108062,39 +108809,63 @@
}
/*
-** An sqlite3_exec() callback for fts3TableExists.
+** Store the current database page-size in bytes in p->nPgsz.
+**
+** If *pRc is non-zero when this function is called, it is a no-op.
+** Otherwise, if an error occurs, an SQLite error code is stored in *pRc
+** before returning.
*/
-static int fts3TableExistsCallback(void *pArg, int n, char **pp1, char **pp2){
- UNUSED_PARAMETER(n);
- UNUSED_PARAMETER(pp1);
- UNUSED_PARAMETER(pp2);
- *(int*)pArg = 1;
- return 1;
+static void fts3DatabasePageSize(int *pRc, Fts3Table *p){
+ if( *pRc==SQLITE_OK ){
+ int rc; /* Return code */
+ char *zSql; /* SQL text "PRAGMA %Q.page_size" */
+ sqlite3_stmt *pStmt; /* Compiled "PRAGMA %Q.page_size" statement */
+
+ zSql = sqlite3_mprintf("PRAGMA %Q.page_size", p->zDb);
+ if( !zSql ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_step(pStmt);
+ p->nPgsz = sqlite3_column_int(pStmt, 0);
+ rc = sqlite3_finalize(pStmt);
+ }
+ }
+ assert( p->nPgsz>0 || rc!=SQLITE_OK );
+ sqlite3_free(zSql);
+ *pRc = rc;
+ }
}
/*
-** Determine if a table currently exists in the database.
+** "Special" FTS4 arguments are column specifications of the following form:
+**
+** <key> = <value>
+**
+** There may not be whitespace surrounding the "=" character. The <value>
+** term may be quoted, but the <key> may not.
*/
-static void fts3TableExists(
- int *pRc, /* Success code */
- sqlite3 *db, /* The database connection to test */
- const char *zDb, /* ATTACHed database within the connection */
- const char *zName, /* Name of the FTS3 table */
- const char *zSuffix, /* Shadow table extension */
- u8 *pResult /* Write results here */
+static int fts3IsSpecialColumn(
+ const char *z,
+ int *pnKey,
+ char **pzValue
){
- int rc = SQLITE_OK;
- int res = 0;
- char *zSql;
- if( *pRc ) return;
- zSql = sqlite3_mprintf(
- "SELECT 1 FROM %Q.sqlite_master WHERE name='%q%s'",
- zDb, zName, zSuffix
- );
- rc = sqlite3_exec(db, zSql, fts3TableExistsCallback, &res, 0);
- sqlite3_free(zSql);
- *pResult = (u8)(res & 0xff);
- if( rc!=SQLITE_ABORT ) *pRc = rc;
+ char *zValue;
+ const char *zCsr = z;
+
+ while( *zCsr!='=' ){
+ if( *zCsr=='\0' ) return 0;
+ zCsr++;
+ }
+
+ *pnKey = (int)(zCsr-z);
+ zValue = sqlite3_mprintf("%s", &zCsr[1]);
+ if( zValue ){
+ sqlite3Fts3Dequote(zValue);
+ }
+ *pzValue = zValue;
+ return 1;
}
/*
@@ -108118,8 +108889,8 @@
char **pzErr /* Write any error message here */
){
Fts3Hash *pHash = (Fts3Hash *)pAux;
- Fts3Table *p; /* Pointer to allocated vtab */
- int rc; /* Return code */
+ Fts3Table *p = 0; /* Pointer to allocated vtab */
+ int rc = SQLITE_OK; /* Return code */
int i; /* Iterator variable */
int nByte; /* Size of allocation used for *p */
int iCol; /* Column index */
@@ -108128,35 +108899,90 @@
char *zCsr; /* Space for holding column names */
int nDb; /* Bytes required to hold database name */
int nName; /* Bytes required to hold table name */
-
- const char *zTokenizer = 0; /* Name of tokenizer to use */
+ int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */
+ int bNoDocsize = 0; /* True to omit %_docsize table */
+ const char **aCol; /* Array of column names */
sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */
+ assert( strlen(argv[0])==4 );
+ assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4)
+ || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4)
+ );
+
nDb = (int)strlen(argv[1]) + 1;
nName = (int)strlen(argv[2]) + 1;
- for(i=3; i<argc; i++){
+
+ aCol = (const char **)sqlite3_malloc(sizeof(const char *) * (argc-2) );
+ if( !aCol ) return SQLITE_NOMEM;
+ memset((void *)aCol, 0, sizeof(const char *) * (argc-2));
+
+ /* Loop through all of the arguments passed by the user to the FTS3/4
+ ** module (i.e. all the column names and special arguments). This loop
+ ** does the following:
+ **
+ ** + Figures out the number of columns the FTSX table will have, and
+ ** the number of bytes of space that must be allocated to store copies
+ ** of the column names.
+ **
+ ** + If there is a tokenizer specification included in the arguments,
+ ** initializes the tokenizer pTokenizer.
+ */
+ for(i=3; rc==SQLITE_OK && i<argc; i++){
char const *z = argv[i];
- rc = sqlite3Fts3InitTokenizer(pHash, z, &pTokenizer, &zTokenizer, pzErr);
- if( rc!=SQLITE_OK ){
- return rc;
+ int nKey;
+ char *zVal;
+
+ /* Check if this is a tokenizer specification */
+ if( !pTokenizer
+ && strlen(z)>8
+ && 0==sqlite3_strnicmp(z, "tokenize", 8)
+ && 0==sqlite3Fts3IsIdChar(z[8])
+ ){
+ rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr);
}
- if( z!=zTokenizer ){
+
+ /* Check if it is an FTS4 special argument. */
+ else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){
+ if( !zVal ){
+ rc = SQLITE_NOMEM;
+ goto fts3_init_out;
+ }
+ if( nKey==9 && 0==sqlite3_strnicmp(z, "matchinfo", 9) ){
+ if( strlen(zVal)==4 && 0==sqlite3_strnicmp(zVal, "fts3", 4) ){
+ bNoDocsize = 1;
+ }else{
+ *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal);
+ rc = SQLITE_ERROR;
+ }
+ }else{
+ *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z);
+ rc = SQLITE_ERROR;
+ }
+ sqlite3_free(zVal);
+ }
+
+ /* Otherwise, the argument is a column name. */
+ else {
nString += (int)(strlen(z) + 1);
+ aCol[nCol++] = z;
}
}
- nCol = argc - 3 - (zTokenizer!=0);
- if( zTokenizer==0 ){
- rc = sqlite3Fts3InitTokenizer(pHash, 0, &pTokenizer, 0, pzErr);
- if( rc!=SQLITE_OK ){
- return rc;
- }
- assert( pTokenizer );
- }
+ if( rc!=SQLITE_OK ) goto fts3_init_out;
if( nCol==0 ){
+ assert( nString==0 );
+ aCol[0] = "content";
+ nString = 8;
nCol = 1;
}
+ if( pTokenizer==0 ){
+ rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr);
+ if( rc!=SQLITE_OK ) goto fts3_init_out;
+ }
+ assert( pTokenizer );
+
+
/* Allocate and populate the Fts3Table structure. */
nByte = sizeof(Fts3Table) + /* Fts3Table */
nCol * sizeof(char *) + /* azColumn */
@@ -108169,7 +108995,6 @@
goto fts3_init_out;
}
memset(p, 0, nByte);
-
p->db = db;
p->nColumn = nCol;
p->nPendingData = 0;
@@ -108177,11 +109002,12 @@
p->pTokenizer = pTokenizer;
p->nNodeSize = 1000;
p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
- zCsr = (char *)&p->azColumn[nCol];
-
+ p->bHasDocsize = (isFts4 && bNoDocsize==0);
+ p->bHasStat = isFts4;
fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1);
/* Fill in the zName and zDb fields of the vtab structure. */
+ zCsr = (char *)&p->azColumn[nCol];
p->zName = zCsr;
memcpy(zCsr, argv[2], nName);
zCsr += nName;
@@ -108190,52 +109016,45 @@
zCsr += nDb;
/* Fill in the azColumn array */
- iCol = 0;
- for(i=3; i<argc; i++){
- if( argv[i]!=zTokenizer ){
- char *z;
- int n;
- z = (char *)sqlite3Fts3NextToken(argv[i], &n);
- memcpy(zCsr, z, n);
- zCsr[n] = '\0';
- sqlite3Fts3Dequote(zCsr);
- p->azColumn[iCol++] = zCsr;
- zCsr += n+1;
- assert( zCsr <= &((char *)p)[nByte] );
- }
- }
- if( iCol==0 ){
- assert( nCol==1 );
- p->azColumn[0] = "content";
+ for(iCol=0; iCol<nCol; iCol++){
+ char *z;
+ int n;
+ z = (char *)sqlite3Fts3NextToken(aCol[iCol], &n);
+ memcpy(zCsr, z, n);
+ zCsr[n] = '\0';
+ sqlite3Fts3Dequote(zCsr);
+ p->azColumn[iCol] = zCsr;
+ zCsr += n+1;
+ assert( zCsr <= &((char *)p)[nByte] );
}
/* If this is an xCreate call, create the underlying tables in the
** database. TODO: For xConnect(), it could verify that said tables exist.
*/
if( isCreate ){
- p->bHasContent = 1;
- p->bHasDocsize = argv[0][3]=='4';
rc = fts3CreateTables(p);
- }else{
- rc = SQLITE_OK;
- fts3TableExists(&rc, db, argv[1], argv[2], "_content", &p->bHasContent);
- fts3TableExists(&rc, db, argv[1], argv[2], "_docsize", &p->bHasDocsize);
}
- if( rc!=SQLITE_OK ) goto fts3_init_out;
- rc = fts3DeclareVtab(p);
- if( rc!=SQLITE_OK ) goto fts3_init_out;
+ /* Figure out the page-size for the database. This is required in order to
+ ** estimate the cost of loading large doclists from the database (see
+ ** function sqlite3Fts3SegReaderCost() for details).
+ */
+ fts3DatabasePageSize(&rc, p);
- *ppVTab = &p->base;
+ /* Declare the table schema to SQLite. */
+ fts3DeclareVtab(&rc, p);
fts3_init_out:
- assert( p || (pTokenizer && rc!=SQLITE_OK) );
+
+ sqlite3_free((void *)aCol);
if( rc!=SQLITE_OK ){
if( p ){
fts3DisconnectMethod((sqlite3_vtab *)p);
- }else{
+ }else if( pTokenizer ){
pTokenizer->pModule->xDestroy(pTokenizer);
}
+ }else{
+ *ppVTab = &p->base;
}
return rc;
}
@@ -108347,10 +109166,12 @@
** Close the cursor. For additional information see the documentation
** on the xClose method of the virtual table interface.
*/
-static int fulltextClose(sqlite3_vtab_cursor *pCursor){
+static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
+ assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
sqlite3_finalize(pCsr->pStmt);
sqlite3Fts3ExprFree(pCsr->pExpr);
+ sqlite3Fts3FreeDeferredTokens(pCsr);
sqlite3_free(pCsr->aDoclist);
sqlite3_free(pCsr->aMatchinfo);
sqlite3_free(pCsr);
@@ -108389,50 +109210,137 @@
}
/*
-** Advance the cursor to the next row in the %_content table that
-** matches the search criteria. For a MATCH search, this will be
-** the next row that matches. For a full-table scan, this will be
-** simply the next row in the %_content table. For a docid lookup,
-** this routine simply sets the EOF flag.
+** This function is used to process a single interior node when searching
+** a b-tree for a term or term prefix. The node data is passed to this
+** function via the zNode/nNode parameters. The term to search for is
+** passed in zTerm/nTerm.
**
-** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned
-** even if we reach end-of-file. The fts3EofMethod() will be called
-** subsequently to determine whether or not an EOF was hit.
+** If piFirst is not NULL, then this function sets *piFirst to the blockid
+** of the child node that heads the sub-tree that may contain the term.
+**
+** If piLast is not NULL, then *piLast is set to the right-most child node
+** that heads a sub-tree that may contain a term for which zTerm/nTerm is
+** a prefix.
+**
+** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK.
*/
-static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
+static int fts3ScanInteriorNode(
+ const char *zTerm, /* Term to select leaves for */
+ int nTerm, /* Size of term zTerm in bytes */
+ const char *zNode, /* Buffer containing segment interior node */
+ int nNode, /* Size of buffer at zNode */
+ sqlite3_int64 *piFirst, /* OUT: Selected child node */
+ sqlite3_int64 *piLast /* OUT: Selected child node */
+){
int rc = SQLITE_OK; /* Return code */
- Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
+ const char *zCsr = zNode; /* Cursor to iterate through node */
+ const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
+ char *zBuffer = 0; /* Buffer to load terms into */
+ int nAlloc = 0; /* Size of allocated buffer */
+ int isFirstTerm = 1; /* True when processing first term on page */
+ sqlite3_int64 iChild; /* Block id of child node to descend to */
- if( pCsr->aDoclist==0 ){
- if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
- pCsr->isEof = 1;
- rc = sqlite3_reset(pCsr->pStmt);
- }
- }else if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){
- pCsr->isEof = 1;
- }else{
- sqlite3_reset(pCsr->pStmt);
- fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId);
- pCsr->isRequireSeek = 1;
- pCsr->isMatchinfoNeeded = 1;
+ /* Skip over the 'height' varint that occurs at the start of every
+ ** interior node. Then load the blockid of the left-child of the b-tree
+ ** node into variable iChild.
+ **
+ ** Even if the data structure on disk is corrupted, this (reading two
+ ** varints from the buffer) does not risk an overread. If zNode is a
+ ** root node, then the buffer comes from a SELECT statement. SQLite does
+ ** not make this guarantee explicitly, but in practice there are always
+ ** either more than 20 bytes of allocated space following the nNode bytes of
+ ** contents, or two zero bytes. Or, if the node is read from the %_segments
+ ** table, then there are always 20 bytes of zeroed padding following the
+ ** nNode bytes of content (see sqlite3Fts3ReadBlock() for details).
+ */
+ zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
+ zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
+ if( zCsr>zEnd ){
+ return SQLITE_CORRUPT;
}
+
+ while( zCsr<zEnd && (piFirst || piLast) ){
+ int cmp; /* memcmp() result */
+ int nSuffix; /* Size of term suffix */
+ int nPrefix = 0; /* Size of term prefix */
+ int nBuffer; /* Total term size */
+
+ /* Load the next term on the node into zBuffer. Use realloc() to expand
+ ** the size of zBuffer if required. */
+ if( !isFirstTerm ){
+ zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix);
+ }
+ isFirstTerm = 0;
+ zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix);
+
+ if( nPrefix<0 || nSuffix<0 || &zCsr[nSuffix]>zEnd ){
+ rc = SQLITE_CORRUPT;
+ goto finish_scan;
+ }
+ if( nPrefix+nSuffix>nAlloc ){
+ char *zNew;
+ nAlloc = (nPrefix+nSuffix) * 2;
+ zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
+ if( !zNew ){
+ rc = SQLITE_NOMEM;
+ goto finish_scan;
+ }
+ zBuffer = zNew;
+ }
+ memcpy(&zBuffer[nPrefix], zCsr, nSuffix);
+ nBuffer = nPrefix + nSuffix;
+ zCsr += nSuffix;
+
+ /* Compare the term we are searching for with the term just loaded from
+ ** the interior node. If the specified term is greater than or equal
+ ** to the term from the interior node, then all terms on the sub-tree
+ ** headed by node iChild are smaller than zTerm. No need to search
+ ** iChild.
+ **
+ ** If the interior node term is larger than the specified term, then
+ ** the tree headed by iChild may contain the specified term.
+ */
+ cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer));
+ if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){
+ *piFirst = iChild;
+ piFirst = 0;
+ }
+
+ if( piLast && cmp<0 ){
+ *piLast = iChild;
+ piLast = 0;
+ }
+
+ iChild++;
+ };
+
+ if( piFirst ) *piFirst = iChild;
+ if( piLast ) *piLast = iChild;
+
+ finish_scan:
+ sqlite3_free(zBuffer);
return rc;
}
/*
-** The buffer pointed to by argument zNode (size nNode bytes) contains the
-** root node of a b-tree segment. The segment is guaranteed to be at least
-** one level high (i.e. the root node is not also a leaf). If successful,
-** this function locates the leaf node of the segment that may contain the
-** term specified by arguments zTerm and nTerm and writes its block number
-** to *piLeaf.
+** The buffer pointed to by argument zNode (size nNode bytes) contains an
+** interior node of a b-tree segment. The zTerm buffer (size nTerm bytes)
+** contains a term. This function searches the sub-tree headed by the zNode
+** node for the range of leaf nodes that may contain the specified term
+** or terms for which the specified term is a prefix.
**
-** It is possible that the returned leaf node does not contain the specified
-** term. However, if the segment does contain said term, it is stored on
-** the identified leaf node. Because this function only inspects interior
-** segment nodes (and never loads leaf nodes into memory), it is not possible
-** to be sure.
+** If piLeaf is not NULL, then *piLeaf is set to the blockid of the
+** left-most leaf node in the tree that may contain the specified term.
+** If piLeaf2 is not NULL, then *piLeaf2 is set to the blockid of the
+** right-most leaf node that may contain a term for which the specified
+** term is a prefix.
+**
+** It is possible that the range of returned leaf nodes does not contain
+** the specified term or any terms for which it is a prefix. However, if the
+** segment does contain any such terms, they are stored within the identified
+** range. Because this function only inspects interior segment nodes (and
+** never loads leaf nodes into memory), it is not possible to be sure.
**
** If an error occurs, an error code other than SQLITE_OK is returned.
*/
@@ -108442,77 +109350,41 @@
int nTerm, /* Size of term zTerm in bytes */
const char *zNode, /* Buffer containing segment interior node */
int nNode, /* Size of buffer at zNode */
- sqlite3_int64 *piLeaf /* Selected leaf node */
+ sqlite3_int64 *piLeaf, /* Selected leaf node */
+ sqlite3_int64 *piLeaf2 /* Selected leaf node */
){
- int rc = SQLITE_OK; /* Return code */
- const char *zCsr = zNode; /* Cursor to iterate through node */
- const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
- char *zBuffer = 0; /* Buffer to load terms into */
- int nAlloc = 0; /* Size of allocated buffer */
+ int rc; /* Return code */
+ int iHeight; /* Height of this node in tree */
- while( 1 ){
- int isFirstTerm = 1; /* True when processing first term on page */
- int iHeight; /* Height of this node in tree */
- sqlite3_int64 iChild; /* Block id of child node to descend to */
- int nBlock; /* Size of child node in bytes */
+ assert( piLeaf || piLeaf2 );
- zCsr += sqlite3Fts3GetVarint32(zCsr, &iHeight);
- zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
-
- while( zCsr<zEnd ){
- int cmp; /* memcmp() result */
- int nSuffix; /* Size of term suffix */
- int nPrefix = 0; /* Size of term prefix */
- int nBuffer; /* Total term size */
-
- /* Load the next term on the node into zBuffer */
- if( !isFirstTerm ){
- zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix);
+ sqlite3Fts3GetVarint32(zNode, &iHeight);
+ rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2);
+ assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) );
+
+ if( rc==SQLITE_OK && iHeight>1 ){
+ char *zBlob = 0; /* Blob read from %_segments table */
+ int nBlob; /* Size of zBlob in bytes */
+
+ if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){
+ rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob);
+ if( rc==SQLITE_OK ){
+ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0);
}
- isFirstTerm = 0;
- zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix);
- if( nPrefix+nSuffix>nAlloc ){
- char *zNew;
- nAlloc = (nPrefix+nSuffix) * 2;
- zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
- if( !zNew ){
- sqlite3_free(zBuffer);
- return SQLITE_NOMEM;
- }
- zBuffer = zNew;
- }
- memcpy(&zBuffer[nPrefix], zCsr, nSuffix);
- nBuffer = nPrefix + nSuffix;
- zCsr += nSuffix;
-
- /* Compare the term we are searching for with the term just loaded from
- ** the interior node. If the specified term is greater than or equal
- ** to the term from the interior node, then all terms on the sub-tree
- ** headed by node iChild are smaller than zTerm. No need to search
- ** iChild.
- **
- ** If the interior node term is larger than the specified term, then
- ** the tree headed by iChild may contain the specified term.
- */
- cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer));
- if( cmp<0 || (cmp==0 && nBuffer>nTerm) ) break;
- iChild++;
- };
-
- /* If (iHeight==1), the children of this interior node are leaves. The
- ** specified term may be present on leaf node iChild.
- */
- if( iHeight==1 ){
- *piLeaf = iChild;
- break;
+ sqlite3_free(zBlob);
+ piLeaf = 0;
+ zBlob = 0;
}
- /* Descend to interior node iChild. */
- rc = sqlite3Fts3ReadBlock(p, iChild, &zCsr, &nBlock);
- if( rc!=SQLITE_OK ) break;
- zEnd = &zCsr[nBlock];
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts3ReadBlock(p, piLeaf ? *piLeaf : *piLeaf2, &zBlob, &nBlob);
+ }
+ if( rc==SQLITE_OK ){
+ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2);
+ }
+ sqlite3_free(zBlob);
}
- sqlite3_free(zBuffer);
+
return rc;
}
@@ -108744,20 +109616,44 @@
/*
** nToken==1 searches for adjacent positions.
+**
+** This function is used to merge two position lists into one. When it is
+** called, *pp1 and *pp2 must both point to position lists. A position-list is
+** the part of a doclist that follows each document id. For example, if a row
+** contains:
+**
+** 'a b c'|'x y z'|'a b b a'
+**
+** Then the position list for this row for token 'b' would consist of:
+**
+** 0x02 0x01 0x02 0x03 0x03 0x00
+**
+** When this function returns, both *pp1 and *pp2 are left pointing to the
+** byte following the 0x00 terminator of their respective position lists.
+**
+** If isSaveLeft is 0, an entry is added to the output position list for
+** each position in *pp2 for which there exists one or more positions in
+** *pp1 so that (pos(*pp2)>pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e.
+** when the *pp1 token appears before the *pp2 token, but not more than nToken
+** slots before it.
*/
static int fts3PoslistPhraseMerge(
- char **pp, /* Output buffer */
+ char **pp, /* IN/OUT: Preallocated output buffer */
int nToken, /* Maximum difference in token positions */
int isSaveLeft, /* Save the left position */
- char **pp1, /* Left input list */
- char **pp2 /* Right input list */
+ int isExact, /* If *pp1 is exactly nTokens before *pp2 */
+ char **pp1, /* IN/OUT: Left input list */
+ char **pp2 /* IN/OUT: Right input list */
){
char *p = (pp ? *pp : 0);
char *p1 = *pp1;
char *p2 = *pp2;
-
int iCol1 = 0;
int iCol2 = 0;
+
+ /* Never set both isSaveLeft and isExact for the same invocation. */
+ assert( isSaveLeft==0 || isExact==0 );
+
assert( *p1!=0 && *p2!=0 );
if( *p1==POS_COLUMN ){
p1++;
@@ -108786,7 +109682,9 @@
fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2;
while( 1 ){
- if( iPos2>iPos1 && iPos2<=iPos1+nToken ){
+ if( iPos2==iPos1+nToken
+ || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken)
+ ){
sqlite3_int64 iSave;
if( !pp ){
fts3PoslistCopy(0, &p2);
@@ -108869,21 +109767,21 @@
char *p2 = *pp2;
if( !pp ){
- if( fts3PoslistPhraseMerge(0, nRight, 0, pp1, pp2) ) return 1;
+ if( fts3PoslistPhraseMerge(0, nRight, 0, 0, pp1, pp2) ) return 1;
*pp1 = p1;
*pp2 = p2;
- return fts3PoslistPhraseMerge(0, nLeft, 0, pp2, pp1);
+ return fts3PoslistPhraseMerge(0, nLeft, 0, 0, pp2, pp1);
}else{
char *pTmp1 = aTmp;
char *pTmp2;
char *aTmp2;
int res = 1;
- fts3PoslistPhraseMerge(&pTmp1, nRight, 0, pp1, pp2);
+ fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2);
aTmp2 = pTmp2 = pTmp1;
*pp1 = p1;
*pp2 = p2;
- fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, pp2, pp1);
+ fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1);
if( pTmp1!=aTmp && pTmp2!=aTmp2 ){
fts3PoslistMerge(pp, &aTmp, &aTmp2);
}else if( pTmp1!=aTmp ){
@@ -108929,7 +109827,8 @@
char *a1, /* Buffer containing first doclist */
int n1, /* Size of buffer a1 */
char *a2, /* Buffer containing second doclist */
- int n2 /* Size of buffer a2 */
+ int n2, /* Size of buffer a2 */
+ int *pnDoc /* OUT: Number of docids in output */
){
sqlite3_int64 i1 = 0;
sqlite3_int64 i2 = 0;
@@ -108940,6 +109839,7 @@
char *p2 = a2;
char *pEnd1 = &a1[n1];
char *pEnd2 = &a2[n2];
+ int nDoc = 0;
assert( mergetype==MERGE_OR || mergetype==MERGE_POS_OR
|| mergetype==MERGE_AND || mergetype==MERGE_NOT
@@ -108983,6 +109883,7 @@
fts3PutDeltaVarint(&p, &iPrev, i1);
fts3GetDeltaVarint2(&p1, pEnd1, &i1);
fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+ nDoc++;
}else if( i1<i2 ){
fts3GetDeltaVarint2(&p1, pEnd1, &i1);
}else{
@@ -109013,9 +109914,11 @@
char *pSave = p;
sqlite3_int64 iPrevSave = iPrev;
fts3PutDeltaVarint(&p, &iPrev, i1);
- if( 0==fts3PoslistPhraseMerge(ppPos, 1, 0, &p1, &p2) ){
+ if( 0==fts3PoslistPhraseMerge(ppPos, nParam1, 0, 1, &p1, &p2) ){
p = pSave;
iPrev = iPrevSave;
+ }else{
+ nDoc++;
}
fts3GetDeltaVarint2(&p1, pEnd1, &i1);
fts3GetDeltaVarint2(&p2, pEnd2, &i2);
@@ -109068,6 +109971,7 @@
}
}
+ if( pnDoc ) *pnDoc = nDoc;
*pnBuffer = (int)(p-aBuffer);
return SQLITE_OK;
}
@@ -109106,7 +110010,7 @@
if( !aOut ){
aOut = pTS->aaOutput[i];
nOut = pTS->anOutput[i];
- pTS->aaOutput[0] = 0;
+ pTS->aaOutput[i] = 0;
}else{
int nNew = nOut + pTS->anOutput[i];
char *aNew = sqlite3_malloc(nNew);
@@ -109115,7 +110019,7 @@
return SQLITE_NOMEM;
}
fts3DoclistMerge(mergetype, 0, 0,
- aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut
+ aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, 0
);
sqlite3_free(pTS->aaOutput[i]);
sqlite3_free(aOut);
@@ -109186,8 +110090,8 @@
}
return SQLITE_NOMEM;
}
- fts3DoclistMerge(mergetype, 0, 0,
- aNew, &nNew, pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge
+ fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew,
+ pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge, 0
);
if( iOut>0 ) sqlite3_free(aMerge);
@@ -109205,6 +110109,161 @@
return SQLITE_OK;
}
+static int fts3DeferredTermSelect(
+ Fts3DeferredToken *pToken, /* Phrase token */
+ int isTermPos, /* True to include positions */
+ int *pnOut, /* OUT: Size of list */
+ char **ppOut /* OUT: Body of list */
+){
+ char *aSource;
+ int nSource;
+
+ aSource = sqlite3Fts3DeferredDoclist(pToken, &nSource);
+ if( !aSource ){
+ *pnOut = 0;
+ *ppOut = 0;
+ }else if( isTermPos ){
+ *ppOut = sqlite3_malloc(nSource);
+ if( !*ppOut ) return SQLITE_NOMEM;
+ memcpy(*ppOut, aSource, nSource);
+ *pnOut = nSource;
+ }else{
+ sqlite3_int64 docid;
+ *pnOut = sqlite3Fts3GetVarint(aSource, &docid);
+ *ppOut = sqlite3_malloc(*pnOut);
+ if( !*ppOut ) return SQLITE_NOMEM;
+ sqlite3Fts3PutVarint(*ppOut, docid);
+ }
+
+ return SQLITE_OK;
+}
+
+/*
+** An Fts3SegReaderArray is used to store an array of Fts3SegReader objects.
+** Elements are added to the array using fts3SegReaderArrayAdd().
+*/
+struct Fts3SegReaderArray {
+ int nSegment; /* Number of valid entries in apSegment[] */
+ int nAlloc; /* Allocated size of apSegment[] */
+ int nCost; /* The cost of executing SegReaderIterate() */
+ Fts3SegReader *apSegment[1]; /* Array of seg-reader objects */
+};
+
+
+/*
+** Free an Fts3SegReaderArray object. Also free all seg-readers in the
+** array (using sqlite3Fts3SegReaderFree()).
+*/
+static void fts3SegReaderArrayFree(Fts3SegReaderArray *pArray){
+ if( pArray ){
+ int i;
+ for(i=0; i<pArray->nSegment; i++){
+ sqlite3Fts3SegReaderFree(pArray->apSegment[i]);
+ }
+ sqlite3_free(pArray);
+ }
+}
+
+static int fts3SegReaderArrayAdd(
+ Fts3SegReaderArray **ppArray,
+ Fts3SegReader *pNew
+){
+ Fts3SegReaderArray *pArray = *ppArray;
+
+ if( !pArray || pArray->nAlloc==pArray->nSegment ){
+ int nNew = (pArray ? pArray->nAlloc+16 : 16);
+ pArray = (Fts3SegReaderArray *)sqlite3_realloc(pArray,
+ sizeof(Fts3SegReaderArray) + (nNew-1) * sizeof(Fts3SegReader*)
+ );
+ if( !pArray ){
+ sqlite3Fts3SegReaderFree(pNew);
+ return SQLITE_NOMEM;
+ }
+ if( nNew==16 ){
+ pArray->nSegment = 0;
+ pArray->nCost = 0;
+ }
+ pArray->nAlloc = nNew;
+ *ppArray = pArray;
+ }
+
+ pArray->apSegment[pArray->nSegment++] = pNew;
+ return SQLITE_OK;
+}
+
+static int fts3TermSegReaderArray(
+ Fts3Cursor *pCsr, /* Virtual table cursor handle */
+ const char *zTerm, /* Term to query for */
+ int nTerm, /* Size of zTerm in bytes */
+ int isPrefix, /* True for a prefix search */
+ Fts3SegReaderArray **ppArray /* OUT: Allocated seg-reader array */
+){
+ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
+ int rc; /* Return code */
+ Fts3SegReaderArray *pArray = 0; /* Array object to build */
+ Fts3SegReader *pReader = 0; /* Seg-reader to add to pArray */
+ sqlite3_stmt *pStmt = 0; /* SQL statement to scan %_segdir table */
+ int iAge = 0; /* Used to assign ages to segments */
+
+ /* Allocate a seg-reader to scan the pending terms, if any. */
+ rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &pReader);
+ if( rc==SQLITE_OK && pReader ) {
+ rc = fts3SegReaderArrayAdd(&pArray, pReader);
+ }
+
+ /* Loop through the entire %_segdir table. For each segment, create a
+ ** Fts3SegReader to iterate through the subset of the segment leaves
+ ** that may contain a term that matches zTerm/nTerm. For non-prefix
+ ** searches, this is always a single leaf. For prefix searches, this
+ ** may be a contiguous block of leaves.
+ */
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts3AllSegdirs(p, &pStmt);
+ }
+ while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
+ Fts3SegReader *pNew = 0;
+ int nRoot = sqlite3_column_bytes(pStmt, 4);
+ char const *zRoot = sqlite3_column_blob(pStmt, 4);
+ if( sqlite3_column_int64(pStmt, 1)==0 ){
+ /* The entire segment is stored on the root node (which must be a
+ ** leaf). Do not bother inspecting any data in this case, just
+ ** create a Fts3SegReader to scan the single leaf.
+ */
+ rc = sqlite3Fts3SegReaderNew(iAge, 0, 0, 0, zRoot, nRoot, &pNew);
+ }else{
+ sqlite3_int64 i1; /* First leaf that may contain zTerm */
+ sqlite3_int64 i2; /* Final leaf that may contain zTerm */
+ rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1, (isPrefix?&i2:0));
+ if( isPrefix==0 ) i2 = i1;
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts3SegReaderNew(iAge, i1, i2, 0, 0, 0, &pNew);
+ }
+ }
+ assert( (pNew==0)==(rc!=SQLITE_OK) );
+
+ /* If a new Fts3SegReader was allocated, add it to the array. */
+ if( rc==SQLITE_OK ){
+ rc = fts3SegReaderArrayAdd(&pArray, pNew);
+ }
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts3SegReaderCost(pCsr, pNew, &pArray->nCost);
+ }
+ iAge++;
+ }
+
+ if( rc==SQLITE_DONE ){
+ rc = sqlite3_reset(pStmt);
+ }else{
+ sqlite3_reset(pStmt);
+ }
+ if( rc!=SQLITE_OK ){
+ fts3SegReaderArrayFree(pArray);
+ pArray = 0;
+ }
+ *ppArray = pArray;
+ return rc;
+}
+
/*
** This function retreives the doclist for the specified term (or term
** prefix) from the database.
@@ -109218,112 +110277,31 @@
*/
static int fts3TermSelect(
Fts3Table *p, /* Virtual table handle */
+ Fts3PhraseToken *pTok, /* Token to query for */
int iColumn, /* Column to query (or -ve for all columns) */
- const char *zTerm, /* Term to query for */
- int nTerm, /* Size of zTerm in bytes */
- int isPrefix, /* True for a prefix search */
int isReqPos, /* True to include position lists in output */
int *pnOut, /* OUT: Size of buffer at *ppOut */
char **ppOut /* OUT: Malloced result buffer */
){
- int i;
- TermSelect tsc;
- Fts3SegFilter filter; /* Segment term filter configuration */
- Fts3SegReader **apSegment; /* Array of segments to read data from */
- int nSegment = 0; /* Size of apSegment array */
- int nAlloc = 16; /* Allocated size of segment array */
int rc; /* Return code */
- sqlite3_stmt *pStmt = 0; /* SQL statement to scan %_segdir table */
- int iAge = 0; /* Used to assign ages to segments */
+ Fts3SegReaderArray *pArray; /* Seg-reader array for this term */
+ TermSelect tsc; /* Context object for fts3TermSelectCb() */
+ Fts3SegFilter filter; /* Segment term filter configuration */
- apSegment = (Fts3SegReader **)sqlite3_malloc(sizeof(Fts3SegReader*)*nAlloc);
- if( !apSegment ) return SQLITE_NOMEM;
- rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &apSegment[0]);
- if( rc!=SQLITE_OK ) goto finished;
- if( apSegment[0] ){
- nSegment = 1;
- }
-
- /* Loop through the entire %_segdir table. For each segment, create a
- ** Fts3SegReader to iterate through the subset of the segment leaves
- ** that may contain a term that matches zTerm/nTerm. For non-prefix
- ** searches, this is always a single leaf. For prefix searches, this
- ** may be a contiguous block of leaves.
- **
- ** The code in this loop does not actually load any leaves into memory
- ** (unless the root node happens to be a leaf). It simply examines the
- ** b-tree structure to determine which leaves need to be inspected.
- */
- rc = sqlite3Fts3AllSegdirs(p, &pStmt);
- while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
- Fts3SegReader *pNew = 0;
- int nRoot = sqlite3_column_bytes(pStmt, 4);
- char const *zRoot = sqlite3_column_blob(pStmt, 4);
- if( sqlite3_column_int64(pStmt, 1)==0 ){
- /* The entire segment is stored on the root node (which must be a
- ** leaf). Do not bother inspecting any data in this case, just
- ** create a Fts3SegReader to scan the single leaf.
- */
- rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew);
- }else{
- int rc2; /* Return value of sqlite3Fts3ReadBlock() */
- sqlite3_int64 i1; /* Blockid of leaf that may contain zTerm */
- rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1);
- if( rc==SQLITE_OK ){
- sqlite3_int64 i2 = sqlite3_column_int64(pStmt, 2);
- rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew);
- }
-
- /* The following call to ReadBlock() serves to reset the SQL statement
- ** used to retrieve blocks of data from the %_segments table. If it is
- ** not reset here, then it may remain classified as an active statement
- ** by SQLite, which may lead to "DROP TABLE" or "DETACH" commands
- ** failing.
- */
- rc2 = sqlite3Fts3ReadBlock(p, 0, 0, 0);
- if( rc==SQLITE_OK ){
- rc = rc2;
- }
- }
- iAge++;
-
- /* If a new Fts3SegReader was allocated, add it to the apSegment array. */
- assert( pNew!=0 || rc!=SQLITE_OK );
- if( pNew ){
- if( nSegment==nAlloc ){
- Fts3SegReader **pArray;
- nAlloc += 16;
- pArray = (Fts3SegReader **)sqlite3_realloc(
- apSegment, nAlloc*sizeof(Fts3SegReader *)
- );
- if( !pArray ){
- sqlite3Fts3SegReaderFree(p, pNew);
- rc = SQLITE_NOMEM;
- goto finished;
- }
- apSegment = pArray;
- }
- apSegment[nSegment++] = pNew;
- }
- }
- if( rc!=SQLITE_DONE ){
- assert( rc!=SQLITE_OK );
- goto finished;
- }
-
+ pArray = pTok->pArray;
memset(&tsc, 0, sizeof(TermSelect));
tsc.isReqPos = isReqPos;
filter.flags = FTS3_SEGMENT_IGNORE_EMPTY
- | (isPrefix ? FTS3_SEGMENT_PREFIX : 0)
+ | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0)
| (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0)
| (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0);
filter.iCol = iColumn;
- filter.zTerm = zTerm;
- filter.nTerm = nTerm;
+ filter.zTerm = pTok->z;
+ filter.nTerm = pTok->n;
- rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment, &filter,
- fts3TermSelectCb, (void *)&tsc
+ rc = sqlite3Fts3SegReaderIterate(p, pArray->apSegment, pArray->nSegment,
+ &filter, fts3TermSelectCb, (void *)&tsc
);
if( rc==SQLITE_OK ){
rc = fts3TermSelectMerge(&tsc);
@@ -109333,26 +110311,112 @@
*ppOut = tsc.aaOutput[0];
*pnOut = tsc.anOutput[0];
}else{
+ int i;
for(i=0; i<SizeofArray(tsc.aaOutput); i++){
sqlite3_free(tsc.aaOutput[i]);
}
}
-finished:
- sqlite3_reset(pStmt);
- for(i=0; i<nSegment; i++){
- sqlite3Fts3SegReaderFree(p, apSegment[i]);
- }
- sqlite3_free(apSegment);
+ fts3SegReaderArrayFree(pArray);
+ pTok->pArray = 0;
return rc;
}
+/*
+** This function counts the total number of docids in the doclist stored
+** in buffer aList[], size nList bytes.
+**
+** If the isPoslist argument is true, then it is assumed that the doclist
+** contains a position-list following each docid. Otherwise, it is assumed
+** that the doclist is simply a list of docids stored as delta encoded
+** varints.
+*/
+static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){
+ int nDoc = 0; /* Return value */
+ if( aList ){
+ char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */
+ char *p = aList; /* Cursor */
+ if( !isPoslist ){
+ /* The number of docids in the list is the same as the number of
+ ** varints. In FTS3 a varint consists of a single byte with the 0x80
+ ** bit cleared and zero or more bytes with the 0x80 bit set. So to
+ ** count the varints in the buffer, just count the number of bytes
+ ** with the 0x80 bit clear. */
+ while( p<aEnd ) nDoc += (((*p++)&0x80)==0);
+ }else{
+ while( p<aEnd ){
+ nDoc++;
+ while( (*p++)&0x80 ); /* Skip docid varint */
+ fts3PoslistCopy(0, &p); /* Skip over position list */
+ }
+ }
+ }
+
+ return nDoc;
+}
+
+/*
+** Call sqlite3Fts3DeferToken() for each token in the expression pExpr.
+*/
+static int fts3DeferExpression(Fts3Cursor *pCsr, Fts3Expr *pExpr){
+ int rc = SQLITE_OK;
+ if( pExpr ){
+ rc = fts3DeferExpression(pCsr, pExpr->pLeft);
+ if( rc==SQLITE_OK ){
+ rc = fts3DeferExpression(pCsr, pExpr->pRight);
+ }
+ if( pExpr->eType==FTSQUERY_PHRASE ){
+ int iCol = pExpr->pPhrase->iColumn;
+ int i;
+ for(i=0; rc==SQLITE_OK && i<pExpr->pPhrase->nToken; i++){
+ Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i];
+ if( pToken->pDeferred==0 ){
+ rc = sqlite3Fts3DeferToken(pCsr, pToken, iCol);
+ }
+ }
+ }
+ }
+ return rc;
+}
+
+/*
+** This function removes the position information from a doclist. When
+** called, buffer aList (size *pnList bytes) contains a doclist that includes
+** position information. This function removes the position information so
+** that aList contains only docids, and adjusts *pnList to reflect the new
+** (possibly reduced) size of the doclist.
+*/
+static void fts3DoclistStripPositions(
+ char *aList, /* IN/OUT: Buffer containing doclist */
+ int *pnList /* IN/OUT: Size of doclist in bytes */
+){
+ if( aList ){
+ char *aEnd = &aList[*pnList]; /* Pointer to one byte after EOF */
+ char *p = aList; /* Input cursor */
+ char *pOut = aList; /* Output cursor */
+
+ while( p<aEnd ){
+ sqlite3_int64 delta;
+ p += sqlite3Fts3GetVarint(p, &delta);
+ fts3PoslistCopy(0, &p);
+ pOut += sqlite3Fts3PutVarint(pOut, delta);
+ }
+
+ *pnList = (int)(pOut - aList);
+ }
+}
/*
** Return a DocList corresponding to the phrase *pPhrase.
+**
+** If this function returns SQLITE_OK, but *pnOut is set to a negative value,
+** then no tokens in the phrase were looked up in the full-text index. This
+** is only possible when this function is called from within xFilter(). The
+** caller should assume that all documents match the phrase. The actual
+** filtering will take place in xNext().
*/
static int fts3PhraseSelect(
- Fts3Table *p, /* Virtual table handle */
+ Fts3Cursor *pCsr, /* Virtual table cursor handle */
Fts3Phrase *pPhrase, /* Phrase to return a doclist for */
int isReqPos, /* True if output should contain positions */
char **paOut, /* OUT: Pointer to malloc'd result buffer */
@@ -109364,42 +110428,137 @@
int ii;
int iCol = pPhrase->iColumn;
int isTermPos = (pPhrase->nToken>1 || isReqPos);
+ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
+ int isFirst = 1;
+
+ int iPrevTok = 0;
+ int nDoc = 0;
+
+ /* If this is an xFilter() evaluation, create a segment-reader for each
+ ** phrase token. Or, if this is an xNext() or snippet/offsets/matchinfo
+ ** evaluation, only create segment-readers if there are no Fts3DeferredToken
+ ** objects attached to the phrase-tokens.
+ */
+ for(ii=0; ii<pPhrase->nToken; ii++){
+ Fts3PhraseToken *pTok = &pPhrase->aToken[ii];
+ if( pTok->pArray==0 ){
+ if( (pCsr->eEvalmode==FTS3_EVAL_FILTER)
+ || (pCsr->eEvalmode==FTS3_EVAL_NEXT && pCsr->pDeferred==0)
+ || (pCsr->eEvalmode==FTS3_EVAL_MATCHINFO && pTok->bFulltext)
+ ){
+ rc = fts3TermSegReaderArray(
+ pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pArray
+ );
+ if( rc!=SQLITE_OK ) return rc;
+ }
+ }
+ }
for(ii=0; ii<pPhrase->nToken; ii++){
- struct PhraseToken *pTok = &pPhrase->aToken[ii];
- char *z = pTok->z; /* Next token of the phrase */
- int n = pTok->n; /* Size of z in bytes */
- int isPrefix = pTok->isPrefix;/* True if token is a prefix */
- char *pList; /* Pointer to token doclist */
- int nList; /* Size of buffer at pList */
+ Fts3PhraseToken *pTok; /* Token to find doclist for */
+ int iTok = 0; /* The token being queried this iteration */
+ char *pList = 0; /* Pointer to token doclist */
+ int nList = 0; /* Size of buffer at pList */
- rc = fts3TermSelect(p, iCol, z, n, isPrefix, isTermPos, &nList, &pList);
+ /* Select a token to process. If this is an xFilter() call, then tokens
+ ** are processed in order from least to most costly. Otherwise, tokens
+ ** are processed in the order in which they occur in the phrase.
+ */
+ if( pCsr->eEvalmode==FTS3_EVAL_MATCHINFO ){
+ assert( isReqPos );
+ iTok = ii;
+ pTok = &pPhrase->aToken[iTok];
+ if( pTok->bFulltext==0 ) continue;
+ }else if( pCsr->eEvalmode==FTS3_EVAL_NEXT || isReqPos ){
+ iTok = ii;
+ pTok = &pPhrase->aToken[iTok];
+ }else{
+ int nMinCost = 0x7FFFFFFF;
+ int jj;
+
+ /* Find the remaining token with the lowest cost. */
+ for(jj=0; jj<pPhrase->nToken; jj++){
+ Fts3SegReaderArray *pArray = pPhrase->aToken[jj].pArray;
+ if( pArray && pArray->nCost<nMinCost ){
+ iTok = jj;
+ nMinCost = pArray->nCost;
+ }
+ }
+ pTok = &pPhrase->aToken[iTok];
+
+ /* This branch is taken if it is determined that loading the doclist
+ ** for the next token would require more IO than loading all documents
+ ** currently identified by doclist pOut/nOut. No further doclists will
+ ** be loaded from the full-text index for this phrase.
+ */
+ if( nMinCost>nDoc && ii>0 ){
+ rc = fts3DeferExpression(pCsr, pCsr->pExpr);
+ break;
+ }
+ }
+
+ if( pCsr->eEvalmode==FTS3_EVAL_NEXT && pTok->pDeferred ){
+ rc = fts3DeferredTermSelect(pTok->pDeferred, isTermPos, &nList, &pList);
+ }else{
+ if( pTok->pArray ){
+ rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList);
+ }
+ pTok->bFulltext = 1;
+ }
+ assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pArray==0 );
if( rc!=SQLITE_OK ) break;
- if( ii==0 ){
+ if( isFirst ){
pOut = pList;
nOut = nList;
- }else{
- /* Merge the new term list and the current output. If this is the
- ** last term in the phrase, and positions are not required in the
- ** output of this function, the positions can be dropped as part
- ** of this merge. Either way, the result of this merge will be
- ** smaller than nList bytes. The code in fts3DoclistMerge() is written
- ** so that it is safe to use pList as the output as well as an input
- ** in this case.
- */
- int mergetype = MERGE_POS_PHRASE;
- if( ii==pPhrase->nToken-1 && !isReqPos ){
- mergetype = MERGE_PHRASE;
+ if( pCsr->eEvalmode==FTS3_EVAL_FILTER && pPhrase->nToken>1 ){
+ nDoc = fts3DoclistCountDocids(1, pOut, nOut);
}
- fts3DoclistMerge(mergetype, 0, 0, pList, &nOut, pOut, nOut, pList, nList);
- sqlite3_free(pOut);
- pOut = pList;
+ isFirst = 0;
+ iPrevTok = iTok;
+ }else{
+ /* Merge the new term list and the current output. */
+ char *aLeft, *aRight;
+ int nLeft, nRight;
+ int nDist;
+ int mt;
+
+ /* If this is the final token of the phrase, and positions were not
+ ** requested by the caller, use MERGE_PHRASE instead of POS_PHRASE.
+ ** This drops the position information from the output list.
+ */
+ mt = MERGE_POS_PHRASE;
+ if( ii==pPhrase->nToken-1 && !isReqPos ) mt = MERGE_PHRASE;
+
+ assert( iPrevTok!=iTok );
+ if( iPrevTok<iTok ){
+ aLeft = pOut;
+ nLeft = nOut;
+ aRight = pList;
+ nRight = nList;
+ nDist = iTok-iPrevTok;
+ iPrevTok = iTok;
+ }else{
+ aRight = pOut;
+ nRight = nOut;
+ aLeft = pList;
+ nLeft = nList;
+ nDist = iPrevTok-iTok;
+ }
+ pOut = aRight;
+ fts3DoclistMerge(
+ mt, nDist, 0, pOut, &nOut, aLeft, nLeft, aRight, nRight, &nDoc
+ );
+ sqlite3_free(aLeft);
}
assert( nOut==0 || pOut!=0 );
}
if( rc==SQLITE_OK ){
+ if( ii!=pPhrase->nToken ){
+ assert( pCsr->eEvalmode==FTS3_EVAL_FILTER && isReqPos==0 );
+ fts3DoclistStripPositions(pOut, &nOut);
+ }
*paOut = pOut;
*pnOut = nOut;
}else{
@@ -109408,6 +110567,14 @@
return rc;
}
+/*
+** This function merges two doclists according to the requirements of a
+** NEAR operator.
+**
+** Both input doclists must include position information. The output doclist
+** includes position information if the first argument to this function
+** is MERGE_POS_NEAR, or does not if it is MERGE_NEAR.
+*/
static int fts3NearMerge(
int mergetype, /* MERGE_POS_NEAR or MERGE_NEAR */
int nNear, /* Parameter to NEAR operator */
@@ -109420,8 +110587,8 @@
char **paOut, /* OUT: Results of merge (malloced) */
int *pnOut /* OUT: Sized of output buffer */
){
- char *aOut;
- int rc;
+ char *aOut; /* Buffer to write output doclist to */
+ int rc; /* Return code */
assert( mergetype==MERGE_POS_NEAR || MERGE_NEAR );
@@ -109430,7 +110597,7 @@
rc = SQLITE_NOMEM;
}else{
rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft,
- aOut, pnOut, aLeft, nLeft, aRight, nRight
+ aOut, pnOut, aLeft, nLeft, aRight, nRight, 0
);
if( rc!=SQLITE_OK ){
sqlite3_free(aOut);
@@ -109442,8 +110609,23 @@
return rc;
}
+/*
+** This function is used as part of the processing for the snippet() and
+** offsets() functions.
+**
+** Both pLeft and pRight are expression nodes of type FTSQUERY_PHRASE. Both
+** have their respective doclists (including position information) loaded
+** in Fts3Expr.aDoclist/nDoclist. This function removes all entries from
+** each doclist that are not within nNear tokens of a corresponding entry
+** in the other doclist.
+*/
SQLITE_PRIVATE int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){
- int rc;
+ int rc; /* Return code */
+
+ assert( pLeft->eType==FTSQUERY_PHRASE );
+ assert( pRight->eType==FTSQUERY_PHRASE );
+ assert( pLeft->isLoaded && pRight->isLoaded );
+
if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){
sqlite3_free(pLeft->aDoclist);
sqlite3_free(pRight->aDoclist);
@@ -109451,8 +110633,8 @@
pLeft->aDoclist = 0;
rc = SQLITE_OK;
}else{
- char *aOut;
- int nOut;
+ char *aOut; /* Buffer in which to assemble new doclist */
+ int nOut; /* Size of buffer aOut in bytes */
rc = fts3NearMerge(MERGE_POS_NEAR, nNear,
pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist,
@@ -109476,14 +110658,156 @@
return rc;
}
+
/*
-** Evaluate the full-text expression pExpr against fts3 table pTab. Store
-** the resulting doclist in *paOut and *pnOut. This routine mallocs for
-** the space needed to store the output. The caller is responsible for
-** freeing the space when it has finished.
+** Allocate an Fts3SegReaderArray for each token in the expression pExpr.
+** The allocated objects are stored in the Fts3PhraseToken.pArray member
+** variables of each token structure.
*/
-static int evalFts3Expr(
- Fts3Table *p, /* Virtual table handle */
+static int fts3ExprAllocateSegReaders(
+ Fts3Cursor *pCsr, /* FTS3 table */
+ Fts3Expr *pExpr, /* Expression to create seg-readers for */
+ int *pnExpr /* OUT: Number of AND'd expressions */
+){
+ int rc = SQLITE_OK; /* Return code */
+
+ assert( pCsr->eEvalmode==FTS3_EVAL_FILTER );
+ if( pnExpr && pExpr->eType!=FTSQUERY_AND ){
+ (*pnExpr)++;
+ pnExpr = 0;
+ }
+
+ if( pExpr->eType==FTSQUERY_PHRASE ){
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
+ int ii;
+
+ for(ii=0; rc==SQLITE_OK && ii<pPhrase->nToken; ii++){
+ Fts3PhraseToken *pTok = &pPhrase->aToken[ii];
+ if( pTok->pArray==0 ){
+ rc = fts3TermSegReaderArray(
+ pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pArray
+ );
+ }
+ }
+ }else{
+ rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pLeft, pnExpr);
+ if( rc==SQLITE_OK ){
+ rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pRight, pnExpr);
+ }
+ }
+ return rc;
+}
+
+/*
+** Free the Fts3SegReaderArray objects associated with each token in the
+** expression pExpr. In other words, this function frees the resources
+** allocated by fts3ExprAllocateSegReaders().
+*/
+static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){
+ if( pExpr ){
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
+ if( pPhrase ){
+ int kk;
+ for(kk=0; kk<pPhrase->nToken; kk++){
+ fts3SegReaderArrayFree(pPhrase->aToken[kk].pArray);
+ pPhrase->aToken[kk].pArray = 0;
+ }
+ }
+ fts3ExprFreeSegReaders(pExpr->pLeft);
+ fts3ExprFreeSegReaders(pExpr->pRight);
+ }
+}
+
+/*
+** Return the sum of the costs of all tokens in the expression pExpr. This
+** function must be called after Fts3SegReaderArrays have been allocated
+** for all tokens using fts3ExprAllocateSegReaders().
+*/
+static int fts3ExprCost(Fts3Expr *pExpr){
+ int nCost; /* Return value */
+ if( pExpr->eType==FTSQUERY_PHRASE ){
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
+ int ii;
+ nCost = 0;
+ for(ii=0; ii<pPhrase->nToken; ii++){
+ Fts3SegReaderArray *pArray = pPhrase->aToken[ii].pArray;
+ if( pArray ){
+ nCost += pPhrase->aToken[ii].pArray->nCost;
+ }
+ }
+ }else{
+ nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight);
+ }
+ return nCost;
+}
+
+/*
+** The following is a helper function (and type) for fts3EvalExpr(). It
+** must be called after Fts3SegReaders have been allocated for every token
+** in the expression. See the context it is called from in fts3EvalExpr()
+** for further explanation.
+*/
+typedef struct ExprAndCost ExprAndCost;
+struct ExprAndCost {
+ Fts3Expr *pExpr;
+ int nCost;
+};
+static void fts3ExprAssignCosts(
+ Fts3Expr *pExpr, /* Expression to create seg-readers for */
+ ExprAndCost **ppExprCost /* OUT: Write to *ppExprCost */
+){
+ if( pExpr->eType==FTSQUERY_AND ){
+ fts3ExprAssignCosts(pExpr->pLeft, ppExprCost);
+ fts3ExprAssignCosts(pExpr->pRight, ppExprCost);
+ }else{
+ (*ppExprCost)->pExpr = pExpr;
+ (*ppExprCost)->nCost = fts3ExprCost(pExpr);
+ (*ppExprCost)++;
+ }
+}
+
+/*
+** Evaluate the full-text expression pExpr against FTS3 table pTab. Store
+** the resulting doclist in *paOut and *pnOut. This routine mallocs for
+** the space needed to store the output. The caller is responsible for
+** freeing the space when it has finished.
+**
+** This function is called in two distinct contexts:
+**
+** * From within the virtual table xFilter() method. In this case, the
+** output doclist contains entries for all rows in the table, based on
+** data read from the full-text index.
+**
+** In this case, if the query expression contains one or more tokens that
+** are very common, then the returned doclist may contain a superset of
+** the documents that actually match the expression.
+**
+** * From within the virtual table xNext() method. This call is only made
+** if the call from within xFilter() found that there were very common
+** tokens in the query expression and did return a superset of the
+** matching documents. In this case the returned doclist contains only
+** entries that correspond to the current row of the table. Instead of
+** reading the data for each token from the full-text index, the data is
+** already available in-memory in the Fts3PhraseToken.pDeferred structures.
+** See fts3EvalDeferred() for how it gets there.
+**
+** In the first case above, Fts3Cursor.doDeferred==0. In the second (if it is
+** required) Fts3Cursor.doDeferred==1.
+**
+** If the SQLite invokes the snippet(), offsets() or matchinfo() function
+** as part of a SELECT on an FTS3 table, this function is called on each
+** individual phrase expression in the query. If there were very common tokens
+** found in the xFilter() call, then this function is called once for phrase
+** for each row visited, and the returned doclist contains entries for the
+** current row only. Otherwise, if there were no very common tokens, then this
+** function is called once only for each phrase in the query and the returned
+** doclist contains entries for all rows of the table.
+**
+** Fts3Cursor.doDeferred==1 when this function is called on phrases as a
+** result of a snippet(), offsets() or matchinfo() invocation.
+*/
+static int fts3EvalExpr(
+ Fts3Cursor *p, /* Virtual table cursor handle */
Fts3Expr *pExpr, /* Parsed fts3 expression */
char **paOut, /* OUT: Pointer to malloc'd result buffer */
int *pnOut, /* OUT: Size of buffer at *paOut */
@@ -109496,33 +110820,102 @@
*pnOut = 0;
if( pExpr ){
- assert( pExpr->eType==FTSQUERY_PHRASE
- || pExpr->eType==FTSQUERY_NEAR
- || isReqPos==0
+ assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR
+ || pExpr->eType==FTSQUERY_AND || pExpr->eType==FTSQUERY_NOT
+ || pExpr->eType==FTSQUERY_PHRASE
);
+ assert( pExpr->eType==FTSQUERY_PHRASE || isReqPos==0 );
+
if( pExpr->eType==FTSQUERY_PHRASE ){
- rc = fts3PhraseSelect(p, pExpr->pPhrase,
+ rc = fts3PhraseSelect(p, pExpr->pPhrase,
isReqPos || (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR),
paOut, pnOut
);
+ fts3ExprFreeSegReaders(pExpr);
+ }else if( p->eEvalmode==FTS3_EVAL_FILTER && pExpr->eType==FTSQUERY_AND ){
+ ExprAndCost *aExpr = 0; /* Array of AND'd expressions and costs */
+ int nExpr = 0; /* Size of aExpr[] */
+ char *aRet = 0; /* Doclist to return to caller */
+ int nRet = 0; /* Length of aRet[] in bytes */
+ int nDoc = 0x7FFFFFFF;
+
+ assert( !isReqPos );
+
+ rc = fts3ExprAllocateSegReaders(p, pExpr, &nExpr);
+ if( rc==SQLITE_OK ){
+ assert( nExpr>1 );
+ aExpr = sqlite3_malloc(sizeof(ExprAndCost) * nExpr);
+ if( !aExpr ) rc = SQLITE_NOMEM;
+ }
+ if( rc==SQLITE_OK ){
+ int ii; /* Used to iterate through expressions */
+
+ fts3ExprAssignCosts(pExpr, &aExpr);
+ aExpr -= nExpr;
+ for(ii=0; ii<nExpr; ii++){
+ char *aNew;
+ int nNew;
+ int jj;
+ ExprAndCost *pBest = 0;
+
+ for(jj=0; jj<nExpr; jj++){
+ ExprAndCost *pCand = &aExpr[jj];
+ if( pCand->pExpr && (pBest==0 || pCand->nCost<pBest->nCost) ){
+ pBest = pCand;
+ }
+ }
+
+ if( pBest->nCost>nDoc ){
+ rc = fts3DeferExpression(p, p->pExpr);
+ break;
+ }else{
+ rc = fts3EvalExpr(p, pBest->pExpr, &aNew, &nNew, 0);
+ if( rc!=SQLITE_OK ) break;
+ pBest->pExpr = 0;
+ if( ii==0 ){
+ aRet = aNew;
+ nRet = nNew;
+ nDoc = fts3DoclistCountDocids(0, aRet, nRet);
+ }else{
+ fts3DoclistMerge(
+ MERGE_AND, 0, 0, aRet, &nRet, aRet, nRet, aNew, nNew, &nDoc
+ );
+ sqlite3_free(aNew);
+ }
+ }
+ }
+ }
+
+ if( rc==SQLITE_OK ){
+ *paOut = aRet;
+ *pnOut = nRet;
+ }else{
+ assert( *paOut==0 );
+ sqlite3_free(aRet);
+ }
+ sqlite3_free(aExpr);
+ fts3ExprFreeSegReaders(pExpr);
+
}else{
char *aLeft;
char *aRight;
int nLeft;
int nRight;
- if( 0==(rc = evalFts3Expr(p, pExpr->pRight, &aRight, &nRight, isReqPos))
- && 0==(rc = evalFts3Expr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos))
+ assert( pExpr->eType==FTSQUERY_NEAR
+ || pExpr->eType==FTSQUERY_OR
+ || pExpr->eType==FTSQUERY_NOT
+ || (pExpr->eType==FTSQUERY_AND && p->eEvalmode==FTS3_EVAL_NEXT)
+ );
+
+ if( 0==(rc = fts3EvalExpr(p, pExpr->pRight, &aRight, &nRight, isReqPos))
+ && 0==(rc = fts3EvalExpr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos))
){
- assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR
- || pExpr->eType==FTSQUERY_AND || pExpr->eType==FTSQUERY_NOT
- );
switch( pExpr->eType ){
case FTSQUERY_NEAR: {
Fts3Expr *pLeft;
Fts3Expr *pRight;
- int mergetype = isReqPos ? MERGE_POS_NEAR : MERGE_NEAR;
-
+ int mergetype = MERGE_NEAR;
if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){
mergetype = MERGE_POS_NEAR;
}
@@ -109551,7 +110944,7 @@
*/
char *aBuffer = sqlite3_malloc(nRight+nLeft+1);
rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut,
- aLeft, nLeft, aRight, nRight
+ aLeft, nLeft, aRight, nRight, 0
);
*paOut = aBuffer;
sqlite3_free(aLeft);
@@ -109561,7 +110954,7 @@
default: {
assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND );
fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut,
- aLeft, nLeft, aRight, nRight
+ aLeft, nLeft, aRight, nRight, 0
);
*paOut = aLeft;
break;
@@ -109572,6 +110965,89 @@
}
}
+ assert( rc==SQLITE_OK || *paOut==0 );
+ return rc;
+}
+
+/*
+** This function is called from within xNext() for each row visited by
+** an FTS3 query. If evaluating the FTS3 query expression within xFilter()
+** was able to determine the exact set of matching rows, this function sets
+** *pbRes to true and returns SQLITE_IO immediately.
+**
+** Otherwise, if evaluating the query expression within xFilter() returned a
+** superset of the matching documents instead of an exact set (this happens
+** when the query includes very common tokens and it is deemed too expensive to
+** load their doclists from disk), this function tests if the current row
+** really does match the FTS3 query.
+**
+** If an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK
+** is returned and *pbRes is set to true if the current row matches the
+** FTS3 query (and should be included in the results returned to SQLite), or
+** false otherwise.
+*/
+static int fts3EvalDeferred(
+ Fts3Cursor *pCsr, /* FTS3 cursor pointing at row to test */
+ int *pbRes /* OUT: Set to true if row is a match */
+){
+ int rc = SQLITE_OK;
+ if( pCsr->pDeferred==0 ){
+ *pbRes = 1;
+ }else{
+ rc = fts3CursorSeek(0, pCsr);
+ if( rc==SQLITE_OK ){
+ sqlite3Fts3FreeDeferredDoclists(pCsr);
+ rc = sqlite3Fts3CacheDeferredDoclists(pCsr);
+ }
+ if( rc==SQLITE_OK ){
+ char *a = 0;
+ int n = 0;
+ rc = fts3EvalExpr(pCsr, pCsr->pExpr, &a, &n, 0);
+ assert( n>=0 );
+ *pbRes = (n>0);
+ sqlite3_free(a);
+ }
+ }
+ return rc;
+}
+
+/*
+** Advance the cursor to the next row in the %_content table that
+** matches the search criteria. For a MATCH search, this will be
+** the next row that matches. For a full-table scan, this will be
+** simply the next row in the %_content table. For a docid lookup,
+** this routine simply sets the EOF flag.
+**
+** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned
+** even if we reach end-of-file. The fts3EofMethod() will be called
+** subsequently to determine whether or not an EOF was hit.
+*/
+static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
+ int res;
+ int rc = SQLITE_OK; /* Return code */
+ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
+
+ pCsr->eEvalmode = FTS3_EVAL_NEXT;
+ do {
+ if( pCsr->aDoclist==0 ){
+ if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
+ pCsr->isEof = 1;
+ rc = sqlite3_reset(pCsr->pStmt);
+ break;
+ }
+ pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0);
+ }else{
+ if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){
+ pCsr->isEof = 1;
+ break;
+ }
+ sqlite3_reset(pCsr->pStmt);
+ fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId);
+ pCsr->isRequireSeek = 1;
+ pCsr->isMatchinfoNeeded = 1;
+ }
+ }while( SQLITE_OK==(rc = fts3EvalDeferred(pCsr, &res)) && res==0 );
+
return rc;
}
@@ -109591,11 +111067,6 @@
** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand
** side of the MATCH operator.
*/
-/* TODO(shess) Upgrade the cursor initialization and destruction to
-** account for fts3FilterMethod() being called multiple times on the
-** same cursor. The current solution is very fragile. Apply fix to
-** fts3 as appropriate.
-*/
static int fts3FilterMethod(
sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
int idxNum, /* Strategy index */
@@ -109618,6 +111089,7 @@
assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) );
assert( nVal==0 || nVal==1 );
assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) );
+ assert( p->pSegments==0 );
/* In case the cursor has been used before, clear it now. */
sqlite3_finalize(pCsr->pStmt);
@@ -109625,24 +111097,7 @@
sqlite3Fts3ExprFree(pCsr->pExpr);
memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
- /* Compile a SELECT statement for this cursor. For a full-table-scan, the
- ** statement loops through all rows of the %_content table. For a
- ** full-text query or docid lookup, the statement retrieves a single
- ** row by docid.
- */
- zSql = sqlite3_mprintf(azSql[idxNum==FTS3_FULLSCAN_SEARCH], p->zDb, p->zName);
- if( !zSql ){
- rc = SQLITE_NOMEM;
- }else{
- rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
- sqlite3_free(zSql);
- }
- if( rc!=SQLITE_OK ) return rc;
- pCsr->eSearch = (i16)idxNum;
-
- if( idxNum==FTS3_DOCID_SEARCH ){
- rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
- }else if( idxNum!=FTS3_FULLSCAN_SEARCH ){
+ if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){
int iCol = idxNum-FTS3_FULLTEXT_SEARCH;
const char *zQuery = (const char *)sqlite3_value_text(apVal[0]);
@@ -109664,11 +111119,30 @@
rc = sqlite3Fts3ReadLock(p);
if( rc!=SQLITE_OK ) return rc;
- rc = evalFts3Expr(p, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0);
+ rc = fts3EvalExpr(pCsr, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0);
+ sqlite3Fts3SegmentsClose(p);
+ if( rc!=SQLITE_OK ) return rc;
pCsr->pNextId = pCsr->aDoclist;
pCsr->iPrevId = 0;
}
+ /* Compile a SELECT statement for this cursor. For a full-table-scan, the
+ ** statement loops through all rows of the %_content table. For a
+ ** full-text query or docid lookup, the statement retrieves a single
+ ** row by docid.
+ */
+ zSql = sqlite3_mprintf(azSql[idxNum==FTS3_FULLSCAN_SEARCH], p->zDb, p->zName);
+ if( !zSql ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
+ sqlite3_free(zSql);
+ }
+ if( rc==SQLITE_OK && idxNum==FTS3_DOCID_SEARCH ){
+ rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
+ }
+ pCsr->eSearch = (i16)idxNum;
+
if( rc!=SQLITE_OK ) return rc;
return fts3NextMethod(pCursor);
}
@@ -109692,6 +111166,11 @@
if( pCsr->aDoclist ){
*pRowid = pCsr->iPrevId;
}else{
+ /* This branch runs if the query is implemented using a full-table scan
+ ** (not using the full-text index). In this case grab the rowid from the
+ ** SELECT statement.
+ */
+ assert( pCsr->isRequireSeek==0 );
*pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
}
return SQLITE_OK;
@@ -109754,7 +111233,9 @@
** hash-table to the database.
*/
static int fts3SyncMethod(sqlite3_vtab *pVtab){
- return sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab);
+ int rc = sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab);
+ sqlite3Fts3SegmentsClose((Fts3Table *)pVtab);
+ return rc;
}
/*
@@ -109792,8 +111273,27 @@
** This is used by the matchinfo(), snippet() and offsets() auxillary
** functions.
*/
-SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Table *pTab, Fts3Expr *pExpr){
- return evalFts3Expr(pTab, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1);
+SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){
+ int rc;
+ assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase );
+ assert( pCsr->eEvalmode==FTS3_EVAL_NEXT );
+ rc = fts3EvalExpr(pCsr, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1);
+ return rc;
+}
+
+SQLITE_PRIVATE int sqlite3Fts3ExprLoadFtDoclist(
+ Fts3Cursor *pCsr,
+ Fts3Expr *pExpr,
+ char **paDoclist,
+ int *pnDoclist
+){
+ int rc;
+ assert( pCsr->eEvalmode==FTS3_EVAL_NEXT );
+ assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase );
+ pCsr->eEvalmode = FTS3_EVAL_MATCHINFO;
+ rc = fts3EvalExpr(pCsr, pExpr, paDoclist, pnDoclist, 1);
+ pCsr->eEvalmode = FTS3_EVAL_NEXT;
+ return rc;
}
/*
@@ -109859,7 +111359,7 @@
sqlite3_context *pContext, /* SQL function call context */
const char *zFunc, /* Function name */
sqlite3_value *pVal, /* argv[0] passed to function */
- Fts3Cursor **ppCsr /* OUT: Store cursor handle here */
+ Fts3Cursor **ppCsr /* OUT: Store cursor handle here */
){
Fts3Cursor *pRet;
if( sqlite3_value_type(pVal)!=SQLITE_BLOB
@@ -109985,15 +111485,13 @@
sqlite3_value **apVal /* Array of arguments */
){
Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */
-
- if( nVal!=1 ){
- sqlite3_result_error(pContext,
- "wrong number of arguments to function matchinfo()", -1);
- return;
- }
-
+ assert( nVal==1 || nVal==2 );
if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){
- sqlite3Fts3Matchinfo(pContext, pCsr);
+ const char *zArg = 0;
+ if( nVal>1 ){
+ zArg = (const char *)sqlite3_value_text(apVal[1]);
+ }
+ sqlite3Fts3Matchinfo(pContext, pCsr, zArg);
}
}
@@ -110054,12 +111552,13 @@
"ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';",
p->zDb, p->zName, zName
);
- if( rc==SQLITE_ERROR ) rc = SQLITE_OK;
if( p->bHasDocsize ){
fts3DbExec(&rc, db,
"ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';",
p->zDb, p->zName, zName
);
+ }
+ if( p->bHasStat ){
fts3DbExec(&rc, db,
"ALTER TABLE %Q.'%q_stat' RENAME TO '%q_stat';",
p->zDb, p->zName, zName
@@ -110084,7 +111583,7 @@
/* xDisconnect */ fts3DisconnectMethod,
/* xDestroy */ fts3DestroyMethod,
/* xOpen */ fts3OpenMethod,
- /* xClose */ fulltextClose,
+ /* xClose */ fts3CloseMethod,
/* xFilter */ fts3FilterMethod,
/* xNext */ fts3NextMethod,
/* xEof */ fts3EofMethod,
@@ -110111,19 +111610,20 @@
}
/*
-** The fts3 built-in tokenizers - "simple" and "porter" - are implemented
-** in files fts3_tokenizer1.c and fts3_porter.c respectively. The following
-** two forward declarations are for functions declared in these files
-** used to retrieve the respective implementations.
+** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are
+** implemented in files fts3_tokenizer1.c, fts3_porter.c and fts3_icu.c
+** respectively. The following three forward declarations are for functions
+** declared in these files used to retrieve the respective implementations.
**
** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed
** to by the argument to point to the "simple" tokenizer implementation.
-** Function ...PorterTokenizerModule() sets *pModule to point to the
-** porter tokenizer/stemmer implementation.
+** And so on.
*/
SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+#ifdef SQLITE_ENABLE_ICU
SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+#endif
/*
** Initialise the fts3 extension. If this extension is built as part
@@ -110179,7 +111679,8 @@
&& SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer"))
&& SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
&& SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1))
- && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", -1))
+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1))
+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 2))
&& SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1))
){
rc = sqlite3_create_module_v2(
@@ -110323,6 +111824,18 @@
}
/*
+** Allocate nByte bytes of memory using sqlite3_malloc(). If successful,
+** zero the memory before returning a pointer to it. If unsuccessful,
+** return NULL.
+*/
+static void *fts3MallocZero(int nByte){
+ void *pRet = sqlite3_malloc(nByte);
+ if( pRet ) memset(pRet, 0, nByte);
+ return pRet;
+}
+
+
+/*
** Extract the next token from buffer z (length n) using the tokenizer
** and other information (column names etc.) in pParse. Create an Fts3Expr
** structure of type FTSQUERY_PHRASE containing a phrase consisting of this
@@ -110359,11 +111872,10 @@
if( rc==SQLITE_OK ){
nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken;
- pRet = (Fts3Expr *)sqlite3_malloc(nByte);
+ pRet = (Fts3Expr *)fts3MallocZero(nByte);
if( !pRet ){
rc = SQLITE_NOMEM;
}else{
- memset(pRet, 0, nByte);
pRet->eType = FTSQUERY_PHRASE;
pRet->pPhrase = (Fts3Phrase *)&pRet[1];
pRet->pPhrase->nToken = 1;
@@ -110439,7 +111951,7 @@
rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos);
if( rc==SQLITE_OK ){
int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
- p = fts3ReallocOrFree(p, nByte+ii*sizeof(struct PhraseToken));
+ p = fts3ReallocOrFree(p, nByte+ii*sizeof(Fts3PhraseToken));
zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken);
if( !p || !zTemp ){
goto no_mem;
@@ -110449,6 +111961,7 @@
p->pPhrase = (Fts3Phrase *)&p[1];
}
p->pPhrase = (Fts3Phrase *)&p[1];
+ memset(&p->pPhrase->aToken[ii], 0, sizeof(Fts3PhraseToken));
p->pPhrase->nToken = ii+1;
p->pPhrase->aToken[ii].n = nToken;
memcpy(&zTemp[nTemp], zToken, nToken);
@@ -110470,7 +111983,7 @@
char *zNew = NULL;
int nNew = 0;
int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
- nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(struct PhraseToken);
+ nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(Fts3PhraseToken);
p = fts3ReallocOrFree(p, nByte + nTemp);
if( !p ){
goto no_mem;
@@ -110588,11 +112101,10 @@
if( fts3isspace(cNext)
|| cNext=='"' || cNext=='(' || cNext==')' || cNext==0
){
- pRet = (Fts3Expr *)sqlite3_malloc(sizeof(Fts3Expr));
+ pRet = (Fts3Expr *)fts3MallocZero(sizeof(Fts3Expr));
if( !pRet ){
return SQLITE_NOMEM;
}
- memset(pRet, 0, sizeof(Fts3Expr));
pRet->eType = pKey->eType;
pRet->nNear = nNear;
*ppExpr = pRet;
@@ -110610,7 +112122,6 @@
if( sqlite3_fts3_enable_parentheses ){
if( *zInput=='(' ){
int nConsumed;
- int rc;
pParse->nNest++;
rc = fts3ExprParse(pParse, &zInput[1], nInput-1, ppExpr, &nConsumed);
if( rc==SQLITE_OK && !*ppExpr ){
@@ -110768,13 +112279,12 @@
&& p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot
){
/* Create an implicit NOT operator. */
- Fts3Expr *pNot = sqlite3_malloc(sizeof(Fts3Expr));
+ Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr));
if( !pNot ){
sqlite3Fts3ExprFree(p);
rc = SQLITE_NOMEM;
goto exprparse_out;
}
- memset(pNot, 0, sizeof(Fts3Expr));
pNot->eType = FTSQUERY_NOT;
pNot->pRight = p;
if( pNotBranch ){
@@ -110802,13 +112312,12 @@
/* Insert an implicit AND operator. */
Fts3Expr *pAnd;
assert( pRet && pPrev );
- pAnd = sqlite3_malloc(sizeof(Fts3Expr));
+ pAnd = fts3MallocZero(sizeof(Fts3Expr));
if( !pAnd ){
sqlite3Fts3ExprFree(p);
rc = SQLITE_NOMEM;
goto exprparse_out;
}
- memset(pAnd, 0, sizeof(Fts3Expr));
pAnd->eType = FTSQUERY_AND;
insertBinaryOperator(&pRet, pPrev, pAnd);
pPrev = pAnd;
@@ -110992,47 +112501,53 @@
}
/*
-** This function is part of the test interface for the query parser. It
-** writes a text representation of the query expression pExpr into the
-** buffer pointed to by argument zBuf. It is assumed that zBuf is large
-** enough to store the required text representation.
+** Return a pointer to a buffer containing a text representation of the
+** expression passed as the first argument. The buffer is obtained from
+** sqlite3_malloc(). It is the responsibility of the caller to use
+** sqlite3_free() to release the memory. If an OOM condition is encountered,
+** NULL is returned.
+**
+** If the second argument is not NULL, then its contents are prepended to
+** the returned expression text and then freed using sqlite3_free().
*/
-static void exprToString(Fts3Expr *pExpr, char *zBuf){
+static char *exprToString(Fts3Expr *pExpr, char *zBuf){
switch( pExpr->eType ){
case FTSQUERY_PHRASE: {
Fts3Phrase *pPhrase = pExpr->pPhrase;
int i;
- zBuf += sprintf(zBuf, "PHRASE %d %d", pPhrase->iColumn, pPhrase->isNot);
- for(i=0; i<pPhrase->nToken; i++){
- zBuf += sprintf(zBuf," %.*s",pPhrase->aToken[i].n,pPhrase->aToken[i].z);
- zBuf += sprintf(zBuf,"%s", (pPhrase->aToken[i].isPrefix?"+":""));
+ zBuf = sqlite3_mprintf(
+ "%zPHRASE %d %d", zBuf, pPhrase->iColumn, pPhrase->isNot);
+ for(i=0; zBuf && i<pPhrase->nToken; i++){
+ zBuf = sqlite3_mprintf("%z %.*s%s", zBuf,
+ pPhrase->aToken[i].n, pPhrase->aToken[i].z,
+ (pPhrase->aToken[i].isPrefix?"+":"")
+ );
}
- return;
+ return zBuf;
}
case FTSQUERY_NEAR:
- zBuf += sprintf(zBuf, "NEAR/%d ", pExpr->nNear);
+ zBuf = sqlite3_mprintf("%zNEAR/%d ", zBuf, pExpr->nNear);
break;
case FTSQUERY_NOT:
- zBuf += sprintf(zBuf, "NOT ");
+ zBuf = sqlite3_mprintf("%zNOT ", zBuf);
break;
case FTSQUERY_AND:
- zBuf += sprintf(zBuf, "AND ");
+ zBuf = sqlite3_mprintf("%zAND ", zBuf);
break;
case FTSQUERY_OR:
- zBuf += sprintf(zBuf, "OR ");
+ zBuf = sqlite3_mprintf("%zOR ", zBuf);
break;
}
- zBuf += sprintf(zBuf, "{");
- exprToString(pExpr->pLeft, zBuf);
- zBuf += strlen(zBuf);
- zBuf += sprintf(zBuf, "} ");
+ if( zBuf ) zBuf = sqlite3_mprintf("%z{", zBuf);
+ if( zBuf ) zBuf = exprToString(pExpr->pLeft, zBuf);
+ if( zBuf ) zBuf = sqlite3_mprintf("%z} {", zBuf);
- zBuf += sprintf(zBuf, "{");
- exprToString(pExpr->pRight, zBuf);
- zBuf += strlen(zBuf);
- zBuf += sprintf(zBuf, "}");
+ if( zBuf ) zBuf = exprToString(pExpr->pRight, zBuf);
+ if( zBuf ) zBuf = sqlite3_mprintf("%z}", zBuf);
+
+ return zBuf;
}
/*
@@ -111063,6 +112578,7 @@
int nCol;
int ii;
Fts3Expr *pExpr;
+ char *zBuf = 0;
sqlite3 *db = sqlite3_context_db_handle(context);
if( argc<3 ){
@@ -111105,18 +112621,17 @@
rc = sqlite3Fts3ExprParse(
pTokenizer, azCol, nCol, nCol, zExpr, nExpr, &pExpr
);
- if( rc==SQLITE_NOMEM ){
- sqlite3_result_error_nomem(context);
- goto exprtest_out;
- }else if( rc==SQLITE_OK ){
- char zBuf[4096];
- exprToString(pExpr, zBuf);
- sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
- sqlite3Fts3ExprFree(pExpr);
- }else{
+ if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){
sqlite3_result_error(context, "Error parsing expression", -1);
+ }else if( rc==SQLITE_NOMEM || !(zBuf = exprToString(pExpr, 0)) ){
+ sqlite3_result_error_nomem(context);
+ }else{
+ sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
+ sqlite3_free(zBuf);
}
+ sqlite3Fts3ExprFree(pExpr);
+
exprtest_out:
if( pModule && pTokenizer ){
rc = pModule->xDestroy(pTokenizer);
@@ -111859,7 +113374,7 @@
int i, j;
char zReverse[28];
char *z, *z2;
- if( nIn<3 || nIn>=sizeof(zReverse)-7 ){
+ if( nIn<3 || nIn>=(int)sizeof(zReverse)-7 ){
/* The word is too big or too small for the porter stemmer.
** Fallback to the copy stemmer */
copy_stemmer(zIn, nIn, zOut, pnOut);
@@ -112258,7 +113773,7 @@
sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
}
-static int fts3IsIdChar(char c){
+SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char c){
static const char isFtsIdChar[] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
@@ -112296,9 +113811,9 @@
break;
default:
- if( fts3IsIdChar(*z1) ){
+ if( sqlite3Fts3IsIdChar(*z1) ){
z2 = &z1[1];
- while( fts3IsIdChar(*z2) ) z2++;
+ while( sqlite3Fts3IsIdChar(*z2) ) z2++;
}else{
z1++;
}
@@ -112311,9 +113826,8 @@
SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(
Fts3Hash *pHash, /* Tokenizer hash table */
- const char *zArg, /* Possible tokenizer specification */
+ const char *zArg, /* Tokenizer name */
sqlite3_tokenizer **ppTok, /* OUT: Tokenizer (if applicable) */
- const char **pzTokenizer, /* OUT: Set to zArg if is tokenizer */
char **pzErr /* OUT: Set to malloced error message */
){
int rc;
@@ -112323,26 +113837,15 @@
char *zEnd; /* Pointer to nul-term of zCopy */
sqlite3_tokenizer_module *m;
- if( !z ){
- zCopy = sqlite3_mprintf("simple");
- }else{
- if( sqlite3_strnicmp(z, "tokenize", 8) || fts3IsIdChar(z[8])){
- return SQLITE_OK;
- }
- zCopy = sqlite3_mprintf("%s", &z[8]);
- *pzTokenizer = zArg;
- }
- if( !zCopy ){
- return SQLITE_NOMEM;
- }
-
+ zCopy = sqlite3_mprintf("%s", zArg);
+ if( !zCopy ) return SQLITE_NOMEM;
zEnd = &zCopy[strlen(zCopy)];
z = (char *)sqlite3Fts3NextToken(zCopy, &n);
z[n] = '\0';
sqlite3Fts3Dequote(z);
- m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, z, (int)strlen(z)+1);
+ m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1);
if( !m ){
*pzErr = sqlite3_mprintf("unknown tokenizer: %s", z);
rc = SQLITE_ERROR;
@@ -112636,15 +114139,23 @@
}
#endif
- if( SQLITE_OK!=rc
- || SQLITE_OK!=(rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0))
- || SQLITE_OK!=(rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0))
+ if( SQLITE_OK==rc ){
+ rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0);
+ }
+ if( SQLITE_OK==rc ){
+ rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0);
+ }
#ifdef SQLITE_TEST
- || SQLITE_OK!=(rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0))
- || SQLITE_OK!=(rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0))
- || SQLITE_OK!=(rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0))
+ if( SQLITE_OK==rc ){
+ rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0);
+ }
+ if( SQLITE_OK==rc ){
+ rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0);
+ }
+ if( SQLITE_OK==rc ){
+ rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0);
+ }
#endif
- );
#ifdef SQLITE_TEST
sqlite3_free(zTest);
@@ -112911,6 +114422,18 @@
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+/*
+** When full-text index nodes are loaded from disk, the buffer that they
+** are loaded into has the following number of bytes of padding at the end
+** of it. i.e. if a full-text index node is 900 bytes in size, then a buffer
+** of 920 bytes is allocated for it.
+**
+** This means that if we have a pointer into a buffer containing node data,
+** it is always safe to read up to two varints from it without risking an
+** overread, even if the node data is corrupted.
+*/
+#define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2)
+
typedef struct PendingList PendingList;
typedef struct SegmentNode SegmentNode;
typedef struct SegmentWriter SegmentWriter;
@@ -112929,6 +114452,17 @@
sqlite3_int64 iLastPos;
};
+
+/*
+** Each cursor has a (possibly empty) linked list of the following objects.
+*/
+struct Fts3DeferredToken {
+ Fts3PhraseToken *pToken; /* Pointer to corresponding expr token */
+ int iCol; /* Column token must occur in */
+ Fts3DeferredToken *pNext; /* Next in list of deferred tokens */
+ PendingList *pList; /* Doclist is assembled here */
+};
+
/*
** An instance of this structure is used to iterate through the terms on
** a contiguous set of segment b-tree leaf nodes. Although the details of
@@ -112938,6 +114472,7 @@
**
** sqlite3Fts3SegReaderNew()
** sqlite3Fts3SegReaderFree()
+** sqlite3Fts3SegReaderCost()
** sqlite3Fts3SegReaderIterate()
**
** Methods used to manipulate Fts3SegReader structures:
@@ -112948,12 +114483,14 @@
*/
struct Fts3SegReader {
int iIdx; /* Index within level, or 0x7FFFFFFF for PT */
- sqlite3_int64 iStartBlock;
- sqlite3_int64 iEndBlock;
- sqlite3_stmt *pStmt; /* SQL Statement to access leaf nodes */
+
+ sqlite3_int64 iStartBlock; /* Rowid of first leaf block to traverse */
+ sqlite3_int64 iLeafEndBlock; /* Rowid of final leaf block to traverse */
+ sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */
+ sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */
+
char *aNode; /* Pointer to node data (or NULL) */
int nNode; /* Size of buffer at aNode (or 0) */
- int nTermAlloc; /* Allocated size of zTerm buffer */
Fts3HashElem **ppNextElem;
/* Variables set by fts3SegReaderNext(). These may be read directly
@@ -112963,6 +114500,7 @@
*/
int nTerm; /* Number of bytes in current term */
char *zTerm; /* Pointer to current term */
+ int nTermAlloc; /* Allocated size of zTerm buffer */
char *aDoclist; /* Pointer to doclist of current entry */
int nDoclist; /* Size of doclist in current entry */
@@ -112972,6 +114510,7 @@
};
#define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0)
+#define fts3SegReaderIsRootOnly(p) ((p)->aNode==(char *)&(p)[1])
/*
** An instance of this structure is used to create a segment b-tree in the
@@ -113040,12 +114579,11 @@
#define SQL_DELETE_SEGDIR_BY_LEVEL 16
#define SQL_DELETE_SEGMENTS_RANGE 17
#define SQL_CONTENT_INSERT 18
-#define SQL_GET_BLOCK 19
-#define SQL_DELETE_DOCSIZE 20
-#define SQL_REPLACE_DOCSIZE 21
-#define SQL_SELECT_DOCSIZE 22
-#define SQL_SELECT_DOCTOTAL 23
-#define SQL_REPLACE_DOCTOTAL 24
+#define SQL_DELETE_DOCSIZE 19
+#define SQL_REPLACE_DOCSIZE 20
+#define SQL_SELECT_DOCSIZE 21
+#define SQL_SELECT_DOCTOTAL 22
+#define SQL_REPLACE_DOCTOTAL 23
/*
** This function is used to obtain an SQLite prepared statement handle
@@ -113090,12 +114628,11 @@
/* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?",
/* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?",
/* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%z)",
-/* 19 */ "SELECT block FROM %Q.'%q_segments' WHERE blockid = ?",
-/* 20 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?",
-/* 21 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",
-/* 22 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
-/* 23 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0",
-/* 24 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)",
+/* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?",
+/* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",
+/* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
+/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0",
+/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)",
};
int rc = SQLITE_OK;
sqlite3_stmt *pStmt;
@@ -113144,6 +114681,51 @@
return rc;
}
+static int fts3SelectDocsize(
+ Fts3Table *pTab, /* FTS3 table handle */
+ int eStmt, /* Either SQL_SELECT_DOCSIZE or DOCTOTAL */
+ sqlite3_int64 iDocid, /* Docid to bind for SQL_SELECT_DOCSIZE */
+ sqlite3_stmt **ppStmt /* OUT: Statement handle */
+){
+ sqlite3_stmt *pStmt = 0; /* Statement requested from fts3SqlStmt() */
+ int rc; /* Return code */
+
+ assert( eStmt==SQL_SELECT_DOCSIZE || eStmt==SQL_SELECT_DOCTOTAL );
+
+ rc = fts3SqlStmt(pTab, eStmt, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ if( eStmt==SQL_SELECT_DOCSIZE ){
+ sqlite3_bind_int64(pStmt, 1, iDocid);
+ }
+ rc = sqlite3_step(pStmt);
+ if( rc!=SQLITE_ROW ){
+ rc = sqlite3_reset(pStmt);
+ if( rc==SQLITE_OK ) rc = SQLITE_CORRUPT;
+ pStmt = 0;
+ }else{
+ rc = SQLITE_OK;
+ }
+ }
+
+ *ppStmt = pStmt;
+ return rc;
+}
+
+SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(
+ Fts3Table *pTab, /* Fts3 table handle */
+ sqlite3_stmt **ppStmt /* OUT: Statement handle */
+){
+ return fts3SelectDocsize(pTab, SQL_SELECT_DOCTOTAL, 0, ppStmt);
+}
+
+SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(
+ Fts3Table *pTab, /* Fts3 table handle */
+ sqlite3_int64 iDocid, /* Docid to read size data for */
+ sqlite3_stmt **ppStmt /* OUT: Statement handle */
+){
+ return fts3SelectDocsize(pTab, SQL_SELECT_DOCSIZE, iDocid, ppStmt);
+}
+
/*
** Similar to fts3SqlStmt(). Except, after binding the parameters in
** array apVal[] to the SQL statement identified by eStmt, the statement
@@ -113171,45 +114753,6 @@
/*
-** Read a single block from the %_segments table. If the specified block
-** does not exist, return SQLITE_CORRUPT. If some other error (malloc, IO
-** etc.) occurs, return the appropriate SQLite error code.
-**
-** Otherwise, if successful, set *pzBlock to point to a buffer containing
-** the block read from the database, and *pnBlock to the size of the read
-** block in bytes.
-**
-** WARNING: The returned buffer is only valid until the next call to
-** sqlite3Fts3ReadBlock().
-*/
-SQLITE_PRIVATE int sqlite3Fts3ReadBlock(
- Fts3Table *p,
- sqlite3_int64 iBlock,
- char const **pzBlock,
- int *pnBlock
-){
- sqlite3_stmt *pStmt;
- int rc = fts3SqlStmt(p, SQL_GET_BLOCK, &pStmt, 0);
- if( rc!=SQLITE_OK ) return rc;
- sqlite3_reset(pStmt);
-
- if( pzBlock ){
- sqlite3_bind_int64(pStmt, 1, iBlock);
- rc = sqlite3_step(pStmt);
- if( rc!=SQLITE_ROW ){
- return (rc==SQLITE_DONE ? SQLITE_CORRUPT : rc);
- }
-
- *pnBlock = sqlite3_column_bytes(pStmt, 0);
- *pzBlock = (char *)sqlite3_column_blob(pStmt, 0);
- if( sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){
- return SQLITE_CORRUPT;
- }
- }
- return SQLITE_OK;
-}
-
-/*
** This function ensures that the caller has obtained a shared-cache
** table-lock on the %_content table. This is required before reading
** data from the fts3 table. If this lock is not acquired first, then
@@ -113377,10 +114920,10 @@
** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code.
*/
static int fts3PendingTermsAdd(
- Fts3Table *p, /* FTS table into which text will be inserted */
- const char *zText, /* Text of document to be inseted */
- int iCol, /* Column number into which text is inserted */
- u32 *pnWord /* OUT: Number of tokens inserted */
+ Fts3Table *p, /* Table into which text will be inserted */
+ const char *zText, /* Text of document to be inserted */
+ int iCol, /* Column into which text is being inserted */
+ u32 *pnWord /* OUT: Number of tokens inserted */
){
int rc;
int iStart;
@@ -113465,6 +115008,9 @@
return SQLITE_OK;
}
+/*
+** Discard the contents of the pending-terms hash table.
+*/
SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){
Fts3HashElem *pElem;
for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){
@@ -113492,6 +115038,7 @@
return rc;
}
}
+ aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]);
}
return SQLITE_OK;
}
@@ -113579,6 +115126,8 @@
fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0);
if( p->bHasDocsize ){
fts3SqlExec(&rc, p, SQL_DELETE_ALL_DOCSIZE, 0);
+ }
+ if( p->bHasStat ){
fts3SqlExec(&rc, p, SQL_DELETE_ALL_STAT, 0);
}
return rc;
@@ -113589,7 +115138,7 @@
** (an integer) of a row about to be deleted. Remove all terms from the
** full-text index.
*/
-static void fts3DeleteTerms(
+static void fts3DeleteTerms(
int *pRC, /* Result code */
Fts3Table *p, /* The FTS table to delete from */
sqlite3_value **apVal, /* apVal[] contains the docid to be deleted */
@@ -113611,6 +115160,7 @@
*pRC = rc;
return;
}
+ aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i);
}
}
rc = sqlite3_reset(pSelect);
@@ -113674,11 +115224,92 @@
}
/*
+** The %_segments table is declared as follows:
+**
+** CREATE TABLE %_segments(blockid INTEGER PRIMARY KEY, block BLOB)
+**
+** This function reads data from a single row of the %_segments table. The
+** specific row is identified by the iBlockid parameter. If paBlob is not
+** NULL, then a buffer is allocated using sqlite3_malloc() and populated
+** with the contents of the blob stored in the "block" column of the
+** identified table row is. Whether or not paBlob is NULL, *pnBlob is set
+** to the size of the blob in bytes before returning.
+**
+** If an error occurs, or the table does not contain the specified row,
+** an SQLite error code is returned. Otherwise, SQLITE_OK is returned. If
+** paBlob is non-NULL, then it is the responsibility of the caller to
+** eventually free the returned buffer.
+**
+** This function may leave an open sqlite3_blob* handle in the
+** Fts3Table.pSegments variable. This handle is reused by subsequent calls
+** to this function. The handle may be closed by calling the
+** sqlite3Fts3SegmentsClose() function. Reusing a blob handle is a handy
+** performance improvement, but the blob handle should always be closed
+** before control is returned to the user (to prevent a lock being held
+** on the database file for longer than necessary). Thus, any virtual table
+** method (xFilter etc.) that may directly or indirectly call this function
+** must call sqlite3Fts3SegmentsClose() before returning.
+*/
+SQLITE_PRIVATE int sqlite3Fts3ReadBlock(
+ Fts3Table *p, /* FTS3 table handle */
+ sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */
+ char **paBlob, /* OUT: Blob data in malloc'd buffer */
+ int *pnBlob /* OUT: Size of blob data */
+){
+ int rc; /* Return code */
+
+ /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */
+ assert( pnBlob);
+
+ if( p->pSegments ){
+ rc = sqlite3_blob_reopen(p->pSegments, iBlockid);
+ }else{
+ if( 0==p->zSegmentsTbl ){
+ p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName);
+ if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM;
+ }
+ rc = sqlite3_blob_open(
+ p->db, p->zDb, p->zSegmentsTbl, "block", iBlockid, 0, &p->pSegments
+ );
+ }
+
+ if( rc==SQLITE_OK ){
+ int nByte = sqlite3_blob_bytes(p->pSegments);
+ if( paBlob ){
+ char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING);
+ if( !aByte ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0);
+ memset(&aByte[nByte], 0, FTS3_NODE_PADDING);
+ if( rc!=SQLITE_OK ){
+ sqlite3_free(aByte);
+ aByte = 0;
+ }
+ }
+ *paBlob = aByte;
+ }
+ *pnBlob = nByte;
+ }
+
+ return rc;
+}
+
+/*
+** Close the blob handle at p->pSegments, if it is open. See comments above
+** the sqlite3Fts3ReadBlock() function for details.
+*/
+SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *p){
+ sqlite3_blob_close(p->pSegments);
+ p->pSegments = 0;
+}
+
+/*
** Move the iterator passed as the first argument to the next term in the
** segment. If successful, SQLITE_OK is returned. If there is no next term,
** SQLITE_DONE. Otherwise, an SQLite error code.
*/
-static int fts3SegReaderNext(Fts3SegReader *pReader){
+static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){
char *pNext; /* Cursor variable */
int nPrefix; /* Number of bytes in term prefix */
int nSuffix; /* Number of bytes in term suffix */
@@ -113690,7 +115321,8 @@
}
if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){
- int rc;
+ int rc; /* Return code from Fts3ReadBlock() */
+
if( fts3SegReaderIsPending(pReader) ){
Fts3HashElem *pElem = *(pReader->ppNextElem);
if( pElem==0 ){
@@ -113706,22 +115338,36 @@
}
return SQLITE_OK;
}
- if( !pReader->pStmt ){
- pReader->aNode = 0;
+
+ if( !fts3SegReaderIsRootOnly(pReader) ){
+ sqlite3_free(pReader->aNode);
+ }
+ pReader->aNode = 0;
+
+ /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf
+ ** blocks have already been traversed. */
+ assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock );
+ if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){
return SQLITE_OK;
}
- rc = sqlite3_step(pReader->pStmt);
- if( rc!=SQLITE_ROW ){
- pReader->aNode = 0;
- return (rc==SQLITE_DONE ? SQLITE_OK : rc);
- }
- pReader->nNode = sqlite3_column_bytes(pReader->pStmt, 0);
- pReader->aNode = (char *)sqlite3_column_blob(pReader->pStmt, 0);
+
+ rc = sqlite3Fts3ReadBlock(
+ p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode
+ );
+ if( rc!=SQLITE_OK ) return rc;
pNext = pReader->aNode;
}
+ /* Because of the FTS3_NODE_PADDING bytes of padding, the following is
+ ** safe (no risk of overread) even if the node data is corrupted.
+ */
pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix);
pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix);
+ if( nPrefix<0 || nSuffix<=0
+ || &pNext[nSuffix]>&pReader->aNode[pReader->nNode]
+ ){
+ return SQLITE_CORRUPT;
+ }
if( nPrefix+nSuffix>pReader->nTermAlloc ){
int nNew = (nPrefix+nSuffix)*2;
@@ -113736,9 +115382,18 @@
pReader->nTerm = nPrefix+nSuffix;
pNext += nSuffix;
pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist);
- assert( pNext<&pReader->aNode[pReader->nNode] );
pReader->aDoclist = pNext;
pReader->pOffsetList = 0;
+
+ /* Check that the doclist does not appear to extend past the end of the
+ ** b-tree node. And that the final byte of the doclist is 0x00. If either
+ ** of these statements is untrue, then the data structure is corrupt.
+ */
+ if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode]
+ || pReader->aDoclist[pReader->nDoclist-1]
+ ){
+ return SQLITE_CORRUPT;
+ }
return SQLITE_OK;
}
@@ -113802,31 +115457,105 @@
}
/*
+** This function is called to estimate the amount of data that will be
+** loaded from the disk If SegReaderIterate() is called on this seg-reader,
+** in units of average document size.
+**
+** This can be used as follows: If the caller has a small doclist that
+** contains references to N documents, and is considering merging it with
+** a large doclist (size X "average documents"), it may opt not to load
+** the large doclist if X>N.
+*/
+SQLITE_PRIVATE int sqlite3Fts3SegReaderCost(
+ Fts3Cursor *pCsr, /* FTS3 cursor handle */
+ Fts3SegReader *pReader, /* Segment-reader handle */
+ int *pnCost /* IN/OUT: Number of bytes read */
+){
+ Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
+ int rc = SQLITE_OK; /* Return code */
+ int nCost = 0; /* Cost in bytes to return */
+ int pgsz = p->nPgsz; /* Database page size */
+
+ /* If this seg-reader is reading the pending-terms table, or if all data
+ ** for the segment is stored on the root page of the b-tree, then the cost
+ ** is zero. In this case all required data is already in main memory.
+ */
+ if( p->bHasStat
+ && !fts3SegReaderIsPending(pReader)
+ && !fts3SegReaderIsRootOnly(pReader)
+ ){
+ int nBlob = 0;
+ sqlite3_int64 iBlock;
+
+ if( pCsr->nRowAvg==0 ){
+ /* The average document size, which is required to calculate the cost
+ ** of each doclist, has not yet been determined. Read the required
+ ** data from the %_stat table to calculate it.
+ **
+ ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3
+ ** varints, where nCol is the number of columns in the FTS3 table.
+ ** The first varint is the number of documents currently stored in
+ ** the table. The following nCol varints contain the total amount of
+ ** data stored in all rows of each column of the table, from left
+ ** to right.
+ */
+ sqlite3_stmt *pStmt;
+ rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0);
+ if( rc ) return rc;
+ if( sqlite3_step(pStmt)==SQLITE_ROW ){
+ sqlite3_int64 nDoc = 0;
+ sqlite3_int64 nByte = 0;
+ const char *a = sqlite3_column_blob(pStmt, 0);
+ if( a ){
+ const char *pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
+ a += sqlite3Fts3GetVarint(a, &nDoc);
+ while( a<pEnd ){
+ a += sqlite3Fts3GetVarint(a, &nByte);
+ }
+ }
+
+ pCsr->nRowAvg = (int)(((nByte / nDoc) + pgsz - 1) / pgsz);
+ }
+ rc = sqlite3_reset(pStmt);
+ if( rc!=SQLITE_OK || pCsr->nRowAvg==0 ) return rc;
+ }
+
+ /* Assume that a blob flows over onto overflow pages if it is larger
+ ** than (pgsz-35) bytes in size (the file-format documentation
+ ** confirms this).
+ */
+ for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){
+ rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob);
+ if( rc!=SQLITE_OK ) break;
+ if( (nBlob+35)>pgsz ){
+ int nOvfl = (nBlob + 34)/pgsz;
+ nCost += ((nOvfl + pCsr->nRowAvg - 1)/pCsr->nRowAvg);
+ }
+ }
+ }
+
+ *pnCost += nCost;
+ return rc;
+}
+
+/*
** Free all allocations associated with the iterator passed as the
** second argument.
*/
-SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3Table *p, Fts3SegReader *pReader){
- if( pReader ){
- if( pReader->pStmt ){
- /* Move the leaf-range SELECT statement to the aLeavesStmt[] array,
- ** so that it can be reused when required by another query.
- */
- assert( p->nLeavesStmt<p->nLeavesTotal );
- sqlite3_reset(pReader->pStmt);
- p->aLeavesStmt[p->nLeavesStmt++] = pReader->pStmt;
+SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){
+ if( pReader && !fts3SegReaderIsPending(pReader) ){
+ sqlite3_free(pReader->zTerm);
+ if( !fts3SegReaderIsRootOnly(pReader) ){
+ sqlite3_free(pReader->aNode);
}
- if( !fts3SegReaderIsPending(pReader) ){
- sqlite3_free(pReader->zTerm);
- }
- sqlite3_free(pReader);
}
+ sqlite3_free(pReader);
}
/*
** Allocate a new SegReader object.
*/
SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(
- Fts3Table *p, /* Virtual table handle */
int iAge, /* Segment "age". */
sqlite3_int64 iStartLeaf, /* First leaf to traverse */
sqlite3_int64 iEndLeaf, /* Final leaf to traverse */
@@ -113839,8 +115568,9 @@
Fts3SegReader *pReader; /* Newly allocated SegReader object */
int nExtra = 0; /* Bytes to allocate segment root node */
+ assert( iStartLeaf<=iEndLeaf );
if( iStartLeaf==0 ){
- nExtra = nRoot;
+ nExtra = nRoot + FTS3_NODE_PADDING;
}
pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra);
@@ -113848,8 +115578,9 @@
return SQLITE_NOMEM;
}
memset(pReader, 0, sizeof(Fts3SegReader));
- pReader->iStartBlock = iStartLeaf;
pReader->iIdx = iAge;
+ pReader->iStartBlock = iStartLeaf;
+ pReader->iLeafEndBlock = iEndLeaf;
pReader->iEndBlock = iEndBlock;
if( nExtra ){
@@ -113857,59 +115588,15 @@
pReader->aNode = (char *)&pReader[1];
pReader->nNode = nRoot;
memcpy(pReader->aNode, zRoot, nRoot);
+ memset(&pReader->aNode[nRoot], 0, FTS3_NODE_PADDING);
}else{
- /* If the text of the SQL statement to iterate through a contiguous
- ** set of entries in the %_segments table has not yet been composed,
- ** compose it now.
- */
- if( !p->zSelectLeaves ){
- p->zSelectLeaves = sqlite3_mprintf(
- "SELECT block FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ? "
- "ORDER BY blockid", p->zDb, p->zName
- );
- if( !p->zSelectLeaves ){
- rc = SQLITE_NOMEM;
- goto finished;
- }
- }
-
- /* If there are no free statements in the aLeavesStmt[] array, prepare
- ** a new statement now. Otherwise, reuse a prepared statement from
- ** aLeavesStmt[].
- */
- if( p->nLeavesStmt==0 ){
- if( p->nLeavesTotal==p->nLeavesAlloc ){
- int nNew = p->nLeavesAlloc + 16;
- sqlite3_stmt **aNew = (sqlite3_stmt **)sqlite3_realloc(
- p->aLeavesStmt, nNew*sizeof(sqlite3_stmt *)
- );
- if( !aNew ){
- rc = SQLITE_NOMEM;
- goto finished;
- }
- p->nLeavesAlloc = nNew;
- p->aLeavesStmt = aNew;
- }
- rc = sqlite3_prepare_v2(p->db, p->zSelectLeaves, -1, &pReader->pStmt, 0);
- if( rc!=SQLITE_OK ){
- goto finished;
- }
- p->nLeavesTotal++;
- }else{
- pReader->pStmt = p->aLeavesStmt[--p->nLeavesStmt];
- }
-
- /* Bind the start and end leaf blockids to the prepared SQL statement. */
- sqlite3_bind_int64(pReader->pStmt, 1, iStartLeaf);
- sqlite3_bind_int64(pReader->pStmt, 2, iEndLeaf);
+ pReader->iCurrentBlock = iStartLeaf-1;
}
- rc = fts3SegReaderNext(pReader);
- finished:
if( rc==SQLITE_OK ){
*ppReader = pReader;
}else{
- sqlite3Fts3SegReaderFree(p, pReader);
+ sqlite3Fts3SegReaderFree(pReader);
}
return rc;
}
@@ -114000,7 +115687,6 @@
pReader->iIdx = 0x7FFFFFFF;
pReader->ppNextElem = (Fts3HashElem **)&pReader[1];
memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *));
- fts3SegReaderNext(pReader);
}
}
@@ -114033,12 +115719,11 @@
** code is returned.
*/
static int fts3SegReaderNew(
- Fts3Table *p, /* Virtual table handle */
sqlite3_stmt *pStmt, /* See above */
int iAge, /* Segment "age". */
Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */
){
- return sqlite3Fts3SegReaderNew(p, iAge,
+ return sqlite3Fts3SegReaderNew(iAge,
sqlite3_column_int64(pStmt, 1),
sqlite3_column_int64(pStmt, 2),
sqlite3_column_int64(pStmt, 3),
@@ -114242,7 +115927,7 @@
** (according to memcmp) than the previous term.
*/
static int fts3NodeAddTerm(
- Fts3Table *p, /* Virtual table handle */
+ Fts3Table *p, /* Virtual table handle */
SegmentNode **ppTree, /* IN/OUT: SegmentNode handle */
int isCopyTerm, /* True if zTerm/nTerm is transient */
const char *zTerm, /* Pointer to buffer containing term */
@@ -114872,15 +116557,14 @@
** unnecessary merge/sort operations for the case where single segment
** b-tree leaf nodes contain more than one term.
*/
- if( pFilter->zTerm ){
+ for(i=0; i<nSegment; i++){
int nTerm = pFilter->nTerm;
const char *zTerm = pFilter->zTerm;
- for(i=0; i<nSegment; i++){
- Fts3SegReader *pSeg = apSegment[i];
- while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ){
- rc = fts3SegReaderNext(pSeg);
- if( rc!=SQLITE_OK ) goto finished; }
- }
+ Fts3SegReader *pSeg = apSegment[i];
+ do {
+ rc = fts3SegReaderNext(p, pSeg);
+ if( rc!=SQLITE_OK ) goto finished;
+ }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 );
}
fts3SegReaderSort(apSegment, nSegment, nSegment, fts3SegReaderCmp);
@@ -114989,7 +116673,7 @@
}
for(i=0; i<nMerge; i++){
- rc = fts3SegReaderNext(apSegment[i]);
+ rc = fts3SegReaderNext(p, apSegment[i]);
if( rc!=SQLITE_OK ) goto finished;
}
fts3SegReaderSort(apSegment, nSegment, nMerge, fts3SegReaderCmp);
@@ -115015,7 +116699,7 @@
int i; /* Iterator variable */
int rc; /* Return code */
int iIdx; /* Index of new segment */
- int iNewLevel; /* Level to create new segment at */
+ int iNewLevel = 0; /* Level to create new segment at */
sqlite3_stmt *pStmt = 0;
SegmentWriter *pWriter = 0;
int nSegment = 0; /* Number of segments being merged */
@@ -115070,7 +116754,7 @@
if( rc!=SQLITE_OK ) goto finished;
sqlite3_bind_int(pStmt, 1, iLevel);
for(i=0; SQLITE_ROW==(sqlite3_step(pStmt)); i++){
- rc = fts3SegReaderNew(p, pStmt, i, &apSegment[i]);
+ rc = fts3SegReaderNew(pStmt, i, &apSegment[i]);
if( rc!=SQLITE_OK ){
goto finished;
}
@@ -115100,11 +116784,11 @@
fts3SegWriterFree(pWriter);
if( apSegment ){
for(i=0; i<nSegment; i++){
- sqlite3Fts3SegReaderFree(p, apSegment[i]);
+ sqlite3Fts3SegReaderFree(apSegment[i]);
}
sqlite3_free(apSegment);
}
- sqlite3Fts3SegReaderFree(p, pPending);
+ sqlite3Fts3SegReaderFree(pPending);
sqlite3_reset(pStmt);
return rc;
}
@@ -115157,7 +116841,7 @@
rc = fts3SegWriterFlush(p, pWriter, 0, idx);
}
fts3SegWriterFree(pWriter);
- sqlite3Fts3SegReaderFree(p, pReader);
+ sqlite3Fts3SegReaderFree(pReader);
if( rc==SQLITE_OK ){
sqlite3Fts3PendingTermsClear(p);
@@ -115201,75 +116885,6 @@
}
/*
-** Fill in the document size auxiliary information for the matchinfo
-** structure. The auxiliary information is:
-**
-** N Total number of documents in the full-text index
-** a0 Average length of column 0 over the whole index
-** n0 Length of column 0 on the matching row
-** ...
-** aM Average length of column M over the whole index
-** nM Length of column M on the matching row
-**
-** The fts3MatchinfoDocsizeLocal() routine fills in the nX values.
-** The fts3MatchinfoDocsizeGlobal() routine fills in N and the aX values.
-*/
-SQLITE_PRIVATE int sqlite3Fts3MatchinfoDocsizeLocal(Fts3Cursor *pCur, u32 *a){
- const char *pBlob; /* The BLOB holding %_docsize info */
- int nBlob; /* Size of the BLOB */
- sqlite3_stmt *pStmt; /* Statement for reading and writing */
- int i, j; /* Loop counters */
- sqlite3_int64 x; /* Varint value */
- int rc; /* Result code from subfunctions */
- Fts3Table *p; /* The FTS table */
-
- p = (Fts3Table*)pCur->base.pVtab;
- rc = fts3SqlStmt(p, SQL_SELECT_DOCSIZE, &pStmt, 0);
- if( rc ){
- return rc;
- }
- sqlite3_bind_int64(pStmt, 1, pCur->iPrevId);
- if( sqlite3_step(pStmt)==SQLITE_ROW ){
- nBlob = sqlite3_column_bytes(pStmt, 0);
- pBlob = (const char*)sqlite3_column_blob(pStmt, 0);
- for(i=j=0; i<p->nColumn && j<nBlob; i++){
- j = sqlite3Fts3GetVarint(&pBlob[j], &x);
- a[2+i*2] = (u32)(x & 0xffffffff);
- }
- }
- sqlite3_reset(pStmt);
- return SQLITE_OK;
-}
-SQLITE_PRIVATE int sqlite3Fts3MatchinfoDocsizeGlobal(Fts3Cursor *pCur, u32 *a){
- const char *pBlob; /* The BLOB holding %_stat info */
- int nBlob; /* Size of the BLOB */
- sqlite3_stmt *pStmt; /* Statement for reading and writing */
- int i, j; /* Loop counters */
- sqlite3_int64 x; /* Varint value */
- int nDoc; /* Number of documents */
- int rc; /* Result code from subfunctions */
- Fts3Table *p; /* The FTS table */
-
- p = (Fts3Table*)pCur->base.pVtab;
- rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0);
- if( rc ){
- return rc;
- }
- if( sqlite3_step(pStmt)==SQLITE_ROW ){
- nBlob = sqlite3_column_bytes(pStmt, 0);
- pBlob = (const char*)sqlite3_column_blob(pStmt, 0);
- j = sqlite3Fts3GetVarint(pBlob, &x);
- a[0] = nDoc = (u32)(x & 0xffffffff);
- for(i=0; i<p->nColumn && j<nBlob; i++){
- j = sqlite3Fts3GetVarint(&pBlob[j], &x);
- a[1+i*2] = ((u32)(x & 0xffffffff) + nDoc/2)/nDoc;
- }
- }
- sqlite3_reset(pStmt);
- return SQLITE_OK;
-}
-
-/*
** Insert the sizes (in tokens) for each column of the document
** with docid equal to p->iPrevDocid. The sizes are encoded as
** a blob of varints.
@@ -115304,16 +116919,26 @@
}
/*
-** Update the 0 record of the %_stat table so that it holds a blob
-** which contains the document count followed by the cumulative
-** document sizes for all columns.
+** Record 0 of the %_stat table contains a blob consisting of N varints,
+** where N is the number of user defined columns in the fts3 table plus
+** two. If nCol is the number of user defined columns, then values of the
+** varints are set as follows:
+**
+** Varint 0: Total number of rows in the table.
+**
+** Varint 1..nCol: For each column, the total number of tokens stored in
+** the column for all rows of the table.
+**
+** Varint 1+nCol: The total size, in bytes, of all text values in all
+** columns of all rows of the table.
+**
*/
static void fts3UpdateDocTotals(
- int *pRC, /* The result code */
- Fts3Table *p, /* Table being updated */
- u32 *aSzIns, /* Size increases */
- u32 *aSzDel, /* Size decreases */
- int nChng /* Change in the number of documents */
+ int *pRC, /* The result code */
+ Fts3Table *p, /* Table being updated */
+ u32 *aSzIns, /* Size increases */
+ u32 *aSzDel, /* Size decreases */
+ int nChng /* Change in the number of documents */
){
char *pBlob; /* Storage for BLOB written into %_stat */
int nBlob; /* Size of BLOB written into %_stat */
@@ -115322,13 +116947,15 @@
int i; /* Loop counter */
int rc; /* Result code from subfunctions */
+ const int nStat = p->nColumn+2;
+
if( *pRC ) return;
- a = sqlite3_malloc( (sizeof(u32)+10)*(p->nColumn+1) );
+ a = sqlite3_malloc( (sizeof(u32)+10)*nStat );
if( a==0 ){
*pRC = SQLITE_NOMEM;
return;
}
- pBlob = (char*)&a[p->nColumn+1];
+ pBlob = (char*)&a[nStat];
rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0);
if( rc ){
sqlite3_free(a);
@@ -115336,11 +116963,11 @@
return;
}
if( sqlite3_step(pStmt)==SQLITE_ROW ){
- fts3DecodeIntArray(p->nColumn+1, a,
+ fts3DecodeIntArray(nStat, a,
sqlite3_column_blob(pStmt, 0),
sqlite3_column_bytes(pStmt, 0));
}else{
- memset(a, 0, sizeof(u32)*(p->nColumn+1) );
+ memset(a, 0, sizeof(u32)*(nStat) );
}
sqlite3_reset(pStmt);
if( nChng<0 && a[0]<(u32)(-nChng) ){
@@ -115348,7 +116975,7 @@
}else{
a[0] += nChng;
}
- for(i=0; i<p->nColumn; i++){
+ for(i=0; i<p->nColumn+1; i++){
u32 x = a[i+1];
if( x+aSzIns[i] < aSzDel[i] ){
x = 0;
@@ -115357,7 +116984,7 @@
}
a[i+1] = x;
}
- fts3EncodeIntArray(p->nColumn+1, a, pBlob, &nBlob);
+ fts3EncodeIntArray(nStat, a, pBlob, &nBlob);
rc = fts3SqlStmt(p, SQL_REPLACE_DOCTOTAL, &pStmt, 0);
if( rc ){
sqlite3_free(a);
@@ -115404,10 +117031,160 @@
rc = SQLITE_ERROR;
}
+ sqlite3Fts3SegmentsClose(p);
return rc;
}
/*
+** Return the deferred doclist associated with deferred token pDeferred.
+** This function assumes that sqlite3Fts3CacheDeferredDoclists() has already
+** been called to allocate and populate the doclist.
+*/
+SQLITE_PRIVATE char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *pDeferred, int *pnByte){
+ if( pDeferred->pList ){
+ *pnByte = pDeferred->pList->nData;
+ return pDeferred->pList->aData;
+ }
+ *pnByte = 0;
+ return 0;
+}
+
+/*
+** Helper fucntion for FreeDeferredDoclists(). This function removes all
+** references to deferred doclists from within the tree of Fts3Expr
+** structures headed by
+*/
+static void fts3DeferredDoclistClear(Fts3Expr *pExpr){
+ if( pExpr ){
+ fts3DeferredDoclistClear(pExpr->pLeft);
+ fts3DeferredDoclistClear(pExpr->pRight);
+ if( pExpr->isLoaded ){
+ sqlite3_free(pExpr->aDoclist);
+ pExpr->isLoaded = 0;
+ pExpr->aDoclist = 0;
+ pExpr->nDoclist = 0;
+ pExpr->pCurrent = 0;
+ pExpr->iCurrent = 0;
+ }
+ }
+}
+
+/*
+** Delete all cached deferred doclists. Deferred doclists are cached
+** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function.
+*/
+SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){
+ Fts3DeferredToken *pDef;
+ for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){
+ sqlite3_free(pDef->pList);
+ pDef->pList = 0;
+ }
+ if( pCsr->pDeferred ){
+ fts3DeferredDoclistClear(pCsr->pExpr);
+ }
+}
+
+/*
+** Free all entries in the pCsr->pDeffered list. Entries are added to
+** this list using sqlite3Fts3DeferToken().
+*/
+SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){
+ Fts3DeferredToken *pDef;
+ Fts3DeferredToken *pNext;
+ for(pDef=pCsr->pDeferred; pDef; pDef=pNext){
+ pNext = pDef->pNext;
+ sqlite3_free(pDef->pList);
+ sqlite3_free(pDef);
+ }
+ pCsr->pDeferred = 0;
+}
+
+/*
+** Generate deferred-doclists for all tokens in the pCsr->pDeferred list
+** based on the row that pCsr currently points to.
+**
+** A deferred-doclist is like any other doclist with position information
+** included, except that it only contains entries for a single row of the
+** table, not for all rows.
+*/
+SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){
+ int rc = SQLITE_OK; /* Return code */
+ if( pCsr->pDeferred ){
+ int i; /* Used to iterate through table columns */
+ sqlite3_int64 iDocid; /* Docid of the row pCsr points to */
+ Fts3DeferredToken *pDef; /* Used to iterate through deferred tokens */
+
+ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
+ sqlite3_tokenizer *pT = p->pTokenizer;
+ sqlite3_tokenizer_module const *pModule = pT->pModule;
+
+ assert( pCsr->isRequireSeek==0 );
+ iDocid = sqlite3_column_int64(pCsr->pStmt, 0);
+
+ for(i=0; i<p->nColumn && rc==SQLITE_OK; i++){
+ const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1);
+ sqlite3_tokenizer_cursor *pTC = 0;
+
+ rc = pModule->xOpen(pT, zText, -1, &pTC);
+ while( rc==SQLITE_OK ){
+ char const *zToken; /* Buffer containing token */
+ int nToken; /* Number of bytes in token */
+ int iDum1, iDum2; /* Dummy variables */
+ int iPos; /* Position of token in zText */
+
+ pTC->pTokenizer = pT;
+ rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos);
+ for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){
+ Fts3PhraseToken *pPT = pDef->pToken;
+ if( (pDef->iCol>=p->nColumn || pDef->iCol==i)
+ && (pPT->n==nToken || (pPT->isPrefix && pPT->n<nToken))
+ && (0==memcmp(zToken, pPT->z, pPT->n))
+ ){
+ fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc);
+ }
+ }
+ }
+ if( pTC ) pModule->xClose(pTC);
+ if( rc==SQLITE_DONE ) rc = SQLITE_OK;
+ }
+
+ for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){
+ if( pDef->pList ){
+ rc = fts3PendingListAppendVarint(&pDef->pList, 0);
+ }
+ }
+ }
+
+ return rc;
+}
+
+/*
+** Add an entry for token pToken to the pCsr->pDeferred list.
+*/
+SQLITE_PRIVATE int sqlite3Fts3DeferToken(
+ Fts3Cursor *pCsr, /* Fts3 table cursor */
+ Fts3PhraseToken *pToken, /* Token to defer */
+ int iCol /* Column that token must appear in (or -1) */
+){
+ Fts3DeferredToken *pDeferred;
+ pDeferred = sqlite3_malloc(sizeof(*pDeferred));
+ if( !pDeferred ){
+ return SQLITE_NOMEM;
+ }
+ memset(pDeferred, 0, sizeof(*pDeferred));
+ pDeferred->pToken = pToken;
+ pDeferred->pNext = pCsr->pDeferred;
+ pDeferred->iCol = iCol;
+ pCsr->pDeferred = pDeferred;
+
+ assert( pToken->pDeferred==0 );
+ pToken->pDeferred = pDeferred;
+
+ return SQLITE_OK;
+}
+
+
+/*
** This function does the work for the xUpdate method of FTS3 virtual
** tables.
*/
@@ -115425,16 +117202,17 @@
u32 *aSzDel; /* Sizes of deleted documents */
int nChng = 0; /* Net change in number of documents */
+ assert( p->pSegments==0 );
/* Allocate space to hold the change in document sizes */
- aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*p->nColumn*2 );
+ aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 );
if( aSzIns==0 ) return SQLITE_NOMEM;
- aSzDel = &aSzIns[p->nColumn];
- memset(aSzIns, 0, sizeof(aSzIns[0])*p->nColumn*2);
+ aSzDel = &aSzIns[p->nColumn+1];
+ memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2);
/* If this is a DELETE or UPDATE operation, remove the old record. */
if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
- int isEmpty;
+ int isEmpty = 0;
rc = fts3IsEmpty(p, apVal, &isEmpty);
if( rc==SQLITE_OK ){
if( isEmpty ){
@@ -115451,8 +117229,8 @@
fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, apVal);
if( p->bHasDocsize ){
fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, apVal);
- nChng--;
}
+ nChng--;
}
}
}else if( sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL ){
@@ -115470,16 +117248,17 @@
rc = fts3InsertTerms(p, apVal, aSzIns);
}
if( p->bHasDocsize ){
- nChng++;
fts3InsertDocsize(&rc, p, aSzIns);
}
+ nChng++;
}
- if( p->bHasDocsize ){
+ if( p->bHasStat ){
fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng);
}
sqlite3_free(aSzIns);
+ sqlite3Fts3SegmentsClose(p);
return rc;
}
@@ -115503,6 +117282,7 @@
sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
}
}
+ sqlite3Fts3SegmentsClose(p);
return rc;
}
@@ -115526,6 +117306,22 @@
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+/*
+** Characters that may appear in the second argument to matchinfo().
+*/
+#define FTS3_MATCHINFO_NPHRASE 'p' /* 1 value */
+#define FTS3_MATCHINFO_NCOL 'c' /* 1 value */
+#define FTS3_MATCHINFO_NDOC 'n' /* 1 value */
+#define FTS3_MATCHINFO_AVGLENGTH 'a' /* nCol values */
+#define FTS3_MATCHINFO_LENGTH 'l' /* nCol values */
+#define FTS3_MATCHINFO_LCS 's' /* nCol values */
+#define FTS3_MATCHINFO_HITS 'x' /* 3*nCol*nPhrase values */
+
+/*
+** The default value for the second argument to matchinfo().
+*/
+#define FTS3_MATCHINFO_DEFAULT "pcx"
+
/*
** Used as an fts3ExprIterate() context when loading phrase doclists to
@@ -115533,7 +117329,7 @@
*/
typedef struct LoadDoclistCtx LoadDoclistCtx;
struct LoadDoclistCtx {
- Fts3Table *pTab; /* FTS3 Table */
+ Fts3Cursor *pCsr; /* FTS3 Cursor */
int nPhrase; /* Number of phrases seen so far */
int nToken; /* Number of tokens seen so far */
};
@@ -115579,6 +117375,8 @@
struct MatchInfo {
Fts3Cursor *pCursor; /* FTS3 Cursor */
int nCol; /* Number of columns in table */
+ int nPhrase; /* Number of matchable phrases in query */
+ sqlite3_int64 nDoc; /* Number of docs in database */
u32 *aMatchinfo; /* Pre-allocated buffer */
};
@@ -115727,7 +117525,7 @@
p->nToken += pExpr->pPhrase->nToken;
if( pExpr->isLoaded==0 ){
- rc = sqlite3Fts3ExprLoadDoclist(p->pTab, pExpr);
+ rc = sqlite3Fts3ExprLoadDoclist(p->pCsr, pExpr);
pExpr->isLoaded = 1;
if( rc==SQLITE_OK ){
rc = fts3ExprNearTrim(pExpr);
@@ -115770,7 +117568,7 @@
){
int rc; /* Return Code */
LoadDoclistCtx sCtx = {0,0,0}; /* Context for fts3ExprIterate() */
- sCtx.pTab = (Fts3Table *)pCsr->base.pVtab;
+ sCtx.pCsr = pCsr;
rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb1, (void *)&sCtx);
if( rc==SQLITE_OK ){
(void)fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb2, 0);
@@ -115780,6 +117578,18 @@
return rc;
}
+static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
+ (*(int *)ctx)++;
+ UNUSED_PARAMETER(pExpr);
+ UNUSED_PARAMETER(iPhrase);
+ return SQLITE_OK;
+}
+static int fts3ExprPhraseCount(Fts3Expr *pExpr){
+ int nPhrase = 0;
+ (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase);
+ return nPhrase;
+}
+
/*
** Advance the position list iterator specified by the first two
** arguments so that it points to the first element with a value greater
@@ -116292,38 +118102,87 @@
/*
** fts3ExprIterate() callback used to collect the "global" matchinfo stats
-** for a single query. The "global" stats are those elements of the matchinfo
-** array that are constant for all rows returned by the current query.
+** for a single query.
+**
+** fts3ExprIterate() callback to load the 'global' elements of a
+** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements
+** of the matchinfo array that are constant for all rows returned by the
+** current query.
+**
+** Argument pCtx is actually a pointer to a struct of type MatchInfo. This
+** function populates Matchinfo.aMatchinfo[] as follows:
+**
+** for(iCol=0; iCol<nCol; iCol++){
+** aMatchinfo[3*iPhrase*nCol + 3*iCol + 1] = X;
+** aMatchinfo[3*iPhrase*nCol + 3*iCol + 2] = Y;
+** }
+**
+** where X is the number of matches for phrase iPhrase is column iCol of all
+** rows of the table. Y is the number of rows for which column iCol contains
+** at least one instance of phrase iPhrase.
+**
+** If the phrase pExpr consists entirely of deferred tokens, then all X and
+** Y values are set to nDoc, where nDoc is the number of documents in the
+** file system. This is done because the full-text index doclist is required
+** to calculate these values properly, and the full-text index doclist is
+** not available for deferred tokens.
*/
-static int fts3ExprGlobalMatchinfoCb(
+static int fts3ExprGlobalHitsCb(
Fts3Expr *pExpr, /* Phrase expression node */
int iPhrase, /* Phrase number (numbered from zero) */
void *pCtx /* Pointer to MatchInfo structure */
){
MatchInfo *p = (MatchInfo *)pCtx;
- char *pCsr;
+ Fts3Cursor *pCsr = p->pCursor;
+ char *pIter;
char *pEnd;
- const int iStart = 2 + (iPhrase * p->nCol * 3) + 1;
+ char *pFree = 0;
+ u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol];
assert( pExpr->isLoaded );
+ assert( pExpr->eType==FTSQUERY_PHRASE );
- /* Fill in the global hit count matrix row for this phrase. */
- pCsr = pExpr->aDoclist;
- pEnd = &pExpr->aDoclist[pExpr->nDoclist];
- while( pCsr<pEnd ){
- while( *pCsr++ & 0x80 ); /* Skip past docid. */
- fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 1);
+ if( pCsr->pDeferred ){
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
+ int ii;
+ for(ii=0; ii<pPhrase->nToken; ii++){
+ if( pPhrase->aToken[ii].bFulltext ) break;
+ }
+ if( ii<pPhrase->nToken ){
+ int nFree = 0;
+ int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree);
+ if( rc!=SQLITE_OK ) return rc;
+ pIter = pFree;
+ pEnd = &pFree[nFree];
+ }else{
+ int iCol; /* Column index */
+ for(iCol=0; iCol<p->nCol; iCol++){
+ aOut[iCol*3 + 1] = (u32)p->nDoc;
+ aOut[iCol*3 + 2] = (u32)p->nDoc;
+ }
+ return SQLITE_OK;
+ }
+ }else{
+ pIter = pExpr->aDoclist;
+ pEnd = &pExpr->aDoclist[pExpr->nDoclist];
}
+ /* Fill in the global hit count matrix row for this phrase. */
+ while( pIter<pEnd ){
+ while( *pIter++ & 0x80 ); /* Skip past docid. */
+ fts3LoadColumnlistCounts(&pIter, &aOut[1], 1);
+ }
+
+ sqlite3_free(pFree);
return SQLITE_OK;
}
/*
-** fts3ExprIterate() callback used to collect the "local" matchinfo stats
-** for a single query. The "local" stats are those elements of the matchinfo
+** fts3ExprIterate() callback used to collect the "local" part of the
+** FTS3_MATCHINFO_HITS array. The local stats are those elements of the
** array that are different for each row returned by the query.
*/
-static int fts3ExprLocalMatchinfoCb(
+static int fts3ExprLocalHitsCb(
Fts3Expr *pExpr, /* Phrase expression node */
int iPhrase, /* Phrase number */
void *pCtx /* Pointer to MatchInfo structure */
@@ -116332,7 +118191,7 @@
if( pExpr->aDoclist ){
char *pCsr;
- int iStart = 2 + (iPhrase * p->nCol * 3);
+ int iStart = iPhrase * p->nCol * 3;
int i;
for(i=0; i<p->nCol; i++) p->aMatchinfo[iStart+i*3] = 0;
@@ -116346,67 +118205,399 @@
return SQLITE_OK;
}
+static int fts3MatchinfoCheck(
+ Fts3Table *pTab,
+ char cArg,
+ char **pzErr
+){
+ if( (cArg==FTS3_MATCHINFO_NPHRASE)
+ || (cArg==FTS3_MATCHINFO_NCOL)
+ || (cArg==FTS3_MATCHINFO_NDOC && pTab->bHasStat)
+ || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bHasStat)
+ || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize)
+ || (cArg==FTS3_MATCHINFO_LCS)
+ || (cArg==FTS3_MATCHINFO_HITS)
+ ){
+ return SQLITE_OK;
+ }
+ *pzErr = sqlite3_mprintf("unrecognized matchinfo request: %c", cArg);
+ return SQLITE_ERROR;
+}
+
+static int fts3MatchinfoSize(MatchInfo *pInfo, char cArg){
+ int nVal; /* Number of integers output by cArg */
+
+ switch( cArg ){
+ case FTS3_MATCHINFO_NDOC:
+ case FTS3_MATCHINFO_NPHRASE:
+ case FTS3_MATCHINFO_NCOL:
+ nVal = 1;
+ break;
+
+ case FTS3_MATCHINFO_AVGLENGTH:
+ case FTS3_MATCHINFO_LENGTH:
+ case FTS3_MATCHINFO_LCS:
+ nVal = pInfo->nCol;
+ break;
+
+ default:
+ assert( cArg==FTS3_MATCHINFO_HITS );
+ nVal = pInfo->nCol * pInfo->nPhrase * 3;
+ break;
+ }
+
+ return nVal;
+}
+
+static int fts3MatchinfoSelectDoctotal(
+ Fts3Table *pTab,
+ sqlite3_stmt **ppStmt,
+ sqlite3_int64 *pnDoc,
+ const char **paLen
+){
+ sqlite3_stmt *pStmt;
+ const char *a;
+ sqlite3_int64 nDoc;
+
+ if( !*ppStmt ){
+ int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt);
+ if( rc!=SQLITE_OK ) return rc;
+ }
+ pStmt = *ppStmt;
+
+ a = sqlite3_column_blob(pStmt, 0);
+ a += sqlite3Fts3GetVarint(a, &nDoc);
+ *pnDoc = (u32)nDoc;
+
+ if( paLen ) *paLen = a;
+ return SQLITE_OK;
+}
+
+/*
+** An instance of the following structure is used to store state while
+** iterating through a multi-column position-list corresponding to the
+** hits for a single phrase on a single row in order to calculate the
+** values for a matchinfo() FTS3_MATCHINFO_LCS request.
+*/
+typedef struct LcsIterator LcsIterator;
+struct LcsIterator {
+ Fts3Expr *pExpr; /* Pointer to phrase expression */
+ char *pRead; /* Cursor used to iterate through aDoclist */
+ int iPosOffset; /* Tokens count up to end of this phrase */
+ int iCol; /* Current column number */
+ int iPos; /* Current position */
+};
+
+/*
+** If LcsIterator.iCol is set to the following value, the iterator has
+** finished iterating through all offsets for all columns.
+*/
+#define LCS_ITERATOR_FINISHED 0x7FFFFFFF;
+
+static int fts3MatchinfoLcsCb(
+ Fts3Expr *pExpr, /* Phrase expression node */
+ int iPhrase, /* Phrase number (numbered from zero) */
+ void *pCtx /* Pointer to MatchInfo structure */
+){
+ LcsIterator *aIter = (LcsIterator *)pCtx;
+ aIter[iPhrase].pExpr = pExpr;
+ return SQLITE_OK;
+}
+
+/*
+** Advance the iterator passed as an argument to the next position. Return
+** 1 if the iterator is at EOF or if it now points to the start of the
+** position list for the next column.
+*/
+static int fts3LcsIteratorAdvance(LcsIterator *pIter){
+ char *pRead = pIter->pRead;
+ sqlite3_int64 iRead;
+ int rc = 0;
+
+ pRead += sqlite3Fts3GetVarint(pRead, &iRead);
+ if( iRead==0 ){
+ pIter->iCol = LCS_ITERATOR_FINISHED;
+ rc = 1;
+ }else{
+ if( iRead==1 ){
+ pRead += sqlite3Fts3GetVarint(pRead, &iRead);
+ pIter->iCol = (int)iRead;
+ pIter->iPos = pIter->iPosOffset;
+ pRead += sqlite3Fts3GetVarint(pRead, &iRead);
+ rc = 1;
+ }
+ pIter->iPos += (int)(iRead-2);
+ }
+
+ pIter->pRead = pRead;
+ return rc;
+}
+
+/*
+** This function implements the FTS3_MATCHINFO_LCS matchinfo() flag.
+**
+** If the call is successful, the longest-common-substring lengths for each
+** column are written into the first nCol elements of the pInfo->aMatchinfo[]
+** array before returning. SQLITE_OK is returned in this case.
+**
+** Otherwise, if an error occurs, an SQLite error code is returned and the
+** data written to the first nCol elements of pInfo->aMatchinfo[] is
+** undefined.
+*/
+static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){
+ LcsIterator *aIter;
+ int i;
+ int iCol;
+ int nToken = 0;
+
+ /* Allocate and populate the array of LcsIterator objects. The array
+ ** contains one element for each matchable phrase in the query.
+ **/
+ aIter = sqlite3_malloc(sizeof(LcsIterator) * pCsr->nPhrase);
+ if( !aIter ) return SQLITE_NOMEM;
+ memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase);
+ (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);
+ for(i=0; i<pInfo->nPhrase; i++){
+ LcsIterator *pIter = &aIter[i];
+ nToken -= pIter->pExpr->pPhrase->nToken;
+ pIter->iPosOffset = nToken;
+ pIter->pRead = sqlite3Fts3FindPositions(pIter->pExpr, pCsr->iPrevId, -1);
+ if( pIter->pRead ){
+ pIter->iPos = pIter->iPosOffset;
+ fts3LcsIteratorAdvance(&aIter[i]);
+ }else{
+ pIter->iCol = LCS_ITERATOR_FINISHED;
+ }
+ }
+
+ for(iCol=0; iCol<pInfo->nCol; iCol++){
+ int nLcs = 0; /* LCS value for this column */
+ int nLive = 0; /* Number of iterators in aIter not at EOF */
+
+ /* Loop through the iterators in aIter[]. Set nLive to the number of
+ ** iterators that point to a position-list corresponding to column iCol.
+ */
+ for(i=0; i<pInfo->nPhrase; i++){
+ assert( aIter[i].iCol>=iCol );
+ if( aIter[i].iCol==iCol ) nLive++;
+ }
+
+ /* The following loop runs until all iterators in aIter[] have finished
+ ** iterating through positions in column iCol. Exactly one of the
+ ** iterators is advanced each time the body of the loop is run.
+ */
+ while( nLive>0 ){
+ LcsIterator *pAdv = 0; /* The iterator to advance by one position */
+ int nThisLcs = 0; /* LCS for the current iterator positions */
+
+ for(i=0; i<pInfo->nPhrase; i++){
+ LcsIterator *pIter = &aIter[i];
+ if( iCol!=pIter->iCol ){
+ /* This iterator is already at EOF for this column. */
+ nThisLcs = 0;
+ }else{
+ if( pAdv==0 || pIter->iPos<pAdv->iPos ){
+ pAdv = pIter;
+ }
+ if( nThisLcs==0 || pIter->iPos==pIter[-1].iPos ){
+ nThisLcs++;
+ }else{
+ nThisLcs = 1;
+ }
+ if( nThisLcs>nLcs ) nLcs = nThisLcs;
+ }
+ }
+ if( fts3LcsIteratorAdvance(pAdv) ) nLive--;
+ }
+
+ pInfo->aMatchinfo[iCol] = nLcs;
+ }
+
+ sqlite3_free(aIter);
+ return SQLITE_OK;
+}
+
+/*
+** Populate the buffer pInfo->aMatchinfo[] with an array of integers to
+** be returned by the matchinfo() function. Argument zArg contains the
+** format string passed as the second argument to matchinfo (or the
+** default value "pcx" if no second argument was specified). The format
+** string has already been validated and the pInfo->aMatchinfo[] array
+** is guaranteed to be large enough for the output.
+**
+** If bGlobal is true, then populate all fields of the matchinfo() output.
+** If it is false, then assume that those fields that do not change between
+** rows (i.e. FTS3_MATCHINFO_NPHRASE, NCOL, NDOC, AVGLENGTH and part of HITS)
+** have already been populated.
+**
+** Return SQLITE_OK if successful, or an SQLite error code if an error
+** occurs. If a value other than SQLITE_OK is returned, the state the
+** pInfo->aMatchinfo[] buffer is left in is undefined.
+*/
+static int fts3MatchinfoValues(
+ Fts3Cursor *pCsr, /* FTS3 cursor object */
+ int bGlobal, /* True to grab the global stats */
+ MatchInfo *pInfo, /* Matchinfo context object */
+ const char *zArg /* Matchinfo format string */
+){
+ int rc = SQLITE_OK;
+ int i;
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
+ sqlite3_stmt *pSelect = 0;
+
+ for(i=0; rc==SQLITE_OK && zArg[i]; i++){
+
+ switch( zArg[i] ){
+ case FTS3_MATCHINFO_NPHRASE:
+ if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase;
+ break;
+
+ case FTS3_MATCHINFO_NCOL:
+ if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nCol;
+ break;
+
+ case FTS3_MATCHINFO_NDOC:
+ if( bGlobal ){
+ sqlite3_int64 nDoc;
+ rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0);
+ pInfo->aMatchinfo[0] = (u32)nDoc;
+ }
+ break;
+
+ case FTS3_MATCHINFO_AVGLENGTH:
+ if( bGlobal ){
+ sqlite3_int64 nDoc; /* Number of rows in table */
+ const char *a; /* Aggregate column length array */
+
+ rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a);
+ if( rc==SQLITE_OK ){
+ int iCol;
+ for(iCol=0; iCol<pInfo->nCol; iCol++){
+ sqlite3_int64 nToken;
+ a += sqlite3Fts3GetVarint(a, &nToken);
+ pInfo->aMatchinfo[iCol] = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc);
+ }
+ }
+ }
+ break;
+
+ case FTS3_MATCHINFO_LENGTH: {
+ sqlite3_stmt *pSelectDocsize = 0;
+ rc = sqlite3Fts3SelectDocsize(pTab, pCsr->iPrevId, &pSelectDocsize);
+ if( rc==SQLITE_OK ){
+ int iCol;
+ const char *a = sqlite3_column_blob(pSelectDocsize, 0);
+ for(iCol=0; iCol<pInfo->nCol; iCol++){
+ sqlite3_int64 nToken;
+ a += sqlite3Fts3GetVarint(a, &nToken);
+ pInfo->aMatchinfo[iCol] = (u32)nToken;
+ }
+ }
+ sqlite3_reset(pSelectDocsize);
+ break;
+ }
+
+ case FTS3_MATCHINFO_LCS:
+ rc = fts3ExprLoadDoclists(pCsr, 0, 0);
+ if( rc==SQLITE_OK ){
+ rc = fts3MatchinfoLcs(pCsr, pInfo);
+ }
+ break;
+
+ default: {
+ Fts3Expr *pExpr;
+ assert( zArg[i]==FTS3_MATCHINFO_HITS );
+ pExpr = pCsr->pExpr;
+ rc = fts3ExprLoadDoclists(pCsr, 0, 0);
+ if( rc!=SQLITE_OK ) break;
+ if( bGlobal ){
+ if( pCsr->pDeferred ){
+ rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc, 0);
+ if( rc!=SQLITE_OK ) break;
+ }
+ rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo);
+ if( rc!=SQLITE_OK ) break;
+ }
+ (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo);
+ break;
+ }
+ }
+
+ pInfo->aMatchinfo += fts3MatchinfoSize(pInfo, zArg[i]);
+ }
+
+ sqlite3_reset(pSelect);
+ return rc;
+}
+
+
/*
** Populate pCsr->aMatchinfo[] with data for the current row. The
** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32).
*/
-static int fts3GetMatchinfo(Fts3Cursor *pCsr){
+static int fts3GetMatchinfo(
+ Fts3Cursor *pCsr, /* FTS3 Cursor object */
+ const char *zArg /* Second argument to matchinfo() function */
+){
MatchInfo sInfo;
Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
int rc = SQLITE_OK;
+ int bGlobal = 0; /* Collect 'global' stats as well as local */
+ memset(&sInfo, 0, sizeof(MatchInfo));
sInfo.pCursor = pCsr;
sInfo.nCol = pTab->nColumn;
+ /* If there is cached matchinfo() data, but the format string for the
+ ** cache does not match the format string for this request, discard
+ ** the cached data. */
+ if( pCsr->zMatchinfo && strcmp(pCsr->zMatchinfo, zArg) ){
+ assert( pCsr->aMatchinfo );
+ sqlite3_free(pCsr->aMatchinfo);
+ pCsr->zMatchinfo = 0;
+ pCsr->aMatchinfo = 0;
+ }
+
+ /* If Fts3Cursor.aMatchinfo[] is NULL, then this is the first time the
+ ** matchinfo function has been called for this query. In this case
+ ** allocate the array used to accumulate the matchinfo data and
+ ** initialize those elements that are constant for every row.
+ */
if( pCsr->aMatchinfo==0 ){
- /* If Fts3Cursor.aMatchinfo[] is NULL, then this is the first time the
- ** matchinfo function has been called for this query. In this case
- ** allocate the array used to accumulate the matchinfo data and
- ** initialize those elements that are constant for every row.
- */
- int nPhrase; /* Number of phrases */
- int nMatchinfo; /* Number of u32 elements in match-info */
+ int nMatchinfo = 0; /* Number of u32 elements in match-info */
+ int nArg; /* Bytes in zArg */
+ int i; /* Used to iterate through zArg */
- /* Load doclists for each phrase in the query. */
- rc = fts3ExprLoadDoclists(pCsr, &nPhrase, 0);
- if( rc!=SQLITE_OK ){
- return rc;
- }
- nMatchinfo = 2 + 3*sInfo.nCol*nPhrase;
- if( pTab->bHasDocsize ){
- nMatchinfo += 1 + 2*pTab->nColumn;
+ /* Determine the number of phrases in the query */
+ pCsr->nPhrase = fts3ExprPhraseCount(pCsr->pExpr);
+ sInfo.nPhrase = pCsr->nPhrase;
+
+ /* Determine the number of integers in the buffer returned by this call. */
+ for(i=0; zArg[i]; i++){
+ nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]);
}
- sInfo.aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo);
- if( !sInfo.aMatchinfo ){
- return SQLITE_NOMEM;
- }
- memset(sInfo.aMatchinfo, 0, sizeof(u32)*nMatchinfo);
+ /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */
+ nArg = (int)strlen(zArg);
+ pCsr->aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo + nArg + 1);
+ if( !pCsr->aMatchinfo ) return SQLITE_NOMEM;
-
- /* First element of match-info is the number of phrases in the query */
- sInfo.aMatchinfo[0] = nPhrase;
- sInfo.aMatchinfo[1] = sInfo.nCol;
- (void)fts3ExprIterate(pCsr->pExpr, fts3ExprGlobalMatchinfoCb,(void*)&sInfo);
- if( pTab->bHasDocsize ){
- int ofst = 2 + 3*sInfo.aMatchinfo[0]*sInfo.aMatchinfo[1];
- rc = sqlite3Fts3MatchinfoDocsizeGlobal(pCsr, &sInfo.aMatchinfo[ofst]);
- }
- pCsr->aMatchinfo = sInfo.aMatchinfo;
+ pCsr->zMatchinfo = (char *)&pCsr->aMatchinfo[nMatchinfo];
+ pCsr->nMatchinfo = nMatchinfo;
+ memcpy(pCsr->zMatchinfo, zArg, nArg+1);
+ memset(pCsr->aMatchinfo, 0, sizeof(u32)*nMatchinfo);
pCsr->isMatchinfoNeeded = 1;
+ bGlobal = 1;
}
sInfo.aMatchinfo = pCsr->aMatchinfo;
- if( rc==SQLITE_OK && pCsr->isMatchinfoNeeded ){
- (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLocalMatchinfoCb, (void*)&sInfo);
- if( pTab->bHasDocsize ){
- int ofst = 2 + 3*sInfo.aMatchinfo[0]*sInfo.aMatchinfo[1];
- rc = sqlite3Fts3MatchinfoDocsizeLocal(pCsr, &sInfo.aMatchinfo[ofst]);
- }
+ sInfo.nPhrase = pCsr->nPhrase;
+ if( pCsr->isMatchinfoNeeded ){
+ rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg);
pCsr->isMatchinfoNeeded = 0;
}
- return SQLITE_OK;
+ return rc;
}
/*
@@ -116467,7 +118658,7 @@
** columns of the FTS3 table. Otherwise, only column iCol is considered.
*/
for(iRead=0; iRead<pTab->nColumn; iRead++){
- SnippetFragment sF;
+ SnippetFragment sF = {0, 0, 0, 0};
int iS;
if( iCol>=0 && iRead!=iCol ) continue;
@@ -116501,6 +118692,7 @@
}
snippet_out:
+ sqlite3Fts3SegmentsClose(pTab);
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pCtx, rc);
sqlite3_free(res.z);
@@ -116680,6 +118872,7 @@
offsets_out:
sqlite3_free(sCtx.aTerm);
assert( rc!=SQLITE_DONE );
+ sqlite3Fts3SegmentsClose(pTab);
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pCtx, rc);
sqlite3_free(res.z);
@@ -116692,21 +118885,43 @@
/*
** Implementation of matchinfo() function.
*/
-SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *pContext, Fts3Cursor *pCsr){
+SQLITE_PRIVATE void sqlite3Fts3Matchinfo(
+ sqlite3_context *pContext, /* Function call context */
+ Fts3Cursor *pCsr, /* FTS3 table cursor */
+ const char *zArg /* Second arg to matchinfo() function */
+){
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
int rc;
+ int i;
+ const char *zFormat;
+
+ if( zArg ){
+ for(i=0; zArg[i]; i++){
+ char *zErr = 0;
+ if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){
+ sqlite3_result_error(pContext, zErr, -1);
+ sqlite3_free(zErr);
+ return;
+ }
+ }
+ zFormat = zArg;
+ }else{
+ zFormat = FTS3_MATCHINFO_DEFAULT;
+ }
+
if( !pCsr->pExpr ){
sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC);
return;
}
- rc = fts3GetMatchinfo(pCsr);
+
+ /* Retrieve matchinfo() data. */
+ rc = fts3GetMatchinfo(pCsr, zFormat);
+ sqlite3Fts3SegmentsClose(pTab);
+
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pContext, rc);
}else{
- Fts3Table *pTab = (Fts3Table*)pCsr->base.pVtab;
- int n = sizeof(u32)*(2+pCsr->aMatchinfo[0]*pCsr->aMatchinfo[1]*3);
- if( pTab->bHasDocsize ){
- n += sizeof(u32)*(1 + 2*pTab->nColumn);
- }
+ int n = pCsr->nMatchinfo * sizeof(u32);
sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT);
}
}
@@ -116826,6 +119041,12 @@
typedef unsigned int u32;
#endif
+/* The following macro is used to suppress compiler warnings.
+*/
+#ifndef UNUSED_PARAMETER
+# define UNUSED_PARAMETER(x) (void)(x)
+#endif
+
typedef struct Rtree Rtree;
typedef struct RtreeCursor RtreeCursor;
typedef struct RtreeNode RtreeNode;
@@ -117600,6 +119821,7 @@
RtreeCell cell;
int ii;
int bRes = 0;
+ int rc = SQLITE_OK;
nodeGetCell(pRtree, pCursor->pNode, pCursor->iCell, &cell);
for(ii=0; bRes==0 && ii<pCursor->nConstraint; ii++){
@@ -117625,12 +119847,8 @@
break;
default: {
- int rc;
assert( p->op==RTREE_MATCH );
rc = testRtreeGeom(pRtree, p, &cell, &bRes);
- if( rc!=SQLITE_OK ){
- return rc;
- }
bRes = !bRes;
break;
}
@@ -117638,7 +119856,7 @@
}
*pbEof = bRes;
- return SQLITE_OK;
+ return rc;
}
/*
@@ -117721,14 +119939,13 @@
rc = testRtreeCell(pRtree, pCursor, &isEof);
}
if( rc!=SQLITE_OK || isEof || iHeight==0 ){
- *pEof = isEof;
- return rc;
+ goto descend_to_cell_out;
}
iRowid = nodeGetRowid(pRtree, pCursor->pNode, pCursor->iCell);
rc = nodeAcquire(pRtree, iRowid, pCursor->pNode, &pChild);
if( rc!=SQLITE_OK ){
- return rc;
+ goto descend_to_cell_out;
}
nodeRelease(pRtree, pCursor->pNode);
@@ -117738,7 +119955,7 @@
pCursor->iCell = ii;
rc = descendToCell(pRtree, pCursor, iHeight-1, &isEof);
if( rc!=SQLITE_OK ){
- return rc;
+ goto descend_to_cell_out;
}
}
@@ -117750,8 +119967,9 @@
pCursor->iCell = iSavedCell;
}
+descend_to_cell_out:
*pEof = isEof;
- return SQLITE_OK;
+ return rc;
}
/*
@@ -117907,7 +120125,7 @@
/* Check that the blob is roughly the right size. */
nBlob = sqlite3_value_bytes(pValue);
- if( nBlob<sizeof(RtreeMatchArg)
+ if( nBlob<(int)sizeof(RtreeMatchArg)
|| ((nBlob-sizeof(RtreeMatchArg))%sizeof(double))!=0
){
return SQLITE_ERROR;
@@ -117922,7 +120140,7 @@
memcpy(p, sqlite3_value_blob(pValue), nBlob);
if( p->magic!=RTREE_GEOMETRY_MAGIC
- || nBlob!=(sizeof(RtreeMatchArg) + (p->nParam-1)*sizeof(double))
+ || nBlob!=(int)(sizeof(RtreeMatchArg) + (p->nParam-1)*sizeof(double))
){
sqlite3_free(pGeom);
return SQLITE_ERROR;
@@ -118068,6 +120286,7 @@
int iIdx = 0;
char zIdxStr[RTREE_MAX_DIMENSIONS*8+1];
memset(zIdxStr, 0, sizeof(zIdxStr));
+ UNUSED_PARAMETER(tab);
assert( pIdxInfo->idxStr==0 );
for(ii=0; ii<pIdxInfo->nConstraint; ii++){
@@ -118241,6 +120460,7 @@
if( ii!=iExclude )
#else
assert( iExclude==-1 );
+ UNUSED_PARAMETER(iExclude);
#endif
{
int jj;
@@ -119362,16 +121582,6 @@
return rc;
}
-#ifndef NDEBUG
-static int hashIsEmpty(Rtree *pRtree){
- int ii;
- for(ii=0; ii<HASHSIZE; ii++){
- assert( !pRtree->aHash[ii] );
- }
- return 1;
-}
-#endif
-
/*
** The xUpdate method for rtree module virtual tables.
*/
@@ -119837,6 +122047,7 @@
Rtree tree;
int ii;
+ UNUSED_PARAMETER(nArg);
memset(&node, 0, sizeof(RtreeNode));
memset(&tree, 0, sizeof(Rtree));
tree.nDim = sqlite3_value_int(apArg[0]);
@@ -119870,6 +122081,7 @@
}
static void rtreedepth(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){
+ UNUSED_PARAMETER(nArg);
if( sqlite3_value_type(apArg[0])!=SQLITE_BLOB
|| sqlite3_value_bytes(apArg[0])<2
){
@@ -119891,7 +122103,6 @@
rc = sqlite3_create_function(db, "rtreenode", 2, utf8, 0, rtreenode, 0, 0);
if( rc==SQLITE_OK ){
- int utf8 = SQLITE_UTF8;
rc = sqlite3_create_function(db, "rtreedepth", 1, utf8, 0,rtreedepth, 0, 0);
}
if( rc==SQLITE_OK ){
diff --git a/dist/sqlite3.h b/dist/sqlite3.h
index ba51372..ac397da 100644
--- a/dist/sqlite3.h
+++ b/dist/sqlite3.h
@@ -107,9 +107,9 @@
** [sqlite3_libversion_number()], [sqlite3_sourceid()],
** [sqlite_version()] and [sqlite_source_id()].
*/
-#define SQLITE_VERSION "3.7.3"
-#define SQLITE_VERSION_NUMBER 3007003
-#define SQLITE_SOURCE_ID "2010-09-28 07:24:50 88108fff672b6012ffe87ba4b1efe6d5a07037d1"
+#define SQLITE_VERSION "3.7.4"
+#define SQLITE_VERSION_NUMBER 3007004
+#define SQLITE_SOURCE_ID "2010-12-07 20:14:09 a586a4deeb25330037a49df295b36aaf624d0f45"
/*
** CAPI3REF: Run-Time Library Version Numbers
@@ -545,6 +545,18 @@
** equal SQLITE_SYNC_NORMAL, that means to use normal fsync() semantics.
** If the lower four bits equal SQLITE_SYNC_FULL, that means
** to use Mac OS X style fullsync instead of fsync().
+**
+** Do not confuse the SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL flags
+** with the [PRAGMA synchronous]=NORMAL and [PRAGMA synchronous]=FULL
+** settings. The [synchronous pragma] determines when calls to the
+** xSync VFS method occur and applies uniformly across all platforms.
+** The SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL flags determine how
+** energetic or rigorous or forceful the sync operations are and
+** only make a difference on Mac OSX for the default SQLite code.
+** (Third-party VFS implementations might also make the distinction
+** between SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL, but among the
+** operating systems natively supported by SQLite, only Mac OSX
+** cares about the difference.)
*/
#define SQLITE_SYNC_NORMAL 0x00002
#define SQLITE_SYNC_FULL 0x00003
@@ -713,6 +725,8 @@
#define SQLITE_LAST_ERRNO 4
#define SQLITE_FCNTL_SIZE_HINT 5
#define SQLITE_FCNTL_CHUNK_SIZE 6
+#define SQLITE_FCNTL_FILE_POINTER 7
+
/*
** CAPI3REF: Mutex Handle
@@ -1739,6 +1753,9 @@
/*
** CAPI3REF: Convenience Routines For Running Queries
**
+** This is a legacy interface that is preserved for backwards compatibility.
+** Use of this interface is not recommended.
+**
** Definition: A <b>result table</b> is memory data structure created by the
** [sqlite3_get_table()] interface. A result table records the
** complete query results from one or more queries.
@@ -1759,7 +1776,7 @@
** It is not safe to pass a result table directly to [sqlite3_free()].
** A result table should be deallocated using [sqlite3_free_table()].
**
-** As an example of the result table format, suppose a query result
+** ^(As an example of the result table format, suppose a query result
** is as follows:
**
** <blockquote><pre>
@@ -1783,7 +1800,7 @@
** azResult[5] = "28";
** azResult[6] = "Cindy";
** azResult[7] = "21";
-** </pre></blockquote>
+** </pre></blockquote>)^
**
** ^The sqlite3_get_table() function evaluates one or more
** semicolon-separated SQL statements in the zero-terminated UTF-8
@@ -1791,19 +1808,19 @@
** pointer given in its 3rd parameter.
**
** After the application has finished with the result from sqlite3_get_table(),
-** it should pass the result table pointer to sqlite3_free_table() in order to
+** it must pass the result table pointer to sqlite3_free_table() in order to
** release the memory that was malloced. Because of the way the
** [sqlite3_malloc()] happens within sqlite3_get_table(), the calling
** function must not try to call [sqlite3_free()] directly. Only
** [sqlite3_free_table()] is able to release the memory properly and safely.
**
-** ^(The sqlite3_get_table() interface is implemented as a wrapper around
+** The sqlite3_get_table() interface is implemented as a wrapper around
** [sqlite3_exec()]. The sqlite3_get_table() routine does not have access
** to any internal data structures of SQLite. It uses only the public
** interface defined here. As a consequence, errors that occur in the
** wrapper layer outside of the internal [sqlite3_exec()] call are not
** reflected in subsequent calls to [sqlite3_errcode()] or
-** [sqlite3_errmsg()].)^
+** [sqlite3_errmsg()].
*/
SQLITE_API int sqlite3_get_table(
sqlite3 *db, /* An open database */
@@ -2634,6 +2651,20 @@
SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt);
/*
+** CAPI3REF: Determine If An SQL Statement Writes The Database
+**
+** ^The sqlite3_stmt_readonly(X) interface returns true (non-zero) if
+** the [prepared statement] X is [SELECT] statement and false (zero) if
+** X is an [INSERT], [UPDATE], [DELETE], CREATE, DROP, [ANALYZE],
+** [ALTER], or [REINDEX] statement.
+** If X is a NULL pointer or any other kind of statement, including but
+** not limited to [ATTACH], [DETACH], [COMMIT], [ROLLBACK], [RELEASE],
+** [SAVEPOINT], [PRAGMA], or [VACUUM] the result of sqlite3_stmt_readonly(X) is
+** undefined.
+*/
+SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt);
+
+/*
** CAPI3REF: Dynamically Typed Value Object
** KEYWORDS: {protected sqlite3_value} {unprotected sqlite3_value}
**
@@ -2732,7 +2763,10 @@
**
** ^The fifth argument to sqlite3_bind_blob(), sqlite3_bind_text(), and
** sqlite3_bind_text16() is a destructor used to dispose of the BLOB or
-** string after SQLite has finished with it. ^If the fifth argument is
+** string after SQLite has finished with it. ^The destructor is called
+** to dispose of the BLOB or string even if the call to sqlite3_bind_blob(),
+** sqlite3_bind_text(), or sqlite3_bind_text16() fails.
+** ^If the fifth argument is
** the special value [SQLITE_STATIC], then SQLite assumes that the
** information is in static, unmanaged space and does not need to be freed.
** ^If the fifth argument has the value [SQLITE_TRANSIENT], then
@@ -3265,17 +3299,26 @@
** CAPI3REF: Destroy A Prepared Statement Object
**
** ^The sqlite3_finalize() function is called to delete a [prepared statement].
-** ^If the statement was executed successfully or not executed at all, then
-** SQLITE_OK is returned. ^If execution of the statement failed then an
-** [error code] or [extended error code] is returned.
+** ^If the most recent evaluation of the statement encountered no errors or
+** or if the statement is never been evaluated, then sqlite3_finalize() returns
+** SQLITE_OK. ^If the most recent evaluation of statement S failed, then
+** sqlite3_finalize(S) returns the appropriate [error code] or
+** [extended error code].
**
-** ^This routine can be called at any point during the execution of the
-** [prepared statement]. ^If the virtual machine has not
-** completed execution when this routine is called, that is like
-** encountering an error or an [sqlite3_interrupt | interrupt].
-** ^Incomplete updates may be rolled back and transactions canceled,
-** depending on the circumstances, and the
-** [error code] returned will be [SQLITE_ABORT].
+** ^The sqlite3_finalize(S) routine can be called at any point during
+** the life cycle of [prepared statement] S:
+** before statement S is ever evaluated, after
+** one or more calls to [sqlite3_reset()], or after any call
+** to [sqlite3_step()] regardless of whether or not the statement has
+** completed execution.
+**
+** ^Invoking sqlite3_finalize() on a NULL pointer is a harmless no-op.
+**
+** The application must finalize every [prepared statement] in order to avoid
+** resource leaks. It is a grievous error for the application to try to use
+** a prepared statement after it has been finalized. Any use of a prepared
+** statement after it has been finalized can result in undefined and
+** undesirable behavior such as segfaults and heap corruption.
*/
SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt);
@@ -3363,12 +3406,15 @@
** SQL function or aggregate, pass NULL poiners for all three function
** callbacks.
**
-** ^If the tenth parameter to sqlite3_create_function_v2() is not NULL,
-** then it is invoked when the function is deleted, either by being
-** overloaded or when the database connection closes.
-** ^When the destructure callback of the tenth parameter is invoked, it
-** is passed a single argument which is a copy of the pointer which was
-** the fifth parameter to sqlite3_create_function_v2().
+** ^(If the tenth parameter to sqlite3_create_function_v2() is not NULL,
+** then it is destructor for the application data pointer.
+** The destructor is invoked when the function is deleted, either by being
+** overloaded or when the database connection closes.)^
+** ^The destructor is also invoked if the call to
+** sqlite3_create_function_v2() fails.
+** ^When the destructor callback of the tenth parameter is invoked, it
+** is passed a single argument which is a copy of the application data
+** pointer which was the fifth parameter to sqlite3_create_function_v2().
**
** ^It is permitted to register multiple implementations of the same
** functions with the same name but with either differing numbers of
@@ -3832,6 +3878,15 @@
** calls to the collation creation functions or when the
** [database connection] is closed using [sqlite3_close()].
**
+** ^The xDestroy callback is <u>not</u> called if the
+** sqlite3_create_collation_v2() function fails. Applications that invoke
+** sqlite3_create_collation_v2() with a non-NULL xDestroy argument should
+** check the return code and dispose of the application data pointer
+** themselves rather than expecting SQLite to deal with it for them.
+** This is different from every other SQLite interface. The inconsistency
+** is unfortunate but cannot be changed without breaking backwards
+** compatibility.
+**
** See also: [sqlite3_collation_needed()] and [sqlite3_collation_needed16()].
*/
SQLITE_API int sqlite3_create_collation(
@@ -4586,7 +4641,9 @@
** ^The sqlite3_create_module_v2() interface has a fifth parameter which
** is a pointer to a destructor for the pClientData. ^SQLite will
** invoke the destructor function (if it is not NULL) when SQLite
-** no longer needs the pClientData pointer. ^The sqlite3_create_module()
+** no longer needs the pClientData pointer. ^The destructor will also
+** be invoked if the call to sqlite3_create_module_v2() fails.
+** ^The sqlite3_create_module()
** interface is equivalent to sqlite3_create_module_v2() with a NULL
** destructor.
*/
@@ -4770,6 +4827,30 @@
);
/*
+** CAPI3REF: Move a BLOB Handle to a New Row
+**
+** ^This function is used to move an existing blob handle so that it points
+** to a different row of the same database table. ^The new row is identified
+** by the rowid value passed as the second argument. Only the row can be
+** changed. ^The database, table and column on which the blob handle is open
+** remain the same. Moving an existing blob handle to a new row can be
+** faster than closing the existing handle and opening a new one.
+**
+** ^(The new row must meet the same criteria as for [sqlite3_blob_open()] -
+** it must exist and there must be either a blob or text value stored in
+** the nominated column.)^ ^If the new row is not present in the table, or if
+** it does not contain a blob or text value, or if another error occurs, an
+** SQLite error code is returned and the blob handle is considered aborted.
+** ^All subsequent calls to [sqlite3_blob_read()], [sqlite3_blob_write()] or
+** [sqlite3_blob_reopen()] on an aborted blob handle immediately return
+** SQLITE_ABORT. ^Calling [sqlite3_blob_bytes()] on an aborted blob handle
+** always returns zero.
+**
+** ^This function sets the database handle error code and message.
+*/
+SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64);
+
+/*
** CAPI3REF: Close A BLOB Handle
**
** ^Closes an open [BLOB handle].
@@ -5176,7 +5257,7 @@
** ^The [sqlite3_file_control()] interface makes a direct call to the
** xFileControl method for the [sqlite3_io_methods] object associated
** with a particular database identified by the second argument. ^The
-** name of the database "main" for the main database or "temp" for the
+** name of the database is "main" for the main database or "temp" for the
** TEMP database, or the name that appears after the AS keyword for
** databases that are added using the [ATTACH] SQL command.
** ^A NULL pointer can be used in place of "main" to refer to the
@@ -5186,6 +5267,12 @@
** the xFileControl method. ^The return value of the xFileControl
** method becomes the return value of this routine.
**
+** ^The SQLITE_FCNTL_FILE_POINTER value for the op parameter causes
+** a pointer to the underlying [sqlite3_file] object to be written into
+** the space pointed to by the 4th parameter. ^The SQLITE_FCNTL_FILE_POINTER
+** case is a short-circuit path which does not actually invoke the
+** underlying sqlite3_io_methods.xFileControl method.
+**
** ^If the second parameter (zDbName) does not match the name of any
** open database file, then SQLITE_ERROR is returned. ^This error
** code is not remembered and will not be recalled by [sqlite3_errcode()]
diff --git a/dist/sqlite3.h.orig b/dist/sqlite3.h.orig
index 9edccf0..47ef256 100644
--- a/dist/sqlite3.h.orig
+++ b/dist/sqlite3.h.orig
@@ -107,9 +107,9 @@
** [sqlite3_libversion_number()], [sqlite3_sourceid()],
** [sqlite_version()] and [sqlite_source_id()].
*/
-#define SQLITE_VERSION "3.7.3"
-#define SQLITE_VERSION_NUMBER 3007003
-#define SQLITE_SOURCE_ID "2010-09-28 07:24:50 88108fff672b6012ffe87ba4b1efe6d5a07037d1"
+#define SQLITE_VERSION "3.7.4"
+#define SQLITE_VERSION_NUMBER 3007004
+#define SQLITE_SOURCE_ID "2010-12-07 20:14:09 a586a4deeb25330037a49df295b36aaf624d0f45"
/*
** CAPI3REF: Run-Time Library Version Numbers
@@ -542,6 +542,18 @@
** equal SQLITE_SYNC_NORMAL, that means to use normal fsync() semantics.
** If the lower four bits equal SQLITE_SYNC_FULL, that means
** to use Mac OS X style fullsync instead of fsync().
+**
+** Do not confuse the SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL flags
+** with the [PRAGMA synchronous]=NORMAL and [PRAGMA synchronous]=FULL
+** settings. The [synchronous pragma] determines when calls to the
+** xSync VFS method occur and applies uniformly across all platforms.
+** The SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL flags determine how
+** energetic or rigorous or forceful the sync operations are and
+** only make a difference on Mac OSX for the default SQLite code.
+** (Third-party VFS implementations might also make the distinction
+** between SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL, but among the
+** operating systems natively supported by SQLite, only Mac OSX
+** cares about the difference.)
*/
#define SQLITE_SYNC_NORMAL 0x00002
#define SQLITE_SYNC_FULL 0x00003
@@ -710,6 +722,8 @@
#define SQLITE_LAST_ERRNO 4
#define SQLITE_FCNTL_SIZE_HINT 5
#define SQLITE_FCNTL_CHUNK_SIZE 6
+#define SQLITE_FCNTL_FILE_POINTER 7
+
/*
** CAPI3REF: Mutex Handle
@@ -1736,6 +1750,9 @@
/*
** CAPI3REF: Convenience Routines For Running Queries
**
+** This is a legacy interface that is preserved for backwards compatibility.
+** Use of this interface is not recommended.
+**
** Definition: A <b>result table</b> is memory data structure created by the
** [sqlite3_get_table()] interface. A result table records the
** complete query results from one or more queries.
@@ -1756,7 +1773,7 @@
** It is not safe to pass a result table directly to [sqlite3_free()].
** A result table should be deallocated using [sqlite3_free_table()].
**
-** As an example of the result table format, suppose a query result
+** ^(As an example of the result table format, suppose a query result
** is as follows:
**
** <blockquote><pre>
@@ -1780,7 +1797,7 @@
** azResult[5] = "28";
** azResult[6] = "Cindy";
** azResult[7] = "21";
-** </pre></blockquote>
+** </pre></blockquote>)^
**
** ^The sqlite3_get_table() function evaluates one or more
** semicolon-separated SQL statements in the zero-terminated UTF-8
@@ -1788,19 +1805,19 @@
** pointer given in its 3rd parameter.
**
** After the application has finished with the result from sqlite3_get_table(),
-** it should pass the result table pointer to sqlite3_free_table() in order to
+** it must pass the result table pointer to sqlite3_free_table() in order to
** release the memory that was malloced. Because of the way the
** [sqlite3_malloc()] happens within sqlite3_get_table(), the calling
** function must not try to call [sqlite3_free()] directly. Only
** [sqlite3_free_table()] is able to release the memory properly and safely.
**
-** ^(The sqlite3_get_table() interface is implemented as a wrapper around
+** The sqlite3_get_table() interface is implemented as a wrapper around
** [sqlite3_exec()]. The sqlite3_get_table() routine does not have access
** to any internal data structures of SQLite. It uses only the public
** interface defined here. As a consequence, errors that occur in the
** wrapper layer outside of the internal [sqlite3_exec()] call are not
** reflected in subsequent calls to [sqlite3_errcode()] or
-** [sqlite3_errmsg()].)^
+** [sqlite3_errmsg()].
*/
SQLITE_API int sqlite3_get_table(
sqlite3 *db, /* An open database */
@@ -2631,6 +2648,20 @@
SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt);
/*
+** CAPI3REF: Determine If An SQL Statement Writes The Database
+**
+** ^The sqlite3_stmt_readonly(X) interface returns true (non-zero) if
+** the [prepared statement] X is [SELECT] statement and false (zero) if
+** X is an [INSERT], [UPDATE], [DELETE], CREATE, DROP, [ANALYZE],
+** [ALTER], or [REINDEX] statement.
+** If X is a NULL pointer or any other kind of statement, including but
+** not limited to [ATTACH], [DETACH], [COMMIT], [ROLLBACK], [RELEASE],
+** [SAVEPOINT], [PRAGMA], or [VACUUM] the result of sqlite3_stmt_readonly(X) is
+** undefined.
+*/
+SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt);
+
+/*
** CAPI3REF: Dynamically Typed Value Object
** KEYWORDS: {protected sqlite3_value} {unprotected sqlite3_value}
**
@@ -2729,7 +2760,10 @@
**
** ^The fifth argument to sqlite3_bind_blob(), sqlite3_bind_text(), and
** sqlite3_bind_text16() is a destructor used to dispose of the BLOB or
-** string after SQLite has finished with it. ^If the fifth argument is
+** string after SQLite has finished with it. ^The destructor is called
+** to dispose of the BLOB or string even if the call to sqlite3_bind_blob(),
+** sqlite3_bind_text(), or sqlite3_bind_text16() fails.
+** ^If the fifth argument is
** the special value [SQLITE_STATIC], then SQLite assumes that the
** information is in static, unmanaged space and does not need to be freed.
** ^If the fifth argument has the value [SQLITE_TRANSIENT], then
@@ -3262,17 +3296,26 @@
** CAPI3REF: Destroy A Prepared Statement Object
**
** ^The sqlite3_finalize() function is called to delete a [prepared statement].
-** ^If the statement was executed successfully or not executed at all, then
-** SQLITE_OK is returned. ^If execution of the statement failed then an
-** [error code] or [extended error code] is returned.
+** ^If the most recent evaluation of the statement encountered no errors or
+** or if the statement is never been evaluated, then sqlite3_finalize() returns
+** SQLITE_OK. ^If the most recent evaluation of statement S failed, then
+** sqlite3_finalize(S) returns the appropriate [error code] or
+** [extended error code].
**
-** ^This routine can be called at any point during the execution of the
-** [prepared statement]. ^If the virtual machine has not
-** completed execution when this routine is called, that is like
-** encountering an error or an [sqlite3_interrupt | interrupt].
-** ^Incomplete updates may be rolled back and transactions canceled,
-** depending on the circumstances, and the
-** [error code] returned will be [SQLITE_ABORT].
+** ^The sqlite3_finalize(S) routine can be called at any point during
+** the life cycle of [prepared statement] S:
+** before statement S is ever evaluated, after
+** one or more calls to [sqlite3_reset()], or after any call
+** to [sqlite3_step()] regardless of whether or not the statement has
+** completed execution.
+**
+** ^Invoking sqlite3_finalize() on a NULL pointer is a harmless no-op.
+**
+** The application must finalize every [prepared statement] in order to avoid
+** resource leaks. It is a grievous error for the application to try to use
+** a prepared statement after it has been finalized. Any use of a prepared
+** statement after it has been finalized can result in undefined and
+** undesirable behavior such as segfaults and heap corruption.
*/
SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt);
@@ -3360,12 +3403,15 @@
** SQL function or aggregate, pass NULL poiners for all three function
** callbacks.
**
-** ^If the tenth parameter to sqlite3_create_function_v2() is not NULL,
-** then it is invoked when the function is deleted, either by being
-** overloaded or when the database connection closes.
-** ^When the destructure callback of the tenth parameter is invoked, it
-** is passed a single argument which is a copy of the pointer which was
-** the fifth parameter to sqlite3_create_function_v2().
+** ^(If the tenth parameter to sqlite3_create_function_v2() is not NULL,
+** then it is destructor for the application data pointer.
+** The destructor is invoked when the function is deleted, either by being
+** overloaded or when the database connection closes.)^
+** ^The destructor is also invoked if the call to
+** sqlite3_create_function_v2() fails.
+** ^When the destructor callback of the tenth parameter is invoked, it
+** is passed a single argument which is a copy of the application data
+** pointer which was the fifth parameter to sqlite3_create_function_v2().
**
** ^It is permitted to register multiple implementations of the same
** functions with the same name but with either differing numbers of
@@ -3829,6 +3875,15 @@
** calls to the collation creation functions or when the
** [database connection] is closed using [sqlite3_close()].
**
+** ^The xDestroy callback is <u>not</u> called if the
+** sqlite3_create_collation_v2() function fails. Applications that invoke
+** sqlite3_create_collation_v2() with a non-NULL xDestroy argument should
+** check the return code and dispose of the application data pointer
+** themselves rather than expecting SQLite to deal with it for them.
+** This is different from every other SQLite interface. The inconsistency
+** is unfortunate but cannot be changed without breaking backwards
+** compatibility.
+**
** See also: [sqlite3_collation_needed()] and [sqlite3_collation_needed16()].
*/
SQLITE_API int sqlite3_create_collation(
@@ -4583,7 +4638,9 @@
** ^The sqlite3_create_module_v2() interface has a fifth parameter which
** is a pointer to a destructor for the pClientData. ^SQLite will
** invoke the destructor function (if it is not NULL) when SQLite
-** no longer needs the pClientData pointer. ^The sqlite3_create_module()
+** no longer needs the pClientData pointer. ^The destructor will also
+** be invoked if the call to sqlite3_create_module_v2() fails.
+** ^The sqlite3_create_module()
** interface is equivalent to sqlite3_create_module_v2() with a NULL
** destructor.
*/
@@ -4767,6 +4824,30 @@
);
/*
+** CAPI3REF: Move a BLOB Handle to a New Row
+**
+** ^This function is used to move an existing blob handle so that it points
+** to a different row of the same database table. ^The new row is identified
+** by the rowid value passed as the second argument. Only the row can be
+** changed. ^The database, table and column on which the blob handle is open
+** remain the same. Moving an existing blob handle to a new row can be
+** faster than closing the existing handle and opening a new one.
+**
+** ^(The new row must meet the same criteria as for [sqlite3_blob_open()] -
+** it must exist and there must be either a blob or text value stored in
+** the nominated column.)^ ^If the new row is not present in the table, or if
+** it does not contain a blob or text value, or if another error occurs, an
+** SQLite error code is returned and the blob handle is considered aborted.
+** ^All subsequent calls to [sqlite3_blob_read()], [sqlite3_blob_write()] or
+** [sqlite3_blob_reopen()] on an aborted blob handle immediately return
+** SQLITE_ABORT. ^Calling [sqlite3_blob_bytes()] on an aborted blob handle
+** always returns zero.
+**
+** ^This function sets the database handle error code and message.
+*/
+SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64);
+
+/*
** CAPI3REF: Close A BLOB Handle
**
** ^Closes an open [BLOB handle].
@@ -5173,7 +5254,7 @@
** ^The [sqlite3_file_control()] interface makes a direct call to the
** xFileControl method for the [sqlite3_io_methods] object associated
** with a particular database identified by the second argument. ^The
-** name of the database "main" for the main database or "temp" for the
+** name of the database is "main" for the main database or "temp" for the
** TEMP database, or the name that appears after the AS keyword for
** databases that are added using the [ATTACH] SQL command.
** ^A NULL pointer can be used in place of "main" to refer to the
@@ -5183,6 +5264,12 @@
** the xFileControl method. ^The return value of the xFileControl
** method becomes the return value of this routine.
**
+** ^The SQLITE_FCNTL_FILE_POINTER value for the op parameter causes
+** a pointer to the underlying [sqlite3_file] object to be written into
+** the space pointed to by the 4th parameter. ^The SQLITE_FCNTL_FILE_POINTER
+** case is a short-circuit path which does not actually invoke the
+** underlying sqlite3_io_methods.xFileControl method.
+**
** ^If the second parameter (zDbName) does not match the name of any
** open database file, then SQLITE_ERROR is returned. ^This error
** code is not remembered and will not be recalled by [sqlite3_errcode()]
diff --git a/dist/version b/dist/version
index dca041c..a03fb9f 100644
--- a/dist/version
+++ b/dist/version
@@ -1,2 +1 @@
-3.6.22 Froyo branch http://www.sqlite.org/src/timeline?t=branch-3.6.22
-at checkin http://www.sqlite.org/src/info/82dd61fccf
+downloaded from http://www.sqlite.org/sqlite-amalgamation-3070400.zip
