000001 /*
000002 ** 2001 September 15
000003 **
000004 ** The author disclaims copyright to this source code. In place of
000005 ** a legal notice, here is a blessing:
000006 **
000007 ** May you do good and not evil.
000008 ** May you find forgiveness for yourself and forgive others.
000009 ** May you share freely, never taking more than you give.
000010 **
000011 *************************************************************************
000012 ** This module contains C code that generates VDBE code used to process
000013 ** the WHERE clause of SQL statements. This module is responsible for
000014 ** generating the code that loops through a table looking for applicable
000015 ** rows. Indices are selected and used to speed the search when doing
000016 ** so is applicable. Because this module is responsible for selecting
000017 ** indices, you might also think of this module as the "query optimizer".
000018 */
000019 #include "sqliteInt.h"
000020 #include "whereInt.h"
000021
000022 /*
000023 ** Extra information appended to the end of sqlite3_index_info but not
000024 ** visible to the xBestIndex function, at least not directly. The
000025 ** sqlite3_vtab_collation() interface knows how to reach it, however.
000026 **
000027 ** This object is not an API and can be changed from one release to the
000028 ** next. As long as allocateIndexInfo() and sqlite3_vtab_collation()
000029 ** agree on the structure, all will be well.
000030 */
000031 typedef struct HiddenIndexInfo HiddenIndexInfo;
000032 struct HiddenIndexInfo {
000033 WhereClause *pWC; /* The Where clause being analyzed */
000034 Parse *pParse; /* The parsing context */
000035 int eDistinct; /* Value to return from sqlite3_vtab_distinct() */
000036 u32 mIn; /* Mask of terms that are <col> IN (...) */
000037 u32 mHandleIn; /* Terms that vtab will handle as <col> IN (...) */
000038 sqlite3_value *aRhs[1]; /* RHS values for constraints. MUST BE LAST
000039 ** because extra space is allocated to hold up
000040 ** to nTerm such values */
000041 };
000042
000043 /* Forward declaration of methods */
000044 static int whereLoopResize(sqlite3*, WhereLoop*, int);
000045
000046 /*
000047 ** Return the estimated number of output rows from a WHERE clause
000048 */
000049 LogEst sqlite3WhereOutputRowCount(WhereInfo *pWInfo){
000050 return pWInfo->nRowOut;
000051 }
000052
000053 /*
000054 ** Return one of the WHERE_DISTINCT_xxxxx values to indicate how this
000055 ** WHERE clause returns outputs for DISTINCT processing.
000056 */
000057 int sqlite3WhereIsDistinct(WhereInfo *pWInfo){
000058 return pWInfo->eDistinct;
000059 }
000060
000061 /*
000062 ** Return the number of ORDER BY terms that are satisfied by the
000063 ** WHERE clause. A return of 0 means that the output must be
000064 ** completely sorted. A return equal to the number of ORDER BY
000065 ** terms means that no sorting is needed at all. A return that
000066 ** is positive but less than the number of ORDER BY terms means that
000067 ** block sorting is required.
000068 */
000069 int sqlite3WhereIsOrdered(WhereInfo *pWInfo){
000070 return pWInfo->nOBSat<0 ? 0 : pWInfo->nOBSat;
000071 }
000072
000073 /*
000074 ** In the ORDER BY LIMIT optimization, if the inner-most loop is known
000075 ** to emit rows in increasing order, and if the last row emitted by the
000076 ** inner-most loop did not fit within the sorter, then we can skip all
000077 ** subsequent rows for the current iteration of the inner loop (because they
000078 ** will not fit in the sorter either) and continue with the second inner
000079 ** loop - the loop immediately outside the inner-most.
000080 **
000081 ** When a row does not fit in the sorter (because the sorter already
000082 ** holds LIMIT+OFFSET rows that are smaller), then a jump is made to the
000083 ** label returned by this function.
000084 **
000085 ** If the ORDER BY LIMIT optimization applies, the jump destination should
000086 ** be the continuation for the second-inner-most loop. If the ORDER BY
000087 ** LIMIT optimization does not apply, then the jump destination should
000088 ** be the continuation for the inner-most loop.
000089 **
000090 ** It is always safe for this routine to return the continuation of the
000091 ** inner-most loop, in the sense that a correct answer will result.
000092 ** Returning the continuation the second inner loop is an optimization
000093 ** that might make the code run a little faster, but should not change
000094 ** the final answer.
000095 */
000096 int sqlite3WhereOrderByLimitOptLabel(WhereInfo *pWInfo){
000097 WhereLevel *pInner;
000098 if( !pWInfo->bOrderedInnerLoop ){
000099 /* The ORDER BY LIMIT optimization does not apply. Jump to the
000100 ** continuation of the inner-most loop. */
000101 return pWInfo->iContinue;
000102 }
000103 pInner = &pWInfo->a[pWInfo->nLevel-1];
000104 assert( pInner->addrNxt!=0 );
000105 return pInner->pRJ ? pWInfo->iContinue : pInner->addrNxt;
000106 }
000107
000108 /*
000109 ** While generating code for the min/max optimization, after handling
000110 ** the aggregate-step call to min() or max(), check to see if any
000111 ** additional looping is required. If the output order is such that
000112 ** we are certain that the correct answer has already been found, then
000113 ** code an OP_Goto to by pass subsequent processing.
000114 **
000115 ** Any extra OP_Goto that is coded here is an optimization. The
000116 ** correct answer should be obtained regardless. This OP_Goto just
000117 ** makes the answer appear faster.
000118 */
000119 void sqlite3WhereMinMaxOptEarlyOut(Vdbe *v, WhereInfo *pWInfo){
000120 WhereLevel *pInner;
000121 int i;
000122 if( !pWInfo->bOrderedInnerLoop ) return;
000123 if( pWInfo->nOBSat==0 ) return;
000124 for(i=pWInfo->nLevel-1; i>=0; i--){
000125 pInner = &pWInfo->a[i];
000126 if( (pInner->pWLoop->wsFlags & WHERE_COLUMN_IN)!=0 ){
000127 sqlite3VdbeGoto(v, pInner->addrNxt);
000128 return;
000129 }
000130 }
000131 sqlite3VdbeGoto(v, pWInfo->iBreak);
000132 }
000133
000134 /*
000135 ** Return the VDBE address or label to jump to in order to continue
000136 ** immediately with the next row of a WHERE clause.
000137 */
000138 int sqlite3WhereContinueLabel(WhereInfo *pWInfo){
000139 assert( pWInfo->iContinue!=0 );
000140 return pWInfo->iContinue;
000141 }
000142
000143 /*
000144 ** Return the VDBE address or label to jump to in order to break
000145 ** out of a WHERE loop.
000146 */
000147 int sqlite3WhereBreakLabel(WhereInfo *pWInfo){
000148 return pWInfo->iBreak;
000149 }
000150
000151 /*
000152 ** Return ONEPASS_OFF (0) if an UPDATE or DELETE statement is unable to
000153 ** operate directly on the rowids returned by a WHERE clause. Return
000154 ** ONEPASS_SINGLE (1) if the statement can operation directly because only
000155 ** a single row is to be changed. Return ONEPASS_MULTI (2) if the one-pass
000156 ** optimization can be used on multiple
000157 **
000158 ** If the ONEPASS optimization is used (if this routine returns true)
000159 ** then also write the indices of open cursors used by ONEPASS
000160 ** into aiCur[0] and aiCur[1]. iaCur[0] gets the cursor of the data
000161 ** table and iaCur[1] gets the cursor used by an auxiliary index.
000162 ** Either value may be -1, indicating that cursor is not used.
000163 ** Any cursors returned will have been opened for writing.
000164 **
000165 ** aiCur[0] and aiCur[1] both get -1 if the where-clause logic is
000166 ** unable to use the ONEPASS optimization.
000167 */
000168 int sqlite3WhereOkOnePass(WhereInfo *pWInfo, int *aiCur){
000169 memcpy(aiCur, pWInfo->aiCurOnePass, sizeof(int)*2);
000170 #ifdef WHERETRACE_ENABLED
000171 if( sqlite3WhereTrace && pWInfo->eOnePass!=ONEPASS_OFF ){
000172 sqlite3DebugPrintf("%s cursors: %d %d\n",
000173 pWInfo->eOnePass==ONEPASS_SINGLE ? "ONEPASS_SINGLE" : "ONEPASS_MULTI",
000174 aiCur[0], aiCur[1]);
000175 }
000176 #endif
000177 return pWInfo->eOnePass;
000178 }
000179
000180 /*
000181 ** Return TRUE if the WHERE loop uses the OP_DeferredSeek opcode to move
000182 ** the data cursor to the row selected by the index cursor.
000183 */
000184 int sqlite3WhereUsesDeferredSeek(WhereInfo *pWInfo){
000185 return pWInfo->bDeferredSeek;
000186 }
000187
000188 /*
000189 ** Move the content of pSrc into pDest
000190 */
000191 static void whereOrMove(WhereOrSet *pDest, WhereOrSet *pSrc){
000192 pDest->n = pSrc->n;
000193 memcpy(pDest->a, pSrc->a, pDest->n*sizeof(pDest->a[0]));
000194 }
000195
000196 /*
000197 ** Try to insert a new prerequisite/cost entry into the WhereOrSet pSet.
000198 **
000199 ** The new entry might overwrite an existing entry, or it might be
000200 ** appended, or it might be discarded. Do whatever is the right thing
000201 ** so that pSet keeps the N_OR_COST best entries seen so far.
000202 */
000203 static int whereOrInsert(
000204 WhereOrSet *pSet, /* The WhereOrSet to be updated */
000205 Bitmask prereq, /* Prerequisites of the new entry */
000206 LogEst rRun, /* Run-cost of the new entry */
000207 LogEst nOut /* Number of outputs for the new entry */
000208 ){
000209 u16 i;
000210 WhereOrCost *p;
000211 for(i=pSet->n, p=pSet->a; i>0; i--, p++){
000212 if( rRun<=p->rRun && (prereq & p->prereq)==prereq ){
000213 goto whereOrInsert_done;
000214 }
000215 if( p->rRun<=rRun && (p->prereq & prereq)==p->prereq ){
000216 return 0;
000217 }
000218 }
000219 if( pSet->n<N_OR_COST ){
000220 p = &pSet->a[pSet->n++];
000221 p->nOut = nOut;
000222 }else{
000223 p = pSet->a;
000224 for(i=1; i<pSet->n; i++){
000225 if( p->rRun>pSet->a[i].rRun ) p = pSet->a + i;
000226 }
000227 if( p->rRun<=rRun ) return 0;
000228 }
000229 whereOrInsert_done:
000230 p->prereq = prereq;
000231 p->rRun = rRun;
000232 if( p->nOut>nOut ) p->nOut = nOut;
000233 return 1;
000234 }
000235
000236 /*
000237 ** Return the bitmask for the given cursor number. Return 0 if
000238 ** iCursor is not in the set.
000239 */
000240 Bitmask sqlite3WhereGetMask(WhereMaskSet *pMaskSet, int iCursor){
000241 int i;
000242 assert( pMaskSet->n<=(int)sizeof(Bitmask)*8 );
000243 assert( pMaskSet->n>0 || pMaskSet->ix[0]<0 );
000244 assert( iCursor>=-1 );
000245 if( pMaskSet->ix[0]==iCursor ){
000246 return 1;
000247 }
000248 for(i=1; i<pMaskSet->n; i++){
000249 if( pMaskSet->ix[i]==iCursor ){
000250 return MASKBIT(i);
000251 }
000252 }
000253 return 0;
000254 }
000255
000256 /* Allocate memory that is automatically freed when pWInfo is freed.
000257 */
000258 void *sqlite3WhereMalloc(WhereInfo *pWInfo, u64 nByte){
000259 WhereMemBlock *pBlock;
000260 pBlock = sqlite3DbMallocRawNN(pWInfo->pParse->db, nByte+sizeof(*pBlock));
000261 if( pBlock ){
000262 pBlock->pNext = pWInfo->pMemToFree;
000263 pBlock->sz = nByte;
000264 pWInfo->pMemToFree = pBlock;
000265 pBlock++;
000266 }
000267 return (void*)pBlock;
000268 }
000269 void *sqlite3WhereRealloc(WhereInfo *pWInfo, void *pOld, u64 nByte){
000270 void *pNew = sqlite3WhereMalloc(pWInfo, nByte);
000271 if( pNew && pOld ){
000272 WhereMemBlock *pOldBlk = (WhereMemBlock*)pOld;
000273 pOldBlk--;
000274 assert( pOldBlk->sz<nByte );
000275 memcpy(pNew, pOld, pOldBlk->sz);
000276 }
000277 return pNew;
000278 }
000279
000280 /*
000281 ** Create a new mask for cursor iCursor.
000282 **
000283 ** There is one cursor per table in the FROM clause. The number of
000284 ** tables in the FROM clause is limited by a test early in the
000285 ** sqlite3WhereBegin() routine. So we know that the pMaskSet->ix[]
000286 ** array will never overflow.
000287 */
000288 static void createMask(WhereMaskSet *pMaskSet, int iCursor){
000289 assert( pMaskSet->n < ArraySize(pMaskSet->ix) );
000290 pMaskSet->ix[pMaskSet->n++] = iCursor;
000291 }
000292
000293 /*
000294 ** If the right-hand branch of the expression is a TK_COLUMN, then return
000295 ** a pointer to the right-hand branch. Otherwise, return NULL.
000296 */
000297 static Expr *whereRightSubexprIsColumn(Expr *p){
000298 p = sqlite3ExprSkipCollateAndLikely(p->pRight);
000299 if( ALWAYS(p!=0) && p->op==TK_COLUMN && !ExprHasProperty(p, EP_FixedCol) ){
000300 return p;
000301 }
000302 return 0;
000303 }
000304
000305 /*
000306 ** Advance to the next WhereTerm that matches according to the criteria
000307 ** established when the pScan object was initialized by whereScanInit().
000308 ** Return NULL if there are no more matching WhereTerms.
000309 */
000310 static WhereTerm *whereScanNext(WhereScan *pScan){
000311 int iCur; /* The cursor on the LHS of the term */
000312 i16 iColumn; /* The column on the LHS of the term. -1 for IPK */
000313 Expr *pX; /* An expression being tested */
000314 WhereClause *pWC; /* Shorthand for pScan->pWC */
000315 WhereTerm *pTerm; /* The term being tested */
000316 int k = pScan->k; /* Where to start scanning */
000317
000318 assert( pScan->iEquiv<=pScan->nEquiv );
000319 pWC = pScan->pWC;
000320 while(1){
000321 iColumn = pScan->aiColumn[pScan->iEquiv-1];
000322 iCur = pScan->aiCur[pScan->iEquiv-1];
000323 assert( pWC!=0 );
000324 assert( iCur>=0 );
000325 do{
000326 for(pTerm=pWC->a+k; k<pWC->nTerm; k++, pTerm++){
000327 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 || pTerm->leftCursor<0 );
000328 if( pTerm->leftCursor==iCur
000329 && pTerm->u.x.leftColumn==iColumn
000330 && (iColumn!=XN_EXPR
000331 || sqlite3ExprCompareSkip(pTerm->pExpr->pLeft,
000332 pScan->pIdxExpr,iCur)==0)
000333 && (pScan->iEquiv<=1 || !ExprHasProperty(pTerm->pExpr, EP_OuterON))
000334 ){
000335 if( (pTerm->eOperator & WO_EQUIV)!=0
000336 && pScan->nEquiv<ArraySize(pScan->aiCur)
000337 && (pX = whereRightSubexprIsColumn(pTerm->pExpr))!=0
000338 ){
000339 int j;
000340 for(j=0; j<pScan->nEquiv; j++){
000341 if( pScan->aiCur[j]==pX->iTable
000342 && pScan->aiColumn[j]==pX->iColumn ){
000343 break;
000344 }
000345 }
000346 if( j==pScan->nEquiv ){
000347 pScan->aiCur[j] = pX->iTable;
000348 pScan->aiColumn[j] = pX->iColumn;
000349 pScan->nEquiv++;
000350 }
000351 }
000352 if( (pTerm->eOperator & pScan->opMask)!=0 ){
000353 /* Verify the affinity and collating sequence match */
000354 if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){
000355 CollSeq *pColl;
000356 Parse *pParse = pWC->pWInfo->pParse;
000357 pX = pTerm->pExpr;
000358 if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){
000359 continue;
000360 }
000361 assert(pX->pLeft);
000362 pColl = sqlite3ExprCompareCollSeq(pParse, pX);
000363 if( pColl==0 ) pColl = pParse->db->pDfltColl;
000364 if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){
000365 continue;
000366 }
000367 }
000368 if( (pTerm->eOperator & (WO_EQ|WO_IS))!=0
000369 && (pX = pTerm->pExpr->pRight, ALWAYS(pX!=0))
000370 && pX->op==TK_COLUMN
000371 && pX->iTable==pScan->aiCur[0]
000372 && pX->iColumn==pScan->aiColumn[0]
000373 ){
000374 testcase( pTerm->eOperator & WO_IS );
000375 continue;
000376 }
000377 pScan->pWC = pWC;
000378 pScan->k = k+1;
000379 #ifdef WHERETRACE_ENABLED
000380 if( sqlite3WhereTrace & 0x20000 ){
000381 int ii;
000382 sqlite3DebugPrintf("SCAN-TERM %p: nEquiv=%d",
000383 pTerm, pScan->nEquiv);
000384 for(ii=0; ii<pScan->nEquiv; ii++){
000385 sqlite3DebugPrintf(" {%d:%d}",
000386 pScan->aiCur[ii], pScan->aiColumn[ii]);
000387 }
000388 sqlite3DebugPrintf("\n");
000389 }
000390 #endif
000391 return pTerm;
000392 }
000393 }
000394 }
000395 pWC = pWC->pOuter;
000396 k = 0;
000397 }while( pWC!=0 );
000398 if( pScan->iEquiv>=pScan->nEquiv ) break;
000399 pWC = pScan->pOrigWC;
000400 k = 0;
000401 pScan->iEquiv++;
000402 }
000403 return 0;
000404 }
000405
000406 /*
000407 ** This is whereScanInit() for the case of an index on an expression.
000408 ** It is factored out into a separate tail-recursion subroutine so that
000409 ** the normal whereScanInit() routine, which is a high-runner, does not
000410 ** need to push registers onto the stack as part of its prologue.
000411 */
000412 static SQLITE_NOINLINE WhereTerm *whereScanInitIndexExpr(WhereScan *pScan){
000413 pScan->idxaff = sqlite3ExprAffinity(pScan->pIdxExpr);
000414 return whereScanNext(pScan);
000415 }
000416
000417 /*
000418 ** Initialize a WHERE clause scanner object. Return a pointer to the
000419 ** first match. Return NULL if there are no matches.
000420 **
000421 ** The scanner will be searching the WHERE clause pWC. It will look
000422 ** for terms of the form "X <op> <expr>" where X is column iColumn of table
000423 ** iCur. Or if pIdx!=0 then X is column iColumn of index pIdx. pIdx
000424 ** must be one of the indexes of table iCur.
000425 **
000426 ** The <op> must be one of the operators described by opMask.
000427 **
000428 ** If the search is for X and the WHERE clause contains terms of the
000429 ** form X=Y then this routine might also return terms of the form
000430 ** "Y <op> <expr>". The number of levels of transitivity is limited,
000431 ** but is enough to handle most commonly occurring SQL statements.
000432 **
000433 ** If X is not the INTEGER PRIMARY KEY then X must be compatible with
000434 ** index pIdx.
000435 */
000436 static WhereTerm *whereScanInit(
000437 WhereScan *pScan, /* The WhereScan object being initialized */
000438 WhereClause *pWC, /* The WHERE clause to be scanned */
000439 int iCur, /* Cursor to scan for */
000440 int iColumn, /* Column to scan for */
000441 u32 opMask, /* Operator(s) to scan for */
000442 Index *pIdx /* Must be compatible with this index */
000443 ){
000444 pScan->pOrigWC = pWC;
000445 pScan->pWC = pWC;
000446 pScan->pIdxExpr = 0;
000447 pScan->idxaff = 0;
000448 pScan->zCollName = 0;
000449 pScan->opMask = opMask;
000450 pScan->k = 0;
000451 pScan->aiCur[0] = iCur;
000452 pScan->nEquiv = 1;
000453 pScan->iEquiv = 1;
000454 if( pIdx ){
000455 int j = iColumn;
000456 iColumn = pIdx->aiColumn[j];
000457 if( iColumn==pIdx->pTable->iPKey ){
000458 iColumn = XN_ROWID;
000459 }else if( iColumn>=0 ){
000460 pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity;
000461 pScan->zCollName = pIdx->azColl[j];
000462 }else if( iColumn==XN_EXPR ){
000463 pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr;
000464 pScan->zCollName = pIdx->azColl[j];
000465 pScan->aiColumn[0] = XN_EXPR;
000466 return whereScanInitIndexExpr(pScan);
000467 }
000468 }else if( iColumn==XN_EXPR ){
000469 return 0;
000470 }
000471 pScan->aiColumn[0] = iColumn;
000472 return whereScanNext(pScan);
000473 }
000474
000475 /*
000476 ** Search for a term in the WHERE clause that is of the form "X <op> <expr>"
000477 ** where X is a reference to the iColumn of table iCur or of index pIdx
000478 ** if pIdx!=0 and <op> is one of the WO_xx operator codes specified by
000479 ** the op parameter. Return a pointer to the term. Return 0 if not found.
000480 **
000481 ** If pIdx!=0 then it must be one of the indexes of table iCur.
000482 ** Search for terms matching the iColumn-th column of pIdx
000483 ** rather than the iColumn-th column of table iCur.
000484 **
000485 ** The term returned might by Y=<expr> if there is another constraint in
000486 ** the WHERE clause that specifies that X=Y. Any such constraints will be
000487 ** identified by the WO_EQUIV bit in the pTerm->eOperator field. The
000488 ** aiCur[]/iaColumn[] arrays hold X and all its equivalents. There are 11
000489 ** slots in aiCur[]/aiColumn[] so that means we can look for X plus up to 10
000490 ** other equivalent values. Hence a search for X will return <expr> if X=A1
000491 ** and A1=A2 and A2=A3 and ... and A9=A10 and A10=<expr>.
000492 **
000493 ** If there are multiple terms in the WHERE clause of the form "X <op> <expr>"
000494 ** then try for the one with no dependencies on <expr> - in other words where
000495 ** <expr> is a constant expression of some kind. Only return entries of
000496 ** the form "X <op> Y" where Y is a column in another table if no terms of
000497 ** the form "X <op> <const-expr>" exist. If no terms with a constant RHS
000498 ** exist, try to return a term that does not use WO_EQUIV.
000499 */
000500 WhereTerm *sqlite3WhereFindTerm(
000501 WhereClause *pWC, /* The WHERE clause to be searched */
000502 int iCur, /* Cursor number of LHS */
000503 int iColumn, /* Column number of LHS */
000504 Bitmask notReady, /* RHS must not overlap with this mask */
000505 u32 op, /* Mask of WO_xx values describing operator */
000506 Index *pIdx /* Must be compatible with this index, if not NULL */
000507 ){
000508 WhereTerm *pResult = 0;
000509 WhereTerm *p;
000510 WhereScan scan;
000511
000512 p = whereScanInit(&scan, pWC, iCur, iColumn, op, pIdx);
000513 op &= WO_EQ|WO_IS;
000514 while( p ){
000515 if( (p->prereqRight & notReady)==0 ){
000516 if( p->prereqRight==0 && (p->eOperator&op)!=0 ){
000517 testcase( p->eOperator & WO_IS );
000518 return p;
000519 }
000520 if( pResult==0 ) pResult = p;
000521 }
000522 p = whereScanNext(&scan);
000523 }
000524 return pResult;
000525 }
000526
000527 /*
000528 ** This function searches pList for an entry that matches the iCol-th column
000529 ** of index pIdx.
000530 **
000531 ** If such an expression is found, its index in pList->a[] is returned. If
000532 ** no expression is found, -1 is returned.
000533 */
000534 static int findIndexCol(
000535 Parse *pParse, /* Parse context */
000536 ExprList *pList, /* Expression list to search */
000537 int iBase, /* Cursor for table associated with pIdx */
000538 Index *pIdx, /* Index to match column of */
000539 int iCol /* Column of index to match */
000540 ){
000541 int i;
000542 const char *zColl = pIdx->azColl[iCol];
000543
000544 for(i=0; i<pList->nExpr; i++){
000545 Expr *p = sqlite3ExprSkipCollateAndLikely(pList->a[i].pExpr);
000546 if( ALWAYS(p!=0)
000547 && (p->op==TK_COLUMN || p->op==TK_AGG_COLUMN)
000548 && p->iColumn==pIdx->aiColumn[iCol]
000549 && p->iTable==iBase
000550 ){
000551 CollSeq *pColl = sqlite3ExprNNCollSeq(pParse, pList->a[i].pExpr);
000552 if( 0==sqlite3StrICmp(pColl->zName, zColl) ){
000553 return i;
000554 }
000555 }
000556 }
000557
000558 return -1;
000559 }
000560
000561 /*
000562 ** Return TRUE if the iCol-th column of index pIdx is NOT NULL
000563 */
000564 static int indexColumnNotNull(Index *pIdx, int iCol){
000565 int j;
000566 assert( pIdx!=0 );
000567 assert( iCol>=0 && iCol<pIdx->nColumn );
000568 j = pIdx->aiColumn[iCol];
000569 if( j>=0 ){
000570 return pIdx->pTable->aCol[j].notNull;
000571 }else if( j==(-1) ){
000572 return 1;
000573 }else{
000574 assert( j==(-2) );
000575 return 0; /* Assume an indexed expression can always yield a NULL */
000576
000577 }
000578 }
000579
000580 /*
000581 ** Return true if the DISTINCT expression-list passed as the third argument
000582 ** is redundant.
000583 **
000584 ** A DISTINCT list is redundant if any subset of the columns in the
000585 ** DISTINCT list are collectively unique and individually non-null.
000586 */
000587 static int isDistinctRedundant(
000588 Parse *pParse, /* Parsing context */
000589 SrcList *pTabList, /* The FROM clause */
000590 WhereClause *pWC, /* The WHERE clause */
000591 ExprList *pDistinct /* The result set that needs to be DISTINCT */
000592 ){
000593 Table *pTab;
000594 Index *pIdx;
000595 int i;
000596 int iBase;
000597
000598 /* If there is more than one table or sub-select in the FROM clause of
000599 ** this query, then it will not be possible to show that the DISTINCT
000600 ** clause is redundant. */
000601 if( pTabList->nSrc!=1 ) return 0;
000602 iBase = pTabList->a[0].iCursor;
000603 pTab = pTabList->a[0].pTab;
000604
000605 /* If any of the expressions is an IPK column on table iBase, then return
000606 ** true. Note: The (p->iTable==iBase) part of this test may be false if the
000607 ** current SELECT is a correlated sub-query.
000608 */
000609 for(i=0; i<pDistinct->nExpr; i++){
000610 Expr *p = sqlite3ExprSkipCollateAndLikely(pDistinct->a[i].pExpr);
000611 if( NEVER(p==0) ) continue;
000612 if( p->op!=TK_COLUMN && p->op!=TK_AGG_COLUMN ) continue;
000613 if( p->iTable==iBase && p->iColumn<0 ) return 1;
000614 }
000615
000616 /* Loop through all indices on the table, checking each to see if it makes
000617 ** the DISTINCT qualifier redundant. It does so if:
000618 **
000619 ** 1. The index is itself UNIQUE, and
000620 **
000621 ** 2. All of the columns in the index are either part of the pDistinct
000622 ** list, or else the WHERE clause contains a term of the form "col=X",
000623 ** where X is a constant value. The collation sequences of the
000624 ** comparison and select-list expressions must match those of the index.
000625 **
000626 ** 3. All of those index columns for which the WHERE clause does not
000627 ** contain a "col=X" term are subject to a NOT NULL constraint.
000628 */
000629 for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
000630 if( !IsUniqueIndex(pIdx) ) continue;
000631 if( pIdx->pPartIdxWhere ) continue;
000632 for(i=0; i<pIdx->nKeyCol; i++){
000633 if( 0==sqlite3WhereFindTerm(pWC, iBase, i, ~(Bitmask)0, WO_EQ, pIdx) ){
000634 if( findIndexCol(pParse, pDistinct, iBase, pIdx, i)<0 ) break;
000635 if( indexColumnNotNull(pIdx, i)==0 ) break;
000636 }
000637 }
000638 if( i==pIdx->nKeyCol ){
000639 /* This index implies that the DISTINCT qualifier is redundant. */
000640 return 1;
000641 }
000642 }
000643
000644 return 0;
000645 }
000646
000647
000648 /*
000649 ** Estimate the logarithm of the input value to base 2.
000650 */
000651 static LogEst estLog(LogEst N){
000652 return N<=10 ? 0 : sqlite3LogEst(N) - 33;
000653 }
000654
000655 /*
000656 ** Convert OP_Column opcodes to OP_Copy in previously generated code.
000657 **
000658 ** This routine runs over generated VDBE code and translates OP_Column
000659 ** opcodes into OP_Copy when the table is being accessed via co-routine
000660 ** instead of via table lookup.
000661 **
000662 ** If the iAutoidxCur is not zero, then any OP_Rowid instructions on
000663 ** cursor iTabCur are transformed into OP_Sequence opcode for the
000664 ** iAutoidxCur cursor, in order to generate unique rowids for the
000665 ** automatic index being generated.
000666 */
000667 static void translateColumnToCopy(
000668 Parse *pParse, /* Parsing context */
000669 int iStart, /* Translate from this opcode to the end */
000670 int iTabCur, /* OP_Column/OP_Rowid references to this table */
000671 int iRegister, /* The first column is in this register */
000672 int iAutoidxCur /* If non-zero, cursor of autoindex being generated */
000673 ){
000674 Vdbe *v = pParse->pVdbe;
000675 VdbeOp *pOp = sqlite3VdbeGetOp(v, iStart);
000676 int iEnd = sqlite3VdbeCurrentAddr(v);
000677 if( pParse->db->mallocFailed ) return;
000678 for(; iStart<iEnd; iStart++, pOp++){
000679 if( pOp->p1!=iTabCur ) continue;
000680 if( pOp->opcode==OP_Column ){
000681 #ifdef SQLITE_DEBUG
000682 if( pParse->db->flags & SQLITE_VdbeAddopTrace ){
000683 printf("TRANSLATE OP_Column to OP_Copy at %d\n", iStart);
000684 }
000685 #endif
000686 pOp->opcode = OP_Copy;
000687 pOp->p1 = pOp->p2 + iRegister;
000688 pOp->p2 = pOp->p3;
000689 pOp->p3 = 0;
000690 pOp->p5 = 2; /* Cause the MEM_Subtype flag to be cleared */
000691 }else if( pOp->opcode==OP_Rowid ){
000692 #ifdef SQLITE_DEBUG
000693 if( pParse->db->flags & SQLITE_VdbeAddopTrace ){
000694 printf("TRANSLATE OP_Rowid to OP_Sequence at %d\n", iStart);
000695 }
000696 #endif
000697 pOp->opcode = OP_Sequence;
000698 pOp->p1 = iAutoidxCur;
000699 #ifdef SQLITE_ALLOW_ROWID_IN_VIEW
000700 if( iAutoidxCur==0 ){
000701 pOp->opcode = OP_Null;
000702 pOp->p3 = 0;
000703 }
000704 #endif
000705 }
000706 }
000707 }
000708
000709 /*
000710 ** Two routines for printing the content of an sqlite3_index_info
000711 ** structure. Used for testing and debugging only. If neither
000712 ** SQLITE_TEST or SQLITE_DEBUG are defined, then these routines
000713 ** are no-ops.
000714 */
000715 #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED)
000716 static void whereTraceIndexInfoInputs(sqlite3_index_info *p){
000717 int i;
000718 if( (sqlite3WhereTrace & 0x10)==0 ) return;
000719 for(i=0; i<p->nConstraint; i++){
000720 sqlite3DebugPrintf(
000721 " constraint[%d]: col=%d termid=%d op=%d usabled=%d collseq=%s\n",
000722 i,
000723 p->aConstraint[i].iColumn,
000724 p->aConstraint[i].iTermOffset,
000725 p->aConstraint[i].op,
000726 p->aConstraint[i].usable,
000727 sqlite3_vtab_collation(p,i));
000728 }
000729 for(i=0; i<p->nOrderBy; i++){
000730 sqlite3DebugPrintf(" orderby[%d]: col=%d desc=%d\n",
000731 i,
000732 p->aOrderBy[i].iColumn,
000733 p->aOrderBy[i].desc);
000734 }
000735 }
000736 static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){
000737 int i;
000738 if( (sqlite3WhereTrace & 0x10)==0 ) return;
000739 for(i=0; i<p->nConstraint; i++){
000740 sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n",
000741 i,
000742 p->aConstraintUsage[i].argvIndex,
000743 p->aConstraintUsage[i].omit);
000744 }
000745 sqlite3DebugPrintf(" idxNum=%d\n", p->idxNum);
000746 sqlite3DebugPrintf(" idxStr=%s\n", p->idxStr);
000747 sqlite3DebugPrintf(" orderByConsumed=%d\n", p->orderByConsumed);
000748 sqlite3DebugPrintf(" estimatedCost=%g\n", p->estimatedCost);
000749 sqlite3DebugPrintf(" estimatedRows=%lld\n", p->estimatedRows);
000750 }
000751 #else
000752 #define whereTraceIndexInfoInputs(A)
000753 #define whereTraceIndexInfoOutputs(A)
000754 #endif
000755
000756 /*
000757 ** We know that pSrc is an operand of an outer join. Return true if
000758 ** pTerm is a constraint that is compatible with that join.
000759 **
000760 ** pTerm must be EP_OuterON if pSrc is the right operand of an
000761 ** outer join. pTerm can be either EP_OuterON or EP_InnerON if pSrc
000762 ** is the left operand of a RIGHT join.
000763 **
000764 ** See https://sqlite.org/forum/forumpost/206d99a16dd9212f
000765 ** for an example of a WHERE clause constraints that may not be used on
000766 ** the right table of a RIGHT JOIN because the constraint implies a
000767 ** not-NULL condition on the left table of the RIGHT JOIN.
000768 */
000769 static int constraintCompatibleWithOuterJoin(
000770 const WhereTerm *pTerm, /* WHERE clause term to check */
000771 const SrcItem *pSrc /* Table we are trying to access */
000772 ){
000773 assert( (pSrc->fg.jointype&(JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 ); /* By caller */
000774 testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LEFT );
000775 testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LTORJ );
000776 testcase( ExprHasProperty(pTerm->pExpr, EP_OuterON) )
000777 testcase( ExprHasProperty(pTerm->pExpr, EP_InnerON) );
000778 if( !ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON)
000779 || pTerm->pExpr->w.iJoin != pSrc->iCursor
000780 ){
000781 return 0;
000782 }
000783 if( (pSrc->fg.jointype & (JT_LEFT|JT_RIGHT))!=0
000784 && ExprHasProperty(pTerm->pExpr, EP_InnerON)
000785 ){
000786 return 0;
000787 }
000788 return 1;
000789 }
000790
000791
000792
000793 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
000794 /*
000795 ** Return TRUE if the WHERE clause term pTerm is of a form where it
000796 ** could be used with an index to access pSrc, assuming an appropriate
000797 ** index existed.
000798 */
000799 static int termCanDriveIndex(
000800 const WhereTerm *pTerm, /* WHERE clause term to check */
000801 const SrcItem *pSrc, /* Table we are trying to access */
000802 const Bitmask notReady /* Tables in outer loops of the join */
000803 ){
000804 char aff;
000805 if( pTerm->leftCursor!=pSrc->iCursor ) return 0;
000806 if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) return 0;
000807 assert( (pSrc->fg.jointype & JT_RIGHT)==0 );
000808 if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0
000809 && !constraintCompatibleWithOuterJoin(pTerm,pSrc)
000810 ){
000811 return 0; /* See https://sqlite.org/forum/forumpost/51e6959f61 */
000812 }
000813 if( (pTerm->prereqRight & notReady)!=0 ) return 0;
000814 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
000815 if( pTerm->u.x.leftColumn<0 ) return 0;
000816 aff = pSrc->pTab->aCol[pTerm->u.x.leftColumn].affinity;
000817 if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0;
000818 testcase( pTerm->pExpr->op==TK_IS );
000819 return 1;
000820 }
000821 #endif
000822
000823
000824 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
000825
000826 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
000827 /*
000828 ** Argument pIdx represents an automatic index that the current statement
000829 ** will create and populate. Add an OP_Explain with text of the form:
000830 **
000831 ** CREATE AUTOMATIC INDEX ON <table>(<cols>) [WHERE <expr>]
000832 **
000833 ** This is only required if sqlite3_stmt_scanstatus() is enabled, to
000834 ** associate an SQLITE_SCANSTAT_NCYCLE and SQLITE_SCANSTAT_NLOOP
000835 ** values with. In order to avoid breaking legacy code and test cases,
000836 ** the OP_Explain is not added if this is an EXPLAIN QUERY PLAN command.
000837 */
000838 static void explainAutomaticIndex(
000839 Parse *pParse,
000840 Index *pIdx, /* Automatic index to explain */
000841 int bPartial, /* True if pIdx is a partial index */
000842 int *pAddrExplain /* OUT: Address of OP_Explain */
000843 ){
000844 if( IS_STMT_SCANSTATUS(pParse->db) && pParse->explain!=2 ){
000845 Table *pTab = pIdx->pTable;
000846 const char *zSep = "";
000847 char *zText = 0;
000848 int ii = 0;
000849 sqlite3_str *pStr = sqlite3_str_new(pParse->db);
000850 sqlite3_str_appendf(pStr,"CREATE AUTOMATIC INDEX ON %s(", pTab->zName);
000851 assert( pIdx->nColumn>1 );
000852 assert( pIdx->aiColumn[pIdx->nColumn-1]==XN_ROWID );
000853 for(ii=0; ii<(pIdx->nColumn-1); ii++){
000854 const char *zName = 0;
000855 int iCol = pIdx->aiColumn[ii];
000856
000857 zName = pTab->aCol[iCol].zCnName;
000858 sqlite3_str_appendf(pStr, "%s%s", zSep, zName);
000859 zSep = ", ";
000860 }
000861 zText = sqlite3_str_finish(pStr);
000862 if( zText==0 ){
000863 sqlite3OomFault(pParse->db);
000864 }else{
000865 *pAddrExplain = sqlite3VdbeExplain(
000866 pParse, 0, "%s)%s", zText, (bPartial ? " WHERE <expr>" : "")
000867 );
000868 sqlite3_free(zText);
000869 }
000870 }
000871 }
000872 #else
000873 # define explainAutomaticIndex(a,b,c,d)
000874 #endif
000875
000876 /*
000877 ** Generate code to construct the Index object for an automatic index
000878 ** and to set up the WhereLevel object pLevel so that the code generator
000879 ** makes use of the automatic index.
000880 */
000881 static SQLITE_NOINLINE void constructAutomaticIndex(
000882 Parse *pParse, /* The parsing context */
000883 WhereClause *pWC, /* The WHERE clause */
000884 const Bitmask notReady, /* Mask of cursors that are not available */
000885 WhereLevel *pLevel /* Write new index here */
000886 ){
000887 int nKeyCol; /* Number of columns in the constructed index */
000888 WhereTerm *pTerm; /* A single term of the WHERE clause */
000889 WhereTerm *pWCEnd; /* End of pWC->a[] */
000890 Index *pIdx; /* Object describing the transient index */
000891 Vdbe *v; /* Prepared statement under construction */
000892 int addrInit; /* Address of the initialization bypass jump */
000893 Table *pTable; /* The table being indexed */
000894 int addrTop; /* Top of the index fill loop */
000895 int regRecord; /* Register holding an index record */
000896 int n; /* Column counter */
000897 int i; /* Loop counter */
000898 int mxBitCol; /* Maximum column in pSrc->colUsed */
000899 CollSeq *pColl; /* Collating sequence to on a column */
000900 WhereLoop *pLoop; /* The Loop object */
000901 char *zNotUsed; /* Extra space on the end of pIdx */
000902 Bitmask idxCols; /* Bitmap of columns used for indexing */
000903 Bitmask extraCols; /* Bitmap of additional columns */
000904 u8 sentWarning = 0; /* True if a warning has been issued */
000905 u8 useBloomFilter = 0; /* True to also add a Bloom filter */
000906 Expr *pPartial = 0; /* Partial Index Expression */
000907 int iContinue = 0; /* Jump here to skip excluded rows */
000908 SrcList *pTabList; /* The complete FROM clause */
000909 SrcItem *pSrc; /* The FROM clause term to get the next index */
000910 int addrCounter = 0; /* Address where integer counter is initialized */
000911 int regBase; /* Array of registers where record is assembled */
000912 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
000913 int addrExp = 0; /* Address of OP_Explain */
000914 #endif
000915
000916 /* Generate code to skip over the creation and initialization of the
000917 ** transient index on 2nd and subsequent iterations of the loop. */
000918 v = pParse->pVdbe;
000919 assert( v!=0 );
000920 addrInit = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
000921
000922 /* Count the number of columns that will be added to the index
000923 ** and used to match WHERE clause constraints */
000924 nKeyCol = 0;
000925 pTabList = pWC->pWInfo->pTabList;
000926 pSrc = &pTabList->a[pLevel->iFrom];
000927 pTable = pSrc->pTab;
000928 pWCEnd = &pWC->a[pWC->nTerm];
000929 pLoop = pLevel->pWLoop;
000930 idxCols = 0;
000931 for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){
000932 Expr *pExpr = pTerm->pExpr;
000933 /* Make the automatic index a partial index if there are terms in the
000934 ** WHERE clause (or the ON clause of a LEFT join) that constrain which
000935 ** rows of the target table (pSrc) that can be used. */
000936 if( (pTerm->wtFlags & TERM_VIRTUAL)==0
000937 && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, pLevel->iFrom)
000938 ){
000939 pPartial = sqlite3ExprAnd(pParse, pPartial,
000940 sqlite3ExprDup(pParse->db, pExpr, 0));
000941 }
000942 if( termCanDriveIndex(pTerm, pSrc, notReady) ){
000943 int iCol;
000944 Bitmask cMask;
000945 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
000946 iCol = pTerm->u.x.leftColumn;
000947 cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol);
000948 testcase( iCol==BMS );
000949 testcase( iCol==BMS-1 );
000950 if( !sentWarning ){
000951 sqlite3_log(SQLITE_WARNING_AUTOINDEX,
000952 "automatic index on %s(%s)", pTable->zName,
000953 pTable->aCol[iCol].zCnName);
000954 sentWarning = 1;
000955 }
000956 if( (idxCols & cMask)==0 ){
000957 if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ){
000958 goto end_auto_index_create;
000959 }
000960 pLoop->aLTerm[nKeyCol++] = pTerm;
000961 idxCols |= cMask;
000962 }
000963 }
000964 }
000965 assert( nKeyCol>0 || pParse->db->mallocFailed );
000966 pLoop->u.btree.nEq = pLoop->nLTerm = nKeyCol;
000967 pLoop->wsFlags = WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WHERE_INDEXED
000968 | WHERE_AUTO_INDEX;
000969
000970 /* Count the number of additional columns needed to create a
000971 ** covering index. A "covering index" is an index that contains all
000972 ** columns that are needed by the query. With a covering index, the
000973 ** original table never needs to be accessed. Automatic indices must
000974 ** be a covering index because the index will not be updated if the
000975 ** original table changes and the index and table cannot both be used
000976 ** if they go out of sync.
000977 */
000978 if( IsView(pTable) ){
000979 extraCols = ALLBITS;
000980 }else{
000981 extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1));
000982 }
000983 mxBitCol = MIN(BMS-1,pTable->nCol);
000984 testcase( pTable->nCol==BMS-1 );
000985 testcase( pTable->nCol==BMS-2 );
000986 for(i=0; i<mxBitCol; i++){
000987 if( extraCols & MASKBIT(i) ) nKeyCol++;
000988 }
000989 if( pSrc->colUsed & MASKBIT(BMS-1) ){
000990 nKeyCol += pTable->nCol - BMS + 1;
000991 }
000992
000993 /* Construct the Index object to describe this index */
000994 pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed);
000995 if( pIdx==0 ) goto end_auto_index_create;
000996 pLoop->u.btree.pIndex = pIdx;
000997 pIdx->zName = "auto-index";
000998 pIdx->pTable = pTable;
000999 n = 0;
001000 idxCols = 0;
001001 for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){
001002 if( termCanDriveIndex(pTerm, pSrc, notReady) ){
001003 int iCol;
001004 Bitmask cMask;
001005 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
001006 iCol = pTerm->u.x.leftColumn;
001007 cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol);
001008 testcase( iCol==BMS-1 );
001009 testcase( iCol==BMS );
001010 if( (idxCols & cMask)==0 ){
001011 Expr *pX = pTerm->pExpr;
001012 idxCols |= cMask;
001013 pIdx->aiColumn[n] = pTerm->u.x.leftColumn;
001014 pColl = sqlite3ExprCompareCollSeq(pParse, pX);
001015 assert( pColl!=0 || pParse->nErr>0 ); /* TH3 collate01.800 */
001016 pIdx->azColl[n] = pColl ? pColl->zName : sqlite3StrBINARY;
001017 n++;
001018 if( ALWAYS(pX->pLeft!=0)
001019 && sqlite3ExprAffinity(pX->pLeft)!=SQLITE_AFF_TEXT
001020 ){
001021 /* TUNING: only use a Bloom filter on an automatic index
001022 ** if one or more key columns has the ability to hold numeric
001023 ** values, since strings all have the same hash in the Bloom
001024 ** filter implementation and hence a Bloom filter on a text column
001025 ** is not usually helpful. */
001026 useBloomFilter = 1;
001027 }
001028 }
001029 }
001030 }
001031 assert( (u32)n==pLoop->u.btree.nEq );
001032
001033 /* Add additional columns needed to make the automatic index into
001034 ** a covering index */
001035 for(i=0; i<mxBitCol; i++){
001036 if( extraCols & MASKBIT(i) ){
001037 pIdx->aiColumn[n] = i;
001038 pIdx->azColl[n] = sqlite3StrBINARY;
001039 n++;
001040 }
001041 }
001042 if( pSrc->colUsed & MASKBIT(BMS-1) ){
001043 for(i=BMS-1; i<pTable->nCol; i++){
001044 pIdx->aiColumn[n] = i;
001045 pIdx->azColl[n] = sqlite3StrBINARY;
001046 n++;
001047 }
001048 }
001049 assert( n==nKeyCol );
001050 pIdx->aiColumn[n] = XN_ROWID;
001051 pIdx->azColl[n] = sqlite3StrBINARY;
001052
001053 /* Create the automatic index */
001054 explainAutomaticIndex(pParse, pIdx, pPartial!=0, &addrExp);
001055 assert( pLevel->iIdxCur>=0 );
001056 pLevel->iIdxCur = pParse->nTab++;
001057 sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1);
001058 sqlite3VdbeSetP4KeyInfo(pParse, pIdx);
001059 VdbeComment((v, "for %s", pTable->zName));
001060 if( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) && useBloomFilter ){
001061 sqlite3WhereExplainBloomFilter(pParse, pWC->pWInfo, pLevel);
001062 pLevel->regFilter = ++pParse->nMem;
001063 sqlite3VdbeAddOp2(v, OP_Blob, 10000, pLevel->regFilter);
001064 }
001065
001066 /* Fill the automatic index with content */
001067 assert( pSrc == &pWC->pWInfo->pTabList->a[pLevel->iFrom] );
001068 if( pSrc->fg.viaCoroutine ){
001069 int regYield = pSrc->regReturn;
001070 addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0);
001071 sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pSrc->addrFillSub);
001072 addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield);
001073 VdbeCoverage(v);
001074 VdbeComment((v, "next row of %s", pSrc->pTab->zName));
001075 }else{
001076 addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v);
001077 }
001078 if( pPartial ){
001079 iContinue = sqlite3VdbeMakeLabel(pParse);
001080 sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL);
001081 pLoop->wsFlags |= WHERE_PARTIALIDX;
001082 }
001083 regRecord = sqlite3GetTempReg(pParse);
001084 regBase = sqlite3GenerateIndexKey(
001085 pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0
001086 );
001087 if( pLevel->regFilter ){
001088 sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0,
001089 regBase, pLoop->u.btree.nEq);
001090 }
001091 sqlite3VdbeScanStatusCounters(v, addrExp, addrExp, sqlite3VdbeCurrentAddr(v));
001092 sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord);
001093 sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
001094 if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue);
001095 if( pSrc->fg.viaCoroutine ){
001096 sqlite3VdbeChangeP2(v, addrCounter, regBase+n);
001097 testcase( pParse->db->mallocFailed );
001098 assert( pLevel->iIdxCur>0 );
001099 translateColumnToCopy(pParse, addrTop, pLevel->iTabCur,
001100 pSrc->regResult, pLevel->iIdxCur);
001101 sqlite3VdbeGoto(v, addrTop);
001102 pSrc->fg.viaCoroutine = 0;
001103 }else{
001104 sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v);
001105 sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX);
001106 }
001107 sqlite3VdbeJumpHere(v, addrTop);
001108 sqlite3ReleaseTempReg(pParse, regRecord);
001109
001110 /* Jump here when skipping the initialization */
001111 sqlite3VdbeJumpHere(v, addrInit);
001112 sqlite3VdbeScanStatusRange(v, addrExp, addrExp, -1);
001113
001114 end_auto_index_create:
001115 sqlite3ExprDelete(pParse->db, pPartial);
001116 }
001117 #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
001118
001119 /*
001120 ** Generate bytecode that will initialize a Bloom filter that is appropriate
001121 ** for pLevel.
001122 **
001123 ** If there are inner loops within pLevel that have the WHERE_BLOOMFILTER
001124 ** flag set, initialize a Bloomfilter for them as well. Except don't do
001125 ** this recursive initialization if the SQLITE_BloomPulldown optimization has
001126 ** been turned off.
001127 **
001128 ** When the Bloom filter is initialized, the WHERE_BLOOMFILTER flag is cleared
001129 ** from the loop, but the regFilter value is set to a register that implements
001130 ** the Bloom filter. When regFilter is positive, the
001131 ** sqlite3WhereCodeOneLoopStart() will generate code to test the Bloom filter
001132 ** and skip the subsequence B-Tree seek if the Bloom filter indicates that
001133 ** no matching rows exist.
001134 **
001135 ** This routine may only be called if it has previously been determined that
001136 ** the loop would benefit from a Bloom filter, and the WHERE_BLOOMFILTER bit
001137 ** is set.
001138 */
001139 static SQLITE_NOINLINE void sqlite3ConstructBloomFilter(
001140 WhereInfo *pWInfo, /* The WHERE clause */
001141 int iLevel, /* Index in pWInfo->a[] that is pLevel */
001142 WhereLevel *pLevel, /* Make a Bloom filter for this FROM term */
001143 Bitmask notReady /* Loops that are not ready */
001144 ){
001145 int addrOnce; /* Address of opening OP_Once */
001146 int addrTop; /* Address of OP_Rewind */
001147 int addrCont; /* Jump here to skip a row */
001148 const WhereTerm *pTerm; /* For looping over WHERE clause terms */
001149 const WhereTerm *pWCEnd; /* Last WHERE clause term */
001150 Parse *pParse = pWInfo->pParse; /* Parsing context */
001151 Vdbe *v = pParse->pVdbe; /* VDBE under construction */
001152 WhereLoop *pLoop = pLevel->pWLoop; /* The loop being coded */
001153 int iCur; /* Cursor for table getting the filter */
001154 IndexedExpr *saved_pIdxEpr; /* saved copy of Parse.pIdxEpr */
001155 IndexedExpr *saved_pIdxPartExpr; /* saved copy of Parse.pIdxPartExpr */
001156
001157 saved_pIdxEpr = pParse->pIdxEpr;
001158 saved_pIdxPartExpr = pParse->pIdxPartExpr;
001159 pParse->pIdxEpr = 0;
001160 pParse->pIdxPartExpr = 0;
001161
001162 assert( pLoop!=0 );
001163 assert( v!=0 );
001164 assert( pLoop->wsFlags & WHERE_BLOOMFILTER );
001165 assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 );
001166
001167 addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
001168 do{
001169 const SrcList *pTabList;
001170 const SrcItem *pItem;
001171 const Table *pTab;
001172 u64 sz;
001173 int iSrc;
001174 sqlite3WhereExplainBloomFilter(pParse, pWInfo, pLevel);
001175 addrCont = sqlite3VdbeMakeLabel(pParse);
001176 iCur = pLevel->iTabCur;
001177 pLevel->regFilter = ++pParse->nMem;
001178
001179 /* The Bloom filter is a Blob held in a register. Initialize it
001180 ** to zero-filled blob of at least 80K bits, but maybe more if the
001181 ** estimated size of the table is larger. We could actually
001182 ** measure the size of the table at run-time using OP_Count with
001183 ** P3==1 and use that value to initialize the blob. But that makes
001184 ** testing complicated. By basing the blob size on the value in the
001185 ** sqlite_stat1 table, testing is much easier.
001186 */
001187 pTabList = pWInfo->pTabList;
001188 iSrc = pLevel->iFrom;
001189 pItem = &pTabList->a[iSrc];
001190 assert( pItem!=0 );
001191 pTab = pItem->pTab;
001192 assert( pTab!=0 );
001193 sz = sqlite3LogEstToInt(pTab->nRowLogEst);
001194 if( sz<10000 ){
001195 sz = 10000;
001196 }else if( sz>10000000 ){
001197 sz = 10000000;
001198 }
001199 sqlite3VdbeAddOp2(v, OP_Blob, (int)sz, pLevel->regFilter);
001200
001201 addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v);
001202 pWCEnd = &pWInfo->sWC.a[pWInfo->sWC.nTerm];
001203 for(pTerm=pWInfo->sWC.a; pTerm<pWCEnd; pTerm++){
001204 Expr *pExpr = pTerm->pExpr;
001205 if( (pTerm->wtFlags & TERM_VIRTUAL)==0
001206 && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, iSrc)
001207 ){
001208 sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL);
001209 }
001210 }
001211 if( pLoop->wsFlags & WHERE_IPK ){
001212 int r1 = sqlite3GetTempReg(pParse);
001213 sqlite3VdbeAddOp2(v, OP_Rowid, iCur, r1);
001214 sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, 1);
001215 sqlite3ReleaseTempReg(pParse, r1);
001216 }else{
001217 Index *pIdx = pLoop->u.btree.pIndex;
001218 int n = pLoop->u.btree.nEq;
001219 int r1 = sqlite3GetTempRange(pParse, n);
001220 int jj;
001221 for(jj=0; jj<n; jj++){
001222 assert( pIdx->pTable==pItem->pTab );
001223 sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iCur, jj, r1+jj);
001224 }
001225 sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, n);
001226 sqlite3ReleaseTempRange(pParse, r1, n);
001227 }
001228 sqlite3VdbeResolveLabel(v, addrCont);
001229 sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1);
001230 VdbeCoverage(v);
001231 sqlite3VdbeJumpHere(v, addrTop);
001232 pLoop->wsFlags &= ~WHERE_BLOOMFILTER;
001233 if( OptimizationDisabled(pParse->db, SQLITE_BloomPulldown) ) break;
001234 while( ++iLevel < pWInfo->nLevel ){
001235 const SrcItem *pTabItem;
001236 pLevel = &pWInfo->a[iLevel];
001237 pTabItem = &pWInfo->pTabList->a[pLevel->iFrom];
001238 if( pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ) ) continue;
001239 pLoop = pLevel->pWLoop;
001240 if( NEVER(pLoop==0) ) continue;
001241 if( pLoop->prereq & notReady ) continue;
001242 if( (pLoop->wsFlags & (WHERE_BLOOMFILTER|WHERE_COLUMN_IN))
001243 ==WHERE_BLOOMFILTER
001244 ){
001245 /* This is a candidate for bloom-filter pull-down (early evaluation).
001246 ** The test that WHERE_COLUMN_IN is omitted is important, as we are
001247 ** not able to do early evaluation of bloom filters that make use of
001248 ** the IN operator */
001249 break;
001250 }
001251 }
001252 }while( iLevel < pWInfo->nLevel );
001253 sqlite3VdbeJumpHere(v, addrOnce);
001254 pParse->pIdxEpr = saved_pIdxEpr;
001255 pParse->pIdxPartExpr = saved_pIdxPartExpr;
001256 }
001257
001258
001259 #ifndef SQLITE_OMIT_VIRTUALTABLE
001260 /*
001261 ** Allocate and populate an sqlite3_index_info structure. It is the
001262 ** responsibility of the caller to eventually release the structure
001263 ** by passing the pointer returned by this function to freeIndexInfo().
001264 */
001265 static sqlite3_index_info *allocateIndexInfo(
001266 WhereInfo *pWInfo, /* The WHERE clause */
001267 WhereClause *pWC, /* The WHERE clause being analyzed */
001268 Bitmask mUnusable, /* Ignore terms with these prereqs */
001269 SrcItem *pSrc, /* The FROM clause term that is the vtab */
001270 u16 *pmNoOmit /* Mask of terms not to omit */
001271 ){
001272 int i, j;
001273 int nTerm;
001274 Parse *pParse = pWInfo->pParse;
001275 struct sqlite3_index_constraint *pIdxCons;
001276 struct sqlite3_index_orderby *pIdxOrderBy;
001277 struct sqlite3_index_constraint_usage *pUsage;
001278 struct HiddenIndexInfo *pHidden;
001279 WhereTerm *pTerm;
001280 int nOrderBy;
001281 sqlite3_index_info *pIdxInfo;
001282 u16 mNoOmit = 0;
001283 const Table *pTab;
001284 int eDistinct = 0;
001285 ExprList *pOrderBy = pWInfo->pOrderBy;
001286
001287 assert( pSrc!=0 );
001288 pTab = pSrc->pTab;
001289 assert( pTab!=0 );
001290 assert( IsVirtual(pTab) );
001291
001292 /* Find all WHERE clause constraints referring to this virtual table.
001293 ** Mark each term with the TERM_OK flag. Set nTerm to the number of
001294 ** terms found.
001295 */
001296 for(i=nTerm=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
001297 pTerm->wtFlags &= ~TERM_OK;
001298 if( pTerm->leftCursor != pSrc->iCursor ) continue;
001299 if( pTerm->prereqRight & mUnusable ) continue;
001300 assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) );
001301 testcase( pTerm->eOperator & WO_IN );
001302 testcase( pTerm->eOperator & WO_ISNULL );
001303 testcase( pTerm->eOperator & WO_IS );
001304 testcase( pTerm->eOperator & WO_ALL );
001305 if( (pTerm->eOperator & ~(WO_EQUIV))==0 ) continue;
001306 if( pTerm->wtFlags & TERM_VNULL ) continue;
001307
001308 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
001309 assert( pTerm->u.x.leftColumn>=XN_ROWID );
001310 assert( pTerm->u.x.leftColumn<pTab->nCol );
001311 if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0
001312 && !constraintCompatibleWithOuterJoin(pTerm,pSrc)
001313 ){
001314 continue;
001315 }
001316 nTerm++;
001317 pTerm->wtFlags |= TERM_OK;
001318 }
001319
001320 /* If the ORDER BY clause contains only columns in the current
001321 ** virtual table then allocate space for the aOrderBy part of
001322 ** the sqlite3_index_info structure.
001323 */
001324 nOrderBy = 0;
001325 if( pOrderBy ){
001326 int n = pOrderBy->nExpr;
001327 for(i=0; i<n; i++){
001328 Expr *pExpr = pOrderBy->a[i].pExpr;
001329 Expr *pE2;
001330
001331 /* Skip over constant terms in the ORDER BY clause */
001332 if( sqlite3ExprIsConstant(pExpr) ){
001333 continue;
001334 }
001335
001336 /* Virtual tables are unable to deal with NULLS FIRST */
001337 if( pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_BIGNULL ) break;
001338
001339 /* First case - a direct column references without a COLLATE operator */
001340 if( pExpr->op==TK_COLUMN && pExpr->iTable==pSrc->iCursor ){
001341 assert( pExpr->iColumn>=XN_ROWID && pExpr->iColumn<pTab->nCol );
001342 continue;
001343 }
001344
001345 /* 2nd case - a column reference with a COLLATE operator. Only match
001346 ** of the COLLATE operator matches the collation of the column. */
001347 if( pExpr->op==TK_COLLATE
001348 && (pE2 = pExpr->pLeft)->op==TK_COLUMN
001349 && pE2->iTable==pSrc->iCursor
001350 ){
001351 const char *zColl; /* The collating sequence name */
001352 assert( !ExprHasProperty(pExpr, EP_IntValue) );
001353 assert( pExpr->u.zToken!=0 );
001354 assert( pE2->iColumn>=XN_ROWID && pE2->iColumn<pTab->nCol );
001355 pExpr->iColumn = pE2->iColumn;
001356 if( pE2->iColumn<0 ) continue; /* Collseq does not matter for rowid */
001357 zColl = sqlite3ColumnColl(&pTab->aCol[pE2->iColumn]);
001358 if( zColl==0 ) zColl = sqlite3StrBINARY;
001359 if( sqlite3_stricmp(pExpr->u.zToken, zColl)==0 ) continue;
001360 }
001361
001362 /* No matches cause a break out of the loop */
001363 break;
001364 }
001365 if( i==n ){
001366 nOrderBy = n;
001367 if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) ){
001368 eDistinct = 2 + ((pWInfo->wctrlFlags & WHERE_SORTBYGROUP)!=0);
001369 }else if( pWInfo->wctrlFlags & WHERE_GROUPBY ){
001370 eDistinct = 1;
001371 }
001372 }
001373 }
001374
001375 /* Allocate the sqlite3_index_info structure
001376 */
001377 pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo)
001378 + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm
001379 + sizeof(*pIdxOrderBy)*nOrderBy + sizeof(*pHidden)
001380 + sizeof(sqlite3_value*)*nTerm );
001381 if( pIdxInfo==0 ){
001382 sqlite3ErrorMsg(pParse, "out of memory");
001383 return 0;
001384 }
001385 pHidden = (struct HiddenIndexInfo*)&pIdxInfo[1];
001386 pIdxCons = (struct sqlite3_index_constraint*)&pHidden->aRhs[nTerm];
001387 pIdxOrderBy = (struct sqlite3_index_orderby*)&pIdxCons[nTerm];
001388 pUsage = (struct sqlite3_index_constraint_usage*)&pIdxOrderBy[nOrderBy];
001389 pIdxInfo->aConstraint = pIdxCons;
001390 pIdxInfo->aOrderBy = pIdxOrderBy;
001391 pIdxInfo->aConstraintUsage = pUsage;
001392 pHidden->pWC = pWC;
001393 pHidden->pParse = pParse;
001394 pHidden->eDistinct = eDistinct;
001395 pHidden->mIn = 0;
001396 for(i=j=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
001397 u16 op;
001398 if( (pTerm->wtFlags & TERM_OK)==0 ) continue;
001399 pIdxCons[j].iColumn = pTerm->u.x.leftColumn;
001400 pIdxCons[j].iTermOffset = i;
001401 op = pTerm->eOperator & WO_ALL;
001402 if( op==WO_IN ){
001403 if( (pTerm->wtFlags & TERM_SLICE)==0 ){
001404 pHidden->mIn |= SMASKBIT32(j);
001405 }
001406 op = WO_EQ;
001407 }
001408 if( op==WO_AUX ){
001409 pIdxCons[j].op = pTerm->eMatchOp;
001410 }else if( op & (WO_ISNULL|WO_IS) ){
001411 if( op==WO_ISNULL ){
001412 pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_ISNULL;
001413 }else{
001414 pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_IS;
001415 }
001416 }else{
001417 pIdxCons[j].op = (u8)op;
001418 /* The direct assignment in the previous line is possible only because
001419 ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The
001420 ** following asserts verify this fact. */
001421 assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ );
001422 assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT );
001423 assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE );
001424 assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT );
001425 assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE );
001426 assert( pTerm->eOperator&(WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_AUX) );
001427
001428 if( op & (WO_LT|WO_LE|WO_GT|WO_GE)
001429 && sqlite3ExprIsVector(pTerm->pExpr->pRight)
001430 ){
001431 testcase( j!=i );
001432 if( j<16 ) mNoOmit |= (1 << j);
001433 if( op==WO_LT ) pIdxCons[j].op = WO_LE;
001434 if( op==WO_GT ) pIdxCons[j].op = WO_GE;
001435 }
001436 }
001437
001438 j++;
001439 }
001440 assert( j==nTerm );
001441 pIdxInfo->nConstraint = j;
001442 for(i=j=0; i<nOrderBy; i++){
001443 Expr *pExpr = pOrderBy->a[i].pExpr;
001444 if( sqlite3ExprIsConstant(pExpr) ) continue;
001445 assert( pExpr->op==TK_COLUMN
001446 || (pExpr->op==TK_COLLATE && pExpr->pLeft->op==TK_COLUMN
001447 && pExpr->iColumn==pExpr->pLeft->iColumn) );
001448 pIdxOrderBy[j].iColumn = pExpr->iColumn;
001449 pIdxOrderBy[j].desc = pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_DESC;
001450 j++;
001451 }
001452 pIdxInfo->nOrderBy = j;
001453
001454 *pmNoOmit = mNoOmit;
001455 return pIdxInfo;
001456 }
001457
001458 /*
001459 ** Free an sqlite3_index_info structure allocated by allocateIndexInfo()
001460 ** and possibly modified by xBestIndex methods.
001461 */
001462 static void freeIndexInfo(sqlite3 *db, sqlite3_index_info *pIdxInfo){
001463 HiddenIndexInfo *pHidden;
001464 int i;
001465 assert( pIdxInfo!=0 );
001466 pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
001467 assert( pHidden->pParse!=0 );
001468 assert( pHidden->pParse->db==db );
001469 for(i=0; i<pIdxInfo->nConstraint; i++){
001470 sqlite3ValueFree(pHidden->aRhs[i]); /* IMP: R-14553-25174 */
001471 pHidden->aRhs[i] = 0;
001472 }
001473 sqlite3DbFree(db, pIdxInfo);
001474 }
001475
001476 /*
001477 ** The table object reference passed as the second argument to this function
001478 ** must represent a virtual table. This function invokes the xBestIndex()
001479 ** method of the virtual table with the sqlite3_index_info object that
001480 ** comes in as the 3rd argument to this function.
001481 **
001482 ** If an error occurs, pParse is populated with an error message and an
001483 ** appropriate error code is returned. A return of SQLITE_CONSTRAINT from
001484 ** xBestIndex is not considered an error. SQLITE_CONSTRAINT indicates that
001485 ** the current configuration of "unusable" flags in sqlite3_index_info can
001486 ** not result in a valid plan.
001487 **
001488 ** Whether or not an error is returned, it is the responsibility of the
001489 ** caller to eventually free p->idxStr if p->needToFreeIdxStr indicates
001490 ** that this is required.
001491 */
001492 static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){
001493 sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab;
001494 int rc;
001495
001496 whereTraceIndexInfoInputs(p);
001497 pParse->db->nSchemaLock++;
001498 rc = pVtab->pModule->xBestIndex(pVtab, p);
001499 pParse->db->nSchemaLock--;
001500 whereTraceIndexInfoOutputs(p);
001501
001502 if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT ){
001503 if( rc==SQLITE_NOMEM ){
001504 sqlite3OomFault(pParse->db);
001505 }else if( !pVtab->zErrMsg ){
001506 sqlite3ErrorMsg(pParse, "%s", sqlite3ErrStr(rc));
001507 }else{
001508 sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg);
001509 }
001510 }
001511 if( pTab->u.vtab.p->bAllSchemas ){
001512 sqlite3VtabUsesAllSchemas(pParse);
001513 }
001514 sqlite3_free(pVtab->zErrMsg);
001515 pVtab->zErrMsg = 0;
001516 return rc;
001517 }
001518 #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */
001519
001520 #ifdef SQLITE_ENABLE_STAT4
001521 /*
001522 ** Estimate the location of a particular key among all keys in an
001523 ** index. Store the results in aStat as follows:
001524 **
001525 ** aStat[0] Est. number of rows less than pRec
001526 ** aStat[1] Est. number of rows equal to pRec
001527 **
001528 ** Return the index of the sample that is the smallest sample that
001529 ** is greater than or equal to pRec. Note that this index is not an index
001530 ** into the aSample[] array - it is an index into a virtual set of samples
001531 ** based on the contents of aSample[] and the number of fields in record
001532 ** pRec.
001533 */
001534 static int whereKeyStats(
001535 Parse *pParse, /* Database connection */
001536 Index *pIdx, /* Index to consider domain of */
001537 UnpackedRecord *pRec, /* Vector of values to consider */
001538 int roundUp, /* Round up if true. Round down if false */
001539 tRowcnt *aStat /* OUT: stats written here */
001540 ){
001541 IndexSample *aSample = pIdx->aSample;
001542 int iCol; /* Index of required stats in anEq[] etc. */
001543 int i; /* Index of first sample >= pRec */
001544 int iSample; /* Smallest sample larger than or equal to pRec */
001545 int iMin = 0; /* Smallest sample not yet tested */
001546 int iTest; /* Next sample to test */
001547 int res; /* Result of comparison operation */
001548 int nField; /* Number of fields in pRec */
001549 tRowcnt iLower = 0; /* anLt[] + anEq[] of largest sample pRec is > */
001550
001551 #ifndef SQLITE_DEBUG
001552 UNUSED_PARAMETER( pParse );
001553 #endif
001554 assert( pRec!=0 );
001555 assert( pIdx->nSample>0 );
001556 assert( pRec->nField>0 );
001557
001558
001559 /* Do a binary search to find the first sample greater than or equal
001560 ** to pRec. If pRec contains a single field, the set of samples to search
001561 ** is simply the aSample[] array. If the samples in aSample[] contain more
001562 ** than one fields, all fields following the first are ignored.
001563 **
001564 ** If pRec contains N fields, where N is more than one, then as well as the
001565 ** samples in aSample[] (truncated to N fields), the search also has to
001566 ** consider prefixes of those samples. For example, if the set of samples
001567 ** in aSample is:
001568 **
001569 ** aSample[0] = (a, 5)
001570 ** aSample[1] = (a, 10)
001571 ** aSample[2] = (b, 5)
001572 ** aSample[3] = (c, 100)
001573 ** aSample[4] = (c, 105)
001574 **
001575 ** Then the search space should ideally be the samples above and the
001576 ** unique prefixes [a], [b] and [c]. But since that is hard to organize,
001577 ** the code actually searches this set:
001578 **
001579 ** 0: (a)
001580 ** 1: (a, 5)
001581 ** 2: (a, 10)
001582 ** 3: (a, 10)
001583 ** 4: (b)
001584 ** 5: (b, 5)
001585 ** 6: (c)
001586 ** 7: (c, 100)
001587 ** 8: (c, 105)
001588 ** 9: (c, 105)
001589 **
001590 ** For each sample in the aSample[] array, N samples are present in the
001591 ** effective sample array. In the above, samples 0 and 1 are based on
001592 ** sample aSample[0]. Samples 2 and 3 on aSample[1] etc.
001593 **
001594 ** Often, sample i of each block of N effective samples has (i+1) fields.
001595 ** Except, each sample may be extended to ensure that it is greater than or
001596 ** equal to the previous sample in the array. For example, in the above,
001597 ** sample 2 is the first sample of a block of N samples, so at first it
001598 ** appears that it should be 1 field in size. However, that would make it
001599 ** smaller than sample 1, so the binary search would not work. As a result,
001600 ** it is extended to two fields. The duplicates that this creates do not
001601 ** cause any problems.
001602 */
001603 if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){
001604 nField = pIdx->nKeyCol;
001605 }else{
001606 nField = pIdx->nColumn;
001607 }
001608 nField = MIN(pRec->nField, nField);
001609 iCol = 0;
001610 iSample = pIdx->nSample * nField;
001611 do{
001612 int iSamp; /* Index in aSample[] of test sample */
001613 int n; /* Number of fields in test sample */
001614
001615 iTest = (iMin+iSample)/2;
001616 iSamp = iTest / nField;
001617 if( iSamp>0 ){
001618 /* The proposed effective sample is a prefix of sample aSample[iSamp].
001619 ** Specifically, the shortest prefix of at least (1 + iTest%nField)
001620 ** fields that is greater than the previous effective sample. */
001621 for(n=(iTest % nField) + 1; n<nField; n++){
001622 if( aSample[iSamp-1].anLt[n-1]!=aSample[iSamp].anLt[n-1] ) break;
001623 }
001624 }else{
001625 n = iTest + 1;
001626 }
001627
001628 pRec->nField = n;
001629 res = sqlite3VdbeRecordCompare(aSample[iSamp].n, aSample[iSamp].p, pRec);
001630 if( res<0 ){
001631 iLower = aSample[iSamp].anLt[n-1] + aSample[iSamp].anEq[n-1];
001632 iMin = iTest+1;
001633 }else if( res==0 && n<nField ){
001634 iLower = aSample[iSamp].anLt[n-1];
001635 iMin = iTest+1;
001636 res = -1;
001637 }else{
001638 iSample = iTest;
001639 iCol = n-1;
001640 }
001641 }while( res && iMin<iSample );
001642 i = iSample / nField;
001643
001644 #ifdef SQLITE_DEBUG
001645 /* The following assert statements check that the binary search code
001646 ** above found the right answer. This block serves no purpose other
001647 ** than to invoke the asserts. */
001648 if( pParse->db->mallocFailed==0 ){
001649 if( res==0 ){
001650 /* If (res==0) is true, then pRec must be equal to sample i. */
001651 assert( i<pIdx->nSample );
001652 assert( iCol==nField-1 );
001653 pRec->nField = nField;
001654 assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)
001655 || pParse->db->mallocFailed
001656 );
001657 }else{
001658 /* Unless i==pIdx->nSample, indicating that pRec is larger than
001659 ** all samples in the aSample[] array, pRec must be smaller than the
001660 ** (iCol+1) field prefix of sample i. */
001661 assert( i<=pIdx->nSample && i>=0 );
001662 pRec->nField = iCol+1;
001663 assert( i==pIdx->nSample
001664 || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0
001665 || pParse->db->mallocFailed );
001666
001667 /* if i==0 and iCol==0, then record pRec is smaller than all samples
001668 ** in the aSample[] array. Otherwise, if (iCol>0) then pRec must
001669 ** be greater than or equal to the (iCol) field prefix of sample i.
001670 ** If (i>0), then pRec must also be greater than sample (i-1). */
001671 if( iCol>0 ){
001672 pRec->nField = iCol;
001673 assert( sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)<=0
001674 || pParse->db->mallocFailed || CORRUPT_DB );
001675 }
001676 if( i>0 ){
001677 pRec->nField = nField;
001678 assert( sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0
001679 || pParse->db->mallocFailed || CORRUPT_DB );
001680 }
001681 }
001682 }
001683 #endif /* ifdef SQLITE_DEBUG */
001684
001685 if( res==0 ){
001686 /* Record pRec is equal to sample i */
001687 assert( iCol==nField-1 );
001688 aStat[0] = aSample[i].anLt[iCol];
001689 aStat[1] = aSample[i].anEq[iCol];
001690 }else{
001691 /* At this point, the (iCol+1) field prefix of aSample[i] is the first
001692 ** sample that is greater than pRec. Or, if i==pIdx->nSample then pRec
001693 ** is larger than all samples in the array. */
001694 tRowcnt iUpper, iGap;
001695 if( i>=pIdx->nSample ){
001696 iUpper = pIdx->nRowEst0;
001697 }else{
001698 iUpper = aSample[i].anLt[iCol];
001699 }
001700
001701 if( iLower>=iUpper ){
001702 iGap = 0;
001703 }else{
001704 iGap = iUpper - iLower;
001705 }
001706 if( roundUp ){
001707 iGap = (iGap*2)/3;
001708 }else{
001709 iGap = iGap/3;
001710 }
001711 aStat[0] = iLower + iGap;
001712 aStat[1] = pIdx->aAvgEq[nField-1];
001713 }
001714
001715 /* Restore the pRec->nField value before returning. */
001716 pRec->nField = nField;
001717 return i;
001718 }
001719 #endif /* SQLITE_ENABLE_STAT4 */
001720
001721 /*
001722 ** If it is not NULL, pTerm is a term that provides an upper or lower
001723 ** bound on a range scan. Without considering pTerm, it is estimated
001724 ** that the scan will visit nNew rows. This function returns the number
001725 ** estimated to be visited after taking pTerm into account.
001726 **
001727 ** If the user explicitly specified a likelihood() value for this term,
001728 ** then the return value is the likelihood multiplied by the number of
001729 ** input rows. Otherwise, this function assumes that an "IS NOT NULL" term
001730 ** has a likelihood of 0.50, and any other term a likelihood of 0.25.
001731 */
001732 static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){
001733 LogEst nRet = nNew;
001734 if( pTerm ){
001735 if( pTerm->truthProb<=0 ){
001736 nRet += pTerm->truthProb;
001737 }else if( (pTerm->wtFlags & TERM_VNULL)==0 ){
001738 nRet -= 20; assert( 20==sqlite3LogEst(4) );
001739 }
001740 }
001741 return nRet;
001742 }
001743
001744
001745 #ifdef SQLITE_ENABLE_STAT4
001746 /*
001747 ** Return the affinity for a single column of an index.
001748 */
001749 char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){
001750 assert( iCol>=0 && iCol<pIdx->nColumn );
001751 if( !pIdx->zColAff ){
001752 if( sqlite3IndexAffinityStr(db, pIdx)==0 ) return SQLITE_AFF_BLOB;
001753 }
001754 assert( pIdx->zColAff[iCol]!=0 );
001755 return pIdx->zColAff[iCol];
001756 }
001757 #endif
001758
001759
001760 #ifdef SQLITE_ENABLE_STAT4
001761 /*
001762 ** This function is called to estimate the number of rows visited by a
001763 ** range-scan on a skip-scan index. For example:
001764 **
001765 ** CREATE INDEX i1 ON t1(a, b, c);
001766 ** SELECT * FROM t1 WHERE a=? AND c BETWEEN ? AND ?;
001767 **
001768 ** Value pLoop->nOut is currently set to the estimated number of rows
001769 ** visited for scanning (a=? AND b=?). This function reduces that estimate
001770 ** by some factor to account for the (c BETWEEN ? AND ?) expression based
001771 ** on the stat4 data for the index. this scan will be performed multiple
001772 ** times (once for each (a,b) combination that matches a=?) is dealt with
001773 ** by the caller.
001774 **
001775 ** It does this by scanning through all stat4 samples, comparing values
001776 ** extracted from pLower and pUpper with the corresponding column in each
001777 ** sample. If L and U are the number of samples found to be less than or
001778 ** equal to the values extracted from pLower and pUpper respectively, and
001779 ** N is the total number of samples, the pLoop->nOut value is adjusted
001780 ** as follows:
001781 **
001782 ** nOut = nOut * ( min(U - L, 1) / N )
001783 **
001784 ** If pLower is NULL, or a value cannot be extracted from the term, L is
001785 ** set to zero. If pUpper is NULL, or a value cannot be extracted from it,
001786 ** U is set to N.
001787 **
001788 ** Normally, this function sets *pbDone to 1 before returning. However,
001789 ** if no value can be extracted from either pLower or pUpper (and so the
001790 ** estimate of the number of rows delivered remains unchanged), *pbDone
001791 ** is left as is.
001792 **
001793 ** If an error occurs, an SQLite error code is returned. Otherwise,
001794 ** SQLITE_OK.
001795 */
001796 static int whereRangeSkipScanEst(
001797 Parse *pParse, /* Parsing & code generating context */
001798 WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */
001799 WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */
001800 WhereLoop *pLoop, /* Update the .nOut value of this loop */
001801 int *pbDone /* Set to true if at least one expr. value extracted */
001802 ){
001803 Index *p = pLoop->u.btree.pIndex;
001804 int nEq = pLoop->u.btree.nEq;
001805 sqlite3 *db = pParse->db;
001806 int nLower = -1;
001807 int nUpper = p->nSample+1;
001808 int rc = SQLITE_OK;
001809 u8 aff = sqlite3IndexColumnAffinity(db, p, nEq);
001810 CollSeq *pColl;
001811
001812 sqlite3_value *p1 = 0; /* Value extracted from pLower */
001813 sqlite3_value *p2 = 0; /* Value extracted from pUpper */
001814 sqlite3_value *pVal = 0; /* Value extracted from record */
001815
001816 pColl = sqlite3LocateCollSeq(pParse, p->azColl[nEq]);
001817 if( pLower ){
001818 rc = sqlite3Stat4ValueFromExpr(pParse, pLower->pExpr->pRight, aff, &p1);
001819 nLower = 0;
001820 }
001821 if( pUpper && rc==SQLITE_OK ){
001822 rc = sqlite3Stat4ValueFromExpr(pParse, pUpper->pExpr->pRight, aff, &p2);
001823 nUpper = p2 ? 0 : p->nSample;
001824 }
001825
001826 if( p1 || p2 ){
001827 int i;
001828 int nDiff;
001829 for(i=0; rc==SQLITE_OK && i<p->nSample; i++){
001830 rc = sqlite3Stat4Column(db, p->aSample[i].p, p->aSample[i].n, nEq, &pVal);
001831 if( rc==SQLITE_OK && p1 ){
001832 int res = sqlite3MemCompare(p1, pVal, pColl);
001833 if( res>=0 ) nLower++;
001834 }
001835 if( rc==SQLITE_OK && p2 ){
001836 int res = sqlite3MemCompare(p2, pVal, pColl);
001837 if( res>=0 ) nUpper++;
001838 }
001839 }
001840 nDiff = (nUpper - nLower);
001841 if( nDiff<=0 ) nDiff = 1;
001842
001843 /* If there is both an upper and lower bound specified, and the
001844 ** comparisons indicate that they are close together, use the fallback
001845 ** method (assume that the scan visits 1/64 of the rows) for estimating
001846 ** the number of rows visited. Otherwise, estimate the number of rows
001847 ** using the method described in the header comment for this function. */
001848 if( nDiff!=1 || pUpper==0 || pLower==0 ){
001849 int nAdjust = (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff));
001850 pLoop->nOut -= nAdjust;
001851 *pbDone = 1;
001852 WHERETRACE(0x20, ("range skip-scan regions: %u..%u adjust=%d est=%d\n",
001853 nLower, nUpper, nAdjust*-1, pLoop->nOut));
001854 }
001855
001856 }else{
001857 assert( *pbDone==0 );
001858 }
001859
001860 sqlite3ValueFree(p1);
001861 sqlite3ValueFree(p2);
001862 sqlite3ValueFree(pVal);
001863
001864 return rc;
001865 }
001866 #endif /* SQLITE_ENABLE_STAT4 */
001867
001868 /*
001869 ** This function is used to estimate the number of rows that will be visited
001870 ** by scanning an index for a range of values. The range may have an upper
001871 ** bound, a lower bound, or both. The WHERE clause terms that set the upper
001872 ** and lower bounds are represented by pLower and pUpper respectively. For
001873 ** example, assuming that index p is on t1(a):
001874 **
001875 ** ... FROM t1 WHERE a > ? AND a < ? ...
001876 ** |_____| |_____|
001877 ** | |
001878 ** pLower pUpper
001879 **
001880 ** If either of the upper or lower bound is not present, then NULL is passed in
001881 ** place of the corresponding WhereTerm.
001882 **
001883 ** The value in (pBuilder->pNew->u.btree.nEq) is the number of the index
001884 ** column subject to the range constraint. Or, equivalently, the number of
001885 ** equality constraints optimized by the proposed index scan. For example,
001886 ** assuming index p is on t1(a, b), and the SQL query is:
001887 **
001888 ** ... FROM t1 WHERE a = ? AND b > ? AND b < ? ...
001889 **
001890 ** then nEq is set to 1 (as the range restricted column, b, is the second
001891 ** left-most column of the index). Or, if the query is:
001892 **
001893 ** ... FROM t1 WHERE a > ? AND a < ? ...
001894 **
001895 ** then nEq is set to 0.
001896 **
001897 ** When this function is called, *pnOut is set to the sqlite3LogEst() of the
001898 ** number of rows that the index scan is expected to visit without
001899 ** considering the range constraints. If nEq is 0, then *pnOut is the number of
001900 ** rows in the index. Assuming no error occurs, *pnOut is adjusted (reduced)
001901 ** to account for the range constraints pLower and pUpper.
001902 **
001903 ** In the absence of sqlite_stat4 ANALYZE data, or if such data cannot be
001904 ** used, a single range inequality reduces the search space by a factor of 4.
001905 ** and a pair of constraints (x>? AND x<?) reduces the expected number of
001906 ** rows visited by a factor of 64.
001907 */
001908 static int whereRangeScanEst(
001909 Parse *pParse, /* Parsing & code generating context */
001910 WhereLoopBuilder *pBuilder,
001911 WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */
001912 WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */
001913 WhereLoop *pLoop /* Modify the .nOut and maybe .rRun fields */
001914 ){
001915 int rc = SQLITE_OK;
001916 int nOut = pLoop->nOut;
001917 LogEst nNew;
001918
001919 #ifdef SQLITE_ENABLE_STAT4
001920 Index *p = pLoop->u.btree.pIndex;
001921 int nEq = pLoop->u.btree.nEq;
001922
001923 if( p->nSample>0 && ALWAYS(nEq<p->nSampleCol)
001924 && OptimizationEnabled(pParse->db, SQLITE_Stat4)
001925 ){
001926 if( nEq==pBuilder->nRecValid ){
001927 UnpackedRecord *pRec = pBuilder->pRec;
001928 tRowcnt a[2];
001929 int nBtm = pLoop->u.btree.nBtm;
001930 int nTop = pLoop->u.btree.nTop;
001931
001932 /* Variable iLower will be set to the estimate of the number of rows in
001933 ** the index that are less than the lower bound of the range query. The
001934 ** lower bound being the concatenation of $P and $L, where $P is the
001935 ** key-prefix formed by the nEq values matched against the nEq left-most
001936 ** columns of the index, and $L is the value in pLower.
001937 **
001938 ** Or, if pLower is NULL or $L cannot be extracted from it (because it
001939 ** is not a simple variable or literal value), the lower bound of the
001940 ** range is $P. Due to a quirk in the way whereKeyStats() works, even
001941 ** if $L is available, whereKeyStats() is called for both ($P) and
001942 ** ($P:$L) and the larger of the two returned values is used.
001943 **
001944 ** Similarly, iUpper is to be set to the estimate of the number of rows
001945 ** less than the upper bound of the range query. Where the upper bound
001946 ** is either ($P) or ($P:$U). Again, even if $U is available, both values
001947 ** of iUpper are requested of whereKeyStats() and the smaller used.
001948 **
001949 ** The number of rows between the two bounds is then just iUpper-iLower.
001950 */
001951 tRowcnt iLower; /* Rows less than the lower bound */
001952 tRowcnt iUpper; /* Rows less than the upper bound */
001953 int iLwrIdx = -2; /* aSample[] for the lower bound */
001954 int iUprIdx = -1; /* aSample[] for the upper bound */
001955
001956 if( pRec ){
001957 testcase( pRec->nField!=pBuilder->nRecValid );
001958 pRec->nField = pBuilder->nRecValid;
001959 }
001960 /* Determine iLower and iUpper using ($P) only. */
001961 if( nEq==0 ){
001962 iLower = 0;
001963 iUpper = p->nRowEst0;
001964 }else{
001965 /* Note: this call could be optimized away - since the same values must
001966 ** have been requested when testing key $P in whereEqualScanEst(). */
001967 whereKeyStats(pParse, p, pRec, 0, a);
001968 iLower = a[0];
001969 iUpper = a[0] + a[1];
001970 }
001971
001972 assert( pLower==0 || (pLower->eOperator & (WO_GT|WO_GE))!=0 );
001973 assert( pUpper==0 || (pUpper->eOperator & (WO_LT|WO_LE))!=0 );
001974 assert( p->aSortOrder!=0 );
001975 if( p->aSortOrder[nEq] ){
001976 /* The roles of pLower and pUpper are swapped for a DESC index */
001977 SWAP(WhereTerm*, pLower, pUpper);
001978 SWAP(int, nBtm, nTop);
001979 }
001980
001981 /* If possible, improve on the iLower estimate using ($P:$L). */
001982 if( pLower ){
001983 int n; /* Values extracted from pExpr */
001984 Expr *pExpr = pLower->pExpr->pRight;
001985 rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nBtm, nEq, &n);
001986 if( rc==SQLITE_OK && n ){
001987 tRowcnt iNew;
001988 u16 mask = WO_GT|WO_LE;
001989 if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT);
001990 iLwrIdx = whereKeyStats(pParse, p, pRec, 0, a);
001991 iNew = a[0] + ((pLower->eOperator & mask) ? a[1] : 0);
001992 if( iNew>iLower ) iLower = iNew;
001993 nOut--;
001994 pLower = 0;
001995 }
001996 }
001997
001998 /* If possible, improve on the iUpper estimate using ($P:$U). */
001999 if( pUpper ){
002000 int n; /* Values extracted from pExpr */
002001 Expr *pExpr = pUpper->pExpr->pRight;
002002 rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nTop, nEq, &n);
002003 if( rc==SQLITE_OK && n ){
002004 tRowcnt iNew;
002005 u16 mask = WO_GT|WO_LE;
002006 if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT);
002007 iUprIdx = whereKeyStats(pParse, p, pRec, 1, a);
002008 iNew = a[0] + ((pUpper->eOperator & mask) ? a[1] : 0);
002009 if( iNew<iUpper ) iUpper = iNew;
002010 nOut--;
002011 pUpper = 0;
002012 }
002013 }
002014
002015 pBuilder->pRec = pRec;
002016 if( rc==SQLITE_OK ){
002017 if( iUpper>iLower ){
002018 nNew = sqlite3LogEst(iUpper - iLower);
002019 /* TUNING: If both iUpper and iLower are derived from the same
002020 ** sample, then assume they are 4x more selective. This brings
002021 ** the estimated selectivity more in line with what it would be
002022 ** if estimated without the use of STAT4 tables. */
002023 if( iLwrIdx==iUprIdx ){ nNew -= 20; }
002024 assert( 20==sqlite3LogEst(4) );
002025 }else{
002026 nNew = 10; assert( 10==sqlite3LogEst(2) );
002027 }
002028 if( nNew<nOut ){
002029 nOut = nNew;
002030 }
002031 WHERETRACE(0x20, ("STAT4 range scan: %u..%u est=%d\n",
002032 (u32)iLower, (u32)iUpper, nOut));
002033 }
002034 }else{
002035 int bDone = 0;
002036 rc = whereRangeSkipScanEst(pParse, pLower, pUpper, pLoop, &bDone);
002037 if( bDone ) return rc;
002038 }
002039 }
002040 #else
002041 UNUSED_PARAMETER(pParse);
002042 UNUSED_PARAMETER(pBuilder);
002043 assert( pLower || pUpper );
002044 #endif
002045 assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 || pParse->nErr>0 );
002046 nNew = whereRangeAdjust(pLower, nOut);
002047 nNew = whereRangeAdjust(pUpper, nNew);
002048
002049 /* TUNING: If there is both an upper and lower limit and neither limit
002050 ** has an application-defined likelihood(), assume the range is
002051 ** reduced by an additional 75%. This means that, by default, an open-ended
002052 ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the
002053 ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to
002054 ** match 1/64 of the index. */
002055 if( pLower && pLower->truthProb>0 && pUpper && pUpper->truthProb>0 ){
002056 nNew -= 20;
002057 }
002058
002059 nOut -= (pLower!=0) + (pUpper!=0);
002060 if( nNew<10 ) nNew = 10;
002061 if( nNew<nOut ) nOut = nNew;
002062 #if defined(WHERETRACE_ENABLED)
002063 if( pLoop->nOut>nOut ){
002064 WHERETRACE(0x20,("Range scan lowers nOut from %d to %d\n",
002065 pLoop->nOut, nOut));
002066 }
002067 #endif
002068 pLoop->nOut = (LogEst)nOut;
002069 return rc;
002070 }
002071
002072 #ifdef SQLITE_ENABLE_STAT4
002073 /*
002074 ** Estimate the number of rows that will be returned based on
002075 ** an equality constraint x=VALUE and where that VALUE occurs in
002076 ** the histogram data. This only works when x is the left-most
002077 ** column of an index and sqlite_stat4 histogram data is available
002078 ** for that index. When pExpr==NULL that means the constraint is
002079 ** "x IS NULL" instead of "x=VALUE".
002080 **
002081 ** Write the estimated row count into *pnRow and return SQLITE_OK.
002082 ** If unable to make an estimate, leave *pnRow unchanged and return
002083 ** non-zero.
002084 **
002085 ** This routine can fail if it is unable to load a collating sequence
002086 ** required for string comparison, or if unable to allocate memory
002087 ** for a UTF conversion required for comparison. The error is stored
002088 ** in the pParse structure.
002089 */
002090 static int whereEqualScanEst(
002091 Parse *pParse, /* Parsing & code generating context */
002092 WhereLoopBuilder *pBuilder,
002093 Expr *pExpr, /* Expression for VALUE in the x=VALUE constraint */
002094 tRowcnt *pnRow /* Write the revised row estimate here */
002095 ){
002096 Index *p = pBuilder->pNew->u.btree.pIndex;
002097 int nEq = pBuilder->pNew->u.btree.nEq;
002098 UnpackedRecord *pRec = pBuilder->pRec;
002099 int rc; /* Subfunction return code */
002100 tRowcnt a[2]; /* Statistics */
002101 int bOk;
002102
002103 assert( nEq>=1 );
002104 assert( nEq<=p->nColumn );
002105 assert( p->aSample!=0 );
002106 assert( p->nSample>0 );
002107 assert( pBuilder->nRecValid<nEq );
002108
002109 /* If values are not available for all fields of the index to the left
002110 ** of this one, no estimate can be made. Return SQLITE_NOTFOUND. */
002111 if( pBuilder->nRecValid<(nEq-1) ){
002112 return SQLITE_NOTFOUND;
002113 }
002114
002115 /* This is an optimization only. The call to sqlite3Stat4ProbeSetValue()
002116 ** below would return the same value. */
002117 if( nEq>=p->nColumn ){
002118 *pnRow = 1;
002119 return SQLITE_OK;
002120 }
002121
002122 rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, 1, nEq-1, &bOk);
002123 pBuilder->pRec = pRec;
002124 if( rc!=SQLITE_OK ) return rc;
002125 if( bOk==0 ) return SQLITE_NOTFOUND;
002126 pBuilder->nRecValid = nEq;
002127
002128 whereKeyStats(pParse, p, pRec, 0, a);
002129 WHERETRACE(0x20,("equality scan regions %s(%d): %d\n",
002130 p->zName, nEq-1, (int)a[1]));
002131 *pnRow = a[1];
002132
002133 return rc;
002134 }
002135 #endif /* SQLITE_ENABLE_STAT4 */
002136
002137 #ifdef SQLITE_ENABLE_STAT4
002138 /*
002139 ** Estimate the number of rows that will be returned based on
002140 ** an IN constraint where the right-hand side of the IN operator
002141 ** is a list of values. Example:
002142 **
002143 ** WHERE x IN (1,2,3,4)
002144 **
002145 ** Write the estimated row count into *pnRow and return SQLITE_OK.
002146 ** If unable to make an estimate, leave *pnRow unchanged and return
002147 ** non-zero.
002148 **
002149 ** This routine can fail if it is unable to load a collating sequence
002150 ** required for string comparison, or if unable to allocate memory
002151 ** for a UTF conversion required for comparison. The error is stored
002152 ** in the pParse structure.
002153 */
002154 static int whereInScanEst(
002155 Parse *pParse, /* Parsing & code generating context */
002156 WhereLoopBuilder *pBuilder,
002157 ExprList *pList, /* The value list on the RHS of "x IN (v1,v2,v3,...)" */
002158 tRowcnt *pnRow /* Write the revised row estimate here */
002159 ){
002160 Index *p = pBuilder->pNew->u.btree.pIndex;
002161 i64 nRow0 = sqlite3LogEstToInt(p->aiRowLogEst[0]);
002162 int nRecValid = pBuilder->nRecValid;
002163 int rc = SQLITE_OK; /* Subfunction return code */
002164 tRowcnt nEst; /* Number of rows for a single term */
002165 tRowcnt nRowEst = 0; /* New estimate of the number of rows */
002166 int i; /* Loop counter */
002167
002168 assert( p->aSample!=0 );
002169 for(i=0; rc==SQLITE_OK && i<pList->nExpr; i++){
002170 nEst = nRow0;
002171 rc = whereEqualScanEst(pParse, pBuilder, pList->a[i].pExpr, &nEst);
002172 nRowEst += nEst;
002173 pBuilder->nRecValid = nRecValid;
002174 }
002175
002176 if( rc==SQLITE_OK ){
002177 if( nRowEst > (tRowcnt)nRow0 ) nRowEst = nRow0;
002178 *pnRow = nRowEst;
002179 WHERETRACE(0x20,("IN row estimate: est=%d\n", nRowEst));
002180 }
002181 assert( pBuilder->nRecValid==nRecValid );
002182 return rc;
002183 }
002184 #endif /* SQLITE_ENABLE_STAT4 */
002185
002186
002187 #ifdef WHERETRACE_ENABLED
002188 /*
002189 ** Print the content of a WhereTerm object
002190 */
002191 void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm){
002192 if( pTerm==0 ){
002193 sqlite3DebugPrintf("TERM-%-3d NULL\n", iTerm);
002194 }else{
002195 char zType[8];
002196 char zLeft[50];
002197 memcpy(zType, "....", 5);
002198 if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V';
002199 if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E';
002200 if( ExprHasProperty(pTerm->pExpr, EP_OuterON) ) zType[2] = 'L';
002201 if( pTerm->wtFlags & TERM_CODED ) zType[3] = 'C';
002202 if( pTerm->eOperator & WO_SINGLE ){
002203 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
002204 sqlite3_snprintf(sizeof(zLeft),zLeft,"left={%d:%d}",
002205 pTerm->leftCursor, pTerm->u.x.leftColumn);
002206 }else if( (pTerm->eOperator & WO_OR)!=0 && pTerm->u.pOrInfo!=0 ){
002207 sqlite3_snprintf(sizeof(zLeft),zLeft,"indexable=0x%llx",
002208 pTerm->u.pOrInfo->indexable);
002209 }else{
002210 sqlite3_snprintf(sizeof(zLeft),zLeft,"left=%d", pTerm->leftCursor);
002211 }
002212 sqlite3DebugPrintf(
002213 "TERM-%-3d %p %s %-12s op=%03x wtFlags=%04x",
002214 iTerm, pTerm, zType, zLeft, pTerm->eOperator, pTerm->wtFlags);
002215 /* The 0x10000 .wheretrace flag causes extra information to be
002216 ** shown about each Term */
002217 if( sqlite3WhereTrace & 0x10000 ){
002218 sqlite3DebugPrintf(" prob=%-3d prereq=%llx,%llx",
002219 pTerm->truthProb, (u64)pTerm->prereqAll, (u64)pTerm->prereqRight);
002220 }
002221 if( (pTerm->eOperator & (WO_OR|WO_AND))==0 && pTerm->u.x.iField ){
002222 sqlite3DebugPrintf(" iField=%d", pTerm->u.x.iField);
002223 }
002224 if( pTerm->iParent>=0 ){
002225 sqlite3DebugPrintf(" iParent=%d", pTerm->iParent);
002226 }
002227 sqlite3DebugPrintf("\n");
002228 sqlite3TreeViewExpr(0, pTerm->pExpr, 0);
002229 }
002230 }
002231 #endif
002232
002233 #ifdef WHERETRACE_ENABLED
002234 /*
002235 ** Show the complete content of a WhereClause
002236 */
002237 void sqlite3WhereClausePrint(WhereClause *pWC){
002238 int i;
002239 for(i=0; i<pWC->nTerm; i++){
002240 sqlite3WhereTermPrint(&pWC->a[i], i);
002241 }
002242 }
002243 #endif
002244
002245 #ifdef WHERETRACE_ENABLED
002246 /*
002247 ** Print a WhereLoop object for debugging purposes
002248 **
002249 ** Format example:
002250 **
002251 ** .--- Position in WHERE clause rSetup, rRun, nOut ---.
002252 ** | |
002253 ** | .--- selfMask nTerm ------. |
002254 ** | | | |
002255 ** | | .-- prereq Idx wsFlags----. | |
002256 ** | | | Name | | |
002257 ** | | | __|__ nEq ---. ___|__ | __|__
002258 ** | / \ / \ / \ | / \ / \ / \
002259 ** 1.002.001 t2.t2xy 2 f 010241 N 2 cost 0,56,31
002260 */
002261 void sqlite3WhereLoopPrint(const WhereLoop *p, const WhereClause *pWC){
002262 if( pWC ){
002263 WhereInfo *pWInfo = pWC->pWInfo;
002264 int nb = 1+(pWInfo->pTabList->nSrc+3)/4;
002265 SrcItem *pItem = pWInfo->pTabList->a + p->iTab;
002266 Table *pTab = pItem->pTab;
002267 Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1;
002268 sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId,
002269 p->iTab, nb, p->maskSelf, nb, p->prereq & mAll);
002270 sqlite3DebugPrintf(" %12s",
002271 pItem->zAlias ? pItem->zAlias : pTab->zName);
002272 }else{
002273 sqlite3DebugPrintf("%c%2d.%03llx.%03llx %c%d",
002274 p->cId, p->iTab, p->maskSelf, p->prereq & 0xfff, p->cId, p->iTab);
002275 }
002276 if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){
002277 const char *zName;
002278 if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){
002279 if( strncmp(zName, "sqlite_autoindex_", 17)==0 ){
002280 int i = sqlite3Strlen30(zName) - 1;
002281 while( zName[i]!='_' ) i--;
002282 zName += i;
002283 }
002284 sqlite3DebugPrintf(".%-16s %2d", zName, p->u.btree.nEq);
002285 }else{
002286 sqlite3DebugPrintf("%20s","");
002287 }
002288 }else{
002289 char *z;
002290 if( p->u.vtab.idxStr ){
002291 z = sqlite3_mprintf("(%d,\"%s\",%#x)",
002292 p->u.vtab.idxNum, p->u.vtab.idxStr, p->u.vtab.omitMask);
002293 }else{
002294 z = sqlite3_mprintf("(%d,%x)", p->u.vtab.idxNum, p->u.vtab.omitMask);
002295 }
002296 sqlite3DebugPrintf(" %-19s", z);
002297 sqlite3_free(z);
002298 }
002299 if( p->wsFlags & WHERE_SKIPSCAN ){
002300 sqlite3DebugPrintf(" f %06x %d-%d", p->wsFlags, p->nLTerm,p->nSkip);
002301 }else{
002302 sqlite3DebugPrintf(" f %06x N %d", p->wsFlags, p->nLTerm);
002303 }
002304 sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut);
002305 if( p->nLTerm && (sqlite3WhereTrace & 0x4000)!=0 ){
002306 int i;
002307 for(i=0; i<p->nLTerm; i++){
002308 sqlite3WhereTermPrint(p->aLTerm[i], i);
002309 }
002310 }
002311 }
002312 void sqlite3ShowWhereLoop(const WhereLoop *p){
002313 if( p ) sqlite3WhereLoopPrint(p, 0);
002314 }
002315 void sqlite3ShowWhereLoopList(const WhereLoop *p){
002316 while( p ){
002317 sqlite3ShowWhereLoop(p);
002318 p = p->pNextLoop;
002319 }
002320 }
002321 #endif
002322
002323 /*
002324 ** Convert bulk memory into a valid WhereLoop that can be passed
002325 ** to whereLoopClear harmlessly.
002326 */
002327 static void whereLoopInit(WhereLoop *p){
002328 p->aLTerm = p->aLTermSpace;
002329 p->nLTerm = 0;
002330 p->nLSlot = ArraySize(p->aLTermSpace);
002331 p->wsFlags = 0;
002332 }
002333
002334 /*
002335 ** Clear the WhereLoop.u union. Leave WhereLoop.pLTerm intact.
002336 */
002337 static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){
002338 if( p->wsFlags & (WHERE_VIRTUALTABLE|WHERE_AUTO_INDEX) ){
002339 if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 && p->u.vtab.needFree ){
002340 sqlite3_free(p->u.vtab.idxStr);
002341 p->u.vtab.needFree = 0;
002342 p->u.vtab.idxStr = 0;
002343 }else if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 && p->u.btree.pIndex!=0 ){
002344 sqlite3DbFree(db, p->u.btree.pIndex->zColAff);
002345 sqlite3DbFreeNN(db, p->u.btree.pIndex);
002346 p->u.btree.pIndex = 0;
002347 }
002348 }
002349 }
002350
002351 /*
002352 ** Deallocate internal memory used by a WhereLoop object. Leave the
002353 ** object in an initialized state, as if it had been newly allocated.
002354 */
002355 static void whereLoopClear(sqlite3 *db, WhereLoop *p){
002356 if( p->aLTerm!=p->aLTermSpace ){
002357 sqlite3DbFreeNN(db, p->aLTerm);
002358 p->aLTerm = p->aLTermSpace;
002359 p->nLSlot = ArraySize(p->aLTermSpace);
002360 }
002361 whereLoopClearUnion(db, p);
002362 p->nLTerm = 0;
002363 p->wsFlags = 0;
002364 }
002365
002366 /*
002367 ** Increase the memory allocation for pLoop->aLTerm[] to be at least n.
002368 */
002369 static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){
002370 WhereTerm **paNew;
002371 if( p->nLSlot>=n ) return SQLITE_OK;
002372 n = (n+7)&~7;
002373 paNew = sqlite3DbMallocRawNN(db, sizeof(p->aLTerm[0])*n);
002374 if( paNew==0 ) return SQLITE_NOMEM_BKPT;
002375 memcpy(paNew, p->aLTerm, sizeof(p->aLTerm[0])*p->nLSlot);
002376 if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFreeNN(db, p->aLTerm);
002377 p->aLTerm = paNew;
002378 p->nLSlot = n;
002379 return SQLITE_OK;
002380 }
002381
002382 /*
002383 ** Transfer content from the second pLoop into the first.
002384 */
002385 static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){
002386 whereLoopClearUnion(db, pTo);
002387 if( pFrom->nLTerm > pTo->nLSlot
002388 && whereLoopResize(db, pTo, pFrom->nLTerm)
002389 ){
002390 memset(pTo, 0, WHERE_LOOP_XFER_SZ);
002391 return SQLITE_NOMEM_BKPT;
002392 }
002393 memcpy(pTo, pFrom, WHERE_LOOP_XFER_SZ);
002394 memcpy(pTo->aLTerm, pFrom->aLTerm, pTo->nLTerm*sizeof(pTo->aLTerm[0]));
002395 if( pFrom->wsFlags & WHERE_VIRTUALTABLE ){
002396 pFrom->u.vtab.needFree = 0;
002397 }else if( (pFrom->wsFlags & WHERE_AUTO_INDEX)!=0 ){
002398 pFrom->u.btree.pIndex = 0;
002399 }
002400 return SQLITE_OK;
002401 }
002402
002403 /*
002404 ** Delete a WhereLoop object
002405 */
002406 static void whereLoopDelete(sqlite3 *db, WhereLoop *p){
002407 assert( db!=0 );
002408 whereLoopClear(db, p);
002409 sqlite3DbNNFreeNN(db, p);
002410 }
002411
002412 /*
002413 ** Free a WhereInfo structure
002414 */
002415 static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){
002416 assert( pWInfo!=0 );
002417 assert( db!=0 );
002418 sqlite3WhereClauseClear(&pWInfo->sWC);
002419 while( pWInfo->pLoops ){
002420 WhereLoop *p = pWInfo->pLoops;
002421 pWInfo->pLoops = p->pNextLoop;
002422 whereLoopDelete(db, p);
002423 }
002424 while( pWInfo->pMemToFree ){
002425 WhereMemBlock *pNext = pWInfo->pMemToFree->pNext;
002426 sqlite3DbNNFreeNN(db, pWInfo->pMemToFree);
002427 pWInfo->pMemToFree = pNext;
002428 }
002429 sqlite3DbNNFreeNN(db, pWInfo);
002430 }
002431
002432 /*
002433 ** Return TRUE if X is a proper subset of Y but is of equal or less cost.
002434 ** In other words, return true if all constraints of X are also part of Y
002435 ** and Y has additional constraints that might speed the search that X lacks
002436 ** but the cost of running X is not more than the cost of running Y.
002437 **
002438 ** In other words, return true if the cost relationwship between X and Y
002439 ** is inverted and needs to be adjusted.
002440 **
002441 ** Case 1:
002442 **
002443 ** (1a) X and Y use the same index.
002444 ** (1b) X has fewer == terms than Y
002445 ** (1c) Neither X nor Y use skip-scan
002446 ** (1d) X does not have a a greater cost than Y
002447 **
002448 ** Case 2:
002449 **
002450 ** (2a) X has the same or lower cost, or returns the same or fewer rows,
002451 ** than Y.
002452 ** (2b) X uses fewer WHERE clause terms than Y
002453 ** (2c) Every WHERE clause term used by X is also used by Y
002454 ** (2d) X skips at least as many columns as Y
002455 ** (2e) If X is a covering index, than Y is too
002456 */
002457 static int whereLoopCheaperProperSubset(
002458 const WhereLoop *pX, /* First WhereLoop to compare */
002459 const WhereLoop *pY /* Compare against this WhereLoop */
002460 ){
002461 int i, j;
002462 if( pX->rRun>pY->rRun && pX->nOut>pY->nOut ) return 0; /* (1d) and (2a) */
002463 assert( (pX->wsFlags & WHERE_VIRTUALTABLE)==0 );
002464 assert( (pY->wsFlags & WHERE_VIRTUALTABLE)==0 );
002465 if( pX->u.btree.nEq < pY->u.btree.nEq /* (1b) */
002466 && pX->u.btree.pIndex==pY->u.btree.pIndex /* (1a) */
002467 && pX->nSkip==0 && pY->nSkip==0 /* (1c) */
002468 ){
002469 return 1; /* Case 1 is true */
002470 }
002471 if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){
002472 return 0; /* (2b) */
002473 }
002474 if( pY->nSkip > pX->nSkip ) return 0; /* (2d) */
002475 for(i=pX->nLTerm-1; i>=0; i--){
002476 if( pX->aLTerm[i]==0 ) continue;
002477 for(j=pY->nLTerm-1; j>=0; j--){
002478 if( pY->aLTerm[j]==pX->aLTerm[i] ) break;
002479 }
002480 if( j<0 ) return 0; /* (2c) */
002481 }
002482 if( (pX->wsFlags&WHERE_IDX_ONLY)!=0
002483 && (pY->wsFlags&WHERE_IDX_ONLY)==0 ){
002484 return 0; /* (2e) */
002485 }
002486 return 1; /* Case 2 is true */
002487 }
002488
002489 /*
002490 ** Try to adjust the cost and number of output rows of WhereLoop pTemplate
002491 ** upwards or downwards so that:
002492 **
002493 ** (1) pTemplate costs less than any other WhereLoops that are a proper
002494 ** subset of pTemplate
002495 **
002496 ** (2) pTemplate costs more than any other WhereLoops for which pTemplate
002497 ** is a proper subset.
002498 **
002499 ** To say "WhereLoop X is a proper subset of Y" means that X uses fewer
002500 ** WHERE clause terms than Y and that every WHERE clause term used by X is
002501 ** also used by Y.
002502 */
002503 static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){
002504 if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return;
002505 for(; p; p=p->pNextLoop){
002506 if( p->iTab!=pTemplate->iTab ) continue;
002507 if( (p->wsFlags & WHERE_INDEXED)==0 ) continue;
002508 if( whereLoopCheaperProperSubset(p, pTemplate) ){
002509 /* Adjust pTemplate cost downward so that it is cheaper than its
002510 ** subset p. */
002511 WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n",
002512 pTemplate->rRun, pTemplate->nOut,
002513 MIN(p->rRun, pTemplate->rRun),
002514 MIN(p->nOut - 1, pTemplate->nOut)));
002515 pTemplate->rRun = MIN(p->rRun, pTemplate->rRun);
002516 pTemplate->nOut = MIN(p->nOut - 1, pTemplate->nOut);
002517 }else if( whereLoopCheaperProperSubset(pTemplate, p) ){
002518 /* Adjust pTemplate cost upward so that it is costlier than p since
002519 ** pTemplate is a proper subset of p */
002520 WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n",
002521 pTemplate->rRun, pTemplate->nOut,
002522 MAX(p->rRun, pTemplate->rRun),
002523 MAX(p->nOut + 1, pTemplate->nOut)));
002524 pTemplate->rRun = MAX(p->rRun, pTemplate->rRun);
002525 pTemplate->nOut = MAX(p->nOut + 1, pTemplate->nOut);
002526 }
002527 }
002528 }
002529
002530 /*
002531 ** Search the list of WhereLoops in *ppPrev looking for one that can be
002532 ** replaced by pTemplate.
002533 **
002534 ** Return NULL if pTemplate does not belong on the WhereLoop list.
002535 ** In other words if pTemplate ought to be dropped from further consideration.
002536 **
002537 ** If pX is a WhereLoop that pTemplate can replace, then return the
002538 ** link that points to pX.
002539 **
002540 ** If pTemplate cannot replace any existing element of the list but needs
002541 ** to be added to the list as a new entry, then return a pointer to the
002542 ** tail of the list.
002543 */
002544 static WhereLoop **whereLoopFindLesser(
002545 WhereLoop **ppPrev,
002546 const WhereLoop *pTemplate
002547 ){
002548 WhereLoop *p;
002549 for(p=(*ppPrev); p; ppPrev=&p->pNextLoop, p=*ppPrev){
002550 if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ){
002551 /* If either the iTab or iSortIdx values for two WhereLoop are different
002552 ** then those WhereLoops need to be considered separately. Neither is
002553 ** a candidate to replace the other. */
002554 continue;
002555 }
002556 /* In the current implementation, the rSetup value is either zero
002557 ** or the cost of building an automatic index (NlogN) and the NlogN
002558 ** is the same for compatible WhereLoops. */
002559 assert( p->rSetup==0 || pTemplate->rSetup==0
002560 || p->rSetup==pTemplate->rSetup );
002561
002562 /* whereLoopAddBtree() always generates and inserts the automatic index
002563 ** case first. Hence compatible candidate WhereLoops never have a larger
002564 ** rSetup. Call this SETUP-INVARIANT */
002565 assert( p->rSetup>=pTemplate->rSetup );
002566
002567 /* Any loop using an application-defined index (or PRIMARY KEY or
002568 ** UNIQUE constraint) with one or more == constraints is better
002569 ** than an automatic index. Unless it is a skip-scan. */
002570 if( (p->wsFlags & WHERE_AUTO_INDEX)!=0
002571 && (pTemplate->nSkip)==0
002572 && (pTemplate->wsFlags & WHERE_INDEXED)!=0
002573 && (pTemplate->wsFlags & WHERE_COLUMN_EQ)!=0
002574 && (p->prereq & pTemplate->prereq)==pTemplate->prereq
002575 ){
002576 break;
002577 }
002578
002579 /* If existing WhereLoop p is better than pTemplate, pTemplate can be
002580 ** discarded. WhereLoop p is better if:
002581 ** (1) p has no more dependencies than pTemplate, and
002582 ** (2) p has an equal or lower cost than pTemplate
002583 */
002584 if( (p->prereq & pTemplate->prereq)==p->prereq /* (1) */
002585 && p->rSetup<=pTemplate->rSetup /* (2a) */
002586 && p->rRun<=pTemplate->rRun /* (2b) */
002587 && p->nOut<=pTemplate->nOut /* (2c) */
002588 ){
002589 return 0; /* Discard pTemplate */
002590 }
002591
002592 /* If pTemplate is always better than p, then cause p to be overwritten
002593 ** with pTemplate. pTemplate is better than p if:
002594 ** (1) pTemplate has no more dependencies than p, and
002595 ** (2) pTemplate has an equal or lower cost than p.
002596 */
002597 if( (p->prereq & pTemplate->prereq)==pTemplate->prereq /* (1) */
002598 && p->rRun>=pTemplate->rRun /* (2a) */
002599 && p->nOut>=pTemplate->nOut /* (2b) */
002600 ){
002601 assert( p->rSetup>=pTemplate->rSetup ); /* SETUP-INVARIANT above */
002602 break; /* Cause p to be overwritten by pTemplate */
002603 }
002604 }
002605 return ppPrev;
002606 }
002607
002608 /*
002609 ** Insert or replace a WhereLoop entry using the template supplied.
002610 **
002611 ** An existing WhereLoop entry might be overwritten if the new template
002612 ** is better and has fewer dependencies. Or the template will be ignored
002613 ** and no insert will occur if an existing WhereLoop is faster and has
002614 ** fewer dependencies than the template. Otherwise a new WhereLoop is
002615 ** added based on the template.
002616 **
002617 ** If pBuilder->pOrSet is not NULL then we care about only the
002618 ** prerequisites and rRun and nOut costs of the N best loops. That
002619 ** information is gathered in the pBuilder->pOrSet object. This special
002620 ** processing mode is used only for OR clause processing.
002621 **
002622 ** When accumulating multiple loops (when pBuilder->pOrSet is NULL) we
002623 ** still might overwrite similar loops with the new template if the
002624 ** new template is better. Loops may be overwritten if the following
002625 ** conditions are met:
002626 **
002627 ** (1) They have the same iTab.
002628 ** (2) They have the same iSortIdx.
002629 ** (3) The template has same or fewer dependencies than the current loop
002630 ** (4) The template has the same or lower cost than the current loop
002631 */
002632 static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){
002633 WhereLoop **ppPrev, *p;
002634 WhereInfo *pWInfo = pBuilder->pWInfo;
002635 sqlite3 *db = pWInfo->pParse->db;
002636 int rc;
002637
002638 /* Stop the search once we hit the query planner search limit */
002639 if( pBuilder->iPlanLimit==0 ){
002640 WHERETRACE(0xffffffff,("=== query planner search limit reached ===\n"));
002641 if( pBuilder->pOrSet ) pBuilder->pOrSet->n = 0;
002642 return SQLITE_DONE;
002643 }
002644 pBuilder->iPlanLimit--;
002645
002646 whereLoopAdjustCost(pWInfo->pLoops, pTemplate);
002647
002648 /* If pBuilder->pOrSet is defined, then only keep track of the costs
002649 ** and prereqs.
002650 */
002651 if( pBuilder->pOrSet!=0 ){
002652 if( pTemplate->nLTerm ){
002653 #if WHERETRACE_ENABLED
002654 u16 n = pBuilder->pOrSet->n;
002655 int x =
002656 #endif
002657 whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun,
002658 pTemplate->nOut);
002659 #if WHERETRACE_ENABLED /* 0x8 */
002660 if( sqlite3WhereTrace & 0x8 ){
002661 sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n);
002662 sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC);
002663 }
002664 #endif
002665 }
002666 return SQLITE_OK;
002667 }
002668
002669 /* Look for an existing WhereLoop to replace with pTemplate
002670 */
002671 ppPrev = whereLoopFindLesser(&pWInfo->pLoops, pTemplate);
002672
002673 if( ppPrev==0 ){
002674 /* There already exists a WhereLoop on the list that is better
002675 ** than pTemplate, so just ignore pTemplate */
002676 #if WHERETRACE_ENABLED /* 0x8 */
002677 if( sqlite3WhereTrace & 0x8 ){
002678 sqlite3DebugPrintf(" skip: ");
002679 sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC);
002680 }
002681 #endif
002682 return SQLITE_OK;
002683 }else{
002684 p = *ppPrev;
002685 }
002686
002687 /* If we reach this point it means that either p[] should be overwritten
002688 ** with pTemplate[] if p[] exists, or if p==NULL then allocate a new
002689 ** WhereLoop and insert it.
002690 */
002691 #if WHERETRACE_ENABLED /* 0x8 */
002692 if( sqlite3WhereTrace & 0x8 ){
002693 if( p!=0 ){
002694 sqlite3DebugPrintf("replace: ");
002695 sqlite3WhereLoopPrint(p, pBuilder->pWC);
002696 sqlite3DebugPrintf(" with: ");
002697 }else{
002698 sqlite3DebugPrintf(" add: ");
002699 }
002700 sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC);
002701 }
002702 #endif
002703 if( p==0 ){
002704 /* Allocate a new WhereLoop to add to the end of the list */
002705 *ppPrev = p = sqlite3DbMallocRawNN(db, sizeof(WhereLoop));
002706 if( p==0 ) return SQLITE_NOMEM_BKPT;
002707 whereLoopInit(p);
002708 p->pNextLoop = 0;
002709 }else{
002710 /* We will be overwriting WhereLoop p[]. But before we do, first
002711 ** go through the rest of the list and delete any other entries besides
002712 ** p[] that are also supplanted by pTemplate */
002713 WhereLoop **ppTail = &p->pNextLoop;
002714 WhereLoop *pToDel;
002715 while( *ppTail ){
002716 ppTail = whereLoopFindLesser(ppTail, pTemplate);
002717 if( ppTail==0 ) break;
002718 pToDel = *ppTail;
002719 if( pToDel==0 ) break;
002720 *ppTail = pToDel->pNextLoop;
002721 #if WHERETRACE_ENABLED /* 0x8 */
002722 if( sqlite3WhereTrace & 0x8 ){
002723 sqlite3DebugPrintf(" delete: ");
002724 sqlite3WhereLoopPrint(pToDel, pBuilder->pWC);
002725 }
002726 #endif
002727 whereLoopDelete(db, pToDel);
002728 }
002729 }
002730 rc = whereLoopXfer(db, p, pTemplate);
002731 if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){
002732 Index *pIndex = p->u.btree.pIndex;
002733 if( pIndex && pIndex->idxType==SQLITE_IDXTYPE_IPK ){
002734 p->u.btree.pIndex = 0;
002735 }
002736 }
002737 return rc;
002738 }
002739
002740 /*
002741 ** Adjust the WhereLoop.nOut value downward to account for terms of the
002742 ** WHERE clause that reference the loop but which are not used by an
002743 ** index.
002744 *
002745 ** For every WHERE clause term that is not used by the index
002746 ** and which has a truth probability assigned by one of the likelihood(),
002747 ** likely(), or unlikely() SQL functions, reduce the estimated number
002748 ** of output rows by the probability specified.
002749 **
002750 ** TUNING: For every WHERE clause term that is not used by the index
002751 ** and which does not have an assigned truth probability, heuristics
002752 ** described below are used to try to estimate the truth probability.
002753 ** TODO --> Perhaps this is something that could be improved by better
002754 ** table statistics.
002755 **
002756 ** Heuristic 1: Estimate the truth probability as 93.75%. The 93.75%
002757 ** value corresponds to -1 in LogEst notation, so this means decrement
002758 ** the WhereLoop.nOut field for every such WHERE clause term.
002759 **
002760 ** Heuristic 2: If there exists one or more WHERE clause terms of the
002761 ** form "x==EXPR" and EXPR is not a constant 0 or 1, then make sure the
002762 ** final output row estimate is no greater than 1/4 of the total number
002763 ** of rows in the table. In other words, assume that x==EXPR will filter
002764 ** out at least 3 out of 4 rows. If EXPR is -1 or 0 or 1, then maybe the
002765 ** "x" column is boolean or else -1 or 0 or 1 is a common default value
002766 ** on the "x" column and so in that case only cap the output row estimate
002767 ** at 1/2 instead of 1/4.
002768 */
002769 static void whereLoopOutputAdjust(
002770 WhereClause *pWC, /* The WHERE clause */
002771 WhereLoop *pLoop, /* The loop to adjust downward */
002772 LogEst nRow /* Number of rows in the entire table */
002773 ){
002774 WhereTerm *pTerm, *pX;
002775 Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf);
002776 int i, j;
002777 LogEst iReduce = 0; /* pLoop->nOut should not exceed nRow-iReduce */
002778
002779 assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 );
002780 for(i=pWC->nBase, pTerm=pWC->a; i>0; i--, pTerm++){
002781 assert( pTerm!=0 );
002782 if( (pTerm->prereqAll & notAllowed)!=0 ) continue;
002783 if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue;
002784 if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) continue;
002785 for(j=pLoop->nLTerm-1; j>=0; j--){
002786 pX = pLoop->aLTerm[j];
002787 if( pX==0 ) continue;
002788 if( pX==pTerm ) break;
002789 if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break;
002790 }
002791 if( j<0 ){
002792 sqlite3ProgressCheck(pWC->pWInfo->pParse);
002793 if( pLoop->maskSelf==pTerm->prereqAll ){
002794 /* If there are extra terms in the WHERE clause not used by an index
002795 ** that depend only on the table being scanned, and that will tend to
002796 ** cause many rows to be omitted, then mark that table as
002797 ** "self-culling".
002798 **
002799 ** 2022-03-24: Self-culling only applies if either the extra terms
002800 ** are straight comparison operators that are non-true with NULL
002801 ** operand, or if the loop is not an OUTER JOIN.
002802 */
002803 if( (pTerm->eOperator & 0x3f)!=0
002804 || (pWC->pWInfo->pTabList->a[pLoop->iTab].fg.jointype
002805 & (JT_LEFT|JT_LTORJ))==0
002806 ){
002807 pLoop->wsFlags |= WHERE_SELFCULL;
002808 }
002809 }
002810 if( pTerm->truthProb<=0 ){
002811 /* If a truth probability is specified using the likelihood() hints,
002812 ** then use the probability provided by the application. */
002813 pLoop->nOut += pTerm->truthProb;
002814 }else{
002815 /* In the absence of explicit truth probabilities, use heuristics to
002816 ** guess a reasonable truth probability. */
002817 pLoop->nOut--;
002818 if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0
002819 && (pTerm->wtFlags & TERM_HIGHTRUTH)==0 /* tag-20200224-1 */
002820 ){
002821 Expr *pRight = pTerm->pExpr->pRight;
002822 int k = 0;
002823 testcase( pTerm->pExpr->op==TK_IS );
002824 if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){
002825 k = 10;
002826 }else{
002827 k = 20;
002828 }
002829 if( iReduce<k ){
002830 pTerm->wtFlags |= TERM_HEURTRUTH;
002831 iReduce = k;
002832 }
002833 }
002834 }
002835 }
002836 }
002837 if( pLoop->nOut > nRow-iReduce ){
002838 pLoop->nOut = nRow - iReduce;
002839 }
002840 }
002841
002842 /*
002843 ** Term pTerm is a vector range comparison operation. The first comparison
002844 ** in the vector can be optimized using column nEq of the index. This
002845 ** function returns the total number of vector elements that can be used
002846 ** as part of the range comparison.
002847 **
002848 ** For example, if the query is:
002849 **
002850 ** WHERE a = ? AND (b, c, d) > (?, ?, ?)
002851 **
002852 ** and the index:
002853 **
002854 ** CREATE INDEX ... ON (a, b, c, d, e)
002855 **
002856 ** then this function would be invoked with nEq=1. The value returned in
002857 ** this case is 3.
002858 */
002859 static int whereRangeVectorLen(
002860 Parse *pParse, /* Parsing context */
002861 int iCur, /* Cursor open on pIdx */
002862 Index *pIdx, /* The index to be used for a inequality constraint */
002863 int nEq, /* Number of prior equality constraints on same index */
002864 WhereTerm *pTerm /* The vector inequality constraint */
002865 ){
002866 int nCmp = sqlite3ExprVectorSize(pTerm->pExpr->pLeft);
002867 int i;
002868
002869 nCmp = MIN(nCmp, (pIdx->nColumn - nEq));
002870 for(i=1; i<nCmp; i++){
002871 /* Test if comparison i of pTerm is compatible with column (i+nEq)
002872 ** of the index. If not, exit the loop. */
002873 char aff; /* Comparison affinity */
002874 char idxaff = 0; /* Indexed columns affinity */
002875 CollSeq *pColl; /* Comparison collation sequence */
002876 Expr *pLhs, *pRhs;
002877
002878 assert( ExprUseXList(pTerm->pExpr->pLeft) );
002879 pLhs = pTerm->pExpr->pLeft->x.pList->a[i].pExpr;
002880 pRhs = pTerm->pExpr->pRight;
002881 if( ExprUseXSelect(pRhs) ){
002882 pRhs = pRhs->x.pSelect->pEList->a[i].pExpr;
002883 }else{
002884 pRhs = pRhs->x.pList->a[i].pExpr;
002885 }
002886
002887 /* Check that the LHS of the comparison is a column reference to
002888 ** the right column of the right source table. And that the sort
002889 ** order of the index column is the same as the sort order of the
002890 ** leftmost index column. */
002891 if( pLhs->op!=TK_COLUMN
002892 || pLhs->iTable!=iCur
002893 || pLhs->iColumn!=pIdx->aiColumn[i+nEq]
002894 || pIdx->aSortOrder[i+nEq]!=pIdx->aSortOrder[nEq]
002895 ){
002896 break;
002897 }
002898
002899 testcase( pLhs->iColumn==XN_ROWID );
002900 aff = sqlite3CompareAffinity(pRhs, sqlite3ExprAffinity(pLhs));
002901 idxaff = sqlite3TableColumnAffinity(pIdx->pTable, pLhs->iColumn);
002902 if( aff!=idxaff ) break;
002903
002904 pColl = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs);
002905 if( pColl==0 ) break;
002906 if( sqlite3StrICmp(pColl->zName, pIdx->azColl[i+nEq]) ) break;
002907 }
002908 return i;
002909 }
002910
002911 /*
002912 ** Adjust the cost C by the costMult factor T. This only occurs if
002913 ** compiled with -DSQLITE_ENABLE_COSTMULT
002914 */
002915 #ifdef SQLITE_ENABLE_COSTMULT
002916 # define ApplyCostMultiplier(C,T) C += T
002917 #else
002918 # define ApplyCostMultiplier(C,T)
002919 #endif
002920
002921 /*
002922 ** We have so far matched pBuilder->pNew->u.btree.nEq terms of the
002923 ** index pIndex. Try to match one more.
002924 **
002925 ** When this function is called, pBuilder->pNew->nOut contains the
002926 ** number of rows expected to be visited by filtering using the nEq
002927 ** terms only. If it is modified, this value is restored before this
002928 ** function returns.
002929 **
002930 ** If pProbe->idxType==SQLITE_IDXTYPE_IPK, that means pIndex is
002931 ** a fake index used for the INTEGER PRIMARY KEY.
002932 */
002933 static int whereLoopAddBtreeIndex(
002934 WhereLoopBuilder *pBuilder, /* The WhereLoop factory */
002935 SrcItem *pSrc, /* FROM clause term being analyzed */
002936 Index *pProbe, /* An index on pSrc */
002937 LogEst nInMul /* log(Number of iterations due to IN) */
002938 ){
002939 WhereInfo *pWInfo = pBuilder->pWInfo; /* WHERE analyze context */
002940 Parse *pParse = pWInfo->pParse; /* Parsing context */
002941 sqlite3 *db = pParse->db; /* Database connection malloc context */
002942 WhereLoop *pNew; /* Template WhereLoop under construction */
002943 WhereTerm *pTerm; /* A WhereTerm under consideration */
002944 int opMask; /* Valid operators for constraints */
002945 WhereScan scan; /* Iterator for WHERE terms */
002946 Bitmask saved_prereq; /* Original value of pNew->prereq */
002947 u16 saved_nLTerm; /* Original value of pNew->nLTerm */
002948 u16 saved_nEq; /* Original value of pNew->u.btree.nEq */
002949 u16 saved_nBtm; /* Original value of pNew->u.btree.nBtm */
002950 u16 saved_nTop; /* Original value of pNew->u.btree.nTop */
002951 u16 saved_nSkip; /* Original value of pNew->nSkip */
002952 u32 saved_wsFlags; /* Original value of pNew->wsFlags */
002953 LogEst saved_nOut; /* Original value of pNew->nOut */
002954 int rc = SQLITE_OK; /* Return code */
002955 LogEst rSize; /* Number of rows in the table */
002956 LogEst rLogSize; /* Logarithm of table size */
002957 WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */
002958
002959 pNew = pBuilder->pNew;
002960 assert( db->mallocFailed==0 || pParse->nErr>0 );
002961 if( pParse->nErr ){
002962 return pParse->rc;
002963 }
002964 WHERETRACE(0x800, ("BEGIN %s.addBtreeIdx(%s), nEq=%d, nSkip=%d, rRun=%d\n",
002965 pProbe->pTable->zName,pProbe->zName,
002966 pNew->u.btree.nEq, pNew->nSkip, pNew->rRun));
002967
002968 assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 );
002969 assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 );
002970 if( pNew->wsFlags & WHERE_BTM_LIMIT ){
002971 opMask = WO_LT|WO_LE;
002972 }else{
002973 assert( pNew->u.btree.nBtm==0 );
002974 opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS;
002975 }
002976 if( pProbe->bUnordered || pProbe->bLowQual ){
002977 if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE);
002978 if( pProbe->bLowQual ) opMask &= ~(WO_EQ|WO_IN|WO_IS);
002979 }
002980
002981 assert( pNew->u.btree.nEq<pProbe->nColumn );
002982 assert( pNew->u.btree.nEq<pProbe->nKeyCol
002983 || pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY );
002984
002985 saved_nEq = pNew->u.btree.nEq;
002986 saved_nBtm = pNew->u.btree.nBtm;
002987 saved_nTop = pNew->u.btree.nTop;
002988 saved_nSkip = pNew->nSkip;
002989 saved_nLTerm = pNew->nLTerm;
002990 saved_wsFlags = pNew->wsFlags;
002991 saved_prereq = pNew->prereq;
002992 saved_nOut = pNew->nOut;
002993 pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, saved_nEq,
002994 opMask, pProbe);
002995 pNew->rSetup = 0;
002996 rSize = pProbe->aiRowLogEst[0];
002997 rLogSize = estLog(rSize);
002998 for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){
002999 u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */
003000 LogEst rCostIdx;
003001 LogEst nOutUnadjusted; /* nOut before IN() and WHERE adjustments */
003002 int nIn = 0;
003003 #ifdef SQLITE_ENABLE_STAT4
003004 int nRecValid = pBuilder->nRecValid;
003005 #endif
003006 if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0)
003007 && indexColumnNotNull(pProbe, saved_nEq)
003008 ){
003009 continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */
003010 }
003011 if( pTerm->prereqRight & pNew->maskSelf ) continue;
003012
003013 /* Do not allow the upper bound of a LIKE optimization range constraint
003014 ** to mix with a lower range bound from some other source */
003015 if( pTerm->wtFlags & TERM_LIKEOPT && pTerm->eOperator==WO_LT ) continue;
003016
003017 if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0
003018 && !constraintCompatibleWithOuterJoin(pTerm,pSrc)
003019 ){
003020 continue;
003021 }
003022 if( IsUniqueIndex(pProbe) && saved_nEq==pProbe->nKeyCol-1 ){
003023 pBuilder->bldFlags1 |= SQLITE_BLDF1_UNIQUE;
003024 }else{
003025 pBuilder->bldFlags1 |= SQLITE_BLDF1_INDEXED;
003026 }
003027 pNew->wsFlags = saved_wsFlags;
003028 pNew->u.btree.nEq = saved_nEq;
003029 pNew->u.btree.nBtm = saved_nBtm;
003030 pNew->u.btree.nTop = saved_nTop;
003031 pNew->nLTerm = saved_nLTerm;
003032 if( pNew->nLTerm>=pNew->nLSlot
003033 && whereLoopResize(db, pNew, pNew->nLTerm+1)
003034 ){
003035 break; /* OOM while trying to enlarge the pNew->aLTerm array */
003036 }
003037 pNew->aLTerm[pNew->nLTerm++] = pTerm;
003038 pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf;
003039
003040 assert( nInMul==0
003041 || (pNew->wsFlags & WHERE_COLUMN_NULL)!=0
003042 || (pNew->wsFlags & WHERE_COLUMN_IN)!=0
003043 || (pNew->wsFlags & WHERE_SKIPSCAN)!=0
003044 );
003045
003046 if( eOp & WO_IN ){
003047 Expr *pExpr = pTerm->pExpr;
003048 if( ExprUseXSelect(pExpr) ){
003049 /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */
003050 int i;
003051 nIn = 46; assert( 46==sqlite3LogEst(25) );
003052
003053 /* The expression may actually be of the form (x, y) IN (SELECT...).
003054 ** In this case there is a separate term for each of (x) and (y).
003055 ** However, the nIn multiplier should only be applied once, not once
003056 ** for each such term. The following loop checks that pTerm is the
003057 ** first such term in use, and sets nIn back to 0 if it is not. */
003058 for(i=0; i<pNew->nLTerm-1; i++){
003059 if( pNew->aLTerm[i] && pNew->aLTerm[i]->pExpr==pExpr ) nIn = 0;
003060 }
003061 }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){
003062 /* "x IN (value, value, ...)" */
003063 nIn = sqlite3LogEst(pExpr->x.pList->nExpr);
003064 }
003065 if( pProbe->hasStat1 && rLogSize>=10 ){
003066 LogEst M, logK, x;
003067 /* Let:
003068 ** N = the total number of rows in the table
003069 ** K = the number of entries on the RHS of the IN operator
003070 ** M = the number of rows in the table that match terms to the
003071 ** to the left in the same index. If the IN operator is on
003072 ** the left-most index column, M==N.
003073 **
003074 ** Given the definitions above, it is better to omit the IN operator
003075 ** from the index lookup and instead do a scan of the M elements,
003076 ** testing each scanned row against the IN operator separately, if:
003077 **
003078 ** M*log(K) < K*log(N)
003079 **
003080 ** Our estimates for M, K, and N might be inaccurate, so we build in
003081 ** a safety margin of 2 (LogEst: 10) that favors using the IN operator
003082 ** with the index, as using an index has better worst-case behavior.
003083 ** If we do not have real sqlite_stat1 data, always prefer to use
003084 ** the index. Do not bother with this optimization on very small
003085 ** tables (less than 2 rows) as it is pointless in that case.
003086 */
003087 M = pProbe->aiRowLogEst[saved_nEq];
003088 logK = estLog(nIn);
003089 /* TUNING v----- 10 to bias toward indexed IN */
003090 x = M + logK + 10 - (nIn + rLogSize);
003091 if( x>=0 ){
003092 WHERETRACE(0x40,
003093 ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d) "
003094 "prefers indexed lookup\n",
003095 saved_nEq, M, logK, nIn, rLogSize, x));
003096 }else if( nInMul<2 && OptimizationEnabled(db, SQLITE_SeekScan) ){
003097 WHERETRACE(0x40,
003098 ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d"
003099 " nInMul=%d) prefers skip-scan\n",
003100 saved_nEq, M, logK, nIn, rLogSize, x, nInMul));
003101 pNew->wsFlags |= WHERE_IN_SEEKSCAN;
003102 }else{
003103 WHERETRACE(0x40,
003104 ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d"
003105 " nInMul=%d) prefers normal scan\n",
003106 saved_nEq, M, logK, nIn, rLogSize, x, nInMul));
003107 continue;
003108 }
003109 }
003110 pNew->wsFlags |= WHERE_COLUMN_IN;
003111 }else if( eOp & (WO_EQ|WO_IS) ){
003112 int iCol = pProbe->aiColumn[saved_nEq];
003113 pNew->wsFlags |= WHERE_COLUMN_EQ;
003114 assert( saved_nEq==pNew->u.btree.nEq );
003115 if( iCol==XN_ROWID
003116 || (iCol>=0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1)
003117 ){
003118 if( iCol==XN_ROWID || pProbe->uniqNotNull
003119 || (pProbe->nKeyCol==1 && pProbe->onError && eOp==WO_EQ)
003120 ){
003121 pNew->wsFlags |= WHERE_ONEROW;
003122 }else{
003123 pNew->wsFlags |= WHERE_UNQ_WANTED;
003124 }
003125 }
003126 if( scan.iEquiv>1 ) pNew->wsFlags |= WHERE_TRANSCONS;
003127 }else if( eOp & WO_ISNULL ){
003128 pNew->wsFlags |= WHERE_COLUMN_NULL;
003129 }else{
003130 int nVecLen = whereRangeVectorLen(
003131 pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm
003132 );
003133 if( eOp & (WO_GT|WO_GE) ){
003134 testcase( eOp & WO_GT );
003135 testcase( eOp & WO_GE );
003136 pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT;
003137 pNew->u.btree.nBtm = nVecLen;
003138 pBtm = pTerm;
003139 pTop = 0;
003140 if( pTerm->wtFlags & TERM_LIKEOPT ){
003141 /* Range constraints that come from the LIKE optimization are
003142 ** always used in pairs. */
003143 pTop = &pTerm[1];
003144 assert( (pTop-(pTerm->pWC->a))<pTerm->pWC->nTerm );
003145 assert( pTop->wtFlags & TERM_LIKEOPT );
003146 assert( pTop->eOperator==WO_LT );
003147 if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */
003148 pNew->aLTerm[pNew->nLTerm++] = pTop;
003149 pNew->wsFlags |= WHERE_TOP_LIMIT;
003150 pNew->u.btree.nTop = 1;
003151 }
003152 }else{
003153 assert( eOp & (WO_LT|WO_LE) );
003154 testcase( eOp & WO_LT );
003155 testcase( eOp & WO_LE );
003156 pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT;
003157 pNew->u.btree.nTop = nVecLen;
003158 pTop = pTerm;
003159 pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ?
003160 pNew->aLTerm[pNew->nLTerm-2] : 0;
003161 }
003162 }
003163
003164 /* At this point pNew->nOut is set to the number of rows expected to
003165 ** be visited by the index scan before considering term pTerm, or the
003166 ** values of nIn and nInMul. In other words, assuming that all
003167 ** "x IN(...)" terms are replaced with "x = ?". This block updates
003168 ** the value of pNew->nOut to account for pTerm (but not nIn/nInMul). */
003169 assert( pNew->nOut==saved_nOut );
003170 if( pNew->wsFlags & WHERE_COLUMN_RANGE ){
003171 /* Adjust nOut using stat4 data. Or, if there is no stat4
003172 ** data, using some other estimate. */
003173 whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew);
003174 }else{
003175 int nEq = ++pNew->u.btree.nEq;
003176 assert( eOp & (WO_ISNULL|WO_EQ|WO_IN|WO_IS) );
003177
003178 assert( pNew->nOut==saved_nOut );
003179 if( pTerm->truthProb<=0 && pProbe->aiColumn[saved_nEq]>=0 ){
003180 assert( (eOp & WO_IN) || nIn==0 );
003181 testcase( eOp & WO_IN );
003182 pNew->nOut += pTerm->truthProb;
003183 pNew->nOut -= nIn;
003184 }else{
003185 #ifdef SQLITE_ENABLE_STAT4
003186 tRowcnt nOut = 0;
003187 if( nInMul==0
003188 && pProbe->nSample
003189 && ALWAYS(pNew->u.btree.nEq<=pProbe->nSampleCol)
003190 && ((eOp & WO_IN)==0 || ExprUseXList(pTerm->pExpr))
003191 && OptimizationEnabled(db, SQLITE_Stat4)
003192 ){
003193 Expr *pExpr = pTerm->pExpr;
003194 if( (eOp & (WO_EQ|WO_ISNULL|WO_IS))!=0 ){
003195 testcase( eOp & WO_EQ );
003196 testcase( eOp & WO_IS );
003197 testcase( eOp & WO_ISNULL );
003198 rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut);
003199 }else{
003200 rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut);
003201 }
003202 if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK;
003203 if( rc!=SQLITE_OK ) break; /* Jump out of the pTerm loop */
003204 if( nOut ){
003205 pNew->nOut = sqlite3LogEst(nOut);
003206 if( nEq==1
003207 /* TUNING: Mark terms as "low selectivity" if they seem likely
003208 ** to be true for half or more of the rows in the table.
003209 ** See tag-202002240-1 */
003210 && pNew->nOut+10 > pProbe->aiRowLogEst[0]
003211 ){
003212 #if WHERETRACE_ENABLED /* 0x01 */
003213 if( sqlite3WhereTrace & 0x20 ){
003214 sqlite3DebugPrintf(
003215 "STAT4 determines term has low selectivity:\n");
003216 sqlite3WhereTermPrint(pTerm, 999);
003217 }
003218 #endif
003219 pTerm->wtFlags |= TERM_HIGHTRUTH;
003220 if( pTerm->wtFlags & TERM_HEURTRUTH ){
003221 /* If the term has previously been used with an assumption of
003222 ** higher selectivity, then set the flag to rerun the
003223 ** loop computations. */
003224 pBuilder->bldFlags2 |= SQLITE_BLDF2_2NDPASS;
003225 }
003226 }
003227 if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut;
003228 pNew->nOut -= nIn;
003229 }
003230 }
003231 if( nOut==0 )
003232 #endif
003233 {
003234 pNew->nOut += (pProbe->aiRowLogEst[nEq] - pProbe->aiRowLogEst[nEq-1]);
003235 if( eOp & WO_ISNULL ){
003236 /* TUNING: If there is no likelihood() value, assume that a
003237 ** "col IS NULL" expression matches twice as many rows
003238 ** as (col=?). */
003239 pNew->nOut += 10;
003240 }
003241 }
003242 }
003243 }
003244
003245 /* Set rCostIdx to the cost of visiting selected rows in index. Add
003246 ** it to pNew->rRun, which is currently set to the cost of the index
003247 ** seek only. Then, if this is a non-covering index, add the cost of
003248 ** visiting the rows in the main table. */
003249 assert( pSrc->pTab->szTabRow>0 );
003250 if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){
003251 /* The pProbe->szIdxRow is low for an IPK table since the interior
003252 ** pages are small. Thus szIdxRow gives a good estimate of seek cost.
003253 ** But the leaf pages are full-size, so pProbe->szIdxRow would badly
003254 ** under-estimate the scanning cost. */
003255 rCostIdx = pNew->nOut + 16;
003256 }else{
003257 rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow;
003258 }
003259 pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx);
003260 if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK|WHERE_EXPRIDX))==0 ){
003261 pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16);
003262 }
003263 ApplyCostMultiplier(pNew->rRun, pProbe->pTable->costMult);
003264
003265 nOutUnadjusted = pNew->nOut;
003266 pNew->rRun += nInMul + nIn;
003267 pNew->nOut += nInMul + nIn;
003268 whereLoopOutputAdjust(pBuilder->pWC, pNew, rSize);
003269 rc = whereLoopInsert(pBuilder, pNew);
003270
003271 if( pNew->wsFlags & WHERE_COLUMN_RANGE ){
003272 pNew->nOut = saved_nOut;
003273 }else{
003274 pNew->nOut = nOutUnadjusted;
003275 }
003276
003277 if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0
003278 && pNew->u.btree.nEq<pProbe->nColumn
003279 && (pNew->u.btree.nEq<pProbe->nKeyCol ||
003280 pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY)
003281 ){
003282 if( pNew->u.btree.nEq>3 ){
003283 sqlite3ProgressCheck(pParse);
003284 }
003285 whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn);
003286 }
003287 pNew->nOut = saved_nOut;
003288 #ifdef SQLITE_ENABLE_STAT4
003289 pBuilder->nRecValid = nRecValid;
003290 #endif
003291 }
003292 pNew->prereq = saved_prereq;
003293 pNew->u.btree.nEq = saved_nEq;
003294 pNew->u.btree.nBtm = saved_nBtm;
003295 pNew->u.btree.nTop = saved_nTop;
003296 pNew->nSkip = saved_nSkip;
003297 pNew->wsFlags = saved_wsFlags;
003298 pNew->nOut = saved_nOut;
003299 pNew->nLTerm = saved_nLTerm;
003300
003301 /* Consider using a skip-scan if there are no WHERE clause constraints
003302 ** available for the left-most terms of the index, and if the average
003303 ** number of repeats in the left-most terms is at least 18.
003304 **
003305 ** The magic number 18 is selected on the basis that scanning 17 rows
003306 ** is almost always quicker than an index seek (even though if the index
003307 ** contains fewer than 2^17 rows we assume otherwise in other parts of
003308 ** the code). And, even if it is not, it should not be too much slower.
003309 ** On the other hand, the extra seeks could end up being significantly
003310 ** more expensive. */
003311 assert( 42==sqlite3LogEst(18) );
003312 if( saved_nEq==saved_nSkip
003313 && saved_nEq+1<pProbe->nKeyCol
003314 && saved_nEq==pNew->nLTerm
003315 && pProbe->noSkipScan==0
003316 && pProbe->hasStat1!=0
003317 && OptimizationEnabled(db, SQLITE_SkipScan)
003318 && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */
003319 && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK
003320 ){
003321 LogEst nIter;
003322 pNew->u.btree.nEq++;
003323 pNew->nSkip++;
003324 pNew->aLTerm[pNew->nLTerm++] = 0;
003325 pNew->wsFlags |= WHERE_SKIPSCAN;
003326 nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1];
003327 pNew->nOut -= nIter;
003328 /* TUNING: Because uncertainties in the estimates for skip-scan queries,
003329 ** add a 1.375 fudge factor to make skip-scan slightly less likely. */
003330 nIter += 5;
003331 whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul);
003332 pNew->nOut = saved_nOut;
003333 pNew->u.btree.nEq = saved_nEq;
003334 pNew->nSkip = saved_nSkip;
003335 pNew->wsFlags = saved_wsFlags;
003336 }
003337
003338 WHERETRACE(0x800, ("END %s.addBtreeIdx(%s), nEq=%d, rc=%d\n",
003339 pProbe->pTable->zName, pProbe->zName, saved_nEq, rc));
003340 return rc;
003341 }
003342
003343 /*
003344 ** Return True if it is possible that pIndex might be useful in
003345 ** implementing the ORDER BY clause in pBuilder.
003346 **
003347 ** Return False if pBuilder does not contain an ORDER BY clause or
003348 ** if there is no way for pIndex to be useful in implementing that
003349 ** ORDER BY clause.
003350 */
003351 static int indexMightHelpWithOrderBy(
003352 WhereLoopBuilder *pBuilder,
003353 Index *pIndex,
003354 int iCursor
003355 ){
003356 ExprList *pOB;
003357 ExprList *aColExpr;
003358 int ii, jj;
003359
003360 if( pIndex->bUnordered ) return 0;
003361 if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0;
003362 for(ii=0; ii<pOB->nExpr; ii++){
003363 Expr *pExpr = sqlite3ExprSkipCollateAndLikely(pOB->a[ii].pExpr);
003364 if( NEVER(pExpr==0) ) continue;
003365 if( pExpr->op==TK_COLUMN && pExpr->iTable==iCursor ){
003366 if( pExpr->iColumn<0 ) return 1;
003367 for(jj=0; jj<pIndex->nKeyCol; jj++){
003368 if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1;
003369 }
003370 }else if( (aColExpr = pIndex->aColExpr)!=0 ){
003371 for(jj=0; jj<pIndex->nKeyCol; jj++){
003372 if( pIndex->aiColumn[jj]!=XN_EXPR ) continue;
003373 if( sqlite3ExprCompareSkip(pExpr,aColExpr->a[jj].pExpr,iCursor)==0 ){
003374 return 1;
003375 }
003376 }
003377 }
003378 }
003379 return 0;
003380 }
003381
003382 /* Check to see if a partial index with pPartIndexWhere can be used
003383 ** in the current query. Return true if it can be and false if not.
003384 */
003385 static int whereUsablePartialIndex(
003386 int iTab, /* The table for which we want an index */
003387 u8 jointype, /* The JT_* flags on the join */
003388 WhereClause *pWC, /* The WHERE clause of the query */
003389 Expr *pWhere /* The WHERE clause from the partial index */
003390 ){
003391 int i;
003392 WhereTerm *pTerm;
003393 Parse *pParse;
003394
003395 if( jointype & JT_LTORJ ) return 0;
003396 pParse = pWC->pWInfo->pParse;
003397 while( pWhere->op==TK_AND ){
003398 if( !whereUsablePartialIndex(iTab,jointype,pWC,pWhere->pLeft) ) return 0;
003399 pWhere = pWhere->pRight;
003400 }
003401 if( pParse->db->flags & SQLITE_EnableQPSG ) pParse = 0;
003402 for(i=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
003403 Expr *pExpr;
003404 pExpr = pTerm->pExpr;
003405 if( (!ExprHasProperty(pExpr, EP_OuterON) || pExpr->w.iJoin==iTab)
003406 && ((jointype & JT_OUTER)==0 || ExprHasProperty(pExpr, EP_OuterON))
003407 && sqlite3ExprImpliesExpr(pParse, pExpr, pWhere, iTab)
003408 && (pTerm->wtFlags & TERM_VNULL)==0
003409 ){
003410 return 1;
003411 }
003412 }
003413 return 0;
003414 }
003415
003416 /*
003417 ** pIdx is an index containing expressions. Check it see if any of the
003418 ** expressions in the index match the pExpr expression.
003419 */
003420 static int exprIsCoveredByIndex(
003421 const Expr *pExpr,
003422 const Index *pIdx,
003423 int iTabCur
003424 ){
003425 int i;
003426 for(i=0; i<pIdx->nColumn; i++){
003427 if( pIdx->aiColumn[i]==XN_EXPR
003428 && sqlite3ExprCompare(0, pExpr, pIdx->aColExpr->a[i].pExpr, iTabCur)==0
003429 ){
003430 return 1;
003431 }
003432 }
003433 return 0;
003434 }
003435
003436 /*
003437 ** Structure passed to the whereIsCoveringIndex Walker callback.
003438 */
003439 typedef struct CoveringIndexCheck CoveringIndexCheck;
003440 struct CoveringIndexCheck {
003441 Index *pIdx; /* The index */
003442 int iTabCur; /* Cursor number for the corresponding table */
003443 u8 bExpr; /* Uses an indexed expression */
003444 u8 bUnidx; /* Uses an unindexed column not within an indexed expr */
003445 };
003446
003447 /*
003448 ** Information passed in is pWalk->u.pCovIdxCk. Call it pCk.
003449 **
003450 ** If the Expr node references the table with cursor pCk->iTabCur, then
003451 ** make sure that column is covered by the index pCk->pIdx. We know that
003452 ** all columns less than 63 (really BMS-1) are covered, so we don't need
003453 ** to check them. But we do need to check any column at 63 or greater.
003454 **
003455 ** If the index does not cover the column, then set pWalk->eCode to
003456 ** non-zero and return WRC_Abort to stop the search.
003457 **
003458 ** If this node does not disprove that the index can be a covering index,
003459 ** then just return WRC_Continue, to continue the search.
003460 **
003461 ** If pCk->pIdx contains indexed expressions and one of those expressions
003462 ** matches pExpr, then prune the search.
003463 */
003464 static int whereIsCoveringIndexWalkCallback(Walker *pWalk, Expr *pExpr){
003465 int i; /* Loop counter */
003466 const Index *pIdx; /* The index of interest */
003467 const i16 *aiColumn; /* Columns contained in the index */
003468 u16 nColumn; /* Number of columns in the index */
003469 CoveringIndexCheck *pCk; /* Info about this search */
003470
003471 pCk = pWalk->u.pCovIdxCk;
003472 pIdx = pCk->pIdx;
003473 if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN) ){
003474 /* if( pExpr->iColumn<(BMS-1) && pIdx->bHasExpr==0 ) return WRC_Continue;*/
003475 if( pExpr->iTable!=pCk->iTabCur ) return WRC_Continue;
003476 pIdx = pWalk->u.pCovIdxCk->pIdx;
003477 aiColumn = pIdx->aiColumn;
003478 nColumn = pIdx->nColumn;
003479 for(i=0; i<nColumn; i++){
003480 if( aiColumn[i]==pExpr->iColumn ) return WRC_Continue;
003481 }
003482 pCk->bUnidx = 1;
003483 return WRC_Abort;
003484 }else if( pIdx->bHasExpr
003485 && exprIsCoveredByIndex(pExpr, pIdx, pWalk->u.pCovIdxCk->iTabCur) ){
003486 pCk->bExpr = 1;
003487 return WRC_Prune;
003488 }
003489 return WRC_Continue;
003490 }
003491
003492
003493 /*
003494 ** pIdx is an index that covers all of the low-number columns used by
003495 ** pWInfo->pSelect (columns from 0 through 62) or an index that has
003496 ** expressions terms. Hence, we cannot determine whether or not it is
003497 ** a covering index by using the colUsed bitmasks. We have to do a search
003498 ** to see if the index is covering. This routine does that search.
003499 **
003500 ** The return value is one of these:
003501 **
003502 ** 0 The index is definitely not a covering index
003503 **
003504 ** WHERE_IDX_ONLY The index is definitely a covering index
003505 **
003506 ** WHERE_EXPRIDX The index is likely a covering index, but it is
003507 ** difficult to determine precisely because of the
003508 ** expressions that are indexed. Score it as a
003509 ** covering index, but still keep the main table open
003510 ** just in case we need it.
003511 **
003512 ** This routine is an optimization. It is always safe to return zero.
003513 ** But returning one of the other two values when zero should have been
003514 ** returned can lead to incorrect bytecode and assertion faults.
003515 */
003516 static SQLITE_NOINLINE u32 whereIsCoveringIndex(
003517 WhereInfo *pWInfo, /* The WHERE clause context */
003518 Index *pIdx, /* Index that is being tested */
003519 int iTabCur /* Cursor for the table being indexed */
003520 ){
003521 int i, rc;
003522 struct CoveringIndexCheck ck;
003523 Walker w;
003524 if( pWInfo->pSelect==0 ){
003525 /* We don't have access to the full query, so we cannot check to see
003526 ** if pIdx is covering. Assume it is not. */
003527 return 0;
003528 }
003529 if( pIdx->bHasExpr==0 ){
003530 for(i=0; i<pIdx->nColumn; i++){
003531 if( pIdx->aiColumn[i]>=BMS-1 ) break;
003532 }
003533 if( i>=pIdx->nColumn ){
003534 /* pIdx does not index any columns greater than 62, but we know from
003535 ** colMask that columns greater than 62 are used, so this is not a
003536 ** covering index */
003537 return 0;
003538 }
003539 }
003540 ck.pIdx = pIdx;
003541 ck.iTabCur = iTabCur;
003542 ck.bExpr = 0;
003543 ck.bUnidx = 0;
003544 memset(&w, 0, sizeof(w));
003545 w.xExprCallback = whereIsCoveringIndexWalkCallback;
003546 w.xSelectCallback = sqlite3SelectWalkNoop;
003547 w.u.pCovIdxCk = &ck;
003548 sqlite3WalkSelect(&w, pWInfo->pSelect);
003549 if( ck.bUnidx ){
003550 rc = 0;
003551 }else if( ck.bExpr ){
003552 rc = WHERE_EXPRIDX;
003553 }else{
003554 rc = WHERE_IDX_ONLY;
003555 }
003556 return rc;
003557 }
003558
003559 /*
003560 ** This is an sqlite3ParserAddCleanup() callback that is invoked to
003561 ** free the Parse->pIdxEpr list when the Parse object is destroyed.
003562 */
003563 static void whereIndexedExprCleanup(sqlite3 *db, void *pObject){
003564 IndexedExpr **pp = (IndexedExpr**)pObject;
003565 while( *pp!=0 ){
003566 IndexedExpr *p = *pp;
003567 *pp = p->pIENext;
003568 sqlite3ExprDelete(db, p->pExpr);
003569 sqlite3DbFreeNN(db, p);
003570 }
003571 }
003572
003573 /*
003574 ** This function is called for a partial index - one with a WHERE clause - in
003575 ** two scenarios. In both cases, it determines whether or not the WHERE
003576 ** clause on the index implies that a column of the table may be safely
003577 ** replaced by a constant expression. For example, in the following
003578 ** SELECT:
003579 **
003580 ** CREATE INDEX i1 ON t1(b, c) WHERE a=<expr>;
003581 ** SELECT a, b, c FROM t1 WHERE a=<expr> AND b=?;
003582 **
003583 ** The "a" in the select-list may be replaced by <expr>, iff:
003584 **
003585 ** (a) <expr> is a constant expression, and
003586 ** (b) The (a=<expr>) comparison uses the BINARY collation sequence, and
003587 ** (c) Column "a" has an affinity other than NONE or BLOB.
003588 **
003589 ** If argument pItem is NULL, then pMask must not be NULL. In this case this
003590 ** function is being called as part of determining whether or not pIdx
003591 ** is a covering index. This function clears any bits in (*pMask)
003592 ** corresponding to columns that may be replaced by constants as described
003593 ** above.
003594 **
003595 ** Otherwise, if pItem is not NULL, then this function is being called
003596 ** as part of coding a loop that uses index pIdx. In this case, add entries
003597 ** to the Parse.pIdxPartExpr list for each column that can be replaced
003598 ** by a constant.
003599 */
003600 static void wherePartIdxExpr(
003601 Parse *pParse, /* Parse context */
003602 Index *pIdx, /* Partial index being processed */
003603 Expr *pPart, /* WHERE clause being processed */
003604 Bitmask *pMask, /* Mask to clear bits in */
003605 int iIdxCur, /* Cursor number for index */
003606 SrcItem *pItem /* The FROM clause entry for the table */
003607 ){
003608 assert( pItem==0 || (pItem->fg.jointype & JT_RIGHT)==0 );
003609 assert( (pItem==0 || pMask==0) && (pMask!=0 || pItem!=0) );
003610
003611 if( pPart->op==TK_AND ){
003612 wherePartIdxExpr(pParse, pIdx, pPart->pRight, pMask, iIdxCur, pItem);
003613 pPart = pPart->pLeft;
003614 }
003615
003616 if( (pPart->op==TK_EQ || pPart->op==TK_IS) ){
003617 Expr *pLeft = pPart->pLeft;
003618 Expr *pRight = pPart->pRight;
003619 u8 aff;
003620
003621 if( pLeft->op!=TK_COLUMN ) return;
003622 if( !sqlite3ExprIsConstant(pRight) ) return;
003623 if( !sqlite3IsBinary(sqlite3ExprCompareCollSeq(pParse, pPart)) ) return;
003624 if( pLeft->iColumn<0 ) return;
003625 aff = pIdx->pTable->aCol[pLeft->iColumn].affinity;
003626 if( aff>=SQLITE_AFF_TEXT ){
003627 if( pItem ){
003628 sqlite3 *db = pParse->db;
003629 IndexedExpr *p = (IndexedExpr*)sqlite3DbMallocRaw(db, sizeof(*p));
003630 if( p ){
003631 int bNullRow = (pItem->fg.jointype&(JT_LEFT|JT_LTORJ))!=0;
003632 p->pExpr = sqlite3ExprDup(db, pRight, 0);
003633 p->iDataCur = pItem->iCursor;
003634 p->iIdxCur = iIdxCur;
003635 p->iIdxCol = pLeft->iColumn;
003636 p->bMaybeNullRow = bNullRow;
003637 p->pIENext = pParse->pIdxPartExpr;
003638 p->aff = aff;
003639 pParse->pIdxPartExpr = p;
003640 if( p->pIENext==0 ){
003641 void *pArg = (void*)&pParse->pIdxPartExpr;
003642 sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pArg);
003643 }
003644 }
003645 }else if( pLeft->iColumn<(BMS-1) ){
003646 *pMask &= ~((Bitmask)1 << pLeft->iColumn);
003647 }
003648 }
003649 }
003650 }
003651
003652
003653 /*
003654 ** Add all WhereLoop objects for a single table of the join where the table
003655 ** is identified by pBuilder->pNew->iTab. That table is guaranteed to be
003656 ** a b-tree table, not a virtual table.
003657 **
003658 ** The costs (WhereLoop.rRun) of the b-tree loops added by this function
003659 ** are calculated as follows:
003660 **
003661 ** For a full scan, assuming the table (or index) contains nRow rows:
003662 **
003663 ** cost = nRow * 3.0 // full-table scan
003664 ** cost = nRow * K // scan of covering index
003665 ** cost = nRow * (K+3.0) // scan of non-covering index
003666 **
003667 ** where K is a value between 1.1 and 3.0 set based on the relative
003668 ** estimated average size of the index and table records.
003669 **
003670 ** For an index scan, where nVisit is the number of index rows visited
003671 ** by the scan, and nSeek is the number of seek operations required on
003672 ** the index b-tree:
003673 **
003674 ** cost = nSeek * (log(nRow) + K * nVisit) // covering index
003675 ** cost = nSeek * (log(nRow) + (K+3.0) * nVisit) // non-covering index
003676 **
003677 ** Normally, nSeek is 1. nSeek values greater than 1 come about if the
003678 ** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when
003679 ** implicit "x IN (SELECT x FROM tbl)" terms are added for skip-scans.
003680 **
003681 ** The estimated values (nRow, nVisit, nSeek) often contain a large amount
003682 ** of uncertainty. For this reason, scoring is designed to pick plans that
003683 ** "do the least harm" if the estimates are inaccurate. For example, a
003684 ** log(nRow) factor is omitted from a non-covering index scan in order to
003685 ** bias the scoring in favor of using an index, since the worst-case
003686 ** performance of using an index is far better than the worst-case performance
003687 ** of a full table scan.
003688 */
003689 static int whereLoopAddBtree(
003690 WhereLoopBuilder *pBuilder, /* WHERE clause information */
003691 Bitmask mPrereq /* Extra prerequisites for using this table */
003692 ){
003693 WhereInfo *pWInfo; /* WHERE analysis context */
003694 Index *pProbe; /* An index we are evaluating */
003695 Index sPk; /* A fake index object for the primary key */
003696 LogEst aiRowEstPk[2]; /* The aiRowLogEst[] value for the sPk index */
003697 i16 aiColumnPk = -1; /* The aColumn[] value for the sPk index */
003698 SrcList *pTabList; /* The FROM clause */
003699 SrcItem *pSrc; /* The FROM clause btree term to add */
003700 WhereLoop *pNew; /* Template WhereLoop object */
003701 int rc = SQLITE_OK; /* Return code */
003702 int iSortIdx = 1; /* Index number */
003703 int b; /* A boolean value */
003704 LogEst rSize; /* number of rows in the table */
003705 WhereClause *pWC; /* The parsed WHERE clause */
003706 Table *pTab; /* Table being queried */
003707
003708 pNew = pBuilder->pNew;
003709 pWInfo = pBuilder->pWInfo;
003710 pTabList = pWInfo->pTabList;
003711 pSrc = pTabList->a + pNew->iTab;
003712 pTab = pSrc->pTab;
003713 pWC = pBuilder->pWC;
003714 assert( !IsVirtual(pSrc->pTab) );
003715
003716 if( pSrc->fg.isIndexedBy ){
003717 assert( pSrc->fg.isCte==0 );
003718 /* An INDEXED BY clause specifies a particular index to use */
003719 pProbe = pSrc->u2.pIBIndex;
003720 }else if( !HasRowid(pTab) ){
003721 pProbe = pTab->pIndex;
003722 }else{
003723 /* There is no INDEXED BY clause. Create a fake Index object in local
003724 ** variable sPk to represent the rowid primary key index. Make this
003725 ** fake index the first in a chain of Index objects with all of the real
003726 ** indices to follow */
003727 Index *pFirst; /* First of real indices on the table */
003728 memset(&sPk, 0, sizeof(Index));
003729 sPk.nKeyCol = 1;
003730 sPk.nColumn = 1;
003731 sPk.aiColumn = &aiColumnPk;
003732 sPk.aiRowLogEst = aiRowEstPk;
003733 sPk.onError = OE_Replace;
003734 sPk.pTable = pTab;
003735 sPk.szIdxRow = 3; /* TUNING: Interior rows of IPK table are very small */
003736 sPk.idxType = SQLITE_IDXTYPE_IPK;
003737 aiRowEstPk[0] = pTab->nRowLogEst;
003738 aiRowEstPk[1] = 0;
003739 pFirst = pSrc->pTab->pIndex;
003740 if( pSrc->fg.notIndexed==0 ){
003741 /* The real indices of the table are only considered if the
003742 ** NOT INDEXED qualifier is omitted from the FROM clause */
003743 sPk.pNext = pFirst;
003744 }
003745 pProbe = &sPk;
003746 }
003747 rSize = pTab->nRowLogEst;
003748
003749 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
003750 /* Automatic indexes */
003751 if( !pBuilder->pOrSet /* Not part of an OR optimization */
003752 && (pWInfo->wctrlFlags & (WHERE_RIGHT_JOIN|WHERE_OR_SUBCLAUSE))==0
003753 && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0
003754 && !pSrc->fg.isIndexedBy /* Has no INDEXED BY clause */
003755 && !pSrc->fg.notIndexed /* Has no NOT INDEXED clause */
003756 && HasRowid(pTab) /* Not WITHOUT ROWID table. (FIXME: Why not?) */
003757 && !pSrc->fg.isCorrelated /* Not a correlated subquery */
003758 && !pSrc->fg.isRecursive /* Not a recursive common table expression. */
003759 && (pSrc->fg.jointype & JT_RIGHT)==0 /* Not the right tab of a RIGHT JOIN */
003760 ){
003761 /* Generate auto-index WhereLoops */
003762 LogEst rLogSize; /* Logarithm of the number of rows in the table */
003763 WhereTerm *pTerm;
003764 WhereTerm *pWCEnd = pWC->a + pWC->nTerm;
003765 rLogSize = estLog(rSize);
003766 for(pTerm=pWC->a; rc==SQLITE_OK && pTerm<pWCEnd; pTerm++){
003767 if( pTerm->prereqRight & pNew->maskSelf ) continue;
003768 if( termCanDriveIndex(pTerm, pSrc, 0) ){
003769 pNew->u.btree.nEq = 1;
003770 pNew->nSkip = 0;
003771 pNew->u.btree.pIndex = 0;
003772 pNew->nLTerm = 1;
003773 pNew->aLTerm[0] = pTerm;
003774 /* TUNING: One-time cost for computing the automatic index is
003775 ** estimated to be X*N*log2(N) where N is the number of rows in
003776 ** the table being indexed and where X is 7 (LogEst=28) for normal
003777 ** tables or 0.5 (LogEst=-10) for views and subqueries. The value
003778 ** of X is smaller for views and subqueries so that the query planner
003779 ** will be more aggressive about generating automatic indexes for
003780 ** those objects, since there is no opportunity to add schema
003781 ** indexes on subqueries and views. */
003782 pNew->rSetup = rLogSize + rSize;
003783 if( !IsView(pTab) && (pTab->tabFlags & TF_Ephemeral)==0 ){
003784 pNew->rSetup += 28;
003785 }else{
003786 pNew->rSetup -= 25; /* Greatly reduced setup cost for auto indexes
003787 ** on ephemeral materializations of views */
003788 }
003789 ApplyCostMultiplier(pNew->rSetup, pTab->costMult);
003790 if( pNew->rSetup<0 ) pNew->rSetup = 0;
003791 /* TUNING: Each index lookup yields 20 rows in the table. This
003792 ** is more than the usual guess of 10 rows, since we have no way
003793 ** of knowing how selective the index will ultimately be. It would
003794 ** not be unreasonable to make this value much larger. */
003795 pNew->nOut = 43; assert( 43==sqlite3LogEst(20) );
003796 pNew->rRun = sqlite3LogEstAdd(rLogSize,pNew->nOut);
003797 pNew->wsFlags = WHERE_AUTO_INDEX;
003798 pNew->prereq = mPrereq | pTerm->prereqRight;
003799 rc = whereLoopInsert(pBuilder, pNew);
003800 }
003801 }
003802 }
003803 #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
003804
003805 /* Loop over all indices. If there was an INDEXED BY clause, then only
003806 ** consider index pProbe. */
003807 for(; rc==SQLITE_OK && pProbe;
003808 pProbe=(pSrc->fg.isIndexedBy ? 0 : pProbe->pNext), iSortIdx++
003809 ){
003810 if( pProbe->pPartIdxWhere!=0
003811 && !whereUsablePartialIndex(pSrc->iCursor, pSrc->fg.jointype, pWC,
003812 pProbe->pPartIdxWhere)
003813 ){
003814 testcase( pNew->iTab!=pSrc->iCursor ); /* See ticket [98d973b8f5] */
003815 continue; /* Partial index inappropriate for this query */
003816 }
003817 if( pProbe->bNoQuery ) continue;
003818 rSize = pProbe->aiRowLogEst[0];
003819 pNew->u.btree.nEq = 0;
003820 pNew->u.btree.nBtm = 0;
003821 pNew->u.btree.nTop = 0;
003822 pNew->nSkip = 0;
003823 pNew->nLTerm = 0;
003824 pNew->iSortIdx = 0;
003825 pNew->rSetup = 0;
003826 pNew->prereq = mPrereq;
003827 pNew->nOut = rSize;
003828 pNew->u.btree.pIndex = pProbe;
003829 b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor);
003830
003831 /* The ONEPASS_DESIRED flags never occurs together with ORDER BY */
003832 assert( (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || b==0 );
003833 if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){
003834 /* Integer primary key index */
003835 pNew->wsFlags = WHERE_IPK;
003836
003837 /* Full table scan */
003838 pNew->iSortIdx = b ? iSortIdx : 0;
003839 /* TUNING: Cost of full table scan is 3.0*N. The 3.0 factor is an
003840 ** extra cost designed to discourage the use of full table scans,
003841 ** since index lookups have better worst-case performance if our
003842 ** stat guesses are wrong. Reduce the 3.0 penalty slightly
003843 ** (to 2.75) if we have valid STAT4 information for the table.
003844 ** At 2.75, a full table scan is preferred over using an index on
003845 ** a column with just two distinct values where each value has about
003846 ** an equal number of appearances. Without STAT4 data, we still want
003847 ** to use an index in that case, since the constraint might be for
003848 ** the scarcer of the two values, and in that case an index lookup is
003849 ** better.
003850 */
003851 #ifdef SQLITE_ENABLE_STAT4
003852 pNew->rRun = rSize + 16 - 2*((pTab->tabFlags & TF_HasStat4)!=0);
003853 #else
003854 pNew->rRun = rSize + 16;
003855 #endif
003856 ApplyCostMultiplier(pNew->rRun, pTab->costMult);
003857 whereLoopOutputAdjust(pWC, pNew, rSize);
003858 rc = whereLoopInsert(pBuilder, pNew);
003859 pNew->nOut = rSize;
003860 if( rc ) break;
003861 }else{
003862 Bitmask m;
003863 if( pProbe->isCovering ){
003864 m = 0;
003865 pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED;
003866 }else{
003867 m = pSrc->colUsed & pProbe->colNotIdxed;
003868 if( pProbe->pPartIdxWhere ){
003869 wherePartIdxExpr(
003870 pWInfo->pParse, pProbe, pProbe->pPartIdxWhere, &m, 0, 0
003871 );
003872 }
003873 pNew->wsFlags = WHERE_INDEXED;
003874 if( m==TOPBIT || (pProbe->bHasExpr && !pProbe->bHasVCol && m!=0) ){
003875 u32 isCov = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor);
003876 if( isCov==0 ){
003877 WHERETRACE(0x200,
003878 ("-> %s is not a covering index"
003879 " according to whereIsCoveringIndex()\n", pProbe->zName));
003880 assert( m!=0 );
003881 }else{
003882 m = 0;
003883 pNew->wsFlags |= isCov;
003884 if( isCov & WHERE_IDX_ONLY ){
003885 WHERETRACE(0x200,
003886 ("-> %s is a covering expression index"
003887 " according to whereIsCoveringIndex()\n", pProbe->zName));
003888 }else{
003889 assert( isCov==WHERE_EXPRIDX );
003890 WHERETRACE(0x200,
003891 ("-> %s might be a covering expression index"
003892 " according to whereIsCoveringIndex()\n", pProbe->zName));
003893 }
003894 }
003895 }else if( m==0 ){
003896 WHERETRACE(0x200,
003897 ("-> %s a covering index according to bitmasks\n",
003898 pProbe->zName, m==0 ? "is" : "is not"));
003899 pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED;
003900 }
003901 }
003902
003903 /* Full scan via index */
003904 if( b
003905 || !HasRowid(pTab)
003906 || pProbe->pPartIdxWhere!=0
003907 || pSrc->fg.isIndexedBy
003908 || ( m==0
003909 && pProbe->bUnordered==0
003910 && (pProbe->szIdxRow<pTab->szTabRow)
003911 && (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0
003912 && sqlite3GlobalConfig.bUseCis
003913 && OptimizationEnabled(pWInfo->pParse->db, SQLITE_CoverIdxScan)
003914 )
003915 ){
003916 pNew->iSortIdx = b ? iSortIdx : 0;
003917
003918 /* The cost of visiting the index rows is N*K, where K is
003919 ** between 1.1 and 3.0, depending on the relative sizes of the
003920 ** index and table rows. */
003921 pNew->rRun = rSize + 1 + (15*pProbe->szIdxRow)/pTab->szTabRow;
003922 if( m!=0 ){
003923 /* If this is a non-covering index scan, add in the cost of
003924 ** doing table lookups. The cost will be 3x the number of
003925 ** lookups. Take into account WHERE clause terms that can be
003926 ** satisfied using just the index, and that do not require a
003927 ** table lookup. */
003928 LogEst nLookup = rSize + 16; /* Base cost: N*3 */
003929 int ii;
003930 int iCur = pSrc->iCursor;
003931 WhereClause *pWC2 = &pWInfo->sWC;
003932 for(ii=0; ii<pWC2->nTerm; ii++){
003933 WhereTerm *pTerm = &pWC2->a[ii];
003934 if( !sqlite3ExprCoveredByIndex(pTerm->pExpr, iCur, pProbe) ){
003935 break;
003936 }
003937 /* pTerm can be evaluated using just the index. So reduce
003938 ** the expected number of table lookups accordingly */
003939 if( pTerm->truthProb<=0 ){
003940 nLookup += pTerm->truthProb;
003941 }else{
003942 nLookup--;
003943 if( pTerm->eOperator & (WO_EQ|WO_IS) ) nLookup -= 19;
003944 }
003945 }
003946
003947 pNew->rRun = sqlite3LogEstAdd(pNew->rRun, nLookup);
003948 }
003949 ApplyCostMultiplier(pNew->rRun, pTab->costMult);
003950 whereLoopOutputAdjust(pWC, pNew, rSize);
003951 if( (pSrc->fg.jointype & JT_RIGHT)!=0 && pProbe->aColExpr ){
003952 /* Do not do an SCAN of a index-on-expression in a RIGHT JOIN
003953 ** because the cursor used to access the index might not be
003954 ** positioned to the correct row during the right-join no-match
003955 ** loop. */
003956 }else{
003957 rc = whereLoopInsert(pBuilder, pNew);
003958 }
003959 pNew->nOut = rSize;
003960 if( rc ) break;
003961 }
003962 }
003963
003964 pBuilder->bldFlags1 = 0;
003965 rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0);
003966 if( pBuilder->bldFlags1==SQLITE_BLDF1_INDEXED ){
003967 /* If a non-unique index is used, or if a prefix of the key for
003968 ** unique index is used (making the index functionally non-unique)
003969 ** then the sqlite_stat1 data becomes important for scoring the
003970 ** plan */
003971 pTab->tabFlags |= TF_StatsUsed;
003972 }
003973 #ifdef SQLITE_ENABLE_STAT4
003974 sqlite3Stat4ProbeFree(pBuilder->pRec);
003975 pBuilder->nRecValid = 0;
003976 pBuilder->pRec = 0;
003977 #endif
003978 }
003979 return rc;
003980 }
003981
003982 #ifndef SQLITE_OMIT_VIRTUALTABLE
003983
003984 /*
003985 ** Return true if pTerm is a virtual table LIMIT or OFFSET term.
003986 */
003987 static int isLimitTerm(WhereTerm *pTerm){
003988 assert( pTerm->eOperator==WO_AUX || pTerm->eMatchOp==0 );
003989 return pTerm->eMatchOp>=SQLITE_INDEX_CONSTRAINT_LIMIT
003990 && pTerm->eMatchOp<=SQLITE_INDEX_CONSTRAINT_OFFSET;
003991 }
003992
003993 /*
003994 ** Argument pIdxInfo is already populated with all constraints that may
003995 ** be used by the virtual table identified by pBuilder->pNew->iTab. This
003996 ** function marks a subset of those constraints usable, invokes the
003997 ** xBestIndex method and adds the returned plan to pBuilder.
003998 **
003999 ** A constraint is marked usable if:
004000 **
004001 ** * Argument mUsable indicates that its prerequisites are available, and
004002 **
004003 ** * It is not one of the operators specified in the mExclude mask passed
004004 ** as the fourth argument (which in practice is either WO_IN or 0).
004005 **
004006 ** Argument mPrereq is a mask of tables that must be scanned before the
004007 ** virtual table in question. These are added to the plans prerequisites
004008 ** before it is added to pBuilder.
004009 **
004010 ** Output parameter *pbIn is set to true if the plan added to pBuilder
004011 ** uses one or more WO_IN terms, or false otherwise.
004012 */
004013 static int whereLoopAddVirtualOne(
004014 WhereLoopBuilder *pBuilder,
004015 Bitmask mPrereq, /* Mask of tables that must be used. */
004016 Bitmask mUsable, /* Mask of usable tables */
004017 u16 mExclude, /* Exclude terms using these operators */
004018 sqlite3_index_info *pIdxInfo, /* Populated object for xBestIndex */
004019 u16 mNoOmit, /* Do not omit these constraints */
004020 int *pbIn, /* OUT: True if plan uses an IN(...) op */
004021 int *pbRetryLimit /* OUT: Retry without LIMIT/OFFSET */
004022 ){
004023 WhereClause *pWC = pBuilder->pWC;
004024 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
004025 struct sqlite3_index_constraint *pIdxCons;
004026 struct sqlite3_index_constraint_usage *pUsage = pIdxInfo->aConstraintUsage;
004027 int i;
004028 int mxTerm;
004029 int rc = SQLITE_OK;
004030 WhereLoop *pNew = pBuilder->pNew;
004031 Parse *pParse = pBuilder->pWInfo->pParse;
004032 SrcItem *pSrc = &pBuilder->pWInfo->pTabList->a[pNew->iTab];
004033 int nConstraint = pIdxInfo->nConstraint;
004034
004035 assert( (mUsable & mPrereq)==mPrereq );
004036 *pbIn = 0;
004037 pNew->prereq = mPrereq;
004038
004039 /* Set the usable flag on the subset of constraints identified by
004040 ** arguments mUsable and mExclude. */
004041 pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
004042 for(i=0; i<nConstraint; i++, pIdxCons++){
004043 WhereTerm *pTerm = &pWC->a[pIdxCons->iTermOffset];
004044 pIdxCons->usable = 0;
004045 if( (pTerm->prereqRight & mUsable)==pTerm->prereqRight
004046 && (pTerm->eOperator & mExclude)==0
004047 && (pbRetryLimit || !isLimitTerm(pTerm))
004048 ){
004049 pIdxCons->usable = 1;
004050 }
004051 }
004052
004053 /* Initialize the output fields of the sqlite3_index_info structure */
004054 memset(pUsage, 0, sizeof(pUsage[0])*nConstraint);
004055 assert( pIdxInfo->needToFreeIdxStr==0 );
004056 pIdxInfo->idxStr = 0;
004057 pIdxInfo->idxNum = 0;
004058 pIdxInfo->orderByConsumed = 0;
004059 pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2;
004060 pIdxInfo->estimatedRows = 25;
004061 pIdxInfo->idxFlags = 0;
004062 pIdxInfo->colUsed = (sqlite3_int64)pSrc->colUsed;
004063 pHidden->mHandleIn = 0;
004064
004065 /* Invoke the virtual table xBestIndex() method */
004066 rc = vtabBestIndex(pParse, pSrc->pTab, pIdxInfo);
004067 if( rc ){
004068 if( rc==SQLITE_CONSTRAINT ){
004069 /* If the xBestIndex method returns SQLITE_CONSTRAINT, that means
004070 ** that the particular combination of parameters provided is unusable.
004071 ** Make no entries in the loop table.
004072 */
004073 WHERETRACE(0xffffffff, (" ^^^^--- non-viable plan rejected!\n"));
004074 return SQLITE_OK;
004075 }
004076 return rc;
004077 }
004078
004079 mxTerm = -1;
004080 assert( pNew->nLSlot>=nConstraint );
004081 memset(pNew->aLTerm, 0, sizeof(pNew->aLTerm[0])*nConstraint );
004082 memset(&pNew->u.vtab, 0, sizeof(pNew->u.vtab));
004083 pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
004084 for(i=0; i<nConstraint; i++, pIdxCons++){
004085 int iTerm;
004086 if( (iTerm = pUsage[i].argvIndex - 1)>=0 ){
004087 WhereTerm *pTerm;
004088 int j = pIdxCons->iTermOffset;
004089 if( iTerm>=nConstraint
004090 || j<0
004091 || j>=pWC->nTerm
004092 || pNew->aLTerm[iTerm]!=0
004093 || pIdxCons->usable==0
004094 ){
004095 sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName);
004096 testcase( pIdxInfo->needToFreeIdxStr );
004097 return SQLITE_ERROR;
004098 }
004099 testcase( iTerm==nConstraint-1 );
004100 testcase( j==0 );
004101 testcase( j==pWC->nTerm-1 );
004102 pTerm = &pWC->a[j];
004103 pNew->prereq |= pTerm->prereqRight;
004104 assert( iTerm<pNew->nLSlot );
004105 pNew->aLTerm[iTerm] = pTerm;
004106 if( iTerm>mxTerm ) mxTerm = iTerm;
004107 testcase( iTerm==15 );
004108 testcase( iTerm==16 );
004109 if( pUsage[i].omit ){
004110 if( i<16 && ((1<<i)&mNoOmit)==0 ){
004111 testcase( i!=iTerm );
004112 pNew->u.vtab.omitMask |= 1<<iTerm;
004113 }else{
004114 testcase( i!=iTerm );
004115 }
004116 if( pTerm->eMatchOp==SQLITE_INDEX_CONSTRAINT_OFFSET ){
004117 pNew->u.vtab.bOmitOffset = 1;
004118 }
004119 }
004120 if( SMASKBIT32(i) & pHidden->mHandleIn ){
004121 pNew->u.vtab.mHandleIn |= MASKBIT32(iTerm);
004122 }else if( (pTerm->eOperator & WO_IN)!=0 ){
004123 /* A virtual table that is constrained by an IN clause may not
004124 ** consume the ORDER BY clause because (1) the order of IN terms
004125 ** is not necessarily related to the order of output terms and
004126 ** (2) Multiple outputs from a single IN value will not merge
004127 ** together. */
004128 pIdxInfo->orderByConsumed = 0;
004129 pIdxInfo->idxFlags &= ~SQLITE_INDEX_SCAN_UNIQUE;
004130 *pbIn = 1; assert( (mExclude & WO_IN)==0 );
004131 }
004132
004133 assert( pbRetryLimit || !isLimitTerm(pTerm) );
004134 if( isLimitTerm(pTerm) && *pbIn ){
004135 /* If there is an IN(...) term handled as an == (separate call to
004136 ** xFilter for each value on the RHS of the IN) and a LIMIT or
004137 ** OFFSET term handled as well, the plan is unusable. Set output
004138 ** variable *pbRetryLimit to true to tell the caller to retry with
004139 ** LIMIT and OFFSET disabled. */
004140 if( pIdxInfo->needToFreeIdxStr ){
004141 sqlite3_free(pIdxInfo->idxStr);
004142 pIdxInfo->idxStr = 0;
004143 pIdxInfo->needToFreeIdxStr = 0;
004144 }
004145 *pbRetryLimit = 1;
004146 return SQLITE_OK;
004147 }
004148 }
004149 }
004150
004151 pNew->nLTerm = mxTerm+1;
004152 for(i=0; i<=mxTerm; i++){
004153 if( pNew->aLTerm[i]==0 ){
004154 /* The non-zero argvIdx values must be contiguous. Raise an
004155 ** error if they are not */
004156 sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName);
004157 testcase( pIdxInfo->needToFreeIdxStr );
004158 return SQLITE_ERROR;
004159 }
004160 }
004161 assert( pNew->nLTerm<=pNew->nLSlot );
004162 pNew->u.vtab.idxNum = pIdxInfo->idxNum;
004163 pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr;
004164 pIdxInfo->needToFreeIdxStr = 0;
004165 pNew->u.vtab.idxStr = pIdxInfo->idxStr;
004166 pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ?
004167 pIdxInfo->nOrderBy : 0);
004168 pNew->rSetup = 0;
004169 pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost);
004170 pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows);
004171
004172 /* Set the WHERE_ONEROW flag if the xBestIndex() method indicated
004173 ** that the scan will visit at most one row. Clear it otherwise. */
004174 if( pIdxInfo->idxFlags & SQLITE_INDEX_SCAN_UNIQUE ){
004175 pNew->wsFlags |= WHERE_ONEROW;
004176 }else{
004177 pNew->wsFlags &= ~WHERE_ONEROW;
004178 }
004179 rc = whereLoopInsert(pBuilder, pNew);
004180 if( pNew->u.vtab.needFree ){
004181 sqlite3_free(pNew->u.vtab.idxStr);
004182 pNew->u.vtab.needFree = 0;
004183 }
004184 WHERETRACE(0xffffffff, (" bIn=%d prereqIn=%04llx prereqOut=%04llx\n",
004185 *pbIn, (sqlite3_uint64)mPrereq,
004186 (sqlite3_uint64)(pNew->prereq & ~mPrereq)));
004187
004188 return rc;
004189 }
004190
004191 /*
004192 ** Return the collating sequence for a constraint passed into xBestIndex.
004193 **
004194 ** pIdxInfo must be an sqlite3_index_info structure passed into xBestIndex.
004195 ** This routine depends on there being a HiddenIndexInfo structure immediately
004196 ** following the sqlite3_index_info structure.
004197 **
004198 ** Return a pointer to the collation name:
004199 **
004200 ** 1. If there is an explicit COLLATE operator on the constraint, return it.
004201 **
004202 ** 2. Else, if the column has an alternative collation, return that.
004203 **
004204 ** 3. Otherwise, return "BINARY".
004205 */
004206 const char *sqlite3_vtab_collation(sqlite3_index_info *pIdxInfo, int iCons){
004207 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
004208 const char *zRet = 0;
004209 if( iCons>=0 && iCons<pIdxInfo->nConstraint ){
004210 CollSeq *pC = 0;
004211 int iTerm = pIdxInfo->aConstraint[iCons].iTermOffset;
004212 Expr *pX = pHidden->pWC->a[iTerm].pExpr;
004213 if( pX->pLeft ){
004214 pC = sqlite3ExprCompareCollSeq(pHidden->pParse, pX);
004215 }
004216 zRet = (pC ? pC->zName : sqlite3StrBINARY);
004217 }
004218 return zRet;
004219 }
004220
004221 /*
004222 ** Return true if constraint iCons is really an IN(...) constraint, or
004223 ** false otherwise. If iCons is an IN(...) constraint, set (if bHandle!=0)
004224 ** or clear (if bHandle==0) the flag to handle it using an iterator.
004225 */
004226 int sqlite3_vtab_in(sqlite3_index_info *pIdxInfo, int iCons, int bHandle){
004227 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
004228 u32 m = SMASKBIT32(iCons);
004229 if( m & pHidden->mIn ){
004230 if( bHandle==0 ){
004231 pHidden->mHandleIn &= ~m;
004232 }else if( bHandle>0 ){
004233 pHidden->mHandleIn |= m;
004234 }
004235 return 1;
004236 }
004237 return 0;
004238 }
004239
004240 /*
004241 ** This interface is callable from within the xBestIndex callback only.
004242 **
004243 ** If possible, set (*ppVal) to point to an object containing the value
004244 ** on the right-hand-side of constraint iCons.
004245 */
004246 int sqlite3_vtab_rhs_value(
004247 sqlite3_index_info *pIdxInfo, /* Copy of first argument to xBestIndex */
004248 int iCons, /* Constraint for which RHS is wanted */
004249 sqlite3_value **ppVal /* Write value extracted here */
004250 ){
004251 HiddenIndexInfo *pH = (HiddenIndexInfo*)&pIdxInfo[1];
004252 sqlite3_value *pVal = 0;
004253 int rc = SQLITE_OK;
004254 if( iCons<0 || iCons>=pIdxInfo->nConstraint ){
004255 rc = SQLITE_MISUSE_BKPT; /* EV: R-30545-25046 */
004256 }else{
004257 if( pH->aRhs[iCons]==0 ){
004258 WhereTerm *pTerm = &pH->pWC->a[pIdxInfo->aConstraint[iCons].iTermOffset];
004259 rc = sqlite3ValueFromExpr(
004260 pH->pParse->db, pTerm->pExpr->pRight, ENC(pH->pParse->db),
004261 SQLITE_AFF_BLOB, &pH->aRhs[iCons]
004262 );
004263 testcase( rc!=SQLITE_OK );
004264 }
004265 pVal = pH->aRhs[iCons];
004266 }
004267 *ppVal = pVal;
004268
004269 if( rc==SQLITE_OK && pVal==0 ){ /* IMP: R-19933-32160 */
004270 rc = SQLITE_NOTFOUND; /* IMP: R-36424-56542 */
004271 }
004272
004273 return rc;
004274 }
004275
004276 /*
004277 ** Return true if ORDER BY clause may be handled as DISTINCT.
004278 */
004279 int sqlite3_vtab_distinct(sqlite3_index_info *pIdxInfo){
004280 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
004281 assert( pHidden->eDistinct>=0 && pHidden->eDistinct<=3 );
004282 return pHidden->eDistinct;
004283 }
004284
004285 /*
004286 ** Cause the prepared statement that is associated with a call to
004287 ** xBestIndex to potentially use all schemas. If the statement being
004288 ** prepared is read-only, then just start read transactions on all
004289 ** schemas. But if this is a write operation, start writes on all
004290 ** schemas.
004291 **
004292 ** This is used by the (built-in) sqlite_dbpage virtual table.
004293 */
004294 void sqlite3VtabUsesAllSchemas(Parse *pParse){
004295 int nDb = pParse->db->nDb;
004296 int i;
004297 for(i=0; i<nDb; i++){
004298 sqlite3CodeVerifySchema(pParse, i);
004299 }
004300 if( DbMaskNonZero(pParse->writeMask) ){
004301 for(i=0; i<nDb; i++){
004302 sqlite3BeginWriteOperation(pParse, 0, i);
004303 }
004304 }
004305 }
004306
004307 /*
004308 ** Add all WhereLoop objects for a table of the join identified by
004309 ** pBuilder->pNew->iTab. That table is guaranteed to be a virtual table.
004310 **
004311 ** If there are no LEFT or CROSS JOIN joins in the query, both mPrereq and
004312 ** mUnusable are set to 0. Otherwise, mPrereq is a mask of all FROM clause
004313 ** entries that occur before the virtual table in the FROM clause and are
004314 ** separated from it by at least one LEFT or CROSS JOIN. Similarly, the
004315 ** mUnusable mask contains all FROM clause entries that occur after the
004316 ** virtual table and are separated from it by at least one LEFT or
004317 ** CROSS JOIN.
004318 **
004319 ** For example, if the query were:
004320 **
004321 ** ... FROM t1, t2 LEFT JOIN t3, t4, vt CROSS JOIN t5, t6;
004322 **
004323 ** then mPrereq corresponds to (t1, t2) and mUnusable to (t5, t6).
004324 **
004325 ** All the tables in mPrereq must be scanned before the current virtual
004326 ** table. So any terms for which all prerequisites are satisfied by
004327 ** mPrereq may be specified as "usable" in all calls to xBestIndex.
004328 ** Conversely, all tables in mUnusable must be scanned after the current
004329 ** virtual table, so any terms for which the prerequisites overlap with
004330 ** mUnusable should always be configured as "not-usable" for xBestIndex.
004331 */
004332 static int whereLoopAddVirtual(
004333 WhereLoopBuilder *pBuilder, /* WHERE clause information */
004334 Bitmask mPrereq, /* Tables that must be scanned before this one */
004335 Bitmask mUnusable /* Tables that must be scanned after this one */
004336 ){
004337 int rc = SQLITE_OK; /* Return code */
004338 WhereInfo *pWInfo; /* WHERE analysis context */
004339 Parse *pParse; /* The parsing context */
004340 WhereClause *pWC; /* The WHERE clause */
004341 SrcItem *pSrc; /* The FROM clause term to search */
004342 sqlite3_index_info *p; /* Object to pass to xBestIndex() */
004343 int nConstraint; /* Number of constraints in p */
004344 int bIn; /* True if plan uses IN(...) operator */
004345 WhereLoop *pNew;
004346 Bitmask mBest; /* Tables used by best possible plan */
004347 u16 mNoOmit;
004348 int bRetry = 0; /* True to retry with LIMIT/OFFSET disabled */
004349
004350 assert( (mPrereq & mUnusable)==0 );
004351 pWInfo = pBuilder->pWInfo;
004352 pParse = pWInfo->pParse;
004353 pWC = pBuilder->pWC;
004354 pNew = pBuilder->pNew;
004355 pSrc = &pWInfo->pTabList->a[pNew->iTab];
004356 assert( IsVirtual(pSrc->pTab) );
004357 p = allocateIndexInfo(pWInfo, pWC, mUnusable, pSrc, &mNoOmit);
004358 if( p==0 ) return SQLITE_NOMEM_BKPT;
004359 pNew->rSetup = 0;
004360 pNew->wsFlags = WHERE_VIRTUALTABLE;
004361 pNew->nLTerm = 0;
004362 pNew->u.vtab.needFree = 0;
004363 nConstraint = p->nConstraint;
004364 if( whereLoopResize(pParse->db, pNew, nConstraint) ){
004365 freeIndexInfo(pParse->db, p);
004366 return SQLITE_NOMEM_BKPT;
004367 }
004368
004369 /* First call xBestIndex() with all constraints usable. */
004370 WHERETRACE(0x800, ("BEGIN %s.addVirtual()\n", pSrc->pTab->zName));
004371 WHERETRACE(0x800, (" VirtualOne: all usable\n"));
004372 rc = whereLoopAddVirtualOne(
004373 pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, &bRetry
004374 );
004375 if( bRetry ){
004376 assert( rc==SQLITE_OK );
004377 rc = whereLoopAddVirtualOne(
004378 pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, 0
004379 );
004380 }
004381
004382 /* If the call to xBestIndex() with all terms enabled produced a plan
004383 ** that does not require any source tables (IOW: a plan with mBest==0)
004384 ** and does not use an IN(...) operator, then there is no point in making
004385 ** any further calls to xBestIndex() since they will all return the same
004386 ** result (if the xBestIndex() implementation is sane). */
004387 if( rc==SQLITE_OK && ((mBest = (pNew->prereq & ~mPrereq))!=0 || bIn) ){
004388 int seenZero = 0; /* True if a plan with no prereqs seen */
004389 int seenZeroNoIN = 0; /* Plan with no prereqs and no IN(...) seen */
004390 Bitmask mPrev = 0;
004391 Bitmask mBestNoIn = 0;
004392
004393 /* If the plan produced by the earlier call uses an IN(...) term, call
004394 ** xBestIndex again, this time with IN(...) terms disabled. */
004395 if( bIn ){
004396 WHERETRACE(0x800, (" VirtualOne: all usable w/o IN\n"));
004397 rc = whereLoopAddVirtualOne(
004398 pBuilder, mPrereq, ALLBITS, WO_IN, p, mNoOmit, &bIn, 0);
004399 assert( bIn==0 );
004400 mBestNoIn = pNew->prereq & ~mPrereq;
004401 if( mBestNoIn==0 ){
004402 seenZero = 1;
004403 seenZeroNoIN = 1;
004404 }
004405 }
004406
004407 /* Call xBestIndex once for each distinct value of (prereqRight & ~mPrereq)
004408 ** in the set of terms that apply to the current virtual table. */
004409 while( rc==SQLITE_OK ){
004410 int i;
004411 Bitmask mNext = ALLBITS;
004412 assert( mNext>0 );
004413 for(i=0; i<nConstraint; i++){
004414 Bitmask mThis = (
004415 pWC->a[p->aConstraint[i].iTermOffset].prereqRight & ~mPrereq
004416 );
004417 if( mThis>mPrev && mThis<mNext ) mNext = mThis;
004418 }
004419 mPrev = mNext;
004420 if( mNext==ALLBITS ) break;
004421 if( mNext==mBest || mNext==mBestNoIn ) continue;
004422 WHERETRACE(0x800, (" VirtualOne: mPrev=%04llx mNext=%04llx\n",
004423 (sqlite3_uint64)mPrev, (sqlite3_uint64)mNext));
004424 rc = whereLoopAddVirtualOne(
004425 pBuilder, mPrereq, mNext|mPrereq, 0, p, mNoOmit, &bIn, 0);
004426 if( pNew->prereq==mPrereq ){
004427 seenZero = 1;
004428 if( bIn==0 ) seenZeroNoIN = 1;
004429 }
004430 }
004431
004432 /* If the calls to xBestIndex() in the above loop did not find a plan
004433 ** that requires no source tables at all (i.e. one guaranteed to be
004434 ** usable), make a call here with all source tables disabled */
004435 if( rc==SQLITE_OK && seenZero==0 ){
004436 WHERETRACE(0x800, (" VirtualOne: all disabled\n"));
004437 rc = whereLoopAddVirtualOne(
004438 pBuilder, mPrereq, mPrereq, 0, p, mNoOmit, &bIn, 0);
004439 if( bIn==0 ) seenZeroNoIN = 1;
004440 }
004441
004442 /* If the calls to xBestIndex() have so far failed to find a plan
004443 ** that requires no source tables at all and does not use an IN(...)
004444 ** operator, make a final call to obtain one here. */
004445 if( rc==SQLITE_OK && seenZeroNoIN==0 ){
004446 WHERETRACE(0x800, (" VirtualOne: all disabled and w/o IN\n"));
004447 rc = whereLoopAddVirtualOne(
004448 pBuilder, mPrereq, mPrereq, WO_IN, p, mNoOmit, &bIn, 0);
004449 }
004450 }
004451
004452 if( p->needToFreeIdxStr ) sqlite3_free(p->idxStr);
004453 freeIndexInfo(pParse->db, p);
004454 WHERETRACE(0x800, ("END %s.addVirtual(), rc=%d\n", pSrc->pTab->zName, rc));
004455 return rc;
004456 }
004457 #endif /* SQLITE_OMIT_VIRTUALTABLE */
004458
004459 /*
004460 ** Add WhereLoop entries to handle OR terms. This works for either
004461 ** btrees or virtual tables.
004462 */
004463 static int whereLoopAddOr(
004464 WhereLoopBuilder *pBuilder,
004465 Bitmask mPrereq,
004466 Bitmask mUnusable
004467 ){
004468 WhereInfo *pWInfo = pBuilder->pWInfo;
004469 WhereClause *pWC;
004470 WhereLoop *pNew;
004471 WhereTerm *pTerm, *pWCEnd;
004472 int rc = SQLITE_OK;
004473 int iCur;
004474 WhereClause tempWC;
004475 WhereLoopBuilder sSubBuild;
004476 WhereOrSet sSum, sCur;
004477 SrcItem *pItem;
004478
004479 pWC = pBuilder->pWC;
004480 pWCEnd = pWC->a + pWC->nTerm;
004481 pNew = pBuilder->pNew;
004482 memset(&sSum, 0, sizeof(sSum));
004483 pItem = pWInfo->pTabList->a + pNew->iTab;
004484 iCur = pItem->iCursor;
004485
004486 /* The multi-index OR optimization does not work for RIGHT and FULL JOIN */
004487 if( pItem->fg.jointype & JT_RIGHT ) return SQLITE_OK;
004488
004489 for(pTerm=pWC->a; pTerm<pWCEnd && rc==SQLITE_OK; pTerm++){
004490 if( (pTerm->eOperator & WO_OR)!=0
004491 && (pTerm->u.pOrInfo->indexable & pNew->maskSelf)!=0
004492 ){
004493 WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc;
004494 WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm];
004495 WhereTerm *pOrTerm;
004496 int once = 1;
004497 int i, j;
004498
004499 sSubBuild = *pBuilder;
004500 sSubBuild.pOrSet = &sCur;
004501
004502 WHERETRACE(0x400, ("Begin processing OR-clause %p\n", pTerm));
004503 for(pOrTerm=pOrWC->a; pOrTerm<pOrWCEnd; pOrTerm++){
004504 if( (pOrTerm->eOperator & WO_AND)!=0 ){
004505 sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc;
004506 }else if( pOrTerm->leftCursor==iCur ){
004507 tempWC.pWInfo = pWC->pWInfo;
004508 tempWC.pOuter = pWC;
004509 tempWC.op = TK_AND;
004510 tempWC.nTerm = 1;
004511 tempWC.nBase = 1;
004512 tempWC.a = pOrTerm;
004513 sSubBuild.pWC = &tempWC;
004514 }else{
004515 continue;
004516 }
004517 sCur.n = 0;
004518 #ifdef WHERETRACE_ENABLED
004519 WHERETRACE(0x400, ("OR-term %d of %p has %d subterms:\n",
004520 (int)(pOrTerm-pOrWC->a), pTerm, sSubBuild.pWC->nTerm));
004521 if( sqlite3WhereTrace & 0x20000 ){
004522 sqlite3WhereClausePrint(sSubBuild.pWC);
004523 }
004524 #endif
004525 #ifndef SQLITE_OMIT_VIRTUALTABLE
004526 if( IsVirtual(pItem->pTab) ){
004527 rc = whereLoopAddVirtual(&sSubBuild, mPrereq, mUnusable);
004528 }else
004529 #endif
004530 {
004531 rc = whereLoopAddBtree(&sSubBuild, mPrereq);
004532 }
004533 if( rc==SQLITE_OK ){
004534 rc = whereLoopAddOr(&sSubBuild, mPrereq, mUnusable);
004535 }
004536 testcase( rc==SQLITE_NOMEM && sCur.n>0 );
004537 testcase( rc==SQLITE_DONE );
004538 if( sCur.n==0 ){
004539 sSum.n = 0;
004540 break;
004541 }else if( once ){
004542 whereOrMove(&sSum, &sCur);
004543 once = 0;
004544 }else{
004545 WhereOrSet sPrev;
004546 whereOrMove(&sPrev, &sSum);
004547 sSum.n = 0;
004548 for(i=0; i<sPrev.n; i++){
004549 for(j=0; j<sCur.n; j++){
004550 whereOrInsert(&sSum, sPrev.a[i].prereq | sCur.a[j].prereq,
004551 sqlite3LogEstAdd(sPrev.a[i].rRun, sCur.a[j].rRun),
004552 sqlite3LogEstAdd(sPrev.a[i].nOut, sCur.a[j].nOut));
004553 }
004554 }
004555 }
004556 }
004557 pNew->nLTerm = 1;
004558 pNew->aLTerm[0] = pTerm;
004559 pNew->wsFlags = WHERE_MULTI_OR;
004560 pNew->rSetup = 0;
004561 pNew->iSortIdx = 0;
004562 memset(&pNew->u, 0, sizeof(pNew->u));
004563 for(i=0; rc==SQLITE_OK && i<sSum.n; i++){
004564 /* TUNING: Currently sSum.a[i].rRun is set to the sum of the costs
004565 ** of all sub-scans required by the OR-scan. However, due to rounding
004566 ** errors, it may be that the cost of the OR-scan is equal to its
004567 ** most expensive sub-scan. Add the smallest possible penalty
004568 ** (equivalent to multiplying the cost by 1.07) to ensure that
004569 ** this does not happen. Otherwise, for WHERE clauses such as the
004570 ** following where there is an index on "y":
004571 **
004572 ** WHERE likelihood(x=?, 0.99) OR y=?
004573 **
004574 ** the planner may elect to "OR" together a full-table scan and an
004575 ** index lookup. And other similarly odd results. */
004576 pNew->rRun = sSum.a[i].rRun + 1;
004577 pNew->nOut = sSum.a[i].nOut;
004578 pNew->prereq = sSum.a[i].prereq;
004579 rc = whereLoopInsert(pBuilder, pNew);
004580 }
004581 WHERETRACE(0x400, ("End processing OR-clause %p\n", pTerm));
004582 }
004583 }
004584 return rc;
004585 }
004586
004587 /*
004588 ** Add all WhereLoop objects for all tables
004589 */
004590 static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
004591 WhereInfo *pWInfo = pBuilder->pWInfo;
004592 Bitmask mPrereq = 0;
004593 Bitmask mPrior = 0;
004594 int iTab;
004595 SrcList *pTabList = pWInfo->pTabList;
004596 SrcItem *pItem;
004597 SrcItem *pEnd = &pTabList->a[pWInfo->nLevel];
004598 sqlite3 *db = pWInfo->pParse->db;
004599 int rc = SQLITE_OK;
004600 int bFirstPastRJ = 0;
004601 int hasRightJoin = 0;
004602 WhereLoop *pNew;
004603
004604
004605 /* Loop over the tables in the join, from left to right */
004606 pNew = pBuilder->pNew;
004607
004608 /* Verify that pNew has already been initialized */
004609 assert( pNew->nLTerm==0 );
004610 assert( pNew->wsFlags==0 );
004611 assert( pNew->nLSlot>=ArraySize(pNew->aLTermSpace) );
004612 assert( pNew->aLTerm!=0 );
004613
004614 pBuilder->iPlanLimit = SQLITE_QUERY_PLANNER_LIMIT;
004615 for(iTab=0, pItem=pTabList->a; pItem<pEnd; iTab++, pItem++){
004616 Bitmask mUnusable = 0;
004617 pNew->iTab = iTab;
004618 pBuilder->iPlanLimit += SQLITE_QUERY_PLANNER_LIMIT_INCR;
004619 pNew->maskSelf = sqlite3WhereGetMask(&pWInfo->sMaskSet, pItem->iCursor);
004620 if( bFirstPastRJ
004621 || (pItem->fg.jointype & (JT_OUTER|JT_CROSS|JT_LTORJ))!=0
004622 ){
004623 /* Add prerequisites to prevent reordering of FROM clause terms
004624 ** across CROSS joins and outer joins. The bFirstPastRJ boolean
004625 ** prevents the right operand of a RIGHT JOIN from being swapped with
004626 ** other elements even further to the right.
004627 **
004628 ** The JT_LTORJ case and the hasRightJoin flag work together to
004629 ** prevent FROM-clause terms from moving from the right side of
004630 ** a LEFT JOIN over to the left side of that join if the LEFT JOIN
004631 ** is itself on the left side of a RIGHT JOIN.
004632 */
004633 if( pItem->fg.jointype & JT_LTORJ ) hasRightJoin = 1;
004634 mPrereq |= mPrior;
004635 bFirstPastRJ = (pItem->fg.jointype & JT_RIGHT)!=0;
004636 }else if( !hasRightJoin ){
004637 mPrereq = 0;
004638 }
004639 #ifndef SQLITE_OMIT_VIRTUALTABLE
004640 if( IsVirtual(pItem->pTab) ){
004641 SrcItem *p;
004642 for(p=&pItem[1]; p<pEnd; p++){
004643 if( mUnusable || (p->fg.jointype & (JT_OUTER|JT_CROSS)) ){
004644 mUnusable |= sqlite3WhereGetMask(&pWInfo->sMaskSet, p->iCursor);
004645 }
004646 }
004647 rc = whereLoopAddVirtual(pBuilder, mPrereq, mUnusable);
004648 }else
004649 #endif /* SQLITE_OMIT_VIRTUALTABLE */
004650 {
004651 rc = whereLoopAddBtree(pBuilder, mPrereq);
004652 }
004653 if( rc==SQLITE_OK && pBuilder->pWC->hasOr ){
004654 rc = whereLoopAddOr(pBuilder, mPrereq, mUnusable);
004655 }
004656 mPrior |= pNew->maskSelf;
004657 if( rc || db->mallocFailed ){
004658 if( rc==SQLITE_DONE ){
004659 /* We hit the query planner search limit set by iPlanLimit */
004660 sqlite3_log(SQLITE_WARNING, "abbreviated query algorithm search");
004661 rc = SQLITE_OK;
004662 }else{
004663 break;
004664 }
004665 }
004666 }
004667
004668 whereLoopClear(db, pNew);
004669 return rc;
004670 }
004671
004672 /*
004673 ** Examine a WherePath (with the addition of the extra WhereLoop of the 6th
004674 ** parameters) to see if it outputs rows in the requested ORDER BY
004675 ** (or GROUP BY) without requiring a separate sort operation. Return N:
004676 **
004677 ** N>0: N terms of the ORDER BY clause are satisfied
004678 ** N==0: No terms of the ORDER BY clause are satisfied
004679 ** N<0: Unknown yet how many terms of ORDER BY might be satisfied.
004680 **
004681 ** Note that processing for WHERE_GROUPBY and WHERE_DISTINCTBY is not as
004682 ** strict. With GROUP BY and DISTINCT the only requirement is that
004683 ** equivalent rows appear immediately adjacent to one another. GROUP BY
004684 ** and DISTINCT do not require rows to appear in any particular order as long
004685 ** as equivalent rows are grouped together. Thus for GROUP BY and DISTINCT
004686 ** the pOrderBy terms can be matched in any order. With ORDER BY, the
004687 ** pOrderBy terms must be matched in strict left-to-right order.
004688 */
004689 static i8 wherePathSatisfiesOrderBy(
004690 WhereInfo *pWInfo, /* The WHERE clause */
004691 ExprList *pOrderBy, /* ORDER BY or GROUP BY or DISTINCT clause to check */
004692 WherePath *pPath, /* The WherePath to check */
004693 u16 wctrlFlags, /* WHERE_GROUPBY or _DISTINCTBY or _ORDERBY_LIMIT */
004694 u16 nLoop, /* Number of entries in pPath->aLoop[] */
004695 WhereLoop *pLast, /* Add this WhereLoop to the end of pPath->aLoop[] */
004696 Bitmask *pRevMask /* OUT: Mask of WhereLoops to run in reverse order */
004697 ){
004698 u8 revSet; /* True if rev is known */
004699 u8 rev; /* Composite sort order */
004700 u8 revIdx; /* Index sort order */
004701 u8 isOrderDistinct; /* All prior WhereLoops are order-distinct */
004702 u8 distinctColumns; /* True if the loop has UNIQUE NOT NULL columns */
004703 u8 isMatch; /* iColumn matches a term of the ORDER BY clause */
004704 u16 eqOpMask; /* Allowed equality operators */
004705 u16 nKeyCol; /* Number of key columns in pIndex */
004706 u16 nColumn; /* Total number of ordered columns in the index */
004707 u16 nOrderBy; /* Number terms in the ORDER BY clause */
004708 int iLoop; /* Index of WhereLoop in pPath being processed */
004709 int i, j; /* Loop counters */
004710 int iCur; /* Cursor number for current WhereLoop */
004711 int iColumn; /* A column number within table iCur */
004712 WhereLoop *pLoop = 0; /* Current WhereLoop being processed. */
004713 WhereTerm *pTerm; /* A single term of the WHERE clause */
004714 Expr *pOBExpr; /* An expression from the ORDER BY clause */
004715 CollSeq *pColl; /* COLLATE function from an ORDER BY clause term */
004716 Index *pIndex; /* The index associated with pLoop */
004717 sqlite3 *db = pWInfo->pParse->db; /* Database connection */
004718 Bitmask obSat = 0; /* Mask of ORDER BY terms satisfied so far */
004719 Bitmask obDone; /* Mask of all ORDER BY terms */
004720 Bitmask orderDistinctMask; /* Mask of all well-ordered loops */
004721 Bitmask ready; /* Mask of inner loops */
004722
004723 /*
004724 ** We say the WhereLoop is "one-row" if it generates no more than one
004725 ** row of output. A WhereLoop is one-row if all of the following are true:
004726 ** (a) All index columns match with WHERE_COLUMN_EQ.
004727 ** (b) The index is unique
004728 ** Any WhereLoop with an WHERE_COLUMN_EQ constraint on the rowid is one-row.
004729 ** Every one-row WhereLoop will have the WHERE_ONEROW bit set in wsFlags.
004730 **
004731 ** We say the WhereLoop is "order-distinct" if the set of columns from
004732 ** that WhereLoop that are in the ORDER BY clause are different for every
004733 ** row of the WhereLoop. Every one-row WhereLoop is automatically
004734 ** order-distinct. A WhereLoop that has no columns in the ORDER BY clause
004735 ** is not order-distinct. To be order-distinct is not quite the same as being
004736 ** UNIQUE since a UNIQUE column or index can have multiple rows that
004737 ** are NULL and NULL values are equivalent for the purpose of order-distinct.
004738 ** To be order-distinct, the columns must be UNIQUE and NOT NULL.
004739 **
004740 ** The rowid for a table is always UNIQUE and NOT NULL so whenever the
004741 ** rowid appears in the ORDER BY clause, the corresponding WhereLoop is
004742 ** automatically order-distinct.
004743 */
004744
004745 assert( pOrderBy!=0 );
004746 if( nLoop && OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return 0;
004747
004748 nOrderBy = pOrderBy->nExpr;
004749 testcase( nOrderBy==BMS-1 );
004750 if( nOrderBy>BMS-1 ) return 0; /* Cannot optimize overly large ORDER BYs */
004751 isOrderDistinct = 1;
004752 obDone = MASKBIT(nOrderBy)-1;
004753 orderDistinctMask = 0;
004754 ready = 0;
004755 eqOpMask = WO_EQ | WO_IS | WO_ISNULL;
004756 if( wctrlFlags & (WHERE_ORDERBY_LIMIT|WHERE_ORDERBY_MAX|WHERE_ORDERBY_MIN) ){
004757 eqOpMask |= WO_IN;
004758 }
004759 for(iLoop=0; isOrderDistinct && obSat<obDone && iLoop<=nLoop; iLoop++){
004760 if( iLoop>0 ) ready |= pLoop->maskSelf;
004761 if( iLoop<nLoop ){
004762 pLoop = pPath->aLoop[iLoop];
004763 if( wctrlFlags & WHERE_ORDERBY_LIMIT ) continue;
004764 }else{
004765 pLoop = pLast;
004766 }
004767 if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){
004768 if( pLoop->u.vtab.isOrdered
004769 && ((wctrlFlags&(WHERE_DISTINCTBY|WHERE_SORTBYGROUP))!=WHERE_DISTINCTBY)
004770 ){
004771 obSat = obDone;
004772 }
004773 break;
004774 }else if( wctrlFlags & WHERE_DISTINCTBY ){
004775 pLoop->u.btree.nDistinctCol = 0;
004776 }
004777 iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor;
004778
004779 /* Mark off any ORDER BY term X that is a column in the table of
004780 ** the current loop for which there is term in the WHERE
004781 ** clause of the form X IS NULL or X=? that reference only outer
004782 ** loops.
004783 */
004784 for(i=0; i<nOrderBy; i++){
004785 if( MASKBIT(i) & obSat ) continue;
004786 pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr);
004787 if( NEVER(pOBExpr==0) ) continue;
004788 if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue;
004789 if( pOBExpr->iTable!=iCur ) continue;
004790 pTerm = sqlite3WhereFindTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn,
004791 ~ready, eqOpMask, 0);
004792 if( pTerm==0 ) continue;
004793 if( pTerm->eOperator==WO_IN ){
004794 /* IN terms are only valid for sorting in the ORDER BY LIMIT
004795 ** optimization, and then only if they are actually used
004796 ** by the query plan */
004797 assert( wctrlFlags &
004798 (WHERE_ORDERBY_LIMIT|WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX) );
004799 for(j=0; j<pLoop->nLTerm && pTerm!=pLoop->aLTerm[j]; j++){}
004800 if( j>=pLoop->nLTerm ) continue;
004801 }
004802 if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 && pOBExpr->iColumn>=0 ){
004803 Parse *pParse = pWInfo->pParse;
004804 CollSeq *pColl1 = sqlite3ExprNNCollSeq(pParse, pOrderBy->a[i].pExpr);
004805 CollSeq *pColl2 = sqlite3ExprCompareCollSeq(pParse, pTerm->pExpr);
004806 assert( pColl1 );
004807 if( pColl2==0 || sqlite3StrICmp(pColl1->zName, pColl2->zName) ){
004808 continue;
004809 }
004810 testcase( pTerm->pExpr->op==TK_IS );
004811 }
004812 obSat |= MASKBIT(i);
004813 }
004814
004815 if( (pLoop->wsFlags & WHERE_ONEROW)==0 ){
004816 if( pLoop->wsFlags & WHERE_IPK ){
004817 pIndex = 0;
004818 nKeyCol = 0;
004819 nColumn = 1;
004820 }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){
004821 return 0;
004822 }else{
004823 nKeyCol = pIndex->nKeyCol;
004824 nColumn = pIndex->nColumn;
004825 assert( nColumn==nKeyCol+1 || !HasRowid(pIndex->pTable) );
004826 assert( pIndex->aiColumn[nColumn-1]==XN_ROWID
004827 || !HasRowid(pIndex->pTable));
004828 /* All relevant terms of the index must also be non-NULL in order
004829 ** for isOrderDistinct to be true. So the isOrderDistint value
004830 ** computed here might be a false positive. Corrections will be
004831 ** made at tag-20210426-1 below */
004832 isOrderDistinct = IsUniqueIndex(pIndex)
004833 && (pLoop->wsFlags & WHERE_SKIPSCAN)==0;
004834 }
004835
004836 /* Loop through all columns of the index and deal with the ones
004837 ** that are not constrained by == or IN.
004838 */
004839 rev = revSet = 0;
004840 distinctColumns = 0;
004841 for(j=0; j<nColumn; j++){
004842 u8 bOnce = 1; /* True to run the ORDER BY search loop */
004843
004844 assert( j>=pLoop->u.btree.nEq
004845 || (pLoop->aLTerm[j]==0)==(j<pLoop->nSkip)
004846 );
004847 if( j<pLoop->u.btree.nEq && j>=pLoop->nSkip ){
004848 u16 eOp = pLoop->aLTerm[j]->eOperator;
004849
004850 /* Skip over == and IS and ISNULL terms. (Also skip IN terms when
004851 ** doing WHERE_ORDERBY_LIMIT processing). Except, IS and ISNULL
004852 ** terms imply that the index is not UNIQUE NOT NULL in which case
004853 ** the loop need to be marked as not order-distinct because it can
004854 ** have repeated NULL rows.
004855 **
004856 ** If the current term is a column of an ((?,?) IN (SELECT...))
004857 ** expression for which the SELECT returns more than one column,
004858 ** check that it is the only column used by this loop. Otherwise,
004859 ** if it is one of two or more, none of the columns can be
004860 ** considered to match an ORDER BY term.
004861 */
004862 if( (eOp & eqOpMask)!=0 ){
004863 if( eOp & (WO_ISNULL|WO_IS) ){
004864 testcase( eOp & WO_ISNULL );
004865 testcase( eOp & WO_IS );
004866 testcase( isOrderDistinct );
004867 isOrderDistinct = 0;
004868 }
004869 continue;
004870 }else if( ALWAYS(eOp & WO_IN) ){
004871 /* ALWAYS() justification: eOp is an equality operator due to the
004872 ** j<pLoop->u.btree.nEq constraint above. Any equality other
004873 ** than WO_IN is captured by the previous "if". So this one
004874 ** always has to be WO_IN. */
004875 Expr *pX = pLoop->aLTerm[j]->pExpr;
004876 for(i=j+1; i<pLoop->u.btree.nEq; i++){
004877 if( pLoop->aLTerm[i]->pExpr==pX ){
004878 assert( (pLoop->aLTerm[i]->eOperator & WO_IN) );
004879 bOnce = 0;
004880 break;
004881 }
004882 }
004883 }
004884 }
004885
004886 /* Get the column number in the table (iColumn) and sort order
004887 ** (revIdx) for the j-th column of the index.
004888 */
004889 if( pIndex ){
004890 iColumn = pIndex->aiColumn[j];
004891 revIdx = pIndex->aSortOrder[j] & KEYINFO_ORDER_DESC;
004892 if( iColumn==pIndex->pTable->iPKey ) iColumn = XN_ROWID;
004893 }else{
004894 iColumn = XN_ROWID;
004895 revIdx = 0;
004896 }
004897
004898 /* An unconstrained column that might be NULL means that this
004899 ** WhereLoop is not well-ordered. tag-20210426-1
004900 */
004901 if( isOrderDistinct ){
004902 if( iColumn>=0
004903 && j>=pLoop->u.btree.nEq
004904 && pIndex->pTable->aCol[iColumn].notNull==0
004905 ){
004906 isOrderDistinct = 0;
004907 }
004908 if( iColumn==XN_EXPR ){
004909 isOrderDistinct = 0;
004910 }
004911 }
004912
004913 /* Find the ORDER BY term that corresponds to the j-th column
004914 ** of the index and mark that ORDER BY term off
004915 */
004916 isMatch = 0;
004917 for(i=0; bOnce && i<nOrderBy; i++){
004918 if( MASKBIT(i) & obSat ) continue;
004919 pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr);
004920 testcase( wctrlFlags & WHERE_GROUPBY );
004921 testcase( wctrlFlags & WHERE_DISTINCTBY );
004922 if( NEVER(pOBExpr==0) ) continue;
004923 if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0;
004924 if( iColumn>=XN_ROWID ){
004925 if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue;
004926 if( pOBExpr->iTable!=iCur ) continue;
004927 if( pOBExpr->iColumn!=iColumn ) continue;
004928 }else{
004929 Expr *pIxExpr = pIndex->aColExpr->a[j].pExpr;
004930 if( sqlite3ExprCompareSkip(pOBExpr, pIxExpr, iCur) ){
004931 continue;
004932 }
004933 }
004934 if( iColumn!=XN_ROWID ){
004935 pColl = sqlite3ExprNNCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr);
004936 if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue;
004937 }
004938 if( wctrlFlags & WHERE_DISTINCTBY ){
004939 pLoop->u.btree.nDistinctCol = j+1;
004940 }
004941 isMatch = 1;
004942 break;
004943 }
004944 if( isMatch && (wctrlFlags & WHERE_GROUPBY)==0 ){
004945 /* Make sure the sort order is compatible in an ORDER BY clause.
004946 ** Sort order is irrelevant for a GROUP BY clause. */
004947 if( revSet ){
004948 if( (rev ^ revIdx)
004949 != (pOrderBy->a[i].fg.sortFlags&KEYINFO_ORDER_DESC)
004950 ){
004951 isMatch = 0;
004952 }
004953 }else{
004954 rev = revIdx ^ (pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_DESC);
004955 if( rev ) *pRevMask |= MASKBIT(iLoop);
004956 revSet = 1;
004957 }
004958 }
004959 if( isMatch && (pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_BIGNULL) ){
004960 if( j==pLoop->u.btree.nEq ){
004961 pLoop->wsFlags |= WHERE_BIGNULL_SORT;
004962 }else{
004963 isMatch = 0;
004964 }
004965 }
004966 if( isMatch ){
004967 if( iColumn==XN_ROWID ){
004968 testcase( distinctColumns==0 );
004969 distinctColumns = 1;
004970 }
004971 obSat |= MASKBIT(i);
004972 }else{
004973 /* No match found */
004974 if( j==0 || j<nKeyCol ){
004975 testcase( isOrderDistinct!=0 );
004976 isOrderDistinct = 0;
004977 }
004978 break;
004979 }
004980 } /* end Loop over all index columns */
004981 if( distinctColumns ){
004982 testcase( isOrderDistinct==0 );
004983 isOrderDistinct = 1;
004984 }
004985 } /* end-if not one-row */
004986
004987 /* Mark off any other ORDER BY terms that reference pLoop */
004988 if( isOrderDistinct ){
004989 orderDistinctMask |= pLoop->maskSelf;
004990 for(i=0; i<nOrderBy; i++){
004991 Expr *p;
004992 Bitmask mTerm;
004993 if( MASKBIT(i) & obSat ) continue;
004994 p = pOrderBy->a[i].pExpr;
004995 mTerm = sqlite3WhereExprUsage(&pWInfo->sMaskSet,p);
004996 if( mTerm==0 && !sqlite3ExprIsConstant(p) ) continue;
004997 if( (mTerm&~orderDistinctMask)==0 ){
004998 obSat |= MASKBIT(i);
004999 }
005000 }
005001 }
005002 } /* End the loop over all WhereLoops from outer-most down to inner-most */
005003 if( obSat==obDone ) return (i8)nOrderBy;
005004 if( !isOrderDistinct ){
005005 for(i=nOrderBy-1; i>0; i--){
005006 Bitmask m = ALWAYS(i<BMS) ? MASKBIT(i) - 1 : 0;
005007 if( (obSat&m)==m ) return i;
005008 }
005009 return 0;
005010 }
005011 return -1;
005012 }
005013
005014
005015 /*
005016 ** If the WHERE_GROUPBY flag is set in the mask passed to sqlite3WhereBegin(),
005017 ** the planner assumes that the specified pOrderBy list is actually a GROUP
005018 ** BY clause - and so any order that groups rows as required satisfies the
005019 ** request.
005020 **
005021 ** Normally, in this case it is not possible for the caller to determine
005022 ** whether or not the rows are really being delivered in sorted order, or
005023 ** just in some other order that provides the required grouping. However,
005024 ** if the WHERE_SORTBYGROUP flag is also passed to sqlite3WhereBegin(), then
005025 ** this function may be called on the returned WhereInfo object. It returns
005026 ** true if the rows really will be sorted in the specified order, or false
005027 ** otherwise.
005028 **
005029 ** For example, assuming:
005030 **
005031 ** CREATE INDEX i1 ON t1(x, Y);
005032 **
005033 ** then
005034 **
005035 ** SELECT * FROM t1 GROUP BY x,y ORDER BY x,y; -- IsSorted()==1
005036 ** SELECT * FROM t1 GROUP BY y,x ORDER BY y,x; -- IsSorted()==0
005037 */
005038 int sqlite3WhereIsSorted(WhereInfo *pWInfo){
005039 assert( pWInfo->wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY) );
005040 assert( pWInfo->wctrlFlags & WHERE_SORTBYGROUP );
005041 return pWInfo->sorted;
005042 }
005043
005044 #ifdef WHERETRACE_ENABLED
005045 /* For debugging use only: */
005046 static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){
005047 static char zName[65];
005048 int i;
005049 for(i=0; i<nLoop; i++){ zName[i] = pPath->aLoop[i]->cId; }
005050 if( pLast ) zName[i++] = pLast->cId;
005051 zName[i] = 0;
005052 return zName;
005053 }
005054 #endif
005055
005056 /*
005057 ** Return the cost of sorting nRow rows, assuming that the keys have
005058 ** nOrderby columns and that the first nSorted columns are already in
005059 ** order.
005060 */
005061 static LogEst whereSortingCost(
005062 WhereInfo *pWInfo, /* Query planning context */
005063 LogEst nRow, /* Estimated number of rows to sort */
005064 int nOrderBy, /* Number of ORDER BY clause terms */
005065 int nSorted /* Number of initial ORDER BY terms naturally in order */
005066 ){
005067 /* Estimated cost of a full external sort, where N is
005068 ** the number of rows to sort is:
005069 **
005070 ** cost = (K * N * log(N)).
005071 **
005072 ** Or, if the order-by clause has X terms but only the last Y
005073 ** terms are out of order, then block-sorting will reduce the
005074 ** sorting cost to:
005075 **
005076 ** cost = (K * N * log(N)) * (Y/X)
005077 **
005078 ** The constant K is at least 2.0 but will be larger if there are a
005079 ** large number of columns to be sorted, as the sorting time is
005080 ** proportional to the amount of content to be sorted. The algorithm
005081 ** does not currently distinguish between fat columns (BLOBs and TEXTs)
005082 ** and skinny columns (INTs). It just uses the number of columns as
005083 ** an approximation for the row width.
005084 **
005085 ** And extra factor of 2.0 or 3.0 is added to the sorting cost if the sort
005086 ** is built using OP_IdxInsert and OP_Sort rather than with OP_SorterInsert.
005087 */
005088 LogEst rSortCost, nCol;
005089 assert( pWInfo->pSelect!=0 );
005090 assert( pWInfo->pSelect->pEList!=0 );
005091 /* TUNING: sorting cost proportional to the number of output columns: */
005092 nCol = sqlite3LogEst((pWInfo->pSelect->pEList->nExpr+59)/30);
005093 rSortCost = nRow + nCol;
005094 if( nSorted>0 ){
005095 /* Scale the result by (Y/X) */
005096 rSortCost += sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66;
005097 }
005098
005099 /* Multiple by log(M) where M is the number of output rows.
005100 ** Use the LIMIT for M if it is smaller. Or if this sort is for
005101 ** a DISTINCT operator, M will be the number of distinct output
005102 ** rows, so fudge it downwards a bit.
005103 */
005104 if( (pWInfo->wctrlFlags & WHERE_USE_LIMIT)!=0 ){
005105 rSortCost += 10; /* TUNING: Extra 2.0x if using LIMIT */
005106 if( nSorted!=0 ){
005107 rSortCost += 6; /* TUNING: Extra 1.5x if also using partial sort */
005108 }
005109 if( pWInfo->iLimit<nRow ){
005110 nRow = pWInfo->iLimit;
005111 }
005112 }else if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT) ){
005113 /* TUNING: In the sort for a DISTINCT operator, assume that the DISTINCT
005114 ** reduces the number of output rows by a factor of 2 */
005115 if( nRow>10 ){ nRow -= 10; assert( 10==sqlite3LogEst(2) ); }
005116 }
005117 rSortCost += estLog(nRow);
005118 return rSortCost;
005119 }
005120
005121 /*
005122 ** Given the list of WhereLoop objects at pWInfo->pLoops, this routine
005123 ** attempts to find the lowest cost path that visits each WhereLoop
005124 ** once. This path is then loaded into the pWInfo->a[].pWLoop fields.
005125 **
005126 ** Assume that the total number of output rows that will need to be sorted
005127 ** will be nRowEst (in the 10*log2 representation). Or, ignore sorting
005128 ** costs if nRowEst==0.
005129 **
005130 ** Return SQLITE_OK on success or SQLITE_NOMEM of a memory allocation
005131 ** error occurs.
005132 */
005133 static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
005134 int mxChoice; /* Maximum number of simultaneous paths tracked */
005135 int nLoop; /* Number of terms in the join */
005136 Parse *pParse; /* Parsing context */
005137 int iLoop; /* Loop counter over the terms of the join */
005138 int ii, jj; /* Loop counters */
005139 int mxI = 0; /* Index of next entry to replace */
005140 int nOrderBy; /* Number of ORDER BY clause terms */
005141 LogEst mxCost = 0; /* Maximum cost of a set of paths */
005142 LogEst mxUnsorted = 0; /* Maximum unsorted cost of a set of path */
005143 int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */
005144 WherePath *aFrom; /* All nFrom paths at the previous level */
005145 WherePath *aTo; /* The nTo best paths at the current level */
005146 WherePath *pFrom; /* An element of aFrom[] that we are working on */
005147 WherePath *pTo; /* An element of aTo[] that we are working on */
005148 WhereLoop *pWLoop; /* One of the WhereLoop objects */
005149 WhereLoop **pX; /* Used to divy up the pSpace memory */
005150 LogEst *aSortCost = 0; /* Sorting and partial sorting costs */
005151 char *pSpace; /* Temporary memory used by this routine */
005152 int nSpace; /* Bytes of space allocated at pSpace */
005153
005154 pParse = pWInfo->pParse;
005155 nLoop = pWInfo->nLevel;
005156 /* TUNING: For simple queries, only the best path is tracked.
005157 ** For 2-way joins, the 5 best paths are followed.
005158 ** For joins of 3 or more tables, track the 10 best paths */
005159 mxChoice = (nLoop<=1) ? 1 : (nLoop==2 ? 5 : 10);
005160 assert( nLoop<=pWInfo->pTabList->nSrc );
005161 WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d, nQueryLoop=%d)\n",
005162 nRowEst, pParse->nQueryLoop));
005163
005164 /* If nRowEst is zero and there is an ORDER BY clause, ignore it. In this
005165 ** case the purpose of this call is to estimate the number of rows returned
005166 ** by the overall query. Once this estimate has been obtained, the caller
005167 ** will invoke this function a second time, passing the estimate as the
005168 ** nRowEst parameter. */
005169 if( pWInfo->pOrderBy==0 || nRowEst==0 ){
005170 nOrderBy = 0;
005171 }else{
005172 nOrderBy = pWInfo->pOrderBy->nExpr;
005173 }
005174
005175 /* Allocate and initialize space for aTo, aFrom and aSortCost[] */
005176 nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2;
005177 nSpace += sizeof(LogEst) * nOrderBy;
005178 pSpace = sqlite3StackAllocRawNN(pParse->db, nSpace);
005179 if( pSpace==0 ) return SQLITE_NOMEM_BKPT;
005180 aTo = (WherePath*)pSpace;
005181 aFrom = aTo+mxChoice;
005182 memset(aFrom, 0, sizeof(aFrom[0]));
005183 pX = (WhereLoop**)(aFrom+mxChoice);
005184 for(ii=mxChoice*2, pFrom=aTo; ii>0; ii--, pFrom++, pX += nLoop){
005185 pFrom->aLoop = pX;
005186 }
005187 if( nOrderBy ){
005188 /* If there is an ORDER BY clause and it is not being ignored, set up
005189 ** space for the aSortCost[] array. Each element of the aSortCost array
005190 ** is either zero - meaning it has not yet been initialized - or the
005191 ** cost of sorting nRowEst rows of data where the first X terms of
005192 ** the ORDER BY clause are already in order, where X is the array
005193 ** index. */
005194 aSortCost = (LogEst*)pX;
005195 memset(aSortCost, 0, sizeof(LogEst) * nOrderBy);
005196 }
005197 assert( aSortCost==0 || &pSpace[nSpace]==(char*)&aSortCost[nOrderBy] );
005198 assert( aSortCost!=0 || &pSpace[nSpace]==(char*)pX );
005199
005200 /* Seed the search with a single WherePath containing zero WhereLoops.
005201 **
005202 ** TUNING: Do not let the number of iterations go above 28. If the cost
005203 ** of computing an automatic index is not paid back within the first 28
005204 ** rows, then do not use the automatic index. */
005205 aFrom[0].nRow = MIN(pParse->nQueryLoop, 48); assert( 48==sqlite3LogEst(28) );
005206 nFrom = 1;
005207 assert( aFrom[0].isOrdered==0 );
005208 if( nOrderBy ){
005209 /* If nLoop is zero, then there are no FROM terms in the query. Since
005210 ** in this case the query may return a maximum of one row, the results
005211 ** are already in the requested order. Set isOrdered to nOrderBy to
005212 ** indicate this. Or, if nLoop is greater than zero, set isOrdered to
005213 ** -1, indicating that the result set may or may not be ordered,
005214 ** depending on the loops added to the current plan. */
005215 aFrom[0].isOrdered = nLoop>0 ? -1 : nOrderBy;
005216 }
005217
005218 /* Compute successively longer WherePaths using the previous generation
005219 ** of WherePaths as the basis for the next. Keep track of the mxChoice
005220 ** best paths at each generation */
005221 for(iLoop=0; iLoop<nLoop; iLoop++){
005222 nTo = 0;
005223 for(ii=0, pFrom=aFrom; ii<nFrom; ii++, pFrom++){
005224 for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){
005225 LogEst nOut; /* Rows visited by (pFrom+pWLoop) */
005226 LogEst rCost; /* Cost of path (pFrom+pWLoop) */
005227 LogEst rUnsorted; /* Unsorted cost of (pFrom+pWLoop) */
005228 i8 isOrdered; /* isOrdered for (pFrom+pWLoop) */
005229 Bitmask maskNew; /* Mask of src visited by (..) */
005230 Bitmask revMask; /* Mask of rev-order loops for (..) */
005231
005232 if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue;
005233 if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue;
005234 if( (pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 && pFrom->nRow<3 ){
005235 /* Do not use an automatic index if the this loop is expected
005236 ** to run less than 1.25 times. It is tempting to also exclude
005237 ** automatic index usage on an outer loop, but sometimes an automatic
005238 ** index is useful in the outer loop of a correlated subquery. */
005239 assert( 10==sqlite3LogEst(2) );
005240 continue;
005241 }
005242
005243 /* At this point, pWLoop is a candidate to be the next loop.
005244 ** Compute its cost */
005245 rUnsorted = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow);
005246 rUnsorted = sqlite3LogEstAdd(rUnsorted, pFrom->rUnsorted);
005247 nOut = pFrom->nRow + pWLoop->nOut;
005248 maskNew = pFrom->maskLoop | pWLoop->maskSelf;
005249 isOrdered = pFrom->isOrdered;
005250 if( isOrdered<0 ){
005251 revMask = 0;
005252 isOrdered = wherePathSatisfiesOrderBy(pWInfo,
005253 pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags,
005254 iLoop, pWLoop, &revMask);
005255 }else{
005256 revMask = pFrom->revLoop;
005257 }
005258 if( isOrdered>=0 && isOrdered<nOrderBy ){
005259 if( aSortCost[isOrdered]==0 ){
005260 aSortCost[isOrdered] = whereSortingCost(
005261 pWInfo, nRowEst, nOrderBy, isOrdered
005262 );
005263 }
005264 /* TUNING: Add a small extra penalty (3) to sorting as an
005265 ** extra encouragement to the query planner to select a plan
005266 ** where the rows emerge in the correct order without any sorting
005267 ** required. */
005268 rCost = sqlite3LogEstAdd(rUnsorted, aSortCost[isOrdered]) + 3;
005269
005270 WHERETRACE(0x002,
005271 ("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n",
005272 aSortCost[isOrdered], (nOrderBy-isOrdered), nOrderBy,
005273 rUnsorted, rCost));
005274 }else{
005275 rCost = rUnsorted;
005276 rUnsorted -= 2; /* TUNING: Slight bias in favor of no-sort plans */
005277 }
005278
005279 /* Check to see if pWLoop should be added to the set of
005280 ** mxChoice best-so-far paths.
005281 **
005282 ** First look for an existing path among best-so-far paths
005283 ** that covers the same set of loops and has the same isOrdered
005284 ** setting as the current path candidate.
005285 **
005286 ** The term "((pTo->isOrdered^isOrdered)&0x80)==0" is equivalent
005287 ** to (pTo->isOrdered==(-1))==(isOrdered==(-1))" for the range
005288 ** of legal values for isOrdered, -1..64.
005289 */
005290 for(jj=0, pTo=aTo; jj<nTo; jj++, pTo++){
005291 if( pTo->maskLoop==maskNew
005292 && ((pTo->isOrdered^isOrdered)&0x80)==0
005293 ){
005294 testcase( jj==nTo-1 );
005295 break;
005296 }
005297 }
005298 if( jj>=nTo ){
005299 /* None of the existing best-so-far paths match the candidate. */
005300 if( nTo>=mxChoice
005301 && (rCost>mxCost || (rCost==mxCost && rUnsorted>=mxUnsorted))
005302 ){
005303 /* The current candidate is no better than any of the mxChoice
005304 ** paths currently in the best-so-far buffer. So discard
005305 ** this candidate as not viable. */
005306 #ifdef WHERETRACE_ENABLED /* 0x4 */
005307 if( sqlite3WhereTrace&0x4 ){
005308 sqlite3DebugPrintf("Skip %s cost=%-3d,%3d,%3d order=%c\n",
005309 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted,
005310 isOrdered>=0 ? isOrdered+'0' : '?');
005311 }
005312 #endif
005313 continue;
005314 }
005315 /* If we reach this points it means that the new candidate path
005316 ** needs to be added to the set of best-so-far paths. */
005317 if( nTo<mxChoice ){
005318 /* Increase the size of the aTo set by one */
005319 jj = nTo++;
005320 }else{
005321 /* New path replaces the prior worst to keep count below mxChoice */
005322 jj = mxI;
005323 }
005324 pTo = &aTo[jj];
005325 #ifdef WHERETRACE_ENABLED /* 0x4 */
005326 if( sqlite3WhereTrace&0x4 ){
005327 sqlite3DebugPrintf("New %s cost=%-3d,%3d,%3d order=%c\n",
005328 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted,
005329 isOrdered>=0 ? isOrdered+'0' : '?');
005330 }
005331 #endif
005332 }else{
005333 /* Control reaches here if best-so-far path pTo=aTo[jj] covers the
005334 ** same set of loops and has the same isOrdered setting as the
005335 ** candidate path. Check to see if the candidate should replace
005336 ** pTo or if the candidate should be skipped.
005337 **
005338 ** The conditional is an expanded vector comparison equivalent to:
005339 ** (pTo->rCost,pTo->nRow,pTo->rUnsorted) <= (rCost,nOut,rUnsorted)
005340 */
005341 if( pTo->rCost<rCost
005342 || (pTo->rCost==rCost
005343 && (pTo->nRow<nOut
005344 || (pTo->nRow==nOut && pTo->rUnsorted<=rUnsorted)
005345 )
005346 )
005347 ){
005348 #ifdef WHERETRACE_ENABLED /* 0x4 */
005349 if( sqlite3WhereTrace&0x4 ){
005350 sqlite3DebugPrintf(
005351 "Skip %s cost=%-3d,%3d,%3d order=%c",
005352 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted,
005353 isOrdered>=0 ? isOrdered+'0' : '?');
005354 sqlite3DebugPrintf(" vs %s cost=%-3d,%3d,%3d order=%c\n",
005355 wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
005356 pTo->rUnsorted, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?');
005357 }
005358 #endif
005359 /* Discard the candidate path from further consideration */
005360 testcase( pTo->rCost==rCost );
005361 continue;
005362 }
005363 testcase( pTo->rCost==rCost+1 );
005364 /* Control reaches here if the candidate path is better than the
005365 ** pTo path. Replace pTo with the candidate. */
005366 #ifdef WHERETRACE_ENABLED /* 0x4 */
005367 if( sqlite3WhereTrace&0x4 ){
005368 sqlite3DebugPrintf(
005369 "Update %s cost=%-3d,%3d,%3d order=%c",
005370 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted,
005371 isOrdered>=0 ? isOrdered+'0' : '?');
005372 sqlite3DebugPrintf(" was %s cost=%-3d,%3d,%3d order=%c\n",
005373 wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
005374 pTo->rUnsorted, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?');
005375 }
005376 #endif
005377 }
005378 /* pWLoop is a winner. Add it to the set of best so far */
005379 pTo->maskLoop = pFrom->maskLoop | pWLoop->maskSelf;
005380 pTo->revLoop = revMask;
005381 pTo->nRow = nOut;
005382 pTo->rCost = rCost;
005383 pTo->rUnsorted = rUnsorted;
005384 pTo->isOrdered = isOrdered;
005385 memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop);
005386 pTo->aLoop[iLoop] = pWLoop;
005387 if( nTo>=mxChoice ){
005388 mxI = 0;
005389 mxCost = aTo[0].rCost;
005390 mxUnsorted = aTo[0].nRow;
005391 for(jj=1, pTo=&aTo[1]; jj<mxChoice; jj++, pTo++){
005392 if( pTo->rCost>mxCost
005393 || (pTo->rCost==mxCost && pTo->rUnsorted>mxUnsorted)
005394 ){
005395 mxCost = pTo->rCost;
005396 mxUnsorted = pTo->rUnsorted;
005397 mxI = jj;
005398 }
005399 }
005400 }
005401 }
005402 }
005403
005404 #ifdef WHERETRACE_ENABLED /* >=2 */
005405 if( sqlite3WhereTrace & 0x02 ){
005406 sqlite3DebugPrintf("---- after round %d ----\n", iLoop);
005407 for(ii=0, pTo=aTo; ii<nTo; ii++, pTo++){
005408 sqlite3DebugPrintf(" %s cost=%-3d nrow=%-3d order=%c",
005409 wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
005410 pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?');
005411 if( pTo->isOrdered>0 ){
005412 sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop);
005413 }else{
005414 sqlite3DebugPrintf("\n");
005415 }
005416 }
005417 }
005418 #endif
005419
005420 /* Swap the roles of aFrom and aTo for the next generation */
005421 pFrom = aTo;
005422 aTo = aFrom;
005423 aFrom = pFrom;
005424 nFrom = nTo;
005425 }
005426
005427 if( nFrom==0 ){
005428 sqlite3ErrorMsg(pParse, "no query solution");
005429 sqlite3StackFreeNN(pParse->db, pSpace);
005430 return SQLITE_ERROR;
005431 }
005432
005433 /* Find the lowest cost path. pFrom will be left pointing to that path */
005434 pFrom = aFrom;
005435 for(ii=1; ii<nFrom; ii++){
005436 if( pFrom->rCost>aFrom[ii].rCost ) pFrom = &aFrom[ii];
005437 }
005438 assert( pWInfo->nLevel==nLoop );
005439 /* Load the lowest cost path into pWInfo */
005440 for(iLoop=0; iLoop<nLoop; iLoop++){
005441 WhereLevel *pLevel = pWInfo->a + iLoop;
005442 pLevel->pWLoop = pWLoop = pFrom->aLoop[iLoop];
005443 pLevel->iFrom = pWLoop->iTab;
005444 pLevel->iTabCur = pWInfo->pTabList->a[pLevel->iFrom].iCursor;
005445 }
005446 if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0
005447 && (pWInfo->wctrlFlags & WHERE_DISTINCTBY)==0
005448 && pWInfo->eDistinct==WHERE_DISTINCT_NOOP
005449 && nRowEst
005450 ){
005451 Bitmask notUsed;
005452 int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pResultSet, pFrom,
005453 WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used);
005454 if( rc==pWInfo->pResultSet->nExpr ){
005455 pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
005456 }
005457 }
005458 pWInfo->bOrderedInnerLoop = 0;
005459 if( pWInfo->pOrderBy ){
005460 pWInfo->nOBSat = pFrom->isOrdered;
005461 if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){
005462 if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){
005463 pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
005464 }
005465 if( pWInfo->pSelect->pOrderBy
005466 && pWInfo->nOBSat > pWInfo->pSelect->pOrderBy->nExpr ){
005467 pWInfo->nOBSat = pWInfo->pSelect->pOrderBy->nExpr;
005468 }
005469 }else{
005470 pWInfo->revMask = pFrom->revLoop;
005471 if( pWInfo->nOBSat<=0 ){
005472 pWInfo->nOBSat = 0;
005473 if( nLoop>0 ){
005474 u32 wsFlags = pFrom->aLoop[nLoop-1]->wsFlags;
005475 if( (wsFlags & WHERE_ONEROW)==0
005476 && (wsFlags&(WHERE_IPK|WHERE_COLUMN_IN))!=(WHERE_IPK|WHERE_COLUMN_IN)
005477 ){
005478 Bitmask m = 0;
005479 int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom,
005480 WHERE_ORDERBY_LIMIT, nLoop-1, pFrom->aLoop[nLoop-1], &m);
005481 testcase( wsFlags & WHERE_IPK );
005482 testcase( wsFlags & WHERE_COLUMN_IN );
005483 if( rc==pWInfo->pOrderBy->nExpr ){
005484 pWInfo->bOrderedInnerLoop = 1;
005485 pWInfo->revMask = m;
005486 }
005487 }
005488 }
005489 }else if( nLoop
005490 && pWInfo->nOBSat==1
005491 && (pWInfo->wctrlFlags & (WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX))!=0
005492 ){
005493 pWInfo->bOrderedInnerLoop = 1;
005494 }
005495 }
005496 if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP)
005497 && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr && nLoop>0
005498 ){
005499 Bitmask revMask = 0;
005500 int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy,
005501 pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], &revMask
005502 );
005503 assert( pWInfo->sorted==0 );
005504 if( nOrder==pWInfo->pOrderBy->nExpr ){
005505 pWInfo->sorted = 1;
005506 pWInfo->revMask = revMask;
005507 }
005508 }
005509 }
005510
005511
005512 pWInfo->nRowOut = pFrom->nRow;
005513
005514 /* Free temporary memory and return success */
005515 sqlite3StackFreeNN(pParse->db, pSpace);
005516 return SQLITE_OK;
005517 }
005518
005519 /*
005520 ** Most queries use only a single table (they are not joins) and have
005521 ** simple == constraints against indexed fields. This routine attempts
005522 ** to plan those simple cases using much less ceremony than the
005523 ** general-purpose query planner, and thereby yield faster sqlite3_prepare()
005524 ** times for the common case.
005525 **
005526 ** Return non-zero on success, if this query can be handled by this
005527 ** no-frills query planner. Return zero if this query needs the
005528 ** general-purpose query planner.
005529 */
005530 static int whereShortCut(WhereLoopBuilder *pBuilder){
005531 WhereInfo *pWInfo;
005532 SrcItem *pItem;
005533 WhereClause *pWC;
005534 WhereTerm *pTerm;
005535 WhereLoop *pLoop;
005536 int iCur;
005537 int j;
005538 Table *pTab;
005539 Index *pIdx;
005540 WhereScan scan;
005541
005542 pWInfo = pBuilder->pWInfo;
005543 if( pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE ) return 0;
005544 assert( pWInfo->pTabList->nSrc>=1 );
005545 pItem = pWInfo->pTabList->a;
005546 pTab = pItem->pTab;
005547 if( IsVirtual(pTab) ) return 0;
005548 if( pItem->fg.isIndexedBy || pItem->fg.notIndexed ){
005549 testcase( pItem->fg.isIndexedBy );
005550 testcase( pItem->fg.notIndexed );
005551 return 0;
005552 }
005553 iCur = pItem->iCursor;
005554 pWC = &pWInfo->sWC;
005555 pLoop = pBuilder->pNew;
005556 pLoop->wsFlags = 0;
005557 pLoop->nSkip = 0;
005558 pTerm = whereScanInit(&scan, pWC, iCur, -1, WO_EQ|WO_IS, 0);
005559 while( pTerm && pTerm->prereqRight ) pTerm = whereScanNext(&scan);
005560 if( pTerm ){
005561 testcase( pTerm->eOperator & WO_IS );
005562 pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW;
005563 pLoop->aLTerm[0] = pTerm;
005564 pLoop->nLTerm = 1;
005565 pLoop->u.btree.nEq = 1;
005566 /* TUNING: Cost of a rowid lookup is 10 */
005567 pLoop->rRun = 33; /* 33==sqlite3LogEst(10) */
005568 }else{
005569 for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
005570 int opMask;
005571 assert( pLoop->aLTermSpace==pLoop->aLTerm );
005572 if( !IsUniqueIndex(pIdx)
005573 || pIdx->pPartIdxWhere!=0
005574 || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace)
005575 ) continue;
005576 opMask = pIdx->uniqNotNull ? (WO_EQ|WO_IS) : WO_EQ;
005577 for(j=0; j<pIdx->nKeyCol; j++){
005578 pTerm = whereScanInit(&scan, pWC, iCur, j, opMask, pIdx);
005579 while( pTerm && pTerm->prereqRight ) pTerm = whereScanNext(&scan);
005580 if( pTerm==0 ) break;
005581 testcase( pTerm->eOperator & WO_IS );
005582 pLoop->aLTerm[j] = pTerm;
005583 }
005584 if( j!=pIdx->nKeyCol ) continue;
005585 pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_ONEROW|WHERE_INDEXED;
005586 if( pIdx->isCovering || (pItem->colUsed & pIdx->colNotIdxed)==0 ){
005587 pLoop->wsFlags |= WHERE_IDX_ONLY;
005588 }
005589 pLoop->nLTerm = j;
005590 pLoop->u.btree.nEq = j;
005591 pLoop->u.btree.pIndex = pIdx;
005592 /* TUNING: Cost of a unique index lookup is 15 */
005593 pLoop->rRun = 39; /* 39==sqlite3LogEst(15) */
005594 break;
005595 }
005596 }
005597 if( pLoop->wsFlags ){
005598 pLoop->nOut = (LogEst)1;
005599 pWInfo->a[0].pWLoop = pLoop;
005600 assert( pWInfo->sMaskSet.n==1 && iCur==pWInfo->sMaskSet.ix[0] );
005601 pLoop->maskSelf = 1; /* sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); */
005602 pWInfo->a[0].iTabCur = iCur;
005603 pWInfo->nRowOut = 1;
005604 if( pWInfo->pOrderBy ) pWInfo->nOBSat = pWInfo->pOrderBy->nExpr;
005605 if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){
005606 pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
005607 }
005608 if( scan.iEquiv>1 ) pLoop->wsFlags |= WHERE_TRANSCONS;
005609 #ifdef SQLITE_DEBUG
005610 pLoop->cId = '0';
005611 #endif
005612 #ifdef WHERETRACE_ENABLED
005613 if( sqlite3WhereTrace & 0x02 ){
005614 sqlite3DebugPrintf("whereShortCut() used to compute solution\n");
005615 }
005616 #endif
005617 return 1;
005618 }
005619 return 0;
005620 }
005621
005622 /*
005623 ** Helper function for exprIsDeterministic().
005624 */
005625 static int exprNodeIsDeterministic(Walker *pWalker, Expr *pExpr){
005626 if( pExpr->op==TK_FUNCTION && ExprHasProperty(pExpr, EP_ConstFunc)==0 ){
005627 pWalker->eCode = 0;
005628 return WRC_Abort;
005629 }
005630 return WRC_Continue;
005631 }
005632
005633 /*
005634 ** Return true if the expression contains no non-deterministic SQL
005635 ** functions. Do not consider non-deterministic SQL functions that are
005636 ** part of sub-select statements.
005637 */
005638 static int exprIsDeterministic(Expr *p){
005639 Walker w;
005640 memset(&w, 0, sizeof(w));
005641 w.eCode = 1;
005642 w.xExprCallback = exprNodeIsDeterministic;
005643 w.xSelectCallback = sqlite3SelectWalkFail;
005644 sqlite3WalkExpr(&w, p);
005645 return w.eCode;
005646 }
005647
005648
005649 #ifdef WHERETRACE_ENABLED
005650 /*
005651 ** Display all WhereLoops in pWInfo
005652 */
005653 static void showAllWhereLoops(WhereInfo *pWInfo, WhereClause *pWC){
005654 if( sqlite3WhereTrace ){ /* Display all of the WhereLoop objects */
005655 WhereLoop *p;
005656 int i;
005657 static const char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz"
005658 "ABCDEFGHIJKLMNOPQRSTUVWYXZ";
005659 for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){
005660 p->cId = zLabel[i%(sizeof(zLabel)-1)];
005661 sqlite3WhereLoopPrint(p, pWC);
005662 }
005663 }
005664 }
005665 # define WHERETRACE_ALL_LOOPS(W,C) showAllWhereLoops(W,C)
005666 #else
005667 # define WHERETRACE_ALL_LOOPS(W,C)
005668 #endif
005669
005670 /* Attempt to omit tables from a join that do not affect the result.
005671 ** For a table to not affect the result, the following must be true:
005672 **
005673 ** 1) The query must not be an aggregate.
005674 ** 2) The table must be the RHS of a LEFT JOIN.
005675 ** 3) Either the query must be DISTINCT, or else the ON or USING clause
005676 ** must contain a constraint that limits the scan of the table to
005677 ** at most a single row.
005678 ** 4) The table must not be referenced by any part of the query apart
005679 ** from its own USING or ON clause.
005680 ** 5) The table must not have an inner-join ON or USING clause if there is
005681 ** a RIGHT JOIN anywhere in the query. Otherwise the ON/USING clause
005682 ** might move from the right side to the left side of the RIGHT JOIN.
005683 ** Note: Due to (2), this condition can only arise if the table is
005684 ** the right-most table of a subquery that was flattened into the
005685 ** main query and that subquery was the right-hand operand of an
005686 ** inner join that held an ON or USING clause.
005687 **
005688 ** For example, given:
005689 **
005690 ** CREATE TABLE t1(ipk INTEGER PRIMARY KEY, v1);
005691 ** CREATE TABLE t2(ipk INTEGER PRIMARY KEY, v2);
005692 ** CREATE TABLE t3(ipk INTEGER PRIMARY KEY, v3);
005693 **
005694 ** then table t2 can be omitted from the following:
005695 **
005696 ** SELECT v1, v3 FROM t1
005697 ** LEFT JOIN t2 ON (t1.ipk=t2.ipk)
005698 ** LEFT JOIN t3 ON (t1.ipk=t3.ipk)
005699 **
005700 ** or from:
005701 **
005702 ** SELECT DISTINCT v1, v3 FROM t1
005703 ** LEFT JOIN t2
005704 ** LEFT JOIN t3 ON (t1.ipk=t3.ipk)
005705 */
005706 static SQLITE_NOINLINE Bitmask whereOmitNoopJoin(
005707 WhereInfo *pWInfo,
005708 Bitmask notReady
005709 ){
005710 int i;
005711 Bitmask tabUsed;
005712 int hasRightJoin;
005713
005714 /* Preconditions checked by the caller */
005715 assert( pWInfo->nLevel>=2 );
005716 assert( OptimizationEnabled(pWInfo->pParse->db, SQLITE_OmitNoopJoin) );
005717
005718 /* These two preconditions checked by the caller combine to guarantee
005719 ** condition (1) of the header comment */
005720 assert( pWInfo->pResultSet!=0 );
005721 assert( 0==(pWInfo->wctrlFlags & WHERE_AGG_DISTINCT) );
005722
005723 tabUsed = sqlite3WhereExprListUsage(&pWInfo->sMaskSet, pWInfo->pResultSet);
005724 if( pWInfo->pOrderBy ){
005725 tabUsed |= sqlite3WhereExprListUsage(&pWInfo->sMaskSet, pWInfo->pOrderBy);
005726 }
005727 hasRightJoin = (pWInfo->pTabList->a[0].fg.jointype & JT_LTORJ)!=0;
005728 for(i=pWInfo->nLevel-1; i>=1; i--){
005729 WhereTerm *pTerm, *pEnd;
005730 SrcItem *pItem;
005731 WhereLoop *pLoop;
005732 pLoop = pWInfo->a[i].pWLoop;
005733 pItem = &pWInfo->pTabList->a[pLoop->iTab];
005734 if( (pItem->fg.jointype & (JT_LEFT|JT_RIGHT))!=JT_LEFT ) continue;
005735 if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)==0
005736 && (pLoop->wsFlags & WHERE_ONEROW)==0
005737 ){
005738 continue;
005739 }
005740 if( (tabUsed & pLoop->maskSelf)!=0 ) continue;
005741 pEnd = pWInfo->sWC.a + pWInfo->sWC.nTerm;
005742 for(pTerm=pWInfo->sWC.a; pTerm<pEnd; pTerm++){
005743 if( (pTerm->prereqAll & pLoop->maskSelf)!=0 ){
005744 if( !ExprHasProperty(pTerm->pExpr, EP_OuterON)
005745 || pTerm->pExpr->w.iJoin!=pItem->iCursor
005746 ){
005747 break;
005748 }
005749 }
005750 if( hasRightJoin
005751 && ExprHasProperty(pTerm->pExpr, EP_InnerON)
005752 && pTerm->pExpr->w.iJoin==pItem->iCursor
005753 ){
005754 break; /* restriction (5) */
005755 }
005756 }
005757 if( pTerm<pEnd ) continue;
005758 WHERETRACE(0xffffffff, ("-> drop loop %c not used\n", pLoop->cId));
005759 notReady &= ~pLoop->maskSelf;
005760 for(pTerm=pWInfo->sWC.a; pTerm<pEnd; pTerm++){
005761 if( (pTerm->prereqAll & pLoop->maskSelf)!=0 ){
005762 pTerm->wtFlags |= TERM_CODED;
005763 }
005764 }
005765 if( i!=pWInfo->nLevel-1 ){
005766 int nByte = (pWInfo->nLevel-1-i) * sizeof(WhereLevel);
005767 memmove(&pWInfo->a[i], &pWInfo->a[i+1], nByte);
005768 }
005769 pWInfo->nLevel--;
005770 assert( pWInfo->nLevel>0 );
005771 }
005772 return notReady;
005773 }
005774
005775 /*
005776 ** Check to see if there are any SEARCH loops that might benefit from
005777 ** using a Bloom filter. Consider a Bloom filter if:
005778 **
005779 ** (1) The SEARCH happens more than N times where N is the number
005780 ** of rows in the table that is being considered for the Bloom
005781 ** filter.
005782 ** (2) Some searches are expected to find zero rows. (This is determined
005783 ** by the WHERE_SELFCULL flag on the term.)
005784 ** (3) Bloom-filter processing is not disabled. (Checked by the
005785 ** caller.)
005786 ** (4) The size of the table being searched is known by ANALYZE.
005787 **
005788 ** This block of code merely checks to see if a Bloom filter would be
005789 ** appropriate, and if so sets the WHERE_BLOOMFILTER flag on the
005790 ** WhereLoop. The implementation of the Bloom filter comes further
005791 ** down where the code for each WhereLoop is generated.
005792 */
005793 static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful(
005794 const WhereInfo *pWInfo
005795 ){
005796 int i;
005797 LogEst nSearch = 0;
005798
005799 assert( pWInfo->nLevel>=2 );
005800 assert( OptimizationEnabled(pWInfo->pParse->db, SQLITE_BloomFilter) );
005801 for(i=0; i<pWInfo->nLevel; i++){
005802 WhereLoop *pLoop = pWInfo->a[i].pWLoop;
005803 const unsigned int reqFlags = (WHERE_SELFCULL|WHERE_COLUMN_EQ);
005804 SrcItem *pItem = &pWInfo->pTabList->a[pLoop->iTab];
005805 Table *pTab = pItem->pTab;
005806 if( (pTab->tabFlags & TF_HasStat1)==0 ) break;
005807 pTab->tabFlags |= TF_StatsUsed;
005808 if( i>=1
005809 && (pLoop->wsFlags & reqFlags)==reqFlags
005810 /* vvvvvv--- Always the case if WHERE_COLUMN_EQ is defined */
005811 && ALWAYS((pLoop->wsFlags & (WHERE_IPK|WHERE_INDEXED))!=0)
005812 ){
005813 if( nSearch > pTab->nRowLogEst ){
005814 testcase( pItem->fg.jointype & JT_LEFT );
005815 pLoop->wsFlags |= WHERE_BLOOMFILTER;
005816 pLoop->wsFlags &= ~WHERE_IDX_ONLY;
005817 WHERETRACE(0xffffffff, (
005818 "-> use Bloom-filter on loop %c because there are ~%.1e "
005819 "lookups into %s which has only ~%.1e rows\n",
005820 pLoop->cId, (double)sqlite3LogEstToInt(nSearch), pTab->zName,
005821 (double)sqlite3LogEstToInt(pTab->nRowLogEst)));
005822 }
005823 }
005824 nSearch += pLoop->nOut;
005825 }
005826 }
005827
005828 /*
005829 ** The index pIdx is used by a query and contains one or more expressions.
005830 ** In other words pIdx is an index on an expression. iIdxCur is the cursor
005831 ** number for the index and iDataCur is the cursor number for the corresponding
005832 ** table.
005833 **
005834 ** This routine adds IndexedExpr entries to the Parse->pIdxEpr field for
005835 ** each of the expressions in the index so that the expression code generator
005836 ** will know to replace occurrences of the indexed expression with
005837 ** references to the corresponding column of the index.
005838 */
005839 static SQLITE_NOINLINE void whereAddIndexedExpr(
005840 Parse *pParse, /* Add IndexedExpr entries to pParse->pIdxEpr */
005841 Index *pIdx, /* The index-on-expression that contains the expressions */
005842 int iIdxCur, /* Cursor number for pIdx */
005843 SrcItem *pTabItem /* The FROM clause entry for the table */
005844 ){
005845 int i;
005846 IndexedExpr *p;
005847 Table *pTab;
005848 assert( pIdx->bHasExpr );
005849 pTab = pIdx->pTable;
005850 for(i=0; i<pIdx->nColumn; i++){
005851 Expr *pExpr;
005852 int j = pIdx->aiColumn[i];
005853 int bMaybeNullRow;
005854 if( j==XN_EXPR ){
005855 pExpr = pIdx->aColExpr->a[i].pExpr;
005856 testcase( pTabItem->fg.jointype & JT_LEFT );
005857 testcase( pTabItem->fg.jointype & JT_RIGHT );
005858 testcase( pTabItem->fg.jointype & JT_LTORJ );
005859 bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0;
005860 }else if( j>=0 && (pTab->aCol[j].colFlags & COLFLAG_VIRTUAL)!=0 ){
005861 pExpr = sqlite3ColumnExpr(pTab, &pTab->aCol[j]);
005862 bMaybeNullRow = 0;
005863 }else{
005864 continue;
005865 }
005866 if( sqlite3ExprIsConstant(pExpr) ) continue;
005867 if( pExpr->op==TK_FUNCTION ){
005868 /* Functions that might set a subtype should not be replaced by the
005869 ** value taken from an expression index since the index omits the
005870 ** subtype. https://sqlite.org/forum/forumpost/68d284c86b082c3e */
005871 int n;
005872 FuncDef *pDef;
005873 sqlite3 *db = pParse->db;
005874 assert( ExprUseXList(pExpr) );
005875 n = pExpr->x.pList ? pExpr->x.pList->nExpr : 0;
005876 pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0);
005877 if( pDef==0 || (pDef->funcFlags & SQLITE_RESULT_SUBTYPE)!=0 ){
005878 continue;
005879 }
005880 }
005881 p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexedExpr));
005882 if( p==0 ) break;
005883 p->pIENext = pParse->pIdxEpr;
005884 #ifdef WHERETRACE_ENABLED
005885 if( sqlite3WhereTrace & 0x200 ){
005886 sqlite3DebugPrintf("New pParse->pIdxEpr term {%d,%d}\n", iIdxCur, i);
005887 if( sqlite3WhereTrace & 0x5000 ) sqlite3ShowExpr(pExpr);
005888 }
005889 #endif
005890 p->pExpr = sqlite3ExprDup(pParse->db, pExpr, 0);
005891 p->iDataCur = pTabItem->iCursor;
005892 p->iIdxCur = iIdxCur;
005893 p->iIdxCol = i;
005894 p->bMaybeNullRow = bMaybeNullRow;
005895 if( sqlite3IndexAffinityStr(pParse->db, pIdx) ){
005896 p->aff = pIdx->zColAff[i];
005897 }
005898 #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
005899 p->zIdxName = pIdx->zName;
005900 #endif
005901 pParse->pIdxEpr = p;
005902 if( p->pIENext==0 ){
005903 void *pArg = (void*)&pParse->pIdxEpr;
005904 sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pArg);
005905 }
005906 }
005907 }
005908
005909 /*
005910 ** Set the reverse-scan order mask to one for all tables in the query
005911 ** with the exception of MATERIALIZED common table expressions that have
005912 ** their own internal ORDER BY clauses.
005913 **
005914 ** This implements the PRAGMA reverse_unordered_selects=ON setting.
005915 ** (Also SQLITE_DBCONFIG_REVERSE_SCANORDER).
005916 */
005917 static SQLITE_NOINLINE void whereReverseScanOrder(WhereInfo *pWInfo){
005918 int ii;
005919 for(ii=0; ii<pWInfo->pTabList->nSrc; ii++){
005920 SrcItem *pItem = &pWInfo->pTabList->a[ii];
005921 if( !pItem->fg.isCte
005922 || pItem->u2.pCteUse->eM10d!=M10d_Yes
005923 || NEVER(pItem->pSelect==0)
005924 || pItem->pSelect->pOrderBy==0
005925 ){
005926 pWInfo->revMask |= MASKBIT(ii);
005927 }
005928 }
005929 }
005930
005931 /*
005932 ** Generate the beginning of the loop used for WHERE clause processing.
005933 ** The return value is a pointer to an opaque structure that contains
005934 ** information needed to terminate the loop. Later, the calling routine
005935 ** should invoke sqlite3WhereEnd() with the return value of this function
005936 ** in order to complete the WHERE clause processing.
005937 **
005938 ** If an error occurs, this routine returns NULL.
005939 **
005940 ** The basic idea is to do a nested loop, one loop for each table in
005941 ** the FROM clause of a select. (INSERT and UPDATE statements are the
005942 ** same as a SELECT with only a single table in the FROM clause.) For
005943 ** example, if the SQL is this:
005944 **
005945 ** SELECT * FROM t1, t2, t3 WHERE ...;
005946 **
005947 ** Then the code generated is conceptually like the following:
005948 **
005949 ** foreach row1 in t1 do \ Code generated
005950 ** foreach row2 in t2 do |-- by sqlite3WhereBegin()
005951 ** foreach row3 in t3 do /
005952 ** ...
005953 ** end \ Code generated
005954 ** end |-- by sqlite3WhereEnd()
005955 ** end /
005956 **
005957 ** Note that the loops might not be nested in the order in which they
005958 ** appear in the FROM clause if a different order is better able to make
005959 ** use of indices. Note also that when the IN operator appears in
005960 ** the WHERE clause, it might result in additional nested loops for
005961 ** scanning through all values on the right-hand side of the IN.
005962 **
005963 ** There are Btree cursors associated with each table. t1 uses cursor
005964 ** number pTabList->a[0].iCursor. t2 uses the cursor pTabList->a[1].iCursor.
005965 ** And so forth. This routine generates code to open those VDBE cursors
005966 ** and sqlite3WhereEnd() generates the code to close them.
005967 **
005968 ** The code that sqlite3WhereBegin() generates leaves the cursors named
005969 ** in pTabList pointing at their appropriate entries. The [...] code
005970 ** can use OP_Column and OP_Rowid opcodes on these cursors to extract
005971 ** data from the various tables of the loop.
005972 **
005973 ** If the WHERE clause is empty, the foreach loops must each scan their
005974 ** entire tables. Thus a three-way join is an O(N^3) operation. But if
005975 ** the tables have indices and there are terms in the WHERE clause that
005976 ** refer to those indices, a complete table scan can be avoided and the
005977 ** code will run much faster. Most of the work of this routine is checking
005978 ** to see if there are indices that can be used to speed up the loop.
005979 **
005980 ** Terms of the WHERE clause are also used to limit which rows actually
005981 ** make it to the "..." in the middle of the loop. After each "foreach",
005982 ** terms of the WHERE clause that use only terms in that loop and outer
005983 ** loops are evaluated and if false a jump is made around all subsequent
005984 ** inner loops (or around the "..." if the test occurs within the inner-
005985 ** most loop)
005986 **
005987 ** OUTER JOINS
005988 **
005989 ** An outer join of tables t1 and t2 is conceptually coded as follows:
005990 **
005991 ** foreach row1 in t1 do
005992 ** flag = 0
005993 ** foreach row2 in t2 do
005994 ** start:
005995 ** ...
005996 ** flag = 1
005997 ** end
005998 ** if flag==0 then
005999 ** move the row2 cursor to a null row
006000 ** goto start
006001 ** fi
006002 ** end
006003 **
006004 ** ORDER BY CLAUSE PROCESSING
006005 **
006006 ** pOrderBy is a pointer to the ORDER BY clause (or the GROUP BY clause
006007 ** if the WHERE_GROUPBY flag is set in wctrlFlags) of a SELECT statement
006008 ** if there is one. If there is no ORDER BY clause or if this routine
006009 ** is called from an UPDATE or DELETE statement, then pOrderBy is NULL.
006010 **
006011 ** The iIdxCur parameter is the cursor number of an index. If
006012 ** WHERE_OR_SUBCLAUSE is set, iIdxCur is the cursor number of an index
006013 ** to use for OR clause processing. The WHERE clause should use this
006014 ** specific cursor. If WHERE_ONEPASS_DESIRED is set, then iIdxCur is
006015 ** the first cursor in an array of cursors for all indices. iIdxCur should
006016 ** be used to compute the appropriate cursor depending on which index is
006017 ** used.
006018 */
006019 WhereInfo *sqlite3WhereBegin(
006020 Parse *pParse, /* The parser context */
006021 SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */
006022 Expr *pWhere, /* The WHERE clause */
006023 ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */
006024 ExprList *pResultSet, /* Query result set. Req'd for DISTINCT */
006025 Select *pSelect, /* The entire SELECT statement */
006026 u16 wctrlFlags, /* The WHERE_* flags defined in sqliteInt.h */
006027 int iAuxArg /* If WHERE_OR_SUBCLAUSE is set, index cursor number
006028 ** If WHERE_USE_LIMIT, then the limit amount */
006029 ){
006030 int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */
006031 int nTabList; /* Number of elements in pTabList */
006032 WhereInfo *pWInfo; /* Will become the return value of this function */
006033 Vdbe *v = pParse->pVdbe; /* The virtual database engine */
006034 Bitmask notReady; /* Cursors that are not yet positioned */
006035 WhereLoopBuilder sWLB; /* The WhereLoop builder */
006036 WhereMaskSet *pMaskSet; /* The expression mask set */
006037 WhereLevel *pLevel; /* A single level in pWInfo->a[] */
006038 WhereLoop *pLoop; /* Pointer to a single WhereLoop object */
006039 int ii; /* Loop counter */
006040 sqlite3 *db; /* Database connection */
006041 int rc; /* Return code */
006042 u8 bFordelete = 0; /* OPFLAG_FORDELETE or zero, as appropriate */
006043
006044 assert( (wctrlFlags & WHERE_ONEPASS_MULTIROW)==0 || (
006045 (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0
006046 && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0
006047 ));
006048
006049 /* Only one of WHERE_OR_SUBCLAUSE or WHERE_USE_LIMIT */
006050 assert( (wctrlFlags & WHERE_OR_SUBCLAUSE)==0
006051 || (wctrlFlags & WHERE_USE_LIMIT)==0 );
006052
006053 /* Variable initialization */
006054 db = pParse->db;
006055 memset(&sWLB, 0, sizeof(sWLB));
006056
006057 /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */
006058 testcase( pOrderBy && pOrderBy->nExpr==BMS-1 );
006059 if( pOrderBy && pOrderBy->nExpr>=BMS ){
006060 pOrderBy = 0;
006061 wctrlFlags &= ~WHERE_WANT_DISTINCT;
006062 }
006063
006064 /* The number of tables in the FROM clause is limited by the number of
006065 ** bits in a Bitmask
006066 */
006067 testcase( pTabList->nSrc==BMS );
006068 if( pTabList->nSrc>BMS ){
006069 sqlite3ErrorMsg(pParse, "at most %d tables in a join", BMS);
006070 return 0;
006071 }
006072
006073 /* This function normally generates a nested loop for all tables in
006074 ** pTabList. But if the WHERE_OR_SUBCLAUSE flag is set, then we should
006075 ** only generate code for the first table in pTabList and assume that
006076 ** any cursors associated with subsequent tables are uninitialized.
006077 */
006078 nTabList = (wctrlFlags & WHERE_OR_SUBCLAUSE) ? 1 : pTabList->nSrc;
006079
006080 /* Allocate and initialize the WhereInfo structure that will become the
006081 ** return value. A single allocation is used to store the WhereInfo
006082 ** struct, the contents of WhereInfo.a[], the WhereClause structure
006083 ** and the WhereMaskSet structure. Since WhereClause contains an 8-byte
006084 ** field (type Bitmask) it must be aligned on an 8-byte boundary on
006085 ** some architectures. Hence the ROUND8() below.
006086 */
006087 nByteWInfo = ROUND8P(sizeof(WhereInfo));
006088 if( nTabList>1 ){
006089 nByteWInfo = ROUND8P(nByteWInfo + (nTabList-1)*sizeof(WhereLevel));
006090 }
006091 pWInfo = sqlite3DbMallocRawNN(db, nByteWInfo + sizeof(WhereLoop));
006092 if( db->mallocFailed ){
006093 sqlite3DbFree(db, pWInfo);
006094 pWInfo = 0;
006095 goto whereBeginError;
006096 }
006097 pWInfo->pParse = pParse;
006098 pWInfo->pTabList = pTabList;
006099 pWInfo->pOrderBy = pOrderBy;
006100 #if WHERETRACE_ENABLED
006101 pWInfo->pWhere = pWhere;
006102 #endif
006103 pWInfo->pResultSet = pResultSet;
006104 pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1;
006105 pWInfo->nLevel = nTabList;
006106 pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(pParse);
006107 pWInfo->wctrlFlags = wctrlFlags;
006108 pWInfo->iLimit = iAuxArg;
006109 pWInfo->savedNQueryLoop = pParse->nQueryLoop;
006110 pWInfo->pSelect = pSelect;
006111 memset(&pWInfo->nOBSat, 0,
006112 offsetof(WhereInfo,sWC) - offsetof(WhereInfo,nOBSat));
006113 memset(&pWInfo->a[0], 0, sizeof(WhereLoop)+nTabList*sizeof(WhereLevel));
006114 assert( pWInfo->eOnePass==ONEPASS_OFF ); /* ONEPASS defaults to OFF */
006115 pMaskSet = &pWInfo->sMaskSet;
006116 pMaskSet->n = 0;
006117 pMaskSet->ix[0] = -99; /* Initialize ix[0] to a value that can never be
006118 ** a valid cursor number, to avoid an initial
006119 ** test for pMaskSet->n==0 in sqlite3WhereGetMask() */
006120 sWLB.pWInfo = pWInfo;
006121 sWLB.pWC = &pWInfo->sWC;
006122 sWLB.pNew = (WhereLoop*)(((char*)pWInfo)+nByteWInfo);
006123 assert( EIGHT_BYTE_ALIGNMENT(sWLB.pNew) );
006124 whereLoopInit(sWLB.pNew);
006125 #ifdef SQLITE_DEBUG
006126 sWLB.pNew->cId = '*';
006127 #endif
006128
006129 /* Split the WHERE clause into separate subexpressions where each
006130 ** subexpression is separated by an AND operator.
006131 */
006132 sqlite3WhereClauseInit(&pWInfo->sWC, pWInfo);
006133 sqlite3WhereSplit(&pWInfo->sWC, pWhere, TK_AND);
006134
006135 /* Special case: No FROM clause
006136 */
006137 if( nTabList==0 ){
006138 if( pOrderBy ) pWInfo->nOBSat = pOrderBy->nExpr;
006139 if( (wctrlFlags & WHERE_WANT_DISTINCT)!=0
006140 && OptimizationEnabled(db, SQLITE_DistinctOpt)
006141 ){
006142 pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
006143 }
006144 ExplainQueryPlan((pParse, 0, "SCAN CONSTANT ROW"));
006145 }else{
006146 /* Assign a bit from the bitmask to every term in the FROM clause.
006147 **
006148 ** The N-th term of the FROM clause is assigned a bitmask of 1<<N.
006149 **
006150 ** The rule of the previous sentence ensures that if X is the bitmask for
006151 ** a table T, then X-1 is the bitmask for all other tables to the left of T.
006152 ** Knowing the bitmask for all tables to the left of a left join is
006153 ** important. Ticket #3015.
006154 **
006155 ** Note that bitmasks are created for all pTabList->nSrc tables in
006156 ** pTabList, not just the first nTabList tables. nTabList is normally
006157 ** equal to pTabList->nSrc but might be shortened to 1 if the
006158 ** WHERE_OR_SUBCLAUSE flag is set.
006159 */
006160 ii = 0;
006161 do{
006162 createMask(pMaskSet, pTabList->a[ii].iCursor);
006163 sqlite3WhereTabFuncArgs(pParse, &pTabList->a[ii], &pWInfo->sWC);
006164 }while( (++ii)<pTabList->nSrc );
006165 #ifdef SQLITE_DEBUG
006166 {
006167 Bitmask mx = 0;
006168 for(ii=0; ii<pTabList->nSrc; ii++){
006169 Bitmask m = sqlite3WhereGetMask(pMaskSet, pTabList->a[ii].iCursor);
006170 assert( m>=mx );
006171 mx = m;
006172 }
006173 }
006174 #endif
006175 }
006176
006177 /* Analyze all of the subexpressions. */
006178 sqlite3WhereExprAnalyze(pTabList, &pWInfo->sWC);
006179 if( pSelect && pSelect->pLimit ){
006180 sqlite3WhereAddLimit(&pWInfo->sWC, pSelect);
006181 }
006182 if( pParse->nErr ) goto whereBeginError;
006183
006184 /* The False-WHERE-Term-Bypass optimization:
006185 **
006186 ** If there are WHERE terms that are false, then no rows will be output,
006187 ** so skip over all of the code generated here.
006188 **
006189 ** Conditions:
006190 **
006191 ** (1) The WHERE term must not refer to any tables in the join.
006192 ** (2) The term must not come from an ON clause on the
006193 ** right-hand side of a LEFT or FULL JOIN.
006194 ** (3) The term must not come from an ON clause, or there must be
006195 ** no RIGHT or FULL OUTER joins in pTabList.
006196 ** (4) If the expression contains non-deterministic functions
006197 ** that are not within a sub-select. This is not required
006198 ** for correctness but rather to preserves SQLite's legacy
006199 ** behaviour in the following two cases:
006200 **
006201 ** WHERE random()>0; -- eval random() once per row
006202 ** WHERE (SELECT random())>0; -- eval random() just once overall
006203 **
006204 ** Note that the Where term need not be a constant in order for this
006205 ** optimization to apply, though it does need to be constant relative to
006206 ** the current subquery (condition 1). The term might include variables
006207 ** from outer queries so that the value of the term changes from one
006208 ** invocation of the current subquery to the next.
006209 */
006210 for(ii=0; ii<sWLB.pWC->nBase; ii++){
006211 WhereTerm *pT = &sWLB.pWC->a[ii]; /* A term of the WHERE clause */
006212 Expr *pX; /* The expression of pT */
006213 if( pT->wtFlags & TERM_VIRTUAL ) continue;
006214 pX = pT->pExpr;
006215 assert( pX!=0 );
006216 assert( pT->prereqAll!=0 || !ExprHasProperty(pX, EP_OuterON) );
006217 if( pT->prereqAll==0 /* Conditions (1) and (2) */
006218 && (nTabList==0 || exprIsDeterministic(pX)) /* Condition (4) */
006219 && !(ExprHasProperty(pX, EP_InnerON) /* Condition (3) */
006220 && (pTabList->a[0].fg.jointype & JT_LTORJ)!=0 )
006221 ){
006222 sqlite3ExprIfFalse(pParse, pX, pWInfo->iBreak, SQLITE_JUMPIFNULL);
006223 pT->wtFlags |= TERM_CODED;
006224 }
006225 }
006226
006227 if( wctrlFlags & WHERE_WANT_DISTINCT ){
006228 if( OptimizationDisabled(db, SQLITE_DistinctOpt) ){
006229 /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via
006230 ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */
006231 wctrlFlags &= ~WHERE_WANT_DISTINCT;
006232 pWInfo->wctrlFlags &= ~WHERE_WANT_DISTINCT;
006233 }else if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pResultSet) ){
006234 /* The DISTINCT marking is pointless. Ignore it. */
006235 pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
006236 }else if( pOrderBy==0 ){
006237 /* Try to ORDER BY the result set to make distinct processing easier */
006238 pWInfo->wctrlFlags |= WHERE_DISTINCTBY;
006239 pWInfo->pOrderBy = pResultSet;
006240 }
006241 }
006242
006243 /* Construct the WhereLoop objects */
006244 #if defined(WHERETRACE_ENABLED)
006245 if( sqlite3WhereTrace & 0xffffffff ){
006246 sqlite3DebugPrintf("*** Optimizer Start *** (wctrlFlags: 0x%x",wctrlFlags);
006247 if( wctrlFlags & WHERE_USE_LIMIT ){
006248 sqlite3DebugPrintf(", limit: %d", iAuxArg);
006249 }
006250 sqlite3DebugPrintf(")\n");
006251 if( sqlite3WhereTrace & 0x8000 ){
006252 Select sSelect;
006253 memset(&sSelect, 0, sizeof(sSelect));
006254 sSelect.selFlags = SF_WhereBegin;
006255 sSelect.pSrc = pTabList;
006256 sSelect.pWhere = pWhere;
006257 sSelect.pOrderBy = pOrderBy;
006258 sSelect.pEList = pResultSet;
006259 sqlite3TreeViewSelect(0, &sSelect, 0);
006260 }
006261 if( sqlite3WhereTrace & 0x4000 ){ /* Display all WHERE clause terms */
006262 sqlite3DebugPrintf("---- WHERE clause at start of analysis:\n");
006263 sqlite3WhereClausePrint(sWLB.pWC);
006264 }
006265 }
006266 #endif
006267
006268 if( nTabList!=1 || whereShortCut(&sWLB)==0 ){
006269 rc = whereLoopAddAll(&sWLB);
006270 if( rc ) goto whereBeginError;
006271
006272 #ifdef SQLITE_ENABLE_STAT4
006273 /* If one or more WhereTerm.truthProb values were used in estimating
006274 ** loop parameters, but then those truthProb values were subsequently
006275 ** changed based on STAT4 information while computing subsequent loops,
006276 ** then we need to rerun the whole loop building process so that all
006277 ** loops will be built using the revised truthProb values. */
006278 if( sWLB.bldFlags2 & SQLITE_BLDF2_2NDPASS ){
006279 WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC);
006280 WHERETRACE(0xffffffff,
006281 ("**** Redo all loop computations due to"
006282 " TERM_HIGHTRUTH changes ****\n"));
006283 while( pWInfo->pLoops ){
006284 WhereLoop *p = pWInfo->pLoops;
006285 pWInfo->pLoops = p->pNextLoop;
006286 whereLoopDelete(db, p);
006287 }
006288 rc = whereLoopAddAll(&sWLB);
006289 if( rc ) goto whereBeginError;
006290 }
006291 #endif
006292 WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC);
006293
006294 wherePathSolver(pWInfo, 0);
006295 if( db->mallocFailed ) goto whereBeginError;
006296 if( pWInfo->pOrderBy ){
006297 wherePathSolver(pWInfo, pWInfo->nRowOut+1);
006298 if( db->mallocFailed ) goto whereBeginError;
006299 }
006300
006301 /* TUNING: Assume that a DISTINCT clause on a subquery reduces
006302 ** the output size by a factor of 8 (LogEst -30).
006303 */
006304 if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0 ){
006305 WHERETRACE(0x0080,("nRowOut reduced from %d to %d due to DISTINCT\n",
006306 pWInfo->nRowOut, pWInfo->nRowOut-30));
006307 pWInfo->nRowOut -= 30;
006308 }
006309
006310 }
006311 assert( pWInfo->pTabList!=0 );
006312 if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){
006313 whereReverseScanOrder(pWInfo);
006314 }
006315 if( pParse->nErr ){
006316 goto whereBeginError;
006317 }
006318 assert( db->mallocFailed==0 );
006319 #ifdef WHERETRACE_ENABLED
006320 if( sqlite3WhereTrace ){
006321 sqlite3DebugPrintf("---- Solution nRow=%d", pWInfo->nRowOut);
006322 if( pWInfo->nOBSat>0 ){
006323 sqlite3DebugPrintf(" ORDERBY=%d,0x%llx", pWInfo->nOBSat, pWInfo->revMask);
006324 }
006325 switch( pWInfo->eDistinct ){
006326 case WHERE_DISTINCT_UNIQUE: {
006327 sqlite3DebugPrintf(" DISTINCT=unique");
006328 break;
006329 }
006330 case WHERE_DISTINCT_ORDERED: {
006331 sqlite3DebugPrintf(" DISTINCT=ordered");
006332 break;
006333 }
006334 case WHERE_DISTINCT_UNORDERED: {
006335 sqlite3DebugPrintf(" DISTINCT=unordered");
006336 break;
006337 }
006338 }
006339 sqlite3DebugPrintf("\n");
006340 for(ii=0; ii<pWInfo->nLevel; ii++){
006341 sqlite3WhereLoopPrint(pWInfo->a[ii].pWLoop, sWLB.pWC);
006342 }
006343 }
006344 #endif
006345
006346 /* Attempt to omit tables from a join that do not affect the result.
006347 ** See the comment on whereOmitNoopJoin() for further information.
006348 **
006349 ** This query optimization is factored out into a separate "no-inline"
006350 ** procedure to keep the sqlite3WhereBegin() procedure from becoming
006351 ** too large. If sqlite3WhereBegin() becomes too large, that prevents
006352 ** some C-compiler optimizers from in-lining the
006353 ** sqlite3WhereCodeOneLoopStart() procedure, and it is important to
006354 ** in-line sqlite3WhereCodeOneLoopStart() for performance reasons.
006355 */
006356 notReady = ~(Bitmask)0;
006357 if( pWInfo->nLevel>=2
006358 && pResultSet!=0 /* these two combine to guarantee */
006359 && 0==(wctrlFlags & WHERE_AGG_DISTINCT) /* condition (1) above */
006360 && OptimizationEnabled(db, SQLITE_OmitNoopJoin)
006361 ){
006362 notReady = whereOmitNoopJoin(pWInfo, notReady);
006363 nTabList = pWInfo->nLevel;
006364 assert( nTabList>0 );
006365 }
006366
006367 /* Check to see if there are any SEARCH loops that might benefit from
006368 ** using a Bloom filter.
006369 */
006370 if( pWInfo->nLevel>=2
006371 && OptimizationEnabled(db, SQLITE_BloomFilter)
006372 ){
006373 whereCheckIfBloomFilterIsUseful(pWInfo);
006374 }
006375
006376 #if defined(WHERETRACE_ENABLED)
006377 if( sqlite3WhereTrace & 0x4000 ){ /* Display all terms of the WHERE clause */
006378 sqlite3DebugPrintf("---- WHERE clause at end of analysis:\n");
006379 sqlite3WhereClausePrint(sWLB.pWC);
006380 }
006381 WHERETRACE(0xffffffff,("*** Optimizer Finished ***\n"));
006382 #endif
006383 pWInfo->pParse->nQueryLoop += pWInfo->nRowOut;
006384
006385 /* If the caller is an UPDATE or DELETE statement that is requesting
006386 ** to use a one-pass algorithm, determine if this is appropriate.
006387 **
006388 ** A one-pass approach can be used if the caller has requested one
006389 ** and either (a) the scan visits at most one row or (b) each
006390 ** of the following are true:
006391 **
006392 ** * the caller has indicated that a one-pass approach can be used
006393 ** with multiple rows (by setting WHERE_ONEPASS_MULTIROW), and
006394 ** * the table is not a virtual table, and
006395 ** * either the scan does not use the OR optimization or the caller
006396 ** is a DELETE operation (WHERE_DUPLICATES_OK is only specified
006397 ** for DELETE).
006398 **
006399 ** The last qualification is because an UPDATE statement uses
006400 ** WhereInfo.aiCurOnePass[1] to determine whether or not it really can
006401 ** use a one-pass approach, and this is not set accurately for scans
006402 ** that use the OR optimization.
006403 */
006404 assert( (wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || pWInfo->nLevel==1 );
006405 if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 ){
006406 int wsFlags = pWInfo->a[0].pWLoop->wsFlags;
006407 int bOnerow = (wsFlags & WHERE_ONEROW)!=0;
006408 assert( !(wsFlags & WHERE_VIRTUALTABLE) || IsVirtual(pTabList->a[0].pTab) );
006409 if( bOnerow || (
006410 0!=(wctrlFlags & WHERE_ONEPASS_MULTIROW)
006411 && !IsVirtual(pTabList->a[0].pTab)
006412 && (0==(wsFlags & WHERE_MULTI_OR) || (wctrlFlags & WHERE_DUPLICATES_OK))
006413 && OptimizationEnabled(db, SQLITE_OnePass)
006414 )){
006415 pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI;
006416 if( HasRowid(pTabList->a[0].pTab) && (wsFlags & WHERE_IDX_ONLY) ){
006417 if( wctrlFlags & WHERE_ONEPASS_MULTIROW ){
006418 bFordelete = OPFLAG_FORDELETE;
006419 }
006420 pWInfo->a[0].pWLoop->wsFlags = (wsFlags & ~WHERE_IDX_ONLY);
006421 }
006422 }
006423 }
006424
006425 /* Open all tables in the pTabList and any indices selected for
006426 ** searching those tables.
006427 */
006428 for(ii=0, pLevel=pWInfo->a; ii<nTabList; ii++, pLevel++){
006429 Table *pTab; /* Table to open */
006430 int iDb; /* Index of database containing table/index */
006431 SrcItem *pTabItem;
006432
006433 pTabItem = &pTabList->a[pLevel->iFrom];
006434 pTab = pTabItem->pTab;
006435 iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
006436 pLoop = pLevel->pWLoop;
006437 if( (pTab->tabFlags & TF_Ephemeral)!=0 || IsView(pTab) ){
006438 /* Do nothing */
006439 }else
006440 #ifndef SQLITE_OMIT_VIRTUALTABLE
006441 if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){
006442 const char *pVTab = (const char *)sqlite3GetVTable(db, pTab);
006443 int iCur = pTabItem->iCursor;
006444 sqlite3VdbeAddOp4(v, OP_VOpen, iCur, 0, 0, pVTab, P4_VTAB);
006445 }else if( IsVirtual(pTab) ){
006446 /* noop */
006447 }else
006448 #endif
006449 if( ((pLoop->wsFlags & WHERE_IDX_ONLY)==0
006450 && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0)
006451 || (pTabItem->fg.jointype & (JT_LTORJ|JT_RIGHT))!=0
006452 ){
006453 int op = OP_OpenRead;
006454 if( pWInfo->eOnePass!=ONEPASS_OFF ){
006455 op = OP_OpenWrite;
006456 pWInfo->aiCurOnePass[0] = pTabItem->iCursor;
006457 };
006458 sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op);
006459 assert( pTabItem->iCursor==pLevel->iTabCur );
006460 testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS-1 );
006461 testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS );
006462 if( pWInfo->eOnePass==ONEPASS_OFF
006463 && pTab->nCol<BMS
006464 && (pTab->tabFlags & (TF_HasGenerated|TF_WithoutRowid))==0
006465 && (pLoop->wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))==0
006466 ){
006467 /* If we know that only a prefix of the record will be used,
006468 ** it is advantageous to reduce the "column count" field in
006469 ** the P4 operand of the OP_OpenRead/Write opcode. */
006470 Bitmask b = pTabItem->colUsed;
006471 int n = 0;
006472 for(; b; b=b>>1, n++){}
006473 sqlite3VdbeChangeP4(v, -1, SQLITE_INT_TO_PTR(n), P4_INT32);
006474 assert( n<=pTab->nCol );
006475 }
006476 #ifdef SQLITE_ENABLE_CURSOR_HINTS
006477 if( pLoop->u.btree.pIndex!=0 && (pTab->tabFlags & TF_WithoutRowid)==0 ){
006478 sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ|bFordelete);
006479 }else
006480 #endif
006481 {
006482 sqlite3VdbeChangeP5(v, bFordelete);
006483 }
006484 #ifdef SQLITE_ENABLE_COLUMN_USED_MASK
006485 sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, pTabItem->iCursor, 0, 0,
006486 (const u8*)&pTabItem->colUsed, P4_INT64);
006487 #endif
006488 }else{
006489 sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName);
006490 }
006491 if( pLoop->wsFlags & WHERE_INDEXED ){
006492 Index *pIx = pLoop->u.btree.pIndex;
006493 int iIndexCur;
006494 int op = OP_OpenRead;
006495 /* iAuxArg is always set to a positive value if ONEPASS is possible */
006496 assert( iAuxArg!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 );
006497 if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIx)
006498 && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0
006499 ){
006500 /* This is one term of an OR-optimization using the PRIMARY KEY of a
006501 ** WITHOUT ROWID table. No need for a separate index */
006502 iIndexCur = pLevel->iTabCur;
006503 op = 0;
006504 }else if( pWInfo->eOnePass!=ONEPASS_OFF ){
006505 Index *pJ = pTabItem->pTab->pIndex;
006506 iIndexCur = iAuxArg;
006507 assert( wctrlFlags & WHERE_ONEPASS_DESIRED );
006508 while( ALWAYS(pJ) && pJ!=pIx ){
006509 iIndexCur++;
006510 pJ = pJ->pNext;
006511 }
006512 op = OP_OpenWrite;
006513 pWInfo->aiCurOnePass[1] = iIndexCur;
006514 }else if( iAuxArg && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ){
006515 iIndexCur = iAuxArg;
006516 op = OP_ReopenIdx;
006517 }else{
006518 iIndexCur = pParse->nTab++;
006519 if( pIx->bHasExpr && OptimizationEnabled(db, SQLITE_IndexedExpr) ){
006520 whereAddIndexedExpr(pParse, pIx, iIndexCur, pTabItem);
006521 }
006522 if( pIx->pPartIdxWhere && (pTabItem->fg.jointype & JT_RIGHT)==0 ){
006523 wherePartIdxExpr(
006524 pParse, pIx, pIx->pPartIdxWhere, 0, iIndexCur, pTabItem
006525 );
006526 }
006527 }
006528 pLevel->iIdxCur = iIndexCur;
006529 assert( pIx!=0 );
006530 assert( pIx->pSchema==pTab->pSchema );
006531 assert( iIndexCur>=0 );
006532 if( op ){
006533 sqlite3VdbeAddOp3(v, op, iIndexCur, pIx->tnum, iDb);
006534 sqlite3VdbeSetP4KeyInfo(pParse, pIx);
006535 if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0
006536 && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0
006537 && (pLoop->wsFlags & WHERE_BIGNULL_SORT)==0
006538 && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0
006539 && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0
006540 && pWInfo->eDistinct!=WHERE_DISTINCT_ORDERED
006541 ){
006542 sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ);
006543 }
006544 VdbeComment((v, "%s", pIx->zName));
006545 #ifdef SQLITE_ENABLE_COLUMN_USED_MASK
006546 {
006547 u64 colUsed = 0;
006548 int ii, jj;
006549 for(ii=0; ii<pIx->nColumn; ii++){
006550 jj = pIx->aiColumn[ii];
006551 if( jj<0 ) continue;
006552 if( jj>63 ) jj = 63;
006553 if( (pTabItem->colUsed & MASKBIT(jj))==0 ) continue;
006554 colUsed |= ((u64)1)<<(ii<63 ? ii : 63);
006555 }
006556 sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, iIndexCur, 0, 0,
006557 (u8*)&colUsed, P4_INT64);
006558 }
006559 #endif /* SQLITE_ENABLE_COLUMN_USED_MASK */
006560 }
006561 }
006562 if( iDb>=0 ) sqlite3CodeVerifySchema(pParse, iDb);
006563 if( (pTabItem->fg.jointype & JT_RIGHT)!=0
006564 && (pLevel->pRJ = sqlite3WhereMalloc(pWInfo, sizeof(WhereRightJoin)))!=0
006565 ){
006566 WhereRightJoin *pRJ = pLevel->pRJ;
006567 pRJ->iMatch = pParse->nTab++;
006568 pRJ->regBloom = ++pParse->nMem;
006569 sqlite3VdbeAddOp2(v, OP_Blob, 65536, pRJ->regBloom);
006570 pRJ->regReturn = ++pParse->nMem;
006571 sqlite3VdbeAddOp2(v, OP_Null, 0, pRJ->regReturn);
006572 assert( pTab==pTabItem->pTab );
006573 if( HasRowid(pTab) ){
006574 KeyInfo *pInfo;
006575 sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRJ->iMatch, 1);
006576 pInfo = sqlite3KeyInfoAlloc(pParse->db, 1, 0);
006577 if( pInfo ){
006578 pInfo->aColl[0] = 0;
006579 pInfo->aSortFlags[0] = 0;
006580 sqlite3VdbeAppendP4(v, pInfo, P4_KEYINFO);
006581 }
006582 }else{
006583 Index *pPk = sqlite3PrimaryKeyIndex(pTab);
006584 sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRJ->iMatch, pPk->nKeyCol);
006585 sqlite3VdbeSetP4KeyInfo(pParse, pPk);
006586 }
006587 pLoop->wsFlags &= ~WHERE_IDX_ONLY;
006588 /* The nature of RIGHT JOIN processing is such that it messes up
006589 ** the output order. So omit any ORDER BY/GROUP BY elimination
006590 ** optimizations. We need to do an actual sort for RIGHT JOIN. */
006591 pWInfo->nOBSat = 0;
006592 pWInfo->eDistinct = WHERE_DISTINCT_UNORDERED;
006593 }
006594 }
006595 pWInfo->iTop = sqlite3VdbeCurrentAddr(v);
006596 if( db->mallocFailed ) goto whereBeginError;
006597
006598 /* Generate the code to do the search. Each iteration of the for
006599 ** loop below generates code for a single nested loop of the VM
006600 ** program.
006601 */
006602 for(ii=0; ii<nTabList; ii++){
006603 int addrExplain;
006604 int wsFlags;
006605 SrcItem *pSrc;
006606 if( pParse->nErr ) goto whereBeginError;
006607 pLevel = &pWInfo->a[ii];
006608 wsFlags = pLevel->pWLoop->wsFlags;
006609 pSrc = &pTabList->a[pLevel->iFrom];
006610 if( pSrc->fg.isMaterialized ){
006611 if( pSrc->fg.isCorrelated ){
006612 sqlite3VdbeAddOp2(v, OP_Gosub, pSrc->regReturn, pSrc->addrFillSub);
006613 }else{
006614 int iOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
006615 sqlite3VdbeAddOp2(v, OP_Gosub, pSrc->regReturn, pSrc->addrFillSub);
006616 sqlite3VdbeJumpHere(v, iOnce);
006617 }
006618 }
006619 assert( pTabList == pWInfo->pTabList );
006620 if( (wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))!=0 ){
006621 if( (wsFlags & WHERE_AUTO_INDEX)!=0 ){
006622 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
006623 constructAutomaticIndex(pParse, &pWInfo->sWC, notReady, pLevel);
006624 #endif
006625 }else{
006626 sqlite3ConstructBloomFilter(pWInfo, ii, pLevel, notReady);
006627 }
006628 if( db->mallocFailed ) goto whereBeginError;
006629 }
006630 addrExplain = sqlite3WhereExplainOneScan(
006631 pParse, pTabList, pLevel, wctrlFlags
006632 );
006633 pLevel->addrBody = sqlite3VdbeCurrentAddr(v);
006634 notReady = sqlite3WhereCodeOneLoopStart(pParse,v,pWInfo,ii,pLevel,notReady);
006635 pWInfo->iContinue = pLevel->addrCont;
006636 if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_OR_SUBCLAUSE)==0 ){
006637 sqlite3WhereAddScanStatus(v, pTabList, pLevel, addrExplain);
006638 }
006639 }
006640
006641 /* Done. */
006642 VdbeModuleComment((v, "Begin WHERE-core"));
006643 pWInfo->iEndWhere = sqlite3VdbeCurrentAddr(v);
006644 return pWInfo;
006645
006646 /* Jump here if malloc fails */
006647 whereBeginError:
006648 if( pWInfo ){
006649 pParse->nQueryLoop = pWInfo->savedNQueryLoop;
006650 whereInfoFree(db, pWInfo);
006651 }
006652 #ifdef WHERETRACE_ENABLED
006653 /* Prevent harmless compiler warnings about debugging routines
006654 ** being declared but never used */
006655 sqlite3ShowWhereLoopList(0);
006656 #endif /* WHERETRACE_ENABLED */
006657 return 0;
006658 }
006659
006660 /*
006661 ** Part of sqlite3WhereEnd() will rewrite opcodes to reference the
006662 ** index rather than the main table. In SQLITE_DEBUG mode, we want
006663 ** to trace those changes if PRAGMA vdbe_addoptrace=on. This routine
006664 ** does that.
006665 */
006666 #ifndef SQLITE_DEBUG
006667 # define OpcodeRewriteTrace(D,K,P) /* no-op */
006668 #else
006669 # define OpcodeRewriteTrace(D,K,P) sqlite3WhereOpcodeRewriteTrace(D,K,P)
006670 static void sqlite3WhereOpcodeRewriteTrace(
006671 sqlite3 *db,
006672 int pc,
006673 VdbeOp *pOp
006674 ){
006675 if( (db->flags & SQLITE_VdbeAddopTrace)==0 ) return;
006676 sqlite3VdbePrintOp(0, pc, pOp);
006677 }
006678 #endif
006679
006680 #ifdef SQLITE_DEBUG
006681 /*
006682 ** Return true if cursor iCur is opened by instruction k of the
006683 ** bytecode. Used inside of assert() only.
006684 */
006685 static int cursorIsOpen(Vdbe *v, int iCur, int k){
006686 while( k>=0 ){
006687 VdbeOp *pOp = sqlite3VdbeGetOp(v,k--);
006688 if( pOp->p1!=iCur ) continue;
006689 if( pOp->opcode==OP_Close ) return 0;
006690 if( pOp->opcode==OP_OpenRead ) return 1;
006691 if( pOp->opcode==OP_OpenWrite ) return 1;
006692 if( pOp->opcode==OP_OpenDup ) return 1;
006693 if( pOp->opcode==OP_OpenAutoindex ) return 1;
006694 if( pOp->opcode==OP_OpenEphemeral ) return 1;
006695 }
006696 return 0;
006697 }
006698 #endif /* SQLITE_DEBUG */
006699
006700 /*
006701 ** Generate the end of the WHERE loop. See comments on
006702 ** sqlite3WhereBegin() for additional information.
006703 */
006704 void sqlite3WhereEnd(WhereInfo *pWInfo){
006705 Parse *pParse = pWInfo->pParse;
006706 Vdbe *v = pParse->pVdbe;
006707 int i;
006708 WhereLevel *pLevel;
006709 WhereLoop *pLoop;
006710 SrcList *pTabList = pWInfo->pTabList;
006711 sqlite3 *db = pParse->db;
006712 int iEnd = sqlite3VdbeCurrentAddr(v);
006713 int nRJ = 0;
006714
006715 /* Generate loop termination code.
006716 */
006717 VdbeModuleComment((v, "End WHERE-core"));
006718 for(i=pWInfo->nLevel-1; i>=0; i--){
006719 int addr;
006720 pLevel = &pWInfo->a[i];
006721 if( pLevel->pRJ ){
006722 /* Terminate the subroutine that forms the interior of the loop of
006723 ** the RIGHT JOIN table */
006724 WhereRightJoin *pRJ = pLevel->pRJ;
006725 sqlite3VdbeResolveLabel(v, pLevel->addrCont);
006726 pLevel->addrCont = 0;
006727 pRJ->endSubrtn = sqlite3VdbeCurrentAddr(v);
006728 sqlite3VdbeAddOp3(v, OP_Return, pRJ->regReturn, pRJ->addrSubrtn, 1);
006729 VdbeCoverage(v);
006730 nRJ++;
006731 }
006732 pLoop = pLevel->pWLoop;
006733 if( pLevel->op!=OP_Noop ){
006734 #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT
006735 int addrSeek = 0;
006736 Index *pIdx;
006737 int n;
006738 if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED
006739 && i==pWInfo->nLevel-1 /* Ticket [ef9318757b152e3] 2017-10-21 */
006740 && (pLoop->wsFlags & WHERE_INDEXED)!=0
006741 && (pIdx = pLoop->u.btree.pIndex)->hasStat1
006742 && (n = pLoop->u.btree.nDistinctCol)>0
006743 && pIdx->aiRowLogEst[n]>=36
006744 ){
006745 int r1 = pParse->nMem+1;
006746 int j, op;
006747 for(j=0; j<n; j++){
006748 sqlite3VdbeAddOp3(v, OP_Column, pLevel->iIdxCur, j, r1+j);
006749 }
006750 pParse->nMem += n+1;
006751 op = pLevel->op==OP_Prev ? OP_SeekLT : OP_SeekGT;
006752 addrSeek = sqlite3VdbeAddOp4Int(v, op, pLevel->iIdxCur, 0, r1, n);
006753 VdbeCoverageIf(v, op==OP_SeekLT);
006754 VdbeCoverageIf(v, op==OP_SeekGT);
006755 sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2);
006756 }
006757 #endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */
006758 /* The common case: Advance to the next row */
006759 if( pLevel->addrCont ) sqlite3VdbeResolveLabel(v, pLevel->addrCont);
006760 sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3);
006761 sqlite3VdbeChangeP5(v, pLevel->p5);
006762 VdbeCoverage(v);
006763 VdbeCoverageIf(v, pLevel->op==OP_Next);
006764 VdbeCoverageIf(v, pLevel->op==OP_Prev);
006765 VdbeCoverageIf(v, pLevel->op==OP_VNext);
006766 if( pLevel->regBignull ){
006767 sqlite3VdbeResolveLabel(v, pLevel->addrBignull);
006768 sqlite3VdbeAddOp2(v, OP_DecrJumpZero, pLevel->regBignull, pLevel->p2-1);
006769 VdbeCoverage(v);
006770 }
006771 #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT
006772 if( addrSeek ) sqlite3VdbeJumpHere(v, addrSeek);
006773 #endif
006774 }else if( pLevel->addrCont ){
006775 sqlite3VdbeResolveLabel(v, pLevel->addrCont);
006776 }
006777 if( (pLoop->wsFlags & WHERE_IN_ABLE)!=0 && pLevel->u.in.nIn>0 ){
006778 struct InLoop *pIn;
006779 int j;
006780 sqlite3VdbeResolveLabel(v, pLevel->addrNxt);
006781 for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){
006782 assert( sqlite3VdbeGetOp(v, pIn->addrInTop+1)->opcode==OP_IsNull
006783 || pParse->db->mallocFailed );
006784 sqlite3VdbeJumpHere(v, pIn->addrInTop+1);
006785 if( pIn->eEndLoopOp!=OP_Noop ){
006786 if( pIn->nPrefix ){
006787 int bEarlyOut =
006788 (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0
006789 && (pLoop->wsFlags & WHERE_IN_EARLYOUT)!=0;
006790 if( pLevel->iLeftJoin ){
006791 /* For LEFT JOIN queries, cursor pIn->iCur may not have been
006792 ** opened yet. This occurs for WHERE clauses such as
006793 ** "a = ? AND b IN (...)", where the index is on (a, b). If
006794 ** the RHS of the (a=?) is NULL, then the "b IN (...)" may
006795 ** never have been coded, but the body of the loop run to
006796 ** return the null-row. So, if the cursor is not open yet,
006797 ** jump over the OP_Next or OP_Prev instruction about to
006798 ** be coded. */
006799 sqlite3VdbeAddOp2(v, OP_IfNotOpen, pIn->iCur,
006800 sqlite3VdbeCurrentAddr(v) + 2 + bEarlyOut);
006801 VdbeCoverage(v);
006802 }
006803 if( bEarlyOut ){
006804 sqlite3VdbeAddOp4Int(v, OP_IfNoHope, pLevel->iIdxCur,
006805 sqlite3VdbeCurrentAddr(v)+2,
006806 pIn->iBase, pIn->nPrefix);
006807 VdbeCoverage(v);
006808 /* Retarget the OP_IsNull against the left operand of IN so
006809 ** it jumps past the OP_IfNoHope. This is because the
006810 ** OP_IsNull also bypasses the OP_Affinity opcode that is
006811 ** required by OP_IfNoHope. */
006812 sqlite3VdbeJumpHere(v, pIn->addrInTop+1);
006813 }
006814 }
006815 sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop);
006816 VdbeCoverage(v);
006817 VdbeCoverageIf(v, pIn->eEndLoopOp==OP_Prev);
006818 VdbeCoverageIf(v, pIn->eEndLoopOp==OP_Next);
006819 }
006820 sqlite3VdbeJumpHere(v, pIn->addrInTop-1);
006821 }
006822 }
006823 sqlite3VdbeResolveLabel(v, pLevel->addrBrk);
006824 if( pLevel->pRJ ){
006825 sqlite3VdbeAddOp3(v, OP_Return, pLevel->pRJ->regReturn, 0, 1);
006826 VdbeCoverage(v);
006827 }
006828 if( pLevel->addrSkip ){
006829 sqlite3VdbeGoto(v, pLevel->addrSkip);
006830 VdbeComment((v, "next skip-scan on %s", pLoop->u.btree.pIndex->zName));
006831 sqlite3VdbeJumpHere(v, pLevel->addrSkip);
006832 sqlite3VdbeJumpHere(v, pLevel->addrSkip-2);
006833 }
006834 #ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS
006835 if( pLevel->addrLikeRep ){
006836 sqlite3VdbeAddOp2(v, OP_DecrJumpZero, (int)(pLevel->iLikeRepCntr>>1),
006837 pLevel->addrLikeRep);
006838 VdbeCoverage(v);
006839 }
006840 #endif
006841 if( pLevel->iLeftJoin ){
006842 int ws = pLoop->wsFlags;
006843 addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v);
006844 assert( (ws & WHERE_IDX_ONLY)==0 || (ws & WHERE_INDEXED)!=0 );
006845 if( (ws & WHERE_IDX_ONLY)==0 ){
006846 assert( pLevel->iTabCur==pTabList->a[pLevel->iFrom].iCursor );
006847 sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iTabCur);
006848 }
006849 if( (ws & WHERE_INDEXED)
006850 || ((ws & WHERE_MULTI_OR) && pLevel->u.pCoveringIdx)
006851 ){
006852 if( ws & WHERE_MULTI_OR ){
006853 Index *pIx = pLevel->u.pCoveringIdx;
006854 int iDb = sqlite3SchemaToIndex(db, pIx->pSchema);
006855 sqlite3VdbeAddOp3(v, OP_ReopenIdx, pLevel->iIdxCur, pIx->tnum, iDb);
006856 sqlite3VdbeSetP4KeyInfo(pParse, pIx);
006857 }
006858 sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur);
006859 }
006860 if( pLevel->op==OP_Return ){
006861 sqlite3VdbeAddOp2(v, OP_Gosub, pLevel->p1, pLevel->addrFirst);
006862 }else{
006863 sqlite3VdbeGoto(v, pLevel->addrFirst);
006864 }
006865 sqlite3VdbeJumpHere(v, addr);
006866 }
006867 VdbeModuleComment((v, "End WHERE-loop%d: %s", i,
006868 pWInfo->pTabList->a[pLevel->iFrom].pTab->zName));
006869 }
006870
006871 assert( pWInfo->nLevel<=pTabList->nSrc );
006872 for(i=0, pLevel=pWInfo->a; i<pWInfo->nLevel; i++, pLevel++){
006873 int k, last;
006874 VdbeOp *pOp, *pLastOp;
006875 Index *pIdx = 0;
006876 SrcItem *pTabItem = &pTabList->a[pLevel->iFrom];
006877 Table *pTab = pTabItem->pTab;
006878 assert( pTab!=0 );
006879 pLoop = pLevel->pWLoop;
006880
006881 /* Do RIGHT JOIN processing. Generate code that will output the
006882 ** unmatched rows of the right operand of the RIGHT JOIN with
006883 ** all of the columns of the left operand set to NULL.
006884 */
006885 if( pLevel->pRJ ){
006886 sqlite3WhereRightJoinLoop(pWInfo, i, pLevel);
006887 continue;
006888 }
006889
006890 /* For a co-routine, change all OP_Column references to the table of
006891 ** the co-routine into OP_Copy of result contained in a register.
006892 ** OP_Rowid becomes OP_Null.
006893 */
006894 if( pTabItem->fg.viaCoroutine ){
006895 testcase( pParse->db->mallocFailed );
006896 translateColumnToCopy(pParse, pLevel->addrBody, pLevel->iTabCur,
006897 pTabItem->regResult, 0);
006898 continue;
006899 }
006900
006901 /* If this scan uses an index, make VDBE code substitutions to read data
006902 ** from the index instead of from the table where possible. In some cases
006903 ** this optimization prevents the table from ever being read, which can
006904 ** yield a significant performance boost.
006905 **
006906 ** Calls to the code generator in between sqlite3WhereBegin and
006907 ** sqlite3WhereEnd will have created code that references the table
006908 ** directly. This loop scans all that code looking for opcodes
006909 ** that reference the table and converts them into opcodes that
006910 ** reference the index.
006911 */
006912 if( pLoop->wsFlags & (WHERE_INDEXED|WHERE_IDX_ONLY) ){
006913 pIdx = pLoop->u.btree.pIndex;
006914 }else if( pLoop->wsFlags & WHERE_MULTI_OR ){
006915 pIdx = pLevel->u.pCoveringIdx;
006916 }
006917 if( pIdx
006918 && !db->mallocFailed
006919 ){
006920 if( pWInfo->eOnePass==ONEPASS_OFF || !HasRowid(pIdx->pTable) ){
006921 last = iEnd;
006922 }else{
006923 last = pWInfo->iEndWhere;
006924 }
006925 if( pIdx->bHasExpr ){
006926 IndexedExpr *p = pParse->pIdxEpr;
006927 while( p ){
006928 if( p->iIdxCur==pLevel->iIdxCur ){
006929 #ifdef WHERETRACE_ENABLED
006930 if( sqlite3WhereTrace & 0x200 ){
006931 sqlite3DebugPrintf("Disable pParse->pIdxEpr term {%d,%d}\n",
006932 p->iIdxCur, p->iIdxCol);
006933 if( sqlite3WhereTrace & 0x5000 ) sqlite3ShowExpr(p->pExpr);
006934 }
006935 #endif
006936 p->iDataCur = -1;
006937 p->iIdxCur = -1;
006938 }
006939 p = p->pIENext;
006940 }
006941 }
006942 k = pLevel->addrBody + 1;
006943 #ifdef SQLITE_DEBUG
006944 if( db->flags & SQLITE_VdbeAddopTrace ){
006945 printf("TRANSLATE cursor %d->%d in opcode range %d..%d\n",
006946 pLevel->iTabCur, pLevel->iIdxCur, k, last-1);
006947 }
006948 /* Proof that the "+1" on the k value above is safe */
006949 pOp = sqlite3VdbeGetOp(v, k - 1);
006950 assert( pOp->opcode!=OP_Column || pOp->p1!=pLevel->iTabCur );
006951 assert( pOp->opcode!=OP_Rowid || pOp->p1!=pLevel->iTabCur );
006952 assert( pOp->opcode!=OP_IfNullRow || pOp->p1!=pLevel->iTabCur );
006953 #endif
006954 pOp = sqlite3VdbeGetOp(v, k);
006955 pLastOp = pOp + (last - k);
006956 assert( pOp<=pLastOp );
006957 do{
006958 if( pOp->p1!=pLevel->iTabCur ){
006959 /* no-op */
006960 }else if( pOp->opcode==OP_Column
006961 #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
006962 || pOp->opcode==OP_Offset
006963 #endif
006964 ){
006965 int x = pOp->p2;
006966 assert( pIdx->pTable==pTab );
006967 #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
006968 if( pOp->opcode==OP_Offset ){
006969 /* Do not need to translate the column number */
006970 }else
006971 #endif
006972 if( !HasRowid(pTab) ){
006973 Index *pPk = sqlite3PrimaryKeyIndex(pTab);
006974 x = pPk->aiColumn[x];
006975 assert( x>=0 );
006976 }else{
006977 testcase( x!=sqlite3StorageColumnToTable(pTab,x) );
006978 x = sqlite3StorageColumnToTable(pTab,x);
006979 }
006980 x = sqlite3TableColumnToIndex(pIdx, x);
006981 if( x>=0 ){
006982 pOp->p2 = x;
006983 pOp->p1 = pLevel->iIdxCur;
006984 OpcodeRewriteTrace(db, k, pOp);
006985 }else{
006986 /* Unable to translate the table reference into an index
006987 ** reference. Verify that this is harmless - that the
006988 ** table being referenced really is open.
006989 */
006990 #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
006991 assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0
006992 || cursorIsOpen(v,pOp->p1,k)
006993 || pOp->opcode==OP_Offset
006994 );
006995 #else
006996 assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0
006997 || cursorIsOpen(v,pOp->p1,k)
006998 );
006999 #endif
007000 }
007001 }else if( pOp->opcode==OP_Rowid ){
007002 pOp->p1 = pLevel->iIdxCur;
007003 pOp->opcode = OP_IdxRowid;
007004 OpcodeRewriteTrace(db, k, pOp);
007005 }else if( pOp->opcode==OP_IfNullRow ){
007006 pOp->p1 = pLevel->iIdxCur;
007007 OpcodeRewriteTrace(db, k, pOp);
007008 }
007009 #ifdef SQLITE_DEBUG
007010 k++;
007011 #endif
007012 }while( (++pOp)<pLastOp );
007013 #ifdef SQLITE_DEBUG
007014 if( db->flags & SQLITE_VdbeAddopTrace ) printf("TRANSLATE complete\n");
007015 #endif
007016 }
007017 }
007018
007019 /* The "break" point is here, just past the end of the outer loop.
007020 ** Set it.
007021 */
007022 sqlite3VdbeResolveLabel(v, pWInfo->iBreak);
007023
007024 /* Final cleanup
007025 */
007026 pParse->nQueryLoop = pWInfo->savedNQueryLoop;
007027 whereInfoFree(db, pWInfo);
007028 pParse->withinRJSubrtn -= nRJ;
007029 return;
007030 }