000001 /*
000002 ** 2001 September 15
000003 **
000004 ** The author disclaims copyright to this source code. In place of
000005 ** a legal notice, here is a blessing:
000006 **
000007 ** May you do good and not evil.
000008 ** May you find forgiveness for yourself and forgive others.
000009 ** May you share freely, never taking more than you give.
000010 **
000011 *************************************************************************
000012 ** This module contains C code that generates VDBE code used to process
000013 ** the WHERE clause of SQL statements. This module is responsible for
000014 ** generating the code that loops through a table looking for applicable
000015 ** rows. Indices are selected and used to speed the search when doing
000016 ** so is applicable. Because this module is responsible for selecting
000017 ** indices, you might also think of this module as the "query optimizer".
000018 */
000019 #include "sqliteInt.h"
000020 #include "whereInt.h"
000021
000022 /*
000023 ** Extra information appended to the end of sqlite3_index_info but not
000024 ** visible to the xBestIndex function, at least not directly. The
000025 ** sqlite3_vtab_collation() interface knows how to reach it, however.
000026 **
000027 ** This object is not an API and can be changed from one release to the
000028 ** next. As long as allocateIndexInfo() and sqlite3_vtab_collation()
000029 ** agree on the structure, all will be well.
000030 */
000031 typedef struct HiddenIndexInfo HiddenIndexInfo;
000032 struct HiddenIndexInfo {
000033 WhereClause *pWC; /* The Where clause being analyzed */
000034 Parse *pParse; /* The parsing context */
000035 int eDistinct; /* Value to return from sqlite3_vtab_distinct() */
000036 u32 mIn; /* Mask of terms that are <col> IN (...) */
000037 u32 mHandleIn; /* Terms that vtab will handle as <col> IN (...) */
000038 sqlite3_value *aRhs[1]; /* RHS values for constraints. MUST BE LAST
000039 ** because extra space is allocated to hold up
000040 ** to nTerm such values */
000041 };
000042
000043 /* Forward declaration of methods */
000044 static int whereLoopResize(sqlite3*, WhereLoop*, int);
000045
000046 /*
000047 ** Return the estimated number of output rows from a WHERE clause
000048 */
000049 LogEst sqlite3WhereOutputRowCount(WhereInfo *pWInfo){
000050 return pWInfo->nRowOut;
000051 }
000052
000053 /*
000054 ** Return one of the WHERE_DISTINCT_xxxxx values to indicate how this
000055 ** WHERE clause returns outputs for DISTINCT processing.
000056 */
000057 int sqlite3WhereIsDistinct(WhereInfo *pWInfo){
000058 return pWInfo->eDistinct;
000059 }
000060
000061 /*
000062 ** Return the number of ORDER BY terms that are satisfied by the
000063 ** WHERE clause. A return of 0 means that the output must be
000064 ** completely sorted. A return equal to the number of ORDER BY
000065 ** terms means that no sorting is needed at all. A return that
000066 ** is positive but less than the number of ORDER BY terms means that
000067 ** block sorting is required.
000068 */
000069 int sqlite3WhereIsOrdered(WhereInfo *pWInfo){
000070 return pWInfo->nOBSat<0 ? 0 : pWInfo->nOBSat;
000071 }
000072
000073 /*
000074 ** In the ORDER BY LIMIT optimization, if the inner-most loop is known
000075 ** to emit rows in increasing order, and if the last row emitted by the
000076 ** inner-most loop did not fit within the sorter, then we can skip all
000077 ** subsequent rows for the current iteration of the inner loop (because they
000078 ** will not fit in the sorter either) and continue with the second inner
000079 ** loop - the loop immediately outside the inner-most.
000080 **
000081 ** When a row does not fit in the sorter (because the sorter already
000082 ** holds LIMIT+OFFSET rows that are smaller), then a jump is made to the
000083 ** label returned by this function.
000084 **
000085 ** If the ORDER BY LIMIT optimization applies, the jump destination should
000086 ** be the continuation for the second-inner-most loop. If the ORDER BY
000087 ** LIMIT optimization does not apply, then the jump destination should
000088 ** be the continuation for the inner-most loop.
000089 **
000090 ** It is always safe for this routine to return the continuation of the
000091 ** inner-most loop, in the sense that a correct answer will result.
000092 ** Returning the continuation the second inner loop is an optimization
000093 ** that might make the code run a little faster, but should not change
000094 ** the final answer.
000095 */
000096 int sqlite3WhereOrderByLimitOptLabel(WhereInfo *pWInfo){
000097 WhereLevel *pInner;
000098 if( !pWInfo->bOrderedInnerLoop ){
000099 /* The ORDER BY LIMIT optimization does not apply. Jump to the
000100 ** continuation of the inner-most loop. */
000101 return pWInfo->iContinue;
000102 }
000103 pInner = &pWInfo->a[pWInfo->nLevel-1];
000104 assert( pInner->addrNxt!=0 );
000105 return pInner->pRJ ? pWInfo->iContinue : pInner->addrNxt;
000106 }
000107
000108 /*
000109 ** While generating code for the min/max optimization, after handling
000110 ** the aggregate-step call to min() or max(), check to see if any
000111 ** additional looping is required. If the output order is such that
000112 ** we are certain that the correct answer has already been found, then
000113 ** code an OP_Goto to by pass subsequent processing.
000114 **
000115 ** Any extra OP_Goto that is coded here is an optimization. The
000116 ** correct answer should be obtained regardless. This OP_Goto just
000117 ** makes the answer appear faster.
000118 */
000119 void sqlite3WhereMinMaxOptEarlyOut(Vdbe *v, WhereInfo *pWInfo){
000120 WhereLevel *pInner;
000121 int i;
000122 if( !pWInfo->bOrderedInnerLoop ) return;
000123 if( pWInfo->nOBSat==0 ) return;
000124 for(i=pWInfo->nLevel-1; i>=0; i--){
000125 pInner = &pWInfo->a[i];
000126 if( (pInner->pWLoop->wsFlags & WHERE_COLUMN_IN)!=0 ){
000127 sqlite3VdbeGoto(v, pInner->addrNxt);
000128 return;
000129 }
000130 }
000131 sqlite3VdbeGoto(v, pWInfo->iBreak);
000132 }
000133
000134 /*
000135 ** Return the VDBE address or label to jump to in order to continue
000136 ** immediately with the next row of a WHERE clause.
000137 */
000138 int sqlite3WhereContinueLabel(WhereInfo *pWInfo){
000139 assert( pWInfo->iContinue!=0 );
000140 return pWInfo->iContinue;
000141 }
000142
000143 /*
000144 ** Return the VDBE address or label to jump to in order to break
000145 ** out of a WHERE loop.
000146 */
000147 int sqlite3WhereBreakLabel(WhereInfo *pWInfo){
000148 return pWInfo->iBreak;
000149 }
000150
000151 /*
000152 ** Return ONEPASS_OFF (0) if an UPDATE or DELETE statement is unable to
000153 ** operate directly on the rowids returned by a WHERE clause. Return
000154 ** ONEPASS_SINGLE (1) if the statement can operation directly because only
000155 ** a single row is to be changed. Return ONEPASS_MULTI (2) if the one-pass
000156 ** optimization can be used on multiple
000157 **
000158 ** If the ONEPASS optimization is used (if this routine returns true)
000159 ** then also write the indices of open cursors used by ONEPASS
000160 ** into aiCur[0] and aiCur[1]. iaCur[0] gets the cursor of the data
000161 ** table and iaCur[1] gets the cursor used by an auxiliary index.
000162 ** Either value may be -1, indicating that cursor is not used.
000163 ** Any cursors returned will have been opened for writing.
000164 **
000165 ** aiCur[0] and aiCur[1] both get -1 if the where-clause logic is
000166 ** unable to use the ONEPASS optimization.
000167 */
000168 int sqlite3WhereOkOnePass(WhereInfo *pWInfo, int *aiCur){
000169 memcpy(aiCur, pWInfo->aiCurOnePass, sizeof(int)*2);
000170 #ifdef WHERETRACE_ENABLED
000171 if( sqlite3WhereTrace && pWInfo->eOnePass!=ONEPASS_OFF ){
000172 sqlite3DebugPrintf("%s cursors: %d %d\n",
000173 pWInfo->eOnePass==ONEPASS_SINGLE ? "ONEPASS_SINGLE" : "ONEPASS_MULTI",
000174 aiCur[0], aiCur[1]);
000175 }
000176 #endif
000177 return pWInfo->eOnePass;
000178 }
000179
000180 /*
000181 ** Return TRUE if the WHERE loop uses the OP_DeferredSeek opcode to move
000182 ** the data cursor to the row selected by the index cursor.
000183 */
000184 int sqlite3WhereUsesDeferredSeek(WhereInfo *pWInfo){
000185 return pWInfo->bDeferredSeek;
000186 }
000187
000188 /*
000189 ** Move the content of pSrc into pDest
000190 */
000191 static void whereOrMove(WhereOrSet *pDest, WhereOrSet *pSrc){
000192 pDest->n = pSrc->n;
000193 memcpy(pDest->a, pSrc->a, pDest->n*sizeof(pDest->a[0]));
000194 }
000195
000196 /*
000197 ** Try to insert a new prerequisite/cost entry into the WhereOrSet pSet.
000198 **
000199 ** The new entry might overwrite an existing entry, or it might be
000200 ** appended, or it might be discarded. Do whatever is the right thing
000201 ** so that pSet keeps the N_OR_COST best entries seen so far.
000202 */
000203 static int whereOrInsert(
000204 WhereOrSet *pSet, /* The WhereOrSet to be updated */
000205 Bitmask prereq, /* Prerequisites of the new entry */
000206 LogEst rRun, /* Run-cost of the new entry */
000207 LogEst nOut /* Number of outputs for the new entry */
000208 ){
000209 u16 i;
000210 WhereOrCost *p;
000211 for(i=pSet->n, p=pSet->a; i>0; i--, p++){
000212 if( rRun<=p->rRun && (prereq & p->prereq)==prereq ){
000213 goto whereOrInsert_done;
000214 }
000215 if( p->rRun<=rRun && (p->prereq & prereq)==p->prereq ){
000216 return 0;
000217 }
000218 }
000219 if( pSet->n<N_OR_COST ){
000220 p = &pSet->a[pSet->n++];
000221 p->nOut = nOut;
000222 }else{
000223 p = pSet->a;
000224 for(i=1; i<pSet->n; i++){
000225 if( p->rRun>pSet->a[i].rRun ) p = pSet->a + i;
000226 }
000227 if( p->rRun<=rRun ) return 0;
000228 }
000229 whereOrInsert_done:
000230 p->prereq = prereq;
000231 p->rRun = rRun;
000232 if( p->nOut>nOut ) p->nOut = nOut;
000233 return 1;
000234 }
000235
000236 /*
000237 ** Return the bitmask for the given cursor number. Return 0 if
000238 ** iCursor is not in the set.
000239 */
000240 Bitmask sqlite3WhereGetMask(WhereMaskSet *pMaskSet, int iCursor){
000241 int i;
000242 assert( pMaskSet->n<=(int)sizeof(Bitmask)*8 );
000243 assert( pMaskSet->n>0 || pMaskSet->ix[0]<0 );
000244 assert( iCursor>=-1 );
000245 if( pMaskSet->ix[0]==iCursor ){
000246 return 1;
000247 }
000248 for(i=1; i<pMaskSet->n; i++){
000249 if( pMaskSet->ix[i]==iCursor ){
000250 return MASKBIT(i);
000251 }
000252 }
000253 return 0;
000254 }
000255
000256 /* Allocate memory that is automatically freed when pWInfo is freed.
000257 */
000258 void *sqlite3WhereMalloc(WhereInfo *pWInfo, u64 nByte){
000259 WhereMemBlock *pBlock;
000260 pBlock = sqlite3DbMallocRawNN(pWInfo->pParse->db, nByte+sizeof(*pBlock));
000261 if( pBlock ){
000262 pBlock->pNext = pWInfo->pMemToFree;
000263 pBlock->sz = nByte;
000264 pWInfo->pMemToFree = pBlock;
000265 pBlock++;
000266 }
000267 return (void*)pBlock;
000268 }
000269 void *sqlite3WhereRealloc(WhereInfo *pWInfo, void *pOld, u64 nByte){
000270 void *pNew = sqlite3WhereMalloc(pWInfo, nByte);
000271 if( pNew && pOld ){
000272 WhereMemBlock *pOldBlk = (WhereMemBlock*)pOld;
000273 pOldBlk--;
000274 assert( pOldBlk->sz<nByte );
000275 memcpy(pNew, pOld, pOldBlk->sz);
000276 }
000277 return pNew;
000278 }
000279
000280 /*
000281 ** Create a new mask for cursor iCursor.
000282 **
000283 ** There is one cursor per table in the FROM clause. The number of
000284 ** tables in the FROM clause is limited by a test early in the
000285 ** sqlite3WhereBegin() routine. So we know that the pMaskSet->ix[]
000286 ** array will never overflow.
000287 */
000288 static void createMask(WhereMaskSet *pMaskSet, int iCursor){
000289 assert( pMaskSet->n < ArraySize(pMaskSet->ix) );
000290 pMaskSet->ix[pMaskSet->n++] = iCursor;
000291 }
000292
000293 /*
000294 ** If the right-hand branch of the expression is a TK_COLUMN, then return
000295 ** a pointer to the right-hand branch. Otherwise, return NULL.
000296 */
000297 static Expr *whereRightSubexprIsColumn(Expr *p){
000298 p = sqlite3ExprSkipCollateAndLikely(p->pRight);
000299 if( ALWAYS(p!=0) && p->op==TK_COLUMN && !ExprHasProperty(p, EP_FixedCol) ){
000300 return p;
000301 }
000302 return 0;
000303 }
000304
000305 /*
000306 ** Term pTerm is guaranteed to be a WO_IN term. It may be a component term
000307 ** of a vector IN expression of the form "(x, y, ...) IN (SELECT ...)".
000308 ** This function checks to see if the term is compatible with an index
000309 ** column with affinity idxaff (one of the SQLITE_AFF_XYZ values). If so,
000310 ** it returns a pointer to the name of the collation sequence (e.g. "BINARY"
000311 ** or "NOCASE") used by the comparison in pTerm. If it is not compatible
000312 ** with affinity idxaff, NULL is returned.
000313 */
000314 static SQLITE_NOINLINE const char *indexInAffinityOk(
000315 Parse *pParse,
000316 WhereTerm *pTerm,
000317 u8 idxaff
000318 ){
000319 Expr *pX = pTerm->pExpr;
000320 Expr inexpr;
000321
000322 assert( pTerm->eOperator & WO_IN );
000323
000324 if( sqlite3ExprIsVector(pX->pLeft) ){
000325 int iField = pTerm->u.x.iField - 1;
000326 inexpr.flags = 0;
000327 inexpr.op = TK_EQ;
000328 inexpr.pLeft = pX->pLeft->x.pList->a[iField].pExpr;
000329 assert( ExprUseXSelect(pX) );
000330 inexpr.pRight = pX->x.pSelect->pEList->a[iField].pExpr;
000331 pX = &inexpr;
000332 }
000333
000334 if( sqlite3IndexAffinityOk(pX, idxaff) ){
000335 CollSeq *pRet = sqlite3ExprCompareCollSeq(pParse, pX);
000336 return pRet ? pRet->zName : sqlite3StrBINARY;
000337 }
000338 return 0;
000339 }
000340
000341 /*
000342 ** Advance to the next WhereTerm that matches according to the criteria
000343 ** established when the pScan object was initialized by whereScanInit().
000344 ** Return NULL if there are no more matching WhereTerms.
000345 */
000346 static WhereTerm *whereScanNext(WhereScan *pScan){
000347 int iCur; /* The cursor on the LHS of the term */
000348 i16 iColumn; /* The column on the LHS of the term. -1 for IPK */
000349 Expr *pX; /* An expression being tested */
000350 WhereClause *pWC; /* Shorthand for pScan->pWC */
000351 WhereTerm *pTerm; /* The term being tested */
000352 int k = pScan->k; /* Where to start scanning */
000353
000354 assert( pScan->iEquiv<=pScan->nEquiv );
000355 pWC = pScan->pWC;
000356 while(1){
000357 iColumn = pScan->aiColumn[pScan->iEquiv-1];
000358 iCur = pScan->aiCur[pScan->iEquiv-1];
000359 assert( pWC!=0 );
000360 assert( iCur>=0 );
000361 do{
000362 for(pTerm=pWC->a+k; k<pWC->nTerm; k++, pTerm++){
000363 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 || pTerm->leftCursor<0 );
000364 if( pTerm->leftCursor==iCur
000365 && pTerm->u.x.leftColumn==iColumn
000366 && (iColumn!=XN_EXPR
000367 || sqlite3ExprCompareSkip(pTerm->pExpr->pLeft,
000368 pScan->pIdxExpr,iCur)==0)
000369 && (pScan->iEquiv<=1 || !ExprHasProperty(pTerm->pExpr, EP_OuterON))
000370 ){
000371 if( (pTerm->eOperator & WO_EQUIV)!=0
000372 && pScan->nEquiv<ArraySize(pScan->aiCur)
000373 && (pX = whereRightSubexprIsColumn(pTerm->pExpr))!=0
000374 ){
000375 int j;
000376 for(j=0; j<pScan->nEquiv; j++){
000377 if( pScan->aiCur[j]==pX->iTable
000378 && pScan->aiColumn[j]==pX->iColumn ){
000379 break;
000380 }
000381 }
000382 if( j==pScan->nEquiv ){
000383 pScan->aiCur[j] = pX->iTable;
000384 pScan->aiColumn[j] = pX->iColumn;
000385 pScan->nEquiv++;
000386 }
000387 }
000388 if( (pTerm->eOperator & pScan->opMask)!=0 ){
000389 /* Verify the affinity and collating sequence match */
000390 if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){
000391 const char *zCollName;
000392 Parse *pParse = pWC->pWInfo->pParse;
000393 pX = pTerm->pExpr;
000394
000395 if( (pTerm->eOperator & WO_IN) ){
000396 zCollName = indexInAffinityOk(pParse, pTerm, pScan->idxaff);
000397 if( !zCollName ) continue;
000398 }else{
000399 CollSeq *pColl;
000400 if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){
000401 continue;
000402 }
000403 assert(pX->pLeft);
000404 pColl = sqlite3ExprCompareCollSeq(pParse, pX);
000405 zCollName = pColl ? pColl->zName : sqlite3StrBINARY;
000406 }
000407
000408 if( sqlite3StrICmp(zCollName, pScan->zCollName) ){
000409 continue;
000410 }
000411 }
000412 if( (pTerm->eOperator & (WO_EQ|WO_IS))!=0
000413 && (pX = pTerm->pExpr->pRight, ALWAYS(pX!=0))
000414 && pX->op==TK_COLUMN
000415 && pX->iTable==pScan->aiCur[0]
000416 && pX->iColumn==pScan->aiColumn[0]
000417 ){
000418 testcase( pTerm->eOperator & WO_IS );
000419 continue;
000420 }
000421 pScan->pWC = pWC;
000422 pScan->k = k+1;
000423 #ifdef WHERETRACE_ENABLED
000424 if( sqlite3WhereTrace & 0x20000 ){
000425 int ii;
000426 sqlite3DebugPrintf("SCAN-TERM %p: nEquiv=%d",
000427 pTerm, pScan->nEquiv);
000428 for(ii=0; ii<pScan->nEquiv; ii++){
000429 sqlite3DebugPrintf(" {%d:%d}",
000430 pScan->aiCur[ii], pScan->aiColumn[ii]);
000431 }
000432 sqlite3DebugPrintf("\n");
000433 }
000434 #endif
000435 return pTerm;
000436 }
000437 }
000438 }
000439 pWC = pWC->pOuter;
000440 k = 0;
000441 }while( pWC!=0 );
000442 if( pScan->iEquiv>=pScan->nEquiv ) break;
000443 pWC = pScan->pOrigWC;
000444 k = 0;
000445 pScan->iEquiv++;
000446 }
000447 return 0;
000448 }
000449
000450 /*
000451 ** This is whereScanInit() for the case of an index on an expression.
000452 ** It is factored out into a separate tail-recursion subroutine so that
000453 ** the normal whereScanInit() routine, which is a high-runner, does not
000454 ** need to push registers onto the stack as part of its prologue.
000455 */
000456 static SQLITE_NOINLINE WhereTerm *whereScanInitIndexExpr(WhereScan *pScan){
000457 pScan->idxaff = sqlite3ExprAffinity(pScan->pIdxExpr);
000458 return whereScanNext(pScan);
000459 }
000460
000461 /*
000462 ** Initialize a WHERE clause scanner object. Return a pointer to the
000463 ** first match. Return NULL if there are no matches.
000464 **
000465 ** The scanner will be searching the WHERE clause pWC. It will look
000466 ** for terms of the form "X <op> <expr>" where X is column iColumn of table
000467 ** iCur. Or if pIdx!=0 then X is column iColumn of index pIdx. pIdx
000468 ** must be one of the indexes of table iCur.
000469 **
000470 ** The <op> must be one of the operators described by opMask.
000471 **
000472 ** If the search is for X and the WHERE clause contains terms of the
000473 ** form X=Y then this routine might also return terms of the form
000474 ** "Y <op> <expr>". The number of levels of transitivity is limited,
000475 ** but is enough to handle most commonly occurring SQL statements.
000476 **
000477 ** If X is not the INTEGER PRIMARY KEY then X must be compatible with
000478 ** index pIdx.
000479 */
000480 static WhereTerm *whereScanInit(
000481 WhereScan *pScan, /* The WhereScan object being initialized */
000482 WhereClause *pWC, /* The WHERE clause to be scanned */
000483 int iCur, /* Cursor to scan for */
000484 int iColumn, /* Column to scan for */
000485 u32 opMask, /* Operator(s) to scan for */
000486 Index *pIdx /* Must be compatible with this index */
000487 ){
000488 pScan->pOrigWC = pWC;
000489 pScan->pWC = pWC;
000490 pScan->pIdxExpr = 0;
000491 pScan->idxaff = 0;
000492 pScan->zCollName = 0;
000493 pScan->opMask = opMask;
000494 pScan->k = 0;
000495 pScan->aiCur[0] = iCur;
000496 pScan->nEquiv = 1;
000497 pScan->iEquiv = 1;
000498 if( pIdx ){
000499 int j = iColumn;
000500 iColumn = pIdx->aiColumn[j];
000501 if( iColumn==pIdx->pTable->iPKey ){
000502 iColumn = XN_ROWID;
000503 }else if( iColumn>=0 ){
000504 pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity;
000505 pScan->zCollName = pIdx->azColl[j];
000506 }else if( iColumn==XN_EXPR ){
000507 pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr;
000508 pScan->zCollName = pIdx->azColl[j];
000509 pScan->aiColumn[0] = XN_EXPR;
000510 return whereScanInitIndexExpr(pScan);
000511 }
000512 }else if( iColumn==XN_EXPR ){
000513 return 0;
000514 }
000515 pScan->aiColumn[0] = iColumn;
000516 return whereScanNext(pScan);
000517 }
000518
000519 /*
000520 ** Search for a term in the WHERE clause that is of the form "X <op> <expr>"
000521 ** where X is a reference to the iColumn of table iCur or of index pIdx
000522 ** if pIdx!=0 and <op> is one of the WO_xx operator codes specified by
000523 ** the op parameter. Return a pointer to the term. Return 0 if not found.
000524 **
000525 ** If pIdx!=0 then it must be one of the indexes of table iCur.
000526 ** Search for terms matching the iColumn-th column of pIdx
000527 ** rather than the iColumn-th column of table iCur.
000528 **
000529 ** The term returned might by Y=<expr> if there is another constraint in
000530 ** the WHERE clause that specifies that X=Y. Any such constraints will be
000531 ** identified by the WO_EQUIV bit in the pTerm->eOperator field. The
000532 ** aiCur[]/iaColumn[] arrays hold X and all its equivalents. There are 11
000533 ** slots in aiCur[]/aiColumn[] so that means we can look for X plus up to 10
000534 ** other equivalent values. Hence a search for X will return <expr> if X=A1
000535 ** and A1=A2 and A2=A3 and ... and A9=A10 and A10=<expr>.
000536 **
000537 ** If there are multiple terms in the WHERE clause of the form "X <op> <expr>"
000538 ** then try for the one with no dependencies on <expr> - in other words where
000539 ** <expr> is a constant expression of some kind. Only return entries of
000540 ** the form "X <op> Y" where Y is a column in another table if no terms of
000541 ** the form "X <op> <const-expr>" exist. If no terms with a constant RHS
000542 ** exist, try to return a term that does not use WO_EQUIV.
000543 */
000544 WhereTerm *sqlite3WhereFindTerm(
000545 WhereClause *pWC, /* The WHERE clause to be searched */
000546 int iCur, /* Cursor number of LHS */
000547 int iColumn, /* Column number of LHS */
000548 Bitmask notReady, /* RHS must not overlap with this mask */
000549 u32 op, /* Mask of WO_xx values describing operator */
000550 Index *pIdx /* Must be compatible with this index, if not NULL */
000551 ){
000552 WhereTerm *pResult = 0;
000553 WhereTerm *p;
000554 WhereScan scan;
000555
000556 p = whereScanInit(&scan, pWC, iCur, iColumn, op, pIdx);
000557 op &= WO_EQ|WO_IS;
000558 while( p ){
000559 if( (p->prereqRight & notReady)==0 ){
000560 if( p->prereqRight==0 && (p->eOperator&op)!=0 ){
000561 testcase( p->eOperator & WO_IS );
000562 return p;
000563 }
000564 if( pResult==0 ) pResult = p;
000565 }
000566 p = whereScanNext(&scan);
000567 }
000568 return pResult;
000569 }
000570
000571 /*
000572 ** This function searches pList for an entry that matches the iCol-th column
000573 ** of index pIdx.
000574 **
000575 ** If such an expression is found, its index in pList->a[] is returned. If
000576 ** no expression is found, -1 is returned.
000577 */
000578 static int findIndexCol(
000579 Parse *pParse, /* Parse context */
000580 ExprList *pList, /* Expression list to search */
000581 int iBase, /* Cursor for table associated with pIdx */
000582 Index *pIdx, /* Index to match column of */
000583 int iCol /* Column of index to match */
000584 ){
000585 int i;
000586 const char *zColl = pIdx->azColl[iCol];
000587
000588 for(i=0; i<pList->nExpr; i++){
000589 Expr *p = sqlite3ExprSkipCollateAndLikely(pList->a[i].pExpr);
000590 if( ALWAYS(p!=0)
000591 && (p->op==TK_COLUMN || p->op==TK_AGG_COLUMN)
000592 && p->iColumn==pIdx->aiColumn[iCol]
000593 && p->iTable==iBase
000594 ){
000595 CollSeq *pColl = sqlite3ExprNNCollSeq(pParse, pList->a[i].pExpr);
000596 if( 0==sqlite3StrICmp(pColl->zName, zColl) ){
000597 return i;
000598 }
000599 }
000600 }
000601
000602 return -1;
000603 }
000604
000605 /*
000606 ** Return TRUE if the iCol-th column of index pIdx is NOT NULL
000607 */
000608 static int indexColumnNotNull(Index *pIdx, int iCol){
000609 int j;
000610 assert( pIdx!=0 );
000611 assert( iCol>=0 && iCol<pIdx->nColumn );
000612 j = pIdx->aiColumn[iCol];
000613 if( j>=0 ){
000614 return pIdx->pTable->aCol[j].notNull;
000615 }else if( j==(-1) ){
000616 return 1;
000617 }else{
000618 assert( j==(-2) );
000619 return 0; /* Assume an indexed expression can always yield a NULL */
000620
000621 }
000622 }
000623
000624 /*
000625 ** Return true if the DISTINCT expression-list passed as the third argument
000626 ** is redundant.
000627 **
000628 ** A DISTINCT list is redundant if any subset of the columns in the
000629 ** DISTINCT list are collectively unique and individually non-null.
000630 */
000631 static int isDistinctRedundant(
000632 Parse *pParse, /* Parsing context */
000633 SrcList *pTabList, /* The FROM clause */
000634 WhereClause *pWC, /* The WHERE clause */
000635 ExprList *pDistinct /* The result set that needs to be DISTINCT */
000636 ){
000637 Table *pTab;
000638 Index *pIdx;
000639 int i;
000640 int iBase;
000641
000642 /* If there is more than one table or sub-select in the FROM clause of
000643 ** this query, then it will not be possible to show that the DISTINCT
000644 ** clause is redundant. */
000645 if( pTabList->nSrc!=1 ) return 0;
000646 iBase = pTabList->a[0].iCursor;
000647 pTab = pTabList->a[0].pSTab;
000648
000649 /* If any of the expressions is an IPK column on table iBase, then return
000650 ** true. Note: The (p->iTable==iBase) part of this test may be false if the
000651 ** current SELECT is a correlated sub-query.
000652 */
000653 for(i=0; i<pDistinct->nExpr; i++){
000654 Expr *p = sqlite3ExprSkipCollateAndLikely(pDistinct->a[i].pExpr);
000655 if( NEVER(p==0) ) continue;
000656 if( p->op!=TK_COLUMN && p->op!=TK_AGG_COLUMN ) continue;
000657 if( p->iTable==iBase && p->iColumn<0 ) return 1;
000658 }
000659
000660 /* Loop through all indices on the table, checking each to see if it makes
000661 ** the DISTINCT qualifier redundant. It does so if:
000662 **
000663 ** 1. The index is itself UNIQUE, and
000664 **
000665 ** 2. All of the columns in the index are either part of the pDistinct
000666 ** list, or else the WHERE clause contains a term of the form "col=X",
000667 ** where X is a constant value. The collation sequences of the
000668 ** comparison and select-list expressions must match those of the index.
000669 **
000670 ** 3. All of those index columns for which the WHERE clause does not
000671 ** contain a "col=X" term are subject to a NOT NULL constraint.
000672 */
000673 for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
000674 if( !IsUniqueIndex(pIdx) ) continue;
000675 if( pIdx->pPartIdxWhere ) continue;
000676 for(i=0; i<pIdx->nKeyCol; i++){
000677 if( 0==sqlite3WhereFindTerm(pWC, iBase, i, ~(Bitmask)0, WO_EQ, pIdx) ){
000678 if( findIndexCol(pParse, pDistinct, iBase, pIdx, i)<0 ) break;
000679 if( indexColumnNotNull(pIdx, i)==0 ) break;
000680 }
000681 }
000682 if( i==pIdx->nKeyCol ){
000683 /* This index implies that the DISTINCT qualifier is redundant. */
000684 return 1;
000685 }
000686 }
000687
000688 return 0;
000689 }
000690
000691
000692 /*
000693 ** Estimate the logarithm of the input value to base 2.
000694 */
000695 static LogEst estLog(LogEst N){
000696 return N<=10 ? 0 : sqlite3LogEst(N) - 33;
000697 }
000698
000699 /*
000700 ** Convert OP_Column opcodes to OP_Copy in previously generated code.
000701 **
000702 ** This routine runs over generated VDBE code and translates OP_Column
000703 ** opcodes into OP_Copy when the table is being accessed via co-routine
000704 ** instead of via table lookup.
000705 **
000706 ** If the iAutoidxCur is not zero, then any OP_Rowid instructions on
000707 ** cursor iTabCur are transformed into OP_Sequence opcode for the
000708 ** iAutoidxCur cursor, in order to generate unique rowids for the
000709 ** automatic index being generated.
000710 */
000711 static void translateColumnToCopy(
000712 Parse *pParse, /* Parsing context */
000713 int iStart, /* Translate from this opcode to the end */
000714 int iTabCur, /* OP_Column/OP_Rowid references to this table */
000715 int iRegister, /* The first column is in this register */
000716 int iAutoidxCur /* If non-zero, cursor of autoindex being generated */
000717 ){
000718 Vdbe *v = pParse->pVdbe;
000719 VdbeOp *pOp = sqlite3VdbeGetOp(v, iStart);
000720 int iEnd = sqlite3VdbeCurrentAddr(v);
000721 if( pParse->db->mallocFailed ) return;
000722 #ifdef SQLITE_DEBUG
000723 if( pParse->db->flags & SQLITE_VdbeAddopTrace ){
000724 printf("CHECKING for column-to-copy on cursor %d for %d..%d\n",
000725 iTabCur, iStart, iEnd);
000726 }
000727 #endif
000728 for(; iStart<iEnd; iStart++, pOp++){
000729 if( pOp->p1!=iTabCur ) continue;
000730 if( pOp->opcode==OP_Column ){
000731 #ifdef SQLITE_DEBUG
000732 if( pParse->db->flags & SQLITE_VdbeAddopTrace ){
000733 printf("TRANSLATE OP_Column to OP_Copy at %d\n", iStart);
000734 }
000735 #endif
000736 pOp->opcode = OP_Copy;
000737 pOp->p1 = pOp->p2 + iRegister;
000738 pOp->p2 = pOp->p3;
000739 pOp->p3 = 0;
000740 pOp->p5 = 2; /* Cause the MEM_Subtype flag to be cleared */
000741 }else if( pOp->opcode==OP_Rowid ){
000742 #ifdef SQLITE_DEBUG
000743 if( pParse->db->flags & SQLITE_VdbeAddopTrace ){
000744 printf("TRANSLATE OP_Rowid to OP_Sequence at %d\n", iStart);
000745 }
000746 #endif
000747 pOp->opcode = OP_Sequence;
000748 pOp->p1 = iAutoidxCur;
000749 #ifdef SQLITE_ALLOW_ROWID_IN_VIEW
000750 if( iAutoidxCur==0 ){
000751 pOp->opcode = OP_Null;
000752 pOp->p3 = 0;
000753 }
000754 #endif
000755 }
000756 }
000757 }
000758
000759 /*
000760 ** Two routines for printing the content of an sqlite3_index_info
000761 ** structure. Used for testing and debugging only. If neither
000762 ** SQLITE_TEST or SQLITE_DEBUG are defined, then these routines
000763 ** are no-ops.
000764 */
000765 #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED)
000766 static void whereTraceIndexInfoInputs(
000767 sqlite3_index_info *p, /* The IndexInfo object */
000768 Table *pTab /* The TABLE that is the virtual table */
000769 ){
000770 int i;
000771 if( (sqlite3WhereTrace & 0x10)==0 ) return;
000772 sqlite3DebugPrintf("sqlite3_index_info inputs for %s:\n", pTab->zName);
000773 for(i=0; i<p->nConstraint; i++){
000774 sqlite3DebugPrintf(
000775 " constraint[%d]: col=%d termid=%d op=%d usabled=%d collseq=%s\n",
000776 i,
000777 p->aConstraint[i].iColumn,
000778 p->aConstraint[i].iTermOffset,
000779 p->aConstraint[i].op,
000780 p->aConstraint[i].usable,
000781 sqlite3_vtab_collation(p,i));
000782 }
000783 for(i=0; i<p->nOrderBy; i++){
000784 sqlite3DebugPrintf(" orderby[%d]: col=%d desc=%d\n",
000785 i,
000786 p->aOrderBy[i].iColumn,
000787 p->aOrderBy[i].desc);
000788 }
000789 }
000790 static void whereTraceIndexInfoOutputs(
000791 sqlite3_index_info *p, /* The IndexInfo object */
000792 Table *pTab /* The TABLE that is the virtual table */
000793 ){
000794 int i;
000795 if( (sqlite3WhereTrace & 0x10)==0 ) return;
000796 sqlite3DebugPrintf("sqlite3_index_info outputs for %s:\n", pTab->zName);
000797 for(i=0; i<p->nConstraint; i++){
000798 sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n",
000799 i,
000800 p->aConstraintUsage[i].argvIndex,
000801 p->aConstraintUsage[i].omit);
000802 }
000803 sqlite3DebugPrintf(" idxNum=%d\n", p->idxNum);
000804 sqlite3DebugPrintf(" idxStr=%s\n", p->idxStr);
000805 sqlite3DebugPrintf(" orderByConsumed=%d\n", p->orderByConsumed);
000806 sqlite3DebugPrintf(" estimatedCost=%g\n", p->estimatedCost);
000807 sqlite3DebugPrintf(" estimatedRows=%lld\n", p->estimatedRows);
000808 }
000809 #else
000810 #define whereTraceIndexInfoInputs(A,B)
000811 #define whereTraceIndexInfoOutputs(A,B)
000812 #endif
000813
000814 /*
000815 ** We know that pSrc is an operand of an outer join. Return true if
000816 ** pTerm is a constraint that is compatible with that join.
000817 **
000818 ** pTerm must be EP_OuterON if pSrc is the right operand of an
000819 ** outer join. pTerm can be either EP_OuterON or EP_InnerON if pSrc
000820 ** is the left operand of a RIGHT join.
000821 **
000822 ** See https://sqlite.org/forum/forumpost/206d99a16dd9212f
000823 ** for an example of a WHERE clause constraints that may not be used on
000824 ** the right table of a RIGHT JOIN because the constraint implies a
000825 ** not-NULL condition on the left table of the RIGHT JOIN.
000826 */
000827 static int constraintCompatibleWithOuterJoin(
000828 const WhereTerm *pTerm, /* WHERE clause term to check */
000829 const SrcItem *pSrc /* Table we are trying to access */
000830 ){
000831 assert( (pSrc->fg.jointype&(JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 ); /* By caller */
000832 testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LEFT );
000833 testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LTORJ );
000834 testcase( ExprHasProperty(pTerm->pExpr, EP_OuterON) )
000835 testcase( ExprHasProperty(pTerm->pExpr, EP_InnerON) );
000836 if( !ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON)
000837 || pTerm->pExpr->w.iJoin != pSrc->iCursor
000838 ){
000839 return 0;
000840 }
000841 if( (pSrc->fg.jointype & (JT_LEFT|JT_RIGHT))!=0
000842 && NEVER(ExprHasProperty(pTerm->pExpr, EP_InnerON))
000843 ){
000844 return 0;
000845 }
000846 return 1;
000847 }
000848
000849 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
000850 /*
000851 ** Return true if column iCol of table pTab seem like it might be a
000852 ** good column to use as part of a query-time index.
000853 **
000854 ** Current algorithm (subject to improvement!):
000855 **
000856 ** 1. If iCol is already the left-most column of some other index,
000857 ** then return false.
000858 **
000859 ** 2. If iCol is part of an existing index that has an aiRowLogEst of
000860 ** more than 20, then return false.
000861 **
000862 ** 3. If no disqualifying conditions above are found, return true.
000863 **
000864 ** 2025-01-03: I experimented with a new rule that returns false if the
000865 ** the datatype of the column is "BOOLEAN". This did not improve
000866 ** performance on any queries at hand, but it did burn CPU cycles, so the
000867 ** idea was not committed.
000868 */
000869 static SQLITE_NOINLINE int columnIsGoodIndexCandidate(
000870 const Table *pTab,
000871 int iCol
000872 ){
000873 const Index *pIdx;
000874 for(pIdx = pTab->pIndex; pIdx!=0; pIdx=pIdx->pNext){
000875 int j;
000876 for(j=0; j<pIdx->nKeyCol; j++){
000877 if( pIdx->aiColumn[j]==iCol ){
000878 if( j==0 ) return 0;
000879 if( pIdx->hasStat1 && pIdx->aiRowLogEst[j+1]>20 ) return 0;
000880 break;
000881 }
000882 }
000883 }
000884 return 1;
000885 }
000886 #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
000887
000888
000889
000890 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
000891 /*
000892 ** Return TRUE if the WHERE clause term pTerm is of a form where it
000893 ** could be used with an index to access pSrc, assuming an appropriate
000894 ** index existed.
000895 */
000896 static int termCanDriveIndex(
000897 const WhereTerm *pTerm, /* WHERE clause term to check */
000898 const SrcItem *pSrc, /* Table we are trying to access */
000899 const Bitmask notReady /* Tables in outer loops of the join */
000900 ){
000901 char aff;
000902 int leftCol;
000903
000904 if( pTerm->leftCursor!=pSrc->iCursor ) return 0;
000905 if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) return 0;
000906 assert( (pSrc->fg.jointype & JT_RIGHT)==0 );
000907 if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0
000908 && !constraintCompatibleWithOuterJoin(pTerm,pSrc)
000909 ){
000910 return 0; /* See https://sqlite.org/forum/forumpost/51e6959f61 */
000911 }
000912 if( (pTerm->prereqRight & notReady)!=0 ) return 0;
000913 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
000914 leftCol = pTerm->u.x.leftColumn;
000915 if( leftCol<0 ) return 0;
000916 aff = pSrc->pSTab->aCol[leftCol].affinity;
000917 if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0;
000918 testcase( pTerm->pExpr->op==TK_IS );
000919 return columnIsGoodIndexCandidate(pSrc->pSTab, leftCol);
000920 }
000921 #endif
000922
000923
000924 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
000925
000926 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
000927 /*
000928 ** Argument pIdx represents an automatic index that the current statement
000929 ** will create and populate. Add an OP_Explain with text of the form:
000930 **
000931 ** CREATE AUTOMATIC INDEX ON <table>(<cols>) [WHERE <expr>]
000932 **
000933 ** This is only required if sqlite3_stmt_scanstatus() is enabled, to
000934 ** associate an SQLITE_SCANSTAT_NCYCLE and SQLITE_SCANSTAT_NLOOP
000935 ** values with. In order to avoid breaking legacy code and test cases,
000936 ** the OP_Explain is not added if this is an EXPLAIN QUERY PLAN command.
000937 */
000938 static void explainAutomaticIndex(
000939 Parse *pParse,
000940 Index *pIdx, /* Automatic index to explain */
000941 int bPartial, /* True if pIdx is a partial index */
000942 int *pAddrExplain /* OUT: Address of OP_Explain */
000943 ){
000944 if( IS_STMT_SCANSTATUS(pParse->db) && pParse->explain!=2 ){
000945 Table *pTab = pIdx->pTable;
000946 const char *zSep = "";
000947 char *zText = 0;
000948 int ii = 0;
000949 sqlite3_str *pStr = sqlite3_str_new(pParse->db);
000950 sqlite3_str_appendf(pStr,"CREATE AUTOMATIC INDEX ON %s(", pTab->zName);
000951 assert( pIdx->nColumn>1 );
000952 assert( pIdx->aiColumn[pIdx->nColumn-1]==XN_ROWID || !HasRowid(pTab) );
000953 for(ii=0; ii<(pIdx->nColumn-1); ii++){
000954 const char *zName = 0;
000955 int iCol = pIdx->aiColumn[ii];
000956
000957 zName = pTab->aCol[iCol].zCnName;
000958 sqlite3_str_appendf(pStr, "%s%s", zSep, zName);
000959 zSep = ", ";
000960 }
000961 zText = sqlite3_str_finish(pStr);
000962 if( zText==0 ){
000963 sqlite3OomFault(pParse->db);
000964 }else{
000965 *pAddrExplain = sqlite3VdbeExplain(
000966 pParse, 0, "%s)%s", zText, (bPartial ? " WHERE <expr>" : "")
000967 );
000968 sqlite3_free(zText);
000969 }
000970 }
000971 }
000972 #else
000973 # define explainAutomaticIndex(a,b,c,d)
000974 #endif
000975
000976 /*
000977 ** Generate code to construct the Index object for an automatic index
000978 ** and to set up the WhereLevel object pLevel so that the code generator
000979 ** makes use of the automatic index.
000980 */
000981 static SQLITE_NOINLINE void constructAutomaticIndex(
000982 Parse *pParse, /* The parsing context */
000983 WhereClause *pWC, /* The WHERE clause */
000984 const Bitmask notReady, /* Mask of cursors that are not available */
000985 WhereLevel *pLevel /* Write new index here */
000986 ){
000987 int nKeyCol; /* Number of columns in the constructed index */
000988 WhereTerm *pTerm; /* A single term of the WHERE clause */
000989 WhereTerm *pWCEnd; /* End of pWC->a[] */
000990 Index *pIdx; /* Object describing the transient index */
000991 Vdbe *v; /* Prepared statement under construction */
000992 int addrInit; /* Address of the initialization bypass jump */
000993 Table *pTable; /* The table being indexed */
000994 int addrTop; /* Top of the index fill loop */
000995 int regRecord; /* Register holding an index record */
000996 int n; /* Column counter */
000997 int i; /* Loop counter */
000998 int mxBitCol; /* Maximum column in pSrc->colUsed */
000999 CollSeq *pColl; /* Collating sequence to on a column */
001000 WhereLoop *pLoop; /* The Loop object */
001001 char *zNotUsed; /* Extra space on the end of pIdx */
001002 Bitmask idxCols; /* Bitmap of columns used for indexing */
001003 Bitmask extraCols; /* Bitmap of additional columns */
001004 u8 sentWarning = 0; /* True if a warning has been issued */
001005 u8 useBloomFilter = 0; /* True to also add a Bloom filter */
001006 Expr *pPartial = 0; /* Partial Index Expression */
001007 int iContinue = 0; /* Jump here to skip excluded rows */
001008 SrcList *pTabList; /* The complete FROM clause */
001009 SrcItem *pSrc; /* The FROM clause term to get the next index */
001010 int addrCounter = 0; /* Address where integer counter is initialized */
001011 int regBase; /* Array of registers where record is assembled */
001012 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
001013 int addrExp = 0; /* Address of OP_Explain */
001014 #endif
001015
001016 /* Generate code to skip over the creation and initialization of the
001017 ** transient index on 2nd and subsequent iterations of the loop. */
001018 v = pParse->pVdbe;
001019 assert( v!=0 );
001020 addrInit = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
001021
001022 /* Count the number of columns that will be added to the index
001023 ** and used to match WHERE clause constraints */
001024 nKeyCol = 0;
001025 pTabList = pWC->pWInfo->pTabList;
001026 pSrc = &pTabList->a[pLevel->iFrom];
001027 pTable = pSrc->pSTab;
001028 pWCEnd = &pWC->a[pWC->nTerm];
001029 pLoop = pLevel->pWLoop;
001030 idxCols = 0;
001031 for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){
001032 Expr *pExpr = pTerm->pExpr;
001033 /* Make the automatic index a partial index if there are terms in the
001034 ** WHERE clause (or the ON clause of a LEFT join) that constrain which
001035 ** rows of the target table (pSrc) that can be used. */
001036 if( (pTerm->wtFlags & TERM_VIRTUAL)==0
001037 && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, pLevel->iFrom, 0)
001038 ){
001039 pPartial = sqlite3ExprAnd(pParse, pPartial,
001040 sqlite3ExprDup(pParse->db, pExpr, 0));
001041 }
001042 if( termCanDriveIndex(pTerm, pSrc, notReady) ){
001043 int iCol;
001044 Bitmask cMask;
001045 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
001046 iCol = pTerm->u.x.leftColumn;
001047 cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol);
001048 testcase( iCol==BMS );
001049 testcase( iCol==BMS-1 );
001050 if( !sentWarning ){
001051 sqlite3_log(SQLITE_WARNING_AUTOINDEX,
001052 "automatic index on %s(%s)", pTable->zName,
001053 pTable->aCol[iCol].zCnName);
001054 sentWarning = 1;
001055 }
001056 if( (idxCols & cMask)==0 ){
001057 if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ){
001058 goto end_auto_index_create;
001059 }
001060 pLoop->aLTerm[nKeyCol++] = pTerm;
001061 idxCols |= cMask;
001062 }
001063 }
001064 }
001065 assert( nKeyCol>0 || pParse->db->mallocFailed );
001066 pLoop->u.btree.nEq = pLoop->nLTerm = nKeyCol;
001067 pLoop->wsFlags = WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WHERE_INDEXED
001068 | WHERE_AUTO_INDEX;
001069
001070 /* Count the number of additional columns needed to create a
001071 ** covering index. A "covering index" is an index that contains all
001072 ** columns that are needed by the query. With a covering index, the
001073 ** original table never needs to be accessed. Automatic indices must
001074 ** be a covering index because the index will not be updated if the
001075 ** original table changes and the index and table cannot both be used
001076 ** if they go out of sync.
001077 */
001078 if( IsView(pTable) ){
001079 extraCols = ALLBITS & ~idxCols;
001080 }else{
001081 extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1));
001082 }
001083 if( !HasRowid(pTable) ){
001084 /* For WITHOUT ROWID tables, ensure that all PRIMARY KEY columns are
001085 ** either in the idxCols mask or in the extraCols mask */
001086 for(i=0; i<pTable->nCol; i++){
001087 if( (pTable->aCol[i].colFlags & COLFLAG_PRIMKEY)==0 ) continue;
001088 if( i>=BMS-1 ){
001089 extraCols |= MASKBIT(BMS-1);
001090 break;
001091 }
001092 if( idxCols & MASKBIT(i) ) continue;
001093 extraCols |= MASKBIT(i);
001094 }
001095 }
001096 mxBitCol = MIN(BMS-1,pTable->nCol);
001097 testcase( pTable->nCol==BMS-1 );
001098 testcase( pTable->nCol==BMS-2 );
001099 for(i=0; i<mxBitCol; i++){
001100 if( extraCols & MASKBIT(i) ) nKeyCol++;
001101 }
001102 if( pSrc->colUsed & MASKBIT(BMS-1) ){
001103 nKeyCol += pTable->nCol - BMS + 1;
001104 }
001105
001106 /* Construct the Index object to describe this index */
001107 pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+HasRowid(pTable),
001108 0, &zNotUsed);
001109 if( pIdx==0 ) goto end_auto_index_create;
001110 pLoop->u.btree.pIndex = pIdx;
001111 pIdx->zName = "auto-index";
001112 pIdx->pTable = pTable;
001113 n = 0;
001114 idxCols = 0;
001115 for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){
001116 if( termCanDriveIndex(pTerm, pSrc, notReady) ){
001117 int iCol;
001118 Bitmask cMask;
001119 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
001120 iCol = pTerm->u.x.leftColumn;
001121 cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol);
001122 testcase( iCol==BMS-1 );
001123 testcase( iCol==BMS );
001124 if( (idxCols & cMask)==0 ){
001125 Expr *pX = pTerm->pExpr;
001126 idxCols |= cMask;
001127 pIdx->aiColumn[n] = pTerm->u.x.leftColumn;
001128 pColl = sqlite3ExprCompareCollSeq(pParse, pX);
001129 assert( pColl!=0 || pParse->nErr>0 ); /* TH3 collate01.800 */
001130 pIdx->azColl[n] = pColl ? pColl->zName : sqlite3StrBINARY;
001131 n++;
001132 if( ALWAYS(pX->pLeft!=0)
001133 && sqlite3ExprAffinity(pX->pLeft)!=SQLITE_AFF_TEXT
001134 ){
001135 /* TUNING: only use a Bloom filter on an automatic index
001136 ** if one or more key columns has the ability to hold numeric
001137 ** values, since strings all have the same hash in the Bloom
001138 ** filter implementation and hence a Bloom filter on a text column
001139 ** is not usually helpful. */
001140 useBloomFilter = 1;
001141 }
001142 }
001143 }
001144 }
001145 assert( (u32)n==pLoop->u.btree.nEq );
001146
001147 /* Add additional columns needed to make the automatic index into
001148 ** a covering index */
001149 for(i=0; i<mxBitCol; i++){
001150 if( extraCols & MASKBIT(i) ){
001151 pIdx->aiColumn[n] = i;
001152 pIdx->azColl[n] = sqlite3StrBINARY;
001153 n++;
001154 }
001155 }
001156 if( pSrc->colUsed & MASKBIT(BMS-1) ){
001157 for(i=BMS-1; i<pTable->nCol; i++){
001158 pIdx->aiColumn[n] = i;
001159 pIdx->azColl[n] = sqlite3StrBINARY;
001160 n++;
001161 }
001162 }
001163 assert( n==nKeyCol );
001164 if( HasRowid(pTable) ){
001165 pIdx->aiColumn[n] = XN_ROWID;
001166 pIdx->azColl[n] = sqlite3StrBINARY;
001167 }
001168
001169 /* Create the automatic index */
001170 explainAutomaticIndex(pParse, pIdx, pPartial!=0, &addrExp);
001171 assert( pLevel->iIdxCur>=0 );
001172 pLevel->iIdxCur = pParse->nTab++;
001173 sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1);
001174 sqlite3VdbeSetP4KeyInfo(pParse, pIdx);
001175 VdbeComment((v, "for %s", pTable->zName));
001176 if( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) && useBloomFilter ){
001177 sqlite3WhereExplainBloomFilter(pParse, pWC->pWInfo, pLevel);
001178 pLevel->regFilter = ++pParse->nMem;
001179 sqlite3VdbeAddOp2(v, OP_Blob, 10000, pLevel->regFilter);
001180 }
001181
001182 /* Fill the automatic index with content */
001183 assert( pSrc == &pWC->pWInfo->pTabList->a[pLevel->iFrom] );
001184 if( pSrc->fg.viaCoroutine ){
001185 int regYield;
001186 Subquery *pSubq;
001187 assert( pSrc->fg.isSubquery );
001188 pSubq = pSrc->u4.pSubq;
001189 assert( pSubq!=0 );
001190 regYield = pSubq->regReturn;
001191 addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0);
001192 sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pSubq->addrFillSub);
001193 addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield);
001194 VdbeCoverage(v);
001195 VdbeComment((v, "next row of %s", pSrc->pSTab->zName));
001196 }else{
001197 addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v);
001198 }
001199 if( pPartial ){
001200 iContinue = sqlite3VdbeMakeLabel(pParse);
001201 sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL);
001202 pLoop->wsFlags |= WHERE_PARTIALIDX;
001203 }
001204 regRecord = sqlite3GetTempReg(pParse);
001205 regBase = sqlite3GenerateIndexKey(
001206 pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0
001207 );
001208 if( pLevel->regFilter ){
001209 sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0,
001210 regBase, pLoop->u.btree.nEq);
001211 }
001212 sqlite3VdbeScanStatusCounters(v, addrExp, addrExp, sqlite3VdbeCurrentAddr(v));
001213 sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord);
001214 sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
001215 if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue);
001216 if( pSrc->fg.viaCoroutine ){
001217 assert( pSrc->fg.isSubquery && pSrc->u4.pSubq!=0 );
001218 sqlite3VdbeChangeP2(v, addrCounter, regBase+n);
001219 testcase( pParse->db->mallocFailed );
001220 assert( pLevel->iIdxCur>0 );
001221 translateColumnToCopy(pParse, addrTop, pLevel->iTabCur,
001222 pSrc->u4.pSubq->regResult, pLevel->iIdxCur);
001223 sqlite3VdbeGoto(v, addrTop);
001224 pSrc->fg.viaCoroutine = 0;
001225 }else{
001226 sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v);
001227 sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX);
001228 }
001229 sqlite3VdbeJumpHere(v, addrTop);
001230 sqlite3ReleaseTempReg(pParse, regRecord);
001231
001232 /* Jump here when skipping the initialization */
001233 sqlite3VdbeJumpHere(v, addrInit);
001234 sqlite3VdbeScanStatusRange(v, addrExp, addrExp, -1);
001235
001236 end_auto_index_create:
001237 sqlite3ExprDelete(pParse->db, pPartial);
001238 }
001239 #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
001240
001241 /*
001242 ** Generate bytecode that will initialize a Bloom filter that is appropriate
001243 ** for pLevel.
001244 **
001245 ** If there are inner loops within pLevel that have the WHERE_BLOOMFILTER
001246 ** flag set, initialize a Bloomfilter for them as well. Except don't do
001247 ** this recursive initialization if the SQLITE_BloomPulldown optimization has
001248 ** been turned off.
001249 **
001250 ** When the Bloom filter is initialized, the WHERE_BLOOMFILTER flag is cleared
001251 ** from the loop, but the regFilter value is set to a register that implements
001252 ** the Bloom filter. When regFilter is positive, the
001253 ** sqlite3WhereCodeOneLoopStart() will generate code to test the Bloom filter
001254 ** and skip the subsequence B-Tree seek if the Bloom filter indicates that
001255 ** no matching rows exist.
001256 **
001257 ** This routine may only be called if it has previously been determined that
001258 ** the loop would benefit from a Bloom filter, and the WHERE_BLOOMFILTER bit
001259 ** is set.
001260 */
001261 static SQLITE_NOINLINE void sqlite3ConstructBloomFilter(
001262 WhereInfo *pWInfo, /* The WHERE clause */
001263 int iLevel, /* Index in pWInfo->a[] that is pLevel */
001264 WhereLevel *pLevel, /* Make a Bloom filter for this FROM term */
001265 Bitmask notReady /* Loops that are not ready */
001266 ){
001267 int addrOnce; /* Address of opening OP_Once */
001268 int addrTop; /* Address of OP_Rewind */
001269 int addrCont; /* Jump here to skip a row */
001270 const WhereTerm *pTerm; /* For looping over WHERE clause terms */
001271 const WhereTerm *pWCEnd; /* Last WHERE clause term */
001272 Parse *pParse = pWInfo->pParse; /* Parsing context */
001273 Vdbe *v = pParse->pVdbe; /* VDBE under construction */
001274 WhereLoop *pLoop = pLevel->pWLoop; /* The loop being coded */
001275 int iCur; /* Cursor for table getting the filter */
001276 IndexedExpr *saved_pIdxEpr; /* saved copy of Parse.pIdxEpr */
001277 IndexedExpr *saved_pIdxPartExpr; /* saved copy of Parse.pIdxPartExpr */
001278
001279 saved_pIdxEpr = pParse->pIdxEpr;
001280 saved_pIdxPartExpr = pParse->pIdxPartExpr;
001281 pParse->pIdxEpr = 0;
001282 pParse->pIdxPartExpr = 0;
001283
001284 assert( pLoop!=0 );
001285 assert( v!=0 );
001286 assert( pLoop->wsFlags & WHERE_BLOOMFILTER );
001287 assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 );
001288
001289 addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
001290 do{
001291 const SrcList *pTabList;
001292 const SrcItem *pItem;
001293 const Table *pTab;
001294 u64 sz;
001295 int iSrc;
001296 sqlite3WhereExplainBloomFilter(pParse, pWInfo, pLevel);
001297 addrCont = sqlite3VdbeMakeLabel(pParse);
001298 iCur = pLevel->iTabCur;
001299 pLevel->regFilter = ++pParse->nMem;
001300
001301 /* The Bloom filter is a Blob held in a register. Initialize it
001302 ** to zero-filled blob of at least 80K bits, but maybe more if the
001303 ** estimated size of the table is larger. We could actually
001304 ** measure the size of the table at run-time using OP_Count with
001305 ** P3==1 and use that value to initialize the blob. But that makes
001306 ** testing complicated. By basing the blob size on the value in the
001307 ** sqlite_stat1 table, testing is much easier.
001308 */
001309 pTabList = pWInfo->pTabList;
001310 iSrc = pLevel->iFrom;
001311 pItem = &pTabList->a[iSrc];
001312 assert( pItem!=0 );
001313 pTab = pItem->pSTab;
001314 assert( pTab!=0 );
001315 sz = sqlite3LogEstToInt(pTab->nRowLogEst);
001316 if( sz<10000 ){
001317 sz = 10000;
001318 }else if( sz>10000000 ){
001319 sz = 10000000;
001320 }
001321 sqlite3VdbeAddOp2(v, OP_Blob, (int)sz, pLevel->regFilter);
001322
001323 addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v);
001324 pWCEnd = &pWInfo->sWC.a[pWInfo->sWC.nTerm];
001325 for(pTerm=pWInfo->sWC.a; pTerm<pWCEnd; pTerm++){
001326 Expr *pExpr = pTerm->pExpr;
001327 if( (pTerm->wtFlags & TERM_VIRTUAL)==0
001328 && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, iSrc, 0)
001329 ){
001330 sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL);
001331 }
001332 }
001333 if( pLoop->wsFlags & WHERE_IPK ){
001334 int r1 = sqlite3GetTempReg(pParse);
001335 sqlite3VdbeAddOp2(v, OP_Rowid, iCur, r1);
001336 sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, 1);
001337 sqlite3ReleaseTempReg(pParse, r1);
001338 }else{
001339 Index *pIdx = pLoop->u.btree.pIndex;
001340 int n = pLoop->u.btree.nEq;
001341 int r1 = sqlite3GetTempRange(pParse, n);
001342 int jj;
001343 for(jj=0; jj<n; jj++){
001344 assert( pIdx->pTable==pItem->pSTab );
001345 sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iCur, jj, r1+jj);
001346 }
001347 sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, n);
001348 sqlite3ReleaseTempRange(pParse, r1, n);
001349 }
001350 sqlite3VdbeResolveLabel(v, addrCont);
001351 sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1);
001352 VdbeCoverage(v);
001353 sqlite3VdbeJumpHere(v, addrTop);
001354 pLoop->wsFlags &= ~WHERE_BLOOMFILTER;
001355 if( OptimizationDisabled(pParse->db, SQLITE_BloomPulldown) ) break;
001356 while( ++iLevel < pWInfo->nLevel ){
001357 const SrcItem *pTabItem;
001358 pLevel = &pWInfo->a[iLevel];
001359 pTabItem = &pWInfo->pTabList->a[pLevel->iFrom];
001360 if( pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ) ) continue;
001361 pLoop = pLevel->pWLoop;
001362 if( NEVER(pLoop==0) ) continue;
001363 if( pLoop->prereq & notReady ) continue;
001364 if( (pLoop->wsFlags & (WHERE_BLOOMFILTER|WHERE_COLUMN_IN))
001365 ==WHERE_BLOOMFILTER
001366 ){
001367 /* This is a candidate for bloom-filter pull-down (early evaluation).
001368 ** The test that WHERE_COLUMN_IN is omitted is important, as we are
001369 ** not able to do early evaluation of bloom filters that make use of
001370 ** the IN operator */
001371 break;
001372 }
001373 }
001374 }while( iLevel < pWInfo->nLevel );
001375 sqlite3VdbeJumpHere(v, addrOnce);
001376 pParse->pIdxEpr = saved_pIdxEpr;
001377 pParse->pIdxPartExpr = saved_pIdxPartExpr;
001378 }
001379
001380
001381 #ifndef SQLITE_OMIT_VIRTUALTABLE
001382 /*
001383 ** Return term iTerm of the WhereClause passed as the first argument. Terms
001384 ** are numbered from 0 upwards, starting with the terms in pWC->a[], then
001385 ** those in pWC->pOuter->a[] (if any), and so on.
001386 */
001387 static WhereTerm *termFromWhereClause(WhereClause *pWC, int iTerm){
001388 WhereClause *p;
001389 for(p=pWC; p; p=p->pOuter){
001390 if( iTerm<p->nTerm ) return &p->a[iTerm];
001391 iTerm -= p->nTerm;
001392 }
001393 return 0;
001394 }
001395
001396 /*
001397 ** Allocate and populate an sqlite3_index_info structure. It is the
001398 ** responsibility of the caller to eventually release the structure
001399 ** by passing the pointer returned by this function to freeIndexInfo().
001400 */
001401 static sqlite3_index_info *allocateIndexInfo(
001402 WhereInfo *pWInfo, /* The WHERE clause */
001403 WhereClause *pWC, /* The WHERE clause being analyzed */
001404 Bitmask mUnusable, /* Ignore terms with these prereqs */
001405 SrcItem *pSrc, /* The FROM clause term that is the vtab */
001406 u16 *pmNoOmit /* Mask of terms not to omit */
001407 ){
001408 int i, j;
001409 int nTerm;
001410 Parse *pParse = pWInfo->pParse;
001411 struct sqlite3_index_constraint *pIdxCons;
001412 struct sqlite3_index_orderby *pIdxOrderBy;
001413 struct sqlite3_index_constraint_usage *pUsage;
001414 struct HiddenIndexInfo *pHidden;
001415 WhereTerm *pTerm;
001416 int nOrderBy;
001417 sqlite3_index_info *pIdxInfo;
001418 u16 mNoOmit = 0;
001419 const Table *pTab;
001420 int eDistinct = 0;
001421 ExprList *pOrderBy = pWInfo->pOrderBy;
001422 WhereClause *p;
001423
001424 assert( pSrc!=0 );
001425 pTab = pSrc->pSTab;
001426 assert( pTab!=0 );
001427 assert( IsVirtual(pTab) );
001428
001429 /* Find all WHERE clause constraints referring to this virtual table.
001430 ** Mark each term with the TERM_OK flag. Set nTerm to the number of
001431 ** terms found.
001432 */
001433 for(p=pWC, nTerm=0; p; p=p->pOuter){
001434 for(i=0, pTerm=p->a; i<p->nTerm; i++, pTerm++){
001435 pTerm->wtFlags &= ~TERM_OK;
001436 if( pTerm->leftCursor != pSrc->iCursor ) continue;
001437 if( pTerm->prereqRight & mUnusable ) continue;
001438 assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) );
001439 testcase( pTerm->eOperator & WO_IN );
001440 testcase( pTerm->eOperator & WO_ISNULL );
001441 testcase( pTerm->eOperator & WO_IS );
001442 testcase( pTerm->eOperator & WO_ALL );
001443 if( (pTerm->eOperator & ~(WO_EQUIV))==0 ) continue;
001444 if( pTerm->wtFlags & TERM_VNULL ) continue;
001445
001446 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
001447 assert( pTerm->u.x.leftColumn>=XN_ROWID );
001448 assert( pTerm->u.x.leftColumn<pTab->nCol );
001449 if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0
001450 && !constraintCompatibleWithOuterJoin(pTerm,pSrc)
001451 ){
001452 continue;
001453 }
001454 nTerm++;
001455 pTerm->wtFlags |= TERM_OK;
001456 }
001457 }
001458
001459 /* If the ORDER BY clause contains only columns in the current
001460 ** virtual table then allocate space for the aOrderBy part of
001461 ** the sqlite3_index_info structure.
001462 */
001463 nOrderBy = 0;
001464 if( pOrderBy ){
001465 int n = pOrderBy->nExpr;
001466 for(i=0; i<n; i++){
001467 Expr *pExpr = pOrderBy->a[i].pExpr;
001468 Expr *pE2;
001469
001470 /* Skip over constant terms in the ORDER BY clause */
001471 if( sqlite3ExprIsConstant(0, pExpr) ){
001472 continue;
001473 }
001474
001475 /* Virtual tables are unable to deal with NULLS FIRST */
001476 if( pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_BIGNULL ) break;
001477
001478 /* First case - a direct column references without a COLLATE operator */
001479 if( pExpr->op==TK_COLUMN && pExpr->iTable==pSrc->iCursor ){
001480 assert( pExpr->iColumn>=XN_ROWID && pExpr->iColumn<pTab->nCol );
001481 continue;
001482 }
001483
001484 /* 2nd case - a column reference with a COLLATE operator. Only match
001485 ** of the COLLATE operator matches the collation of the column. */
001486 if( pExpr->op==TK_COLLATE
001487 && (pE2 = pExpr->pLeft)->op==TK_COLUMN
001488 && pE2->iTable==pSrc->iCursor
001489 ){
001490 const char *zColl; /* The collating sequence name */
001491 assert( !ExprHasProperty(pExpr, EP_IntValue) );
001492 assert( pExpr->u.zToken!=0 );
001493 assert( pE2->iColumn>=XN_ROWID && pE2->iColumn<pTab->nCol );
001494 pExpr->iColumn = pE2->iColumn;
001495 if( pE2->iColumn<0 ) continue; /* Collseq does not matter for rowid */
001496 zColl = sqlite3ColumnColl(&pTab->aCol[pE2->iColumn]);
001497 if( zColl==0 ) zColl = sqlite3StrBINARY;
001498 if( sqlite3_stricmp(pExpr->u.zToken, zColl)==0 ) continue;
001499 }
001500
001501 /* No matches cause a break out of the loop */
001502 break;
001503 }
001504 if( i==n ){
001505 nOrderBy = n;
001506 if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) && !pSrc->fg.rowidUsed ){
001507 eDistinct = 2 + ((pWInfo->wctrlFlags & WHERE_SORTBYGROUP)!=0);
001508 }else if( pWInfo->wctrlFlags & WHERE_GROUPBY ){
001509 eDistinct = 1;
001510 }
001511 }
001512 }
001513
001514 /* Allocate the sqlite3_index_info structure
001515 */
001516 pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo)
001517 + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm
001518 + sizeof(*pIdxOrderBy)*nOrderBy + sizeof(*pHidden)
001519 + sizeof(sqlite3_value*)*nTerm );
001520 if( pIdxInfo==0 ){
001521 sqlite3ErrorMsg(pParse, "out of memory");
001522 return 0;
001523 }
001524 pHidden = (struct HiddenIndexInfo*)&pIdxInfo[1];
001525 pIdxCons = (struct sqlite3_index_constraint*)&pHidden->aRhs[nTerm];
001526 pIdxOrderBy = (struct sqlite3_index_orderby*)&pIdxCons[nTerm];
001527 pUsage = (struct sqlite3_index_constraint_usage*)&pIdxOrderBy[nOrderBy];
001528 pIdxInfo->aConstraint = pIdxCons;
001529 pIdxInfo->aOrderBy = pIdxOrderBy;
001530 pIdxInfo->aConstraintUsage = pUsage;
001531 pIdxInfo->colUsed = (sqlite3_int64)pSrc->colUsed;
001532 if( HasRowid(pTab)==0 ){
001533 /* Ensure that all bits associated with PK columns are set. This is to
001534 ** ensure they are available for cases like RIGHT joins or OR loops. */
001535 Index *pPk = sqlite3PrimaryKeyIndex((Table*)pTab);
001536 assert( pPk!=0 );
001537 for(i=0; i<pPk->nKeyCol; i++){
001538 int iCol = pPk->aiColumn[i];
001539 assert( iCol>=0 );
001540 if( iCol>=BMS-1 ) iCol = BMS-1;
001541 pIdxInfo->colUsed |= MASKBIT(iCol);
001542 }
001543 }
001544 pHidden->pWC = pWC;
001545 pHidden->pParse = pParse;
001546 pHidden->eDistinct = eDistinct;
001547 pHidden->mIn = 0;
001548 for(p=pWC, i=j=0; p; p=p->pOuter){
001549 int nLast = i+p->nTerm;;
001550 for(pTerm=p->a; i<nLast; i++, pTerm++){
001551 u16 op;
001552 if( (pTerm->wtFlags & TERM_OK)==0 ) continue;
001553 pIdxCons[j].iColumn = pTerm->u.x.leftColumn;
001554 pIdxCons[j].iTermOffset = i;
001555 op = pTerm->eOperator & WO_ALL;
001556 if( op==WO_IN ){
001557 if( (pTerm->wtFlags & TERM_SLICE)==0 ){
001558 pHidden->mIn |= SMASKBIT32(j);
001559 }
001560 op = WO_EQ;
001561 }
001562 if( op==WO_AUX ){
001563 pIdxCons[j].op = pTerm->eMatchOp;
001564 }else if( op & (WO_ISNULL|WO_IS) ){
001565 if( op==WO_ISNULL ){
001566 pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_ISNULL;
001567 }else{
001568 pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_IS;
001569 }
001570 }else{
001571 pIdxCons[j].op = (u8)op;
001572 /* The direct assignment in the previous line is possible only because
001573 ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The
001574 ** following asserts verify this fact. */
001575 assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ );
001576 assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT );
001577 assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE );
001578 assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT );
001579 assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE );
001580 assert( pTerm->eOperator&(WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_AUX) );
001581
001582 if( op & (WO_LT|WO_LE|WO_GT|WO_GE)
001583 && sqlite3ExprIsVector(pTerm->pExpr->pRight)
001584 ){
001585 testcase( j!=i );
001586 if( j<16 ) mNoOmit |= (1 << j);
001587 if( op==WO_LT ) pIdxCons[j].op = WO_LE;
001588 if( op==WO_GT ) pIdxCons[j].op = WO_GE;
001589 }
001590 }
001591
001592 j++;
001593 }
001594 }
001595 assert( j==nTerm );
001596 pIdxInfo->nConstraint = j;
001597 for(i=j=0; i<nOrderBy; i++){
001598 Expr *pExpr = pOrderBy->a[i].pExpr;
001599 if( sqlite3ExprIsConstant(0, pExpr) ) continue;
001600 assert( pExpr->op==TK_COLUMN
001601 || (pExpr->op==TK_COLLATE && pExpr->pLeft->op==TK_COLUMN
001602 && pExpr->iColumn==pExpr->pLeft->iColumn) );
001603 pIdxOrderBy[j].iColumn = pExpr->iColumn;
001604 pIdxOrderBy[j].desc = pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_DESC;
001605 j++;
001606 }
001607 pIdxInfo->nOrderBy = j;
001608
001609 *pmNoOmit = mNoOmit;
001610 return pIdxInfo;
001611 }
001612
001613 /*
001614 ** Free and zero the sqlite3_index_info.idxStr value if needed.
001615 */
001616 static void freeIdxStr(sqlite3_index_info *pIdxInfo){
001617 if( pIdxInfo->needToFreeIdxStr ){
001618 sqlite3_free(pIdxInfo->idxStr);
001619 pIdxInfo->idxStr = 0;
001620 pIdxInfo->needToFreeIdxStr = 0;
001621 }
001622 }
001623
001624 /*
001625 ** Free an sqlite3_index_info structure allocated by allocateIndexInfo()
001626 ** and possibly modified by xBestIndex methods.
001627 */
001628 static void freeIndexInfo(sqlite3 *db, sqlite3_index_info *pIdxInfo){
001629 HiddenIndexInfo *pHidden;
001630 int i;
001631 assert( pIdxInfo!=0 );
001632 pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
001633 assert( pHidden->pParse!=0 );
001634 assert( pHidden->pParse->db==db );
001635 for(i=0; i<pIdxInfo->nConstraint; i++){
001636 sqlite3ValueFree(pHidden->aRhs[i]); /* IMP: R-14553-25174 */
001637 pHidden->aRhs[i] = 0;
001638 }
001639 freeIdxStr(pIdxInfo);
001640 sqlite3DbFree(db, pIdxInfo);
001641 }
001642
001643 /*
001644 ** The table object reference passed as the second argument to this function
001645 ** must represent a virtual table. This function invokes the xBestIndex()
001646 ** method of the virtual table with the sqlite3_index_info object that
001647 ** comes in as the 3rd argument to this function.
001648 **
001649 ** If an error occurs, pParse is populated with an error message and an
001650 ** appropriate error code is returned. A return of SQLITE_CONSTRAINT from
001651 ** xBestIndex is not considered an error. SQLITE_CONSTRAINT indicates that
001652 ** the current configuration of "unusable" flags in sqlite3_index_info can
001653 ** not result in a valid plan.
001654 **
001655 ** Whether or not an error is returned, it is the responsibility of the
001656 ** caller to eventually free p->idxStr if p->needToFreeIdxStr indicates
001657 ** that this is required.
001658 */
001659 static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){
001660 int rc;
001661 sqlite3_vtab *pVtab;
001662
001663 assert( IsVirtual(pTab) );
001664 pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab;
001665 whereTraceIndexInfoInputs(p, pTab);
001666 pParse->db->nSchemaLock++;
001667 rc = pVtab->pModule->xBestIndex(pVtab, p);
001668 pParse->db->nSchemaLock--;
001669 whereTraceIndexInfoOutputs(p, pTab);
001670
001671 if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT ){
001672 if( rc==SQLITE_NOMEM ){
001673 sqlite3OomFault(pParse->db);
001674 }else if( !pVtab->zErrMsg ){
001675 sqlite3ErrorMsg(pParse, "%s", sqlite3ErrStr(rc));
001676 }else{
001677 sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg);
001678 }
001679 }
001680 if( pTab->u.vtab.p->bAllSchemas ){
001681 sqlite3VtabUsesAllSchemas(pParse);
001682 }
001683 sqlite3_free(pVtab->zErrMsg);
001684 pVtab->zErrMsg = 0;
001685 return rc;
001686 }
001687 #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */
001688
001689 #ifdef SQLITE_ENABLE_STAT4
001690 /*
001691 ** Estimate the location of a particular key among all keys in an
001692 ** index. Store the results in aStat as follows:
001693 **
001694 ** aStat[0] Est. number of rows less than pRec
001695 ** aStat[1] Est. number of rows equal to pRec
001696 **
001697 ** Return the index of the sample that is the smallest sample that
001698 ** is greater than or equal to pRec. Note that this index is not an index
001699 ** into the aSample[] array - it is an index into a virtual set of samples
001700 ** based on the contents of aSample[] and the number of fields in record
001701 ** pRec.
001702 */
001703 static int whereKeyStats(
001704 Parse *pParse, /* Database connection */
001705 Index *pIdx, /* Index to consider domain of */
001706 UnpackedRecord *pRec, /* Vector of values to consider */
001707 int roundUp, /* Round up if true. Round down if false */
001708 tRowcnt *aStat /* OUT: stats written here */
001709 ){
001710 IndexSample *aSample = pIdx->aSample;
001711 int iCol; /* Index of required stats in anEq[] etc. */
001712 int i; /* Index of first sample >= pRec */
001713 int iSample; /* Smallest sample larger than or equal to pRec */
001714 int iMin = 0; /* Smallest sample not yet tested */
001715 int iTest; /* Next sample to test */
001716 int res; /* Result of comparison operation */
001717 int nField; /* Number of fields in pRec */
001718 tRowcnt iLower = 0; /* anLt[] + anEq[] of largest sample pRec is > */
001719
001720 #ifndef SQLITE_DEBUG
001721 UNUSED_PARAMETER( pParse );
001722 #endif
001723 assert( pRec!=0 );
001724 assert( pIdx->nSample>0 );
001725 assert( pRec->nField>0 );
001726
001727
001728 /* Do a binary search to find the first sample greater than or equal
001729 ** to pRec. If pRec contains a single field, the set of samples to search
001730 ** is simply the aSample[] array. If the samples in aSample[] contain more
001731 ** than one fields, all fields following the first are ignored.
001732 **
001733 ** If pRec contains N fields, where N is more than one, then as well as the
001734 ** samples in aSample[] (truncated to N fields), the search also has to
001735 ** consider prefixes of those samples. For example, if the set of samples
001736 ** in aSample is:
001737 **
001738 ** aSample[0] = (a, 5)
001739 ** aSample[1] = (a, 10)
001740 ** aSample[2] = (b, 5)
001741 ** aSample[3] = (c, 100)
001742 ** aSample[4] = (c, 105)
001743 **
001744 ** Then the search space should ideally be the samples above and the
001745 ** unique prefixes [a], [b] and [c]. But since that is hard to organize,
001746 ** the code actually searches this set:
001747 **
001748 ** 0: (a)
001749 ** 1: (a, 5)
001750 ** 2: (a, 10)
001751 ** 3: (a, 10)
001752 ** 4: (b)
001753 ** 5: (b, 5)
001754 ** 6: (c)
001755 ** 7: (c, 100)
001756 ** 8: (c, 105)
001757 ** 9: (c, 105)
001758 **
001759 ** For each sample in the aSample[] array, N samples are present in the
001760 ** effective sample array. In the above, samples 0 and 1 are based on
001761 ** sample aSample[0]. Samples 2 and 3 on aSample[1] etc.
001762 **
001763 ** Often, sample i of each block of N effective samples has (i+1) fields.
001764 ** Except, each sample may be extended to ensure that it is greater than or
001765 ** equal to the previous sample in the array. For example, in the above,
001766 ** sample 2 is the first sample of a block of N samples, so at first it
001767 ** appears that it should be 1 field in size. However, that would make it
001768 ** smaller than sample 1, so the binary search would not work. As a result,
001769 ** it is extended to two fields. The duplicates that this creates do not
001770 ** cause any problems.
001771 */
001772 if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){
001773 nField = pIdx->nKeyCol;
001774 }else{
001775 nField = pIdx->nColumn;
001776 }
001777 nField = MIN(pRec->nField, nField);
001778 iCol = 0;
001779 iSample = pIdx->nSample * nField;
001780 do{
001781 int iSamp; /* Index in aSample[] of test sample */
001782 int n; /* Number of fields in test sample */
001783
001784 iTest = (iMin+iSample)/2;
001785 iSamp = iTest / nField;
001786 if( iSamp>0 ){
001787 /* The proposed effective sample is a prefix of sample aSample[iSamp].
001788 ** Specifically, the shortest prefix of at least (1 + iTest%nField)
001789 ** fields that is greater than the previous effective sample. */
001790 for(n=(iTest % nField) + 1; n<nField; n++){
001791 if( aSample[iSamp-1].anLt[n-1]!=aSample[iSamp].anLt[n-1] ) break;
001792 }
001793 }else{
001794 n = iTest + 1;
001795 }
001796
001797 pRec->nField = n;
001798 res = sqlite3VdbeRecordCompare(aSample[iSamp].n, aSample[iSamp].p, pRec);
001799 if( res<0 ){
001800 iLower = aSample[iSamp].anLt[n-1] + aSample[iSamp].anEq[n-1];
001801 iMin = iTest+1;
001802 }else if( res==0 && n<nField ){
001803 iLower = aSample[iSamp].anLt[n-1];
001804 iMin = iTest+1;
001805 res = -1;
001806 }else{
001807 iSample = iTest;
001808 iCol = n-1;
001809 }
001810 }while( res && iMin<iSample );
001811 i = iSample / nField;
001812
001813 #ifdef SQLITE_DEBUG
001814 /* The following assert statements check that the binary search code
001815 ** above found the right answer. This block serves no purpose other
001816 ** than to invoke the asserts. */
001817 if( pParse->db->mallocFailed==0 ){
001818 if( res==0 ){
001819 /* If (res==0) is true, then pRec must be equal to sample i. */
001820 assert( i<pIdx->nSample );
001821 assert( iCol==nField-1 );
001822 pRec->nField = nField;
001823 assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)
001824 || pParse->db->mallocFailed
001825 );
001826 }else{
001827 /* Unless i==pIdx->nSample, indicating that pRec is larger than
001828 ** all samples in the aSample[] array, pRec must be smaller than the
001829 ** (iCol+1) field prefix of sample i. */
001830 assert( i<=pIdx->nSample && i>=0 );
001831 pRec->nField = iCol+1;
001832 assert( i==pIdx->nSample
001833 || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0
001834 || pParse->db->mallocFailed );
001835
001836 /* if i==0 and iCol==0, then record pRec is smaller than all samples
001837 ** in the aSample[] array. Otherwise, if (iCol>0) then pRec must
001838 ** be greater than or equal to the (iCol) field prefix of sample i.
001839 ** If (i>0), then pRec must also be greater than sample (i-1). */
001840 if( iCol>0 ){
001841 pRec->nField = iCol;
001842 assert( sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)<=0
001843 || pParse->db->mallocFailed || CORRUPT_DB );
001844 }
001845 if( i>0 ){
001846 pRec->nField = nField;
001847 assert( sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0
001848 || pParse->db->mallocFailed || CORRUPT_DB );
001849 }
001850 }
001851 }
001852 #endif /* ifdef SQLITE_DEBUG */
001853
001854 if( res==0 ){
001855 /* Record pRec is equal to sample i */
001856 assert( iCol==nField-1 );
001857 aStat[0] = aSample[i].anLt[iCol];
001858 aStat[1] = aSample[i].anEq[iCol];
001859 }else{
001860 /* At this point, the (iCol+1) field prefix of aSample[i] is the first
001861 ** sample that is greater than pRec. Or, if i==pIdx->nSample then pRec
001862 ** is larger than all samples in the array. */
001863 tRowcnt iUpper, iGap;
001864 if( i>=pIdx->nSample ){
001865 iUpper = pIdx->nRowEst0;
001866 }else{
001867 iUpper = aSample[i].anLt[iCol];
001868 }
001869
001870 if( iLower>=iUpper ){
001871 iGap = 0;
001872 }else{
001873 iGap = iUpper - iLower;
001874 }
001875 if( roundUp ){
001876 iGap = (iGap*2)/3;
001877 }else{
001878 iGap = iGap/3;
001879 }
001880 aStat[0] = iLower + iGap;
001881 aStat[1] = pIdx->aAvgEq[nField-1];
001882 }
001883
001884 /* Restore the pRec->nField value before returning. */
001885 pRec->nField = nField;
001886 return i;
001887 }
001888 #endif /* SQLITE_ENABLE_STAT4 */
001889
001890 /*
001891 ** If it is not NULL, pTerm is a term that provides an upper or lower
001892 ** bound on a range scan. Without considering pTerm, it is estimated
001893 ** that the scan will visit nNew rows. This function returns the number
001894 ** estimated to be visited after taking pTerm into account.
001895 **
001896 ** If the user explicitly specified a likelihood() value for this term,
001897 ** then the return value is the likelihood multiplied by the number of
001898 ** input rows. Otherwise, this function assumes that an "IS NOT NULL" term
001899 ** has a likelihood of 0.50, and any other term a likelihood of 0.25.
001900 */
001901 static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){
001902 LogEst nRet = nNew;
001903 if( pTerm ){
001904 if( pTerm->truthProb<=0 ){
001905 nRet += pTerm->truthProb;
001906 }else if( (pTerm->wtFlags & TERM_VNULL)==0 ){
001907 nRet -= 20; assert( 20==sqlite3LogEst(4) );
001908 }
001909 }
001910 return nRet;
001911 }
001912
001913
001914 #ifdef SQLITE_ENABLE_STAT4
001915 /*
001916 ** Return the affinity for a single column of an index.
001917 */
001918 char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){
001919 assert( iCol>=0 && iCol<pIdx->nColumn );
001920 if( !pIdx->zColAff ){
001921 if( sqlite3IndexAffinityStr(db, pIdx)==0 ) return SQLITE_AFF_BLOB;
001922 }
001923 assert( pIdx->zColAff[iCol]!=0 );
001924 return pIdx->zColAff[iCol];
001925 }
001926 #endif
001927
001928
001929 #ifdef SQLITE_ENABLE_STAT4
001930 /*
001931 ** This function is called to estimate the number of rows visited by a
001932 ** range-scan on a skip-scan index. For example:
001933 **
001934 ** CREATE INDEX i1 ON t1(a, b, c);
001935 ** SELECT * FROM t1 WHERE a=? AND c BETWEEN ? AND ?;
001936 **
001937 ** Value pLoop->nOut is currently set to the estimated number of rows
001938 ** visited for scanning (a=? AND b=?). This function reduces that estimate
001939 ** by some factor to account for the (c BETWEEN ? AND ?) expression based
001940 ** on the stat4 data for the index. this scan will be performed multiple
001941 ** times (once for each (a,b) combination that matches a=?) is dealt with
001942 ** by the caller.
001943 **
001944 ** It does this by scanning through all stat4 samples, comparing values
001945 ** extracted from pLower and pUpper with the corresponding column in each
001946 ** sample. If L and U are the number of samples found to be less than or
001947 ** equal to the values extracted from pLower and pUpper respectively, and
001948 ** N is the total number of samples, the pLoop->nOut value is adjusted
001949 ** as follows:
001950 **
001951 ** nOut = nOut * ( min(U - L, 1) / N )
001952 **
001953 ** If pLower is NULL, or a value cannot be extracted from the term, L is
001954 ** set to zero. If pUpper is NULL, or a value cannot be extracted from it,
001955 ** U is set to N.
001956 **
001957 ** Normally, this function sets *pbDone to 1 before returning. However,
001958 ** if no value can be extracted from either pLower or pUpper (and so the
001959 ** estimate of the number of rows delivered remains unchanged), *pbDone
001960 ** is left as is.
001961 **
001962 ** If an error occurs, an SQLite error code is returned. Otherwise,
001963 ** SQLITE_OK.
001964 */
001965 static int whereRangeSkipScanEst(
001966 Parse *pParse, /* Parsing & code generating context */
001967 WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */
001968 WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */
001969 WhereLoop *pLoop, /* Update the .nOut value of this loop */
001970 int *pbDone /* Set to true if at least one expr. value extracted */
001971 ){
001972 Index *p = pLoop->u.btree.pIndex;
001973 int nEq = pLoop->u.btree.nEq;
001974 sqlite3 *db = pParse->db;
001975 int nLower = -1;
001976 int nUpper = p->nSample+1;
001977 int rc = SQLITE_OK;
001978 u8 aff = sqlite3IndexColumnAffinity(db, p, nEq);
001979 CollSeq *pColl;
001980
001981 sqlite3_value *p1 = 0; /* Value extracted from pLower */
001982 sqlite3_value *p2 = 0; /* Value extracted from pUpper */
001983 sqlite3_value *pVal = 0; /* Value extracted from record */
001984
001985 pColl = sqlite3LocateCollSeq(pParse, p->azColl[nEq]);
001986 if( pLower ){
001987 rc = sqlite3Stat4ValueFromExpr(pParse, pLower->pExpr->pRight, aff, &p1);
001988 nLower = 0;
001989 }
001990 if( pUpper && rc==SQLITE_OK ){
001991 rc = sqlite3Stat4ValueFromExpr(pParse, pUpper->pExpr->pRight, aff, &p2);
001992 nUpper = p2 ? 0 : p->nSample;
001993 }
001994
001995 if( p1 || p2 ){
001996 int i;
001997 int nDiff;
001998 for(i=0; rc==SQLITE_OK && i<p->nSample; i++){
001999 rc = sqlite3Stat4Column(db, p->aSample[i].p, p->aSample[i].n, nEq, &pVal);
002000 if( rc==SQLITE_OK && p1 ){
002001 int res = sqlite3MemCompare(p1, pVal, pColl);
002002 if( res>=0 ) nLower++;
002003 }
002004 if( rc==SQLITE_OK && p2 ){
002005 int res = sqlite3MemCompare(p2, pVal, pColl);
002006 if( res>=0 ) nUpper++;
002007 }
002008 }
002009 nDiff = (nUpper - nLower);
002010 if( nDiff<=0 ) nDiff = 1;
002011
002012 /* If there is both an upper and lower bound specified, and the
002013 ** comparisons indicate that they are close together, use the fallback
002014 ** method (assume that the scan visits 1/64 of the rows) for estimating
002015 ** the number of rows visited. Otherwise, estimate the number of rows
002016 ** using the method described in the header comment for this function. */
002017 if( nDiff!=1 || pUpper==0 || pLower==0 ){
002018 int nAdjust = (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff));
002019 pLoop->nOut -= nAdjust;
002020 *pbDone = 1;
002021 WHERETRACE(0x20, ("range skip-scan regions: %u..%u adjust=%d est=%d\n",
002022 nLower, nUpper, nAdjust*-1, pLoop->nOut));
002023 }
002024
002025 }else{
002026 assert( *pbDone==0 );
002027 }
002028
002029 sqlite3ValueFree(p1);
002030 sqlite3ValueFree(p2);
002031 sqlite3ValueFree(pVal);
002032
002033 return rc;
002034 }
002035 #endif /* SQLITE_ENABLE_STAT4 */
002036
002037 /*
002038 ** This function is used to estimate the number of rows that will be visited
002039 ** by scanning an index for a range of values. The range may have an upper
002040 ** bound, a lower bound, or both. The WHERE clause terms that set the upper
002041 ** and lower bounds are represented by pLower and pUpper respectively. For
002042 ** example, assuming that index p is on t1(a):
002043 **
002044 ** ... FROM t1 WHERE a > ? AND a < ? ...
002045 ** |_____| |_____|
002046 ** | |
002047 ** pLower pUpper
002048 **
002049 ** If either of the upper or lower bound is not present, then NULL is passed in
002050 ** place of the corresponding WhereTerm.
002051 **
002052 ** The value in (pBuilder->pNew->u.btree.nEq) is the number of the index
002053 ** column subject to the range constraint. Or, equivalently, the number of
002054 ** equality constraints optimized by the proposed index scan. For example,
002055 ** assuming index p is on t1(a, b), and the SQL query is:
002056 **
002057 ** ... FROM t1 WHERE a = ? AND b > ? AND b < ? ...
002058 **
002059 ** then nEq is set to 1 (as the range restricted column, b, is the second
002060 ** left-most column of the index). Or, if the query is:
002061 **
002062 ** ... FROM t1 WHERE a > ? AND a < ? ...
002063 **
002064 ** then nEq is set to 0.
002065 **
002066 ** When this function is called, *pnOut is set to the sqlite3LogEst() of the
002067 ** number of rows that the index scan is expected to visit without
002068 ** considering the range constraints. If nEq is 0, then *pnOut is the number of
002069 ** rows in the index. Assuming no error occurs, *pnOut is adjusted (reduced)
002070 ** to account for the range constraints pLower and pUpper.
002071 **
002072 ** In the absence of sqlite_stat4 ANALYZE data, or if such data cannot be
002073 ** used, a single range inequality reduces the search space by a factor of 4.
002074 ** and a pair of constraints (x>? AND x<?) reduces the expected number of
002075 ** rows visited by a factor of 64.
002076 */
002077 static int whereRangeScanEst(
002078 Parse *pParse, /* Parsing & code generating context */
002079 WhereLoopBuilder *pBuilder,
002080 WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */
002081 WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */
002082 WhereLoop *pLoop /* Modify the .nOut and maybe .rRun fields */
002083 ){
002084 int rc = SQLITE_OK;
002085 int nOut = pLoop->nOut;
002086 LogEst nNew;
002087
002088 #ifdef SQLITE_ENABLE_STAT4
002089 Index *p = pLoop->u.btree.pIndex;
002090 int nEq = pLoop->u.btree.nEq;
002091
002092 if( p->nSample>0 && ALWAYS(nEq<p->nSampleCol)
002093 && OptimizationEnabled(pParse->db, SQLITE_Stat4)
002094 ){
002095 if( nEq==pBuilder->nRecValid ){
002096 UnpackedRecord *pRec = pBuilder->pRec;
002097 tRowcnt a[2];
002098 int nBtm = pLoop->u.btree.nBtm;
002099 int nTop = pLoop->u.btree.nTop;
002100
002101 /* Variable iLower will be set to the estimate of the number of rows in
002102 ** the index that are less than the lower bound of the range query. The
002103 ** lower bound being the concatenation of $P and $L, where $P is the
002104 ** key-prefix formed by the nEq values matched against the nEq left-most
002105 ** columns of the index, and $L is the value in pLower.
002106 **
002107 ** Or, if pLower is NULL or $L cannot be extracted from it (because it
002108 ** is not a simple variable or literal value), the lower bound of the
002109 ** range is $P. Due to a quirk in the way whereKeyStats() works, even
002110 ** if $L is available, whereKeyStats() is called for both ($P) and
002111 ** ($P:$L) and the larger of the two returned values is used.
002112 **
002113 ** Similarly, iUpper is to be set to the estimate of the number of rows
002114 ** less than the upper bound of the range query. Where the upper bound
002115 ** is either ($P) or ($P:$U). Again, even if $U is available, both values
002116 ** of iUpper are requested of whereKeyStats() and the smaller used.
002117 **
002118 ** The number of rows between the two bounds is then just iUpper-iLower.
002119 */
002120 tRowcnt iLower; /* Rows less than the lower bound */
002121 tRowcnt iUpper; /* Rows less than the upper bound */
002122 int iLwrIdx = -2; /* aSample[] for the lower bound */
002123 int iUprIdx = -1; /* aSample[] for the upper bound */
002124
002125 if( pRec ){
002126 testcase( pRec->nField!=pBuilder->nRecValid );
002127 pRec->nField = pBuilder->nRecValid;
002128 }
002129 /* Determine iLower and iUpper using ($P) only. */
002130 if( nEq==0 ){
002131 iLower = 0;
002132 iUpper = p->nRowEst0;
002133 }else{
002134 /* Note: this call could be optimized away - since the same values must
002135 ** have been requested when testing key $P in whereEqualScanEst(). */
002136 whereKeyStats(pParse, p, pRec, 0, a);
002137 iLower = a[0];
002138 iUpper = a[0] + a[1];
002139 }
002140
002141 assert( pLower==0 || (pLower->eOperator & (WO_GT|WO_GE))!=0 );
002142 assert( pUpper==0 || (pUpper->eOperator & (WO_LT|WO_LE))!=0 );
002143 assert( p->aSortOrder!=0 );
002144 if( p->aSortOrder[nEq] ){
002145 /* The roles of pLower and pUpper are swapped for a DESC index */
002146 SWAP(WhereTerm*, pLower, pUpper);
002147 SWAP(int, nBtm, nTop);
002148 }
002149
002150 /* If possible, improve on the iLower estimate using ($P:$L). */
002151 if( pLower ){
002152 int n; /* Values extracted from pExpr */
002153 Expr *pExpr = pLower->pExpr->pRight;
002154 rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nBtm, nEq, &n);
002155 if( rc==SQLITE_OK && n ){
002156 tRowcnt iNew;
002157 u16 mask = WO_GT|WO_LE;
002158 if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT);
002159 iLwrIdx = whereKeyStats(pParse, p, pRec, 0, a);
002160 iNew = a[0] + ((pLower->eOperator & mask) ? a[1] : 0);
002161 if( iNew>iLower ) iLower = iNew;
002162 nOut--;
002163 pLower = 0;
002164 }
002165 }
002166
002167 /* If possible, improve on the iUpper estimate using ($P:$U). */
002168 if( pUpper ){
002169 int n; /* Values extracted from pExpr */
002170 Expr *pExpr = pUpper->pExpr->pRight;
002171 rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nTop, nEq, &n);
002172 if( rc==SQLITE_OK && n ){
002173 tRowcnt iNew;
002174 u16 mask = WO_GT|WO_LE;
002175 if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT);
002176 iUprIdx = whereKeyStats(pParse, p, pRec, 1, a);
002177 iNew = a[0] + ((pUpper->eOperator & mask) ? a[1] : 0);
002178 if( iNew<iUpper ) iUpper = iNew;
002179 nOut--;
002180 pUpper = 0;
002181 }
002182 }
002183
002184 pBuilder->pRec = pRec;
002185 if( rc==SQLITE_OK ){
002186 if( iUpper>iLower ){
002187 nNew = sqlite3LogEst(iUpper - iLower);
002188 /* TUNING: If both iUpper and iLower are derived from the same
002189 ** sample, then assume they are 4x more selective. This brings
002190 ** the estimated selectivity more in line with what it would be
002191 ** if estimated without the use of STAT4 tables. */
002192 if( iLwrIdx==iUprIdx ){ nNew -= 20; }
002193 assert( 20==sqlite3LogEst(4) );
002194 }else{
002195 nNew = 10; assert( 10==sqlite3LogEst(2) );
002196 }
002197 if( nNew<nOut ){
002198 nOut = nNew;
002199 }
002200 WHERETRACE(0x20, ("STAT4 range scan: %u..%u est=%d\n",
002201 (u32)iLower, (u32)iUpper, nOut));
002202 }
002203 }else{
002204 int bDone = 0;
002205 rc = whereRangeSkipScanEst(pParse, pLower, pUpper, pLoop, &bDone);
002206 if( bDone ) return rc;
002207 }
002208 }
002209 #else
002210 UNUSED_PARAMETER(pParse);
002211 UNUSED_PARAMETER(pBuilder);
002212 assert( pLower || pUpper );
002213 #endif
002214 assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 || pParse->nErr>0 );
002215 nNew = whereRangeAdjust(pLower, nOut);
002216 nNew = whereRangeAdjust(pUpper, nNew);
002217
002218 /* TUNING: If there is both an upper and lower limit and neither limit
002219 ** has an application-defined likelihood(), assume the range is
002220 ** reduced by an additional 75%. This means that, by default, an open-ended
002221 ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the
002222 ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to
002223 ** match 1/64 of the index. */
002224 if( pLower && pLower->truthProb>0 && pUpper && pUpper->truthProb>0 ){
002225 nNew -= 20;
002226 }
002227
002228 nOut -= (pLower!=0) + (pUpper!=0);
002229 if( nNew<10 ) nNew = 10;
002230 if( nNew<nOut ) nOut = nNew;
002231 #if defined(WHERETRACE_ENABLED)
002232 if( pLoop->nOut>nOut ){
002233 WHERETRACE(0x20,("Range scan lowers nOut from %d to %d\n",
002234 pLoop->nOut, nOut));
002235 }
002236 #endif
002237 pLoop->nOut = (LogEst)nOut;
002238 return rc;
002239 }
002240
002241 #ifdef SQLITE_ENABLE_STAT4
002242 /*
002243 ** Estimate the number of rows that will be returned based on
002244 ** an equality constraint x=VALUE and where that VALUE occurs in
002245 ** the histogram data. This only works when x is the left-most
002246 ** column of an index and sqlite_stat4 histogram data is available
002247 ** for that index. When pExpr==NULL that means the constraint is
002248 ** "x IS NULL" instead of "x=VALUE".
002249 **
002250 ** Write the estimated row count into *pnRow and return SQLITE_OK.
002251 ** If unable to make an estimate, leave *pnRow unchanged and return
002252 ** non-zero.
002253 **
002254 ** This routine can fail if it is unable to load a collating sequence
002255 ** required for string comparison, or if unable to allocate memory
002256 ** for a UTF conversion required for comparison. The error is stored
002257 ** in the pParse structure.
002258 */
002259 static int whereEqualScanEst(
002260 Parse *pParse, /* Parsing & code generating context */
002261 WhereLoopBuilder *pBuilder,
002262 Expr *pExpr, /* Expression for VALUE in the x=VALUE constraint */
002263 tRowcnt *pnRow /* Write the revised row estimate here */
002264 ){
002265 Index *p = pBuilder->pNew->u.btree.pIndex;
002266 int nEq = pBuilder->pNew->u.btree.nEq;
002267 UnpackedRecord *pRec = pBuilder->pRec;
002268 int rc; /* Subfunction return code */
002269 tRowcnt a[2]; /* Statistics */
002270 int bOk;
002271
002272 assert( nEq>=1 );
002273 assert( nEq<=p->nColumn );
002274 assert( p->aSample!=0 );
002275 assert( p->nSample>0 );
002276 assert( pBuilder->nRecValid<nEq );
002277
002278 /* If values are not available for all fields of the index to the left
002279 ** of this one, no estimate can be made. Return SQLITE_NOTFOUND. */
002280 if( pBuilder->nRecValid<(nEq-1) ){
002281 return SQLITE_NOTFOUND;
002282 }
002283
002284 /* This is an optimization only. The call to sqlite3Stat4ProbeSetValue()
002285 ** below would return the same value. */
002286 if( nEq>=p->nColumn ){
002287 *pnRow = 1;
002288 return SQLITE_OK;
002289 }
002290
002291 rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, 1, nEq-1, &bOk);
002292 pBuilder->pRec = pRec;
002293 if( rc!=SQLITE_OK ) return rc;
002294 if( bOk==0 ) return SQLITE_NOTFOUND;
002295 pBuilder->nRecValid = nEq;
002296
002297 whereKeyStats(pParse, p, pRec, 0, a);
002298 WHERETRACE(0x20,("equality scan regions %s(%d): %d\n",
002299 p->zName, nEq-1, (int)a[1]));
002300 *pnRow = a[1];
002301
002302 return rc;
002303 }
002304 #endif /* SQLITE_ENABLE_STAT4 */
002305
002306 #ifdef SQLITE_ENABLE_STAT4
002307 /*
002308 ** Estimate the number of rows that will be returned based on
002309 ** an IN constraint where the right-hand side of the IN operator
002310 ** is a list of values. Example:
002311 **
002312 ** WHERE x IN (1,2,3,4)
002313 **
002314 ** Write the estimated row count into *pnRow and return SQLITE_OK.
002315 ** If unable to make an estimate, leave *pnRow unchanged and return
002316 ** non-zero.
002317 **
002318 ** This routine can fail if it is unable to load a collating sequence
002319 ** required for string comparison, or if unable to allocate memory
002320 ** for a UTF conversion required for comparison. The error is stored
002321 ** in the pParse structure.
002322 */
002323 static int whereInScanEst(
002324 Parse *pParse, /* Parsing & code generating context */
002325 WhereLoopBuilder *pBuilder,
002326 ExprList *pList, /* The value list on the RHS of "x IN (v1,v2,v3,...)" */
002327 tRowcnt *pnRow /* Write the revised row estimate here */
002328 ){
002329 Index *p = pBuilder->pNew->u.btree.pIndex;
002330 i64 nRow0 = sqlite3LogEstToInt(p->aiRowLogEst[0]);
002331 int nRecValid = pBuilder->nRecValid;
002332 int rc = SQLITE_OK; /* Subfunction return code */
002333 tRowcnt nEst; /* Number of rows for a single term */
002334 tRowcnt nRowEst = 0; /* New estimate of the number of rows */
002335 int i; /* Loop counter */
002336
002337 assert( p->aSample!=0 );
002338 for(i=0; rc==SQLITE_OK && i<pList->nExpr; i++){
002339 nEst = nRow0;
002340 rc = whereEqualScanEst(pParse, pBuilder, pList->a[i].pExpr, &nEst);
002341 nRowEst += nEst;
002342 pBuilder->nRecValid = nRecValid;
002343 }
002344
002345 if( rc==SQLITE_OK ){
002346 if( nRowEst > (tRowcnt)nRow0 ) nRowEst = nRow0;
002347 *pnRow = nRowEst;
002348 WHERETRACE(0x20,("IN row estimate: est=%d\n", nRowEst));
002349 }
002350 assert( pBuilder->nRecValid==nRecValid );
002351 return rc;
002352 }
002353 #endif /* SQLITE_ENABLE_STAT4 */
002354
002355
002356 #if defined(WHERETRACE_ENABLED) || defined(SQLITE_DEBUG)
002357 /*
002358 ** Print the content of a WhereTerm object
002359 */
002360 void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm){
002361 if( pTerm==0 ){
002362 sqlite3DebugPrintf("TERM-%-3d NULL\n", iTerm);
002363 }else{
002364 char zType[8];
002365 char zLeft[50];
002366 memcpy(zType, "....", 5);
002367 if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V';
002368 if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E';
002369 if( ExprHasProperty(pTerm->pExpr, EP_OuterON) ) zType[2] = 'L';
002370 if( pTerm->wtFlags & TERM_CODED ) zType[3] = 'C';
002371 if( pTerm->eOperator & WO_SINGLE ){
002372 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
002373 sqlite3_snprintf(sizeof(zLeft),zLeft,"left={%d:%d}",
002374 pTerm->leftCursor, pTerm->u.x.leftColumn);
002375 }else if( (pTerm->eOperator & WO_OR)!=0 && pTerm->u.pOrInfo!=0 ){
002376 sqlite3_snprintf(sizeof(zLeft),zLeft,"indexable=0x%llx",
002377 pTerm->u.pOrInfo->indexable);
002378 }else{
002379 sqlite3_snprintf(sizeof(zLeft),zLeft,"left=%d", pTerm->leftCursor);
002380 }
002381 sqlite3DebugPrintf(
002382 "TERM-%-3d %p %s %-12s op=%03x wtFlags=%04x",
002383 iTerm, pTerm, zType, zLeft, pTerm->eOperator, pTerm->wtFlags);
002384 /* The 0x10000 .wheretrace flag causes extra information to be
002385 ** shown about each Term */
002386 if( sqlite3WhereTrace & 0x10000 ){
002387 sqlite3DebugPrintf(" prob=%-3d prereq=%llx,%llx",
002388 pTerm->truthProb, (u64)pTerm->prereqAll, (u64)pTerm->prereqRight);
002389 }
002390 if( (pTerm->eOperator & (WO_OR|WO_AND))==0 && pTerm->u.x.iField ){
002391 sqlite3DebugPrintf(" iField=%d", pTerm->u.x.iField);
002392 }
002393 if( pTerm->iParent>=0 ){
002394 sqlite3DebugPrintf(" iParent=%d", pTerm->iParent);
002395 }
002396 sqlite3DebugPrintf("\n");
002397 sqlite3TreeViewExpr(0, pTerm->pExpr, 0);
002398 }
002399 }
002400 void sqlite3ShowWhereTerm(WhereTerm *pTerm){
002401 sqlite3WhereTermPrint(pTerm, 0);
002402 }
002403 #endif
002404
002405 #ifdef WHERETRACE_ENABLED
002406 /*
002407 ** Show the complete content of a WhereClause
002408 */
002409 void sqlite3WhereClausePrint(WhereClause *pWC){
002410 int i;
002411 for(i=0; i<pWC->nTerm; i++){
002412 sqlite3WhereTermPrint(&pWC->a[i], i);
002413 }
002414 }
002415 #endif
002416
002417 #ifdef WHERETRACE_ENABLED
002418 /*
002419 ** Print a WhereLoop object for debugging purposes
002420 **
002421 ** Format example:
002422 **
002423 ** .--- Position in WHERE clause rSetup, rRun, nOut ---.
002424 ** | |
002425 ** | .--- selfMask nTerm ------. |
002426 ** | | | |
002427 ** | | .-- prereq Idx wsFlags----. | |
002428 ** | | | Name | | |
002429 ** | | | __|__ nEq ---. ___|__ | __|__
002430 ** | / \ / \ / \ | / \ / \ / \
002431 ** 1.002.001 t2.t2xy 2 f 010241 N 2 cost 0,56,31
002432 */
002433 void sqlite3WhereLoopPrint(const WhereLoop *p, const WhereClause *pWC){
002434 WhereInfo *pWInfo;
002435 if( pWC ){
002436 pWInfo = pWC->pWInfo;
002437 int nb = 1+(pWInfo->pTabList->nSrc+3)/4;
002438 SrcItem *pItem = pWInfo->pTabList->a + p->iTab;
002439 Table *pTab = pItem->pSTab;
002440 Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1;
002441 sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId,
002442 p->iTab, nb, p->maskSelf, nb, p->prereq & mAll);
002443 sqlite3DebugPrintf(" %12s",
002444 pItem->zAlias ? pItem->zAlias : pTab->zName);
002445 }else{
002446 pWInfo = 0;
002447 sqlite3DebugPrintf("%c%2d.%03llx.%03llx %c%d",
002448 p->cId, p->iTab, p->maskSelf, p->prereq & 0xfff, p->cId, p->iTab);
002449 }
002450 if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){
002451 const char *zName;
002452 if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){
002453 if( strncmp(zName, "sqlite_autoindex_", 17)==0 ){
002454 int i = sqlite3Strlen30(zName) - 1;
002455 while( zName[i]!='_' ) i--;
002456 zName += i;
002457 }
002458 sqlite3DebugPrintf(".%-16s %2d", zName, p->u.btree.nEq);
002459 }else{
002460 sqlite3DebugPrintf("%20s","");
002461 }
002462 }else{
002463 char *z;
002464 if( p->u.vtab.idxStr ){
002465 z = sqlite3_mprintf("(%d,\"%s\",%#x)",
002466 p->u.vtab.idxNum, p->u.vtab.idxStr, p->u.vtab.omitMask);
002467 }else{
002468 z = sqlite3_mprintf("(%d,%x)", p->u.vtab.idxNum, p->u.vtab.omitMask);
002469 }
002470 sqlite3DebugPrintf(" %-19s", z);
002471 sqlite3_free(z);
002472 }
002473 if( p->wsFlags & WHERE_SKIPSCAN ){
002474 sqlite3DebugPrintf(" f %06x %d-%d", p->wsFlags, p->nLTerm,p->nSkip);
002475 }else{
002476 sqlite3DebugPrintf(" f %06x N %d", p->wsFlags, p->nLTerm);
002477 }
002478 if( pWInfo && pWInfo->bStarUsed && p->rStarDelta!=0 ){
002479 sqlite3DebugPrintf(" cost %d,%d,%d delta=%d\n",
002480 p->rSetup, p->rRun, p->nOut, p->rStarDelta);
002481 }else{
002482 sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut);
002483 }
002484 if( p->nLTerm && (sqlite3WhereTrace & 0x4000)!=0 ){
002485 int i;
002486 for(i=0; i<p->nLTerm; i++){
002487 sqlite3WhereTermPrint(p->aLTerm[i], i);
002488 }
002489 }
002490 }
002491 void sqlite3ShowWhereLoop(const WhereLoop *p){
002492 if( p ) sqlite3WhereLoopPrint(p, 0);
002493 }
002494 void sqlite3ShowWhereLoopList(const WhereLoop *p){
002495 while( p ){
002496 sqlite3ShowWhereLoop(p);
002497 p = p->pNextLoop;
002498 }
002499 }
002500 #endif
002501
002502 /*
002503 ** Convert bulk memory into a valid WhereLoop that can be passed
002504 ** to whereLoopClear harmlessly.
002505 */
002506 static void whereLoopInit(WhereLoop *p){
002507 p->aLTerm = p->aLTermSpace;
002508 p->nLTerm = 0;
002509 p->nLSlot = ArraySize(p->aLTermSpace);
002510 p->wsFlags = 0;
002511 }
002512
002513 /*
002514 ** Clear the WhereLoop.u union. Leave WhereLoop.pLTerm intact.
002515 */
002516 static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){
002517 if( p->wsFlags & (WHERE_VIRTUALTABLE|WHERE_AUTO_INDEX) ){
002518 if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 && p->u.vtab.needFree ){
002519 sqlite3_free(p->u.vtab.idxStr);
002520 p->u.vtab.needFree = 0;
002521 p->u.vtab.idxStr = 0;
002522 }else if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 && p->u.btree.pIndex!=0 ){
002523 sqlite3DbFree(db, p->u.btree.pIndex->zColAff);
002524 sqlite3DbFreeNN(db, p->u.btree.pIndex);
002525 p->u.btree.pIndex = 0;
002526 }
002527 }
002528 }
002529
002530 /*
002531 ** Deallocate internal memory used by a WhereLoop object. Leave the
002532 ** object in an initialized state, as if it had been newly allocated.
002533 */
002534 static void whereLoopClear(sqlite3 *db, WhereLoop *p){
002535 if( p->aLTerm!=p->aLTermSpace ){
002536 sqlite3DbFreeNN(db, p->aLTerm);
002537 p->aLTerm = p->aLTermSpace;
002538 p->nLSlot = ArraySize(p->aLTermSpace);
002539 }
002540 whereLoopClearUnion(db, p);
002541 p->nLTerm = 0;
002542 p->wsFlags = 0;
002543 }
002544
002545 /*
002546 ** Increase the memory allocation for pLoop->aLTerm[] to be at least n.
002547 */
002548 static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){
002549 WhereTerm **paNew;
002550 if( p->nLSlot>=n ) return SQLITE_OK;
002551 n = (n+7)&~7;
002552 paNew = sqlite3DbMallocRawNN(db, sizeof(p->aLTerm[0])*n);
002553 if( paNew==0 ) return SQLITE_NOMEM_BKPT;
002554 memcpy(paNew, p->aLTerm, sizeof(p->aLTerm[0])*p->nLSlot);
002555 if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFreeNN(db, p->aLTerm);
002556 p->aLTerm = paNew;
002557 p->nLSlot = n;
002558 return SQLITE_OK;
002559 }
002560
002561 /*
002562 ** Transfer content from the second pLoop into the first.
002563 */
002564 static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){
002565 whereLoopClearUnion(db, pTo);
002566 if( pFrom->nLTerm > pTo->nLSlot
002567 && whereLoopResize(db, pTo, pFrom->nLTerm)
002568 ){
002569 memset(pTo, 0, WHERE_LOOP_XFER_SZ);
002570 return SQLITE_NOMEM_BKPT;
002571 }
002572 memcpy(pTo, pFrom, WHERE_LOOP_XFER_SZ);
002573 memcpy(pTo->aLTerm, pFrom->aLTerm, pTo->nLTerm*sizeof(pTo->aLTerm[0]));
002574 if( pFrom->wsFlags & WHERE_VIRTUALTABLE ){
002575 pFrom->u.vtab.needFree = 0;
002576 }else if( (pFrom->wsFlags & WHERE_AUTO_INDEX)!=0 ){
002577 pFrom->u.btree.pIndex = 0;
002578 }
002579 return SQLITE_OK;
002580 }
002581
002582 /*
002583 ** Delete a WhereLoop object
002584 */
002585 static void whereLoopDelete(sqlite3 *db, WhereLoop *p){
002586 assert( db!=0 );
002587 whereLoopClear(db, p);
002588 sqlite3DbNNFreeNN(db, p);
002589 }
002590
002591 /*
002592 ** Free a WhereInfo structure
002593 */
002594 static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){
002595 assert( pWInfo!=0 );
002596 assert( db!=0 );
002597 sqlite3WhereClauseClear(&pWInfo->sWC);
002598 while( pWInfo->pLoops ){
002599 WhereLoop *p = pWInfo->pLoops;
002600 pWInfo->pLoops = p->pNextLoop;
002601 whereLoopDelete(db, p);
002602 }
002603 while( pWInfo->pMemToFree ){
002604 WhereMemBlock *pNext = pWInfo->pMemToFree->pNext;
002605 sqlite3DbNNFreeNN(db, pWInfo->pMemToFree);
002606 pWInfo->pMemToFree = pNext;
002607 }
002608 sqlite3DbNNFreeNN(db, pWInfo);
002609 }
002610
002611 /*
002612 ** Return TRUE if X is a proper subset of Y but is of equal or less cost.
002613 ** In other words, return true if all constraints of X are also part of Y
002614 ** and Y has additional constraints that might speed the search that X lacks
002615 ** but the cost of running X is not more than the cost of running Y.
002616 **
002617 ** In other words, return true if the cost relationship between X and Y
002618 ** is inverted and needs to be adjusted.
002619 **
002620 ** Case 1:
002621 **
002622 ** (1a) X and Y use the same index.
002623 ** (1b) X has fewer == terms than Y
002624 ** (1c) Neither X nor Y use skip-scan
002625 ** (1d) X does not have a a greater cost than Y
002626 **
002627 ** Case 2:
002628 **
002629 ** (2a) X has the same or lower cost, or returns the same or fewer rows,
002630 ** than Y.
002631 ** (2b) X uses fewer WHERE clause terms than Y
002632 ** (2c) Every WHERE clause term used by X is also used by Y
002633 ** (2d) X skips at least as many columns as Y
002634 ** (2e) If X is a covering index, than Y is too
002635 */
002636 static int whereLoopCheaperProperSubset(
002637 const WhereLoop *pX, /* First WhereLoop to compare */
002638 const WhereLoop *pY /* Compare against this WhereLoop */
002639 ){
002640 int i, j;
002641 if( pX->rRun>pY->rRun && pX->nOut>pY->nOut ) return 0; /* (1d) and (2a) */
002642 assert( (pX->wsFlags & WHERE_VIRTUALTABLE)==0 );
002643 assert( (pY->wsFlags & WHERE_VIRTUALTABLE)==0 );
002644 if( pX->u.btree.nEq < pY->u.btree.nEq /* (1b) */
002645 && pX->u.btree.pIndex==pY->u.btree.pIndex /* (1a) */
002646 && pX->nSkip==0 && pY->nSkip==0 /* (1c) */
002647 ){
002648 return 1; /* Case 1 is true */
002649 }
002650 if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){
002651 return 0; /* (2b) */
002652 }
002653 if( pY->nSkip > pX->nSkip ) return 0; /* (2d) */
002654 for(i=pX->nLTerm-1; i>=0; i--){
002655 if( pX->aLTerm[i]==0 ) continue;
002656 for(j=pY->nLTerm-1; j>=0; j--){
002657 if( pY->aLTerm[j]==pX->aLTerm[i] ) break;
002658 }
002659 if( j<0 ) return 0; /* (2c) */
002660 }
002661 if( (pX->wsFlags&WHERE_IDX_ONLY)!=0
002662 && (pY->wsFlags&WHERE_IDX_ONLY)==0 ){
002663 return 0; /* (2e) */
002664 }
002665 return 1; /* Case 2 is true */
002666 }
002667
002668 /*
002669 ** Try to adjust the cost and number of output rows of WhereLoop pTemplate
002670 ** upwards or downwards so that:
002671 **
002672 ** (1) pTemplate costs less than any other WhereLoops that are a proper
002673 ** subset of pTemplate
002674 **
002675 ** (2) pTemplate costs more than any other WhereLoops for which pTemplate
002676 ** is a proper subset.
002677 **
002678 ** To say "WhereLoop X is a proper subset of Y" means that X uses fewer
002679 ** WHERE clause terms than Y and that every WHERE clause term used by X is
002680 ** also used by Y.
002681 */
002682 static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){
002683 if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return;
002684 for(; p; p=p->pNextLoop){
002685 if( p->iTab!=pTemplate->iTab ) continue;
002686 if( (p->wsFlags & WHERE_INDEXED)==0 ) continue;
002687 if( whereLoopCheaperProperSubset(p, pTemplate) ){
002688 /* Adjust pTemplate cost downward so that it is cheaper than its
002689 ** subset p. */
002690 WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n",
002691 pTemplate->rRun, pTemplate->nOut,
002692 MIN(p->rRun, pTemplate->rRun),
002693 MIN(p->nOut - 1, pTemplate->nOut)));
002694 pTemplate->rRun = MIN(p->rRun, pTemplate->rRun);
002695 pTemplate->nOut = MIN(p->nOut - 1, pTemplate->nOut);
002696 }else if( whereLoopCheaperProperSubset(pTemplate, p) ){
002697 /* Adjust pTemplate cost upward so that it is costlier than p since
002698 ** pTemplate is a proper subset of p */
002699 WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n",
002700 pTemplate->rRun, pTemplate->nOut,
002701 MAX(p->rRun, pTemplate->rRun),
002702 MAX(p->nOut + 1, pTemplate->nOut)));
002703 pTemplate->rRun = MAX(p->rRun, pTemplate->rRun);
002704 pTemplate->nOut = MAX(p->nOut + 1, pTemplate->nOut);
002705 }
002706 }
002707 }
002708
002709 /*
002710 ** Search the list of WhereLoops in *ppPrev looking for one that can be
002711 ** replaced by pTemplate.
002712 **
002713 ** Return NULL if pTemplate does not belong on the WhereLoop list.
002714 ** In other words if pTemplate ought to be dropped from further consideration.
002715 **
002716 ** If pX is a WhereLoop that pTemplate can replace, then return the
002717 ** link that points to pX.
002718 **
002719 ** If pTemplate cannot replace any existing element of the list but needs
002720 ** to be added to the list as a new entry, then return a pointer to the
002721 ** tail of the list.
002722 */
002723 static WhereLoop **whereLoopFindLesser(
002724 WhereLoop **ppPrev,
002725 const WhereLoop *pTemplate
002726 ){
002727 WhereLoop *p;
002728 for(p=(*ppPrev); p; ppPrev=&p->pNextLoop, p=*ppPrev){
002729 if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ){
002730 /* If either the iTab or iSortIdx values for two WhereLoop are different
002731 ** then those WhereLoops need to be considered separately. Neither is
002732 ** a candidate to replace the other. */
002733 continue;
002734 }
002735 /* In the current implementation, the rSetup value is either zero
002736 ** or the cost of building an automatic index (NlogN) and the NlogN
002737 ** is the same for compatible WhereLoops. */
002738 assert( p->rSetup==0 || pTemplate->rSetup==0
002739 || p->rSetup==pTemplate->rSetup );
002740
002741 /* whereLoopAddBtree() always generates and inserts the automatic index
002742 ** case first. Hence compatible candidate WhereLoops never have a larger
002743 ** rSetup. Call this SETUP-INVARIANT */
002744 assert( p->rSetup>=pTemplate->rSetup );
002745
002746 /* Any loop using an application-defined index (or PRIMARY KEY or
002747 ** UNIQUE constraint) with one or more == constraints is better
002748 ** than an automatic index. Unless it is a skip-scan. */
002749 if( (p->wsFlags & WHERE_AUTO_INDEX)!=0
002750 && (pTemplate->nSkip)==0
002751 && (pTemplate->wsFlags & WHERE_INDEXED)!=0
002752 && (pTemplate->wsFlags & WHERE_COLUMN_EQ)!=0
002753 && (p->prereq & pTemplate->prereq)==pTemplate->prereq
002754 ){
002755 break;
002756 }
002757
002758 /* If existing WhereLoop p is better than pTemplate, pTemplate can be
002759 ** discarded. WhereLoop p is better if:
002760 ** (1) p has no more dependencies than pTemplate, and
002761 ** (2) p has an equal or lower cost than pTemplate
002762 */
002763 if( (p->prereq & pTemplate->prereq)==p->prereq /* (1) */
002764 && p->rSetup<=pTemplate->rSetup /* (2a) */
002765 && p->rRun<=pTemplate->rRun /* (2b) */
002766 && p->nOut<=pTemplate->nOut /* (2c) */
002767 ){
002768 return 0; /* Discard pTemplate */
002769 }
002770
002771 /* If pTemplate is always better than p, then cause p to be overwritten
002772 ** with pTemplate. pTemplate is better than p if:
002773 ** (1) pTemplate has no more dependencies than p, and
002774 ** (2) pTemplate has an equal or lower cost than p.
002775 */
002776 if( (p->prereq & pTemplate->prereq)==pTemplate->prereq /* (1) */
002777 && p->rRun>=pTemplate->rRun /* (2a) */
002778 && p->nOut>=pTemplate->nOut /* (2b) */
002779 ){
002780 assert( p->rSetup>=pTemplate->rSetup ); /* SETUP-INVARIANT above */
002781 break; /* Cause p to be overwritten by pTemplate */
002782 }
002783 }
002784 return ppPrev;
002785 }
002786
002787 /*
002788 ** Insert or replace a WhereLoop entry using the template supplied.
002789 **
002790 ** An existing WhereLoop entry might be overwritten if the new template
002791 ** is better and has fewer dependencies. Or the template will be ignored
002792 ** and no insert will occur if an existing WhereLoop is faster and has
002793 ** fewer dependencies than the template. Otherwise a new WhereLoop is
002794 ** added based on the template.
002795 **
002796 ** If pBuilder->pOrSet is not NULL then we care about only the
002797 ** prerequisites and rRun and nOut costs of the N best loops. That
002798 ** information is gathered in the pBuilder->pOrSet object. This special
002799 ** processing mode is used only for OR clause processing.
002800 **
002801 ** When accumulating multiple loops (when pBuilder->pOrSet is NULL) we
002802 ** still might overwrite similar loops with the new template if the
002803 ** new template is better. Loops may be overwritten if the following
002804 ** conditions are met:
002805 **
002806 ** (1) They have the same iTab.
002807 ** (2) They have the same iSortIdx.
002808 ** (3) The template has same or fewer dependencies than the current loop
002809 ** (4) The template has the same or lower cost than the current loop
002810 */
002811 static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){
002812 WhereLoop **ppPrev, *p;
002813 WhereInfo *pWInfo = pBuilder->pWInfo;
002814 sqlite3 *db = pWInfo->pParse->db;
002815 int rc;
002816
002817 /* Stop the search once we hit the query planner search limit */
002818 if( pBuilder->iPlanLimit==0 ){
002819 WHERETRACE(0xffffffff,("=== query planner search limit reached ===\n"));
002820 if( pBuilder->pOrSet ) pBuilder->pOrSet->n = 0;
002821 return SQLITE_DONE;
002822 }
002823 pBuilder->iPlanLimit--;
002824
002825 whereLoopAdjustCost(pWInfo->pLoops, pTemplate);
002826
002827 /* If pBuilder->pOrSet is defined, then only keep track of the costs
002828 ** and prereqs.
002829 */
002830 if( pBuilder->pOrSet!=0 ){
002831 if( pTemplate->nLTerm ){
002832 #if WHERETRACE_ENABLED
002833 u16 n = pBuilder->pOrSet->n;
002834 int x =
002835 #endif
002836 whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun,
002837 pTemplate->nOut);
002838 #if WHERETRACE_ENABLED /* 0x8 */
002839 if( sqlite3WhereTrace & 0x8 ){
002840 sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n);
002841 sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC);
002842 }
002843 #endif
002844 }
002845 return SQLITE_OK;
002846 }
002847
002848 /* Look for an existing WhereLoop to replace with pTemplate
002849 */
002850 ppPrev = whereLoopFindLesser(&pWInfo->pLoops, pTemplate);
002851
002852 if( ppPrev==0 ){
002853 /* There already exists a WhereLoop on the list that is better
002854 ** than pTemplate, so just ignore pTemplate */
002855 #if WHERETRACE_ENABLED /* 0x8 */
002856 if( sqlite3WhereTrace & 0x8 ){
002857 sqlite3DebugPrintf(" skip: ");
002858 sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC);
002859 }
002860 #endif
002861 return SQLITE_OK;
002862 }else{
002863 p = *ppPrev;
002864 }
002865
002866 /* If we reach this point it means that either p[] should be overwritten
002867 ** with pTemplate[] if p[] exists, or if p==NULL then allocate a new
002868 ** WhereLoop and insert it.
002869 */
002870 #if WHERETRACE_ENABLED /* 0x8 */
002871 if( sqlite3WhereTrace & 0x8 ){
002872 if( p!=0 ){
002873 sqlite3DebugPrintf("replace: ");
002874 sqlite3WhereLoopPrint(p, pBuilder->pWC);
002875 sqlite3DebugPrintf(" with: ");
002876 }else{
002877 sqlite3DebugPrintf(" add: ");
002878 }
002879 sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC);
002880 }
002881 #endif
002882 if( p==0 ){
002883 /* Allocate a new WhereLoop to add to the end of the list */
002884 *ppPrev = p = sqlite3DbMallocRawNN(db, sizeof(WhereLoop));
002885 if( p==0 ) return SQLITE_NOMEM_BKPT;
002886 whereLoopInit(p);
002887 p->pNextLoop = 0;
002888 }else{
002889 /* We will be overwriting WhereLoop p[]. But before we do, first
002890 ** go through the rest of the list and delete any other entries besides
002891 ** p[] that are also supplanted by pTemplate */
002892 WhereLoop **ppTail = &p->pNextLoop;
002893 WhereLoop *pToDel;
002894 while( *ppTail ){
002895 ppTail = whereLoopFindLesser(ppTail, pTemplate);
002896 if( ppTail==0 ) break;
002897 pToDel = *ppTail;
002898 if( pToDel==0 ) break;
002899 *ppTail = pToDel->pNextLoop;
002900 #if WHERETRACE_ENABLED /* 0x8 */
002901 if( sqlite3WhereTrace & 0x8 ){
002902 sqlite3DebugPrintf(" delete: ");
002903 sqlite3WhereLoopPrint(pToDel, pBuilder->pWC);
002904 }
002905 #endif
002906 whereLoopDelete(db, pToDel);
002907 }
002908 }
002909 rc = whereLoopXfer(db, p, pTemplate);
002910 if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){
002911 Index *pIndex = p->u.btree.pIndex;
002912 if( pIndex && pIndex->idxType==SQLITE_IDXTYPE_IPK ){
002913 p->u.btree.pIndex = 0;
002914 }
002915 }
002916 return rc;
002917 }
002918
002919 /*
002920 ** Adjust the WhereLoop.nOut value downward to account for terms of the
002921 ** WHERE clause that reference the loop but which are not used by an
002922 ** index.
002923 *
002924 ** For every WHERE clause term that is not used by the index
002925 ** and which has a truth probability assigned by one of the likelihood(),
002926 ** likely(), or unlikely() SQL functions, reduce the estimated number
002927 ** of output rows by the probability specified.
002928 **
002929 ** TUNING: For every WHERE clause term that is not used by the index
002930 ** and which does not have an assigned truth probability, heuristics
002931 ** described below are used to try to estimate the truth probability.
002932 ** TODO --> Perhaps this is something that could be improved by better
002933 ** table statistics.
002934 **
002935 ** Heuristic 1: Estimate the truth probability as 93.75%. The 93.75%
002936 ** value corresponds to -1 in LogEst notation, so this means decrement
002937 ** the WhereLoop.nOut field for every such WHERE clause term.
002938 **
002939 ** Heuristic 2: If there exists one or more WHERE clause terms of the
002940 ** form "x==EXPR" and EXPR is not a constant 0 or 1, then make sure the
002941 ** final output row estimate is no greater than 1/4 of the total number
002942 ** of rows in the table. In other words, assume that x==EXPR will filter
002943 ** out at least 3 out of 4 rows. If EXPR is -1 or 0 or 1, then maybe the
002944 ** "x" column is boolean or else -1 or 0 or 1 is a common default value
002945 ** on the "x" column and so in that case only cap the output row estimate
002946 ** at 1/2 instead of 1/4.
002947 */
002948 static void whereLoopOutputAdjust(
002949 WhereClause *pWC, /* The WHERE clause */
002950 WhereLoop *pLoop, /* The loop to adjust downward */
002951 LogEst nRow /* Number of rows in the entire table */
002952 ){
002953 WhereTerm *pTerm, *pX;
002954 Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf);
002955 int i, j;
002956 LogEst iReduce = 0; /* pLoop->nOut should not exceed nRow-iReduce */
002957
002958 assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 );
002959 for(i=pWC->nBase, pTerm=pWC->a; i>0; i--, pTerm++){
002960 assert( pTerm!=0 );
002961 if( (pTerm->prereqAll & notAllowed)!=0 ) continue;
002962 if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue;
002963 if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) continue;
002964 for(j=pLoop->nLTerm-1; j>=0; j--){
002965 pX = pLoop->aLTerm[j];
002966 if( pX==0 ) continue;
002967 if( pX==pTerm ) break;
002968 if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break;
002969 }
002970 if( j<0 ){
002971 sqlite3ProgressCheck(pWC->pWInfo->pParse);
002972 if( pLoop->maskSelf==pTerm->prereqAll ){
002973 /* If there are extra terms in the WHERE clause not used by an index
002974 ** that depend only on the table being scanned, and that will tend to
002975 ** cause many rows to be omitted, then mark that table as
002976 ** "self-culling".
002977 **
002978 ** 2022-03-24: Self-culling only applies if either the extra terms
002979 ** are straight comparison operators that are non-true with NULL
002980 ** operand, or if the loop is not an OUTER JOIN.
002981 */
002982 if( (pTerm->eOperator & 0x3f)!=0
002983 || (pWC->pWInfo->pTabList->a[pLoop->iTab].fg.jointype
002984 & (JT_LEFT|JT_LTORJ))==0
002985 ){
002986 pLoop->wsFlags |= WHERE_SELFCULL;
002987 }
002988 }
002989 if( pTerm->truthProb<=0 ){
002990 /* If a truth probability is specified using the likelihood() hints,
002991 ** then use the probability provided by the application. */
002992 pLoop->nOut += pTerm->truthProb;
002993 }else{
002994 /* In the absence of explicit truth probabilities, use heuristics to
002995 ** guess a reasonable truth probability. */
002996 pLoop->nOut--;
002997 if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0
002998 && (pTerm->wtFlags & TERM_HIGHTRUTH)==0 /* tag-20200224-1 */
002999 ){
003000 Expr *pRight = pTerm->pExpr->pRight;
003001 int k = 0;
003002 testcase( pTerm->pExpr->op==TK_IS );
003003 if( sqlite3ExprIsInteger(pRight, &k, 0) && k>=(-1) && k<=1 ){
003004 k = 10;
003005 }else{
003006 k = 20;
003007 }
003008 if( iReduce<k ){
003009 pTerm->wtFlags |= TERM_HEURTRUTH;
003010 iReduce = k;
003011 }
003012 }
003013 }
003014 }
003015 }
003016 if( pLoop->nOut > nRow-iReduce ){
003017 pLoop->nOut = nRow - iReduce;
003018 }
003019 }
003020
003021 /*
003022 ** Term pTerm is a vector range comparison operation. The first comparison
003023 ** in the vector can be optimized using column nEq of the index. This
003024 ** function returns the total number of vector elements that can be used
003025 ** as part of the range comparison.
003026 **
003027 ** For example, if the query is:
003028 **
003029 ** WHERE a = ? AND (b, c, d) > (?, ?, ?)
003030 **
003031 ** and the index:
003032 **
003033 ** CREATE INDEX ... ON (a, b, c, d, e)
003034 **
003035 ** then this function would be invoked with nEq=1. The value returned in
003036 ** this case is 3.
003037 */
003038 static int whereRangeVectorLen(
003039 Parse *pParse, /* Parsing context */
003040 int iCur, /* Cursor open on pIdx */
003041 Index *pIdx, /* The index to be used for a inequality constraint */
003042 int nEq, /* Number of prior equality constraints on same index */
003043 WhereTerm *pTerm /* The vector inequality constraint */
003044 ){
003045 int nCmp = sqlite3ExprVectorSize(pTerm->pExpr->pLeft);
003046 int i;
003047
003048 nCmp = MIN(nCmp, (pIdx->nColumn - nEq));
003049 for(i=1; i<nCmp; i++){
003050 /* Test if comparison i of pTerm is compatible with column (i+nEq)
003051 ** of the index. If not, exit the loop. */
003052 char aff; /* Comparison affinity */
003053 char idxaff = 0; /* Indexed columns affinity */
003054 CollSeq *pColl; /* Comparison collation sequence */
003055 Expr *pLhs, *pRhs;
003056
003057 assert( ExprUseXList(pTerm->pExpr->pLeft) );
003058 pLhs = pTerm->pExpr->pLeft->x.pList->a[i].pExpr;
003059 pRhs = pTerm->pExpr->pRight;
003060 if( ExprUseXSelect(pRhs) ){
003061 pRhs = pRhs->x.pSelect->pEList->a[i].pExpr;
003062 }else{
003063 pRhs = pRhs->x.pList->a[i].pExpr;
003064 }
003065
003066 /* Check that the LHS of the comparison is a column reference to
003067 ** the right column of the right source table. And that the sort
003068 ** order of the index column is the same as the sort order of the
003069 ** leftmost index column. */
003070 if( pLhs->op!=TK_COLUMN
003071 || pLhs->iTable!=iCur
003072 || pLhs->iColumn!=pIdx->aiColumn[i+nEq]
003073 || pIdx->aSortOrder[i+nEq]!=pIdx->aSortOrder[nEq]
003074 ){
003075 break;
003076 }
003077
003078 testcase( pLhs->iColumn==XN_ROWID );
003079 aff = sqlite3CompareAffinity(pRhs, sqlite3ExprAffinity(pLhs));
003080 idxaff = sqlite3TableColumnAffinity(pIdx->pTable, pLhs->iColumn);
003081 if( aff!=idxaff ) break;
003082
003083 pColl = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs);
003084 if( pColl==0 ) break;
003085 if( sqlite3StrICmp(pColl->zName, pIdx->azColl[i+nEq]) ) break;
003086 }
003087 return i;
003088 }
003089
003090 /*
003091 ** Adjust the cost C by the costMult factor T. This only occurs if
003092 ** compiled with -DSQLITE_ENABLE_COSTMULT
003093 */
003094 #ifdef SQLITE_ENABLE_COSTMULT
003095 # define ApplyCostMultiplier(C,T) C += T
003096 #else
003097 # define ApplyCostMultiplier(C,T)
003098 #endif
003099
003100 /*
003101 ** We have so far matched pBuilder->pNew->u.btree.nEq terms of the
003102 ** index pIndex. Try to match one more.
003103 **
003104 ** When this function is called, pBuilder->pNew->nOut contains the
003105 ** number of rows expected to be visited by filtering using the nEq
003106 ** terms only. If it is modified, this value is restored before this
003107 ** function returns.
003108 **
003109 ** If pProbe->idxType==SQLITE_IDXTYPE_IPK, that means pIndex is
003110 ** a fake index used for the INTEGER PRIMARY KEY.
003111 */
003112 static int whereLoopAddBtreeIndex(
003113 WhereLoopBuilder *pBuilder, /* The WhereLoop factory */
003114 SrcItem *pSrc, /* FROM clause term being analyzed */
003115 Index *pProbe, /* An index on pSrc */
003116 LogEst nInMul /* log(Number of iterations due to IN) */
003117 ){
003118 WhereInfo *pWInfo = pBuilder->pWInfo; /* WHERE analyze context */
003119 Parse *pParse = pWInfo->pParse; /* Parsing context */
003120 sqlite3 *db = pParse->db; /* Database connection malloc context */
003121 WhereLoop *pNew; /* Template WhereLoop under construction */
003122 WhereTerm *pTerm; /* A WhereTerm under consideration */
003123 int opMask; /* Valid operators for constraints */
003124 WhereScan scan; /* Iterator for WHERE terms */
003125 Bitmask saved_prereq; /* Original value of pNew->prereq */
003126 u16 saved_nLTerm; /* Original value of pNew->nLTerm */
003127 u16 saved_nEq; /* Original value of pNew->u.btree.nEq */
003128 u16 saved_nBtm; /* Original value of pNew->u.btree.nBtm */
003129 u16 saved_nTop; /* Original value of pNew->u.btree.nTop */
003130 u16 saved_nSkip; /* Original value of pNew->nSkip */
003131 u32 saved_wsFlags; /* Original value of pNew->wsFlags */
003132 LogEst saved_nOut; /* Original value of pNew->nOut */
003133 int rc = SQLITE_OK; /* Return code */
003134 LogEst rSize; /* Number of rows in the table */
003135 LogEst rLogSize; /* Logarithm of table size */
003136 WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */
003137
003138 pNew = pBuilder->pNew;
003139 assert( db->mallocFailed==0 || pParse->nErr>0 );
003140 if( pParse->nErr ){
003141 return pParse->rc;
003142 }
003143 WHERETRACE(0x800, ("BEGIN %s.addBtreeIdx(%s), nEq=%d, nSkip=%d, rRun=%d\n",
003144 pProbe->pTable->zName,pProbe->zName,
003145 pNew->u.btree.nEq, pNew->nSkip, pNew->rRun));
003146
003147 assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 );
003148 assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 );
003149 if( pNew->wsFlags & WHERE_BTM_LIMIT ){
003150 opMask = WO_LT|WO_LE;
003151 }else{
003152 assert( pNew->u.btree.nBtm==0 );
003153 opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS;
003154 }
003155 if( pProbe->bUnordered || pProbe->bLowQual ){
003156 if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE);
003157 if( pProbe->bLowQual && pSrc->fg.isIndexedBy==0 ){
003158 opMask &= ~(WO_EQ|WO_IN|WO_IS);
003159 }
003160 }
003161
003162 assert( pNew->u.btree.nEq<pProbe->nColumn );
003163 assert( pNew->u.btree.nEq<pProbe->nKeyCol
003164 || pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY );
003165
003166 saved_nEq = pNew->u.btree.nEq;
003167 saved_nBtm = pNew->u.btree.nBtm;
003168 saved_nTop = pNew->u.btree.nTop;
003169 saved_nSkip = pNew->nSkip;
003170 saved_nLTerm = pNew->nLTerm;
003171 saved_wsFlags = pNew->wsFlags;
003172 saved_prereq = pNew->prereq;
003173 saved_nOut = pNew->nOut;
003174 pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, saved_nEq,
003175 opMask, pProbe);
003176 pNew->rSetup = 0;
003177 rSize = pProbe->aiRowLogEst[0];
003178 rLogSize = estLog(rSize);
003179 for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){
003180 u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */
003181 LogEst rCostIdx;
003182 LogEst nOutUnadjusted; /* nOut before IN() and WHERE adjustments */
003183 int nIn = 0;
003184 #ifdef SQLITE_ENABLE_STAT4
003185 int nRecValid = pBuilder->nRecValid;
003186 #endif
003187 if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0)
003188 && indexColumnNotNull(pProbe, saved_nEq)
003189 ){
003190 continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */
003191 }
003192 if( pTerm->prereqRight & pNew->maskSelf ) continue;
003193
003194 /* Do not allow the upper bound of a LIKE optimization range constraint
003195 ** to mix with a lower range bound from some other source */
003196 if( pTerm->wtFlags & TERM_LIKEOPT && pTerm->eOperator==WO_LT ) continue;
003197
003198 if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0
003199 && !constraintCompatibleWithOuterJoin(pTerm,pSrc)
003200 ){
003201 continue;
003202 }
003203 if( IsUniqueIndex(pProbe) && saved_nEq==pProbe->nKeyCol-1 ){
003204 pBuilder->bldFlags1 |= SQLITE_BLDF1_UNIQUE;
003205 }else{
003206 pBuilder->bldFlags1 |= SQLITE_BLDF1_INDEXED;
003207 }
003208 pNew->wsFlags = saved_wsFlags;
003209 pNew->u.btree.nEq = saved_nEq;
003210 pNew->u.btree.nBtm = saved_nBtm;
003211 pNew->u.btree.nTop = saved_nTop;
003212 pNew->nLTerm = saved_nLTerm;
003213 if( pNew->nLTerm>=pNew->nLSlot
003214 && whereLoopResize(db, pNew, pNew->nLTerm+1)
003215 ){
003216 break; /* OOM while trying to enlarge the pNew->aLTerm array */
003217 }
003218 pNew->aLTerm[pNew->nLTerm++] = pTerm;
003219 pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf;
003220
003221 assert( nInMul==0
003222 || (pNew->wsFlags & WHERE_COLUMN_NULL)!=0
003223 || (pNew->wsFlags & WHERE_COLUMN_IN)!=0
003224 || (pNew->wsFlags & WHERE_SKIPSCAN)!=0
003225 );
003226
003227 if( eOp & WO_IN ){
003228 Expr *pExpr = pTerm->pExpr;
003229 if( ExprUseXSelect(pExpr) ){
003230 /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */
003231 int i;
003232 nIn = 46; assert( 46==sqlite3LogEst(25) );
003233
003234 /* The expression may actually be of the form (x, y) IN (SELECT...).
003235 ** In this case there is a separate term for each of (x) and (y).
003236 ** However, the nIn multiplier should only be applied once, not once
003237 ** for each such term. The following loop checks that pTerm is the
003238 ** first such term in use, and sets nIn back to 0 if it is not. */
003239 for(i=0; i<pNew->nLTerm-1; i++){
003240 if( pNew->aLTerm[i] && pNew->aLTerm[i]->pExpr==pExpr ) nIn = 0;
003241 }
003242 }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){
003243 /* "x IN (value, value, ...)" */
003244 nIn = sqlite3LogEst(pExpr->x.pList->nExpr);
003245 }
003246 if( pProbe->hasStat1 && rLogSize>=10 ){
003247 LogEst M, logK, x;
003248 /* Let:
003249 ** N = the total number of rows in the table
003250 ** K = the number of entries on the RHS of the IN operator
003251 ** M = the number of rows in the table that match terms to the
003252 ** to the left in the same index. If the IN operator is on
003253 ** the left-most index column, M==N.
003254 **
003255 ** Given the definitions above, it is better to omit the IN operator
003256 ** from the index lookup and instead do a scan of the M elements,
003257 ** testing each scanned row against the IN operator separately, if:
003258 **
003259 ** M*log(K) < K*log(N)
003260 **
003261 ** Our estimates for M, K, and N might be inaccurate, so we build in
003262 ** a safety margin of 2 (LogEst: 10) that favors using the IN operator
003263 ** with the index, as using an index has better worst-case behavior.
003264 ** If we do not have real sqlite_stat1 data, always prefer to use
003265 ** the index. Do not bother with this optimization on very small
003266 ** tables (less than 2 rows) as it is pointless in that case.
003267 */
003268 M = pProbe->aiRowLogEst[saved_nEq];
003269 logK = estLog(nIn);
003270 /* TUNING v----- 10 to bias toward indexed IN */
003271 x = M + logK + 10 - (nIn + rLogSize);
003272 if( x>=0 ){
003273 WHERETRACE(0x40,
003274 ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d) "
003275 "prefers indexed lookup\n",
003276 saved_nEq, M, logK, nIn, rLogSize, x));
003277 }else if( nInMul<2 && OptimizationEnabled(db, SQLITE_SeekScan) ){
003278 WHERETRACE(0x40,
003279 ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d"
003280 " nInMul=%d) prefers skip-scan\n",
003281 saved_nEq, M, logK, nIn, rLogSize, x, nInMul));
003282 pNew->wsFlags |= WHERE_IN_SEEKSCAN;
003283 }else{
003284 WHERETRACE(0x40,
003285 ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d"
003286 " nInMul=%d) prefers normal scan\n",
003287 saved_nEq, M, logK, nIn, rLogSize, x, nInMul));
003288 continue;
003289 }
003290 }
003291 pNew->wsFlags |= WHERE_COLUMN_IN;
003292 }else if( eOp & (WO_EQ|WO_IS) ){
003293 int iCol = pProbe->aiColumn[saved_nEq];
003294 pNew->wsFlags |= WHERE_COLUMN_EQ;
003295 assert( saved_nEq==pNew->u.btree.nEq );
003296 if( iCol==XN_ROWID
003297 || (iCol>=0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1)
003298 ){
003299 if( iCol==XN_ROWID || pProbe->uniqNotNull
003300 || (pProbe->nKeyCol==1 && pProbe->onError && (eOp & WO_EQ))
003301 ){
003302 pNew->wsFlags |= WHERE_ONEROW;
003303 }else{
003304 pNew->wsFlags |= WHERE_UNQ_WANTED;
003305 }
003306 }
003307 if( scan.iEquiv>1 ) pNew->wsFlags |= WHERE_TRANSCONS;
003308 }else if( eOp & WO_ISNULL ){
003309 pNew->wsFlags |= WHERE_COLUMN_NULL;
003310 }else{
003311 int nVecLen = whereRangeVectorLen(
003312 pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm
003313 );
003314 if( eOp & (WO_GT|WO_GE) ){
003315 testcase( eOp & WO_GT );
003316 testcase( eOp & WO_GE );
003317 pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT;
003318 pNew->u.btree.nBtm = nVecLen;
003319 pBtm = pTerm;
003320 pTop = 0;
003321 if( pTerm->wtFlags & TERM_LIKEOPT ){
003322 /* Range constraints that come from the LIKE optimization are
003323 ** always used in pairs. */
003324 pTop = &pTerm[1];
003325 assert( (pTop-(pTerm->pWC->a))<pTerm->pWC->nTerm );
003326 assert( pTop->wtFlags & TERM_LIKEOPT );
003327 assert( pTop->eOperator==WO_LT );
003328 if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */
003329 pNew->aLTerm[pNew->nLTerm++] = pTop;
003330 pNew->wsFlags |= WHERE_TOP_LIMIT;
003331 pNew->u.btree.nTop = 1;
003332 }
003333 }else{
003334 assert( eOp & (WO_LT|WO_LE) );
003335 testcase( eOp & WO_LT );
003336 testcase( eOp & WO_LE );
003337 pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT;
003338 pNew->u.btree.nTop = nVecLen;
003339 pTop = pTerm;
003340 pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ?
003341 pNew->aLTerm[pNew->nLTerm-2] : 0;
003342 }
003343 }
003344
003345 /* At this point pNew->nOut is set to the number of rows expected to
003346 ** be visited by the index scan before considering term pTerm, or the
003347 ** values of nIn and nInMul. In other words, assuming that all
003348 ** "x IN(...)" terms are replaced with "x = ?". This block updates
003349 ** the value of pNew->nOut to account for pTerm (but not nIn/nInMul). */
003350 assert( pNew->nOut==saved_nOut );
003351 if( pNew->wsFlags & WHERE_COLUMN_RANGE ){
003352 /* Adjust nOut using stat4 data. Or, if there is no stat4
003353 ** data, using some other estimate. */
003354 whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew);
003355 }else{
003356 int nEq = ++pNew->u.btree.nEq;
003357 assert( eOp & (WO_ISNULL|WO_EQ|WO_IN|WO_IS) );
003358
003359 assert( pNew->nOut==saved_nOut );
003360 if( pTerm->truthProb<=0 && pProbe->aiColumn[saved_nEq]>=0 ){
003361 assert( (eOp & WO_IN) || nIn==0 );
003362 testcase( eOp & WO_IN );
003363 pNew->nOut += pTerm->truthProb;
003364 pNew->nOut -= nIn;
003365 }else{
003366 #ifdef SQLITE_ENABLE_STAT4
003367 tRowcnt nOut = 0;
003368 if( nInMul==0
003369 && pProbe->nSample
003370 && ALWAYS(pNew->u.btree.nEq<=pProbe->nSampleCol)
003371 && ((eOp & WO_IN)==0 || ExprUseXList(pTerm->pExpr))
003372 && OptimizationEnabled(db, SQLITE_Stat4)
003373 ){
003374 Expr *pExpr = pTerm->pExpr;
003375 if( (eOp & (WO_EQ|WO_ISNULL|WO_IS))!=0 ){
003376 testcase( eOp & WO_EQ );
003377 testcase( eOp & WO_IS );
003378 testcase( eOp & WO_ISNULL );
003379 rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut);
003380 }else{
003381 rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut);
003382 }
003383 if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK;
003384 if( rc!=SQLITE_OK ) break; /* Jump out of the pTerm loop */
003385 if( nOut ){
003386 pNew->nOut = sqlite3LogEst(nOut);
003387 if( nEq==1
003388 /* TUNING: Mark terms as "low selectivity" if they seem likely
003389 ** to be true for half or more of the rows in the table.
003390 ** See tag-202002240-1 */
003391 && pNew->nOut+10 > pProbe->aiRowLogEst[0]
003392 ){
003393 #if WHERETRACE_ENABLED /* 0x01 */
003394 if( sqlite3WhereTrace & 0x20 ){
003395 sqlite3DebugPrintf(
003396 "STAT4 determines term has low selectivity:\n");
003397 sqlite3WhereTermPrint(pTerm, 999);
003398 }
003399 #endif
003400 pTerm->wtFlags |= TERM_HIGHTRUTH;
003401 if( pTerm->wtFlags & TERM_HEURTRUTH ){
003402 /* If the term has previously been used with an assumption of
003403 ** higher selectivity, then set the flag to rerun the
003404 ** loop computations. */
003405 pBuilder->bldFlags2 |= SQLITE_BLDF2_2NDPASS;
003406 }
003407 }
003408 if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut;
003409 pNew->nOut -= nIn;
003410 }
003411 }
003412 if( nOut==0 )
003413 #endif
003414 {
003415 pNew->nOut += (pProbe->aiRowLogEst[nEq] - pProbe->aiRowLogEst[nEq-1]);
003416 if( eOp & WO_ISNULL ){
003417 /* TUNING: If there is no likelihood() value, assume that a
003418 ** "col IS NULL" expression matches twice as many rows
003419 ** as (col=?). */
003420 pNew->nOut += 10;
003421 }
003422 }
003423 }
003424 }
003425
003426 /* Set rCostIdx to the estimated cost of visiting selected rows in the
003427 ** index. The estimate is the sum of two values:
003428 ** 1. The cost of doing one search-by-key to find the first matching
003429 ** entry
003430 ** 2. Stepping forward in the index pNew->nOut times to find all
003431 ** additional matching entries.
003432 */
003433 assert( pSrc->pSTab->szTabRow>0 );
003434 if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){
003435 /* The pProbe->szIdxRow is low for an IPK table since the interior
003436 ** pages are small. Thus szIdxRow gives a good estimate of seek cost.
003437 ** But the leaf pages are full-size, so pProbe->szIdxRow would badly
003438 ** under-estimate the scanning cost. */
003439 rCostIdx = pNew->nOut + 16;
003440 }else{
003441 rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pSTab->szTabRow;
003442 }
003443 rCostIdx = sqlite3LogEstAdd(rLogSize, rCostIdx);
003444
003445 /* Estimate the cost of running the loop. If all data is coming
003446 ** from the index, then this is just the cost of doing the index
003447 ** lookup and scan. But if some data is coming out of the main table,
003448 ** we also have to add in the cost of doing pNew->nOut searches to
003449 ** locate the row in the main table that corresponds to the index entry.
003450 */
003451 pNew->rRun = rCostIdx;
003452 if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK|WHERE_EXPRIDX))==0 ){
003453 pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16);
003454 }
003455 ApplyCostMultiplier(pNew->rRun, pProbe->pTable->costMult);
003456
003457 nOutUnadjusted = pNew->nOut;
003458 pNew->rRun += nInMul + nIn;
003459 pNew->nOut += nInMul + nIn;
003460 whereLoopOutputAdjust(pBuilder->pWC, pNew, rSize);
003461 rc = whereLoopInsert(pBuilder, pNew);
003462
003463 if( pNew->wsFlags & WHERE_COLUMN_RANGE ){
003464 pNew->nOut = saved_nOut;
003465 }else{
003466 pNew->nOut = nOutUnadjusted;
003467 }
003468
003469 if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0
003470 && pNew->u.btree.nEq<pProbe->nColumn
003471 && (pNew->u.btree.nEq<pProbe->nKeyCol ||
003472 pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY)
003473 ){
003474 if( pNew->u.btree.nEq>3 ){
003475 sqlite3ProgressCheck(pParse);
003476 }
003477 whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn);
003478 }
003479 pNew->nOut = saved_nOut;
003480 #ifdef SQLITE_ENABLE_STAT4
003481 pBuilder->nRecValid = nRecValid;
003482 #endif
003483 }
003484 pNew->prereq = saved_prereq;
003485 pNew->u.btree.nEq = saved_nEq;
003486 pNew->u.btree.nBtm = saved_nBtm;
003487 pNew->u.btree.nTop = saved_nTop;
003488 pNew->nSkip = saved_nSkip;
003489 pNew->wsFlags = saved_wsFlags;
003490 pNew->nOut = saved_nOut;
003491 pNew->nLTerm = saved_nLTerm;
003492
003493 /* Consider using a skip-scan if there are no WHERE clause constraints
003494 ** available for the left-most terms of the index, and if the average
003495 ** number of repeats in the left-most terms is at least 18.
003496 **
003497 ** The magic number 18 is selected on the basis that scanning 17 rows
003498 ** is almost always quicker than an index seek (even though if the index
003499 ** contains fewer than 2^17 rows we assume otherwise in other parts of
003500 ** the code). And, even if it is not, it should not be too much slower.
003501 ** On the other hand, the extra seeks could end up being significantly
003502 ** more expensive. */
003503 assert( 42==sqlite3LogEst(18) );
003504 if( saved_nEq==saved_nSkip
003505 && saved_nEq+1<pProbe->nKeyCol
003506 && saved_nEq==pNew->nLTerm
003507 && pProbe->noSkipScan==0
003508 && pProbe->hasStat1!=0
003509 && OptimizationEnabled(db, SQLITE_SkipScan)
003510 && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */
003511 && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK
003512 ){
003513 LogEst nIter;
003514 pNew->u.btree.nEq++;
003515 pNew->nSkip++;
003516 pNew->aLTerm[pNew->nLTerm++] = 0;
003517 pNew->wsFlags |= WHERE_SKIPSCAN;
003518 nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1];
003519 pNew->nOut -= nIter;
003520 /* TUNING: Because uncertainties in the estimates for skip-scan queries,
003521 ** add a 1.375 fudge factor to make skip-scan slightly less likely. */
003522 nIter += 5;
003523 whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul);
003524 pNew->nOut = saved_nOut;
003525 pNew->u.btree.nEq = saved_nEq;
003526 pNew->nSkip = saved_nSkip;
003527 pNew->wsFlags = saved_wsFlags;
003528 }
003529
003530 WHERETRACE(0x800, ("END %s.addBtreeIdx(%s), nEq=%d, rc=%d\n",
003531 pProbe->pTable->zName, pProbe->zName, saved_nEq, rc));
003532 return rc;
003533 }
003534
003535 /*
003536 ** Return True if it is possible that pIndex might be useful in
003537 ** implementing the ORDER BY clause in pBuilder.
003538 **
003539 ** Return False if pBuilder does not contain an ORDER BY clause or
003540 ** if there is no way for pIndex to be useful in implementing that
003541 ** ORDER BY clause.
003542 */
003543 static int indexMightHelpWithOrderBy(
003544 WhereLoopBuilder *pBuilder,
003545 Index *pIndex,
003546 int iCursor
003547 ){
003548 ExprList *pOB;
003549 ExprList *aColExpr;
003550 int ii, jj;
003551
003552 if( pIndex->bUnordered ) return 0;
003553 if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0;
003554 for(ii=0; ii<pOB->nExpr; ii++){
003555 Expr *pExpr = sqlite3ExprSkipCollateAndLikely(pOB->a[ii].pExpr);
003556 if( NEVER(pExpr==0) ) continue;
003557 if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN)
003558 && pExpr->iTable==iCursor
003559 ){
003560 if( pExpr->iColumn<0 ) return 1;
003561 for(jj=0; jj<pIndex->nKeyCol; jj++){
003562 if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1;
003563 }
003564 }else if( (aColExpr = pIndex->aColExpr)!=0 ){
003565 for(jj=0; jj<pIndex->nKeyCol; jj++){
003566 if( pIndex->aiColumn[jj]!=XN_EXPR ) continue;
003567 if( sqlite3ExprCompareSkip(pExpr,aColExpr->a[jj].pExpr,iCursor)==0 ){
003568 return 1;
003569 }
003570 }
003571 }
003572 }
003573 return 0;
003574 }
003575
003576 /* Check to see if a partial index with pPartIndexWhere can be used
003577 ** in the current query. Return true if it can be and false if not.
003578 */
003579 static int whereUsablePartialIndex(
003580 int iTab, /* The table for which we want an index */
003581 u8 jointype, /* The JT_* flags on the join */
003582 WhereClause *pWC, /* The WHERE clause of the query */
003583 Expr *pWhere /* The WHERE clause from the partial index */
003584 ){
003585 int i;
003586 WhereTerm *pTerm;
003587 Parse *pParse;
003588
003589 if( jointype & JT_LTORJ ) return 0;
003590 pParse = pWC->pWInfo->pParse;
003591 while( pWhere->op==TK_AND ){
003592 if( !whereUsablePartialIndex(iTab,jointype,pWC,pWhere->pLeft) ) return 0;
003593 pWhere = pWhere->pRight;
003594 }
003595 for(i=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
003596 Expr *pExpr;
003597 pExpr = pTerm->pExpr;
003598 if( (!ExprHasProperty(pExpr, EP_OuterON) || pExpr->w.iJoin==iTab)
003599 && ((jointype & JT_OUTER)==0 || ExprHasProperty(pExpr, EP_OuterON))
003600 && sqlite3ExprImpliesExpr(pParse, pExpr, pWhere, iTab)
003601 && (pTerm->wtFlags & TERM_VNULL)==0
003602 ){
003603 return 1;
003604 }
003605 }
003606 return 0;
003607 }
003608
003609 /*
003610 ** pIdx is an index containing expressions. Check it see if any of the
003611 ** expressions in the index match the pExpr expression.
003612 */
003613 static int exprIsCoveredByIndex(
003614 const Expr *pExpr,
003615 const Index *pIdx,
003616 int iTabCur
003617 ){
003618 int i;
003619 for(i=0; i<pIdx->nColumn; i++){
003620 if( pIdx->aiColumn[i]==XN_EXPR
003621 && sqlite3ExprCompare(0, pExpr, pIdx->aColExpr->a[i].pExpr, iTabCur)==0
003622 ){
003623 return 1;
003624 }
003625 }
003626 return 0;
003627 }
003628
003629 /*
003630 ** Structure passed to the whereIsCoveringIndex Walker callback.
003631 */
003632 typedef struct CoveringIndexCheck CoveringIndexCheck;
003633 struct CoveringIndexCheck {
003634 Index *pIdx; /* The index */
003635 int iTabCur; /* Cursor number for the corresponding table */
003636 u8 bExpr; /* Uses an indexed expression */
003637 u8 bUnidx; /* Uses an unindexed column not within an indexed expr */
003638 };
003639
003640 /*
003641 ** Information passed in is pWalk->u.pCovIdxCk. Call it pCk.
003642 **
003643 ** If the Expr node references the table with cursor pCk->iTabCur, then
003644 ** make sure that column is covered by the index pCk->pIdx. We know that
003645 ** all columns less than 63 (really BMS-1) are covered, so we don't need
003646 ** to check them. But we do need to check any column at 63 or greater.
003647 **
003648 ** If the index does not cover the column, then set pWalk->eCode to
003649 ** non-zero and return WRC_Abort to stop the search.
003650 **
003651 ** If this node does not disprove that the index can be a covering index,
003652 ** then just return WRC_Continue, to continue the search.
003653 **
003654 ** If pCk->pIdx contains indexed expressions and one of those expressions
003655 ** matches pExpr, then prune the search.
003656 */
003657 static int whereIsCoveringIndexWalkCallback(Walker *pWalk, Expr *pExpr){
003658 int i; /* Loop counter */
003659 const Index *pIdx; /* The index of interest */
003660 const i16 *aiColumn; /* Columns contained in the index */
003661 u16 nColumn; /* Number of columns in the index */
003662 CoveringIndexCheck *pCk; /* Info about this search */
003663
003664 pCk = pWalk->u.pCovIdxCk;
003665 pIdx = pCk->pIdx;
003666 if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN) ){
003667 /* if( pExpr->iColumn<(BMS-1) && pIdx->bHasExpr==0 ) return WRC_Continue;*/
003668 if( pExpr->iTable!=pCk->iTabCur ) return WRC_Continue;
003669 pIdx = pWalk->u.pCovIdxCk->pIdx;
003670 aiColumn = pIdx->aiColumn;
003671 nColumn = pIdx->nColumn;
003672 for(i=0; i<nColumn; i++){
003673 if( aiColumn[i]==pExpr->iColumn ) return WRC_Continue;
003674 }
003675 pCk->bUnidx = 1;
003676 return WRC_Abort;
003677 }else if( pIdx->bHasExpr
003678 && exprIsCoveredByIndex(pExpr, pIdx, pWalk->u.pCovIdxCk->iTabCur) ){
003679 pCk->bExpr = 1;
003680 return WRC_Prune;
003681 }
003682 return WRC_Continue;
003683 }
003684
003685
003686 /*
003687 ** pIdx is an index that covers all of the low-number columns used by
003688 ** pWInfo->pSelect (columns from 0 through 62) or an index that has
003689 ** expressions terms. Hence, we cannot determine whether or not it is
003690 ** a covering index by using the colUsed bitmasks. We have to do a search
003691 ** to see if the index is covering. This routine does that search.
003692 **
003693 ** The return value is one of these:
003694 **
003695 ** 0 The index is definitely not a covering index
003696 **
003697 ** WHERE_IDX_ONLY The index is definitely a covering index
003698 **
003699 ** WHERE_EXPRIDX The index is likely a covering index, but it is
003700 ** difficult to determine precisely because of the
003701 ** expressions that are indexed. Score it as a
003702 ** covering index, but still keep the main table open
003703 ** just in case we need it.
003704 **
003705 ** This routine is an optimization. It is always safe to return zero.
003706 ** But returning one of the other two values when zero should have been
003707 ** returned can lead to incorrect bytecode and assertion faults.
003708 */
003709 static SQLITE_NOINLINE u32 whereIsCoveringIndex(
003710 WhereInfo *pWInfo, /* The WHERE clause context */
003711 Index *pIdx, /* Index that is being tested */
003712 int iTabCur /* Cursor for the table being indexed */
003713 ){
003714 int i, rc;
003715 struct CoveringIndexCheck ck;
003716 Walker w;
003717 if( pWInfo->pSelect==0 ){
003718 /* We don't have access to the full query, so we cannot check to see
003719 ** if pIdx is covering. Assume it is not. */
003720 return 0;
003721 }
003722 if( pIdx->bHasExpr==0 ){
003723 for(i=0; i<pIdx->nColumn; i++){
003724 if( pIdx->aiColumn[i]>=BMS-1 ) break;
003725 }
003726 if( i>=pIdx->nColumn ){
003727 /* pIdx does not index any columns greater than 62, but we know from
003728 ** colMask that columns greater than 62 are used, so this is not a
003729 ** covering index */
003730 return 0;
003731 }
003732 }
003733 ck.pIdx = pIdx;
003734 ck.iTabCur = iTabCur;
003735 ck.bExpr = 0;
003736 ck.bUnidx = 0;
003737 memset(&w, 0, sizeof(w));
003738 w.xExprCallback = whereIsCoveringIndexWalkCallback;
003739 w.xSelectCallback = sqlite3SelectWalkNoop;
003740 w.u.pCovIdxCk = &ck;
003741 sqlite3WalkSelect(&w, pWInfo->pSelect);
003742 if( ck.bUnidx ){
003743 rc = 0;
003744 }else if( ck.bExpr ){
003745 rc = WHERE_EXPRIDX;
003746 }else{
003747 rc = WHERE_IDX_ONLY;
003748 }
003749 return rc;
003750 }
003751
003752 /*
003753 ** This is an sqlite3ParserAddCleanup() callback that is invoked to
003754 ** free the Parse->pIdxEpr list when the Parse object is destroyed.
003755 */
003756 static void whereIndexedExprCleanup(sqlite3 *db, void *pObject){
003757 IndexedExpr **pp = (IndexedExpr**)pObject;
003758 while( *pp!=0 ){
003759 IndexedExpr *p = *pp;
003760 *pp = p->pIENext;
003761 sqlite3ExprDelete(db, p->pExpr);
003762 sqlite3DbFreeNN(db, p);
003763 }
003764 }
003765
003766 /*
003767 ** This function is called for a partial index - one with a WHERE clause - in
003768 ** two scenarios. In both cases, it determines whether or not the WHERE
003769 ** clause on the index implies that a column of the table may be safely
003770 ** replaced by a constant expression. For example, in the following
003771 ** SELECT:
003772 **
003773 ** CREATE INDEX i1 ON t1(b, c) WHERE a=<expr>;
003774 ** SELECT a, b, c FROM t1 WHERE a=<expr> AND b=?;
003775 **
003776 ** The "a" in the select-list may be replaced by <expr>, iff:
003777 **
003778 ** (a) <expr> is a constant expression, and
003779 ** (b) The (a=<expr>) comparison uses the BINARY collation sequence, and
003780 ** (c) Column "a" has an affinity other than NONE or BLOB.
003781 **
003782 ** If argument pItem is NULL, then pMask must not be NULL. In this case this
003783 ** function is being called as part of determining whether or not pIdx
003784 ** is a covering index. This function clears any bits in (*pMask)
003785 ** corresponding to columns that may be replaced by constants as described
003786 ** above.
003787 **
003788 ** Otherwise, if pItem is not NULL, then this function is being called
003789 ** as part of coding a loop that uses index pIdx. In this case, add entries
003790 ** to the Parse.pIdxPartExpr list for each column that can be replaced
003791 ** by a constant.
003792 */
003793 static void wherePartIdxExpr(
003794 Parse *pParse, /* Parse context */
003795 Index *pIdx, /* Partial index being processed */
003796 Expr *pPart, /* WHERE clause being processed */
003797 Bitmask *pMask, /* Mask to clear bits in */
003798 int iIdxCur, /* Cursor number for index */
003799 SrcItem *pItem /* The FROM clause entry for the table */
003800 ){
003801 assert( pItem==0 || (pItem->fg.jointype & JT_RIGHT)==0 );
003802 assert( (pItem==0 || pMask==0) && (pMask!=0 || pItem!=0) );
003803
003804 if( pPart->op==TK_AND ){
003805 wherePartIdxExpr(pParse, pIdx, pPart->pRight, pMask, iIdxCur, pItem);
003806 pPart = pPart->pLeft;
003807 }
003808
003809 if( (pPart->op==TK_EQ || pPart->op==TK_IS) ){
003810 Expr *pLeft = pPart->pLeft;
003811 Expr *pRight = pPart->pRight;
003812 u8 aff;
003813
003814 if( pLeft->op!=TK_COLUMN ) return;
003815 if( !sqlite3ExprIsConstant(0, pRight) ) return;
003816 if( !sqlite3IsBinary(sqlite3ExprCompareCollSeq(pParse, pPart)) ) return;
003817 if( pLeft->iColumn<0 ) return;
003818 aff = pIdx->pTable->aCol[pLeft->iColumn].affinity;
003819 if( aff>=SQLITE_AFF_TEXT ){
003820 if( pItem ){
003821 sqlite3 *db = pParse->db;
003822 IndexedExpr *p = (IndexedExpr*)sqlite3DbMallocRaw(db, sizeof(*p));
003823 if( p ){
003824 int bNullRow = (pItem->fg.jointype&(JT_LEFT|JT_LTORJ))!=0;
003825 p->pExpr = sqlite3ExprDup(db, pRight, 0);
003826 p->iDataCur = pItem->iCursor;
003827 p->iIdxCur = iIdxCur;
003828 p->iIdxCol = pLeft->iColumn;
003829 p->bMaybeNullRow = bNullRow;
003830 p->pIENext = pParse->pIdxPartExpr;
003831 p->aff = aff;
003832 pParse->pIdxPartExpr = p;
003833 if( p->pIENext==0 ){
003834 void *pArg = (void*)&pParse->pIdxPartExpr;
003835 sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pArg);
003836 }
003837 }
003838 }else if( pLeft->iColumn<(BMS-1) ){
003839 *pMask &= ~((Bitmask)1 << pLeft->iColumn);
003840 }
003841 }
003842 }
003843 }
003844
003845
003846 /*
003847 ** Add all WhereLoop objects for a single table of the join where the table
003848 ** is identified by pBuilder->pNew->iTab. That table is guaranteed to be
003849 ** a b-tree table, not a virtual table.
003850 **
003851 ** The costs (WhereLoop.rRun) of the b-tree loops added by this function
003852 ** are calculated as follows:
003853 **
003854 ** For a full scan, assuming the table (or index) contains nRow rows:
003855 **
003856 ** cost = nRow * 3.0 // full-table scan
003857 ** cost = nRow * K // scan of covering index
003858 ** cost = nRow * (K+3.0) // scan of non-covering index
003859 **
003860 ** where K is a value between 1.1 and 3.0 set based on the relative
003861 ** estimated average size of the index and table records.
003862 **
003863 ** For an index scan, where nVisit is the number of index rows visited
003864 ** by the scan, and nSeek is the number of seek operations required on
003865 ** the index b-tree:
003866 **
003867 ** cost = nSeek * (log(nRow) + K * nVisit) // covering index
003868 ** cost = nSeek * (log(nRow) + (K+3.0) * nVisit) // non-covering index
003869 **
003870 ** Normally, nSeek is 1. nSeek values greater than 1 come about if the
003871 ** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when
003872 ** implicit "x IN (SELECT x FROM tbl)" terms are added for skip-scans.
003873 **
003874 ** The estimated values (nRow, nVisit, nSeek) often contain a large amount
003875 ** of uncertainty. For this reason, scoring is designed to pick plans that
003876 ** "do the least harm" if the estimates are inaccurate. For example, a
003877 ** log(nRow) factor is omitted from a non-covering index scan in order to
003878 ** bias the scoring in favor of using an index, since the worst-case
003879 ** performance of using an index is far better than the worst-case performance
003880 ** of a full table scan.
003881 */
003882 static int whereLoopAddBtree(
003883 WhereLoopBuilder *pBuilder, /* WHERE clause information */
003884 Bitmask mPrereq /* Extra prerequisites for using this table */
003885 ){
003886 WhereInfo *pWInfo; /* WHERE analysis context */
003887 Index *pProbe; /* An index we are evaluating */
003888 Index sPk; /* A fake index object for the primary key */
003889 LogEst aiRowEstPk[2]; /* The aiRowLogEst[] value for the sPk index */
003890 i16 aiColumnPk = -1; /* The aColumn[] value for the sPk index */
003891 SrcList *pTabList; /* The FROM clause */
003892 SrcItem *pSrc; /* The FROM clause btree term to add */
003893 WhereLoop *pNew; /* Template WhereLoop object */
003894 int rc = SQLITE_OK; /* Return code */
003895 int iSortIdx = 1; /* Index number */
003896 int b; /* A boolean value */
003897 LogEst rSize; /* number of rows in the table */
003898 WhereClause *pWC; /* The parsed WHERE clause */
003899 Table *pTab; /* Table being queried */
003900
003901 pNew = pBuilder->pNew;
003902 pWInfo = pBuilder->pWInfo;
003903 pTabList = pWInfo->pTabList;
003904 pSrc = pTabList->a + pNew->iTab;
003905 pTab = pSrc->pSTab;
003906 pWC = pBuilder->pWC;
003907 assert( !IsVirtual(pSrc->pSTab) );
003908
003909 if( pSrc->fg.isIndexedBy ){
003910 assert( pSrc->fg.isCte==0 );
003911 /* An INDEXED BY clause specifies a particular index to use */
003912 pProbe = pSrc->u2.pIBIndex;
003913 }else if( !HasRowid(pTab) ){
003914 pProbe = pTab->pIndex;
003915 }else{
003916 /* There is no INDEXED BY clause. Create a fake Index object in local
003917 ** variable sPk to represent the rowid primary key index. Make this
003918 ** fake index the first in a chain of Index objects with all of the real
003919 ** indices to follow */
003920 Index *pFirst; /* First of real indices on the table */
003921 memset(&sPk, 0, sizeof(Index));
003922 sPk.nKeyCol = 1;
003923 sPk.nColumn = 1;
003924 sPk.aiColumn = &aiColumnPk;
003925 sPk.aiRowLogEst = aiRowEstPk;
003926 sPk.onError = OE_Replace;
003927 sPk.pTable = pTab;
003928 sPk.szIdxRow = 3; /* TUNING: Interior rows of IPK table are very small */
003929 sPk.idxType = SQLITE_IDXTYPE_IPK;
003930 aiRowEstPk[0] = pTab->nRowLogEst;
003931 aiRowEstPk[1] = 0;
003932 pFirst = pSrc->pSTab->pIndex;
003933 if( pSrc->fg.notIndexed==0 ){
003934 /* The real indices of the table are only considered if the
003935 ** NOT INDEXED qualifier is omitted from the FROM clause */
003936 sPk.pNext = pFirst;
003937 }
003938 pProbe = &sPk;
003939 }
003940 rSize = pTab->nRowLogEst;
003941
003942 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
003943 /* Automatic indexes */
003944 if( !pBuilder->pOrSet /* Not part of an OR optimization */
003945 && (pWInfo->wctrlFlags & (WHERE_RIGHT_JOIN|WHERE_OR_SUBCLAUSE))==0
003946 && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0
003947 && !pSrc->fg.isIndexedBy /* Has no INDEXED BY clause */
003948 && !pSrc->fg.notIndexed /* Has no NOT INDEXED clause */
003949 && !pSrc->fg.isCorrelated /* Not a correlated subquery */
003950 && !pSrc->fg.isRecursive /* Not a recursive common table expression. */
003951 && (pSrc->fg.jointype & JT_RIGHT)==0 /* Not the right tab of a RIGHT JOIN */
003952 ){
003953 /* Generate auto-index WhereLoops */
003954 LogEst rLogSize; /* Logarithm of the number of rows in the table */
003955 WhereTerm *pTerm;
003956 WhereTerm *pWCEnd = pWC->a + pWC->nTerm;
003957 rLogSize = estLog(rSize);
003958 for(pTerm=pWC->a; rc==SQLITE_OK && pTerm<pWCEnd; pTerm++){
003959 if( pTerm->prereqRight & pNew->maskSelf ) continue;
003960 if( termCanDriveIndex(pTerm, pSrc, 0) ){
003961 pNew->u.btree.nEq = 1;
003962 pNew->nSkip = 0;
003963 pNew->u.btree.pIndex = 0;
003964 pNew->nLTerm = 1;
003965 pNew->aLTerm[0] = pTerm;
003966 /* TUNING: One-time cost for computing the automatic index is
003967 ** estimated to be X*N*log2(N) where N is the number of rows in
003968 ** the table being indexed and where X is 7 (LogEst=28) for normal
003969 ** tables or 0.5 (LogEst=-10) for views and subqueries. The value
003970 ** of X is smaller for views and subqueries so that the query planner
003971 ** will be more aggressive about generating automatic indexes for
003972 ** those objects, since there is no opportunity to add schema
003973 ** indexes on subqueries and views. */
003974 pNew->rSetup = rLogSize + rSize;
003975 if( !IsView(pTab) && (pTab->tabFlags & TF_Ephemeral)==0 ){
003976 pNew->rSetup += 28;
003977 }else{
003978 pNew->rSetup -= 25; /* Greatly reduced setup cost for auto indexes
003979 ** on ephemeral materializations of views */
003980 }
003981 ApplyCostMultiplier(pNew->rSetup, pTab->costMult);
003982 if( pNew->rSetup<0 ) pNew->rSetup = 0;
003983 /* TUNING: Each index lookup yields 20 rows in the table. This
003984 ** is more than the usual guess of 10 rows, since we have no way
003985 ** of knowing how selective the index will ultimately be. It would
003986 ** not be unreasonable to make this value much larger. */
003987 pNew->nOut = 43; assert( 43==sqlite3LogEst(20) );
003988 pNew->rRun = sqlite3LogEstAdd(rLogSize,pNew->nOut);
003989 pNew->wsFlags = WHERE_AUTO_INDEX;
003990 pNew->prereq = mPrereq | pTerm->prereqRight;
003991 rc = whereLoopInsert(pBuilder, pNew);
003992 }
003993 }
003994 }
003995 #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
003996
003997 /* Loop over all indices. If there was an INDEXED BY clause, then only
003998 ** consider index pProbe. */
003999 for(; rc==SQLITE_OK && pProbe;
004000 pProbe=(pSrc->fg.isIndexedBy ? 0 : pProbe->pNext), iSortIdx++
004001 ){
004002 if( pProbe->pPartIdxWhere!=0
004003 && !whereUsablePartialIndex(pSrc->iCursor, pSrc->fg.jointype, pWC,
004004 pProbe->pPartIdxWhere)
004005 ){
004006 testcase( pNew->iTab!=pSrc->iCursor ); /* See ticket [98d973b8f5] */
004007 continue; /* Partial index inappropriate for this query */
004008 }
004009 if( pProbe->bNoQuery ) continue;
004010 rSize = pProbe->aiRowLogEst[0];
004011 pNew->u.btree.nEq = 0;
004012 pNew->u.btree.nBtm = 0;
004013 pNew->u.btree.nTop = 0;
004014 pNew->nSkip = 0;
004015 pNew->nLTerm = 0;
004016 pNew->iSortIdx = 0;
004017 pNew->rSetup = 0;
004018 pNew->prereq = mPrereq;
004019 pNew->nOut = rSize;
004020 pNew->u.btree.pIndex = pProbe;
004021 pNew->u.btree.pOrderBy = 0;
004022 b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor);
004023
004024 /* The ONEPASS_DESIRED flags never occurs together with ORDER BY */
004025 assert( (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || b==0 );
004026 if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){
004027 /* Integer primary key index */
004028 pNew->wsFlags = WHERE_IPK;
004029
004030 /* Full table scan */
004031 pNew->iSortIdx = b ? iSortIdx : 0;
004032 /* TUNING: Cost of full table scan is 3.0*N. The 3.0 factor is an
004033 ** extra cost designed to discourage the use of full table scans,
004034 ** since index lookups have better worst-case performance if our
004035 ** stat guesses are wrong. Reduce the 3.0 penalty slightly
004036 ** (to 2.75) if we have valid STAT4 information for the table.
004037 ** At 2.75, a full table scan is preferred over using an index on
004038 ** a column with just two distinct values where each value has about
004039 ** an equal number of appearances. Without STAT4 data, we still want
004040 ** to use an index in that case, since the constraint might be for
004041 ** the scarcer of the two values, and in that case an index lookup is
004042 ** better.
004043 */
004044 #ifdef SQLITE_ENABLE_STAT4
004045 pNew->rRun = rSize + 16 - 2*((pTab->tabFlags & TF_HasStat4)!=0);
004046 #else
004047 pNew->rRun = rSize + 16;
004048 #endif
004049 ApplyCostMultiplier(pNew->rRun, pTab->costMult);
004050 whereLoopOutputAdjust(pWC, pNew, rSize);
004051 if( pSrc->fg.isSubquery ){
004052 if( pSrc->fg.viaCoroutine ) pNew->wsFlags |= WHERE_COROUTINE;
004053 pNew->u.btree.pOrderBy = pSrc->u4.pSubq->pSelect->pOrderBy;
004054 }
004055 rc = whereLoopInsert(pBuilder, pNew);
004056 pNew->nOut = rSize;
004057 if( rc ) break;
004058 }else{
004059 Bitmask m;
004060 if( pProbe->isCovering ){
004061 m = 0;
004062 pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED;
004063 }else{
004064 m = pSrc->colUsed & pProbe->colNotIdxed;
004065 if( pProbe->pPartIdxWhere ){
004066 wherePartIdxExpr(
004067 pWInfo->pParse, pProbe, pProbe->pPartIdxWhere, &m, 0, 0
004068 );
004069 }
004070 pNew->wsFlags = WHERE_INDEXED;
004071 if( m==TOPBIT || (pProbe->bHasExpr && !pProbe->bHasVCol && m!=0) ){
004072 u32 isCov = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor);
004073 if( isCov==0 ){
004074 WHERETRACE(0x200,
004075 ("-> %s is not a covering index"
004076 " according to whereIsCoveringIndex()\n", pProbe->zName));
004077 assert( m!=0 );
004078 }else{
004079 m = 0;
004080 pNew->wsFlags |= isCov;
004081 if( isCov & WHERE_IDX_ONLY ){
004082 WHERETRACE(0x200,
004083 ("-> %s is a covering expression index"
004084 " according to whereIsCoveringIndex()\n", pProbe->zName));
004085 }else{
004086 assert( isCov==WHERE_EXPRIDX );
004087 WHERETRACE(0x200,
004088 ("-> %s might be a covering expression index"
004089 " according to whereIsCoveringIndex()\n", pProbe->zName));
004090 }
004091 }
004092 }else if( m==0
004093 && (HasRowid(pTab) || pWInfo->pSelect!=0 || sqlite3FaultSim(700))
004094 ){
004095 WHERETRACE(0x200,
004096 ("-> %s a covering index according to bitmasks\n",
004097 pProbe->zName, m==0 ? "is" : "is not"));
004098 pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED;
004099 }
004100 }
004101
004102 /* Full scan via index */
004103 if( b
004104 || !HasRowid(pTab)
004105 || pProbe->pPartIdxWhere!=0
004106 || pSrc->fg.isIndexedBy
004107 || ( m==0
004108 && pProbe->bUnordered==0
004109 && (pProbe->szIdxRow<pTab->szTabRow)
004110 && (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0
004111 && sqlite3GlobalConfig.bUseCis
004112 && OptimizationEnabled(pWInfo->pParse->db, SQLITE_CoverIdxScan)
004113 )
004114 ){
004115 pNew->iSortIdx = b ? iSortIdx : 0;
004116
004117 /* The cost of visiting the index rows is N*K, where K is
004118 ** between 1.1 and 3.0, depending on the relative sizes of the
004119 ** index and table rows. */
004120 pNew->rRun = rSize + 1 + (15*pProbe->szIdxRow)/pTab->szTabRow;
004121 if( m!=0 ){
004122 /* If this is a non-covering index scan, add in the cost of
004123 ** doing table lookups. The cost will be 3x the number of
004124 ** lookups. Take into account WHERE clause terms that can be
004125 ** satisfied using just the index, and that do not require a
004126 ** table lookup. */
004127 LogEst nLookup = rSize + 16; /* Base cost: N*3 */
004128 int ii;
004129 int iCur = pSrc->iCursor;
004130 WhereClause *pWC2 = &pWInfo->sWC;
004131 for(ii=0; ii<pWC2->nTerm; ii++){
004132 WhereTerm *pTerm = &pWC2->a[ii];
004133 if( !sqlite3ExprCoveredByIndex(pTerm->pExpr, iCur, pProbe) ){
004134 break;
004135 }
004136 /* pTerm can be evaluated using just the index. So reduce
004137 ** the expected number of table lookups accordingly */
004138 if( pTerm->truthProb<=0 ){
004139 nLookup += pTerm->truthProb;
004140 }else{
004141 nLookup--;
004142 if( pTerm->eOperator & (WO_EQ|WO_IS) ) nLookup -= 19;
004143 }
004144 }
004145
004146 pNew->rRun = sqlite3LogEstAdd(pNew->rRun, nLookup);
004147 }
004148 ApplyCostMultiplier(pNew->rRun, pTab->costMult);
004149 whereLoopOutputAdjust(pWC, pNew, rSize);
004150 if( (pSrc->fg.jointype & JT_RIGHT)!=0 && pProbe->aColExpr ){
004151 /* Do not do an SCAN of a index-on-expression in a RIGHT JOIN
004152 ** because the cursor used to access the index might not be
004153 ** positioned to the correct row during the right-join no-match
004154 ** loop. */
004155 }else{
004156 rc = whereLoopInsert(pBuilder, pNew);
004157 }
004158 pNew->nOut = rSize;
004159 if( rc ) break;
004160 }
004161 }
004162
004163 pBuilder->bldFlags1 = 0;
004164 rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0);
004165 if( pBuilder->bldFlags1==SQLITE_BLDF1_INDEXED ){
004166 /* If a non-unique index is used, or if a prefix of the key for
004167 ** unique index is used (making the index functionally non-unique)
004168 ** then the sqlite_stat1 data becomes important for scoring the
004169 ** plan */
004170 pTab->tabFlags |= TF_MaybeReanalyze;
004171 }
004172 #ifdef SQLITE_ENABLE_STAT4
004173 sqlite3Stat4ProbeFree(pBuilder->pRec);
004174 pBuilder->nRecValid = 0;
004175 pBuilder->pRec = 0;
004176 #endif
004177 }
004178 return rc;
004179 }
004180
004181 #ifndef SQLITE_OMIT_VIRTUALTABLE
004182
004183 /*
004184 ** Return true if pTerm is a virtual table LIMIT or OFFSET term.
004185 */
004186 static int isLimitTerm(WhereTerm *pTerm){
004187 assert( pTerm->eOperator==WO_AUX || pTerm->eMatchOp==0 );
004188 return pTerm->eMatchOp>=SQLITE_INDEX_CONSTRAINT_LIMIT
004189 && pTerm->eMatchOp<=SQLITE_INDEX_CONSTRAINT_OFFSET;
004190 }
004191
004192 /*
004193 ** Return true if the first nCons constraints in the pUsage array are
004194 ** marked as in-use (have argvIndex>0). False otherwise.
004195 */
004196 static int allConstraintsUsed(
004197 struct sqlite3_index_constraint_usage *aUsage,
004198 int nCons
004199 ){
004200 int ii;
004201 for(ii=0; ii<nCons; ii++){
004202 if( aUsage[ii].argvIndex<=0 ) return 0;
004203 }
004204 return 1;
004205 }
004206
004207 /*
004208 ** Argument pIdxInfo is already populated with all constraints that may
004209 ** be used by the virtual table identified by pBuilder->pNew->iTab. This
004210 ** function marks a subset of those constraints usable, invokes the
004211 ** xBestIndex method and adds the returned plan to pBuilder.
004212 **
004213 ** A constraint is marked usable if:
004214 **
004215 ** * Argument mUsable indicates that its prerequisites are available, and
004216 **
004217 ** * It is not one of the operators specified in the mExclude mask passed
004218 ** as the fourth argument (which in practice is either WO_IN or 0).
004219 **
004220 ** Argument mPrereq is a mask of tables that must be scanned before the
004221 ** virtual table in question. These are added to the plans prerequisites
004222 ** before it is added to pBuilder.
004223 **
004224 ** Output parameter *pbIn is set to true if the plan added to pBuilder
004225 ** uses one or more WO_IN terms, or false otherwise.
004226 */
004227 static int whereLoopAddVirtualOne(
004228 WhereLoopBuilder *pBuilder,
004229 Bitmask mPrereq, /* Mask of tables that must be used. */
004230 Bitmask mUsable, /* Mask of usable tables */
004231 u16 mExclude, /* Exclude terms using these operators */
004232 sqlite3_index_info *pIdxInfo, /* Populated object for xBestIndex */
004233 u16 mNoOmit, /* Do not omit these constraints */
004234 int *pbIn, /* OUT: True if plan uses an IN(...) op */
004235 int *pbRetryLimit /* OUT: Retry without LIMIT/OFFSET */
004236 ){
004237 WhereClause *pWC = pBuilder->pWC;
004238 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
004239 struct sqlite3_index_constraint *pIdxCons;
004240 struct sqlite3_index_constraint_usage *pUsage = pIdxInfo->aConstraintUsage;
004241 int i;
004242 int mxTerm;
004243 int rc = SQLITE_OK;
004244 WhereLoop *pNew = pBuilder->pNew;
004245 Parse *pParse = pBuilder->pWInfo->pParse;
004246 SrcItem *pSrc = &pBuilder->pWInfo->pTabList->a[pNew->iTab];
004247 int nConstraint = pIdxInfo->nConstraint;
004248
004249 assert( (mUsable & mPrereq)==mPrereq );
004250 *pbIn = 0;
004251 pNew->prereq = mPrereq;
004252
004253 /* Set the usable flag on the subset of constraints identified by
004254 ** arguments mUsable and mExclude. */
004255 pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
004256 for(i=0; i<nConstraint; i++, pIdxCons++){
004257 WhereTerm *pTerm = termFromWhereClause(pWC, pIdxCons->iTermOffset);
004258 pIdxCons->usable = 0;
004259 if( (pTerm->prereqRight & mUsable)==pTerm->prereqRight
004260 && (pTerm->eOperator & mExclude)==0
004261 && (pbRetryLimit || !isLimitTerm(pTerm))
004262 ){
004263 pIdxCons->usable = 1;
004264 }
004265 }
004266
004267 /* Initialize the output fields of the sqlite3_index_info structure */
004268 memset(pUsage, 0, sizeof(pUsage[0])*nConstraint);
004269 assert( pIdxInfo->needToFreeIdxStr==0 );
004270 pIdxInfo->idxStr = 0;
004271 pIdxInfo->idxNum = 0;
004272 pIdxInfo->orderByConsumed = 0;
004273 pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2;
004274 pIdxInfo->estimatedRows = 25;
004275 pIdxInfo->idxFlags = 0;
004276 pHidden->mHandleIn = 0;
004277
004278 /* Invoke the virtual table xBestIndex() method */
004279 rc = vtabBestIndex(pParse, pSrc->pSTab, pIdxInfo);
004280 if( rc ){
004281 if( rc==SQLITE_CONSTRAINT ){
004282 /* If the xBestIndex method returns SQLITE_CONSTRAINT, that means
004283 ** that the particular combination of parameters provided is unusable.
004284 ** Make no entries in the loop table.
004285 */
004286 WHERETRACE(0xffffffff, (" ^^^^--- non-viable plan rejected!\n"));
004287 freeIdxStr(pIdxInfo);
004288 return SQLITE_OK;
004289 }
004290 return rc;
004291 }
004292
004293 mxTerm = -1;
004294 assert( pNew->nLSlot>=nConstraint );
004295 memset(pNew->aLTerm, 0, sizeof(pNew->aLTerm[0])*nConstraint );
004296 memset(&pNew->u.vtab, 0, sizeof(pNew->u.vtab));
004297 pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
004298 for(i=0; i<nConstraint; i++, pIdxCons++){
004299 int iTerm;
004300 if( (iTerm = pUsage[i].argvIndex - 1)>=0 ){
004301 WhereTerm *pTerm;
004302 int j = pIdxCons->iTermOffset;
004303 if( iTerm>=nConstraint
004304 || j<0
004305 || (pTerm = termFromWhereClause(pWC, j))==0
004306 || pNew->aLTerm[iTerm]!=0
004307 || pIdxCons->usable==0
004308 ){
004309 sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pSTab->zName);
004310 freeIdxStr(pIdxInfo);
004311 return SQLITE_ERROR;
004312 }
004313 testcase( iTerm==nConstraint-1 );
004314 testcase( j==0 );
004315 testcase( j==pWC->nTerm-1 );
004316 pNew->prereq |= pTerm->prereqRight;
004317 assert( iTerm<pNew->nLSlot );
004318 pNew->aLTerm[iTerm] = pTerm;
004319 if( iTerm>mxTerm ) mxTerm = iTerm;
004320 testcase( iTerm==15 );
004321 testcase( iTerm==16 );
004322 if( pUsage[i].omit ){
004323 if( i<16 && ((1<<i)&mNoOmit)==0 ){
004324 testcase( i!=iTerm );
004325 pNew->u.vtab.omitMask |= 1<<iTerm;
004326 }else{
004327 testcase( i!=iTerm );
004328 }
004329 if( pTerm->eMatchOp==SQLITE_INDEX_CONSTRAINT_OFFSET ){
004330 pNew->u.vtab.bOmitOffset = 1;
004331 }
004332 }
004333 if( SMASKBIT32(i) & pHidden->mHandleIn ){
004334 pNew->u.vtab.mHandleIn |= MASKBIT32(iTerm);
004335 }else if( (pTerm->eOperator & WO_IN)!=0 ){
004336 /* A virtual table that is constrained by an IN clause may not
004337 ** consume the ORDER BY clause because (1) the order of IN terms
004338 ** is not necessarily related to the order of output terms and
004339 ** (2) Multiple outputs from a single IN value will not merge
004340 ** together. */
004341 pIdxInfo->orderByConsumed = 0;
004342 pIdxInfo->idxFlags &= ~SQLITE_INDEX_SCAN_UNIQUE;
004343 *pbIn = 1; assert( (mExclude & WO_IN)==0 );
004344 }
004345
004346 /* Unless pbRetryLimit is non-NULL, there should be no LIMIT/OFFSET
004347 ** terms. And if there are any, they should follow all other terms. */
004348 assert( pbRetryLimit || !isLimitTerm(pTerm) );
004349 assert( !isLimitTerm(pTerm) || i>=nConstraint-2 );
004350 assert( !isLimitTerm(pTerm) || i==nConstraint-1 || isLimitTerm(pTerm+1) );
004351
004352 if( isLimitTerm(pTerm) && (*pbIn || !allConstraintsUsed(pUsage, i)) ){
004353 /* If there is an IN(...) term handled as an == (separate call to
004354 ** xFilter for each value on the RHS of the IN) and a LIMIT or
004355 ** OFFSET term handled as well, the plan is unusable. Similarly,
004356 ** if there is a LIMIT/OFFSET and there are other unused terms,
004357 ** the plan cannot be used. In these cases set variable *pbRetryLimit
004358 ** to true to tell the caller to retry with LIMIT and OFFSET
004359 ** disabled. */
004360 freeIdxStr(pIdxInfo);
004361 *pbRetryLimit = 1;
004362 return SQLITE_OK;
004363 }
004364 }
004365 }
004366
004367 pNew->nLTerm = mxTerm+1;
004368 for(i=0; i<=mxTerm; i++){
004369 if( pNew->aLTerm[i]==0 ){
004370 /* The non-zero argvIdx values must be contiguous. Raise an
004371 ** error if they are not */
004372 sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pSTab->zName);
004373 freeIdxStr(pIdxInfo);
004374 return SQLITE_ERROR;
004375 }
004376 }
004377 assert( pNew->nLTerm<=pNew->nLSlot );
004378 pNew->u.vtab.idxNum = pIdxInfo->idxNum;
004379 pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr;
004380 pIdxInfo->needToFreeIdxStr = 0;
004381 pNew->u.vtab.idxStr = pIdxInfo->idxStr;
004382 pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ?
004383 pIdxInfo->nOrderBy : 0);
004384 pNew->u.vtab.bIdxNumHex = (pIdxInfo->idxFlags&SQLITE_INDEX_SCAN_HEX)!=0;
004385 pNew->rSetup = 0;
004386 pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost);
004387 pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows);
004388
004389 /* Set the WHERE_ONEROW flag if the xBestIndex() method indicated
004390 ** that the scan will visit at most one row. Clear it otherwise. */
004391 if( pIdxInfo->idxFlags & SQLITE_INDEX_SCAN_UNIQUE ){
004392 pNew->wsFlags |= WHERE_ONEROW;
004393 }else{
004394 pNew->wsFlags &= ~WHERE_ONEROW;
004395 }
004396 rc = whereLoopInsert(pBuilder, pNew);
004397 if( pNew->u.vtab.needFree ){
004398 sqlite3_free(pNew->u.vtab.idxStr);
004399 pNew->u.vtab.needFree = 0;
004400 }
004401 WHERETRACE(0xffffffff, (" bIn=%d prereqIn=%04llx prereqOut=%04llx\n",
004402 *pbIn, (sqlite3_uint64)mPrereq,
004403 (sqlite3_uint64)(pNew->prereq & ~mPrereq)));
004404
004405 return rc;
004406 }
004407
004408 /*
004409 ** Return the collating sequence for a constraint passed into xBestIndex.
004410 **
004411 ** pIdxInfo must be an sqlite3_index_info structure passed into xBestIndex.
004412 ** This routine depends on there being a HiddenIndexInfo structure immediately
004413 ** following the sqlite3_index_info structure.
004414 **
004415 ** Return a pointer to the collation name:
004416 **
004417 ** 1. If there is an explicit COLLATE operator on the constraint, return it.
004418 **
004419 ** 2. Else, if the column has an alternative collation, return that.
004420 **
004421 ** 3. Otherwise, return "BINARY".
004422 */
004423 const char *sqlite3_vtab_collation(sqlite3_index_info *pIdxInfo, int iCons){
004424 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
004425 const char *zRet = 0;
004426 if( iCons>=0 && iCons<pIdxInfo->nConstraint ){
004427 CollSeq *pC = 0;
004428 int iTerm = pIdxInfo->aConstraint[iCons].iTermOffset;
004429 Expr *pX = termFromWhereClause(pHidden->pWC, iTerm)->pExpr;
004430 if( pX->pLeft ){
004431 pC = sqlite3ExprCompareCollSeq(pHidden->pParse, pX);
004432 }
004433 zRet = (pC ? pC->zName : sqlite3StrBINARY);
004434 }
004435 return zRet;
004436 }
004437
004438 /*
004439 ** Return true if constraint iCons is really an IN(...) constraint, or
004440 ** false otherwise. If iCons is an IN(...) constraint, set (if bHandle!=0)
004441 ** or clear (if bHandle==0) the flag to handle it using an iterator.
004442 */
004443 int sqlite3_vtab_in(sqlite3_index_info *pIdxInfo, int iCons, int bHandle){
004444 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
004445 u32 m = SMASKBIT32(iCons);
004446 if( m & pHidden->mIn ){
004447 if( bHandle==0 ){
004448 pHidden->mHandleIn &= ~m;
004449 }else if( bHandle>0 ){
004450 pHidden->mHandleIn |= m;
004451 }
004452 return 1;
004453 }
004454 return 0;
004455 }
004456
004457 /*
004458 ** This interface is callable from within the xBestIndex callback only.
004459 **
004460 ** If possible, set (*ppVal) to point to an object containing the value
004461 ** on the right-hand-side of constraint iCons.
004462 */
004463 int sqlite3_vtab_rhs_value(
004464 sqlite3_index_info *pIdxInfo, /* Copy of first argument to xBestIndex */
004465 int iCons, /* Constraint for which RHS is wanted */
004466 sqlite3_value **ppVal /* Write value extracted here */
004467 ){
004468 HiddenIndexInfo *pH = (HiddenIndexInfo*)&pIdxInfo[1];
004469 sqlite3_value *pVal = 0;
004470 int rc = SQLITE_OK;
004471 if( iCons<0 || iCons>=pIdxInfo->nConstraint ){
004472 rc = SQLITE_MISUSE_BKPT; /* EV: R-30545-25046 */
004473 }else{
004474 if( pH->aRhs[iCons]==0 ){
004475 WhereTerm *pTerm = termFromWhereClause(
004476 pH->pWC, pIdxInfo->aConstraint[iCons].iTermOffset
004477 );
004478 rc = sqlite3ValueFromExpr(
004479 pH->pParse->db, pTerm->pExpr->pRight, ENC(pH->pParse->db),
004480 SQLITE_AFF_BLOB, &pH->aRhs[iCons]
004481 );
004482 testcase( rc!=SQLITE_OK );
004483 }
004484 pVal = pH->aRhs[iCons];
004485 }
004486 *ppVal = pVal;
004487
004488 if( rc==SQLITE_OK && pVal==0 ){ /* IMP: R-19933-32160 */
004489 rc = SQLITE_NOTFOUND; /* IMP: R-36424-56542 */
004490 }
004491
004492 return rc;
004493 }
004494
004495 /*
004496 ** Return true if ORDER BY clause may be handled as DISTINCT.
004497 */
004498 int sqlite3_vtab_distinct(sqlite3_index_info *pIdxInfo){
004499 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
004500 assert( pHidden->eDistinct>=0 && pHidden->eDistinct<=3 );
004501 return pHidden->eDistinct;
004502 }
004503
004504 /*
004505 ** Cause the prepared statement that is associated with a call to
004506 ** xBestIndex to potentially use all schemas. If the statement being
004507 ** prepared is read-only, then just start read transactions on all
004508 ** schemas. But if this is a write operation, start writes on all
004509 ** schemas.
004510 **
004511 ** This is used by the (built-in) sqlite_dbpage virtual table.
004512 */
004513 void sqlite3VtabUsesAllSchemas(Parse *pParse){
004514 int nDb = pParse->db->nDb;
004515 int i;
004516 for(i=0; i<nDb; i++){
004517 sqlite3CodeVerifySchema(pParse, i);
004518 }
004519 if( DbMaskNonZero(pParse->writeMask) ){
004520 for(i=0; i<nDb; i++){
004521 sqlite3BeginWriteOperation(pParse, 0, i);
004522 }
004523 }
004524 }
004525
004526 /*
004527 ** Add all WhereLoop objects for a table of the join identified by
004528 ** pBuilder->pNew->iTab. That table is guaranteed to be a virtual table.
004529 **
004530 ** If there are no LEFT or CROSS JOIN joins in the query, both mPrereq and
004531 ** mUnusable are set to 0. Otherwise, mPrereq is a mask of all FROM clause
004532 ** entries that occur before the virtual table in the FROM clause and are
004533 ** separated from it by at least one LEFT or CROSS JOIN. Similarly, the
004534 ** mUnusable mask contains all FROM clause entries that occur after the
004535 ** virtual table and are separated from it by at least one LEFT or
004536 ** CROSS JOIN.
004537 **
004538 ** For example, if the query were:
004539 **
004540 ** ... FROM t1, t2 LEFT JOIN t3, t4, vt CROSS JOIN t5, t6;
004541 **
004542 ** then mPrereq corresponds to (t1, t2) and mUnusable to (t5, t6).
004543 **
004544 ** All the tables in mPrereq must be scanned before the current virtual
004545 ** table. So any terms for which all prerequisites are satisfied by
004546 ** mPrereq may be specified as "usable" in all calls to xBestIndex.
004547 ** Conversely, all tables in mUnusable must be scanned after the current
004548 ** virtual table, so any terms for which the prerequisites overlap with
004549 ** mUnusable should always be configured as "not-usable" for xBestIndex.
004550 */
004551 static int whereLoopAddVirtual(
004552 WhereLoopBuilder *pBuilder, /* WHERE clause information */
004553 Bitmask mPrereq, /* Tables that must be scanned before this one */
004554 Bitmask mUnusable /* Tables that must be scanned after this one */
004555 ){
004556 int rc = SQLITE_OK; /* Return code */
004557 WhereInfo *pWInfo; /* WHERE analysis context */
004558 Parse *pParse; /* The parsing context */
004559 WhereClause *pWC; /* The WHERE clause */
004560 SrcItem *pSrc; /* The FROM clause term to search */
004561 sqlite3_index_info *p; /* Object to pass to xBestIndex() */
004562 int nConstraint; /* Number of constraints in p */
004563 int bIn; /* True if plan uses IN(...) operator */
004564 WhereLoop *pNew;
004565 Bitmask mBest; /* Tables used by best possible plan */
004566 u16 mNoOmit;
004567 int bRetry = 0; /* True to retry with LIMIT/OFFSET disabled */
004568
004569 assert( (mPrereq & mUnusable)==0 );
004570 pWInfo = pBuilder->pWInfo;
004571 pParse = pWInfo->pParse;
004572 pWC = pBuilder->pWC;
004573 pNew = pBuilder->pNew;
004574 pSrc = &pWInfo->pTabList->a[pNew->iTab];
004575 assert( IsVirtual(pSrc->pSTab) );
004576 p = allocateIndexInfo(pWInfo, pWC, mUnusable, pSrc, &mNoOmit);
004577 if( p==0 ) return SQLITE_NOMEM_BKPT;
004578 pNew->rSetup = 0;
004579 pNew->wsFlags = WHERE_VIRTUALTABLE;
004580 pNew->nLTerm = 0;
004581 pNew->u.vtab.needFree = 0;
004582 nConstraint = p->nConstraint;
004583 if( whereLoopResize(pParse->db, pNew, nConstraint) ){
004584 freeIndexInfo(pParse->db, p);
004585 return SQLITE_NOMEM_BKPT;
004586 }
004587
004588 /* First call xBestIndex() with all constraints usable. */
004589 WHERETRACE(0x800, ("BEGIN %s.addVirtual()\n", pSrc->pSTab->zName));
004590 WHERETRACE(0x800, (" VirtualOne: all usable\n"));
004591 rc = whereLoopAddVirtualOne(
004592 pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, &bRetry
004593 );
004594 if( bRetry ){
004595 assert( rc==SQLITE_OK );
004596 rc = whereLoopAddVirtualOne(
004597 pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, 0
004598 );
004599 }
004600
004601 /* If the call to xBestIndex() with all terms enabled produced a plan
004602 ** that does not require any source tables (IOW: a plan with mBest==0)
004603 ** and does not use an IN(...) operator, then there is no point in making
004604 ** any further calls to xBestIndex() since they will all return the same
004605 ** result (if the xBestIndex() implementation is sane). */
004606 if( rc==SQLITE_OK && ((mBest = (pNew->prereq & ~mPrereq))!=0 || bIn) ){
004607 int seenZero = 0; /* True if a plan with no prereqs seen */
004608 int seenZeroNoIN = 0; /* Plan with no prereqs and no IN(...) seen */
004609 Bitmask mPrev = 0;
004610 Bitmask mBestNoIn = 0;
004611
004612 /* If the plan produced by the earlier call uses an IN(...) term, call
004613 ** xBestIndex again, this time with IN(...) terms disabled. */
004614 if( bIn ){
004615 WHERETRACE(0x800, (" VirtualOne: all usable w/o IN\n"));
004616 rc = whereLoopAddVirtualOne(
004617 pBuilder, mPrereq, ALLBITS, WO_IN, p, mNoOmit, &bIn, 0);
004618 assert( bIn==0 );
004619 mBestNoIn = pNew->prereq & ~mPrereq;
004620 if( mBestNoIn==0 ){
004621 seenZero = 1;
004622 seenZeroNoIN = 1;
004623 }
004624 }
004625
004626 /* Call xBestIndex once for each distinct value of (prereqRight & ~mPrereq)
004627 ** in the set of terms that apply to the current virtual table. */
004628 while( rc==SQLITE_OK ){
004629 int i;
004630 Bitmask mNext = ALLBITS;
004631 assert( mNext>0 );
004632 for(i=0; i<nConstraint; i++){
004633 int iTerm = p->aConstraint[i].iTermOffset;
004634 Bitmask mThis = termFromWhereClause(pWC, iTerm)->prereqRight & ~mPrereq;
004635 if( mThis>mPrev && mThis<mNext ) mNext = mThis;
004636 }
004637 mPrev = mNext;
004638 if( mNext==ALLBITS ) break;
004639 if( mNext==mBest || mNext==mBestNoIn ) continue;
004640 WHERETRACE(0x800, (" VirtualOne: mPrev=%04llx mNext=%04llx\n",
004641 (sqlite3_uint64)mPrev, (sqlite3_uint64)mNext));
004642 rc = whereLoopAddVirtualOne(
004643 pBuilder, mPrereq, mNext|mPrereq, 0, p, mNoOmit, &bIn, 0);
004644 if( pNew->prereq==mPrereq ){
004645 seenZero = 1;
004646 if( bIn==0 ) seenZeroNoIN = 1;
004647 }
004648 }
004649
004650 /* If the calls to xBestIndex() in the above loop did not find a plan
004651 ** that requires no source tables at all (i.e. one guaranteed to be
004652 ** usable), make a call here with all source tables disabled */
004653 if( rc==SQLITE_OK && seenZero==0 ){
004654 WHERETRACE(0x800, (" VirtualOne: all disabled\n"));
004655 rc = whereLoopAddVirtualOne(
004656 pBuilder, mPrereq, mPrereq, 0, p, mNoOmit, &bIn, 0);
004657 if( bIn==0 ) seenZeroNoIN = 1;
004658 }
004659
004660 /* If the calls to xBestIndex() have so far failed to find a plan
004661 ** that requires no source tables at all and does not use an IN(...)
004662 ** operator, make a final call to obtain one here. */
004663 if( rc==SQLITE_OK && seenZeroNoIN==0 ){
004664 WHERETRACE(0x800, (" VirtualOne: all disabled and w/o IN\n"));
004665 rc = whereLoopAddVirtualOne(
004666 pBuilder, mPrereq, mPrereq, WO_IN, p, mNoOmit, &bIn, 0);
004667 }
004668 }
004669
004670 freeIndexInfo(pParse->db, p);
004671 WHERETRACE(0x800, ("END %s.addVirtual(), rc=%d\n", pSrc->pSTab->zName, rc));
004672 return rc;
004673 }
004674 #endif /* SQLITE_OMIT_VIRTUALTABLE */
004675
004676 /*
004677 ** Add WhereLoop entries to handle OR terms. This works for either
004678 ** btrees or virtual tables.
004679 */
004680 static int whereLoopAddOr(
004681 WhereLoopBuilder *pBuilder,
004682 Bitmask mPrereq,
004683 Bitmask mUnusable
004684 ){
004685 WhereInfo *pWInfo = pBuilder->pWInfo;
004686 WhereClause *pWC;
004687 WhereLoop *pNew;
004688 WhereTerm *pTerm, *pWCEnd;
004689 int rc = SQLITE_OK;
004690 int iCur;
004691 WhereClause tempWC;
004692 WhereLoopBuilder sSubBuild;
004693 WhereOrSet sSum, sCur;
004694 SrcItem *pItem;
004695
004696 pWC = pBuilder->pWC;
004697 pWCEnd = pWC->a + pWC->nTerm;
004698 pNew = pBuilder->pNew;
004699 memset(&sSum, 0, sizeof(sSum));
004700 pItem = pWInfo->pTabList->a + pNew->iTab;
004701 iCur = pItem->iCursor;
004702
004703 /* The multi-index OR optimization does not work for RIGHT and FULL JOIN */
004704 if( pItem->fg.jointype & JT_RIGHT ) return SQLITE_OK;
004705
004706 for(pTerm=pWC->a; pTerm<pWCEnd && rc==SQLITE_OK; pTerm++){
004707 if( (pTerm->eOperator & WO_OR)!=0
004708 && (pTerm->u.pOrInfo->indexable & pNew->maskSelf)!=0
004709 ){
004710 WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc;
004711 WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm];
004712 WhereTerm *pOrTerm;
004713 int once = 1;
004714 int i, j;
004715
004716 sSubBuild = *pBuilder;
004717 sSubBuild.pOrSet = &sCur;
004718
004719 WHERETRACE(0x400, ("Begin processing OR-clause %p\n", pTerm));
004720 for(pOrTerm=pOrWC->a; pOrTerm<pOrWCEnd; pOrTerm++){
004721 if( (pOrTerm->eOperator & WO_AND)!=0 ){
004722 sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc;
004723 }else if( pOrTerm->leftCursor==iCur ){
004724 tempWC.pWInfo = pWC->pWInfo;
004725 tempWC.pOuter = pWC;
004726 tempWC.op = TK_AND;
004727 tempWC.nTerm = 1;
004728 tempWC.nBase = 1;
004729 tempWC.a = pOrTerm;
004730 sSubBuild.pWC = &tempWC;
004731 }else{
004732 continue;
004733 }
004734 sCur.n = 0;
004735 #ifdef WHERETRACE_ENABLED
004736 WHERETRACE(0x400, ("OR-term %d of %p has %d subterms:\n",
004737 (int)(pOrTerm-pOrWC->a), pTerm, sSubBuild.pWC->nTerm));
004738 if( sqlite3WhereTrace & 0x20000 ){
004739 sqlite3WhereClausePrint(sSubBuild.pWC);
004740 }
004741 #endif
004742 #ifndef SQLITE_OMIT_VIRTUALTABLE
004743 if( IsVirtual(pItem->pSTab) ){
004744 rc = whereLoopAddVirtual(&sSubBuild, mPrereq, mUnusable);
004745 }else
004746 #endif
004747 {
004748 rc = whereLoopAddBtree(&sSubBuild, mPrereq);
004749 }
004750 if( rc==SQLITE_OK ){
004751 rc = whereLoopAddOr(&sSubBuild, mPrereq, mUnusable);
004752 }
004753 testcase( rc==SQLITE_NOMEM && sCur.n>0 );
004754 testcase( rc==SQLITE_DONE );
004755 if( sCur.n==0 ){
004756 sSum.n = 0;
004757 break;
004758 }else if( once ){
004759 whereOrMove(&sSum, &sCur);
004760 once = 0;
004761 }else{
004762 WhereOrSet sPrev;
004763 whereOrMove(&sPrev, &sSum);
004764 sSum.n = 0;
004765 for(i=0; i<sPrev.n; i++){
004766 for(j=0; j<sCur.n; j++){
004767 whereOrInsert(&sSum, sPrev.a[i].prereq | sCur.a[j].prereq,
004768 sqlite3LogEstAdd(sPrev.a[i].rRun, sCur.a[j].rRun),
004769 sqlite3LogEstAdd(sPrev.a[i].nOut, sCur.a[j].nOut));
004770 }
004771 }
004772 }
004773 }
004774 pNew->nLTerm = 1;
004775 pNew->aLTerm[0] = pTerm;
004776 pNew->wsFlags = WHERE_MULTI_OR;
004777 pNew->rSetup = 0;
004778 pNew->iSortIdx = 0;
004779 memset(&pNew->u, 0, sizeof(pNew->u));
004780 for(i=0; rc==SQLITE_OK && i<sSum.n; i++){
004781 /* TUNING: Currently sSum.a[i].rRun is set to the sum of the costs
004782 ** of all sub-scans required by the OR-scan. However, due to rounding
004783 ** errors, it may be that the cost of the OR-scan is equal to its
004784 ** most expensive sub-scan. Add the smallest possible penalty
004785 ** (equivalent to multiplying the cost by 1.07) to ensure that
004786 ** this does not happen. Otherwise, for WHERE clauses such as the
004787 ** following where there is an index on "y":
004788 **
004789 ** WHERE likelihood(x=?, 0.99) OR y=?
004790 **
004791 ** the planner may elect to "OR" together a full-table scan and an
004792 ** index lookup. And other similarly odd results. */
004793 pNew->rRun = sSum.a[i].rRun + 1;
004794 pNew->nOut = sSum.a[i].nOut;
004795 pNew->prereq = sSum.a[i].prereq;
004796 rc = whereLoopInsert(pBuilder, pNew);
004797 }
004798 WHERETRACE(0x400, ("End processing OR-clause %p\n", pTerm));
004799 }
004800 }
004801 return rc;
004802 }
004803
004804 /*
004805 ** Add all WhereLoop objects for all tables
004806 */
004807 static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
004808 WhereInfo *pWInfo = pBuilder->pWInfo;
004809 Bitmask mPrereq = 0;
004810 Bitmask mPrior = 0;
004811 int iTab;
004812 SrcList *pTabList = pWInfo->pTabList;
004813 SrcItem *pItem;
004814 SrcItem *pEnd = &pTabList->a[pWInfo->nLevel];
004815 sqlite3 *db = pWInfo->pParse->db;
004816 int rc = SQLITE_OK;
004817 int bFirstPastRJ = 0;
004818 int hasRightJoin = 0;
004819 WhereLoop *pNew;
004820
004821
004822 /* Loop over the tables in the join, from left to right */
004823 pNew = pBuilder->pNew;
004824
004825 /* Verify that pNew has already been initialized */
004826 assert( pNew->nLTerm==0 );
004827 assert( pNew->wsFlags==0 );
004828 assert( pNew->nLSlot>=ArraySize(pNew->aLTermSpace) );
004829 assert( pNew->aLTerm!=0 );
004830
004831 pBuilder->iPlanLimit = SQLITE_QUERY_PLANNER_LIMIT;
004832 for(iTab=0, pItem=pTabList->a; pItem<pEnd; iTab++, pItem++){
004833 Bitmask mUnusable = 0;
004834 pNew->iTab = iTab;
004835 pBuilder->iPlanLimit += SQLITE_QUERY_PLANNER_LIMIT_INCR;
004836 pNew->maskSelf = sqlite3WhereGetMask(&pWInfo->sMaskSet, pItem->iCursor);
004837 if( bFirstPastRJ
004838 || (pItem->fg.jointype & (JT_OUTER|JT_CROSS|JT_LTORJ))!=0
004839 ){
004840 /* Add prerequisites to prevent reordering of FROM clause terms
004841 ** across CROSS joins and outer joins. The bFirstPastRJ boolean
004842 ** prevents the right operand of a RIGHT JOIN from being swapped with
004843 ** other elements even further to the right.
004844 **
004845 ** The JT_LTORJ case and the hasRightJoin flag work together to
004846 ** prevent FROM-clause terms from moving from the right side of
004847 ** a LEFT JOIN over to the left side of that join if the LEFT JOIN
004848 ** is itself on the left side of a RIGHT JOIN.
004849 */
004850 if( pItem->fg.jointype & JT_LTORJ ) hasRightJoin = 1;
004851 mPrereq |= mPrior;
004852 bFirstPastRJ = (pItem->fg.jointype & JT_RIGHT)!=0;
004853 }else if( !hasRightJoin ){
004854 mPrereq = 0;
004855 }
004856 #ifndef SQLITE_OMIT_VIRTUALTABLE
004857 if( IsVirtual(pItem->pSTab) ){
004858 SrcItem *p;
004859 for(p=&pItem[1]; p<pEnd; p++){
004860 if( mUnusable || (p->fg.jointype & (JT_OUTER|JT_CROSS)) ){
004861 mUnusable |= sqlite3WhereGetMask(&pWInfo->sMaskSet, p->iCursor);
004862 }
004863 }
004864 rc = whereLoopAddVirtual(pBuilder, mPrereq, mUnusable);
004865 }else
004866 #endif /* SQLITE_OMIT_VIRTUALTABLE */
004867 {
004868 rc = whereLoopAddBtree(pBuilder, mPrereq);
004869 }
004870 if( rc==SQLITE_OK && pBuilder->pWC->hasOr ){
004871 rc = whereLoopAddOr(pBuilder, mPrereq, mUnusable);
004872 }
004873 mPrior |= pNew->maskSelf;
004874 if( rc || db->mallocFailed ){
004875 if( rc==SQLITE_DONE ){
004876 /* We hit the query planner search limit set by iPlanLimit */
004877 sqlite3_log(SQLITE_WARNING, "abbreviated query algorithm search");
004878 rc = SQLITE_OK;
004879 }else{
004880 break;
004881 }
004882 }
004883 }
004884
004885 whereLoopClear(db, pNew);
004886 return rc;
004887 }
004888
004889 /* Implementation of the order-by-subquery optimization:
004890 **
004891 ** WhereLoop pLoop, which the iLoop-th term of the nested loop, is really
004892 ** a subquery or CTE that has an ORDER BY clause. See if any of the terms
004893 ** in the subquery ORDER BY clause will satisfy pOrderBy from the outer
004894 ** query. Mark off all satisfied terms (by setting bits in *pOBSat) and
004895 ** return TRUE if they do. If not, return false.
004896 **
004897 ** Example:
004898 **
004899 ** CREATE TABLE t1(a,b,c, PRIMARY KEY(a,b));
004900 ** CREATE TABLE t2(x,y);
004901 ** WITH t3(p,q) AS MATERIALIZED (SELECT x+y, x-y FROM t2 ORDER BY x+y)
004902 ** SELECT * FROM t3 JOIN t1 ON a=q ORDER BY p, b;
004903 **
004904 ** The CTE named "t3" comes out in the natural order of "p", so the first
004905 ** first them of "ORDER BY p,b" is satisfied by a sequential scan of "t3"
004906 ** and sorting only needs to occur on the second term "b".
004907 **
004908 ** Limitations:
004909 **
004910 ** (1) The optimization is not applied if the outer ORDER BY contains
004911 ** a COLLATE clause. The optimization might be applied if the
004912 ** outer ORDER BY uses NULLS FIRST, NULLS LAST, ASC, and/or DESC as
004913 ** long as the subquery ORDER BY does the same. But if the
004914 ** outer ORDER BY uses COLLATE, even a redundant COLLATE, the
004915 ** optimization is bypassed.
004916 **
004917 ** (2) The subquery ORDER BY terms must exactly match subquery result
004918 ** columns, including any COLLATE annotations. This routine relies
004919 ** on iOrderByCol to do matching between order by terms and result
004920 ** columns, and iOrderByCol will not be set if the result column
004921 ** and ORDER BY collations differ.
004922 **
004923 ** (3) The subquery and outer ORDER BY can be in opposite directions as
004924 ** long as the subquery is materialized. If the subquery is
004925 ** implemented as a co-routine, the sort orders must be in the same
004926 ** direction because there is no way to run a co-routine backwards.
004927 */
004928 static SQLITE_NOINLINE int wherePathMatchSubqueryOB(
004929 WhereInfo *pWInfo, /* The WHERE clause */
004930 WhereLoop *pLoop, /* The nested loop term that is a subquery */
004931 int iLoop, /* Which level of the nested loop. 0==outermost */
004932 int iCur, /* Cursor used by the this loop */
004933 ExprList *pOrderBy, /* The ORDER BY clause on the whole query */
004934 Bitmask *pRevMask, /* When loops need to go in reverse order */
004935 Bitmask *pOBSat /* Which terms of pOrderBy are satisfied so far */
004936 ){
004937 int iOB; /* Index into pOrderBy->a[] */
004938 int jSub; /* Index into pSubOB->a[] */
004939 u8 rev = 0; /* True if iOB and jSub sort in opposite directions */
004940 u8 revIdx = 0; /* Sort direction for jSub */
004941 Expr *pOBExpr; /* Current term of outer ORDER BY */
004942 ExprList *pSubOB; /* Complete ORDER BY on the subquery */
004943
004944 pSubOB = pLoop->u.btree.pOrderBy;
004945 assert( pSubOB!=0 );
004946 for(iOB=0; (MASKBIT(iOB) & *pOBSat)!=0; iOB++){}
004947 for(jSub=0; jSub<pSubOB->nExpr && iOB<pOrderBy->nExpr; jSub++, iOB++){
004948 if( pSubOB->a[jSub].u.x.iOrderByCol==0 ) break;
004949 pOBExpr = pOrderBy->a[iOB].pExpr;
004950 if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) break;
004951 if( pOBExpr->iTable!=iCur ) break;
004952 if( pOBExpr->iColumn!=pSubOB->a[jSub].u.x.iOrderByCol-1 ) break;
004953 if( (pWInfo->wctrlFlags & WHERE_GROUPBY)==0 ){
004954 u8 sfOB = pOrderBy->a[iOB].fg.sortFlags; /* sortFlags for iOB */
004955 u8 sfSub = pSubOB->a[jSub].fg.sortFlags; /* sortFlags for jSub */
004956 if( (sfSub & KEYINFO_ORDER_BIGNULL) != (sfOB & KEYINFO_ORDER_BIGNULL) ){
004957 break;
004958 }
004959 revIdx = sfSub & KEYINFO_ORDER_DESC;
004960 if( jSub>0 ){
004961 if( (rev^revIdx)!=(sfOB & KEYINFO_ORDER_DESC) ){
004962 break;
004963 }
004964 }else{
004965 rev = revIdx ^ (sfOB & KEYINFO_ORDER_DESC);
004966 if( rev ){
004967 if( (pLoop->wsFlags & WHERE_COROUTINE)!=0 ){
004968 /* Cannot run a co-routine in reverse order */
004969 break;
004970 }
004971 *pRevMask |= MASKBIT(iLoop);
004972 }
004973 }
004974 }
004975 *pOBSat |= MASKBIT(iOB);
004976 }
004977 return jSub>0;
004978 }
004979
004980 /*
004981 ** Examine a WherePath (with the addition of the extra WhereLoop of the 6th
004982 ** parameters) to see if it outputs rows in the requested ORDER BY
004983 ** (or GROUP BY) without requiring a separate sort operation. Return N:
004984 **
004985 ** N>0: N terms of the ORDER BY clause are satisfied
004986 ** N==0: No terms of the ORDER BY clause are satisfied
004987 ** N<0: Unknown yet how many terms of ORDER BY might be satisfied.
004988 **
004989 ** Note that processing for WHERE_GROUPBY and WHERE_DISTINCTBY is not as
004990 ** strict. With GROUP BY and DISTINCT the only requirement is that
004991 ** equivalent rows appear immediately adjacent to one another. GROUP BY
004992 ** and DISTINCT do not require rows to appear in any particular order as long
004993 ** as equivalent rows are grouped together. Thus for GROUP BY and DISTINCT
004994 ** the pOrderBy terms can be matched in any order. With ORDER BY, the
004995 ** pOrderBy terms must be matched in strict left-to-right order.
004996 */
004997 static i8 wherePathSatisfiesOrderBy(
004998 WhereInfo *pWInfo, /* The WHERE clause */
004999 ExprList *pOrderBy, /* ORDER BY or GROUP BY or DISTINCT clause to check */
005000 WherePath *pPath, /* The WherePath to check */
005001 u16 wctrlFlags, /* WHERE_GROUPBY or _DISTINCTBY or _ORDERBY_LIMIT */
005002 u16 nLoop, /* Number of entries in pPath->aLoop[] */
005003 WhereLoop *pLast, /* Add this WhereLoop to the end of pPath->aLoop[] */
005004 Bitmask *pRevMask /* OUT: Mask of WhereLoops to run in reverse order */
005005 ){
005006 u8 revSet; /* True if rev is known */
005007 u8 rev; /* Composite sort order */
005008 u8 revIdx; /* Index sort order */
005009 u8 isOrderDistinct; /* All prior WhereLoops are order-distinct */
005010 u8 distinctColumns; /* True if the loop has UNIQUE NOT NULL columns */
005011 u8 isMatch; /* iColumn matches a term of the ORDER BY clause */
005012 u16 eqOpMask; /* Allowed equality operators */
005013 u16 nKeyCol; /* Number of key columns in pIndex */
005014 u16 nColumn; /* Total number of ordered columns in the index */
005015 u16 nOrderBy; /* Number terms in the ORDER BY clause */
005016 int iLoop; /* Index of WhereLoop in pPath being processed */
005017 int i, j; /* Loop counters */
005018 int iCur; /* Cursor number for current WhereLoop */
005019 int iColumn; /* A column number within table iCur */
005020 WhereLoop *pLoop = 0; /* Current WhereLoop being processed. */
005021 WhereTerm *pTerm; /* A single term of the WHERE clause */
005022 Expr *pOBExpr; /* An expression from the ORDER BY clause */
005023 CollSeq *pColl; /* COLLATE function from an ORDER BY clause term */
005024 Index *pIndex; /* The index associated with pLoop */
005025 sqlite3 *db = pWInfo->pParse->db; /* Database connection */
005026 Bitmask obSat = 0; /* Mask of ORDER BY terms satisfied so far */
005027 Bitmask obDone; /* Mask of all ORDER BY terms */
005028 Bitmask orderDistinctMask; /* Mask of all well-ordered loops */
005029 Bitmask ready; /* Mask of inner loops */
005030
005031 /*
005032 ** We say the WhereLoop is "one-row" if it generates no more than one
005033 ** row of output. A WhereLoop is one-row if all of the following are true:
005034 ** (a) All index columns match with WHERE_COLUMN_EQ.
005035 ** (b) The index is unique
005036 ** Any WhereLoop with an WHERE_COLUMN_EQ constraint on the rowid is one-row.
005037 ** Every one-row WhereLoop will have the WHERE_ONEROW bit set in wsFlags.
005038 **
005039 ** We say the WhereLoop is "order-distinct" if the set of columns from
005040 ** that WhereLoop that are in the ORDER BY clause are different for every
005041 ** row of the WhereLoop. Every one-row WhereLoop is automatically
005042 ** order-distinct. A WhereLoop that has no columns in the ORDER BY clause
005043 ** is not order-distinct. To be order-distinct is not quite the same as being
005044 ** UNIQUE since a UNIQUE column or index can have multiple rows that
005045 ** are NULL and NULL values are equivalent for the purpose of order-distinct.
005046 ** To be order-distinct, the columns must be UNIQUE and NOT NULL.
005047 **
005048 ** The rowid for a table is always UNIQUE and NOT NULL so whenever the
005049 ** rowid appears in the ORDER BY clause, the corresponding WhereLoop is
005050 ** automatically order-distinct.
005051 */
005052
005053 assert( pOrderBy!=0 );
005054 if( nLoop && OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return 0;
005055
005056 nOrderBy = pOrderBy->nExpr;
005057 testcase( nOrderBy==BMS-1 );
005058 if( nOrderBy>BMS-1 ) return 0; /* Cannot optimize overly large ORDER BYs */
005059 isOrderDistinct = 1;
005060 obDone = MASKBIT(nOrderBy)-1;
005061 orderDistinctMask = 0;
005062 ready = 0;
005063 eqOpMask = WO_EQ | WO_IS | WO_ISNULL;
005064 if( wctrlFlags & (WHERE_ORDERBY_LIMIT|WHERE_ORDERBY_MAX|WHERE_ORDERBY_MIN) ){
005065 eqOpMask |= WO_IN;
005066 }
005067 for(iLoop=0; isOrderDistinct && obSat<obDone && iLoop<=nLoop; iLoop++){
005068 if( iLoop>0 ) ready |= pLoop->maskSelf;
005069 if( iLoop<nLoop ){
005070 pLoop = pPath->aLoop[iLoop];
005071 if( wctrlFlags & WHERE_ORDERBY_LIMIT ) continue;
005072 }else{
005073 pLoop = pLast;
005074 }
005075 if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){
005076 if( pLoop->u.vtab.isOrdered
005077 && ((wctrlFlags&(WHERE_DISTINCTBY|WHERE_SORTBYGROUP))!=WHERE_DISTINCTBY)
005078 ){
005079 obSat = obDone;
005080 }
005081 break;
005082 }else if( wctrlFlags & WHERE_DISTINCTBY ){
005083 pLoop->u.btree.nDistinctCol = 0;
005084 }
005085 iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor;
005086
005087 /* Mark off any ORDER BY term X that is a column in the table of
005088 ** the current loop for which there is term in the WHERE
005089 ** clause of the form X IS NULL or X=? that reference only outer
005090 ** loops.
005091 */
005092 for(i=0; i<nOrderBy; i++){
005093 if( MASKBIT(i) & obSat ) continue;
005094 pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr);
005095 if( NEVER(pOBExpr==0) ) continue;
005096 if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue;
005097 if( pOBExpr->iTable!=iCur ) continue;
005098 pTerm = sqlite3WhereFindTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn,
005099 ~ready, eqOpMask, 0);
005100 if( pTerm==0 ) continue;
005101 if( pTerm->eOperator==WO_IN ){
005102 /* IN terms are only valid for sorting in the ORDER BY LIMIT
005103 ** optimization, and then only if they are actually used
005104 ** by the query plan */
005105 assert( wctrlFlags &
005106 (WHERE_ORDERBY_LIMIT|WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX) );
005107 for(j=0; j<pLoop->nLTerm && pTerm!=pLoop->aLTerm[j]; j++){}
005108 if( j>=pLoop->nLTerm ) continue;
005109 }
005110 if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 && pOBExpr->iColumn>=0 ){
005111 Parse *pParse = pWInfo->pParse;
005112 CollSeq *pColl1 = sqlite3ExprNNCollSeq(pParse, pOrderBy->a[i].pExpr);
005113 CollSeq *pColl2 = sqlite3ExprCompareCollSeq(pParse, pTerm->pExpr);
005114 assert( pColl1 );
005115 if( pColl2==0 || sqlite3StrICmp(pColl1->zName, pColl2->zName) ){
005116 continue;
005117 }
005118 testcase( pTerm->pExpr->op==TK_IS );
005119 }
005120 obSat |= MASKBIT(i);
005121 }
005122
005123 if( (pLoop->wsFlags & WHERE_ONEROW)==0 ){
005124 if( pLoop->wsFlags & WHERE_IPK ){
005125 if( pLoop->u.btree.pOrderBy
005126 && OptimizationEnabled(db, SQLITE_OrderBySubq)
005127 && wherePathMatchSubqueryOB(pWInfo,pLoop,iLoop,iCur,
005128 pOrderBy,pRevMask, &obSat)
005129 ){
005130 nColumn = 0;
005131 isOrderDistinct = 0;
005132 }else{
005133 nColumn = 1;
005134 }
005135 pIndex = 0;
005136 nKeyCol = 0;
005137 }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){
005138 return 0;
005139 }else{
005140 nKeyCol = pIndex->nKeyCol;
005141 nColumn = pIndex->nColumn;
005142 assert( nColumn==nKeyCol+1 || !HasRowid(pIndex->pTable) );
005143 assert( pIndex->aiColumn[nColumn-1]==XN_ROWID
005144 || !HasRowid(pIndex->pTable));
005145 /* All relevant terms of the index must also be non-NULL in order
005146 ** for isOrderDistinct to be true. So the isOrderDistinct value
005147 ** computed here might be a false positive. Corrections will be
005148 ** made at tag-20210426-1 below */
005149 isOrderDistinct = IsUniqueIndex(pIndex)
005150 && (pLoop->wsFlags & WHERE_SKIPSCAN)==0;
005151 }
005152
005153 /* Loop through all columns of the index and deal with the ones
005154 ** that are not constrained by == or IN.
005155 */
005156 rev = revSet = 0;
005157 distinctColumns = 0;
005158 for(j=0; j<nColumn; j++){
005159 u8 bOnce = 1; /* True to run the ORDER BY search loop */
005160
005161 assert( j>=pLoop->u.btree.nEq
005162 || (pLoop->aLTerm[j]==0)==(j<pLoop->nSkip)
005163 );
005164 if( j<pLoop->u.btree.nEq && j>=pLoop->nSkip ){
005165 u16 eOp = pLoop->aLTerm[j]->eOperator;
005166
005167 /* Skip over == and IS and ISNULL terms. (Also skip IN terms when
005168 ** doing WHERE_ORDERBY_LIMIT processing). Except, IS and ISNULL
005169 ** terms imply that the index is not UNIQUE NOT NULL in which case
005170 ** the loop need to be marked as not order-distinct because it can
005171 ** have repeated NULL rows.
005172 **
005173 ** If the current term is a column of an ((?,?) IN (SELECT...))
005174 ** expression for which the SELECT returns more than one column,
005175 ** check that it is the only column used by this loop. Otherwise,
005176 ** if it is one of two or more, none of the columns can be
005177 ** considered to match an ORDER BY term.
005178 */
005179 if( (eOp & eqOpMask)!=0 ){
005180 if( eOp & (WO_ISNULL|WO_IS) ){
005181 testcase( eOp & WO_ISNULL );
005182 testcase( eOp & WO_IS );
005183 testcase( isOrderDistinct );
005184 isOrderDistinct = 0;
005185 }
005186 continue;
005187 }else if( ALWAYS(eOp & WO_IN) ){
005188 /* ALWAYS() justification: eOp is an equality operator due to the
005189 ** j<pLoop->u.btree.nEq constraint above. Any equality other
005190 ** than WO_IN is captured by the previous "if". So this one
005191 ** always has to be WO_IN. */
005192 Expr *pX = pLoop->aLTerm[j]->pExpr;
005193 for(i=j+1; i<pLoop->u.btree.nEq; i++){
005194 if( pLoop->aLTerm[i]->pExpr==pX ){
005195 assert( (pLoop->aLTerm[i]->eOperator & WO_IN) );
005196 bOnce = 0;
005197 break;
005198 }
005199 }
005200 }
005201 }
005202
005203 /* Get the column number in the table (iColumn) and sort order
005204 ** (revIdx) for the j-th column of the index.
005205 */
005206 if( pIndex ){
005207 iColumn = pIndex->aiColumn[j];
005208 revIdx = pIndex->aSortOrder[j] & KEYINFO_ORDER_DESC;
005209 if( iColumn==pIndex->pTable->iPKey ) iColumn = XN_ROWID;
005210 }else{
005211 iColumn = XN_ROWID;
005212 revIdx = 0;
005213 }
005214
005215 /* An unconstrained column that might be NULL means that this
005216 ** WhereLoop is not well-ordered. tag-20210426-1
005217 */
005218 if( isOrderDistinct ){
005219 if( iColumn>=0
005220 && j>=pLoop->u.btree.nEq
005221 && pIndex->pTable->aCol[iColumn].notNull==0
005222 ){
005223 isOrderDistinct = 0;
005224 }
005225 if( iColumn==XN_EXPR ){
005226 isOrderDistinct = 0;
005227 }
005228 }
005229
005230 /* Find the ORDER BY term that corresponds to the j-th column
005231 ** of the index and mark that ORDER BY term having been satisfied.
005232 */
005233 isMatch = 0;
005234 for(i=0; bOnce && i<nOrderBy; i++){
005235 if( MASKBIT(i) & obSat ) continue;
005236 pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr);
005237 testcase( wctrlFlags & WHERE_GROUPBY );
005238 testcase( wctrlFlags & WHERE_DISTINCTBY );
005239 if( NEVER(pOBExpr==0) ) continue;
005240 if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0;
005241 if( iColumn>=XN_ROWID ){
005242 if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue;
005243 if( pOBExpr->iTable!=iCur ) continue;
005244 if( pOBExpr->iColumn!=iColumn ) continue;
005245 }else{
005246 Expr *pIxExpr = pIndex->aColExpr->a[j].pExpr;
005247 if( sqlite3ExprCompareSkip(pOBExpr, pIxExpr, iCur) ){
005248 continue;
005249 }
005250 }
005251 if( iColumn!=XN_ROWID ){
005252 pColl = sqlite3ExprNNCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr);
005253 if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue;
005254 }
005255 if( wctrlFlags & WHERE_DISTINCTBY ){
005256 pLoop->u.btree.nDistinctCol = j+1;
005257 }
005258 isMatch = 1;
005259 break;
005260 }
005261 if( isMatch && (wctrlFlags & WHERE_GROUPBY)==0 ){
005262 /* Make sure the sort order is compatible in an ORDER BY clause.
005263 ** Sort order is irrelevant for a GROUP BY clause. */
005264 if( revSet ){
005265 if( (rev ^ revIdx)
005266 != (pOrderBy->a[i].fg.sortFlags&KEYINFO_ORDER_DESC)
005267 ){
005268 isMatch = 0;
005269 }
005270 }else{
005271 rev = revIdx ^ (pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_DESC);
005272 if( rev ) *pRevMask |= MASKBIT(iLoop);
005273 revSet = 1;
005274 }
005275 }
005276 if( isMatch && (pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_BIGNULL) ){
005277 if( j==pLoop->u.btree.nEq ){
005278 pLoop->wsFlags |= WHERE_BIGNULL_SORT;
005279 }else{
005280 isMatch = 0;
005281 }
005282 }
005283 if( isMatch ){
005284 if( iColumn==XN_ROWID ){
005285 testcase( distinctColumns==0 );
005286 distinctColumns = 1;
005287 }
005288 obSat |= MASKBIT(i);
005289 }else{
005290 /* No match found */
005291 if( j==0 || j<nKeyCol ){
005292 testcase( isOrderDistinct!=0 );
005293 isOrderDistinct = 0;
005294 }
005295 break;
005296 }
005297 } /* end Loop over all index columns */
005298 if( distinctColumns ){
005299 testcase( isOrderDistinct==0 );
005300 isOrderDistinct = 1;
005301 }
005302 } /* end-if not one-row */
005303
005304 /* Mark off any other ORDER BY terms that reference pLoop */
005305 if( isOrderDistinct ){
005306 orderDistinctMask |= pLoop->maskSelf;
005307 for(i=0; i<nOrderBy; i++){
005308 Expr *p;
005309 Bitmask mTerm;
005310 if( MASKBIT(i) & obSat ) continue;
005311 p = pOrderBy->a[i].pExpr;
005312 mTerm = sqlite3WhereExprUsage(&pWInfo->sMaskSet,p);
005313 if( mTerm==0 && !sqlite3ExprIsConstant(0,p) ) continue;
005314 if( (mTerm&~orderDistinctMask)==0 ){
005315 obSat |= MASKBIT(i);
005316 }
005317 }
005318 }
005319 } /* End the loop over all WhereLoops from outer-most down to inner-most */
005320 if( obSat==obDone ) return (i8)nOrderBy;
005321 if( !isOrderDistinct ){
005322 for(i=nOrderBy-1; i>0; i--){
005323 Bitmask m = ALWAYS(i<BMS) ? MASKBIT(i) - 1 : 0;
005324 if( (obSat&m)==m ) return i;
005325 }
005326 return 0;
005327 }
005328 return -1;
005329 }
005330
005331
005332 /*
005333 ** If the WHERE_GROUPBY flag is set in the mask passed to sqlite3WhereBegin(),
005334 ** the planner assumes that the specified pOrderBy list is actually a GROUP
005335 ** BY clause - and so any order that groups rows as required satisfies the
005336 ** request.
005337 **
005338 ** Normally, in this case it is not possible for the caller to determine
005339 ** whether or not the rows are really being delivered in sorted order, or
005340 ** just in some other order that provides the required grouping. However,
005341 ** if the WHERE_SORTBYGROUP flag is also passed to sqlite3WhereBegin(), then
005342 ** this function may be called on the returned WhereInfo object. It returns
005343 ** true if the rows really will be sorted in the specified order, or false
005344 ** otherwise.
005345 **
005346 ** For example, assuming:
005347 **
005348 ** CREATE INDEX i1 ON t1(x, Y);
005349 **
005350 ** then
005351 **
005352 ** SELECT * FROM t1 GROUP BY x,y ORDER BY x,y; -- IsSorted()==1
005353 ** SELECT * FROM t1 GROUP BY y,x ORDER BY y,x; -- IsSorted()==0
005354 */
005355 int sqlite3WhereIsSorted(WhereInfo *pWInfo){
005356 assert( pWInfo->wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY) );
005357 assert( pWInfo->wctrlFlags & WHERE_SORTBYGROUP );
005358 return pWInfo->sorted;
005359 }
005360
005361 #ifdef WHERETRACE_ENABLED
005362 /* For debugging use only: */
005363 static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){
005364 static char zName[65];
005365 int i;
005366 for(i=0; i<nLoop; i++){ zName[i] = pPath->aLoop[i]->cId; }
005367 if( pLast ) zName[i++] = pLast->cId;
005368 zName[i] = 0;
005369 return zName;
005370 }
005371 #endif
005372
005373 /*
005374 ** Return the cost of sorting nRow rows, assuming that the keys have
005375 ** nOrderby columns and that the first nSorted columns are already in
005376 ** order.
005377 */
005378 static LogEst whereSortingCost(
005379 WhereInfo *pWInfo, /* Query planning context */
005380 LogEst nRow, /* Estimated number of rows to sort */
005381 int nOrderBy, /* Number of ORDER BY clause terms */
005382 int nSorted /* Number of initial ORDER BY terms naturally in order */
005383 ){
005384 /* Estimated cost of a full external sort, where N is
005385 ** the number of rows to sort is:
005386 **
005387 ** cost = (K * N * log(N)).
005388 **
005389 ** Or, if the order-by clause has X terms but only the last Y
005390 ** terms are out of order, then block-sorting will reduce the
005391 ** sorting cost to:
005392 **
005393 ** cost = (K * N * log(N)) * (Y/X)
005394 **
005395 ** The constant K is at least 2.0 but will be larger if there are a
005396 ** large number of columns to be sorted, as the sorting time is
005397 ** proportional to the amount of content to be sorted. The algorithm
005398 ** does not currently distinguish between fat columns (BLOBs and TEXTs)
005399 ** and skinny columns (INTs). It just uses the number of columns as
005400 ** an approximation for the row width.
005401 **
005402 ** And extra factor of 2.0 or 3.0 is added to the sorting cost if the sort
005403 ** is built using OP_IdxInsert and OP_Sort rather than with OP_SorterInsert.
005404 */
005405 LogEst rSortCost, nCol;
005406 assert( pWInfo->pSelect!=0 );
005407 assert( pWInfo->pSelect->pEList!=0 );
005408 /* TUNING: sorting cost proportional to the number of output columns: */
005409 nCol = sqlite3LogEst((pWInfo->pSelect->pEList->nExpr+59)/30);
005410 rSortCost = nRow + nCol;
005411 if( nSorted>0 ){
005412 /* Scale the result by (Y/X) */
005413 rSortCost += sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66;
005414 }
005415
005416 /* Multiple by log(M) where M is the number of output rows.
005417 ** Use the LIMIT for M if it is smaller. Or if this sort is for
005418 ** a DISTINCT operator, M will be the number of distinct output
005419 ** rows, so fudge it downwards a bit.
005420 */
005421 if( (pWInfo->wctrlFlags & WHERE_USE_LIMIT)!=0 ){
005422 rSortCost += 10; /* TUNING: Extra 2.0x if using LIMIT */
005423 if( nSorted!=0 ){
005424 rSortCost += 6; /* TUNING: Extra 1.5x if also using partial sort */
005425 }
005426 if( pWInfo->iLimit<nRow ){
005427 nRow = pWInfo->iLimit;
005428 }
005429 }else if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT) ){
005430 /* TUNING: In the sort for a DISTINCT operator, assume that the DISTINCT
005431 ** reduces the number of output rows by a factor of 2 */
005432 if( nRow>10 ){ nRow -= 10; assert( 10==sqlite3LogEst(2) ); }
005433 }
005434 rSortCost += estLog(nRow);
005435 return rSortCost;
005436 }
005437
005438 /*
005439 ** Compute the maximum number of paths in the solver algorithm, for
005440 ** queries that have three or more terms in the FROM clause. Queries with
005441 ** two or fewer FROM clause terms are handled by the caller.
005442 **
005443 ** Query planning is NP-hard. We must limit the number of paths at
005444 ** each step of the solver search algorithm to avoid exponential behavior.
005445 **
005446 ** The value returned is a tuning parameter. Currently the value is:
005447 **
005448 ** 18 for star queries
005449 ** 12 otherwise
005450 **
005451 ** For the purposes of this heuristic, a star-query is defined as a query
005452 ** with a large central table that is joined using an INNER JOIN,
005453 ** not CROSS or OUTER JOINs, against four or more smaller tables.
005454 ** The central table is called the "fact" table. The smaller tables
005455 ** that get joined are "dimension tables". Also, any table that is
005456 ** self-joined cannot be a dimension table; we assume that dimension
005457 ** tables may only be joined against fact tables.
005458 **
005459 ** SIDE EFFECT: (and really the whole point of this subroutine)
005460 **
005461 ** If pWInfo describes a star-query, then the cost for SCANs of dimension
005462 ** WhereLoops is increased to be slightly larger than the cost of a SCAN
005463 ** in the fact table. Only SCAN costs are increased. SEARCH costs are
005464 ** unchanged. This heuristic helps keep fact tables in outer loops. Without
005465 ** this heuristic, paths with fact tables in outer loops tend to get pruned
005466 ** by the mxChoice limit on the number of paths, resulting in poor query
005467 ** plans. See the starschema1.test test module for examples of queries
005468 ** that need this heuristic to find good query plans.
005469 **
005470 ** This heuristic can be completely disabled, so that no query is
005471 ** considered a star-query, using SQLITE_TESTCTRL_OPTIMIZATION to
005472 ** disable the SQLITE_StarQuery optimization. In the CLI, the command
005473 ** to do that is: ".testctrl opt -starquery".
005474 **
005475 ** HISTORICAL NOTES:
005476 **
005477 ** This optimization was first added on 2024-05-09 by check-in 38db9b5c83d.
005478 ** The original optimization reduced the cost and output size estimate for
005479 ** fact tables to help them move to outer loops. But months later (as people
005480 ** started upgrading) performance regression reports started caming in,
005481 ** including:
005482 **
005483 ** forum post b18ef983e68d06d1 (2024-12-21)
005484 ** forum post 0025389d0860af82 (2025-01-14)
005485 ** forum post d87570a145599033 (2025-01-17)
005486 **
005487 ** To address these, the criteria for a star-query was tightened to exclude
005488 ** cases where the fact and dimensions are separated by an outer join, and
005489 ** the affect of star-schema detection was changed to increase the rRun cost
005490 ** on just full table scans of dimension tables, rather than reducing costs
005491 ** in the all access methods of the fact table.
005492 */
005493 static int computeMxChoice(WhereInfo *pWInfo){
005494 int nLoop = pWInfo->nLevel; /* Number of terms in the join */
005495 WhereLoop *pWLoop; /* For looping over WhereLoops */
005496
005497 #ifdef SQLITE_DEBUG
005498 /* The star-query detection code below makes use of the following
005499 ** properties of the WhereLoop list, so verify them before
005500 ** continuing:
005501 ** (1) .maskSelf is the bitmask corresponding to .iTab
005502 ** (2) The WhereLoop list is in ascending .iTab order
005503 */
005504 for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){
005505 assert( pWLoop->maskSelf==MASKBIT(pWLoop->iTab) );
005506 assert( pWLoop->pNextLoop==0 || pWLoop->iTab<=pWLoop->pNextLoop->iTab );
005507 }
005508 #endif /* SQLITE_DEBUG */
005509
005510 if( nLoop>=5
005511 && !pWInfo->bStarDone
005512 && OptimizationEnabled(pWInfo->pParse->db, SQLITE_StarQuery)
005513 ){
005514 SrcItem *aFromTabs; /* All terms of the FROM clause */
005515 int iFromIdx; /* Term of FROM clause is the candidate fact-table */
005516 Bitmask m; /* Bitmask for candidate fact-table */
005517 Bitmask mSelfJoin = 0; /* Tables that cannot be dimension tables */
005518 WhereLoop *pStart; /* Where to start searching for dimension-tables */
005519
005520 pWInfo->bStarDone = 1; /* Only do this computation once */
005521
005522 /* Look for fact tables with four or more dimensions where the
005523 ** dimension tables are not separately from the fact tables by an outer
005524 ** or cross join. Adjust cost weights if found.
005525 */
005526 assert( !pWInfo->bStarUsed );
005527 aFromTabs = pWInfo->pTabList->a;
005528 pStart = pWInfo->pLoops;
005529 for(iFromIdx=0, m=1; iFromIdx<nLoop; iFromIdx++, m<<=1){
005530 int nDep = 0; /* Number of dimension tables */
005531 LogEst mxRun; /* Maximum SCAN cost of a fact table */
005532 Bitmask mSeen = 0; /* Mask of dimension tables */
005533 SrcItem *pFactTab; /* The candidate fact table */
005534
005535 pFactTab = aFromTabs + iFromIdx;
005536 if( (pFactTab->fg.jointype & (JT_OUTER|JT_CROSS))!=0 ){
005537 /* If the candidate fact-table is the right table of an outer join
005538 ** restrict the search for dimension-tables to be tables to the right
005539 ** of the fact-table. */
005540 if( iFromIdx+4 > nLoop ) break; /* Impossible to reach nDep>=4 */
005541 while( pStart && pStart->iTab<=iFromIdx ){
005542 pStart = pStart->pNextLoop;
005543 }
005544 }
005545 for(pWLoop=pStart; pWLoop; pWLoop=pWLoop->pNextLoop){
005546 if( (aFromTabs[pWLoop->iTab].fg.jointype & (JT_OUTER|JT_CROSS))!=0 ){
005547 /* Fact-tables and dimension-tables cannot be separated by an
005548 ** outer join (at least for the definition of fact- and dimension-
005549 ** used by this heuristic). */
005550 break;
005551 }
005552 if( (pWLoop->prereq & m)!=0 /* pWInfo depends on iFromIdx */
005553 && (pWLoop->maskSelf & mSeen)==0 /* pWInfo not already a dependency */
005554 && (pWLoop->maskSelf & mSelfJoin)==0 /* Not a self-join */
005555 ){
005556 if( aFromTabs[pWLoop->iTab].pSTab==pFactTab->pSTab ){
005557 mSelfJoin |= m;
005558 }else{
005559 nDep++;
005560 mSeen |= pWLoop->maskSelf;
005561 }
005562 }
005563 }
005564 if( nDep<=3 ) continue;
005565
005566 /* If we reach this point, it means that pFactTab is a fact table
005567 ** with four or more dimensions connected by inner joins. Proceed
005568 ** to make cost adjustments. */
005569
005570 #ifdef WHERETRACE_ENABLED
005571 /* Make sure rStarDelta values are initialized */
005572 if( !pWInfo->bStarUsed ){
005573 for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){
005574 pWLoop->rStarDelta = 0;
005575 }
005576 }
005577 #endif
005578 pWInfo->bStarUsed = 1;
005579
005580 /* Compute the maximum cost of any WhereLoop for the
005581 ** fact table plus one epsilon */
005582 mxRun = LOGEST_MIN;
005583 for(pWLoop=pStart; pWLoop; pWLoop=pWLoop->pNextLoop){
005584 if( pWLoop->iTab<iFromIdx ) continue;
005585 if( pWLoop->iTab>iFromIdx ) break;
005586 if( pWLoop->rRun>mxRun ) mxRun = pWLoop->rRun;
005587 }
005588 if( ALWAYS(mxRun<LOGEST_MAX) ) mxRun++;
005589
005590 /* Increase the cost of table scans for dimension tables to be
005591 ** slightly more than the maximum cost of the fact table */
005592 for(pWLoop=pStart; pWLoop; pWLoop=pWLoop->pNextLoop){
005593 if( (pWLoop->maskSelf & mSeen)==0 ) continue;
005594 if( pWLoop->nLTerm ) continue;
005595 if( pWLoop->rRun<mxRun ){
005596 #ifdef WHERETRACE_ENABLED /* 0x80000 */
005597 if( sqlite3WhereTrace & 0x80000 ){
005598 SrcItem *pDim = aFromTabs + pWLoop->iTab;
005599 sqlite3DebugPrintf(
005600 "Increase SCAN cost of dimension %s(%d) of fact %s(%d) to %d\n",
005601 pDim->zAlias ? pDim->zAlias: pDim->pSTab->zName, pWLoop->iTab,
005602 pFactTab->zAlias ? pFactTab->zAlias : pFactTab->pSTab->zName,
005603 iFromIdx, mxRun
005604 );
005605 }
005606 pWLoop->rStarDelta = mxRun - pWLoop->rRun;
005607 #endif /* WHERETRACE_ENABLED */
005608 pWLoop->rRun = mxRun;
005609 }
005610 }
005611 }
005612 #ifdef WHERETRACE_ENABLED /* 0x80000 */
005613 if( (sqlite3WhereTrace & 0x80000)!=0 && pWInfo->bStarUsed ){
005614 sqlite3DebugPrintf("WhereLoops changed by star-query heuristic:\n");
005615 for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){
005616 if( pWLoop->rStarDelta ){
005617 sqlite3WhereLoopPrint(pWLoop, &pWInfo->sWC);
005618 }
005619 }
005620 }
005621 #endif
005622 }
005623 return pWInfo->bStarUsed ? 18 : 12;
005624 }
005625
005626 /*
005627 ** Two WhereLoop objects, pCandidate and pBaseline, are known to have the
005628 ** same cost. Look deep into each to see if pCandidate is even slightly
005629 ** better than pBaseline. Return false if it is, if pCandidate is is preferred.
005630 ** Return true if pBaseline is preferred or if we cannot tell the difference.
005631 **
005632 ** Result Meaning
005633 ** -------- ----------------------------------------------------------
005634 ** true We cannot tell the difference in pCandidate and pBaseline
005635 ** false pCandidate seems like a better choice than pBaseline
005636 */
005637 static SQLITE_NOINLINE int whereLoopIsNoBetter(
005638 const WhereLoop *pCandidate,
005639 const WhereLoop *pBaseline
005640 ){
005641 if( (pCandidate->wsFlags & WHERE_INDEXED)==0 ) return 1;
005642 if( (pBaseline->wsFlags & WHERE_INDEXED)==0 ) return 1;
005643 if( pCandidate->u.btree.pIndex->szIdxRow <
005644 pBaseline->u.btree.pIndex->szIdxRow ) return 0;
005645 return 1;
005646 }
005647
005648 /*
005649 ** Given the list of WhereLoop objects at pWInfo->pLoops, this routine
005650 ** attempts to find the lowest cost path that visits each WhereLoop
005651 ** once. This path is then loaded into the pWInfo->a[].pWLoop fields.
005652 **
005653 ** Assume that the total number of output rows that will need to be sorted
005654 ** will be nRowEst (in the 10*log2 representation). Or, ignore sorting
005655 ** costs if nRowEst==0.
005656 **
005657 ** Return SQLITE_OK on success or SQLITE_NOMEM of a memory allocation
005658 ** error occurs.
005659 */
005660 static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
005661 int mxChoice; /* Maximum number of simultaneous paths tracked */
005662 int nLoop; /* Number of terms in the join */
005663 Parse *pParse; /* Parsing context */
005664 int iLoop; /* Loop counter over the terms of the join */
005665 int ii, jj; /* Loop counters */
005666 int mxI = 0; /* Index of next entry to replace */
005667 int nOrderBy; /* Number of ORDER BY clause terms */
005668 LogEst mxCost = 0; /* Maximum cost of a set of paths */
005669 LogEst mxUnsort = 0; /* Maximum unsorted cost of a set of path */
005670 int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */
005671 WherePath *aFrom; /* All nFrom paths at the previous level */
005672 WherePath *aTo; /* The nTo best paths at the current level */
005673 WherePath *pFrom; /* An element of aFrom[] that we are working on */
005674 WherePath *pTo; /* An element of aTo[] that we are working on */
005675 WhereLoop *pWLoop; /* One of the WhereLoop objects */
005676 WhereLoop **pX; /* Used to divy up the pSpace memory */
005677 LogEst *aSortCost = 0; /* Sorting and partial sorting costs */
005678 char *pSpace; /* Temporary memory used by this routine */
005679 int nSpace; /* Bytes of space allocated at pSpace */
005680
005681 pParse = pWInfo->pParse;
005682 nLoop = pWInfo->nLevel;
005683 WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d, nQueryLoop=%d)\n",
005684 nRowEst, pParse->nQueryLoop));
005685 /* TUNING: mxChoice is the maximum number of possible paths to preserve
005686 ** at each step. Based on the number of loops in the FROM clause:
005687 **
005688 ** nLoop mxChoice
005689 ** ----- --------
005690 ** 1 1 // the most common case
005691 ** 2 5
005692 ** 3+ 12 or 18 // see computeMxChoice()
005693 */
005694 if( nLoop<=1 ){
005695 mxChoice = 1;
005696 }else if( nLoop==2 ){
005697 mxChoice = 5;
005698 }else if( pParse->nErr ){
005699 mxChoice = 1;
005700 }else{
005701 mxChoice = computeMxChoice(pWInfo);
005702 }
005703 assert( nLoop<=pWInfo->pTabList->nSrc );
005704
005705 /* If nRowEst is zero and there is an ORDER BY clause, ignore it. In this
005706 ** case the purpose of this call is to estimate the number of rows returned
005707 ** by the overall query. Once this estimate has been obtained, the caller
005708 ** will invoke this function a second time, passing the estimate as the
005709 ** nRowEst parameter. */
005710 if( pWInfo->pOrderBy==0 || nRowEst==0 ){
005711 nOrderBy = 0;
005712 }else{
005713 nOrderBy = pWInfo->pOrderBy->nExpr;
005714 }
005715
005716 /* Allocate and initialize space for aTo, aFrom and aSortCost[] */
005717 nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2;
005718 nSpace += sizeof(LogEst) * nOrderBy;
005719 pSpace = sqlite3StackAllocRawNN(pParse->db, nSpace);
005720 if( pSpace==0 ) return SQLITE_NOMEM_BKPT;
005721 aTo = (WherePath*)pSpace;
005722 aFrom = aTo+mxChoice;
005723 memset(aFrom, 0, sizeof(aFrom[0]));
005724 pX = (WhereLoop**)(aFrom+mxChoice);
005725 for(ii=mxChoice*2, pFrom=aTo; ii>0; ii--, pFrom++, pX += nLoop){
005726 pFrom->aLoop = pX;
005727 }
005728 if( nOrderBy ){
005729 /* If there is an ORDER BY clause and it is not being ignored, set up
005730 ** space for the aSortCost[] array. Each element of the aSortCost array
005731 ** is either zero - meaning it has not yet been initialized - or the
005732 ** cost of sorting nRowEst rows of data where the first X terms of
005733 ** the ORDER BY clause are already in order, where X is the array
005734 ** index. */
005735 aSortCost = (LogEst*)pX;
005736 memset(aSortCost, 0, sizeof(LogEst) * nOrderBy);
005737 }
005738 assert( aSortCost==0 || &pSpace[nSpace]==(char*)&aSortCost[nOrderBy] );
005739 assert( aSortCost!=0 || &pSpace[nSpace]==(char*)pX );
005740
005741 /* Seed the search with a single WherePath containing zero WhereLoops.
005742 **
005743 ** TUNING: Do not let the number of iterations go above 28. If the cost
005744 ** of computing an automatic index is not paid back within the first 28
005745 ** rows, then do not use the automatic index. */
005746 aFrom[0].nRow = MIN(pParse->nQueryLoop, 48); assert( 48==sqlite3LogEst(28) );
005747 nFrom = 1;
005748 assert( aFrom[0].isOrdered==0 );
005749 if( nOrderBy ){
005750 /* If nLoop is zero, then there are no FROM terms in the query. Since
005751 ** in this case the query may return a maximum of one row, the results
005752 ** are already in the requested order. Set isOrdered to nOrderBy to
005753 ** indicate this. Or, if nLoop is greater than zero, set isOrdered to
005754 ** -1, indicating that the result set may or may not be ordered,
005755 ** depending on the loops added to the current plan. */
005756 aFrom[0].isOrdered = nLoop>0 ? -1 : nOrderBy;
005757 }
005758
005759 /* Compute successively longer WherePaths using the previous generation
005760 ** of WherePaths as the basis for the next. Keep track of the mxChoice
005761 ** best paths at each generation */
005762 for(iLoop=0; iLoop<nLoop; iLoop++){
005763 nTo = 0;
005764 for(ii=0, pFrom=aFrom; ii<nFrom; ii++, pFrom++){
005765 for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){
005766 LogEst nOut; /* Rows visited by (pFrom+pWLoop) */
005767 LogEst rCost; /* Cost of path (pFrom+pWLoop) */
005768 LogEst rUnsort; /* Unsorted cost of (pFrom+pWLoop) */
005769 i8 isOrdered; /* isOrdered for (pFrom+pWLoop) */
005770 Bitmask maskNew; /* Mask of src visited by (..) */
005771 Bitmask revMask; /* Mask of rev-order loops for (..) */
005772
005773 if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue;
005774 if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue;
005775 if( (pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 && pFrom->nRow<3 ){
005776 /* Do not use an automatic index if the this loop is expected
005777 ** to run less than 1.25 times. It is tempting to also exclude
005778 ** automatic index usage on an outer loop, but sometimes an automatic
005779 ** index is useful in the outer loop of a correlated subquery. */
005780 assert( 10==sqlite3LogEst(2) );
005781 continue;
005782 }
005783
005784 /* At this point, pWLoop is a candidate to be the next loop.
005785 ** Compute its cost */
005786 rUnsort = pWLoop->rRun + pFrom->nRow;
005787 if( pWLoop->rSetup ){
005788 rUnsort = sqlite3LogEstAdd(pWLoop->rSetup, rUnsort);
005789 }
005790 rUnsort = sqlite3LogEstAdd(rUnsort, pFrom->rUnsort);
005791 nOut = pFrom->nRow + pWLoop->nOut;
005792 maskNew = pFrom->maskLoop | pWLoop->maskSelf;
005793 isOrdered = pFrom->isOrdered;
005794 if( isOrdered<0 ){
005795 revMask = 0;
005796 isOrdered = wherePathSatisfiesOrderBy(pWInfo,
005797 pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags,
005798 iLoop, pWLoop, &revMask);
005799 }else{
005800 revMask = pFrom->revLoop;
005801 }
005802 if( isOrdered>=0 && isOrdered<nOrderBy ){
005803 if( aSortCost[isOrdered]==0 ){
005804 aSortCost[isOrdered] = whereSortingCost(
005805 pWInfo, nRowEst, nOrderBy, isOrdered
005806 );
005807 }
005808 /* TUNING: Add a small extra penalty (3) to sorting as an
005809 ** extra encouragement to the query planner to select a plan
005810 ** where the rows emerge in the correct order without any sorting
005811 ** required. */
005812 rCost = sqlite3LogEstAdd(rUnsort, aSortCost[isOrdered]) + 3;
005813
005814 WHERETRACE(0x002,
005815 ("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n",
005816 aSortCost[isOrdered], (nOrderBy-isOrdered), nOrderBy,
005817 rUnsort, rCost));
005818 }else{
005819 rCost = rUnsort;
005820 rUnsort -= 2; /* TUNING: Slight bias in favor of no-sort plans */
005821 }
005822
005823 /* Check to see if pWLoop should be added to the set of
005824 ** mxChoice best-so-far paths.
005825 **
005826 ** First look for an existing path among best-so-far paths
005827 ** that covers the same set of loops and has the same isOrdered
005828 ** setting as the current path candidate.
005829 **
005830 ** The term "((pTo->isOrdered^isOrdered)&0x80)==0" is equivalent
005831 ** to (pTo->isOrdered==(-1))==(isOrdered==(-1))" for the range
005832 ** of legal values for isOrdered, -1..64.
005833 */
005834 testcase( nTo==0 );
005835 for(jj=0, pTo=aTo; jj<nTo; jj++, pTo++){
005836 if( pTo->maskLoop==maskNew
005837 && ((pTo->isOrdered^isOrdered)&0x80)==0
005838 ){
005839 testcase( jj==nTo-1 );
005840 break;
005841 }
005842 }
005843 if( jj>=nTo ){
005844 /* None of the existing best-so-far paths match the candidate. */
005845 if( nTo>=mxChoice
005846 && (rCost>mxCost || (rCost==mxCost && rUnsort>=mxUnsort))
005847 ){
005848 /* The current candidate is no better than any of the mxChoice
005849 ** paths currently in the best-so-far buffer. So discard
005850 ** this candidate as not viable. */
005851 #ifdef WHERETRACE_ENABLED /* 0x4 */
005852 if( sqlite3WhereTrace&0x4 ){
005853 sqlite3DebugPrintf("Skip %s cost=%-3d,%3d,%3d order=%c\n",
005854 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsort,
005855 isOrdered>=0 ? isOrdered+'0' : '?');
005856 }
005857 #endif
005858 continue;
005859 }
005860 /* If we reach this points it means that the new candidate path
005861 ** needs to be added to the set of best-so-far paths. */
005862 if( nTo<mxChoice ){
005863 /* Increase the size of the aTo set by one */
005864 jj = nTo++;
005865 }else{
005866 /* New path replaces the prior worst to keep count below mxChoice */
005867 jj = mxI;
005868 }
005869 pTo = &aTo[jj];
005870 #ifdef WHERETRACE_ENABLED /* 0x4 */
005871 if( sqlite3WhereTrace&0x4 ){
005872 sqlite3DebugPrintf("New %s cost=%-3d,%3d,%3d order=%c\n",
005873 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsort,
005874 isOrdered>=0 ? isOrdered+'0' : '?');
005875 }
005876 #endif
005877 }else{
005878 /* Control reaches here if best-so-far path pTo=aTo[jj] covers the
005879 ** same set of loops and has the same isOrdered setting as the
005880 ** candidate path. Check to see if the candidate should replace
005881 ** pTo or if the candidate should be skipped.
005882 **
005883 ** The conditional is an expanded vector comparison equivalent to:
005884 ** (pTo->rCost,pTo->nRow,pTo->rUnsort) <= (rCost,nOut,rUnsort)
005885 */
005886 if( (pTo->rCost<rCost)
005887 || (pTo->rCost==rCost && pTo->nRow<nOut)
005888 || (pTo->rCost==rCost && pTo->nRow==nOut && pTo->rUnsort<rUnsort)
005889 || (pTo->rCost==rCost && pTo->nRow==nOut && pTo->rUnsort==rUnsort
005890 && whereLoopIsNoBetter(pWLoop, pTo->aLoop[iLoop]) )
005891 ){
005892 #ifdef WHERETRACE_ENABLED /* 0x4 */
005893 if( sqlite3WhereTrace&0x4 ){
005894 sqlite3DebugPrintf(
005895 "Skip %s cost=%-3d,%3d,%3d order=%c",
005896 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsort,
005897 isOrdered>=0 ? isOrdered+'0' : '?');
005898 sqlite3DebugPrintf(" vs %s cost=%-3d,%3d,%3d order=%c\n",
005899 wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
005900 pTo->rUnsort, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?');
005901 }
005902 #endif
005903 /* Discard the candidate path from further consideration */
005904 testcase( pTo->rCost==rCost );
005905 continue;
005906 }
005907 testcase( pTo->rCost==rCost+1 );
005908 /* Control reaches here if the candidate path is better than the
005909 ** pTo path. Replace pTo with the candidate. */
005910 #ifdef WHERETRACE_ENABLED /* 0x4 */
005911 if( sqlite3WhereTrace&0x4 ){
005912 sqlite3DebugPrintf(
005913 "Update %s cost=%-3d,%3d,%3d order=%c",
005914 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsort,
005915 isOrdered>=0 ? isOrdered+'0' : '?');
005916 sqlite3DebugPrintf(" was %s cost=%-3d,%3d,%3d order=%c\n",
005917 wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
005918 pTo->rUnsort, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?');
005919 }
005920 #endif
005921 }
005922 /* pWLoop is a winner. Add it to the set of best so far */
005923 pTo->maskLoop = pFrom->maskLoop | pWLoop->maskSelf;
005924 pTo->revLoop = revMask;
005925 pTo->nRow = nOut;
005926 pTo->rCost = rCost;
005927 pTo->rUnsort = rUnsort;
005928 pTo->isOrdered = isOrdered;
005929 memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop);
005930 pTo->aLoop[iLoop] = pWLoop;
005931 if( nTo>=mxChoice ){
005932 mxI = 0;
005933 mxCost = aTo[0].rCost;
005934 mxUnsort = aTo[0].nRow;
005935 for(jj=1, pTo=&aTo[1]; jj<mxChoice; jj++, pTo++){
005936 if( pTo->rCost>mxCost
005937 || (pTo->rCost==mxCost && pTo->rUnsort>mxUnsort)
005938 ){
005939 mxCost = pTo->rCost;
005940 mxUnsort = pTo->rUnsort;
005941 mxI = jj;
005942 }
005943 }
005944 }
005945 }
005946 }
005947
005948 #ifdef WHERETRACE_ENABLED /* >=2 */
005949 if( sqlite3WhereTrace & 0x02 ){
005950 LogEst rMin, rFloor = 0;
005951 int nDone = 0;
005952 int nProgress;
005953 sqlite3DebugPrintf("---- after round %d ----\n", iLoop);
005954 do{
005955 nProgress = 0;
005956 rMin = 0x7fff;
005957 for(ii=0, pTo=aTo; ii<nTo; ii++, pTo++){
005958 if( pTo->rCost>rFloor && pTo->rCost<rMin ) rMin = pTo->rCost;
005959 }
005960 for(ii=0, pTo=aTo; ii<nTo; ii++, pTo++){
005961 if( pTo->rCost==rMin ){
005962 sqlite3DebugPrintf(" %s cost=%-3d nrow=%-3d order=%c",
005963 wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
005964 pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?');
005965 if( pTo->isOrdered>0 ){
005966 sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop);
005967 }else{
005968 sqlite3DebugPrintf("\n");
005969 }
005970 nDone++;
005971 nProgress++;
005972 }
005973 }
005974 rFloor = rMin;
005975 }while( nDone<nTo && nProgress>0 );
005976 }
005977 #endif
005978
005979 /* Swap the roles of aFrom and aTo for the next generation */
005980 pFrom = aTo;
005981 aTo = aFrom;
005982 aFrom = pFrom;
005983 nFrom = nTo;
005984 }
005985
005986 if( nFrom==0 ){
005987 sqlite3ErrorMsg(pParse, "no query solution");
005988 sqlite3StackFreeNN(pParse->db, pSpace);
005989 return SQLITE_ERROR;
005990 }
005991
005992 /* Find the lowest cost path. pFrom will be left pointing to that path */
005993 pFrom = aFrom;
005994 for(ii=1; ii<nFrom; ii++){
005995 if( pFrom->rCost>aFrom[ii].rCost ) pFrom = &aFrom[ii];
005996 }
005997 assert( pWInfo->nLevel==nLoop );
005998 /* Load the lowest cost path into pWInfo */
005999 for(iLoop=0; iLoop<nLoop; iLoop++){
006000 WhereLevel *pLevel = pWInfo->a + iLoop;
006001 pLevel->pWLoop = pWLoop = pFrom->aLoop[iLoop];
006002 pLevel->iFrom = pWLoop->iTab;
006003 pLevel->iTabCur = pWInfo->pTabList->a[pLevel->iFrom].iCursor;
006004 }
006005 if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0
006006 && (pWInfo->wctrlFlags & WHERE_DISTINCTBY)==0
006007 && pWInfo->eDistinct==WHERE_DISTINCT_NOOP
006008 && nRowEst
006009 ){
006010 Bitmask notUsed;
006011 int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pResultSet, pFrom,
006012 WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used);
006013 if( rc==pWInfo->pResultSet->nExpr ){
006014 pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
006015 }
006016 }
006017 pWInfo->bOrderedInnerLoop = 0;
006018 if( pWInfo->pOrderBy ){
006019 pWInfo->nOBSat = pFrom->isOrdered;
006020 if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){
006021 if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){
006022 pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
006023 }
006024 /* vvv--- See check-in [12ad822d9b827777] on 2023-03-16 ---vvv */
006025 assert( pWInfo->pSelect->pOrderBy==0
006026 || pWInfo->nOBSat <= pWInfo->pSelect->pOrderBy->nExpr );
006027 }else{
006028 pWInfo->revMask = pFrom->revLoop;
006029 if( pWInfo->nOBSat<=0 ){
006030 pWInfo->nOBSat = 0;
006031 if( nLoop>0 ){
006032 u32 wsFlags = pFrom->aLoop[nLoop-1]->wsFlags;
006033 if( (wsFlags & WHERE_ONEROW)==0
006034 && (wsFlags&(WHERE_IPK|WHERE_COLUMN_IN))!=(WHERE_IPK|WHERE_COLUMN_IN)
006035 ){
006036 Bitmask m = 0;
006037 int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom,
006038 WHERE_ORDERBY_LIMIT, nLoop-1, pFrom->aLoop[nLoop-1], &m);
006039 testcase( wsFlags & WHERE_IPK );
006040 testcase( wsFlags & WHERE_COLUMN_IN );
006041 if( rc==pWInfo->pOrderBy->nExpr ){
006042 pWInfo->bOrderedInnerLoop = 1;
006043 pWInfo->revMask = m;
006044 }
006045 }
006046 }
006047 }else if( nLoop
006048 && pWInfo->nOBSat==1
006049 && (pWInfo->wctrlFlags & (WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX))!=0
006050 ){
006051 pWInfo->bOrderedInnerLoop = 1;
006052 }
006053 }
006054 if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP)
006055 && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr && nLoop>0
006056 ){
006057 Bitmask revMask = 0;
006058 int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy,
006059 pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], &revMask
006060 );
006061 assert( pWInfo->sorted==0 );
006062 if( nOrder==pWInfo->pOrderBy->nExpr ){
006063 pWInfo->sorted = 1;
006064 pWInfo->revMask = revMask;
006065 }
006066 }
006067 }
006068
006069 pWInfo->nRowOut = pFrom->nRow;
006070 #ifdef WHERETRACE_ENABLED
006071 pWInfo->rTotalCost = pFrom->rCost;
006072 #endif
006073
006074 /* Free temporary memory and return success */
006075 sqlite3StackFreeNN(pParse->db, pSpace);
006076 return SQLITE_OK;
006077 }
006078
006079 /*
006080 ** This routine implements a heuristic designed to improve query planning.
006081 ** This routine is called in between the first and second call to
006082 ** wherePathSolver(). Hence the name "Interstage" "Heuristic".
006083 **
006084 ** The first call to wherePathSolver() (hereafter just "solver()") computes
006085 ** the best path without regard to the order of the outputs. The second call
006086 ** to the solver() builds upon the first call to try to find an alternative
006087 ** path that satisfies the ORDER BY clause.
006088 **
006089 ** This routine looks at the results of the first solver() run, and for
006090 ** every FROM clause term in the resulting query plan that uses an equality
006091 ** constraint against an index, disable other WhereLoops for that same
006092 ** FROM clause term that would try to do a full-table scan. This prevents
006093 ** an index search from being converted into a full-table scan in order to
006094 ** satisfy an ORDER BY clause, since even though we might get slightly better
006095 ** performance using the full-scan without sorting if the output size
006096 ** estimates are very precise, we might also get severe performance
006097 ** degradation using the full-scan if the output size estimate is too large.
006098 ** It is better to err on the side of caution.
006099 **
006100 ** Except, if the first solver() call generated a full-table scan in an outer
006101 ** loop then stop this analysis at the first full-scan, since the second
006102 ** solver() run might try to swap that full-scan for another in order to
006103 ** get the output into the correct order. In other words, we allow a
006104 ** rewrite like this:
006105 **
006106 ** First Solver() Second Solver()
006107 ** |-- SCAN t1 |-- SCAN t2
006108 ** |-- SEARCH t2 `-- SEARCH t1
006109 ** `-- SORT USING B-TREE
006110 **
006111 ** The purpose of this routine is to disallow rewrites such as:
006112 **
006113 ** First Solver() Second Solver()
006114 ** |-- SEARCH t1 |-- SCAN t2 <--- bad!
006115 ** |-- SEARCH t2 `-- SEARCH t1
006116 ** `-- SORT USING B-TREE
006117 **
006118 ** See test cases in test/whereN.test for the real-world query that
006119 ** originally provoked this heuristic.
006120 */
006121 static SQLITE_NOINLINE void whereInterstageHeuristic(WhereInfo *pWInfo){
006122 int i;
006123 #ifdef WHERETRACE_ENABLED
006124 int once = 0;
006125 #endif
006126 for(i=0; i<pWInfo->nLevel; i++){
006127 WhereLoop *p = pWInfo->a[i].pWLoop;
006128 if( p==0 ) break;
006129 if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 ) continue;
006130 if( (p->wsFlags & (WHERE_COLUMN_EQ|WHERE_COLUMN_NULL|WHERE_COLUMN_IN))!=0 ){
006131 u8 iTab = p->iTab;
006132 WhereLoop *pLoop;
006133 for(pLoop=pWInfo->pLoops; pLoop; pLoop=pLoop->pNextLoop){
006134 if( pLoop->iTab!=iTab ) continue;
006135 if( (pLoop->wsFlags & (WHERE_CONSTRAINT|WHERE_AUTO_INDEX))!=0 ){
006136 /* Auto-index and index-constrained loops allowed to remain */
006137 continue;
006138 }
006139 #ifdef WHERETRACE_ENABLED
006140 if( sqlite3WhereTrace & 0x80 ){
006141 if( once==0 ){
006142 sqlite3DebugPrintf("Loops disabled by interstage heuristic:\n");
006143 once = 1;
006144 }
006145 sqlite3WhereLoopPrint(pLoop, &pWInfo->sWC);
006146 }
006147 #endif /* WHERETRACE_ENABLED */
006148 pLoop->prereq = ALLBITS; /* Prevent 2nd solver() from using this one */
006149 }
006150 }else{
006151 break;
006152 }
006153 }
006154 }
006155
006156 /*
006157 ** Most queries use only a single table (they are not joins) and have
006158 ** simple == constraints against indexed fields. This routine attempts
006159 ** to plan those simple cases using much less ceremony than the
006160 ** general-purpose query planner, and thereby yield faster sqlite3_prepare()
006161 ** times for the common case.
006162 **
006163 ** Return non-zero on success, if this query can be handled by this
006164 ** no-frills query planner. Return zero if this query needs the
006165 ** general-purpose query planner.
006166 */
006167 static int whereShortCut(WhereLoopBuilder *pBuilder){
006168 WhereInfo *pWInfo;
006169 SrcItem *pItem;
006170 WhereClause *pWC;
006171 WhereTerm *pTerm;
006172 WhereLoop *pLoop;
006173 int iCur;
006174 int j;
006175 Table *pTab;
006176 Index *pIdx;
006177 WhereScan scan;
006178
006179 pWInfo = pBuilder->pWInfo;
006180 if( pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE ) return 0;
006181 assert( pWInfo->pTabList->nSrc>=1 );
006182 pItem = pWInfo->pTabList->a;
006183 pTab = pItem->pSTab;
006184 if( IsVirtual(pTab) ) return 0;
006185 if( pItem->fg.isIndexedBy || pItem->fg.notIndexed ){
006186 testcase( pItem->fg.isIndexedBy );
006187 testcase( pItem->fg.notIndexed );
006188 return 0;
006189 }
006190 iCur = pItem->iCursor;
006191 pWC = &pWInfo->sWC;
006192 pLoop = pBuilder->pNew;
006193 pLoop->wsFlags = 0;
006194 pLoop->nSkip = 0;
006195 pTerm = whereScanInit(&scan, pWC, iCur, -1, WO_EQ|WO_IS, 0);
006196 while( pTerm && pTerm->prereqRight ) pTerm = whereScanNext(&scan);
006197 if( pTerm ){
006198 testcase( pTerm->eOperator & WO_IS );
006199 pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW;
006200 pLoop->aLTerm[0] = pTerm;
006201 pLoop->nLTerm = 1;
006202 pLoop->u.btree.nEq = 1;
006203 /* TUNING: Cost of a rowid lookup is 10 */
006204 pLoop->rRun = 33; /* 33==sqlite3LogEst(10) */
006205 }else{
006206 for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
006207 int opMask;
006208 assert( pLoop->aLTermSpace==pLoop->aLTerm );
006209 if( !IsUniqueIndex(pIdx)
006210 || pIdx->pPartIdxWhere!=0
006211 || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace)
006212 ) continue;
006213 opMask = pIdx->uniqNotNull ? (WO_EQ|WO_IS) : WO_EQ;
006214 for(j=0; j<pIdx->nKeyCol; j++){
006215 pTerm = whereScanInit(&scan, pWC, iCur, j, opMask, pIdx);
006216 while( pTerm && pTerm->prereqRight ) pTerm = whereScanNext(&scan);
006217 if( pTerm==0 ) break;
006218 testcase( pTerm->eOperator & WO_IS );
006219 pLoop->aLTerm[j] = pTerm;
006220 }
006221 if( j!=pIdx->nKeyCol ) continue;
006222 pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_ONEROW|WHERE_INDEXED;
006223 if( pIdx->isCovering || (pItem->colUsed & pIdx->colNotIdxed)==0 ){
006224 pLoop->wsFlags |= WHERE_IDX_ONLY;
006225 }
006226 pLoop->nLTerm = j;
006227 pLoop->u.btree.nEq = j;
006228 pLoop->u.btree.pIndex = pIdx;
006229 /* TUNING: Cost of a unique index lookup is 15 */
006230 pLoop->rRun = 39; /* 39==sqlite3LogEst(15) */
006231 break;
006232 }
006233 }
006234 if( pLoop->wsFlags ){
006235 pLoop->nOut = (LogEst)1;
006236 pWInfo->a[0].pWLoop = pLoop;
006237 assert( pWInfo->sMaskSet.n==1 && iCur==pWInfo->sMaskSet.ix[0] );
006238 pLoop->maskSelf = 1; /* sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); */
006239 pWInfo->a[0].iTabCur = iCur;
006240 pWInfo->nRowOut = 1;
006241 if( pWInfo->pOrderBy ) pWInfo->nOBSat = pWInfo->pOrderBy->nExpr;
006242 if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){
006243 pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
006244 }
006245 if( scan.iEquiv>1 ) pLoop->wsFlags |= WHERE_TRANSCONS;
006246 #ifdef SQLITE_DEBUG
006247 pLoop->cId = '0';
006248 #endif
006249 #ifdef WHERETRACE_ENABLED
006250 if( sqlite3WhereTrace & 0x02 ){
006251 sqlite3DebugPrintf("whereShortCut() used to compute solution\n");
006252 }
006253 #endif
006254 return 1;
006255 }
006256 return 0;
006257 }
006258
006259 /*
006260 ** Helper function for exprIsDeterministic().
006261 */
006262 static int exprNodeIsDeterministic(Walker *pWalker, Expr *pExpr){
006263 if( pExpr->op==TK_FUNCTION && ExprHasProperty(pExpr, EP_ConstFunc)==0 ){
006264 pWalker->eCode = 0;
006265 return WRC_Abort;
006266 }
006267 return WRC_Continue;
006268 }
006269
006270 /*
006271 ** Return true if the expression contains no non-deterministic SQL
006272 ** functions. Do not consider non-deterministic SQL functions that are
006273 ** part of sub-select statements.
006274 */
006275 static int exprIsDeterministic(Expr *p){
006276 Walker w;
006277 memset(&w, 0, sizeof(w));
006278 w.eCode = 1;
006279 w.xExprCallback = exprNodeIsDeterministic;
006280 w.xSelectCallback = sqlite3SelectWalkFail;
006281 sqlite3WalkExpr(&w, p);
006282 return w.eCode;
006283 }
006284
006285
006286 #ifdef WHERETRACE_ENABLED
006287 /*
006288 ** Display all WhereLoops in pWInfo
006289 */
006290 static void showAllWhereLoops(WhereInfo *pWInfo, WhereClause *pWC){
006291 if( sqlite3WhereTrace ){ /* Display all of the WhereLoop objects */
006292 WhereLoop *p;
006293 int i;
006294 static const char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz"
006295 "ABCDEFGHIJKLMNOPQRSTUVWYXZ";
006296 for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){
006297 p->cId = zLabel[i%(sizeof(zLabel)-1)];
006298 sqlite3WhereLoopPrint(p, pWC);
006299 }
006300 }
006301 }
006302 # define WHERETRACE_ALL_LOOPS(W,C) showAllWhereLoops(W,C)
006303 #else
006304 # define WHERETRACE_ALL_LOOPS(W,C)
006305 #endif
006306
006307 /* Attempt to omit tables from a join that do not affect the result.
006308 ** For a table to not affect the result, the following must be true:
006309 **
006310 ** 1) The query must not be an aggregate.
006311 ** 2) The table must be the RHS of a LEFT JOIN.
006312 ** 3) Either the query must be DISTINCT, or else the ON or USING clause
006313 ** must contain a constraint that limits the scan of the table to
006314 ** at most a single row.
006315 ** 4) The table must not be referenced by any part of the query apart
006316 ** from its own USING or ON clause.
006317 ** 5) The table must not have an inner-join ON or USING clause if there is
006318 ** a RIGHT JOIN anywhere in the query. Otherwise the ON/USING clause
006319 ** might move from the right side to the left side of the RIGHT JOIN.
006320 ** Note: Due to (2), this condition can only arise if the table is
006321 ** the right-most table of a subquery that was flattened into the
006322 ** main query and that subquery was the right-hand operand of an
006323 ** inner join that held an ON or USING clause.
006324 ** 6) The ORDER BY clause has 63 or fewer terms
006325 ** 7) The omit-noop-join optimization is enabled.
006326 **
006327 ** Items (1), (6), and (7) are checked by the caller.
006328 **
006329 ** For example, given:
006330 **
006331 ** CREATE TABLE t1(ipk INTEGER PRIMARY KEY, v1);
006332 ** CREATE TABLE t2(ipk INTEGER PRIMARY KEY, v2);
006333 ** CREATE TABLE t3(ipk INTEGER PRIMARY KEY, v3);
006334 **
006335 ** then table t2 can be omitted from the following:
006336 **
006337 ** SELECT v1, v3 FROM t1
006338 ** LEFT JOIN t2 ON (t1.ipk=t2.ipk)
006339 ** LEFT JOIN t3 ON (t1.ipk=t3.ipk)
006340 **
006341 ** or from:
006342 **
006343 ** SELECT DISTINCT v1, v3 FROM t1
006344 ** LEFT JOIN t2
006345 ** LEFT JOIN t3 ON (t1.ipk=t3.ipk)
006346 */
006347 static SQLITE_NOINLINE Bitmask whereOmitNoopJoin(
006348 WhereInfo *pWInfo,
006349 Bitmask notReady
006350 ){
006351 int i;
006352 Bitmask tabUsed;
006353 int hasRightJoin;
006354
006355 /* Preconditions checked by the caller */
006356 assert( pWInfo->nLevel>=2 );
006357 assert( OptimizationEnabled(pWInfo->pParse->db, SQLITE_OmitNoopJoin) );
006358
006359 /* These two preconditions checked by the caller combine to guarantee
006360 ** condition (1) of the header comment */
006361 assert( pWInfo->pResultSet!=0 );
006362 assert( 0==(pWInfo->wctrlFlags & WHERE_AGG_DISTINCT) );
006363
006364 tabUsed = sqlite3WhereExprListUsage(&pWInfo->sMaskSet, pWInfo->pResultSet);
006365 if( pWInfo->pOrderBy ){
006366 tabUsed |= sqlite3WhereExprListUsage(&pWInfo->sMaskSet, pWInfo->pOrderBy);
006367 }
006368 hasRightJoin = (pWInfo->pTabList->a[0].fg.jointype & JT_LTORJ)!=0;
006369 for(i=pWInfo->nLevel-1; i>=1; i--){
006370 WhereTerm *pTerm, *pEnd;
006371 SrcItem *pItem;
006372 WhereLoop *pLoop;
006373 Bitmask m1;
006374 pLoop = pWInfo->a[i].pWLoop;
006375 pItem = &pWInfo->pTabList->a[pLoop->iTab];
006376 if( (pItem->fg.jointype & (JT_LEFT|JT_RIGHT))!=JT_LEFT ) continue;
006377 if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)==0
006378 && (pLoop->wsFlags & WHERE_ONEROW)==0
006379 ){
006380 continue;
006381 }
006382 if( (tabUsed & pLoop->maskSelf)!=0 ) continue;
006383 pEnd = pWInfo->sWC.a + pWInfo->sWC.nTerm;
006384 for(pTerm=pWInfo->sWC.a; pTerm<pEnd; pTerm++){
006385 if( (pTerm->prereqAll & pLoop->maskSelf)!=0 ){
006386 if( !ExprHasProperty(pTerm->pExpr, EP_OuterON)
006387 || pTerm->pExpr->w.iJoin!=pItem->iCursor
006388 ){
006389 break;
006390 }
006391 }
006392 if( hasRightJoin
006393 && ExprHasProperty(pTerm->pExpr, EP_InnerON)
006394 && NEVER(pTerm->pExpr->w.iJoin==pItem->iCursor)
006395 ){
006396 break; /* restriction (5) */
006397 }
006398 }
006399 if( pTerm<pEnd ) continue;
006400 WHERETRACE(0xffffffff,("-> omit unused FROM-clause term %c\n",pLoop->cId));
006401 m1 = MASKBIT(i)-1;
006402 testcase( ((pWInfo->revMask>>1) & ~m1)!=0 );
006403 pWInfo->revMask = (m1 & pWInfo->revMask) | ((pWInfo->revMask>>1) & ~m1);
006404 notReady &= ~pLoop->maskSelf;
006405 for(pTerm=pWInfo->sWC.a; pTerm<pEnd; pTerm++){
006406 if( (pTerm->prereqAll & pLoop->maskSelf)!=0 ){
006407 pTerm->wtFlags |= TERM_CODED;
006408 }
006409 }
006410 if( i!=pWInfo->nLevel-1 ){
006411 int nByte = (pWInfo->nLevel-1-i) * sizeof(WhereLevel);
006412 memmove(&pWInfo->a[i], &pWInfo->a[i+1], nByte);
006413 }
006414 pWInfo->nLevel--;
006415 assert( pWInfo->nLevel>0 );
006416 }
006417 return notReady;
006418 }
006419
006420 /*
006421 ** Check to see if there are any SEARCH loops that might benefit from
006422 ** using a Bloom filter. Consider a Bloom filter if:
006423 **
006424 ** (1) The SEARCH happens more than N times where N is the number
006425 ** of rows in the table that is being considered for the Bloom
006426 ** filter.
006427 ** (2) Some searches are expected to find zero rows. (This is determined
006428 ** by the WHERE_SELFCULL flag on the term.)
006429 ** (3) Bloom-filter processing is not disabled. (Checked by the
006430 ** caller.)
006431 ** (4) The size of the table being searched is known by ANALYZE.
006432 **
006433 ** This block of code merely checks to see if a Bloom filter would be
006434 ** appropriate, and if so sets the WHERE_BLOOMFILTER flag on the
006435 ** WhereLoop. The implementation of the Bloom filter comes further
006436 ** down where the code for each WhereLoop is generated.
006437 */
006438 static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful(
006439 const WhereInfo *pWInfo
006440 ){
006441 int i;
006442 LogEst nSearch = 0;
006443
006444 assert( pWInfo->nLevel>=2 );
006445 assert( OptimizationEnabled(pWInfo->pParse->db, SQLITE_BloomFilter) );
006446 for(i=0; i<pWInfo->nLevel; i++){
006447 WhereLoop *pLoop = pWInfo->a[i].pWLoop;
006448 const unsigned int reqFlags = (WHERE_SELFCULL|WHERE_COLUMN_EQ);
006449 SrcItem *pItem = &pWInfo->pTabList->a[pLoop->iTab];
006450 Table *pTab = pItem->pSTab;
006451 if( (pTab->tabFlags & TF_HasStat1)==0 ) break;
006452 pTab->tabFlags |= TF_MaybeReanalyze;
006453 if( i>=1
006454 && (pLoop->wsFlags & reqFlags)==reqFlags
006455 /* vvvvvv--- Always the case if WHERE_COLUMN_EQ is defined */
006456 && ALWAYS((pLoop->wsFlags & (WHERE_IPK|WHERE_INDEXED))!=0)
006457 ){
006458 if( nSearch > pTab->nRowLogEst ){
006459 testcase( pItem->fg.jointype & JT_LEFT );
006460 pLoop->wsFlags |= WHERE_BLOOMFILTER;
006461 pLoop->wsFlags &= ~WHERE_IDX_ONLY;
006462 WHERETRACE(0xffffffff, (
006463 "-> use Bloom-filter on loop %c because there are ~%.1e "
006464 "lookups into %s which has only ~%.1e rows\n",
006465 pLoop->cId, (double)sqlite3LogEstToInt(nSearch), pTab->zName,
006466 (double)sqlite3LogEstToInt(pTab->nRowLogEst)));
006467 }
006468 }
006469 nSearch += pLoop->nOut;
006470 }
006471 }
006472
006473 /*
006474 ** The index pIdx is used by a query and contains one or more expressions.
006475 ** In other words pIdx is an index on an expression. iIdxCur is the cursor
006476 ** number for the index and iDataCur is the cursor number for the corresponding
006477 ** table.
006478 **
006479 ** This routine adds IndexedExpr entries to the Parse->pIdxEpr field for
006480 ** each of the expressions in the index so that the expression code generator
006481 ** will know to replace occurrences of the indexed expression with
006482 ** references to the corresponding column of the index.
006483 */
006484 static SQLITE_NOINLINE void whereAddIndexedExpr(
006485 Parse *pParse, /* Add IndexedExpr entries to pParse->pIdxEpr */
006486 Index *pIdx, /* The index-on-expression that contains the expressions */
006487 int iIdxCur, /* Cursor number for pIdx */
006488 SrcItem *pTabItem /* The FROM clause entry for the table */
006489 ){
006490 int i;
006491 IndexedExpr *p;
006492 Table *pTab;
006493 assert( pIdx->bHasExpr );
006494 pTab = pIdx->pTable;
006495 for(i=0; i<pIdx->nColumn; i++){
006496 Expr *pExpr;
006497 int j = pIdx->aiColumn[i];
006498 if( j==XN_EXPR ){
006499 pExpr = pIdx->aColExpr->a[i].pExpr;
006500 }else if( j>=0 && (pTab->aCol[j].colFlags & COLFLAG_VIRTUAL)!=0 ){
006501 pExpr = sqlite3ColumnExpr(pTab, &pTab->aCol[j]);
006502 }else{
006503 continue;
006504 }
006505 if( sqlite3ExprIsConstant(0,pExpr) ) continue;
006506 p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexedExpr));
006507 if( p==0 ) break;
006508 p->pIENext = pParse->pIdxEpr;
006509 #ifdef WHERETRACE_ENABLED
006510 if( sqlite3WhereTrace & 0x200 ){
006511 sqlite3DebugPrintf("New pParse->pIdxEpr term {%d,%d}\n", iIdxCur, i);
006512 if( sqlite3WhereTrace & 0x5000 ) sqlite3ShowExpr(pExpr);
006513 }
006514 #endif
006515 p->pExpr = sqlite3ExprDup(pParse->db, pExpr, 0);
006516 p->iDataCur = pTabItem->iCursor;
006517 p->iIdxCur = iIdxCur;
006518 p->iIdxCol = i;
006519 p->bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0;
006520 if( sqlite3IndexAffinityStr(pParse->db, pIdx) ){
006521 p->aff = pIdx->zColAff[i];
006522 }
006523 #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
006524 p->zIdxName = pIdx->zName;
006525 #endif
006526 pParse->pIdxEpr = p;
006527 if( p->pIENext==0 ){
006528 void *pArg = (void*)&pParse->pIdxEpr;
006529 sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pArg);
006530 }
006531 }
006532 }
006533
006534 /*
006535 ** Set the reverse-scan order mask to one for all tables in the query
006536 ** with the exception of MATERIALIZED common table expressions that have
006537 ** their own internal ORDER BY clauses.
006538 **
006539 ** This implements the PRAGMA reverse_unordered_selects=ON setting.
006540 ** (Also SQLITE_DBCONFIG_REVERSE_SCANORDER).
006541 */
006542 static SQLITE_NOINLINE void whereReverseScanOrder(WhereInfo *pWInfo){
006543 int ii;
006544 for(ii=0; ii<pWInfo->pTabList->nSrc; ii++){
006545 SrcItem *pItem = &pWInfo->pTabList->a[ii];
006546 if( !pItem->fg.isCte
006547 || pItem->u2.pCteUse->eM10d!=M10d_Yes
006548 || NEVER(pItem->fg.isSubquery==0)
006549 || pItem->u4.pSubq->pSelect->pOrderBy==0
006550 ){
006551 pWInfo->revMask |= MASKBIT(ii);
006552 }
006553 }
006554 }
006555
006556 /*
006557 ** Generate the beginning of the loop used for WHERE clause processing.
006558 ** The return value is a pointer to an opaque structure that contains
006559 ** information needed to terminate the loop. Later, the calling routine
006560 ** should invoke sqlite3WhereEnd() with the return value of this function
006561 ** in order to complete the WHERE clause processing.
006562 **
006563 ** If an error occurs, this routine returns NULL.
006564 **
006565 ** The basic idea is to do a nested loop, one loop for each table in
006566 ** the FROM clause of a select. (INSERT and UPDATE statements are the
006567 ** same as a SELECT with only a single table in the FROM clause.) For
006568 ** example, if the SQL is this:
006569 **
006570 ** SELECT * FROM t1, t2, t3 WHERE ...;
006571 **
006572 ** Then the code generated is conceptually like the following:
006573 **
006574 ** foreach row1 in t1 do \ Code generated
006575 ** foreach row2 in t2 do |-- by sqlite3WhereBegin()
006576 ** foreach row3 in t3 do /
006577 ** ...
006578 ** end \ Code generated
006579 ** end |-- by sqlite3WhereEnd()
006580 ** end /
006581 **
006582 ** Note that the loops might not be nested in the order in which they
006583 ** appear in the FROM clause if a different order is better able to make
006584 ** use of indices. Note also that when the IN operator appears in
006585 ** the WHERE clause, it might result in additional nested loops for
006586 ** scanning through all values on the right-hand side of the IN.
006587 **
006588 ** There are Btree cursors associated with each table. t1 uses cursor
006589 ** number pTabList->a[0].iCursor. t2 uses the cursor pTabList->a[1].iCursor.
006590 ** And so forth. This routine generates code to open those VDBE cursors
006591 ** and sqlite3WhereEnd() generates the code to close them.
006592 **
006593 ** The code that sqlite3WhereBegin() generates leaves the cursors named
006594 ** in pTabList pointing at their appropriate entries. The [...] code
006595 ** can use OP_Column and OP_Rowid opcodes on these cursors to extract
006596 ** data from the various tables of the loop.
006597 **
006598 ** If the WHERE clause is empty, the foreach loops must each scan their
006599 ** entire tables. Thus a three-way join is an O(N^3) operation. But if
006600 ** the tables have indices and there are terms in the WHERE clause that
006601 ** refer to those indices, a complete table scan can be avoided and the
006602 ** code will run much faster. Most of the work of this routine is checking
006603 ** to see if there are indices that can be used to speed up the loop.
006604 **
006605 ** Terms of the WHERE clause are also used to limit which rows actually
006606 ** make it to the "..." in the middle of the loop. After each "foreach",
006607 ** terms of the WHERE clause that use only terms in that loop and outer
006608 ** loops are evaluated and if false a jump is made around all subsequent
006609 ** inner loops (or around the "..." if the test occurs within the inner-
006610 ** most loop)
006611 **
006612 ** OUTER JOINS
006613 **
006614 ** An outer join of tables t1 and t2 is conceptually coded as follows:
006615 **
006616 ** foreach row1 in t1 do
006617 ** flag = 0
006618 ** foreach row2 in t2 do
006619 ** start:
006620 ** ...
006621 ** flag = 1
006622 ** end
006623 ** if flag==0 then
006624 ** move the row2 cursor to a null row
006625 ** goto start
006626 ** fi
006627 ** end
006628 **
006629 ** ORDER BY CLAUSE PROCESSING
006630 **
006631 ** pOrderBy is a pointer to the ORDER BY clause (or the GROUP BY clause
006632 ** if the WHERE_GROUPBY flag is set in wctrlFlags) of a SELECT statement
006633 ** if there is one. If there is no ORDER BY clause or if this routine
006634 ** is called from an UPDATE or DELETE statement, then pOrderBy is NULL.
006635 **
006636 ** The iIdxCur parameter is the cursor number of an index. If
006637 ** WHERE_OR_SUBCLAUSE is set, iIdxCur is the cursor number of an index
006638 ** to use for OR clause processing. The WHERE clause should use this
006639 ** specific cursor. If WHERE_ONEPASS_DESIRED is set, then iIdxCur is
006640 ** the first cursor in an array of cursors for all indices. iIdxCur should
006641 ** be used to compute the appropriate cursor depending on which index is
006642 ** used.
006643 */
006644 WhereInfo *sqlite3WhereBegin(
006645 Parse *pParse, /* The parser context */
006646 SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */
006647 Expr *pWhere, /* The WHERE clause */
006648 ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */
006649 ExprList *pResultSet, /* Query result set. Req'd for DISTINCT */
006650 Select *pSelect, /* The entire SELECT statement */
006651 u16 wctrlFlags, /* The WHERE_* flags defined in sqliteInt.h */
006652 int iAuxArg /* If WHERE_OR_SUBCLAUSE is set, index cursor number
006653 ** If WHERE_USE_LIMIT, then the limit amount */
006654 ){
006655 int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */
006656 int nTabList; /* Number of elements in pTabList */
006657 WhereInfo *pWInfo; /* Will become the return value of this function */
006658 Vdbe *v = pParse->pVdbe; /* The virtual database engine */
006659 Bitmask notReady; /* Cursors that are not yet positioned */
006660 WhereLoopBuilder sWLB; /* The WhereLoop builder */
006661 WhereMaskSet *pMaskSet; /* The expression mask set */
006662 WhereLevel *pLevel; /* A single level in pWInfo->a[] */
006663 WhereLoop *pLoop; /* Pointer to a single WhereLoop object */
006664 int ii; /* Loop counter */
006665 sqlite3 *db; /* Database connection */
006666 int rc; /* Return code */
006667 u8 bFordelete = 0; /* OPFLAG_FORDELETE or zero, as appropriate */
006668
006669 assert( (wctrlFlags & WHERE_ONEPASS_MULTIROW)==0 || (
006670 (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0
006671 && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0
006672 ));
006673
006674 /* Only one of WHERE_OR_SUBCLAUSE or WHERE_USE_LIMIT */
006675 assert( (wctrlFlags & WHERE_OR_SUBCLAUSE)==0
006676 || (wctrlFlags & WHERE_USE_LIMIT)==0 );
006677
006678 /* Variable initialization */
006679 db = pParse->db;
006680 memset(&sWLB, 0, sizeof(sWLB));
006681
006682 /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */
006683 testcase( pOrderBy && pOrderBy->nExpr==BMS-1 );
006684 if( pOrderBy && pOrderBy->nExpr>=BMS ){
006685 pOrderBy = 0;
006686 wctrlFlags &= ~WHERE_WANT_DISTINCT;
006687 wctrlFlags |= WHERE_KEEP_ALL_JOINS; /* Disable omit-noop-join opt */
006688 }
006689
006690 /* The number of tables in the FROM clause is limited by the number of
006691 ** bits in a Bitmask
006692 */
006693 testcase( pTabList->nSrc==BMS );
006694 if( pTabList->nSrc>BMS ){
006695 sqlite3ErrorMsg(pParse, "at most %d tables in a join", BMS);
006696 return 0;
006697 }
006698
006699 /* This function normally generates a nested loop for all tables in
006700 ** pTabList. But if the WHERE_OR_SUBCLAUSE flag is set, then we should
006701 ** only generate code for the first table in pTabList and assume that
006702 ** any cursors associated with subsequent tables are uninitialized.
006703 */
006704 nTabList = (wctrlFlags & WHERE_OR_SUBCLAUSE) ? 1 : pTabList->nSrc;
006705
006706 /* Allocate and initialize the WhereInfo structure that will become the
006707 ** return value. A single allocation is used to store the WhereInfo
006708 ** struct, the contents of WhereInfo.a[], the WhereClause structure
006709 ** and the WhereMaskSet structure. Since WhereClause contains an 8-byte
006710 ** field (type Bitmask) it must be aligned on an 8-byte boundary on
006711 ** some architectures. Hence the ROUND8() below.
006712 */
006713 nByteWInfo = ROUND8P(sizeof(WhereInfo));
006714 if( nTabList>1 ){
006715 nByteWInfo = ROUND8P(nByteWInfo + (nTabList-1)*sizeof(WhereLevel));
006716 }
006717 pWInfo = sqlite3DbMallocRawNN(db, nByteWInfo + sizeof(WhereLoop));
006718 if( db->mallocFailed ){
006719 sqlite3DbFree(db, pWInfo);
006720 pWInfo = 0;
006721 goto whereBeginError;
006722 }
006723 pWInfo->pParse = pParse;
006724 pWInfo->pTabList = pTabList;
006725 pWInfo->pOrderBy = pOrderBy;
006726 #if WHERETRACE_ENABLED
006727 pWInfo->pWhere = pWhere;
006728 #endif
006729 pWInfo->pResultSet = pResultSet;
006730 pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1;
006731 pWInfo->nLevel = nTabList;
006732 pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(pParse);
006733 pWInfo->wctrlFlags = wctrlFlags;
006734 pWInfo->iLimit = iAuxArg;
006735 pWInfo->savedNQueryLoop = pParse->nQueryLoop;
006736 pWInfo->pSelect = pSelect;
006737 memset(&pWInfo->nOBSat, 0,
006738 offsetof(WhereInfo,sWC) - offsetof(WhereInfo,nOBSat));
006739 memset(&pWInfo->a[0], 0, sizeof(WhereLoop)+nTabList*sizeof(WhereLevel));
006740 assert( pWInfo->eOnePass==ONEPASS_OFF ); /* ONEPASS defaults to OFF */
006741 pMaskSet = &pWInfo->sMaskSet;
006742 pMaskSet->n = 0;
006743 pMaskSet->ix[0] = -99; /* Initialize ix[0] to a value that can never be
006744 ** a valid cursor number, to avoid an initial
006745 ** test for pMaskSet->n==0 in sqlite3WhereGetMask() */
006746 sWLB.pWInfo = pWInfo;
006747 sWLB.pWC = &pWInfo->sWC;
006748 sWLB.pNew = (WhereLoop*)(((char*)pWInfo)+nByteWInfo);
006749 assert( EIGHT_BYTE_ALIGNMENT(sWLB.pNew) );
006750 whereLoopInit(sWLB.pNew);
006751 #ifdef SQLITE_DEBUG
006752 sWLB.pNew->cId = '*';
006753 #endif
006754
006755 /* Split the WHERE clause into separate subexpressions where each
006756 ** subexpression is separated by an AND operator.
006757 */
006758 sqlite3WhereClauseInit(&pWInfo->sWC, pWInfo);
006759 sqlite3WhereSplit(&pWInfo->sWC, pWhere, TK_AND);
006760
006761 /* Special case: No FROM clause
006762 */
006763 if( nTabList==0 ){
006764 if( pOrderBy ) pWInfo->nOBSat = pOrderBy->nExpr;
006765 if( (wctrlFlags & WHERE_WANT_DISTINCT)!=0
006766 && OptimizationEnabled(db, SQLITE_DistinctOpt)
006767 ){
006768 pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
006769 }
006770 if( ALWAYS(pWInfo->pSelect)
006771 && (pWInfo->pSelect->selFlags & SF_MultiValue)==0
006772 ){
006773 ExplainQueryPlan((pParse, 0, "SCAN CONSTANT ROW"));
006774 }
006775 }else{
006776 /* Assign a bit from the bitmask to every term in the FROM clause.
006777 **
006778 ** The N-th term of the FROM clause is assigned a bitmask of 1<<N.
006779 **
006780 ** The rule of the previous sentence ensures that if X is the bitmask for
006781 ** a table T, then X-1 is the bitmask for all other tables to the left of T.
006782 ** Knowing the bitmask for all tables to the left of a left join is
006783 ** important. Ticket #3015.
006784 **
006785 ** Note that bitmasks are created for all pTabList->nSrc tables in
006786 ** pTabList, not just the first nTabList tables. nTabList is normally
006787 ** equal to pTabList->nSrc but might be shortened to 1 if the
006788 ** WHERE_OR_SUBCLAUSE flag is set.
006789 */
006790 ii = 0;
006791 do{
006792 createMask(pMaskSet, pTabList->a[ii].iCursor);
006793 sqlite3WhereTabFuncArgs(pParse, &pTabList->a[ii], &pWInfo->sWC);
006794 }while( (++ii)<pTabList->nSrc );
006795 #ifdef SQLITE_DEBUG
006796 {
006797 Bitmask mx = 0;
006798 for(ii=0; ii<pTabList->nSrc; ii++){
006799 Bitmask m = sqlite3WhereGetMask(pMaskSet, pTabList->a[ii].iCursor);
006800 assert( m>=mx );
006801 mx = m;
006802 }
006803 }
006804 #endif
006805 }
006806
006807 /* Analyze all of the subexpressions. */
006808 sqlite3WhereExprAnalyze(pTabList, &pWInfo->sWC);
006809 if( pSelect && pSelect->pLimit ){
006810 sqlite3WhereAddLimit(&pWInfo->sWC, pSelect);
006811 }
006812 if( pParse->nErr ) goto whereBeginError;
006813
006814 /* The False-WHERE-Term-Bypass optimization:
006815 **
006816 ** If there are WHERE terms that are false, then no rows will be output,
006817 ** so skip over all of the code generated here.
006818 **
006819 ** Conditions:
006820 **
006821 ** (1) The WHERE term must not refer to any tables in the join.
006822 ** (2) The term must not come from an ON clause on the
006823 ** right-hand side of a LEFT or FULL JOIN.
006824 ** (3) The term must not come from an ON clause, or there must be
006825 ** no RIGHT or FULL OUTER joins in pTabList.
006826 ** (4) If the expression contains non-deterministic functions
006827 ** that are not within a sub-select. This is not required
006828 ** for correctness but rather to preserves SQLite's legacy
006829 ** behaviour in the following two cases:
006830 **
006831 ** WHERE random()>0; -- eval random() once per row
006832 ** WHERE (SELECT random())>0; -- eval random() just once overall
006833 **
006834 ** Note that the Where term need not be a constant in order for this
006835 ** optimization to apply, though it does need to be constant relative to
006836 ** the current subquery (condition 1). The term might include variables
006837 ** from outer queries so that the value of the term changes from one
006838 ** invocation of the current subquery to the next.
006839 */
006840 for(ii=0; ii<sWLB.pWC->nBase; ii++){
006841 WhereTerm *pT = &sWLB.pWC->a[ii]; /* A term of the WHERE clause */
006842 Expr *pX; /* The expression of pT */
006843 if( pT->wtFlags & TERM_VIRTUAL ) continue;
006844 pX = pT->pExpr;
006845 assert( pX!=0 );
006846 assert( pT->prereqAll!=0 || !ExprHasProperty(pX, EP_OuterON) );
006847 if( pT->prereqAll==0 /* Conditions (1) and (2) */
006848 && (nTabList==0 || exprIsDeterministic(pX)) /* Condition (4) */
006849 && !(ExprHasProperty(pX, EP_InnerON) /* Condition (3) */
006850 && (pTabList->a[0].fg.jointype & JT_LTORJ)!=0 )
006851 ){
006852 sqlite3ExprIfFalse(pParse, pX, pWInfo->iBreak, SQLITE_JUMPIFNULL);
006853 pT->wtFlags |= TERM_CODED;
006854 }
006855 }
006856
006857 if( wctrlFlags & WHERE_WANT_DISTINCT ){
006858 if( OptimizationDisabled(db, SQLITE_DistinctOpt) ){
006859 /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via
006860 ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */
006861 wctrlFlags &= ~WHERE_WANT_DISTINCT;
006862 pWInfo->wctrlFlags &= ~WHERE_WANT_DISTINCT;
006863 }else if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pResultSet) ){
006864 /* The DISTINCT marking is pointless. Ignore it. */
006865 pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
006866 }else if( pOrderBy==0 ){
006867 /* Try to ORDER BY the result set to make distinct processing easier */
006868 pWInfo->wctrlFlags |= WHERE_DISTINCTBY;
006869 pWInfo->pOrderBy = pResultSet;
006870 }
006871 }
006872
006873 /* Construct the WhereLoop objects */
006874 #if defined(WHERETRACE_ENABLED)
006875 if( sqlite3WhereTrace & 0xffffffff ){
006876 sqlite3DebugPrintf("*** Optimizer Start *** (wctrlFlags: 0x%x",wctrlFlags);
006877 if( wctrlFlags & WHERE_USE_LIMIT ){
006878 sqlite3DebugPrintf(", limit: %d", iAuxArg);
006879 }
006880 sqlite3DebugPrintf(")\n");
006881 if( sqlite3WhereTrace & 0x8000 ){
006882 Select sSelect;
006883 memset(&sSelect, 0, sizeof(sSelect));
006884 sSelect.selFlags = SF_WhereBegin;
006885 sSelect.pSrc = pTabList;
006886 sSelect.pWhere = pWhere;
006887 sSelect.pOrderBy = pOrderBy;
006888 sSelect.pEList = pResultSet;
006889 sqlite3TreeViewSelect(0, &sSelect, 0);
006890 }
006891 if( sqlite3WhereTrace & 0x4000 ){ /* Display all WHERE clause terms */
006892 sqlite3DebugPrintf("---- WHERE clause at start of analysis:\n");
006893 sqlite3WhereClausePrint(sWLB.pWC);
006894 }
006895 }
006896 #endif
006897
006898 if( nTabList!=1 || whereShortCut(&sWLB)==0 ){
006899 rc = whereLoopAddAll(&sWLB);
006900 if( rc ) goto whereBeginError;
006901
006902 #ifdef SQLITE_ENABLE_STAT4
006903 /* If one or more WhereTerm.truthProb values were used in estimating
006904 ** loop parameters, but then those truthProb values were subsequently
006905 ** changed based on STAT4 information while computing subsequent loops,
006906 ** then we need to rerun the whole loop building process so that all
006907 ** loops will be built using the revised truthProb values. */
006908 if( sWLB.bldFlags2 & SQLITE_BLDF2_2NDPASS ){
006909 WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC);
006910 WHERETRACE(0xffffffff,
006911 ("**** Redo all loop computations due to"
006912 " TERM_HIGHTRUTH changes ****\n"));
006913 while( pWInfo->pLoops ){
006914 WhereLoop *p = pWInfo->pLoops;
006915 pWInfo->pLoops = p->pNextLoop;
006916 whereLoopDelete(db, p);
006917 }
006918 rc = whereLoopAddAll(&sWLB);
006919 if( rc ) goto whereBeginError;
006920 }
006921 #endif
006922 WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC);
006923
006924 wherePathSolver(pWInfo, 0);
006925 if( db->mallocFailed ) goto whereBeginError;
006926 if( pWInfo->pOrderBy ){
006927 whereInterstageHeuristic(pWInfo);
006928 wherePathSolver(pWInfo, pWInfo->nRowOut<0 ? 1 : pWInfo->nRowOut+1);
006929 if( db->mallocFailed ) goto whereBeginError;
006930 }
006931
006932 /* TUNING: Assume that a DISTINCT clause on a subquery reduces
006933 ** the output size by a factor of 8 (LogEst -30).
006934 */
006935 if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0 ){
006936 WHERETRACE(0x0080,("nRowOut reduced from %d to %d due to DISTINCT\n",
006937 pWInfo->nRowOut, pWInfo->nRowOut-30));
006938 pWInfo->nRowOut -= 30;
006939 }
006940
006941 }
006942 assert( pWInfo->pTabList!=0 );
006943 if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){
006944 whereReverseScanOrder(pWInfo);
006945 }
006946 if( pParse->nErr ){
006947 goto whereBeginError;
006948 }
006949 assert( db->mallocFailed==0 );
006950 #ifdef WHERETRACE_ENABLED
006951 if( sqlite3WhereTrace ){
006952 sqlite3DebugPrintf("---- Solution cost=%d, nRow=%d",
006953 pWInfo->rTotalCost, pWInfo->nRowOut);
006954 if( pWInfo->nOBSat>0 ){
006955 sqlite3DebugPrintf(" ORDERBY=%d,0x%llx", pWInfo->nOBSat, pWInfo->revMask);
006956 }
006957 switch( pWInfo->eDistinct ){
006958 case WHERE_DISTINCT_UNIQUE: {
006959 sqlite3DebugPrintf(" DISTINCT=unique");
006960 break;
006961 }
006962 case WHERE_DISTINCT_ORDERED: {
006963 sqlite3DebugPrintf(" DISTINCT=ordered");
006964 break;
006965 }
006966 case WHERE_DISTINCT_UNORDERED: {
006967 sqlite3DebugPrintf(" DISTINCT=unordered");
006968 break;
006969 }
006970 }
006971 sqlite3DebugPrintf("\n");
006972 for(ii=0; ii<pWInfo->nLevel; ii++){
006973 sqlite3WhereLoopPrint(pWInfo->a[ii].pWLoop, sWLB.pWC);
006974 }
006975 }
006976 #endif
006977
006978 /* Attempt to omit tables from a join that do not affect the result.
006979 ** See the comment on whereOmitNoopJoin() for further information.
006980 **
006981 ** This query optimization is factored out into a separate "no-inline"
006982 ** procedure to keep the sqlite3WhereBegin() procedure from becoming
006983 ** too large. If sqlite3WhereBegin() becomes too large, that prevents
006984 ** some C-compiler optimizers from in-lining the
006985 ** sqlite3WhereCodeOneLoopStart() procedure, and it is important to
006986 ** in-line sqlite3WhereCodeOneLoopStart() for performance reasons.
006987 */
006988 notReady = ~(Bitmask)0;
006989 if( pWInfo->nLevel>=2 /* Must be a join, or this opt8n is pointless */
006990 && pResultSet!=0 /* Condition (1) */
006991 && 0==(wctrlFlags & (WHERE_AGG_DISTINCT|WHERE_KEEP_ALL_JOINS)) /* (1),(6) */
006992 && OptimizationEnabled(db, SQLITE_OmitNoopJoin) /* (7) */
006993 ){
006994 notReady = whereOmitNoopJoin(pWInfo, notReady);
006995 nTabList = pWInfo->nLevel;
006996 assert( nTabList>0 );
006997 }
006998
006999 /* Check to see if there are any SEARCH loops that might benefit from
007000 ** using a Bloom filter.
007001 */
007002 if( pWInfo->nLevel>=2
007003 && OptimizationEnabled(db, SQLITE_BloomFilter)
007004 ){
007005 whereCheckIfBloomFilterIsUseful(pWInfo);
007006 }
007007
007008 #if defined(WHERETRACE_ENABLED)
007009 if( sqlite3WhereTrace & 0x4000 ){ /* Display all terms of the WHERE clause */
007010 sqlite3DebugPrintf("---- WHERE clause at end of analysis:\n");
007011 sqlite3WhereClausePrint(sWLB.pWC);
007012 }
007013 WHERETRACE(0xffffffff,("*** Optimizer Finished ***\n"));
007014 #endif
007015 pWInfo->pParse->nQueryLoop += pWInfo->nRowOut;
007016
007017 /* If the caller is an UPDATE or DELETE statement that is requesting
007018 ** to use a one-pass algorithm, determine if this is appropriate.
007019 **
007020 ** A one-pass approach can be used if the caller has requested one
007021 ** and either (a) the scan visits at most one row or (b) each
007022 ** of the following are true:
007023 **
007024 ** * the caller has indicated that a one-pass approach can be used
007025 ** with multiple rows (by setting WHERE_ONEPASS_MULTIROW), and
007026 ** * the table is not a virtual table, and
007027 ** * either the scan does not use the OR optimization or the caller
007028 ** is a DELETE operation (WHERE_DUPLICATES_OK is only specified
007029 ** for DELETE).
007030 **
007031 ** The last qualification is because an UPDATE statement uses
007032 ** WhereInfo.aiCurOnePass[1] to determine whether or not it really can
007033 ** use a one-pass approach, and this is not set accurately for scans
007034 ** that use the OR optimization.
007035 */
007036 assert( (wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || pWInfo->nLevel==1 );
007037 if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 ){
007038 int wsFlags = pWInfo->a[0].pWLoop->wsFlags;
007039 int bOnerow = (wsFlags & WHERE_ONEROW)!=0;
007040 assert( !(wsFlags&WHERE_VIRTUALTABLE) || IsVirtual(pTabList->a[0].pSTab) );
007041 if( bOnerow || (
007042 0!=(wctrlFlags & WHERE_ONEPASS_MULTIROW)
007043 && !IsVirtual(pTabList->a[0].pSTab)
007044 && (0==(wsFlags & WHERE_MULTI_OR) || (wctrlFlags & WHERE_DUPLICATES_OK))
007045 && OptimizationEnabled(db, SQLITE_OnePass)
007046 )){
007047 pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI;
007048 if( HasRowid(pTabList->a[0].pSTab) && (wsFlags & WHERE_IDX_ONLY) ){
007049 if( wctrlFlags & WHERE_ONEPASS_MULTIROW ){
007050 bFordelete = OPFLAG_FORDELETE;
007051 }
007052 pWInfo->a[0].pWLoop->wsFlags = (wsFlags & ~WHERE_IDX_ONLY);
007053 }
007054 }
007055 }
007056
007057 /* Open all tables in the pTabList and any indices selected for
007058 ** searching those tables.
007059 */
007060 for(ii=0, pLevel=pWInfo->a; ii<nTabList; ii++, pLevel++){
007061 Table *pTab; /* Table to open */
007062 int iDb; /* Index of database containing table/index */
007063 SrcItem *pTabItem;
007064
007065 pTabItem = &pTabList->a[pLevel->iFrom];
007066 pTab = pTabItem->pSTab;
007067 iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
007068 pLoop = pLevel->pWLoop;
007069 if( (pTab->tabFlags & TF_Ephemeral)!=0 || IsView(pTab) ){
007070 /* Do nothing */
007071 }else
007072 #ifndef SQLITE_OMIT_VIRTUALTABLE
007073 if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){
007074 const char *pVTab = (const char *)sqlite3GetVTable(db, pTab);
007075 int iCur = pTabItem->iCursor;
007076 sqlite3VdbeAddOp4(v, OP_VOpen, iCur, 0, 0, pVTab, P4_VTAB);
007077 }else if( IsVirtual(pTab) ){
007078 /* noop */
007079 }else
007080 #endif
007081 if( ((pLoop->wsFlags & WHERE_IDX_ONLY)==0
007082 && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0)
007083 || (pTabItem->fg.jointype & (JT_LTORJ|JT_RIGHT))!=0
007084 ){
007085 int op = OP_OpenRead;
007086 if( pWInfo->eOnePass!=ONEPASS_OFF ){
007087 op = OP_OpenWrite;
007088 pWInfo->aiCurOnePass[0] = pTabItem->iCursor;
007089 };
007090 sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op);
007091 assert( pTabItem->iCursor==pLevel->iTabCur );
007092 testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS-1 );
007093 testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS );
007094 if( pWInfo->eOnePass==ONEPASS_OFF
007095 && pTab->nCol<BMS
007096 && (pTab->tabFlags & (TF_HasGenerated|TF_WithoutRowid))==0
007097 && (pLoop->wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))==0
007098 ){
007099 /* If we know that only a prefix of the record will be used,
007100 ** it is advantageous to reduce the "column count" field in
007101 ** the P4 operand of the OP_OpenRead/Write opcode. */
007102 Bitmask b = pTabItem->colUsed;
007103 int n = 0;
007104 for(; b; b=b>>1, n++){}
007105 sqlite3VdbeChangeP4(v, -1, SQLITE_INT_TO_PTR(n), P4_INT32);
007106 assert( n<=pTab->nCol );
007107 }
007108 #ifdef SQLITE_ENABLE_CURSOR_HINTS
007109 if( pLoop->u.btree.pIndex!=0 && (pTab->tabFlags & TF_WithoutRowid)==0 ){
007110 sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ|bFordelete);
007111 }else
007112 #endif
007113 {
007114 sqlite3VdbeChangeP5(v, bFordelete);
007115 }
007116 #ifdef SQLITE_ENABLE_COLUMN_USED_MASK
007117 sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, pTabItem->iCursor, 0, 0,
007118 (const u8*)&pTabItem->colUsed, P4_INT64);
007119 #endif
007120 }else{
007121 sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName);
007122 }
007123 if( pLoop->wsFlags & WHERE_INDEXED ){
007124 Index *pIx = pLoop->u.btree.pIndex;
007125 int iIndexCur;
007126 int op = OP_OpenRead;
007127 /* iAuxArg is always set to a positive value if ONEPASS is possible */
007128 assert( iAuxArg!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 );
007129 if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIx)
007130 && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0
007131 ){
007132 /* This is one term of an OR-optimization using the PRIMARY KEY of a
007133 ** WITHOUT ROWID table. No need for a separate index */
007134 iIndexCur = pLevel->iTabCur;
007135 op = 0;
007136 }else if( pWInfo->eOnePass!=ONEPASS_OFF ){
007137 Index *pJ = pTabItem->pSTab->pIndex;
007138 iIndexCur = iAuxArg;
007139 assert( wctrlFlags & WHERE_ONEPASS_DESIRED );
007140 while( ALWAYS(pJ) && pJ!=pIx ){
007141 iIndexCur++;
007142 pJ = pJ->pNext;
007143 }
007144 op = OP_OpenWrite;
007145 pWInfo->aiCurOnePass[1] = iIndexCur;
007146 }else if( iAuxArg && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ){
007147 iIndexCur = iAuxArg;
007148 op = OP_ReopenIdx;
007149 }else{
007150 iIndexCur = pParse->nTab++;
007151 if( pIx->bHasExpr && OptimizationEnabled(db, SQLITE_IndexedExpr) ){
007152 whereAddIndexedExpr(pParse, pIx, iIndexCur, pTabItem);
007153 }
007154 if( pIx->pPartIdxWhere && (pTabItem->fg.jointype & JT_RIGHT)==0 ){
007155 wherePartIdxExpr(
007156 pParse, pIx, pIx->pPartIdxWhere, 0, iIndexCur, pTabItem
007157 );
007158 }
007159 }
007160 pLevel->iIdxCur = iIndexCur;
007161 assert( pIx!=0 );
007162 assert( pIx->pSchema==pTab->pSchema );
007163 assert( iIndexCur>=0 );
007164 if( op ){
007165 sqlite3VdbeAddOp3(v, op, iIndexCur, pIx->tnum, iDb);
007166 sqlite3VdbeSetP4KeyInfo(pParse, pIx);
007167 if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0
007168 && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0
007169 && (pLoop->wsFlags & WHERE_BIGNULL_SORT)==0
007170 && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0
007171 && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0
007172 && pWInfo->eDistinct!=WHERE_DISTINCT_ORDERED
007173 ){
007174 sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ);
007175 }
007176 VdbeComment((v, "%s", pIx->zName));
007177 #ifdef SQLITE_ENABLE_COLUMN_USED_MASK
007178 {
007179 u64 colUsed = 0;
007180 int ii, jj;
007181 for(ii=0; ii<pIx->nColumn; ii++){
007182 jj = pIx->aiColumn[ii];
007183 if( jj<0 ) continue;
007184 if( jj>63 ) jj = 63;
007185 if( (pTabItem->colUsed & MASKBIT(jj))==0 ) continue;
007186 colUsed |= ((u64)1)<<(ii<63 ? ii : 63);
007187 }
007188 sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, iIndexCur, 0, 0,
007189 (u8*)&colUsed, P4_INT64);
007190 }
007191 #endif /* SQLITE_ENABLE_COLUMN_USED_MASK */
007192 }
007193 }
007194 if( iDb>=0 ) sqlite3CodeVerifySchema(pParse, iDb);
007195 if( (pTabItem->fg.jointype & JT_RIGHT)!=0
007196 && (pLevel->pRJ = sqlite3WhereMalloc(pWInfo, sizeof(WhereRightJoin)))!=0
007197 ){
007198 WhereRightJoin *pRJ = pLevel->pRJ;
007199 pRJ->iMatch = pParse->nTab++;
007200 pRJ->regBloom = ++pParse->nMem;
007201 sqlite3VdbeAddOp2(v, OP_Blob, 65536, pRJ->regBloom);
007202 pRJ->regReturn = ++pParse->nMem;
007203 sqlite3VdbeAddOp2(v, OP_Null, 0, pRJ->regReturn);
007204 assert( pTab==pTabItem->pSTab );
007205 if( HasRowid(pTab) ){
007206 KeyInfo *pInfo;
007207 sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRJ->iMatch, 1);
007208 pInfo = sqlite3KeyInfoAlloc(pParse->db, 1, 0);
007209 if( pInfo ){
007210 pInfo->aColl[0] = 0;
007211 pInfo->aSortFlags[0] = 0;
007212 sqlite3VdbeAppendP4(v, pInfo, P4_KEYINFO);
007213 }
007214 }else{
007215 Index *pPk = sqlite3PrimaryKeyIndex(pTab);
007216 sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRJ->iMatch, pPk->nKeyCol);
007217 sqlite3VdbeSetP4KeyInfo(pParse, pPk);
007218 }
007219 pLoop->wsFlags &= ~WHERE_IDX_ONLY;
007220 /* The nature of RIGHT JOIN processing is such that it messes up
007221 ** the output order. So omit any ORDER BY/GROUP BY elimination
007222 ** optimizations. We need to do an actual sort for RIGHT JOIN. */
007223 pWInfo->nOBSat = 0;
007224 pWInfo->eDistinct = WHERE_DISTINCT_UNORDERED;
007225 }
007226 }
007227 pWInfo->iTop = sqlite3VdbeCurrentAddr(v);
007228 if( db->mallocFailed ) goto whereBeginError;
007229
007230 /* Generate the code to do the search. Each iteration of the for
007231 ** loop below generates code for a single nested loop of the VM
007232 ** program.
007233 */
007234 for(ii=0; ii<nTabList; ii++){
007235 int addrExplain;
007236 int wsFlags;
007237 SrcItem *pSrc;
007238 if( pParse->nErr ) goto whereBeginError;
007239 pLevel = &pWInfo->a[ii];
007240 wsFlags = pLevel->pWLoop->wsFlags;
007241 pSrc = &pTabList->a[pLevel->iFrom];
007242 if( pSrc->fg.isMaterialized ){
007243 Subquery *pSubq;
007244 int iOnce = 0;
007245 assert( pSrc->fg.isSubquery );
007246 pSubq = pSrc->u4.pSubq;
007247 if( pSrc->fg.isCorrelated==0 ){
007248 iOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
007249 }else{
007250 iOnce = 0;
007251 }
007252 sqlite3VdbeAddOp2(v, OP_Gosub, pSubq->regReturn, pSubq->addrFillSub);
007253 VdbeComment((v, "materialize %!S", pSrc));
007254 if( iOnce ) sqlite3VdbeJumpHere(v, iOnce);
007255 }
007256 assert( pTabList == pWInfo->pTabList );
007257 if( (wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))!=0 ){
007258 if( (wsFlags & WHERE_AUTO_INDEX)!=0 ){
007259 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
007260 constructAutomaticIndex(pParse, &pWInfo->sWC, notReady, pLevel);
007261 #endif
007262 }else{
007263 sqlite3ConstructBloomFilter(pWInfo, ii, pLevel, notReady);
007264 }
007265 if( db->mallocFailed ) goto whereBeginError;
007266 }
007267 addrExplain = sqlite3WhereExplainOneScan(
007268 pParse, pTabList, pLevel, wctrlFlags
007269 );
007270 pLevel->addrBody = sqlite3VdbeCurrentAddr(v);
007271 notReady = sqlite3WhereCodeOneLoopStart(pParse,v,pWInfo,ii,pLevel,notReady);
007272 pWInfo->iContinue = pLevel->addrCont;
007273 if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_OR_SUBCLAUSE)==0 ){
007274 sqlite3WhereAddScanStatus(v, pTabList, pLevel, addrExplain);
007275 }
007276 }
007277
007278 /* Done. */
007279 VdbeModuleComment((v, "Begin WHERE-core"));
007280 pWInfo->iEndWhere = sqlite3VdbeCurrentAddr(v);
007281 return pWInfo;
007282
007283 /* Jump here if malloc fails */
007284 whereBeginError:
007285 if( pWInfo ){
007286 pParse->nQueryLoop = pWInfo->savedNQueryLoop;
007287 whereInfoFree(db, pWInfo);
007288 }
007289 #ifdef WHERETRACE_ENABLED
007290 /* Prevent harmless compiler warnings about debugging routines
007291 ** being declared but never used */
007292 sqlite3ShowWhereLoopList(0);
007293 #endif /* WHERETRACE_ENABLED */
007294 return 0;
007295 }
007296
007297 /*
007298 ** Part of sqlite3WhereEnd() will rewrite opcodes to reference the
007299 ** index rather than the main table. In SQLITE_DEBUG mode, we want
007300 ** to trace those changes if PRAGMA vdbe_addoptrace=on. This routine
007301 ** does that.
007302 */
007303 #ifndef SQLITE_DEBUG
007304 # define OpcodeRewriteTrace(D,K,P) /* no-op */
007305 #else
007306 # define OpcodeRewriteTrace(D,K,P) sqlite3WhereOpcodeRewriteTrace(D,K,P)
007307 static void sqlite3WhereOpcodeRewriteTrace(
007308 sqlite3 *db,
007309 int pc,
007310 VdbeOp *pOp
007311 ){
007312 if( (db->flags & SQLITE_VdbeAddopTrace)==0 ) return;
007313 sqlite3VdbePrintOp(0, pc, pOp);
007314 sqlite3ShowWhereTerm(0); /* So compiler won't complain about unused func */
007315 }
007316 #endif
007317
007318 /*
007319 ** Generate the end of the WHERE loop. See comments on
007320 ** sqlite3WhereBegin() for additional information.
007321 */
007322 void sqlite3WhereEnd(WhereInfo *pWInfo){
007323 Parse *pParse = pWInfo->pParse;
007324 Vdbe *v = pParse->pVdbe;
007325 int i;
007326 WhereLevel *pLevel;
007327 WhereLoop *pLoop;
007328 SrcList *pTabList = pWInfo->pTabList;
007329 sqlite3 *db = pParse->db;
007330 int iEnd = sqlite3VdbeCurrentAddr(v);
007331 int nRJ = 0;
007332
007333 /* Generate loop termination code.
007334 */
007335 VdbeModuleComment((v, "End WHERE-core"));
007336 for(i=pWInfo->nLevel-1; i>=0; i--){
007337 int addr;
007338 pLevel = &pWInfo->a[i];
007339 if( pLevel->pRJ ){
007340 /* Terminate the subroutine that forms the interior of the loop of
007341 ** the RIGHT JOIN table */
007342 WhereRightJoin *pRJ = pLevel->pRJ;
007343 sqlite3VdbeResolveLabel(v, pLevel->addrCont);
007344 pLevel->addrCont = 0;
007345 pRJ->endSubrtn = sqlite3VdbeCurrentAddr(v);
007346 sqlite3VdbeAddOp3(v, OP_Return, pRJ->regReturn, pRJ->addrSubrtn, 1);
007347 VdbeCoverage(v);
007348 nRJ++;
007349 }
007350 pLoop = pLevel->pWLoop;
007351 if( pLevel->op!=OP_Noop ){
007352 #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT
007353 int addrSeek = 0;
007354 Index *pIdx;
007355 int n;
007356 if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED
007357 && i==pWInfo->nLevel-1 /* Ticket [ef9318757b152e3] 2017-10-21 */
007358 && (pLoop->wsFlags & WHERE_INDEXED)!=0
007359 && (pIdx = pLoop->u.btree.pIndex)->hasStat1
007360 && (n = pLoop->u.btree.nDistinctCol)>0
007361 && pIdx->aiRowLogEst[n]>=36
007362 ){
007363 int r1 = pParse->nMem+1;
007364 int j, op;
007365 for(j=0; j<n; j++){
007366 sqlite3VdbeAddOp3(v, OP_Column, pLevel->iIdxCur, j, r1+j);
007367 }
007368 pParse->nMem += n+1;
007369 op = pLevel->op==OP_Prev ? OP_SeekLT : OP_SeekGT;
007370 addrSeek = sqlite3VdbeAddOp4Int(v, op, pLevel->iIdxCur, 0, r1, n);
007371 VdbeCoverageIf(v, op==OP_SeekLT);
007372 VdbeCoverageIf(v, op==OP_SeekGT);
007373 sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2);
007374 }
007375 #endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */
007376 /* The common case: Advance to the next row */
007377 if( pLevel->addrCont ) sqlite3VdbeResolveLabel(v, pLevel->addrCont);
007378 sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3);
007379 sqlite3VdbeChangeP5(v, pLevel->p5);
007380 VdbeCoverage(v);
007381 VdbeCoverageIf(v, pLevel->op==OP_Next);
007382 VdbeCoverageIf(v, pLevel->op==OP_Prev);
007383 VdbeCoverageIf(v, pLevel->op==OP_VNext);
007384 if( pLevel->regBignull ){
007385 sqlite3VdbeResolveLabel(v, pLevel->addrBignull);
007386 sqlite3VdbeAddOp2(v, OP_DecrJumpZero, pLevel->regBignull, pLevel->p2-1);
007387 VdbeCoverage(v);
007388 }
007389 #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT
007390 if( addrSeek ) sqlite3VdbeJumpHere(v, addrSeek);
007391 #endif
007392 }else if( pLevel->addrCont ){
007393 sqlite3VdbeResolveLabel(v, pLevel->addrCont);
007394 }
007395 if( (pLoop->wsFlags & WHERE_IN_ABLE)!=0 && pLevel->u.in.nIn>0 ){
007396 struct InLoop *pIn;
007397 int j;
007398 sqlite3VdbeResolveLabel(v, pLevel->addrNxt);
007399 for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){
007400 assert( sqlite3VdbeGetOp(v, pIn->addrInTop+1)->opcode==OP_IsNull
007401 || pParse->db->mallocFailed );
007402 sqlite3VdbeJumpHere(v, pIn->addrInTop+1);
007403 if( pIn->eEndLoopOp!=OP_Noop ){
007404 if( pIn->nPrefix ){
007405 int bEarlyOut =
007406 (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0
007407 && (pLoop->wsFlags & WHERE_IN_EARLYOUT)!=0;
007408 if( pLevel->iLeftJoin ){
007409 /* For LEFT JOIN queries, cursor pIn->iCur may not have been
007410 ** opened yet. This occurs for WHERE clauses such as
007411 ** "a = ? AND b IN (...)", where the index is on (a, b). If
007412 ** the RHS of the (a=?) is NULL, then the "b IN (...)" may
007413 ** never have been coded, but the body of the loop run to
007414 ** return the null-row. So, if the cursor is not open yet,
007415 ** jump over the OP_Next or OP_Prev instruction about to
007416 ** be coded. */
007417 sqlite3VdbeAddOp2(v, OP_IfNotOpen, pIn->iCur,
007418 sqlite3VdbeCurrentAddr(v) + 2 + bEarlyOut);
007419 VdbeCoverage(v);
007420 }
007421 if( bEarlyOut ){
007422 sqlite3VdbeAddOp4Int(v, OP_IfNoHope, pLevel->iIdxCur,
007423 sqlite3VdbeCurrentAddr(v)+2,
007424 pIn->iBase, pIn->nPrefix);
007425 VdbeCoverage(v);
007426 /* Retarget the OP_IsNull against the left operand of IN so
007427 ** it jumps past the OP_IfNoHope. This is because the
007428 ** OP_IsNull also bypasses the OP_Affinity opcode that is
007429 ** required by OP_IfNoHope. */
007430 sqlite3VdbeJumpHere(v, pIn->addrInTop+1);
007431 }
007432 }
007433 sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop);
007434 VdbeCoverage(v);
007435 VdbeCoverageIf(v, pIn->eEndLoopOp==OP_Prev);
007436 VdbeCoverageIf(v, pIn->eEndLoopOp==OP_Next);
007437 }
007438 sqlite3VdbeJumpHere(v, pIn->addrInTop-1);
007439 }
007440 }
007441 sqlite3VdbeResolveLabel(v, pLevel->addrBrk);
007442 if( pLevel->pRJ ){
007443 sqlite3VdbeAddOp3(v, OP_Return, pLevel->pRJ->regReturn, 0, 1);
007444 VdbeCoverage(v);
007445 }
007446 if( pLevel->addrSkip ){
007447 sqlite3VdbeGoto(v, pLevel->addrSkip);
007448 VdbeComment((v, "next skip-scan on %s", pLoop->u.btree.pIndex->zName));
007449 sqlite3VdbeJumpHere(v, pLevel->addrSkip);
007450 sqlite3VdbeJumpHere(v, pLevel->addrSkip-2);
007451 }
007452 #ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS
007453 if( pLevel->addrLikeRep ){
007454 sqlite3VdbeAddOp2(v, OP_DecrJumpZero, (int)(pLevel->iLikeRepCntr>>1),
007455 pLevel->addrLikeRep);
007456 VdbeCoverage(v);
007457 }
007458 #endif
007459 if( pLevel->iLeftJoin ){
007460 int ws = pLoop->wsFlags;
007461 addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v);
007462 assert( (ws & WHERE_IDX_ONLY)==0 || (ws & WHERE_INDEXED)!=0 );
007463 if( (ws & WHERE_IDX_ONLY)==0 ){
007464 SrcItem *pSrc = &pTabList->a[pLevel->iFrom];
007465 assert( pLevel->iTabCur==pSrc->iCursor );
007466 if( pSrc->fg.viaCoroutine ){
007467 int m, n;
007468 assert( pSrc->fg.isSubquery );
007469 n = pSrc->u4.pSubq->regResult;
007470 assert( pSrc->pSTab!=0 );
007471 m = pSrc->pSTab->nCol;
007472 sqlite3VdbeAddOp3(v, OP_Null, 0, n, n+m-1);
007473 }
007474 sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iTabCur);
007475 }
007476 if( (ws & WHERE_INDEXED)
007477 || ((ws & WHERE_MULTI_OR) && pLevel->u.pCoveringIdx)
007478 ){
007479 if( ws & WHERE_MULTI_OR ){
007480 Index *pIx = pLevel->u.pCoveringIdx;
007481 int iDb = sqlite3SchemaToIndex(db, pIx->pSchema);
007482 sqlite3VdbeAddOp3(v, OP_ReopenIdx, pLevel->iIdxCur, pIx->tnum, iDb);
007483 sqlite3VdbeSetP4KeyInfo(pParse, pIx);
007484 }
007485 sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur);
007486 }
007487 if( pLevel->op==OP_Return ){
007488 sqlite3VdbeAddOp2(v, OP_Gosub, pLevel->p1, pLevel->addrFirst);
007489 }else{
007490 sqlite3VdbeGoto(v, pLevel->addrFirst);
007491 }
007492 sqlite3VdbeJumpHere(v, addr);
007493 }
007494 VdbeModuleComment((v, "End WHERE-loop%d: %s", i,
007495 pWInfo->pTabList->a[pLevel->iFrom].pSTab->zName));
007496 }
007497
007498 assert( pWInfo->nLevel<=pTabList->nSrc );
007499 for(i=0, pLevel=pWInfo->a; i<pWInfo->nLevel; i++, pLevel++){
007500 int k, last;
007501 VdbeOp *pOp, *pLastOp;
007502 Index *pIdx = 0;
007503 SrcItem *pTabItem = &pTabList->a[pLevel->iFrom];
007504 Table *pTab = pTabItem->pSTab;
007505 assert( pTab!=0 );
007506 pLoop = pLevel->pWLoop;
007507
007508 /* Do RIGHT JOIN processing. Generate code that will output the
007509 ** unmatched rows of the right operand of the RIGHT JOIN with
007510 ** all of the columns of the left operand set to NULL.
007511 */
007512 if( pLevel->pRJ ){
007513 sqlite3WhereRightJoinLoop(pWInfo, i, pLevel);
007514 continue;
007515 }
007516
007517 /* For a co-routine, change all OP_Column references to the table of
007518 ** the co-routine into OP_Copy of result contained in a register.
007519 ** OP_Rowid becomes OP_Null.
007520 */
007521 if( pTabItem->fg.viaCoroutine ){
007522 testcase( pParse->db->mallocFailed );
007523 assert( pTabItem->fg.isSubquery );
007524 assert( pTabItem->u4.pSubq->regResult>=0 );
007525 translateColumnToCopy(pParse, pLevel->addrBody, pLevel->iTabCur,
007526 pTabItem->u4.pSubq->regResult, 0);
007527 continue;
007528 }
007529
007530 /* If this scan uses an index, make VDBE code substitutions to read data
007531 ** from the index instead of from the table where possible. In some cases
007532 ** this optimization prevents the table from ever being read, which can
007533 ** yield a significant performance boost.
007534 **
007535 ** Calls to the code generator in between sqlite3WhereBegin and
007536 ** sqlite3WhereEnd will have created code that references the table
007537 ** directly. This loop scans all that code looking for opcodes
007538 ** that reference the table and converts them into opcodes that
007539 ** reference the index.
007540 */
007541 if( pLoop->wsFlags & (WHERE_INDEXED|WHERE_IDX_ONLY) ){
007542 pIdx = pLoop->u.btree.pIndex;
007543 }else if( pLoop->wsFlags & WHERE_MULTI_OR ){
007544 pIdx = pLevel->u.pCoveringIdx;
007545 }
007546 if( pIdx
007547 && !db->mallocFailed
007548 ){
007549 if( pWInfo->eOnePass==ONEPASS_OFF || !HasRowid(pIdx->pTable) ){
007550 last = iEnd;
007551 }else{
007552 last = pWInfo->iEndWhere;
007553 }
007554 if( pIdx->bHasExpr ){
007555 IndexedExpr *p = pParse->pIdxEpr;
007556 while( p ){
007557 if( p->iIdxCur==pLevel->iIdxCur ){
007558 #ifdef WHERETRACE_ENABLED
007559 if( sqlite3WhereTrace & 0x200 ){
007560 sqlite3DebugPrintf("Disable pParse->pIdxEpr term {%d,%d}\n",
007561 p->iIdxCur, p->iIdxCol);
007562 if( sqlite3WhereTrace & 0x5000 ) sqlite3ShowExpr(p->pExpr);
007563 }
007564 #endif
007565 p->iDataCur = -1;
007566 p->iIdxCur = -1;
007567 }
007568 p = p->pIENext;
007569 }
007570 }
007571 k = pLevel->addrBody + 1;
007572 #ifdef SQLITE_DEBUG
007573 if( db->flags & SQLITE_VdbeAddopTrace ){
007574 printf("TRANSLATE cursor %d->%d in opcode range %d..%d\n",
007575 pLevel->iTabCur, pLevel->iIdxCur, k, last-1);
007576 }
007577 /* Proof that the "+1" on the k value above is safe */
007578 pOp = sqlite3VdbeGetOp(v, k - 1);
007579 assert( pOp->opcode!=OP_Column || pOp->p1!=pLevel->iTabCur );
007580 assert( pOp->opcode!=OP_Rowid || pOp->p1!=pLevel->iTabCur );
007581 assert( pOp->opcode!=OP_IfNullRow || pOp->p1!=pLevel->iTabCur );
007582 #endif
007583 pOp = sqlite3VdbeGetOp(v, k);
007584 pLastOp = pOp + (last - k);
007585 assert( pOp<=pLastOp );
007586 do{
007587 if( pOp->p1!=pLevel->iTabCur ){
007588 /* no-op */
007589 }else if( pOp->opcode==OP_Column
007590 #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
007591 || pOp->opcode==OP_Offset
007592 #endif
007593 ){
007594 int x = pOp->p2;
007595 assert( pIdx->pTable==pTab );
007596 #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
007597 if( pOp->opcode==OP_Offset ){
007598 /* Do not need to translate the column number */
007599 }else
007600 #endif
007601 if( !HasRowid(pTab) ){
007602 Index *pPk = sqlite3PrimaryKeyIndex(pTab);
007603 x = pPk->aiColumn[x];
007604 assert( x>=0 );
007605 }else{
007606 testcase( x!=sqlite3StorageColumnToTable(pTab,x) );
007607 x = sqlite3StorageColumnToTable(pTab,x);
007608 }
007609 x = sqlite3TableColumnToIndex(pIdx, x);
007610 if( x>=0 ){
007611 pOp->p2 = x;
007612 pOp->p1 = pLevel->iIdxCur;
007613 OpcodeRewriteTrace(db, k, pOp);
007614 }else if( pLoop->wsFlags & (WHERE_IDX_ONLY|WHERE_EXPRIDX) ){
007615 if( pLoop->wsFlags & WHERE_IDX_ONLY ){
007616 /* An error. pLoop is supposed to be a covering index loop,
007617 ** and yet the VM code refers to a column of the table that
007618 ** is not part of the index. */
007619 sqlite3ErrorMsg(pParse, "internal query planner error");
007620 pParse->rc = SQLITE_INTERNAL;
007621 }else{
007622 /* The WHERE_EXPRIDX flag is set by the planner when it is likely
007623 ** that pLoop is a covering index loop, but it is not possible
007624 ** to be 100% sure. In this case, any OP_Explain opcode
007625 ** corresponding to this loop describes the index as a "COVERING
007626 ** INDEX". But, pOp proves that pLoop is not actually a covering
007627 ** index loop. So clear the WHERE_EXPRIDX flag and rewrite the
007628 ** text that accompanies the OP_Explain opcode, if any. */
007629 pLoop->wsFlags &= ~WHERE_EXPRIDX;
007630 sqlite3WhereAddExplainText(pParse,
007631 pLevel->addrBody-1,
007632 pTabList,
007633 pLevel,
007634 pWInfo->wctrlFlags
007635 );
007636 }
007637 }
007638 }else if( pOp->opcode==OP_Rowid ){
007639 pOp->p1 = pLevel->iIdxCur;
007640 pOp->opcode = OP_IdxRowid;
007641 OpcodeRewriteTrace(db, k, pOp);
007642 }else if( pOp->opcode==OP_IfNullRow ){
007643 pOp->p1 = pLevel->iIdxCur;
007644 OpcodeRewriteTrace(db, k, pOp);
007645 }
007646 #ifdef SQLITE_DEBUG
007647 k++;
007648 #endif
007649 }while( (++pOp)<pLastOp );
007650 #ifdef SQLITE_DEBUG
007651 if( db->flags & SQLITE_VdbeAddopTrace ) printf("TRANSLATE complete\n");
007652 #endif
007653 }
007654 }
007655
007656 /* The "break" point is here, just past the end of the outer loop.
007657 ** Set it.
007658 */
007659 sqlite3VdbeResolveLabel(v, pWInfo->iBreak);
007660
007661 /* Final cleanup
007662 */
007663 pParse->nQueryLoop = pWInfo->savedNQueryLoop;
007664 whereInfoFree(db, pWInfo);
007665 pParse->withinRJSubrtn -= nRJ;
007666 return;
007667 }