SQLite

Changes On Branch default-in-values-2
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch default-in-values-2 Excluding Merge-Ins

This is equivalent to a diff from 1860ea060b to 9349398e15

2025-02-16
10:57
Add a typecast to avoid 32-bit integer overflow in the concat_ws() function with an enormous separator values and many arguments. (check-in: 498e3f1cf5 user: drh tags: trunk)
10:50
Cleanup of the changes for this branch. (Leaf check-in: 9349398e15 user: drh tags: default-in-values-2)
2025-02-15
23:47
Bug fixes in the INSERT logic for VALUES containing DEFAULT terms. (check-in: 2b129c3761 user: drh tags: default-in-values-2)
20:31
An alternative implementation of the default-in-values feature that is cleaner (it avoids dodgy poking about in the parser LALR stack looking for errors) and has less performance impact in the common case where DEFAULT is not used. (check-in: a3d831378d user: drh tags: default-in-values-2)
17:29
Remove tea version check from tool/srctree-check.tcl, as it's obsoleted by [be265559]. (check-in: 1860ea060b user: stephan tags: trunk)
16:24
Automate update of the library version number in autoconf/tea/configure.ac as part of the tool/mkautoconfamal.sh process, per /chat discussion. (check-in: be265559a3 user: stephan tags: trunk)

Changes to src/expr.c.
4891
4892
4893
4894
4895
4896
4897







4898
4899
4900
4901
4902
4903
4904
      return target;
    }
#endif
    case TK_STRING: {
      assert( !ExprHasProperty(pExpr, EP_IntValue) );
      sqlite3VdbeLoadString(v, target, pExpr->u.zToken);
      return target;







    }
    default: {
      /* Make NULL the default case so that if a bug causes an illegal
      ** Expr node to be passed into this function, it will be handled
      ** sanely and not crash.  But keep the assert() to bring the problem
      ** to the attention of the developers. */
      assert( op==TK_NULL || op==TK_ERROR || pParse->db->mallocFailed );







>
>
>
>
>
>
>







4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
      return target;
    }
#endif
    case TK_STRING: {
      assert( !ExprHasProperty(pExpr, EP_IntValue) );
      sqlite3VdbeLoadString(v, target, pExpr->u.zToken);
      return target;
    }
    case TK_DEFAULT: {
      /* If the DEFAULT keyword ever gets this far, that means it was used
      ** in a context that is not supported.  So raise an error. */
      sqlite3ErrorMsg(pParse, "near \"default\": syntax error");
      sqlite3RecordErrorOffsetOfExpr(pParse->db, pExpr);
      return target;
    }
    default: {
      /* Make NULL the default case so that if a bug causes an illegal
      ** Expr node to be passed into this function, it will be handled
      ** sanely and not crash.  But keep the assert() to bring the problem
      ** to the attention of the developers. */
      assert( op==TK_NULL || op==TK_ERROR || pParse->db->mallocFailed );
Changes to src/insert.c.
667
668
669
670
671
672
673
674






675
676
677
678

679
680
681
682
683
684
685
**       the VALUES clause is part of a correlated sub-query).
**
**    d) One or more of the values in the first row of the VALUES clause
**       has an affinity (i.e. is a CAST expression). This causes problems
**       because the complex rules SQLite uses (see function 
**       sqlite3SubqueryColumnTypes() in select.c) to determine the effective
**       affinity of such a column for all rows require access to all values in
**       the column simultaneously. 






*/
Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList *pRow){

  if( pParse->bHasWith                   /* condition (a) above */

   || pParse->db->init.busy              /* condition (b) above */
   || exprListIsConstant(pParse,pRow)==0 /* condition (c) above */
   || (pLeft->pSrc->nSrc==0 &&
       exprListIsNoAffinity(pParse,pLeft->pEList)==0) /* condition (d) above */
   || IN_SPECIAL_PARSE
  ){
    /* The co-routine method cannot be used. Fall back to UNION ALL. */







|
>
>
>
>
>
>




>







667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
**       the VALUES clause is part of a correlated sub-query).
**
**    d) One or more of the values in the first row of the VALUES clause
**       has an affinity (i.e. is a CAST expression). This causes problems
**       because the complex rules SQLite uses (see function 
**       sqlite3SubqueryColumnTypes() in select.c) to determine the effective
**       affinity of such a column for all rows require access to all values in
**       the column simultaneously.
**
**    e) The DEFAULT keyword cannot have been used in any of the terms
**       of the VALUES clause.  This optimization won't work in that case
**       because we do not yet know the mapping from VALUES-clause terms
**       into table columns, so we cannot figure out which column default
**       value to use in place of the DEFAULT keyword.
*/
Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList *pRow){

  if( pParse->bHasWith                   /* condition (a) above */
   || pParse->bDfltInExpr                /* condition (e) above */
   || pParse->db->init.busy              /* condition (b) above */
   || exprListIsConstant(pParse,pRow)==0 /* condition (c) above */
   || (pLeft->pSrc->nSrc==0 &&
       exprListIsNoAffinity(pParse,pLeft->pEList)==0) /* condition (d) above */
   || IN_SPECIAL_PARSE
  ){
    /* The co-routine method cannot be used. Fall back to UNION ALL. */
777
778
779
780
781
782
783














































































































784
785
786
787
788
789
790
      }
    }
    sqlite3ExprListDelete(pParse->db, pRow);
  }

  return pLeft;
}















































































































/* Forward declaration */
static int xferOptimization(
  Parse *pParse,        /* Parser context */
  Table *pDest,         /* The table we are inserting into */
  Select *pSelect,      /* A SELECT statement to use as the data source */
  int onError,          /* How to handle constraint errors */







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
      }
    }
    sqlite3ExprListDelete(pParse->db, pRow);
  }

  return pLeft;
}

/*
** The aTabColMap[] array maps table columns into terms of the IDLIST
** of an INSERT.  For example, if we have:
**
**      CREATE TABLE t1(a,b,c,d,e,f,g);
**                   \/ \___________/
**           pTab----'        `-------- pTab->aCol[]
**
** And we do:
**
**      INSERT INTO t1(e,b,g) ....
**                  \/ \___/
**           pTab---'    `----- IDLIST
**
** Then aTabColMap[] contains: { 0, 2, 0, 0, 1, 0, 3 }
** Thus aTabColMap provides a one-based mapping of table column indexes into
** IDLIST entries.  0 means "none" because there are often table columns
** that are not in the IDLIST.  The left-most table columns is 1, and
** the next table columns is 2 and so forth.
**
** This routine creates a new array (obtained from sqlite3DbMalloc() - 
** the caller must free it) that inverts the mapping.  The returned
** array aColTabMap[] would be {4, 1, 6}.  This new mapping is zero-based.
**
** The aTabColMap input might both be NULL.  This means that the IDLIST
** on the INSERT is omitted.  This routine still constructs a column map,
** but in this case it maps "insertable" columns of the table into actual
** columns.  Hidden and computed columns are not "insertable" and are
** thus skipped.
*/
static int *computeColTabMap(
  Parse *pParse,            /* Parsing context */
  int *const aTabColMap,    /* Mapping from table column to IDList column */
  Table *pTab,              /* The table */
  int nId                   /* Number of columns in the IDLIST */
){
  int *aColTabMap;

  if( pParse->nErr ) return 0;
  assert( pTab->nCol>0 );
  aColTabMap = sqlite3DbMallocZero(pParse->db, sizeof(int)*nId);
  if( aColTabMap==0 ) return 0;
  if( aTabColMap ){
    /* Invert the aTabColMap[] */
    int i;
    assert( sqlite3DbMallocSize(pParse->db, aTabColMap) >=
            sizeof(int)*pTab->nCol );
    for(i=0; i<pTab->nCol; i++){
      if( aTabColMap[i]>0 ){
        aColTabMap[aTabColMap[i]-1] = i;
      }
    }
  }else{
    /* Construct a new mapping that merely skips over non-insertable terms */
    int i, j;
    for(i=j=0; i<pTab->nCol; i++){
      if( (pTab->aCol[i].colFlags & COLFLAG_NOINSERT)==0 ){
        aColTabMap[j++] = i;
      }
    }
  }
  return aColTabMap;
}

/*
** Expression pExpr is a DEFAULT keyword.  Transform that expression to
** an expansion of the actual default value for the iCol-th column of
** table pTab.
*/
void sqlite3ExpandDefaultValue(
  Parse *pParse,    /* Parsing context */
  Expr *pExpr,      /* Expression to be transformed */
  Table *pTab,      /* Table that supplies the new value to pExpr */
  int iCol          /* Column of pTab that contains the new value for pExpr */
){
  assert( pExpr->op==TK_DEFAULT );
  assert( iCol>=0 && iCol<pTab->nCol );
  if( pTab->aCol[iCol].iDflt==0 ){
    pExpr->op = TK_NULL;
  }else{
    Expr *pDfltVal = sqlite3ColumnExpr(pTab, &pTab->aCol[iCol]);
    pExpr->op = TK_UPLUS;
    assert( pExpr->pLeft==0 );
    pExpr->pLeft = sqlite3ExprDup(pParse->db, pDfltVal, 0);
  }
}

/*
** Scan all expressions in pList.  If any is just a DEFAULT keyword,
** convert that expression into a new expression that evaluates to
** the default value of the corresponding table.
*/
static void convertDefaultExpr(
  Parse *pParse,    /* Parsing context */
  int *aColTabMap,  /* Mapping from pList entry to pTab column number */
  Table *pTab,      /* Table being inserted into */
  ExprList *pList,  /* The list to scan */
  int nId           /* Number of columns in aColTabMap */
){
  int i;
  assert( aColTabMap!=0 );
  assert( sqlite3DbMallocSize(pParse->db, aColTabMap) >= sizeof(int)*nId );
  for(i=0; i<pList->nExpr && i<nId; i++){
    Expr *p = pList->a[i].pExpr;
    if( p->op==TK_DEFAULT ){
      sqlite3ExpandDefaultValue(pParse, p, pTab, aColTabMap[i]);
    }
  }
}

/* Forward declaration */
static int xferOptimization(
  Parse *pParse,        /* Parser context */
  Table *pDest,         /* The table we are inserting into */
  Select *pSelect,      /* A SELECT statement to use as the data source */
  int onError,          /* How to handle constraint errors */
841
842
843
844
845
846
847
848

849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
**           open a read cursor on the corresponding <table2> index
**           transfer all records from the read to the write cursors
**           close cursors
**         end foreach
**
** The 3rd template is for when the second template does not apply
** and the SELECT clause does not read from <table> at any time.
** The generated code follows this template:

**
**         X <- A
**         goto B
**      A: setup for the SELECT
**         loop over the rows in the SELECT
**           load values into registers R..R+n
**           yield X
**         end loop
**         cleanup after the SELECT
**         end-coroutine X
**      B: open write cursor to <table> and its indices
**      C: yield X, at EOF goto D
**         insert the select result into <table> from R..R+n
**         goto C
**      D: cleanup
**
** The 4th template is used if the insert statement takes its
** values from a SELECT but the data is being inserted into a table
** that is also read as part of the SELECT.  In the third form,
** we have to use an intermediate table to store the results of
** the select.  The template is like this:
**
**         X <- A
**         goto B
**      A: setup for the SELECT
**         loop over the tables in the SELECT







|
>


















|







958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
**           open a read cursor on the corresponding <table2> index
**           transfer all records from the read to the write cursors
**           close cursors
**         end foreach
**
** The 3rd template is for when the second template does not apply
** and the SELECT clause does not read from <table> at any time.
** This template is also used when the INSERT has a VALUES clause with
** two or more rows.  Pseudocode for this, the 3rd template is:
**
**         X <- A
**         goto B
**      A: setup for the SELECT
**         loop over the rows in the SELECT
**           load values into registers R..R+n
**           yield X
**         end loop
**         cleanup after the SELECT
**         end-coroutine X
**      B: open write cursor to <table> and its indices
**      C: yield X, at EOF goto D
**         insert the select result into <table> from R..R+n
**         goto C
**      D: cleanup
**
** The 4th template is used if the insert statement takes its
** values from a SELECT but the data is being inserted into a table
** that is also read as part of the SELECT.  In the fourth form,
** we have to use an intermediate table to store the results of
** the select.  The template is like this:
**
**         X <- A
**         goto B
**      A: setup for the SELECT
**         loop over the tables in the SELECT
1099
1100
1101
1102
1103
1104
1105


























1106
1107
1108
1109
1110
1111
1112
              pTabList->a, pColumn->a[i].zName);
          pParse->checkSchema = 1;
          goto insert_cleanup;
        }
      }
    }
  }



























  /* Figure out how many columns of data are supplied.  If the data
  ** is coming from a SELECT statement, then generate a co-routine that
  ** produces a single row of the SELECT on each invocation.  The
  ** co-routine is the common header to the 3rd and 4th templates.
  */
  if( pSelect ){







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
              pTabList->a, pColumn->a[i].zName);
          pParse->checkSchema = 1;
          goto insert_cleanup;
        }
      }
    }
  }

  /* If there are DEFAULT keywords within VALUES clauses on the right-hand
  ** side of this INSERT, convert them into the corresponding column default
  ** values.
  */
  if( pParse->bDfltInExpr && (pList || pSelect) ){
    int nId;
    int *aColTabMap;
    nId = pColumn ? pColumn->nId : 0;
    if( pTab->nCol > nId ) nId = pTab->nCol;
    aColTabMap = computeColTabMap(pParse, aTabColMap, pTab, nId);
    if( aColTabMap==0 ){
      assert( pParse->nErr );
    }else if( pSelect==0 ){
      /* A single-row VALUES clause in pList */
      convertDefaultExpr(pParse, aColTabMap, pTab, pList, nId);
    }else if( (pSelect->selFlags & SF_Values)!=0 ){
      /* A multi-row VALUES clause in pSelect */
      Select *pS = pSelect;
      do{
        convertDefaultExpr(pParse, aColTabMap, pTab, pS->pEList, nId);
        pS = pS->pPrior;
      }while( pS );
    }
    sqlite3DbFree(db, aColTabMap);
  }

  /* Figure out how many columns of data are supplied.  If the data
  ** is coming from a SELECT statement, then generate a co-routine that
  ** produces a single row of the SELECT on each invocation.  The
  ** co-routine is the common header to the 3rd and 4th templates.
  */
  if( pSelect ){
Changes to src/parse.y.
1399
1400
1401
1402
1403
1404
1405







1406
1407
1408
1409
1410
1411
1412
}

expr(A) ::= expr(B) PTR(C) expr(D). {
  ExprList *pList = sqlite3ExprListAppend(pParse, 0, B);
  pList = sqlite3ExprListAppend(pParse, pList, D);
  A = sqlite3ExprFunction(pParse, pList, &C, 0);
}








%type between_op {int}
between_op(A) ::= BETWEEN.     {A = 0;}
between_op(A) ::= NOT BETWEEN. {A = 1;}
expr(A) ::= expr(A) between_op(N) expr(X) AND expr(Y). [BETWEEN] {
  ExprList *pList = sqlite3ExprListAppend(pParse,0, X);
  pList = sqlite3ExprListAppend(pParse,pList, Y);







>
>
>
>
>
>
>







1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
}

expr(A) ::= expr(B) PTR(C) expr(D). {
  ExprList *pList = sqlite3ExprListAppend(pParse, 0, B);
  pList = sqlite3ExprListAppend(pParse, pList, D);
  A = sqlite3ExprFunction(pParse, pList, &C, 0);
}

expr(A) ::= DEFAULT(X). {
  Expr *p = sqlite3Expr(pParse->db, TK_DEFAULT, 0);
  sqlite3ExprSetErrorOffset(p, (int)(X.z - pParse->zTail));
  pParse->bDfltInExpr = 1;
  A = p;
}

%type between_op {int}
between_op(A) ::= BETWEEN.     {A = 0;}
between_op(A) ::= NOT BETWEEN. {A = 1;}
expr(A) ::= expr(A) between_op(N) expr(X) AND expr(Y). [BETWEEN] {
  ExprList *pList = sqlite3ExprListAppend(pParse,0, X);
  pList = sqlite3ExprListAppend(pParse,pList, Y);
Changes to src/sqliteInt.h.
3851
3852
3853
3854
3855
3856
3857

3858
3859
3860
3861
3862
3863
3864
  u8 isCreate;         /* CREATE TABLE, INDEX, or VIEW (but not TRIGGER)
                       ** and ALTER TABLE ADD COLUMN. */
#endif
  bft colNamesSet :1;   /* TRUE after OP_ColumnName has been issued to pVdbe */
  bft bHasWith :1;      /* True if statement contains WITH */
  bft okConstFactor :1; /* OK to factor out constants */
  bft checkSchema :1;   /* Causes schema cookie check after an error */

  int nRangeReg;       /* Size of the temporary register block */
  int iRangeReg;       /* First register in temporary register block */
  int nErr;            /* Number of errors seen */
  int nTab;            /* Number of previously allocated VDBE cursors */
  int nMem;            /* Number of memory cells used so far */
  int szOpAlloc;       /* Bytes of memory space allocated for Vdbe.aOp[] */
  int iSelfTab;        /* Table associated with an index on expr, or negative







>







3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
  u8 isCreate;         /* CREATE TABLE, INDEX, or VIEW (but not TRIGGER)
                       ** and ALTER TABLE ADD COLUMN. */
#endif
  bft colNamesSet :1;   /* TRUE after OP_ColumnName has been issued to pVdbe */
  bft bHasWith :1;      /* True if statement contains WITH */
  bft okConstFactor :1; /* OK to factor out constants */
  bft checkSchema :1;   /* Causes schema cookie check after an error */
  bft bDfltInExpr :1;   /* DEFAULT keyword occurs in an expression */
  int nRangeReg;       /* Size of the temporary register block */
  int iRangeReg;       /* First register in temporary register block */
  int nErr;            /* Number of errors seen */
  int nTab;            /* Number of previously allocated VDBE cursors */
  int nMem;            /* Number of memory cells used so far */
  int szOpAlloc;       /* Bytes of memory space allocated for Vdbe.aOp[] */
  int iSelfTab;        /* Table associated with an index on expr, or negative
4971
4972
4973
4974
4975
4976
4977

4978
4979
4980
4981
4982
4983
4984
#ifndef SQLITE_OMIT_AUTOINCREMENT
  void sqlite3AutoincrementBegin(Parse *pParse);
  void sqlite3AutoincrementEnd(Parse *pParse);
#else
# define sqlite3AutoincrementBegin(X)
# define sqlite3AutoincrementEnd(X)
#endif

void sqlite3Insert(Parse*, SrcList*, Select*, IdList*, int, Upsert*);
#ifndef SQLITE_OMIT_GENERATED_COLUMNS
  void sqlite3ComputeGeneratedColumns(Parse*, int, Table*);
#endif
void *sqlite3ArrayAllocate(sqlite3*,void*,int,int*,int*);
IdList *sqlite3IdListAppend(Parse*, IdList*, Token*);
int sqlite3IdListIndex(IdList*,const char*);







>







4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
#ifndef SQLITE_OMIT_AUTOINCREMENT
  void sqlite3AutoincrementBegin(Parse *pParse);
  void sqlite3AutoincrementEnd(Parse *pParse);
#else
# define sqlite3AutoincrementBegin(X)
# define sqlite3AutoincrementEnd(X)
#endif
void sqlite3ExpandDefaultValue(Parse*, Expr*, Table*, int);
void sqlite3Insert(Parse*, SrcList*, Select*, IdList*, int, Upsert*);
#ifndef SQLITE_OMIT_GENERATED_COLUMNS
  void sqlite3ComputeGeneratedColumns(Parse*, int, Table*);
#endif
void *sqlite3ArrayAllocate(sqlite3*,void*,int,int*,int*);
IdList *sqlite3IdListAppend(Parse*, IdList*, Token*);
int sqlite3IdListIndex(IdList*,const char*);
Changes to src/update.c.
486
487
488
489
490
491
492



493
494
495
496
497
498
499
        sqlite3ErrorMsg(pParse,
           "cannot UPDATE generated column \"%s\"",
           pTab->aCol[j].zCnName);
        goto update_cleanup;
      }
#endif
      aXRef[j] = i;



    }else{
      if( pPk==0 && sqlite3IsRowid(pChanges->a[i].zEName) ){
        j = -1;
        chngRowid = 1;
        pRowidExpr = pChanges->a[i].pExpr;
        iRowidExpr = i;
      }else{







>
>
>







486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
        sqlite3ErrorMsg(pParse,
           "cannot UPDATE generated column \"%s\"",
           pTab->aCol[j].zCnName);
        goto update_cleanup;
      }
#endif
      aXRef[j] = i;
      if( pChanges->a[i].pExpr->op==TK_DEFAULT ){
        sqlite3ExpandDefaultValue(pParse, pChanges->a[i].pExpr, pTab, j);
      }
    }else{
      if( pPk==0 && sqlite3IsRowid(pChanges->a[i].zEName) ){
        j = -1;
        chngRowid = 1;
        pRowidExpr = pChanges->a[i].pExpr;
        iRowidExpr = i;
      }else{