SQLite

Changes On Branch blob-as-json
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch blob-as-json Excluding Merge-Ins

This is equivalent to a diff from 4dc00f5776 to 1f09541eb4

2024-01-23
13:51
When a JSON input is a blob, but it looks like valid JSON when cast to text, then accept it as valid JSON. This replicates a long-standing bug in the behavior of JSON routines, and thus avoids breaking legacy apps. [forum:/forumpost/012136abd5292b8d|Forum thread 012136abd5292b8d]. (check-in: e5dc81d5c7 user: drh tags: trunk)
13:28
Fix typo in the tag-20240123-a in a comment. (Closed-Leaf check-in: 1f09541eb4 user: drh tags: blob-as-json)
13:21
If a BLOB looks like JSON when cast to text, then treat it as if it really were JSON. This replicates a long-standing bug in the JSON processing routines, and thereby avoids breaking legacy. (check-in: d79a37690c user: drh tags: blob-as-json)
12:51
Change the shell's --help flag to exit with code 0 instead of 1, per /forum and /chat discussions. (check-in: df563a1857 user: stephan tags: trunk)
2024-01-22
14:16
The -DSQLITE_JSON_BLOB_INPUT_BUG_COMPATIBLE compile-time option causes blob inputs to JSON functions that are not JSONB to be processed as if they where text, immulating historical bugging behavior which some applications have come to rely upon. See [forum:/forumpost/012136abd5292b8d|forum thread 012136abd5292b8d] for discussion. (check-in: 6557222358 user: drh tags: blob-as-json)
14:01
Fix test script literal.test so that it works with SQLITE_OMIT_ALTER_TABLE builds. (check-in: 4dc00f5776 user: dan tags: trunk)
12:56
Fix userauth so that it works together with SQLITE_OMIT_SHARED_CACHE. ([forum:/forumpost/0bfc5888a384d430|Forum post 0bfc5888a384d430]). However, also change to code to issue a deprecation warning whenever SQLITE_USER_AUTHENTICATION is used. (check-in: 249048b0cb user: drh tags: trunk)
11:12
Fix test script literal.test so that it works with SQLITE_OMIT_ALTER_TABLE builds. (check-in: b57327be4b user: dan tags: digit-separators)

Changes to src/json.c.
3212
3213
3214
3215
3216
3217
3218
































3219
3220
3221
3222
3223
3224
3225
  if( rc==JSON_LOOKUP_ERROR ){
    sqlite3_result_error(ctx, "malformed JSON", -1);
  }else{
    jsonBadPathError(ctx, zPath);
  }
  return;
}

































/*
** Generate a JsonParse object, containing valid JSONB in aBlob and nBlob,
** from the SQL function argument pArg.  Return a pointer to the new
** JsonParse object.
**
** Ownership of the new JsonParse object is passed to the caller.  The







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
  if( rc==JSON_LOOKUP_ERROR ){
    sqlite3_result_error(ctx, "malformed JSON", -1);
  }else{
    jsonBadPathError(ctx, zPath);
  }
  return;
}

/*
** If pArg is a blob that seems like a JSONB blob, then initialize
** p to point to that JSONB and return TRUE.  If pArg does not seem like
** a JSONB blob, then return FALSE;
**
** This routine is only called if it is already known that pArg is a
** blob.  The only open question is whether or not the blob appears
** to be a JSONB blob.
*/
static int jsonArgIsJsonb(sqlite3_value *pArg, JsonParse *p){
  u32 n, sz = 0;
  p->aBlob = (u8*)sqlite3_value_blob(pArg);
  p->nBlob = (u32)sqlite3_value_bytes(pArg);
  if( p->nBlob==0 ){
    p->aBlob = 0;
    return 0;
  }
  if( NEVER(p->aBlob==0) ){
    return 0;
  }
  if( (p->aBlob[0] & 0x0f)<=JSONB_OBJECT
   && (n = jsonbPayloadSize(p, 0, &sz))>0
   && sz+n==p->nBlob
   && ((p->aBlob[0] & 0x0f)>JSONB_FALSE || sz==0)
  ){
    return 1;
  }
  p->aBlob = 0;
  p->nBlob = 0;
  return 0;
}

/*
** Generate a JsonParse object, containing valid JSONB in aBlob and nBlob,
** from the SQL function argument pArg.  Return a pointer to the new
** JsonParse object.
**
** Ownership of the new JsonParse object is passed to the caller.  The
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298













3299
3300
3301
3302
3303
3304
3305
    memcpy(p->aBlob, pFromCache->aBlob, nBlob);
    p->nBlobAlloc = p->nBlob = nBlob;
    p->hasNonstd = pFromCache->hasNonstd;
    jsonParseFree(pFromCache);
    return p;
  }
  if( eType==SQLITE_BLOB ){
    u32 n, sz = 0;
    p->aBlob = (u8*)sqlite3_value_blob(pArg);
    p->nBlob = (u32)sqlite3_value_bytes(pArg);
    if( p->nBlob==0 ){
      goto json_pfa_malformed;
    }
    if( NEVER(p->aBlob==0) ){
      goto json_pfa_oom;
    }
    if( (p->aBlob[0] & 0x0f)>JSONB_OBJECT ){
      goto json_pfa_malformed;
    }
    n = jsonbPayloadSize(p, 0, &sz);
    if( n==0 
     || sz+n!=p->nBlob
     || ((p->aBlob[0] & 0x0f)<=JSONB_FALSE && sz>0)
    ){
      goto json_pfa_malformed;
    }
    if( (flgs & JSON_EDITABLE)!=0 && jsonBlobMakeEditable(p, 0)==0 ){
      goto json_pfa_oom;
    }
    return p;













  }
  p->zJson = (char*)sqlite3_value_text(pArg);
  p->nJson = sqlite3_value_bytes(pArg);
  if( p->nJson==0 ) goto json_pfa_malformed;
  if( NEVER(p->zJson==0) ) goto json_pfa_oom;
  if( jsonConvertTextToBlob(p, (flgs & JSON_KEEPERROR) ? 0 : ctx) ){
    if( flgs & JSON_KEEPERROR ){







<
<
<
<
<
<
<
<
<
<
<
<
<
|
<
<
<
<
<
|
|
|
|
>
>
>
>
>
>
>
>
>
>
>
>
>







3301
3302
3303
3304
3305
3306
3307













3308





3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
    memcpy(p->aBlob, pFromCache->aBlob, nBlob);
    p->nBlobAlloc = p->nBlob = nBlob;
    p->hasNonstd = pFromCache->hasNonstd;
    jsonParseFree(pFromCache);
    return p;
  }
  if( eType==SQLITE_BLOB ){













    if( jsonArgIsJsonb(pArg,p) ){





      if( (flgs & JSON_EDITABLE)!=0 && jsonBlobMakeEditable(p, 0)==0 ){
        goto json_pfa_oom;
      }
      return p;
    }
    /* If the blob is not valid JSONB, fall through into trying to cast
    ** the blob into text which is then interpreted as JSON.  (tag-20240123-a)
    **
    ** This goes against all historical documentation about how the SQLite
    ** JSON functions were suppose to work.  From the beginning, blob was
    ** reserved for expansion and a blob value should have raised an error.
    ** But it did not, due to a bug.  And many applications came to depend
    ** upon this buggy behavior, espeically when using the CLI and reading
    ** JSON text using readfile(), which returns a blob.  For this reason
    ** we will continue to support the bug moving forward.
    ** See for example https://sqlite.org/forum/forumpost/012136abd5292b8d
    */
  }
  p->zJson = (char*)sqlite3_value_text(pArg);
  p->nJson = sqlite3_value_bytes(pArg);
  if( p->nJson==0 ) goto json_pfa_malformed;
  if( NEVER(p->zJson==0) ) goto json_pfa_oom;
  if( jsonConvertTextToBlob(p, (flgs & JSON_KEEPERROR) ? 0 : ctx) ){
    if( flgs & JSON_KEEPERROR ){
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289

4290


4291
4292
4293
4294
4295
4296
4297
4298
#ifdef SQLITE_LEGACY_JSON_VALID
      /* Incorrect legacy behavior was to return FALSE for a NULL input */
      sqlite3_result_int(ctx, 0);
#endif
      return;
    }
    case SQLITE_BLOB: {
      if( (flags & 0x0c)!=0 && jsonFuncArgMightBeBinary(argv[0]) ){
        if( flags & 0x04 ){
          /* Superficial checking only - accomplished by the
          ** jsonFuncArgMightBeBinary() call above. */
          res = 1;
        }else{
          /* Strict checking.  Check by translating BLOB->TEXT->BLOB.  If
          ** no errors occur, call that a "strict check". */
          JsonParse px;
          u32 iErr;
          memset(&px, 0, sizeof(px));
          px.aBlob = (u8*)sqlite3_value_blob(argv[0]);
          px.nBlob = sqlite3_value_bytes(argv[0]);
          iErr = jsonbValidityCheck(&px, 0, px.nBlob, 1);
          res = iErr==0;
        }

      }


      break;
    }
    default: {
      JsonParse px;
      if( (flags & 0x3)==0 ) break;
      memset(&px, 0, sizeof(px));
     
      p = jsonParseFuncArg(ctx, argv[0], JSON_KEEPERROR);







|




|










>

>
>
|







4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
#ifdef SQLITE_LEGACY_JSON_VALID
      /* Incorrect legacy behavior was to return FALSE for a NULL input */
      sqlite3_result_int(ctx, 0);
#endif
      return;
    }
    case SQLITE_BLOB: {
      if( jsonFuncArgMightBeBinary(argv[0]) ){
        if( flags & 0x04 ){
          /* Superficial checking only - accomplished by the
          ** jsonFuncArgMightBeBinary() call above. */
          res = 1;
        }else if( flags & 0x08 ){
          /* Strict checking.  Check by translating BLOB->TEXT->BLOB.  If
          ** no errors occur, call that a "strict check". */
          JsonParse px;
          u32 iErr;
          memset(&px, 0, sizeof(px));
          px.aBlob = (u8*)sqlite3_value_blob(argv[0]);
          px.nBlob = sqlite3_value_bytes(argv[0]);
          iErr = jsonbValidityCheck(&px, 0, px.nBlob, 1);
          res = iErr==0;
        }
        break;
      }
      /* Fall through into interpreting the input as text.  See note
      ** above at tag-20240123-a. */
      /* no break */ deliberate_fall_through
    }
    default: {
      JsonParse px;
      if( (flags & 0x3)==0 ) break;
      memset(&px, 0, sizeof(px));
     
      p = jsonParseFuncArg(ctx, argv[0], JSON_KEEPERROR);
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
  UNUSED_PARAMETER(idxStr);
  UNUSED_PARAMETER(argc);
  jsonEachCursorReset(p);
  if( idxNum==0 ) return SQLITE_OK;
  memset(&p->sParse, 0, sizeof(p->sParse));
  p->sParse.nJPRef = 1;
  p->sParse.db = p->db;
  if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){
    if( jsonFuncArgMightBeBinary(argv[0]) ){
      p->sParse.nBlob = sqlite3_value_bytes(argv[0]);
      p->sParse.aBlob = (u8*)sqlite3_value_blob(argv[0]);
    }else{
      goto json_each_malformed_input;
    }
  }else{
    p->sParse.zJson = (char*)sqlite3_value_text(argv[0]);
    p->sParse.nJson = sqlite3_value_bytes(argv[0]);
    if( p->sParse.zJson==0 ){
      p->i = p->iEnd = 0;
      return SQLITE_OK;
    }      







<
|
|
|
<
<
<







5050
5051
5052
5053
5054
5055
5056

5057
5058
5059



5060
5061
5062
5063
5064
5065
5066
  UNUSED_PARAMETER(idxStr);
  UNUSED_PARAMETER(argc);
  jsonEachCursorReset(p);
  if( idxNum==0 ) return SQLITE_OK;
  memset(&p->sParse, 0, sizeof(p->sParse));
  p->sParse.nJPRef = 1;
  p->sParse.db = p->db;

  if( jsonFuncArgMightBeBinary(argv[0]) ){
    p->sParse.nBlob = sqlite3_value_bytes(argv[0]);
    p->sParse.aBlob = (u8*)sqlite3_value_blob(argv[0]);



  }else{
    p->sParse.zJson = (char*)sqlite3_value_text(argv[0]);
    p->sParse.nJson = sqlite3_value_bytes(argv[0]);
    if( p->sParse.zJson==0 ){
      p->i = p->iEnd = 0;
      return SQLITE_OK;
    }      
Added test/json107.test.












































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
# 2024-01-23
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# 
# Legacy JSON bug:  If the input is a BLOB that when cast into TEXT looks
# like valid JSON, then treat it as valid JSON.
#
# The original intent of the JSON functions was to raise an error on any
# BLOB input.  That intent was clearly documented, but the code failed to
# to implement it.  Subsequently, many applications began to depend on the
# incorrect behavior, especially apps that used readfile() to read JSON
# content, since readfile() returns a BLOB.  So we need to support the
# bug moving forward.
#
# The tests in this fail verify that the original buggy behavior is
# preserved.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix json107

if {[db one {PRAGMA encoding}]!="UTF-8"} {
  # These tests only work for a UTF-8 encoding.
  finish_test
  return
}

do_execsql_test 1.1 {
  SELECT json_valid( CAST('{"a":1}' AS BLOB) );
} 1
do_execsql_test 1.1.1 {
  SELECT json_valid( CAST('{"a":1}' AS BLOB), 1);
} 1
do_execsql_test 1.1.2 {
  SELECT json_valid( CAST('{"a":1}' AS BLOB), 2);
} 1
do_execsql_test 1.1.4 {
  SELECT json_valid( CAST('{"a":1}' AS BLOB), 4);
} 0
do_execsql_test 1.1.8 {
  SELECT json_valid( CAST('{"a":1}' AS BLOB), 8);
} 0

do_execsql_test 1.2.1 {
  SELECT CAST('{"a":123}' AS blob) -> 'a';
} 123
do_execsql_test 1.2.2 {
  SELECT CAST('{"a":123}' AS blob) ->> 'a';
} 123
do_execsql_test 1.2.3 {
  SELECT json_extract(CAST('{"a":123}' AS blob), '$.a');
} 123
do_execsql_test 1.3 {
  SELECT json_insert(CAST('{"a":123}' AS blob),'$.b',456);
} {{{"a":123,"b":456}}}
do_execsql_test 1.4 {
  SELECT json_remove(CAST('{"a":123,"b":456}' AS blob),'$.a');
} {{{"b":456}}}
do_execsql_test 1.5 {
  SELECT json_set(CAST('{"a":123,"b":456}' AS blob),'$.a',789);
} {{{"a":789,"b":456}}}
do_execsql_test 1.6 {
  SELECT json_replace(CAST('{"a":123,"b":456}' AS blob),'$.a',789);
} {{{"a":789,"b":456}}}
do_execsql_test 1.7 {
  SELECT json_type(CAST('{"a":123,"b":456}' AS blob));
} object
do_execsql_test 1.8 {
  SELECT json(CAST('{"a":123,"b":456}' AS blob));
} {{{"a":123,"b":456}}}

ifcapable vtab {
  do_execsql_test 2.1 {
    SELECT key, value FROM json_tree( CAST('{"a":123,"b":456}' AS blob) )
      WHERE atom;
  } {a 123 b 456}
} 
finish_test