SQLite

Check-in [d79a37690c]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:If a BLOB looks like JSON when cast to text, then treat it as if it really were JSON. This replicates a long-standing bug in the JSON processing routines, and thereby avoids breaking legacy.
Timelines: family | ancestors | descendants | both | blob-as-json
Files: files | file ages | folders
SHA3-256: d79a37690ce7ebb91df203170d73511da44546328043c2b3fe1786b2f0087093
User & Date: drh 2024-01-23 13:21:40.846
Context
2024-01-23
13:28
Fix typo in the tag-20240123-a in a comment. (Closed-Leaf check-in: 1f09541eb4 user: drh tags: blob-as-json)
13:21
If a BLOB looks like JSON when cast to text, then treat it as if it really were JSON. This replicates a long-standing bug in the JSON processing routines, and thereby avoids breaking legacy. (check-in: d79a37690c user: drh tags: blob-as-json)
2024-01-22
14:16
The -DSQLITE_JSON_BLOB_INPUT_BUG_COMPATIBLE compile-time option causes blob inputs to JSON functions that are not JSONB to be processed as if they where text, immulating historical bugging behavior which some applications have come to rely upon. See [forum:/forumpost/012136abd5292b8d|forum thread 012136abd5292b8d] for discussion. (check-in: 6557222358 user: drh tags: blob-as-json)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/json.c.
3307
3308
3309
3310
3311
3312
3313
3314
3315

3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327

3328
3329
3330
3331
3332
3333
3334
  if( eType==SQLITE_BLOB ){
    if( jsonArgIsJsonb(pArg,p) ){
      if( (flgs & JSON_EDITABLE)!=0 && jsonBlobMakeEditable(p, 0)==0 ){
        goto json_pfa_oom;
      }
      return p;
    }
#if defined(SQLITE_JSON_BLOB_INPUT_BUG_COMPATIBLE)
    /* If the input is a BLOB that is not JSONB, fall through into trying

    ** to process that BLOB as if it where text.  This goes against all
    ** historical documentation about how the SQLite JSON functions are 
    ** suppose to work.  Nevertheless, many SQLite implementations prior to
    ** version 3.45.0 contained a bug such that they did behave this way
    ** and some applications came to depend upon this buggy behavior.  The
    ** SQLITE_JSON_BLOB_INPUT_BUG_COMPATIBLE compile-time option provides
    ** a mechanism for those applications to continue working even after
    ** the bug was fixed.  See
    ** https://sqlite.org/forum/forumpost/012136abd5292b8d */
#else
    goto json_pfa_malformed;
#endif

  }
  p->zJson = (char*)sqlite3_value_text(pArg);
  p->nJson = sqlite3_value_bytes(pArg);
  if( p->nJson==0 ) goto json_pfa_malformed;
  if( NEVER(p->zJson==0) ) goto json_pfa_oom;
  if( jsonConvertTextToBlob(p, (flgs & JSON_KEEPERROR) ? 0 : ctx) ){
    if( flgs & JSON_KEEPERROR ){







<
|
>
|
|
|
|
|
|
|
|
|
<
<
<
>







3307
3308
3309
3310
3311
3312
3313

3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324



3325
3326
3327
3328
3329
3330
3331
3332
  if( eType==SQLITE_BLOB ){
    if( jsonArgIsJsonb(pArg,p) ){
      if( (flgs & JSON_EDITABLE)!=0 && jsonBlobMakeEditable(p, 0)==0 ){
        goto json_pfa_oom;
      }
      return p;
    }

    /* If the blob is not valid JSONB, fall through into trying to cast
    ** the blob into text which is then interpreted as JSON.  (tag-20240123-a)
    **
    ** This goes against all historical documentation about how the SQLite
    ** JSON functions were suppose to work.  From the beginning, blob was
    ** reserved for expansion and a blob value should have raised an error.
    ** But it did not, due to a bug.  And many applications came to depend
    ** upon this buggy behavior, espeically when using the CLI and reading
    ** JSON text using readfile(), which returns a blob.  For this reason
    ** we will continue to support the bug moving forward.
    ** See for example https://sqlite.org/forum/forumpost/012136abd5292b8d



    */
  }
  p->zJson = (char*)sqlite3_value_text(pArg);
  p->nJson = sqlite3_value_bytes(pArg);
  if( p->nJson==0 ) goto json_pfa_malformed;
  if( NEVER(p->zJson==0) ) goto json_pfa_oom;
  if( jsonConvertTextToBlob(p, (flgs & JSON_KEEPERROR) ? 0 : ctx) ){
    if( flgs & JSON_KEEPERROR ){
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318

4319


4320
4321
4322
4323
4324
4325
4326
4327
#ifdef SQLITE_LEGACY_JSON_VALID
      /* Incorrect legacy behavior was to return FALSE for a NULL input */
      sqlite3_result_int(ctx, 0);
#endif
      return;
    }
    case SQLITE_BLOB: {
      if( (flags & 0x0c)!=0 && jsonFuncArgMightBeBinary(argv[0]) ){
        if( flags & 0x04 ){
          /* Superficial checking only - accomplished by the
          ** jsonFuncArgMightBeBinary() call above. */
          res = 1;
        }else{
          /* Strict checking.  Check by translating BLOB->TEXT->BLOB.  If
          ** no errors occur, call that a "strict check". */
          JsonParse px;
          u32 iErr;
          memset(&px, 0, sizeof(px));
          px.aBlob = (u8*)sqlite3_value_blob(argv[0]);
          px.nBlob = sqlite3_value_bytes(argv[0]);
          iErr = jsonbValidityCheck(&px, 0, px.nBlob, 1);
          res = iErr==0;
        }

      }


      break;
    }
    default: {
      JsonParse px;
      if( (flags & 0x3)==0 ) break;
      memset(&px, 0, sizeof(px));
     
      p = jsonParseFuncArg(ctx, argv[0], JSON_KEEPERROR);







|




|










>

>
>
|







4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
#ifdef SQLITE_LEGACY_JSON_VALID
      /* Incorrect legacy behavior was to return FALSE for a NULL input */
      sqlite3_result_int(ctx, 0);
#endif
      return;
    }
    case SQLITE_BLOB: {
      if( jsonFuncArgMightBeBinary(argv[0]) ){
        if( flags & 0x04 ){
          /* Superficial checking only - accomplished by the
          ** jsonFuncArgMightBeBinary() call above. */
          res = 1;
        }else if( flags & 0x08 ){
          /* Strict checking.  Check by translating BLOB->TEXT->BLOB.  If
          ** no errors occur, call that a "strict check". */
          JsonParse px;
          u32 iErr;
          memset(&px, 0, sizeof(px));
          px.aBlob = (u8*)sqlite3_value_blob(argv[0]);
          px.nBlob = sqlite3_value_bytes(argv[0]);
          iErr = jsonbValidityCheck(&px, 0, px.nBlob, 1);
          res = iErr==0;
        }
        break;
      }
      /* Fall through into interpreting the input as text.  See note
      ** above at tag-20240124-a. */
      /* no break */ deliberate_fall_through
    }
    default: {
      JsonParse px;
      if( (flags & 0x3)==0 ) break;
      memset(&px, 0, sizeof(px));
     
      p = jsonParseFuncArg(ctx, argv[0], JSON_KEEPERROR);
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
  UNUSED_PARAMETER(idxStr);
  UNUSED_PARAMETER(argc);
  jsonEachCursorReset(p);
  if( idxNum==0 ) return SQLITE_OK;
  memset(&p->sParse, 0, sizeof(p->sParse));
  p->sParse.nJPRef = 1;
  p->sParse.db = p->db;
  if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){
    if( jsonFuncArgMightBeBinary(argv[0]) ){
      p->sParse.nBlob = sqlite3_value_bytes(argv[0]);
      p->sParse.aBlob = (u8*)sqlite3_value_blob(argv[0]);
    }else{
      goto json_each_malformed_input;
    }
  }else{
    p->sParse.zJson = (char*)sqlite3_value_text(argv[0]);
    p->sParse.nJson = sqlite3_value_bytes(argv[0]);
    if( p->sParse.zJson==0 ){
      p->i = p->iEnd = 0;
      return SQLITE_OK;
    }      







<
|
|
|
<
<
<







5050
5051
5052
5053
5054
5055
5056

5057
5058
5059



5060
5061
5062
5063
5064
5065
5066
  UNUSED_PARAMETER(idxStr);
  UNUSED_PARAMETER(argc);
  jsonEachCursorReset(p);
  if( idxNum==0 ) return SQLITE_OK;
  memset(&p->sParse, 0, sizeof(p->sParse));
  p->sParse.nJPRef = 1;
  p->sParse.db = p->db;

  if( jsonFuncArgMightBeBinary(argv[0]) ){
    p->sParse.nBlob = sqlite3_value_bytes(argv[0]);
    p->sParse.aBlob = (u8*)sqlite3_value_blob(argv[0]);



  }else{
    p->sParse.zJson = (char*)sqlite3_value_text(argv[0]);
    p->sParse.nJson = sqlite3_value_bytes(argv[0]);
    if( p->sParse.zJson==0 ){
      p->i = p->iEnd = 0;
      return SQLITE_OK;
    }      
Added test/json107.test.












































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
# 2024-01-23
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# 
# Legacy JSON bug:  If the input is a BLOB that when cast into TEXT looks
# like valid JSON, then treat it as valid JSON.
#
# The original intent of the JSON functions was to raise an error on any
# BLOB input.  That intent was clearly documented, but the code failed to
# to implement it.  Subsequently, many applications began to depend on the
# incorrect behavior, especially apps that used readfile() to read JSON
# content, since readfile() returns a BLOB.  So we need to support the
# bug moving forward.
#
# The tests in this fail verify that the original buggy behavior is
# preserved.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix json107

if {[db one {PRAGMA encoding}]!="UTF-8"} {
  # These tests only work for a UTF-8 encoding.
  finish_test
  return
}

do_execsql_test 1.1 {
  SELECT json_valid( CAST('{"a":1}' AS BLOB) );
} 1
do_execsql_test 1.1.1 {
  SELECT json_valid( CAST('{"a":1}' AS BLOB), 1);
} 1
do_execsql_test 1.1.2 {
  SELECT json_valid( CAST('{"a":1}' AS BLOB), 2);
} 1
do_execsql_test 1.1.4 {
  SELECT json_valid( CAST('{"a":1}' AS BLOB), 4);
} 0
do_execsql_test 1.1.8 {
  SELECT json_valid( CAST('{"a":1}' AS BLOB), 8);
} 0

do_execsql_test 1.2.1 {
  SELECT CAST('{"a":123}' AS blob) -> 'a';
} 123
do_execsql_test 1.2.2 {
  SELECT CAST('{"a":123}' AS blob) ->> 'a';
} 123
do_execsql_test 1.2.3 {
  SELECT json_extract(CAST('{"a":123}' AS blob), '$.a');
} 123
do_execsql_test 1.3 {
  SELECT json_insert(CAST('{"a":123}' AS blob),'$.b',456);
} {{{"a":123,"b":456}}}
do_execsql_test 1.4 {
  SELECT json_remove(CAST('{"a":123,"b":456}' AS blob),'$.a');
} {{{"b":456}}}
do_execsql_test 1.5 {
  SELECT json_set(CAST('{"a":123,"b":456}' AS blob),'$.a',789);
} {{{"a":789,"b":456}}}
do_execsql_test 1.6 {
  SELECT json_replace(CAST('{"a":123,"b":456}' AS blob),'$.a',789);
} {{{"a":789,"b":456}}}
do_execsql_test 1.7 {
  SELECT json_type(CAST('{"a":123,"b":456}' AS blob));
} object
do_execsql_test 1.8 {
  SELECT json(CAST('{"a":123,"b":456}' AS blob));
} {{{"a":123,"b":456}}}

ifcapable vtab {
  do_execsql_test 2.1 {
    SELECT key, value FROM json_tree( CAST('{"a":123,"b":456}' AS blob) )
      WHERE atom;
  } {a 123 b 456}
} 
finish_test