Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | If a BLOB looks like JSON when cast to text, then treat it as if it really were JSON. This replicates a long-standing bug in the JSON processing routines, and thereby avoids breaking legacy. |
---|---|
Timelines: | family | ancestors | descendants | both | blob-as-json |
Files: | files | file ages | folders |
SHA3-256: |
d79a37690ce7ebb91df203170d73511d |
User & Date: | drh 2024-01-23 13:21:40.846 |
Context
2024-01-23
| ||
13:28 | Fix typo in the tag-20240123-a in a comment. (Closed-Leaf check-in: 1f09541eb4 user: drh tags: blob-as-json) | |
13:21 | If a BLOB looks like JSON when cast to text, then treat it as if it really were JSON. This replicates a long-standing bug in the JSON processing routines, and thereby avoids breaking legacy. (check-in: d79a37690c user: drh tags: blob-as-json) | |
2024-01-22
| ||
14:16 | The -DSQLITE_JSON_BLOB_INPUT_BUG_COMPATIBLE compile-time option causes blob inputs to JSON functions that are not JSONB to be processed as if they where text, immulating historical bugging behavior which some applications have come to rely upon. See [forum:/forumpost/012136abd5292b8d|forum thread 012136abd5292b8d] for discussion. (check-in: 6557222358 user: drh tags: blob-as-json) | |
Changes
Changes to src/json.c.
︙ | ︙ | |||
3307 3308 3309 3310 3311 3312 3313 | if( eType==SQLITE_BLOB ){ if( jsonArgIsJsonb(pArg,p) ){ if( (flgs & JSON_EDITABLE)!=0 && jsonBlobMakeEditable(p, 0)==0 ){ goto json_pfa_oom; } return p; } | < | > | | | | | | | | | < < < > | 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 | if( eType==SQLITE_BLOB ){ if( jsonArgIsJsonb(pArg,p) ){ if( (flgs & JSON_EDITABLE)!=0 && jsonBlobMakeEditable(p, 0)==0 ){ goto json_pfa_oom; } return p; } /* If the blob is not valid JSONB, fall through into trying to cast ** the blob into text which is then interpreted as JSON. (tag-20240123-a) ** ** This goes against all historical documentation about how the SQLite ** JSON functions were suppose to work. From the beginning, blob was ** reserved for expansion and a blob value should have raised an error. ** But it did not, due to a bug. And many applications came to depend ** upon this buggy behavior, espeically when using the CLI and reading ** JSON text using readfile(), which returns a blob. For this reason ** we will continue to support the bug moving forward. ** See for example https://sqlite.org/forum/forumpost/012136abd5292b8d */ } p->zJson = (char*)sqlite3_value_text(pArg); p->nJson = sqlite3_value_bytes(pArg); if( p->nJson==0 ) goto json_pfa_malformed; if( NEVER(p->zJson==0) ) goto json_pfa_oom; if( jsonConvertTextToBlob(p, (flgs & JSON_KEEPERROR) ? 0 : ctx) ){ if( flgs & JSON_KEEPERROR ){ |
︙ | ︙ | |||
4296 4297 4298 4299 4300 4301 4302 | #ifdef SQLITE_LEGACY_JSON_VALID /* Incorrect legacy behavior was to return FALSE for a NULL input */ sqlite3_result_int(ctx, 0); #endif return; } case SQLITE_BLOB: { | | | > > > | | 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 | #ifdef SQLITE_LEGACY_JSON_VALID /* Incorrect legacy behavior was to return FALSE for a NULL input */ sqlite3_result_int(ctx, 0); #endif return; } case SQLITE_BLOB: { if( jsonFuncArgMightBeBinary(argv[0]) ){ if( flags & 0x04 ){ /* Superficial checking only - accomplished by the ** jsonFuncArgMightBeBinary() call above. */ res = 1; }else if( flags & 0x08 ){ /* Strict checking. Check by translating BLOB->TEXT->BLOB. If ** no errors occur, call that a "strict check". */ JsonParse px; u32 iErr; memset(&px, 0, sizeof(px)); px.aBlob = (u8*)sqlite3_value_blob(argv[0]); px.nBlob = sqlite3_value_bytes(argv[0]); iErr = jsonbValidityCheck(&px, 0, px.nBlob, 1); res = iErr==0; } break; } /* Fall through into interpreting the input as text. See note ** above at tag-20240124-a. */ /* no break */ deliberate_fall_through } default: { JsonParse px; if( (flags & 0x3)==0 ) break; memset(&px, 0, sizeof(px)); p = jsonParseFuncArg(ctx, argv[0], JSON_KEEPERROR); |
︙ | ︙ | |||
5049 5050 5051 5052 5053 5054 5055 | UNUSED_PARAMETER(idxStr); UNUSED_PARAMETER(argc); jsonEachCursorReset(p); if( idxNum==0 ) return SQLITE_OK; memset(&p->sParse, 0, sizeof(p->sParse)); p->sParse.nJPRef = 1; p->sParse.db = p->db; | < | | | < < < | 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059 5060 5061 5062 5063 5064 5065 5066 | UNUSED_PARAMETER(idxStr); UNUSED_PARAMETER(argc); jsonEachCursorReset(p); if( idxNum==0 ) return SQLITE_OK; memset(&p->sParse, 0, sizeof(p->sParse)); p->sParse.nJPRef = 1; p->sParse.db = p->db; if( jsonFuncArgMightBeBinary(argv[0]) ){ p->sParse.nBlob = sqlite3_value_bytes(argv[0]); p->sParse.aBlob = (u8*)sqlite3_value_blob(argv[0]); }else{ p->sParse.zJson = (char*)sqlite3_value_text(argv[0]); p->sParse.nJson = sqlite3_value_bytes(argv[0]); if( p->sParse.zJson==0 ){ p->i = p->iEnd = 0; return SQLITE_OK; } |
︙ | ︙ |
Added test/json107.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 | # 2024-01-23 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # # Legacy JSON bug: If the input is a BLOB that when cast into TEXT looks # like valid JSON, then treat it as valid JSON. # # The original intent of the JSON functions was to raise an error on any # BLOB input. That intent was clearly documented, but the code failed to # to implement it. Subsequently, many applications began to depend on the # incorrect behavior, especially apps that used readfile() to read JSON # content, since readfile() returns a BLOB. So we need to support the # bug moving forward. # # The tests in this fail verify that the original buggy behavior is # preserved. # set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix json107 if {[db one {PRAGMA encoding}]!="UTF-8"} { # These tests only work for a UTF-8 encoding. finish_test return } do_execsql_test 1.1 { SELECT json_valid( CAST('{"a":1}' AS BLOB) ); } 1 do_execsql_test 1.1.1 { SELECT json_valid( CAST('{"a":1}' AS BLOB), 1); } 1 do_execsql_test 1.1.2 { SELECT json_valid( CAST('{"a":1}' AS BLOB), 2); } 1 do_execsql_test 1.1.4 { SELECT json_valid( CAST('{"a":1}' AS BLOB), 4); } 0 do_execsql_test 1.1.8 { SELECT json_valid( CAST('{"a":1}' AS BLOB), 8); } 0 do_execsql_test 1.2.1 { SELECT CAST('{"a":123}' AS blob) -> 'a'; } 123 do_execsql_test 1.2.2 { SELECT CAST('{"a":123}' AS blob) ->> 'a'; } 123 do_execsql_test 1.2.3 { SELECT json_extract(CAST('{"a":123}' AS blob), '$.a'); } 123 do_execsql_test 1.3 { SELECT json_insert(CAST('{"a":123}' AS blob),'$.b',456); } {{{"a":123,"b":456}}} do_execsql_test 1.4 { SELECT json_remove(CAST('{"a":123,"b":456}' AS blob),'$.a'); } {{{"b":456}}} do_execsql_test 1.5 { SELECT json_set(CAST('{"a":123,"b":456}' AS blob),'$.a',789); } {{{"a":789,"b":456}}} do_execsql_test 1.6 { SELECT json_replace(CAST('{"a":123,"b":456}' AS blob),'$.a',789); } {{{"a":789,"b":456}}} do_execsql_test 1.7 { SELECT json_type(CAST('{"a":123,"b":456}' AS blob)); } object do_execsql_test 1.8 { SELECT json(CAST('{"a":123,"b":456}' AS blob)); } {{{"a":123,"b":456}}} ifcapable vtab { do_execsql_test 2.1 { SELECT key, value FROM json_tree( CAST('{"a":123,"b":456}' AS blob) ) WHERE atom; } {a 123 b 456} } finish_test |