Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | When a JSON input is a blob, but it looks like valid JSON when cast to text, then accept it as valid JSON. This replicates a long-standing bug in the behavior of JSON routines, and thus avoids breaking legacy apps. [forum:/forumpost/012136abd5292b8d|Forum thread 012136abd5292b8d]. |
---|---|
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA3-256: |
e5dc81d5c7ee97866feb688dfa9b6fc2 |
User & Date: | drh 2024-01-23 13:51:55.791 |
Context
2024-01-23
| ||
14:45 | Improved error message when a double-quoted identifier name cannot be resolved - ask the user if they intended to use a string literal. (check-in: efc5c3c5e6 user: drh tags: trunk) | |
13:53 | When a JSON input is a blob, but it looks like valid JSON when cast to text, then accept it as valid JSON. This replicates a long-standing bug in the behavior of JSON routines, and thus avoids breaking legacy apps. (check-in: 4c2c1b97dc user: drh tags: branch-3.45) | |
13:51 | When a JSON input is a blob, but it looks like valid JSON when cast to text, then accept it as valid JSON. This replicates a long-standing bug in the behavior of JSON routines, and thus avoids breaking legacy apps. [forum:/forumpost/012136abd5292b8d|Forum thread 012136abd5292b8d]. (check-in: e5dc81d5c7 user: drh tags: trunk) | |
13:28 | Fix typo in the tag-20240123-a in a comment. (Closed-Leaf check-in: 1f09541eb4 user: drh tags: blob-as-json) | |
12:51 | Change the shell's --help flag to exit with code 0 instead of 1, per /forum and /chat discussions. (check-in: df563a1857 user: stephan tags: trunk) | |
Changes
Changes to src/json.c.
︙ | ︙ | |||
3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 | if( rc==JSON_LOOKUP_ERROR ){ sqlite3_result_error(ctx, "malformed JSON", -1); }else{ jsonBadPathError(ctx, zPath); } return; } /* ** Generate a JsonParse object, containing valid JSONB in aBlob and nBlob, ** from the SQL function argument pArg. Return a pointer to the new ** JsonParse object. ** ** Ownership of the new JsonParse object is passed to the caller. The | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 | if( rc==JSON_LOOKUP_ERROR ){ sqlite3_result_error(ctx, "malformed JSON", -1); }else{ jsonBadPathError(ctx, zPath); } return; } /* ** If pArg is a blob that seems like a JSONB blob, then initialize ** p to point to that JSONB and return TRUE. If pArg does not seem like ** a JSONB blob, then return FALSE; ** ** This routine is only called if it is already known that pArg is a ** blob. The only open question is whether or not the blob appears ** to be a JSONB blob. */ static int jsonArgIsJsonb(sqlite3_value *pArg, JsonParse *p){ u32 n, sz = 0; p->aBlob = (u8*)sqlite3_value_blob(pArg); p->nBlob = (u32)sqlite3_value_bytes(pArg); if( p->nBlob==0 ){ p->aBlob = 0; return 0; } if( NEVER(p->aBlob==0) ){ return 0; } if( (p->aBlob[0] & 0x0f)<=JSONB_OBJECT && (n = jsonbPayloadSize(p, 0, &sz))>0 && sz+n==p->nBlob && ((p->aBlob[0] & 0x0f)>JSONB_FALSE || sz==0) ){ return 1; } p->aBlob = 0; p->nBlob = 0; return 0; } /* ** Generate a JsonParse object, containing valid JSONB in aBlob and nBlob, ** from the SQL function argument pArg. Return a pointer to the new ** JsonParse object. ** ** Ownership of the new JsonParse object is passed to the caller. The |
︙ | ︙ | |||
3269 3270 3271 3272 3273 3274 3275 | memcpy(p->aBlob, pFromCache->aBlob, nBlob); p->nBlobAlloc = p->nBlob = nBlob; p->hasNonstd = pFromCache->hasNonstd; jsonParseFree(pFromCache); return p; } if( eType==SQLITE_BLOB ){ | < < < < < < < < < < < < < | < < < < < | | | | > > > > > > > > > > > > > | 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 | memcpy(p->aBlob, pFromCache->aBlob, nBlob); p->nBlobAlloc = p->nBlob = nBlob; p->hasNonstd = pFromCache->hasNonstd; jsonParseFree(pFromCache); return p; } if( eType==SQLITE_BLOB ){ if( jsonArgIsJsonb(pArg,p) ){ if( (flgs & JSON_EDITABLE)!=0 && jsonBlobMakeEditable(p, 0)==0 ){ goto json_pfa_oom; } return p; } /* If the blob is not valid JSONB, fall through into trying to cast ** the blob into text which is then interpreted as JSON. (tag-20240123-a) ** ** This goes against all historical documentation about how the SQLite ** JSON functions were suppose to work. From the beginning, blob was ** reserved for expansion and a blob value should have raised an error. ** But it did not, due to a bug. And many applications came to depend ** upon this buggy behavior, espeically when using the CLI and reading ** JSON text using readfile(), which returns a blob. For this reason ** we will continue to support the bug moving forward. ** See for example https://sqlite.org/forum/forumpost/012136abd5292b8d */ } p->zJson = (char*)sqlite3_value_text(pArg); p->nJson = sqlite3_value_bytes(pArg); if( p->nJson==0 ) goto json_pfa_malformed; if( NEVER(p->zJson==0) ) goto json_pfa_oom; if( jsonConvertTextToBlob(p, (flgs & JSON_KEEPERROR) ? 0 : ctx) ){ if( flgs & JSON_KEEPERROR ){ |
︙ | ︙ | |||
4267 4268 4269 4270 4271 4272 4273 | #ifdef SQLITE_LEGACY_JSON_VALID /* Incorrect legacy behavior was to return FALSE for a NULL input */ sqlite3_result_int(ctx, 0); #endif return; } case SQLITE_BLOB: { | | | > > > | | 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 | #ifdef SQLITE_LEGACY_JSON_VALID /* Incorrect legacy behavior was to return FALSE for a NULL input */ sqlite3_result_int(ctx, 0); #endif return; } case SQLITE_BLOB: { if( jsonFuncArgMightBeBinary(argv[0]) ){ if( flags & 0x04 ){ /* Superficial checking only - accomplished by the ** jsonFuncArgMightBeBinary() call above. */ res = 1; }else if( flags & 0x08 ){ /* Strict checking. Check by translating BLOB->TEXT->BLOB. If ** no errors occur, call that a "strict check". */ JsonParse px; u32 iErr; memset(&px, 0, sizeof(px)); px.aBlob = (u8*)sqlite3_value_blob(argv[0]); px.nBlob = sqlite3_value_bytes(argv[0]); iErr = jsonbValidityCheck(&px, 0, px.nBlob, 1); res = iErr==0; } break; } /* Fall through into interpreting the input as text. See note ** above at tag-20240123-a. */ /* no break */ deliberate_fall_through } default: { JsonParse px; if( (flags & 0x3)==0 ) break; memset(&px, 0, sizeof(px)); p = jsonParseFuncArg(ctx, argv[0], JSON_KEEPERROR); |
︙ | ︙ | |||
5020 5021 5022 5023 5024 5025 5026 | UNUSED_PARAMETER(idxStr); UNUSED_PARAMETER(argc); jsonEachCursorReset(p); if( idxNum==0 ) return SQLITE_OK; memset(&p->sParse, 0, sizeof(p->sParse)); p->sParse.nJPRef = 1; p->sParse.db = p->db; | < | | | < < < | 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059 5060 5061 5062 5063 5064 5065 5066 | UNUSED_PARAMETER(idxStr); UNUSED_PARAMETER(argc); jsonEachCursorReset(p); if( idxNum==0 ) return SQLITE_OK; memset(&p->sParse, 0, sizeof(p->sParse)); p->sParse.nJPRef = 1; p->sParse.db = p->db; if( jsonFuncArgMightBeBinary(argv[0]) ){ p->sParse.nBlob = sqlite3_value_bytes(argv[0]); p->sParse.aBlob = (u8*)sqlite3_value_blob(argv[0]); }else{ p->sParse.zJson = (char*)sqlite3_value_text(argv[0]); p->sParse.nJson = sqlite3_value_bytes(argv[0]); if( p->sParse.zJson==0 ){ p->i = p->iEnd = 0; return SQLITE_OK; } |
︙ | ︙ |
Added test/json107.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 | # 2024-01-23 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # # Legacy JSON bug: If the input is a BLOB that when cast into TEXT looks # like valid JSON, then treat it as valid JSON. # # The original intent of the JSON functions was to raise an error on any # BLOB input. That intent was clearly documented, but the code failed to # to implement it. Subsequently, many applications began to depend on the # incorrect behavior, especially apps that used readfile() to read JSON # content, since readfile() returns a BLOB. So we need to support the # bug moving forward. # # The tests in this fail verify that the original buggy behavior is # preserved. # set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix json107 if {[db one {PRAGMA encoding}]!="UTF-8"} { # These tests only work for a UTF-8 encoding. finish_test return } do_execsql_test 1.1 { SELECT json_valid( CAST('{"a":1}' AS BLOB) ); } 1 do_execsql_test 1.1.1 { SELECT json_valid( CAST('{"a":1}' AS BLOB), 1); } 1 do_execsql_test 1.1.2 { SELECT json_valid( CAST('{"a":1}' AS BLOB), 2); } 1 do_execsql_test 1.1.4 { SELECT json_valid( CAST('{"a":1}' AS BLOB), 4); } 0 do_execsql_test 1.1.8 { SELECT json_valid( CAST('{"a":1}' AS BLOB), 8); } 0 do_execsql_test 1.2.1 { SELECT CAST('{"a":123}' AS blob) -> 'a'; } 123 do_execsql_test 1.2.2 { SELECT CAST('{"a":123}' AS blob) ->> 'a'; } 123 do_execsql_test 1.2.3 { SELECT json_extract(CAST('{"a":123}' AS blob), '$.a'); } 123 do_execsql_test 1.3 { SELECT json_insert(CAST('{"a":123}' AS blob),'$.b',456); } {{{"a":123,"b":456}}} do_execsql_test 1.4 { SELECT json_remove(CAST('{"a":123,"b":456}' AS blob),'$.a'); } {{{"b":456}}} do_execsql_test 1.5 { SELECT json_set(CAST('{"a":123,"b":456}' AS blob),'$.a',789); } {{{"a":789,"b":456}}} do_execsql_test 1.6 { SELECT json_replace(CAST('{"a":123,"b":456}' AS blob),'$.a',789); } {{{"a":789,"b":456}}} do_execsql_test 1.7 { SELECT json_type(CAST('{"a":123,"b":456}' AS blob)); } object do_execsql_test 1.8 { SELECT json(CAST('{"a":123,"b":456}' AS blob)); } {{{"a":123,"b":456}}} ifcapable vtab { do_execsql_test 2.1 { SELECT key, value FROM json_tree( CAST('{"a":123,"b":456}' AS blob) ) WHERE atom; } {a 123 b 456} } finish_test |