SQLite

Check-in [b0a70a2356]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a problem causing one wal file to be deleted without being checkpointed on close in cases where the last connection to close the db does so with an out of date wal-index header in memory.
Timelines: family | ancestors | descendants | both | wal2
Files: files | file ages | folders
SHA3-256: b0a70a2356c44d65c54c6d9bdf05972071462e72c28d6c6e593147ffa3d27ef2
User & Date: dan 2021-12-15 12:51:02.972
Context
2022-01-17
19:36
Merge the latest trunk enhancements into the wal2 branch. (check-in: 82f031b41d user: drh tags: wal2)
2021-12-15
13:42
Merge checkpoint-on-close fix from wal2 branch. (check-in: a92eca6c9c user: dan tags: begin-concurrent-pnu-wal2)
12:51
Fix a problem causing one wal file to be deleted without being checkpointed on close in cases where the last connection to close the db does so with an out of date wal-index header in memory. (check-in: b0a70a2356 user: dan tags: wal2)
2021-11-27
15:00
Merge version 3.37.0 into the wal2 branch. (check-in: 3425af625b user: drh tags: wal2)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/wal.c.
790
791
792
793
794
795
796

797
798
799
800
801
802
803
  u32 nCkpt;                 /* Checkpoint sequence counter in the wal-header */
#ifdef SQLITE_DEBUG
  u8 lockError;              /* True if a locking error has occurred */
#endif
#ifdef SQLITE_ENABLE_SNAPSHOT
  WalIndexHdr *pSnapshot;    /* Start transaction here if not NULL */
#endif

  int bWal2;                 /* bWal2 flag passed to WalOpen() */
#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
  sqlite3 *db;
#endif
};

/*







>







790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
  u32 nCkpt;                 /* Checkpoint sequence counter in the wal-header */
#ifdef SQLITE_DEBUG
  u8 lockError;              /* True if a locking error has occurred */
#endif
#ifdef SQLITE_ENABLE_SNAPSHOT
  WalIndexHdr *pSnapshot;    /* Start transaction here if not NULL */
#endif
  int bClosing;              /* Set to true at start of sqlite3WalClose() */
  int bWal2;                 /* bWal2 flag passed to WalOpen() */
#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
  sqlite3 *db;
#endif
};

/*
2846
2847
2848
2849
2850
2851
2852

2853
2854
2855
2856
2857
2858
2859
  int sync_flags,                 /* Flags to pass to OsSync() (or 0) */
  int nBuf,
  u8 *zBuf                        /* Buffer of at least nBuf bytes */
){
  int rc = SQLITE_OK;
  if( pWal ){
    int isDelete = 0;             /* True to unlink wal and wal-index files */


    /* If an EXCLUSIVE lock can be obtained on the database file (using the
    ** ordinary, rollback-mode locking methods, this guarantees that the
    ** connection associated with this log file is the only connection to
    ** the database. In this case checkpoint the database and unlink both
    ** the wal and wal-index files.
    **







>







2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
  int sync_flags,                 /* Flags to pass to OsSync() (or 0) */
  int nBuf,
  u8 *zBuf                        /* Buffer of at least nBuf bytes */
){
  int rc = SQLITE_OK;
  if( pWal ){
    int isDelete = 0;             /* True to unlink wal and wal-index files */
    pWal->bClosing = 1;

    /* If an EXCLUSIVE lock can be obtained on the database file (using the
    ** ordinary, rollback-mode locking methods, this guarantees that the
    ** connection associated with this log file is the only connection to
    ** the database. In this case checkpoint the database and unlink both
    ** the wal and wal-index files.
    **
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779




4780
4781
4782
4783
4784
4785
4786
        }else{
          *pnCkpt = walCkptInfo(pWal)->nBackfill;
        }
      }
    }
  }

  if( isChanged ){
    /* If a new wal-index header was loaded before the checkpoint was 
    ** performed, then the pager-cache associated with pWal is now
    ** out of date. So zero the cached wal-index header to ensure that
    ** next time the pager opens a snapshot on this database it knows that
    ** the cache needs to be reset.
    */




    memset(&pWal->hdr, 0, sizeof(WalIndexHdr));
  }

  walDisableBlocking(pWal);
  sqlite3WalDb(pWal, 0);

  /* Release the locks. */







|





|
>
>
>
>







4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
        }else{
          *pnCkpt = walCkptInfo(pWal)->nBackfill;
        }
      }
    }
  }

  if( isChanged && pWal->bClosing==0 ){
    /* If a new wal-index header was loaded before the checkpoint was 
    ** performed, then the pager-cache associated with pWal is now
    ** out of date. So zero the cached wal-index header to ensure that
    ** next time the pager opens a snapshot on this database it knows that
    ** the cache needs to be reset.
    **
    ** Except, do not do this if the wal is being closed. In this case
    ** the caller needs the wal-index header to check if the database is
    ** in wal2 mode and the "other" wal file also needs to be checkpointed.
    ** Besides, the pager cache will not be used again in this case. */
    memset(&pWal->hdr, 0, sizeof(WalIndexHdr));
  }

  walDisableBlocking(pWal);
  sqlite3WalDb(pWal, 0);

  /* Release the locks. */
Added test/wal2openclose.test.


































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
# 2017 September 19
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this file is testing the operation of the library in
# "PRAGMA journal_mode=WAL2" mode.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/lock_common.tcl
source $testdir/malloc_common.tcl
source $testdir/wal_common.tcl

set testprefix wal2openclose
ifcapable !wal {finish_test ; return }

do_execsql_test 1.0 {
  CREATE TABLE t1(a, b, c);
  PRAGMA journal_mode = wal2;
  PRAGMA wal_autocheckpoint = 0;
  PRAGMA journal_size_limit = 75000;
} {wal2 0 75000}

do_test 1.1 {
  for {set ii 1} {$ii <= 200} {incr ii} {
    execsql {
      INSERT INTO t1 VALUES($ii, $ii, $ii);
    }
  }
  expr ([file size test.db-wal2] - 75000) > 30000
} {1}

do_test 1.2 {
  db close
  list [file exists test.db-wal] [file exists test.db-wal2]
} {0 0}

sqlite3 db test.db
do_execsql_test 1.3 {
  SELECT sum(c) FROM t1
} {20100}
db close

#-------------------------------------------------------------------------
reset_db
do_execsql_test 2.0 {
  CREATE TABLE t1(a, b, c);
  PRAGMA journal_mode = wal2;
  INSERT INTO t1 VALUES(1, 2, 3);
} {wal2}
db_save_and_close

db_restore_and_reopen
do_execsql_test 2.1 {
  SELECT * FROM t1;
} {1 2 3}

do_test 2.2 {
  sqlite3 db2 test.db
  db2 eval {INSERT INTO t1 VALUES(4, 5, 6)}
  db2 close
} {}

breakpoint
db close
sqlite3 db test.db
do_execsql_test 2.2 {
  SELECT * FROM t1;
} {1 2 3 4 5 6}



finish_test
Changes to test/wal2simple.test.
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
    INSERT INTO t1 VALUES(randomblob(500));
  }
  execsql {
    BEGIN;
      SELECT count(*) FROM t1;
  } db2
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 3176 11264}
do_test 7.3.2 {
  execsql { PRAGMA wal_checkpoint }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 3176 21504}
do_test 7.3.3 {
  execsql { 
    INSERT INTO t1 VALUES(randomblob(10000));
    INSERT INTO t1 VALUES(randomblob(500));
  }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 18896 21504}
do_test 7.3.4 {
  execsql END db2
  execsql { INSERT INTO t1 VALUES(randomblob(5000)); }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 18896 21504}

# Connection db2 is holding a PART2 lock. 
#
#   7.4.2: Test that the lock does not prevent writer switching to test.db-wal.
#
#   7.3.3: Test that the lock does prevent checkpointing of test.db-wal2.
#







|



|






|




|







357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
    INSERT INTO t1 VALUES(randomblob(500));
  }
  execsql {
    BEGIN;
      SELECT count(*) FROM t1;
  } db2
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 3176 12288}
do_test 7.3.2 {
  execsql { PRAGMA wal_checkpoint }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 3176 22528}
do_test 7.3.3 {
  execsql { 
    INSERT INTO t1 VALUES(randomblob(10000));
    INSERT INTO t1 VALUES(randomblob(500));
  }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 18896 22528}
do_test 7.3.4 {
  execsql END db2
  execsql { INSERT INTO t1 VALUES(randomblob(5000)); }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 18896 22528}

# Connection db2 is holding a PART2 lock. 
#
#   7.4.2: Test that the lock does not prevent writer switching to test.db-wal.
#
#   7.3.3: Test that the lock does prevent checkpointing of test.db-wal2.
#
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
    PRAGMA wal_checkpoint;
  }
  execsql {
    BEGIN;
      SELECT count(*) FROM t1;
  } db2
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 12608 44032}
do_test 7.4.2 {
  execsql { 
    INSERT INTO t1 VALUES(randomblob(5000));
  }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 12608 44032}
do_test 7.4.3 {
  execsql { PRAGMA wal_checkpoint }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 12608 44032}
do_test 7.4.4 {
  execsql END db2
  execsql { PRAGMA wal_checkpoint }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 12608 54272}

# Connection db2 is holding a PART1_FULL2 lock. 
#
#   7.5.2: Test that the lock does not prevent a checkpoint of test.db-wal2.
#
#   7.5.3: Test that the lock does prevent the writer from overwriting
#          test.db-wal2.







|





|



|




|







402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
    PRAGMA wal_checkpoint;
  }
  execsql {
    BEGIN;
      SELECT count(*) FROM t1;
  } db2
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 12608 50176}
do_test 7.4.2 {
  execsql { 
    INSERT INTO t1 VALUES(randomblob(5000));
  }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 12608 50176}
do_test 7.4.3 {
  execsql { PRAGMA wal_checkpoint }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 12608 50176}
do_test 7.4.4 {
  execsql END db2
  execsql { PRAGMA wal_checkpoint }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 12608 60416}

# Connection db2 is holding a PART1_FULL2 lock. 
#
#   7.5.2: Test that the lock does not prevent a checkpoint of test.db-wal2.
#
#   7.5.3: Test that the lock does prevent the writer from overwriting
#          test.db-wal2.
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
    INSERT INTO t1 VALUES(randomblob(5000));
  }
  execsql {
    BEGIN;
      SELECT count(*) FROM t1;
  } db2
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 12608 64512}
do_test 7.5.2 {
  execsql { PRAGMA wal_checkpoint }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 12608 75776}
do_test 7.5.3.1 {
  execsql { INSERT INTO t1 VALUES(randomblob(5000)) }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {14704 12608 75776}
do_test 7.5.3.2 {
  execsql { INSERT INTO t1 VALUES(randomblob(5000)) }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {22040 12608 75776}
do_test 7.5.4 {
  execsql END db2
  execsql { INSERT INTO t1 VALUES(randomblob(5000)) }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {22040 12608 75776}


finish_test








|



|



|



|




|




447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
    INSERT INTO t1 VALUES(randomblob(5000));
  }
  execsql {
    BEGIN;
      SELECT count(*) FROM t1;
  } db2
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 12608 76800}
do_test 7.5.2 {
  execsql { PRAGMA wal_checkpoint }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 12608 87040}
do_test 7.5.3.1 {
  execsql { INSERT INTO t1 VALUES(randomblob(5000)) }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {14704 12608 87040}
do_test 7.5.3.2 {
  execsql { INSERT INTO t1 VALUES(randomblob(5000)) }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {22040 12608 87040}
do_test 7.5.4 {
  execsql END db2
  execsql { INSERT INTO t1 VALUES(randomblob(5000)) }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {22040 12608 87040}


finish_test