mirror of
https://github.com/narrrl/proton-drive-linux.git
synced 2026-11-03 16:41:26 +01:00
A file or folder made on the mount used to get a minted `local~<ms>-<n>` placeholder. It now gets `local~<lid>`, a function of the row it is given first (`Db::add_local_row`), so the stand-in never changes and always leads back to the row, and through it to the real uid once the node lands. `Db::lid_of` reads it without a lookup. Migration V41 rewrites the placeholders still stored to their row's stand-in: the row and its node_json, its children, the ops on it or into it (uid, parent, sidecar uid, a rename's original parent), and its pins. Step 6a of the plan for 3.0.0 phase 3.
1305 lines
54 KiB
Rust
1305 lines
54 KiB
Rust
//! Forward-only schema migrations.
|
|
//!
|
|
//! Each `MIGRATION_V*` is the SQL that moves the schema from version N-1 to N.
|
|
//! They are applied in order and never edited once shipped: a released version
|
|
//! is already on disk somewhere, so a change here would silently diverge from
|
|
//! what an existing install actually has. Add a new one instead.
|
|
|
|
use rusqlite::OptionalExtension;
|
|
|
|
use super::Db;
|
|
use crate::Result;
|
|
|
|
/// Current schema version. Bump on every forward migration added below.
|
|
pub(super) const SCHEMA_VERSION: i64 = 41;
|
|
|
|
impl Db {
|
|
pub(super) fn migrate(&self) -> Result<()> {
|
|
let mut conn = self.conn.lock();
|
|
|
|
// `sync_state` is the key/value table holding `schema_version` and the
|
|
// event cursor (later). Create it first so we can read the version.
|
|
conn.execute_batch(
|
|
"CREATE TABLE IF NOT EXISTS sync_state (key TEXT PRIMARY KEY, value TEXT);",
|
|
)?;
|
|
|
|
let current: i64 = conn
|
|
.query_row(
|
|
"SELECT value FROM sync_state WHERE key = 'schema_version'",
|
|
[],
|
|
|row| row.get::<_, String>(0),
|
|
)
|
|
.ok()
|
|
.and_then(|v| v.parse().ok())
|
|
.unwrap_or(0);
|
|
|
|
if current > SCHEMA_VERSION {
|
|
return Err(crate::Error::Other(format!(
|
|
"database schema {current} is newer than this build supports ({SCHEMA_VERSION}); refusing to open it to avoid corrupting user data"
|
|
)));
|
|
}
|
|
if current == SCHEMA_VERSION {
|
|
return Ok(());
|
|
}
|
|
|
|
let tx = conn.transaction()?;
|
|
if current < 1 {
|
|
tx.execute_batch(MIGRATION_V1)?;
|
|
}
|
|
if current < 2 {
|
|
tx.execute_batch(MIGRATION_V2)?;
|
|
}
|
|
if current < 3 {
|
|
tx.execute_batch(MIGRATION_V3)?;
|
|
}
|
|
if current < 4 {
|
|
tx.execute_batch(MIGRATION_V4)?;
|
|
}
|
|
if current < 5 {
|
|
tx.execute_batch(MIGRATION_V5)?;
|
|
}
|
|
if current < 6 {
|
|
tx.execute_batch(MIGRATION_V6)?;
|
|
}
|
|
if current < 7 {
|
|
tx.execute_batch(MIGRATION_V7)?;
|
|
}
|
|
if current < 8 {
|
|
tx.execute_batch(MIGRATION_V8)?;
|
|
}
|
|
if current < 9 {
|
|
tx.execute_batch(MIGRATION_V9)?;
|
|
}
|
|
if current < 10 {
|
|
tx.execute_batch(MIGRATION_V10)?;
|
|
}
|
|
if current < 11 {
|
|
tx.execute_batch(MIGRATION_V11)?;
|
|
}
|
|
if current < 12 {
|
|
tx.execute_batch(MIGRATION_V12)?;
|
|
}
|
|
if current < 13 {
|
|
tx.execute_batch(MIGRATION_V13)?;
|
|
}
|
|
if current < 14 {
|
|
tx.execute_batch(MIGRATION_V14)?;
|
|
}
|
|
if current < 15 {
|
|
tx.execute_batch(MIGRATION_V15)?;
|
|
}
|
|
if current < 16 {
|
|
tx.execute_batch(MIGRATION_V16)?;
|
|
}
|
|
if current < 17 {
|
|
tx.execute_batch(MIGRATION_V17)?;
|
|
}
|
|
if current < 18 {
|
|
tx.execute_batch(MIGRATION_V18)?;
|
|
}
|
|
if current < 19 {
|
|
tx.execute_batch(MIGRATION_V19)?;
|
|
}
|
|
if current < 20 {
|
|
tx.execute_batch(MIGRATION_V20)?;
|
|
}
|
|
if current < 21 {
|
|
// A V17-and-older fixture may carry only the tables its own test
|
|
// cares about; there is nothing to reindex without `nodes`.
|
|
let has_nodes: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'nodes'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
if has_nodes {
|
|
tx.execute_batch(MIGRATION_V21)?;
|
|
}
|
|
}
|
|
if current < 22 {
|
|
// Older fixtures (and a database that never grew a table because
|
|
// its feature was never used) may be missing any of these, and an
|
|
// index on an absent table is a hard error rather than a no-op.
|
|
for (table, sql) in MIGRATION_V22 {
|
|
let present: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?1",
|
|
[table],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
if present {
|
|
tx.execute_batch(sql)?;
|
|
}
|
|
}
|
|
// `activity_time` indexes `time DESC` while both activity queries
|
|
// order by `id`: it has only ever cost writes.
|
|
tx.execute_batch("DROP INDEX IF EXISTS activity_time;")?;
|
|
}
|
|
if current < 23 {
|
|
// One last full rebuild of the local index, because from here on the
|
|
// scan maintains it incrementally (`local_upsert_batch`) and so can
|
|
// no longer repair a `local_fts` that drifted from `local_files`.
|
|
// Whatever it costs, it costs once, at startup, before the mount
|
|
// serves anything.
|
|
let present: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'local_fts'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
if present {
|
|
tx.execute_batch("INSERT INTO local_fts(local_fts) VALUES('rebuild');")?;
|
|
}
|
|
}
|
|
if current < 24 {
|
|
tx.execute_batch(MIGRATION_V24)?;
|
|
}
|
|
if current < 25 {
|
|
// Same guard as V21: an old fixture may not have `nodes` at all,
|
|
// and there is no path to materialise without it.
|
|
let has_nodes: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'nodes'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
if has_nodes {
|
|
tx.execute_batch(MIGRATION_V25)?;
|
|
}
|
|
}
|
|
if current < 26 {
|
|
// Same guard as V22: a fixture that never exercised the queue may
|
|
// not have `pending_op`, and altering an absent table is an error
|
|
// rather than a no-op.
|
|
// A fixture built from the current schema and then rewound to an
|
|
// older `schema_version` already has the column, and `ADD COLUMN`
|
|
// has no `IF NOT EXISTS`.
|
|
let has_column: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM pragma_table_info('pending_op') WHERE name = 'claimed_at'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
let has_ops: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'pending_op'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
if has_ops && !has_column {
|
|
tx.execute_batch(MIGRATION_V26)?;
|
|
}
|
|
}
|
|
if current < 27 {
|
|
// Same guards as V26, for the same two reasons: the table may not
|
|
// exist in an old fixture, and a fixture rewound from the current
|
|
// schema already has the column.
|
|
let has_column: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM pragma_table_info('pending_op') \
|
|
WHERE name = 'access_deferred_since'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
let has_ops: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'pending_op'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
if has_ops && !has_column {
|
|
tx.execute_batch(MIGRATION_V27)?;
|
|
}
|
|
}
|
|
if current < 28 {
|
|
// Same guards as V26/V27, for the same two reasons.
|
|
let has_column: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM pragma_table_info('sync_entry') \
|
|
WHERE name = 'local_mtime_ns'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
let has_entries: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'sync_entry'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
if has_entries && !has_column {
|
|
tx.execute_batch(MIGRATION_V28)?;
|
|
}
|
|
}
|
|
if current < 29 {
|
|
// Same guards as V26/V27/V28: an old fixture may not have `trash`,
|
|
// and a fixture rewound from the current schema already has the
|
|
// column.
|
|
let has_column: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM pragma_table_info('trash') WHERE name = 'parent_uid'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
let has_trash: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'trash'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
if has_trash && !has_column {
|
|
tx.execute_batch(MIGRATION_V29)?;
|
|
}
|
|
}
|
|
if current < 30 {
|
|
// Same guards as V26-V29: the column may already be there on a
|
|
// fixture rewound from the current schema, and a very old fixture
|
|
// may not have `photos` at all.
|
|
let has_column: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM pragma_table_info('photos') WHERE name = 'group_key'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
let has_photos: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'photos'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
if has_photos && !has_column {
|
|
tx.execute_batch(MIGRATION_V30)?;
|
|
}
|
|
}
|
|
if current < 31 {
|
|
// Same guards as V26-V30.
|
|
let has_column: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM pragma_table_info('photos') WHERE name = 'resolved_at'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
let has_photos: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'photos'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
if has_photos && !has_column {
|
|
tx.execute_batch(MIGRATION_V31)?;
|
|
}
|
|
}
|
|
if current < 32 {
|
|
// Data only, so safe to repeat; each table is guarded for a very
|
|
// old fixture that never had it.
|
|
for table in ["photos", "album_photos"] {
|
|
let has_table: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?1",
|
|
[table],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
if has_table {
|
|
tx.execute(&MIGRATION_V32.replace("{table}", table), [])?;
|
|
}
|
|
}
|
|
}
|
|
if current < 33 {
|
|
// Same guards as V26-V31.
|
|
let has_column: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM pragma_table_info('photos') WHERE name = 'place_id'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
let has_photos: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'photos'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
if has_photos && !has_column {
|
|
tx.execute_batch(MIGRATION_V33)?;
|
|
}
|
|
}
|
|
if current < 34 {
|
|
// Same guards as V26-V33.
|
|
let has_column: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM pragma_table_info('photos') WHERE name = 'similar_hash'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
let has_photos: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'photos'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
if has_photos && !has_column {
|
|
tx.execute_batch(MIGRATION_V34)?;
|
|
}
|
|
}
|
|
if current < 35 {
|
|
// Same guards as V26-V34.
|
|
let has_column: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM pragma_table_info('trash') WHERE name = 'trashed_at'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
let has_trash: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'trash'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
if has_trash && !has_column {
|
|
tx.execute_batch(MIGRATION_V35)?;
|
|
}
|
|
}
|
|
if current < 36 {
|
|
// Same guards as V26-V35.
|
|
let has_column: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM pragma_table_info('nodes') WHERE name = 'lid'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
let has_nodes: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'nodes'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
if has_nodes && !has_column {
|
|
tx.execute_batch(MIGRATION_V36)?;
|
|
}
|
|
}
|
|
if current < 37 {
|
|
// Same guards as V26-V36.
|
|
let sql: Option<String> = tx
|
|
.query_row(
|
|
"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'pending_op'",
|
|
[],
|
|
|row| row.get(0),
|
|
)
|
|
.optional()?;
|
|
if sql.is_some_and(|sql| !sql.contains("AUTOINCREMENT")) {
|
|
tx.execute_batch(MIGRATION_V37)?;
|
|
}
|
|
}
|
|
if current < 38 {
|
|
// Same guards as V26-V37.
|
|
let has_column: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM pragma_table_info('pending_op') WHERE name = 'issue'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
let has_ops: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'pending_op'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
if has_ops && !has_column {
|
|
tx.execute_batch(MIGRATION_V38)?;
|
|
}
|
|
}
|
|
if current < 39 {
|
|
// Same guards as V26-V38.
|
|
let has_column: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM pragma_table_info('pending_op') WHERE name = 'lid'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
let has_ops: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'pending_op'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
if has_ops && !has_column {
|
|
tx.execute_batch(MIGRATION_V39)?;
|
|
}
|
|
}
|
|
if current < 40 {
|
|
// Same guards as V26-V39.
|
|
let has_column: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM pragma_table_info('nodes') WHERE name = 'parent_lid'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
let has_nodes: bool = tx.query_row(
|
|
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'nodes'",
|
|
[],
|
|
|row| row.get::<_, i64>(0),
|
|
)? > 0;
|
|
if has_nodes && !has_column {
|
|
tx.execute_batch(MIGRATION_V40)?;
|
|
}
|
|
}
|
|
if current < 41 {
|
|
// Needs both tables; on a database that never had them there is
|
|
// nothing to rewrite.
|
|
let tables: i64 = tx.query_row(
|
|
"SELECT COUNT(*) FROM sqlite_master
|
|
WHERE type = 'table' AND name IN ('nodes', 'pending_op', 'pins')",
|
|
[],
|
|
|row| row.get(0),
|
|
)?;
|
|
if tables == 3 {
|
|
tx.execute_batch(MIGRATION_V41)?;
|
|
}
|
|
}
|
|
tx.execute(
|
|
"INSERT INTO sync_state (key, value) VALUES ('schema_version', ?1)
|
|
ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
|
[SCHEMA_VERSION.to_string()],
|
|
)?;
|
|
tx.commit()?;
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
/// Schema v1: nodes + FTS5 trigram index + cache LRU. `sync_state` is created
|
|
/// in [`Db::migrate`] before this runs.
|
|
const MIGRATION_V1: &str = "
|
|
CREATE TABLE nodes (
|
|
uid TEXT PRIMARY KEY,
|
|
parent_uid TEXT,
|
|
name TEXT NOT NULL,
|
|
is_dir INTEGER NOT NULL,
|
|
size INTEGER,
|
|
mtime INTEGER NOT NULL,
|
|
revision_hash TEXT,
|
|
trashed INTEGER NOT NULL DEFAULT 0
|
|
);
|
|
CREATE INDEX idx_nodes_parent ON nodes(parent_uid);
|
|
|
|
CREATE VIRTUAL TABLE nodes_fts USING fts5(
|
|
name, path, content='', tokenize='trigram'
|
|
);
|
|
|
|
CREATE TABLE cache_entries (
|
|
cache_key TEXT PRIMARY KEY,
|
|
size_bytes INTEGER,
|
|
last_accessed INTEGER,
|
|
is_pinned INTEGER NOT NULL DEFAULT 0
|
|
);
|
|
";
|
|
|
|
/// Schema v2: node write-through (P1). `node_json` stores the full [`Node`] so
|
|
/// the hot-cache maps rehydrate losslessly; `listed` records whether a folder's
|
|
/// child listing was complete when persisted.
|
|
const MIGRATION_V2: &str = "
|
|
ALTER TABLE nodes ADD COLUMN node_json TEXT;
|
|
ALTER TABLE nodes ADD COLUMN listed INTEGER NOT NULL DEFAULT 0;
|
|
";
|
|
|
|
/// Schema v3: FTS search (P3). The v1 `nodes_fts` was contentless (`content=''`)
|
|
/// and never populated — drop it and recreate as a self-contained trigram index
|
|
/// carrying the `uid` (UNINDEXED, so retrievable but not tokenized) alongside the
|
|
/// indexed `name`. Backfill from the nodes already persisted by P1.
|
|
const MIGRATION_V3: &str = "
|
|
DROP TABLE IF EXISTS nodes_fts;
|
|
CREATE VIRTUAL TABLE nodes_fts USING fts5(
|
|
uid UNINDEXED, name, tokenize='trigram'
|
|
);
|
|
INSERT INTO nodes_fts (uid, name)
|
|
SELECT uid, name FROM nodes WHERE trashed = 0;
|
|
";
|
|
|
|
/// Schema v4: content-cache LRU index (P4). The v1 `cache_entries` keyed only by
|
|
/// `cache_key`; add a `kind` discriminator so the whole-file blob pool and the
|
|
/// block pool — which carry separate byte budgets — can be summed and evicted
|
|
/// independently. Existing rows (none in practice, the table was never written)
|
|
/// default to `'blob'`. The daemon rebuilds the index from disk on open, so no
|
|
/// backfill is needed here.
|
|
const MIGRATION_V4: &str = "
|
|
ALTER TABLE cache_entries ADD COLUMN kind TEXT NOT NULL DEFAULT 'blob';
|
|
";
|
|
|
|
/// Schema v5: pins move out of `pins.json` into the DB (P5). One row per
|
|
/// directly-pinned node, keyed by uid display string so a pin survives
|
|
/// renames/moves. `recursive` marks a folder pin whose whole subtree is kept;
|
|
/// descendants are resolved on demand against `nodes` via a CTE rather than
|
|
/// being expanded into rows here. `ContentCache::open` imports any legacy
|
|
/// `pins.json` into this table once, then deletes the file.
|
|
const MIGRATION_V5: &str = "
|
|
CREATE TABLE pins (
|
|
uid TEXT PRIMARY KEY,
|
|
path TEXT NOT NULL,
|
|
recursive INTEGER NOT NULL DEFAULT 0
|
|
);
|
|
";
|
|
|
|
/// Schema v6: the index of *local* (non-Drive) files, so the launcher prompt can
|
|
/// search the machine alongside Drive. `local_files` is keyed by absolute path;
|
|
/// `scan_gen` stamps the scan that last saw a row, so a rescan prunes vanished
|
|
/// files with one `DELETE` instead of diffing.
|
|
///
|
|
/// `local_fts` is an *external-content* FTS5 index over `local_files.name`: the
|
|
/// text lives once in the base table and the index is rebuilt in bulk at the end
|
|
/// of a scan (`INSERT INTO local_fts(local_fts) VALUES('rebuild')`). That is far
|
|
/// cheaper than the delete-then-insert-per-row dance `nodes_fts` needs, because
|
|
/// a scan rewrites most rows at once rather than trickling single updates.
|
|
const MIGRATION_V6: &str = "
|
|
CREATE TABLE local_files (
|
|
id INTEGER PRIMARY KEY,
|
|
path TEXT NOT NULL UNIQUE,
|
|
name TEXT NOT NULL,
|
|
is_dir INTEGER NOT NULL,
|
|
size INTEGER NOT NULL DEFAULT 0,
|
|
mtime INTEGER NOT NULL DEFAULT 0,
|
|
scan_gen INTEGER NOT NULL DEFAULT 0
|
|
);
|
|
|
|
CREATE VIRTUAL TABLE local_fts USING fts5(
|
|
name, content='local_files', content_rowid='id', tokenize='trigram'
|
|
);
|
|
";
|
|
|
|
/// Schema v7: the photos timeline and the trash listing become persistent, so
|
|
/// opening the app paints them from disk instead of re-fetching the world (both
|
|
/// were memory-only: the timeline behind a 60 s TTL, the trash not cached at all).
|
|
///
|
|
/// `photos.seq` preserves the server's newest-first order, which is the only order
|
|
/// the timeline has — `capture_time` ties are common (a burst of shots) and would
|
|
/// otherwise shuffle between refreshes. `ratio` and `thumb_state` are *locally
|
|
/// learned*: they cost a download to rediscover, so `photos_replace` carries them
|
|
/// across a refresh while capture times and order come from the server.
|
|
const MIGRATION_V7: &str = "
|
|
CREATE TABLE photos (
|
|
uid TEXT PRIMARY KEY,
|
|
capture_time INTEGER NOT NULL,
|
|
name TEXT,
|
|
ratio REAL,
|
|
thumb_state INTEGER NOT NULL DEFAULT 0,
|
|
seq INTEGER NOT NULL
|
|
);
|
|
CREATE INDEX idx_photos_seq ON photos(seq);
|
|
|
|
CREATE TABLE trash (
|
|
uid TEXT PRIMARY KEY,
|
|
name TEXT NOT NULL,
|
|
is_dir INTEGER NOT NULL,
|
|
size INTEGER NOT NULL DEFAULT 0,
|
|
mtime INTEGER NOT NULL DEFAULT 0
|
|
);
|
|
";
|
|
|
|
/// Schema v8: device sync (devices.md). This machine registers as one Proton Drive
|
|
/// **Device** (a share + root folder on the main volume); `device` is a singleton
|
|
/// row cached so we reuse the same device across restarts instead of creating a
|
|
/// new one each run. `sync_folder` is one row per local folder the user added,
|
|
/// each mapped to a folder under the device root; `mode` is `mirror` (full local
|
|
/// copy, two-way synced) or `ondemand` (FUSE mount at `local_path`, no local
|
|
/// storage). `sync_entry` is the per-file baseline for three-way merge — added in
|
|
/// this migration so Phase 2 has the table, though Phase 1 leaves it empty.
|
|
const MIGRATION_V8: &str = "
|
|
CREATE TABLE device (
|
|
uid TEXT PRIMARY KEY,
|
|
share_id TEXT NOT NULL,
|
|
root_uid TEXT NOT NULL,
|
|
name TEXT NOT NULL,
|
|
created INTEGER NOT NULL DEFAULT 0
|
|
);
|
|
|
|
CREATE TABLE sync_folder (
|
|
id INTEGER PRIMARY KEY,
|
|
local_path TEXT NOT NULL UNIQUE,
|
|
remote_uid TEXT NOT NULL,
|
|
remote_share_id TEXT NOT NULL,
|
|
mode TEXT NOT NULL DEFAULT 'mirror',
|
|
state TEXT NOT NULL DEFAULT 'idle',
|
|
last_sync INTEGER NOT NULL DEFAULT 0
|
|
);
|
|
|
|
CREATE TABLE sync_entry (
|
|
folder_id INTEGER NOT NULL,
|
|
rel_path TEXT NOT NULL,
|
|
remote_uid TEXT,
|
|
local_mtime INTEGER NOT NULL DEFAULT 0,
|
|
local_size INTEGER NOT NULL DEFAULT 0,
|
|
remote_hash TEXT,
|
|
remote_rev TEXT,
|
|
PRIMARY KEY (folder_id, rel_path)
|
|
);
|
|
";
|
|
|
|
const MIGRATION_V9: &str = "
|
|
CREATE TABLE activity (
|
|
id INTEGER PRIMARY KEY,
|
|
time INTEGER NOT NULL,
|
|
kind TEXT NOT NULL,
|
|
target TEXT NOT NULL,
|
|
detail TEXT NOT NULL DEFAULT '',
|
|
ok INTEGER NOT NULL DEFAULT 1
|
|
);
|
|
|
|
CREATE INDEX activity_time ON activity(time DESC);
|
|
";
|
|
|
|
/// Schema v10: a mode switch the user asked for is queued rather than rejected
|
|
/// when the folder is mid-pass or has un-uploaded changes, so `pending_mode`
|
|
/// records the intent until the engine can act on it. NULL is the resting state:
|
|
/// the folder is already where the user wants it.
|
|
const MIGRATION_V10: &str = "
|
|
ALTER TABLE sync_folder ADD COLUMN pending_mode TEXT;
|
|
";
|
|
|
|
/// Schema v11: writes no longer upload inside the FUSE handler. `release` stages
|
|
/// the bytes and records the intended upload here, and a drain worker performs it
|
|
/// (offline.md Phase 3). The row outlives the process, so a write survives both a
|
|
/// dead network and a restart.
|
|
///
|
|
/// `blob_path` points into the content cache's `staging/` dir and `meta_json` is
|
|
/// the [`StagedWrite`] sidecar describing which of its bytes are real — an
|
|
/// incomplete blob must be gap-filled from the remote base before it can be
|
|
/// uploaded. `next_attempt_at` is a ms deadline implementing retry backoff.
|
|
///
|
|
/// [`StagedWrite`]: crate::cache::StagedWrite
|
|
const MIGRATION_V11: &str = "
|
|
CREATE TABLE pending_op (
|
|
id INTEGER PRIMARY KEY,
|
|
kind TEXT NOT NULL,
|
|
uid TEXT NOT NULL,
|
|
blob_path TEXT,
|
|
meta_json TEXT,
|
|
created_at INTEGER NOT NULL,
|
|
attempts INTEGER NOT NULL DEFAULT 0,
|
|
last_error TEXT,
|
|
next_attempt_at INTEGER NOT NULL DEFAULT 0
|
|
);
|
|
|
|
CREATE INDEX pending_op_uid ON pending_op(uid);
|
|
";
|
|
|
|
/// Schema v12: `pending_op` also carries mutations that *create* a node, which a
|
|
/// revision op never had to describe — it always addressed a node the server had
|
|
/// already minted a uid for (offline.md Phase 3b).
|
|
///
|
|
/// An offline `create`/`mkdir` cannot get a real uid, so the node is invented
|
|
/// locally under a `local~<uuid>` placeholder and the op records where it goes
|
|
/// (`parent_uid`) and what it is called (`name`). Both are nullable because a
|
|
/// revision op sets neither.
|
|
///
|
|
/// `parent_uid` is indexed: draining a folder rewrites every queued child that
|
|
/// points at its placeholder.
|
|
const MIGRATION_V12: &str = "
|
|
ALTER TABLE pending_op ADD COLUMN parent_uid TEXT;
|
|
ALTER TABLE pending_op ADD COLUMN name TEXT;
|
|
|
|
CREATE INDEX pending_op_parent ON pending_op(parent_uid);
|
|
";
|
|
|
|
/// Schema v13: photos record their media type and a derived `kind` so the
|
|
/// timeline can be split into Photos / Videos / Raw tabs. `media_type` is
|
|
/// nullable — it is filled once the daemon resolves a photo's node — while
|
|
/// `kind` (see [`crate::control::PhotoKind`]) defaults to `0` (still photo), the
|
|
/// classification anything unresolved falls back to. Existing rows are
|
|
/// reclassified for free on the next timeline refresh, so no backfill is needed.
|
|
const MIGRATION_V13: &str = "
|
|
ALTER TABLE photos ADD COLUMN media_type TEXT;
|
|
ALTER TABLE photos ADD COLUMN kind INTEGER NOT NULL DEFAULT 0;
|
|
";
|
|
|
|
/// Schema v14: index the content-cache LRU. `cache_entries` carried only its
|
|
/// `cache_key` primary key, so every budget check and every eviction pass was a
|
|
/// full scan plus a sort — on a path that runs once per cached 4 MiB block,
|
|
/// under the connection lock every FUSE metadata call also needs. `(kind,
|
|
/// last_accessed)` turns both into a range scan: the `SUM` reads one kind's
|
|
/// slice, and eviction reads its front and stops.
|
|
const MIGRATION_V14: &str = "
|
|
CREATE INDEX cache_entries_lru ON cache_entries(kind, last_accessed);
|
|
";
|
|
|
|
/// Schema v15: key the search index by rowid so a row can be replaced without
|
|
/// scanning the whole index (B12).
|
|
///
|
|
/// v3 built `nodes_fts` with the node's `uid` as an `UNINDEXED` column and
|
|
/// `upsert_nodes` refreshed a row with `DELETE FROM nodes_fts WHERE uid = ?`.
|
|
/// `UNINDEXED` means exactly what it says — the column is retrievable but not
|
|
/// searchable — so that predicate could only ever be a full scan of the index
|
|
/// (`EXPLAIN QUERY PLAN`: `SCAN nodes_fts VIRTUAL TABLE`). Every node written
|
|
/// scanned every node indexed: measured at 6.5 ms per node against a 17k-node
|
|
/// index, about half the cost of a cold listing, and *growing with the size of
|
|
/// the account* rather than the size of the listing.
|
|
///
|
|
/// FTS5 deletes by `rowid` efficiently, and `nodes` is an ordinary rowid table
|
|
/// (`uid` is its `TEXT PRIMARY KEY`, not `WITHOUT ROWID`), so its rowid is a
|
|
/// stable per-node key that `ON CONFLICT DO UPDATE` preserves. Rebuild the index
|
|
/// keyed by it and drop the `uid` column: searches now join on `rowid`, which is
|
|
/// also a cheaper join than the old one on a TEXT uid.
|
|
const MIGRATION_V15: &str = "
|
|
DROP TABLE IF EXISTS nodes_fts;
|
|
CREATE VIRTUAL TABLE nodes_fts USING fts5(
|
|
name, tokenize='trigram'
|
|
);
|
|
INSERT INTO nodes_fts (rowid, name)
|
|
SELECT rowid, name FROM nodes WHERE trashed = 0;
|
|
";
|
|
|
|
/// Schema v16: fuzzy search needs a bounded candidate set that includes parent
|
|
/// paths. Keeping the path in the trigram indexes lets SQLite narrow both name
|
|
/// and path candidates without walking either base table for every keystroke.
|
|
/// Node paths are initially backfilled below and subsequently maintained by the
|
|
/// node write path (including descendants after a folder move/rename).
|
|
const MIGRATION_V16: &str = "
|
|
DROP TABLE IF EXISTS nodes_fts;
|
|
CREATE VIRTUAL TABLE nodes_fts USING fts5(
|
|
name, path, tokenize='trigram'
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_nodes_name_nocase ON nodes(name COLLATE NOCASE);
|
|
INSERT INTO nodes_fts (rowid, name, path)
|
|
WITH RECURSIVE paths(rowid, uid, path) AS (
|
|
SELECT n.rowid, n.uid, '' FROM nodes n
|
|
WHERE n.parent_uid IS NULL
|
|
OR NOT EXISTS (SELECT 1 FROM nodes p WHERE p.uid = n.parent_uid)
|
|
UNION ALL
|
|
SELECT n.rowid, n.uid,
|
|
CASE WHEN paths.path = '' THEN n.name ELSE paths.path || '/' || n.name END
|
|
FROM nodes n JOIN paths ON n.parent_uid = paths.uid
|
|
)
|
|
SELECT n.rowid, n.name, paths.path
|
|
FROM nodes n JOIN paths ON paths.rowid = n.rowid
|
|
WHERE n.trashed = 0;
|
|
|
|
DROP TABLE IF EXISTS local_fts;
|
|
CREATE VIRTUAL TABLE local_fts USING fts5(
|
|
name, path, content='local_files', content_rowid='id', tokenize='trigram'
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_local_files_name_nocase
|
|
ON local_files(name COLLATE NOCASE);
|
|
INSERT INTO local_fts(local_fts) VALUES('rebuild');
|
|
";
|
|
|
|
/// Schema v17: effective access for roots shared with this account. The role is
|
|
/// persisted separately from `nodes.node_json` so a cold/offline mount can
|
|
/// enforce the last known permission even when the SDK cannot refresh metadata.
|
|
const MIGRATION_V17: &str = "
|
|
CREATE TABLE share_access (
|
|
root_uid TEXT PRIMARY KEY,
|
|
access TEXT NOT NULL
|
|
CHECK (access IN ('owner', 'editor', 'viewer', 'unknown'))
|
|
);
|
|
";
|
|
|
|
/// Schema v18: unified location presentation. Device rows carry only their
|
|
/// `sync_folder` identity; reads join all device state from that authoritative
|
|
/// table. The trigger projects future folders, while the foreign key removes a
|
|
/// presentation row with its sync folder.
|
|
const MIGRATION_V18: &str = "
|
|
CREATE TABLE mount (
|
|
id INTEGER PRIMARY KEY,
|
|
kind TEXT NOT NULL
|
|
CHECK (kind IN ('myfiles', 'device', 'shared')),
|
|
sync_folder_id INTEGER REFERENCES sync_folder(id) ON DELETE CASCADE,
|
|
share_root_uid TEXT,
|
|
local_path TEXT NOT NULL DEFAULT '',
|
|
root_uid TEXT NOT NULL DEFAULT '',
|
|
root_share_id TEXT NOT NULL DEFAULT '',
|
|
mode TEXT NOT NULL DEFAULT 'unknown',
|
|
access TEXT NOT NULL DEFAULT 'rw'
|
|
CHECK (access IN ('rw', 'ro')),
|
|
CHECK (
|
|
(kind = 'myfiles' AND sync_folder_id IS NULL AND share_root_uid IS NULL)
|
|
OR
|
|
(kind = 'device' AND sync_folder_id IS NOT NULL AND share_root_uid IS NULL)
|
|
OR
|
|
(kind = 'shared' AND sync_folder_id IS NULL AND share_root_uid IS NOT NULL)
|
|
)
|
|
);
|
|
|
|
CREATE UNIQUE INDEX mount_myfiles
|
|
ON mount(kind) WHERE kind = 'myfiles';
|
|
CREATE UNIQUE INDEX mount_device
|
|
ON mount(sync_folder_id) WHERE kind = 'device';
|
|
CREATE UNIQUE INDEX mount_shared
|
|
ON mount(share_root_uid) WHERE kind = 'shared';
|
|
|
|
INSERT INTO mount (kind, sync_folder_id)
|
|
SELECT 'device', id FROM sync_folder;
|
|
|
|
CREATE TRIGGER mount_sync_folder_insert
|
|
AFTER INSERT ON sync_folder
|
|
BEGIN
|
|
INSERT OR IGNORE INTO mount (kind, sync_folder_id)
|
|
VALUES ('device', NEW.id);
|
|
END;
|
|
";
|
|
|
|
/// Schema v19: photo albums and their contents. Both are projections of the
|
|
/// server's listings — like `photos`, they are replaced wholesale on refresh —
|
|
/// so the Albums view opens from disk instead of re-enumerating every launch.
|
|
///
|
|
/// `album_photos` carries its own `ratio` / `thumb_state` rather than joining
|
|
/// `photos`: an album shared with us lives on the sharer's volume, so its
|
|
/// photos are not in our timeline at all and would otherwise have nowhere to
|
|
/// remember what a thumbnail attempt learned.
|
|
const MIGRATION_V19: &str = "
|
|
CREATE TABLE albums (
|
|
uid TEXT PRIMARY KEY,
|
|
name TEXT NOT NULL,
|
|
photo_count INTEGER NOT NULL DEFAULT 0,
|
|
cover_uid TEXT,
|
|
last_activity INTEGER,
|
|
shared INTEGER NOT NULL DEFAULT 0,
|
|
seq INTEGER NOT NULL
|
|
);
|
|
|
|
CREATE TABLE album_photos (
|
|
album_uid TEXT NOT NULL,
|
|
uid TEXT NOT NULL,
|
|
capture_time INTEGER NOT NULL,
|
|
name TEXT,
|
|
media_type TEXT,
|
|
kind INTEGER NOT NULL DEFAULT 0,
|
|
ratio REAL,
|
|
thumb_state INTEGER NOT NULL DEFAULT 0,
|
|
seq INTEGER NOT NULL,
|
|
PRIMARY KEY (album_uid, uid)
|
|
);
|
|
|
|
CREATE INDEX idx_album_photos_seq ON album_photos(album_uid, seq);
|
|
CREATE INDEX idx_album_photos_uid ON album_photos(uid);
|
|
";
|
|
|
|
/// V20: whether a photo is a favorite.
|
|
///
|
|
/// A column rather than a tag table: `Favorite` is the only Proton photo tag a
|
|
/// user sets by hand (the rest — Video, Screenshot, Selfie, Raw… — are the
|
|
/// server's own classification, and the Photos page already derives its tabs
|
|
/// from the media type). Defaulting to 0 means an existing timeline stays
|
|
/// intact and each row learns its real value on the next refresh, the same way
|
|
/// `media_type` did.
|
|
const MIGRATION_V20: &str = "
|
|
ALTER TABLE photos ADD COLUMN favorite INTEGER NOT NULL DEFAULT 0;
|
|
CREATE INDEX IF NOT EXISTS idx_photos_favorite ON photos(favorite, seq);
|
|
";
|
|
|
|
/// V21: rebuild the node search index after fixing what counts as indexable.
|
|
///
|
|
/// The v16 backfill rooted its path walk at any node whose parent is null *or*
|
|
/// not cached, but the write path's `node_is_indexable_tx` demanded a
|
|
/// null-parent ancestor. Only the My Files root has one — a device folder's
|
|
/// root is stored in `device`/`sync_folder`, never in `nodes` — so every node
|
|
/// under a device folder was dropped from `nodes_fts` the first time it was
|
|
/// upserted after the backfill, and was unfindable from the prompt (2,705 of
|
|
/// 9,252 nodes on the account this was diagnosed against).
|
|
///
|
|
/// The rule now matches the backfill, so this replays it: same recursive walk
|
|
/// as v16, over the current `nodes`.
|
|
/// Two details the v16 backfill did not carry, both cheap here: a node under a
|
|
/// trashed folder stays out (the write path has always excluded it, and
|
|
/// `tombstone_subtree_tx` only marks a subtree it was told about), and the walk
|
|
/// is depth-capped so a parent cycle in a corrupted database cannot spin the
|
|
/// migration forever.
|
|
const MIGRATION_V21: &str = "
|
|
DROP TABLE IF EXISTS nodes_fts;
|
|
CREATE VIRTUAL TABLE nodes_fts USING fts5(
|
|
name, path, tokenize='trigram'
|
|
);
|
|
INSERT INTO nodes_fts (rowid, name, path)
|
|
WITH RECURSIVE paths(rowid, uid, path, trashed, depth) AS (
|
|
-- Only a true root (no parent at all) contributes nothing to the path: its
|
|
-- name is the mount. A row whose parent is merely uncached is an ordinary
|
|
-- folder and keeps its name, which is what `path_of` does at runtime.
|
|
SELECT n.rowid, n.uid,
|
|
CASE WHEN n.parent_uid IS NULL THEN '' ELSE n.name END, n.trashed, 0
|
|
FROM nodes n
|
|
WHERE n.parent_uid IS NULL
|
|
OR NOT EXISTS (SELECT 1 FROM nodes p WHERE p.uid = n.parent_uid)
|
|
UNION ALL
|
|
SELECT n.rowid, n.uid,
|
|
CASE WHEN paths.path = '' THEN n.name ELSE paths.path || '/' || n.name END,
|
|
MAX(paths.trashed, n.trashed), paths.depth + 1
|
|
FROM nodes n JOIN paths ON n.parent_uid = paths.uid
|
|
WHERE paths.depth < 256
|
|
)
|
|
SELECT n.rowid, n.name, paths.path
|
|
FROM nodes n JOIN paths ON paths.rowid = n.rowid
|
|
WHERE paths.trashed = 0;
|
|
";
|
|
|
|
/// V22: indexes for the lookups that were scanning, each paired with the table
|
|
/// it needs to exist for.
|
|
///
|
|
/// Each backs a query on a hot path that had no index to use:
|
|
///
|
|
/// - `sync_entry(remote_uid)` — the reconcile resolves a remote uid to its
|
|
/// local entry once per node, per pass.
|
|
/// - `local_files(scan_gen)` — the sweep that retires the previous generation
|
|
/// after a local index scan.
|
|
/// - `nodes(parent_uid) WHERE trashed = 0` — every listing, with the partial
|
|
/// index keeping it to the live rows a listing actually wants.
|
|
///
|
|
/// Deliberately *not* here: an index on `pending_op(next_attempt_at, id)`.
|
|
/// `next_due_op` orders by `id`, which is the rowid, so it already stops at the
|
|
/// first due row; offering the planner a `next_attempt_at` index makes it scan
|
|
/// that index and then sort the matches by `id` instead —
|
|
/// `next_due_op_does_not_scale_with_queue_length` measures the regression.
|
|
const MIGRATION_V22: [(&str, &str); 3] = [
|
|
(
|
|
"sync_entry",
|
|
"CREATE INDEX IF NOT EXISTS idx_sync_entry_remote_uid ON sync_entry(remote_uid);",
|
|
),
|
|
(
|
|
"local_files",
|
|
"CREATE INDEX IF NOT EXISTS idx_local_files_scan_gen ON local_files(scan_gen);",
|
|
),
|
|
(
|
|
"nodes",
|
|
"CREATE INDEX IF NOT EXISTS idx_nodes_parent_live ON nodes(parent_uid) WHERE trashed = 0;",
|
|
),
|
|
];
|
|
|
|
/// V24: the revisions this daemon sealed itself, so restarting does not reopen
|
|
/// the conflict hole B70 layer B closed.
|
|
///
|
|
/// `Core::own_sealed_revs` held these in memory: a queued write whose baseline
|
|
/// names an earlier revision is a conflict *unless* the remote sits at a
|
|
/// revision we sealed, in which case it is one writer stalling and resuming and
|
|
/// must supersede rather than fork a `(sync-conflict)` copy. Losing the map at
|
|
/// shutdown meant the first drain after a restart could not tell the two apart.
|
|
///
|
|
/// `sealed_at` is what bounds the table: entries older than the window in which
|
|
/// a stall→resume is plausible are pruned on write.
|
|
const MIGRATION_V24: &str = "
|
|
CREATE TABLE IF NOT EXISTS own_sealed_rev (
|
|
uid TEXT PRIMARY KEY,
|
|
revision_id TEXT NOT NULL,
|
|
sealed_at INTEGER NOT NULL
|
|
);
|
|
";
|
|
|
|
/// V25: `nodes.path` — a node's mountpoint-relative path, stored rather than
|
|
/// walked.
|
|
///
|
|
/// `path_of` was a recursive CTE per node, and search ran it once per candidate
|
|
/// (up to a thousand per keystroke) on top of the one the write path already ran
|
|
/// per descendant of every folder upsert. The value only changes when a node's
|
|
/// name or parent changes, which is exactly when the write path is already
|
|
/// walking the subtree — so it belongs in a column.
|
|
///
|
|
/// The backfill is the same walk V21 used for `nodes_fts`, minus the trashed
|
|
/// filter: a trashed node stays out of the index but still has a path. Roots get
|
|
/// `''` (their name is the mount), and a node whose parent is merely uncached
|
|
/// keeps its own name — both matching what `path_of` returned at runtime.
|
|
///
|
|
/// Materialised into a temp table rather than written by a correlated subquery,
|
|
/// so the recursive walk runs once instead of once per row.
|
|
const MIGRATION_V25: &str = "
|
|
ALTER TABLE nodes ADD COLUMN path TEXT;
|
|
|
|
CREATE TEMP TABLE node_paths (uid TEXT PRIMARY KEY, path TEXT NOT NULL);
|
|
|
|
INSERT INTO node_paths (uid, path)
|
|
WITH RECURSIVE paths(uid, path, depth) AS (
|
|
SELECT n.uid, CASE WHEN n.parent_uid IS NULL THEN '' ELSE n.name END, 0
|
|
FROM nodes n
|
|
WHERE n.parent_uid IS NULL
|
|
OR NOT EXISTS (SELECT 1 FROM nodes p WHERE p.uid = n.parent_uid)
|
|
UNION ALL
|
|
SELECT n.uid,
|
|
CASE WHEN paths.path = '' THEN n.name ELSE paths.path || '/' || n.name END,
|
|
paths.depth + 1
|
|
FROM nodes n JOIN paths ON n.parent_uid = paths.uid
|
|
WHERE paths.depth < 256
|
|
)
|
|
SELECT uid, path FROM paths;
|
|
|
|
UPDATE nodes
|
|
SET path = (SELECT p.path FROM node_paths p WHERE p.uid = nodes.uid)
|
|
WHERE EXISTS (SELECT 1 FROM node_paths p WHERE p.uid = nodes.uid);
|
|
|
|
DROP TABLE node_paths;
|
|
";
|
|
|
|
/// V26: `pending_op.claimed_at` — the drain's claim column.
|
|
///
|
|
/// The drain was one thread taking one op at a time, so a 10 GiB upload held
|
|
/// every queued rename and small write behind it for as long as it ran. Several
|
|
/// workers can share the queue only if a claim is visible to the others: this
|
|
/// column is what makes "already being drained by someone" a fact about the row
|
|
/// rather than about one thread's local variable.
|
|
///
|
|
/// `0` is unclaimed; anything else is the ms at which a worker took it. A
|
|
/// claim is process-local state that outlives nothing — the single-writer lock
|
|
/// means a non-zero value found at open belongs to a crashed run, and
|
|
/// [`Db::clear_op_claims`](super::Db::clear_op_claims) drops the lot.
|
|
///
|
|
/// The index is partial so it costs nothing on the ordinary row: only the
|
|
/// handful of claimed rows are in it, which is exactly the set the claim query
|
|
/// has to subtract to keep two workers off one node.
|
|
const MIGRATION_V26: &str = "
|
|
ALTER TABLE pending_op ADD COLUMN claimed_at INTEGER NOT NULL DEFAULT 0;
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_pending_op_claimed
|
|
ON pending_op(uid) WHERE claimed_at <> 0;
|
|
";
|
|
|
|
/// Schema v27: when an op was first deferred by the local access check.
|
|
///
|
|
/// An access deferral consumes no attempt and records no error — the op has not
|
|
/// failed remotely, it has not been sent at all. That is right for a permission
|
|
/// change the user is about to undo, and wrong for a condition that never
|
|
/// clears: the row is re-deferred every
|
|
/// [`DRAIN_ACCESS_RECHECK`](../../../pdfs_fuse/drain/constant.DRAIN_ACCESS_RECHECK.html)
|
|
/// forever while `attempts`, `last_error`, `failing` and the logs all stay
|
|
/// empty, so a queue that can never drain is indistinguishable from a busy one.
|
|
/// Found in the field as 28 revision ops holding 18 GiB of accepted writes for
|
|
/// 30 days, none of them visible anywhere.
|
|
///
|
|
/// `0` means "not currently access-deferred"; anything else is the ms at which
|
|
/// the first consecutive deferral happened, which is what
|
|
/// [`Db::defer_op_for_access`](super::Db::defer_op_for_access) measures the
|
|
/// deferral against.
|
|
const MIGRATION_V27: &str = "
|
|
ALTER TABLE pending_op ADD COLUMN access_deferred_since INTEGER NOT NULL DEFAULT 0;
|
|
";
|
|
|
|
/// Schema v28: sub-second precision for a mirror folder's local baseline.
|
|
///
|
|
/// A reconcile pass decides whether the local side of a path changed by
|
|
/// comparing `(local_mtime, local_size)` against the baseline. Both sides of
|
|
/// that comparison were whole seconds, so an edit that landed in the same second
|
|
/// as the last sync and left the file the same length — a fixed-size record
|
|
/// rewritten, an image re-exported, a database page updated in place — read as
|
|
/// *unchanged* and was never uploaded. The remote copy silently stayed at the
|
|
/// older content until something else about the file moved (bugs.md B25).
|
|
///
|
|
/// Nullable rather than a converted `local_mtime * 1_000_000_000`: the
|
|
/// nanosecond part of an existing baseline is genuinely unknown, and inventing
|
|
/// zero for it would make every already-synced file compare as changed and
|
|
/// re-upload an entire mirror on first start. `NULL` means "this row predates
|
|
/// the column", which the comparison reads as "fall back to whole seconds" —
|
|
/// exactly the behaviour that row was written under. Each row gains a real value
|
|
/// the next time its path is synced.
|
|
const MIGRATION_V28: &str = "
|
|
ALTER TABLE sync_entry ADD COLUMN local_mtime_ns INTEGER;
|
|
";
|
|
|
|
/// Schema v29: the trash listing remembers each trashed node's parent, so a
|
|
/// restore can reason about the shape of what it is putting back.
|
|
///
|
|
/// Trash is a flat list of uids on the wire, but the user deletes a *tree*: a
|
|
/// folder and, often, items inside it that were trashed separately. Restoring
|
|
/// one of those in isolation is what the parent link fixes — a child restored
|
|
/// under a still-trashed parent lands somewhere invisible, and a folder restored
|
|
/// without its separately-trashed contents comes back empty.
|
|
///
|
|
/// `NULL` means "this row predates the column" (or the node is a volume root);
|
|
/// the next trash refresh fills it in, and until then the restore behaves as it
|
|
/// did before.
|
|
const MIGRATION_V29: &str = "
|
|
ALTER TABLE trash ADD COLUMN parent_uid TEXT;
|
|
CREATE INDEX IF NOT EXISTS idx_trash_parent ON trash(parent_uid);
|
|
";
|
|
|
|
/// Schema v30: a RAW and the JPEG of the same shot are one photo to the person
|
|
/// who took it, so the timeline records what ties them together.
|
|
///
|
|
/// `content_hash` and `main_uid` come from the server's own photo properties —
|
|
/// the duplicate-detection hash, and the main photo a related one belongs to.
|
|
/// `group_key` is what the gallery pages by: every member of one shot carries
|
|
/// the same key, and the member the grid shows is the one whose `main_uid` is
|
|
/// its own uid.
|
|
///
|
|
/// All three are `NULL` until the next timeline refresh resolves the photo's
|
|
/// node, and a row without a `group_key` is its own group — which is exactly how
|
|
/// the gallery behaved before this column existed.
|
|
const MIGRATION_V30: &str = "
|
|
ALTER TABLE photos ADD COLUMN content_hash TEXT;
|
|
ALTER TABLE photos ADD COLUMN main_uid TEXT;
|
|
ALTER TABLE photos ADD COLUMN group_key TEXT;
|
|
CREATE INDEX IF NOT EXISTS idx_photos_group ON photos(group_key);
|
|
";
|
|
|
|
/// Schema v31: which photos a refresh still has to read from the server.
|
|
///
|
|
/// A refresh used to resolve every photo's node to learn its name, media type,
|
|
/// favorite tag, content hash and photo relation — minutes of round-trips for a
|
|
/// library of any size, repeated every time the timeline went stale. `resolved_at`
|
|
/// records that the work was already done, so a refresh only reads the photos it
|
|
/// has never read and the ones a remote event marked stale (`resolved_at = NULL`).
|
|
///
|
|
/// The backfill treats an existing timeline as resolved, so an upgrade does not
|
|
/// re-read the whole library once. `WHERE name IS NOT NULL` keeps the photos
|
|
/// whose earlier resolve had failed eligible: a nameless row is one nothing was
|
|
/// ever learned about.
|
|
const MIGRATION_V31: &str = "
|
|
ALTER TABLE photos ADD COLUMN resolved_at INTEGER;
|
|
CREATE INDEX IF NOT EXISTS idx_photos_unresolved ON photos(resolved_at);
|
|
UPDATE photos SET resolved_at = strftime('%s','now') WHERE name IS NOT NULL;
|
|
";
|
|
|
|
/// V32: give videos written off as un-thumbnailable another chance.
|
|
///
|
|
/// Before local video thumbnails, the fallback fed a video's bytes to the image
|
|
/// decoder, which could only fail, and that failure was recorded as the
|
|
/// permanent "no thumbnail" verdict. Run once per table in `{table}`.
|
|
const MIGRATION_V32: &str = "UPDATE {table} SET thumb_state = 0 WHERE kind = 1 AND thumb_state = 2";
|
|
|
|
/// Schema v33: where each photo was taken.
|
|
///
|
|
/// `latitude` and `longitude` come from the location other clients write into
|
|
/// a photo's extended attributes; `place_id` is the GeoNames id of the nearest
|
|
/// town (see `crate::places`), so the Places view is a plain `GROUP BY`.
|
|
///
|
|
/// Every photo is marked unresolved, so the next refresh reads the location of
|
|
/// the whole library once. That is the backfill: the metadata is on the node
|
|
/// the refresh reads anyway, and nothing has to be downloaded.
|
|
const MIGRATION_V33: &str = "
|
|
ALTER TABLE photos ADD COLUMN latitude REAL;
|
|
ALTER TABLE photos ADD COLUMN longitude REAL;
|
|
ALTER TABLE photos ADD COLUMN place_id INTEGER;
|
|
CREATE INDEX IF NOT EXISTS idx_photos_place ON photos(place_id);
|
|
UPDATE photos SET resolved_at = NULL;
|
|
";
|
|
|
|
/// Schema v34: what each photo looks like, for finding similar ones.
|
|
///
|
|
/// `similar_hash` is a 64-bit difference hash of the photo's thumbnail (see
|
|
/// `crate::similar`): resized and re-encoded copies of one picture land a few
|
|
/// bits apart, where the server's `content_hash` only matches byte-identical
|
|
/// files. `NULL` until the daemon has hashed the thumbnail, which it does as
|
|
/// thumbnails arrive and when the similar-photo finder asks for the rest.
|
|
const MIGRATION_V34: &str = "
|
|
ALTER TABLE photos ADD COLUMN similar_hash INTEGER;
|
|
";
|
|
|
|
/// Schema v35: when each trashed node went to the trash.
|
|
///
|
|
/// The API does not tell this client, so `trashed_at` is when the daemon first
|
|
/// saw the node in the trash, in epoch seconds. `NULL` when it cannot say: rows
|
|
/// from before this version, and nodes found by a refresh that did not follow a
|
|
/// complete one, which may have been in the trash for a long time already.
|
|
const MIGRATION_V35: &str = "
|
|
ALTER TABLE trash ADD COLUMN trashed_at INTEGER;
|
|
";
|
|
|
|
/// Schema v36: every node row has a local id (`lid`), phase 2 of
|
|
/// docs/MILESTONE-3.0.0.md.
|
|
///
|
|
/// The `uid` was the row's key, so a node had no identity until Drive minted
|
|
/// one, and a create that landed moved its row to a new key. The `lid` is given
|
|
/// when the row is first written and never changes or comes back, which is what
|
|
/// `AUTOINCREMENT` adds over a plain rowid. It takes the old rowid, so the
|
|
/// search index, keyed by rowid since v15, stays valid. The `uid` becomes
|
|
/// nullable for a node that is not on Drive yet.
|
|
const MIGRATION_V36: &str = "
|
|
CREATE TABLE nodes_v36 (
|
|
lid INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
uid TEXT UNIQUE,
|
|
parent_uid TEXT,
|
|
name TEXT NOT NULL,
|
|
is_dir INTEGER NOT NULL,
|
|
size INTEGER,
|
|
mtime INTEGER NOT NULL,
|
|
revision_hash TEXT,
|
|
trashed INTEGER NOT NULL DEFAULT 0,
|
|
node_json TEXT,
|
|
listed INTEGER NOT NULL DEFAULT 0,
|
|
path TEXT
|
|
);
|
|
INSERT INTO nodes_v36
|
|
(lid, uid, parent_uid, name, is_dir, size, mtime, revision_hash, trashed, node_json, listed,
|
|
path)
|
|
SELECT rowid, uid, parent_uid, name, is_dir, size, mtime, revision_hash, trashed, node_json,
|
|
listed, path
|
|
FROM nodes;
|
|
DROP TABLE nodes;
|
|
ALTER TABLE nodes_v36 RENAME TO nodes;
|
|
CREATE INDEX idx_nodes_parent ON nodes(parent_uid);
|
|
CREATE INDEX idx_nodes_name_nocase ON nodes(name COLLATE NOCASE);
|
|
CREATE INDEX idx_nodes_parent_live ON nodes(parent_uid) WHERE trashed = 0;
|
|
";
|
|
|
|
/// Schema v37: a `pending_op` id is never given out twice.
|
|
///
|
|
/// A plain rowid is the largest one plus one, so an op queued right after the
|
|
/// newest one was dropped took its id. The drain retires the op it is
|
|
/// working on by id, and so deleted the new op: a trash queued over a rename
|
|
/// on the wire was lost when the rename landed. `AUTOINCREMENT` never reuses
|
|
/// an id. Ids are kept, as is every index.
|
|
const MIGRATION_V37: &str = "
|
|
CREATE TABLE pending_op_v37 (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
kind TEXT NOT NULL,
|
|
uid TEXT NOT NULL,
|
|
blob_path TEXT,
|
|
meta_json TEXT,
|
|
created_at INTEGER NOT NULL,
|
|
attempts INTEGER NOT NULL DEFAULT 0,
|
|
last_error TEXT,
|
|
next_attempt_at INTEGER NOT NULL DEFAULT 0,
|
|
parent_uid TEXT,
|
|
name TEXT,
|
|
claimed_at INTEGER NOT NULL DEFAULT 0,
|
|
access_deferred_since INTEGER NOT NULL DEFAULT 0
|
|
);
|
|
INSERT INTO pending_op_v37
|
|
(id, kind, uid, blob_path, meta_json, created_at, attempts, last_error, next_attempt_at,
|
|
parent_uid, name, claimed_at, access_deferred_since)
|
|
SELECT id, kind, uid, blob_path, meta_json, created_at, attempts, last_error,
|
|
next_attempt_at, parent_uid, name, claimed_at, access_deferred_since
|
|
FROM pending_op;
|
|
DROP TABLE pending_op;
|
|
ALTER TABLE pending_op_v37 RENAME TO pending_op;
|
|
CREATE INDEX pending_op_uid ON pending_op(uid);
|
|
CREATE INDEX pending_op_parent ON pending_op(parent_uid);
|
|
CREATE INDEX idx_pending_op_claimed
|
|
ON pending_op(uid) WHERE claimed_at <> 0;
|
|
";
|
|
|
|
/// Schema v38: why Drive refused a queued op, when it did (a sync issue).
|
|
/// `NULL` for an op that has not been refused, which is every op migrated.
|
|
const MIGRATION_V38: &str = "
|
|
ALTER TABLE pending_op ADD COLUMN issue TEXT;
|
|
";
|
|
|
|
/// Schema v39: a queued op names its node and its parent by local id too.
|
|
///
|
|
/// A create made inside a folder that had not landed named the folder by its
|
|
/// placeholder uid, and every landing had to find and rewrite those strings.
|
|
/// The local id stays the same when the folder lands. An op whose node has no
|
|
/// row keeps `NULL` and is matched by uid, as before.
|
|
const MIGRATION_V39: &str = "
|
|
ALTER TABLE pending_op ADD COLUMN lid INTEGER;
|
|
ALTER TABLE pending_op ADD COLUMN parent_lid INTEGER;
|
|
UPDATE pending_op
|
|
SET lid = (SELECT lid FROM nodes WHERE nodes.uid = pending_op.uid),
|
|
parent_lid = (SELECT lid FROM nodes WHERE nodes.uid = pending_op.parent_uid);
|
|
CREATE INDEX pending_op_lid ON pending_op(lid);
|
|
CREATE INDEX pending_op_parent_lid ON pending_op(parent_lid);
|
|
";
|
|
|
|
/// Schema v40: a node row names its parent by local id too.
|
|
///
|
|
/// The local id stays when a folder made here lands and its row takes the uid
|
|
/// Drive gave it, so a child linked by it needs no rewrite. `NULL` while the
|
|
/// parent has no row, as `parent_uid` then names nothing either.
|
|
const MIGRATION_V40: &str = "
|
|
ALTER TABLE nodes ADD COLUMN parent_lid INTEGER;
|
|
UPDATE nodes SET parent_lid = (SELECT p.lid FROM nodes p WHERE p.uid = nodes.parent_uid);
|
|
CREATE INDEX idx_nodes_parent_lid ON nodes(parent_lid);
|
|
";
|
|
|
|
/// Schema v41: a node made on this machine goes by `local~<lid>` until it
|
|
/// lands, a function of its row, where it used to get a `local~<ms>-<n>`
|
|
/// placeholder. Every placeholder still stored is rewritten to its row's
|
|
/// stand-in: the row itself and its `node_json`, its children, the ops on it
|
|
/// or into it, and its pins. A placeholder op with no row keeps its uid and
|
|
/// drains as before.
|
|
const MIGRATION_V41: &str = "
|
|
CREATE TEMP TABLE local_v41 AS
|
|
SELECT uid AS old, 'local~' || lid AS new, lid FROM nodes
|
|
WHERE uid LIKE 'local~%' AND uid <> 'local~' || lid;
|
|
UPDATE pending_op SET uid = (SELECT new FROM local_v41 WHERE old = pending_op.uid)
|
|
WHERE uid IN (SELECT old FROM local_v41);
|
|
UPDATE pending_op SET parent_uid = (SELECT new FROM local_v41 WHERE old = pending_op.parent_uid)
|
|
WHERE parent_uid IN (SELECT old FROM local_v41);
|
|
UPDATE pending_op
|
|
SET meta_json = json_set(meta_json, '$.uid',
|
|
(SELECT new FROM local_v41 WHERE old = json_extract(pending_op.meta_json, '$.uid')))
|
|
WHERE json_valid(meta_json)
|
|
AND json_extract(meta_json, '$.uid') IN (SELECT old FROM local_v41);
|
|
UPDATE pending_op
|
|
SET meta_json = json_set(meta_json, '$.original_parent_uid',
|
|
(SELECT new FROM local_v41
|
|
WHERE old = json_extract(pending_op.meta_json, '$.original_parent_uid')))
|
|
WHERE json_valid(meta_json)
|
|
AND json_extract(meta_json, '$.original_parent_uid') IN (SELECT old FROM local_v41);
|
|
UPDATE pins SET uid = (SELECT new FROM local_v41 WHERE old = pins.uid)
|
|
WHERE uid IN (SELECT old FROM local_v41);
|
|
UPDATE nodes
|
|
SET node_json = json_set(node_json, '$.parent_uid.link_id',
|
|
(SELECT CAST(lid AS TEXT) FROM local_v41 WHERE old = nodes.parent_uid))
|
|
WHERE parent_uid IN (SELECT old FROM local_v41)
|
|
AND json_valid(node_json)
|
|
AND json_extract(node_json, '$.parent_uid.volume_id') = 'local';
|
|
UPDATE nodes SET parent_uid = (SELECT new FROM local_v41 WHERE old = nodes.parent_uid)
|
|
WHERE parent_uid IN (SELECT old FROM local_v41);
|
|
UPDATE nodes
|
|
SET node_json = json_set(node_json, '$.uid.link_id', CAST(lid AS TEXT))
|
|
WHERE lid IN (SELECT lid FROM local_v41)
|
|
AND json_valid(node_json)
|
|
AND json_extract(node_json, '$.uid.volume_id') = 'local';
|
|
UPDATE nodes SET uid = 'local~' || lid WHERE lid IN (SELECT lid FROM local_v41);
|
|
DROP TABLE local_v41;
|
|
";
|