feat(core,fuse,gui): find and trash duplicate photos

A "Find duplicates" button in the Photos header opens a review of every
set of files that share a content hash. Each set shows its copies side by
side with a Keep toggle; the copy in the most albums, then in the larger
RAW+JPEG group, then the oldest, is kept by default, and a set can't lose
its last kept copy. The copies not kept move to Trash one file at a time.

- core: Db::photo_duplicates, Request::PhotoDuplicates, and a files_only
  flag on TrashNodes (defaults to false, so old clients still trash whole
  groups).
- core: deleting a group's representative now clears the group key of its
  remaining files instead of leaving them pointing at a missing uid.
- fuse: handlers for both requests.

crates/pdfs-core/src/control.rs

@@ -88,6 +88,9 @@ pub enum Request {
     /// `limit` of them — the "On this day" strip above the timeline. Replies
     /// with [`Response::Photos`]; a front-end groups them by year itself.
     PhotosOnThisDay { limit: usize },
+    /// Sets of photos that are byte-identical copies of each other, going by
+    /// the server's content hash. Replies with [`Response::PhotoDuplicates`].
+    PhotoDuplicates,
     /// List the account's photo albums, newest activity first, including the
     /// albums other people share with us. Metadata only — an album's cover
     /// thumbnail is fetched with [`Request::PhotoThumbs`] like any other photo.
@@ -134,8 +137,14 @@ pub enum Request {
     ///
     /// A photo that belongs to a group takes the whole group with it: trashing
     /// the JPEG of a shot and leaving its RAW behind is not what anyone means by
-    /// deleting that photo.
-    TrashNodes { uids: Vec<String> },
+    /// deleting that photo. With `files_only` set, exactly the listed files
+    /// go and their groups stay: the duplicate finder removes one copy of a
+    /// shot, not the shot. Older front-ends omit it and trash whole groups.
+    TrashNodes {
+        uids: Vec<String>,
+        #[serde(default)]
+        files_only: bool,
+    },
     /// Fetch thumbnails for ordinary Drive image files shown outside the Photos
     /// timeline. The modification time is the cache validity tag, so replacing
     /// an image can never reuse its previous revision's thumbnail.
@@ -1784,6 +1793,9 @@ pub enum Response {
     /// The months the timeline spans (reply to [`Request::PhotoMonths`]),
     /// newest first.
     PhotoMonths { months: Vec<PhotoMonth> },
+    /// The duplicate sets (reply to [`Request::PhotoDuplicates`]), newest
+    /// first. Each set lists the copy worth keeping first.
+    PhotoDuplicates { sets: Vec<Vec<PhotoItem>> },
     /// Thumbnails for a [`Request::PhotoThumbs`] batch.
     Thumbs { items: Vec<PhotoThumb> },
     /// Per-node outcome of a [`Request::TrashNodes`] batch. Reported per uid
@@ -2180,7 +2192,17 @@ mod tests {
     fn trash_requests_and_outcomes_roundtrip() {
         let request = Request::TrashNodes {
             uids: vec!["volume~first".into(), "volume~second".into()],
+            files_only: false,
         };
+        // An older front-end's request trashes whole groups.
+        let old: Request = serde_json::from_str(r#"{"TrashNodes":{"uids":["v~a"]}}"#).unwrap();
+        assert!(matches!(
+            old,
+            Request::TrashNodes {
+                files_only: false,
+                ..
+            }
+        ));
         let line = serde_json::to_string(&request).unwrap();
         assert!(!line.contains('\n'), "wire form must be a single line");
         let decoded: Request = serde_json::from_str(&line).unwrap();

crates/pdfs-core/src/db/photos.rs

@@ -3,7 +3,7 @@
 
 use rusqlite::params;
 
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 
 use super::Db;
 use crate::Result;
@@ -447,6 +447,14 @@ impl Db {
                 albums.execute([uid])?;
             }
         }
+        // A group whose representative went is left pointing at nothing, which
+        // drops its other files off every page. Make them stand alone until the
+        // next refresh groups them again.
+        tx.execute(
+            "UPDATE photos SET group_key = NULL WHERE group_key IS NOT NULL \
+             AND group_key NOT IN (SELECT uid FROM photos)",
+            [],
+        )?;
         tx.commit()?;
         Ok(removed)
     }
@@ -609,6 +617,57 @@ impl Db {
         Ok(rows)
     }
 
+    /// Sets of photos with the same content hash, each set ordered with the
+    /// copy worth keeping first, and the sets newest first.
+    ///
+    /// The copy to keep is the one in the most albums, then the one that is part
+    /// of a larger group (the JPEG's RAW is filed with it, a stray re-upload is
+    /// not), then the earliest in server order. At most one file per group is
+    /// listed: the files of one RAW+JPEG shot are not copies of each other, and a
+    /// set left with one member is no set.
+    pub fn photo_duplicates(&self) -> Result<Vec<Vec<StoredPhoto>>> {
+        let rows: Vec<(String, String, String)> = {
+            let conn = self.read();
+            let mut stmt = conn.prepare(
+                "SELECT p.uid, p.content_hash, COALESCE(p.group_key, p.uid) AS k FROM photos p \
+                 WHERE p.content_hash IN (SELECT content_hash FROM photos \
+                     WHERE content_hash IS NOT NULL GROUP BY content_hash HAVING COUNT(*) > 1) \
+                 ORDER BY p.content_hash, \
+                     (SELECT COUNT(*) FROM album_photos a WHERE a.uid = p.uid) DESC, \
+                     (SELECT COUNT(*) FROM photos m WHERE COALESCE(m.group_key, m.uid) = COALESCE(p.group_key, p.uid)) DESC, \
+                     p.seq",
+            )?;
+            stmt.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?
+                .collect::<rusqlite::Result<_>>()?
+        };
+        let uids: Vec<String> = rows.iter().map(|(uid, _, _)| uid.clone()).collect();
+        let mut stored: HashMap<String, StoredPhoto> = self
+            .photos_by_uid(&uids)?
+            .into_iter()
+            .map(|photo| (photo.uid.clone(), photo))
+            .collect();
+
+        let mut sets: Vec<Vec<StoredPhoto>> = Vec::new();
+        let mut hash_of_set: Option<&str> = None;
+        let mut groups_in_set: HashSet<&str> = HashSet::new();
+        for (uid, hash, group) in &rows {
+            if hash_of_set != Some(hash.as_str()) {
+                hash_of_set = Some(hash);
+                groups_in_set.clear();
+                sets.push(Vec::new());
+            }
+            if !groups_in_set.insert(group) {
+                continue;
+            }
+            if let (Some(set), Some(photo)) = (sets.last_mut(), stored.remove(uid)) {
+                set.push(photo);
+            }
+        }
+        sets.retain(|set| set.len() > 1);
+        sets.sort_by_key(|set| std::cmp::Reverse(set.iter().map(|p| p.capture_time).max()));
+        Ok(sets)
+    }
+
     /// Per-tab counts for the Photos page subtitle: `(photos, videos, raw)`.
     pub fn photos_counts(&self) -> Result<(usize, usize, usize)> {
         use crate::control::PhotoKind;

crates/pdfs-core/src/db/tests.rs

@@ -285,6 +285,64 @@ fn on_this_day_finds_the_same_date_in_earlier_years() {
     );
 }
 
+fn hashed(uid: &str, capture_time: i64, name: &str, hash: &str) -> TimelineRow {
+    TimelineRow {
+        name: Some(name.into()),
+        content_hash: Some(hash.into()),
+        ..TimelineRow::new(uid, capture_time)
+    }
+}
+
+/// A copy an album holds is the one kept; the files of one RAW+JPEG shot are
+/// never copies of each other; a hash only one file carries is no set.
+#[test]
+fn duplicates_list_the_copy_to_keep_first() {
+    let db = Db::open_in_memory().unwrap();
+    db.photos_replace(&[
+        hashed("copy", 500, "IMG_2.JPG", "h2"),
+        hashed("filed", 400, "IMG_2(1).JPG", "h2"),
+        hashed("jpeg", 300, "IMG_1.JPG", "h1"),
+        hashed("raw", 300, "IMG_1.CR2", "h1"),
+        hashed("stray", 200, "IMG_1(1).CR2", "h1"),
+        hashed("alone", 100, "IMG_3.JPG", "h3"),
+    ])
+    .unwrap();
+    db.album_photos_replace("a1", &[("filed".into(), 400, None, None)])
+        .unwrap();
+
+    let sets = db.photo_duplicates().unwrap();
+    let uids: Vec<Vec<&str>> = sets
+        .iter()
+        .map(|set| set.iter().map(|p| p.uid.as_str()).collect())
+        .collect();
+    assert_eq!(uids, [vec!["filed", "copy"], vec!["jpeg", "stray"]]);
+}
+
+/// Trashing the file a group is shown as leaves the rest of the group on the
+/// timeline instead of pointing at a photo that is gone.
+#[test]
+fn deleting_a_group_representative_keeps_its_other_files() {
+    let db = Db::open_in_memory().unwrap();
+    db.photos_replace(&[
+        hashed("jpeg", 300, "IMG_1.JPG", "h1"),
+        hashed("raw", 300, "IMG_1.CR2", "h2"),
+    ])
+    .unwrap();
+    assert_eq!(
+        db.photos_page(0, 10, None, None, false, false)
+            .unwrap()
+            .len(),
+        1
+    );
+
+    db.photos_delete(&["jpeg".into()]).unwrap();
+    let page = db.photos_page(0, 10, None, None, false, false).unwrap();
+    assert_eq!(
+        page.iter().map(|p| p.uid.as_str()).collect::<Vec<_>>(),
+        ["raw"]
+    );
+}
+
 #[test]
 fn not_in_album_hides_photos_any_album_holds() {
     let db = Db::open_in_memory().unwrap();