diff --git a/Cargo.lock b/Cargo.lock index e345842..95e7d89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1114,6 +1114,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + [[package]] name = "dbl" version = "0.3.2" @@ -2902,6 +2908,32 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "lofty" +version = "0.25.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35d45c446382e04663705aa8e786157f64ef1fa85755d410121ffbf08f54b0db" +dependencies = [ + "byteorder", + "data-encoding", + "flate2", + "lofty_attr", + "log", + "ogg_pager", + "paste", +] + +[[package]] +name = "lofty_attr" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdbca1b60ba7cea959ffc84ac27f3dbdbacd6dfda168c7eabbc00e48654bb9fd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + [[package]] name = "log" version = "0.4.35" @@ -3323,6 +3355,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "ogg_pager" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "166611b491f939446e64c6be930843a31eb8e6798204ec3e9f57e084c18e7446" +dependencies = [ + "byteorder", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -3594,6 +3635,7 @@ dependencies = [ "futures", "image", "libc", + "lofty", "notify", "parking_lot 0.13.0", "pdfs-core", diff --git a/Cargo.toml b/Cargo.toml index 34728a3..6929f66 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,6 +50,10 @@ sha1 = "0.11" # formats a camera or phone actually produces — the rest of the crate's codecs # would be dead weight in the daemon. image = { version = "0.25", default-features = false, features = ["jpeg", "png", "webp", "gif", "tiff", "bmp", "rayon"] } +# Cover pictures of songs, read from their tags (ID3, FLAC, Ogg, MP4, APE) for +# the song's thumbnail. Its default feature is required: 0.25 does not build +# without it. +lofty = "0.25" rusqlite = { version = "0.41", features = ["bundled"] } # keyring 4 split into `keyring-core` plus per-platform stores. The libdbus # Secret Service store is the one keyring 3's `sync-secret-service` wrapped, and diff --git a/crates/pdfs-core/src/control.rs b/crates/pdfs-core/src/control.rs index 8075e50..34cb9fc 100644 --- a/crates/pdfs-core/src/control.rs +++ b/crates/pdfs-core/src/control.rs @@ -1881,11 +1881,26 @@ pub fn is_video_file_name(name: &str, size: Option) -> bool { !transport_stream || size.is_none_or(|size| size >= TS_VIDEO_MIN_BYTES) } +/// Whether `name` is a song or other recording in a format that can carry +/// its own cover picture: ID3 (MP3, AAC, WAV, AIFF), FLAC and Ogg pictures, +/// MP4 `covr` (M4A, M4B), or APE tags (WavPack, Monkey's Audio). +pub fn is_audio_file_name(name: &str) -> bool { + let Some((_, extension)) = name.rsplit_once('.') else { + return false; + }; + AUDIO_COVER_EXTENSIONS.contains(&extension.to_ascii_lowercase().as_str()) +} + +const AUDIO_COVER_EXTENSIONS: &[&str] = &[ + "aac", "aif", "aiff", "ape", "flac", "m4a", "m4b", "mp3", "oga", "ogg", "opus", "wav", "wv", +]; + /// Whether file listings show a thumbnail for a file named `name`, of `size` -/// bytes when known: an image [`is_thumbnail_image_name`] accepts, or a video, -/// whose thumbnail is a frame the daemon takes with ffmpeg. +/// bytes when known: an image [`is_thumbnail_image_name`] accepts, a video, +/// whose thumbnail is a frame the daemon takes with ffmpeg, or a song, whose +/// thumbnail is its cover: the picture in its tags, or a cover image beside it. pub fn is_thumbnail_media_name(name: &str, size: Option) -> bool { - is_thumbnail_image_name(name) || is_video_file_name(name, size) + is_thumbnail_image_name(name) || is_video_file_name(name, size) || is_audio_file_name(name) } /// Whether `name` is one of the concrete RAW formats handled by the exiftool @@ -2741,7 +2756,7 @@ mod tests { } #[test] - fn listings_thumbnail_images_and_videos() { + fn listings_thumbnail_images_videos_and_songs() { assert!(is_thumbnail_media_name("photo.JPG", Some(10))); assert!(is_thumbnail_media_name("camera.nef", None)); assert!(is_thumbnail_media_name("clip.mp4", Some(10))); @@ -2750,13 +2765,24 @@ mod tests { Some(64 * 1024 * 1024) )); assert!(!is_thumbnail_media_name("app.ts", Some(2048))); - assert!(!is_thumbnail_media_name( - "song.flac", - Some(64 * 1024 * 1024) - )); + assert!(is_thumbnail_media_name("song.flac", Some(64 * 1024 * 1024))); + assert!(is_thumbnail_media_name("Track 01.MP3", None)); assert!(!is_thumbnail_media_name("notes.txt", None)); } + #[test] + fn songs_are_told_by_their_extension() { + for name in [ + "a.mp3", "b.FLAC", "c.ogg", "d.oga", "e.opus", "f.m4a", "g.m4b", "h.aac", "i.wav", + "j.aiff", "k.aif", "l.wv", "m.ape", + ] { + assert!(is_audio_file_name(name), "{name}"); + } + for name in ["mp3", "song.mp3.zip", "clip.mp4", "notes.txt", "cover.jpg"] { + assert!(!is_audio_file_name(name), "{name}"); + } + } + /// Keep every extension accepted by the shared classifier covered. Drive /// activation uses this classification before it knows whether opening a /// file should stream through FUSE or materialise a complete cache entry; diff --git a/crates/pdfs-fuse/Cargo.toml b/crates/pdfs-fuse/Cargo.toml index 7a44d36..48a5d70 100644 --- a/crates/pdfs-fuse/Cargo.toml +++ b/crates/pdfs-fuse/Cargo.toml @@ -23,6 +23,7 @@ libc.workspace = true serde_json.workspace = true base64.workspace = true image.workspace = true +lofty.workspace = true notify = "8" parking_lot.workspace = true diff --git a/crates/pdfs-fuse/src/photos.rs b/crates/pdfs-fuse/src/photos.rs index 5639795..557b1f0 100644 --- a/crates/pdfs-fuse/src/photos.rs +++ b/crates/pdfs-fuse/src/photos.rs @@ -19,7 +19,7 @@ use std::time::{Duration, Instant}; use base64::Engine as _; use pdfs_core::control::{ FileThumbRequest, MappingProgress, PhotoItem, PhotoKind, PhotoLocation, PhotoThumb, PlaceInfo, - ThumbnailBuildStatus, Topic, is_raw_image_name, is_thumbnail_image_name, + ThumbnailBuildStatus, Topic, is_audio_file_name, is_raw_image_name, is_thumbnail_image_name, is_thumbnail_media_name, }; use pdfs_core::db::{self, StoredPhoto}; @@ -76,6 +76,36 @@ const VIDEO_BOX_WALK_MAX: usize = 32; /// A single frame is seconds of work even for 4K; this only stops a wedged /// ffmpeg from holding a thumbnail permit. const FFMPEG_TIMEOUT: Duration = Duration::from_secs(30); +/// A song no larger than this is downloaded whole for its cover. +const AUDIO_WHOLE_MAX_BYTES: u64 = 4 * 1024 * 1024; +/// How much of a song's start is read first, to learn where its tags end. +const AUDIO_PROBE_BYTES: u64 = 64 * 1024; +/// How much of a song's start is read when its first bytes don't say where its +/// tags end: an Ogg comment packet, which holds the cover, has no size up front. +const AUDIO_HEAD_BYTES: u64 = 4 * 1024 * 1024; +/// Read past the tags at a song's start, so the first audio frames are there +/// for a reader that looks for them. +const AUDIO_HEAD_SLACK: u64 = 64 * 1024; +/// Largest start of a song read for its tags. A cover bigger than this is not +/// worth the download for a 512-pixel thumbnail. +const AUDIO_TAG_MAX_BYTES: u64 = 16 * 1024 * 1024; +/// How much of a song's end is read in formats that keep their tags there +/// (APE tags in WavPack and Monkey's Audio, `id3 ` chunks in WAV and AIFF). +const AUDIO_TAIL_BYTES: u64 = 4 * 1024 * 1024; +/// Names of a folder's cover image, best first, matched without case and +/// extension. A song without a picture in its tags gets this image's. +const FOLDER_COVER_STEMS: [&str; 6] = [ + "cover", + "folder", + "front", + "album", + "albumart", + "albumartsmall", +]; +/// Image types a folder cover is taken in. +const FOLDER_COVER_EXTENSIONS: [&str; 4] = ["jpg", "jpeg", "png", "webp"]; +/// Largest folder cover image downloaded whole for a song's thumbnail. +const FOLDER_COVER_MAX_BYTES: u64 = 32 * 1024 * 1024; /// Largest thumbnail attached to an upload. Proton's own clients hold theirs to /// this, stepping the JPEG quality down until it fits. const UPLOAD_THUMB_MAX_BYTES: usize = 60 * 1024; @@ -630,6 +660,7 @@ enum Fetched { /// A whole image. Whole(Vec), Video(VideoSample), + Audio(AudioSample), } /// The parts of a video its thumbnail needs, downloaded: the start of the @@ -766,13 +797,181 @@ fn video_sample_thumbnail( video_file_thumbnail(&file.0, ffmpeg, timeout) } +/// The parts of a song its cover needs, downloaded: the start, where most +/// formats keep their tags, an M4A's index, and for formats that keep their +/// tags at the end, the end. +struct AudioSample { + /// The whole file's length. + size: u64, + /// Each part and the offset it came from. + parts: Vec<(u64, Vec)>, +} + +/// Where the tags at the start of a song end, read from its first `probe` +/// bytes: past an ID3v2 tag, and past FLAC's metadata blocks, which hold its +/// picture. `None` when the start says nothing, as for Ogg. A FLAC block +/// header past `probe` stops the walk with a guess of [`AUDIO_HEAD_BYTES`] +/// more. +fn audio_tags_end(probe: &[u8]) -> Option { + let mut offset = 0u64; + let mut known = false; + if probe.get(0..3) == Some(b"ID3") { + let size = probe.get(6..10)?; + if size.iter().any(|byte| byte & 0x80 != 0) { + return None; + } + let size = size + .iter() + .fold(0u64, |acc, byte| (acc << 7) | u64::from(*byte)); + let footer = probe.get(5).is_some_and(|flags| flags & 0x10 != 0); + offset = 10 + size + if footer { 10 } else { 0 }; + known = true; + } + let start = usize::try_from(offset).ok()?; + if probe.get(start..start + 4) == Some(b"fLaC") { + offset += 4; + loop { + let at = usize::try_from(offset).ok()?; + let Some(header) = probe.get(at..at + 4) else { + return Some(offset + AUDIO_HEAD_BYTES); + }; + let len = u64::from_be_bytes([0, 0, 0, 0, 0, header[1], header[2], header[3]]); + offset += 4 + len; + if header[0] & 0x80 != 0 { + return Some(offset); + } + } + } + known.then_some(offset) +} + +/// Whether a song in format `name` may keep its tags at its end. +fn audio_tags_at_end(name: &str) -> bool { + let extension = name + .rsplit_once('.') + .map(|(_, extension)| extension.to_ascii_lowercase()) + .unwrap_or_default(); + matches!(extension.as_str(), "wv" | "ape" | "wav" | "aif" | "aiff") +} + +/// Download what the cover of the song `reader` reads, named `name`, needs. A +/// song of up to [`AUDIO_WHOLE_MAX_BYTES`] comes whole; a larger one brings its +/// tags, never its audio. +async fn sample_audio(reader: &dyn RevisionRead, name: &str) -> DriveResult { + let size = reader.size(); + if size <= AUDIO_WHOLE_MAX_BYTES { + let whole = reader.read_at(0, size).await?; + return Ok(AudioSample { + size, + parts: vec![(0, whole)], + }); + } + let probe = reader.read_at(0, AUDIO_PROBE_BYTES.min(size)).await?; + // An M4A keeps its cover in its index, fetched below wherever it sits. + let iso_media = probe.get(4..8) == Some(b"ftyp"); + let want = match audio_tags_end(&probe) { + _ if iso_media => 0, + Some(end) => end.saturating_add(AUDIO_HEAD_SLACK), + None => AUDIO_HEAD_BYTES, + } + .min(AUDIO_TAG_MAX_BYTES) + .min(size); + let head = if want > probe.len() as u64 { + reader.read_at(0, want).await? + } else { + probe + }; + let mut parts = Vec::new(); + match iso_media_index(reader, &head, size).await? { + Some((offset, len)) if offset.saturating_add(len) > head.len() as u64 => { + parts.push((offset, reader.read_at(offset, len).await?)); + } + _ => {} + } + let head_len = head.len() as u64; + parts.push((0, head)); + if audio_tags_at_end(name) { + let start = size.saturating_sub(AUDIO_TAIL_BYTES).max(head_len); + if start < size { + parts.push((start, reader.read_at(start, size - start).await?)); + } + } + Ok(AudioSample { size, parts }) +} + +/// The picture a song's tags carry: the front cover, or else the first one. +/// `None` for a song without one, or one lofty can't read. +fn embedded_cover(path: &Path) -> Option> { + use lofty::config::{ParseOptions, ParsingMode}; + use lofty::file::TaggedFileExt as _; + use lofty::picture::PictureType; + + let tagged = lofty::probe::Probe::open(path) + .ok()? + .options( + ParseOptions::new() + .read_properties(false) + .parsing_mode(ParsingMode::Relaxed), + ) + .guess_file_type() + .ok()? + .read() + .ok()?; + let pictures = || tagged.tags().iter().flat_map(|tag| tag.pictures()); + pictures() + .find(|picture| picture.pic_type() == PictureType::CoverFront) + .or_else(|| pictures().next()) + .map(|picture| picture.data().to_vec()) + .filter(|data| !data.is_empty()) +} + +/// A thumbnail from the cover in a downloaded [`AudioSample`]'s tags. The +/// sample is laid out in a staging file at the offsets it came from, so the +/// tags are where the file's own headers say they are; the audio between +/// holds no data and costs no disk. A song without a cover, or with one the +/// image decoder can't read, is [`ScaleAttempt::Undecodable`]. +fn audio_cover_thumbnail(sample: &AudioSample, name: &str, staging_dir: &Path) -> ScaleAttempt { + let Some(file) = RawTempFile::create_in(staging_dir, &[], name) else { + return ScaleAttempt::Unavailable; + }; + let laid_out = OpenOptions::new() + .write(true) + .open(&file.0) + .and_then(|staged| { + for (offset, bytes) in &sample.parts { + staged.write_all_at(bytes, *offset)?; + } + staged.set_len(sample.size) + }); + if laid_out.is_err() { + return ScaleAttempt::Unavailable; + } + match embedded_cover(&file.0) { + Some(picture) => scale_thumbnail(&picture, "cover", staging_dir), + None => ScaleAttempt::Undecodable, + } +} + +/// How good a folder cover `name` is: its place in [`FOLDER_COVER_STEMS`], +/// lowest best, or `None` for a file that is no cover image. +fn cover_rank(name: &str) -> Option { + let (stem, extension) = name.rsplit_once('.')?; + let extension = extension.to_ascii_lowercase(); + if !FOLDER_COVER_EXTENSIONS.contains(&extension.as_str()) { + return None; + } + let stem = stem.to_ascii_lowercase(); + FOLDER_COVER_STEMS.iter().position(|known| *known == stem) +} + /// The thumbnail to attach to an upload of the local file at `path`, which is /// named `name` in Drive. Without one, Drive's own apps show a generic icon and /// the gallery has to download the whole file to make its own. /// -/// Best effort: a file that isn't an image or a video, is too large to read, or -/// can't be decoded uploads without a thumbnail, as it always did. Blocking and -/// CPU-bound, so async callers run it on the blocking pool. +/// Best effort: a file that isn't an image, a video or a song with a cover in +/// its tags, is too large to read, or can't be decoded uploads without a +/// thumbnail, as it always did. Blocking and CPU-bound, so async callers run it +/// on the blocking pool. pub(crate) fn upload_thumbnails(path: &Path, name: &str, staging_dir: &Path) -> Vec { if is_video_name(name) && !ffmpeg_available() { return Vec::new(); @@ -806,6 +1005,14 @@ fn upload_thumbnails_with( return Vec::new(); }; scale_thumbnail_unbudgeted(&bytes, name, staging_dir) + } else if is_audio_file_name(name) { + // Only the song's own picture: a cover image beside it is not known + // here, where `path` may be a staging copy rather than the song's + // folder. The daemon finds that one when a listing asks. + match embedded_cover(path) { + Some(picture) => scale_thumbnail(&picture, "cover", staging_dir), + None => return Vec::new(), + } } else { return Vec::new(); }; @@ -1703,6 +1910,7 @@ impl Core { return (uid, ThumbAttempt::Cancelled); } let video = is_video_name(&name); + let audio = is_audio_file_name(&name); // Without ffmpeg a video can still have its uploader's // thumbnail, but Photos has asked the server already. let can_sample = !video || ffmpeg_available(); @@ -1757,6 +1965,7 @@ impl Core { let download_uid = uid.clone(); let drive = core.drive.clone(); + let audio_name = name.clone(); let download = async move { // Photos asked the server in one batch already. A // file view's or a build's file may carry a @@ -1776,6 +1985,11 @@ impl Core { if video { let reader = drive.open_revision(&download_uid).await?; sample_video(reader.as_ref()).await.map(Fetched::Video) + } else if audio { + let reader = drive.open_revision(&download_uid).await?; + sample_audio(reader.as_ref(), &audio_name) + .await + .map(Fetched::Audio) } else { client .download_file(&download_uid) @@ -1833,7 +2047,11 @@ impl Core { // Decoding + scaling a 20 MP JPEG is CPU-bound and would // stall the runtime's worker; hand it to the blocking pool. let staging_dir = core.cache.raw_thumbnail_staging_dir().to_path_buf(); + let staging = staging_dir.clone(); let made = tokio::task::spawn_blocking(move || match fetched { + Fetched::Audio(sample) => { + audio_cover_thumbnail(&sample, &name, &staging_dir) + } Fetched::Video(sample) => video_sample_thumbnail( &sample, &name, @@ -1850,6 +2068,14 @@ impl Core { }) .await .unwrap_or(ScaleAttempt::Unavailable); + // A song without a picture of its own shows its + // folder's cover. + let made = match made { + ScaleAttempt::Undecodable if audio && core.thumb_job_current(job) => { + core.folder_cover_thumbnail(&uid, staging).await + } + made => made, + }; if !core.thumb_job_current(job) { return (uid, ThumbAttempt::Cancelled); } @@ -1918,6 +2144,77 @@ impl Core { summary } + /// The cover image in the folder of the song `uid`: the best-named one of + /// [`FOLDER_COVER_STEMS`] among its siblings the database knows. + fn folder_cover(&self, uid: &NodeUid) -> Option { + let node = self.db.node_by_uid(&uid.to_string()).ok()??; + let parent = node.parent_uid?; + let siblings = match self.db.children_if_listed(&parent) { + Ok(Some(children)) => children, + _ => self.db.known_children(&parent).ok()?, + }; + siblings + .into_iter() + .filter(|sibling| !sibling.is_folder() && !sibling.trashed) + .filter_map(|sibling| cover_rank(&sibling.name).map(|rank| (rank, sibling))) + .min_by(|(a, x), (b, y)| a.cmp(b).then_with(|| x.name.cmp(&y.name))) + .map(|(_, cover)| cover) + } + + /// A thumbnail of the song `uid` made from its folder's cover image: the + /// cover's own cached thumbnail when it has one, else the thumbnail its + /// uploader attached, else the image itself, up to + /// [`FOLDER_COVER_MAX_BYTES`]. + /// + /// The result is stored under the song, so a cover added to the folder or + /// replaced later shows only once the song changes or the daemon restarts + /// and forgets the songs it found no cover for. + async fn folder_cover_thumbnail(&self, uid: &NodeUid, staging_dir: PathBuf) -> ScaleAttempt { + let Some(cover) = self.folder_cover(uid) else { + return ScaleAttempt::Undecodable; + }; + let ttype = ThumbnailType::Thumbnail; + let cached = self + .cache + .cached_thumbnail_path(&cover.uid, ttype.as_i32(), cover.modification_time) + .and_then(|path| std::fs::read(path).ok()); + let ready = match cached { + Some(bytes) => Some(bytes), + None => match self.drive.download_thumbnail(&cover.uid, ttype).await { + Ok(bytes) => bytes.filter(|bytes| !bytes.is_empty()), + Err(e) => { + warn!(cover = %cover.uid, error = %e, "folder cover thumbnail download failed"); + return ScaleAttempt::Unavailable; + } + }, + }; + if let Some(bytes) = ready { + return match ratio_of(&bytes) { + Some(ratio) => ScaleAttempt::Made(GeneratedThumb { bytes, ratio }), + None => ScaleAttempt::Undecodable, + }; + } + let read = async { + let reader = self.drive.open_revision(&cover.uid).await?; + if reader.size() > FOLDER_COVER_MAX_BYTES { + return Ok(None); + } + reader.read_at(0, reader.size()).await.map(Some) + }; + let bytes = match read.await { + Ok(Some(bytes)) => bytes, + Ok(None) => return ScaleAttempt::Undecodable, + Err(e) => { + warn!(cover = %cover.uid, error = %e, "folder cover download failed"); + return ScaleAttempt::Unavailable; + } + }; + let name = cover.name; + tokio::task::spawn_blocking(move || scale_thumbnail(&bytes, &name, &staging_dir)) + .await + .unwrap_or(ScaleAttempt::Unavailable) + } + pub(crate) fn thumbnail_build_status(&self) -> ThumbnailBuildStatus { self.thumbnail_build.lock().clone() } @@ -2459,6 +2756,10 @@ mod decode_budget_tests { #[cfg(test)] mod thumb_tests { + use super::{ + AUDIO_HEAD_BYTES, AUDIO_TAIL_BYTES, AudioSample, audio_cover_thumbnail, audio_tags_end, + cover_rank, sample_audio, + }; use super::{ RawTempFile, ScaleAttempt, THUMB_EDGE, UPLOAD_IMAGE_MAX_BYTES, UPLOAD_THUMB_MAX_BYTES, VIDEO_HEAD_BYTES, VIDEO_WHOLE_MAX_BYTES, VideoSample, can_make_thumbnail, exiftool_binary, @@ -2970,6 +3271,28 @@ mod thumb_tests { assert!(thumb.ratio > 0.0); } + /// Optional check against real songs, through the same partial download a + /// Drive song gets. `PDFS_AUDIO_FIXTURE` lists files separated by `:`. + #[test] + #[ignore = "set PDFS_AUDIO_FIXTURE to songs with a cover in their tags"] + fn real_audio_fixture_yields_its_cover() { + let paths = std::env::var_os("PDFS_AUDIO_FIXTURE").expect("PDFS_AUDIO_FIXTURE"); + for path in std::env::split_paths(&paths) { + let name = path.file_name().unwrap().to_string_lossy().into_owned(); + let bytes = std::fs::read(&path).unwrap(); + let revision = FakeRevision::new(bytes.len() as u64).with(0, bytes); + let sample = block_on(sample_audio(&revision, &name)).unwrap(); + let read = revision.read.load(Ordering::Relaxed); + let attempt = audio_cover_thumbnail(&sample, &name, &std::env::temp_dir()); + assert!( + matches!(attempt, ScaleAttempt::Made(_)), + "{name}: no cover, read {read} of {} bytes", + revision.size + ); + eprintln!("{name}: read {read} of {} bytes", revision.size); + } + } + #[test] fn videos_are_told_apart_by_name() { for name in [ @@ -3126,4 +3449,201 @@ mod thumb_tests { "a small one is untouched" ); } + + /// A `width`×`height` PNG, the cover a tag carries. + fn png(width: u32, height: u32) -> Vec { + let mut bytes = Vec::new(); + image::DynamicImage::ImageRgb8(image::RgbImage::new(width, height)) + .write_to( + &mut std::io::Cursor::new(&mut bytes), + image::ImageFormat::Png, + ) + .unwrap(); + bytes + } + + /// An ID3v2.3 tag holding `picture` as its front cover. + fn id3_with_cover(picture: &[u8]) -> Vec { + let mut body = vec![0u8]; + body.extend_from_slice(b"image/png\0"); + body.push(3); + body.push(0); + body.extend_from_slice(picture); + let mut frame = b"APIC".to_vec(); + frame.extend_from_slice(&(body.len() as u32).to_be_bytes()); + frame.extend_from_slice(&[0, 0]); + frame.extend_from_slice(&body); + let len = frame.len() as u32; + let mut tag = b"ID3\x03\x00\x00".to_vec(); + tag.extend((0..4).rev().map(|i| ((len >> (7 * i)) & 0x7f) as u8)); + tag.extend_from_slice(&frame); + tag + } + + /// Ten silent MPEG-1 Layer III frames at 128 kbit/s and 44.1 kHz. + fn mpeg_frames() -> Vec { + let mut frame = vec![0u8; 417]; + frame[..4].copy_from_slice(&[0xFF, 0xFB, 0x90, 0x64]); + frame.repeat(10) + } + + /// A FLAC stream's metadata: STREAMINFO, then `picture` as its front cover. + fn flac_with_cover(picture: &[u8]) -> Vec { + let mut info = vec![0x10, 0x00, 0x10, 0x00, 0, 0, 0, 0, 0, 0]; + let packed: u64 = (44_100 << 44) | (1 << 41) | (15 << 36); + info.extend_from_slice(&packed.to_be_bytes()); + info.extend_from_slice(&[0; 16]); + let mut block = Vec::new(); + block.extend_from_slice(&3u32.to_be_bytes()); + block.extend_from_slice(&9u32.to_be_bytes()); + block.extend_from_slice(b"image/png"); + block.extend_from_slice(&0u32.to_be_bytes()); + for value in [16u32, 8, 24, 0, picture.len() as u32] { + block.extend_from_slice(&value.to_be_bytes()); + } + block.extend_from_slice(picture); + let mut out = b"fLaC".to_vec(); + out.push(0); + out.extend_from_slice(&(info.len() as u32).to_be_bytes()[1..]); + out.extend_from_slice(&info); + out.push(0x80 | 6); + out.extend_from_slice(&(block.len() as u32).to_be_bytes()[1..]); + out.extend_from_slice(&block); + out + } + + fn whole_song(bytes: Vec) -> AudioSample { + AudioSample { + size: bytes.len() as u64, + parts: vec![(0, bytes)], + } + } + + #[test] + fn a_song_shows_the_cover_in_its_tags() { + let tmp = scratch("song-cover"); + std::fs::create_dir_all(&tmp).unwrap(); + let mut mp3 = id3_with_cover(&png(16, 8)); + mp3.extend(mpeg_frames()); + let ScaleAttempt::Made(thumb) = audio_cover_thumbnail(&whole_song(mp3), "song.mp3", &tmp) + else { + panic!("an ID3 cover scales"); + }; + assert!((thumb.ratio - 2.0).abs() < 1e-6); + + let ScaleAttempt::Made(thumb) = + audio_cover_thumbnail(&whole_song(flac_with_cover(&png(8, 16))), "song.flac", &tmp) + else { + panic!("a FLAC picture scales"); + }; + assert!((thumb.ratio - 0.5).abs() < 1e-6); + std::fs::remove_dir_all(&tmp).unwrap(); + } + + #[test] + fn a_song_without_a_cover_is_no_thumbnail_of_its_own() { + let tmp = scratch("song-bare"); + std::fs::create_dir_all(&tmp).unwrap(); + let attempt = audio_cover_thumbnail(&whole_song(mpeg_frames()), "song.mp3", &tmp); + assert!(matches!(attempt, ScaleAttempt::Undecodable)); + std::fs::remove_dir_all(&tmp).unwrap(); + } + + #[test] + fn the_start_of_a_song_says_where_its_tags_end() { + let tag = id3_with_cover(&png(4, 4)); + assert_eq!(audio_tags_end(&tag), Some(tag.len() as u64)); + let flac = flac_with_cover(&png(4, 4)); + assert_eq!(audio_tags_end(&flac), Some(flac.len() as u64)); + // FLAC after an ID3 tag, as some taggers write it. + let both = [tag.clone(), flac.clone()].concat(); + assert_eq!(audio_tags_end(&both), Some(both.len() as u64)); + // The picture's block header cut off by the end of the probe, just + // past STREAMINFO: a guess past it. + let cut = &flac[..4 + 4 + 34 + 2]; + assert!(audio_tags_end(cut).is_some_and(|end| end > cut.len() as u64)); + assert_eq!(audio_tags_end(b"OggS\0\0\0\0"), None); + assert_eq!( + audio_tags_end(b"ID3\x03\x00\x00\x80\0\0\0"), + None, + "not synchsafe" + ); + } + + #[test] + fn a_large_song_downloads_its_tags_not_its_audio() { + const SIZE: u64 = 300 << 20; + let mut start = id3_with_cover(&png(16, 8)); + start.extend(mpeg_frames()); + let revision = FakeRevision::new(SIZE).with(0, start); + let sample = block_on(sample_audio(&revision, "long mix.mp3")).unwrap(); + let read = revision.read.load(Ordering::Relaxed); + assert!(read < 1 << 20, "read {read} bytes"); + + let tmp = scratch("song-sparse"); + std::fs::create_dir_all(&tmp).unwrap(); + let ScaleAttempt::Made(thumb) = audio_cover_thumbnail(&sample, "long mix.mp3", &tmp) else { + panic!("the cover comes from the start alone"); + }; + assert!((thumb.ratio - 2.0).abs() < 1e-6); + std::fs::remove_dir_all(&tmp).unwrap(); + + // Ogg says nothing up front: a bounded head. WavPack keeps its tags at + // the end: the tail too. + let revision = FakeRevision::new(SIZE); + block_on(sample_audio(&revision, "live.ogg")).unwrap(); + assert!(revision.read.load(Ordering::Relaxed) <= AUDIO_HEAD_BYTES + (64 << 10)); + let revision = FakeRevision::new(SIZE); + let sample = block_on(sample_audio(&revision, "live.wv")).unwrap(); + assert!( + sample + .parts + .iter() + .any(|(offset, bytes)| *offset == SIZE - AUDIO_TAIL_BYTES + && bytes.len() as u64 == AUDIO_TAIL_BYTES) + ); + } + + #[test] + fn an_uploaded_song_carries_its_cover() { + let tmp = scratch("upload-song"); + std::fs::create_dir_all(&tmp).unwrap(); + // A staging blob has no extension: the format comes from the bytes. + let blob = tmp.join("blob"); + let mut mp3 = id3_with_cover(&png(64, 64)); + mp3.extend(mpeg_frames()); + std::fs::write(&blob, mp3).unwrap(); + let ffmpeg = fake_ffmpeg(&tmp, "exit 1"); + let thumbs = upload_thumbnails_with(&blob, "song.mp3", &tmp, ffmpeg.as_os_str()); + let [thumbnail] = thumbs.as_slice() else { + panic!("one thumbnail, got {}", thumbs.len()); + }; + assert!(matches!(thumbnail.thumbnail_type, ThumbnailType::Thumbnail)); + assert_eq!(ratio_of(&thumbnail.content), Some(1.0)); + + std::fs::write(&blob, mpeg_frames()).unwrap(); + assert!( + upload_thumbnails_with(&blob, "song.mp3", &tmp, ffmpeg.as_os_str()).is_empty(), + "a song without a cover uploads bare" + ); + let _ = std::fs::remove_dir_all(&tmp); + } + + #[test] + fn a_folder_cover_is_found_by_its_usual_names() { + assert_eq!(cover_rank("cover.jpg"), Some(0)); + assert_eq!(cover_rank("Cover.JPG"), Some(0)); + assert_eq!(cover_rank("folder.png"), Some(1)); + assert_eq!(cover_rank("AlbumArtSmall.jpg"), Some(5)); + assert!(cover_rank("cover.jpg") < cover_rank("front.webp")); + for name in [ + "cover.txt", + "my cover.jpg", + "cover", + "back.jpg", + "cover.jpg.part", + ] { + assert_eq!(cover_rank(name), None, "{name}"); + } + } } diff --git a/crates/pdfs-fuse/src/sim/daemon.rs b/crates/pdfs-fuse/src/sim/daemon.rs index e0fb73d..b897306 100644 --- a/crates/pdfs-fuse/src/sim/daemon.rs +++ b/crates/pdfs-fuse/src/sim/daemon.rs @@ -3732,6 +3732,88 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + /// A song with no picture in its tags shows the cover image in its folder, + /// and the daemon tells a following front-end that a thumbnail landed, so + /// a folder on screen fills in without being opened again. + #[test] + #[ignore = "mounts FUSE: run with `cargo test -p pdfs-fuse sim:: -- --ignored`"] + fn a_song_without_a_cover_shows_its_folders_and_says_so() { + let dir = scratch("song-cover"); + std::fs::create_dir_all(&dir).unwrap(); + // Silent MPEG-1 Layer III frames, without any tag. + let mut frame = vec![0u8; 417]; + frame[..4].copy_from_slice(&[0xFF, 0xFB, 0x90, 0x64]); + let song = frame.repeat(64); + let mut cover = Vec::new(); + image::codecs::jpeg::JpegEncoder::new(&mut cover) + .encode_image(&image::RgbImage::new(300, 300)) + .unwrap(); + + let drive = FakeDrive::new(); + drive.device().write("track one.mp3", &song); + drive.device().write("Cover.jpg", &cover); + let client = drive.client(1, Faults::lan()); + let daemon = Daemon::start(&dir.join("daemon"), client).unwrap(); + + let landed = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let mut subscription = control::subscribe(&daemon.socket).unwrap(); + let flag = landed.clone(); + std::thread::spawn(move || { + // The first change names every topic: a new subscription catches up. + let mut caught_up = false; + while let Ok(Some(event)) = subscription.next_event() { + if let control::Event::Changed { topics } = event { + if caught_up && topics.contains(&control::Topic::Thumbnails) { + flag.store(true, std::sync::atomic::Ordering::SeqCst); + return; + } + caught_up = true; + } + } + }); + + let Ok(Response::Entries { entries }) = daemon.request(&Request::ListDir { + path: String::new(), + }) else { + panic!("listing"); + }; + let song = entries + .iter() + .find(|entry| entry.name == "track one.mp3") + .expect("the song is listed"); + let items = vec![control::FileThumbRequest { + uid: song.uid.clone(), + modified: song.modified, + name: song.name.clone(), + }]; + let Ok(Response::FileThumbGeneration { generation }) = + daemon.request(&Request::ReserveFileThumbGeneration) + else { + panic!("generation"); + }; + let mut path = None; + let settled = wait_until(Duration::from_secs(60), || { + let Ok(Response::Thumbs { items: replies }) = daemon.request(&Request::FileThumbs { + items: items.clone(), + generation, + }) else { + return false; + }; + path = replies.first().and_then(|reply| reply.path.clone()); + path.is_some() + }); + assert!(settled, "the song got no thumbnail"); + let thumb = image::load_from_memory(&std::fs::read(path.unwrap()).unwrap()).unwrap(); + assert_eq!(thumb.width(), thumb.height(), "the square cover"); + assert!( + wait_until(Duration::from_secs(15), || landed + .load(std::sync::atomic::Ordering::SeqCst)), + "no thumbnails change was published" + ); + assert!(matches!(daemon.stop(), Ok(MountOutcome::Shutdown))); + let _ = std::fs::remove_dir_all(&dir); + } + /// A video in My files gets a thumbnail from a frame of its own, but a /// recording can be gigabytes: only its start, and an MP4's index, are /// downloaded for it. A video with a thumbnail from its uploader is not diff --git a/crates/pdfs-gui/resources/style.css b/crates/pdfs-gui/resources/style.css index 6f5e7ba..0a242f9 100644 --- a/crates/pdfs-gui/resources/style.css +++ b/crates/pdfs-gui/resources/style.css @@ -63,6 +63,7 @@ scale.browser-status-meter trough { min-width: 140px; min-height: 6px; } .photo-viewer-window { background-color: #111014; } .media-player-window video { background-color: black; } .media-player-note { color: rgba(255, 255, 255, 0.5); } +.media-player-note .file-thumbnail { border-radius: 12px; } .viewer-top-bar { background: linear-gradient(to bottom, rgba(0, 0, 0, 0.85), rgba(0, 0, 0, 0.45) 60%, rgba(0, 0, 0, 0)); padding: 10px 16px 40px 20px; color: white; } .viewer-title { font-weight: 700; font-size: 1.05rem; text-shadow: 0 1px 3px rgba(0, 0, 0, 0.9); color: white; } .viewer-counter { font-size: 0.8rem; color: rgba(255, 255, 255, 0.72); text-shadow: 0 1px 3px rgba(0, 0, 0, 0.9); } diff --git a/crates/pdfs-gui/src/app/widgets/player.rs b/crates/pdfs-gui/src/app/widgets/player.rs index 2676223..894c3a1 100644 --- a/crates/pdfs-gui/src/app/widgets/player.rs +++ b/crates/pdfs-gui/src/app/widgets/player.rs @@ -19,6 +19,9 @@ const SEEK_SHORT: i64 = 5; /// How much ↑ and ↓ change the volume in the My files player. const VOLUME_STEP: f64 = 0.1; +/// The edge of a song's cover in the My files player, in logical pixels. +const COVER_SIZE: i32 = 320; + thread_local! { /// The volume and mute the user last set, applied to the next stream so /// muting one video doesn't leave the next one blaring. Kept for the run @@ -202,16 +205,21 @@ fn build_media_player(ui: &Rc, entry: &DirEntry, path: &Path) { wire_player(&video, >k_window); // A song has no picture, which would leave a black window above the - // controls: a note stands in for it. - let note = gtk4::Image::builder() - .icon_name("audio-x-generic-symbolic") - .pixel_size(128) - .halign(gtk4::Align::Center) - .valign(gtk4::Align::Center) - .can_target(false) - .visible(false) - .build(); + // controls: its cover stands in for it, the one its listing tile shows, + // and a note while there is none. + let note = file_thumbnail_widget(COVER_SIZE, 128); + note.set_can_target(false); + note.set_visible(false); note.add_css_class("media-player-note"); + if let Some(cover) = note + .child() + .and_then(|fallback| fallback.next_sibling()) + .and_downcast::() + { + // The whole cover, not a square cut out of it. + cover.set_content_fit(gtk4::ContentFit::Contain); + } + bind_file_thumbnail(ui, ¬e, entry, true); let overlay = gtk4::Overlay::new(); overlay.set_child(Some(&video)); overlay.add_overlay(¬e); diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 8bb490f..3f6ca4e 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -104,6 +104,14 @@ Existing folders stay device backups. Older builds cannot open the upgraded data 10 seconds, and `M` to mute. The volume carries over from one video to the next, and the thumbnail strip in Photos stays off a video, where it covered the seek bar. +- **Album covers for songs.** A song in My files, in search and in Shared shows its cover as its + thumbnail, and the player shows it above the controls instead of a music note. The cover is the + picture in the song's tags (MP3, FLAC, Ogg, Opus, M4A, WAV, AIFF, WavPack, Monkey's Audio), or + else a `cover`, `folder`, `front` or `album` image (`.jpg`, `.png`, `.webp`) in the same folder. + Only the song's tags are downloaded for it, not the whole song. Songs uploaded from this computer + carry their cover as their thumbnail, so they show it at once on other devices too. A cover + image added to a folder later is picked up once the song changes or the service restarts. + ### Changed - **`proton-sdk` and `proton-drive-rs` 0.11.0.** Timestamps are converted where they enter