diff --git a/android/app/src/main/cpp/pstr_mpv.cpp b/android/app/src/main/cpp/pstr_mpv.cpp index d5930d1..dffa0ba 100644 --- a/android/app/src/main/cpp/pstr_mpv.cpp +++ b/android/app/src/main/cpp/pstr_mpv.cpp @@ -211,6 +211,9 @@ struct PlaybackState { /// episode one — which is the difference between resuming an episode where /// the viewer left it and resuming it where the *previous* one ended. int64_t generation = 0; + /// mpv has loaded this generation's file and made its own track pick, + /// which is the first moment the pick can be second-guessed. + bool loaded = false; }; /// A subtitle file published for mpv, and what to call it. @@ -428,6 +431,8 @@ class Player { char *lang_raw = nullptr; char *title_raw = nullptr; int selected = 0; + int is_default = 0; + int forced = 0; mpv_get_property(mpv_, (base + "id").c_str(), MPV_FORMAT_INT64, &id); mpv_get_property(mpv_, (base + "type").c_str(), MPV_FORMAT_STRING, &type_raw); MpvString type(type_raw); @@ -436,10 +441,14 @@ class Player { mpv_get_property(mpv_, (base + "title").c_str(), MPV_FORMAT_STRING, &title_raw); MpvString title(title_raw); mpv_get_property(mpv_, (base + "selected").c_str(), MPV_FORMAT_FLAG, &selected); + mpv_get_property(mpv_, (base + "default").c_str(), MPV_FORMAT_FLAG, &is_default); + mpv_get_property(mpv_, (base + "forced").c_str(), MPV_FORMAT_FLAG, &forced); if (i) json += ','; json += "{\"id\":" + std::to_string(id) + ",\"type\":\"" + escape(type.get()) + "\",\"language\":\"" + escape(lang.get()) + "\",\"title\":\"" + - escape(title.get()) + "\",\"selected\":" + (selected ? "true" : "false") + "}"; + escape(title.get()) + "\",\"selected\":" + (selected ? "true" : "false") + + ",\"default\":" + (is_default ? "true" : "false") + + ",\"forced\":" + (forced ? "true" : "false") + "}"; } return json + ']'; } @@ -602,6 +611,7 @@ class Player { // Mid-load, this is the outgoing file finishing its load. if (!state_.loading) { loaded_generation_ = state_.generation; + state_.loaded = true; ready.swap(pending_subtitles_); } } @@ -970,8 +980,9 @@ extern "C" JNIEXPORT jdoubleArray JNICALL Java_io_narl_protonstream_playback_Nat state.video_width, state.video_height, static_cast(state.end_reason), state.buffering ? 1.0 : 0.0, state.cache_percent, - state.seeking ? 1.0 : 0.0, state.cached_until}; - constexpr jsize kFields = 14; + state.seeking ? 1.0 : 0.0, state.cached_until, + state.loaded ? 1.0 : 0.0}; + constexpr jsize kFields = 15; jdoubleArray result = env->NewDoubleArray(kFields); if (!result) return static_cast(nullptr); env->SetDoubleArrayRegion(result, 0, kFields, values); diff --git a/android/app/src/main/java/io/narl/protonstream/playback/NativeMpvHost.kt b/android/app/src/main/java/io/narl/protonstream/playback/NativeMpvHost.kt index b9dfdae..1043831 100644 --- a/android/app/src/main/java/io/narl/protonstream/playback/NativeMpvHost.kt +++ b/android/app/src/main/java/io/narl/protonstream/playback/NativeMpvHost.kt @@ -20,7 +20,9 @@ import kotlin.concurrent.withLock import uniffi.pstr_android.ChapterPlan import uniffi.pstr_android.ChapterRecord import uniffi.pstr_android.SubtitleFilesRecord +import uniffi.pstr_android.SubtitleTrackRecord import uniffi.pstr_android.chapterPlan +import uniffi.pstr_android.fullSubtitlesInstead /** * Why the open file ended, mirroring `pstr_player::EndReason` so both clients @@ -108,6 +110,10 @@ data class MpvTrack( val language: String, val title: String, val selected: Boolean, + /** The container marks it as the default choice. */ + val isDefault: Boolean = false, + /** The container marks it as forced: on-screen text only, by convention. */ + val forced: Boolean = false, ) { val label: String get() = listOf(language, title).filter(String::isNotBlank) .joinToString(" — ").ifBlank { "${type.replaceFirstChar(Char::uppercase)} $id" } @@ -129,6 +135,8 @@ class NativeMpvHost private constructor(private val nativeHandle: Long) : Libmpv @Volatile private var currentMedia: String? = null @Volatile private var currentGeneration: Double = -1.0 @Volatile private var plannedFor: Pair? = null + /** The load generation whose subtitle pick has been looked at. */ + @Volatile private var subtitlesCheckedFor: Double = -1.0 var onPlaybackStarted: (() -> Unit)? = null var onStateChanged: ((MpvPlaybackState) -> Unit)? = null var onExplicitStop: (() -> Unit)? = null @@ -347,6 +355,12 @@ class NativeMpvHost private constructor(private val nativeHandle: Long) : Libmpv // both of which are the main thread's. withContext(Dispatchers.Main) { onStateChanged?.invoke(next) } refreshChapters(next) + if (values[LOADED] != 0.0 && values.generation() == currentGeneration && + subtitlesCheckedFor != currentGeneration + ) { + subtitlesCheckedFor = currentGeneration + preferFullSubtitles() + } } withOpenHandle(::nativeTakeError)?.let { mutableProblem.value = it } if (tick++ % 4 == 0) refreshTracks() @@ -384,13 +398,32 @@ class NativeMpvHost private constructor(private val nativeHandle: Long) : Libmpv } } + /** + * Play the dialogue rather than the signs and songs mpv picked by language + * (`pstr_core::subtitles`). Not remembered: it is the choice mpv should + * have made, not the viewer's. + */ + private fun preferFullSubtitles() { + refreshTracks() + val subtitles = mutableTracks.value.filter { it.type == "sub" } + val id = fullSubtitlesInstead( + subtitles.map { + SubtitleTrackRecord(it.id, it.language, it.title, it.isDefault, it.forced, it.selected) + }, + ) ?: return + subtitles.firstOrNull { it.id == id }?.let(::selectTrack) + } + private fun refreshTracks() { runCatching { val encoded = withOpenHandle(::nativeTracks) ?: return val values = JSONArray(encoded) List(values.length()) { index -> values.getJSONObject(index).run { - MpvTrack(getLong("id"), getString("type"), getString("language"), getString("title"), getBoolean("selected")) + MpvTrack( + getLong("id"), getString("type"), getString("language"), getString("title"), + getBoolean("selected"), optBoolean("default"), optBoolean("forced"), + ) } } }.onSuccess { mutableTracks.value = it } @@ -467,7 +500,7 @@ class NativeMpvHost private constructor(private val nativeHandle: Long) : Libmpv companion object { /** Fields `nativeState` fills, in the order `pstr_mpv.cpp` writes them. */ - private const val FIELDS = 14 + private const val FIELDS = 15 private const val GENERATION = 6 private const val WIDTH = 7 private const val HEIGHT = 8 @@ -476,6 +509,7 @@ class NativeMpvHost private constructor(private val nativeHandle: Long) : Libmpv private const val CACHE_PERCENT = 11 private const val SEEKING = 12 private const val CACHED_UNTIL = 13 + private const val LOADED = 14 init { System.loadLibrary("pstr_android") diff --git a/crates/pstr-android/src/lib.rs b/crates/pstr-android/src/lib.rs index 789b879..00089fc 100644 --- a/crates/pstr-android/src/lib.rs +++ b/crates/pstr-android/src/lib.rs @@ -27,6 +27,7 @@ use pstr_core::proton_drive_rs::ThumbnailType; use pstr_core::proton_sdk::api::HumanVerificationCredential; use pstr_core::proton_sdk::ids::{LinkId, VolumeId}; use pstr_core::shares::AccountFolder; +use pstr_core::subtitles::SubtitleTrack; use pstr_core::sync::WatchSync; use pstr_core::{ Account, AccountStore, SecretStore, Share, ShareClient, ShareStore, SharedLibrary, @@ -739,6 +740,38 @@ fn chapter_kind(role: ChapterRole) -> ChapterKind { } } +/// One subtitle track as the player read it out of mpv's `track-list`. +#[derive(Debug, Clone, uniffi::Record)] +pub struct SubtitleTrackRecord { + pub id: i64, + pub language: Option, + pub title: Option, + pub default: bool, + pub forced: bool, + pub selected: bool, +} + +/// The dialogue track to play in place of the subtitle track mpv chose, when +/// mpv chose the signs and songs (`pstr_core::subtitles`). Asked once per file, +/// as soon as mpv has loaded it and made its own pick. +#[uniffi::export] +pub fn full_subtitles_instead(tracks: Vec) -> Option { + let tracks: Vec = tracks + .into_iter() + .map(|track| SubtitleTrack { + id: track.id, + language: track + .language + .filter(|language| !language.trim().is_empty()), + title: track.title.filter(|title| !title.trim().is_empty()), + default: track.default, + forced: track.forced, + selected: track.selected, + }) + .collect(); + pstr_core::subtitles::full_instead(&tracks) +} + /// What the app is holding on disk. Offline episodes are the viewer's, the /// block cache is not: only the latter may be reclaimed without asking. #[derive(Debug, Clone, uniffi::Record)] diff --git a/crates/pstr-app/src/playback.rs b/crates/pstr-app/src/playback.rs index c80c504..70edb6e 100644 --- a/crates/pstr-app/src/playback.rs +++ b/crates/pstr-app/src/playback.rs @@ -578,6 +578,15 @@ fn run( if let Some(show) = target.track_prefs.as_deref() { restore_tracks(&player, show); } + // A subtitle track picked by name stays; one mpv picked by + // language may be the signs and songs, not the dialogue. + let picked = target + .track_prefs + .as_deref() + .is_some_and(|show| show.subtitle_title.is_some()); + if !picked { + prefer_full_subtitles(&player); + } emit_tracks(&engine, id, &player); } PlayerEvent::Duration(value) => duration = Some(*value), @@ -735,6 +744,16 @@ fn restore_tracks(player: &Player, show: &pstr_core::catalog::TitleTrackPrefs) { } } +/// Play the dialogue rather than the signs and songs mpv picked by language. +/// Not remembered: it is the choice mpv should have made, not the viewer's. +fn prefer_full_subtitles(player: &Player) { + if let Some(track) = pstr_player::full_subtitles_instead(&player.tracks()) + && let Err(error) = player.select_track(TrackKind::Subtitle, Some(track)) + { + tracing::warn!("select dialogue subtitle {track}: {error}"); + } +} + /// Hand mpv the subtitle files paired with the video it has just loaded. /// /// One is turned on only when subtitles are on and the video has none of its diff --git a/crates/pstr-core/src/lib.rs b/crates/pstr-core/src/lib.rs index c998b35..465ff68 100644 --- a/crates/pstr-core/src/lib.rs +++ b/crates/pstr-core/src/lib.rs @@ -21,6 +21,7 @@ pub mod prefs; pub mod profile; pub mod shares; pub mod shelves; +pub mod subtitles; pub mod sync; pub use account::{Account, AccountStore}; diff --git a/crates/pstr-core/src/subtitles.rs b/crates/pstr-core/src/subtitles.rs new file mode 100644 index 0000000..5f720ff --- /dev/null +++ b/crates/pstr-core/src/subtitles.rs @@ -0,0 +1,202 @@ +//! Which of a file's subtitle tracks is the one to read. +//! +//! Anime releases often carry two subtitle tracks in one language: the +//! dialogue, and a "Signs & Songs" track that only translates the on-screen +//! text and the lyrics. The second is the one marked default or forced as +//! often as not, and mpv, picking by language, plays it — an episode where +//! nobody seems to speak. A language cannot tell the two apart; the track's +//! name and its forced flag can. +//! +//! Nothing here talks to a player. Every client reads the track list out of +//! mpv once a file is loaded and asks [`full_instead`] whether to switch. + +use crate::language::same_language; + +/// One subtitle track of the open file, as mpv lists it. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct SubtitleTrack { + /// mpv's id for it, what `sid` is set to. + pub id: i64, + pub language: Option, + /// The name the muxer gave it: "Dialogue", "Signs & Songs". + pub title: Option, + /// The container marks it as the default choice. + pub default: bool, + /// The container marks it as forced: on-screen text only, by convention. + pub forced: bool, + /// It is the one playing. + pub selected: bool, +} + +/// How much of what is said a track carries, as far as its name and flags say. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Coverage { + /// Signs, songs, or forced: not the dialogue. + Partial, + /// Nothing says either way. + Unknown, + /// Named as the dialogue: "Dialogue", "Full Subtitles", "SDH". + Full, +} + +/// Words in a track's name that make it the whole dialogue. They win over the +/// partial ones: "Full (with Signs)" is still the full track. +const FULL_WORDS: [&str; 6] = ["full", "dialogue", "dialog", "dialogues", "sdh", "cc"]; + +/// Words in a track's name that make it something other than the dialogue. +const PARTIAL_WORDS: [&str; 13] = [ + "sign", "signs", "song", "songs", "s&s", "forced", "karaoke", "lyrics", "op", "ed", "oped", + "opening", "ending", +]; + +impl SubtitleTrack { + pub fn coverage(&self) -> Coverage { + let title = self.title.as_deref().unwrap_or_default().to_lowercase(); + let words: Vec<&str> = title + .split(|c: char| !c.is_alphanumeric() && c != '&') + .filter(|word| !word.is_empty()) + .collect(); + let has = |list: &[&str]| words.iter().any(|word| list.contains(word)); + if has(&FULL_WORDS) { + Coverage::Full + } else if self.forced || has(&PARTIAL_WORDS) { + Coverage::Partial + } else { + Coverage::Unknown + } + } +} + +/// The track to play in place of the one mpv chose, if it chose the wrong one. +/// +/// Only a track in the same language as the playing one is considered — the +/// language is the viewer's choice, and this only corrects which of its tracks +/// was taken. The playing track is replaced when it is partial and the file has +/// one in its language that is not, or when nothing names it and another is +/// named as the dialogue. Among the rest, a track named as the dialogue comes +/// first, then the container's default, then the file's order. `None` leaves +/// the selection as it is, which includes subtitles being off. +pub fn full_instead(tracks: &[SubtitleTrack]) -> Option { + let playing = tracks.iter().find(|track| track.selected)?; + let current = playing.coverage(); + if current == Coverage::Full { + return None; + } + let same = |track: &SubtitleTrack| match (&playing.language, &track.language) { + (Some(a), Some(b)) => same_language(a, b), + (None, None) => true, + _ => false, + }; + tracks + .iter() + .filter(|track| track.id != playing.id && same(track)) + .filter(|track| track.coverage() > current) + // `max_by_key` keeps the last of equals; reversed, the first. + .rev() + .max_by_key(|track| (track.coverage(), track.default)) + .map(|track| track.id) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn track(id: i64, language: &str, title: &str) -> SubtitleTrack { + SubtitleTrack { + id, + language: Some(language.into()), + title: (!title.is_empty()).then(|| title.into()), + ..SubtitleTrack::default() + } + } + + fn playing(mut track: SubtitleTrack) -> SubtitleTrack { + track.selected = true; + track + } + + #[test] + fn signs_and_songs_playing_gives_way_to_the_dialogue_in_its_language() { + let tracks = [ + playing(track(1, "eng", "Signs & Songs")), + track(2, "eng", "Dialogue"), + ]; + assert_eq!(full_instead(&tracks), Some(2)); + } + + #[test] + fn an_unnamed_track_gives_way_to_one_named_as_the_dialogue() { + let tracks = [playing(track(1, "en", "")), track(2, "en", "Dialogue")]; + assert_eq!(full_instead(&tracks), Some(2)); + } + + #[test] + fn a_forced_track_gives_way_to_an_unnamed_one_and_the_default_comes_first() { + let mut forced = playing(track(1, "eng", "English")); + forced.forced = true; + let mut default = track(3, "eng", ""); + default.default = true; + let tracks = [forced, track(2, "eng", ""), default]; + assert_eq!(full_instead(&tracks), Some(3)); + } + + #[test] + fn equal_candidates_resolve_to_the_first_in_the_file() { + let tracks = [ + playing(track(1, "eng", "Songs")), + track(2, "eng", ""), + track(3, "eng", ""), + ]; + assert_eq!(full_instead(&tracks), Some(2)); + } + + #[test] + fn a_track_in_another_language_is_never_switched_to() { + let tracks = [ + playing(track(1, "eng", "Signs")), + track(2, "ger", "Dialogue"), + ]; + assert_eq!(full_instead(&tracks), None); + } + + #[test] + fn two_letter_and_three_letter_tags_are_one_language() { + let tracks = [playing(track(1, "en", "OP/ED")), track(2, "eng", "")]; + assert_eq!(full_instead(&tracks), Some(2)); + } + + #[test] + fn the_dialogue_playing_stays_and_subtitles_off_stay_off() { + let tracks = [ + track(1, "eng", "Signs & Songs"), + playing(track(2, "eng", "Dialogue")), + ]; + assert_eq!(full_instead(&tracks), None); + let off = [ + track(1, "eng", "Signs & Songs"), + track(2, "eng", "Dialogue"), + ]; + assert_eq!(full_instead(&off), None); + } + + #[test] + fn a_full_word_outweighs_a_partial_one_in_the_same_name() { + assert_eq!( + track(1, "eng", "Full (with Signs)").coverage(), + Coverage::Full + ); + assert_eq!(track(1, "eng", "Signs/Songs").coverage(), Coverage::Partial); + assert_eq!(track(1, "eng", "English").coverage(), Coverage::Unknown); + // A word inside another is not that word. + assert_eq!( + track(1, "eng", "Designer cut").coverage(), + Coverage::Unknown + ); + } + + #[test] + fn a_partial_track_with_only_partial_company_stays() { + let tracks = [playing(track(1, "eng", "Signs")), track(2, "eng", "Songs")]; + assert_eq!(full_instead(&tracks), None); + } +} diff --git a/crates/pstr-player/src/lib.rs b/crates/pstr-player/src/lib.rs index 1005b5d..0b85a6a 100644 --- a/crates/pstr-player/src/lib.rs +++ b/crates/pstr-player/src/lib.rs @@ -52,4 +52,4 @@ pub use player::{ }; pub use registry::{PROTOCOL, StreamHandle, StreamRegistry}; pub use render::VideoRenderer; -pub use tracks::{LANGUAGES, Track, TrackKind, language_name, pick_track}; +pub use tracks::{LANGUAGES, Track, TrackKind, full_subtitles_instead, language_name, pick_track}; diff --git a/crates/pstr-player/src/tracks.rs b/crates/pstr-player/src/tracks.rs index ea1121a..e5d7082 100644 --- a/crates/pstr-player/src/tracks.rs +++ b/crates/pstr-player/src/tracks.rs @@ -12,6 +12,7 @@ //! an empty string. use libmpv2::Mpv; +use pstr_core::subtitles::{SubtitleTrack, full_instead}; /// What a track carries. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -69,6 +70,9 @@ pub struct Track { pub selected: bool, /// Whether the container marks it as the default choice. pub default: bool, + /// Whether the container marks it as forced: a subtitle track of the + /// on-screen text alone. + pub forced: bool, /// True for a track added from outside the file — a sidecar subtitle. None /// of ours are, yet. pub external: bool, @@ -152,6 +156,24 @@ pub fn pick_track( ) } +/// The dialogue track to play in place of the subtitle track mpv chose, when +/// mpv chose signs and songs; see [`pstr_core::subtitles::full_instead`]. +pub fn full_subtitles_instead(tracks: &[Track]) -> Option { + let subtitles: Vec<_> = tracks + .iter() + .filter(|track| track.kind == TrackKind::Subtitle) + .map(|track| SubtitleTrack { + id: track.id, + language: track.language.clone(), + title: track.title.clone(), + default: track.default, + forced: track.forced, + selected: track.selected, + }) + .collect(); + full_instead(&subtitles) +} + fn read_one(mpv: &Mpv, index: i64) -> Option { let kind = TrackKind::from_mpv(&text(mpv, index, "type")?)?; let id: i64 = mpv @@ -167,6 +189,7 @@ fn read_one(mpv: &Mpv, index: i64) -> Option { codec: text(mpv, index, "codec"), selected: flag(mpv, index, "selected"), default: flag(mpv, index, "default"), + forced: flag(mpv, index, "forced"), external: flag(mpv, index, "external"), }) } @@ -200,6 +223,7 @@ mod tests { codec: None, selected: false, default: false, + forced: false, external: false, } } diff --git a/ios/ProtonStream/Player/MpvPlayer.swift b/ios/ProtonStream/Player/MpvPlayer.swift index 93a44df..1fce6c8 100644 --- a/ios/ProtonStream/Player/MpvPlayer.swift +++ b/ios/ProtonStream/Player/MpvPlayer.swift @@ -33,6 +33,10 @@ struct MpvTrack: Identifiable, Hashable { let language: String let title: String let selected: Bool + /// The container marks it as the default choice. + var isDefault = false + /// The container marks it as forced: on-screen text only, by convention. + var forced = false var label: String { let parts = [language, title].filter { !$0.trimmingCharacters(in: .whitespaces).isEmpty } @@ -91,6 +95,8 @@ final class MpvPlayer { /// Called when the duration or the pause state changes: what the lock /// screen shows, and whether the screen may sleep. @ObservationIgnored var onStateChanged: (() -> Void)? + /// Called once per file, when mpv has loaded it and picked its tracks. + @ObservationIgnored var onLoaded: (() -> Void)? @ObservationIgnored let layer = MpvMetalLayer() @ObservationIgnored private var mpv: OpaquePointer? @@ -473,6 +479,7 @@ final class MpvPlayer { let ready = pendingSubtitles pendingSubtitles = [] attach(ready) + onLoaded?() } case .restarted: seeking = false @@ -511,7 +518,9 @@ final class MpvPlayer { type: getString(base + "type") ?? "", language: getString(base + "lang") ?? "", title: getString(base + "title") ?? "", - selected: getString(base + "selected") == "yes" + selected: getString(base + "selected") == "yes", + isDefault: getString(base + "default") == "yes", + forced: getString(base + "forced") == "yes" ) } } diff --git a/ios/ProtonStream/Player/PlayerHost.swift b/ios/ProtonStream/Player/PlayerHost.swift index e8a3e38..5f35127 100644 --- a/ios/ProtonStream/Player/PlayerHost.swift +++ b/ios/ProtonStream/Player/PlayerHost.swift @@ -256,6 +256,7 @@ final class PlayerHost { player?.onEnded = { [weak self] reason, media in self?.ended(reason, media) } player?.onTick = { [weak self] in self?.tick() } player?.onStateChanged = { [weak self] in self?.publishNowPlaying() } + player?.onLoaded = { [weak self] in self?.preferFullSubtitles() } } guard let player else { playbackError = "Playback is unavailable: libmpv did not start." @@ -433,6 +434,18 @@ final class PlayerHost { chapters = chapterPlan(chapters: player.chapters, duration: player.duration > 0 ? player.duration : nil) } + /// Play the dialogue rather than the signs and songs mpv picked by + /// language (`pstr_core::subtitles`). Not remembered: it is the choice mpv + /// should have made, not the viewer's. + private func preferFullSubtitles() { + guard let player, playingOpenEpisode else { return } + let subtitles = player.tracks.filter { $0.type == "sub" }.map { + SubtitleTrackRecord(id: $0.id, language: $0.language, title: $0.title, default: $0.isDefault, forced: $0.forced, selected: $0.selected) + } + guard let id = fullSubtitlesInstead(tracks: subtitles), let track = player.tracks.first(where: { $0.type == "sub" && $0.id == id }) else { return } + player.select(type: "sub", track: track) + } + /// The skip button's offer at the current position, if any. var offer: SkipOffer? { guard let player, playingOpenEpisode else { return nil }