feat(core,app,android,ios): play the dialogue, not the signs and songs

Anime releases often carry two subtitle tracks in one language, the
dialogue and a "Signs & Songs" track, and the second is as often as not
the one marked default or forced. mpv, picking by language, played it.

pstr_core::subtitles reads a track's name and forced flag for whether it
is the dialogue, and every client asks it once per file, after mpv has
made its own pick, whether to switch to a track in the same language
that is. A subtitle track the viewer chose by name stays, and subtitles
that are off stay off.
This commit is contained in:
nvrl committed 2026-10-14 18:18:40 +02:00
1 parent 8d615579e2
commit b1ea076ea3
10 files changed
+353 -7

No files matched your search

+14 -3
View File
@@ -211,6 +211,9 @@ struct PlaybackState {
/// episode one — which is the difference between resuming an episode where
/// the viewer left it and resuming it where the *previous* one ended.
int64_t generation = 0;
/// mpv has loaded this generation's file and made its own track pick,
/// which is the first moment the pick can be second-guessed.
bool loaded = false;
};
/// A subtitle file published for mpv, and what to call it.
@@ -428,6 +431,8 @@ class Player {
char *lang_raw = nullptr;
char *title_raw = nullptr;
int selected = 0;
int is_default = 0;
int forced = 0;
mpv_get_property(mpv_, (base + "id").c_str(), MPV_FORMAT_INT64, &id);
mpv_get_property(mpv_, (base + "type").c_str(), MPV_FORMAT_STRING, &type_raw);
MpvString type(type_raw);
@@ -436,10 +441,14 @@ class Player {
mpv_get_property(mpv_, (base + "title").c_str(), MPV_FORMAT_STRING, &title_raw);
MpvString title(title_raw);
mpv_get_property(mpv_, (base + "selected").c_str(), MPV_FORMAT_FLAG, &selected);
mpv_get_property(mpv_, (base + "default").c_str(), MPV_FORMAT_FLAG, &is_default);
mpv_get_property(mpv_, (base + "forced").c_str(), MPV_FORMAT_FLAG, &forced);
if (i) json += ',';
json += "{\"id\":" + std::to_string(id) + ",\"type\":\"" + escape(type.get()) +
"\",\"language\":\"" + escape(lang.get()) + "\",\"title\":\"" +
escape(title.get()) + "\",\"selected\":" + (selected ? "true" : "false") + "}";
escape(title.get()) + "\",\"selected\":" + (selected ? "true" : "false") +
",\"default\":" + (is_default ? "true" : "false") +
",\"forced\":" + (forced ? "true" : "false") + "}";
}
return json + ']';
}
@@ -602,6 +611,7 @@ class Player {
// Mid-load, this is the outgoing file finishing its load.
if (!state_.loading) {
loaded_generation_ = state_.generation;
state_.loaded = true;
ready.swap(pending_subtitles_);
}
}
@@ -970,8 +980,9 @@ extern "C" JNIEXPORT jdoubleArray JNICALL Java_io_narl_protonstream_playback_Nat
state.video_width, state.video_height,
static_cast<jdouble>(state.end_reason),
state.buffering ? 1.0 : 0.0, state.cache_percent,
state.seeking ? 1.0 : 0.0, state.cached_until};
constexpr jsize kFields = 14;
state.seeking ? 1.0 : 0.0, state.cached_until,
state.loaded ? 1.0 : 0.0};
constexpr jsize kFields = 15;
jdoubleArray result = env->NewDoubleArray(kFields);
if (!result) return static_cast<jdoubleArray>(nullptr);
env->SetDoubleArrayRegion(result, 0, kFields, values);
@@ -20,7 +20,9 @@ import kotlin.concurrent.withLock
import uniffi.pstr_android.ChapterPlan
import uniffi.pstr_android.ChapterRecord
import uniffi.pstr_android.SubtitleFilesRecord
import uniffi.pstr_android.SubtitleTrackRecord
import uniffi.pstr_android.chapterPlan
import uniffi.pstr_android.fullSubtitlesInstead
/**
* Why the open file ended, mirroring `pstr_player::EndReason` so both clients
@@ -108,6 +110,10 @@ data class MpvTrack(
val language: String,
val title: String,
val selected: Boolean,
/** The container marks it as the default choice. */
val isDefault: Boolean = false,
/** The container marks it as forced: on-screen text only, by convention. */
val forced: Boolean = false,
) {
val label: String get() = listOf(language, title).filter(String::isNotBlank)
.joinToString(" — ").ifBlank { "${type.replaceFirstChar(Char::uppercase)} $id" }
@@ -129,6 +135,8 @@ class NativeMpvHost private constructor(private val nativeHandle: Long) : Libmpv
@Volatile private var currentMedia: String? = null
@Volatile private var currentGeneration: Double = -1.0
@Volatile private var plannedFor: Pair<Double, Long>? = null
/** The load generation whose subtitle pick has been looked at. */
@Volatile private var subtitlesCheckedFor: Double = -1.0
var onPlaybackStarted: (() -> Unit)? = null
var onStateChanged: ((MpvPlaybackState) -> Unit)? = null
var onExplicitStop: (() -> Unit)? = null
@@ -347,6 +355,12 @@ class NativeMpvHost private constructor(private val nativeHandle: Long) : Libmpv
// both of which are the main thread's.
withContext(Dispatchers.Main) { onStateChanged?.invoke(next) }
refreshChapters(next)
if (values[LOADED] != 0.0 && values.generation() == currentGeneration &&
subtitlesCheckedFor != currentGeneration
) {
subtitlesCheckedFor = currentGeneration
preferFullSubtitles()
}
}
withOpenHandle(::nativeTakeError)?.let { mutableProblem.value = it }
if (tick++ % 4 == 0) refreshTracks()
@@ -384,13 +398,32 @@ class NativeMpvHost private constructor(private val nativeHandle: Long) : Libmpv
}
}
/**
* Play the dialogue rather than the signs and songs mpv picked by language
* (`pstr_core::subtitles`). Not remembered: it is the choice mpv should
* have made, not the viewer's.
*/
private fun preferFullSubtitles() {
refreshTracks()
val subtitles = mutableTracks.value.filter { it.type == "sub" }
val id = fullSubtitlesInstead(
subtitles.map {
SubtitleTrackRecord(it.id, it.language, it.title, it.isDefault, it.forced, it.selected)
},
) ?: return
subtitles.firstOrNull { it.id == id }?.let(::selectTrack)
}
private fun refreshTracks() {
runCatching {
val encoded = withOpenHandle(::nativeTracks) ?: return
val values = JSONArray(encoded)
List(values.length()) { index ->
values.getJSONObject(index).run {
MpvTrack(getLong("id"), getString("type"), getString("language"), getString("title"), getBoolean("selected"))
MpvTrack(
getLong("id"), getString("type"), getString("language"), getString("title"),
getBoolean("selected"), optBoolean("default"), optBoolean("forced"),
)
}
}
}.onSuccess { mutableTracks.value = it }
@@ -467,7 +500,7 @@ class NativeMpvHost private constructor(private val nativeHandle: Long) : Libmpv
companion object {
/** Fields `nativeState` fills, in the order `pstr_mpv.cpp` writes them. */
private const val FIELDS = 14
private const val FIELDS = 15
private const val GENERATION = 6
private const val WIDTH = 7
private const val HEIGHT = 8
@@ -476,6 +509,7 @@ class NativeMpvHost private constructor(private val nativeHandle: Long) : Libmpv
private const val CACHE_PERCENT = 11
private const val SEEKING = 12
private const val CACHED_UNTIL = 13
private const val LOADED = 14
init {
System.loadLibrary("pstr_android")
+33
View File
@@ -27,6 +27,7 @@ use pstr_core::proton_drive_rs::ThumbnailType;
use pstr_core::proton_sdk::api::HumanVerificationCredential;
use pstr_core::proton_sdk::ids::{LinkId, VolumeId};
use pstr_core::shares::AccountFolder;
use pstr_core::subtitles::SubtitleTrack;
use pstr_core::sync::WatchSync;
use pstr_core::{
Account, AccountStore, SecretStore, Share, ShareClient, ShareStore, SharedLibrary,
@@ -739,6 +740,38 @@ fn chapter_kind(role: ChapterRole) -> ChapterKind {
}
}
/// One subtitle track as the player read it out of mpv's `track-list`.
#[derive(Debug, Clone, uniffi::Record)]
pub struct SubtitleTrackRecord {
pub id: i64,
pub language: Option<String>,
pub title: Option<String>,
pub default: bool,
pub forced: bool,
pub selected: bool,
}
/// The dialogue track to play in place of the subtitle track mpv chose, when
/// mpv chose the signs and songs (`pstr_core::subtitles`). Asked once per file,
/// as soon as mpv has loaded it and made its own pick.
#[uniffi::export]
pub fn full_subtitles_instead(tracks: Vec<SubtitleTrackRecord>) -> Option<i64> {
let tracks: Vec<SubtitleTrack> = tracks
.into_iter()
.map(|track| SubtitleTrack {
id: track.id,
language: track
.language
.filter(|language| !language.trim().is_empty()),
title: track.title.filter(|title| !title.trim().is_empty()),
default: track.default,
forced: track.forced,
selected: track.selected,
})
.collect();
pstr_core::subtitles::full_instead(&tracks)
}
/// What the app is holding on disk. Offline episodes are the viewer's, the
/// block cache is not: only the latter may be reclaimed without asking.
#[derive(Debug, Clone, uniffi::Record)]
+19
View File
@@ -578,6 +578,15 @@ fn run(
if let Some(show) = target.track_prefs.as_deref() {
restore_tracks(&player, show);
}
// A subtitle track picked by name stays; one mpv picked by
// language may be the signs and songs, not the dialogue.
let picked = target
.track_prefs
.as_deref()
.is_some_and(|show| show.subtitle_title.is_some());
if !picked {
prefer_full_subtitles(&player);
}
emit_tracks(&engine, id, &player);
}
PlayerEvent::Duration(value) => duration = Some(*value),
@@ -735,6 +744,16 @@ fn restore_tracks(player: &Player, show: &pstr_core::catalog::TitleTrackPrefs) {
}
}
/// Play the dialogue rather than the signs and songs mpv picked by language.
/// Not remembered: it is the choice mpv should have made, not the viewer's.
fn prefer_full_subtitles(player: &Player) {
if let Some(track) = pstr_player::full_subtitles_instead(&player.tracks())
&& let Err(error) = player.select_track(TrackKind::Subtitle, Some(track))
{
tracing::warn!("select dialogue subtitle {track}: {error}");
}
}
/// Hand mpv the subtitle files paired with the video it has just loaded.
///
/// One is turned on only when subtitles are on and the video has none of its
+1
View File
@@ -21,6 +21,7 @@ pub mod prefs;
pub mod profile;
pub mod shares;
pub mod shelves;
pub mod subtitles;
pub mod sync;
pub use account::{Account, AccountStore};
+202
View File
@@ -0,0 +1,202 @@
//! Which of a file's subtitle tracks is the one to read.
//!
//! Anime releases often carry two subtitle tracks in one language: the
//! dialogue, and a "Signs & Songs" track that only translates the on-screen
//! text and the lyrics. The second is the one marked default or forced as
//! often as not, and mpv, picking by language, plays it — an episode where
//! nobody seems to speak. A language cannot tell the two apart; the track's
//! name and its forced flag can.
//!
//! Nothing here talks to a player. Every client reads the track list out of
//! mpv once a file is loaded and asks [`full_instead`] whether to switch.
use crate::language::same_language;
/// One subtitle track of the open file, as mpv lists it.
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct SubtitleTrack {
/// mpv's id for it, what `sid` is set to.
pub id: i64,
pub language: Option<String>,
/// The name the muxer gave it: "Dialogue", "Signs & Songs".
pub title: Option<String>,
/// The container marks it as the default choice.
pub default: bool,
/// The container marks it as forced: on-screen text only, by convention.
pub forced: bool,
/// It is the one playing.
pub selected: bool,
}
/// How much of what is said a track carries, as far as its name and flags say.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum Coverage {
/// Signs, songs, or forced: not the dialogue.
Partial,
/// Nothing says either way.
Unknown,
/// Named as the dialogue: "Dialogue", "Full Subtitles", "SDH".
Full,
}
/// Words in a track's name that make it the whole dialogue. They win over the
/// partial ones: "Full (with Signs)" is still the full track.
const FULL_WORDS: [&str; 6] = ["full", "dialogue", "dialog", "dialogues", "sdh", "cc"];
/// Words in a track's name that make it something other than the dialogue.
const PARTIAL_WORDS: [&str; 13] = [
"sign", "signs", "song", "songs", "s&s", "forced", "karaoke", "lyrics", "op", "ed", "oped",
"opening", "ending",
];
impl SubtitleTrack {
pub fn coverage(&self) -> Coverage {
let title = self.title.as_deref().unwrap_or_default().to_lowercase();
let words: Vec<&str> = title
.split(|c: char| !c.is_alphanumeric() && c != '&')
.filter(|word| !word.is_empty())
.collect();
let has = |list: &[&str]| words.iter().any(|word| list.contains(word));
if has(&FULL_WORDS) {
Coverage::Full
} else if self.forced || has(&PARTIAL_WORDS) {
Coverage::Partial
} else {
Coverage::Unknown
}
}
}
/// The track to play in place of the one mpv chose, if it chose the wrong one.
///
/// Only a track in the same language as the playing one is considered — the
/// language is the viewer's choice, and this only corrects which of its tracks
/// was taken. The playing track is replaced when it is partial and the file has
/// one in its language that is not, or when nothing names it and another is
/// named as the dialogue. Among the rest, a track named as the dialogue comes
/// first, then the container's default, then the file's order. `None` leaves
/// the selection as it is, which includes subtitles being off.
pub fn full_instead(tracks: &[SubtitleTrack]) -> Option<i64> {
let playing = tracks.iter().find(|track| track.selected)?;
let current = playing.coverage();
if current == Coverage::Full {
return None;
}
let same = |track: &SubtitleTrack| match (&playing.language, &track.language) {
(Some(a), Some(b)) => same_language(a, b),
(None, None) => true,
_ => false,
};
tracks
.iter()
.filter(|track| track.id != playing.id && same(track))
.filter(|track| track.coverage() > current)
// `max_by_key` keeps the last of equals; reversed, the first.
.rev()
.max_by_key(|track| (track.coverage(), track.default))
.map(|track| track.id)
}
#[cfg(test)]
mod tests {
use super::*;
fn track(id: i64, language: &str, title: &str) -> SubtitleTrack {
SubtitleTrack {
id,
language: Some(language.into()),
title: (!title.is_empty()).then(|| title.into()),
..SubtitleTrack::default()
}
}
fn playing(mut track: SubtitleTrack) -> SubtitleTrack {
track.selected = true;
track
}
#[test]
fn signs_and_songs_playing_gives_way_to_the_dialogue_in_its_language() {
let tracks = [
playing(track(1, "eng", "Signs & Songs")),
track(2, "eng", "Dialogue"),
];
assert_eq!(full_instead(&tracks), Some(2));
}
#[test]
fn an_unnamed_track_gives_way_to_one_named_as_the_dialogue() {
let tracks = [playing(track(1, "en", "")), track(2, "en", "Dialogue")];
assert_eq!(full_instead(&tracks), Some(2));
}
#[test]
fn a_forced_track_gives_way_to_an_unnamed_one_and_the_default_comes_first() {
let mut forced = playing(track(1, "eng", "English"));
forced.forced = true;
let mut default = track(3, "eng", "");
default.default = true;
let tracks = [forced, track(2, "eng", ""), default];
assert_eq!(full_instead(&tracks), Some(3));
}
#[test]
fn equal_candidates_resolve_to_the_first_in_the_file() {
let tracks = [
playing(track(1, "eng", "Songs")),
track(2, "eng", ""),
track(3, "eng", ""),
];
assert_eq!(full_instead(&tracks), Some(2));
}
#[test]
fn a_track_in_another_language_is_never_switched_to() {
let tracks = [
playing(track(1, "eng", "Signs")),
track(2, "ger", "Dialogue"),
];
assert_eq!(full_instead(&tracks), None);
}
#[test]
fn two_letter_and_three_letter_tags_are_one_language() {
let tracks = [playing(track(1, "en", "OP/ED")), track(2, "eng", "")];
assert_eq!(full_instead(&tracks), Some(2));
}
#[test]
fn the_dialogue_playing_stays_and_subtitles_off_stay_off() {
let tracks = [
track(1, "eng", "Signs & Songs"),
playing(track(2, "eng", "Dialogue")),
];
assert_eq!(full_instead(&tracks), None);
let off = [
track(1, "eng", "Signs & Songs"),
track(2, "eng", "Dialogue"),
];
assert_eq!(full_instead(&off), None);
}
#[test]
fn a_full_word_outweighs_a_partial_one_in_the_same_name() {
assert_eq!(
track(1, "eng", "Full (with Signs)").coverage(),
Coverage::Full
);
assert_eq!(track(1, "eng", "Signs/Songs").coverage(), Coverage::Partial);
assert_eq!(track(1, "eng", "English").coverage(), Coverage::Unknown);
// A word inside another is not that word.
assert_eq!(
track(1, "eng", "Designer cut").coverage(),
Coverage::Unknown
);
}
#[test]
fn a_partial_track_with_only_partial_company_stays() {
let tracks = [playing(track(1, "eng", "Signs")), track(2, "eng", "Songs")];
assert_eq!(full_instead(&tracks), None);
}
}
+1 -1
View File
@@ -52,4 +52,4 @@ pub use player::{
};
pub use registry::{PROTOCOL, StreamHandle, StreamRegistry};
pub use render::VideoRenderer;
pub use tracks::{LANGUAGES, Track, TrackKind, language_name, pick_track};
pub use tracks::{LANGUAGES, Track, TrackKind, full_subtitles_instead, language_name, pick_track};
+24
View File
@@ -12,6 +12,7 @@
//! an empty string.
use libmpv2::Mpv;
use pstr_core::subtitles::{SubtitleTrack, full_instead};
/// What a track carries.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -69,6 +70,9 @@ pub struct Track {
pub selected: bool,
/// Whether the container marks it as the default choice.
pub default: bool,
/// Whether the container marks it as forced: a subtitle track of the
/// on-screen text alone.
pub forced: bool,
/// True for a track added from outside the file — a sidecar subtitle. None
/// of ours are, yet.
pub external: bool,
@@ -152,6 +156,24 @@ pub fn pick_track(
)
}
/// The dialogue track to play in place of the subtitle track mpv chose, when
/// mpv chose signs and songs; see [`pstr_core::subtitles::full_instead`].
pub fn full_subtitles_instead(tracks: &[Track]) -> Option<i64> {
let subtitles: Vec<_> = tracks
.iter()
.filter(|track| track.kind == TrackKind::Subtitle)
.map(|track| SubtitleTrack {
id: track.id,
language: track.language.clone(),
title: track.title.clone(),
default: track.default,
forced: track.forced,
selected: track.selected,
})
.collect();
full_instead(&subtitles)
}
fn read_one(mpv: &Mpv, index: i64) -> Option<Track> {
let kind = TrackKind::from_mpv(&text(mpv, index, "type")?)?;
let id: i64 = mpv
@@ -167,6 +189,7 @@ fn read_one(mpv: &Mpv, index: i64) -> Option<Track> {
codec: text(mpv, index, "codec"),
selected: flag(mpv, index, "selected"),
default: flag(mpv, index, "default"),
forced: flag(mpv, index, "forced"),
external: flag(mpv, index, "external"),
})
}
@@ -200,6 +223,7 @@ mod tests {
codec: None,
selected: false,
default: false,
forced: false,
external: false,
}
}
+10 -1
View File
@@ -33,6 +33,10 @@ struct MpvTrack: Identifiable, Hashable {
let language: String
let title: String
let selected: Bool
/// The container marks it as the default choice.
var isDefault = false
/// The container marks it as forced: on-screen text only, by convention.
var forced = false
var label: String {
let parts = [language, title].filter { !$0.trimmingCharacters(in: .whitespaces).isEmpty }
@@ -91,6 +95,8 @@ final class MpvPlayer {
/// Called when the duration or the pause state changes: what the lock
/// screen shows, and whether the screen may sleep.
@ObservationIgnored var onStateChanged: (() -> Void)?
/// Called once per file, when mpv has loaded it and picked its tracks.
@ObservationIgnored var onLoaded: (() -> Void)?
@ObservationIgnored let layer = MpvMetalLayer()
@ObservationIgnored private var mpv: OpaquePointer?
@@ -473,6 +479,7 @@ final class MpvPlayer {
let ready = pendingSubtitles
pendingSubtitles = []
attach(ready)
onLoaded?()
}
case .restarted:
seeking = false
@@ -511,7 +518,9 @@ final class MpvPlayer {
type: getString(base + "type") ?? "",
language: getString(base + "lang") ?? "",
title: getString(base + "title") ?? "",
selected: getString(base + "selected") == "yes"
selected: getString(base + "selected") == "yes",
isDefault: getString(base + "default") == "yes",
forced: getString(base + "forced") == "yes"
)
}
}
+13
View File
@@ -256,6 +256,7 @@ final class PlayerHost {
player?.onEnded = { [weak self] reason, media in self?.ended(reason, media) }
player?.onTick = { [weak self] in self?.tick() }
player?.onStateChanged = { [weak self] in self?.publishNowPlaying() }
player?.onLoaded = { [weak self] in self?.preferFullSubtitles() }
}
guard let player else {
playbackError = "Playback is unavailable: libmpv did not start."
@@ -433,6 +434,18 @@ final class PlayerHost {
chapters = chapterPlan(chapters: player.chapters, duration: player.duration > 0 ? player.duration : nil)
}
/// Play the dialogue rather than the signs and songs mpv picked by
/// language (`pstr_core::subtitles`). Not remembered: it is the choice mpv
/// should have made, not the viewer's.
private func preferFullSubtitles() {
guard let player, playingOpenEpisode else { return }
let subtitles = player.tracks.filter { $0.type == "sub" }.map {
SubtitleTrackRecord(id: $0.id, language: $0.language, title: $0.title, default: $0.isDefault, forced: $0.forced, selected: $0.selected)
}
guard let id = fullSubtitlesInstead(tracks: subtitles), let track = player.tracks.first(where: { $0.type == "sub" && $0.id == id }) else { return }
player.select(type: "sub", track: track)
}
/// The skip button's offer at the current position, if any.
var offer: SkipOffer? {
guard let player, playingOpenEpisode else { return nil }