feat(android,ios): play the subtitle files beside an episode

The bridge's open_subtitles opens and publishes them and names the one to
turn on. The native player holds them until mpv has the episode loaded,
adds them with sub-add, and releases them instead when another episode
was loaded in the meantime.

android/app/src/main/cpp/pstr_mpv.cpp

@@ -27,6 +27,7 @@ extern "C" {
 #include <string_view>
 #include <thread>
 #include <utility>
+#include <vector>
 
 namespace {
 
@@ -212,6 +213,14 @@ struct PlaybackState {
     int64_t generation = 0;
 };
 
+/// A subtitle file published for mpv, and what to call it.
+struct Subtitle {
+    uint64_t handle = 0;
+    std::string title;
+    std::string language;
+    bool select = false;
+};
+
 class Player {
   public:
     Player() {
@@ -338,7 +347,10 @@ class Player {
             state_.generation = generation;
             state_.loading = true;
             error_.clear();
+            for (const auto &subtitle : pending_subtitles_) released_.push_back(subtitle.handle);
+            pending_subtitles_.clear();
         }
+        flush_released();
         const std::string url = "pstr://" + std::to_string(handle);
         const char *command[] = {"loadfile", url.c_str(), nullptr};
         if (mpv_command(mpv_, command) < 0) {
@@ -372,6 +384,27 @@ class Player {
         mpv_command(mpv_, command);
     }
 
+    /// A subtitle file beside the file of load `generation`, added once mpv
+    /// has that file loaded: an external track belongs to the file open at
+    /// the time. Handed over for a file already replaced, it is released
+    /// instead. `select` turns it on, but only when the file has no subtitle
+    /// of its own playing.
+    void add_subtitle(int64_t generation, Subtitle subtitle) {
+        bool now = false;
+        {
+            std::lock_guard lock(state_mutex_);
+            if (generation != state_.generation) {
+                released_.push_back(subtitle.handle);
+            } else if (loaded_generation_ == generation) {
+                now = true;
+            } else {
+                pending_subtitles_.push_back(std::move(subtitle));
+            }
+        }
+        flush_released();
+        if (now) attach_subtitle(subtitle);
+    }
+
     PlaybackState state() const {
         std::lock_guard lock(state_mutex_);
         return state_;
@@ -434,6 +467,35 @@ class Player {
     }
 
   private:
+    void attach_subtitle(const Subtitle &subtitle) {
+        bool showing = false;
+        if (char *sid = mpv_get_property_string(mpv_, "sid")) {
+            showing = std::strcmp(sid, "no") != 0;
+            mpv_free(sid);
+        }
+        const std::string url = "pstr://" + std::to_string(subtitle.handle);
+        std::vector<const char *> command{
+            "sub-add", url.c_str(), subtitle.select && !showing ? "select" : "auto",
+            subtitle.title.c_str()};
+        if (!subtitle.language.empty()) command.push_back(subtitle.language.c_str());
+        command.push_back(nullptr);
+        // On failure mpv may never have opened it, so nothing will close it.
+        // Releasing one it did open and close is a no-op.
+        if (mpv_command(mpv_, command.data()) < 0) pstr_android_stream_release(subtitle.handle);
+    }
+
+    /// Release the subtitle streams queued under `state_mutex_`, outside it:
+    /// a release runs Rust destructors, which must not run under a lock a
+    /// reader of the state may be waiting on.
+    void flush_released() {
+        std::vector<uint64_t> handles;
+        {
+            std::lock_guard lock(state_mutex_);
+            handles.swap(released_);
+        }
+        for (const uint64_t handle : handles) pstr_android_stream_release(handle);
+    }
+
     struct MpvFree {
         void operator()(char *value) const noexcept { if (value) mpv_free(value); }
     };
@@ -534,6 +596,16 @@ class Player {
             if (event->event_id == MPV_EVENT_FILE_LOADED) {
                 const double start = pending_start_.exchange(0.0);
                 if (start > 0.0) set_double("time-pos", start);
+                std::vector<Subtitle> ready;
+                {
+                    std::lock_guard lock(state_mutex_);
+                    // Mid-load, this is the outgoing file finishing its load.
+                    if (!state_.loading) {
+                        loaded_generation_ = state_.generation;
+                        ready.swap(pending_subtitles_);
+                    }
+                }
+                for (const auto &subtitle : ready) attach_subtitle(subtitle);
             }
             if (event->event_id == MPV_EVENT_END_FILE) {
                 const auto *end = static_cast<mpv_event_end_file *>(event->data);
@@ -757,6 +829,12 @@ class Player {
     bool surface_changed_ = false;
     std::atomic<bool> frame_ready_{false};
     std::atomic<double> pending_start_{0.0};
+    /// The load generation whose file mpv has loaded, under `state_mutex_`.
+    int64_t loaded_generation_ = -1;
+    /// Subtitles handed over before their file loaded, under `state_mutex_`.
+    std::vector<Subtitle> pending_subtitles_;
+    /// Streams to release once `state_mutex_` is let go.
+    std::vector<uint64_t> released_;
 };
 
 Player *from(jlong handle) { return reinterpret_cast<Player *>(handle); }
@@ -850,6 +928,17 @@ extern "C" JNIEXPORT jboolean JNICALL Java_io_narl_protonstream_playback_NativeM
                 : JNI_FALSE);
     });
 }
+extern "C" JNIEXPORT void JNICALL Java_io_narl_protonstream_playback_NativeMpvHost_nativeAddSubtitle(JNIEnv *env, jobject, jlong handle, jlong generation, jlong stream, jstring title, jstring language, jboolean select) noexcept {
+    c_boundary([=] {
+        Subtitle subtitle{static_cast<uint64_t>(stream), {}, {}, select == JNI_TRUE};
+        auto *p = from(handle);
+        if (!p || !utf8(env, title, subtitle.title) || !utf8(env, language, subtitle.language)) {
+            pstr_android_stream_release(subtitle.handle);
+            return;
+        }
+        p->add_subtitle(generation, std::move(subtitle));
+    });
+}
 extern "C" JNIEXPORT void JNICALL Java_io_narl_protonstream_playback_NativeMpvHost_nativePause(JNIEnv *, jobject, jlong h, jboolean value) noexcept {
     c_boundary([=] { if (auto *p = from(h)) p->pause(value == JNI_TRUE); });
 }

android/app/src/main/java/io/narl/protonstream/playback/NativeMpvHost.kt

@@ -19,6 +19,7 @@ import java.util.concurrent.locks.ReentrantLock
 import kotlin.concurrent.withLock
 import uniffi.pstr_android.ChapterPlan
 import uniffi.pstr_android.ChapterRecord
+import uniffi.pstr_android.SubtitleFilesRecord
 import uniffi.pstr_android.chapterPlan
 
 /**
@@ -292,6 +293,30 @@ class NativeMpvHost private constructor(private val nativeHandle: Long) : Libmpv
 
     override fun releaseNativeStream(nativeHandle: ULong) = pstrAndroidStreamRelease(nativeHandle.toLong())
 
+    /**
+     * Hand mpv the subtitle files beside [media], the episode [play] loaded.
+     *
+     * Native code holds them until mpv has the file loaded, since an external
+     * track belongs to the file open at the time, and releases them instead
+     * if another episode has been loaded since. [SubtitleFilesRecord.preferred]
+     * is turned on only when the file has no subtitle of its own playing;
+     * pass [subtitles] false to turn none on.
+     */
+    fun addSubtitles(media: String, files: SubtitleFilesRecord, subtitles: Boolean) {
+        val generation = currentGeneration.toLong()
+        val added = withOpenHandle { handle ->
+            files.files.forEachIndexed { index, file ->
+                if (currentMedia == media) {
+                    val select = subtitles && files.preferred == index.toUInt()
+                    nativeAddSubtitle(handle, generation, file.handle.toLong(), file.title, file.language, select)
+                } else {
+                    pstrAndroidStreamRelease(file.handle.toLong())
+                }
+            }
+        }
+        if (added == null) files.files.forEach { pstrAndroidStreamRelease(it.handle.toLong()) }
+    }
+
     private fun startPolling() {
         if (poller?.isActive == true) return
         // Off the main thread deliberately: `nativeState` and `nativeTracks`
@@ -414,6 +439,14 @@ class NativeMpvHost private constructor(private val nativeHandle: Long) : Libmpv
         subtitles: Boolean,
         hardwareDecoding: Boolean,
     ): Boolean
+    private external fun nativeAddSubtitle(
+        handle: Long,
+        generation: Long,
+        stream: Long,
+        title: String,
+        language: String?,
+        select: Boolean,
+    )
     private external fun nativePause(handle: Long, paused: Boolean)
     private external fun nativeSeek(handle: Long, position: Double)
     private external fun nativeVolume(handle: Long, volume: Double)

android/app/src/main/java/io/narl/protonstream/playback/PlayerScreen.kt

@@ -250,11 +250,19 @@ fun PlayerScreen(
                 startup.audioLanguage, startup.subtitleLanguage, startup.subtitles,
                 SettingsStore(context).hardwareDecoding,
             )
+
             // After the load, because mpv keeps none of the three across one.
             nativeHost.setVolume(startup.volume)
             nativeHost.setMuted(startup.muted)
             nativeHost.setSpeed(startup.speed)
             session.transferToPlayer()
+            // While mpv opens the episode, and off the spinner's path: the host
+            // holds them until it has the file.
+            launch(Dispatchers.IO) {
+                val files = NativeRuntime.engine()
+                    .openSubtitles(episode.shareId, episode.linkId, startup.subtitleLanguage)
+                nativeHost.addSubtitles(key, files, startup.subtitles)
+            }
         }.onFailure { playbackError = it.message ?: "Playback failed" }
         opening = false
     }

crates/pstr-android/src/lib.rs

@@ -13,7 +13,9 @@ use parking_lot::Mutex;
 use pstr_core::account::{DriveEntry, PendingSignIn, PlaceKind, SignIn};
 use pstr_core::appearance::{Accent, Appearance, Flavor, Palette};
 use pstr_core::browse::{Filter, Sort};
-use pstr_core::catalog::{Catalog, OfflineFile, TitleTrackPrefs, WatchState, build_rows};
+use pstr_core::catalog::{
+    Catalog, OfflineFile, TitleTrackPrefs, WatchState, build_rows, build_sidecars,
+};
 use pstr_core::chapters::{Chapter, ChapterRole};
 use pstr_core::config::AppDirs;
 use pstr_core::library::{Episode, Library, Title, TitleKind};
@@ -102,6 +104,26 @@ pub struct AndroidPaths {
     pub cache: String,
 }
 
+/// A subtitle file beside an episode, opened and published for the native
+/// player like the episode itself. The player releases `handle` when mpv
+/// closes it; one never handed to mpv goes back through
+/// `pstr_android_stream_release`.
+#[derive(Debug, Clone, uniffi::Record)]
+pub struct SubtitleFileRecord {
+    pub handle: u64,
+    /// What else the file name says — "forced", "SDH" — or "External".
+    pub title: String,
+    pub language: Option<String>,
+}
+
+/// An episode's subtitle files, and which to turn on when the episode has
+/// none of its own playing.
+#[derive(Debug, Clone, Default, uniffi::Record)]
+pub struct SubtitleFilesRecord {
+    pub files: Vec<SubtitleFileRecord>,
+    pub preferred: Option<u32>,
+}
+
 /// Android implements this with an AES-GCM key held by Android Keystore.
 #[uniffi::export(callback_interface)]
 pub trait AndroidSecretStore: Send + Sync {
@@ -1972,6 +1994,83 @@ impl AndroidEngine {
         .map_err(BridgeError::from_display)?
     }
 
+    /// Open the subtitle files paired with an episode, for the native player
+    /// to add once it has the episode loaded. `language` is the subtitle
+    /// language in effect for it, which picks [`SubtitleFilesRecord::preferred`].
+    ///
+    /// Through the session already open: a downloaded episode played without
+    /// one goes without its subtitles rather than waiting on a handshake. One
+    /// that fails to open is left out; the episode plays either way.
+    pub async fn open_subtitles(
+        self: Arc<Self>,
+        share_id: String,
+        link_id: String,
+        language: Option<String>,
+    ) -> SubtitleFilesRecord {
+        let runtime = Arc::clone(&self.runtime);
+        let opened = spawned(&runtime, async move {
+            let sidecars = self
+                .catalog
+                .lock()
+                .sidecars(&share_id, &link_id)
+                .unwrap_or_default();
+            if sidecars.is_empty() {
+                return SubtitleFilesRecord::default();
+            }
+            let Some(connection) = self.cached_connection() else {
+                return SubtitleFilesRecord::default();
+            };
+            let (engine, connection) = (&self, &connection);
+            let opening = sidecars.into_iter().map(|sidecar| async move {
+                let uid = node_uid(&sidecar.volume_id, &sidecar.link_id);
+                match engine
+                    .open_from_source(connection, &sidecar.share_id, &uid)
+                    .await
+                {
+                    Ok(stream) => {
+                        stream.warm_edges();
+                        Some((sidecar, stream))
+                    }
+                    Err(error) => {
+                        log::debug!("opening subtitle {} failed: {error}", sidecar.name);
+                        None
+                    }
+                }
+            });
+            let opened: Vec<_> = futures::future::join_all(opening)
+                .await
+                .into_iter()
+                .flatten()
+                .collect();
+            let sidecars: Vec<_> = opened.iter().map(|(sidecar, _)| sidecar.clone()).collect();
+            let preferred = pstr_core::catalog::preferred_sidecar(&sidecars, language.as_deref())
+                .and_then(|index| u32::try_from(index).ok());
+            let files = opened
+                .into_iter()
+                .map(|(sidecar, stream)| {
+                    let handle = Arc::new(AndroidStream {
+                        runtime: Arc::clone(&self.runtime),
+                        stream,
+                        cancel: tokio::sync::Notify::new(),
+                        native_id: Mutex::new(None),
+                    })
+                    .native_handle();
+                    SubtitleFileRecord {
+                        handle,
+                        title: sidecar.title.unwrap_or_else(|| "External".to_owned()),
+                        language: sidecar.language,
+                    }
+                })
+                .collect();
+            SubtitleFilesRecord { files, preferred }
+        })
+        .await;
+        opened.unwrap_or_else(|error| {
+            log::debug!("opening subtitles stopped: {error}");
+            SubtitleFilesRecord::default()
+        })
+    }
+
     /// Fetch the blocks an episode opens with into the cache, without playing
     /// it: the next episode, near the end of this one, so that autoplay or
     /// "Play now" starts from the cache rather than from two round trips.
@@ -2645,10 +2744,14 @@ impl AndroidEngine {
                 }
             };
             let rows = build_rows(&id, &nodes);
-            self.catalog
-                .lock()
-                .replace_share(&id, &rows)
-                .map_err(BridgeError::from_display)?;
+            let sidecars = build_sidecars(&id, &nodes);
+            {
+                let mut catalog = self.catalog.lock();
+                catalog
+                    .replace_share(&id, &rows)
+                    .and_then(|()| catalog.replace_sidecars(&id, &sidecars))
+                    .map_err(BridgeError::from_display)?;
+            }
             let after = self
                 .catalog
                 .lock()

ios/ProtonStream/Player/MpvPlayer.swift

@@ -99,6 +99,11 @@ final class MpvPlayer {
     /// `END_FILE(stop)` that stopping it emits.
     @ObservationIgnored private var loading = false
     @ObservationIgnored private var pendingMedia: String?
+    /// The load generation whose file mpv has loaded.
+    @ObservationIgnored private var loadedGeneration = -1
+    /// Subtitle files handed over before their file loaded, and whether each
+    /// is the one to turn on.
+    @ObservationIgnored private var pendingSubtitles: [(SubtitleFileRecord, Bool)] = []
 
     /// A core, or nil when libmpv would not start.
     static func create() -> MpvPlayer? {
@@ -209,6 +214,9 @@ final class MpvPlayer {
         problem = nil
         self.media = nil
         generation += 1
+        // Subtitles for the file being replaced will never be added.
+        pendingSubtitles.forEach { pstr_android_stream_release($0.0.handle) }
+        pendingSubtitles = []
         pendingMedia = media
         loading = true
         if command(["loadfile", "pstr://\(handle)", "replace"]) < 0 {
@@ -219,6 +227,46 @@ final class MpvPlayer {
         return true
     }
 
+    /// Add the subtitle files beside the file of load `generation`.
+    ///
+    /// Held until mpv has that file loaded, since an external track belongs to
+    /// the file open at the time, and released instead if another file has
+    /// been loaded since. `files.preferred` is turned on only when the file
+    /// has no subtitle of its own playing; `subtitles` false turns none on.
+    func addSubtitles(generation: Int, files: SubtitleFilesRecord, subtitles: Bool) {
+        let marked = files.files.enumerated().map { index, file in
+            (file, subtitles && files.preferred == UInt32(index))
+        }
+        if generation != self.generation {
+            marked.forEach { pstr_android_stream_release($0.0.handle) }
+        } else if loadedGeneration == generation {
+            attach(marked)
+        } else {
+            pendingSubtitles += marked
+        }
+    }
+
+    /// `sub-add` off the main thread: mpv reads the whole file before it
+    /// returns, through the stream protocol and so from the network.
+    private func attach(_ subtitles: [(SubtitleFileRecord, Bool)]) {
+        guard let mpv, !subtitles.isEmpty else { return }
+        let showing = getString("sid").map { $0 != "no" } ?? false
+        let core = UncheckedCore(mpv)
+        DispatchQueue.global(qos: .userInitiated).async {
+            for (file, select) in subtitles {
+                var args = ["sub-add", "pstr://\(file.handle)", select && !showing ? "select" : "auto", file.title]
+                if let language = file.language { args.append(language) }
+                var pointers = args.map { UnsafePointer<CChar>(strdup($0)) }
+                pointers.append(nil)
+                let status = mpv_command(core.pointer, &pointers)
+                pointers.forEach { free(UnsafeMutablePointer(mutating: $0)) }
+                // On failure mpv may never have opened it, so nothing will
+                // close it. Releasing one it did open and close is a no-op.
+                if status < 0 { pstr_android_stream_release(file.handle) }
+            }
+        }
+    }
+
     /// Written through rather than left to the observer: a tap that only shows
     /// its effect on mpv's next property event reads as a button that missed.
     func setPaused(_ value: Bool) {
@@ -370,6 +418,13 @@ final class MpvPlayer {
         case .loaded:
             refreshTracks()
             refreshChapters()
+            // Mid-load, this is the outgoing file finishing its load.
+            if !loading {
+                loadedGeneration = generation
+                let ready = pendingSubtitles
+                pendingSubtitles = []
+                attach(ready)
+            }
         case .restarted:
             seeking = false
         case let .ended(reason):
@@ -464,3 +519,11 @@ final class MpvPlayer {
         return String(cString: raw)
     }
 }
+
+/// mpv's handle, for a command issued off the main thread. libmpv's client API
+/// is thread-safe, and the core is only destroyed when it shuts down, which
+/// nothing in the app asks it to.
+private struct UncheckedCore: @unchecked Sendable {
+    let pointer: OpaquePointer
+    init(_ pointer: OpaquePointer) { self.pointer = pointer }
+}

ios/ProtonStream/Player/PlayerHost.swift

@@ -305,6 +305,16 @@ final class PlayerHost {
                 player.setMuted(global.muted)
                 player.setSpeed(global.speed)
                 player.setPaused(false)
+                // While mpv opens the episode; the player holds them until it has.
+                let generation = player.generation
+                let subtitleLanguage = show?.subtitleLanguage ?? global.subtitleLanguage
+                let subtitles = show?.subtitles ?? global.subtitles
+                Task {
+                    guard let files = try? await NativeRuntime.engine().openSubtitles(
+                        shareId: episode.shareId, linkId: episode.linkId, language: subtitleLanguage
+                    ) else { return }
+                    player.addSubtitles(generation: generation, files: files, subtitles: subtitles)
+                }
                 startSaving()
             } catch {
                 if Task.isCancelled || self.session !== session {