initial commit

.envrc

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# the shebang is ignored, but nice for editors
+
+if type -P lorri &>/dev/null; then
+  eval "$(lorri direnv)"
+else
+  echo 'while direnv evaluated .envrc, could not find the command "lorri" [https://github.com/nix-community/lorri]'
+  use nix
+fi

.gitignore

@@ -0,0 +1 @@
+/target

Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "cushy_video"
+version = "0.1.0"
+edition = "2021"
+exclude = ["media"]
+
+[dependencies]
+cushy = { git = "https://github.com/khonsulabs/cushy.git", branch = "main", features = [
+    "tokio",
+    "tokio-multi-thread",
+] }
+gstreamer = "0.23" # video decoder
+gstreamer-app = "0.23" # appsink
+gstreamer-base = "0.23" # basesrc
+glib = "0.20" # gobject traits and error type
+url = "2" # URL parsing
+thiserror = "1" # error handling
+html-escape = "0.2.13" # subtitle unescaping
+log = "0.4"

README.md

@@ -0,0 +1,3 @@
+# Cushy video player
+
+A [cushy](https://github.com/khonsulabs/cushy) port of [iced_video_player](https://github.com/jazzfool/iced_video_player).

examples/base.rs

@@ -0,0 +1,23 @@
+use cushy::{widget::MakeWidget, Run};
+use cushy_video::{player::VideoPlayer, video::Video};
+
+fn main() -> cushy::Result {
+    VideoPlayer::new(
+        Video::new(
+            &url::Url::from_file_path(
+                std::path::PathBuf::from(file!())
+                    .parent()
+                    .unwrap()
+                    .join("../media/big-buck-bunny-480p-30sec.mp4")
+                    .canonicalize()
+                    .unwrap(),
+            )
+            .unwrap(),
+        )
+        .unwrap(),
+    )
+    .contain()
+    .pad()
+    .expand()
+    .run()
+}

shell.nix

@@ -0,0 +1,45 @@
+{ pkgs ? import <nixpkgs> {} }:
+let
+    # rust-rover things
+    fenix = import (fetchTarball "https://github.com/nix-community/fenix/archive/main.tar.gz") { };
+    rust-toolchain =
+        fenix.default.toolchain;
+in
+pkgs.mkShell rec {
+    buildInputs = with pkgs;[
+        openssl
+        pkg-config
+        cmake
+        zlib
+        rust-toolchain
+
+        dbus
+
+        # common glutin
+        libxkbcommon
+        libGL
+
+        # video
+        glib
+        gst_all_1.gstreamer
+        gst_all_1.gst-plugins-base
+
+        # winit wayland
+        wayland
+
+        # winit x11
+        xorg.libXcursor
+        xorg.libXrandr
+        xorg.libXi
+        xorg.libX11
+    ];
+    nativeBuildInputs = with pkgs; [
+        pkg-config
+        fontconfig
+    ];
+    LD_LIBRARY_PATH = "${pkgs.lib.makeLibraryPath buildInputs}";
+    OPENSSL_DIR="${pkgs.openssl.dev}";
+    OPENSSL_LIB_DIR="${pkgs.openssl.out}/lib";
+    RUST_SRC_PATH = "${pkgs.rust.packages.stable.rustPlatform.rustLibSrc}";
+    GST_PLUGIN_PATH = "${pkgs.gst_all_1.gstreamer}:${pkgs.gst_all_1.gst-plugins-bad}:${pkgs.gst_all_1.gst-plugins-ugly}:${pkgs.gst_all_1.gst-plugins-good}:${pkgs.gst_all_1.gst-plugins-base}";
+}

src/lib.rs

@@ -0,0 +1,36 @@
+use gstreamer as gst;
+use thiserror::Error;
+
+pub mod pipeline;
+pub mod player;
+pub mod video;
+
+#[derive(Debug, Error)]
+pub enum Error {
+    #[error("{0}")]
+    Glib(#[from] glib::Error),
+    #[error("{0}")]
+    Bool(#[from] glib::BoolError),
+    #[error("failed to get the gstreamer bus")]
+    Bus,
+    #[error("failed to get AppSink element with name='{0}' from gstreamer pipeline")]
+    AppSink(String),
+    #[error("{0}")]
+    StateChange(#[from] gst::StateChangeError),
+    #[error("failed to cast gstreamer element")]
+    Cast,
+    #[error("{0}")]
+    Io(#[from] std::io::Error),
+    #[error("invalid URI")]
+    Uri,
+    #[error("failed to get media capabilities")]
+    Caps,
+    #[error("failed to query media duration or position")]
+    Duration,
+    #[error("failed to sync with playback")]
+    Sync,
+    #[error("failed to lock internal sync primitive")]
+    Lock,
+    #[error("invalid framerate: {0}")]
+    Framerate(f64),
+}

src/pipeline.rs

@@ -0,0 +1,440 @@
+use std::{
+    collections::{btree_map::Entry, BTreeMap},
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc, Mutex,
+    },
+};
+
+use cushy::{
+    figures::{units::UPx, Rect},
+    kludgine::{self, wgpu},
+    RenderOperation,
+};
+
+#[repr(C)]
+struct Uniforms {
+    rect: [f32; 4],
+}
+
+struct VideoEntry {
+    texture_y: wgpu::Texture,
+    texture_uv: wgpu::Texture,
+    uniforms: wgpu::Buffer,
+    bg0: wgpu::BindGroup,
+    alive: Arc<AtomicBool>,
+}
+
+struct VideoPipeline {
+    pipeline: wgpu::RenderPipeline,
+    bg0_layout: wgpu::BindGroupLayout,
+    sampler: wgpu::Sampler,
+    videos: BTreeMap<u64, VideoEntry>,
+}
+
+impl VideoPipeline {
+    fn new(graphics: &mut kludgine::Graphics<'_>) -> Self {
+        let device = graphics.device();
+        let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
+            label: Some("iced_video_player shader"),
+            source: wgpu::ShaderSource::Wgsl(include_str!("shader.wgsl").into()),
+        });
+
+        let bg0_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
+            label: Some("iced_video_player bind group 0 layout"),
+            entries: &[
+                wgpu::BindGroupLayoutEntry {
+                    binding: 0,
+                    visibility: wgpu::ShaderStages::FRAGMENT,
+                    ty: wgpu::BindingType::Texture {
+                        sample_type: wgpu::TextureSampleType::Float { filterable: true },
+                        view_dimension: wgpu::TextureViewDimension::D2,
+                        multisampled: false,
+                    },
+                    count: None,
+                },
+                wgpu::BindGroupLayoutEntry {
+                    binding: 1,
+                    visibility: wgpu::ShaderStages::FRAGMENT,
+                    ty: wgpu::BindingType::Texture {
+                        sample_type: wgpu::TextureSampleType::Float { filterable: true },
+                        view_dimension: wgpu::TextureViewDimension::D2,
+                        multisampled: false,
+                    },
+                    count: None,
+                },
+                wgpu::BindGroupLayoutEntry {
+                    binding: 2,
+                    visibility: wgpu::ShaderStages::FRAGMENT,
+                    ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering),
+                    count: None,
+                },
+                wgpu::BindGroupLayoutEntry {
+                    binding: 3,
+                    visibility: wgpu::ShaderStages::VERTEX,
+                    ty: wgpu::BindingType::Buffer {
+                        ty: wgpu::BufferBindingType::Uniform,
+                        has_dynamic_offset: false,
+                        min_binding_size: None,
+                    },
+                    count: None,
+                },
+            ],
+        });
+
+        let layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
+            label: Some("iced_video_player pipeline layout"),
+            bind_group_layouts: &[&bg0_layout],
+            push_constant_ranges: &[],
+        });
+
+        let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
+            label: Some("iced_video_player pipeline"),
+            layout: Some(&layout),
+            vertex: wgpu::VertexState {
+                module: &shader,
+                entry_point: Some("vs_main"),
+                buffers: &[],
+                compilation_options: Default::default(),
+            },
+            primitive: wgpu::PrimitiveState::default(),
+            depth_stencil: None,
+            multisample: graphics.multisample_state(),
+            fragment: Some(wgpu::FragmentState {
+                module: &shader,
+                entry_point: Some("fs_main"),
+                targets: &[Some(wgpu::ColorTargetState {
+                    format: graphics.texture_format(),
+                    blend: None,
+                    write_mask: wgpu::ColorWrites::ALL,
+                })],
+                compilation_options: Default::default(),
+            }),
+            multiview: None,
+            cache: None,
+        });
+
+        let sampler = device.create_sampler(&wgpu::SamplerDescriptor {
+            label: Some("iced_video_player sampler"),
+            address_mode_u: wgpu::AddressMode::ClampToEdge,
+            address_mode_v: wgpu::AddressMode::ClampToEdge,
+            address_mode_w: wgpu::AddressMode::ClampToEdge,
+            mag_filter: wgpu::FilterMode::Linear,
+            min_filter: wgpu::FilterMode::Linear,
+            mipmap_filter: wgpu::FilterMode::Nearest,
+            lod_min_clamp: 0.0,
+            lod_max_clamp: 1.0,
+            compare: None,
+            anisotropy_clamp: 1,
+            border_color: None,
+        });
+
+        VideoPipeline {
+            pipeline,
+            bg0_layout,
+            sampler,
+            videos: BTreeMap::new(),
+        }
+    }
+
+    fn upload(
+        &mut self,
+        device: &wgpu::Device,
+        queue: &wgpu::Queue,
+        video_id: u64,
+        alive: &Arc<AtomicBool>,
+        (width, height): (u32, u32),
+        frame: &[u8],
+    ) {
+        if let Entry::Vacant(entry) = self.videos.entry(video_id) {
+            let texture_y = device.create_texture(&wgpu::TextureDescriptor {
+                label: Some("iced_video_player texture"),
+                size: wgpu::Extent3d {
+                    width,
+                    height,
+                    depth_or_array_layers: 1,
+                },
+                mip_level_count: 1,
+                sample_count: 1,
+                dimension: wgpu::TextureDimension::D2,
+                format: wgpu::TextureFormat::R8Unorm,
+                usage: wgpu::TextureUsages::COPY_DST | wgpu::TextureUsages::TEXTURE_BINDING,
+                view_formats: &[],
+            });
+
+            let texture_uv = device.create_texture(&wgpu::TextureDescriptor {
+                label: Some("iced_video_player texture"),
+                size: wgpu::Extent3d {
+                    width: width / 2,
+                    height: height / 2,
+                    depth_or_array_layers: 1,
+                },
+                mip_level_count: 1,
+                sample_count: 1,
+                dimension: wgpu::TextureDimension::D2,
+                format: wgpu::TextureFormat::Rg8Unorm,
+                usage: wgpu::TextureUsages::COPY_DST | wgpu::TextureUsages::TEXTURE_BINDING,
+                view_formats: &[],
+            });
+
+            let view_y = texture_y.create_view(&wgpu::TextureViewDescriptor {
+                label: Some("iced_video_player texture view"),
+                format: None,
+                dimension: None,
+                aspect: wgpu::TextureAspect::All,
+                base_mip_level: 0,
+                mip_level_count: None,
+                base_array_layer: 0,
+                array_layer_count: None,
+            });
+
+            let view_uv = texture_uv.create_view(&wgpu::TextureViewDescriptor {
+                label: Some("iced_video_player texture view"),
+                format: None,
+                dimension: None,
+                aspect: wgpu::TextureAspect::All,
+                base_mip_level: 0,
+                mip_level_count: None,
+                base_array_layer: 0,
+                array_layer_count: None,
+            });
+
+            let buffer = device.create_buffer(&wgpu::BufferDescriptor {
+                label: Some("iced_video_player uniform buffer"),
+                size: std::mem::size_of::<Uniforms>() as _,
+                usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::UNIFORM,
+                mapped_at_creation: false,
+            });
+
+            let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
+                label: Some("iced_video_player bind group"),
+                layout: &self.bg0_layout,
+                entries: &[
+                    wgpu::BindGroupEntry {
+                        binding: 0,
+                        resource: wgpu::BindingResource::TextureView(&view_y),
+                    },
+                    wgpu::BindGroupEntry {
+                        binding: 1,
+                        resource: wgpu::BindingResource::TextureView(&view_uv),
+                    },
+                    wgpu::BindGroupEntry {
+                        binding: 2,
+                        resource: wgpu::BindingResource::Sampler(&self.sampler),
+                    },
+                    wgpu::BindGroupEntry {
+                        binding: 3,
+                        resource: wgpu::BindingResource::Buffer(wgpu::BufferBinding {
+                            buffer: &buffer,
+                            offset: 0,
+                            size: None,
+                        }),
+                    },
+                ],
+            });
+
+            entry.insert(VideoEntry {
+                texture_y,
+                texture_uv,
+                uniforms: buffer,
+                bg0: bind_group,
+                alive: Arc::clone(alive),
+            });
+        }
+
+        let VideoEntry {
+            texture_y,
+            texture_uv,
+            ..
+        } = self.videos.get(&video_id).unwrap();
+
+        queue.write_texture(
+            wgpu::ImageCopyTexture {
+                texture: texture_y,
+                mip_level: 0,
+                origin: wgpu::Origin3d::ZERO,
+                aspect: wgpu::TextureAspect::All,
+            },
+            &frame[..(width * height) as usize],
+            wgpu::ImageDataLayout {
+                offset: 0,
+                bytes_per_row: Some(width),
+                rows_per_image: Some(height),
+            },
+            wgpu::Extent3d {
+                width,
+                height,
+                depth_or_array_layers: 1,
+            },
+        );
+
+        queue.write_texture(
+            wgpu::ImageCopyTexture {
+                texture: texture_uv,
+                mip_level: 0,
+                origin: wgpu::Origin3d::ZERO,
+                aspect: wgpu::TextureAspect::All,
+            },
+            &frame[(width * height) as usize..],
+            wgpu::ImageDataLayout {
+                offset: 0,
+                bytes_per_row: Some(width),
+                rows_per_image: Some(height / 2),
+            },
+            wgpu::Extent3d {
+                width: width / 2,
+                height: height / 2,
+                depth_or_array_layers: 1,
+            },
+        );
+    }
+
+    fn cleanup(&mut self) {
+        let ids: Vec<_> = self
+            .videos
+            .iter()
+            .filter_map(|(id, entry)| (!entry.alive.load(Ordering::SeqCst)).then_some(*id))
+            .collect();
+        for id in ids {
+            if let Some(video) = self.videos.remove(&id) {
+                video.texture_y.destroy();
+                video.texture_uv.destroy();
+                video.uniforms.destroy();
+            }
+        }
+    }
+
+    fn prepare(&mut self, queue: &wgpu::Queue, video_id: u64, bounds: Rect<UPx>) {
+        if let Some(video) = self.videos.get(&video_id) {
+            let uniforms = Uniforms {
+                rect: [
+                    bounds.origin.x.into(),
+                    bounds.origin.y.into(),
+                    (bounds.origin.x + bounds.size.width).into(),
+                    (bounds.origin.y + bounds.size.height).into(),
+                ],
+            };
+            queue.write_buffer(&video.uniforms, 0, unsafe {
+                std::slice::from_raw_parts(
+                    &uniforms as *const _ as *const u8,
+                    std::mem::size_of::<Uniforms>(),
+                )
+            });
+        }
+
+        self.cleanup();
+    }
+
+    fn draw(&self, pass: &mut wgpu::RenderPass, viewport: Rect<UPx>, video_id: u64) {
+        if let Some(video) = self.videos.get(&video_id) {
+            // let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
+            //     label: Some("iced_video_player render pass"),
+            //     color_attachments: &[Some(wgpu::RenderPassColorAttachment {
+            //         view: target,
+            //         resolve_target: None,
+            //         ops: wgpu::Operations {
+            //             load: wgpu::LoadOp::Load,
+            //             store: wgpu::StoreOp::Store,
+            //         },
+            //     })],
+            //     depth_stencil_attachment: None,
+            //     timestamp_writes: None,
+            //     occlusion_query_set: None,
+            // });
+
+            pass.set_pipeline(&self.pipeline);
+            pass.set_bind_group(0, &video.bg0, &[]);
+            // pass.set_viewport(
+            //     viewport.origin.x as _,
+            //     viewport.origin.y as _,
+            //     viewport.size.width as _,
+            //     viewport.size.height as _,
+            //     0.0,
+            //     1.0,
+            // );
+            pass.draw(0..4, 0..1);
+        }
+    }
+}
+
+pub(crate) struct VideoRO {
+    pipeline: Option<VideoPipeline>,
+}
+
+impl RenderOperation for VideoRO {
+    type DrawInfo = VideoPrimitive;
+    type Prepared = VideoPrimitive;
+
+    fn new(graphics: &mut cushy::kludgine::Graphics<'_>) -> Self {
+        VideoRO { pipeline: None }
+    }
+
+    fn prepare(
+        &mut self,
+        context: Self::DrawInfo,
+        origin: cushy::figures::Point<cushy::figures::units::Px>,
+        graphics: &mut cushy::kludgine::Graphics<'_>,
+    ) -> Self::Prepared {
+        let pipeline = self
+            .pipeline
+            .get_or_insert_with(|| VideoPipeline::new(graphics));
+
+        if context.upload_frame {
+            pipeline.upload(
+                graphics.device(),
+                graphics.queue(),
+                context.video_id,
+                &context.alive,
+                context.size,
+                context.frame.lock().expect("lock frame mutex").as_slice(),
+            );
+        }
+
+        pipeline.prepare(graphics.queue(), context.video_id, graphics.clip_rect());
+        context
+    }
+
+    fn render(
+        &self,
+        prepared: &Self::Prepared,
+        origin: cushy::figures::Point<cushy::figures::units::Px>,
+        opacity: f32,
+        graphics: &mut cushy::kludgine::RenderingGraphics<'_, '_>,
+    ) {
+        let pipeline = self.pipeline.as_ref().expect("prepare sets pipeline");
+        let rect = graphics.clip_rect();
+        pipeline.draw(
+            // target,
+            graphics.pass_mut(),
+            rect,
+            prepared.video_id,
+        );
+    }
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct VideoPrimitive {
+    video_id: u64,
+    alive: Arc<AtomicBool>,
+    frame: Arc<Mutex<Vec<u8>>>,
+    size: (u32, u32),
+    upload_frame: bool,
+}
+
+impl VideoPrimitive {
+    pub fn new(
+        video_id: u64,
+        alive: Arc<AtomicBool>,
+        frame: Arc<Mutex<Vec<u8>>>,
+        size: (u32, u32),
+        upload_frame: bool,
+    ) -> Self {
+        VideoPrimitive {
+            video_id,
+            alive,
+            frame,
+            size,
+            upload_frame,
+        }
+    }
+}

src/player.rs

@@ -0,0 +1,100 @@
+use std::{
+    sync::{atomic::Ordering, Arc},
+    time::Instant,
+};
+
+use cushy::{
+    context::{GraphicsContext, LayoutContext},
+    figures::{units::UPx, Size},
+    value::{Destination, Dynamic, DynamicReader, Generation, Source},
+    widget::Widget,
+    ConstraintLimit,
+};
+
+use crate::{
+    pipeline::{VideoPrimitive, VideoRO},
+    video::Video,
+    Error,
+};
+
+/// A video player widget, with no builtin controls.
+/// Autoplays by default.
+/// Supports subtitles, but doesn't render them - see [VideoPlayer::get_subtitles]
+#[derive(Debug)]
+pub struct VideoPlayer {
+    video: Video,
+    subtitles: Dynamic<Option<String>>,
+    frame: Dynamic<()>,
+    last_frame: Generation,
+}
+
+impl VideoPlayer {
+    pub fn new(video: Video) -> Self {
+        let subtitles = video.0.read().unwrap().subtitles.clone();
+        let frame = video.0.read().unwrap().upload_frame.clone();
+        Self {
+            subtitles,
+            video,
+            last_frame: Generation::default(),
+            frame,
+        }
+    }
+
+    pub fn from_url(url: &url::Url) -> Result<Self, Error> {
+        Ok(Self::new(Video::new(url)?))
+    }
+
+    /// Returns a dynamic source that can be used to get the subtitles, if present.
+    /// Currently, HTML entities are unescaped, but no other processing is done. No rich text support.
+    #[must_use]
+    pub fn get_subtitles(&self) -> DynamicReader<Option<String>> {
+        self.subtitles.clone().into_reader()
+    }
+
+    /// Gets a dynamic reader that can be used to listen to frame changes.
+    #[must_use]
+    pub fn on_frame(&self) -> DynamicReader<()> {
+        self.frame.clone().into_reader()
+    }
+
+    /// Gets a read handle on the inner Video object. Can be used to control playback and get metadata.
+    pub fn video(&self) -> &Video {
+        &self.video
+    }
+}
+
+impl Widget for VideoPlayer {
+    fn redraw(&mut self, context: &mut GraphicsContext<'_, '_, '_, '_>) {
+        let mut inner = self.video.write();
+        let frame = inner.upload_frame.generation();
+        let _ = inner.upload_frame.get_tracking_redraw(context); // no data here, just to trigger redraw
+
+        let upload_frame = frame != self.last_frame;
+        self.last_frame = frame;
+
+        if upload_frame {
+            let last_frame_time = inner
+                .last_frame_time
+                .lock()
+                .map(|time| *time)
+                .unwrap_or_else(|_| Instant::now());
+            inner.set_av_offset(Instant::now() - last_frame_time);
+        }
+
+        context.gfx.draw_with::<VideoRO>(VideoPrimitive::new(
+            inner.id,
+            Arc::clone(&inner.alive),
+            Arc::clone(&inner.frame),
+            (inner.width as _, inner.height as _),
+            upload_frame,
+        ));
+    }
+
+    fn layout(
+        &mut self,
+        available_space: Size<ConstraintLimit>,
+        _context: &mut LayoutContext<'_, '_, '_, '_>,
+    ) -> Size<UPx> {
+        available_space.map(ConstraintLimit::max)
+    }
+}

src/shader.wgsl

@@ -0,0 +1,63 @@
+struct VertexOutput {
+    @builtin(position) position: vec4<f32>,
+    @location(0) uv: vec2<f32>,
+}
+
+struct Uniforms {
+    rect: vec4<f32>,
+}
+
+@group(0) @binding(0)
+var tex_y: texture_2d<f32>;
+
+@group(0) @binding(1)
+var tex_uv: texture_2d<f32>;
+
+@group(0) @binding(2)
+var s: sampler;
+
+@group(0) @binding(3)
+var<uniform> uniforms: Uniforms;
+
+@vertex
+fn vs_main(@builtin(vertex_index) in_vertex_index: u32) -> VertexOutput {
+    let quad = array<vec2<f32>, 6>(
+        uniforms.rect.xy,
+        uniforms.rect.zy,
+        uniforms.rect.xw,
+        uniforms.rect.zy,
+        uniforms.rect.zw,
+        uniforms.rect.xw,
+    );
+
+    var out: VertexOutput;
+    out.uv = vec2<f32>(0.0);
+    out.uv.x = select(0.0, 2.0, in_vertex_index == 1u);
+    out.uv.y = select(0.0, 2.0, in_vertex_index == 2u);
+    out.position = vec4<f32>(out.uv * vec2<f32>(2.0, -2.0) + vec2<f32>(-1.0, 1.0), 1.0, 1.0);
+    return out;
+}
+
+@fragment
+fn fs_main(in: VertexOutput) -> @location(0) vec4<f32> {
+    let yuv2r = vec3<f32>(1.164, 0.0, 1.596);
+    let yuv2g = vec3<f32>(1.164, -0.391, -0.813);
+    let yuv2b = vec3<f32>(1.164, 2.018, 0.0);
+
+    var yuv = vec3<f32>(0.0);
+    yuv.x = textureSample(tex_y, s, in.uv).r - 0.0625;
+    yuv.y = textureSample(tex_uv, s, in.uv).r - 0.5;
+    yuv.z = textureSample(tex_uv, s, in.uv).g - 0.5;
+
+    var rgb = vec3<f32>(0.0);
+    rgb.x = dot(yuv, yuv2r);
+    rgb.y = dot(yuv, yuv2g);
+    rgb.z = dot(yuv, yuv2b);
+
+    let threshold = rgb <= vec3<f32>(0.04045);
+    let hi = pow((rgb + vec3<f32>(0.055)) / vec3<f32>(1.055), vec3<f32>(2.4));
+    let lo = rgb * vec3<f32>(1.0 / 12.92);
+    rgb = select(hi, lo, threshold);
+
+    return vec4<f32>(rgb, 1.0);
+}

src/video.rs

@@ -0,0 +1,603 @@
+// This file is taken nearly one to one from https://github.com/jazzfool/iced_video_player
+use crate::Error;
+use cushy::value::{Destination, Dynamic};
+use gstreamer as gst;
+use gstreamer_app as gst_app;
+use gstreamer_app::prelude::*;
+use std::num::NonZeroU8;
+use std::ops::{Deref, DerefMut};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::{Arc, Mutex, RwLock};
+use std::time::{Duration, Instant};
+
+/// Position in the media.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum Position {
+    /// Position based on time.
+    ///
+    /// Not the most accurate format for videos.
+    Time(Duration),
+    /// Position based on nth frame.
+    Frame(u64),
+}
+
+impl From<Position> for gst::GenericFormattedValue {
+    fn from(pos: Position) -> Self {
+        match pos {
+            Position::Time(t) => gst::ClockTime::from_nseconds(t.as_nanos() as _).into(),
+            Position::Frame(f) => gst::format::Default::from_u64(f).into(),
+        }
+    }
+}
+
+impl From<Duration> for Position {
+    fn from(t: Duration) -> Self {
+        Position::Time(t)
+    }
+}
+
+impl From<u64> for Position {
+    fn from(f: u64) -> Self {
+        Position::Frame(f)
+    }
+}
+
+#[derive(Debug)]
+pub(crate) struct Internal {
+    pub(crate) id: u64,
+
+    pub(crate) bus: gst::Bus,
+    pub(crate) source: gst::Pipeline,
+    pub(crate) alive: Arc<AtomicBool>,
+    pub(crate) worker: Option<std::thread::JoinHandle<()>>,
+
+    pub(crate) width: i32,
+    pub(crate) height: i32,
+    pub(crate) framerate: f64,
+    pub(crate) duration: Duration,
+    pub(crate) speed: f64,
+    pub(crate) sync_av: bool,
+
+    pub(crate) frame: Arc<Mutex<Vec<u8>>>,
+    pub(crate) last_frame_time: Arc<Mutex<Instant>>,
+    pub(crate) looping: bool,
+    pub(crate) is_eos: bool,
+    pub(crate) restart_stream: bool,
+    pub(crate) sync_av_avg: u64,
+    pub(crate) sync_av_counter: u64,
+
+    pub(crate) upload_frame: Dynamic<()>,
+    pub(crate) subtitles: Dynamic<Option<String>>,
+}
+
+impl Internal {
+    pub(crate) fn seek(&self, position: impl Into<Position>, accurate: bool) -> Result<(), Error> {
+        let position = position.into();
+
+        // gstreamer complains if the start & end value types aren't the same
+        match &position {
+            Position::Time(_) => self.source.seek(
+                self.speed,
+                gst::SeekFlags::FLUSH
+                    | if accurate {
+                        gst::SeekFlags::ACCURATE
+                    } else {
+                        gst::SeekFlags::empty()
+                    },
+                gst::SeekType::Set,
+                gst::GenericFormattedValue::from(position),
+                gst::SeekType::Set,
+                gst::ClockTime::NONE,
+            )?,
+            Position::Frame(_) => self.source.seek(
+                self.speed,
+                gst::SeekFlags::FLUSH
+                    | if accurate {
+                        gst::SeekFlags::ACCURATE
+                    } else {
+                        gst::SeekFlags::empty()
+                    },
+                gst::SeekType::Set,
+                gst::GenericFormattedValue::from(position),
+                gst::SeekType::Set,
+                gst::format::Default::NONE,
+            )?,
+        };
+
+        Ok(())
+    }
+
+    pub(crate) fn set_speed(&mut self, speed: f64) -> Result<(), Error> {
+        let Some(position) = self.source.query_position::<gst::ClockTime>() else {
+            return Err(Error::Caps);
+        };
+        if speed > 0.0 {
+            self.source.seek(
+                speed,
+                gst::SeekFlags::FLUSH | gst::SeekFlags::ACCURATE,
+                gst::SeekType::Set,
+                position,
+                gst::SeekType::End,
+                gst::ClockTime::from_seconds(0),
+            )?;
+        } else {
+            self.source.seek(
+                speed,
+                gst::SeekFlags::FLUSH | gst::SeekFlags::ACCURATE,
+                gst::SeekType::Set,
+                gst::ClockTime::from_seconds(0),
+                gst::SeekType::Set,
+                position,
+            )?;
+        }
+        self.speed = speed;
+        Ok(())
+    }
+
+    pub(crate) fn restart_stream(&mut self) -> Result<(), Error> {
+        self.is_eos = false;
+        self.set_paused(false);
+        self.seek(0, false)?;
+        Ok(())
+    }
+
+    pub(crate) fn set_paused(&mut self, paused: bool) {
+        self.source
+            .set_state(if paused {
+                gst::State::Paused
+            } else {
+                gst::State::Playing
+            })
+            .unwrap(/* state was changed in ctor; state errors caught there */);
+
+        // Set restart_stream flag to make the stream restart on the next Message::NextFrame
+        if self.is_eos && !paused {
+            self.restart_stream = true;
+        }
+    }
+
+    pub(crate) fn paused(&self) -> bool {
+        self.source.state(gst::ClockTime::ZERO).1 == gst::State::Paused
+    }
+
+    /// Syncs audio with video when there is (inevitably) latency presenting the frame.
+    pub(crate) fn set_av_offset(&mut self, offset: Duration) {
+        if self.sync_av {
+            self.sync_av_counter += 1;
+            self.sync_av_avg = self.sync_av_avg * (self.sync_av_counter - 1) / self.sync_av_counter
+                + offset.as_nanos() as u64 / self.sync_av_counter;
+            if self.sync_av_counter % 128 == 0 {
+                self.source
+                    .set_property("av-offset", -(self.sync_av_avg as i64));
+            }
+        }
+    }
+}
+
+/// A multimedia video loaded from a URI (e.g., a local file path or HTTP stream).
+#[derive(Debug)]
+pub struct Video(pub(crate) RwLock<Internal>);
+
+impl Drop for Video {
+    fn drop(&mut self) {
+        let inner = self.0.get_mut().expect("failed to lock");
+
+        inner
+            .source
+            .set_state(gst::State::Null)
+            .expect("failed to set state");
+
+        inner.alive.store(false, Ordering::SeqCst);
+        if let Some(worker) = inner.worker.take() {
+            worker.join().expect("failed to stop video thread");
+        }
+    }
+}
+
+impl Video {
+    /// Create a new video player from a given video which loads from `uri`.
+    /// Note that live sources will report the duration to be zero.
+    pub fn new(uri: &url::Url) -> Result<Self, Error> {
+        gst::init()?;
+
+        let pipeline = format!("playbin uri=\"{}\" text-sink=\"appsink name=iced_text sync=true caps=text/x-raw\" video-sink=\"videoscale ! videoconvert ! appsink name=cushy_video drop=true caps=video/x-raw,format=NV12,pixel-aspect-ratio=1/1\"", uri.as_str());
+        let pipeline = gst::parse::launch(pipeline.as_ref())?
+            .downcast::<gst::Pipeline>()
+            .unwrap();
+        // .map_err(|_| Error::Cast)?;
+
+        let video_sink: gst::Element = pipeline.property("video-sink");
+        let pad = video_sink.pads().first().cloned().unwrap();
+        let pad = pad.dynamic_cast::<gst::GhostPad>().unwrap();
+        let bin = pad
+            .parent_element()
+            .unwrap()
+            .downcast::<gst::Bin>()
+            .unwrap();
+        let video_sink = bin.by_name("cushy_video").unwrap();
+        let video_sink = video_sink.downcast::<gst_app::AppSink>().unwrap();
+
+        let text_sink: gst::Element = pipeline.property("text-sink");
+        let text_sink = text_sink.downcast::<gst_app::AppSink>().unwrap();
+
+        Self::from_gst_pipeline(pipeline, video_sink, Some(text_sink))
+    }
+
+    /// Creates a new video based on an existing GStreamer pipeline and appsink.
+    /// Expects an `appsink` plugin with `caps=video/x-raw,format=NV12`.
+    ///
+    /// An optional `text_sink` can be provided, which enables subtitle messages
+    /// to be emitted.
+    ///
+    /// **Note:** Many functions of [`Video`] assume a `playbin` pipeline.
+    /// Non-`playbin` pipelines given here may not have full functionality.
+    pub fn from_gst_pipeline(
+        pipeline: gst::Pipeline,
+        video_sink: gst_app::AppSink,
+        text_sink: Option<gst_app::AppSink>,
+    ) -> Result<Self, Error> {
+        gst::init()?;
+        static NEXT_ID: AtomicU64 = AtomicU64::new(0);
+        let id = NEXT_ID.fetch_add(1, Ordering::SeqCst);
+
+        let pad = video_sink.pads().first().cloned().unwrap();
+
+        pipeline.set_state(gst::State::Playing)?;
+
+        // wait for up to 5 seconds until the decoder gets the source capabilities
+        pipeline.state(gst::ClockTime::from_seconds(5)).0?;
+
+        // extract resolution and framerate
+        // TODO(jazzfool): maybe we want to extract some other information too?
+        let caps = pad.current_caps().ok_or(Error::Caps)?;
+        let s = caps.structure(0).ok_or(Error::Caps)?;
+        let width = s.get::<i32>("width").map_err(|_| Error::Caps)?;
+        let height = s.get::<i32>("height").map_err(|_| Error::Caps)?;
+        // resolution should be mod4
+        let width = ((width + 4 - 1) / 4) * 4;
+        let framerate = s
+            .get::<gst::Fraction>("framerate")
+            .map_err(|_| Error::Caps)?;
+        let framerate = framerate.numer() as f64 / framerate.denom() as f64;
+
+        if framerate.is_nan()
+            || framerate.is_infinite()
+            || framerate < 0.0
+            || framerate.abs() < f64::EPSILON
+        {
+            return Err(Error::Framerate(framerate));
+        }
+
+        let duration = Duration::from_nanos(
+            pipeline
+                .query_duration::<gst::ClockTime>()
+                .map(|duration| duration.nseconds())
+                .unwrap_or(0),
+        );
+
+        let sync_av = pipeline.has_property("av-offset", None);
+
+        // NV12 = 12bpp
+        let frame = Arc::new(Mutex::new(vec![
+            0u8;
+            (width as usize * height as usize * 3)
+                .div_ceil(2)
+        ]));
+        let alive = Arc::new(AtomicBool::new(true));
+        let last_frame_time = Arc::new(Mutex::new(Instant::now()));
+
+        let subtitles = Dynamic::new(None);
+        let upload_frame = Dynamic::new(());
+
+        let frame_ref = Arc::clone(&frame);
+        let alive_ref = Arc::clone(&alive);
+        let last_frame_time_ref = Arc::clone(&last_frame_time);
+
+        let subtitles_ref = subtitles.clone();
+        let upload_frame_ref = upload_frame.clone();
+
+        let pipeline_ref = pipeline.clone();
+
+        let worker = std::thread::spawn(move || {
+            let mut clear_subtitles_at = None;
+
+            while alive_ref.load(Ordering::Acquire) {
+                if let Err(gst::FlowError::Error) = (|| -> Result<(), gst::FlowError> {
+                    let sample =
+                        if pipeline_ref.state(gst::ClockTime::ZERO).1 != gst::State::Playing {
+                            video_sink
+                                .try_pull_preroll(gst::ClockTime::from_mseconds(16))
+                                .ok_or(gst::FlowError::Eos)?
+                        } else {
+                            video_sink
+                                .try_pull_sample(gst::ClockTime::from_mseconds(16))
+                                .ok_or(gst::FlowError::Eos)?
+                        };
+
+                    *last_frame_time_ref
+                        .lock()
+                        .map_err(|_| gst::FlowError::Error)? = Instant::now();
+
+                    let buffer = sample.buffer().ok_or(gst::FlowError::Error)?;
+                    let pts = buffer.pts().unwrap_or_default();
+                    let map = buffer.map_readable().map_err(|_| gst::FlowError::Error)?;
+
+                    let mut frame = frame_ref.lock().map_err(|_| gst::FlowError::Error)?;
+                    let frame_len = frame.len();
+                    frame.copy_from_slice(&map.as_slice()[..frame_len]);
+
+                    upload_frame_ref.map_mut(|mut f| *f = ());
+
+                    if let Some(at) = clear_subtitles_at {
+                        if pts >= at {
+                            subtitles_ref.set(None);
+                            clear_subtitles_at = None;
+                        }
+                    }
+
+                    let text = text_sink
+                        .as_ref()
+                        .and_then(|sink| sink.try_pull_sample(gst::ClockTime::from_seconds(0)));
+                    if let Some(text) = text {
+                        let text = text.buffer().ok_or(gst::FlowError::Error)?;
+                        let pts = text.pts().unwrap_or_default();
+                        let duration = text.duration().unwrap_or(gst::ClockTime::ZERO);
+                        let map = text.map_readable().map_err(|_| gst::FlowError::Error)?;
+
+                        let text = html_escape::decode_html_entities(
+                            std::str::from_utf8(map.as_slice())
+                                .map_err(|_| gst::FlowError::Error)?,
+                        )
+                        .to_string();
+                        subtitles_ref.set(Some(text));
+
+                        clear_subtitles_at = Some(pts + duration);
+                    }
+
+                    Ok(())
+                })() {
+                    log::error!("error pulling frame");
+                }
+            }
+        });
+
+        Ok(Video(RwLock::new(Internal {
+            id,
+
+            bus: pipeline.bus().unwrap(),
+            source: pipeline,
+            alive,
+            worker: Some(worker),
+
+            width,
+            height,
+            framerate,
+            duration,
+            speed: 1.0,
+            sync_av,
+
+            frame,
+            last_frame_time,
+            looping: false,
+            is_eos: false,
+            restart_stream: false,
+            sync_av_avg: 0,
+            sync_av_counter: 0,
+
+            subtitles,
+            upload_frame,
+        })))
+    }
+
+    pub(crate) fn read(&self) -> impl Deref<Target = Internal> + '_ {
+        self.0.read().expect("lock")
+    }
+
+    pub(crate) fn write(&self) -> impl DerefMut<Target = Internal> + '_ {
+        self.0.write().expect("lock")
+    }
+
+    pub(crate) fn get_mut(&mut self) -> impl DerefMut<Target = Internal> + '_ {
+        self.0.get_mut().expect("lock")
+    }
+
+    /// Get the size/resolution of the video as `(width, height)`.
+    pub fn size(&self) -> (i32, i32) {
+        (self.read().width, self.read().height)
+    }
+
+    /// Get the framerate of the video as frames per second.
+    pub fn framerate(&self) -> f64 {
+        self.read().framerate
+    }
+
+    /// Set the volume multiplier of the audio.
+    /// `0.0` = 0% volume, `1.0` = 100% volume.
+    ///
+    /// This uses a linear scale, for example `0.5` is perceived as half as loud.
+    pub fn set_volume(&mut self, volume: f64) {
+        self.get_mut().source.set_property("volume", volume);
+        self.set_muted(self.muted()); // for some reason gstreamer unmutes when changing volume?
+    }
+
+    /// Get the volume multiplier of the audio.
+    pub fn volume(&self) -> f64 {
+        self.read().source.property("volume")
+    }
+
+    /// Set if the audio is muted or not, without changing the volume.
+    pub fn set_muted(&mut self, muted: bool) {
+        self.get_mut().source.set_property("mute", muted);
+    }
+
+    /// Get if the audio is muted or not.
+    pub fn muted(&self) -> bool {
+        self.read().source.property("mute")
+    }
+
+    /// Get if the stream ended or not.
+    pub fn eos(&self) -> bool {
+        self.read().is_eos
+    }
+
+    /// Get if the media will loop or not.
+    pub fn looping(&self) -> bool {
+        self.read().looping
+    }
+
+    /// Set if the media will loop or not.
+    pub fn set_looping(&mut self, looping: bool) {
+        self.get_mut().looping = looping;
+    }
+
+    /// Set if the media is paused or not.
+    pub fn set_paused(&mut self, paused: bool) {
+        self.get_mut().set_paused(paused)
+    }
+
+    /// Get if the media is paused or not.
+    pub fn paused(&self) -> bool {
+        self.read().paused()
+    }
+
+    /// Jumps to a specific position in the media.
+    /// Passing `true` to the `accurate` parameter will result in more accurate seeking,
+    /// however, it is also slower. For most seeks (e.g., scrubbing) this is not needed.
+    pub fn seek(&mut self, position: impl Into<Position>, accurate: bool) -> Result<(), Error> {
+        self.get_mut().seek(position, accurate)
+    }
+
+    /// Set the playback speed of the media.
+    /// The default speed is `1.0`.
+    pub fn set_speed(&mut self, speed: f64) -> Result<(), Error> {
+        self.get_mut().set_speed(speed)
+    }
+
+    /// Get the current playback speed.
+    pub fn speed(&self) -> f64 {
+        self.read().speed
+    }
+
+    /// Get the current playback position in time.
+    pub fn position(&self) -> Duration {
+        Duration::from_nanos(
+            self.read()
+                .source
+                .query_position::<gst::ClockTime>()
+                .map_or(0, |pos| pos.nseconds()),
+        )
+    }
+
+    /// Get the media duration.
+    pub fn duration(&self) -> Duration {
+        self.read().duration
+    }
+
+    /// Restarts a stream; seeks to the first frame and unpauses, sets the `eos` flag to false.
+    pub fn restart_stream(&mut self) -> Result<(), Error> {
+        self.get_mut().restart_stream()
+    }
+
+    /// Set the subtitle URL to display.
+    pub fn set_subtitle_url(&mut self, url: &url::Url) -> Result<(), Error> {
+        let paused = self.paused();
+        let mut inner = self.get_mut();
+        inner.source.set_state(gst::State::Ready)?;
+        inner.source.set_property("suburi", url.as_str());
+        inner.set_paused(paused);
+        Ok(())
+    }
+
+    /// Get the current subtitle URL.
+    pub fn subtitle_url(&self) -> Option<url::Url> {
+        url::Url::parse(&self.read().source.property::<String>("suburi")).ok()
+    }
+
+    /// Get the underlying GStreamer pipeline.
+    pub fn pipeline(&self) -> gst::Pipeline {
+        self.read().source.clone()
+    }
+
+    // /// Generates a list of thumbnails based on a set of positions in the media, downscaled by a given factor.
+    // ///
+    // /// Slow; only needs to be called once for each instance.
+    // /// It's best to call this at the very start of playback, otherwise the position may shift.
+    // pub fn thumbnails<I>(
+    //     &mut self,
+    //     positions: I,
+    //     downscale: NonZeroU8,
+    // ) -> Result<Vec<img::Handle>, Error>
+    // where
+    //     I: IntoIterator<Item = Position>,
+    // {
+    //     let downscale = u8::from(downscale) as u32;
+
+    //     let paused = self.paused();
+    //     let muted = self.muted();
+    //     let pos = self.position();
+
+    //     self.set_paused(false);
+    //     self.set_muted(true);
+
+    //     let out = {
+    //         let inner = self.read();
+    //         let width = inner.width;
+    //         let height = inner.height;
+    //         positions
+    //             .into_iter()
+    //             .map(|pos| {
+    //                 inner.seek(pos, true)?;
+    //                 inner.upload_frame.store(false, Ordering::SeqCst);
+    //                 while !inner.upload_frame.load(Ordering::SeqCst) {
+    //                     std::hint::spin_loop();
+    //                 }
+    //                 Ok(img::Handle::from_rgba(
+    //                     inner.width as u32 / downscale,
+    //                     inner.height as u32 / downscale,
+    //                     yuv_to_rgba(
+    //                         &inner.frame.lock().map_err(|_| Error::Lock)?,
+    //                         width as _,
+    //                         height as _,
+    //                         downscale,
+    //                     ),
+    //                 ))
+    //             })
+    //             .collect()
+    //     };
+
+    //     self.set_paused(paused);
+    //     self.set_muted(muted);
+    //     self.seek(pos, true)?;
+
+    //     out
+    // }
+}
+
+// fn yuv_to_rgba(yuv: &[u8], width: u32, height: u32, downscale: u32) -> Vec<u8> {
+//     let uv_start = width * height;
+//     let mut rgba = vec![];
+
+//     for y in 0..height / downscale {
+//         for x in 0..width / downscale {
+//             let x_src = x * downscale;
+//             let y_src = y * downscale;
+
+//             let uv_i = uv_start + width * (y_src / 2) + x_src / 2 * 2;
+
+//             let y = yuv[(y_src * width + x_src) as usize] as f32;
+//             let u = yuv[uv_i as usize] as f32;
+//             let v = yuv[(uv_i + 1) as usize] as f32;
+
+//             let r = 1.164 * (y - 16.0) + 1.596 * (v - 128.0);
+//             let g = 1.164 * (y - 16.0) - 0.813 * (v - 128.0) - 0.391 * (u - 128.0);
+//             let b = 1.164 * (y - 16.0) + 2.018 * (u - 128.0);
+
+//             rgba.push(r as u8);
+//             rgba.push(g as u8);
+//             rgba.push(b as u8);
+//             rgba.push(0xFF);
+//         }
+//     }
+
+//     rgba
+// }