1
//! Shared core for the "video-ish" widgets (camera / screencap / video).
2
//!
3
//! All three are identical in architecture (RefAny dataset + AfterMount
4
//! background capture/decode thread + writeback that uploads each frame into a
5
//! stable external GL texture + recomposites). Only the *config* and the
6
//! *worker* differ. This module holds the duplicated pieces - the [`VideoFrame`]
7
//! the worker produces and [`present_frame`], the GL writeback core - so each
8
//! widget is a thin config+worker wrapper and there's a single place for GL
9
//! fixes + the real platform workers (AVFoundation / ScreenCaptureKit /
10
//! vk-video) to plug in.
11
//!
12
//! NOTE: GL code - compile-verified here; the actual texture rendering must be
13
//! verified on a machine with a window + GPU.
14

            
15
use azul_core::animation::UpdateImageType;
16
use azul_core::callbacks::Update;
17
use azul_core::gl::gl::{RGBA, TEXTURE_2D, UNSIGNED_BYTE};
18
use azul_core::gl::{GlContextPtr, OptionU8VecRef, Texture, U8VecRef};
19
use azul_core::geom::PhysicalSizeU32;
20
use azul_core::refany::RefAny;
21
use azul_core::resources::ImageRef;
22
use azul_core::video::VideoFrame;
23
use azul_css::impl_option_inner; // brought into scope for impl_widget_callback!'s impl_option!
24
use azul_css::props::basic::ColorU;
25

            
26
use crate::callbacks::CallbackInfo;
27

            
28
/// User hook fired once per captured/decoded frame - the backreference
29
/// dependency-injection pattern (see `architecture.md`). A capture widget's
30
/// private writeback invokes it with each [`VideoFrame`], so application code
31
/// can apply effects, save the frame into its own data model, or send it over
32
/// the network (azul-meet). Returns `Update` like any callback. Wired via
33
/// `CameraWidget::with_on_frame` / `ScreenCaptureWidget::with_on_frame` /
34
/// `VideoWidget::with_on_frame`.
35
pub type OnVideoFrameCallbackType = extern "C" fn(RefAny, CallbackInfo, VideoFrame) -> Update;
36
impl_widget_callback!(
37
    OnVideoFrame,
38
    OptionOnVideoFrame,
39
    OnVideoFrameCallback,
40
    OnVideoFrameCallbackType
41
);
42

            
43
// Host-invoker plumbing for managed-FFI bindings - see core/src/host_invoker.rs.
44
azul_core::impl_managed_callback! {
45
    wrapper:        OnVideoFrameCallback,
46
    info_ty:        CallbackInfo,
47
    return_ty:      Update,
48
    default_ret:    Update::DoNothing,
49
    invoker_static: ON_VIDEO_FRAME_INVOKER,
50
    invoker_ty:     AzOnVideoFrameCallbackInvoker,
51
    thunk_fn:       az_on_video_frame_callback_thunk,
52
    setter_fn:      AzApp_setOnVideoFrameCallbackInvoker,
53
    from_handle_fn: AzOnVideoFrameCallback_createFromHostHandle,
54
    extra_args:     [ frame: VideoFrame ],
55
}
56

            
57
/// Invoke a capture widget's optional `on_frame` hook with `frame`, returning
58
/// the user's `Update` (`DoNothing` when no hook is set). Shared by all three
59
/// capture widgets' writebacks.
60
pub fn invoke_on_frame(
61
    hook: &OptionOnVideoFrame,
62
    info: &mut CallbackInfo,
63
    frame: &VideoFrame,
64
) -> Update {
65
    match hook {
66
        OptionOnVideoFrame::Some(h) => {
67
            (h.callback.cb)(h.refany.clone(), info.clone(), frame.clone())
68
        }
69
        OptionOnVideoFrame::None => Update::DoNothing,
70
    }
71
}
72

            
73
/// Present `frame` for a video-ish widget and return the (stable) GL texture
74
/// id to store back in the widget's state.
75
///
76
/// - First frame (`current_id` is `None`): allocate a GL texture, upload, wrap
77
///   in an external-texture `ImageRef`, and install it on the widget's node
78
///   **once** via `change_node_image` (the node is found via
79
///   `get_node_id_of_root_dataset(dataset)`). Returns `Some(new_id)`.
80
/// - Every frame after: re-upload into the same texture id + recomposite
81
///   (`update_all_image_callbacks` -> `ShouldReRenderCurrentWindow`) - no
82
///   relayout, no display-list rebuild, since the external texture's wr key
83
///   (= the `ImageRef` data pointer) stays stable. Returns `current_id`.
84
/// - No GL context (cpurender): returns `current_id` unchanged (a CPU upload
85
///   path is a follow-up).
86
pub fn present_frame(
87
    info: &mut CallbackInfo,
88
    dataset: RefAny,
89
    current_id: Option<u32>,
90
    frame: &VideoFrame,
91
) -> Option<u32> {
92
    let gl = match info.get_gl_context().into_option() {
93
        Some(g) => g,
94
        None => return current_id,
95
    };
96

            
97
    match current_id {
98
        Some(id) => {
99
            upload_rgba(&gl, id, frame);
100
            info.update_all_image_callbacks();
101
            Some(id)
102
        }
103
        None => {
104
            let tex = Texture::allocate_rgba8(
105
                gl.clone(),
106
                PhysicalSizeU32 {
107
                    width: frame.width,
108
                    height: frame.height,
109
                },
110
                ColorU {
111
                    r: 0,
112
                    g: 0,
113
                    b: 0,
114
                    a: 0,
115
                },
116
            );
117
            let id = tex.texture_id;
118
            upload_rgba(&gl, id, frame);
119
            let image = ImageRef::new_gltexture(tex);
120
            if let Some(node) = info.get_node_id_of_root_dataset(dataset) {
121
                if let Some(nid) = node.node.into_crate_internal() {
122
                    info.change_node_image(node.dom, nid, image, UpdateImageType::Content);
123
                }
124
            }
125
            Some(id)
126
        }
127
    }
128
}
129

            
130
/// Upload tightly-packed RGBA8 pixels into the GL texture `texture_id`.
131
pub fn upload_rgba(gl: &GlContextPtr, texture_id: u32, frame: &VideoFrame) {
132
    gl.bind_texture(TEXTURE_2D, texture_id);
133
    gl.tex_image_2d(
134
        TEXTURE_2D,
135
        0,
136
        RGBA as i32,
137
        frame.width as i32,
138
        frame.height as i32,
139
        0,
140
        RGBA,
141
        UNSIGNED_BYTE,
142
        OptionU8VecRef::Some(U8VecRef::from(frame.bytes.as_ref())),
143
    );
144
}
145

            
146
/// A platform frame-capture backend (camera / screen), registered by the dll at
147
/// startup so the cross-platform capture widgets can pull **real** frames
148
/// instead of their built-in test pattern. The dll provides one per OS (v4l2 on
149
/// Linux, AVFoundation on macOS, Media Foundation on Windows, ScreenCaptureKit /
150
/// PipeWire / DXGI for screens, ...). These are plain Rust fn pointers - the dll
151
/// links azul-layout statically, so registering + calling is a Rust-to-Rust
152
/// call, no `extern "C"`/trait-object dance.
153
#[derive(Clone, Copy)]
154
pub struct CaptureVTable {
155
    /// Open source `index` (camera device / display index) at the requested
156
    /// `width` x `height`. Returns an opaque handle, or `0` on failure (the
157
    /// worker then falls back to the test pattern).
158
    pub open: fn(index: u32, width: u32, height: u32) -> u64,
159
    /// Block for the next frame, writing tightly-packed RGBA8 into `out`
160
    /// (resized as needed). Returns the actual frame `(width, height)`, or
161
    /// `(0, 0)` on end-of-stream / error (the worker then stops + closes).
162
    pub read: fn(handle: u64, out: &mut alloc::vec::Vec<u8>) -> (u32, u32),
163
    /// Close + free the source.
164
    pub close: fn(handle: u64),
165
}
166

            
167
static CAMERA_BACKEND: std::sync::OnceLock<CaptureVTable> = std::sync::OnceLock::new();
168
static SCREEN_BACKEND: std::sync::OnceLock<CaptureVTable> = std::sync::OnceLock::new();
169

            
170
/// Register the platform **camera** capture backend (called once by the dll at
171
/// startup; the first registration wins). Without it, `CameraWidget` shows its
172
/// test pattern.
173
pub fn register_camera_backend(vtable: CaptureVTable) {
174
    let _ = CAMERA_BACKEND.set(vtable);
175
}
176

            
177
/// Register the platform **screen** capture backend (for `ScreenCaptureWidget`).
178
pub fn register_screen_backend(vtable: CaptureVTable) {
179
    let _ = SCREEN_BACKEND.set(vtable);
180
}
181

            
182
/// The registered camera backend, if the dll provided one for this platform.
183
pub fn camera_backend() -> Option<CaptureVTable> {
184
    CAMERA_BACKEND.get().copied()
185
}
186

            
187
/// The registered screen-capture backend, if any.
188
pub fn screen_backend() -> Option<CaptureVTable> {
189
    SCREEN_BACKEND.get().copied()
190
}
191

            
192
/// A platform **audio**-capture backend (microphone), registered by the dll so
193
/// `MicrophoneWidget` can pull real samples instead of the test tone. Like
194
/// [`CaptureVTable`] but yields interleaved `f32` audio rather than RGBA video.
195
#[derive(Clone, Copy)]
196
pub struct AudioCaptureVTable {
197
    /// Open the default mic at `sample_rate` x `channels`. Opaque handle, or
198
    /// `0` on failure.
199
    pub open: fn(sample_rate: u32, channels: u16) -> u64,
200
    /// Block for the next chunk, writing interleaved `f32` into `out` (resized).
201
    /// Returns the frame count (`out.len() / channels`), or `0` on error / EOF
202
    /// (the worker then stops + closes).
203
    pub read: fn(handle: u64, out: &mut alloc::vec::Vec<f32>) -> u32,
204
    /// Close + free the source.
205
    pub close: fn(handle: u64),
206
}
207

            
208
static MIC_BACKEND: std::sync::OnceLock<AudioCaptureVTable> = std::sync::OnceLock::new();
209

            
210
/// Register the platform microphone-capture backend (called once by the dll).
211
pub fn register_mic_backend(vtable: AudioCaptureVTable) {
212
    let _ = MIC_BACKEND.set(vtable);
213
}
214

            
215
/// The registered mic-capture backend, if the dll provided one for this platform.
216
pub fn mic_backend() -> Option<AudioCaptureVTable> {
217
    MIC_BACKEND.get().copied()
218
}