1
//! Video-playback widget — a "dumb widget" identical in architecture to the
2
//! [`CameraWidget`](super::camera) / [`ScreenCaptureWidget`](super::screencap),
3
//! only the source differs (a video URL/file decoded via vk-video).
4
//! SUPER_PLAN_2 §4 P6, widget pivot.
5
//!
6
//! `VideoWidget::create(config).dom()` → an `<img>` a background decode thread
7
//! keeps fed; each frame goes through [`super::capture_common::present_frame`]
8
//! (GL-texture install-once / re-upload + recomposite). Shared core in
9
//! `capture_common`; this widget is its config + worker. Test-pattern worker
10
//! (scrolling SMPTE colour bars) stands in for the real vk-video decode worker.
11

            
12
use alloc::vec::Vec;
13

            
14
use azul_core::callbacks::{Update, VirtualViewCallbackInfo, VirtualViewReturn};
15
use azul_core::dom::{ComponentEventFilter, DatasetMergeCallbackType, Dom, EventFilter, OptionDom};
16
use azul_core::geom::LogicalPosition;
17
use azul_core::refany::{OptionRefAny, RefAny};
18
use azul_core::resources::{ImageRef, RawImage, RawImageData, RawImageFormat};
19
use azul_core::task::{ThreadId, ThreadReceiver, ThreadSendMsg};
20
use azul_core::video::{VideoConfig, VideoFrame};
21

            
22
use super::capture_common::{
23
    invoke_on_frame, OnVideoFrame, OnVideoFrameCallback, OptionOnVideoFrame,
24
};
25
use crate::callbacks::{Callback, CallbackInfo, CallbackType};
26
use crate::thread::{
27
    Thread, ThreadCallback, ThreadReceiveMsg, ThreadSender, ThreadWriteBackMsg, WriteBackCallback,
28
};
29

            
30
/// Default decode size for the test pattern (the real decoder reports the
31
/// stream's actual size).
32
const DEFAULT_W: u32 = 1280;
33
const DEFAULT_H: u32 = 720;
34

            
35
/// Live state for one video widget, carried across relayout by
36
/// [`merge_video_state`].
37
pub struct VideoWidgetState {
38
    /// The requested playback configuration (source + autoplay/loop).
39
    pub config: VideoConfig,
40
    /// `true` once the decode thread has been started.
41
    pub started: bool,
42
    /// The stable external GL texture id once installed.
43
    pub gl_texture_id: Option<u32>,
44
    /// Optional user hook invoked with each decoded frame (effects / save /
45
    /// send). Re-set on every fresh build (see [`merge_video_state`]).
46
    pub on_frame: OptionOnVideoFrame,
47
    /// Optional pre-decoded frames to replay (a `RefAny` holding a
48
    /// `Vec<VideoFrame>`); when set, the replay worker cycles these instead of
49
    /// the built-in test pattern. Carried forward by [`merge_video_state`].
50
    pub frames: OptionRefAny,
51
    /// The off-main-thread streaming decode worker (mirrors the map widget's
52
    /// `fetch_callback`). Set via [`VideoWidget::dom_with_decoder`]. When present,
53
    /// AfterMount spawns it on a background `Thread` instead of the replay /
54
    /// test-pattern workers, so the VK decode runs off the main thread.
55
    pub decode_callback: Option<ThreadCallback>,
56
    /// The latest decoded frame to display, as a CPU `ImageRef` (RGBA8). The
57
    /// VirtualView render callback ([`video_widget_render`]) reads this on each
58
    /// re-render; [`video_writeback`] stores it and triggers an in-place
59
    /// VirtualView re-render — so the frame renders on cpurender AND webrender,
60
    /// exactly like the map widget's tile cache. (Replaces the GL `present_frame`
61
    /// path for video; camera/screencap still use present_frame.)
62
    pub current_frame: Option<ImageRef>,
63
    /// The decode worker's `ThreadId` (set by AfterMount). Lets the resize callback
64
    /// message the running worker (`info.get_thread(id).sender.send(..)`) so it can
65
    /// re-target the decoder to the new physical-pixel size — a cheap image swap, no
66
    /// relayout. Carried across relayout by [`merge_video_state`].
67
    pub thread_id: Option<ThreadId>,
68
    /// Clone of the worker's main→worker `Sender` (set by AfterMount, carried by
69
    /// merge). Lets [`merge_video_state`] — which has no `CallbackInfo` — push a
70
    /// seek to the running worker when `config.timestamp` changes (scrubbing).
71
    pub seek_sender: Option<std::sync::mpsc::Sender<ThreadSendMsg>>,
72
}
73

            
74
/// A video-playback widget. `create(config).dom()` yields an `<img>` the
75
/// decode thread keeps fed.
76
#[repr(C)]
77
pub struct VideoWidget {
78
    /// Source URL + autoplay/loop + format.
79
    pub config: VideoConfig,
80
    /// Optional per-frame user hook (effects / save / send - azul-meet).
81
    pub on_frame: OptionOnVideoFrame,
82
    /// Optional pre-decoded frames to replay (a `RefAny` holding a
83
    /// `Vec<VideoFrame>`); set via [`with_frames`](Self::with_frames). When
84
    /// present the widget cycles these instead of the test pattern.
85
    pub frames: OptionRefAny,
86
}
87

            
88
impl VideoWidget {
89
    /// Create a video widget for the given config.
90
    pub fn create(config: VideoConfig) -> Self {
91
        Self {
92
            config,
93
            on_frame: OptionOnVideoFrame::None,
94
            frames: OptionRefAny::None,
95
        }
96
    }
97

            
98
    /// Set a hook invoked with every decoded frame - for live effects, saving
99
    /// frames into your data model, or sending them over the network
100
    /// (azul-meet). The backreference DI pattern (see `architecture.md`).
101
    pub fn set_on_frame<C: Into<OnVideoFrameCallback>>(&mut self, data: RefAny, on_frame: C) {
102
        self.on_frame = Some(OnVideoFrame {
103
            refany: data,
104
            callback: on_frame.into(),
105
        })
106
        .into();
107
    }
108

            
109
    /// Builder form of [`set_on_frame`](Self::set_on_frame).
110
    pub fn with_on_frame<C: Into<OnVideoFrameCallback>>(
111
        mut self,
112
        data: RefAny,
113
        on_frame: C,
114
    ) -> Self {
115
        self.set_on_frame(data, on_frame);
116
        self
117
    }
118

            
119
    /// Replay a list of already-decoded frames instead of the built-in test
120
    /// pattern: `frames` is a [`RefAny`] holding a `Vec<VideoFrame>`. The
121
    /// background worker cycles them through the shared GL presenter (the same
122
    /// `present_frame` path the camera/screencap widgets use), so callers that
123
    /// decode a clip up front (e.g. `decode_mp4_h264_bytes`) get real pixels on
124
    /// screen. The `RefAny` must carry a `Vec<VideoFrame>`, else playback is
125
    /// skipped and the test pattern shows instead.
126
    pub fn with_frames(mut self, frames: RefAny) -> Self {
127
        self.frames = Some(frames).into();
128
        self
129
    }
130

            
131
    fn build_dom(self, decode_cb: Option<ThreadCallback>) -> Dom {
132
        let state = VideoWidgetState {
133
            config: self.config,
134
            started: false,
135
            gl_texture_id: None,
136
            on_frame: self.on_frame,
137
            frames: self.frames,
138
            decode_callback: decode_cb,
139
            current_frame: None,
140
            thread_id: None,
141
            seek_sender: None,
142
        };
143
        let dataset = RefAny::new(state);
144
        let vv_data = dataset.clone();
145

            
146
        // The body is a VirtualView (exactly like the map widget): its render
147
        // callback re-reads `current_frame` from the dataset each re-render and
148
        // builds the `<img>`, so streamed frames render on BOTH cpurender and
149
        // webrender. The background decode worker is started on AfterMount and
150
        // `WriteBack`s frames into `current_frame` + triggers a VirtualView
151
        // re-render in place (no DOM rebuild) — see `video_writeback`. The caller
152
        // sizes the outer node via `.with_css(...)` on the returned Dom.
153
        Dom::create_div()
154
            .with_dataset(OptionRefAny::Some(dataset.clone()))
155
            .with_merge_callback(merge_video_state as DatasetMergeCallbackType)
156
            .with_callback(
157
                EventFilter::Component(ComponentEventFilter::AfterMount),
158
                dataset.clone(),
159
                Callback::from(video_on_after_mount as CallbackType),
160
            )
161
            // Window/layout resize → re-target the decoder to the new physical size
162
            // (a cheap image swap, no relayout). See `video_on_resize`.
163
            .with_callback(
164
                EventFilter::Component(ComponentEventFilter::NodeResized),
165
                dataset,
166
                Callback::from(video_on_resize as CallbackType),
167
            )
168
            .with_child(
169
                Dom::create_virtual_view(
170
                    vv_data,
171
                    video_widget_render as azul_core::callbacks::VirtualViewCallbackType,
172
                )
173
                .with_css("width: 100%; height: 100%; overflow: hidden;"),
174
            )
175
    }
176

            
177
    /// Build the widget's DOM: a single `<img>` node a background thread keeps
178
    /// fed. Replays pre-decoded [`with_frames`](Self::with_frames) if given, else
179
    /// shows the built-in test pattern.
180
    pub fn dom(self) -> Dom {
181
        self.build_dom(None)
182
    }
183

            
184
    /// Build the widget's DOM and wire a background **streaming** decode worker —
185
    /// mirrors `MapWidget::dom_with_fetch`. `cb` runs on a framework `Thread` OFF
186
    /// the main thread: it reads the `VideoConfig` (its typed `VideoSource` —
187
    /// URL / file / bytes), runs the VK decode incrementally (no up-front decode),
188
    /// and `WriteBack`s frames to the `<img>` paced by wall-clock (dropping late
189
    /// frames). The standard worker is
190
    /// `azul_dll::desktop::extra::video_codec::stream::video_decode_worker`; wrap
191
    /// it in a `ThreadCallback` to pass it here.
192
    pub fn dom_with_decoder(self, cb: ThreadCallback) -> Dom {
193
        self.build_dom(Some(cb))
194
    }
195
}
196

            
197
/// VirtualView render callback (mirrors `map_widget_render`): build the `<img>`
198
/// for the latest decoded frame, re-read from the widget's dataset on every
199
/// re-render. The decode worker stores frames into `current_frame` and triggers
200
/// the re-render in place (see [`video_writeback`]), so this renders on both the
201
/// CPU and GPU renderers with no DOM rebuild.
202
extern "C" fn video_widget_render(
203
    mut data: RefAny,
204
    info: VirtualViewCallbackInfo,
205
) -> VirtualViewReturn {
206
    let bounds = info.get_bounds().get_logical_size();
207
    if std::env::var("AZ_VIDEO_FRAMELOG").is_ok() {
208
        eprintln!("[vrender] bounds {}x{}", bounds.width, bounds.height);
209
    }
210
    // Defensive (like map_widget_render): a non-finite / non-positive box (layout
211
    // not yet settled, e.g. flex-grow before the parent height resolves) would
212
    // produce a garbage `<img>` size — render nothing until it settles.
213
    let dom = if !bounds.width.is_finite()
214
        || !bounds.height.is_finite()
215
        || bounds.width <= 0.0
216
        || bounds.height <= 0.0
217
    {
218
        OptionDom::None
219
    } else {
220
        match data.downcast_ref::<VideoWidgetState>() {
221
            Some(s) => match &s.current_frame {
222
                Some(img) => OptionDom::Some(
223
                    Dom::create_image(img.clone()).with_css("width: 100%; height: 100%;"),
224
                ),
225
                None => OptionDom::None,
226
            },
227
            None => OptionDom::None,
228
        }
229
    };
230
    VirtualViewReturn {
231
        dom,
232
        scroll_size: bounds,
233
        scroll_offset: LogicalPosition::zero(),
234
        virtual_scroll_size: bounds,
235
        virtual_scroll_offset: LogicalPosition::zero(),
236
    }
237
}
238

            
239
/// AfterMount: start the background decode thread exactly once.
240
extern "C" fn video_on_after_mount(mut data: RefAny, mut info: CallbackInfo) -> Update {
241
    // Mark started exactly once; pull out the streaming decode worker (if any),
242
    // its source, and any pre-decoded replay frames.
243
    let (decode_cb, config, frames) = {
244
        let mut s = match data.downcast_mut::<VideoWidgetState>() {
245
            Some(s) => s,
246
            None => return Update::DoNothing,
247
        };
248
        if s.started {
249
            return Update::DoNothing;
250
        }
251
        s.started = true;
252
        let frames = match &s.frames {
253
            OptionRefAny::Some(f) => Some(f.clone()),
254
            OptionRefAny::None => None,
255
        };
256
        (s.decode_callback.clone(), s.config.clone(), frames)
257
    };
258
    // Priority: off-main streaming decode worker > replay pre-decoded frames >
259
    // built-in test pattern. All feed the same WriteBack -> video_writeback path.
260
    if let Some(cb) = decode_cb {
261
        // The worker's thread-init is the `VideoConfig` itself: it matches on
262
        // `config.source` (typed — no RefAny downcast) and reads `config.timestamp`.
263
        let init = RefAny::new(config);
264
        let tid = ThreadId::unique();
265
        let thread = Thread::create(init, data.clone(), cb);
266
        // Grab the main→worker sender BEFORE add_thread moves the Thread, so the
267
        // merge callback can push seeks to the worker (scrubbing).
268
        let seek_sender = thread.clone_sender();
269
        info.add_thread(tid, thread);
270
        // Remember the worker's id (resize messaging) + sender (seek messaging).
271
        if let Some(mut s) = data.downcast_mut::<VideoWidgetState>() {
272
            s.thread_id = Some(tid);
273
            s.seek_sender = seek_sender;
274
        }
275
    } else if let Some(frames) = frames {
276
        info.add_thread(
277
            ThreadId::unique(),
278
            Thread::create(frames, data.clone(), ThreadCallback::new(video_replay_worker)),
279
        );
280
    } else {
281
        info.add_thread(
282
            ThreadId::unique(),
283
            Thread::create(
284
                RefAny::new(()),
285
                data.clone(),
286
                ThreadCallback::new(video_test_worker),
287
            ),
288
        );
289
    }
290
    Update::DoNothing
291
}
292

            
293
/// NodeResized: the video box changed physical size (window resize / relayout). Tell
294
/// the running decode worker the new target size via its `ThreadSender` so it scales
295
/// frames to fit OFF the main thread — the UI then does a cheap image swap with no
296
/// interpolation. This is a message, NOT a relayout: returns `DoNothing`.
297
extern "C" fn video_on_resize(mut data: RefAny, mut info: CallbackInfo) -> Update {
298
    let tid = match data.downcast_ref::<VideoWidgetState>() {
299
        Some(s) => s.thread_id,
300
        None => return Update::DoNothing,
301
    };
302
    let tid = match tid {
303
        Some(t) => t,
304
        None => return Update::DoNothing,
305
    };
306
    let node = info.get_hit_node();
307
    let size = match info.get_node_size(node) {
308
        Some(s) => s,
309
        None => return Update::DoNothing,
310
    };
311
    let target = (size.width.max(1.0) as u32, size.height.max(1.0) as u32);
312
    if let Some(thread) = info.get_thread(&tid) {
313
        thread.send_message(ThreadSendMsg::Custom(RefAny::new(target)));
314
    }
315
    Update::DoNothing
316
}
317

            
318
/// Background worker (test pattern): SMPTE-style colour bars scrolling
319
/// horizontally ~30×/s. Replaced by the real vk-video decode worker later.
320
extern "C" fn video_test_worker(_init: RefAny, mut sender: ThreadSender, _recv: ThreadReceiver) {
321
    const BARS: [[u8; 3]; 7] = [
322
        [235, 235, 235],
323
        [235, 235, 16],
324
        [16, 235, 235],
325
        [16, 235, 16],
326
        [235, 16, 235],
327
        [235, 16, 16],
328
        [16, 16, 235],
329
    ];
330
    let (w, h) = (DEFAULT_W as usize, DEFAULT_H as usize);
331
    let mut tick: u32 = 0;
332
    loop {
333
        let shift = (tick as usize / 4) % 7;
334
        let mut bytes = Vec::with_capacity(w * h * 4);
335
        for _y in 0..h {
336
            for x in 0..w {
337
                let c = BARS[((x * 7 / w) + shift) % 7];
338
                bytes.extend_from_slice(&[c[0], c[1], c[2], 255]);
339
            }
340
        }
341
        let frame = VideoFrame {
342
            width: w as u32,
343
            height: h as u32,
344
            bytes: bytes.into(),
345
        };
346
        let sent = sender.send(ThreadReceiveMsg::WriteBack(ThreadWriteBackMsg::new(
347
            WriteBackCallback::new(video_writeback),
348
            RefAny::new(frame),
349
        )));
350
        if !sent {
351
            break;
352
        }
353
        std::thread::sleep(std::time::Duration::from_millis(33));
354
        tick = tick.wrapping_add(2);
355
    }
356
}
357

            
358
/// Background worker (replay): cycle a caller-supplied `Vec<VideoFrame>` (e.g. a
359
/// clip decoded up front via `decode_mp4_h264_bytes`) ~30x/s through the same
360
/// WriteBack -> [`video_writeback`] -> [`super::capture_common::present_frame`]
361
/// path as the test pattern, so real decoded pixels land in the shared GL
362
/// texture. `init` is the `RefAny` handed to
363
/// [`VideoWidget::with_frames`](VideoWidget::with_frames); if it doesn't hold a
364
/// non-empty `Vec<VideoFrame>` the worker just returns.
365
extern "C" fn video_replay_worker(mut init: RefAny, mut sender: ThreadSender, _recv: ThreadReceiver) {
366
    let frames: Vec<VideoFrame> = match init.downcast_ref::<Vec<VideoFrame>>() {
367
        Some(f) => f.clone(),
368
        None => return,
369
    };
370
    if frames.is_empty() {
371
        return;
372
    }
373
    let mut idx: usize = 0;
374
    loop {
375
        let frame = frames[idx % frames.len()].clone();
376
        let sent = sender.send(ThreadReceiveMsg::WriteBack(ThreadWriteBackMsg::new(
377
            WriteBackCallback::new(video_writeback),
378
            RefAny::new(frame),
379
        )));
380
        if !sent {
381
            break;
382
        }
383
        std::thread::sleep(std::time::Duration::from_millis(33));
384
        idx = idx.wrapping_add(1);
385
    }
386
}
387

            
388
/// Writeback (main thread): store the decoded frame as the widget's
389
/// `current_frame` (a CPU `ImageRef`) and re-render the VirtualView in place so it
390
/// re-reads it — exactly like `map_tile_writeback`. Renders on cpurender AND
391
/// webrender (no GL `present_frame`, no DOM rebuild).
392
pub extern "C" fn video_writeback(
393
    mut writeback_data: RefAny,
394
    mut frame_data: RefAny,
395
    mut info: CallbackInfo,
396
) -> Update {
397
    let hook = match writeback_data.downcast_ref::<VideoWidgetState>() {
398
        Some(s) => s.on_frame.clone(),
399
        None => OptionOnVideoFrame::None,
400
    };
401
    let mut user_update = Update::DoNothing;
402
    match frame_data.downcast_ref::<VideoFrame>() {
403
        Some(frame) => {
404
            if let Some(img) = ImageRef::new_rawimage(RawImage {
405
                pixels: RawImageData::U8(frame.bytes.clone()),
406
                width: frame.width as usize,
407
                height: frame.height as usize,
408
                premultiplied_alpha: false,
409
                data_format: RawImageFormat::RGBA8,
410
                tag: b"azul-video-frame".to_vec().into(),
411
            }) {
412
                if let Some(mut s) = writeback_data.downcast_mut::<VideoWidgetState>() {
413
                    s.current_frame = Some(img);
414
                }
415
            }
416
            user_update = invoke_on_frame(&hook, &mut info, &frame);
417
        }
418
        None => return Update::DoNothing,
419
    }
420
    // Re-render the VirtualView(s) in place so the content callback re-reads the
421
    // freshly-stored `current_frame` (NOT RefreshDom — that would rebuild the DOM
422
    // and orphan the worker's dataset clone). Same trick as `map_tile_writeback`.
423
    info.trigger_all_virtual_view_rerender();
424
    user_update
425
}
426

            
427
/// Carry live state forward across relayout.
428
extern "C" fn merge_video_state(mut new_data: RefAny, mut old_data: RefAny) -> RefAny {
429
    {
430
        let new_guard = new_data.downcast_mut::<VideoWidgetState>();
431
        let old_guard = old_data.downcast_ref::<VideoWidgetState>();
432
        if let (Some(mut new_g), Some(old_g)) = (new_guard, old_guard) {
433
            new_g.started = old_g.started;
434
            new_g.gl_texture_id = old_g.gl_texture_id;
435
            new_g.frames = old_g.frames.clone();
436
            new_g.decode_callback = old_g.decode_callback.clone();
437
            new_g.current_frame = old_g.current_frame.clone();
438
            new_g.thread_id = old_g.thread_id;
439
            new_g.seek_sender = old_g.seek_sender.clone();
440
            // Scrubbing: a changed `config.timestamp` across this relayout → tell the
441
            // worker to seek. Cheap wall-clock reposition (the worker already has the
442
            // decoded frames), result comes back as an image swap — no re-decode here.
443
            if old_g.config.timestamp != new_g.config.timestamp {
444
                if let Some(snd) = new_g.seek_sender.as_ref() {
445
                    let _ = snd.send(ThreadSendMsg::Custom(RefAny::new(new_g.config.timestamp)));
446
                }
447
            }
448
            // Input-source change → tell the worker to re-init the decode (it
449
            // re-resolves/demuxes/decodes the new source); the frame swaps in when ready.
450
            if old_g.config.source != new_g.config.source {
451
                if let Some(snd) = new_g.seek_sender.as_ref() {
452
                    let _ =
453
                        snd.send(ThreadSendMsg::Custom(RefAny::new(new_g.config.source.clone())));
454
                }
455
            }
456
        }
457
    }
458
    new_data
459
}