matrixfurry.com
The open source OpenXR runtime
1// Copyright 2020-2021, Collabora, Ltd.
2// SPDX-License-Identifier: BSL-1.0
3/*!
4 * @file
5 * @brief Video file frameserver implementation
6 * @author Christoph Haag <christoph.haag@collabora.com>
7 * @author Pete Black <pblack@collabora.com>
8 * @author Jakob Bornecrantz <jakob@collabora.com>
9 * @ingroup drv_vf
10 */
11
12#include "vf_interface.h" // IWYU pragma: associated
13
14#include "os/os_time.h"
15#include "os/os_threading.h"
16
17#include "util/u_trace_marker.h"
18#include "util/u_var.h"
19#include "util/u_misc.h"
20#include "util/u_debug.h"
21#include "util/u_format.h"
22#include "util/u_frame.h"
23#include "util/u_logging.h"
24#include "util/u_trace_marker.h"
25
26
27#include <stdio.h>
28#include <assert.h>
29
30#include <glib.h>
31#include <gst/gst.h>
32#include <gst/app/gstappsink.h>
33#include <gst/video/video-frame.h>
34
35
36/*
37 *
38 * Defines.
39 *
40 */
41
42/*
43 *
44 * Printing functions.
45 *
46 */
47
48#define VF_TRACE(d, ...) U_LOG_IFL_T(d->log_level, __VA_ARGS__)
49#define VF_DEBUG(d, ...) U_LOG_IFL_D(d->log_level, __VA_ARGS__)
50#define VF_INFO(d, ...) U_LOG_IFL_I(d->log_level, __VA_ARGS__)
51#define VF_WARN(d, ...) U_LOG_IFL_W(d->log_level, __VA_ARGS__)
52#define VF_ERROR(d, ...) U_LOG_IFL_E(d->log_level, __VA_ARGS__)
53
54DEBUG_GET_ONCE_LOG_OPTION(vf_log, "VF_LOG", U_LOGGING_WARN)
55
56/*!
57 * A frame server operating on a video file.
58 *
59 * @implements xrt_frame_node
60 * @implements xrt_fs
61 */
62struct vf_fs
63{
64 struct xrt_fs base;
65
66 struct os_thread_helper play_thread;
67
68 GMainLoop *loop;
69 GstElement *source;
70 GstElement *testsink;
71 bool got_sample;
72 int width;
73 int height;
74 enum xrt_format format;
75 enum xrt_stereo_format stereo_format;
76
77 struct xrt_frame_node node;
78
79 struct
80 {
81 bool extended_format;
82 bool timeperframe;
83 } has;
84
85 enum xrt_fs_capture_type capture_type;
86 struct xrt_frame_sink *sink;
87
88 uint32_t selected;
89
90 struct xrt_fs_capture_parameters capture_params;
91
92 bool is_configured;
93 bool is_running;
94 enum u_logging_level log_level;
95};
96
97/*!
98 * Frame wrapping a GstSample/GstBuffer.
99 *
100 * @implements xrt_frame
101 */
102struct vf_frame
103{
104 struct xrt_frame base;
105
106 GstSample *sample;
107
108 GstVideoFrame frame;
109};
110
111
112/*
113 *
114 * Cast helpers.
115 *
116 */
117
118/*!
119 * Cast to derived type.
120 */
121static inline struct vf_fs *
122vf_fs(struct xrt_fs *xfs)
123{
124 return (struct vf_fs *)xfs;
125}
126
127/*!
128 * Cast to derived type.
129 */
130static inline struct vf_frame *
131vf_frame(struct xrt_frame *xf)
132{
133 return (struct vf_frame *)xf;
134}
135
136
137/*
138 *
139 * Frame methods.
140 *
141 */
142
143static void
144vf_frame_destroy(struct xrt_frame *xf)
145{
146 SINK_TRACE_MARKER();
147
148 struct vf_frame *vff = vf_frame(xf);
149
150 gst_video_frame_unmap(&vff->frame);
151
152 if (vff->sample != NULL) {
153 gst_sample_unref(vff->sample);
154 vff->sample = NULL;
155 }
156
157 free(vff);
158}
159
160
161/*
162 *
163 * Misc helper functions
164 *
165 */
166
167
168static void
169vf_fs_frame(struct vf_fs *vid, GstSample *sample)
170{
171 SINK_TRACE_MARKER();
172
173 // Noop.
174 if (!vid->sink) {
175 return;
176 }
177
178 GstVideoInfo info;
179 GstBuffer *buffer;
180 GstCaps *caps;
181 buffer = gst_sample_get_buffer(sample);
182 caps = gst_sample_get_caps(sample);
183
184 gst_video_info_init(&info);
185 gst_video_info_from_caps(&info, caps);
186
187 static int seq = 0;
188 struct vf_frame *vff = U_TYPED_CALLOC(struct vf_frame);
189
190 if (!gst_video_frame_map(&vff->frame, &info, buffer, GST_MAP_READ)) {
191 VF_ERROR(vid, "Failed to map frame %d", seq);
192 // Yes, we should do this here because we don't want the destroy function to run.
193 free(vff);
194 return;
195 }
196
197 // We now want to hold onto the sample for as long as the frame lives.
198 gst_sample_ref(sample);
199 vff->sample = sample;
200
201 // Hardcoded first plane.
202 int plane = 0;
203
204 struct xrt_frame *xf = &vff->base;
205 xf->reference.count = 1;
206 xf->destroy = vf_frame_destroy;
207 xf->width = vid->width;
208 xf->height = vid->height;
209 xf->format = vid->format;
210 xf->stride = info.stride[plane];
211 xf->data = vff->frame.data[plane];
212 xf->stereo_format = vid->stereo_format;
213 xf->size = info.size;
214 xf->source_id = vid->base.source_id;
215
216 //! @todo Proper sequence number and timestamp.
217 xf->source_sequence = seq;
218 xf->timestamp = os_monotonic_get_ns();
219
220 xrt_sink_push_frame(vid->sink, &vff->base);
221
222 xrt_frame_reference(&xf, NULL);
223 vff = NULL;
224
225 seq++;
226}
227
228static GstFlowReturn
229on_new_sample_from_sink(GstElement *elt, struct vf_fs *vid)
230{
231 SINK_TRACE_MARKER();
232 GstSample *sample;
233 sample = gst_app_sink_pull_sample(GST_APP_SINK(elt));
234
235 if (!vid->got_sample) {
236 gint width;
237 gint height;
238
239 GstCaps *caps = gst_sample_get_caps(sample);
240 GstStructure *structure = gst_caps_get_structure(caps, 0);
241
242 gst_structure_get_int(structure, "width", &width);
243 gst_structure_get_int(structure, "height", &height);
244
245 VF_DEBUG(vid, "video size is %dx%d", width, height);
246 vid->got_sample = true;
247 vid->width = width;
248 vid->height = height;
249
250 // first sample is only used for getting metadata
251 return GST_FLOW_OK;
252 }
253
254 // Takes ownership of the sample.
255 vf_fs_frame(vid, sample);
256
257 // Done with sample now.
258 gst_sample_unref(sample);
259
260 return GST_FLOW_OK;
261}
262
263static void
264print_gst_error(GstMessage *message)
265{
266 GError *err = NULL;
267 gchar *dbg_info = NULL;
268
269 gst_message_parse_error(message, &err, &dbg_info);
270 U_LOG_E("ERROR from element %s: %s", GST_OBJECT_NAME(message->src), err->message);
271 U_LOG_E("Debugging info: %s", (dbg_info) ? dbg_info : "none");
272 g_error_free(err);
273 g_free(dbg_info);
274}
275
276static gboolean
277on_source_message(GstBus *bus, GstMessage *message, struct vf_fs *vid)
278{
279 /* nil */
280 switch (GST_MESSAGE_TYPE(message)) {
281 case GST_MESSAGE_EOS:
282 VF_DEBUG(vid, "Finished playback.");
283 g_main_loop_quit(vid->loop);
284 break;
285 case GST_MESSAGE_ERROR:
286 VF_ERROR(vid, "Received error.");
287 print_gst_error(message);
288 g_main_loop_quit(vid->loop);
289 break;
290 default: break;
291 }
292 return TRUE;
293}
294
295static void *
296vf_fs_mainloop(void *ptr)
297{
298 SINK_TRACE_MARKER();
299
300 struct vf_fs *vid = (struct vf_fs *)ptr;
301
302 VF_DEBUG(vid, "Let's run!");
303 g_main_loop_run(vid->loop);
304 VF_DEBUG(vid, "Going out!");
305
306 gst_object_unref(vid->testsink);
307 gst_element_set_state(vid->source, GST_STATE_NULL);
308
309
310 gst_object_unref(vid->source);
311 g_main_loop_unref(vid->loop);
312
313 return NULL;
314}
315
316
317/*
318 *
319 * Frame server methods.
320 *
321 */
322
323static bool
324vf_fs_enumerate_modes(struct xrt_fs *xfs, struct xrt_fs_mode **out_modes, uint32_t *out_count)
325{
326 struct vf_fs *vid = vf_fs(xfs);
327
328 struct xrt_fs_mode *modes = U_TYPED_ARRAY_CALLOC(struct xrt_fs_mode, 1);
329 if (modes == NULL) {
330 return false;
331 }
332
333 modes[0].width = vid->width;
334 modes[0].height = vid->height;
335 modes[0].format = vid->format;
336 modes[0].stereo_format = vid->stereo_format;
337
338 *out_modes = modes;
339 *out_count = 1;
340
341 return true;
342}
343
344static bool
345vf_fs_configure_capture(struct xrt_fs *xfs, struct xrt_fs_capture_parameters *cp)
346{
347 // struct vf_fs *vid = vf_fs(xfs);
348 //! @todo
349 return false;
350}
351
352static bool
353vf_fs_stream_start(struct xrt_fs *xfs,
354 struct xrt_frame_sink *xs,
355 enum xrt_fs_capture_type capture_type,
356 uint32_t descriptor_index)
357{
358 struct vf_fs *vid = vf_fs(xfs);
359
360 vid->sink = xs;
361 vid->is_running = true;
362 vid->capture_type = capture_type;
363 vid->selected = descriptor_index;
364
365 gst_element_set_state(vid->source, GST_STATE_PLAYING);
366
367 VF_TRACE(vid, "info: Started!");
368
369 // we're off to the races!
370 return true;
371}
372
373static bool
374vf_fs_stream_stop(struct xrt_fs *xfs)
375{
376 struct vf_fs *vid = vf_fs(xfs);
377
378 if (!vid->is_running) {
379 return true;
380 }
381
382 vid->is_running = false;
383 gst_element_set_state(vid->source, GST_STATE_PAUSED);
384
385 return true;
386}
387
388static bool
389vf_fs_is_running(struct xrt_fs *xfs)
390{
391 struct vf_fs *vid = vf_fs(xfs);
392
393 GstState current = GST_STATE_NULL;
394 GstState pending;
395 gst_element_get_state(vid->source, ¤t, &pending, 0);
396
397 return current == GST_STATE_PLAYING;
398}
399
400static void
401vf_fs_destroy(struct vf_fs *vid)
402{
403 g_main_loop_quit(vid->loop);
404
405 // Destroy also stops the thread.
406 os_thread_helper_destroy(&vid->play_thread);
407
408 free(vid);
409}
410
411
412/*
413 *
414 * Node methods.
415 *
416 */
417
418static void
419vf_fs_node_break_apart(struct xrt_frame_node *node)
420{
421 struct vf_fs *vid = container_of(node, struct vf_fs, node);
422 vf_fs_stream_stop(&vid->base);
423}
424
425static void
426vf_fs_node_destroy(struct xrt_frame_node *node)
427{
428 struct vf_fs *vid = container_of(node, struct vf_fs, node);
429 vf_fs_destroy(vid);
430}
431
432
433/*
434 *
435 * Exported create functions and helper.
436 *
437 */
438
439static struct xrt_fs *
440alloc_and_init_common(struct xrt_frame_context *xfctx, //
441 enum xrt_format format, //
442 enum xrt_stereo_format stereo_format, //
443 gchar *pipeline_string) //
444{
445 struct vf_fs *vid = U_TYPED_CALLOC(struct vf_fs);
446 vid->got_sample = false;
447 vid->format = format;
448 vid->stereo_format = stereo_format;
449
450 GstBus *bus = NULL;
451
452 int ret = os_thread_helper_init(&vid->play_thread);
453 if (ret < 0) {
454 VF_ERROR(vid, "Failed to init thread");
455 g_free(pipeline_string);
456 free(vid);
457 return NULL;
458 }
459
460 vid->loop = g_main_loop_new(NULL, FALSE);
461 VF_DEBUG(vid, "Pipeline: %s", pipeline_string);
462
463 vid->source = gst_parse_launch(pipeline_string, NULL);
464 g_free(pipeline_string);
465
466 if (vid->source == NULL) {
467 VF_ERROR(vid, "Bad source");
468 g_main_loop_unref(vid->loop);
469 free(vid);
470 return NULL;
471 }
472
473 vid->testsink = gst_bin_get_by_name(GST_BIN(vid->source), "testsink");
474 g_object_set(G_OBJECT(vid->testsink), "emit-signals", TRUE, "sync", TRUE, NULL);
475 g_signal_connect(vid->testsink, "new-sample", G_CALLBACK(on_new_sample_from_sink), vid);
476
477 bus = gst_element_get_bus(vid->source);
478 gst_bus_add_watch(bus, (GstBusFunc)on_source_message, vid);
479 gst_object_unref(bus);
480
481 ret = os_thread_helper_start(&vid->play_thread, vf_fs_mainloop, vid);
482 if (ret != 0) {
483 VF_ERROR(vid, "Failed to start thread '%i'", ret);
484 g_main_loop_unref(vid->loop);
485 free(vid);
486 return NULL;
487 }
488
489 // We need one sample to determine frame size.
490 VF_DEBUG(vid, "Waiting for frame");
491 gst_element_set_state(vid->source, GST_STATE_PLAYING);
492 while (!vid->got_sample) {
493 os_nanosleep(100 * 1000 * 1000);
494 }
495 VF_DEBUG(vid, "Got first sample");
496 gst_element_set_state(vid->source, GST_STATE_PAUSED);
497
498 vid->base.enumerate_modes = vf_fs_enumerate_modes;
499 vid->base.configure_capture = vf_fs_configure_capture;
500 vid->base.stream_start = vf_fs_stream_start;
501 vid->base.stream_stop = vf_fs_stream_stop;
502 vid->base.is_running = vf_fs_is_running;
503 vid->node.break_apart = vf_fs_node_break_apart;
504 vid->node.destroy = vf_fs_node_destroy;
505 vid->log_level = debug_get_log_option_vf_log();
506
507 // It's now safe to add it to the context.
508 xrt_frame_context_add(xfctx, &vid->node);
509
510 // Start the variable tracking after we know what device we have.
511 // clang-format off
512 u_var_add_root(vid, "Video File Frameserver", true);
513 u_var_add_ro_text(vid, vid->base.name, "Card");
514 u_var_add_log_level(vid, &vid->log_level, "Log Level");
515 // clang-format on
516
517 return &(vid->base);
518}
519
520struct xrt_fs *
521vf_fs_videotestsource(struct xrt_frame_context *xfctx, uint32_t width, uint32_t height)
522{
523 gst_init(0, NULL);
524
525 enum xrt_format format = XRT_FORMAT_R8G8B8;
526 enum xrt_stereo_format stereo_format = XRT_STEREO_FORMAT_NONE;
527
528 gchar *pipeline_string = g_strdup_printf(
529 "videotestsrc name=source ! "
530 "clockoverlay ! "
531 "videoconvert ! "
532 "videoscale ! "
533 "video/x-raw,format=RGB,width=%u,height=%u ! "
534 "appsink name=testsink",
535 width, height);
536
537 return alloc_and_init_common(xfctx, format, stereo_format, pipeline_string);
538}
539
540struct xrt_fs *
541vf_fs_open_file(struct xrt_frame_context *xfctx, const char *path)
542{
543 if (path == NULL) {
544 U_LOG_E("No path given");
545 return NULL;
546 }
547
548 gst_init(0, NULL);
549
550 if (!g_file_test(path, G_FILE_TEST_EXISTS)) {
551 U_LOG_E("File %s does not exist", path);
552 return NULL;
553 }
554
555#if 0
556 const gchar *caps = "video/x-raw,format=RGB";
557 enum xrt_format format = XRT_FORMAT_R8G8B8;
558 enum xrt_stereo_format stereo_format = XRT_STEREO_FORMAT_NONE;
559#endif
560
561#if 0
562 // For hand tracking
563 const gchar *caps = "video/x-raw,format=RGB";
564 enum xrt_format format = XRT_FORMAT_R8G8B8;
565 enum xrt_stereo_format stereo_format = XRT_STEREO_FORMAT_SBS;
566#endif
567
568#if 1
569 const gchar *caps = "video/x-raw,format=YUY2";
570 enum xrt_format format = XRT_FORMAT_YUYV422;
571 enum xrt_stereo_format stereo_format = XRT_STEREO_FORMAT_SBS;
572#endif
573
574 gchar *loop = "false";
575
576 gchar *pipeline_string = g_strdup_printf(
577 "multifilesrc location=\"%s\" loop=%s ! "
578 "decodebin ! "
579 "videoconvert ! "
580 "appsink caps=\"%s\" name=testsink",
581 path, loop, caps);
582
583 return alloc_and_init_common(xfctx, format, stereo_format, pipeline_string);
584}