feat: use gpu texture in screen consumer

niklaspandersson · niklaspandersson · commit 39af23c8eab6 · 2025-06-26T08:52:18.000+02:00
* Add texture property to const_frame that references the texture on the gpu
* Replace GPU-CPU-GPU copy with GPU texture in screen consumer
diff --git a/src/accelerator/ogl/image/image_mixer.cpp b/src/accelerator/ogl/image/image_mixer.cpp
@@ -84,22 +84,29 @@ class image_renderer
     {
     }
 
-    std::future<array<const std::uint8_t>> operator()(std::vector<layer>             layers,
-                                                      const core::video_format_desc& format_desc)
+    std::tuple<std::future<array<const std::uint8_t>>, std::shared_ptr<texture>>
+    operator()(std::vector<layer> layers, const core::video_format_desc& format_desc)
     {
         if (layers.empty()) { // Bypass GPU with empty frame.
-            static const std::vector<uint8_t, boost::alignment::aligned_allocator<uint8_t, 32>> buffer(max_frame_size_, 0);
-            return make_ready_future(array<const std::uint8_t>(buffer.data(), format_desc.size, true));
+            static const std::vector<uint8_t, boost::alignment::aligned_allocator<uint8_t, 32>> buffer(max_frame_size_,
+                                                                                                       0);
+            return {make_ready_future(array<const std::uint8_t>(buffer.data(), format_desc.size, true)), nullptr};
         }
 
-        return flatten(ogl_->dispatch_async(
-            [=, layers = std::move(layers)]() mutable -> std::shared_future<array<const std::uint8_t>> {
-                auto target_texture = ogl_->create_texture(format_desc.width, format_desc.height, 4, depth_);
+        std::shared_ptr<texture>               target_texture;
+        std::future<array<const std::uint8_t>> data_future =
+            ogl_->dispatch_async([=,
+                                  &target_texture,
+                                  layers = std::move(layers)]() mutable -> std::future<array<const std::uint8_t>> {
+                    target_texture = ogl_->create_texture(format_desc.width, format_desc.height, 4, depth_);
 
-                draw(target_texture, std::move(layers), format_desc);
+                    draw(target_texture, std::move(layers), format_desc);
 
-                return ogl_->copy_async(target_texture);
-            }));
+                    return ogl_->copy_async(target_texture);
+                })
+                .get();
+
+        return {std::move(data_future), target_texture};
     }
 
     common::bit_depth depth() const { return depth_; }
@@ -318,7 +325,8 @@ struct image_mixer::impl
         layer_stack_.resize(transform_stack_.back().image_transform.layer_depth);
     }
 
-    std::future<array<const std::uint8_t>> render(const core::video_format_desc& format_desc)
+    std::tuple<std::future<array<const std::uint8_t>>, std::shared_ptr<texture>>
+    render(const core::video_format_desc& format_desc)
     {
         return renderer_(std::move(layers_), format_desc);
     }
@@ -374,7 +382,8 @@ void image_mixer::push(const core::frame_transform& transform) { impl_->push(tra
 void image_mixer::visit(const core::const_frame& frame) { impl_->visit(frame); }
 void image_mixer::pop() { impl_->pop(); }
 void image_mixer::update_aspect_ratio(double aspect_ratio) { impl_->update_aspect_ratio(aspect_ratio); }
-std::future<array<const std::uint8_t>> image_mixer::render(const core::video_format_desc& format_desc)
+std::tuple<std::future<array<const std::uint8_t>>, std::shared_ptr<core::texture>>
+image_mixer::render(const core::video_format_desc& format_desc)
 {
     return impl_->render(format_desc);
 }
diff --git a/src/accelerator/ogl/image/image_mixer.h b/src/accelerator/ogl/image/image_mixer.h
@@ -47,8 +47,9 @@ class image_mixer final : public core::image_mixer
 
     image_mixer& operator=(const image_mixer&) = delete;
 
-    std::future<array<const std::uint8_t>> render(const core::video_format_desc& format_desc) override;
-    core::mutable_frame                    create_frame(const void* tag, const core::pixel_format_desc& desc) override;
+    std::tuple<std::future<array<const std::uint8_t>>, std::shared_ptr<core::texture>>
+                        render(const core::video_format_desc& format_desc) override;
+    core::mutable_frame create_frame(const void* tag, const core::pixel_format_desc& desc) override;
     core::mutable_frame
     create_frame(const void* video_stream_tag, const core::pixel_format_desc& desc, common::bit_depth depth) override;
 
diff --git a/src/accelerator/ogl/util/context_egl.cpp b/src/accelerator/ogl/util/context_egl.cpp
@@ -10,6 +10,10 @@ namespace caspar::accelerator::ogl {
 
 struct device_context::impl
 {
+private:
+    static EGLContext Prev_Created_Context;
+
+public:
     EGLDisplay eglDisplay_;
     EGLContext eglContext_;
 
@@ -46,10 +50,12 @@ struct device_context::impl
             CASPAR_THROW_EXCEPTION(gl::ogl_exception() << msg_info("Failed to initialize OpenGL: eglBindAPI"));
         }
 
-        eglContext_ = eglCreateContext(eglDisplay_, eglConfig, EGL_NO_CONTEXT, NULL);
+        eglContext_ = eglCreateContext(eglDisplay_, eglConfig, device_context::impl::Prev_Created_Context, NULL);
         if (eglContext_ == EGL_NO_CONTEXT) {
             CASPAR_THROW_EXCEPTION(gl::ogl_exception() << msg_info("Failed to initialize OpenGL: eglCreateContext"));
         }
+        // TODO: Access ought to be protexted bu mutex
+        device_context::impl::Prev_Created_Context = eglContext_;
 
         if (!eglMakeCurrent(eglDisplay_, EGL_NO_SURFACE, EGL_NO_SURFACE, eglContext_)) {
             CASPAR_THROW_EXCEPTION(gl::ogl_exception() << msg_info("Failed to initialize OpenGL: eglMakeCurrent"));
@@ -68,6 +74,8 @@ struct device_context::impl
     }
 };
 
+EGLContext device_context::impl::Prev_Created_Context = EGL_NO_CONTEXT;
+
 device_context::device_context()
     : impl_(new impl())
 {
diff --git a/src/accelerator/ogl/util/texture.h b/src/accelerator/ogl/util/texture.h
@@ -22,11 +22,12 @@
 #pragma once
 
 #include <common/bit_depth.h>
+#include <core/frame/frame.h>
 #include <memory>
 
 namespace caspar { namespace accelerator { namespace ogl {
 
-class texture final
+class texture final : public core::texture
 {
   public:
     texture(int width, int height, int stride, common::bit_depth depth = common::bit_depth::bit8);
@@ -45,8 +46,9 @@ class texture final
 
     void attach();
     void clear();
-    void bind(int index);
-    void unbind();
+
+    virtual void bind(int index) override;
+    virtual void unbind() override;
 
     int               width() const;
     int               height() const;
diff --git a/src/core/frame/frame.cpp b/src/core/frame/frame.cpp
@@ -92,32 +92,35 @@ struct const_frame::impl
 {
     std::vector<array<const std::uint8_t>> image_data_;
     array<const std::int32_t>              audio_data_;
-    core::pixel_format_desc                desc_     = core::pixel_format_desc(pixel_format::invalid);
+    core::pixel_format_desc                desc_ = core::pixel_format_desc(pixel_format::invalid);
     const void*                            tag_;
     frame_geometry                         geometry_ = frame_geometry::get_default();
     std::any                               opaque_;
+    std::shared_ptr<core::texture>         texture_;
 
     impl(const void*                            tag,
          std::vector<array<const std::uint8_t>> image_data,
          array<const std::int32_t>              audio_data,
-         const core::pixel_format_desc&         desc)
+         const core::pixel_format_desc&         desc,
+         std::shared_ptr<core::texture>         texture)
         : image_data_(std::move(image_data))
         , audio_data_(std::move(audio_data))
         , desc_(desc)
         , tag_(tag)
+        , texture_(std::move(texture))
     {
         if (desc_.planes.size() != image_data_.size()) {
             CASPAR_THROW_EXCEPTION(invalid_argument());
         }
     }
 
-
     impl(mutable_frame&& other)
         : image_data_(std::make_move_iterator(other.impl_->image_data_.begin()),
                       std::make_move_iterator(other.impl_->image_data_.end()))
         , audio_data_(std::move(other.impl_->audio_data_))
         , desc_(std::move(other.impl_->desc_))
         , tag_(other.stream_tag())
+        , texture_(nullptr)
         , geometry_(std::move(other.impl_->geometry_))
     {
         if (desc_.planes.size() != image_data_.size() && !other.impl_->commit_) {
@@ -130,6 +133,8 @@ struct const_frame::impl
 
     const array<const std::uint8_t>& image_data(std::size_t index) const { return image_data_.at(index); }
 
+    std::shared_ptr<core::texture> texture() { return texture_; }
+
     std::size_t width() const { return desc_.planes.at(0).width; }
 
     std::size_t height() const { return desc_.planes.at(0).height; }
@@ -141,8 +146,9 @@ const_frame::const_frame() {}
 const_frame::const_frame(const void*                            tag,
                          std::vector<array<const std::uint8_t>> image_data,
                          array<const std::int32_t>              audio_data,
-                         const core::pixel_format_desc&         desc)
-    : impl_(new impl(tag, std::move(image_data), std::move(audio_data), desc))
+                         const core::pixel_format_desc&         desc,
+                         std::shared_ptr<core::texture>         texture)
+    : impl_(new impl(tag, std::move(image_data), std::move(audio_data), desc, texture))
 {
 }
 const_frame::const_frame(mutable_frame&& other)
@@ -166,26 +172,28 @@ bool                     const_frame::operator>(const const_frame& other) const
 const pixel_format_desc& const_frame::pixel_format_desc() const { return impl_->desc_; }
 const array<const std::uint8_t>& const_frame::image_data(std::size_t index) const { return impl_->image_data(index); }
 const array<const std::int32_t>& const_frame::audio_data() const { return impl_->audio_data_; }
+std::shared_ptr<core::texture>   const_frame::texture() { return impl_->texture(); }
 std::size_t                      const_frame::width() const { return impl_->width(); }
 std::size_t                      const_frame::height() const { return impl_->height(); }
 std::size_t                      const_frame::size() const { return impl_->size(); }
 const void*                      const_frame::stream_tag() const { return impl_->tag_; }
-const_frame                      const_frame::with_tag(const void* new_tag) const {
+const_frame                      const_frame::with_tag(const void* new_tag) const
+{
     if (!impl_) {
         return const_frame();
     }
-    
+
     std::vector<array<const std::uint8_t>> image_data_copy = impl_->image_data_;
     auto new_frame = const_frame(new_tag, std::move(image_data_copy), impl_->audio_data_, impl_->desc_);
-    
+
     new_frame.impl_->geometry_ = impl_->geometry_;
     if (impl_->opaque_.has_value()) {
         new_frame.impl_->opaque_ = impl_->opaque_;
     }
-    
+
     return new_frame;
 }
-const frame_geometry&            const_frame::geometry() const { return impl_->geometry_; }
-const std::any&                  const_frame::opaque() const { return impl_->opaque_; }
+const frame_geometry& const_frame::geometry() const { return impl_->geometry_; }
+const std::any&       const_frame::opaque() const { return impl_->opaque_; }
 const_frame::operator bool() const { return impl_ != nullptr && impl_->desc_.format != core::pixel_format::invalid; }
 }} // namespace caspar::core
diff --git a/src/core/frame/frame.h b/src/core/frame/frame.h
@@ -11,6 +11,14 @@
 
 namespace caspar { namespace core {
 
+class texture
+{
+  public:
+    virtual ~texture() {}
+    virtual void bind(int index) = 0;
+    virtual void unbind()        = 0;
+};
+
 class mutable_frame final
 {
     friend class const_frame;
@@ -62,7 +70,8 @@ class const_frame final
     explicit const_frame(const void*                            tag,
                          std::vector<array<const std::uint8_t>> image_data,
                          array<const std::int32_t>              audio_data,
-                         const struct pixel_format_desc&        desc);
+                         const struct pixel_format_desc&        desc,
+                         std::shared_ptr<core::texture>         texture = nullptr);
     const_frame(const const_frame& other);
     const_frame(mutable_frame&& other);
 
@@ -76,6 +85,8 @@ class const_frame final
 
     const array<const std::int32_t>& audio_data() const;
 
+    std::shared_ptr<core::texture> texture();
+
     std::size_t width() const;
 
     std::size_t height() const;
diff --git a/src/core/mixer/image/image_mixer.h b/src/core/mixer/image/image_mixer.h
@@ -48,7 +48,8 @@ class image_mixer
 
     virtual void update_aspect_ratio(double aspect_ratio) = 0;
 
-    virtual std::future<array<const uint8_t>> render(const struct video_format_desc& format_desc) = 0;
+    virtual std::tuple<std::future<array<const std::uint8_t>>, std::shared_ptr<core::texture>>
+    render(const struct video_format_desc& format_desc) = 0;
 
     class mutable_frame create_frame(const void* tag, const struct pixel_format_desc& desc) override = 0;
     class mutable_frame create_frame(const void*                     video_stream_tag,
diff --git a/src/core/mixer/mixer.cpp b/src/core/mixer/mixer.cpp
@@ -71,17 +71,20 @@ struct mixer::impl
             frame.accept(*image_mixer_);
         }
 
-        auto image = image_mixer_->render(format_desc);
-        auto audio = audio_mixer_(format_desc, nb_samples);
+        auto result  = image_mixer_->render(format_desc);
+        auto image   = std::move(std::get<0>(result));
+        auto texture = std::move(std::get<1>(result));
+        auto audio   = audio_mixer_(format_desc, nb_samples);
 
         state_["audio"] = audio_mixer_.state();
 
         auto depth = image_mixer_->depth();
 
         buffer_.push(std::async(std::launch::deferred,
-                                [image = std::move(image),
-                                 audio = std::move(audio),
-                                 graph = graph_,
+                                [image   = std::move(image),
+                                 audio   = std::move(audio),
+                                 texture = std::move(texture),
+                                 graph   = graph_,
                                  depth,
                                  format_desc,
                                  tag = this]() mutable {
@@ -90,7 +93,7 @@ struct mixer::impl
                                         pixel_format_desc::plane(format_desc.width, format_desc.height, 4, depth));
                                     std::vector<array<const uint8_t>> image_data;
                                     image_data.emplace_back(std::move(image.get()));
-                                    return const_frame(tag, std::move(image_data), std::move(audio), desc);
+                                    return const_frame(tag, std::move(image_data), std::move(audio), desc, texture);
                                 }));
 
         if (buffer_.size() <= format_desc.field_count) {
diff --git a/src/modules/screen/consumer/screen_consumer.cpp b/src/modules/screen/consumer/screen_consumer.cpp

Original file line number	Diff line number	Diff line change
`@@ -84,22 +84,29 @@ class image_renderer`
`84`	`84`	`{`
`85`	`85`	`}`
`86`	`86`
`87`		`- std::future<array<const std::uint8_t>> operator()(std::vector<layer> layers,`
`88`		`- const core::video_format_desc& format_desc)`
	`87`	`+ std::tuple<std::future<array<const std::uint8_t>>, std::shared_ptr<texture>>`
	`88`	`+ operator()(std::vector<layer> layers, const core::video_format_desc& format_desc)`
`89`	`89`	`{`
`90`	`90`	`if (layers.empty()) { // Bypass GPU with empty frame.`
`91`		`- static const std::vector<uint8_t, boost::alignment::aligned_allocator<uint8_t, 32>> buffer(max_frame_size_, 0);`
`92`		`- return make_ready_future(array<const std::uint8_t>(buffer.data(), format_desc.size, true));`
	`91`	`+ static const std::vector<uint8_t, boost::alignment::aligned_allocator<uint8_t, 32>> buffer(max_frame_size_,`
	`92`	`+ 0);`
	`93`	`+ return {make_ready_future(array<const std::uint8_t>(buffer.data(), format_desc.size, true)), nullptr};`
`93`	`94`	`}`
`94`	`95`
`95`		`- return flatten(ogl_->dispatch_async(`
`96`		`- [=, layers = std::move(layers)]() mutable -> std::shared_future<array<const std::uint8_t>> {`
`97`		`- auto target_texture = ogl_->create_texture(format_desc.width, format_desc.height, 4, depth_);`
	`96`	`+ std::shared_ptr<texture> target_texture;`
	`97`	`+ std::future<array<const std::uint8_t>> data_future =`
	`98`	`+ ogl_->dispatch_async([=,`
	`99`	`+ &target_texture,`
	`100`	`+ layers = std::move(layers)]() mutable -> std::future<array<const std::uint8_t>> {`
	`101`	`+ target_texture = ogl_->create_texture(format_desc.width, format_desc.height, 4, depth_);`
`98`	`102`
`99`		`- draw(target_texture, std::move(layers), format_desc);`
	`103`	`+ draw(target_texture, std::move(layers), format_desc);`
`100`	`104`
`101`		`- return ogl_->copy_async(target_texture);`
`102`		`- }));`
	`105`	`+ return ogl_->copy_async(target_texture);`
	`106`	`+ })`
	`107`	`+ .get();`
	`108`	`+`
	`109`	`+ return {std::move(data_future), target_texture};`
`103`	`110`	`}`
`104`	`111`
`105`	`112`	`common::bit_depth depth() const { return depth_; }`
`@@ -318,7 +325,8 @@ struct image_mixer::impl`
`318`	`325`	`layer_stack_.resize(transform_stack_.back().image_transform.layer_depth);`
`319`	`326`	`}`
`320`	`327`
`321`		`- std::future<array<const std::uint8_t>> render(const core::video_format_desc& format_desc)`
	`328`	`+ std::tuple<std::future<array<const std::uint8_t>>, std::shared_ptr<texture>>`
	`329`	`+ render(const core::video_format_desc& format_desc)`
`322`	`330`	`{`
`323`	`331`	`return renderer_(std::move(layers_), format_desc);`
`324`	`332`	`}`
`@@ -374,7 +382,8 @@ void image_mixer::push(const core::frame_transform& transform) { impl_->push(tra`
`374`	`382`	`void image_mixer::visit(const core::const_frame& frame) { impl_->visit(frame); }`
`375`	`383`	`void image_mixer::pop() { impl_->pop(); }`
`376`	`384`	`void image_mixer::update_aspect_ratio(double aspect_ratio) { impl_->update_aspect_ratio(aspect_ratio); }`
`377`		`-std::future<array<const std::uint8_t>> image_mixer::render(const core::video_format_desc& format_desc)`
	`385`	`+std::tuple<std::future<array<const std::uint8_t>>, std::shared_ptr<core::texture>>`
	`386`	`+image_mixer::render(const core::video_format_desc& format_desc)`
`378`	`387`	`{`
`379`	`388`	`return impl_->render(format_desc);`
`380`	`389`	`}`
Original file line number	Diff line number	Diff line change
`@@ -10,6 +10,10 @@ namespace caspar::accelerator::ogl {`
`10`	`10`
`11`	`11`	`struct device_context::impl`
`12`	`12`	`{`
	`13`	`+private:`
	`14`	`+ static EGLContext Prev_Created_Context;`
	`15`	`+`
	`16`	`+public:`
`13`	`17`	`EGLDisplay eglDisplay_;`
`14`	`18`	`EGLContext eglContext_;`
`15`	`19`
`@@ -46,10 +50,12 @@ struct device_context::impl`
`46`	`50`	`CASPAR_THROW_EXCEPTION(gl::ogl_exception() << msg_info("Failed to initialize OpenGL: eglBindAPI"));`
`47`	`51`	`}`
`48`	`52`
`49`		`- eglContext_ = eglCreateContext(eglDisplay_, eglConfig, EGL_NO_CONTEXT, NULL);`
	`53`	`+ eglContext_ = eglCreateContext(eglDisplay_, eglConfig, device_context::impl::Prev_Created_Context, NULL);`
`50`	`54`	`if (eglContext_ == EGL_NO_CONTEXT) {`
`51`	`55`	`CASPAR_THROW_EXCEPTION(gl::ogl_exception() << msg_info("Failed to initialize OpenGL: eglCreateContext"));`
`52`	`56`	`}`
	`57`	`+ // TODO: Access ought to be protexted bu mutex`
	`58`	`+ device_context::impl::Prev_Created_Context = eglContext_;`
`53`	`59`
`54`	`60`	`if (!eglMakeCurrent(eglDisplay_, EGL_NO_SURFACE, EGL_NO_SURFACE, eglContext_)) {`
`55`	`61`	`CASPAR_THROW_EXCEPTION(gl::ogl_exception() << msg_info("Failed to initialize OpenGL: eglMakeCurrent"));`
`@@ -68,6 +74,8 @@ struct device_context::impl`
`68`	`74`	`}`
`69`	`75`	`};`
`70`	`76`
	`77`	`+EGLContext device_context::impl::Prev_Created_Context = EGL_NO_CONTEXT;`
	`78`	`+`
`71`	`79`	`device_context::device_context()`
`72`	`80`	`: impl_(new impl())`
`73`	`81`	`{`