From e1614a280f7cd97788ec1868114099694cd3758d Mon Sep 17 00:00:00 2001 From: John Preston Date: Sat, 29 May 2021 17:00:09 +0400 Subject: [PATCH] Add blur effect for video tile background. --- .../calls/group/calls_group_viewport.cpp | 17 +- .../calls/group/calls_group_viewport.h | 4 +- .../group/calls_group_viewport_opengl.cpp | 534 +++++++++++++----- .../calls/group/calls_group_viewport_opengl.h | 44 +- .../calls/group/calls_group_viewport_tile.cpp | 17 - .../calls/group/calls_group_viewport_tile.h | 18 - Telegram/lib_ui | 2 +- 7 files changed, 440 insertions(+), 196 deletions(-) diff --git a/Telegram/SourceFiles/calls/group/calls_group_viewport.cpp b/Telegram/SourceFiles/calls/group/calls_group_viewport.cpp index 5717e64333..4da576a4ca 100644 --- a/Telegram/SourceFiles/calls/group/calls_group_viewport.cpp +++ b/Telegram/SourceFiles/calls/group/calls_group_viewport.cpp @@ -40,13 +40,7 @@ Viewport::Viewport(not_null parent, PanelMode mode) setup(); } -Viewport::~Viewport() { - for (const auto &tile : base::take(_tiles)) { - if (auto textures = tile->takeTextures()) { - _freeTextures(base::take(textures)); - } - } -} +Viewport::~Viewport() = default; not_null Viewport::widget() const { return _content->rpWidget(); @@ -237,9 +231,6 @@ void Viewport::remove(const VideoEndpoint &endpoint) { } if (_pressed.tile == removing) { setPressed({}); - } - if (const auto textures = removing->takeTextures()) { - } _tiles.erase(i); updateTilesGeometry(); @@ -511,9 +502,7 @@ Ui::GL::ChosenRenderer Viewport::chooseRenderer( LOG(("OpenGL: %1 (Calls::Group::Viewport)").arg(Logs::b(use))); if (use) { auto renderer = std::make_unique(this); - _freeTextures = [raw = renderer.get()](const Textures &textures) { - raw->free(textures); - }; + _opengl = true; return { .renderer = std::move(renderer), .backend = Ui::GL::Backend::OpenGL, @@ -526,7 +515,7 @@ Ui::GL::ChosenRenderer Viewport::chooseRenderer( } bool Viewport::requireARGB32() const { - return !_freeTextures; + return !_opengl; } int Viewport::fullHeight() const { diff --git a/Telegram/SourceFiles/calls/group/calls_group_viewport.h b/Telegram/SourceFiles/calls/group/calls_group_viewport.h index a2dfb7fbf3..63aa9b79e1 100644 --- a/Telegram/SourceFiles/calls/group/calls_group_viewport.h +++ b/Telegram/SourceFiles/calls/group/calls_group_viewport.h @@ -89,6 +89,8 @@ public: [[nodiscard]] rpl::producer qualityRequests() const; [[nodiscard]] rpl::producer mouseInsideValue() const; + [[nodiscard]] uint32 defaultFramebufferObject() const; + [[nodiscard]] rpl::lifetime &lifetime(); private: @@ -134,7 +136,7 @@ private: [[nodiscard]] Ui::GL::ChosenRenderer chooseRenderer( Ui::GL::Capabilities capabilities); - Fn _freeTextures; + bool _opengl = false; PanelMode _mode = PanelMode(); bool _geometryStaleAfterModeChange = false; const std::unique_ptr _content; diff --git a/Telegram/SourceFiles/calls/group/calls_group_viewport_opengl.cpp b/Telegram/SourceFiles/calls/group/calls_group_viewport_opengl.cpp index 7e84ffea47..eaf85ea912 100644 --- a/Telegram/SourceFiles/calls/group/calls_group_viewport_opengl.cpp +++ b/Telegram/SourceFiles/calls/group/calls_group_viewport_opengl.cpp @@ -22,12 +22,50 @@ namespace { using namespace Ui::GL; +constexpr auto kScaleForBlurTextureIndex = 3; +constexpr auto kFirstBlurPassTextureIndex = 4; +constexpr auto kBlurTextureSizeFactor = 1.7; +constexpr auto kBlurOpacity = 0.5; + +ShaderPart FragmentBlurTexture(bool vertical, char prefix = 'v') { + const auto offsets = (vertical ? QString("0, 1") : QString("1, 0")); + const auto name = prefix + QString("_texcoord"); + return { + .header = R"( +varying vec2 )" + name + R"(; +uniform sampler2D b_texture; +uniform float texelOffset; +const vec3 satLuminanceWeighting = vec3(0.2126, 0.7152, 0.0722); +const vec2 offsets = vec2()" + offsets + R"(); +const int radius = 15; +const int diameter = 2 * radius + 1; +)", + .body = R"( + vec4 accumulated = vec4(0.); + for (int i = 0; i != diameter; i++) { + float stepOffset = float(i - radius) * texelOffset; + vec2 offset = vec2(stepOffset) * offsets; + vec4 sampled = vec4(texture2D(b_texture, )" + name + R"( + offset)); + float fradius = float(radius); + float boxWeight = fradius + 1.0 - abs(float(i) - fradius); + accumulated += sampled * boxWeight; + } + vec3 blurred = accumulated.rgb / accumulated.a; + float satLuminance = dot(blurred, satLuminanceWeighting); + vec3 mixinColor = vec3(satLuminance); + result = vec4(clamp(mix(mixinColor, blurred, 1.1), 0.0, 1.0), 1.0); +)", + }; +} + // Depends on FragmetSampleTexture(). [[nodiscard]] ShaderPart FragmentFrameColor() { + const auto blur = FragmentBlurTexture(true, 'b'); return { .header = R"( uniform vec4 frameBg; uniform vec3 shadow; // fullHeight, shown, maxOpacity +const float backgroundOpacity = )" + QString::number(kBlurOpacity) + R"(; float insideTexture() { vec2 textureHalf = vec2(0.5, 0.5); vec2 fromTextureCenter = abs(v_texcoord - textureHalf); @@ -35,10 +73,18 @@ float insideTexture() { float outsideCheck = dot(fromTextureEdge, fromTextureEdge); return step(outsideCheck, 0); } +)" + blur.header + R"( +vec4 background() { + vec4 result; +)" + blur.body + R"( + return result; +} )", .body = R"( float inside = insideTexture(); - result = result * inside + frameBg * (1. - inside); + result = result * inside + + (1. - inside) * (backgroundOpacity * background() + + (1. - backgroundOpacity) * frameBg); float shadowCoord = gl_FragCoord.y - roundRect.y; float shadowValue = max(1. - (shadowCoord / shadow.x), 0.); @@ -48,6 +94,51 @@ float insideTexture() { }; } +[[nodiscard]] QSize NonEmpty(QSize size) { + return QSize(std::max(size.width(), 1), std::max(size.height(), 1)); +} + +[[nodiscard]] QSize CountBlurredSize( + QSize unscaled, + QSize viewport, + float factor) { + factor *= kBlurTextureSizeFactor; // The more the scale - more blurred the image. + const auto area = viewport / int(std::round(factor * cScale() / 100)); + const auto scaled = unscaled.scaled(area, Qt::KeepAspectRatio); + return (scaled.width() > unscaled.width() + || scaled.height() > unscaled.height()) + ? unscaled + : NonEmpty(scaled); +} + +[[nodiscard]] std::array, 4> CountTexCoords( + QSize unscaled, + QSize size, + bool expand, + bool swap = false) { + const auto scaled = NonEmpty(unscaled.scaled( + size, + expand ? Qt::KeepAspectRatioByExpanding : Qt::KeepAspectRatio)); + const auto left = (size.width() - scaled.width()) / 2; + const auto top = (size.height() - scaled.height()) / 2; + const auto right = left + scaled.width(); + const auto bottom = top + scaled.height(); + auto dleft = float(left) / scaled.width(); + auto dright = float(size.width() - left) / scaled.width(); + auto dtop = float(top) / scaled.height(); + auto dbottom = float(size.height() - top) / scaled.height(); + if (swap) { + std::swap(dleft, dtop); + std::swap(dright, dbottom); + } + return { { + { { -dleft, 1.f + dtop } }, + { { dright, 1.f + dtop } }, + { { dright, 1.f - dbottom } }, + { { -dleft, 1.f - dbottom } }, + } }; +} + void FillRectVertices(GLfloat *coords, Rect rect) { coords[0] = coords[10] = rect.left(); coords[1] = coords[11] = rect.top(); @@ -116,7 +207,7 @@ void FillTexturedRectangle( shift(0)); f.glEnableVertexAttribArray(position); - GLint texcoord = program->attributeLocation("texcoord"); + GLint texcoord = program->attributeLocation("v_texcoordIn"); f.glVertexAttribPointer( texcoord, 2, @@ -150,10 +241,6 @@ Viewport::RendererGL::RendererGL(not_null owner) }, _lifetime); } -void Viewport::RendererGL::free(const Textures &textures) { - _texturesToFree.push_back(textures); -} - void Viewport::RendererGL::init( not_null widget, QOpenGLFunctions &f) { @@ -162,13 +249,34 @@ void Viewport::RendererGL::init( _frameBuffer->setUsagePattern(QOpenGLBuffer::DynamicDraw); _frameBuffer->create(); _frameBuffer->bind(); - _frameBuffer->allocate(64 * sizeof(GLfloat)); - _yuv420Program.emplace(); + constexpr auto kQuads = 6; + constexpr auto kQuadVertices = kQuads * 4; + constexpr auto kQuadValues = kQuadVertices * 4; + constexpr auto kValues = kQuadValues + 8; // Blur texture coordinates. + _frameBuffer->allocate(kValues * sizeof(GLfloat)); + _downscaleProgram.yuv420.emplace(); + _downscaleVertexShader = LinkProgram( + &*_downscaleProgram.yuv420, + VertexShader({ + VertexPassTextureCoord(), + }), + FragmentShader({ + FragmentSampleYUV420Texture(), + })).vertex; + _blurProgram.emplace(); + LinkProgram( + &*_blurProgram, + _downscaleVertexShader, + FragmentShader({ + FragmentBlurTexture(false), + })); + _frameProgram.yuv420.emplace(); _frameVertexShader = LinkProgram( - &*_yuv420Program, + &*_frameProgram.yuv420, VertexShader({ VertexViewportTransform(), VertexPassTextureCoord(), + VertexPassTextureCoord('b'), }), FragmentShader({ FragmentSampleYUV420Texture(), @@ -195,11 +303,20 @@ void Viewport::RendererGL::init( } void Viewport::RendererGL::ensureARGB32Program() { + Expects(_downscaleVertexShader != nullptr); Expects(_frameVertexShader != nullptr); - _argb32Program.emplace(); + _downscaleProgram.argb32.emplace(); LinkProgram( - &*_argb32Program, + &*_downscaleProgram.argb32, + _downscaleVertexShader, + FragmentShader({ + FragmentSampleARGB32Texture(), + })); + + _frameProgram.argb32.emplace(); + LinkProgram( + &*_frameProgram.argb32, _frameVertexShader, FragmentShader({ FragmentSampleARGB32Texture(), @@ -216,14 +333,16 @@ void Viewport::RendererGL::deinit( _frameVertexShader = nullptr; _bgProgram = std::nullopt; _imageProgram = std::nullopt; - _argb32Program = std::nullopt; - _yuv420Program = std::nullopt; - for (const auto &tile : _owner->_tiles) { - if (const auto textures = tile->takeTextures()) { - free(textures); - } + _downscaleProgram.argb32 = std::nullopt; + _downscaleProgram.yuv420 = std::nullopt; + _blurProgram = std::nullopt; + _frameProgram.argb32 = std::nullopt; + _frameProgram.yuv420 = std::nullopt; + for (auto &data : _tileData) { + data.textures.destroy(f); } - freeTextures(f); + _tileData.clear(); + _tileDataIndices.clear(); _buttons.destroy(f); } @@ -234,7 +353,12 @@ void Viewport::RendererGL::resize( int h) { _factor = widget->devicePixelRatio(); _viewport = QSize(w, h); - f.glViewport(0, 0, w * _factor, h * _factor); + setDefaultViewport(f); +} + +void Viewport::RendererGL::setDefaultViewport(QOpenGLFunctions &f) { + const auto size = _viewport * _factor; + f.glViewport(0, 0, size.width(), size.height()); } void Viewport::RendererGL::paint( @@ -243,13 +367,15 @@ void Viewport::RendererGL::paint( _factor = widget->devicePixelRatio(); validateDatas(); fillBackground(f); + const auto defaultFramebufferObject = widget->defaultFramebufferObject(); auto index = 0; for (const auto &tile : _owner->_tiles) { - auto &data = _tileData[_tileDataIndices[index++]]; - validateOutlineAnimation(tile.get(), data); - paintTile(f, tile.get(), data); + paintTile( + f, + defaultFramebufferObject, + tile.get(), + _tileData[_tileDataIndices[index++]]); } - freeTextures(f); } void Viewport::RendererGL::fillBackground(QOpenGLFunctions &f) { @@ -279,18 +405,21 @@ void Viewport::RendererGL::fillBackground(QOpenGLFunctions &f) { void Viewport::RendererGL::paintTile( QOpenGLFunctions &f, + GLuint defaultFramebufferObject, not_null tile, - const TileData &tileData) { + TileData &tileData) { const auto track = tile->track(); const auto data = track->frameWithInfo(false); if (data.format == Webrtc::FrameFormat::None) { return; } - + Assert(!data.yuv420->size.isEmpty()); const auto geometry = tile->geometry(); if (geometry.isEmpty()) { return; } + + _rgbaFrame = (data.format == Webrtc::FrameFormat::ARGB32); const auto x = geometry.x(); const auto y = geometry.y(); const auto width = geometry.width(); @@ -302,43 +431,37 @@ void Viewport::RendererGL::paintTile( const auto row = tile->row(); const auto style = row->computeIconState(MembersRowStyle::LargeVideo); + validateOutlineAnimation(tile, tileData); + const auto outline = tileData.outlined.value(tileData.outline ? 1. : 0.); + ensureButtonsImage(); // Frame. - const auto expand = !_owner->wide()/* && !tile->screencast()*/; - const auto scaled = Media::View::FlipSizeByRotation( + const auto unscaled = Media::View::FlipSizeByRotation( data.yuv420->size, - data.rotation - ).scaled( - QSize(width, height), - (expand ? Qt::KeepAspectRatioByExpanding : Qt::KeepAspectRatio)); - const auto good = !scaled.isEmpty(); - const auto left = (width - scaled.width()) / 2; - const auto top = (height - scaled.height()) / 2; - const auto right = left + scaled.width(); - const auto bottom = top + scaled.height(); - const auto radius = GLfloat(st::roundRadiusLarge); - auto dleft = good ? (float(left) / scaled.width()) : 0.f; - auto dright = good ? (float(width - left) / scaled.width()) : 1.f; - auto dtop = good ? (float(top) / scaled.height()) : 0.f; - auto dbottom = good ? (float(height - top) / scaled.height()) : 1.f; + data.rotation); const auto swap = (((data.rotation / 90) % 2) == 1); - if (swap) { - std::swap(dleft, dtop); - std::swap(dright, dbottom); - } + const auto expand = !_owner->wide()/* && !tile->screencast()*/; + auto texCoords = CountTexCoords(unscaled, geometry.size(), expand, swap); + auto blurTexCoords = expand + ? texCoords + : CountTexCoords(unscaled, geometry.size(), true); const auto rect = transformRect(geometry); - auto texCoord = std::array, 4> { { - { { -dleft, 1.f + dtop } }, - { { dright, 1.f + dtop } }, - { { dright, 1.f - dbottom } }, - { { -dleft, 1.f - dbottom } }, + auto toBlurTexCoords = std::array, 4> { { + { { 0.f, 1.f } }, + { { 1.f, 1.f } }, + { { 1.f, 0.f } }, + { { 0.f, 0.f } }, } }; - if (data.rotation > 0) { + if (const auto shift = (data.rotation / 90); shift > 0) { std::rotate( - texCoord.begin(), - texCoord.begin() + (data.rotation / 90), - texCoord.end()); + toBlurTexCoords.begin(), + toBlurTexCoords.begin() + shift, + toBlurTexCoords.end()); + std::rotate( + texCoords.begin(), + texCoords.begin() + shift, + texCoords.end()); } // Pin. @@ -377,18 +500,52 @@ void Viewport::RendererGL::paintTile( const auto nameRect = transformRect(name.geometry); const GLfloat coords[] = { + // YUV -> RGB-for-blur quad. + -1.f, 1.f, + toBlurTexCoords[0][0], toBlurTexCoords[0][1], + + 1.f, 1.f, + toBlurTexCoords[1][0], toBlurTexCoords[1][1], + + 1.f, -1.f, + toBlurTexCoords[2][0], toBlurTexCoords[2][1], + + -1.f, -1.f, + toBlurTexCoords[3][0], toBlurTexCoords[3][1], + + // First RGB -> RGB blur pass. + -1.f, 1.f, + 0.f, 1.f, + + 1.f, 1.f, + 1.f, 1.f, + + 1.f, -1.f, + 1.f, 0.f, + + -1.f, -1.f, + 0.f, 0.f, + + // Second blur pass + paint final frame. rect.left(), rect.top(), - texCoord[0][0], texCoord[0][1], + texCoords[0][0], texCoords[0][1], rect.right(), rect.top(), - texCoord[1][0], texCoord[1][1], + texCoords[1][0], texCoords[1][1], rect.right(), rect.bottom(), - texCoord[2][0], texCoord[2][1], + texCoords[2][0], texCoords[2][1], rect.left(), rect.bottom(), - texCoord[3][0], texCoord[3][1], + texCoords[3][0], texCoords[3][1], + // Additional blurred background texture coordinates. + blurTexCoords[0][0], blurTexCoords[0][1], + blurTexCoords[1][0], blurTexCoords[1][1], + blurTexCoords[2][0], blurTexCoords[2][1], + blurTexCoords[3][0], blurTexCoords[3][1], + + // Pin button. pinRect.left(), pinRect.top(), pin.texture.left(), pin.texture.bottom(), @@ -401,6 +558,7 @@ void Viewport::RendererGL::paintTile( pinRect.left(), pinRect.bottom(), pin.texture.left(), pin.texture.top(), + // Mute icon. muteRect.left(), muteRect.top(), mute.texture.left(), mute.texture.bottom(), @@ -413,6 +571,7 @@ void Viewport::RendererGL::paintTile( muteRect.left(), muteRect.bottom(), mute.texture.left(), mute.texture.top(), + // Name. nameRect.left(), nameRect.top(), name.texture.left(), name.texture.bottom(), @@ -426,84 +585,33 @@ void Viewport::RendererGL::paintTile( name.texture.left(), name.texture.top(), }; - tile->ensureTexturesCreated(f); - const auto &textures = tile->textures(); - const auto upload = (textures.trackIndex != data.index); - const auto uploadOne = [&]( - GLint internalformat, - GLint format, - QSize size, - int stride, - const void *data) { - f.glPixelStorei(GL_UNPACK_ROW_LENGTH, stride); - f.glTexImage2D( - GL_TEXTURE_2D, - 0, - internalformat, - size.width(), - size.height(), - 0, - format, - GL_UNSIGNED_BYTE, - data); - f.glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); - }; - if (upload) { - textures.textureIndex = 1 - textures.textureIndex; - } - const auto rgba = (data.format == Webrtc::FrameFormat::ARGB32); - if (rgba) { - ensureARGB32Program(); - f.glUseProgram(_argb32Program->programId()); - f.glActiveTexture(GL_TEXTURE0); - textures.values.bind(f, textures.textureIndex); - if (upload) { - const auto &image = data.original; - const auto stride = image.bytesPerLine() / 4; - const auto data = image.constBits(); - uploadOne(GL_RGB, GL_RGBA, image.size(), stride, data); - } - _argb32Program->setUniformValue("s_texture", GLint(0)); - } else { - const auto yuv = data.yuv420; - const auto otherSize = yuv->chromaSize; - f.glUseProgram(_yuv420Program->programId()); - f.glActiveTexture(GL_TEXTURE0); - textures.values.bind(f, textures.textureIndex * 3 + 0); - if (upload) { - f.glPixelStorei(GL_UNPACK_ALIGNMENT, 1); - uploadOne(GL_RED, GL_RED, yuv->size, yuv->y.stride, yuv->y.data); - } - f.glActiveTexture(GL_TEXTURE1); - textures.values.bind(f, textures.textureIndex * 3 + 1); - if (upload) { - uploadOne(GL_RED, GL_RED, otherSize, yuv->u.stride, yuv->u.data); - } - f.glActiveTexture(GL_TEXTURE2); - textures.values.bind(f, textures.textureIndex * 3 + 2); - if (upload) { - uploadOne(GL_RED, GL_RED, otherSize, yuv->v.stride, yuv->v.data); - f.glPixelStorei(GL_UNPACK_ALIGNMENT, 4); - } - _yuv420Program->setUniformValue("y_texture", GLint(0)); - _yuv420Program->setUniformValue("u_texture", GLint(1)); - _yuv420Program->setUniformValue("v_texture", GLint(2)); - } - tile->track()->markFrameShown(); + const auto blurSize = CountBlurredSize(unscaled, _viewport, _factor); + prepareObjects(f, tileData, blurSize); + f.glViewport(0, 0, blurSize.width(), blurSize.height()); _frameBuffer->bind(); _frameBuffer->write(0, coords, sizeof(coords)); - const auto program = rgba ? &*_argb32Program : &*_yuv420Program; + bindFrame(f, data, tileData, _downscaleProgram); + tile->track()->markFrameShown(); + + drawDownscalePass(f, tileData); + drawFirstBlurPass(f, tileData, blurSize); + + f.glBindFramebuffer(GL_FRAMEBUFFER, defaultFramebufferObject); + setDefaultViewport(f); + + bindFrame(f, data, tileData, _frameProgram); + + const auto program = _rgbaFrame + ? &*_frameProgram.argb32 + : &*_frameProgram.yuv420; const auto uniformViewport = QSizeF(_viewport * _factor); program->setUniformValue("viewport", uniformViewport); - program->setUniformValue( - "frameBg", - Uniform(st::groupCallMembersBg->c)); - const auto outline = tileData.outlined.value(tileData.outline ? 1. : 0.); + program->setUniformValue("frameBg", Uniform(st::groupCallBg->c)); program->setUniformValue("radiusOutline", QVector2D( - radius * _factor, + GLfloat(st::roundRadiusLarge * _factor), (outline > 0) ? (st::groupCallOutline * _factor) : 0.f)); program->setUniformValue("roundRect", Uniform(rect)); program->setUniformValue("roundBg", Uniform(st::groupCallBg->c)); @@ -519,7 +627,25 @@ void Viewport::RendererGL::paintTile( "shadow", QVector3D(shadowHeight, shown, shadowAlpha)); - FillTexturedRectangle(f, program); + f.glActiveTexture(_rgbaFrame ? GL_TEXTURE1 : GL_TEXTURE3); + tileData.textures.bind( + f, + tileData.textureIndex * 5 + kFirstBlurPassTextureIndex); + program->setUniformValue("b_texture", GLint(_rgbaFrame ? 1 : 3)); + program->setUniformValue( + "texelOffset", + GLfloat(1.f / blurSize.height())); + GLint blurTexcoord = program->attributeLocation("b_texcoordIn"); + f.glVertexAttribPointer( + blurTexcoord, + 2, + GL_FLOAT, + GL_FALSE, + 2 * sizeof(GLfloat), + reinterpret_cast(48 * sizeof(GLfloat))); + f.glEnableVertexAttribArray(blurTexcoord); + FillTexturedRectangle(f, program, 8); + f.glDisableVertexAttribArray(blurTexcoord); const auto pinVisible = _owner->wide() && (pin.geometry.bottom() > y); @@ -541,7 +667,7 @@ void Viewport::RendererGL::paintTile( _buttons.bind(f); if (pinVisible) { - FillTexturedRectangle(f, &*_imageProgram, 4); + FillTexturedRectangle(f, &*_imageProgram, 14); } if (nameShift == fullNameShift) { @@ -550,16 +676,154 @@ void Viewport::RendererGL::paintTile( // Mute. if (!muteRect.empty()) { - FillTexturedRectangle(f, &*_imageProgram, 8); + FillTexturedRectangle(f, &*_imageProgram, 18); } // Name. if (!nameRect.empty()) { _names.bind(f); - FillTexturedRectangle(f, &*_imageProgram, 12); + FillTexturedRectangle(f, &*_imageProgram, 22); } } +void Viewport::RendererGL::prepareObjects( + QOpenGLFunctions &f, + TileData &tileData, + QSize blurSize) { + tileData.textures.ensureCreated(f); + tileData.framebuffers.ensureCreated(f); + + const auto create = [&](int index) { + tileData.textures.bind(f, tileData.textureIndex * 5 + index); + f.glTexImage2D( + GL_TEXTURE_2D, + 0, + GL_RGB, + blurSize.width(), + blurSize.height(), + 0, + GL_RGB, + GL_UNSIGNED_BYTE, + nullptr); + }; + create(kScaleForBlurTextureIndex); + create(kFirstBlurPassTextureIndex); +} + +void Viewport::RendererGL::bindFrame( + QOpenGLFunctions &f, + const Webrtc::FrameWithInfo &data, + TileData &tileData, + Program &program) { + const auto upload = (tileData.trackIndex != data.index); + tileData.trackIndex = data.index; + const auto uploadOne = [&]( + GLint internalformat, + GLint format, + QSize size, + int stride, + const void *data) { + f.glPixelStorei(GL_UNPACK_ROW_LENGTH, stride); + f.glTexImage2D( + GL_TEXTURE_2D, + 0, + internalformat, + size.width(), + size.height(), + 0, + format, + GL_UNSIGNED_BYTE, + data); + f.glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); + }; + //if (upload) { + // tileData.textureIndex = 1 - tileData.textureIndex; + //} + if (_rgbaFrame) { + ensureARGB32Program(); + f.glUseProgram(program.argb32->programId()); + f.glActiveTexture(GL_TEXTURE0); + tileData.textures.bind(f, tileData.textureIndex); + if (upload) { + const auto &image = data.original; + const auto stride = image.bytesPerLine() / 4; + const auto data = image.constBits(); + uploadOne(GL_RGB, GL_RGBA, image.size(), stride, data); + } + program.argb32->setUniformValue("s_texture", GLint(0)); + } else { + const auto yuv = data.yuv420; + const auto otherSize = yuv->chromaSize; + f.glUseProgram(program.yuv420->programId()); + f.glActiveTexture(GL_TEXTURE0); + tileData.textures.bind(f, tileData.textureIndex * 5 + 0); + if (upload) { + f.glPixelStorei(GL_UNPACK_ALIGNMENT, 1); + uploadOne(GL_RED, GL_RED, yuv->size, yuv->y.stride, yuv->y.data); + } + f.glActiveTexture(GL_TEXTURE1); + tileData.textures.bind(f, tileData.textureIndex * 5 + 1); + if (upload) { + uploadOne(GL_RED, GL_RED, otherSize, yuv->u.stride, yuv->u.data); + } + f.glActiveTexture(GL_TEXTURE2); + tileData.textures.bind(f, tileData.textureIndex * 5 + 2); + if (upload) { + uploadOne(GL_RED, GL_RED, otherSize, yuv->v.stride, yuv->v.data); + f.glPixelStorei(GL_UNPACK_ALIGNMENT, 4); + } + program.yuv420->setUniformValue("y_texture", GLint(0)); + program.yuv420->setUniformValue("u_texture", GLint(1)); + program.yuv420->setUniformValue("v_texture", GLint(2)); + } +} + +void Viewport::RendererGL::drawDownscalePass( + QOpenGLFunctions &f, + TileData &tileData) { + tileData.framebuffers.bind(f, 0); + f.glFramebufferTexture2D( + GL_FRAMEBUFFER, + GL_COLOR_ATTACHMENT0, + GL_TEXTURE_2D, + tileData.textures.id( + tileData.textureIndex * 5 + kScaleForBlurTextureIndex), + 0); + + const auto program = _rgbaFrame + ? &*_downscaleProgram.argb32 + : &*_downscaleProgram.yuv420; + + FillTexturedRectangle(f, program); +} + +void Viewport::RendererGL::drawFirstBlurPass( + QOpenGLFunctions &f, + TileData &tileData, + QSize blurSize) { + tileData.framebuffers.bind(f, 1); + f.glFramebufferTexture2D( + GL_FRAMEBUFFER, + GL_COLOR_ATTACHMENT0, + GL_TEXTURE_2D, + tileData.textures.id( + tileData.textureIndex * 5 + kFirstBlurPassTextureIndex), + 0); + + f.glUseProgram(_blurProgram->programId()); + f.glActiveTexture(GL_TEXTURE0); + tileData.textures.bind( + f, + tileData.textureIndex * 5 + kScaleForBlurTextureIndex); + + _blurProgram->setUniformValue("b_texture", GLint(0)); + _blurProgram->setUniformValue( + "texelOffset", + GLfloat(1.f / blurSize.width())); + + FillTexturedRectangle(f, &*_blurProgram, 4); +} + Rect Viewport::RendererGL::transformRect(const Rect &raster) const { return { raster.left() * _factor, @@ -578,12 +842,6 @@ Rect Viewport::RendererGL::transformRect(const QRect &raster) const { }; } -void Viewport::RendererGL::freeTextures(QOpenGLFunctions &f) { - for (auto &textures : base::take(_texturesToFree)) { - textures.values.destroy(f); - } -} - void Viewport::RendererGL::ensureButtonsImage() { if (_buttons) { return; diff --git a/Telegram/SourceFiles/calls/group/calls_group_viewport_opengl.h b/Telegram/SourceFiles/calls/group/calls_group_viewport_opengl.h index f06e80b35f..d821fb387f 100644 --- a/Telegram/SourceFiles/calls/group/calls_group_viewport_opengl.h +++ b/Telegram/SourceFiles/calls/group/calls_group_viewport_opengl.h @@ -17,14 +17,16 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include #include +namespace Webrtc { +struct FrameWithInfo; +} // namespace Webrtc + namespace Calls::Group { class Viewport::RendererGL final : public Ui::GL::Renderer { public: explicit RendererGL(not_null owner); - void free(const Textures &textures); - void init( not_null widget, QOpenGLFunctions &f) override; @@ -46,24 +48,50 @@ public: private: struct TileData { not_null peer; + Ui::GL::Textures<5> textures; + Ui::GL::Framebuffers<2> framebuffers; + mutable int textureIndex = 0; + mutable int trackIndex = -1; Ui::Animations::Simple outlined; QRect nameRect; int nameVersion = 0; bool stale = false; bool outline = false; }; + struct Program { + std::optional argb32; + std::optional yuv420; + }; + + void setDefaultViewport(QOpenGLFunctions &f); void fillBackground(QOpenGLFunctions &f); void paintTile( QOpenGLFunctions &f, + GLuint defaultFramebufferObject, not_null tile, - const TileData &nameData); - void freeTextures(QOpenGLFunctions &f); + TileData &nameData); [[nodiscard]] Ui::GL::Rect transformRect(const QRect &raster) const; [[nodiscard]] Ui::GL::Rect transformRect( const Ui::GL::Rect &raster) const; void ensureARGB32Program(); void ensureButtonsImage(); + void prepareObjects( + QOpenGLFunctions &f, + TileData &tileData, + QSize blurSize); + void bindFrame( + QOpenGLFunctions &f, + const Webrtc::FrameWithInfo &data, + TileData &tileData, + Program &program); + void drawDownscalePass( + QOpenGLFunctions &f, + TileData &tileData); + void drawFirstBlurPass( + QOpenGLFunctions &f, + TileData &tileData, + QSize blurSize); void validateDatas(); void validateOutlineAnimation( not_null tile, @@ -73,12 +101,15 @@ private: GLfloat _factor = 1.; QSize _viewport; + bool _rgbaFrame = false; std::optional _frameBuffer; std::optional _bgBuffer; - std::optional _argb32Program; - std::optional _yuv420Program; + Program _downscaleProgram; + std::optional _blurProgram; + Program _frameProgram; std::optional _imageProgram; std::optional _bgProgram; + QOpenGLShader *_downscaleVertexShader = nullptr; QOpenGLShader *_frameVertexShader = nullptr; Ui::GL::Image _buttons; @@ -92,7 +123,6 @@ private: std::vector _tileDataIndices; std::vector _bgTriangles; - std::vector _texturesToFree; Ui::CrossLineAnimation _pinIcon; Ui::CrossLineAnimation _muteIcon; Ui::RoundRect _pinBackground; diff --git a/Telegram/SourceFiles/calls/group/calls_group_viewport_tile.cpp b/Telegram/SourceFiles/calls/group/calls_group_viewport_tile.cpp index fa32418e70..464cb33459 100644 --- a/Telegram/SourceFiles/calls/group/calls_group_viewport_tile.cpp +++ b/Telegram/SourceFiles/calls/group/calls_group_viewport_tile.cpp @@ -31,10 +31,6 @@ Viewport::VideoTile::VideoTile( setup(std::move(pinned)); } -Viewport::VideoTile::~VideoTile() { - Expects(!_textures); -} - QRect Viewport::VideoTile::pinInner() const { return _pinInner.translated(0, -pinSlide()); } @@ -167,17 +163,4 @@ void Viewport::VideoTile::setup(rpl::producer pinned) { } } -void Viewport::VideoTile::ensureTexturesCreated( - QOpenGLFunctions &f) { - _textures.values.ensureCreated(f); -} - -const Viewport::Textures &Viewport::VideoTile::textures() const { - return _textures; -} - -Viewport::Textures Viewport::VideoTile::takeTextures() { - return base::take(_textures); -} - } // namespace Calls::Group diff --git a/Telegram/SourceFiles/calls/group/calls_group_viewport_tile.h b/Telegram/SourceFiles/calls/group/calls_group_viewport_tile.h index 4920ea87b6..a63431d70c 100644 --- a/Telegram/SourceFiles/calls/group/calls_group_viewport_tile.h +++ b/Telegram/SourceFiles/calls/group/calls_group_viewport_tile.h @@ -10,7 +10,6 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL #include "calls/group/calls_group_viewport.h" #include "calls/group/calls_group_call.h" #include "ui/effects/animations.h" -#include "ui/gl/gl_image.h" class Painter; class QOpenGLFunctions; @@ -22,16 +21,6 @@ class RoundRect; namespace Calls::Group { -struct Viewport::Textures { - Ui::GL::Textures<6> values; - mutable int textureIndex = 0; - mutable int trackIndex = -1; - - explicit operator bool() const { - return values.created(); - } -}; - class Viewport::VideoTile final { public: VideoTile( @@ -39,7 +28,6 @@ public: LargeVideoTrack track, rpl::producer pinned, Fn update); - ~VideoTile(); [[nodiscard]] not_null track() const { return _track.track; @@ -69,10 +57,6 @@ public: void togglePinShown(bool shown); bool updateRequestedQuality(VideoQuality quality); - void ensureTexturesCreated(QOpenGLFunctions &f); - [[nodiscard]] const Textures &textures() const; - [[nodiscard]] Textures takeTextures(); - [[nodiscard]] rpl::lifetime &lifetime() { return _lifetime; } @@ -105,8 +89,6 @@ private: bool _pinned = false; std::optional _quality; - Textures _textures; - rpl::lifetime _lifetime; }; diff --git a/Telegram/lib_ui b/Telegram/lib_ui index 5389de6b96..6fa01d0a1e 160000 --- a/Telegram/lib_ui +++ b/Telegram/lib_ui @@ -1 +1 @@ -Subproject commit 5389de6b96f241d992eb93e515e4aefaaf4d86f9 +Subproject commit 6fa01d0a1eaa5fb5809d7a57e3f96058f81795cd