qffmpegconverter.cpp

Go to the documentation of this file.

// Copyright (C) 2024 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
#include "qffmpegconverter_p.h"
#include <QtMultimedia/qvideoframeformat.h>
#include <QtMultimedia/qvideoframe.h>
#include <QtCore/qloggingcategory.h>
#include <private/qvideotexturehelper_p.h>
 
extern "C" {
#include <libswscale/swscale.h>
}
 
QT_BEGIN_NAMESPACE
 
namespace {
 
Q_LOGGING_CATEGORY(lc, "qt.multimedia.ffmpeg.converter");
 
 
// Converts to FFmpeg pixel format. This function differs from
// QFFmpegVideoBuffer::toAVPixelFormat which only covers the subset
// of pixel formats required for encoding. Here we need to cover more
// pixel formats to be able to generate test images for decoding/display
AVPixelFormat toAVPixelFormat(QVideoFrameFormat::PixelFormat pixelFormat)
{
    switch (pixelFormat) {
    default:
    case QVideoFrameFormat::Format_Invalid:
        return AV_PIX_FMT_NONE;
    case QVideoFrameFormat::Format_AYUV:
    case QVideoFrameFormat::Format_AYUV_Premultiplied:
        return AV_PIX_FMT_NONE; // TODO: Fixme (No corresponding FFmpeg format available)
    case QVideoFrameFormat::Format_YV12:
    case QVideoFrameFormat::Format_IMC1:
    case QVideoFrameFormat::Format_IMC3:
    case QVideoFrameFormat::Format_IMC2:
    case QVideoFrameFormat::Format_IMC4:
        return AV_PIX_FMT_YUV420P;
    case QVideoFrameFormat::Format_Jpeg:
        return AV_PIX_FMT_BGRA;
    case QVideoFrameFormat::Format_ARGB8888:
        return AV_PIX_FMT_ARGB;
    case QVideoFrameFormat::Format_ARGB8888_Premultiplied:
    case QVideoFrameFormat::Format_XRGB8888:
        return AV_PIX_FMT_0RGB;
    case QVideoFrameFormat::Format_BGRA8888:
        return AV_PIX_FMT_BGRA;
    case QVideoFrameFormat::Format_BGRA8888_Premultiplied:
    case QVideoFrameFormat::Format_BGRX8888:
        return AV_PIX_FMT_BGR0;
    case QVideoFrameFormat::Format_ABGR8888:
        return AV_PIX_FMT_ABGR;
    case QVideoFrameFormat::Format_XBGR8888:
        return AV_PIX_FMT_0BGR;
    case QVideoFrameFormat::Format_RGBA8888:
        return AV_PIX_FMT_RGBA;
    case QVideoFrameFormat::Format_RGBX8888:
        return AV_PIX_FMT_RGB0;
    case QVideoFrameFormat::Format_YUV422P:
        return AV_PIX_FMT_YUV422P;
    case QVideoFrameFormat::Format_YUV420P:
        return AV_PIX_FMT_YUV420P;
    case QVideoFrameFormat::Format_YUV420P10:
        return AV_PIX_FMT_YUV420P10;
    case QVideoFrameFormat::Format_UYVY:
        return AV_PIX_FMT_UYVY422;
    case QVideoFrameFormat::Format_YUYV:
        return AV_PIX_FMT_YUYV422;
    case QVideoFrameFormat::Format_NV12:
        return AV_PIX_FMT_NV12;
    case QVideoFrameFormat::Format_NV21:
        return AV_PIX_FMT_NV21;
    case QVideoFrameFormat::Format_Y8:
        return AV_PIX_FMT_GRAY8;
    case QVideoFrameFormat::Format_Y16:
        return AV_PIX_FMT_GRAY16;
    case QVideoFrameFormat::Format_P010:
        return AV_PIX_FMT_P010;
    case QVideoFrameFormat::Format_P016:
        return AV_PIX_FMT_P016;
    case QVideoFrameFormat::Format_SamplerExternalOES:
        return AV_PIX_FMT_MEDIACODEC;
    }
}
 
struct SwsFrameData
{
    static constexpr int arraySize = 4; // Array size required by sws_scale
    std::array<uchar *, arraySize> bits;
    std::array<int, arraySize> stride;
};
 
SwsFrameData getSwsData(QVideoFrame &dst)
{
    switch (dst.pixelFormat()) {
    case QVideoFrameFormat::Format_YV12:
    case QVideoFrameFormat::Format_IMC1:
        return { { dst.bits(0), dst.bits(2), dst.bits(1), nullptr },
                 { dst.bytesPerLine(0), dst.bytesPerLine(2), dst.bytesPerLine(1), 0 } };
 
    case QVideoFrameFormat::Format_IMC2:
        return { { dst.bits(0), dst.bits(1) + dst.bytesPerLine(1) / 2, dst.bits(1), nullptr },
                 { dst.bytesPerLine(0), dst.bytesPerLine(1), dst.bytesPerLine(1), 0 } };
 
    case QVideoFrameFormat::Format_IMC4:
        return { { dst.bits(0), dst.bits(1), dst.bits(1) + dst.bytesPerLine(1) / 2, nullptr },
                 { dst.bytesPerLine(0), dst.bytesPerLine(1), dst.bytesPerLine(1), 0 } };
    default:
        return { { dst.bits(0), dst.bits(1), dst.bits(2), nullptr },
                 { dst.bytesPerLine(0), dst.bytesPerLine(1), dst.bytesPerLine(2), 0 } };
    }
}
 
struct SwsColorSpace
{
    int colorSpace;
    int colorRange; // 0 - mpeg/video, 1 - jpeg/full
};
 
// Qt heuristics for determining color space requires checking
// both frame color space and range. This function mimics logic
// used elsewhere in Qt Multimedia.
SwsColorSpace toSwsColorSpace(QVideoFrameFormat::ColorRange colorRange,
                              QVideoFrameFormat::ColorSpace colorSpace)
{
    const int avRange = colorRange == QVideoFrameFormat::ColorRange_Video ? 0 : 1;
 
    switch (colorSpace) {
    case QVideoFrameFormat::ColorSpace_BT601:
        if (colorRange == QVideoFrameFormat::ColorRange_Full)
            return { SWS_CS_ITU709, 1 }; // TODO: FIXME - Not exact match
        return { SWS_CS_ITU601, 0 };
    case QVideoFrameFormat::ColorSpace_BT709:
        return { SWS_CS_ITU709, avRange };
    case QVideoFrameFormat::ColorSpace_AdobeRgb:
        return { SWS_CS_ITU601, 1 }; // TODO: Why do ITU601 and Adobe RGB match well?
    case QVideoFrameFormat::ColorSpace_BT2020:
        return { SWS_CS_BT2020, avRange };
    case QVideoFrameFormat::ColorSpace_Undefined:
    default:
        return { SWS_CS_DEFAULT, avRange };
    }
}
 
using SwsContextUPtr = std::unique_ptr<SwsContext, decltype(&sws_freeContext)>;
using PixelFormat = QVideoFrameFormat::PixelFormat;
 
// clang-format off
 
SwsContextUPtr createConverter(const QSize &srcSize, PixelFormat srcPixFmt,
                               const QSize &dstSize, PixelFormat dstPixFmt)
{
    SwsContext* context = sws_getContext(
        srcSize.width(), srcSize.height(), toAVPixelFormat(srcPixFmt),
        dstSize.width(), dstSize.height(), toAVPixelFormat(dstPixFmt),
        SWS_BILINEAR, nullptr, nullptr, nullptr);
 
    return { context, &sws_freeContext };
}
 
bool setColorSpaceDetails(SwsContext *context,
                          const QVideoFrameFormat &srcFormat,
                          const QVideoFrameFormat &dstFormat)
{
    const SwsColorSpace src = toSwsColorSpace(srcFormat.colorRange(), srcFormat.colorSpace());
    const SwsColorSpace dst = toSwsColorSpace(dstFormat.colorRange(), dstFormat.colorSpace());
 
    constexpr int brightness = 0;
    constexpr int contrast = 0;
    constexpr int saturation = 0;
    const int status = sws_setColorspaceDetails(context,
        sws_getCoefficients(src.colorSpace), src.colorRange,
        sws_getCoefficients(dst.colorSpace), dst.colorRange,
        brightness, contrast, saturation);
 
    return status == 0;
}
 
bool convert(SwsContext *context, QVideoFrame &src, int srcHeight, QVideoFrame &dst)
{
    if (!src.map(QtVideo::MapMode::ReadOnly))
        return false;
 
    QScopeGuard unmapSrc{[&] {
        src.unmap();
    }};
 
    if (!dst.map(QtVideo::MapMode::WriteOnly))
        return false;
 
    QScopeGuard unmapDst{[&] {
        dst.unmap();
    }};
 
    const SwsFrameData srcData = getSwsData(src);
    const SwsFrameData dstData = getSwsData(dst);
 
    constexpr int firstSrcSliceRow = 0;
    const int scaledHeight = sws_scale(context,
        srcData.bits.data(), srcData.stride.data(),
        firstSrcSliceRow, srcHeight,
        dstData.bits.data(), dstData.stride.data());
 
    if (scaledHeight != srcHeight)
        return false;
 
    return true;
}
 
// Ensure even size if using planar format with chroma subsampling
QSize adjustSize(const QSize& size, PixelFormat srcFmt, PixelFormat dstFmt)
{
    const auto* srcDesc = QVideoTextureHelper::textureDescription(srcFmt);
    const auto* dstDesc = QVideoTextureHelper::textureDescription(dstFmt);
 
    QSize output = size;
    for (const auto desc : { srcDesc, dstDesc }) {
        for (int i = 0; i < desc->nplanes; ++i) {
            // TODO: Assumes that max subsampling is 2
            if (desc->sizeScale[i].x != 1)
                output.setWidth(output.width() & ~1); // Make even
 
            if (desc->sizeScale[i].y != 1)
                output.setHeight(output.height() & ~1); // Make even
        }
    }
 
    return output;
}
 
} // namespace
 
// Converts a video frame to the dstFormat video frame format.
QVideoFrame convertFrame(QVideoFrame &src, const QVideoFrameFormat &dstFormat)
{
    if (src.size() != dstFormat.frameSize()) {
        qCCritical(lc) << "Resizing is not supported";
        return {};
    }
 
    // Adjust size to even width/height if we have chroma subsampling
    const QSize size = adjustSize(src.size(), src.pixelFormat(), dstFormat.pixelFormat());
    if (size != src.size())
        qCWarning(lc) << "Input truncated to even width/height";
 
    const SwsContextUPtr conv = createConverter(
        size, src.pixelFormat(), size, dstFormat.pixelFormat());
 
    if (!conv) {
        qCCritical(lc) << "Failed to create SW converter";
        return {};
    }
 
    if (!setColorSpaceDetails(conv.get(), src.surfaceFormat(), dstFormat)) {
        qCCritical(lc) << "Failed to set color space details";
        return {};
    }
 
    QVideoFrame dst{ dstFormat };
 
    if (!convert(conv.get(), src, size.height(), dst)) {
        qCCritical(lc) << "Frame conversion failed";
        return {};
    }
 
    return dst;
}
 
// clang-format on
 
QT_END_NAMESPACE