avutil/hwcontext_videotoolbox: implement hwupload to convert AVFrame to CVPixelBuffer
Teach AV_HWDEVICE_TYPE_VIDEOTOOLBOX to be able to create AVFrames of type
AV_PIX_FMT_VIDEOTOOLBOX. This can be used to hwupload a regular AVFrame
into its CVPixelBuffer equivalent.
ffmpeg -init_hw_device videotoolbox -f lavfi -i color=black:640x480 -vf hwupload -c:v h264_videotoolbox -f null -y /dev/null
Signed-off-by: Aman Karmani <aman@tmm1.net>
This commit is contained in:
Aman Karmani
parent
997f9bdb99
commit
504c60660d
|
|
@ -24,6 +24,7 @@
|
|||
#include <VideoToolbox/VideoToolbox.h>
|
||||
|
||||
#include "buffer.h"
|
||||
#include "buffer_internal.h"
|
||||
#include "common.h"
|
||||
#include "hwcontext.h"
|
||||
#include "hwcontext_internal.h"
|
||||
|
|
@ -32,6 +33,10 @@
|
|||
#include "pixfmt.h"
|
||||
#include "pixdesc.h"
|
||||
|
||||
typedef struct VTFramesContext {
|
||||
CVPixelBufferPoolRef pool;
|
||||
} VTFramesContext;
|
||||
|
||||
static const struct {
|
||||
uint32_t cv_fmt;
|
||||
bool full_range;
|
||||
|
|
@ -50,6 +55,39 @@ static const struct {
|
|||
#endif
|
||||
};
|
||||
|
||||
static const enum AVPixelFormat supported_formats[] = {
|
||||
AV_PIX_FMT_NV12,
|
||||
AV_PIX_FMT_YUV420P,
|
||||
AV_PIX_FMT_UYVY422,
|
||||
AV_PIX_FMT_P010,
|
||||
AV_PIX_FMT_BGRA,
|
||||
};
|
||||
|
||||
static int vt_frames_get_constraints(AVHWDeviceContext *ctx,
|
||||
const void *hwconfig,
|
||||
AVHWFramesConstraints *constraints)
|
||||
{
|
||||
int i;
|
||||
|
||||
constraints->valid_sw_formats = av_malloc_array(FF_ARRAY_ELEMS(supported_formats) + 1,
|
||||
sizeof(*constraints->valid_sw_formats));
|
||||
if (!constraints->valid_sw_formats)
|
||||
return AVERROR(ENOMEM);
|
||||
|
||||
for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++)
|
||||
constraints->valid_sw_formats[i] = supported_formats[i];
|
||||
constraints->valid_sw_formats[FF_ARRAY_ELEMS(supported_formats)] = AV_PIX_FMT_NONE;
|
||||
|
||||
constraints->valid_hw_formats = av_malloc_array(2, sizeof(*constraints->valid_hw_formats));
|
||||
if (!constraints->valid_hw_formats)
|
||||
return AVERROR(ENOMEM);
|
||||
|
||||
constraints->valid_hw_formats[0] = AV_PIX_FMT_VIDEOTOOLBOX;
|
||||
constraints->valid_hw_formats[1] = AV_PIX_FMT_NONE;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
enum AVPixelFormat av_map_videotoolbox_format_to_pixfmt(uint32_t cv_fmt)
|
||||
{
|
||||
int i;
|
||||
|
|
@ -75,11 +113,134 @@ uint32_t av_map_videotoolbox_format_from_pixfmt2(enum AVPixelFormat pix_fmt, boo
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int vt_pool_alloc(AVHWFramesContext *ctx)
|
||||
{
|
||||
VTFramesContext *fctx = ctx->internal->priv;
|
||||
CVReturn err;
|
||||
CFNumberRef w, h, pixfmt;
|
||||
uint32_t cv_pixfmt;
|
||||
CFMutableDictionaryRef attributes, iosurface_properties;
|
||||
|
||||
attributes = CFDictionaryCreateMutable(
|
||||
NULL,
|
||||
2,
|
||||
&kCFTypeDictionaryKeyCallBacks,
|
||||
&kCFTypeDictionaryValueCallBacks);
|
||||
|
||||
cv_pixfmt = av_map_videotoolbox_format_from_pixfmt(ctx->sw_format);
|
||||
pixfmt = CFNumberCreate(NULL, kCFNumberSInt32Type, &cv_pixfmt);
|
||||
CFDictionarySetValue(
|
||||
attributes,
|
||||
kCVPixelBufferPixelFormatTypeKey,
|
||||
pixfmt);
|
||||
CFRelease(pixfmt);
|
||||
|
||||
iosurface_properties = CFDictionaryCreateMutable(
|
||||
NULL,
|
||||
0,
|
||||
&kCFTypeDictionaryKeyCallBacks,
|
||||
&kCFTypeDictionaryValueCallBacks);
|
||||
CFDictionarySetValue(attributes, kCVPixelBufferIOSurfacePropertiesKey, iosurface_properties);
|
||||
CFRelease(iosurface_properties);
|
||||
|
||||
w = CFNumberCreate(NULL, kCFNumberSInt32Type, &ctx->width);
|
||||
h = CFNumberCreate(NULL, kCFNumberSInt32Type, &ctx->height);
|
||||
CFDictionarySetValue(attributes, kCVPixelBufferWidthKey, w);
|
||||
CFDictionarySetValue(attributes, kCVPixelBufferHeightKey, h);
|
||||
CFRelease(w);
|
||||
CFRelease(h);
|
||||
|
||||
err = CVPixelBufferPoolCreate(
|
||||
NULL,
|
||||
NULL,
|
||||
attributes,
|
||||
&fctx->pool);
|
||||
CFRelease(attributes);
|
||||
|
||||
if (err == kCVReturnSuccess)
|
||||
return 0;
|
||||
|
||||
av_log(ctx, AV_LOG_ERROR, "Error creating CVPixelBufferPool: %d\n", err);
|
||||
return AVERROR_EXTERNAL;
|
||||
}
|
||||
|
||||
static AVBufferRef *vt_dummy_pool_alloc(void *opaque, size_t size)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void vt_frames_uninit(AVHWFramesContext *ctx)
|
||||
{
|
||||
VTFramesContext *fctx = ctx->internal->priv;
|
||||
if (fctx->pool) {
|
||||
CVPixelBufferPoolRelease(fctx->pool);
|
||||
fctx->pool = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static int vt_frames_init(AVHWFramesContext *ctx)
|
||||
{
|
||||
int i, ret;
|
||||
|
||||
for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++) {
|
||||
if (ctx->sw_format == supported_formats[i])
|
||||
break;
|
||||
}
|
||||
if (i == FF_ARRAY_ELEMS(supported_formats)) {
|
||||
av_log(ctx, AV_LOG_ERROR, "Pixel format '%s' is not supported\n",
|
||||
av_get_pix_fmt_name(ctx->sw_format));
|
||||
return AVERROR(ENOSYS);
|
||||
}
|
||||
|
||||
// create a dummy pool so av_hwframe_get_buffer doesn't EINVAL
|
||||
if (!ctx->pool) {
|
||||
ctx->internal->pool_internal = av_buffer_pool_init2(0, ctx, vt_dummy_pool_alloc, NULL);
|
||||
if (!ctx->internal->pool_internal)
|
||||
return AVERROR(ENOMEM);
|
||||
}
|
||||
|
||||
ret = vt_pool_alloc(ctx);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void videotoolbox_buffer_release(void *opaque, uint8_t *data)
|
||||
{
|
||||
CVPixelBufferRelease((CVPixelBufferRef)data);
|
||||
}
|
||||
|
||||
static int vt_get_buffer(AVHWFramesContext *ctx, AVFrame *frame)
|
||||
{
|
||||
frame->buf[0] = av_buffer_pool_get(ctx->pool);
|
||||
if (!frame->buf[0])
|
||||
return AVERROR(ENOMEM);
|
||||
VTFramesContext *fctx = ctx->internal->priv;
|
||||
|
||||
if (ctx->pool && ctx->pool->size != 0) {
|
||||
frame->buf[0] = av_buffer_pool_get(ctx->pool);
|
||||
if (!frame->buf[0])
|
||||
return AVERROR(ENOMEM);
|
||||
} else {
|
||||
CVPixelBufferRef pixbuf;
|
||||
AVBufferRef *buf = NULL;
|
||||
CVReturn err;
|
||||
|
||||
err = CVPixelBufferPoolCreatePixelBuffer(
|
||||
NULL,
|
||||
fctx->pool,
|
||||
&pixbuf
|
||||
);
|
||||
if (err != kCVReturnSuccess) {
|
||||
av_log(ctx, AV_LOG_ERROR, "Failed to create pixel buffer from pool: %d\n", err);
|
||||
return AVERROR_EXTERNAL;
|
||||
}
|
||||
|
||||
buf = av_buffer_create((uint8_t *)pixbuf, 1, videotoolbox_buffer_release, NULL, 0);
|
||||
if (!buf) {
|
||||
CVPixelBufferRelease(pixbuf);
|
||||
return AVERROR(ENOMEM);
|
||||
}
|
||||
frame->buf[0] = buf;
|
||||
}
|
||||
|
||||
frame->data[3] = frame->buf[0]->data;
|
||||
frame->format = AV_PIX_FMT_VIDEOTOOLBOX;
|
||||
|
|
@ -111,6 +272,248 @@ static void vt_unmap(AVHWFramesContext *ctx, HWMapDescriptor *hwmap)
|
|||
CVPixelBufferUnlockBaseAddress(pixbuf, (uintptr_t)hwmap->priv);
|
||||
}
|
||||
|
||||
static int vt_pixbuf_set_par(AVHWFramesContext *hwfc,
|
||||
CVPixelBufferRef pixbuf, const AVFrame *src)
|
||||
{
|
||||
CFMutableDictionaryRef par = NULL;
|
||||
CFNumberRef num = NULL, den = NULL;
|
||||
AVRational avpar = src->sample_aspect_ratio;
|
||||
|
||||
if (avpar.num == 0)
|
||||
return 0;
|
||||
|
||||
av_reduce(&avpar.num, &avpar.den,
|
||||
avpar.num, avpar.den,
|
||||
0xFFFFFFFF);
|
||||
|
||||
num = CFNumberCreate(kCFAllocatorDefault,
|
||||
kCFNumberIntType,
|
||||
&avpar.num);
|
||||
|
||||
den = CFNumberCreate(kCFAllocatorDefault,
|
||||
kCFNumberIntType,
|
||||
&avpar.den);
|
||||
|
||||
par = CFDictionaryCreateMutable(kCFAllocatorDefault,
|
||||
2,
|
||||
&kCFCopyStringDictionaryKeyCallBacks,
|
||||
&kCFTypeDictionaryValueCallBacks);
|
||||
|
||||
if (!par || !num || !den) {
|
||||
if (par) CFRelease(par);
|
||||
if (num) CFRelease(num);
|
||||
if (den) CFRelease(den);
|
||||
return AVERROR(ENOMEM);
|
||||
}
|
||||
|
||||
CFDictionarySetValue(
|
||||
par,
|
||||
kCVImageBufferPixelAspectRatioHorizontalSpacingKey,
|
||||
num);
|
||||
CFDictionarySetValue(
|
||||
par,
|
||||
kCVImageBufferPixelAspectRatioVerticalSpacingKey,
|
||||
den);
|
||||
|
||||
CVBufferSetAttachment(
|
||||
pixbuf,
|
||||
kCVImageBufferPixelAspectRatioKey,
|
||||
par,
|
||||
kCVAttachmentMode_ShouldPropagate
|
||||
);
|
||||
|
||||
CFRelease(par);
|
||||
CFRelease(num);
|
||||
CFRelease(den);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int vt_pixbuf_set_chromaloc(AVHWFramesContext *hwfc,
|
||||
CVPixelBufferRef pixbuf, const AVFrame *src)
|
||||
{
|
||||
CFStringRef loc = NULL;
|
||||
|
||||
switch (src->chroma_location) {
|
||||
case AVCHROMA_LOC_LEFT:
|
||||
loc = kCVImageBufferChromaLocation_Left;
|
||||
break;
|
||||
case AVCHROMA_LOC_CENTER:
|
||||
loc = kCVImageBufferChromaLocation_Center;
|
||||
break;
|
||||
case AVCHROMA_LOC_TOP:
|
||||
loc = kCVImageBufferChromaLocation_Top;
|
||||
break;
|
||||
case AVCHROMA_LOC_BOTTOM:
|
||||
loc = kCVImageBufferChromaLocation_Bottom;
|
||||
break;
|
||||
case AVCHROMA_LOC_TOPLEFT:
|
||||
loc = kCVImageBufferChromaLocation_TopLeft;
|
||||
break;
|
||||
case AVCHROMA_LOC_BOTTOMLEFT:
|
||||
loc = kCVImageBufferChromaLocation_BottomLeft;
|
||||
break;
|
||||
}
|
||||
|
||||
if (loc) {
|
||||
CVBufferSetAttachment(
|
||||
pixbuf,
|
||||
kCVImageBufferChromaLocationTopFieldKey,
|
||||
loc,
|
||||
kCVAttachmentMode_ShouldPropagate);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int vt_pixbuf_set_colorspace(AVHWFramesContext *hwfc,
|
||||
CVPixelBufferRef pixbuf, const AVFrame *src)
|
||||
{
|
||||
CFStringRef colormatrix = NULL, colorpri = NULL, colortrc = NULL;
|
||||
Float32 gamma = 0;
|
||||
|
||||
switch (src->colorspace) {
|
||||
case AVCOL_SPC_BT2020_CL:
|
||||
case AVCOL_SPC_BT2020_NCL:
|
||||
if (__builtin_available(macOS 10.11, *))
|
||||
colormatrix = kCVImageBufferYCbCrMatrix_ITU_R_2020;
|
||||
else
|
||||
colormatrix = CFSTR("ITU_R_2020");
|
||||
break;
|
||||
case AVCOL_SPC_BT470BG:
|
||||
case AVCOL_SPC_SMPTE170M:
|
||||
colormatrix = kCVImageBufferYCbCrMatrix_ITU_R_601_4;
|
||||
break;
|
||||
case AVCOL_SPC_BT709:
|
||||
colormatrix = kCVImageBufferYCbCrMatrix_ITU_R_709_2;
|
||||
break;
|
||||
case AVCOL_SPC_SMPTE240M:
|
||||
colormatrix = kCVImageBufferYCbCrMatrix_SMPTE_240M_1995;
|
||||
break;
|
||||
case AVCOL_SPC_UNSPECIFIED:
|
||||
break;
|
||||
default:
|
||||
av_log(hwfc, AV_LOG_WARNING, "Color space %s is not supported.\n", av_color_space_name(src->colorspace));
|
||||
}
|
||||
|
||||
switch (src->color_primaries) {
|
||||
case AVCOL_PRI_BT2020:
|
||||
if (__builtin_available(macOS 10.11, *))
|
||||
colorpri = kCVImageBufferColorPrimaries_ITU_R_2020;
|
||||
else
|
||||
colorpri = CFSTR("ITU_R_2020");
|
||||
break;
|
||||
case AVCOL_PRI_BT709:
|
||||
colorpri = kCVImageBufferColorPrimaries_ITU_R_709_2;
|
||||
break;
|
||||
case AVCOL_PRI_SMPTE170M:
|
||||
colorpri = kCVImageBufferColorPrimaries_SMPTE_C;
|
||||
break;
|
||||
case AVCOL_PRI_BT470BG:
|
||||
colorpri = kCVImageBufferColorPrimaries_EBU_3213;
|
||||
break;
|
||||
case AVCOL_PRI_UNSPECIFIED:
|
||||
break;
|
||||
default:
|
||||
av_log(hwfc, AV_LOG_WARNING, "Color primaries %s is not supported.\n", av_color_primaries_name(src->color_primaries));
|
||||
}
|
||||
|
||||
switch (src->color_trc) {
|
||||
case AVCOL_TRC_SMPTE2084:
|
||||
if (__builtin_available(macOS 10.13, *))
|
||||
colortrc = kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ;
|
||||
else
|
||||
colortrc = CFSTR("SMPTE_ST_2084_PQ");
|
||||
break;
|
||||
case AVCOL_TRC_BT2020_10:
|
||||
case AVCOL_TRC_BT2020_12:
|
||||
if (__builtin_available(macOS 10.11, *))
|
||||
colortrc = kCVImageBufferTransferFunction_ITU_R_2020;
|
||||
else
|
||||
colortrc = CFSTR("ITU_R_2020");
|
||||
break;
|
||||
case AVCOL_TRC_BT709:
|
||||
colortrc = kCVImageBufferTransferFunction_ITU_R_709_2;
|
||||
break;
|
||||
case AVCOL_TRC_SMPTE240M:
|
||||
colortrc = kCVImageBufferTransferFunction_SMPTE_240M_1995;
|
||||
break;
|
||||
case AVCOL_TRC_SMPTE428:
|
||||
if (__builtin_available(macOS 10.12, *))
|
||||
colortrc = kCVImageBufferTransferFunction_SMPTE_ST_428_1;
|
||||
else
|
||||
colortrc = CFSTR("SMPTE_ST_428_1");
|
||||
break;
|
||||
case AVCOL_TRC_ARIB_STD_B67:
|
||||
if (__builtin_available(macOS 10.13, *))
|
||||
colortrc = kCVImageBufferTransferFunction_ITU_R_2100_HLG;
|
||||
else
|
||||
colortrc = CFSTR("ITU_R_2100_HLG");
|
||||
break;
|
||||
case AVCOL_TRC_GAMMA22:
|
||||
gamma = 2.2;
|
||||
colortrc = kCVImageBufferTransferFunction_UseGamma;
|
||||
break;
|
||||
case AVCOL_TRC_GAMMA28:
|
||||
gamma = 2.8;
|
||||
colortrc = kCVImageBufferTransferFunction_UseGamma;
|
||||
break;
|
||||
case AVCOL_TRC_UNSPECIFIED:
|
||||
break;
|
||||
default:
|
||||
av_log(hwfc, AV_LOG_WARNING, "Color transfer function %s is not supported.\n", av_color_transfer_name(src->color_trc));
|
||||
}
|
||||
|
||||
if (colormatrix) {
|
||||
CVBufferSetAttachment(
|
||||
pixbuf,
|
||||
kCVImageBufferYCbCrMatrixKey,
|
||||
colormatrix,
|
||||
kCVAttachmentMode_ShouldPropagate);
|
||||
}
|
||||
if (colorpri) {
|
||||
CVBufferSetAttachment(
|
||||
pixbuf,
|
||||
kCVImageBufferColorPrimariesKey,
|
||||
colorpri,
|
||||
kCVAttachmentMode_ShouldPropagate);
|
||||
}
|
||||
if (colortrc) {
|
||||
CVBufferSetAttachment(
|
||||
pixbuf,
|
||||
kCVImageBufferTransferFunctionKey,
|
||||
colortrc,
|
||||
kCVAttachmentMode_ShouldPropagate);
|
||||
}
|
||||
if (gamma != 0) {
|
||||
CFNumberRef gamma_level = CFNumberCreate(NULL, kCFNumberFloat32Type, &gamma);
|
||||
CVBufferSetAttachment(
|
||||
pixbuf,
|
||||
kCVImageBufferGammaLevelKey,
|
||||
gamma_level,
|
||||
kCVAttachmentMode_ShouldPropagate);
|
||||
CFRelease(gamma_level);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int vt_pixbuf_set_attachments(AVHWFramesContext *hwfc,
|
||||
CVPixelBufferRef pixbuf, const AVFrame *src)
|
||||
{
|
||||
int ret;
|
||||
ret = vt_pixbuf_set_par(hwfc, pixbuf, src);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
ret = vt_pixbuf_set_colorspace(hwfc, pixbuf, src);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
ret = vt_pixbuf_set_chromaloc(hwfc, pixbuf, src);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int vt_map_frame(AVHWFramesContext *ctx, AVFrame *dst, const AVFrame *src,
|
||||
int flags)
|
||||
{
|
||||
|
|
@ -223,6 +626,10 @@ static int vt_transfer_data_to(AVHWFramesContext *hwfc,
|
|||
if (err)
|
||||
goto fail;
|
||||
|
||||
err = vt_pixbuf_set_attachments(hwfc, (CVPixelBufferRef)dst->data[3], src);
|
||||
if (err)
|
||||
goto fail;
|
||||
|
||||
err = 0;
|
||||
fail:
|
||||
av_frame_free(&map);
|
||||
|
|
@ -244,8 +651,13 @@ const HWContextType ff_hwcontext_type_videotoolbox = {
|
|||
.type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
|
||||
.name = "videotoolbox",
|
||||
|
||||
.frames_priv_size = sizeof(VTFramesContext),
|
||||
|
||||
.device_create = vt_device_create,
|
||||
.frames_init = vt_frames_init,
|
||||
.frames_get_buffer = vt_get_buffer,
|
||||
.frames_get_constraints = vt_frames_get_constraints,
|
||||
.frames_uninit = vt_frames_uninit,
|
||||
.transfer_get_formats = vt_transfer_get_formats,
|
||||
.transfer_data_to = vt_transfer_data_to,
|
||||
.transfer_data_from = vt_transfer_data_from,
|
||||
|
|
|
|||
|
|
@ -29,11 +29,14 @@
|
|||
* @file
|
||||
* An API-specific header for AV_HWDEVICE_TYPE_VIDEOTOOLBOX.
|
||||
*
|
||||
* This API currently does not support frame allocation, as the raw VideoToolbox
|
||||
* API does allocation, and FFmpeg itself never has the need to allocate frames.
|
||||
* This API supports frame allocation using a native CVPixelBufferPool
|
||||
* instead of an AVBufferPool.
|
||||
*
|
||||
* If the API user sets a custom pool, AVHWFramesContext.pool must return
|
||||
* AVBufferRefs whose data pointer is a CVImageBufferRef or CVPixelBufferRef.
|
||||
* Note that the underlying CVPixelBuffer could be retained by OS frameworks
|
||||
* depending on application usage, so it is preferable to let CoreVideo manage
|
||||
* the pool using the default implementation.
|
||||
*
|
||||
* Currently AVHWDeviceContext.hwctx and AVHWFramesContext.hwctx are always
|
||||
* NULL.
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@
|
|||
|
||||
#define LIBAVUTIL_VERSION_MAJOR 57
|
||||
#define LIBAVUTIL_VERSION_MINOR 0
|
||||
#define LIBAVUTIL_VERSION_MICRO 100
|
||||
#define LIBAVUTIL_VERSION_MICRO 101
|
||||
|
||||
#define LIBAVUTIL_VERSION_INT AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \
|
||||
LIBAVUTIL_VERSION_MINOR, \
|
||||
|
|
|
|||
Loading…
Reference in New Issue