From 4f9ee87253e021f43b1eaf40e57c2fb10af2be88 Mon Sep 17 00:00:00 2001 From: Lynne Date: Thu, 12 Nov 2020 12:44:30 +0100 Subject: [PATCH] libavutil: introduce AVFilmGrainParams side data This patch introduces a new frame side data type AVFilmGrainParams for use with video codecs which support it. It can save a lot of memory used for duplicate processed reference frames and reduce copies when applying film grain during presentation. --- doc/APIchanges | 4 + libavutil/Makefile | 2 + libavutil/film_grain_params.c | 42 +++++++++ libavutil/film_grain_params.h | 167 ++++++++++++++++++++++++++++++++++ libavutil/frame.c | 1 + libavutil/frame.h | 6 ++ libavutil/version.h | 2 +- 7 files changed, 223 insertions(+), 1 deletion(-) create mode 100644 libavutil/film_grain_params.c create mode 100644 libavutil/film_grain_params.h diff --git a/doc/APIchanges b/doc/APIchanges index b70c78a483..58837090cf 100644 --- a/doc/APIchanges +++ b/doc/APIchanges @@ -15,6 +15,10 @@ libavutil: 2017-10-21 API changes, most recent first: +2020-11-25 - xxxxxxxxxx - lavu 56.61.100 - film_grain_params.h + Adds a new API for extracting codec film grain parameters as side data. + Adds a new AVFrameSideDataType entry AV_FRAME_DATA_FILM_GRAIN_PARAMS for it. + 2020-xx-xx - xxxxxxxxxx - lavf 58.64.100 - avformat.h Add AVSTREAM_EVENT_FLAG_NEW_PACKETS. diff --git a/libavutil/Makefile b/libavutil/Makefile index 9b08372eb2..27bafe9e12 100644 --- a/libavutil/Makefile +++ b/libavutil/Makefile @@ -84,6 +84,7 @@ HEADERS = adler32.h \ xtea.h \ tea.h \ tx.h \ + film_grain_params.h \ HEADERS-$(CONFIG_LZO) += lzo.h @@ -170,6 +171,7 @@ OBJS = adler32.o \ tx_double.o \ tx_int32.o \ video_enc_params.o \ + film_grain_params.o \ OBJS-$(CONFIG_CUDA) += hwcontext_cuda.o diff --git a/libavutil/film_grain_params.c b/libavutil/film_grain_params.c new file mode 100644 index 0000000000..930d23c7fe --- /dev/null +++ b/libavutil/film_grain_params.c @@ -0,0 +1,42 @@ +/** + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "film_grain_params.h" + +AVFilmGrainParams *av_film_grain_params_alloc(size_t *size) +{ + AVFilmGrainParams *params = av_mallocz(sizeof(AVFilmGrainParams)); + + if (size) + *size = sizeof(*params); + + return params; +} + +AVFilmGrainParams *av_film_grain_params_create_side_data(AVFrame *frame) +{ + AVFrameSideData *side_data = av_frame_new_side_data(frame, + AV_FRAME_DATA_FILM_GRAIN_PARAMS, + sizeof(AVFilmGrainParams)); + if (!side_data) + return NULL; + + memset(side_data->data, 0, sizeof(AVFilmGrainParams)); + + return (AVFilmGrainParams *)side_data->data; +} diff --git a/libavutil/film_grain_params.h b/libavutil/film_grain_params.h new file mode 100644 index 0000000000..cdc8f623c6 --- /dev/null +++ b/libavutil/film_grain_params.h @@ -0,0 +1,167 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVUTIL_FILM_GRAIN_PARAMS_H +#define AVUTIL_FILM_GRAIN_PARAMS_H + +#include "frame.h" + +enum AVFilmGrainParamsType { + AV_FILM_GRAM_PARAMS_NONE = 0, + + /** + * The union is valid when interpreted as AVFilmGrainAOMParams (codec.aom) + */ + AV_FILM_GRAM_PARAMS_AV1, +}; + +/** + * This structure describes how to handle film grain synthesis for AOM codecs. + * + * @note The struct must be allocated as part of AVFilmGrainParams using + * av_film_grain_params_alloc(). Its size is not a part of the public ABI. + */ +typedef struct AVFilmGrainAOMParams { + /** + * Number of points, and the scale and value for each point of the + * piecewise linear scaling function for the uma plane. + */ + int num_y_points; + uint8_t y_points[14][2 /* value, scaling */]; + + /** + * Signals whether to derive the chroma scaling function from the luma. + * Not equivalent to copying the luma values and scales. + */ + int chroma_scaling_from_luma; + + /** + * If chroma_scaling_from_luma is set to 0, signals the chroma scaling + * function parameters. + */ + int num_uv_points[2]; + uint8_t uv_points[2][10][2 /* value, scaling */]; + + /** + * Specifies the shift applied to the chroma components. For AV1, its within + * [8; 11] and determines the range and quantization of the film grain. + */ + int scaling_shift; + + /** + * Specifies the auto-regression lag. The number of coefficients is given by + * 2*ar_coeff_lag(ar_coeff_lag - 1), with an extra one for the chroma. + */ + int ar_coeff_lag; + + /** + * Luma auto-regression coefficients. + */ + int8_t ar_coeffs_y[24]; + + /** + * Chroma auto-regression coefficients. + */ + int8_t ar_coeffs_uv[2][25]; + + /** + * Specifies the range of the auto-regressive coefficients. Values of 6, + * 7, 8 and so on represent a range of [-2, 2), [-1, 1), [-0.5, 0.5) and + * so on. For AV1 must be between 6 and 9. + */ + int ar_coeff_shift; + + /** + * Signals the down shift applied to the generated gaussian numbers during + * synthesis. + */ + int grain_scale_shift; + + /** + * Specifies the luma/chroma multipliers for the index to the component + * scaling function. + */ + int uv_mult[2]; + int uv_mult_luma[2]; + + /** + * Offset used for component scaling function. For AV1 its a 9-bit value + * with a range [-256, 255] + */ + int uv_offset[2]; + + /** + * Signals whether to overlap film grain blocks. + */ + int overlap_flag; + + /** + * Signals to clip to limited color levels after film grain application. + */ + int limit_output_range; +} AVFilmGrainAOMParams; + +/** + * This structure describes how to handle film grain synthesis in video + * for specific codecs. Must be present on every frame where film grain is + * meant to be synthesised for correct presentation. + * + * @note The struct must be allocated with av_film_grain_params_alloc() and + * its size is not a part of the public ABI. + */ +typedef struct AVFilmGrainParams { + /** + * Specifies the codec for which this structure is valid. + */ + enum AVFilmGrainParamsType type; + + /** + * Seed to use for the synthesis process, if the codec allows for it. + */ + uint64_t seed; + + /** + * Additional fields may be added both here and in any structure included. + * If a codec's film grain structure differs slightly over another + * codec's, fields within may change meaning depending on the type. + */ + union { + AVFilmGrainAOMParams aom; + } codec; +} AVFilmGrainParams; + +/** + * Allocate an AVFilmGrainParams structure and set its fields to + * default values. The resulting struct can be freed using av_freep(). + * If size is not NULL it will be set to the number of bytes allocated. + * + * @return An AVFilmGrainParams filled with default values or NULL + * on failure. + */ +AVFilmGrainParams *av_film_grain_params_alloc(size_t *size); + +/** + * Allocate a complete AVFilmGrainParams and add it to the frame. + * + * @param frame The frame which side data is added to. + * + * @return The AVFilmGrainParams structure to be filled by caller. + */ +AVFilmGrainParams *av_film_grain_params_create_side_data(AVFrame *frame); + +#endif /* AVUTIL_FILM_GRAIN_PARAMS_H */ diff --git a/libavutil/frame.c b/libavutil/frame.c index b019779b1a..eab51b6a32 100644 --- a/libavutil/frame.c +++ b/libavutil/frame.c @@ -851,6 +851,7 @@ const char *av_frame_side_data_name(enum AVFrameSideDataType type) case AV_FRAME_DATA_REGIONS_OF_INTEREST: return "Regions Of Interest"; case AV_FRAME_DATA_VIDEO_ENC_PARAMS: return "Video encoding parameters"; case AV_FRAME_DATA_SEI_UNREGISTERED: return "H.26[45] User Data Unregistered SEI message"; + case AV_FRAME_DATA_FILM_GRAIN_PARAMS: return "Film grain parameters"; } return NULL; } diff --git a/libavutil/frame.h b/libavutil/frame.h index 3bd240fc97..392315f40f 100644 --- a/libavutil/frame.h +++ b/libavutil/frame.h @@ -192,6 +192,12 @@ enum AVFrameSideDataType { * uuid_iso_iec_11578 followed by AVFrameSideData.size - 16 bytes of user_data_payload_byte. */ AV_FRAME_DATA_SEI_UNREGISTERED, + + /** + * Film grain parameters for a frame, described by AVFilmGrainParameters. + * Must be present for every frame which should have film grain applied. + */ + AV_FRAME_DATA_FILM_GRAIN_PARAMS, }; enum AVActiveFormatDescription { diff --git a/libavutil/version.h b/libavutil/version.h index 1aca823650..55a59d6c58 100644 --- a/libavutil/version.h +++ b/libavutil/version.h @@ -79,7 +79,7 @@ */ #define LIBAVUTIL_VERSION_MAJOR 56 -#define LIBAVUTIL_VERSION_MINOR 60 +#define LIBAVUTIL_VERSION_MINOR 61 #define LIBAVUTIL_VERSION_MICRO 100 #define LIBAVUTIL_VERSION_INT AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \