From 38d553322891c8e47182f05199d19888422167dc Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Wed, 1 Feb 2012 15:32:21 +0100 Subject: [PATCH] pixdesc: mark pseudopaletted formats with a special flag. This makes it possible to dintinguish them from PAL8. Fixes an invalid write in avpicture_layout(). --- doc/APIchanges | 3 +++ libavcodec/imgconvert.c | 8 +------- libavcodec/rawdec.c | 3 +-- libavfilter/vf_crop.c | 3 ++- libavfilter/vf_pixdesctest.c | 3 ++- libavfilter/vf_scale.c | 3 ++- libavutil/avutil.h | 2 +- libavutil/imgutils.c | 9 ++++++--- libavutil/pixdesc.c | 8 ++++---- libavutil/pixdesc.h | 6 ++++++ libswscale/swscale_internal.h | 4 +++- 11 files changed, 31 insertions(+), 21 deletions(-) diff --git a/doc/APIchanges b/doc/APIchanges index 8da67d0ae2..f962432651 100644 --- a/doc/APIchanges +++ b/doc/APIchanges @@ -13,6 +13,9 @@ libavutil: 2011-04-18 API changes, most recent first: +2012-02-xx - xxxxxxx - lavu 51.22.1 - pixdesc.h + Add PIX_FMT_PSEUDOPAL flag. + 2012-02-01 - xxxxxxx - lavc 54.01.0 Add avcodec_encode_video2() and deprecate avcodec_encode_video(). diff --git a/libavcodec/imgconvert.c b/libavcodec/imgconvert.c index d90254ca76..90c9b7b255 100644 --- a/libavcodec/imgconvert.c +++ b/libavcodec/imgconvert.c @@ -471,15 +471,9 @@ int avpicture_get_size(enum PixelFormat pix_fmt, int width, int height) AVPicture dummy_pict; if(av_image_check_size(width, height, 0, NULL)) return -1; - switch (pix_fmt) { - case PIX_FMT_RGB8: - case PIX_FMT_BGR8: - case PIX_FMT_RGB4_BYTE: - case PIX_FMT_BGR4_BYTE: - case PIX_FMT_GRAY8: + if (av_pix_fmt_descriptors[pix_fmt].flags & PIX_FMT_PSEUDOPAL) // do not include palette for these pseudo-paletted formats return width * height; - } return avpicture_fill(&dummy_pict, NULL, pix_fmt, width, height); } diff --git a/libavcodec/rawdec.c b/libavcodec/rawdec.c index 427d109a2b..bb93129027 100644 --- a/libavcodec/rawdec.c +++ b/libavcodec/rawdec.c @@ -158,8 +158,7 @@ static int raw_decode(AVCodecContext *avctx, avpicture_fill(picture, buf, avctx->pix_fmt, avctx->width, avctx->height); if((avctx->pix_fmt==PIX_FMT_PAL8 && buf_size < context->length) || - (avctx->pix_fmt!=PIX_FMT_PAL8 && - (av_pix_fmt_descriptors[avctx->pix_fmt].flags & PIX_FMT_PAL))){ + (av_pix_fmt_descriptors[avctx->pix_fmt].flags & PIX_FMT_PSEUDOPAL)) { frame->data[1]= context->palette; } if (avctx->pix_fmt == PIX_FMT_PAL8) { diff --git a/libavfilter/vf_crop.c b/libavfilter/vf_crop.c index 0880d4e5f9..cb01bd41cb 100644 --- a/libavfilter/vf_crop.c +++ b/libavfilter/vf_crop.c @@ -272,7 +272,8 @@ static void start_frame(AVFilterLink *link, AVFilterBufferRef *picref) ref2->data[0] += crop->y * ref2->linesize[0]; ref2->data[0] += crop->x * crop->max_step[0]; - if (!(av_pix_fmt_descriptors[link->format].flags & PIX_FMT_PAL)) { + if (!(av_pix_fmt_descriptors[link->format].flags & PIX_FMT_PAL || + av_pix_fmt_descriptors[link->format].flags & PIX_FMT_PSEUDOPAL)) { for (i = 1; i < 3; i ++) { if (ref2->data[i]) { ref2->data[i] += (crop->y >> crop->vsub) * ref2->linesize[i]; diff --git a/libavfilter/vf_pixdesctest.c b/libavfilter/vf_pixdesctest.c index 344f6648f1..cf7dfeda3e 100644 --- a/libavfilter/vf_pixdesctest.c +++ b/libavfilter/vf_pixdesctest.c @@ -72,7 +72,8 @@ static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *picref) } /* copy palette */ - if (priv->pix_desc->flags & PIX_FMT_PAL) + if (priv->pix_desc->flags & PIX_FMT_PAL || + priv->pix_desc->flags & PIX_FMT_PSEUDOPAL) memcpy(outpicref->data[1], outpicref->data[1], 256*4); avfilter_start_frame(outlink, avfilter_ref_buffer(outpicref, ~0)); diff --git a/libavfilter/vf_scale.c b/libavfilter/vf_scale.c index dd2f7e18a5..37a6f8e386 100644 --- a/libavfilter/vf_scale.c +++ b/libavfilter/vf_scale.c @@ -208,7 +208,8 @@ static int config_props(AVFilterLink *outlink) outlink->w, outlink->h, av_pix_fmt_descriptors[outlink->format].name, scale->flags); - scale->input_is_pal = av_pix_fmt_descriptors[inlink->format].flags & PIX_FMT_PAL; + scale->input_is_pal = av_pix_fmt_descriptors[inlink->format].flags & PIX_FMT_PAL || + av_pix_fmt_descriptors[inlink->format].flags & PIX_FMT_PSEUDOPAL; if (scale->sws) sws_freeContext(scale->sws); diff --git a/libavutil/avutil.h b/libavutil/avutil.h index 0e62b4a13f..05e9248375 100644 --- a/libavutil/avutil.h +++ b/libavutil/avutil.h @@ -155,7 +155,7 @@ #define LIBAVUTIL_VERSION_MAJOR 51 #define LIBAVUTIL_VERSION_MINOR 22 -#define LIBAVUTIL_VERSION_MICRO 0 +#define LIBAVUTIL_VERSION_MICRO 1 #define LIBAVUTIL_VERSION_INT AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \ LIBAVUTIL_VERSION_MINOR, \ diff --git a/libavutil/imgutils.c b/libavutil/imgutils.c index b734db7bb1..8c8251bddb 100644 --- a/libavutil/imgutils.c +++ b/libavutil/imgutils.c @@ -108,7 +108,8 @@ int av_image_fill_pointers(uint8_t *data[4], enum PixelFormat pix_fmt, int heigh return AVERROR(EINVAL); size[0] = linesizes[0] * height; - if (desc->flags & PIX_FMT_PAL) { + if (desc->flags & PIX_FMT_PAL || + desc->flags & PIX_FMT_PSEUDOPAL) { size[0] = (size[0] + 3) & ~3; data[1] = ptr + size[0]; /* palette is stored here as 256 32 bits words */ return size[0] + 256 * 4; @@ -196,7 +197,8 @@ int av_image_alloc(uint8_t *pointers[4], int linesizes[4], av_free(buf); return ret; } - if (av_pix_fmt_descriptors[pix_fmt].flags & PIX_FMT_PAL) + if (av_pix_fmt_descriptors[pix_fmt].flags & PIX_FMT_PAL || + av_pix_fmt_descriptors[pix_fmt].flags & PIX_FMT_PSEUDOPAL) ff_set_systematic_pal2((uint32_t*)pointers[1], pix_fmt); return ret; @@ -243,7 +245,8 @@ void av_image_copy(uint8_t *dst_data[4], int dst_linesizes[4], if (desc->flags & PIX_FMT_HWACCEL) return; - if (desc->flags & PIX_FMT_PAL) { + if (desc->flags & PIX_FMT_PAL || + desc->flags & PIX_FMT_PSEUDOPAL) { av_image_copy_plane(dst_data[0], dst_linesizes[0], src_data[0], src_linesizes[0], width, height); diff --git a/libavutil/pixdesc.c b/libavutil/pixdesc.c index 6e1f23b97f..8e08b5a989 100644 --- a/libavutil/pixdesc.c +++ b/libavutil/pixdesc.c @@ -327,7 +327,7 @@ const AVPixFmtDescriptor av_pix_fmt_descriptors[PIX_FMT_NB] = { { 0, 0, 1, 3, 2 }, /* G */ { 0, 0, 1, 0, 2 }, /* R */ }, - .flags = PIX_FMT_PAL | PIX_FMT_RGB, + .flags = PIX_FMT_RGB | PIX_FMT_PSEUDOPAL, }, [PIX_FMT_BGR4] = { .name = "bgr4", @@ -351,7 +351,7 @@ const AVPixFmtDescriptor av_pix_fmt_descriptors[PIX_FMT_NB] = { { 0, 0, 1, 1, 1 }, /* G */ { 0, 0, 1, 0, 0 }, /* R */ }, - .flags = PIX_FMT_PAL | PIX_FMT_RGB, + .flags = PIX_FMT_RGB | PIX_FMT_PSEUDOPAL, }, [PIX_FMT_RGB8] = { .name = "rgb8", @@ -363,7 +363,7 @@ const AVPixFmtDescriptor av_pix_fmt_descriptors[PIX_FMT_NB] = { { 0, 0, 1, 3, 2 }, /* G */ { 0, 0, 1, 0, 2 }, /* B */ }, - .flags = PIX_FMT_PAL | PIX_FMT_RGB, + .flags = PIX_FMT_RGB | PIX_FMT_PSEUDOPAL, }, [PIX_FMT_RGB4] = { .name = "rgb4", @@ -387,7 +387,7 @@ const AVPixFmtDescriptor av_pix_fmt_descriptors[PIX_FMT_NB] = { { 0, 0, 1, 1, 1 }, /* G */ { 0, 0, 1, 0, 0 }, /* B */ }, - .flags = PIX_FMT_PAL | PIX_FMT_RGB, + .flags = PIX_FMT_RGB | PIX_FMT_PSEUDOPAL, }, [PIX_FMT_NV12] = { .name = "nv12", diff --git a/libavutil/pixdesc.h b/libavutil/pixdesc.h index b5972c78ff..23a97ee231 100644 --- a/libavutil/pixdesc.h +++ b/libavutil/pixdesc.h @@ -89,6 +89,12 @@ typedef struct AVPixFmtDescriptor{ #define PIX_FMT_HWACCEL 8 ///< Pixel format is an HW accelerated format. #define PIX_FMT_PLANAR 16 ///< At least one pixel component is not in the first data plane #define PIX_FMT_RGB 32 ///< The pixel format contains RGB-like data (as opposed to YUV/grayscale) +/** + * The pixel format is "pseudo-paletted". This means that Libav treats it as + * paletted internally, but the palette is generated by the decoder and is not + * stored in the file. + */ +#define PIX_FMT_PSEUDOPAL 64 /** * The array of all the pixel format descriptors. diff --git a/libswscale/swscale_internal.h b/libswscale/swscale_internal.h index 0f5404e154..bc36826ea2 100644 --- a/libswscale/swscale_internal.h +++ b/libswscale/swscale_internal.h @@ -627,7 +627,9 @@ const char *sws_format_name(enum PixelFormat format); (av_pix_fmt_descriptors[x].nb_components >= 2 && \ (av_pix_fmt_descriptors[x].flags & PIX_FMT_PLANAR)) -#define usePal(x) ((av_pix_fmt_descriptors[x].flags & PIX_FMT_PAL) || (x) == PIX_FMT_Y400A) +#define usePal(x) ((av_pix_fmt_descriptors[x].flags & PIX_FMT_PAL) || \ + (av_pix_fmt_descriptors[x].flags & PIX_FMT_PSEUDOPAL) || \ + (x) == PIX_FMT_Y400A) extern const uint64_t ff_dither4[2]; extern const uint64_t ff_dither8[2];