MrBesen/ffmpeg
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

avfilter/vf_spp: fix overflows with depth > 8

Browse Source 
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
This commit is contained in:
Michael Niedermayer 2014-12-20 20:39:13 +01:00
parent 5318cf521f
commit 1ac5abb1d0
1 changed files with 11 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
libavfilter/vf_spp.c
View File

@ -186,16 +186,17 @@ static void store_slice_c(uint8_t *dst, const uint16_t *src,
}
}
static void store_slice16_c(uint16_t *dst, const uint16_t *src,
static void store_slice16_c(uint16_t *dst, const int16_t *src,
int dst_linesize, int src_linesize,
int width, int height, int log2_scale,
const uint8_t dither[8][8])
const uint8_t dither[8][8], int depth)
{
int y, x;
unsigned int mask = -1<<depth;
#define STORE16(pos) do { \
temp = ((src[x + y*src_linesize + pos] << log2_scale) + (d[pos]>>1)) >> 5; \
if (temp & 0x400) \
if (temp & mask ) \
temp = ~(temp >> 31); \
dst[x + y*dst_linesize + pos] = temp; \
} while (0)
@ -242,7 +243,7 @@ static inline int norm_qscale(int qscale, int type)
static void filter(SPPContext *p, uint8_t *dst, uint8_t *src,
int dst_linesize, int src_linesize, int width, int height,
const uint8_t *qp_table, int qp_stride, int is_luma, int sample_bytes)
const uint8_t *qp_table, int qp_stride, int is_luma, int depth)
{
int x, y, i;
const int count = 1 << p->log2_count;
@ -251,6 +252,7 @@ static void filter(SPPContext *p, uint8_t *dst, uint8_t *src,
int16_t *block = (int16_t *)block_align;
int16_t *block2 = (int16_t *)(block_align + 16);
uint16_t *psrc16 = (uint16_t*)p->src;
const int sample_bytes = (depth+7) / 8;
for (y = 0; y < height; y++) {
int index = 8 + 8*linesize + y*linesize;
@ -305,7 +307,7 @@ static void filter(SPPContext *p, uint8_t *dst, uint8_t *src,
store_slice16_c((uint16_t*)(dst + (y - 8) * dst_linesize), p->temp + 8 + y*linesize,
dst_linesize/2, linesize, width,
FFMIN(8, height + 8 - y), MAX_LEVEL - p->log2_count,
ldither);
ldither, depth);
}
}
}
@ -366,7 +368,7 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in)
int qp_stride = 0;
const int8_t *qp_table = NULL;
const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
const int sample_bytes = desc->comp[0].depth_minus1 < 8 ? 1 : 2;
const int depth = desc->comp[0].depth_minus1 + 1;
/* if we are not in a constant user quantizer mode and we don't want to use
* the quantizers from the B-frames (B-frames often have a higher QP), we
@ -426,11 +428,11 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in)
out->height = in->height;
}
filter(spp, out->data[0], in->data[0], out->linesize[0], in->linesize[0], inlink->w, inlink->h, qp_table, qp_stride, 1, sample_bytes);
filter(spp, out->data[0], in->data[0], out->linesize[0], in->linesize[0], inlink->w, inlink->h, qp_table, qp_stride, 1, depth);
if (out->data[2]) {
filter(spp, out->data[1], in->data[1], out->linesize[1], in->linesize[1], cw, ch, qp_table, qp_stride, 0, sample_bytes);
filter(spp, out->data[2], in->data[2], out->linesize[2], in->linesize[2], cw, ch, qp_table, qp_stride, 0, sample_bytes);
filter(spp, out->data[1], in->data[1], out->linesize[1], in->linesize[1], cw, ch, qp_table, qp_stride, 0, depth);
filter(spp, out->data[2], in->data[2], out->linesize[2], in->linesize[2], cw, ch, qp_table, qp_stride, 0, depth);
}
emms_c();
}