avcodec/apedec: properly calculate and store absolute value

This commit is contained in:
Paul B Mahol 2020-10-06 01:24:42 +02:00
parent ed0001482a
commit ea0972f6dd

View File

@ -1311,7 +1311,7 @@ static void do_apply_filter(APEContext *ctx, int version, APEFilter *f,
int32_t *data, int count, int order, int fracbits)
{
int res;
int absres;
unsigned absres;
while (count--) {
/* round fixedpoint scalar product */
@ -1335,7 +1335,7 @@ static void do_apply_filter(APEContext *ctx, int version, APEFilter *f,
/* Version 3.98 and later files */
/* Update the adaption coefficients */
absres = res < 0 ? -(unsigned)res : res;
absres = FFABS(res);
if (absres)
*f->adaptcoeffs = APESIGN(res) *
(8 << ((absres > f->avg * 3) + (absres > f->avg * 4 / 3)));