avformat/lxfdec: Fix multiple integer overflows related to track_size

Fixes: signed integer overflow: 538976288 * 8 cannot be represented in type 'int'
Fixes: 26910/clusterfuzz-testcase-minimized-ffmpeg_dem_LXF_fuzzer-6634030636335104

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
This commit is contained in:
Michael Niedermayer 2021-01-14 22:08:25 +01:00
parent 6c5121379a
commit 7819412f44

View File

@ -195,7 +195,7 @@ static int get_packet_header(AVFormatContext *s)
return AVERROR_PATCHWELCOME;
}
samples = track_size * 8 / st->codecpar->bits_per_coded_sample;
samples = track_size * 8LL / st->codecpar->bits_per_coded_sample;
//use audio packet size to determine video standard
//for NTSC we have one 8008-sample audio frame per five video frames
@ -210,6 +210,8 @@ static int get_packet_header(AVFormatContext *s)
avpriv_set_pts_info(s->streams[0], 64, 1, 25);
}
if (av_popcount(channels) * (uint64_t)track_size > INT_MAX)
return AVERROR_INVALIDDATA;
//TODO: warning if track mask != (1 << channels) - 1?
ret = av_popcount(channels) * track_size;