avcodec/cbs: Fix potential overflow

The number of bits in a PutBitContext must fit into an int, yet nothing
guaranteed the size argument cbs_write_unit_data() uses in init_put_bits()
to be in the range 0..INT_MAX / 8. This has been changed.

Furthermore, the check 8 * data_size > data_bit_start that there is
data beyond the initial padding when writing mpeg2 or H.264/5 slices
could also overflow, so divide it by 8 to get an equivalent check
without this problem.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@gmail.com>
This commit is contained in:
Andreas Rheinhardt 2019-11-17 08:34:36 +01:00 committed by Mark Thompson
parent 7c92eaace2
commit cda3e8ca04
3 changed files with 5 additions and 3 deletions

View File

@ -309,7 +309,9 @@ static int cbs_write_unit_data(CodedBitstreamContext *ctx,
if (ret < 0) {
if (ret == AVERROR(ENOSPC)) {
// Overflow.
ctx->write_buffer_size *= 2;
if (ctx->write_buffer_size == INT_MAX / 8)
return AVERROR(ENOMEM);
ctx->write_buffer_size = FFMIN(2 * ctx->write_buffer_size, INT_MAX / 8);
goto reallocate_and_try_again;
}
// Write failed for some other reason.

View File

@ -1101,7 +1101,7 @@ static int cbs_h2645_write_slice_data(CodedBitstreamContext *ctx,
const uint8_t *pos = data + data_bit_start / 8;
av_assert0(data_bit_start >= 0 &&
8 * data_size > data_bit_start);
data_size > data_bit_start / 8);
if (data_size * 8 + 8 > put_bits_left(pbc))
return AVERROR(ENOSPC);

View File

@ -337,7 +337,7 @@ static int cbs_mpeg2_write_slice(CodedBitstreamContext *ctx,
uint8_t *pos = slice->data + slice->data_bit_start / 8;
av_assert0(slice->data_bit_start >= 0 &&
8 * slice->data_size > slice->data_bit_start);
slice->data_size > slice->data_bit_start / 8);
if (slice->data_size * 8 + 8 > put_bits_left(pbc))
return AVERROR(ENOSPC);