sanity checks, some might have been exploitable ...

Originally committed as revision 5369 to svn://svn.ffmpeg.org/ffmpeg/trunk

libavcodec/4xm.c

@@ -606,7 +606,7 @@ static int decode_frame(AVCodecContext *avctx,
     int i, frame_4cc, frame_size;
 
     frame_4cc= get32(buf);
-    if(buf_size != get32(buf+4)+8){
+    if(buf_size != get32(buf+4)+8 || buf_size < 20){
         av_log(f->avctx, AV_LOG_ERROR, "size mismatch %d %d\n", buf_size, get32(buf+4));
     }
 
@@ -634,6 +634,10 @@ static int decode_frame(AVCodecContext *avctx,
         cfrm= &f->cfrm[i];
 
         cfrm->data= av_fast_realloc(cfrm->data, &cfrm->allocated_size, cfrm->size + data_size + FF_INPUT_BUFFER_PADDING_SIZE);
+        if(!cfrm->data){ //explicit check needed as memcpy below might not catch a NULL
+            av_log(f->avctx, AV_LOG_ERROR, "realloc falure");
+            return -1;
+        }
 
         memcpy(cfrm->data + cfrm->size, buf+20, data_size);
         cfrm->size += data_size;

libavcodec/alac.c

@@ -100,7 +100,7 @@ static void allocate_buffers(ALACContext *alac)
     alac->outputsamples_buffer_b = av_malloc(alac->setinfo_max_samples_per_frame * 4);
 }
 
-static void alac_set_info(ALACContext *alac)
+static int alac_set_info(ALACContext *alac)
 {
     unsigned char *ptr = alac->avctx->extradata;
 
@@ -108,6 +108,10 @@ static void alac_set_info(ALACContext *alac)
     ptr += 4; /* alac */
     ptr += 4; /* 0 ? */
 
+    if(BE_32(ptr) >= UINT_MAX/4){
+        av_log(alac->avctx, AV_LOG_ERROR, "setinfo_max_samples_per_frame too large\n");
+        return -1;
+    }
     alac->setinfo_max_samples_per_frame = BE_32(ptr); /* buffer size / 2 ? */
     ptr += 4;
     alac->setinfo_7a = *ptr++;
@@ -126,6 +130,8 @@ static void alac_set_info(ALACContext *alac)
     ptr += 4;
 
     allocate_buffers(alac);
+
+    return 0;
 }
 
 /* hideously inefficient. could use a bitmask search,

libavcodec/cook.c

@@ -1253,6 +1253,10 @@ static int cook_decode_init(AVCodecContext *avctx)
     if (init_cook_vlc_tables(q) != 0)
         return -1;
 
+
+    if(avctx->block_align >= UINT_MAX/2)
+        return -1;
+
     /* Pad the databuffer with FF_INPUT_BUFFER_PADDING_SIZE,
        this is for the bitstreamreader. */
     if ((q->decoded_bytes_buffer = av_mallocz((avctx->block_align+(4-avctx->block_align%4) + FF_INPUT_BUFFER_PADDING_SIZE)*sizeof(uint8_t)))  == NULL)

libavcodec/shorten.c

@@ -106,18 +106,27 @@ static int shorten_decode_init(AVCodecContext * avctx)
     return 0;
 }
 
-static void allocate_buffers(ShortenContext *s)
+static int allocate_buffers(ShortenContext *s)
 {
     int i, chan;
     for (chan=0; chan<s->channels; chan++) {
+        if(FFMAX(1, s->nmean) >= UINT_MAX/sizeof(int32_t)){
+            av_log(s->avctx, AV_LOG_ERROR, "nmean too large\n");
+            return -1;
+        }
+        if(s->blocksize + s->nwrap >= UINT_MAX/sizeof(int32_t) || s->blocksize + s->nwrap <= (unsigned)s->nwrap){
+            av_log(s->avctx, AV_LOG_ERROR, "s->blocksize + s->nwrap too large\n");
+            return -1;
+        }
+
         s->offset[chan] = av_realloc(s->offset[chan], sizeof(int32_t)*FFMAX(1, s->nmean));
 
         s->decoded[chan] = av_realloc(s->decoded[chan], sizeof(int32_t)*(s->blocksize + s->nwrap));
         for (i=0; i<s->nwrap; i++)
             s->decoded[chan][i] = 0;
         s->decoded[chan] += s->nwrap;
-
     }
+    return 0;
 }
 
 

libavcodec/smacker.c

@@ -177,6 +177,11 @@ static int smacker_decode_header_tree(SmackVContext *smk, GetBitContext *gb, int
     int escapes[3];
     DBCtx ctx;
 
+    if(size >= UINT_MAX>>4){ // (((size + 3) >> 2) + 3) << 2 must not overflow
+        av_log(smk->avctx, AV_LOG_ERROR, "size too large\n");
+        return -1;
+    }
+
     tmp1.length = 256;
     tmp1.maxlength = 0;
     tmp1.current = 0;

libavcodec/snow.c

@@ -3712,7 +3712,7 @@ static int decode_header(SnowContext *s){
     s->mv_scale= get_symbol(&s->c, s->header_state, 0);
     s->qbias= get_symbol(&s->c, s->header_state, 1);
     s->block_max_depth= get_symbol(&s->c, s->header_state, 0);
-    if(s->block_max_depth > 1){
+    if(s->block_max_depth > 1 || s->block_max_depth < 0){
         av_log(s->avctx, AV_LOG_ERROR, "block_max_depth= %d is too large", s->block_max_depth);
         s->block_max_depth= 0;
         return -1;

libavcodec/tta.c

@@ -238,6 +238,10 @@ static int tta_decode_init(AVCodecContext * avctx)
         avctx->bits_per_sample = get_le16(&s->gb);
         s->bps = (avctx->bits_per_sample + 7) / 8;
         avctx->sample_rate = get_le32(&s->gb);
+        if(avctx->sample_rate > 1000000){ //prevent FRAME_TIME * avctx->sample_rate from overflowing and sanity check
+            av_log(avctx, AV_LOG_ERROR, "sample_rate too large\n");
+            return -1;
+        }
         s->data_length = get_le32(&s->gb);
         skip_bits(&s->gb, 32); // CRC32 of header
 
@@ -276,6 +280,11 @@ static int tta_decode_init(AVCodecContext * avctx)
             skip_bits(&s->gb, 32);
         skip_bits(&s->gb, 32); // CRC32 of seektable
 
+        if(s->frame_length >= UINT_MAX / (s->channels * sizeof(int32_t))){
+            av_log(avctx, AV_LOG_ERROR, "frame_length too large\n");
+            return -1;
+        }
+
         s->decode_buffer = av_mallocz(sizeof(int32_t)*s->frame_length*s->channels);
     } else {
         av_log(avctx, AV_LOG_ERROR, "Wrong extradata present\n");