shithub: libvpx

Info  •  Files  •  Log  •  Branches

Fix VP8 encoder UBSAN/IOC errors

1. vp8/decoder/dboolhuff.c
2. vp8/decoder/dboolhuff.h
3. vp8/encoder/bitstream.c
4. vp8/encoder/boolhuff.h
5. vp8/encoder/rdopt.c

BUG=https://bugs.chromium.org/p/webm/issues/detail?id=1218

Change-Id: I5d315d63fd7aeaee6f3bd79178e593f3db38a6b1

--- a/vp8/decoder/dboolhuff.c

+++ b/vp8/decoder/dboolhuff.c

@@ -44,7 +44,7 @@

     int shift = VP8_BD_VALUE_SIZE - CHAR_BIT - (count + CHAR_BIT);

     size_t bytes_left = br->user_buffer_end - bufptr;

     size_t bits_left = bytes_left * CHAR_BIT;

-    int x = (int)(shift + CHAR_BIT - bits_left);

+    int x = shift + CHAR_BIT - (int)bits_left;

     int loop_end = 0;

     unsigned char decrypted[sizeof(VP8_BD_VALUE) + 1];

--- a/vp8/decoder/dboolhuff.h

+++ b/vp8/decoder/dboolhuff.h

@@ -83,7 +83,7 @@

     }

     {

-        register unsigned int shift = vp8_norm[range];

+        register int shift = vp8_norm[range];

         range <<= shift;

         value <<= shift;

         count -= shift;

--- a/vp8/encoder/bitstream.c

+++ b/vp8/encoder/bitstream.c

@@ -163,7 +163,7 @@

 {

     const TOKENEXTRA *stop = p + xcount;

     unsigned int split;

-    unsigned int shift;

+    int shift;

     int count = w->count;

     unsigned int range = w->range;

     unsigned int lowvalue = w->lowvalue;

--- a/vp8/encoder/boolhuff.h

+++ b/vp8/encoder/boolhuff.h

@@ -65,7 +65,7 @@

     int count = br->count;

     unsigned int range = br->range;

     unsigned int lowvalue = br->lowvalue;

-    register unsigned int shift;

+    register int shift;

 #ifdef VP8_ENTROPY_STATS

 #if defined(SECTIONBITS_OUTPUT)

--- a/vp8/encoder/rdopt.c

+++ b/vp8/encoder/rdopt.c

@@ -1899,7 +1899,8 @@

                     int prob_skip_cost;

                     prob_skip_cost = vp8_cost_bit(cpi->prob_skip_false, 1);

-                    prob_skip_cost -= vp8_cost_bit(cpi->prob_skip_false, 0);

+                    prob_skip_cost -=

+                        (int)vp8_cost_bit(cpi->prob_skip_false, 0);

                     rd->rate2 += prob_skip_cost;

                     *other_cost += prob_skip_cost;

                 }