shithub: libvpx

Info  •  Files  •  Log  •  Branches

fail allocation of buffers if size_t < frame_size

Change-Id: I25c595e8c197ab0a9955d2373f1a74d42fbd1638

--- a/vpx_scale/generic/yv12config.c

+++ b/vpx_scale/generic/yv12config.c

@@ -142,13 +142,15 @@

     const int aligned_width = (width + 7) & ~7;

     const int aligned_height = (height + 7) & ~7;

     const int y_stride = ((aligned_width + 2 * border) + 31) & ~31;

-    const int yplane_size = (aligned_height + 2 * border) * y_stride;

+    const uint64_t yplane_size = (aligned_height + 2 * border) *

+                                 (uint64_t)y_stride;

     const int uv_width = aligned_width >> ss_x;

     const int uv_height = aligned_height >> ss_y;

     const int uv_stride = y_stride >> ss_x;

     const int uv_border_w = border >> ss_x;

     const int uv_border_h = border >> ss_y;

-    const int uvplane_size = (uv_height + 2 * uv_border_h) * uv_stride;

+    const uint64_t uvplane_size = (uv_height + 2 * uv_border_h) *

+                                  (uint64_t)uv_stride;

 #if CONFIG_ALPHA

     const int alpha_width = aligned_width;

     const int alpha_height = aligned_height;

@@ -155,21 +157,24 @@

     const int alpha_stride = y_stride;

     const int alpha_border_w = border;

     const int alpha_border_h = border;

-    const int alpha_plane_size = (alpha_height + 2 * alpha_border_h) *

-                                 alpha_stride;

-    const int frame_size = yplane_size + 2 * uvplane_size +

-                           alpha_plane_size;

+    const uint64_t alpha_plane_size = (alpha_height + 2 * alpha_border_h) *

+                                      (uint64_t)alpha_stride;

+    const uint64_t frame_size = yplane_size + 2 * uvplane_size +

+                                alpha_plane_size;

 #else

-    const int frame_size = yplane_size + 2 * uvplane_size;

+    const uint64_t frame_size = yplane_size + 2 * uvplane_size;

 #endif

     if (cb != NULL) {

       const int align_addr_extra_size = 31;

-      const size_t external_frame_size = frame_size + align_addr_extra_size;

+      const uint64_t external_frame_size = frame_size + align_addr_extra_size;

       assert(fb != NULL);

+      if (external_frame_size != (size_t)external_frame_size)

+        return -1;

+

       // Allocation to hold larger frame, or first allocation.

-      if (cb(cb_priv, external_frame_size, fb) < 0)

+      if (cb(cb_priv, (size_t)external_frame_size, fb) < 0)

         return -1;

       if (fb->data == NULL || fb->size < external_frame_size)

@@ -181,10 +186,15 @@

       vpx_memset(fb->data, 0, fb->size);

       ybf->buffer_alloc = (uint8_t *)yv12_align_addr(fb->data, 32);

-    } else if (frame_size > ybf->buffer_alloc_sz) {

+    } else if (frame_size > (size_t)ybf->buffer_alloc_sz) {

       // Allocation to hold larger frame, or first allocation.

       vpx_free(ybf->buffer_alloc);

-      ybf->buffer_alloc = (uint8_t *)vpx_memalign(32, frame_size);

+      ybf->buffer_alloc = NULL;

+

+      if (frame_size != (size_t)frame_size)

+        return -1;

+

+      ybf->buffer_alloc = (uint8_t *)vpx_memalign(32, (size_t)frame_size);

       if (!ybf->buffer_alloc)

         return -1;