ref: fbb62c6d2bd1951cb1a69bdba13c582d2939f93e
parent: 8c2e5e49645feea4c9bff63a324200823812c7c1
author: Yaowu Xu <[email protected]>
date: Fri Sep 20 09:29:59 EDT 2013
fix integer overflow in vp8 Change-Id: I62550a7a883115f3ce878710cf3bb039bea54390
--- a/vp8/common/findnearmv.h
+++ b/vp8/common/findnearmv.h
@@ -124,7 +124,7 @@
b += 16;
}
- return (cur_mb->bmi + b - 4)->mv.as_int;
+ return (cur_mb->bmi + (b - 4))->mv.as_int;
}
static B_PREDICTION_MODE left_block_mode(const MODE_INFO *cur_mb, int b)
{
--- a/vp8/common/reconinter.c
+++ b/vp8/common/reconinter.c
@@ -138,14 +138,10 @@
{
for (r = 0; r < 4; r++)
{
-#if !(CONFIG_FAST_UNALIGNED)
pred_ptr[0] = ptr[0];
pred_ptr[1] = ptr[1];
pred_ptr[2] = ptr[2];
pred_ptr[3] = ptr[3];
-#else
- *(uint32_t *)pred_ptr = *(uint32_t *)ptr ;
-#endif
pred_ptr += pitch;
ptr += pre_stride;
}
@@ -196,16 +192,12 @@
{
for (r = 0; r < 4; r++)
{
-#if !(CONFIG_FAST_UNALIGNED)
dst[0] = ptr[0];
dst[1] = ptr[1];
dst[2] = ptr[2];
dst[3] = ptr[3];
-#else
- *(uint32_t *)dst = *(uint32_t *)ptr ;
-#endif
- dst += dst_stride;
- ptr += pre_stride;
+ dst += dst_stride;
+ ptr += pre_stride;
}
}
}
@@ -270,7 +262,7 @@
+ x->block[yoffset+4].bmi.mv.as_mv.row
+ x->block[yoffset+5].bmi.mv.as_mv.row;
- temp += 4 + ((temp >> (sizeof(int) * CHAR_BIT - 1)) << 3);
+ temp += 4 + ((temp >> (sizeof(temp) * CHAR_BIT - 1)) * 8);
x->block[uoffset].bmi.mv.as_mv.row = (temp / 8) & x->fullpixel_mask;
@@ -279,7 +271,7 @@
+ x->block[yoffset+4].bmi.mv.as_mv.col
+ x->block[yoffset+5].bmi.mv.as_mv.col;
- temp += 4 + ((temp >> (sizeof(int) * CHAR_BIT - 1)) << 3);
+ temp += 4 + ((temp >> (sizeof(temp) * CHAR_BIT - 1)) * 8);
x->block[uoffset].bmi.mv.as_mv.col = (temp / 8) & x->fullpixel_mask;
@@ -558,7 +550,7 @@
+ x->mode_info_context->bmi[yoffset + 4].mv.as_mv.row
+ x->mode_info_context->bmi[yoffset + 5].mv.as_mv.row;
- temp += 4 + ((temp >> (sizeof(int) * CHAR_BIT - 1)) << 3);
+ temp += 4 + ((temp >> (sizeof(temp) * CHAR_BIT - 1)) * 8);
x->block[uoffset].bmi.mv.as_mv.row = (temp / 8) & x->fullpixel_mask;
@@ -567,7 +559,7 @@
+ x->mode_info_context->bmi[yoffset + 4].mv.as_mv.col
+ x->mode_info_context->bmi[yoffset + 5].mv.as_mv.col;
- temp += 4 + ((temp >> (sizeof(int) * CHAR_BIT - 1)) << 3);
+ temp += 4 + ((temp >> (sizeof(temp) * CHAR_BIT - 1)) * 8);
x->block[uoffset].bmi.mv.as_mv.col = (temp / 8) & x->fullpixel_mask;
--- a/vp8/decoder/decodemv.c
+++ b/vp8/decoder/decodemv.c
@@ -110,8 +110,8 @@
static void read_mv(vp8_reader *r, MV *mv, const MV_CONTEXT *mvc)
{
- mv->row = (short)(read_mvcomponent(r, mvc) << 1);
- mv->col = (short)(read_mvcomponent(r, ++mvc) << 1);
+ mv->row = (short)(read_mvcomponent(r, mvc) * 2);
+ mv->col = (short)(read_mvcomponent(r, ++mvc) * 2);
}
@@ -292,9 +292,9 @@
blockmv.as_int = 0;
if( vp8_read(bc, prob[2]) )
{
- blockmv.as_mv.row = read_mvcomponent(bc, &mvc[0]) << 1;
+ blockmv.as_mv.row = read_mvcomponent(bc, &mvc[0]) * 2;
blockmv.as_mv.row += best_mv.as_mv.row;
- blockmv.as_mv.col = read_mvcomponent(bc, &mvc[1]) << 1;
+ blockmv.as_mv.col = read_mvcomponent(bc, &mvc[1]) * 2;
blockmv.as_mv.col += best_mv.as_mv.col;
}
}
--- a/vp8/decoder/decodframe.c
+++ b/vp8/decoder/decodframe.c
@@ -576,7 +576,7 @@
xd->left_available = 0;
- xd->mb_to_top_edge = -((mb_row * 16)) << 3;
+ xd->mb_to_top_edge = -((mb_row * 16) << 3);
xd->mb_to_bottom_edge = ((pc->mb_rows - 1 - mb_row) * 16) << 3;
xd->recon_above[0] = dst_buffer[0] + recon_yoffset;
--- a/vp8/encoder/bitstream.c
+++ b/vp8/encoder/bitstream.c
@@ -432,7 +432,7 @@
assert(NEARESTMV <= m && m <= SPLITMV);
#endif
vp8_write_token(w, vp8_mv_ref_tree, p,
- vp8_mv_ref_encoding_array - NEARESTMV + m);
+ vp8_mv_ref_encoding_array + (m - NEARESTMV));
}
static void write_sub_mv_ref
@@ -444,7 +444,7 @@
assert(LEFT4X4 <= m && m <= NEW4X4);
#endif
vp8_write_token(w, vp8_sub_mv_ref_tree, p,
- vp8_sub_mv_ref_encoding_array - LEFT4X4 + m);
+ vp8_sub_mv_ref_encoding_array + (m - LEFT4X4));
}
static void write_mv
@@ -577,7 +577,7 @@
*/
xd->mb_to_left_edge = -((mb_col * 16) << 3);
xd->mb_to_right_edge = ((pc->mb_cols - 1 - mb_col) * 16) << 3;
- xd->mb_to_top_edge = -((mb_row * 16)) << 3;
+ xd->mb_to_top_edge = -((mb_row * 16) << 3);
xd->mb_to_bottom_edge = ((pc->mb_rows - 1 - mb_row) * 16) << 3;
#ifdef VP8_ENTROPY_STATS
--- a/vp8/encoder/firstpass.c
+++ b/vp8/encoder/firstpass.c
@@ -711,8 +711,8 @@
neutral_count++;
}
- d->bmi.mv.as_mv.row <<= 3;
- d->bmi.mv.as_mv.col <<= 3;
+ d->bmi.mv.as_mv.row *= 8;
+ d->bmi.mv.as_mv.col *= 8;
this_error = motion_error;
vp8_set_mbmode_and_mvs(x, NEWMV, &d->bmi.mv);
vp8_encode_inter16x16y(x);
@@ -915,7 +915,7 @@
FIRSTPASS_STATS * fpstats)
{
int mv_cost;
- int mode_cost;
+ int64_t mode_cost;
double av_pct_inter = fpstats->pcnt_inter / fpstats->count;
double av_pct_motion = fpstats->pcnt_motion / fpstats->count;
@@ -937,10 +937,9 @@
/* Crude estimate of overhead cost from modes
* << 9 is the normalization to (bits * 512) used in vp8_bits_per_mb
*/
- mode_cost =
- (int)( ( ((av_pct_inter - av_pct_motion) * zz_cost) +
- (av_pct_motion * motion_cost) +
- (av_intra * intra_cost) ) * cpi->common.MBs ) << 9;
+ mode_cost =((((av_pct_inter - av_pct_motion) * zz_cost) +
+ (av_pct_motion * motion_cost) +
+ (av_intra * intra_cost)) * cpi->common.MBs) * 512;
return mv_cost + mode_cost;
}
--- a/vp8/encoder/mcomp.c
+++ b/vp8/encoder/mcomp.c
@@ -210,7 +210,7 @@
unsigned char *z = (*(b->base_src) + b->src);
int rr = ref_mv->as_mv.row >> 1, rc = ref_mv->as_mv.col >> 1;
- int br = bestmv->as_mv.row << 2, bc = bestmv->as_mv.col << 2;
+ int br = bestmv->as_mv.row * 4, bc = bestmv->as_mv.col * 4;
int tr = br, tc = bc;
unsigned int besterr;
unsigned int left, right, up, down, diag;
@@ -220,10 +220,14 @@
unsigned int quarteriters = 4;
int thismse;
- int minc = MAX(x->mv_col_min << 2, (ref_mv->as_mv.col >> 1) - ((1 << mvlong_width) - 1));
- int maxc = MIN(x->mv_col_max << 2, (ref_mv->as_mv.col >> 1) + ((1 << mvlong_width) - 1));
- int minr = MAX(x->mv_row_min << 2, (ref_mv->as_mv.row >> 1) - ((1 << mvlong_width) - 1));
- int maxr = MIN(x->mv_row_max << 2, (ref_mv->as_mv.row >> 1) + ((1 << mvlong_width) - 1));
+ int minc = MAX(x->mv_col_min * 4,
+ (ref_mv->as_mv.col >> 1) - ((1 << mvlong_width) - 1));
+ int maxc = MIN(x->mv_col_max * 4,
+ (ref_mv->as_mv.col >> 1) + ((1 << mvlong_width) - 1));
+ int minr = MAX(x->mv_row_min * 4,
+ (ref_mv->as_mv.row >> 1) - ((1 << mvlong_width) - 1));
+ int maxr = MIN(x->mv_row_max * 4,
+ (ref_mv->as_mv.row >> 1) + ((1 << mvlong_width) - 1));
int y_stride;
int offset;
@@ -254,8 +258,8 @@
offset = (bestmv->as_mv.row) * y_stride + bestmv->as_mv.col;
/* central mv */
- bestmv->as_mv.row <<= 3;
- bestmv->as_mv.col <<= 3;
+ bestmv->as_mv.row *= 8;
+ bestmv->as_mv.col *= 8;
/* calculate central point error */
besterr = vfp->vf(y, y_stride, z, b->src_stride, sse1);
@@ -337,8 +341,8 @@
tc = bc;
}
- bestmv->as_mv.row = br << 1;
- bestmv->as_mv.col = bc << 1;
+ bestmv->as_mv.row = br * 2;
+ bestmv->as_mv.col = bc * 2;
if ((abs(bestmv->as_mv.col - ref_mv->as_mv.col) > (MAX_FULL_PEL_VAL<<3)) ||
(abs(bestmv->as_mv.row - ref_mv->as_mv.row) > (MAX_FULL_PEL_VAL<<3)))
@@ -699,8 +703,8 @@
#endif
/* central mv */
- bestmv->as_mv.row <<= 3;
- bestmv->as_mv.col <<= 3;
+ bestmv->as_mv.row *= 8;
+ bestmv->as_mv.col *= 8;
startmv = *bestmv;
/* calculate central point error */
@@ -1315,8 +1319,8 @@
(*num00)++;
}
- this_mv.as_mv.row = best_mv->as_mv.row << 3;
- this_mv.as_mv.col = best_mv->as_mv.col << 3;
+ this_mv.as_mv.row = best_mv->as_mv.row * 8;
+ this_mv.as_mv.col = best_mv->as_mv.col * 8;
return fn_ptr->vf(what, what_stride, best_address, in_what_stride, &thissad)
+ mv_err_cost(&this_mv, center_mv, mvcost, x->errorperbit);
@@ -1709,8 +1713,8 @@
}
}
- this_mv.as_mv.row = best_mv->as_mv.row << 3;
- this_mv.as_mv.col = best_mv->as_mv.col << 3;
+ this_mv.as_mv.row = best_mv->as_mv.row * 8;
+ this_mv.as_mv.col = best_mv->as_mv.col * 8;
return fn_ptr->vf(what, what_stride, bestaddress, in_what_stride, &thissad)
+ mv_err_cost(&this_mv, center_mv, mvcost, x->errorperbit);
@@ -1905,8 +1909,8 @@
}
}
- this_mv.as_mv.row = ref_mv->as_mv.row << 3;
- this_mv.as_mv.col = ref_mv->as_mv.col << 3;
+ this_mv.as_mv.row = ref_mv->as_mv.row * 8;
+ this_mv.as_mv.col = ref_mv->as_mv.col * 8;
return fn_ptr->vf(what, what_stride, best_address, in_what_stride, &thissad)
+ mv_err_cost(&this_mv, center_mv, mvcost, x->errorperbit);
--- a/vp8/encoder/rdopt.c
+++ b/vp8/encoder/rdopt.c
@@ -935,7 +935,7 @@
assert(NEARESTMV <= m && m <= SPLITMV);
vp8_mv_ref_probs(p, near_mv_ref_ct);
return vp8_cost_token(vp8_mv_ref_tree, p,
- vp8_mv_ref_encoding_array - NEARESTMV + m);
+ vp8_mv_ref_encoding_array + (m - NEARESTMV));
}
void vp8_set_mbmode_and_mvs(MACROBLOCK *x, MB_PREDICTION_MODE mb, int_mv *mv)