ref: 715bab6e5554b1632c3c183782a9c9d01c3460cc
parent: a2517555007044381c8fdae9c85551d0da43ffdd
parent: 8db245b6a1754849ebd305f93dc011275b2d56ed
author: Marco Paniconi <[email protected]>
date: Fri Aug 15 04:51:08 EDT 2014
Merge "Add an adaptive denoising mode."
--- a/examples/vpx_temporal_svc_encoder.c
+++ b/examples/vpx_temporal_svc_encoder.c
@@ -37,7 +37,8 @@
kDenoiserOff,
kDenoiserOnYOnly,
kDenoiserOnYUV,
- kDenoiserOnYUVAggressive // Aggressive mode not implemented currently.
+ kDenoiserOnYUVAggressive,
+ kDenoiserOnAdaptive
};
static int mode_to_num_layers[12] = {1, 2, 2, 3, 3, 3, 3, 5, 2, 3, 3, 3};--- a/vp8/common/onyx.h
+++ b/vp8/common/onyx.h
@@ -108,7 +108,8 @@
* For temporal denoiser: noise_sensitivity = 0 means off,
* noise_sensitivity = 1 means temporal denoiser on for Y channel only,
* noise_sensitivity = 2 means temporal denoiser on for all channels.
- * noise_sensitivity >= 3 means aggressive denoising mode.
+ * noise_sensitivity = 3 means aggressive denoising mode.
+ * noise_sensitivity >= 4 means adaptive denoising mode.
* Temporal denoiser is enabled via the configuration option:
* CONFIG_TEMPORAL_DENOISING.
* For spatial denoiser: noise_sensitivity controls the amount of
--- a/vp8/encoder/denoising.c
+++ b/vp8/encoder/denoising.c
@@ -341,8 +341,10 @@
denoiser->denoiser_mode = kDenoiserOnYOnly;
} else if (mode == 2) {denoiser->denoiser_mode = kDenoiserOnYUV;
- } else {+ } else if (mode == 3) {denoiser->denoiser_mode = kDenoiserOnYUVAggressive;
+ } else {+ denoiser->denoiser_mode = kDenoiserOnAdaptive;
}
if (denoiser->denoiser_mode != kDenoiserOnYUVAggressive) {denoiser->denoise_pars.scale_sse_thresh = 1;
@@ -397,9 +399,23 @@
vpx_memset(denoiser->yv12_mc_running_avg.buffer_alloc, 0,
denoiser->yv12_mc_running_avg.frame_size);
+ if (vp8_yv12_alloc_frame_buffer(&denoiser->yv12_last_source, width,
+ height, VP8BORDERINPIXELS) < 0) {+ vp8_denoiser_free(denoiser);
+ return 1;
+ }
+ vpx_memset(denoiser->yv12_last_source.buffer_alloc, 0,
+ denoiser->yv12_last_source.frame_size);
+
denoiser->denoise_state = vpx_calloc((num_mb_rows * num_mb_cols), 1);
vpx_memset(denoiser->denoise_state, 0, (num_mb_rows * num_mb_cols));
vp8_denoiser_set_parameters(denoiser, mode);
+ denoiser->nmse_source_diff = 0;
+ denoiser->nmse_source_diff_count = 0;
+ // TODO(marpan): Adjust thresholds, including effect on resolution.
+ denoiser->threshold_aggressive_mode = 40;
+ if (width * height > 640 * 480)
+ denoiser->threshold_aggressive_mode = 180;
return 0;
}
--- a/vp8/encoder/denoising.h
+++ b/vp8/encoder/denoising.h
@@ -24,7 +24,7 @@
#define SUM_DIFF_THRESHOLD_UV (96) // (8 * 8 * 1.5)
#define SUM_DIFF_THRESHOLD_HIGH_UV (8 * 8 * 2)
-#define SUM_DIFF_FROM_AVG_THRESH_UV (8 * 8 * 4)
+#define SUM_DIFF_FROM_AVG_THRESH_UV (8 * 8 * 8)
#define MOTION_MAGNITUDE_THRESHOLD_UV (8*3)
enum vp8_denoiser_decision
@@ -43,7 +43,8 @@
kDenoiserOff,
kDenoiserOnYOnly,
kDenoiserOnYUV,
- kDenoiserOnYUVAggressive
+ kDenoiserOnYUVAggressive,
+ kDenoiserOnAdaptive
};
typedef struct {@@ -72,9 +73,14 @@
{YV12_BUFFER_CONFIG yv12_running_avg[MAX_REF_FRAMES];
YV12_BUFFER_CONFIG yv12_mc_running_avg;
+ // TODO(marpan): Should remove yv12_last_source and use vp8_lookahead_peak.
+ YV12_BUFFER_CONFIG yv12_last_source;
unsigned char* denoise_state;
int num_mb_cols;
int denoiser_mode;
+ int threshold_aggressive_mode;
+ int nmse_source_diff;
+ int nmse_source_diff_count;
denoise_params denoise_pars;
} VP8_DENOISER;
@@ -82,6 +88,8 @@
int num_mb_rows, int num_mb_cols, int mode);
void vp8_denoiser_free(VP8_DENOISER *denoiser);
+
+void vp8_denoiser_set_parameters(VP8_DENOISER *denoiser, int mode);
void vp8_denoiser_denoise_mb(VP8_DENOISER *denoiser,
MACROBLOCK *x,
--- a/vp8/encoder/onyx_if.c
+++ b/vp8/encoder/onyx_if.c
@@ -3283,6 +3283,8 @@
&cpi->denoiser.yv12_running_avg[LAST_FRAME]);
}
}
+ if (cpi->oxcf.noise_sensitivity == 4)
+ vp8_yv12_copy_frame(cpi->Source, &cpi->denoiser.yv12_last_source);
}
#endif
@@ -3289,6 +3291,111 @@
}
+static void process_denoiser_mode_change(VP8_COMP *cpi) {+ const VP8_COMMON *const cm = &cpi->common;
+ int i, j;
+ int total = 0;
+ int num_blocks = 0;
+ // Number of blocks skipped along row/column in computing the
+ // nmse (normalized mean square error) of source.
+ int skip = 2;
+ // Only select blocks for computing nmse that have been encoded
+ // as ZERO LAST min_consec_zero_last frames in a row.
+ int min_consec_zero_last = 10;
+ // Decision is tested for changing the denoising mode every
+ // num_mode_change times this function is called. Note that this
+ // function called every 8 frames, so (8 * num_mode_change) is number
+ // of frames where denoising mode change is tested for switch.
+ int num_mode_change = 15;
+ // Framerate factor, to compensate for larger mse at lower framerates.
+ // TODO(marpan): Adjust this factor,
+ int fac_framerate = cpi->output_framerate < 25.0f ? 80 : 100;
+ int tot_num_blocks = cm->mb_rows * cm->mb_cols;
+ int ystride = cpi->Source->y_stride;
+ unsigned char *src = cpi->Source->y_buffer;
+ unsigned char *dst = cpi->denoiser.yv12_last_source.y_buffer;
+ static const unsigned char const_source[16] = {+ 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128};
+
+ // Loop through the Y plane, every skip blocks along rows and columns,
+ // summing the normalized mean square error, only for blocks that have
+ // been encoded as ZEROMV LAST at least min_consec_zero_last least frames in
+ // a row and have small sum difference between current and previous frame.
+ // Normalization here is by the contrast of the current frame block.
+ for (i = 0; i < cm->Height; i += 16 * skip) {+ int block_index_row = (i >> 4) * cm->mb_cols;
+ for (j = 0; j < cm->Width; j += 16 * skip) {+ int index = block_index_row + (j >> 4);
+ if (cpi->consec_zero_last[index] >= min_consec_zero_last) {+ unsigned int sse;
+ const unsigned int mse = vp8_mse16x16(src + j,
+ ystride,
+ dst + j,
+ ystride,
+ &sse);
+ const unsigned int var = vp8_variance16x16(src + j,
+ ystride,
+ dst + j,
+ ystride,
+ &sse);
+ // Only consider this block as valid for noise measurement
+ // if the sum_diff average of the current and previous frame
+ // is small (to avoid effects from lighting change).
+ if ((mse - var) < 256) {+ const unsigned int act = vp8_variance16x16(src + j,
+ ystride,
+ const_source,
+ 0,
+ &sse);
+ if (act > 0)
+ total += mse / act;
+ num_blocks++;
+ }
+ }
+ }
+ src += 16 * skip * ystride;
+ dst += 16 * skip * ystride;
+ }
+ total = total * fac_framerate / 100;
+
+ // Only consider this frame as valid sample if we have computed nmse over
+ // at least ~1/16 blocks, and Total > 0 (Total == 0 can happen if the
+ // application inputs duplicate frames, or contrast is all zero).
+ if (total > 0 &&
+ (num_blocks > (tot_num_blocks >> 4))) {+ // Update the recursive mean square source_diff.
+ if (cpi->denoiser.nmse_source_diff_count == 0)
+ // First sample in new interval.
+ cpi->denoiser.nmse_source_diff = total;
+ else
+ // For subsequent samples, use average with weight ~1/4 for new sample.
+ cpi->denoiser.nmse_source_diff = (int)((total >> 2) +
+ 3 * (cpi->denoiser.nmse_source_diff >> 2));
+ cpi->denoiser.nmse_source_diff_count++;
+ }
+ // Check for changing the denoiser mode, when we have obtained #samples =
+ // num_mode_change.
+ if (cpi->denoiser.nmse_source_diff_count == num_mode_change) {+ // Check for going up: from normal to aggressive mode.
+ if ((cpi->denoiser.denoiser_mode = kDenoiserOnYUV) &&
+ (cpi->denoiser.nmse_source_diff >
+ cpi->denoiser.threshold_aggressive_mode)) {+ vp8_denoiser_set_parameters(&cpi->denoiser, kDenoiserOnYUVAggressive);
+ } else {+ // Check for going down: from aggressive to normal mode.
+ if ((cpi->denoiser.denoiser_mode = kDenoiserOnYUVAggressive) &&
+ (cpi->denoiser.nmse_source_diff <
+ cpi->denoiser.threshold_aggressive_mode)) {+ vp8_denoiser_set_parameters(&cpi->denoiser, kDenoiserOnYUV);
+ }
+ }
+ // Reset metric and counter for next interval.
+ cpi->denoiser.nmse_source_diff = 0;
+ cpi->denoiser.nmse_source_diff_count = 0;
+ }
+}
+
void vp8_loopfilter_frame(VP8_COMP *cpi, VP8_COMMON *cm)
{const FRAME_TYPE frame_type = cm->frame_type;
@@ -3445,6 +3552,12 @@
{/* Key frame from VFW/auto-keyframe/first frame */
cm->frame_type = KEY_FRAME;
+#if CONFIG_TEMPORAL_DENOISING
+ if (cpi->oxcf.noise_sensitivity == 4) {+ // For adaptive mode, reset denoiser to normal mode on key frame.
+ vp8_denoiser_set_parameters(&cpi->denoiser, kDenoiserOnYUV);
+ }
+#endif
}
#if CONFIG_MULTI_RES_ENCODING
@@ -4471,6 +4584,21 @@
cm->copy_buffer_to_arf = 0;
cm->frame_to_show = &cm->yv12_fb[cm->new_fb_idx];
+
+#if CONFIG_TEMPORAL_DENOISING
+ // For the adaptive denoising mode (noise_sensitivity == 4), sample the mse
+ // of source diff (between current and previous frame), and determine if we
+ // should switch the denoiser mode. Sampling refers to computing the mse for
+ // a sub-sample of the frame (i.e., skip x blocks along row/column), and
+ // only for blocks in that set that have used ZEROMV LAST, along with some
+ // constraint on the sum diff between blocks. This process is called every
+ // ~8 frames, to further reduce complexity.
+ if (cpi->oxcf.noise_sensitivity == 4 &&
+ cpi->frames_since_key % 8 == 0 &&
+ cm->frame_type != KEY_FRAME) {+ process_denoiser_mode_change(cpi);
+ }
+#endif
#if CONFIG_MULTITHREAD
if (cpi->b_multi_threaded)