ref: 806e59fcb18e95227272ae31f6bd0f59b384a174
dir: /module/gmp-openh264.cpp/
/*! * \copy * Copyright (c) 2009-2014, Cisco Systems * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. * * ************************************************************************************* */ #include <stdint.h> #include <time.h> #include <cstdio> #include <cstring> #include <iostream> #include <string> #include <memory> #include <assert.h> #include <limits.h> #include "gmp-platform.h" #include "gmp-video-host.h" #include "gmp-video-encode.h" #include "gmp-video-decode.h" #include "gmp-video-frame-i420.h" #include "gmp-video-frame-encoded.h" #include "codec_def.h" #include "codec_app_def.h" #include "codec_api.h" #include "task_utils.h" #if defined(_MSC_VER) #define PUBLIC_FUNC __declspec(dllexport) #else #define PUBLIC_FUNC #endif // This is for supporting older versions which do not have support for nullptr. #if defined(__clang__) # ifndef __has_extension # define __has_extension __has_feature # endif # if __has_extension(cxx_nullptr) # define GMP_HAVE_NULLPTR # endif #elif defined(__GNUC__) # if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L # if (__GNU_C__ >=4) # if (__GNU_C_MINOR__ >= 6) # define GMP_HAVE_NULLPTR # endif # endif # endif #elif defined(_MSC_VER) # define GMP_HAVE_NULLPTR #endif #if !defined (GMP_HAVE_NULLPTR) # define nullptr __null #endif static int g_log_level = 0; #define GMPLOG(l, x) do { \ if (l <= g_log_level) { \ const char *log_string = "unknown"; \ if ((l >= 0) && (l <= 3)) { \ log_string = kLogStrings[l]; \ } \ std::cerr << log_string << ": " << x << std::endl; \ } \ } while(0) #define GL_CRIT 0 #define GL_ERROR 1 #define GL_INFO 2 #define GL_DEBUG 3 const char* kLogStrings[] = { "Critical", "Error", "Info", "Debug" }; static GMPPlatformAPI* g_platform_api = nullptr; class OpenH264VideoEncoder; template <typename T> class SelfDestruct { public: SelfDestruct (T* t) : t_ (t) {} ~SelfDestruct() { if (t_) { t_->Destroy(); } } T* forget() { T* t = t_; t_ = nullptr; return t; } private: T* t_; }; class FrameStats { public: FrameStats (const char* type) : frames_in_ (0), frames_out_ (0), start_time_ (time (0)), last_time_ (start_time_), type_ (type) {} void FrameIn() { ++frames_in_; time_t now = time (0); if (now == last_time_) { return; } if (! (frames_in_ % 10)) { GMPLOG (GL_INFO, type_ << ": " << now << " Frame count " << frames_in_ << "(" << (frames_in_ / (now - start_time_)) << "/" << (30 / (now - last_time_)) << ")" << " -- " << frames_out_); last_time_ = now; } } void FrameOut() { ++frames_out_; } private: uint64_t frames_in_; uint64_t frames_out_; time_t start_time_; time_t last_time_; const std::string type_; }; class OpenH264VideoEncoder : public GMPVideoEncoder { public: OpenH264VideoEncoder (GMPVideoHost* hostAPI) : host_ (hostAPI), worker_thread_ (nullptr), encoder_ (nullptr), max_payload_size_ (0), callback_ (nullptr), stats_ ("Encoder") {} virtual ~OpenH264VideoEncoder() { worker_thread_->Join(); } virtual GMPVideoErr InitEncode (const GMPVideoCodec& codecSettings, GMPEncoderCallback* callback, int32_t numberOfCores, uint32_t maxPayloadSize) { GMPErr err = g_platform_api->createthread (&worker_thread_); if (err != GMPNoErr) { GMPLOG (GL_ERROR, "Couldn't create new thread"); return GMPVideoGenericErr; } int rv = WelsCreateSVCEncoder (&encoder_); if (rv) { return GMPVideoGenericErr; } SEncParamBase param; memset (¶m, 0, sizeof (param)); GMPLOG (GL_INFO, "Initializing encoder at " << codecSettings.mWidth << "x" << codecSettings.mHeight << "@" << static_cast<int> (codecSettings.mMaxFramerate) << "max payload size=" << maxPayloadSize); // Translate parameters. param.iUsageType = CAMERA_VIDEO_REAL_TIME; param.iPicWidth = codecSettings.mWidth; param.iPicHeight = codecSettings.mHeight; param.iTargetBitrate = codecSettings.mStartBitrate * 1000; GMPLOG (GL_INFO, "Initializing Bit Rate at: Start: " << codecSettings.mStartBitrate << "; Min: " << codecSettings.mMinBitrate << "; Max: " << codecSettings.mMaxBitrate); param.iRCMode = RC_BITRATE_MODE; // TODO([email protected]). Scary conversion from unsigned char to float below. param.fMaxFrameRate = static_cast<float> (codecSettings.mMaxFramerate); param.iInputCsp = videoFormatI420; rv = encoder_->Initialize (¶m); if (rv) { GMPLOG (GL_ERROR, "Couldn't initialize encoder"); return GMPVideoGenericErr; } max_payload_size_ = maxPayloadSize; callback_ = callback; GMPLOG (GL_INFO, "Initialized encoder"); return GMPVideoNoErr; } virtual GMPVideoErr Encode (GMPVideoi420Frame* inputImage, const GMPCodecSpecificInfo& codecSpecificInfo, const std::vector<GMPVideoFrameType>& frameTypes) { GMPLOG (GL_DEBUG, __FUNCTION__ << " size=" << inputImage->Width() << "x" << inputImage->Height()); stats_.FrameIn(); assert (!frameTypes.empty()); if (frameTypes.empty()) { GMPLOG (GL_ERROR, "No frame types provided"); return GMPVideoGenericErr; } worker_thread_->Post (WrapTask ( this, &OpenH264VideoEncoder::Encode_w, inputImage, (frameTypes)[0])); return GMPVideoGenericErr; } void Encode_w (GMPVideoi420Frame* inputImage, GMPVideoFrameType frame_type) { SFrameBSInfo encoded; if (frame_type == kGMPKeyFrame) { encoder_->ForceIntraFrame (true); if (!inputImage) return; } if (!inputImage) { GMPLOG (GL_ERROR, "no input image"); return; } SSourcePicture src; src.iColorFormat = videoFormatI420; src.iStride[0] = inputImage->Stride (kGMPYPlane); src.pData[0] = reinterpret_cast<unsigned char*> ( const_cast<uint8_t*> (inputImage->Buffer (kGMPYPlane))); src.iStride[1] = inputImage->Stride (kGMPUPlane); src.pData[1] = reinterpret_cast<unsigned char*> ( const_cast<uint8_t*> (inputImage->Buffer (kGMPUPlane))); src.iStride[2] = inputImage->Stride (kGMPVPlane); src.pData[2] = reinterpret_cast<unsigned char*> ( const_cast<uint8_t*> (inputImage->Buffer (kGMPVPlane))); src.iStride[3] = 0; src.pData[3] = nullptr; src.iPicWidth = inputImage->Width(); src.iPicHeight = inputImage->Height(); const SSourcePicture* pics = &src; int result = encoder_->EncodeFrame (pics, &encoded); if (result != cmResultSuccess) { GMPLOG (GL_ERROR, "Couldn't encode frame. Error = " << result); } // Translate int to enum GMPVideoFrameType encoded_type; bool has_frame = false; switch (encoded.eFrameType) { case videoFrameTypeIDR: encoded_type = kGMPKeyFrame; has_frame = true; break; case videoFrameTypeI: encoded_type = kGMPKeyFrame; has_frame = true; break; case videoFrameTypeP: encoded_type = kGMPDeltaFrame; has_frame = true; break; case videoFrameTypeSkip: // Can skip the call back since no actual bitstream will be generated break; case videoFrameTypeIPMixed://this type is currently not suppported case videoFrameTypeInvalid: GMPLOG (GL_ERROR, "Couldn't encode frame. Type = " << encoded.eFrameType); break; default: // The API is defined as returning a type. assert (false); break; } if (!has_frame) { // This frame must be destroyed on the main thread. g_platform_api->syncrunonmainthread (WrapTask ( this, &OpenH264VideoEncoder::DestroyInputFrame_m, inputImage)); return; } // Synchronously send this back to the main thread for delivery. g_platform_api->syncrunonmainthread (WrapTask ( this, &OpenH264VideoEncoder::Encode_m, inputImage, &encoded, encoded_type)); } void Encode_m (GMPVideoi420Frame* frame, SFrameBSInfo* encoded, GMPVideoFrameType frame_type) { // Now return the encoded data back to the parent. GMPVideoFrame* ftmp; GMPVideoErr err = host_->CreateFrame (kGMPEncodedVideoFrame, &ftmp); if (err != GMPVideoNoErr) { GMPLOG (GL_ERROR, "Error creating encoded frame"); return; } GMPVideoEncodedFrame* f = static_cast<GMPVideoEncodedFrame*> (ftmp); // Buffer up the data. uint32_t length = 0; std::vector<uint32_t> lengths; for (int i = 0; i < encoded->iLayerNum; ++i) { lengths.push_back (0); for (int j = 0; j < encoded->sLayerInfo[i].iNalCount; ++j) { lengths[i] += encoded->sLayerInfo[i].pNalLengthInByte[j]; length += encoded->sLayerInfo[i].pNalLengthInByte[j]; } } // TODO start-code to length conversion here when gmp // stops doing it for us before this call. err = f->CreateEmptyFrame (length); if (err != GMPVideoNoErr) { GMPLOG (GL_ERROR, "Error allocating frame data"); f->Destroy(); return; } // Copy the data. // Here we concatenate into one big buffer uint8_t* tmp = f->Buffer(); for (int i = 0; i < encoded->iLayerNum; ++i) { memcpy (tmp, encoded->sLayerInfo[i].pBsBuf, lengths[i]); tmp += lengths[i]; } f->SetEncodedWidth (frame->Width()); f->SetEncodedHeight (frame->Height()); f->SetTimeStamp (frame->Timestamp()); f->SetFrameType (frame_type); f->SetCompleteFrame (true); GMPLOG (GL_DEBUG, "Encoding complete. type= " << f->FrameType() << " length=" << f->Size() << " timestamp=" << f->TimeStamp()); // Destroy the frame. frame->Destroy(); // Return the encoded frame. GMPCodecSpecificInfo info; memset (&info, 0, sizeof (info)); // TODO need to set what goes in this info structure. callback_->Encoded (f, info); stats_.FrameOut(); } // These frames must be destroyed on the main thread. void DestroyInputFrame_m (GMPVideoi420Frame* frame) { frame->Destroy(); } virtual GMPVideoErr SetChannelParameters (uint32_t aPacketLoss, uint32_t aRTT) { return GMPVideoNoErr; } virtual GMPVideoErr SetRates (uint32_t aNewBitRate, uint32_t aFrameRate) { GMPLOG (GL_INFO, "[SetRates] Begin with: " << aNewBitRate << " , " << aFrameRate); //update bitrate if needed const int32_t newBitRate = aNewBitRate * 1000; //kbps->bps SBitrateInfo existEncoderBitRate; existEncoderBitRate.iLayer = SPATIAL_LAYER_ALL; int rv = encoder_->GetOption (ENCODER_OPTION_BITRATE, &existEncoderBitRate); if (rv != cmResultSuccess) { GMPLOG (GL_ERROR, "[SetRates] Error in Getting Bit Rate at Layer:" << rv << " ; Layer = " << existEncoderBitRate.iLayer << " ; BR = " << existEncoderBitRate.iBitrate); return GMPVideoGenericErr; } if (rv == cmResultSuccess && existEncoderBitRate.iBitrate != newBitRate) { SBitrateInfo newEncoderBitRate; newEncoderBitRate.iLayer = SPATIAL_LAYER_ALL; newEncoderBitRate.iBitrate = newBitRate; rv = encoder_->SetOption (ENCODER_OPTION_BITRATE, &newEncoderBitRate); if (rv == cmResultSuccess) { GMPLOG (GL_INFO, "[SetRates] Update Encoder Bandwidth (AllLayers): ReturnValue: " << rv << "BitRate(kbps): " << aNewBitRate); } else { GMPLOG (GL_ERROR, "[SetRates] Error in Setting Bit Rate at Layer:" << rv << " ; Layer = " << newEncoderBitRate.iLayer << " ; BR = " << newEncoderBitRate.iBitrate); return GMPVideoGenericErr; } } //update framerate if needed float existFrameRate = 0; rv = encoder_->GetOption (ENCODER_OPTION_FRAME_RATE, &existFrameRate); if (rv != cmResultSuccess) { GMPLOG (GL_ERROR, "[SetRates] Error in Getting Frame Rate:" << rv << " FrameRate: " << existFrameRate); return GMPVideoGenericErr; } if (rv == cmResultSuccess && (aFrameRate - existFrameRate > 0.001f || existFrameRate - aFrameRate > 0.001f)) { float newFrameRate = static_cast<float> (aFrameRate); rv = encoder_->SetOption (ENCODER_OPTION_FRAME_RATE, &newFrameRate); if (rv == cmResultSuccess) { GMPLOG (GL_INFO, "[SetRates] Update Encoder Frame Rate: ReturnValue: " << rv << " FrameRate: " << aFrameRate); } else { GMPLOG (GL_ERROR, "[SetRates] Error in Setting Frame Rate: ReturnValue: " << rv << " FrameRate: " << aFrameRate); return GMPVideoGenericErr; } } return GMPVideoNoErr; } virtual GMPVideoErr SetPeriodicKeyFrames (bool aEnable) { return GMPVideoNoErr; } virtual void EncodingComplete() { delete this; } private: GMPVideoHost* host_; GMPThread* worker_thread_; ISVCEncoder* encoder_; uint32_t max_payload_size_; GMPEncoderCallback* callback_; FrameStats stats_; }; class OpenH264VideoDecoder : public GMPVideoDecoder { public: OpenH264VideoDecoder (GMPVideoHost* hostAPI) : host_ (hostAPI), worker_thread_ (nullptr), callback_ (nullptr), decoder_ (nullptr), stats_ ("Decoder") {} virtual ~OpenH264VideoDecoder() { } virtual GMPVideoErr InitDecode (const GMPVideoCodec& codecSettings, GMPDecoderCallback* callback, int32_t coreCount) { GMPLOG (GL_INFO, "InitDecode"); GMPErr err = g_platform_api->createthread (&worker_thread_); if (err != GMPNoErr) { GMPLOG (GL_ERROR, "Couldn't create new thread"); return GMPVideoGenericErr; } if (WelsCreateDecoder (&decoder_)) { GMPLOG (GL_ERROR, "Couldn't create decoder"); return GMPVideoGenericErr; } if (!decoder_) { GMPLOG (GL_ERROR, "Couldn't create decoder"); return GMPVideoGenericErr; } SDecodingParam param; memset (¶m, 0, sizeof (param)); param.iOutputColorFormat = videoFormatI420; param.uiTargetDqLayer = UCHAR_MAX; // Default value param.uiEcActiveFlag = 1; // Error concealment on. param.sVideoProperty.eVideoBsType = VIDEO_BITSTREAM_DEFAULT; if (decoder_->Initialize (¶m)) { GMPLOG (GL_ERROR, "Couldn't initialize decoder"); return GMPVideoGenericErr; } callback_ = callback; return GMPVideoNoErr; } virtual GMPVideoErr Decode (GMPVideoEncodedFrame* inputFrame, bool missingFrames, const GMPCodecSpecificInfo& codecSpecificInfo, int64_t renderTimeMs = -1) { GMPLOG (GL_DEBUG, __FUNCTION__ << "Decoding frame size=" << inputFrame->Size() << " timestamp=" << inputFrame->TimeStamp()); stats_.FrameIn(); worker_thread_->Post (WrapTask ( this, &OpenH264VideoDecoder::Decode_w, inputFrame, missingFrames, renderTimeMs)); return GMPVideoNoErr; } virtual GMPVideoErr Reset() { return GMPVideoNoErr; } virtual GMPVideoErr Drain() { return GMPVideoNoErr; } virtual void DecodingComplete() { delete this; } private: void Decode_w (GMPVideoEncodedFrame* inputFrame, bool missingFrames, int64_t renderTimeMs = -1) { GMPLOG (GL_DEBUG, "Frame decode on worker thread length = " << inputFrame->Size()); SBufferInfo decoded; bool valid = false; memset (&decoded, 0, sizeof (decoded)); unsigned char* data[3] = {nullptr, nullptr, nullptr}; int rv = decoder_->DecodeFrame2 (inputFrame->Buffer(), inputFrame->Size(), data, &decoded); if (rv) { GMPLOG (GL_ERROR, "Decoding error rv=" << rv); } else { valid = true; } g_platform_api->syncrunonmainthread (WrapTask ( this, &OpenH264VideoDecoder::Decode_m, inputFrame, &decoded, data, renderTimeMs, valid)); } // Return the decoded data back to the parent. void Decode_m (GMPVideoEncodedFrame* inputFrame, SBufferInfo* decoded, unsigned char* data[3], int64_t renderTimeMs, bool valid) { // Attach a self-destructor so that this dies on return. SelfDestruct<GMPVideoEncodedFrame> ifd (inputFrame); // If we don't actually have data, just abort. if (!valid) { return; } if (decoded->iBufferStatus != 1) { return; } int width = decoded->UsrData.sSystemBuffer.iWidth; int height = decoded->UsrData.sSystemBuffer.iHeight; int ystride = decoded->UsrData.sSystemBuffer.iStride[0]; int uvstride = decoded->UsrData.sSystemBuffer.iStride[1]; GMPLOG (GL_DEBUG, "Video frame ready for display " << width << "x" << height << " timestamp=" << inputFrame->TimeStamp()); GMPVideoFrame* ftmp = nullptr; // Translate the image. GMPVideoErr err = host_->CreateFrame (kGMPI420VideoFrame, &ftmp); if (err != GMPVideoNoErr) { GMPLOG (GL_ERROR, "Couldn't allocate empty I420 frame"); return; } GMPVideoi420Frame* frame = static_cast<GMPVideoi420Frame*> (ftmp); err = frame->CreateFrame ( ystride * height, static_cast<uint8_t*> (data[0]), uvstride * height / 2, static_cast<uint8_t*> (data[1]), uvstride * height / 2, static_cast<uint8_t*> (data[2]), width, height, ystride, uvstride, uvstride); if (err != GMPVideoNoErr) { GMPLOG (GL_ERROR, "Couldn't make decoded frame"); return; } GMPLOG (GL_DEBUG, "Allocated size = " << frame->AllocatedSize (kGMPYPlane)); frame->SetTimestamp (inputFrame->TimeStamp()); frame->SetRenderTime_ms (renderTimeMs); callback_->Decoded (frame); stats_.FrameOut(); } GMPVideoHost* host_; GMPThread* worker_thread_; GMPDecoderCallback* callback_; ISVCDecoder* decoder_; FrameStats stats_; }; extern "C" { PUBLIC_FUNC GMPErr GMPInit (GMPPlatformAPI* aPlatformAPI) { g_platform_api = aPlatformAPI; return GMPNoErr; } PUBLIC_FUNC GMPErr GMPGetAPI (const char* aApiName, void* aHostAPI, void** aPluginApi) { if (!strcmp (aApiName, "decode-video")) { *aPluginApi = new OpenH264VideoDecoder (static_cast<GMPVideoHost*> (aHostAPI)); return GMPNoErr; } else if (!strcmp (aApiName, "encode-video")) { *aPluginApi = new OpenH264VideoEncoder (static_cast<GMPVideoHost*> (aHostAPI)); return GMPNoErr; } return GMPGenericErr; } PUBLIC_FUNC void GMPShutdown (void) { g_platform_api = nullptr; } } // extern "C"