From ef8b008d280d74125de72a9d4546ace0949d3510 Mon Sep 17 00:00:00 2001 From: Leon Styhre Date: Sun, 9 May 2021 22:52:26 +0200 Subject: [PATCH] Added an experimental FFmpeg video player. --- CMake/Packages/FindFFmpeg.cmake | 88 +++ CMakeLists.txt | 3 + es-core/CMakeLists.txt | 2 + .../src/components/VideoFFmpegComponent.cpp | 733 ++++++++++++++++++ es-core/src/components/VideoFFmpegComponent.h | 118 +++ 5 files changed, 944 insertions(+) create mode 100644 CMake/Packages/FindFFmpeg.cmake create mode 100644 es-core/src/components/VideoFFmpegComponent.cpp create mode 100644 es-core/src/components/VideoFFmpegComponent.h diff --git a/CMake/Packages/FindFFmpeg.cmake b/CMake/Packages/FindFFmpeg.cmake new file mode 100644 index 000000000..b97210742 --- /dev/null +++ b/CMake/Packages/FindFFmpeg.cmake @@ -0,0 +1,88 @@ +# Copyright (c) 2014 Matt Coffin +# +# This software is provided 'as-is', without any express or implied +# warranty. In no event will the authors be held liable for any damages +# arising from the use of this software. +# +# Permission is granted to anyone to use this software for any purpose, +# including commercial applications, and to alter it and redistribute it +# freely, subject to the following restrictions: +# +# 1. The origin of this software must not be misrepresented; you must not +# claim that you wrote the original software. If you use this software +# in a product, an acknowledgment in the product documentation would be +# appreciated but is not required. +# 2. Altered source versions must be plainly marked as such, and must not be +# misrepresented as being the original software. +# 3. This notice may not be removed or altered from any source distribution. + +# - Try to find FFmpeg +# Once done this will define +# FFmpeg_FOUND +# FFmpeg_INCLUDE_DIRS +# FFmpeg_LIBRARIES +# FFmpeg_INCLUDE_FILES +# Author: Matt Coffin + +include(FindPackageHandleStandardArgs) + +if (NOT FFmpeg_FIND_COMPONENTS) + set(FFmpeg_FIND_COMPONENTS avcodec avformat avutil swresample swscale) +endif(NOT FFmpeg_FIND_COMPONENTS) + +# Generate component include files and requirements +foreach(comp ${FFmpeg_FIND_COMPONENTS}) + if(FFmpeg_FIND_REQUIRED_${comp}) + list(APPEND required "FFmpeg_${comp}_FOUND") + endif() +endforeach(comp) + +if(CMAKE_SIZEOF_VOID_P EQUAL 8) + set(_lib_suffix 64) +else() + set(_lib_suffix 32) +endif() + +set(FFMPEG_PATH_ARCH FFmpegPath${_lib_suffix}) +# Find libraries +find_package(PkgConfig QUIET) +foreach(comp ${FFmpeg_FIND_COMPONENTS}) + if(PKG_CONFIG_FOUND) + pkg_check_modules(_${comp} QUIET lib${comp}) + endif() + find_path(FFmpeg_${comp}_INCLUDE_DIR + NAMES "lib${comp}/${comp}.h" + HINTS + ${_${comp}_INCLUDE_DIRS} + ENV FFmpegPath + ENV ${FFMPEG_PATH_ARCH} + PATHS + /usr/include /usr/local/include /opt/local/include /sw/include + PATH_SUFFIXES ffmpeg libav + DOC "FFmpeg include directory") + find_library(FFmpeg_${comp}_LIBRARY + NAMES ${comp} ${comp}-ffmpeg ${_${comp}_LIBRARIES} + HINTS + ${_${comp}_LIBRARY_DIRS} + "${FFmpeg_${comp}_INCLUDE_DIR}/../lib" + "${FFmpeg_${comp}_INCLUDE_DIR}/../lib${_lib_suffix}" + "${FFmpeg_${comp}_INCLUDE_DIR}/../libs${_lib_suffix}" + "${FFmpeg_${comp}_INCLUDE_DIR}/lib" + "${FFmpeg_${comp}_INCLUDE_DIR}/lib${_lib_suffix}" + PATHS + /usr/lib /usr/local/lib /opt/local/lib /sw/lib + PATH_SUFFIXES ${comp} lib${comp} + DOC "FFmpeg ${comp} library") + find_package_handle_standard_args(FFmpeg_${comp} + FOUND_VAR FFmpeg_${comp}_FOUND + REQUIRED_VARS FFmpeg_${comp}_LIBRARY FFmpeg_${comp}_INCLUDE_DIR) + if(${FFmpeg_${comp}_FOUND}) + list(APPEND FFmpeg_INCLUDE_DIRS ${FFmpeg_${comp}_INCLUDE_DIR}) + list(APPEND FFmpeg_LIBRARIES ${FFmpeg_${comp}_LIBRARY}) + endif() +endforeach(comp) + +# Run checks via find_package_handle_standard_args +find_package_handle_standard_args(FFmpeg + FOUND_VAR FFmpeg_FOUND + REQUIRED_VARS ${required} FFmpeg_INCLUDE_DIRS FFmpeg_LIBRARIES) \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 564e22808..98cc1fbf2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,6 +72,7 @@ endif() # Skip package dependency checks if we're on Windows. if(NOT WIN32) find_package(CURL REQUIRED) + find_package(FFmpeg REQUIRED) find_package(FreeImage REQUIRED) find_package(Freetype REQUIRED) find_package(Pugixml REQUIRED) @@ -218,6 +219,7 @@ endif() set(COMMON_INCLUDE_DIRS ${CURL_INCLUDE_DIR} + ${FFMPEG_INCLUDE_DIRS} ${FreeImage_INCLUDE_DIRS} ${FREETYPE_INCLUDE_DIRS} ${PUGIXML_INCLUDE_DIRS} @@ -280,6 +282,7 @@ endif() if(NOT WIN32) set(COMMON_LIBRARIES ${CURL_LIBRARIES} + ${FFmpeg_LIBRARIES} ${FreeImage_LIBRARIES} ${FREETYPE_LIBRARIES} ${PUGIXML_LIBRARIES} diff --git a/es-core/CMakeLists.txt b/es-core/CMakeLists.txt index e4625c27e..0f9f56719 100644 --- a/es-core/CMakeLists.txt +++ b/es-core/CMakeLists.txt @@ -47,6 +47,7 @@ set(CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/src/components/TextEditComponent.h ${CMAKE_CURRENT_SOURCE_DIR}/src/components/TextListComponent.h ${CMAKE_CURRENT_SOURCE_DIR}/src/components/VideoComponent.h + ${CMAKE_CURRENT_SOURCE_DIR}/src/components/VideoFFmpegComponent.h ${CMAKE_CURRENT_SOURCE_DIR}/src/components/VideoOmxComponent.h ${CMAKE_CURRENT_SOURCE_DIR}/src/components/VideoVlcComponent.h @@ -123,6 +124,7 @@ set(CORE_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/components/TextComponent.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/components/TextEditComponent.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/components/VideoComponent.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/components/VideoFFmpegComponent.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/components/VideoOmxComponent.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/components/VideoVlcComponent.cpp diff --git a/es-core/src/components/VideoFFmpegComponent.cpp b/es-core/src/components/VideoFFmpegComponent.cpp new file mode 100644 index 000000000..38d7c6809 --- /dev/null +++ b/es-core/src/components/VideoFFmpegComponent.cpp @@ -0,0 +1,733 @@ +// SPDX-License-Identifier: MIT +// +// EmulationStation Desktop Edition +// VideoFFmpegComponent.cpp +// +// Video player based on FFmpeg. +// + +#include "components/VideoFFmpegComponent.h" + +#include "resources/TextureResource.h" +#include "AudioManager.h" +#include "Settings.h" +#include "Window.h" + +VideoFFmpegComponent::VideoFFmpegComponent( + Window* window) + : VideoComponent(window), + mFormatContext(nullptr), + mVideoStream(nullptr), + mAudioStream(nullptr), + mVideoCodec(nullptr), + mAudioCodec(nullptr), + mVideoCodecContext(nullptr), + mAudioCodecContext(nullptr), + mVideoTimeBase(0.0l), + mVideoMinQueueSize(0), + mAccumulatedTime(0), + mStartTimeAccumulation(false), + mDecodedFrame(false), + mEndOfVideo(false) +{ + // Get an empty texture for rendering the video. + mTexture = TextureResource::get(""); +} + +VideoFFmpegComponent::~VideoFFmpegComponent() +{ + stopVideo(); + mTexture.reset(); +} + +void VideoFFmpegComponent::setResize(float width, float height) +{ + // This resize function is used when stretching videos to full screen in the video screensaver. + mTargetSize = Vector2f(width, height); + mTargetIsMax = false; + mStaticImage.setResize(width, height); + resize(); +} + +void VideoFFmpegComponent::setMaxSize(float width, float height) +{ + // This resize function is used in most instances, such as non-stretched video screensaver + // and the gamelist videos. + mTargetSize = Vector2f(width, height); + mTargetIsMax = true; + mStaticImage.setMaxSize(width, height); + resize(); +} + +void VideoFFmpegComponent::resize() +{ + if (!mTexture) + return; + + const Vector2f textureSize(static_cast(mVideoWidth), static_cast(mVideoHeight)); + + if (textureSize == Vector2f::Zero()) + return; + + if (mTargetIsMax) { + mSize = textureSize; + + Vector2f resizeScale((mTargetSize.x() / mSize.x()), (mTargetSize.y() / mSize.y())); + + if (resizeScale.x() < resizeScale.y()) { + mSize[0] *= resizeScale.x(); + mSize[1] *= resizeScale.x(); + } + else { + mSize[0] *= resizeScale.y(); + mSize[1] *= resizeScale.y(); + } + + mSize[1] = std::round(mSize[1]); + mSize[0] = (mSize[1] / textureSize.y()) * textureSize.x(); + + } + else { + // If both components are set, we just stretch. + // If no components are set, we don't resize at all. + mSize = mTargetSize == Vector2f::Zero() ? textureSize : mTargetSize; + + // If only one component is set, we resize in a way that maintains aspect ratio. + if (!mTargetSize.x() && mTargetSize.y()) { + mSize[1] = std::round(mTargetSize.y()); + mSize[0] = (mSize.y() / textureSize.y()) * textureSize.x(); + } + else if (mTargetSize.x() && !mTargetSize.y()) { + mSize[1] = std::round((mTargetSize.x() / textureSize.x()) * textureSize.y()); + mSize[0] = (mSize.y() / textureSize.y()) * textureSize.x(); + } + } + + onSizeChanged(); +} + +void VideoFFmpegComponent::render(const Transform4x4f& parentTrans) +{ + VideoComponent::render(parentTrans); + Transform4x4f trans = parentTrans * getTransform(); + GuiComponent::renderChildren(trans); + + if (mIsPlaying && mFormatContext) { + unsigned int color; + if (mFadeIn < 1) { + const unsigned int fadeIn = static_cast(mFadeIn * 255.0f); + color = Renderer::convertRGBAToABGR((fadeIn << 24) | + (fadeIn << 16) | (fadeIn << 8) | 255); + } + else { + color = 0xFFFFFFFF; + } + Renderer::Vertex vertices[4]; + Renderer::setMatrix(parentTrans); + + // Render the black rectangle behind the video. + if (mVideoRectangleCoords.size() == 4) { + Renderer::drawRect(mVideoRectangleCoords[0], mVideoRectangleCoords[1], + mVideoRectangleCoords[2], mVideoRectangleCoords[3], 0x000000FF, 0x000000FF); + } + + vertices[0] = { { 0.0f , 0.0f }, { 0.0f, 0.0f }, color }; + vertices[1] = { { 0.0f , mSize.y() }, { 0.0f, 1.0f }, color }; + vertices[2] = { { mSize.x(), 0.0f }, { 1.0f, 0.0f }, color }; + vertices[3] = { { mSize.x(), mSize.y() }, { 1.0f, 1.0f }, color }; + + // Round vertices. + for (int i = 0; i < 4; i++) + vertices[i].pos.round(); + + if (!mDecodedFrame) + return; + + mTexture->bind(); + + #if defined(USE_OPENGL_21) + // Render scanlines if this option is enabled. However, if this is the video + // screensaver, then skip this as screensaver scanline rendering is handled in + // SystemScreenSaver as a postprocessing step. + if (!mScreensaverMode && Settings::getInstance()->getBool("GamelistVideoScanlines")) + vertices[0].shaders = Renderer::SHADER_SCANLINES; + #endif + + // Render it. + Renderer::setMatrix(trans); + Renderer::drawTriangleStrips(&vertices[0], 4, trans); + } + else { + VideoComponent::renderSnapshot(parentTrans); + } +} + +void VideoFFmpegComponent::update(int deltaTime) +{ + if (mPause || !mFormatContext) + return; + + if (mIsActuallyPlaying && mStartTimeAccumulation) { + mAccumulatedTime += static_cast( + std::chrono::duration_cast + (std::chrono::high_resolution_clock::now() - + mTimeReference).count()) / 1000000000.0l; + } + + mTimeReference = std::chrono::high_resolution_clock::now(); + + readFrames(); + + if (!mEndOfVideo && mIsActuallyPlaying && mVideoFrameQueue.empty() && mAudioFrameQueue.empty()) + mEndOfVideo = true; + + processFrames(); +} + +void VideoFFmpegComponent::readFrames() +{ + if (mVideoCodecContext && mFormatContext) { + if (mVideoFrameQueue.size() < mVideoMinQueueSize || (mAudioStreamIndex >= 0 && + mAudioFrameQueue.size() < mAudioMinQueueSize)) { + while(av_read_frame(mFormatContext, mPacket) >= 0) { + if (mPacket->stream_index == mVideoStreamIndex) { + if (!avcodec_send_packet(mVideoCodecContext, mPacket) && + !avcodec_receive_frame(mVideoCodecContext, mVideoFrame)) { + + // We have a video frame that needs conversion to RGBA format. + int dst_linesize[4]; + uint8_t* frameRGB[4]; + + // The pts value is the presentation time, i.e. the time stamp when + // the frame (picture) should be displayed. + double pts = 0.0l; + // This is needed to avoid a potential divide by zero. + if (mVideoFrame->pkt_duration) + pts = static_cast(mVideoFrame->pts) * mVideoTimeBase / + static_cast(mVideoFrame->pkt_duration); + else + pts = static_cast(mVideoFrame->pts) * mVideoTimeBase; + + // Conversion using libswscale. Bicubic interpolation gives a good + // balance between speed and image quality. + struct SwsContext* conversionContext = + sws_getContext(mVideoCodecContext->width, + mVideoCodecContext->height, + mVideoCodecContext->pix_fmt, + mVideoCodecContext->width, + mVideoCodecContext->height, + AV_PIX_FMT_RGBA, + SWS_BICUBIC, + nullptr, + nullptr, + nullptr); + + av_image_alloc( + frameRGB, + dst_linesize, + mVideoFrame->width, + mVideoFrame->height, + AV_PIX_FMT_RGB32, + 1); + + sws_scale( + conversionContext, + const_cast(mVideoFrame->data), + mVideoFrame->linesize, + 0, + mVideoCodecContext->height, + frameRGB, + dst_linesize); + + VideoFrame currFrame; + + // Save the frame into the queue for later processing. + currFrame.width = mVideoFrame->width; + currFrame.height = mVideoFrame->height; + currFrame.frameRGBA.insert(currFrame.frameRGBA.begin(), &frameRGB[0][0], + &frameRGB[0][currFrame.width * currFrame.height * 4]); + currFrame.pts = pts; + + mVideoFrameQueue.push(currFrame); + + av_freep(&frameRGB[0]); + sws_freeContext(conversionContext); + av_packet_unref(mPacket); + break; + } + } + else if (mPacket->stream_index == mAudioStreamIndex) { + if (!avcodec_send_packet(mAudioCodecContext, mPacket) && + !avcodec_receive_frame(mAudioCodecContext, mAudioFrame)) { + + // We have a audio frame that needs to be converted using libswresample. + SwrContext* resampleContext = nullptr; + uint8_t** convertedData = nullptr; + int numConvertedSamples = 0; + int resampledDataSize = 0; + + enum AVSampleFormat outSampleFormat = AV_SAMPLE_FMT_FLT; + int outSampleRate = mAudioCodecContext->sample_rate; + + int64_t inChannelLayout = mAudioCodecContext->channel_layout; + int64_t outChannelLayout = AV_CH_LAYOUT_STEREO; + int outNumChannels = 0; + int outChannels = 2; + int inNumSamples = 0; + int outMaxNumSamples = 0; + int outLineSize = 0; + + // The pts value is the presentation time, i.e. the time stamp when + // the audio should be played. + double timeBase = av_q2d(mAudioStream->time_base); + double pts = mAudioFrame->pts * av_q2d(mAudioStream->time_base); + + // Audio resampler setup. We only perform channel rematrixing and + // format conversion here, the sample rate is left untouched. + // There is a sample rate conversion in AudioManager and we don't + // want to resample twice. And for some files there may not be any + // resampling needed at all if the format is the same as the output + // format for the application. + int outNumSamples = av_rescale_rnd(mAudioFrame->nb_samples, + outSampleRate, mAudioCodecContext->sample_rate, AV_ROUND_UP); + + resampleContext = swr_alloc(); + if (!resampleContext) { + LOG(LogError) << "VideoFFmpegComponent::readFrames() Couldn't " + "allocate audio resample context"; + } + + inChannelLayout = (mAudioCodecContext->channels == + av_get_channel_layout_nb_channels( + mAudioCodecContext->channel_layout)) ? + mAudioCodecContext->channel_layout : + av_get_default_channel_layout(mAudioCodecContext->channels); + + if (outChannels == 1) + outChannelLayout = AV_CH_LAYOUT_MONO; + else if (outChannels == 2) + outChannelLayout = AV_CH_LAYOUT_STEREO; + else + outChannelLayout = AV_CH_LAYOUT_SURROUND; + + inNumSamples = mAudioFrame->nb_samples; + + av_opt_set_int(resampleContext, "in_channel_layout", inChannelLayout, 0); + av_opt_set_int(resampleContext, "in_sample_rate", + mAudioCodecContext->sample_rate, 0); + av_opt_set_sample_fmt(resampleContext, "in_sample_fmt", + mAudioCodecContext->sample_fmt, 0); + + av_opt_set_int(resampleContext, "out_channel_layout", outChannelLayout, 0); + av_opt_set_int(resampleContext, "out_sample_rate", outSampleRate, 0); + av_opt_set_sample_fmt(resampleContext, "out_sample_fmt", + outSampleFormat, 0); + + if (swr_init(resampleContext) < 0) { + LOG(LogError) << "VideoFFmpegComponent::readFrames() Couldn't " + "initialize the resampling context"; + } + + outMaxNumSamples = outNumSamples = av_rescale_rnd(inNumSamples, + outSampleRate, mAudioCodecContext->sample_rate, AV_ROUND_UP); + + outNumChannels = av_get_channel_layout_nb_channels(outChannelLayout); + + av_samples_alloc_array_and_samples( + &convertedData, + &outLineSize, + outNumChannels, + outNumSamples, + outSampleFormat, + 1); + + outNumSamples = av_rescale_rnd(swr_get_delay(resampleContext, + mAudioCodecContext->sample_rate) + inNumSamples, + outSampleRate, mAudioCodecContext->sample_rate, AV_ROUND_UP); + + if (outNumSamples > outMaxNumSamples) { + av_freep(&convertedData[0]); + av_samples_alloc( + convertedData, + &outLineSize, + outNumChannels, + outNumSamples, + outSampleFormat, + 1); + outMaxNumSamples = outNumSamples; + } + + // Perform the actual conversion. + if (resampleContext) { + numConvertedSamples = swr_convert(resampleContext, convertedData, + outNumSamples, const_cast(mAudioFrame->data), + mAudioFrame->nb_samples); + if (numConvertedSamples < 0) { + LOG(LogError) << "VideoFFmpegComponent::readFrames() Audio " + "resampling failed"; + } + + resampledDataSize = av_samples_get_buffer_size( + &outLineSize, + outNumChannels, + numConvertedSamples, + outSampleFormat, + 1); + if (resampledDataSize < 0) { + LOG(LogError) << "VideoFFmpegComponent::readFrames() Audio " + "resampling did not generated any output"; + } + } + + AudioFrame currFrame; + + // Save the frame into the queue for later processing. + currFrame.resampledData.insert(currFrame.resampledData.begin(), + &convertedData[0][0], &convertedData[0][resampledDataSize]); + currFrame.resampledDataSize = resampledDataSize; + currFrame.pts = pts; + + mAudioFrameQueue.push(currFrame); + + if (convertedData) { + av_freep(&convertedData[0]); + av_freep(&convertedData); + } + + if (resampleContext) + swr_free(&resampleContext); + + av_packet_unref(mPacket); + continue; + } + } + } + } + } +} + +void VideoFFmpegComponent::processFrames() +{ + // Check if we should start counting the time (i.e. start playing the video). + // The audio stream controls when the playback and time counting starts, assuming + // there is an audio track. + if (!mAudioCodecContext || (mAudioCodecContext && !mAudioFrameQueue.empty())) { + if (!mStartTimeAccumulation) { + mTimeReference = std::chrono::high_resolution_clock::now(); + mStartTimeAccumulation = true; + mIsActuallyPlaying = true; + } + } + + // Process all available audio frames that have a pts value below mAccumulatedTime. + while (!mAudioFrameQueue.empty()) { + if (mAudioFrameQueue.front().pts < mAccumulatedTime) { + + // Enable only when needed, as this generates a lot of debug output. +// LOG(LogDebug) << "Processing audio frame with PTS: " << +// mAudioFrameQueue.front().pts; +// LOG(LogDebug) << "Total audio frames processed / audio frame queue size: " << +// mAudioFrameCount << " / " << std::to_string(mAudioFrameQueue.size()); + + if ((Settings::getInstance()->getBool("GamelistVideoAudio") && + !mScreensaverMode) || (Settings::getInstance()-> + getBool("ScreensaverVideoAudio") && mScreensaverMode)) { + AudioManager::getInstance()->processStream( + &mAudioFrameQueue.front().resampledData.at(0), + mAudioFrameQueue.front().resampledDataSize); + } + mAudioFrameQueue.pop(); + mAudioFrameCount++; + } + else { + break; + } + } + + // Process all available video frames that have a pts value below mAccumulatedTime. + // But if more than one frame is processed here, it means that the computer can't keep + // up for some reason. + while (mIsActuallyPlaying && !mVideoFrameQueue.empty()) { + if (mVideoFrameQueue.front().pts < mAccumulatedTime) { + + // Enable only when needed, as this generates a lot of debug output. +// LOG(LogDebug) << "Processing video frame with PTS: " << +// mVideoFrameQueue.front().pts; +// LOG(LogDebug) << "Total video frames processed / video frame queue size: " << +// mVideoFrameCount << " / " << std::to_string(mVideoFrameQueue.size()); + + // Build a texture for the video frame. + mTexture->initFromPixels(&mVideoFrameQueue.front().frameRGBA.at(0), + mVideoFrameQueue.front().width, mVideoFrameQueue.front().height); + mVideoFrameQueue.pop(); + mVideoFrameCount++; + mDecodedFrame = true; + } + else { + break; + } + } +} + +void VideoFFmpegComponent::calculateBlackRectangle() +{ + // Calculate the position and size for the black rectangle that will be rendered behind + // videos. If the option to display pillarboxes (and letterboxes) is enabled, then this + // would extend to the entire md_video area (if above the threshold as defined below) or + // otherwise it will exactly match the video size. The reason to add a black rectangle + // behind videos in this second instance is that the scanline rendering will make the + // video partially transparent so this may avoid some unforseen issues with some themes. + if (mVideoAreaPos != 0 && mVideoAreaSize != 0) { + mVideoRectangleCoords.clear(); + + if (Settings::getInstance()->getBool("GamelistVideoPillarbox")) { + float rectHeight; + float rectWidth; + // Video is in landscape orientation. + if (mSize.x() > mSize.y()) { + // Checking the Y size should not normally be required as landscape format + // should mean the height can't be higher than the max size defined by the + // theme. But as the height in mSize is provided by FFmpeg in integer format + // and then scaled, there could be rounding errors that make the video height + // slightly higher than allowed. It's only a single pixel or a few pixels, but + // it's still visible for some videos. + if (mSize.y() < mVideoAreaSize.y() && mSize.y() / mVideoAreaSize.y() < 0.90) + rectHeight = mVideoAreaSize.y(); + else + rectHeight = mSize.y(); + // Don't add a black border that is too narrow, that's what the 0.85 constant + // takes care of. + if (mSize.x() < mVideoAreaSize.x() && mSize.x() / mVideoAreaSize.x() < 0.85) + rectWidth = mVideoAreaSize.x(); + else + rectWidth = mSize.x(); + } + // Video is in portrait orientation (or completely square). + else { + rectWidth = mVideoAreaSize.x(); + rectHeight = mSize.y(); + } + // Populate the rectangle coordinates to be used in render(). + mVideoRectangleCoords.push_back(std::round(mVideoAreaPos.x() - + rectWidth * mOrigin.x())); + mVideoRectangleCoords.push_back(std::round(mVideoAreaPos.y() - + rectHeight * mOrigin.y())); + mVideoRectangleCoords.push_back(std::round(rectWidth)); + mVideoRectangleCoords.push_back(std::round(rectHeight)); + } + // If the option to display pillarboxes is disabled, then make the rectangle equivalent + // to the size of the video. + else { + mVideoRectangleCoords.push_back(std::round(mPosition.x() - mSize.x() * mOrigin.x())); + mVideoRectangleCoords.push_back(std::round(mPosition.y() - mSize.y() * mOrigin.y())); + mVideoRectangleCoords.push_back(std::round(mSize.x())); + mVideoRectangleCoords.push_back(std::round(mSize.y())); + } + } +} + +void VideoFFmpegComponent::startVideo() +{ + if (!mFormatContext) { + mVideoWidth = 0; + mVideoHeight = 0; + mAccumulatedTime = 0; + mStartTimeAccumulation = false; + mDecodedFrame = false; + mEndOfVideo = false; + mVideoFrameCount = 0; + mAudioFrameCount = 0; + + // This is used for the audio and video synchronization. + mTimeReference = std::chrono::high_resolution_clock::now(); + + // Clear the video and audio frame queues. + std::queue().swap(mVideoFrameQueue); + std::queue().swap(mAudioFrameQueue); + + #if defined(_WIN64) + mVideoPath = path(Utils::String::replace(mVideoPath, "/", "\\")); + #endif + + std::string filePath = "file:" + mVideoPath; + + // This will disable the FFmpeg logging, so comment this out if debug info is needed. + av_log_set_callback(nullptr); + + // File operations and basic setup. + + if (avformat_open_input(&mFormatContext, filePath.c_str(), nullptr, nullptr)) { + LOG(LogError) << "VideoFFmpegComponent::startVideo(): Couldn't open video file \"" << + mVideoPath << "\""; + return; + } + + if (avformat_find_stream_info(mFormatContext, nullptr)) { + LOG(LogError) << "VideoFFmpegComponent::startVideo(): Couldn't read stream information" + "from video file \"" << mVideoPath << "\""; + return; + } + + mVideoStreamIndex = -1; + mAudioStreamIndex = -1; + + // Video stream setup. + + mVideoStreamIndex = av_find_best_stream( + mFormatContext, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0); + + if (mVideoStreamIndex < 0) { + LOG(LogError) << "VideoFFmpegComponent::startVideo(): Couldn't retrieve video stream " + "for file \"" << mVideoPath << "\""; + return; + } + + mVideoStream = mFormatContext->streams[mVideoStreamIndex]; + mVideoWidth = mFormatContext->streams[mVideoStreamIndex]->codecpar->width; + mVideoHeight = mFormatContext->streams[mVideoStreamIndex]->codecpar->height; + + mVideoCodec = avcodec_find_decoder(mVideoStream->codecpar->codec_id); + + if (!mVideoCodec) { + LOG(LogError) << "VideoFFmpegComponent::startVideo(): Couldn't find a suitable video " + "codec for file \"" << mVideoPath << "\""; + return; + } + + mVideoCodecContext = avcodec_alloc_context3(mVideoCodec); + + if (!mVideoCodec) { + LOG(LogError) << "VideoFFmpegComponent::startVideo(): Couldn't allocate video " + "codec context for file \"" << mVideoPath << "\""; + return; + } + + if (mVideoCodec->capabilities & AV_CODEC_CAP_TRUNCATED) + mVideoCodecContext->flags |= AV_CODEC_FLAG_TRUNCATED; + + if (avcodec_parameters_to_context(mVideoCodecContext, mVideoStream->codecpar)) { + LOG(LogError) << "VideoFFmpegComponent::startVideo(): Couldn't fill the video " + "codec context parameters for file \"" << mVideoPath << "\""; + return; + } + + if (avcodec_open2(mVideoCodecContext, mVideoCodec, nullptr)) { + LOG(LogError) << "VideoFFmpegComponent::startVideo(): Couldn't initialize the " + "video codec context for file \"" << mVideoPath << "\""; + return; + } + + // Audio stream setup, optional as some videos may not have any audio tracks. + + mAudioStreamIndex = av_find_best_stream( + mFormatContext, AVMEDIA_TYPE_AUDIO, -1, -1, nullptr, 0); + + if (mAudioStreamIndex < 0) { + LOG(LogDebug) << "VideoFFmpegComponent::startVideo(): Couldn't retrieve audio stream " + "for file \"" << mVideoPath << "\""; + } + + if (mAudioStreamIndex >= 0) { + mAudioStream = mFormatContext->streams[mAudioStreamIndex]; + mAudioCodec = avcodec_find_decoder(mAudioStream->codecpar->codec_id); + + if (!mAudioCodec) { + LOG(LogError) << "Couldn't find a suitable audio codec for file \"" << + mVideoPath << "\""; + return; + } + + mAudioCodecContext = avcodec_alloc_context3(mAudioCodec); + + if (mAudioCodec->capabilities & AV_CODEC_CAP_TRUNCATED) + mAudioCodecContext->flags |= AV_CODEC_FLAG_TRUNCATED; + + // Some formats want separate stream headers. + if (mAudioCodecContext->flags & AVFMT_GLOBALHEADER) + mAudioCodecContext->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; + + if (avcodec_parameters_to_context(mAudioCodecContext, mAudioStream->codecpar)) { + LOG(LogError) << "VideoFFmpegComponent::startVideo(): Couldn't fill the audio " + "codec context parameters for file \"" << mVideoPath << "\""; + return; + } + + if (avcodec_open2(mAudioCodecContext, mAudioCodec, nullptr)) { + LOG(LogError) << "VideoFFmpegComponent::startVideo(): Couldn't initialize the " + "audio codec context for file \"" << mVideoPath << "\""; + return; + } + + AudioManager::getInstance()->setupAudioStream(mAudioCodecContext->sample_rate); + } + + mVideoTimeBase = 1.0l / av_q2d(mVideoStream->avg_frame_rate); + // Set some reasonable minimum queue sizes (buffers). + mVideoMinQueueSize = static_cast(av_q2d(mVideoStream->avg_frame_rate) / 2.0l); + if (mAudioStreamIndex >=0) + mAudioMinQueueSize = mAudioStream->codecpar->channels * 15; + else + mAudioMinQueueSize = 30; + + mPacket = av_packet_alloc(); + mVideoFrame = av_frame_alloc(); + mAudioFrame = av_frame_alloc(); + + // Resize the video surface, which is needed both for the gamelist view and for the + // video screeensaver. + resize(); + + // Calculate pillarbox/letterbox sizes. + calculateBlackRectangle(); + + mIsPlaying = true; + } +} + +void VideoFFmpegComponent::stopVideo() +{ + mIsPlaying = false; + mIsActuallyPlaying = false; + mStartDelayed = false; + mPause = false; + mEndOfVideo = false; + + // Clear the video and audio frame queues. + std::queue().swap(mVideoFrameQueue); + std::queue().swap(mAudioFrameQueue); + + if (mFormatContext) { + av_frame_free(&mVideoFrame); + av_frame_free(&mAudioFrame); + av_packet_unref(mPacket); + av_packet_free(&mPacket); + + avcodec_free_context(&mVideoCodecContext); + avcodec_free_context(&mAudioCodecContext); + avformat_close_input(&mFormatContext); + avformat_free_context(mFormatContext); + mVideoCodecContext = nullptr; + mAudioCodecContext = nullptr; + mFormatContext = nullptr; + } +} + +void VideoFFmpegComponent::pauseVideo() +{ +} + +void VideoFFmpegComponent::handleLooping() +{ + if (mIsPlaying && mEndOfVideo) { + // If the screensaver video swap time is set to 0, it means we should + // skip to the next game when the video has finished playing. + if (mScreensaverMode && + Settings::getInstance()->getInt("ScreensaverSwapVideoTimeout") == 0) { + mWindow->screensaverTriggerNextGame(); + } + else { + stopVideo(); + startVideo(); + } + } +} diff --git a/es-core/src/components/VideoFFmpegComponent.h b/es-core/src/components/VideoFFmpegComponent.h new file mode 100644 index 000000000..63fd53f1b --- /dev/null +++ b/es-core/src/components/VideoFFmpegComponent.h @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: MIT +// +// EmulationStation Desktop Edition +// VideoFFmpegComponent.h +// +// Video player based on FFmpeg. +// + +#ifndef ES_CORE_COMPONENTS_VIDEO_FFMPEG_COMPONENT_H +#define ES_CORE_COMPONENTS_VIDEO_FFMPEG_COMPONENT_H + +#define VIDEO_FRAME_QUEUE_SIZE 3 + +#include "VideoComponent.h" + +extern "C" +{ +#include +#include +#include +#include +#include +#include +} + +#include +#include + +class VideoFFmpegComponent : public VideoComponent +{ +public: + VideoFFmpegComponent(Window* window); + virtual ~VideoFFmpegComponent(); + + // Resize the video to fit this size. If one axis is zero, scale that axis to maintain + // aspect ratio. If both are non-zero, potentially break the aspect ratio. If both are + // zero, no resizing. This can be set before or after a video is loaded. + // setMaxSize() and setResize() are mutually exclusive. + void setResize(float width, float height) override; + + // Resize the video to be as large as possible but fit within a box of this size. + // This can be set before or after a video is loaded. + // Never breaks the aspect ratio. setMaxSize() and setResize() are mutually exclusive. + void setMaxSize(float width, float height) override; + +private: + // Calculates the correct mSize from our resizing information (set by setResize/setMaxSize). + // Used internally whenever the resizing parameters or texture change. + void resize(); + + void render(const Transform4x4f& parentTrans) override; + void update(int deltaTime) override; + + void readFrames(); + void processFrames(); + + void calculateBlackRectangle(); + + // Start the video immediately. + virtual void startVideo() override; + // Stop the video. + virtual void stopVideo() override; + // Pause the video when a game has been launched. + virtual void pauseVideo() override; + // Handle looping the video. Must be called periodically. + virtual void handleLooping() override; + + std::shared_ptr mTexture; + std::vector mVideoRectangleCoords; + + AVFormatContext* mFormatContext; + AVStream* mVideoStream; + AVStream* mAudioStream; + AVCodec *mVideoCodec; + AVCodec *mAudioCodec; + AVCodecContext* mVideoCodecContext; + AVCodecContext* mAudioCodecContext; + int mVideoStreamIndex; + int mAudioStreamIndex; + + AVPacket* mPacket; + AVFrame* mVideoFrame; + AVFrame* mAudioFrame; + + struct VideoFrame { + std::vector frameRGBA; + int width; + int height; + double pts; + }; + + struct AudioFrame { + std::vector resampledData; + int resampledDataSize; + double pts; + }; + + std::queue mVideoFrameQueue; + std::queue mAudioFrameQueue; + + int mVideoMinQueueSize; + int mAudioMinQueueSize; + double mVideoTimeBase; + + // Used for audio and video synchronization. + std::chrono::high_resolution_clock::time_point mTimeReference; + + double mAccumulatedTime; + bool mStartTimeAccumulation; + bool mDecodedFrame; + bool mEndOfVideo; + + // These are only used for debugging. + int mAudioFrameCount; + int mVideoFrameCount; +}; + +#endif // ES_CORE_COMPONENTS_VIDEO_FFMPEG_COMPONENT_H