Xargon/ES-DE
1
0
Fork 0
mirror of https://github.com/RetroDECK/ES-DE.git synced 2024-11-21 21:55:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Added an experimental FFmpeg video player.

Browse source
This commit is contained in:
Leon Styhre 2021-05-09 22:52:26 +02:00
parent fbbb6aece1
commit ef8b008d28
5 changed files with 944 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

88
CMake/Packages/FindFFmpeg.cmake Normal file
View file

@ -0,0 +1,88 @@
# Copyright (c) 2014 Matt Coffin <mcoffin13@gmail.com>
#
# This software is provided 'as-is', without any express or implied
# warranty. In no event will the authors be held liable for any damages
# arising from the use of this software.
#
# Permission is granted to anyone to use this software for any purpose,
# including commercial applications, and to alter it and redistribute it
# freely, subject to the following restrictions:
#
# 1. The origin of this software must not be misrepresented; you must not
# claim that you wrote the original software. If you use this software
# in a product, an acknowledgment in the product documentation would be
# appreciated but is not required.
# 2. Altered source versions must be plainly marked as such, and must not be
# misrepresented as being the original software.
# 3. This notice may not be removed or altered from any source distribution.
# - Try to find FFmpeg
# Once done this will define
# FFmpeg_FOUND
# FFmpeg_INCLUDE_DIRS
# FFmpeg_LIBRARIES
# FFmpeg_INCLUDE_FILES
# Author: Matt Coffin <mcoffin13@gmail.com>
include(FindPackageHandleStandardArgs)
if (NOT FFmpeg_FIND_COMPONENTS)
set(FFmpeg_FIND_COMPONENTS avcodec avformat avutil swresample swscale)
endif(NOT FFmpeg_FIND_COMPONENTS)
# Generate component include files and requirements
foreach(comp ${FFmpeg_FIND_COMPONENTS})
if(FFmpeg_FIND_REQUIRED_${comp})
list(APPEND required "FFmpeg_${comp}_FOUND")
endif()
endforeach(comp)
if(CMAKE_SIZEOF_VOID_P EQUAL 8)
set(_lib_suffix 64)
else()
set(_lib_suffix 32)
endif()
set(FFMPEG_PATH_ARCH FFmpegPath${_lib_suffix})
# Find libraries
find_package(PkgConfig QUIET)
foreach(comp ${FFmpeg_FIND_COMPONENTS})
if(PKG_CONFIG_FOUND)
pkg_check_modules(_${comp} QUIET lib${comp})
endif()
find_path(FFmpeg_${comp}_INCLUDE_DIR
NAMES "lib${comp}/${comp}.h"
HINTS
${_${comp}_INCLUDE_DIRS}
ENV FFmpegPath
ENV ${FFMPEG_PATH_ARCH}
PATHS
/usr/include /usr/local/include /opt/local/include /sw/include
PATH_SUFFIXES ffmpeg libav
DOC "FFmpeg include directory")
find_library(FFmpeg_${comp}_LIBRARY
NAMES ${comp} ${comp}-ffmpeg ${_${comp}_LIBRARIES}
HINTS
${_${comp}_LIBRARY_DIRS}
"${FFmpeg_${comp}_INCLUDE_DIR}/../lib"
"${FFmpeg_${comp}_INCLUDE_DIR}/../lib${_lib_suffix}"
"${FFmpeg_${comp}_INCLUDE_DIR}/../libs${_lib_suffix}"
"${FFmpeg_${comp}_INCLUDE_DIR}/lib"
"${FFmpeg_${comp}_INCLUDE_DIR}/lib${_lib_suffix}"
PATHS
/usr/lib /usr/local/lib /opt/local/lib /sw/lib
PATH_SUFFIXES ${comp} lib${comp}
DOC "FFmpeg ${comp} library")
find_package_handle_standard_args(FFmpeg_${comp}
FOUND_VAR FFmpeg_${comp}_FOUND
REQUIRED_VARS FFmpeg_${comp}_LIBRARY FFmpeg_${comp}_INCLUDE_DIR)
if(${FFmpeg_${comp}_FOUND})
list(APPEND FFmpeg_INCLUDE_DIRS ${FFmpeg_${comp}_INCLUDE_DIR})
list(APPEND FFmpeg_LIBRARIES ${FFmpeg_${comp}_LIBRARY})
endif()
endforeach(comp)
# Run checks via find_package_handle_standard_args
find_package_handle_standard_args(FFmpeg
FOUND_VAR FFmpeg_FOUND
REQUIRED_VARS ${required} FFmpeg_INCLUDE_DIRS FFmpeg_LIBRARIES)

3
CMakeLists.txt
View file

@ -72,6 +72,7 @@ endif()
# Skip package dependency checks if we're on Windows.
if(NOT WIN32)
find_package(CURL REQUIRED)
find_package(FFmpeg REQUIRED)
find_package(FreeImage REQUIRED)
find_package(Freetype REQUIRED)
find_package(Pugixml REQUIRED)
@ -218,6 +219,7 @@ endif()
set(COMMON_INCLUDE_DIRS
${CURL_INCLUDE_DIR}
${FFMPEG_INCLUDE_DIRS}
${FreeImage_INCLUDE_DIRS}
${FREETYPE_INCLUDE_DIRS}
${PUGIXML_INCLUDE_DIRS}
@ -280,6 +282,7 @@ endif()
if(NOT WIN32)
set(COMMON_LIBRARIES
${CURL_LIBRARIES}
${FFmpeg_LIBRARIES}
${FreeImage_LIBRARIES}
${FREETYPE_LIBRARIES}
${PUGIXML_LIBRARIES}

2
es-core/CMakeLists.txt
View file

@ -47,6 +47,7 @@ set(CORE_HEADERS
${CMAKE_CURRENT_SOURCE_DIR}/src/components/TextEditComponent.h
${CMAKE_CURRENT_SOURCE_DIR}/src/components/TextListComponent.h
${CMAKE_CURRENT_SOURCE_DIR}/src/components/VideoComponent.h
${CMAKE_CURRENT_SOURCE_DIR}/src/components/VideoFFmpegComponent.h
${CMAKE_CURRENT_SOURCE_DIR}/src/components/VideoOmxComponent.h
${CMAKE_CURRENT_SOURCE_DIR}/src/components/VideoVlcComponent.h
@ -123,6 +124,7 @@ set(CORE_SOURCES
${CMAKE_CURRENT_SOURCE_DIR}/src/components/TextComponent.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/components/TextEditComponent.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/components/VideoComponent.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/components/VideoFFmpegComponent.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/components/VideoOmxComponent.cpp
${CMAKE_CURRENT_SOURCE_DIR}/src/components/VideoVlcComponent.cpp

733
es-core/src/components/VideoFFmpegComponent.cpp Normal file
View file

@ -0,0 +1,733 @@
// SPDX-License-Identifier: MIT
//
// EmulationStation Desktop Edition
// VideoFFmpegComponent.cpp
//
// Video player based on FFmpeg.
//
#include "components/VideoFFmpegComponent.h"
#include "resources/TextureResource.h"
#include "AudioManager.h"
#include "Settings.h"
#include "Window.h"
VideoFFmpegComponent::VideoFFmpegComponent(
Window* window)
: VideoComponent(window),
mFormatContext(nullptr),
mVideoStream(nullptr),
mAudioStream(nullptr),
mVideoCodec(nullptr),
mAudioCodec(nullptr),
mVideoCodecContext(nullptr),
mAudioCodecContext(nullptr),
mVideoTimeBase(0.0l),
mVideoMinQueueSize(0),
mAccumulatedTime(0),
mStartTimeAccumulation(false),
mDecodedFrame(false),
mEndOfVideo(false)
{
// Get an empty texture for rendering the video.
mTexture = TextureResource::get("");
}
VideoFFmpegComponent::~VideoFFmpegComponent()
{
stopVideo();
mTexture.reset();
}
void VideoFFmpegComponent::setResize(float width, float height)
{
// This resize function is used when stretching videos to full screen in the video screensaver.
mTargetSize = Vector2f(width, height);
mTargetIsMax = false;
mStaticImage.setResize(width, height);
resize();
}
void VideoFFmpegComponent::setMaxSize(float width, float height)
{
// This resize function is used in most instances, such as non-stretched video screensaver
// and the gamelist videos.
mTargetSize = Vector2f(width, height);
mTargetIsMax = true;
mStaticImage.setMaxSize(width, height);
resize();
}
void VideoFFmpegComponent::resize()
{
if (!mTexture)
return;
const Vector2f textureSize(static_cast<float>(mVideoWidth), static_cast<float>(mVideoHeight));
if (textureSize == Vector2f::Zero())
return;
if (mTargetIsMax) {
mSize = textureSize;
Vector2f resizeScale((mTargetSize.x() / mSize.x()), (mTargetSize.y() / mSize.y()));
if (resizeScale.x() < resizeScale.y()) {
mSize[0] *= resizeScale.x();
mSize[1] *= resizeScale.x();
}
else {
mSize[0] *= resizeScale.y();
mSize[1] *= resizeScale.y();
}
mSize[1] = std::round(mSize[1]);
mSize[0] = (mSize[1] / textureSize.y()) * textureSize.x();
}
else {
// If both components are set, we just stretch.
// If no components are set, we don't resize at all.
mSize = mTargetSize == Vector2f::Zero() ? textureSize : mTargetSize;
// If only one component is set, we resize in a way that maintains aspect ratio.
if (!mTargetSize.x() && mTargetSize.y()) {
mSize[1] = std::round(mTargetSize.y());
mSize[0] = (mSize.y() / textureSize.y()) * textureSize.x();
}
else if (mTargetSize.x() && !mTargetSize.y()) {
mSize[1] = std::round((mTargetSize.x() / textureSize.x()) * textureSize.y());
mSize[0] = (mSize.y() / textureSize.y()) * textureSize.x();
}
}
onSizeChanged();
}
void VideoFFmpegComponent::render(const Transform4x4f& parentTrans)
{
VideoComponent::render(parentTrans);
Transform4x4f trans = parentTrans * getTransform();
GuiComponent::renderChildren(trans);
if (mIsPlaying && mFormatContext) {
unsigned int color;
if (mFadeIn < 1) {
const unsigned int fadeIn = static_cast<int>(mFadeIn * 255.0f);
color = Renderer::convertRGBAToABGR((fadeIn << 24) |
(fadeIn << 16) | (fadeIn << 8) | 255);
}
else {
color = 0xFFFFFFFF;
}
Renderer::Vertex vertices[4];
Renderer::setMatrix(parentTrans);
// Render the black rectangle behind the video.
if (mVideoRectangleCoords.size() == 4) {
Renderer::drawRect(mVideoRectangleCoords[0], mVideoRectangleCoords[1],
mVideoRectangleCoords[2], mVideoRectangleCoords[3], 0x000000FF, 0x000000FF);
}
vertices[0] = { { 0.0f , 0.0f }, { 0.0f, 0.0f }, color };
vertices[1] = { { 0.0f , mSize.y() }, { 0.0f, 1.0f }, color };
vertices[2] = { { mSize.x(), 0.0f }, { 1.0f, 0.0f }, color };
vertices[3] = { { mSize.x(), mSize.y() }, { 1.0f, 1.0f }, color };
// Round vertices.
for (int i = 0; i < 4; i++)
vertices[i].pos.round();
if (!mDecodedFrame)
return;
mTexture->bind();
#if defined(USE_OPENGL_21)
// Render scanlines if this option is enabled. However, if this is the video
// screensaver, then skip this as screensaver scanline rendering is handled in
// SystemScreenSaver as a postprocessing step.
if (!mScreensaverMode && Settings::getInstance()->getBool("GamelistVideoScanlines"))
vertices[0].shaders = Renderer::SHADER_SCANLINES;
#endif
// Render it.
Renderer::setMatrix(trans);
Renderer::drawTriangleStrips(&vertices[0], 4, trans);
}
else {
VideoComponent::renderSnapshot(parentTrans);
}
}
void VideoFFmpegComponent::update(int deltaTime)
{
if (mPause || !mFormatContext)
return;
if (mIsActuallyPlaying && mStartTimeAccumulation) {
mAccumulatedTime += static_cast<double>(
std::chrono::duration_cast<std::chrono::nanoseconds>
(std::chrono::high_resolution_clock::now() -
mTimeReference).count()) / 1000000000.0l;
}
mTimeReference = std::chrono::high_resolution_clock::now();
readFrames();
if (!mEndOfVideo && mIsActuallyPlaying && mVideoFrameQueue.empty() && mAudioFrameQueue.empty())
mEndOfVideo = true;
processFrames();
}
void VideoFFmpegComponent::readFrames()
{
if (mVideoCodecContext && mFormatContext) {
if (mVideoFrameQueue.size() < mVideoMinQueueSize || (mAudioStreamIndex >= 0 &&
mAudioFrameQueue.size() < mAudioMinQueueSize)) {
while(av_read_frame(mFormatContext, mPacket) >= 0) {
if (mPacket->stream_index == mVideoStreamIndex) {
if (!avcodec_send_packet(mVideoCodecContext, mPacket) &&
!avcodec_receive_frame(mVideoCodecContext, mVideoFrame)) {
// We have a video frame that needs conversion to RGBA format.
int dst_linesize[4];
uint8_t* frameRGB[4];
// The pts value is the presentation time, i.e. the time stamp when
// the frame (picture) should be displayed.
double pts = 0.0l;
// This is needed to avoid a potential divide by zero.
if (mVideoFrame->pkt_duration)
pts = static_cast<double>(mVideoFrame->pts) * mVideoTimeBase /
static_cast<double>(mVideoFrame->pkt_duration);
else
pts = static_cast<double>(mVideoFrame->pts) * mVideoTimeBase;
// Conversion using libswscale. Bicubic interpolation gives a good
// balance between speed and image quality.
struct SwsContext* conversionContext =
sws_getContext(mVideoCodecContext->width,
mVideoCodecContext->height,
mVideoCodecContext->pix_fmt,
mVideoCodecContext->width,
mVideoCodecContext->height,
AV_PIX_FMT_RGBA,
SWS_BICUBIC,
nullptr,
nullptr,
nullptr);
av_image_alloc(
frameRGB,
dst_linesize,
mVideoFrame->width,
mVideoFrame->height,
AV_PIX_FMT_RGB32,
1);
sws_scale(
conversionContext,
const_cast<uint8_t const* const*>(mVideoFrame->data),
mVideoFrame->linesize,
0,
mVideoCodecContext->height,
frameRGB,
dst_linesize);
VideoFrame currFrame;
// Save the frame into the queue for later processing.
currFrame.width = mVideoFrame->width;
currFrame.height = mVideoFrame->height;
currFrame.frameRGBA.insert(currFrame.frameRGBA.begin(), &frameRGB[0][0],
&frameRGB[0][currFrame.width * currFrame.height * 4]);
currFrame.pts = pts;
mVideoFrameQueue.push(currFrame);
av_freep(&frameRGB[0]);
sws_freeContext(conversionContext);
av_packet_unref(mPacket);
break;
}
}
else if (mPacket->stream_index == mAudioStreamIndex) {
if (!avcodec_send_packet(mAudioCodecContext, mPacket) &&
!avcodec_receive_frame(mAudioCodecContext, mAudioFrame)) {
// We have a audio frame that needs to be converted using libswresample.
SwrContext* resampleContext = nullptr;
uint8_t** convertedData = nullptr;
int numConvertedSamples = 0;
int resampledDataSize = 0;
enum AVSampleFormat outSampleFormat = AV_SAMPLE_FMT_FLT;
int outSampleRate = mAudioCodecContext->sample_rate;
int64_t inChannelLayout = mAudioCodecContext->channel_layout;
int64_t outChannelLayout = AV_CH_LAYOUT_STEREO;
int outNumChannels = 0;
int outChannels = 2;
int inNumSamples = 0;
int outMaxNumSamples = 0;
int outLineSize = 0;
// The pts value is the presentation time, i.e. the time stamp when
// the audio should be played.
double timeBase = av_q2d(mAudioStream->time_base);
double pts = mAudioFrame->pts * av_q2d(mAudioStream->time_base);
// Audio resampler setup. We only perform channel rematrixing and
// format conversion here, the sample rate is left untouched.
// There is a sample rate conversion in AudioManager and we don't
// want to resample twice. And for some files there may not be any
// resampling needed at all if the format is the same as the output
// format for the application.
int outNumSamples = av_rescale_rnd(mAudioFrame->nb_samples,
outSampleRate, mAudioCodecContext->sample_rate, AV_ROUND_UP);
resampleContext = swr_alloc();
if (!resampleContext) {
LOG(LogError) << "VideoFFmpegComponent::readFrames() Couldn't "
"allocate audio resample context";
}
inChannelLayout = (mAudioCodecContext->channels ==
av_get_channel_layout_nb_channels(
mAudioCodecContext->channel_layout)) ?
mAudioCodecContext->channel_layout :
av_get_default_channel_layout(mAudioCodecContext->channels);
if (outChannels == 1)
outChannelLayout = AV_CH_LAYOUT_MONO;
else if (outChannels == 2)
outChannelLayout = AV_CH_LAYOUT_STEREO;
else
outChannelLayout = AV_CH_LAYOUT_SURROUND;
inNumSamples = mAudioFrame->nb_samples;
av_opt_set_int(resampleContext, "in_channel_layout", inChannelLayout, 0);
av_opt_set_int(resampleContext, "in_sample_rate",
mAudioCodecContext->sample_rate, 0);
av_opt_set_sample_fmt(resampleContext, "in_sample_fmt",
mAudioCodecContext->sample_fmt, 0);
av_opt_set_int(resampleContext, "out_channel_layout", outChannelLayout, 0);
av_opt_set_int(resampleContext, "out_sample_rate", outSampleRate, 0);
av_opt_set_sample_fmt(resampleContext, "out_sample_fmt",
outSampleFormat, 0);
if (swr_init(resampleContext) < 0) {
LOG(LogError) << "VideoFFmpegComponent::readFrames() Couldn't "
"initialize the resampling context";
}
outMaxNumSamples = outNumSamples = av_rescale_rnd(inNumSamples,
outSampleRate, mAudioCodecContext->sample_rate, AV_ROUND_UP);
outNumChannels = av_get_channel_layout_nb_channels(outChannelLayout);
av_samples_alloc_array_and_samples(
&convertedData,
&outLineSize,
outNumChannels,
outNumSamples,
outSampleFormat,
1);
outNumSamples = av_rescale_rnd(swr_get_delay(resampleContext,
mAudioCodecContext->sample_rate) + inNumSamples,
outSampleRate, mAudioCodecContext->sample_rate, AV_ROUND_UP);
if (outNumSamples > outMaxNumSamples) {
av_freep(&convertedData[0]);
av_samples_alloc(
convertedData,
&outLineSize,
outNumChannels,
outNumSamples,
outSampleFormat,
1);
outMaxNumSamples = outNumSamples;
}
// Perform the actual conversion.
if (resampleContext) {
numConvertedSamples = swr_convert(resampleContext, convertedData,
outNumSamples, const_cast<const uint8_t**>(mAudioFrame->data),
mAudioFrame->nb_samples);
if (numConvertedSamples < 0) {
LOG(LogError) << "VideoFFmpegComponent::readFrames() Audio "
"resampling failed";
}
resampledDataSize = av_samples_get_buffer_size(
&outLineSize,
outNumChannels,
numConvertedSamples,
outSampleFormat,
1);
if (resampledDataSize < 0) {
LOG(LogError) << "VideoFFmpegComponent::readFrames() Audio "
"resampling did not generated any output";
}
}
AudioFrame currFrame;
// Save the frame into the queue for later processing.
currFrame.resampledData.insert(currFrame.resampledData.begin(),
&convertedData[0][0], &convertedData[0][resampledDataSize]);
currFrame.resampledDataSize = resampledDataSize;
currFrame.pts = pts;
mAudioFrameQueue.push(currFrame);
if (convertedData) {
av_freep(&convertedData[0]);
av_freep(&convertedData);
}
if (resampleContext)
swr_free(&resampleContext);
av_packet_unref(mPacket);
continue;
}
}
}
}
}
}
void VideoFFmpegComponent::processFrames()
{
// Check if we should start counting the time (i.e. start playing the video).
// The audio stream controls when the playback and time counting starts, assuming
// there is an audio track.
if (!mAudioCodecContext || (mAudioCodecContext && !mAudioFrameQueue.empty())) {
if (!mStartTimeAccumulation) {
mTimeReference = std::chrono::high_resolution_clock::now();
mStartTimeAccumulation = true;
mIsActuallyPlaying = true;
}
}
// Process all available audio frames that have a pts value below mAccumulatedTime.
while (!mAudioFrameQueue.empty()) {
if (mAudioFrameQueue.front().pts < mAccumulatedTime) {
// Enable only when needed, as this generates a lot of debug output.
// LOG(LogDebug) << "Processing audio frame with PTS: " <<
// mAudioFrameQueue.front().pts;
// LOG(LogDebug) << "Total audio frames processed / audio frame queue size: " <<
// mAudioFrameCount << " / " << std::to_string(mAudioFrameQueue.size());
if ((Settings::getInstance()->getBool("GamelistVideoAudio") &&
!mScreensaverMode) || (Settings::getInstance()->
getBool("ScreensaverVideoAudio") && mScreensaverMode)) {
AudioManager::getInstance()->processStream(
&mAudioFrameQueue.front().resampledData.at(0),
mAudioFrameQueue.front().resampledDataSize);
}
mAudioFrameQueue.pop();
mAudioFrameCount++;
}
else {
break;
}
}
// Process all available video frames that have a pts value below mAccumulatedTime.
// But if more than one frame is processed here, it means that the computer can't keep
// up for some reason.
while (mIsActuallyPlaying && !mVideoFrameQueue.empty()) {
if (mVideoFrameQueue.front().pts < mAccumulatedTime) {
// Enable only when needed, as this generates a lot of debug output.
// LOG(LogDebug) << "Processing video frame with PTS: " <<
// mVideoFrameQueue.front().pts;
// LOG(LogDebug) << "Total video frames processed / video frame queue size: " <<
// mVideoFrameCount << " / " << std::to_string(mVideoFrameQueue.size());
// Build a texture for the video frame.
mTexture->initFromPixels(&mVideoFrameQueue.front().frameRGBA.at(0),
mVideoFrameQueue.front().width, mVideoFrameQueue.front().height);
mVideoFrameQueue.pop();
mVideoFrameCount++;
mDecodedFrame = true;
}
else {
break;
}
}
}
void VideoFFmpegComponent::calculateBlackRectangle()
{
// Calculate the position and size for the black rectangle that will be rendered behind
// videos. If the option to display pillarboxes (and letterboxes) is enabled, then this
// would extend to the entire md_video area (if above the threshold as defined below) or
// otherwise it will exactly match the video size. The reason to add a black rectangle
// behind videos in this second instance is that the scanline rendering will make the
// video partially transparent so this may avoid some unforseen issues with some themes.
if (mVideoAreaPos != 0 && mVideoAreaSize != 0) {
mVideoRectangleCoords.clear();
if (Settings::getInstance()->getBool("GamelistVideoPillarbox")) {
float rectHeight;
float rectWidth;
// Video is in landscape orientation.
if (mSize.x() > mSize.y()) {
// Checking the Y size should not normally be required as landscape format
// should mean the height can't be higher than the max size defined by the
// theme. But as the height in mSize is provided by FFmpeg in integer format
// and then scaled, there could be rounding errors that make the video height
// slightly higher than allowed. It's only a single pixel or a few pixels, but
// it's still visible for some videos.
if (mSize.y() < mVideoAreaSize.y() && mSize.y() / mVideoAreaSize.y() < 0.90)
rectHeight = mVideoAreaSize.y();
else
rectHeight = mSize.y();
// Don't add a black border that is too narrow, that's what the 0.85 constant
// takes care of.
if (mSize.x() < mVideoAreaSize.x() && mSize.x() / mVideoAreaSize.x() < 0.85)
rectWidth = mVideoAreaSize.x();
else
rectWidth = mSize.x();
}
// Video is in portrait orientation (or completely square).
else {
rectWidth = mVideoAreaSize.x();
rectHeight = mSize.y();
}
// Populate the rectangle coordinates to be used in render().
mVideoRectangleCoords.push_back(std::round(mVideoAreaPos.x() -
rectWidth * mOrigin.x()));
mVideoRectangleCoords.push_back(std::round(mVideoAreaPos.y() -
rectHeight * mOrigin.y()));
mVideoRectangleCoords.push_back(std::round(rectWidth));
mVideoRectangleCoords.push_back(std::round(rectHeight));
}
// If the option to display pillarboxes is disabled, then make the rectangle equivalent
// to the size of the video.
else {
mVideoRectangleCoords.push_back(std::round(mPosition.x() - mSize.x() * mOrigin.x()));
mVideoRectangleCoords.push_back(std::round(mPosition.y() - mSize.y() * mOrigin.y()));
mVideoRectangleCoords.push_back(std::round(mSize.x()));
mVideoRectangleCoords.push_back(std::round(mSize.y()));
}
}
}
void VideoFFmpegComponent::startVideo()
{
if (!mFormatContext) {
mVideoWidth = 0;
mVideoHeight = 0;
mAccumulatedTime = 0;
mStartTimeAccumulation = false;
mDecodedFrame = false;
mEndOfVideo = false;
mVideoFrameCount = 0;
mAudioFrameCount = 0;
// This is used for the audio and video synchronization.
mTimeReference = std::chrono::high_resolution_clock::now();
// Clear the video and audio frame queues.
std::queue<VideoFrame>().swap(mVideoFrameQueue);
std::queue<AudioFrame>().swap(mAudioFrameQueue);
#if defined(_WIN64)
mVideoPath = path(Utils::String::replace(mVideoPath, "/", "\\"));
#endif
std::string filePath = "file:" + mVideoPath;
// This will disable the FFmpeg logging, so comment this out if debug info is needed.
av_log_set_callback(nullptr);
// File operations and basic setup.
if (avformat_open_input(&mFormatContext, filePath.c_str(), nullptr, nullptr)) {
LOG(LogError) << "VideoFFmpegComponent::startVideo(): Couldn't open video file \"" <<
mVideoPath << "\"";
return;
}
if (avformat_find_stream_info(mFormatContext, nullptr)) {
LOG(LogError) << "VideoFFmpegComponent::startVideo(): Couldn't read stream information"
"from video file \"" << mVideoPath << "\"";
return;
}
mVideoStreamIndex = -1;
mAudioStreamIndex = -1;
// Video stream setup.
mVideoStreamIndex = av_find_best_stream(
mFormatContext, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0);
if (mVideoStreamIndex < 0) {
LOG(LogError) << "VideoFFmpegComponent::startVideo(): Couldn't retrieve video stream "
"for file \"" << mVideoPath << "\"";
return;
}
mVideoStream = mFormatContext->streams[mVideoStreamIndex];
mVideoWidth = mFormatContext->streams[mVideoStreamIndex]->codecpar->width;
mVideoHeight = mFormatContext->streams[mVideoStreamIndex]->codecpar->height;
mVideoCodec = avcodec_find_decoder(mVideoStream->codecpar->codec_id);
if (!mVideoCodec) {
LOG(LogError) << "VideoFFmpegComponent::startVideo(): Couldn't find a suitable video "
"codec for file \"" << mVideoPath << "\"";
return;
}
mVideoCodecContext = avcodec_alloc_context3(mVideoCodec);
if (!mVideoCodec) {
LOG(LogError) << "VideoFFmpegComponent::startVideo(): Couldn't allocate video "
"codec context for file \"" << mVideoPath << "\"";
return;
}
if (mVideoCodec->capabilities & AV_CODEC_CAP_TRUNCATED)
mVideoCodecContext->flags |= AV_CODEC_FLAG_TRUNCATED;
if (avcodec_parameters_to_context(mVideoCodecContext, mVideoStream->codecpar)) {
LOG(LogError) << "VideoFFmpegComponent::startVideo(): Couldn't fill the video "
"codec context parameters for file \"" << mVideoPath << "\"";
return;
}
if (avcodec_open2(mVideoCodecContext, mVideoCodec, nullptr)) {
LOG(LogError) << "VideoFFmpegComponent::startVideo(): Couldn't initialize the "
"video codec context for file \"" << mVideoPath << "\"";
return;
}
// Audio stream setup, optional as some videos may not have any audio tracks.
mAudioStreamIndex = av_find_best_stream(
mFormatContext, AVMEDIA_TYPE_AUDIO, -1, -1, nullptr, 0);
if (mAudioStreamIndex < 0) {
LOG(LogDebug) << "VideoFFmpegComponent::startVideo(): Couldn't retrieve audio stream "
"for file \"" << mVideoPath << "\"";
}
if (mAudioStreamIndex >= 0) {
mAudioStream = mFormatContext->streams[mAudioStreamIndex];
mAudioCodec = avcodec_find_decoder(mAudioStream->codecpar->codec_id);
if (!mAudioCodec) {
LOG(LogError) << "Couldn't find a suitable audio codec for file \"" <<
mVideoPath << "\"";
return;
}
mAudioCodecContext = avcodec_alloc_context3(mAudioCodec);
if (mAudioCodec->capabilities & AV_CODEC_CAP_TRUNCATED)
mAudioCodecContext->flags |= AV_CODEC_FLAG_TRUNCATED;
// Some formats want separate stream headers.
if (mAudioCodecContext->flags & AVFMT_GLOBALHEADER)
mAudioCodecContext->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
if (avcodec_parameters_to_context(mAudioCodecContext, mAudioStream->codecpar)) {
LOG(LogError) << "VideoFFmpegComponent::startVideo(): Couldn't fill the audio "
"codec context parameters for file \"" << mVideoPath << "\"";
return;
}
if (avcodec_open2(mAudioCodecContext, mAudioCodec, nullptr)) {
LOG(LogError) << "VideoFFmpegComponent::startVideo(): Couldn't initialize the "
"audio codec context for file \"" << mVideoPath << "\"";
return;
}
AudioManager::getInstance()->setupAudioStream(mAudioCodecContext->sample_rate);
}
mVideoTimeBase = 1.0l / av_q2d(mVideoStream->avg_frame_rate);
// Set some reasonable minimum queue sizes (buffers).
mVideoMinQueueSize = static_cast<int>(av_q2d(mVideoStream->avg_frame_rate) / 2.0l);
if (mAudioStreamIndex >=0)
mAudioMinQueueSize = mAudioStream->codecpar->channels * 15;
else
mAudioMinQueueSize = 30;
mPacket = av_packet_alloc();
mVideoFrame = av_frame_alloc();
mAudioFrame = av_frame_alloc();
// Resize the video surface, which is needed both for the gamelist view and for the
// video screeensaver.
resize();
// Calculate pillarbox/letterbox sizes.
calculateBlackRectangle();
mIsPlaying = true;
}
}
void VideoFFmpegComponent::stopVideo()
{
mIsPlaying = false;
mIsActuallyPlaying = false;
mStartDelayed = false;
mPause = false;
mEndOfVideo = false;
// Clear the video and audio frame queues.
std::queue<VideoFrame>().swap(mVideoFrameQueue);
std::queue<AudioFrame>().swap(mAudioFrameQueue);
if (mFormatContext) {
av_frame_free(&mVideoFrame);
av_frame_free(&mAudioFrame);
av_packet_unref(mPacket);
av_packet_free(&mPacket);
avcodec_free_context(&mVideoCodecContext);
avcodec_free_context(&mAudioCodecContext);
avformat_close_input(&mFormatContext);
avformat_free_context(mFormatContext);
mVideoCodecContext = nullptr;
mAudioCodecContext = nullptr;
mFormatContext = nullptr;
}
}
void VideoFFmpegComponent::pauseVideo()
{
}
void VideoFFmpegComponent::handleLooping()
{
if (mIsPlaying && mEndOfVideo) {
// If the screensaver video swap time is set to 0, it means we should
// skip to the next game when the video has finished playing.
if (mScreensaverMode &&
Settings::getInstance()->getInt("ScreensaverSwapVideoTimeout") == 0) {
mWindow->screensaverTriggerNextGame();
}
else {
stopVideo();
startVideo();
}
}
}

118
es-core/src/components/VideoFFmpegComponent.h Normal file
View file

@ -0,0 +1,118 @@
// SPDX-License-Identifier: MIT
//
// EmulationStation Desktop Edition
// VideoFFmpegComponent.h
//
// Video player based on FFmpeg.
//
#ifndef ES_CORE_COMPONENTS_VIDEO_FFMPEG_COMPONENT_H
#define ES_CORE_COMPONENTS_VIDEO_FFMPEG_COMPONENT_H
#define VIDEO_FRAME_QUEUE_SIZE 3
#include "VideoComponent.h"
extern "C"
{
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/opt.h>
#include <libavutil/imgutils.h>
#include <libswresample/swresample.h>
#include <libswscale/swscale.h>
}
#include <chrono>
#include <queue>
class VideoFFmpegComponent : public VideoComponent
{
public:
VideoFFmpegComponent(Window* window);
virtual ~VideoFFmpegComponent();
// Resize the video to fit this size. If one axis is zero, scale that axis to maintain
// aspect ratio. If both are non-zero, potentially break the aspect ratio. If both are
// zero, no resizing. This can be set before or after a video is loaded.
// setMaxSize() and setResize() are mutually exclusive.
void setResize(float width, float height) override;
// Resize the video to be as large as possible but fit within a box of this size.
// This can be set before or after a video is loaded.
// Never breaks the aspect ratio. setMaxSize() and setResize() are mutually exclusive.
void setMaxSize(float width, float height) override;
private:
// Calculates the correct mSize from our resizing information (set by setResize/setMaxSize).
// Used internally whenever the resizing parameters or texture change.
void resize();
void render(const Transform4x4f& parentTrans) override;
void update(int deltaTime) override;
void readFrames();
void processFrames();
void calculateBlackRectangle();
// Start the video immediately.
virtual void startVideo() override;
// Stop the video.
virtual void stopVideo() override;
// Pause the video when a game has been launched.
virtual void pauseVideo() override;
// Handle looping the video. Must be called periodically.
virtual void handleLooping() override;
std::shared_ptr<TextureResource> mTexture;
std::vector<float> mVideoRectangleCoords;
AVFormatContext* mFormatContext;
AVStream* mVideoStream;
AVStream* mAudioStream;
AVCodec *mVideoCodec;
AVCodec *mAudioCodec;
AVCodecContext* mVideoCodecContext;
AVCodecContext* mAudioCodecContext;
int mVideoStreamIndex;
int mAudioStreamIndex;
AVPacket* mPacket;
AVFrame* mVideoFrame;
AVFrame* mAudioFrame;
struct VideoFrame {
std::vector<uint8_t> frameRGBA;
int width;
int height;
double pts;
};
struct AudioFrame {
std::vector<uint8_t> resampledData;
int resampledDataSize;
double pts;
};
std::queue<VideoFrame> mVideoFrameQueue;
std::queue<AudioFrame> mAudioFrameQueue;
int mVideoMinQueueSize;
int mAudioMinQueueSize;
double mVideoTimeBase;
// Used for audio and video synchronization.
std::chrono::high_resolution_clock::time_point mTimeReference;
double mAccumulatedTime;
bool mStartTimeAccumulation;
bool mDecodedFrame;
bool mEndOfVideo;
// These are only used for debugging.
int mAudioFrameCount;
int mVideoFrameCount;
};
#endif // ES_CORE_COMPONENTS_VIDEO_FFMPEG_COMPONENT_H