From fbbb6aece13d62bca4e5816b22032818de991b79 Mon Sep 17 00:00:00 2001 From: Leon Styhre Date: Sun, 9 May 2021 22:47:46 +0200 Subject: [PATCH] Improved the audio streaming in AudioManager. --- es-app/src/views/ViewController.cpp | 2 + es-core/src/AudioManager.cpp | 110 ++++++++++--------- es-core/src/AudioManager.h | 4 +- es-core/src/components/VideoVlcComponent.cpp | 9 +- es-core/src/components/VideoVlcComponent.h | 2 +- 5 files changed, 69 insertions(+), 58 deletions(-) diff --git a/es-app/src/views/ViewController.cpp b/es-app/src/views/ViewController.cpp index 4ac76121d..b1028e445 100644 --- a/es-app/src/views/ViewController.cpp +++ b/es-app/src/views/ViewController.cpp @@ -23,6 +23,7 @@ #include "views/gamelist/VideoGameListView.h" #include "views/SystemView.h" #include "views/UIModeController.h" +#include "AudioManager.h" #include "FileFilterIndex.h" #include "InputManager.h" #include "Log.h" @@ -319,6 +320,7 @@ void ViewController::restoreViewPosition() void ViewController::goToSystemView(SystemData* system, bool playTransition) { + AudioManager::getInstance()->clearStream(); bool applicationStartup = false; if (mState.viewing == NOTHING) diff --git a/es-core/src/AudioManager.cpp b/es-core/src/AudioManager.cpp index 490b9d93a..58b57dd89 100644 --- a/es-core/src/AudioManager.cpp +++ b/es-core/src/AudioManager.cpp @@ -115,13 +115,9 @@ void AudioManager::init() if (Settings::getInstance()->getInt("SoundVolumeVideos") < 0) Settings::getInstance()->setInt("SoundVolumeVideos", 0); - // Used for streaming audio from videos. - sConversionStream = SDL_NewAudioStream(AUDIO_S16, 2, 44100, sAudioFormat.format, - sAudioFormat.channels, sAudioFormat.freq); - if (sConversionStream == nullptr) { - LOG(LogError) << "Failed to create audio conversion stream:"; - LOG(LogError) << SDL_GetError(); - } + // Use the default sample rate initially, this will possibly be changed by the + // video player when a video is started. + setupAudioStream(sRequestedAudioFormat.freq); } void AudioManager::deinit() @@ -136,6 +132,7 @@ void AudioManager::mixAudio(void* /*unused*/, Uint8* stream, int len) { // Process navigation sounds. bool stillPlaying = false; + bool playedSample = false; // Initialize the buffer to "silence". SDL_memset(stream, 0, len); @@ -145,6 +142,7 @@ void AudioManager::mixAudio(void* /*unused*/, Uint8* stream, int len) while (soundIt != sSoundVector.cend()) { std::shared_ptr sound = *soundIt; if (sound->isPlaying()) { + playedSample = true; // Calculate rest length of current sample. Uint32 restLength = (sound->getLength() - sound->getPosition()); if (restLength > static_cast(len)) { @@ -167,58 +165,49 @@ void AudioManager::mixAudio(void* /*unused*/, Uint8* stream, int len) soundIt++; } + if (playedSample) { + // We have processed all samples. check if some will continue playing. + if (!stillPlaying) { + // Nothing is playing, pause the audio until Sound::play() wakes us up. + SDL_PauseAudioDevice(sAudioDevice, 1); + } + return; + } + // Process video stream audio. - // The calling function in VideoVlcComponent is currently disabled as the internal - // handling of audio streaming from videos does not work correctly. + // Currently only used by VideoFFmpegComponent as the audio streaming seems to + // be broken for libVLC. int chunkLength = SDL_AudioStreamAvailable(sConversionStream); - if (chunkLength != 0) { - // Initialize the buffer to "silence". - SDL_memset(stream, 0, len); + if (chunkLength == 0) + return; - Uint8* converted; - converted = new Uint8[chunkLength]; - int chunkSegment; + // Cap the chunk length to the buffer size. + if (chunkLength > len) + chunkLength = len; - // Break down the chunk into segments that do not exceed the buffer size. - while (chunkLength > 0) { - if (chunkLength > len) { - chunkSegment = len; - chunkLength -= len; - } - else { - chunkSegment = chunkLength; - chunkLength = 0; - } + std::vector converted(chunkLength); - int processedLength = SDL_AudioStreamGet(sConversionStream, converted, chunkSegment); - if (processedLength == -1) { - LOG(LogError) << "Failed to convert sound chunk:"; - LOG(LogError) << SDL_GetError(); - delete[] converted; - return; - } + int processedLength = SDL_AudioStreamGet(sConversionStream, + static_cast(&converted.at(0)), chunkLength); - // Enable only when needed, as it generates a lot of debug output. -// LOG(LogDebug) << "AudioManager::mixAudio(): chunkLength / chunkSegment " -// "/ processedLength: " << chunkLength << " / " << chunkSegment << -// " / " << processedLength; - - if (processedLength > 0) - SDL_MixAudioFormat(stream, converted, sAudioFormat.format, processedLength, - static_cast(Settings::getInstance()-> - getInt("SoundVolumeVideos") * 1.28f)); - } - - delete[] converted; - SDL_PauseAudioDevice(sAudioDevice, 1); + if (processedLength == -1) { + LOG(LogError) << "AudioManager::mixAudio(): Failed to convert sound chunk:"; + LOG(LogError) << SDL_GetError(); + return; } - // We have processed all samples. check if some will still be playing. - if (!stillPlaying) { - // Nothing is playing, pause the audio until Sound::play() wakes us up. + // Enable only when needed, as this generates a lot of debug output. +// LOG(LogDebug) << "AudioManager::mixAudio(): chunkLength" +// "/ processedLength: " << chunkLength << " / " << +// " / " << processedLength; + + SDL_MixAudioFormat(stream, &converted.at(0), sAudioFormat.format, processedLength, + static_cast(Settings::getInstance()-> + getInt("SoundVolumeVideos") * 1.28f)); + + if (SDL_AudioStreamAvailable(sConversionStream) == 0) SDL_PauseAudioDevice(sAudioDevice, 1); - } } void AudioManager::registerSound(std::shared_ptr& sound) @@ -255,7 +244,20 @@ void AudioManager::stop() SDL_PauseAudioDevice(sAudioDevice, 1); } -void AudioManager::processStream(const void *samples, unsigned count) +void AudioManager::setupAudioStream(int sampleRate) +{ + SDL_FreeAudioStream(sConversionStream); + + // Used for streaming audio from videos. + sConversionStream = SDL_NewAudioStream(AUDIO_F32, 2, sampleRate, sAudioFormat.format, + sAudioFormat.channels, sAudioFormat.freq); + if (sConversionStream == nullptr) { + LOG(LogError) << "Failed to create audio conversion stream:"; + LOG(LogError) << SDL_GetError(); + } +} + +void AudioManager::processStream(const void* samples, unsigned count) { if (SDL_AudioStreamPut(sConversionStream, samples, count * sizeof(Uint8)) == -1) { LOG(LogError) << "Failed to put samples in the conversion stream:"; @@ -263,5 +265,11 @@ void AudioManager::processStream(const void *samples, unsigned count) return; } - SDL_PauseAudioDevice(sAudioDevice, 0); + if (count > 0) + SDL_PauseAudioDevice(sAudioDevice, 0); +} + +void AudioManager::clearStream() +{ + SDL_AudioStreamClear(sConversionStream); } diff --git a/es-core/src/AudioManager.h b/es-core/src/AudioManager.h index c1f919a9d..5a39dc04d 100644 --- a/es-core/src/AudioManager.h +++ b/es-core/src/AudioManager.h @@ -31,7 +31,9 @@ public: void stop(); // Used for streaming audio from videos. - void processStream(const void *samples, unsigned count); + void setupAudioStream(int sampleRate); + void processStream(const void* samples, unsigned count); + void clearStream(); bool getHasAudioDevice() { return sHasAudioDevice; }; diff --git a/es-core/src/components/VideoVlcComponent.cpp b/es-core/src/components/VideoVlcComponent.cpp index 3b81abd7d..8438bbbe7 100644 --- a/es-core/src/components/VideoVlcComponent.cpp +++ b/es-core/src/components/VideoVlcComponent.cpp @@ -3,7 +3,7 @@ // EmulationStation Desktop Edition // VideoVlcComponent.cpp // -// Video playing using libVLC. +// Video player based on libVLC. // #include "components/VideoVlcComponent.h" @@ -380,12 +380,11 @@ void VideoVlcComponent::startVideo() // The code below enables the libVLC audio output to be processed inside ES-DE. // Unfortunately this causes excessive stuttering for some reason that I still - // don't understand, so at the moment this code is disabled. A proper mixer - // such as SDL_mixer would be needed anyway to fully support this. + // don't understand, so at the moment this code is disabled. // auto audioFormatCallback = [](void **data, char *format, // unsigned *rate, unsigned *channels) -> int { -// format = const_cast("S16N"); -// *rate = 44100; +// format = const_cast("F32L"); +// *rate = 48000; // *channels = 2; // return 0; // }; diff --git a/es-core/src/components/VideoVlcComponent.h b/es-core/src/components/VideoVlcComponent.h index df5947810..c4b8f15cd 100644 --- a/es-core/src/components/VideoVlcComponent.h +++ b/es-core/src/components/VideoVlcComponent.h @@ -3,7 +3,7 @@ // EmulationStation Desktop Edition // VideoVlcComponent.h // -// Video playing using libVLC. +// Video player based on libVLC. // #ifndef ES_CORE_COMPONENTS_VIDEO_VLC_COMPONENT_H