Improved the audio streaming in AudioManager.

es-app/src/views/ViewController.cpp

@@ -23,6 +23,7 @@
 #include "views/gamelist/VideoGameListView.h"
 #include "views/SystemView.h"
 #include "views/UIModeController.h"
+#include "AudioManager.h"
 #include "FileFilterIndex.h"
 #include "InputManager.h"
 #include "Log.h"
@@ -319,6 +320,7 @@ void ViewController::restoreViewPosition()
 
 void ViewController::goToSystemView(SystemData* system, bool playTransition)
 {
+    AudioManager::getInstance()->clearStream();
     bool applicationStartup = false;
 
     if (mState.viewing == NOTHING)

es-core/src/AudioManager.cpp

@@ -115,13 +115,9 @@ void AudioManager::init()
     if (Settings::getInstance()->getInt("SoundVolumeVideos") < 0)
         Settings::getInstance()->setInt("SoundVolumeVideos", 0);
 
-    // Used for streaming audio from videos.
+    // Use the default sample rate initially, this will possibly be changed by the
-    sConversionStream = SDL_NewAudioStream(AUDIO_S16, 2, 44100, sAudioFormat.format,
+    // video player when a video is started.
-            sAudioFormat.channels, sAudioFormat.freq);
+    setupAudioStream(sRequestedAudioFormat.freq);
-    if (sConversionStream == nullptr) {
-        LOG(LogError) << "Failed to create audio conversion stream:";
-        LOG(LogError) << SDL_GetError();
-    }
 }
 
 void AudioManager::deinit()
@@ -136,6 +132,7 @@ void AudioManager::mixAudio(void* /*unused*/, Uint8* stream, int len)
 {
     // Process navigation sounds.
     bool stillPlaying = false;
+    bool playedSample = false;
 
     // Initialize the buffer to "silence".
     SDL_memset(stream, 0, len);
@@ -145,6 +142,7 @@ void AudioManager::mixAudio(void* /*unused*/, Uint8* stream, int len)
     while (soundIt != sSoundVector.cend()) {
         std::shared_ptr<Sound> sound = *soundIt;
         if (sound->isPlaying()) {
+            playedSample = true;
             // Calculate rest length of current sample.
             Uint32 restLength = (sound->getLength() - sound->getPosition());
             if (restLength > static_cast<Uint32>(len)) {
@@ -167,58 +165,49 @@ void AudioManager::mixAudio(void* /*unused*/, Uint8* stream, int len)
         soundIt++;
     }
 
+    if (playedSample) {
+        // We have processed all samples. check if some will continue playing.
+        if (!stillPlaying) {
+            // Nothing is playing, pause the audio until Sound::play() wakes us up.
+            SDL_PauseAudioDevice(sAudioDevice, 1);
+        }
+        return;
+    }
+
     // Process video stream audio.
-    // The calling function in VideoVlcComponent is currently disabled as the internal
+    // Currently only used by VideoFFmpegComponent as the audio streaming seems to
-    // handling of audio streaming from videos does not work correctly.
+    // be broken for libVLC.
     int chunkLength = SDL_AudioStreamAvailable(sConversionStream);
 
-    if (chunkLength != 0) {
+    if (chunkLength == 0)
-        // Initialize the buffer to "silence".
+        return;
-        SDL_memset(stream, 0, len);
 
-        Uint8* converted;
+    // Cap the chunk length to the buffer size.
-        converted = new Uint8[chunkLength];
+    if (chunkLength > len)
-        int chunkSegment;
+        chunkLength = len;
 
-        // Break down the chunk into segments that do not exceed the buffer size.
+    std::vector<Uint8> converted(chunkLength);
-        while (chunkLength > 0) {
-            if (chunkLength > len) {
-                chunkSegment = len;
-                chunkLength -= len;
-            }
-            else {
-                chunkSegment = chunkLength;
-                chunkLength = 0;
-            }
 
-            int processedLength = SDL_AudioStreamGet(sConversionStream, converted, chunkSegment);
+    int processedLength = SDL_AudioStreamGet(sConversionStream,
-            if (processedLength == -1) {
+            static_cast<void*>(&converted.at(0)), chunkLength);
-                LOG(LogError) << "Failed to convert sound chunk:";
-                LOG(LogError) << SDL_GetError();
-                delete[] converted;
-                return;
-            }
 
-            // Enable only when needed, as it generates a lot of debug output.
+    if (processedLength == -1) {
-//            LOG(LogDebug) << "AudioManager::mixAudio(): chunkLength / chunkSegment "
+        LOG(LogError) << "AudioManager::mixAudio(): Failed to convert sound chunk:";
-//                    "/ processedLength: " << chunkLength << " / " << chunkSegment <<
+        LOG(LogError) << SDL_GetError();
-//                    " / " << processedLength;
+        return;
-
-            if (processedLength > 0)
-                SDL_MixAudioFormat(stream, converted, sAudioFormat.format, processedLength,
-                        static_cast<int>(Settings::getInstance()->
-                        getInt("SoundVolumeVideos") * 1.28f));
-        }
-
-        delete[] converted;
-        SDL_PauseAudioDevice(sAudioDevice, 1);
     }
 
-    // We have processed all samples. check if some will still be playing.
+    // Enable only when needed, as this generates a lot of debug output.
-    if (!stillPlaying) {
+//    LOG(LogDebug) << "AudioManager::mixAudio(): chunkLength"
-        // Nothing is playing, pause the audio until Sound::play() wakes us up.
+//            "/ processedLength: " << chunkLength << " / " <<
+//            " / " << processedLength;
+
+    SDL_MixAudioFormat(stream, &converted.at(0), sAudioFormat.format, processedLength,
+            static_cast<int>(Settings::getInstance()->
+            getInt("SoundVolumeVideos") * 1.28f));
+
+    if (SDL_AudioStreamAvailable(sConversionStream) == 0)
         SDL_PauseAudioDevice(sAudioDevice, 1);
-    }
 }
 
 void AudioManager::registerSound(std::shared_ptr<Sound>& sound)
@@ -255,7 +244,20 @@ void AudioManager::stop()
     SDL_PauseAudioDevice(sAudioDevice, 1);
 }
 
-void AudioManager::processStream(const void *samples, unsigned count)
+void AudioManager::setupAudioStream(int sampleRate)
+{
+    SDL_FreeAudioStream(sConversionStream);
+
+    // Used for streaming audio from videos.
+    sConversionStream = SDL_NewAudioStream(AUDIO_F32, 2, sampleRate, sAudioFormat.format,
+            sAudioFormat.channels, sAudioFormat.freq);
+    if (sConversionStream == nullptr) {
+        LOG(LogError) << "Failed to create audio conversion stream:";
+        LOG(LogError) << SDL_GetError();
+    }
+}
+
+void AudioManager::processStream(const void* samples, unsigned count)
 {
     if (SDL_AudioStreamPut(sConversionStream, samples, count * sizeof(Uint8)) == -1) {
         LOG(LogError) << "Failed to put samples in the conversion stream:";
@@ -263,5 +265,11 @@ void AudioManager::processStream(const void *samples, unsigned count)
         return;
     }
 
-    SDL_PauseAudioDevice(sAudioDevice, 0);
+    if (count > 0)
+        SDL_PauseAudioDevice(sAudioDevice, 0);
+}
+
+void AudioManager::clearStream()
+{
+    SDL_AudioStreamClear(sConversionStream);
 }

es-core/src/AudioManager.h

@@ -31,7 +31,9 @@ public:
     void stop();
 
     // Used for streaming audio from videos.
-    void processStream(const void *samples, unsigned count);
+    void setupAudioStream(int sampleRate);
+    void processStream(const void* samples, unsigned count);
+    void clearStream();
 
     bool getHasAudioDevice() { return sHasAudioDevice; };
 

es-core/src/components/VideoVlcComponent.cpp

@@ -3,7 +3,7 @@
 //  EmulationStation Desktop Edition
 //  VideoVlcComponent.cpp
 //
-//  Video playing using libVLC.
+//  Video player based on libVLC.
 //
 
 #include "components/VideoVlcComponent.h"
@@ -380,12 +380,11 @@ void VideoVlcComponent::startVideo()
 
                     // The code below enables the libVLC audio output to be processed inside ES-DE.
                     // Unfortunately this causes excessive stuttering for some reason that I still
-                    // don't understand, so at the moment this code is disabled. A proper mixer
+                    // don't understand, so at the moment this code is disabled.
-                    // such as SDL_mixer would be needed anyway to fully support this.
 //                    auto audioFormatCallback = [](void **data, char *format,
 //                            unsigned *rate, unsigned *channels) -> int {
-//                        format = const_cast<char*>("S16N");
+//                        format = const_cast<char*>("F32L");
-//                        *rate = 44100;
+//                        *rate = 48000;
 //                        *channels = 2;
 //                        return 0;
 //                    };

es-core/src/components/VideoVlcComponent.h

@@ -3,7 +3,7 @@
 //  EmulationStation Desktop Edition
 //  VideoVlcComponent.h
 //
-//  Video playing using libVLC.
+//  Video player based on libVLC.
 //
 
 #ifndef ES_CORE_COMPONENTS_VIDEO_VLC_COMPONENT_H