Further modernizations of the audio handling code.

This commit is contained in:
Leon Styhre 2020-12-22 23:27:23 +01:00
parent 214a7861f9
commit bde34ddffd
6 changed files with 273 additions and 185 deletions

View file

@ -18,46 +18,7 @@ std::shared_ptr<AudioManager> AudioManager::sInstance;
std::vector<std::shared_ptr<Sound>> AudioManager::sSoundVector;
SDL_AudioDeviceID AudioManager::sAudioDevice = 0;
SDL_AudioSpec AudioManager::sAudioFormat;
void AudioManager::mixAudio(void* /*unused*/, Uint8* stream, int len)
{
bool stillPlaying = false;
// Initialize the buffer to "silence".
SDL_memset(stream, 0, len);
// Iterate through all our samples.
std::vector<std::shared_ptr<Sound>>::const_iterator soundIt = sSoundVector.cbegin();
while (soundIt != sSoundVector.cend()) {
std::shared_ptr<Sound> sound = *soundIt;
if (sound->isPlaying()) {
// Calculate rest length of current sample.
Uint32 restLength = (sound->getLength() - sound->getPosition());
if (restLength > static_cast<Uint32>(len)) {
// If stream length is smaller than sample length, clip it.
restLength = len;
}
// Mix sample into stream.
SDL_MixAudioFormat(stream, &(sound->getData()[sound->getPosition()]), AUDIO_S16,
restLength, Settings::getInstance()->getInt("SoundVolumeNavigation") * 1.28);
if (sound->getPosition() + restLength < sound->getLength()) {
//sample hasn't ended yet
stillPlaying = true;
}
// Set new sound position. if this is at or beyond the end of the sample,
// it will stop automatically.
sound->setPosition(sound->getPosition() + restLength);
}
// Advance to next sound.
++soundIt;
}
// We have processed all samples. check if some will still be playing.
if (!stillPlaying) {
// Nothing is playing, pause the audio until Sound::play() wakes us up.
SDL_PauseAudioDevice(sAudioDevice, 1);
}
}
SDL_AudioStream* AudioManager::sConversionStream;
AudioManager::AudioManager()
{
@ -71,7 +32,7 @@ AudioManager::~AudioManager()
std::shared_ptr<AudioManager>& AudioManager::getInstance()
{
// Check if an AudioManager instance is already created, if not create it.
// Check if an AudioManager instance is already created, if not, create it.
if (sInstance == nullptr)
sInstance = std::shared_ptr<AudioManager>(new AudioManager);
@ -100,9 +61,9 @@ void AudioManager::init()
// Set up format and callback. Play 16-bit stereo audio at 44.1Khz.
sRequestedAudioFormat.freq = 44100;
sRequestedAudioFormat.format = AUDIO_S16;
sRequestedAudioFormat.format = AUDIO_F32;
sRequestedAudioFormat.channels = 2;
sRequestedAudioFormat.samples = 4096;
sRequestedAudioFormat.samples = 1024;
sRequestedAudioFormat.callback = mixAudio;
sRequestedAudioFormat.userdata = nullptr;
@ -119,20 +80,29 @@ void AudioManager::init()
}
if (sAudioFormat.freq != sRequestedAudioFormat.freq) {
LOG(LogDebug) << "AudioManager::init(): Requested frequency 44100 could not be "
LOG(LogDebug) << "AudioManager::init(): Requested sampling rate " <<
std::to_string(sRequestedAudioFormat.freq) << " could not be "
"set, obtained " << std::to_string(sAudioFormat.freq) << ".";
}
if (sAudioFormat.format != sRequestedAudioFormat.format) {
LOG(LogDebug) << "AudioManager::init(): Requested format " << AUDIO_S16 << " could not be "
LOG(LogDebug) << "AudioManager::init(): Requested format " <<
std::to_string(sRequestedAudioFormat.format) << " could not be "
"set, obtained " << std::to_string(sAudioFormat.format) << ".";
}
if (sAudioFormat.channels != sRequestedAudioFormat.channels) {
LOG(LogDebug) << "AudioManager::init(): Requested channel count 2 could not be "
LOG(LogDebug) << "AudioManager::init(): Requested channel count " <<
std::to_string(sRequestedAudioFormat.channels) << " could not be "
"set, obtained " << std::to_string(sAudioFormat.channels) << ".";
}
#if defined(_WIN64)
// Beats me why the buffer size is not divided by the channel count on Windows.
if (sAudioFormat.samples != sRequestedAudioFormat.samples) {
LOG(LogDebug) << "AudioManager::init(): Requested sample buffer size 4096 could not be "
"set, obtained " << std::to_string(sAudioFormat.samples) << ".";
#else
if (sAudioFormat.samples != sRequestedAudioFormat.samples / sRequestedAudioFormat.channels) {
#endif
LOG(LogDebug) << "AudioManager::init(): Requested sample buffer size " <<
std::to_string(sRequestedAudioFormat.samples / sRequestedAudioFormat.channels) <<
" could not be set, obtained " << std::to_string(sAudioFormat.samples) << ".";
}
// Just in case someone changed the es_settings.cfg file manually to invalid values.
@ -144,19 +114,111 @@ void AudioManager::init()
Settings::getInstance()->setInt("SoundVolumeVideos", 100);
if (Settings::getInstance()->getInt("SoundVolumeVideos") < 0)
Settings::getInstance()->setInt("SoundVolumeVideos", 0);
// Used for streaming audio from videos.
sConversionStream = SDL_NewAudioStream(AUDIO_S16, 2, 44100, sAudioFormat.format,
sAudioFormat.channels, sAudioFormat.freq);
if (sConversionStream == nullptr) {
LOG(LogError) << "Failed to create audio conversion stream:";
LOG(LogError) << SDL_GetError();
}
}
void AudioManager::deinit()
{
// Stop all playback.
stop();
// Completely tear down SDL audio. else SDL hogs audio resources and
// emulators might fail to start...
SDL_FreeAudioStream(sConversionStream);
SDL_CloseAudio();
SDL_QuitSubSystem(SDL_INIT_AUDIO);
sInstance = nullptr;
}
void AudioManager::mixAudio(void* /*unused*/, Uint8* stream, int len)
{
// Process navigation sounds.
bool stillPlaying = false;
// Initialize the buffer to "silence".
SDL_memset(stream, 0, len);
// Iterate through all our samples.
std::vector<std::shared_ptr<Sound>>::const_iterator soundIt = sSoundVector.cbegin();
while (soundIt != sSoundVector.cend()) {
std::shared_ptr<Sound> sound = *soundIt;
if (sound->isPlaying()) {
// Calculate rest length of current sample.
Uint32 restLength = (sound->getLength() - sound->getPosition());
if (restLength > static_cast<Uint32>(len)) {
// If stream length is smaller than sample length, clip it.
restLength = len;
}
// Mix sample into stream.
SDL_MixAudioFormat(stream, &(sound->getData()[sound->getPosition()]),
sAudioFormat.format, restLength,
Settings::getInstance()->getInt("SoundVolumeNavigation") * 1.28);
if (sound->getPosition() + restLength < sound->getLength()) {
// Sample hasn't ended yet.
stillPlaying = true;
}
// Set new sound position. if this is at or beyond the end of the sample,
// it will stop automatically.
sound->setPosition(sound->getPosition() + restLength);
}
// Advance to next sound.
soundIt++;
}
// Process video stream audio.
int chunkLength = SDL_AudioStreamAvailable(sConversionStream);
if (chunkLength != 0) {
// Initialize the buffer to "silence".
SDL_memset(stream, 0, len);
Uint8* converted;
converted = new Uint8[chunkLength];
int chunkSegment;
// Break down the chunk into segments that do not exceed the buffer size.
while (chunkLength > 0) {
if (chunkLength > len) {
chunkSegment = len;
chunkLength -= len;
}
else {
chunkSegment = chunkLength;
chunkLength = 0;
}
int processedLength = SDL_AudioStreamGet(sConversionStream, converted, chunkSegment);
if (processedLength == -1) {
LOG(LogError) << "Failed to convert sound chunk:";
LOG(LogError) << SDL_GetError();
delete[] converted;
return;
}
// Currently disabled as it generates a lot of debug output.
// LOG(LogDebug) << "AudioManager::mixAudio(): chunkLength / chunkSegment "
// "/ processedLength: " << chunkLength << " / " << chunkSegment <<
// " / " << processedLength;
if (processedLength > 0)
SDL_MixAudioFormat(stream, converted, sAudioFormat.format, processedLength, 128);
}
delete[] converted;
SDL_PauseAudioDevice(sAudioDevice, 1);
}
// We have processed all samples. check if some will still be playing.
if (!stillPlaying) {
// Nothing is playing, pause the audio until Sound::play() wakes us up.
SDL_PauseAudioDevice(sAudioDevice, 1);
}
}
void AudioManager::registerSound(std::shared_ptr<Sound>& sound)
{
sSoundVector.push_back(sound);
@ -184,9 +246,20 @@ void AudioManager::stop()
{
// Stop playing all Sounds.
for (unsigned int i = 0; i < sSoundVector.size(); i++) {
if (sSoundVector.at(i)->isPlaying())
if (sSoundVector.at(i) && sSoundVector.at(i)->isPlaying())
sSoundVector[i]->stop();
}
// Pause audio.
SDL_PauseAudioDevice(sAudioDevice, 1);
}
void AudioManager::processStream(const void *samples, unsigned count)
{
if (SDL_AudioStreamPut(sConversionStream, samples, count * sizeof(Uint8)) == -1) {
LOG(LogError) << "Failed to put samples in the conversion stream:";
LOG(LogError) << SDL_GetError();
return;
}
SDL_PauseAudioDevice(sAudioDevice, 0);
}

View file

@ -17,16 +17,8 @@ class Sound;
class AudioManager
{
static SDL_AudioSpec sAudioFormat;
static std::vector<std::shared_ptr<Sound>> sSoundVector;
static std::shared_ptr<AudioManager> sInstance;
static void mixAudio(void* unused, Uint8* stream, int len);
AudioManager();
public:
static SDL_AudioDeviceID sAudioDevice;
virtual ~AudioManager();
static std::shared_ptr<AudioManager>& getInstance();
void init();
@ -38,7 +30,20 @@ public:
void play();
void stop();
virtual ~AudioManager();
// Used for streaming audio from videos.
void processStream(const void *samples, unsigned count);
static SDL_AudioDeviceID sAudioDevice;
static SDL_AudioSpec sAudioFormat;
private:
AudioManager();
static void mixAudio(void* unused, Uint8* stream, int len);
static void mixAudio2(void* unused, Uint8* stream, int len);
static SDL_AudioStream* sConversionStream;
static std::vector<std::shared_ptr<Sound>> sSoundVector;
static std::shared_ptr<AudioManager> sInstance;
};
#endif // ES_CORE_AUDIO_MANAGER_H

View file

@ -48,43 +48,6 @@ std::shared_ptr<Sound> Sound::getFromTheme(const std::shared_ptr<ThemeData>& the
return get(elem->get<std::string>("path"));
}
NavigationSounds* NavigationSounds::getInstance()
{
if (sInstance == nullptr)
sInstance = new NavigationSounds();
return sInstance;
}
void NavigationSounds::deinit()
{
if (sInstance)
delete sInstance;
sInstance = nullptr;
}
void NavigationSounds::loadThemeNavigationSounds(const std::shared_ptr<ThemeData>& theme)
{
navigationSounds.push_back(Sound::getFromTheme(theme, "all", "systembrowse"));
navigationSounds.push_back(Sound::getFromTheme(theme, "all", "quicksysselect"));
navigationSounds.push_back(Sound::getFromTheme(theme, "all", "select"));
navigationSounds.push_back(Sound::getFromTheme(theme, "all", "back"));
navigationSounds.push_back(Sound::getFromTheme(theme, "all", "scroll"));
navigationSounds.push_back(Sound::getFromTheme(theme, "all", "favorite"));
navigationSounds.push_back(Sound::getFromTheme(theme, "all", "launch"));
}
void NavigationSounds::playThemeNavigationSound(NavigationSoundsID soundID)
{
NavigationSounds::getInstance()->navigationSounds[soundID]->play();
}
bool NavigationSounds::isPlayingThemeNavigationSound(NavigationSoundsID soundID)
{
return NavigationSounds::getInstance()->navigationSounds[soundID]->isPlaying();
}
Sound::Sound(
const std::string& path)
: mSampleData(nullptr),
@ -123,31 +86,43 @@ void Sound::init()
LOG(LogError) << SDL_GetError();
return;
}
// Build conversion buffer.
SDL_AudioCVT cvt;
SDL_BuildAudioCVT(&cvt, wave.format, wave.channels, wave.freq, AUDIO_S16, 2, 44100);
// Copy data to conversion buffer.
cvt.len = dlen;
cvt.buf = new Uint8[cvt.len * cvt.len_mult];
memcpy(cvt.buf, data, dlen);
// Convert buffer to stereo, 16bit, 44.1kHz.
if (SDL_ConvertAudio(&cvt) < 0) {
LOG(LogError) << "Error converting sound \"" << mPath <<
"\" to 44.1kHz, 16bit, stereo format: " << SDL_GetError();
delete[] cvt.buf;
// Convert sound file to the format required by ES-DE.
SDL_AudioStream *conversionStream = SDL_NewAudioStream(wave.format, wave.channels, wave.freq,
AudioManager::sAudioFormat.format, AudioManager::sAudioFormat.channels,
AudioManager::sAudioFormat.freq);
if (conversionStream == nullptr) {
LOG(LogError) << "Failed to create sample conversion stream:";
LOG(LogError) << SDL_GetError();
return;
}
else {
// Worked. set up member data.
SDL_LockAudioDevice(AudioManager::sAudioDevice);
mSampleData = cvt.buf;
mSampleLength = cvt.len_cvt;
mSamplePos = 0;
mSampleFormat.channels = 2;
mSampleFormat.freq = 44100;
mSampleFormat.format = AUDIO_S16;
SDL_UnlockAudioDevice(AudioManager::sAudioDevice);
if (SDL_AudioStreamPut(conversionStream, data, dlen) == -1) {
LOG(LogError) << "Failed to put samples in the conversion stream:";
LOG(LogError) << SDL_GetError();
SDL_FreeAudioStream(conversionStream);
return;
}
// Free WAV data now.
int sampleLength = SDL_AudioStreamAvailable(conversionStream);
Uint8* converted = new Uint8[sampleLength];
if (SDL_AudioStreamGet(conversionStream, converted, sampleLength) == -1) {
LOG(LogError) << "Failed to convert sound file '" << mPath << "':";
LOG(LogError) << SDL_GetError();
SDL_FreeAudioStream(conversionStream);
delete[] converted;
return;
}
mSampleData = converted;
mSampleLength = sampleLength;
mSamplePos = 0;
mSampleFormat.freq = AudioManager::sAudioFormat.freq;
mSampleFormat.channels = AudioManager::sAudioFormat.channels;
mSampleFormat.format = AudioManager::sAudioFormat.format;
SDL_FreeAudioStream(conversionStream);
SDL_FreeWAV(data);
}
@ -226,9 +201,39 @@ Uint32 Sound::getLength() const
return mSampleLength;
}
Uint32 Sound::getLengthMS() const
NavigationSounds* NavigationSounds::getInstance()
{
// 44100 samples per second, 2 channels (stereo).
// I have no idea why the *0.75 is necessary, but otherwise it's inaccurate.
return static_cast<Uint32>((mSampleLength / 44100.0f / 2.0f * 0.75f) * 1000);
if (sInstance == nullptr)
sInstance = new NavigationSounds();
return sInstance;
}
void NavigationSounds::deinit()
{
if (sInstance)
delete sInstance;
sInstance = nullptr;
}
void NavigationSounds::loadThemeNavigationSounds(const std::shared_ptr<ThemeData>& theme)
{
navigationSounds.push_back(Sound::getFromTheme(theme, "all", "systembrowse"));
navigationSounds.push_back(Sound::getFromTheme(theme, "all", "quicksysselect"));
navigationSounds.push_back(Sound::getFromTheme(theme, "all", "select"));
navigationSounds.push_back(Sound::getFromTheme(theme, "all", "back"));
navigationSounds.push_back(Sound::getFromTheme(theme, "all", "scroll"));
navigationSounds.push_back(Sound::getFromTheme(theme, "all", "favorite"));
navigationSounds.push_back(Sound::getFromTheme(theme, "all", "launch"));
}
void NavigationSounds::playThemeNavigationSound(NavigationSoundsID soundID)
{
NavigationSounds::getInstance()->navigationSounds[soundID]->play();
}
bool NavigationSounds::isPlayingThemeNavigationSound(NavigationSoundsID soundID)
{
return NavigationSounds::getInstance()->navigationSounds[soundID]->isPlaying();
}

View file

@ -10,13 +10,10 @@
#ifndef ES_CORE_SOUND_H
#define ES_CORE_SOUND_H
#if defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__)
#include <sstream>
#endif
#include <SDL2/SDL_audio.h>
#include <map>
#include <memory>
#include <sstream>
#include <string>
#include <vector>
@ -24,18 +21,7 @@ class ThemeData;
class Sound
{
std::string mPath;
SDL_AudioSpec mSampleFormat;
Uint8* mSampleData;
Uint32 mSamplePos;
Uint32 mSampleLength;
bool playing;
public:
static std::shared_ptr<Sound> get(const std::string& path);
static std::shared_ptr<Sound> getFromTheme(const std::shared_ptr<ThemeData>& theme,
const std::string& view, const std::string& elem);
~Sound();
void init();
@ -51,11 +37,21 @@ public:
Uint32 getPosition() const;
void setPosition(Uint32 newPosition);
Uint32 getLength() const;
Uint32 getLengthMS() const;
static std::shared_ptr<Sound> get(const std::string& path);
static std::shared_ptr<Sound> getFromTheme(const std::shared_ptr<ThemeData>& theme,
const std::string& view, const std::string& elem);
private:
Sound(const std::string& path = "");
static std::map<std::string, std::shared_ptr<Sound>> sMap;
std::string mPath;
SDL_AudioSpec mSampleFormat;
Uint8* mSampleData;
Uint32 mSamplePos;
Uint32 mSampleLength;
bool playing;
};
enum NavigationSoundsID {
@ -83,6 +79,4 @@ private:
std::vector<std::shared_ptr<Sound>> navigationSounds;
};
extern NavigationSounds navigationsounds;
#endif // ES_CORE_SOUND_H

View file

@ -11,6 +11,7 @@
#include "renderers/Renderer.h"
#include "resources/TextureResource.h"
#include "utils/StringUtil.h"
#include "AudioManager.h"
#include "Settings.h"
#include "Window.h"
@ -29,30 +30,6 @@
libvlc_instance_t* VideoVlcComponent::mVLC = nullptr;
// VLC prepares to render a video frame.
static void* lock(void* data, void** p_pixels)
{
struct VideoContext* c = reinterpret_cast<struct VideoContext*>(data);
SDL_LockMutex(c->mutex);
SDL_LockSurface(c->surface);
*p_pixels = c->surface->pixels;
return nullptr; // Picture identifier, not needed here.
}
// VLC just rendered a video frame.
static void unlock(void* data, void* /*id*/, void *const* /*p_pixels*/)
{
struct VideoContext* c = reinterpret_cast<struct VideoContext*>(data);
SDL_UnlockSurface(c->surface);
SDL_UnlockMutex(c->mutex);
}
// VLC wants to display a video frame.
static void display(void* /*data*/, void* /*id*/)
{
// Data to be displayed.
}
VideoVlcComponent::VideoVlcComponent(Window* window)
: VideoComponent(window), mMediaPlayer(nullptr), mContext({})
{
@ -71,6 +48,7 @@ VideoVlcComponent::~VideoVlcComponent()
void VideoVlcComponent::setResize(float width, float height)
{
// This resize function is used when stretching videos to full screen in the video screensaver.
mTargetSize = Vector2f(width, height);
mTargetIsMax = false;
mStaticImage.setResize(width, height);
@ -79,6 +57,8 @@ void VideoVlcComponent::setResize(float width, float height)
void VideoVlcComponent::setMaxSize(float width, float height)
{
// This resize function is used in most instances, such as non-stretched video screensaver
// and the gamelist videos.
mTargetSize = Vector2f(width, height);
mTargetIsMax = true;
mStaticImage.setMaxSize(width, height);
@ -136,11 +116,6 @@ void VideoVlcComponent::resize()
if (textureSize == Vector2f::Zero())
return;
// SVG rasterization is determined by height and rasterization is done in terms of pixels.
// If rounding is off enough in the rasterization step (for images with extreme aspect
// ratios), it can cause cutoff when the aspect ratio breaks.
// So we always make sure the resultant height is an integer to make sure cutoff doesn't
// happen, and scale width from that.
if (mTargetIsMax) {
mSize = textureSize;
@ -155,7 +130,6 @@ void VideoVlcComponent::resize()
mSize[1] *= resizeScale.y();
}
// For SVG rasterization, always calculate width from rounded height.
mSize[1] = Math::round(mSize[1]);
mSize[0] = (mSize[1] / textureSize.y()) * textureSize.x();
@ -335,6 +309,8 @@ void VideoVlcComponent::startVideo()
if (!parseResult) {
// Wait for a maximum of 1 second for the media parsing.
// This maximum time is quite excessive as this step should normally
// be completed in 15 - 30 ms or so.
for (int i = 0; i < 200; i++) {
if (libvlc_media_get_parsed_status(mMedia))
break;
@ -360,12 +336,55 @@ void VideoVlcComponent::startVideo()
// Setup the media player.
mMediaPlayer = libvlc_media_player_new_from_media(mMedia);
libvlc_media_player_play(mMediaPlayer);
libvlc_video_set_callbacks(mMediaPlayer, lock, unlock, display,
reinterpret_cast<void*>(&mContext));
// The code below enables the libVLC audio output to be processed inside ES-DE.
// Unfortunately this causes excessive stuttering for some reason that I still
// don't understand, so at the moment this code is disabled. A proper mixer
// such as SDL_mixer would be needed anyway to fully support this.
// auto audioFormatCallback = [](void **data, char *format,
// unsigned *rate, unsigned *channels) -> int {
// format = const_cast<char*>("S16N");
// *rate = 44100;
// *channels = 2;
// return 0;
// };
//
// libvlc_audio_set_format_callbacks(mMediaPlayer,
// audioFormatCallback, nullptr);
//
// auto audioPlayCallback = [](void* data, const void* samples,
// unsigned count, int64_t pts) {
// AudioManager::getInstance()->processStream(samples, count);
// };
//
// libvlc_audio_set_callbacks(mMediaPlayer, audioPlayCallback,
// nullptr, nullptr, nullptr, nullptr, this);
libvlc_video_set_format(mMediaPlayer, "RGBA", static_cast<int>(mVideoWidth),
static_cast<int>(mVideoHeight), static_cast<int>(mVideoWidth * 4));
// Lock video memory as a preparation for rendering a frame.
auto videoLockCallback = [](void* data, void** p_pixels) -> void* {
struct VideoContext* videoContext =
reinterpret_cast<struct VideoContext*>(data);
SDL_LockMutex(videoContext->mutex);
SDL_LockSurface(videoContext->surface);
*p_pixels = videoContext->surface->pixels;
return nullptr; // Picture identifier, not needed here.
};
// Unlock the video memory after rendering a frame.
auto videoUnlockCallback = [](void* data, void*, void *const*) {
struct VideoContext* videoContext =
reinterpret_cast<struct VideoContext*>(data);
SDL_UnlockSurface(videoContext->surface);
SDL_UnlockMutex(videoContext->mutex);
};
libvlc_video_set_callbacks(mMediaPlayer, videoLockCallback,
videoUnlockCallback, nullptr, reinterpret_cast<void*>(&mContext));
libvlc_media_player_play(mMediaPlayer);
if ((!Settings::getInstance()->getBool("GamelistVideoAudio") &&
!mScreensaverMode) ||
(!Settings::getInstance()->getBool("ScreensaverVideoAudio") &&

View file

@ -27,14 +27,6 @@ struct VideoContext {
class VideoVlcComponent : public VideoComponent
{
// Structure that groups together the configuration of the video component.
struct Configuration {
unsigned startDelay;
bool showSnapshotNoVideo;
bool showSnapshotDelay;
std::string defaultVideoPath;
};
public:
VideoVlcComponent(Window* window);
virtual ~VideoVlcComponent();
@ -73,7 +65,7 @@ private:
static void VlcMediaParseCallback(const libvlc_event_t *event, void *user_data) {};
private:
static VideoVlcComponent* sInstance;
static libvlc_instance_t* mVLC;
libvlc_media_t* mMedia;
libvlc_media_player_t* mMediaPlayer;