Podcasts on Linux: added audio support

src/podcast/UBPodcastController.cpp

@@ -66,6 +66,7 @@
     #include "quicktime/UBAudioQueueRecorder.h"
 #elif defined(Q_OS_LINUX)
     #include "ffmpeg/UBFFmpegVideoEncoder.h"
+    #include "ffmpeg/UBMicrophoneInput.h"
 #endif
 
 #include "core/memcheck.h"
@@ -808,6 +809,8 @@ QStringList UBPodcastController::audioRecordingDevices()
     devices = UBWaveRecorder::waveInDevices();
 #elif defined(Q_OS_OSX)
     devices = UBAudioQueueRecorder::waveInDevices();
+#elif defined(Q_OS_LINUX)
+    devices = UBMicrophoneInput::availableDevicesNames();
 #endif
 
     return devices;

src/podcast/ffmpeg/UBFFmpegVideoEncoder.h

@@ -5,20 +5,23 @@ extern "C" {
     #include <libavcodec/avcodec.h>
     #include <libavformat/avformat.h>
     #include <libavformat/avio.h>
+    #include <libavutil/audio_fifo.h>
     #include <libavutil/avutil.h>
     #include <libavutil/imgutils.h>
     #include <libavutil/opt.h>
     #include <libavutil/mathematics.h>
+    #include <libavutil/time.h>
     #include <libswscale/swscale.h>
+    #include <libswresample/swresample.h>
 }
 
 #include <atomic>
-#include <stdio.h>
 
 #include <QtCore>
 #include <QImage>
 
 #include "podcast/UBAbstractVideoEncoder.h"
+#include "podcast/ffmpeg/UBMicrophoneInput.h"
 
 class UBFFmpegVideoEncoderWorker;
 class UBPodcastController;
@@ -45,7 +48,6 @@ public:
 
     void setRecordAudio(bool pRecordAudio) { mShouldRecordAudio = pRecordAudio; }
 
-
 signals:
 
     void encodingFinished(bool ok);
@@ -53,6 +55,7 @@ signals:
 private slots:
 
     void setLastErrorMessage(const QString& pMessage);
+    void onAudioAvailable(QByteArray data);
     void finishEncoding();
 
 private:
@@ -63,32 +66,42 @@ private:
         long timestamp; // unit: ms
     };
 
-    AVFrame* convertFrame(ImageFrame frame);
+    AVFrame* convertImageFrame(ImageFrame frame);
+    AVFrame* convertAudio(QByteArray data);
+    void processAudio(QByteArray& data);
     bool init();
 
-    // Queue for any pixmap that might be sent before the encoder is ready
-    QQueue<ImageFrame> mPendingFrames;
-
     QString mLastErrorMessage;
-    bool mShouldRecordAudio;
 
     QThread* mVideoEncoderThread;
     UBFFmpegVideoEncoderWorker* mVideoWorker;
 
     // Muxer
+    // ------------------------------------------
     AVFormatContext* mOutputFormatContext;
-    int mTimebase;
+    AVStream* mVideoStream;
+    AVStream* mAudioStream;
 
     // Video
-    AVStream* mVideoStream;
+    // ------------------------------------------
+    QQueue<ImageFrame> mPendingFrames;
     struct SwsContext * mSwsContext;
 
-    // Audio
-    AVStream* mAudioStream;
+    int mVideoTimebase;
 
+    // Audio
+    // ------------------------------------------
+    bool mShouldRecordAudio;
 
-    FILE * mFile;
+    UBMicrophoneInput * mAudioInput;
+    struct SwrContext * mSwrContext;
+    /// Queue for audio that has been rescaled/converted but not encoded yet
+    AVAudioFifo *mAudioOutBuffer;
 
+    /// Sample rate for encoded audio
+    int mAudioSampleRate;
+    /// Total audio frames sent to encoder
+    int mAudioFrameCount;
 };
 
 
@@ -105,6 +118,7 @@ public:
     bool isRunning() { return mIsRunning; }
 
     void queueFrame(AVFrame* frame);
+    void queueAudio(AVFrame *frame);
 
 public slots:
     void runEncoding();
@@ -117,19 +131,23 @@ signals:
 
 private:
     void writeLatestVideoFrame();
+    void writeLatestAudioFrame();
 
     UBFFmpegVideoEncoder* mController;
 
     // std::atomic is C++11. This won't work with msvc2010, so a
-    // newer compiler must be used if this is to be used on Windows
+    // newer compiler must be used if this class is to be used on Windows
     std::atomic<bool> mStopRequested;
     std::atomic<bool> mIsRunning;
 
-    QQueue<AVFrame*> mFrameQueue;
+    QQueue<AVFrame*> mImageQueue;
+    QQueue<AVFrame*> mAudioQueue;
+
     QMutex mFrameQueueMutex;
     QWaitCondition mWaitCondition;
 
-    AVPacket* mPacket;
+    AVPacket* mVideoPacket;
+    AVPacket* mAudioPacket;
 };
 
 #endif // UBFFMPEGVIDEOENCODER_H

src/podcast/ffmpeg/UBMicrophoneInput.cpp

+#include "UBMicrophoneInput.h"
+
+UBMicrophoneInput::UBMicrophoneInput()
+    : mAudioInput(NULL)
+    , mIODevice(NULL)
+    , mSeekPos(0)
+{
+}
+
+UBMicrophoneInput::~UBMicrophoneInput()
+{
+    if (mAudioInput)
+        delete mAudioInput;
+}
+
+int UBMicrophoneInput::channelCount()
+{
+    return mAudioFormat.channelCount();
+}
+
+int UBMicrophoneInput::sampleRate()
+{
+    return mAudioFormat.sampleRate();
+}
+
+/* Return the sample size in bits */
+int UBMicrophoneInput::sampleSize()
+{
+    return mAudioFormat.sampleSize();
+}
+
+/** Return the sample format in FFMpeg style (AVSampleFormat enum) */
+int UBMicrophoneInput::sampleFormat()
+{
+    enum AVSampleFormat {
+        AV_SAMPLE_FMT_NONE = -1,
+        AV_SAMPLE_FMT_U8,
+        AV_SAMPLE_FMT_S16,
+        AV_SAMPLE_FMT_S32,
+        AV_SAMPLE_FMT_FLT,
+        AV_SAMPLE_FMT_DBL,
+        AV_SAMPLE_FMT_U8P,
+        AV_SAMPLE_FMT_S16P,
+        AV_SAMPLE_FMT_S32P,
+        AV_SAMPLE_FMT_FLTP,
+        AV_SAMPLE_FMT_DBLP,
+        AV_SAMPLE_FMT_NB
+    };
+
+    int sampleSize = mAudioFormat.sampleSize();
+    QAudioFormat::SampleType sampleType = mAudioFormat.sampleType();
+
+    // qDebug() << "Input sample format: " << sampleSize << "bits " << sampleType;
+
+    switch (sampleType) {
+        case QAudioFormat::Unknown:
+            return AV_SAMPLE_FMT_NONE;
+
+        case QAudioFormat::SignedInt:
+            if (sampleSize == 16)
+                return AV_SAMPLE_FMT_S16;
+            if (sampleSize == 32)
+                return AV_SAMPLE_FMT_S32;
+
+        case QAudioFormat::UnSignedInt:
+            if (sampleSize == 8)
+                return AV_SAMPLE_FMT_U8;
+
+        case QAudioFormat::Float:
+            return AV_SAMPLE_FMT_FLT;
+
+        default:
+            return AV_SAMPLE_FMT_NONE;
+    }
+}
+
+QString UBMicrophoneInput::codec()
+{
+    return mAudioFormat.codec();
+}
+
+qint64 UBMicrophoneInput::processUSecs() const
+{
+    return mAudioInput->processedUSecs();
+}
+
+bool UBMicrophoneInput::init()
+{
+    if (mAudioDeviceInfo.isNull()) {
+        qWarning("No audio input device selected; using default");
+        mAudioDeviceInfo = QAudioDeviceInfo::defaultInputDevice();
+    }
+
+    qDebug() << "Input device name: " << mAudioDeviceInfo.deviceName();
+
+    mAudioFormat = mAudioDeviceInfo.preferredFormat();
+
+    mAudioInput = new QAudioInput(mAudioDeviceInfo, mAudioFormat, NULL);
+    //mAudioInput->setNotifyInterval(100);
+
+    connect(mAudioInput, SIGNAL(stateChanged(QAudio::State)),
+            this, SLOT(onAudioInputStateChanged(QAudio::State)));
+
+    return true;
+}
+
+void UBMicrophoneInput::start()
+{
+    qDebug() << "starting audio input";
+
+    mIODevice = mAudioInput->start();
+
+    connect(mIODevice, SIGNAL(readyRead()),
+            this, SLOT(onDataReady()));
+
+    if (mAudioInput->error() == QAudio::OpenError)
+        qWarning() << "Error opening audio input";
+}
+
+void UBMicrophoneInput::stop()
+{
+    mAudioInput->stop();
+}
+
+QStringList UBMicrophoneInput::availableDevicesNames()
+{
+    QStringList names;
+    QList<QAudioDeviceInfo> devices = QAudioDeviceInfo::availableDevices(QAudio::AudioInput);
+
+    foreach (QAudioDeviceInfo device, devices) {
+        names.push_back(device.deviceName());
+    }
+
+    return names;
+}
+
+void UBMicrophoneInput::setInputDevice(QString name)
+{
+    if (name.isEmpty()) {
+        mAudioDeviceInfo = QAudioDeviceInfo::defaultInputDevice();
+        return;
+    }
+
+    QList<QAudioDeviceInfo> devices = QAudioDeviceInfo::availableDevices(QAudio::AudioInput);
+    bool found = false;
+
+    foreach (QAudioDeviceInfo device, devices) {
+        if (device.deviceName() == name) {
+            mAudioDeviceInfo = device;
+            found = true;
+            break;
+        }
+    }
+
+    if (!found) {
+        qWarning() << "Audio input device not found; using default instead";
+        mAudioDeviceInfo = QAudioDeviceInfo::defaultInputDevice();
+    }
+
+}
+
+void UBMicrophoneInput::onDataReady()
+{
+    int numBytes = mAudioInput->bytesReady();
+
+    if (numBytes > 0)
+        emit dataAvailable(mIODevice->read(numBytes));
+}
+
+void UBMicrophoneInput::onAudioInputStateChanged(QAudio::State state)
+{
+    qDebug() << "Audio input state changed to " << state;
+    switch (state) {
+        case QAudio::StoppedState:
+            if (mAudioInput->error() != QAudio::NoError) {
+                emit error(getErrorString(mAudioInput->error()));
+            }
+            break;
+
+        // handle other states?
+
+        default:
+            break;
+    }
+}
+
+
+/**
+ * @brief Return a meaningful error string based on QAudio error codes
+ */
+QString UBMicrophoneInput::getErrorString(QAudio::Error errorCode)
+{
+    switch (errorCode) {
+        case QAudio::NoError :
+            return "";
+
+        case QAudio::OpenError :
+            return "Couldn't open the audio device";
+
+        case QAudio::IOError :
+            return "Error reading from audio device";
+
+        case QAudio::UnderrunError :
+            return "Underrun error";
+
+        case QAudio::FatalError :
+            return "Fatal error; audio device unusable";
+
+    }
+    return "";
+}

src/podcast/ffmpeg/UBMicrophoneInput.h

+#ifndef UBMICROPHONEINPUT_H
+#define UBMICROPHONEINPUT_H
+
+#include <QtCore>
+#include <QAudioInput>
+
+/**
+ * @brief The UBMicrophoneInput class captures uncompressed sound from a microphone
+ */
+class UBMicrophoneInput : public QObject
+{
+    Q_OBJECT
+
+public:
+    UBMicrophoneInput();
+    virtual ~UBMicrophoneInput();
+
+    bool init();
+    void start();
+    void stop();
+
+    static QStringList availableDevicesNames();
+    void setInputDevice(QString name = "");
+
+    int channelCount();
+    int sampleRate();
+    int sampleSize();
+    int sampleFormat();
+    QString codec();
+
+    qint64 processUSecs() const;
+
+signals:
+    /// Send the new volume, between 0 and 255
+    void audioLevelChanged(quint8 level);
+
+    /// Emitted when new audio data is available
+    void dataAvailable(QByteArray data);
+
+    void error(QString message);
+
+private slots:
+    void onAudioInputStateChanged(QAudio::State state);
+    void onDataReady();
+
+private:
+    QString getErrorString(QAudio::Error errorCode);
+
+    QAudioInput* mAudioInput;
+    QIODevice * mIODevice;
+    QAudioDeviceInfo mAudioDeviceInfo;
+    QAudioFormat mAudioFormat;
+
+    qint64 mSeekPos;
+};
+
+#endif // UBMICROPHONEINPUT_H

src/podcast/podcast.pri

@@ -3,13 +3,15 @@ HEADERS      += src/podcast/UBPodcastController.h \
                 src/podcast/UBAbstractVideoEncoder.h \
                 src/podcast/UBPodcastRecordingPalette.h \
                 src/podcast/youtube/UBYouTubePublisher.h \
-                src/podcast/intranet/UBIntranetPodcastPublisher.h
+                src/podcast/intranet/UBIntranetPodcastPublisher.h \
+    $$PWD/ffmpeg/UBMicrophoneInput.h
                 
 SOURCES      += src/podcast/UBPodcastController.cpp \
                 src/podcast/UBAbstractVideoEncoder.cpp \
                 src/podcast/UBPodcastRecordingPalette.cpp \
                 src/podcast/youtube/UBYouTubePublisher.cpp \
-                src/podcast/intranet/UBIntranetPodcastPublisher.cpp
+                src/podcast/intranet/UBIntranetPodcastPublisher.cpp \
+    $$PWD/ffmpeg/UBMicrophoneInput.cpp
 
 win32 {