Merge branch 'ffmpeg-podcast' into 1.4-dev

This adds screencast support on Linux. This is based on
ffmpeg for the encoding & muxing (default format is MP4 with AAC audio
and H264 video).
The microphone sound is grabbed using QAudioInput. The encoder should
be able to handle virtually any input format (it has been tested with
a stereo input at different sampling rates with one format (16-bit
signed) but it should work fine with any number of channels, sample
format etc.).

The only problems I have run into so far are that desktop recording is
very slow (compared to OS X) and that the last few video frames are
usually not included in the video. This may be due to GOPs not being
complete, but that's just a wild guess.

src/core/UBApplication.cpp

@@ -520,10 +520,8 @@ void UBApplication::decorateActionMenu(QAction* action)
 
             menu->addSeparator();
 
-#ifndef Q_OS_LINUX // No Podcast on Linux yet
             menu->addAction(mainWindow->actionPodcast);
             mainWindow->actionPodcast->setText(tr("Podcast"));
-#endif
 
             menu->addSeparator();
             menu->addAction(mainWindow->actionQuit);

src/podcast/UBPodcastController.cpp

@@ -64,6 +64,9 @@
 #elif defined(Q_OS_OSX)
     #include "quicktime/UBQuickTimeVideoEncoder.h"
     #include "quicktime/UBAudioQueueRecorder.h"
+#elif defined(Q_OS_LINUX)
+    #include "ffmpeg/UBFFmpegVideoEncoder.h"
+    #include "ffmpeg/UBMicrophoneInput.h"
 #endif
 
 #include "core/memcheck.h"
@@ -309,6 +312,8 @@ void UBPodcastController::start()
         mVideoEncoder = new UBWindowsMediaVideoEncoder(this);  //deleted on stop
 #elif defined(Q_OS_OSX)
         mVideoEncoder = new UBQuickTimeVideoEncoder(this);  //deleted on stop
+#elif defined(Q_OS_LINUX)
+        mVideoEncoder = new UBFFmpegVideoEncoder(this);
 #endif
 
         if (mVideoEncoder)
@@ -804,6 +809,8 @@ QStringList UBPodcastController::audioRecordingDevices()
     devices = UBWaveRecorder::waveInDevices();
 #elif defined(Q_OS_OSX)
     devices = UBAudioQueueRecorder::waveInDevices();
+#elif defined(Q_OS_LINUX)
+    devices = UBMicrophoneInput::availableDevicesNames();
 #endif
 
     return devices;

src/podcast/ffmpeg/UBFFmpegVideoEncoder.h

+/*
+ * Copyright (C) 2015-2016 Département de l'Instruction Publique (DIP-SEM)
+ *
+ * This file is part of OpenBoard.
+ *
+ * OpenBoard is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License,
+ * with a specific linking exception for the OpenSSL project's
+ * "OpenSSL" library (or with modified versions of it that use the
+ * same license as the "OpenSSL" library).
+ *
+ * OpenBoard is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with OpenBoard. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef UBFFMPEGVIDEOENCODER_H
+#define UBFFMPEGVIDEOENCODER_H
+
+extern "C" {
+    #include <libavcodec/avcodec.h>
+    #include <libavformat/avformat.h>
+    #include <libavformat/avio.h>
+    #include <libavutil/audio_fifo.h>
+    #include <libavutil/avutil.h>
+    #include <libavutil/imgutils.h>
+    #include <libavutil/opt.h>
+    #include <libavutil/mathematics.h>
+    #include <libavutil/time.h>
+    #include <libswscale/swscale.h>
+    #include <libswresample/swresample.h>
+}
+
+#include <atomic>
+
+#include <QtCore>
+#include <QImage>
+
+#include "podcast/UBAbstractVideoEncoder.h"
+#include "podcast/ffmpeg/UBMicrophoneInput.h"
+
+class UBFFmpegVideoEncoderWorker;
+class UBPodcastController;
+
+/**
+ * This class provides an interface between the podcast controller and the ffmpeg
+ * back-end.
+ * It includes all the necessary objects and methods to record video (muxer, audio and
+ * video streams and encoders, etc) from inputs consisting of raw PCM audio and raw RGBA
+ * images.
+ *
+ * A worker thread is used to encode and write the audio and video on-the-fly.
+ */
+
+class UBFFmpegVideoEncoder : public UBAbstractVideoEncoder
+{
+    Q_OBJECT
+
+    friend class UBFFmpegVideoEncoderWorker;
+
+public:
+
+    UBFFmpegVideoEncoder(QObject* parent = NULL);
+    virtual ~UBFFmpegVideoEncoder();
+
+    bool start();
+    bool stop();
+
+    void newPixmap(const QImage& pImage, long timestamp);
+
+    QString videoFileExtension() const { return "mp4"; }
+
+    QString lastErrorMessage() { return mLastErrorMessage; }
+
+    void setRecordAudio(bool pRecordAudio) { mShouldRecordAudio = pRecordAudio; }
+
+signals:
+
+    void encodingFinished(bool ok);
+
+private slots:
+
+    void setLastErrorMessage(const QString& pMessage);
+    void onAudioAvailable(QByteArray data);
+    void finishEncoding();
+
+private:
+
+    struct ImageFrame
+    {
+        QImage image;
+        long timestamp; // unit: ms
+    };
+
+    AVFrame* convertImageFrame(ImageFrame frame);
+    AVFrame* convertAudio(QByteArray data);
+    void processAudio(QByteArray& data);
+    bool init();
+
+    QString mLastErrorMessage;
+
+    QThread* mVideoEncoderThread;
+    UBFFmpegVideoEncoderWorker* mVideoWorker;
+
+    // Muxer
+    // ------------------------------------------
+    AVFormatContext* mOutputFormatContext;
+    AVStream* mVideoStream;
+    AVStream* mAudioStream;
+
+    // Video
+    // ------------------------------------------
+    QQueue<ImageFrame> mPendingFrames;
+    struct SwsContext * mSwsContext;
+
+    int mVideoTimebase;
+
+    // Audio
+    // ------------------------------------------
+    bool mShouldRecordAudio;
+
+    UBMicrophoneInput * mAudioInput;
+    struct SwrContext * mSwrContext;
+    /// Queue for audio that has been rescaled/converted but not encoded yet
+    AVAudioFifo *mAudioOutBuffer;
+
+    /// Sample rate for encoded audio
+    int mAudioSampleRate;
+    /// Total audio frames sent to encoder
+    int mAudioFrameCount;
+};
+
+
+class UBFFmpegVideoEncoderWorker : public QObject
+{
+    Q_OBJECT
+
+    friend class UBFFmpegVideoEncoder;
+
+public:
+    UBFFmpegVideoEncoderWorker(UBFFmpegVideoEncoder* controller);
+    ~UBFFmpegVideoEncoderWorker();
+
+    bool isRunning() { return mIsRunning; }
+
+    void queueVideoFrame(AVFrame* frame);
+    void queueAudioFrame(AVFrame* frame);
+
+public slots:
+    void runEncoding();
+    void stopEncoding();
+
+signals:
+    void encodingFinished();
+    void error(QString message);
+
+private:
+    void writeLatestVideoFrame();
+    void writeLatestAudioFrame();
+
+    UBFFmpegVideoEncoder* mController;
+
+    // std::atomic is C++11. This won't work with msvc2010, so a
+    // newer compiler must be used if this class is to be used on Windows
+    std::atomic<bool> mStopRequested;
+    std::atomic<bool> mIsRunning;
+
+    QQueue<AVFrame*> mImageQueue;
+    QQueue<AVFrame*> mAudioQueue;
+
+    QMutex mFrameQueueMutex;
+    QWaitCondition mWaitCondition;
+
+    AVPacket* mVideoPacket;
+    AVPacket* mAudioPacket;
+};
+
+#endif // UBFFMPEGVIDEOENCODER_H

src/podcast/ffmpeg/UBMicrophoneInput.cpp

+/*
+ * Copyright (C) 2015-2016 Département de l'Instruction Publique (DIP-SEM)
+ *
+ * This file is part of OpenBoard.
+ *
+ * OpenBoard is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License,
+ * with a specific linking exception for the OpenSSL project's
+ * "OpenSSL" library (or with modified versions of it that use the
+ * same license as the "OpenSSL" library).
+ *
+ * OpenBoard is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with OpenBoard. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "UBMicrophoneInput.h"
+
+UBMicrophoneInput::UBMicrophoneInput()
+    : mAudioInput(NULL)
+    , mIODevice(NULL)
+    , mSeekPos(0)
+{
+}
+
+UBMicrophoneInput::~UBMicrophoneInput()
+{
+    if (mAudioInput)
+        delete mAudioInput;
+}
+
+bool UBMicrophoneInput::init()
+{
+    if (mAudioDeviceInfo.isNull()) {
+        qWarning("No audio input device selected; using default");
+        mAudioDeviceInfo = QAudioDeviceInfo::defaultInputDevice();
+    }
+
+    mAudioFormat = mAudioDeviceInfo.preferredFormat();
+    mAudioInput = new QAudioInput(mAudioDeviceInfo, mAudioFormat, NULL);
+
+    connect(mAudioInput, SIGNAL(stateChanged(QAudio::State)),
+            this, SLOT(onAudioInputStateChanged(QAudio::State)));
+
+
+    qDebug() << "Input device name: " << mAudioDeviceInfo.deviceName();
+    qDebug() << "Input sample format: " << mAudioFormat.sampleSize() << "bit"
+             << mAudioFormat.sampleType() << "at" << mAudioFormat.sampleRate() << "Hz"
+             << "; codec: " << mAudioFormat.codec();
+
+    return true;
+}
+
+void UBMicrophoneInput::start()
+{
+    mIODevice = mAudioInput->start();
+
+    connect(mIODevice, SIGNAL(readyRead()),
+            this, SLOT(onDataReady()));
+
+    if (mAudioInput->error() == QAudio::OpenError)
+        qWarning() << "Error opening audio input";
+}
+
+void UBMicrophoneInput::stop()
+{
+    mAudioInput->stop();
+}
+
+QStringList UBMicrophoneInput::availableDevicesNames()
+{
+    QStringList names;
+    QList<QAudioDeviceInfo> devices = QAudioDeviceInfo::availableDevices(QAudio::AudioInput);
+
+    foreach (QAudioDeviceInfo device, devices) {
+        names.push_back(device.deviceName());
+    }
+
+    return names;
+}
+
+void UBMicrophoneInput::setInputDevice(QString name)
+{
+    if (name.isEmpty()) {
+        mAudioDeviceInfo = QAudioDeviceInfo::defaultInputDevice();
+        return;
+    }
+
+    QList<QAudioDeviceInfo> devices = QAudioDeviceInfo::availableDevices(QAudio::AudioInput);
+    bool found = false;
+
+    foreach (QAudioDeviceInfo device, devices) {
+        if (device.deviceName() == name) {
+            mAudioDeviceInfo = device;
+            found = true;
+            break;
+        }
+    }
+
+    if (!found) {
+        qWarning() << "Audio input device not found; using default instead";
+        mAudioDeviceInfo = QAudioDeviceInfo::defaultInputDevice();
+    }
+
+}
+
+int UBMicrophoneInput::channelCount()
+{
+    return mAudioFormat.channelCount();
+}
+
+int UBMicrophoneInput::sampleRate()
+{
+    return mAudioFormat.sampleRate();
+}
+
+/* Return the sample size in bits */
+int UBMicrophoneInput::sampleSize()
+{
+    return mAudioFormat.sampleSize();
+}
+
+/** Return the sample format in FFMpeg style (AVSampleFormat enum) */
+int UBMicrophoneInput::sampleFormat()
+{
+    enum AVSampleFormat {
+        AV_SAMPLE_FMT_NONE = -1,
+        AV_SAMPLE_FMT_U8,
+        AV_SAMPLE_FMT_S16,
+        AV_SAMPLE_FMT_S32,
+        AV_SAMPLE_FMT_FLT,
+        AV_SAMPLE_FMT_DBL,
+        AV_SAMPLE_FMT_U8P,
+        AV_SAMPLE_FMT_S16P,
+        AV_SAMPLE_FMT_S32P,
+        AV_SAMPLE_FMT_FLTP,
+        AV_SAMPLE_FMT_DBLP,
+        AV_SAMPLE_FMT_NB
+    };
+
+    int sampleSize = mAudioFormat.sampleSize();
+    QAudioFormat::SampleType sampleType = mAudioFormat.sampleType();
+
+    switch (sampleType) {
+        case QAudioFormat::Unknown:
+            return AV_SAMPLE_FMT_NONE;
+
+        case QAudioFormat::SignedInt:
+            if (sampleSize == 16)
+                return AV_SAMPLE_FMT_S16;
+            if (sampleSize == 32)
+                return AV_SAMPLE_FMT_S32;
+            break;
+
+        case QAudioFormat::UnSignedInt:
+            if (sampleSize == 8)
+                return AV_SAMPLE_FMT_U8;
+            break;
+
+        case QAudioFormat::Float:
+            return AV_SAMPLE_FMT_FLT;
+
+        default:
+            return AV_SAMPLE_FMT_NONE;
+    }
+
+    return AV_SAMPLE_FMT_NONE;
+}
+
+QString UBMicrophoneInput::codec()
+{
+    return mAudioFormat.codec();
+}
+
+static qint64 uSecsElapsed = 0;
+void UBMicrophoneInput::onDataReady()
+{
+    int numBytes = mAudioInput->bytesReady();
+
+    uSecsElapsed += mAudioFormat.durationForBytes(numBytes);
+
+    // Only emit data every 100ms
+    if (uSecsElapsed > 100000) {
+        uSecsElapsed = 0;
+        QByteArray data = mIODevice->read(numBytes);
+
+        quint8 level = audioLevel(data);
+        if (level != mLastAudioLevel) {
+            mLastAudioLevel = level;
+            emit audioLevelChanged(level);
+        }
+
+        emit dataAvailable(data);
+    }
+}
+
+void UBMicrophoneInput::onAudioInputStateChanged(QAudio::State state)
+{
+    switch (state) {
+        case QAudio::StoppedState:
+            if (mAudioInput->error() != QAudio::NoError) {
+                emit error(getErrorString(mAudioInput->error()));
+            }
+            break;
+
+        // handle other states?
+
+        default:
+            break;
+    }
+}
+
+/**
+ * @brief Calculate the current audio level of an array of samples and return it
+ * @param data An array of audio samples
+ * @return A value between 0 and 255
+ *
+ * Audio level is calculated as the RMS (root mean square) of the samples
+ * in the supplied array.
+ */
+quint8 UBMicrophoneInput::audioLevel(const QByteArray &data)
+{
+    int bytesPerSample = mAudioFormat.bytesPerFrame() / mAudioFormat.channelCount();
+
+    const char * ptr = data.constData();
+    double sum = 0;
+    int n_samples = data.size() / bytesPerSample;
+
+    for (int i(0); i < (data.size() - bytesPerSample); i += bytesPerSample) {
+        sum += pow(sampleRelativeLevel(ptr + i), 2);
+    }
+
+    double rms = sqrt(sum/n_samples);
+
+    // The vu meter looks a bit better when the RMS isn't displayed linearly, as perceived sound
+    // level increases logarithmically. So here RMS is substituted by rms^(1/e)
+    rms = pow(rms, 1./exp(1));
+
+    return UINT8_MAX * rms;
+}
+
+/**
+ * @brief Calculate one sample's level relative to its maximum value
+ * @param sample One sample, in the format specified by mAudioFormat
+ * @return A double between 0 and 1.0, where 1.0 is the maximum value the sample can take,
+ *         or -1 if the value couldn't be calculated.
+ */
+double UBMicrophoneInput::sampleRelativeLevel(const char* sample)
+{
+    QAudioFormat::SampleType type =  mAudioFormat.sampleType();
+    int sampleSize = mAudioFormat.sampleSize();
+
+    if (sampleSize == 16 && type == QAudioFormat::SignedInt)
+        return double(*reinterpret_cast<const int16_t*>(sample))/INT16_MAX;
+
+    if (sampleSize == 8 && type == QAudioFormat::SignedInt)
+        return double(*reinterpret_cast<const int8_t*>(sample))/INT8_MAX;
+
+    if (sampleSize == 16 && type == QAudioFormat::UnSignedInt)
+        return double(*reinterpret_cast<const uint16_t*>(sample))/UINT16_MAX;
+
+    if (sampleSize == 8 && type == QAudioFormat::UnSignedInt)
+        return double(*reinterpret_cast<const uint8_t*>(sample))/UINT8_MAX;
+
+    if (type == QAudioFormat::Float)
+        return (*reinterpret_cast<const float*>(sample) + 1.0)/2.;
+
+    return -1;
+}
+
+/**
+ * @brief Return a meaningful error string based on QAudio error codes
+ */
+QString UBMicrophoneInput::getErrorString(QAudio::Error errorCode)
+{
+    switch (errorCode) {
+        case QAudio::NoError :
+            return "";
+
+        case QAudio::OpenError :
+            return "Couldn't open the audio device";
+
+        case QAudio::IOError :
+            return "Error reading from audio device";
+
+        case QAudio::UnderrunError :
+            return "Underrun error";
+
+        case QAudio::FatalError :
+            return "Fatal error; audio device unusable";
+
+    }
+    return "";
+}

src/podcast/ffmpeg/UBMicrophoneInput.h

+/*
+ * Copyright (C) 2015-2016 Département de l'Instruction Publique (DIP-SEM)
+ *
+ * This file is part of OpenBoard.
+ *
+ * OpenBoard is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License,
+ * with a specific linking exception for the OpenSSL project's
+ * "OpenSSL" library (or with modified versions of it that use the
+ * same license as the "OpenSSL" library).
+ *
+ * OpenBoard is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with OpenBoard. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef UBMICROPHONEINPUT_H
+#define UBMICROPHONEINPUT_H
+
+#include <QtCore>
+#include <QAudioInput>
+
+/**
+ * @brief The UBMicrophoneInput class captures uncompressed sound from a microphone.
+ *
+ * Audio samples can be read by connecting to the dataAvailable signal.
+ */
+class UBMicrophoneInput : public QObject
+{
+    Q_OBJECT
+
+public:
+    UBMicrophoneInput();
+    virtual ~UBMicrophoneInput();
+
+    bool init();
+    void start();
+    void stop();
+
+    static QStringList availableDevicesNames();
+    void setInputDevice(QString name = "");
+
+    int channelCount();
+    int sampleRate();
+    int sampleSize();
+    int sampleFormat();
+    QString codec();
+
+signals:
+    /// Send the new audio level, between 0 and 255
+    void audioLevelChanged(quint8 level);
+
+    /// Emitted when new audio data is available
+    void dataAvailable(QByteArray data);
+
+    void error(QString message);
+
+private slots:
+    void onAudioInputStateChanged(QAudio::State state);
+    void onDataReady();
+
+private:
+    double sampleRelativeLevel(const char* sample);
+    quint8 audioLevel(const QByteArray& data);
+    QString getErrorString(QAudio::Error errorCode);
+
+    QAudioInput* mAudioInput;
+    QIODevice * mIODevice;
+    QAudioDeviceInfo mAudioDeviceInfo;
+    QAudioFormat mAudioFormat;
+
+    qint64 mSeekPos;
+    quint8 mLastAudioLevel;
+};
+
+#endif // UBMICROPHONEINPUT_H

src/podcast/podcast.pri

@@ -3,13 +3,15 @@ HEADERS      += src/podcast/UBPodcastController.h \
                 src/podcast/UBAbstractVideoEncoder.h \
                 src/podcast/UBPodcastRecordingPalette.h \
                 src/podcast/youtube/UBYouTubePublisher.h \
-                src/podcast/intranet/UBIntranetPodcastPublisher.h
+                src/podcast/intranet/UBIntranetPodcastPublisher.h \
+    $$PWD/ffmpeg/UBMicrophoneInput.h
                 
 SOURCES      += src/podcast/UBPodcastController.cpp \
                 src/podcast/UBAbstractVideoEncoder.cpp \
                 src/podcast/UBPodcastRecordingPalette.cpp \
                 src/podcast/youtube/UBYouTubePublisher.cpp \
-                src/podcast/intranet/UBIntranetPodcastPublisher.cpp
+                src/podcast/intranet/UBIntranetPodcastPublisher.cpp \
+    $$PWD/ffmpeg/UBMicrophoneInput.cpp
 
 win32 {
 
@@ -33,3 +35,28 @@ macx {
                 
     OBJECTIVE_SOURCES += src/podcast/quicktime/UBQuickTimeFile.mm
 }
+
+linux-g++* {
+    HEADERS  += src/podcast/ffmpeg/UBFFmpegVideoEncoder.h
+
+    SOURCES  += src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp
+
+
+    FFMPEG = /opt/ffmpeg
+
+    INCLUDEPATH += $${FFMPEG}/include
+    DEPENDPATH += /usr/lib/x86_64-linux-gnu
+
+    LIBS += -L $${FFMPEG}/lib -lavformat \
+            -L $${FFMPEG}/lib -lavcodec \
+            -L $${FFMPEG}/lib -lswscale \
+            -L $${FFMPEG}/lib -lavutil \
+            -lva-x11 \
+            -lva \
+            -lxcb-shm \
+            -lxcb-xfixes \
+            -lxcb-render -lxcb-shape -lxcb -lX11 -lasound -lSDL -lx264 -lpthread -lvpx -lvorbisenc -lvorbis -ltheoraenc -ltheoradec -logg -lopus -lmp3lame -lfreetype -lfdk-aac -lass -llzma -lbz2 -lz -ldl -lswresample -lswscale -lavutil -lm
+
+
+    QMAKE_CXXFLAGS += -std=c++11 # move this to OpenBoard.pro when we can use C++11 on all platforms
+}