PipeWire audio capture (#247)

* Add base AudioReader class for audio backends

* Add PipeWire audio backend

* Drop video frames before receiving first audio frame

Should help with audio/video sync.

* pulse: add get_time_base()

* add setting for default audio backend and improve selection logic

---------

Co-authored-by: Ilia Bozhinov <ammen99@gmail.com>

.github/workflows/build.yaml

@@ -21,7 +21,7 @@ jobs:
               'pkgconfig(wayland-client)' 'pkgconfig(wayland-protocols)' 'pkgconfig(libpulse-simple)'
               'pkgconfig(libavutil)' 'pkgconfig(libavcodec)' 'pkgconfig(libavformat)'
               'pkgconfig(libavdevice)' 'pkgconfig(libavfilter)' 'pkgconfig(libswresample)'
-              'pkgconfig(gbm)' 'pkgconfig(libdrm)'
+              'pkgconfig(gbm)' 'pkgconfig(libdrm)' 'pkgconfig(libpipewire-0.3)'
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0  # Shallow clones speed things up

config.h.in

@@ -2,10 +2,13 @@
 
 #define DEFAULT_CODEC "@default_codec@"
 #define DEFAULT_PIX_FMT "@default_pix_fmt@"
+#define DEFAULT_AUDIO_BACKEND "@default_audio_backend@"
 #define DEFAULT_AUDIO_CODEC "@default_audio_codec@"
 #define DEFAULT_AUDIO_SAMPLE_RATE @default_audio_sample_rate@
 #define DEFAULT_CONTAINER_FORMAT "@default_container_format@"
 #define FALLBACK_AUDIO_SAMPLE_FMT "@fallback_audio_sample_fmt@"
+#mesondefine HAVE_AUDIO
 #mesondefine HAVE_PULSE
+#mesondefine HAVE_PIPEWIRE
 #mesondefine HAVE_OPENCL
 #mesondefine HAVE_LIBAVDEVICE

manpage/wf-recorder.1

@@ -25,6 +25,7 @@
 .Op Fl p, -codec-param Op Ar option_param=option_value
 .Op Fl v, -version
 .Op Fl x, -pixel-format
+.Op Fl -audio-backend Ar audio_backend
 .Op Fl C, -audio-codec Ar output_audio_codec
 .Op Fl P, -audio-codec-param Op Ar option_param=option_value
 .Op Fl R, -sample-rate Ar sample_rate
@@ -147,6 +148,9 @@ Set the output pixel format.
 List available formats using
 .Dl $ ffmpeg -pix_fmts
 .Pp
+.It Fl  -audio-backend Ar audio_backend
+Specifies the audio backend to be used when -a is set.
+.Pp
 .It Fl C , -audio-codec Ar output_audio_codec
 Specifies the codec of the audio.
 .Pp

meson.build

@@ -6,7 +6,7 @@ project(
 	license: 'MIT',
 	meson_version: '>=0.54.0',
 	default_options: [
-		'cpp_std=c++11',
+		'cpp_std=c++17',
         'c_std=c11',
 		'warning_level=2',
 		'werror=false',
@@ -47,13 +47,67 @@ project_sources = ['src/frame-writer.cpp', 'src/main.cpp', 'src/averr.c']
 wayland_client = dependency('wayland-client', version: '>=1.20')
 wayland_protos = dependency('wayland-protocols', version: '>=1.14')
 
-pulse = dependency('libpulse-simple', required : get_option('pulse'))
-
-if pulse.found()
-    conf_data.set('HAVE_PULSE', true)
-    project_sources += 'src/pulse.cpp'
+audio_backends = {
+    'pulse': {
+      'dependency': dependency('libpulse-simple', required: false),
+      'sources': ['src/pulse.cpp'],
+      'define': 'HAVE_PULSE'
+    },
+    'pipewire': {
+      'dependency': dependency('libpipewire-0.3', required: false),
+      'sources': ['src/pipewire.cpp'],
+      'define': 'HAVE_PIPEWIRE'
+    }
+}
+
+default_audio_backend = get_option('default_audio_backend')
+message('Using default audio backend: @0@'.format(default_audio_backend))
+have_audio = false
+audio_deps = []
+
+foreach backend_name, backend_data : audio_backends
+  if default_audio_backend == backend_name and not backend_data['dependency'].found()
+    error('Default audio backend set to @0@, but @1@ dependency was not found!'.format(backend_name, backend_data['dependency'].name()))
+  endif
+
+  if default_audio_backend == backend_name and get_option(backend_name).disabled()
+    error('Default audio backend set to @0@, but @1@ support is disabled!'.format(backend_name, backend_name))
+  endif
+
+  if get_option(backend_name).enabled() and not backend_data['dependency'].found()
+    error('@0@ support is enabled, but @1@ dependency was not found!'.format(backend_name, backend_data['dependency'].name()))
+  endif
+
+  if backend_data['dependency'].found() and not get_option(backend_name).disabled()
+    conf_data.set(backend_data['define'], true)
+    project_sources += backend_data['sources']
+    audio_deps += backend_data['dependency']
+    have_audio = true
+  else
+    conf_data.set(backend_data['define'], false)
+  endif
+endforeach
+
+if have_audio
+    conf_data.set('HAVE_AUDIO', true)
+    project_sources += 'src/audio.cpp'
+
+    if default_audio_backend == 'auto'
+      if conf_data.get('HAVE_PULSE')
+        default_audio_backend = 'pulse'
+      else
+        foreach backend_name, backend_data : audio_backends
+          if conf_data.get(backend_data['define'])
+            default_audio_backend = backend_name
+            break
+          endif
+        endforeach
+      endif
+    endif
 endif
 
+conf_data.set('default_audio_backend', default_audio_backend)
+
 libavutil = dependency('libavutil')
 libavcodec = dependency('libavcodec')
 libavformat = dependency('libavformat')
@@ -78,9 +132,22 @@ subdir('proto')
 dependencies = [
     wayland_client, wayland_protos,
     libavutil, libavcodec, libavformat, libavdevice, libavfilter,
-    wf_protos, threads, pulse, swr, gbm, drm
-]
+    wf_protos, threads, swr, gbm, drm
+] + audio_deps
 
 executable('wf-recorder', project_sources,
         dependencies: dependencies,
         install: true)
+
+summary = [
+	'',
+	'----------------',
+	'wf-recorder @0@'.format(meson.project_version()),
+    '----------------',
+    'Default audio backend: @0@'.format(default_audio_backend),
+]
+
+foreach backend_name, backend_data : audio_backends
+  summary += ['  - @0@: @1@'.format(backend_name, conf_data.get(backend_data['define']))]
+endforeach
+message('\n'.join(summary))

meson_options.txt

@@ -5,3 +5,5 @@ option('default_audio_sample_rate', type: 'integer', value: 48000, description:
 option('default_container_format', type: 'string', value: 'mkv', description: 'Container file format that will be used by default')
 option('fallback_audio_sample_fmt', type: 'string', value: 's16', description: 'Fallback audio sample format that will be used if wf-recorder cannot determine the sample formats supported by a codec')
 option('pulse', type: 'feature', value: 'auto', description: 'Enable Pulseaudio')
+option('pipewire', type: 'feature', value: 'auto', description: 'Enable PipeWire')
+option('default_audio_backend', type: 'combo', choices: ['auto', 'pulse', 'pipewire'], value: 'auto', description: 'Default audio backend')

src/audio.cpp

@@ -0,0 +1,33 @@
+#include "audio.hpp"
+#include "config.h"
+
+#ifdef HAVE_PULSE
+#include "pulse.hpp"
+#endif
+
+#ifdef HAVE_PIPEWIRE
+#include "pipewire.hpp"
+#endif
+
+AudioReader *AudioReader::create(AudioReaderParams params)
+{
+#ifdef HAVE_PIPEWIRE
+    if (params.audio_backend == "pipewire") {
+        AudioReader *pw = new PipeWireReader;
+        pw->params = params;
+        if (pw->init())
+            return pw;
+        delete pw;
+    }
+#endif
+#ifdef HAVE_PULSE
+    if (params.audio_backend == "pulse") {
+        AudioReader *pa = new PulseReader;
+        pa->params = params;
+        if (pa->init())
+            return pa;
+        delete pa;
+    }
+#endif
+    return nullptr;
+}

src/audio.hpp

@@ -0,0 +1,30 @@
+#ifndef AUDIO_HPP
+#define AUDIO_HPP
+
+#include <stdlib.h>
+#include <stdint.h>
+#include "config.h"
+#include <string>
+
+struct AudioReaderParams
+{
+    size_t audio_frame_size;
+    uint32_t sample_rate;
+    /* Can be NULL */
+    char *audio_source;
+
+    std::string audio_backend = DEFAULT_AUDIO_BACKEND;
+};
+
+class AudioReader
+{
+public:
+    virtual ~AudioReader() {}
+    virtual bool init() = 0;
+    virtual void start() = 0;
+    AudioReaderParams params;
+    static AudioReader *create(AudioReaderParams params);
+    virtual uint64_t get_time_base() const { return 0; }
+};
+
+#endif /* end of include guard: AUDIO_HPP */

src/frame-writer.cpp

@@ -446,7 +446,7 @@ void FrameWriter::init_video_stream()
     }
 }
 
-#ifdef HAVE_PULSE
+#ifdef HAVE_AUDIO
 #if HAVE_CH_LAYOUT
 static uint64_t get_codec_channel_layout(const AVCodec *codec)
 {
@@ -608,7 +608,7 @@ void FrameWriter::init_audio_stream()
 void FrameWriter::init_codecs()
 {
     init_video_stream();
-#ifdef HAVE_PULSE
+#ifdef HAVE_AUDIO
     if (params.enable_audio)
         init_audio_stream();
 #endif
@@ -839,7 +839,7 @@ bool FrameWriter::add_frame(struct gbm_bo *bo, int64_t usec, bool y_invert)
     return push_frame(frame, usec);
 }
 
-#ifdef HAVE_PULSE
+#ifdef HAVE_AUDIO
 #define SRC_RATE 1e6
 #define DST_RATE 1e3
 
@@ -918,7 +918,7 @@ void FrameWriter::finish_frame(AVCodecContext *enc_ctx, AVPacket& pkt)
         av_packet_rescale_ts(&pkt, videoCodecCtx->time_base, videoStream->time_base);
         pkt.stream_index = videoStream->index;
     }
-#ifdef HAVE_PULSE
+#ifdef HAVE_AUDIO
     else
     {
         av_packet_rescale_ts(&pkt, (AVRational){ 1, 1000 }, audioStream->time_base);
@@ -939,7 +939,7 @@ void FrameWriter::finish_frame(AVCodecContext *enc_ctx, AVPacket& pkt)
         params.write_aborted_flag = true;
     }
     av_packet_unref(&pkt);
-#ifdef HAVE_PULSE
+#ifdef HAVE_AUDIO
     if (params.enable_audio)
         fmt_mutex.unlock();
 #endif
@@ -951,7 +951,7 @@ FrameWriter::~FrameWriter()
     AVPacket *pkt = av_packet_alloc();
 
     encode(videoCodecCtx, NULL, pkt);
-#ifdef HAVE_PULSE
+#ifdef HAVE_AUDIO
     if (params.enable_audio)
     {
         encode(audioCodecCtx, NULL, pkt);
@@ -966,7 +966,7 @@ FrameWriter::~FrameWriter()
 
     // Freeing all the allocated memory:
     avcodec_free_context(&videoCodecCtx);
-#ifdef HAVE_PULSE
+#ifdef HAVE_AUDIO
     if (params.enable_audio)
         avcodec_free_context(&audioCodecCtx);
 #endif

src/frame-writer.hpp

@@ -112,7 +112,7 @@ class FrameWriter
 
     void encode(AVCodecContext *enc_ctx, AVFrame *frame, AVPacket *pkt);
 
-#ifdef HAVE_PULSE
+#ifdef HAVE_AUDIO
     SwrContext *swrCtx;
     AVStream *audioStream;
     AVCodecContext *audioCodecCtx;
@@ -128,7 +128,7 @@ class FrameWriter
     bool add_frame(const uint8_t* pixels, int64_t usec, bool y_invert);
     bool add_frame(struct gbm_bo *bo, int64_t usec, bool y_invert);
 
-#ifdef HAVE_PULSE
+#ifdef HAVE_AUDIO
     /* Buffer must have size get_audio_buffer_size() */
     void add_audio(const void* buffer);
     size_t get_audio_buffer_size();