Audio buffer support in filter graph

av/audio/frame.pyx

@@ -49,6 +49,7 @@ cdef class AudioFrame(Frame):
         self.ptr.nb_samples = nb_samples
         self.ptr.format = <int>format
         self.ptr.channel_layout = layout
+        self.ptr.channels = lib.av_get_channel_layout_nb_channels(layout)
 
         # HACK: It really sucks to do this twice.
         self._init_user_attributes()

av/filter/graph.pyx

@@ -1,10 +1,13 @@
-from libc.string cimport memcpy
+from fractions import Fraction
 
+from av.audio.format cimport AudioFormat
+from av.audio.frame cimport AudioFrame
+from av.audio.layout cimport AudioLayout
 from av.filter.context cimport FilterContext, wrap_filter_context
 from av.filter.filter cimport Filter, wrap_filter
 from av.utils cimport err_check
-from av.video.frame cimport VideoFrame, alloc_video_frame
 from av.video.format cimport VideoFormat
+from av.video.frame cimport VideoFrame
 
 cdef class Graph(object):
 
@@ -143,12 +146,53 @@ cdef class Graph(object):
 
         return self.add('buffer', args, name=name)
 
+    def add_abuffer(self, template=None, sample_rate=None, format=None, layout=None, channels=None, name=None, time_base=None):
+        """
+        Convenience method for adding `abuffer <https://ffmpeg.org/ffmpeg-filters.html#abuffer>`_.
+        """
+
+        if template is not None:
+            if sample_rate is None:
+                sample_rate = template.sample_rate
+            if format is None:
+                format = template.format
+            if layout is None:
+                layout = template.layout.name
+            if channels is None:
+                channels = template.channels
+            if time_base is None:
+                time_base = Fraction(template.time_base.numerator, template.time_base.denominator)
+
+        if sample_rate is None:
+            raise ValueError('missing sample_rate')
+        if format is None:
+            raise ValueError('missing format')
+        if layout is None and channels is None:
+            raise ValueError('missing layout or channels')
+        if time_base is None:
+            time_base = Fraction(1, sample_rate)
+
+        args = "sample_rate=%d:sample_fmt=%s:time_base=%d/%d" % (sample_rate,
+                                                                 AudioFormat(format).name,
+                                                                 time_base.numerator,
+                                                                 time_base.denominator)
+        if layout:
+            # Use AudioLayout constructor to handle AudioLayout, numerical and string layout descriptors
+            # see av/audio/layout.pyx
+            args += ":channel_layout=" + AudioLayout(layout).name
+        if channels:
+            args += ":channels=" + str(channels)
+
+        return self.add('abuffer', args, name=name)
+
     def push(self, frame):
 
         if isinstance(frame, VideoFrame):
             contexts = self._context_by_type.get('buffer', [])
+        elif isinstance(frame, AudioFrame):
+            contexts = self._context_by_type.get('abuffer', [])
         else:
-            raise ValueError('can only push VideoFrame', type(frame))
+            raise ValueError('can only push VideoFrame or AudioFrame', type(frame))
 
         if len(contexts) != 1:
             raise ValueError('can only auto-push with single buffer; found %s' % len(contexts))

av/utils.pxd

@@ -3,6 +3,33 @@ from libc.stdint cimport int64_t, uint8_t, uint64_t
 cimport libav as lib
 
 
+cdef extern from "libavutil/error.h" nogil:
+    cdef int AVERROR_BSF_NOT_FOUND
+    cdef int AVERROR_BUG
+    cdef int AVERROR_BUFFER_TOO_SMALL
+    cdef int AVERROR_DECODER_NOT_FOUND
+    cdef int AVERROR_DEMUXER_NOT_FOUND
+    cdef int AVERROR_ENCODER_NOT_FOUND
+    cdef int AVERROR_EOF
+    cdef int AVERROR_EXIT
+    cdef int AVERROR_EXTERNAL
+    cdef int AVERROR_FILTER_NOT_FOUND
+    cdef int AVERROR_INVALIDDATA
+    cdef int AVERROR_MUXER_NOT_FOUND
+    cdef int AVERROR_OPTION_NOT_FOUND
+    cdef int AVERROR_PATCHWELCOME
+    cdef int AVERROR_PROTOCOL_NOT_FOUND
+    cdef int AVERROR_UNKNOWN
+    cdef int AVERROR_EXPERIMENTAL
+    cdef int AVERROR_INPUT_CHANGED
+    cdef int AVERROR_OUTPUT_CHANGED
+    cdef int AVERROR_HTTP_BAD_REQUEST
+    cdef int AVERROR_HTTP_UNAUTHORIZED
+    cdef int AVERROR_HTTP_FORBIDDEN
+    cdef int AVERROR_HTTP_NOT_FOUND
+    cdef int AVERROR_HTTP_OTHER_4XX
+    cdef int AVERROR_HTTP_SERVER_ERROR
+
 cdef int stash_exception(exc_info=*)
 
 cpdef int err_check(int res=*, filename=*) except -1

av/utils.pyx

@@ -13,6 +13,32 @@ from av.logging cimport get_last_error
 # === ERROR HANDLING ===
 # ======================
 
+AV_ERROR_BSF_NOT_FOUND = AVERROR_BSF_NOT_FOUND
+AV_ERROR_BUG = AVERROR_BUG
+AV_ERROR_BUFFER_TOO_SMALL = AVERROR_BUFFER_TOO_SMALL
+AV_ERROR_DECODER_NOT_FOUND = AVERROR_DECODER_NOT_FOUND
+AV_ERROR_DEMUXER_NOT_FOUND = AVERROR_DEMUXER_NOT_FOUND
+AV_ERROR_ENCODER_NOT_FOUND = AVERROR_ENCODER_NOT_FOUND
+AV_ERROR_EOF = AVERROR_EOF
+AV_ERROR_EXIT = AVERROR_EXIT
+AV_ERROR_EXTERNAL = AVERROR_EXTERNAL
+AV_ERROR_FILTER_NOT_FOUND = AVERROR_FILTER_NOT_FOUND
+AV_ERROR_INVALIDDATA = AVERROR_INVALIDDATA
+AV_ERROR_MUXER_NOT_FOUND = AVERROR_MUXER_NOT_FOUND
+AV_ERROR_OPTION_NOT_FOUND = AVERROR_OPTION_NOT_FOUND
+AV_ERROR_PATCHWELCOME = AVERROR_PATCHWELCOME
+AV_ERROR_PROTOCOL_NOT_FOUND = AVERROR_PROTOCOL_NOT_FOUND
+AV_ERROR_UNKNOWN = AVERROR_UNKNOWN
+AV_ERROR_EXPERIMENTAL = AVERROR_EXPERIMENTAL
+AV_ERROR_INPUT_CHANGED = AVERROR_INPUT_CHANGED
+AV_ERROR_OUTPUT_CHANGED = AVERROR_OUTPUT_CHANGED
+AV_ERROR_HTTP_BAD_REQUEST = AVERROR_HTTP_BAD_REQUEST
+AV_ERROR_HTTP_UNAUTHORIZED = AVERROR_HTTP_UNAUTHORIZED
+AV_ERROR_HTTP_FORBIDDEN = AVERROR_HTTP_FORBIDDEN
+AV_ERROR_HTTP_NOT_FOUND = AVERROR_HTTP_NOT_FOUND
+AV_ERROR_HTTP_OTHER_4XX = AVERROR_HTTP_OTHER_4XX
+AV_ERROR_HTTP_SERVER_ERROR = AVERROR_HTTP_SERVER_ERROR
+
 # Would love to use the built-in constant, but it doesn't appear to
 # exist on Travis, or my Linux workstation. Could this be because they
 # are actually libav?

examples/filter_audio.py

@@ -0,0 +1,132 @@
+"""
+Simple audio filtering example ported from C code:
+   https://github.com/FFmpeg/FFmpeg/blob/master/doc/examples/filter_audio.c
+"""
+from __future__ import division, print_function
+
+import errno
+import hashlib
+import sys
+from fractions import Fraction
+
+import numpy as np
+
+import av
+import av.audio.frame as af
+import av.filter
+from av.utils import AV_ERROR_EOF
+
+
+FRAME_SIZE = 1024
+
+INPUT_SAMPLE_RATE = 48000
+INPUT_FORMAT = 'fltp'
+INPUT_CHANNEL_LAYOUT = '5.0(side)'  # -> AV_CH_LAYOUT_5POINT0
+
+OUTPUT_SAMPLE_RATE = 44100
+OUTPUT_FORMAT = 's16'  # notice, packed audio format, expect only one plane in output
+OUTPUT_CHANNEL_LAYOUT = 'stereo'  # -> AV_CH_LAYOUT_STEREO
+
+VOLUME_VAL = 0.90
+
+
+def init_filter_graph():
+    graph = av.filter.Graph()
+
+    output_format = 'sample_fmts={}:sample_rates={}:channel_layouts={}'.format(
+        OUTPUT_FORMAT,
+        OUTPUT_SAMPLE_RATE,
+        OUTPUT_CHANNEL_LAYOUT
+    )
+    print('Output format: {}'.format(output_format))
+
+    # initialize filters
+    filter_chain = [
+        graph.add_abuffer(format=INPUT_FORMAT,
+                          sample_rate=INPUT_SAMPLE_RATE,
+                          layout=INPUT_CHANNEL_LAYOUT,
+                          time_base=Fraction(1, INPUT_SAMPLE_RATE)),
+        # initialize filter with keyword parameters
+        graph.add('volume', volume=str(VOLUME_VAL)),
+        # or compound string configuration
+        graph.add('aformat', output_format),
+        graph.add('abuffersink')
+    ]
+
+    # link up the filters into a chain
+    print('Filter graph:')
+    for c, n in zip(filter_chain, filter_chain[1:]):
+        print('\t{} -> {}'.format(c, n))
+        c.link_to(n)
+
+    # initialize the filter graph
+    graph.configure()
+
+    return graph
+
+
+def get_input(frame_num):
+    """
+    Manually construct and update AudioFrame.
+    Consider using AudioFrame.from_ndarry for most real life numpy->AudioFrame conversions.
+
+    :param frame_num:
+    :return:
+    """
+    frame = av.AudioFrame(format=INPUT_FORMAT, layout=INPUT_CHANNEL_LAYOUT, samples=FRAME_SIZE)
+    frame.sample_rate = INPUT_SAMPLE_RATE
+    frame.pts = frame_num * FRAME_SIZE
+
+    for i in range(len(frame.layout.channels)):
+        data = np.zeros(FRAME_SIZE, dtype=af.format_dtypes[INPUT_FORMAT])
+        for j in range(FRAME_SIZE):
+            data[j] = np.sin(2 * np.pi * (frame_num + j) * (i + 1) / float(FRAME_SIZE))
+        frame.planes[i].update(data)
+
+    return frame
+
+
+def process_output(frame):
+    data = frame.to_ndarray()
+    for i in range(data.shape[0]):
+        m = hashlib.md5(data[i, :].tobytes())
+        print('Plane: {:0d} checksum: {}'.format(i, m.hexdigest()))
+
+
+def main(duration):
+    frames_count = int(duration * INPUT_SAMPLE_RATE / FRAME_SIZE)
+
+    graph = init_filter_graph()
+
+    for f in range(frames_count):
+        frame = get_input(f)
+
+        # submit the frame for processing
+        graph.push(frame)
+
+        # pull frames from graph until graph has done processing or is waiting for a new input
+        while True:
+            try:
+                out_frame = graph.pull()
+                process_output(out_frame)
+            except av.AVError as ex:
+                if ex.errno == errno.EAGAIN or ex.errno == AV_ERROR_EOF:
+                    break
+                else:
+                    raise ex
+
+    # process any remaining buffered frames
+    while True:
+        try:
+            out_frame = graph.pull()
+            process_output(out_frame)
+        except av.AVError as ex:
+            if ex.errno == errno.EAGAIN or ex.errno == AV_ERROR_EOF:
+                break
+            else:
+                raise ex
+
+
+if __name__ == '__main__':
+    duration = 1.0 if len(sys.argv) < 2 else float(sys.argv[1])
+    main(duration)

scripts/activate.sh

@@ -39,7 +39,7 @@ fi
 export PYAV_PYTHON
 export PYAV_PIP="${PYAV_PIP-$PYAV_PYTHON -m pip}"
 
-if [[ "$TRAVIS" ]]; then
+if [[ "$TRAVIS" ]] && [[ "$TRAVIS_OS_NAME" != "osx" ]]; then
 
     # Travis as a very self-contained environment. Lets just work in that.
     echo "We're on Travis, so not setting up another virtualenv."