Dear nia, On 2020-03-14, nia wrote: > On Sat, Mar 14, 2020 at 12:20:11AM +0200, Yorick Hardy wrote: > > You are correct. I threw together a NetBSD audio driver based on the oss > > driver, but it had exactly the same problem. Strangely, I have been unable to > > reproduce the problem on an old i386 netbook (so far). > > > > I wrote a test program to try and reproduce what ffmpeg is doing, and > > (I am not sure yet) it seems like non-blocking reads is causing the > > distortion. The same test program with blocking reads seems to work > > okay. > > > > I will look into it a bit more, and then report back. > > Right, /dev/audio doesn't support non-blocking I/O. But you're supposed to > do short enough reads and writes that it shouldn't matter. That might be > the cause of the worst of the problems. Oops, I think the man page might need to be updated then. I managed to convince my test program to correctly record with non-blocking I/O (perhaps by accident?) by working a bit differently to ffmpeg, but I am not sure how to adjust ffmpeg in this way. I will try blocking reads (presumably reading blocksize bytes at a time). Re: audacity (earlier in the thread), audacity hangs whenever I try to record. I probably need to update all of my packages - but I am not doing any long builds at the moment due to unpredictable electricity supply! > Do you want to work on this together somewhere? Yes, that would be great! As long as you don't mind someone who is extremely unresponsive most of the week! I have quite a few deadlines in the next week, and will probably ignore most things while I am doing that work. Attached are the patches for my "testing" version of the ffmpeg backend (heavily based on the OSS backend). I am sure it should be renamed to "netbsd", initially I was trying for sun compatibility - but I am not sure that makes sense. -- Kind regards, Yorick Hardy
$NetBSD$
--- configure.orig 2019-08-05 21:11:40.000000000 +0000
+++ configure
@@ -2115,6 +2115,7 @@ HEADERS_LIST="
opencv2_core_core_c_h
OpenGL_gl3_h
poll_h
+ sys_audioio_h
sys_param_h
sys_resource_h
sys_select_h
@@ -3306,6 +3307,8 @@ android_camera_indev_deps="android camer
android_camera_indev_extralibs="-landroid -lcamera2ndk -lmediandk"
alsa_indev_deps="alsa"
alsa_outdev_deps="alsa"
+audioio_indev_deps_any="sys_audioio_h"
+audioio_outdev_deps_any="sys_audioio_h"
avfoundation_indev_deps="avfoundation corevideo coremedia pthreads"
avfoundation_indev_suggest="coregraphics applicationservices"
avfoundation_indev_extralibs="-framework Foundation"
@@ -6461,6 +6464,10 @@ check_headers "dev/bktr/ioctl_meteor.h d
check_headers "dev/video/meteor/ioctl_meteor.h dev/video/bktr/ioctl_bt848.h" ||
check_headers "dev/ic/bt8xx.h"
+if check_struct sys/audioio.h audio_info_t play; then
+ enable_sanitized sys/audioio.h
+fi
+
if check_struct sys/soundcard.h audio_buf_info bytes; then
enable_sanitized sys/soundcard.h
else
Attachment:
patch-doc_indevs.texi
Description: TeXInfo document
Attachment:
patch-doc_outdevs.texi
Description: TeXInfo document
$NetBSD$ --- libavdevice/Makefile.orig 2019-08-05 20:52:21.000000000 +0000 +++ libavdevice/Makefile @@ -15,6 +15,8 @@ OBJS-$(CONFIG_SHARED) OBJS-$(CONFIG_ALSA_INDEV) += alsa_dec.o alsa.o timefilter.o OBJS-$(CONFIG_ALSA_OUTDEV) += alsa_enc.o alsa.o OBJS-$(CONFIG_ANDROID_CAMERA_INDEV) += android_camera.o +OBJS-$(CONFIG_AUDIOIO_INDEV) += audioio_dec.o audioio.o +OBJS-$(CONFIG_AUDIOIO_OUTDEV) += audioio_enc.o audioio.o OBJS-$(CONFIG_AVFOUNDATION_INDEV) += avfoundation.o OBJS-$(CONFIG_BKTR_INDEV) += bktr.o OBJS-$(CONFIG_CACA_OUTDEV) += caca.o
$NetBSD$ --- libavdevice/alldevices.c.orig 2019-08-05 20:52:21.000000000 +0000 +++ libavdevice/alldevices.c @@ -27,6 +27,8 @@ extern AVInputFormat ff_alsa_demuxer; extern AVOutputFormat ff_alsa_muxer; extern AVInputFormat ff_android_camera_demuxer; +extern AVInputFormat ff_audioio_demuxer; +extern AVOutputFormat ff_audioio_muxer; extern AVInputFormat ff_avfoundation_demuxer; extern AVInputFormat ff_bktr_demuxer; extern AVOutputFormat ff_caca_muxer;
--- /dev/null 2020-03-11 11:29:21.727010528 +0200
+++ libavdevice/audioio.c 2020-03-11 11:41:43.710391401 +0200
@@ -0,0 +1,125 @@
+/*
+ * Sun and NetBSD play and grab interface
+ * Copyright (c) 2020 Yorick Hardy
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "config.h"
+
+#include <string.h>
+
+#if HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <fcntl.h>
+#include <sys/audioio.h>
+#include <sys/ioctl.h>
+
+#include "libavutil/log.h"
+
+#include "libavcodec/avcodec.h"
+#include "avdevice.h"
+
+#include "audioio.h"
+
+int ff_audioio_audio_open(AVFormatContext *s1, int is_output,
+ const char *audio_device)
+{
+ AudioIOAudioData *s = s1->priv_data;
+ audio_info_t info;
+ struct audio_prinfo *prinfo;
+ int audio_fd;
+ int tmp, err;
+
+ if (is_output)
+ audio_fd = avpriv_open(audio_device, O_WRONLY);
+ else
+ audio_fd = avpriv_open(audio_device, O_RDONLY);
+ if (audio_fd < 0) {
+ av_log(s1, AV_LOG_ERROR, "%s: %s\n", audio_device, av_err2str(AVERROR(errno)));
+ return AVERROR(EIO);
+ }
+
+ /* non blocking mode */
+ if (!is_output) {
+ if (fcntl(audio_fd, F_SETFL, O_NONBLOCK) < 0) {
+ av_log(s1, AV_LOG_WARNING, "%s: Could not enable non block mode (%s)\n", audio_device, av_err2str(AVERROR(errno)));
+ }
+ }
+
+ s->frame_size = AUDIOIO_AUDIO_BLOCK_SIZE;
+
+#define CHECK_IOCTL_ERROR(event) \
+ if (err < 0) { \
+ av_log(s1, AV_LOG_ERROR, #event ": %s\n", av_err2str(AVERROR(errno)));\
+ goto fail; \
+ }
+
+ AUDIO_INITINFO(&info);
+
+ if (is_output) {
+ prinfo = &(info.play);
+ } else {
+ prinfo = &(info.record);
+ }
+
+ prinfo->encoding = AUDIO_ENCODING_SLINEAR;
+ prinfo->sample_rate = s->sample_rate;
+ prinfo->channels = s->channels;
+ prinfo->precision = 16;
+
+ err=ioctl(audio_fd, AUDIO_SETINFO, &info);
+ CHECK_IOCTL_ERROR(AUDIO_SETINFO)
+
+ err=ioctl(audio_fd, AUDIO_GETINFO, &info);
+ CHECK_IOCTL_ERROR(AUDIO_GETINFO)
+
+ s->fd = audio_fd;
+ if (is_output) {
+ s->sample_rate = prinfo->sample_rate;
+ s->channels = prinfo->channels;
+ } else {
+ s->sample_rate = prinfo->sample_rate;
+ s->channels = prinfo->channels;
+ }
+
+ switch(prinfo->encoding) {
+ case AUDIO_ENCODING_SLINEAR_LE:
+ s->codec_id = AV_CODEC_ID_PCM_S16LE;
+ break;
+ case AUDIO_ENCODING_SLINEAR_BE:
+ s->codec_id = AV_CODEC_ID_PCM_S16BE;
+ break;
+ default:
+ av_log(s1, AV_LOG_ERROR, "Soundcard does not support signed 16 bit sample format\n");
+ close(audio_fd);
+ return AVERROR(EIO);
+ }
+
+ return 0;
+ fail:
+ close(audio_fd);
+ return AVERROR(EIO);
+#undef CHECK_IOCTL_ERROR
+}
+
+int ff_audioio_audio_close(AudioIOAudioData *s)
+{
+ close(s->fd);
+ return 0;
+}
--- /dev/null 2020-03-11 11:29:21.727010528 +0200
+++ libavdevice/audioio.h 2020-03-11 11:22:01.834099909 +0200
@@ -0,0 +1,44 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef AVDEVICE_AUDIOIO_H
+#define AVDEVICE_AUDIOIO_H
+
+#include "libavcodec/avcodec.h"
+
+#include "libavformat/avformat.h"
+
+#define AUDIOIO_AUDIO_BLOCK_SIZE 4096
+
+typedef struct AudioIOAudioData {
+ AVClass *class;
+ int fd;
+ int sample_rate;
+ int channels;
+ int frame_size; /* in bytes ! */
+ enum AVCodecID codec_id;
+ uint8_t buffer[AUDIOIO_AUDIO_BLOCK_SIZE];
+ int buffer_ptr;
+} AudioIOAudioData;
+
+int ff_audioio_audio_open(AVFormatContext *s1, int is_output,
+ const char *audio_device);
+
+int ff_audioio_audio_close(AudioIOAudioData *s);
+
+#endif /* AVDEVICE_AUDIOIO_H */
--- /dev/null 2020-03-11 11:29:21.727010528 +0200
+++ libavdevice/audioio_dec.c 2020-03-11 11:37:55.348444643 +0200
@@ -0,0 +1,135 @@
+/*
+ * Sun and NetBSD play and grab interface
+ * Copyright (c) 2020 Yorick Hardy
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "config.h"
+
+#include <stdint.h>
+
+#if HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <fcntl.h>
+#include <sys/audioio.h>
+#include <sys/ioctl.h>
+
+#include "libavutil/internal.h"
+#include "libavutil/opt.h"
+#include "libavutil/time.h"
+
+#include "libavcodec/avcodec.h"
+
+#include "avdevice.h"
+#include "libavformat/internal.h"
+
+#include "audioio.h"
+
+static int audio_read_header(AVFormatContext *s1)
+{
+ AudioIOAudioData *s = s1->priv_data;
+ AVStream *st;
+ int ret;
+
+ st = avformat_new_stream(s1, NULL);
+ if (!st) {
+ return AVERROR(ENOMEM);
+ }
+
+ ret = ff_audioio_audio_open(s1, 0, s1->url);
+ if (ret < 0) {
+ return AVERROR(EIO);
+ }
+
+ /* take real parameters */
+ st->codecpar->codec_type = AVMEDIA_TYPE_AUDIO;
+ st->codecpar->codec_id = s->codec_id;
+ st->codecpar->sample_rate = s->sample_rate;
+ st->codecpar->channels = s->channels;
+
+ avpriv_set_pts_info(st, 64, 1, 1000000); /* 64 bits pts in us */
+ return 0;
+}
+
+static int audio_read_packet(AVFormatContext *s1, AVPacket *pkt)
+{
+ AudioIOAudioData *s = s1->priv_data;
+ audio_info_t info;
+ int ret, bdelay;
+ int64_t cur_time;
+
+ if ((ret=av_new_packet(pkt, s->frame_size)) < 0)
+ return ret;
+
+ ret = read(s->fd, pkt->data, pkt->size);
+ if (ret <= 0){
+ av_packet_unref(pkt);
+ pkt->size = 0;
+ if (ret<0) return AVERROR(errno);
+ else return AVERROR_EOF;
+ }
+ pkt->size = ret;
+
+ /* compute pts of the start of the packet */
+ cur_time = av_gettime();
+ bdelay = ret;
+ if (ioctl(s->fd, AUDIO_GETBUFINFO, &info) == 0) {
+ bdelay += info.record.seek;
+ }
+ /* subtract time represented by the number of bytes in the audio fifo */
+ cur_time -= (bdelay * 1000000LL) / (s->sample_rate * s->channels);
+
+ /* convert to wanted units */
+ pkt->pts = cur_time;
+
+ return 0;
+}
+
+static int audio_read_close(AVFormatContext *s1)
+{
+ AudioIOAudioData *s = s1->priv_data;
+
+ ff_audioio_audio_close(s);
+ return 0;
+}
+
+static const AVOption options[] = {
+ { "sample_rate", "", offsetof(AudioIOAudioData, sample_rate), AV_OPT_TYPE_INT, {.i64 = 48000}, 1, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },
+ { "channels", "", offsetof(AudioIOAudioData, channels), AV_OPT_TYPE_INT, {.i64 = 2}, 1, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },
+ { NULL },
+};
+
+static const AVClass audioio_demuxer_class = {
+ .class_name = "Sun audio demuxer",
+ .item_name = av_default_item_name,
+ .option = options,
+ .version = LIBAVUTIL_VERSION_INT,
+ .category = AV_CLASS_CATEGORY_DEVICE_AUDIO_INPUT,
+};
+
+AVInputFormat ff_audioio_demuxer = {
+ .name = "audioio",
+ .long_name = NULL_IF_CONFIG_SMALL("Sun audio capture"),
+ .priv_data_size = sizeof(AudioIOAudioData),
+ .read_header = audio_read_header,
+ .read_packet = audio_read_packet,
+ .read_close = audio_read_close,
+ .flags = AVFMT_NOFILE,
+ .priv_class = &audioio_demuxer_class,
+};
--- /dev/null 2020-03-11 11:29:21.727010528 +0200
+++ libavdevice/audioio_enc.c 2020-03-11 11:22:01.839672461 +0200
@@ -0,0 +1,113 @@
+/*
+ * Sun and NetBSD play and grab interface
+ * Copyright (c) 2020 Yorick Hardy
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "config.h"
+
+#if HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <fcntl.h>
+#include <sys/audioio.h>
+#include <sys/ioctl.h>
+
+#include "libavutil/internal.h"
+
+#include "libavcodec/avcodec.h"
+
+#include "avdevice.h"
+#include "libavformat/internal.h"
+
+#include "audioio.h"
+
+static int audio_write_header(AVFormatContext *s1)
+{
+ AudioIOAudioData *s = s1->priv_data;
+ AVStream *st;
+ int ret;
+
+ st = s1->streams[0];
+ s->sample_rate = st->codecpar->sample_rate;
+ s->channels = st->codecpar->channels;
+ ret = ff_audioio_audio_open(s1, 1, s1->url);
+ if (ret < 0) {
+ return AVERROR(EIO);
+ } else {
+ return 0;
+ }
+}
+
+static int audio_write_packet(AVFormatContext *s1, AVPacket *pkt)
+{
+ AudioIOAudioData *s = s1->priv_data;
+ int len, ret;
+ int size= pkt->size;
+ uint8_t *buf= pkt->data;
+
+ while (size > 0) {
+ len = FFMIN(AUDIOIO_AUDIO_BLOCK_SIZE - s->buffer_ptr, size);
+ memcpy(s->buffer + s->buffer_ptr, buf, len);
+ s->buffer_ptr += len;
+ if (s->buffer_ptr >= AUDIOIO_AUDIO_BLOCK_SIZE) {
+ for(;;) {
+ ret = write(s->fd, s->buffer, AUDIOIO_AUDIO_BLOCK_SIZE);
+ if (ret > 0)
+ break;
+ if (ret < 0 && (errno != EAGAIN && errno != EINTR))
+ return AVERROR(EIO);
+ }
+ s->buffer_ptr = 0;
+ }
+ buf += len;
+ size -= len;
+ }
+ return 0;
+}
+
+static int audio_write_trailer(AVFormatContext *s1)
+{
+ AudioIOAudioData *s = s1->priv_data;
+
+ ff_audioio_audio_close(s);
+ return 0;
+}
+
+static const AVClass audioio_muxer_class = {
+ .class_name = "Sun audio muxer",
+ .item_name = av_default_item_name,
+ .version = LIBAVUTIL_VERSION_INT,
+ .category = AV_CLASS_CATEGORY_DEVICE_AUDIO_OUTPUT,
+};
+
+AVOutputFormat ff_audioio_muxer = {
+ .name = "audioio",
+ .long_name = NULL_IF_CONFIG_SMALL("Sun audio playback"),
+ .priv_data_size = sizeof(AudioIOAudioData),
+ /* XXX: we make the assumption that the soundcard accepts this format */
+ /* XXX: find better solution with "preinit" method, needed also in
+ other formats */
+ .audio_codec = AV_NE(AV_CODEC_ID_PCM_S16BE, AV_CODEC_ID_PCM_S16LE),
+ .video_codec = AV_CODEC_ID_NONE,
+ .write_header = audio_write_header,
+ .write_packet = audio_write_packet,
+ .write_trailer = audio_write_trailer,
+ .flags = AVFMT_NOFILE,
+ .priv_class = &audioio_muxer_class,
+};