FFmpeg  4.4.6
af_headphone.c
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2017 Paul B Mahol
3  * Copyright (C) 2013-2015 Andreas Fuchs, Wolfgang Hrauda
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #include <math.h>
22 
23 #include "libavutil/avstring.h"
25 #include "libavutil/float_dsp.h"
26 #include "libavutil/intmath.h"
27 #include "libavutil/opt.h"
28 #include "libavcodec/avfft.h"
29 
30 #include "avfilter.h"
31 #include "filters.h"
32 #include "internal.h"
33 #include "audio.h"
34 
35 #define TIME_DOMAIN 0
36 #define FREQUENCY_DOMAIN 1
37 
38 #define HRIR_STEREO 0
39 #define HRIR_MULTI 1
40 
41 typedef struct HeadphoneContext {
42  const AVClass *class;
43 
44  char *map;
45  int type;
46 
48 
50  int eof_hrirs;
51 
52  int ir_len;
53  int air_len;
54 
56 
57  int nb_irs;
58 
59  float gain;
61 
62  float *ringbuffer[2];
63  int write[2];
64 
66  int n_fft;
67  int size;
68  int hrir_fmt;
69 
70  float *data_ir[2];
71  float *temp_src[2];
74 
75  FFTContext *fft[2], *ifft[2];
77 
78  float (*scalarproduct_float)(const float *v1, const float *v2, int len);
79  struct hrir_inputs {
80  int ir_len;
81  int eof;
82  } hrir_in[64];
83  uint64_t mapping[64];
85 
86 static int parse_channel_name(const char *arg, uint64_t *rchannel)
87 {
88  uint64_t layout = av_get_channel_layout(arg);
89 
91  return AVERROR(EINVAL);
92  *rchannel = layout;
93  return 0;
94 }
95 
97 {
98  HeadphoneContext *s = ctx->priv;
99  char *arg, *tokenizer, *p;
100  uint64_t used_channels = 0;
101 
102  p = s->map;
103  while ((arg = av_strtok(p, "|", &tokenizer))) {
104  uint64_t out_channel;
105 
106  p = NULL;
107  if (parse_channel_name(arg, &out_channel)) {
108  av_log(ctx, AV_LOG_WARNING, "Failed to parse \'%s\' as channel name.\n", arg);
109  continue;
110  }
111  if (used_channels & out_channel) {
112  av_log(ctx, AV_LOG_WARNING, "Ignoring duplicate channel '%s'.\n", arg);
113  continue;
114  }
115  used_channels |= out_channel;
116  s->mapping[s->nb_irs] = out_channel;
117  s->nb_irs++;
118  }
119 
120  if (s->hrir_fmt == HRIR_MULTI)
121  s->nb_hrir_inputs = 1;
122  else
123  s->nb_hrir_inputs = s->nb_irs;
124 }
125 
126 typedef struct ThreadData {
127  AVFrame *in, *out;
128  int *write;
129  float **ir;
131  float **ringbuffer;
132  float **temp_src;
135 } ThreadData;
136 
137 static int headphone_convolute(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
138 {
139  HeadphoneContext *s = ctx->priv;
140  ThreadData *td = arg;
141  AVFrame *in = td->in, *out = td->out;
142  int offset = jobnr;
143  int *write = &td->write[jobnr];
144  const float *const ir = td->ir[jobnr];
145  int *n_clippings = &td->n_clippings[jobnr];
146  float *ringbuffer = td->ringbuffer[jobnr];
147  float *temp_src = td->temp_src[jobnr];
148  const int ir_len = s->ir_len;
149  const int air_len = s->air_len;
150  const float *src = (const float *)in->data[0];
151  float *dst = (float *)out->data[0];
152  const int in_channels = in->channels;
153  const int buffer_length = s->buffer_length;
154  const uint32_t modulo = (uint32_t)buffer_length - 1;
155  float *buffer[64];
156  int wr = *write;
157  int read;
158  int i, l;
159 
160  dst += offset;
161  for (l = 0; l < in_channels; l++) {
162  buffer[l] = ringbuffer + l * buffer_length;
163  }
164 
165  for (i = 0; i < in->nb_samples; i++) {
166  const float *cur_ir = ir;
167 
168  *dst = 0;
169  for (l = 0; l < in_channels; l++) {
170  *(buffer[l] + wr) = src[l];
171  }
172 
173  for (l = 0; l < in_channels; cur_ir += air_len, l++) {
174  const float *const bptr = buffer[l];
175 
176  if (l == s->lfe_channel) {
177  *dst += *(buffer[s->lfe_channel] + wr) * s->gain_lfe;
178  continue;
179  }
180 
181  read = (wr - (ir_len - 1)) & modulo;
182 
183  if (read + ir_len < buffer_length) {
184  memcpy(temp_src, bptr + read, ir_len * sizeof(*temp_src));
185  } else {
186  int len = FFMIN(air_len - (read % ir_len), buffer_length - read);
187 
188  memcpy(temp_src, bptr + read, len * sizeof(*temp_src));
189  memcpy(temp_src + len, bptr, (air_len - len) * sizeof(*temp_src));
190  }
191 
192  dst[0] += s->scalarproduct_float(cur_ir, temp_src, FFALIGN(ir_len, 32));
193  }
194 
195  if (fabsf(dst[0]) > 1)
196  n_clippings[0]++;
197 
198  dst += 2;
199  src += in_channels;
200  wr = (wr + 1) & modulo;
201  }
202 
203  *write = wr;
204 
205  return 0;
206 }
207 
208 static int headphone_fast_convolute(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
209 {
210  HeadphoneContext *s = ctx->priv;
211  ThreadData *td = arg;
212  AVFrame *in = td->in, *out = td->out;
213  int offset = jobnr;
214  int *write = &td->write[jobnr];
215  FFTComplex *hrtf = s->data_hrtf[jobnr];
216  int *n_clippings = &td->n_clippings[jobnr];
217  float *ringbuffer = td->ringbuffer[jobnr];
218  const int ir_len = s->ir_len;
219  const float *src = (const float *)in->data[0];
220  float *dst = (float *)out->data[0];
221  const int in_channels = in->channels;
222  const int buffer_length = s->buffer_length;
223  const uint32_t modulo = (uint32_t)buffer_length - 1;
224  FFTComplex *fft_in = s->temp_fft[jobnr];
225  FFTComplex *fft_acc = s->temp_afft[jobnr];
226  FFTContext *ifft = s->ifft[jobnr];
227  FFTContext *fft = s->fft[jobnr];
228  const int n_fft = s->n_fft;
229  const float fft_scale = 1.0f / s->n_fft;
230  FFTComplex *hrtf_offset;
231  int wr = *write;
232  int n_read;
233  int i, j;
234 
235  dst += offset;
236 
237  n_read = FFMIN(ir_len, in->nb_samples);
238  for (j = 0; j < n_read; j++) {
239  dst[2 * j] = ringbuffer[wr];
240  ringbuffer[wr] = 0.0;
241  wr = (wr + 1) & modulo;
242  }
243 
244  for (j = n_read; j < in->nb_samples; j++) {
245  dst[2 * j] = 0;
246  }
247 
248  memset(fft_acc, 0, sizeof(FFTComplex) * n_fft);
249 
250  for (i = 0; i < in_channels; i++) {
251  if (i == s->lfe_channel) {
252  for (j = 0; j < in->nb_samples; j++) {
253  dst[2 * j] += src[i + j * in_channels] * s->gain_lfe;
254  }
255  continue;
256  }
257 
258  offset = i * n_fft;
259  hrtf_offset = hrtf + offset;
260 
261  memset(fft_in, 0, sizeof(FFTComplex) * n_fft);
262 
263  for (j = 0; j < in->nb_samples; j++) {
264  fft_in[j].re = src[j * in_channels + i];
265  }
266 
267  av_fft_permute(fft, fft_in);
268  av_fft_calc(fft, fft_in);
269  for (j = 0; j < n_fft; j++) {
270  const FFTComplex *hcomplex = hrtf_offset + j;
271  const float re = fft_in[j].re;
272  const float im = fft_in[j].im;
273 
274  fft_acc[j].re += re * hcomplex->re - im * hcomplex->im;
275  fft_acc[j].im += re * hcomplex->im + im * hcomplex->re;
276  }
277  }
278 
279  av_fft_permute(ifft, fft_acc);
280  av_fft_calc(ifft, fft_acc);
281 
282  for (j = 0; j < in->nb_samples; j++) {
283  dst[2 * j] += fft_acc[j].re * fft_scale;
284  if (fabsf(dst[2 * j]) > 1)
285  n_clippings[0]++;
286  }
287 
288  for (j = 0; j < ir_len - 1; j++) {
289  int write_pos = (wr + j) & modulo;
290 
291  *(ringbuffer + write_pos) += fft_acc[in->nb_samples + j].re * fft_scale;
292  }
293 
294  *write = wr;
295 
296  return 0;
297 }
298 
299 static int check_ir(AVFilterLink *inlink, int input_number)
300 {
301  AVFilterContext *ctx = inlink->dst;
302  HeadphoneContext *s = ctx->priv;
303  int ir_len, max_ir_len;
304 
305  ir_len = ff_inlink_queued_samples(inlink);
306  max_ir_len = 65536;
307  if (ir_len > max_ir_len) {
308  av_log(ctx, AV_LOG_ERROR, "Too big length of IRs: %d > %d.\n", ir_len, max_ir_len);
309  return AVERROR(EINVAL);
310  }
311  s->hrir_in[input_number].ir_len = ir_len;
312  s->ir_len = FFMAX(ir_len, s->ir_len);
313 
314  return 0;
315 }
316 
318 {
319  AVFilterContext *ctx = outlink->src;
320  int n_clippings[2] = { 0 };
321  ThreadData td;
322  AVFrame *out;
323 
324  out = ff_get_audio_buffer(outlink, in->nb_samples);
325  if (!out) {
326  av_frame_free(&in);
327  return AVERROR(ENOMEM);
328  }
329  out->pts = in->pts;
330 
331  td.in = in; td.out = out; td.write = s->write;
332  td.ir = s->data_ir; td.n_clippings = n_clippings;
333  td.ringbuffer = s->ringbuffer; td.temp_src = s->temp_src;
334  td.temp_fft = s->temp_fft;
335  td.temp_afft = s->temp_afft;
336 
337  if (s->type == TIME_DOMAIN) {
338  ctx->internal->execute(ctx, headphone_convolute, &td, NULL, 2);
339  } else {
340  ctx->internal->execute(ctx, headphone_fast_convolute, &td, NULL, 2);
341  }
342  emms_c();
343 
344  if (n_clippings[0] + n_clippings[1] > 0) {
345  av_log(ctx, AV_LOG_WARNING, "%d of %d samples clipped. Please reduce gain.\n",
346  n_clippings[0] + n_clippings[1], out->nb_samples * 2);
347  }
348 
349  av_frame_free(&in);
350  return ff_filter_frame(outlink, out);
351 }
352 
354 {
355  struct HeadphoneContext *s = ctx->priv;
356  const int ir_len = s->ir_len;
357  int nb_input_channels = ctx->inputs[0]->channels;
358  float gain_lin = expf((s->gain - 3 * nb_input_channels) / 20 * M_LN10);
359  AVFrame *frame;
360  int ret = 0;
361  int n_fft;
362  int i, j, k;
363 
364  s->air_len = 1 << (32 - ff_clz(ir_len));
365  if (s->type == TIME_DOMAIN) {
366  s->air_len = FFALIGN(s->air_len, 32);
367  }
368  s->buffer_length = 1 << (32 - ff_clz(s->air_len));
369  s->n_fft = n_fft = 1 << (32 - ff_clz(ir_len + s->size));
370 
371  if (s->type == FREQUENCY_DOMAIN) {
372  s->fft[0] = av_fft_init(av_log2(s->n_fft), 0);
373  s->fft[1] = av_fft_init(av_log2(s->n_fft), 0);
374  s->ifft[0] = av_fft_init(av_log2(s->n_fft), 1);
375  s->ifft[1] = av_fft_init(av_log2(s->n_fft), 1);
376 
377  if (!s->fft[0] || !s->fft[1] || !s->ifft[0] || !s->ifft[1]) {
378  av_log(ctx, AV_LOG_ERROR, "Unable to create FFT contexts of size %d.\n", s->n_fft);
379  ret = AVERROR(ENOMEM);
380  goto fail;
381  }
382  }
383 
384  if (s->type == TIME_DOMAIN) {
385  s->ringbuffer[0] = av_calloc(s->buffer_length, sizeof(float) * nb_input_channels);
386  s->ringbuffer[1] = av_calloc(s->buffer_length, sizeof(float) * nb_input_channels);
387  } else {
388  s->ringbuffer[0] = av_calloc(s->buffer_length, sizeof(float));
389  s->ringbuffer[1] = av_calloc(s->buffer_length, sizeof(float));
390  s->temp_fft[0] = av_calloc(s->n_fft, sizeof(FFTComplex));
391  s->temp_fft[1] = av_calloc(s->n_fft, sizeof(FFTComplex));
392  s->temp_afft[0] = av_calloc(s->n_fft, sizeof(FFTComplex));
393  s->temp_afft[1] = av_calloc(s->n_fft, sizeof(FFTComplex));
394  if (!s->temp_fft[0] || !s->temp_fft[1] ||
395  !s->temp_afft[0] || !s->temp_afft[1]) {
396  ret = AVERROR(ENOMEM);
397  goto fail;
398  }
399  }
400 
401  if (!s->ringbuffer[0] || !s->ringbuffer[1]) {
402  ret = AVERROR(ENOMEM);
403  goto fail;
404  }
405 
406  if (s->type == TIME_DOMAIN) {
407  s->temp_src[0] = av_calloc(s->air_len, sizeof(float));
408  s->temp_src[1] = av_calloc(s->air_len, sizeof(float));
409 
410  s->data_ir[0] = av_calloc(nb_input_channels * s->air_len, sizeof(*s->data_ir[0]));
411  s->data_ir[1] = av_calloc(nb_input_channels * s->air_len, sizeof(*s->data_ir[1]));
412  if (!s->data_ir[0] || !s->data_ir[1] || !s->temp_src[0] || !s->temp_src[1]) {
413  ret = AVERROR(ENOMEM);
414  goto fail;
415  }
416  } else {
417  s->data_hrtf[0] = av_calloc(n_fft, sizeof(*s->data_hrtf[0]) * nb_input_channels);
418  s->data_hrtf[1] = av_calloc(n_fft, sizeof(*s->data_hrtf[1]) * nb_input_channels);
419  if (!s->data_hrtf[0] || !s->data_hrtf[1]) {
420  ret = AVERROR(ENOMEM);
421  goto fail;
422  }
423  }
424 
425  for (i = 0; i < s->nb_hrir_inputs; av_frame_free(&frame), i++) {
426  int len = s->hrir_in[i].ir_len;
427  float *ptr;
428 
429  ret = ff_inlink_consume_samples(ctx->inputs[i + 1], len, len, &frame);
430  if (ret < 0)
431  goto fail;
432  ptr = (float *)frame->extended_data[0];
433 
434  if (s->hrir_fmt == HRIR_STEREO) {
436  s->mapping[i]);
437  if (idx < 0)
438  continue;
439  if (s->type == TIME_DOMAIN) {
440  float *data_ir_l = s->data_ir[0] + idx * s->air_len;
441  float *data_ir_r = s->data_ir[1] + idx * s->air_len;
442 
443  for (j = 0; j < len; j++) {
444  data_ir_l[j] = ptr[len * 2 - j * 2 - 2] * gain_lin;
445  data_ir_r[j] = ptr[len * 2 - j * 2 - 1] * gain_lin;
446  }
447  } else {
448  FFTComplex *fft_in_l = s->data_hrtf[0] + idx * n_fft;
449  FFTComplex *fft_in_r = s->data_hrtf[1] + idx * n_fft;
450 
451  for (j = 0; j < len; j++) {
452  fft_in_l[j].re = ptr[j * 2 ] * gain_lin;
453  fft_in_r[j].re = ptr[j * 2 + 1] * gain_lin;
454  }
455 
456  av_fft_permute(s->fft[0], fft_in_l);
457  av_fft_calc(s->fft[0], fft_in_l);
458  av_fft_permute(s->fft[0], fft_in_r);
459  av_fft_calc(s->fft[0], fft_in_r);
460  }
461  } else {
462  int I, N = ctx->inputs[1]->channels;
463 
464  for (k = 0; k < N / 2; k++) {
466  s->mapping[k]);
467  if (idx < 0)
468  continue;
469 
470  I = k * 2;
471  if (s->type == TIME_DOMAIN) {
472  float *data_ir_l = s->data_ir[0] + idx * s->air_len;
473  float *data_ir_r = s->data_ir[1] + idx * s->air_len;
474 
475  for (j = 0; j < len; j++) {
476  data_ir_l[j] = ptr[len * N - j * N - N + I ] * gain_lin;
477  data_ir_r[j] = ptr[len * N - j * N - N + I + 1] * gain_lin;
478  }
479  } else {
480  FFTComplex *fft_in_l = s->data_hrtf[0] + idx * n_fft;
481  FFTComplex *fft_in_r = s->data_hrtf[1] + idx * n_fft;
482 
483  for (j = 0; j < len; j++) {
484  fft_in_l[j].re = ptr[j * N + I ] * gain_lin;
485  fft_in_r[j].re = ptr[j * N + I + 1] * gain_lin;
486  }
487 
488  av_fft_permute(s->fft[0], fft_in_l);
489  av_fft_calc(s->fft[0], fft_in_l);
490  av_fft_permute(s->fft[0], fft_in_r);
491  av_fft_calc(s->fft[0], fft_in_r);
492  }
493  }
494  }
495  }
496 
497  s->have_hrirs = 1;
498 
499 fail:
500  return ret;
501 }
502 
504 {
505  HeadphoneContext *s = ctx->priv;
506  AVFilterLink *inlink = ctx->inputs[0];
507  AVFilterLink *outlink = ctx->outputs[0];
508  AVFrame *in = NULL;
509  int i, ret;
510 
512  if (!s->eof_hrirs) {
513  int eof = 1;
514  for (i = 0; i < s->nb_hrir_inputs; i++) {
515  AVFilterLink *input = ctx->inputs[i + 1];
516 
517  if (s->hrir_in[i].eof)
518  continue;
519 
520  if ((ret = check_ir(input, i)) < 0)
521  return ret;
522 
523  if (ff_outlink_get_status(input) == AVERROR_EOF) {
524  if (!ff_inlink_queued_samples(input)) {
525  av_log(ctx, AV_LOG_ERROR, "No samples provided for "
526  "HRIR stream %d.\n", i);
527  return AVERROR_INVALIDDATA;
528  }
529  s->hrir_in[i].eof = 1;
530  } else {
531  if (ff_outlink_frame_wanted(ctx->outputs[0]))
533  eof = 0;
534  }
535  }
536  if (!eof)
537  return 0;
538  s->eof_hrirs = 1;
539 
540  ret = convert_coeffs(ctx, inlink);
541  if (ret < 0)
542  return ret;
543  } else if (!s->have_hrirs)
544  return AVERROR_EOF;
545 
546  if ((ret = ff_inlink_consume_samples(ctx->inputs[0], s->size, s->size, &in)) > 0) {
547  ret = headphone_frame(s, in, outlink);
548  if (ret < 0)
549  return ret;
550  }
551 
552  if (ret < 0)
553  return ret;
554 
555  FF_FILTER_FORWARD_STATUS(ctx->inputs[0], ctx->outputs[0]);
556  if (ff_outlink_frame_wanted(ctx->outputs[0]))
557  ff_inlink_request_frame(ctx->inputs[0]);
558 
559  return 0;
560 }
561 
563 {
564  struct HeadphoneContext *s = ctx->priv;
567  AVFilterChannelLayouts *stereo_layout = NULL;
568  AVFilterChannelLayouts *hrir_layouts = NULL;
569  int ret, i;
570 
572  if (ret)
573  return ret;
575  if (ret)
576  return ret;
577 
579  if (!layouts)
580  return AVERROR(ENOMEM);
581 
582  ret = ff_channel_layouts_ref(layouts, &ctx->inputs[0]->outcfg.channel_layouts);
583  if (ret)
584  return ret;
585 
586  ret = ff_add_channel_layout(&stereo_layout, AV_CH_LAYOUT_STEREO);
587  if (ret)
588  return ret;
589  ret = ff_channel_layouts_ref(stereo_layout, &ctx->outputs[0]->incfg.channel_layouts);
590  if (ret)
591  return ret;
592 
593  if (s->hrir_fmt == HRIR_MULTI) {
594  hrir_layouts = ff_all_channel_counts();
595  if (!hrir_layouts)
596  return AVERROR(ENOMEM);
597  ret = ff_channel_layouts_ref(hrir_layouts, &ctx->inputs[1]->outcfg.channel_layouts);
598  if (ret)
599  return ret;
600  } else {
601  for (i = 1; i <= s->nb_hrir_inputs; i++) {
602  ret = ff_channel_layouts_ref(stereo_layout, &ctx->inputs[i]->outcfg.channel_layouts);
603  if (ret)
604  return ret;
605  }
606  }
607 
609  if (!formats)
610  return AVERROR(ENOMEM);
612 }
613 
614 static int config_input(AVFilterLink *inlink)
615 {
616  AVFilterContext *ctx = inlink->dst;
617  HeadphoneContext *s = ctx->priv;
618 
619  if (s->nb_irs < inlink->channels) {
620  av_log(ctx, AV_LOG_ERROR, "Number of HRIRs must be >= %d.\n", inlink->channels);
621  return AVERROR(EINVAL);
622  }
623 
626  return 0;
627 }
628 
630 {
631  HeadphoneContext *s = ctx->priv;
632  int i, ret;
633 
634  AVFilterPad pad = {
635  .name = "in0",
636  .type = AVMEDIA_TYPE_AUDIO,
637  .config_props = config_input,
638  };
639  if ((ret = ff_insert_inpad(ctx, 0, &pad)) < 0)
640  return ret;
641 
642  if (!s->map) {
643  av_log(ctx, AV_LOG_ERROR, "Valid mapping must be set.\n");
644  return AVERROR(EINVAL);
645  }
646 
647  parse_map(ctx);
648 
649  for (i = 0; i < s->nb_hrir_inputs; i++) {
650  char *name = av_asprintf("hrir%d", i);
651  AVFilterPad pad = {
652  .name = name,
653  .type = AVMEDIA_TYPE_AUDIO,
654  };
655  if (!name)
656  return AVERROR(ENOMEM);
657  if ((ret = ff_insert_inpad(ctx, i + 1, &pad)) < 0) {
658  av_freep(&pad.name);
659  return ret;
660  }
661  }
662 
663  if (s->type == TIME_DOMAIN) {
665  if (!fdsp)
666  return AVERROR(ENOMEM);
667  s->scalarproduct_float = fdsp->scalarproduct_float;
668  av_free(fdsp);
669  }
670 
671  return 0;
672 }
673 
674 static int config_output(AVFilterLink *outlink)
675 {
676  AVFilterContext *ctx = outlink->src;
677  HeadphoneContext *s = ctx->priv;
678  AVFilterLink *inlink = ctx->inputs[0];
679 
680  if (s->hrir_fmt == HRIR_MULTI) {
681  AVFilterLink *hrir_link = ctx->inputs[1];
682 
683  if (hrir_link->channels < inlink->channels * 2) {
684  av_log(ctx, AV_LOG_ERROR, "Number of channels in HRIR stream must be >= %d.\n", inlink->channels * 2);
685  return AVERROR(EINVAL);
686  }
687  }
688 
689  s->gain_lfe = expf((s->gain - 3 * inlink->channels + s->lfe_gain) / 20 * M_LN10);
690 
691  return 0;
692 }
693 
695 {
696  HeadphoneContext *s = ctx->priv;
697 
698  av_fft_end(s->ifft[0]);
699  av_fft_end(s->ifft[1]);
700  av_fft_end(s->fft[0]);
701  av_fft_end(s->fft[1]);
702  av_freep(&s->data_ir[0]);
703  av_freep(&s->data_ir[1]);
704  av_freep(&s->ringbuffer[0]);
705  av_freep(&s->ringbuffer[1]);
706  av_freep(&s->temp_src[0]);
707  av_freep(&s->temp_src[1]);
708  av_freep(&s->temp_fft[0]);
709  av_freep(&s->temp_fft[1]);
710  av_freep(&s->temp_afft[0]);
711  av_freep(&s->temp_afft[1]);
712  av_freep(&s->data_hrtf[0]);
713  av_freep(&s->data_hrtf[1]);
714 
715  for (unsigned i = 1; i < ctx->nb_inputs; i++)
716  av_freep(&ctx->input_pads[i].name);
717 }
718 
719 #define OFFSET(x) offsetof(HeadphoneContext, x)
720 #define FLAGS AV_OPT_FLAG_AUDIO_PARAM|AV_OPT_FLAG_FILTERING_PARAM
721 
722 static const AVOption headphone_options[] = {
723  { "map", "set channels convolution mappings", OFFSET(map), AV_OPT_TYPE_STRING, {.str=NULL}, .flags = FLAGS },
724  { "gain", "set gain in dB", OFFSET(gain), AV_OPT_TYPE_FLOAT, {.dbl=0}, -20, 40, .flags = FLAGS },
725  { "lfe", "set lfe gain in dB", OFFSET(lfe_gain), AV_OPT_TYPE_FLOAT, {.dbl=0}, -20, 40, .flags = FLAGS },
726  { "type", "set processing", OFFSET(type), AV_OPT_TYPE_INT, {.i64=1}, 0, 1, .flags = FLAGS, "type" },
727  { "time", "time domain", 0, AV_OPT_TYPE_CONST, {.i64=0}, 0, 0, .flags = FLAGS, "type" },
728  { "freq", "frequency domain", 0, AV_OPT_TYPE_CONST, {.i64=1}, 0, 0, .flags = FLAGS, "type" },
729  { "size", "set frame size", OFFSET(size), AV_OPT_TYPE_INT, {.i64=1024},1024,96000, .flags = FLAGS },
730  { "hrir", "set hrir format", OFFSET(hrir_fmt), AV_OPT_TYPE_INT, {.i64=HRIR_STEREO}, 0, 1, .flags = FLAGS, "hrir" },
731  { "stereo", "hrir files have exactly 2 channels", 0, AV_OPT_TYPE_CONST, {.i64=HRIR_STEREO}, 0, 0, .flags = FLAGS, "hrir" },
732  { "multich", "single multichannel hrir file", 0, AV_OPT_TYPE_CONST, {.i64=HRIR_MULTI}, 0, 0, .flags = FLAGS, "hrir" },
733  { NULL }
734 };
735 
737 
738 static const AVFilterPad outputs[] = {
739  {
740  .name = "default",
741  .type = AVMEDIA_TYPE_AUDIO,
742  .config_props = config_output,
743  },
744  { NULL }
745 };
746 
748  .name = "headphone",
749  .description = NULL_IF_CONFIG_SMALL("Apply headphone binaural spatialization with HRTFs in additional streams."),
750  .priv_size = sizeof(HeadphoneContext),
751  .priv_class = &headphone_class,
752  .init = init,
753  .uninit = uninit,
755  .activate = activate,
756  .inputs = NULL,
757  .outputs = outputs,
759 };
static const AVFilterPad inputs[]
Definition: af_acontrast.c:193
#define HRIR_MULTI
Definition: af_headphone.c:39
static int convert_coeffs(AVFilterContext *ctx, AVFilterLink *inlink)
Definition: af_headphone.c:353
static int headphone_convolute(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
Definition: af_headphone.c:137
static int headphone_frame(HeadphoneContext *s, AVFrame *in, AVFilterLink *outlink)
Definition: af_headphone.c:317
static int query_formats(AVFilterContext *ctx)
Definition: af_headphone.c:562
static int config_input(AVFilterLink *inlink)
Definition: af_headphone.c:614
static int parse_channel_name(const char *arg, uint64_t *rchannel)
Definition: af_headphone.c:86
#define FLAGS
Definition: af_headphone.c:720
#define TIME_DOMAIN
Definition: af_headphone.c:35
static const AVFilterPad outputs[]
Definition: af_headphone.c:738
static const AVOption headphone_options[]
Definition: af_headphone.c:722
static int check_ir(AVFilterLink *inlink, int input_number)
Definition: af_headphone.c:299
static int activate(AVFilterContext *ctx)
Definition: af_headphone.c:503
static av_cold int init(AVFilterContext *ctx)
Definition: af_headphone.c:629
static av_cold void uninit(AVFilterContext *ctx)
Definition: af_headphone.c:694
AVFilter ff_af_headphone
Definition: af_headphone.c:747
static void parse_map(AVFilterContext *ctx)
Definition: af_headphone.c:96
#define FREQUENCY_DOMAIN
Definition: af_headphone.c:36
#define OFFSET(x)
Definition: af_headphone.c:719
static int config_output(AVFilterLink *outlink)
Definition: af_headphone.c:674
#define HRIR_STEREO
Definition: af_headphone.c:38
AVFILTER_DEFINE_CLASS(headphone)
static int headphone_fast_convolute(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
Definition: af_headphone.c:208
#define N
Definition: af_mcompand.c:54
#define av_cold
Definition: attributes.h:88
AVFrame * ff_get_audio_buffer(AVFilterLink *link, int nb_samples)
Request an audio samples buffer with a specific set of permissions.
Definition: audio.c:86
uint8_t pi<< 24) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_U8, uint8_t,(*(const uint8_t *) pi - 0x80) *(1.0f/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_U8, uint8_t,(*(const uint8_t *) pi - 0x80) *(1.0/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S16, int16_t,(*(const int16_t *) pi >> 8)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S16, int16_t, *(const int16_t *) pi *(1.0f/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S16, int16_t, *(const int16_t *) pi *(1.0/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S32, int32_t,(*(const int32_t *) pi >> 24)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S32, int32_t, *(const int32_t *) pi *(1.0f/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S32, int32_t, *(const int32_t *) pi *(1.0/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_FLT, float, av_clip_uint8(lrintf(*(const float *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_FLT, float, av_clip_int16(lrintf(*(const float *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_FLT, float, av_clipl_int32(llrintf(*(const float *) pi *(1U<< 31)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_DBL, double, av_clip_uint8(lrint(*(const double *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_DBL, double, av_clip_int16(lrint(*(const double *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_DBL, double, av_clipl_int32(llrint(*(const double *) pi *(1U<< 31)))) #define SET_CONV_FUNC_GROUP(ofmt, ifmt) static void set_generic_function(AudioConvert *ac) { } void ff_audio_convert_free(AudioConvert **ac) { if(! *ac) return;ff_dither_free(&(*ac) ->dc);av_freep(ac);} AudioConvert *ff_audio_convert_alloc(AVAudioResampleContext *avr, enum AVSampleFormat out_fmt, enum AVSampleFormat in_fmt, int channels, int sample_rate, int apply_map) { AudioConvert *ac;int in_planar, out_planar;ac=av_mallocz(sizeof(*ac));if(!ac) return NULL;ac->avr=avr;ac->out_fmt=out_fmt;ac->in_fmt=in_fmt;ac->channels=channels;ac->apply_map=apply_map;if(avr->dither_method !=AV_RESAMPLE_DITHER_NONE &&av_get_packed_sample_fmt(out_fmt)==AV_SAMPLE_FMT_S16 &&av_get_bytes_per_sample(in_fmt) > 2) { ac->dc=ff_dither_alloc(avr, out_fmt, in_fmt, channels, sample_rate, apply_map);if(!ac->dc) { av_free(ac);return NULL;} return ac;} in_planar=ff_sample_fmt_is_planar(in_fmt, channels);out_planar=ff_sample_fmt_is_planar(out_fmt, channels);if(in_planar==out_planar) { ac->func_type=CONV_FUNC_TYPE_FLAT;ac->planes=in_planar ? ac->channels :1;} else if(in_planar) ac->func_type=CONV_FUNC_TYPE_INTERLEAVE;else ac->func_type=CONV_FUNC_TYPE_DEINTERLEAVE;set_generic_function(ac);if(ARCH_AARCH64) ff_audio_convert_init_aarch64(ac);if(ARCH_ARM) ff_audio_convert_init_arm(ac);if(ARCH_X86) ff_audio_convert_init_x86(ac);return ac;} int ff_audio_convert(AudioConvert *ac, AudioData *out, AudioData *in) { int use_generic=1;int len=in->nb_samples;int p;if(ac->dc) { av_log(ac->avr, AV_LOG_TRACE, "%d samples - audio_convert: %s to %s (dithered)\n", len, av_get_sample_fmt_name(ac->in_fmt), av_get_sample_fmt_name(ac->out_fmt));return ff_convert_dither(ac-> in
FFT functions.
int ff_outlink_get_status(AVFilterLink *link)
Get the status on an output link.
Definition: avfilter.c:1643
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1096
int ff_inlink_consume_samples(AVFilterLink *link, unsigned min, unsigned max, AVFrame **rframe)
Take samples from the link's FIFO and update the link's stats.
Definition: avfilter.c:1513
int ff_inlink_queued_samples(AVFilterLink *link)
Definition: avfilter.c:1474
void ff_inlink_request_frame(AVFilterLink *link)
Mark that a frame is wanted on the link.
Definition: avfilter.c:1620
Main libavfilter public API header.
char * av_asprintf(const char *fmt,...)
Definition: avstring.c:113
#define flags(name, subs,...)
Definition: cbs_av1.c:572
#define s(width, name)
Definition: cbs_vp9.c:257
uint64_t layout
audio channel layout utility functions
#define fail()
Definition: checkasm.h:133
#define FFMIN(a, b)
Definition: common.h:105
#define FFMAX(a, b)
Definition: common.h:103
#define NULL
Definition: coverity.c:32
static __device__ float fabsf(float a)
Definition: cuda_runtime.h:181
static AVFrame * frame
float im
Definition: fft.c:82
float re
Definition: fft.c:82
#define FF_FILTER_FORWARD_STATUS(inlink, outlink)
Acknowledge the status on an input link and forward it to an output link.
Definition: filters.h:226
#define FF_FILTER_FORWARD_STATUS_BACK_ALL(outlink, filter)
Forward the status on an output link to all input links.
Definition: filters.h:212
static int ff_outlink_frame_wanted(AVFilterLink *link)
Test if a frame is wanted on an output link.
Definition: filters.h:172
AVFilterChannelLayouts * ff_all_channel_counts(void)
Construct an AVFilterChannelLayouts coding for any channel layout, with known or unknown disposition.
Definition: formats.c:436
int ff_add_channel_layout(AVFilterChannelLayouts **l, uint64_t channel_layout)
Definition: formats.c:338
int ff_set_common_formats(AVFilterContext *ctx, AVFilterFormats *formats)
A helper for query_formats() which sets all links to the same list of formats.
Definition: formats.c:587
int ff_add_format(AVFilterFormats **avff, int64_t fmt)
Add fmt to the list of media formats contained in *avff.
Definition: formats.c:332
AVFilterChannelLayouts * ff_all_channel_layouts(void)
Construct an empty AVFilterChannelLayouts/AVFilterFormats struct – representing any channel layout (w...
Definition: formats.c:427
int ff_set_common_samplerates(AVFilterContext *ctx, AVFilterFormats *samplerates)
Definition: formats.c:575
int ff_channel_layouts_ref(AVFilterChannelLayouts *f, AVFilterChannelLayouts **ref)
Add *ref as a new reference to f.
Definition: formats.c:461
AVFilterFormats * ff_all_samplerates(void)
Definition: formats.c:421
@ AV_OPT_TYPE_CONST
Definition: opt.h:234
@ AV_OPT_TYPE_INT
Definition: opt.h:225
@ AV_OPT_TYPE_FLOAT
Definition: opt.h:228
@ AV_OPT_TYPE_STRING
Definition: opt.h:229
int av_get_channel_layout_channel_index(uint64_t channel_layout, uint64_t channel)
Get the index of a channel in channel_layout.
uint64_t av_get_channel_layout(const char *name)
Return a channel layout id that matches name, or 0 if no match is found.
#define AV_CH_LAYOUT_STEREO
int av_get_channel_layout_nb_channels(uint64_t channel_layout)
Return the number of channels in the channel layout.
#define AV_CH_LOW_FREQUENCY
void av_fft_permute(FFTContext *s, FFTComplex *z)
Do the permutation needed BEFORE calling ff_fft_calc().
Definition: avfft.c:38
void av_fft_calc(FFTContext *s, FFTComplex *z)
Do a complex FFT with the parameters defined in av_fft_init().
Definition: avfft.c:43
FFTContext * av_fft_init(int nbits, int inverse)
Set up a complex FFT.
Definition: avfft.c:28
av_cold void av_fft_end(FFTContext *s)
Definition: avfft.c:48
#define AVFILTER_FLAG_SLICE_THREADS
The filter supports multithreading by splitting frames into multiple parts and processing them concur...
Definition: avfilter.h:117
#define AVFILTER_FLAG_DYNAMIC_INPUTS
The number of the filter inputs is not determined just by AVFilter.inputs.
Definition: avfilter.h:106
#define AVERROR_INVALIDDATA
Invalid data found when processing input.
Definition: error.h:59
#define AVERROR_EOF
End of file.
Definition: error.h:55
#define AVERROR(e)
Definition: error.h:43
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:203
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:200
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:194
#define ff_clz
Definition: intmath.h:142
void * av_calloc(size_t nmemb, size_t size)
Non-inlined equivalent of av_mallocz_array().
Definition: mem.c:245
@ AVMEDIA_TYPE_AUDIO
Definition: avutil.h:202
@ AV_SAMPLE_FMT_FLT
float
Definition: samplefmt.h:63
char * av_strtok(char *s, const char *delim, char **saveptr)
Split the string into several tokens which can be accessed by successive calls to av_strtok().
Definition: avstring.c:186
cl_device_type type
const VDPAUPixFmtMap * map
int i
Definition: input.c:407
#define av_log2
Definition: intmath.h:83
const char * arg
Definition: jacosubdec.c:66
static int ff_insert_inpad(AVFilterContext *f, unsigned index, AVFilterPad *p)
Insert a new input pad for the filter.
Definition: internal.h:240
av_cold AVFloatDSPContext * avpriv_float_dsp_alloc(int bit_exact)
Allocate a float DSP context.
Definition: float_dsp.c:135
common internal API header
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification.
Definition: internal.h:117
#define emms_c()
Definition: internal.h:54
#define expf(x)
Definition: libm.h:283
#define FFALIGN(x, a)
Definition: macros.h:48
#define M_LN10
Definition: mathematics.h:43
enum MovChannelLayoutTag * layouts
Definition: mov_chan.c:434
AVOptions.
const char * name
Definition: qsvenc.c:46
#define td
Definition: regdef.h:70
static char buffer[20]
Definition: seek.c:32
formats
Definition: signature.h:48
Describe the class of an AVClass context structure.
Definition: log.h:67
A list of supported channel layouts.
Definition: formats.h:86
An instance of a filter.
Definition: avfilter.h:341
A list of supported formats for one end of a filter link.
Definition: formats.h:65
A filter pad used for either input or output.
Definition: internal.h:54
const char * name
Pad name.
Definition: internal.h:60
Filter definition.
Definition: avfilter.h:145
const char * name
Filter name.
Definition: avfilter.h:149
float(* scalarproduct_float)(const float *v1, const float *v2, int len)
Calculate the scalar product of two vectors of floats.
Definition: float_dsp.h:175
AVFormatInternal * internal
An opaque field for libavformat internal usage.
Definition: avformat.h:1699
This structure describes decoded (raw) audio or video data.
Definition: frame.h:318
uint8_t ** extended_data
pointers to the data planes/channels.
Definition: frame.h:365
AVOption.
Definition: opt.h:248
FFTSample re
Definition: avfft.h:38
FFTSample im
Definition: avfft.h:38
Definition: fft.h:83
float * ringbuffer[2]
Definition: af_headphone.c:62
FFTComplex * temp_fft[2]
Definition: af_headphone.c:72
FFTContext * fft[2]
Definition: af_headphone.c:75
FFTComplex * data_hrtf[2]
Definition: af_headphone.c:76
struct HeadphoneContext::hrir_inputs hrir_in[64]
FFTComplex * temp_afft[2]
Definition: af_headphone.c:73
FFTContext * ifft[2]
Definition: af_headphone.c:75
float(* scalarproduct_float)(const float *v1, const float *v2, int len)
Definition: af_headphone.c:78
float * temp_src[2]
Definition: af_headphone.c:71
float * data_ir[2]
Definition: af_headphone.c:70
uint64_t mapping[64]
Definition: af_headphone.c:83
Used for passing data between threads.
Definition: dsddec.c:67
FFTComplex ** temp_fft
Definition: af_headphone.c:133
int * n_clippings
Definition: af_headphone.c:130
AVFrame * out
Definition: af_adeclick.c:502
float ** ir
Definition: af_headphone.c:129
AVFrame * in
Definition: af_adenorm.c:223
float ** temp_src
Definition: af_headphone.c:132
FFTComplex ** temp_afft
Definition: af_headphone.c:134
float ** ringbuffer
Definition: af_headphone.c:131
#define av_free(p)
#define av_freep(p)
#define av_log(a,...)
#define src
Definition: vp8dsp.c:255
FILE * out
Definition: movenc.c:54
AVFormatContext * ctx
Definition: movenc.c:48
int size
if(ret< 0)
Definition: vf_mcdeint.c:282
static const uint8_t offset[127][2]
Definition: vf_spp.c:107
int len