forked from asterisk/asterisk
-
Notifications
You must be signed in to change notification settings - Fork 0
/
bridge_softmix.c
2894 lines (2519 loc) · 97.1 KB
/
bridge_softmix.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*
* Asterisk -- An open source telephony toolkit.
*
* Copyright (C) 2011, Digium, Inc.
*
* Joshua Colp <[email protected]>
* David Vossel <[email protected]>
*
* See http://www.asterisk.org for more information about
* the Asterisk project. Please do not directly contact
* any of the maintainers of this project for assistance;
* the project provides a web site, mailing lists and IRC
* channels for your use.
*
* This program is free software, distributed under the terms of
* the GNU General Public License Version 2. See the LICENSE file
* at the top of the source tree.
*/
/*! \file
*
* \brief Multi-party software based channel mixing
*
* \author Joshua Colp <[email protected]>
* \author David Vossel <[email protected]>
*
* \ingroup bridges
*/
/*** MODULEINFO
<support_level>core</support_level>
***/
#include "asterisk.h"
#include <math.h>
#include "asterisk/stream.h"
#include "asterisk/test.h"
#include "asterisk/vector.h"
#include "asterisk/message.h"
#include "bridge_softmix/include/bridge_softmix_internal.h"
/*! The minimum sample rate of the bridge. */
#define SOFTMIX_MIN_SAMPLE_RATE 8000 /* 8 kHz sample rate */
/*! \brief Interval at which mixing will take place. Valid options are 10, 20, and 40. */
#define DEFAULT_SOFTMIX_INTERVAL 20
/*! \brief Size of the buffer used for sample manipulation */
#define SOFTMIX_DATALEN(rate, interval) ((rate/50) * (interval / 10))
/*! \brief Number of samples we are dealing with */
#define SOFTMIX_SAMPLES(rate, interval) (SOFTMIX_DATALEN(rate, interval) / 2)
/*! \brief Number of mixing iterations to perform between gathering statistics. */
#define SOFTMIX_STAT_INTERVAL 100
/*!
* \brief Default time in ms of silence necessary to declare talking stopped by the bridge.
*
* \details
* This is the time at which a channel's own audio will stop getting
* mixed out of its own write audio stream because it is no longer talking.
*/
#define DEFAULT_SOFTMIX_SILENCE_THRESHOLD 2500
/*! Default minimum average magnitude threshold to determine talking by the DSP. */
#define DEFAULT_SOFTMIX_TALKING_THRESHOLD 160
#define SOFTBRIDGE_VIDEO_DEST_PREFIX "softbridge_dest"
#define SOFTBRIDGE_VIDEO_DEST_LEN strlen(SOFTBRIDGE_VIDEO_DEST_PREFIX)
#define SOFTBRIDGE_VIDEO_DEST_SEPARATOR '_'
struct softmix_remb_collector {
/*! The frame which will be given to each source stream */
struct ast_frame frame;
/*! The REMB to send to the source which is collecting REMB reports */
struct ast_rtp_rtcp_feedback feedback;
/*! The maximum bitrate (A single precision floating point is big enough) */
float bitrate;
};
struct softmix_stats {
/*! Each index represents a sample rate used above the internal rate. */
unsigned int sample_rates[16];
/*! Each index represents the number of channels using the same index in the sample_rates array. */
unsigned int num_channels[16];
/*! The number of channels above the internal sample rate */
unsigned int num_above_internal_rate;
/*! The number of channels above the maximum sample rate */
unsigned int num_above_maximum_rate;
/*! The number of channels at the internal sample rate */
unsigned int num_at_internal_rate;
/*! The absolute highest sample rate preferred by any channel in the bridge */
unsigned int highest_supported_rate;
/*! Is the sample rate locked by the bridge, if so what is that rate.*/
unsigned int locked_rate;
/*! The maximum sample rate the bridge may use */
unsigned int maximum_rate;
};
struct softmix_translate_helper_entry {
int num_times_requested; /*!< Once this entry is no longer requested, free the trans_pvt
and re-init if it was usable. */
struct ast_format *dst_format; /*!< The destination format for this helper */
struct ast_trans_pvt *trans_pvt; /*!< the translator for this slot. */
struct ast_frame *out_frame; /*!< The output frame from the last translation */
AST_LIST_ENTRY(softmix_translate_helper_entry) entry;
};
struct softmix_translate_helper {
struct ast_format *slin_src; /*!< the source format expected for all the translators */
AST_LIST_HEAD_NOLOCK(, softmix_translate_helper_entry) entries;
};
static struct softmix_translate_helper_entry *softmix_translate_helper_entry_alloc(struct ast_format *dst)
{
struct softmix_translate_helper_entry *entry;
if (!(entry = ast_calloc(1, sizeof(*entry)))) {
return NULL;
}
entry->dst_format = ao2_bump(dst);
/* initialize this to one so that the first time through the cleanup code after
allocation it won't be removed from the entry list */
entry->num_times_requested = 1;
return entry;
}
static void *softmix_translate_helper_free_entry(struct softmix_translate_helper_entry *entry)
{
ao2_cleanup(entry->dst_format);
if (entry->trans_pvt) {
ast_translator_free_path(entry->trans_pvt);
}
if (entry->out_frame) {
ast_frfree(entry->out_frame);
}
ast_free(entry);
return NULL;
}
static void softmix_translate_helper_init(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
{
memset(trans_helper, 0, sizeof(*trans_helper));
trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
}
static void softmix_translate_helper_destroy(struct softmix_translate_helper *trans_helper)
{
struct softmix_translate_helper_entry *entry;
while ((entry = AST_LIST_REMOVE_HEAD(&trans_helper->entries, entry))) {
softmix_translate_helper_free_entry(entry);
}
}
static void softmix_translate_helper_change_rate(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
{
struct softmix_translate_helper_entry *entry;
trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
if (entry->trans_pvt) {
ast_translator_free_path(entry->trans_pvt);
if (!(entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src))) {
AST_LIST_REMOVE_CURRENT(entry);
entry = softmix_translate_helper_free_entry(entry);
}
}
}
AST_LIST_TRAVERSE_SAFE_END;
}
/*!
* \internal
* \brief Get the next available audio on the softmix channel's read stream
* and determine if it should be mixed out or not on the write stream.
*
* \retval pointer to buffer containing the exact number of samples requested on success.
* \retval NULL if no samples are present
*/
static int16_t *softmix_process_read_audio(struct softmix_channel *sc, unsigned int num_samples)
{
if ((ast_slinfactory_available(&sc->factory) >= num_samples) &&
ast_slinfactory_read(&sc->factory, sc->our_buf, num_samples)) {
sc->have_audio = 1;
return sc->our_buf;
}
sc->have_audio = 0;
return NULL;
}
/*!
* \internal
* \brief Process a softmix channel's write audio
*
* \details This function will remove the channel's talking from its own audio if present and
* possibly even do the channel's write translation for it depending on how many other
* channels use the same write format.
*/
static void softmix_process_write_audio(struct softmix_translate_helper *trans_helper,
struct ast_format *raw_write_fmt,
struct softmix_channel *sc, unsigned int default_sample_size)
{
struct softmix_translate_helper_entry *entry = NULL;
int i;
/* If we provided any audio then take it out while in slinear format. */
if (sc->have_audio && !sc->binaural) {
for (i = 0; i < sc->write_frame.samples; i++) {
ast_slinear_saturated_subtract(&sc->final_buf[i], &sc->our_buf[i]);
}
/* check to see if any entries exist for the format. if not we'll want
to remove it during cleanup */
AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
++entry->num_times_requested;
break;
}
}
/* do not do any special write translate optimization if we had to make
* a special mix for them to remove their own audio. */
return;
} else if (sc->have_audio && sc->binaural > 0) {
/*
* Binaural audio requires special saturated substract since we have two
* audio signals per channel now.
*/
softmix_process_write_binaural_audio(sc, default_sample_size);
return;
}
/* Attempt to optimize channels using the same translation path/codec. Build a list of entries
of translation paths and track the number of references for each type. Each one of the same
type should be able to use the same out_frame. Since the optimization is only necessary for
multiple channels (>=2) using the same codec make sure resources are allocated only when
needed and released when not (see also softmix_translate_helper_cleanup */
AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
if (sc->binaural != 0) {
continue;
}
if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
entry->num_times_requested++;
} else {
continue;
}
if (!entry->trans_pvt && (entry->num_times_requested > 1)) {
entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src);
}
if (entry->trans_pvt && !entry->out_frame) {
entry->out_frame = ast_translate(entry->trans_pvt, &sc->write_frame, 0);
}
if (entry->out_frame && entry->out_frame->frametype == AST_FRAME_VOICE
&& entry->out_frame->datalen < MAX_DATALEN) {
ao2_replace(sc->write_frame.subclass.format, entry->out_frame->subclass.format);
memcpy(sc->final_buf, entry->out_frame->data.ptr, entry->out_frame->datalen);
sc->write_frame.datalen = entry->out_frame->datalen;
sc->write_frame.samples = entry->out_frame->samples;
}
break;
}
/* add new entry into list if this format destination was not matched. */
if (!entry && (entry = softmix_translate_helper_entry_alloc(raw_write_fmt))) {
AST_LIST_INSERT_HEAD(&trans_helper->entries, entry, entry);
}
}
static void softmix_translate_helper_cleanup(struct softmix_translate_helper *trans_helper)
{
struct softmix_translate_helper_entry *entry;
AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
/* if it hasn't been requested then remove it */
if (!entry->num_times_requested) {
AST_LIST_REMOVE_CURRENT(entry);
softmix_translate_helper_free_entry(entry);
continue;
}
if (entry->out_frame) {
ast_frfree(entry->out_frame);
entry->out_frame = NULL;
}
/* nothing is optimized for a single path reference, so there is
no reason to continue to hold onto the codec */
if (entry->num_times_requested == 1 && entry->trans_pvt) {
ast_translator_free_path(entry->trans_pvt);
entry->trans_pvt = NULL;
}
/* for each iteration (a mixing run) in the bridge softmix thread the number
of references to a given entry is recalculated, so reset the number of
times requested */
entry->num_times_requested = 0;
}
AST_LIST_TRAVERSE_SAFE_END;
}
static void set_softmix_bridge_data(int rate, int interval, struct ast_bridge_channel *bridge_channel, int reset, int set_binaural, int binaural_pos_id, int is_announcement)
{
struct softmix_channel *sc = bridge_channel->tech_pvt;
struct ast_format *slin_format;
int setup_fail;
#ifdef BINAURAL_RENDERING
if (interval != BINAURAL_MIXING_INTERVAL) {
interval = BINAURAL_MIXING_INTERVAL;
}
#endif
/* The callers have already ensured that sc is never NULL. */
ast_assert(sc != NULL);
slin_format = ast_format_cache_get_slin_by_rate(rate);
ast_mutex_lock(&sc->lock);
if (reset) {
ast_slinfactory_destroy(&sc->factory);
ast_dsp_free(sc->dsp);
}
/* Setup write frame parameters */
sc->write_frame.frametype = AST_FRAME_VOICE;
/*
* NOTE: The write_frame format holds a reference because translation
* could be needed and the format changed to the translated format
* for the channel. The translated format may not be a
* static cached format.
*/
ao2_replace(sc->write_frame.subclass.format, slin_format);
sc->write_frame.data.ptr = sc->final_buf;
sc->write_frame.datalen = SOFTMIX_DATALEN(rate, interval);
sc->write_frame.samples = SOFTMIX_SAMPLES(rate, interval);
/* We will store the rate here cause we need to set the data again when a channel is unsuspended */
sc->rate = rate;
/* If the channel will contain binaural data we will set a identifier in the channel
* if set_binaural == -1 this is just a sample rate update, will ignore it. */
if (set_binaural == 1) {
sc->binaural = 1;
} else if (set_binaural == 0) {
sc->binaural = 0;
}
/* Setting the binaural position. This doesn't require a change of the overlaying channel infos
* and doesn't have to be done if we just updating sample rates. */
if (binaural_pos_id != -1) {
sc->binaural_pos = binaural_pos_id;
}
if (is_announcement != -1) {
sc->is_announcement = is_announcement;
}
/*
* NOTE: The read_slin_format does not hold a reference because it
* will always be a signed linear format.
*/
sc->read_slin_format = slin_format;
/* Setup smoother */
setup_fail = ast_slinfactory_init_with_format(&sc->factory, slin_format);
/* set new read and write formats on channel. */
ast_channel_lock(bridge_channel->chan);
setup_fail |= ast_set_read_format_path(bridge_channel->chan,
ast_channel_rawreadformat(bridge_channel->chan), slin_format);
ast_channel_unlock(bridge_channel->chan);
/* If channel contains binaural data we will set it here for the trans_pvt. */
if (set_binaural == 1 || (set_binaural == -1 && sc->binaural == 1)) {
setup_fail |= ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
} else if (set_binaural == 0) {
setup_fail |= ast_set_write_format(bridge_channel->chan, slin_format);
}
/* set up new DSP. This is on the read side only right before the read frame enters the smoother. */
sc->dsp = ast_dsp_new_with_rate(rate);
if (setup_fail || !sc->dsp) {
/* Bad news. Could not setup the channel for softmix. */
ast_mutex_unlock(&sc->lock);
ast_bridge_channel_leave_bridge(bridge_channel, BRIDGE_CHANNEL_STATE_END, 0);
return;
}
/* we want to aggressively detect silence to avoid feedback */
if (bridge_channel->tech_args.talking_threshold) {
ast_dsp_set_threshold(sc->dsp, bridge_channel->tech_args.talking_threshold);
} else {
ast_dsp_set_threshold(sc->dsp, DEFAULT_SOFTMIX_TALKING_THRESHOLD);
}
ast_mutex_unlock(&sc->lock);
}
/*!
* \internal
* \brief Poke the mixing thread in case it is waiting for an active channel.
* \since 12.0.0
*
* \param softmix_data Bridge mixing data.
*/
static void softmix_poke_thread(struct softmix_bridge_data *softmix_data)
{
ast_mutex_lock(&softmix_data->lock);
ast_cond_signal(&softmix_data->cond);
ast_mutex_unlock(&softmix_data->lock);
}
/*! \brief Function called when a channel is unsuspended from the bridge */
static void softmix_bridge_unsuspend(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
{
#ifdef BINAURAL_RENDERING
struct softmix_channel *sc = bridge_channel->tech_pvt;
if (sc->binaural) {
/* Restore some usefull data if it was a binaural channel */
struct ast_format *slin_format;
slin_format = ast_format_cache_get_slin_by_rate(sc->rate);
ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
}
#endif
if (bridge->tech_pvt) {
softmix_poke_thread(bridge->tech_pvt);
}
}
/*!
* \brief Determine if a stream is a video source stream.
*
* \param stream The stream to test
* \retval 1 The stream is a video source
* \retval 0 The stream is not a video source
*/
static int is_video_source(const struct ast_stream *stream)
{
if (ast_stream_get_state(stream) != AST_STREAM_STATE_REMOVED
&& ast_stream_get_type(stream) == AST_MEDIA_TYPE_VIDEO
&& strncmp(ast_stream_get_name(stream), SOFTBRIDGE_VIDEO_DEST_PREFIX,
SOFTBRIDGE_VIDEO_DEST_LEN)) {
return 1;
}
return 0;
}
/*!
* \brief Determine if a stream is a video destination stream.
*
* A source channel name can be provided to narrow this to a destination stream
* for a particular source channel. Further, a source stream name can be provided
* to narrow this to a particular source stream's destination. However, empty strings
* can be provided to match any destination video stream, regardless of source channel
* or source stream.
*
* \param stream The stream to test
* \param source_channel_name The name of a source video channel to match
* \param source_channel_stream_position The position of the video on the source channel
* \retval 1 The stream is a video destination stream
* \retval 0 The stream is not a video destination stream
*/
static int is_video_dest(const struct ast_stream *stream, const char *source_channel_name,
int source_channel_stream_position)
{
char *dest_video_name;
size_t dest_video_name_len;
if (ast_stream_get_state(stream) == AST_STREAM_STATE_REMOVED
|| ast_stream_get_type(stream) != AST_MEDIA_TYPE_VIDEO) {
return 0;
}
dest_video_name_len = SOFTBRIDGE_VIDEO_DEST_LEN + 1;
if (!ast_strlen_zero(source_channel_name)) {
dest_video_name_len += strlen(source_channel_name) + 1;
if (source_channel_stream_position != -1) {
dest_video_name_len += 11;
}
dest_video_name = ast_alloca(dest_video_name_len);
if (source_channel_stream_position != -1) {
/* We are looking for an exact stream position */
snprintf(dest_video_name, dest_video_name_len, "%s%c%s%c%d",
SOFTBRIDGE_VIDEO_DEST_PREFIX, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
source_channel_name, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
source_channel_stream_position);
return !strcmp(ast_stream_get_name(stream), dest_video_name);
}
snprintf(dest_video_name, dest_video_name_len, "%s%c%s",
SOFTBRIDGE_VIDEO_DEST_PREFIX, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
source_channel_name);
} else {
dest_video_name = SOFTBRIDGE_VIDEO_DEST_PREFIX;
}
return !strncmp(ast_stream_get_name(stream), dest_video_name, dest_video_name_len - 1);
}
static int append_source_stream(struct ast_stream_topology *dest,
const char *channel_name, const char *sdp_label,
struct ast_stream *stream, int index)
{
char *stream_clone_name = NULL;
struct ast_stream *stream_clone;
/* We use the stream topology index for the stream to uniquely identify and recognize it.
* This is guaranteed to remain the same across renegotiation of the source channel and
* ensures that the stream name is unique.
*/
if (ast_asprintf(&stream_clone_name, "%s%c%s%c%d", SOFTBRIDGE_VIDEO_DEST_PREFIX,
SOFTBRIDGE_VIDEO_DEST_SEPARATOR, channel_name, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
index) < 0) {
return -1;
}
stream_clone = ast_stream_clone(stream, stream_clone_name);
ast_free(stream_clone_name);
if (!stream_clone) {
return -1;
}
/* Sends an "a:label" attribute in the SDP for participant event correlation */
if (!ast_strlen_zero(sdp_label)) {
ast_stream_set_metadata(stream_clone, "SDP:LABEL", sdp_label);
}
/* We will be sending them a stream and not expecting anything in return */
ast_stream_set_state(stream_clone, AST_STREAM_STATE_SENDONLY);
if (ast_stream_topology_append_stream(dest, stream_clone) < 0) {
ast_stream_free(stream_clone);
return -1;
}
return 0;
}
static int append_source_streams(struct ast_stream_topology *dest,
const char *channel_name, const char *sdp_label,
const struct ast_stream_topology *source)
{
int i;
for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
struct ast_stream *stream;
stream = ast_stream_topology_get_stream(source, i);
if (!is_video_source(stream)) {
continue;
}
if (append_source_stream(dest, channel_name, sdp_label, stream, i)) {
return -1;
}
}
return 0;
}
static int append_all_streams(struct ast_stream_topology *dest,
const struct ast_stream_topology *source)
{
int i;
int dest_index = 0;
for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
struct ast_stream *clone;
int added = 0;
clone = ast_stream_clone(ast_stream_topology_get_stream(source, i), NULL);
if (!clone) {
return -1;
}
/* If we can reuse an existing removed stream then do so */
while (dest_index < ast_stream_topology_get_count(dest)) {
struct ast_stream *stream = ast_stream_topology_get_stream(dest, dest_index);
dest_index++;
if (ast_stream_get_state(stream) == AST_STREAM_STATE_REMOVED) {
/* This cannot fail because dest_index - 1 is less than the
* current count in dest. */
ast_stream_topology_set_stream(dest, dest_index - 1, clone);
added = 1;
break;
}
}
/* If no removed stream exists that we took the place of append the stream */
if (!added && ast_stream_topology_append_stream(dest, clone) < 0) {
ast_stream_free(clone);
return -1;
}
}
return 0;
}
/*!
* \brief Issue channel stream topology change requests.
*
* When in SFU mode, each participant needs to be able to
* send video directly to other participants in the bridge.
* This means that all participants need to have their topologies
* updated. The joiner needs to have destination streams for
* all current participants, and the current participants need
* to have destinations streams added for the joiner's sources.
*
* \param bridge
* \param joiner The channel that is joining the softmix bridge
*/
static void sfu_topologies_on_join(struct ast_bridge *bridge,
struct ast_bridge_channel *joiner)
{
RAII_VAR(struct ast_stream_topology *, joiner_video, NULL, ast_stream_topology_free);
struct ast_bridge_channels_list *participants = &bridge->channels;
struct ast_bridge_channel *participant;
int res;
struct softmix_channel *sc;
SCOPE_ENTER(3, "%s: \n", ast_channel_name(joiner->chan));
joiner_video = ast_stream_topology_alloc();
if (!joiner_video) {
SCOPE_EXIT_LOG_RTN(LOG_ERROR, "%s: Couldn't alloc topology\n", ast_channel_name(joiner->chan));
}
sc = joiner->tech_pvt;
ast_channel_lock(joiner->chan);
res = append_source_streams(joiner_video, ast_channel_name(joiner->chan),
bridge->softmix.send_sdp_label ? ast_channel_uniqueid(joiner->chan) : NULL,
ast_channel_get_stream_topology(joiner->chan));
sc->topology = ast_stream_topology_clone(ast_channel_get_stream_topology(joiner->chan));
ast_channel_unlock(joiner->chan);
if (res || !sc->topology) {
SCOPE_EXIT_LOG_RTN(LOG_ERROR, "%s: Couldn't append source streams\n", ast_channel_name(joiner->chan));
}
AST_LIST_TRAVERSE(participants, participant, entry) {
if (participant == joiner) {
continue;
}
ast_trace(-1, "%s: Appending existing participant %s\n", ast_channel_name(joiner->chan),
ast_channel_name(participant->chan));
ast_channel_lock(participant->chan);
res = append_source_streams(sc->topology, ast_channel_name(participant->chan),
bridge->softmix.send_sdp_label ? ast_channel_uniqueid(participant->chan) : NULL,
ast_channel_get_stream_topology(participant->chan));
ast_channel_unlock(participant->chan);
if (res) {
SCOPE_EXIT_LOG_RTN(LOG_ERROR, "%s/%s: Couldn't append source streams\n",
ast_channel_name(participant->chan), ast_channel_name(joiner->chan));
}
}
ast_trace(-1, "%s: Requesting topology change.\n", ast_channel_name(joiner->chan));
res = ast_channel_request_stream_topology_change(joiner->chan, sc->topology, NULL);
if (res) {
/* There are conditions under which this is expected so no need to log an ERROR. */
SCOPE_EXIT_RTN("%s: Couldn't request topology change\n", ast_channel_name(joiner->chan));
}
AST_LIST_TRAVERSE(participants, participant, entry) {
if (participant == joiner) {
continue;
}
sc = participant->tech_pvt;
ast_trace(-1, "%s: Appending joiner %s\n", ast_channel_name(participant->chan),
ast_channel_name(joiner->chan));
if (append_all_streams(sc->topology, joiner_video)) {
SCOPE_EXIT_LOG_RTN(LOG_ERROR, "%s/%s: Couldn't append streams\n",
ast_channel_name(participant->chan), ast_channel_name(joiner->chan));
}
ast_trace(-1, "%s: Requesting topology change\n", ast_channel_name(participant->chan));
res = ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
if (res) {
ast_trace(-1, "%s/%s: Couldn't request topology change\n",
ast_channel_name(participant->chan), ast_channel_name(joiner->chan));
}
}
SCOPE_EXIT();
}
/*! \brief Function called when a channel is joined into the bridge */
static int softmix_bridge_join(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
{
struct softmix_channel *sc;
struct softmix_bridge_data *softmix_data;
int set_binaural = 0;
/*
* If false, the channel will be convolved, but since it is a non stereo channel, output
* will be mono.
*/
int skip_binaural_output = 1;
int pos_id;
int is_announcement = 0;
int samplerate_change;
SCOPE_ENTER(3, "%s:\n", ast_channel_name(bridge_channel->chan));
softmix_data = bridge->tech_pvt;
if (!softmix_data) {
SCOPE_EXIT_RTN_VALUE(-1, "No tech_pvt\n");
}
/* Create a new softmix_channel structure and allocate various things on it */
if (!(sc = ast_calloc(1, sizeof(*sc)))) {
SCOPE_EXIT_RTN_VALUE(-1, "Couldn't alloc tech_pvt\n");
}
samplerate_change = softmix_data->internal_rate;
pos_id = -1;
if (bridge->softmix.binaural_active) {
if (strncmp(ast_channel_name(bridge_channel->chan), "CBAnn", 5) != 0) {
set_binaural = ast_format_get_channel_count(bridge_channel->write_format) > 1 ? 1 : 0;
if (set_binaural) {
softmix_data->internal_rate = samplerate_change;
}
skip_binaural_output = 0;
} else {
is_announcement = 1;
}
if (set_binaural) {
softmix_data->convolve.binaural_active = 1;
}
if (!skip_binaural_output) {
pos_id = set_binaural_data_join(&softmix_data->convolve, softmix_data->default_sample_size);
if (pos_id == -1) {
ast_log(LOG_ERROR, "Bridge %s: Failed to join channel %s. "
"Could not allocate enough memory.\n", bridge->uniqueid,
ast_channel_name(bridge_channel->chan));
ast_free(sc);
SCOPE_EXIT_RTN_VALUE(-1, "Couldn't do binaural join\n");
}
}
}
/* Can't forget the lock */
ast_mutex_init(&sc->lock);
/* Can't forget to record our pvt structure within the bridged channel structure */
bridge_channel->tech_pvt = sc;
set_softmix_bridge_data(softmix_data->internal_rate,
softmix_data->internal_mixing_interval
? softmix_data->internal_mixing_interval
: DEFAULT_SOFTMIX_INTERVAL,
bridge_channel, 0, set_binaural, pos_id, is_announcement);
if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
sfu_topologies_on_join(bridge, bridge_channel);
}
/* Complete any active hold before entering, or transitioning to softmix. */
if (ast_channel_hold_state(bridge_channel->chan) == AST_CONTROL_HOLD) {
ast_debug(1, "Channel %s simulating UNHOLD for bridge softmix join.\n",
ast_channel_name(bridge_channel->chan));
ast_indicate(bridge_channel->chan, AST_CONTROL_UNHOLD);
}
softmix_poke_thread(softmix_data);
SCOPE_EXIT_RTN_VALUE(0);
}
static int remove_destination_streams(struct ast_stream_topology *topology,
const char *channel_name)
{
int i;
int stream_removed = 0;
for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
struct ast_stream *stream;
stream = ast_stream_topology_get_stream(topology, i);
if (is_video_dest(stream, channel_name, -1)) {
ast_stream_set_state(stream, AST_STREAM_STATE_REMOVED);
stream_removed = 1;
}
}
return stream_removed;
}
static int sfu_topologies_on_leave(struct ast_bridge_channel *leaver, struct ast_bridge_channels_list *participants)
{
struct ast_bridge_channel *participant;
struct softmix_channel *sc;
AST_LIST_TRAVERSE(participants, participant, entry) {
sc = participant->tech_pvt;
if (!remove_destination_streams(sc->topology, ast_channel_name(leaver->chan))) {
continue;
}
ast_channel_request_stream_topology_change(participant->chan, sc->topology, NULL);
}
sc = leaver->tech_pvt;
if (remove_destination_streams(sc->topology, "")) {
ast_channel_request_stream_topology_change(leaver->chan, sc->topology, NULL);
}
return 0;
}
/*! \brief Function called when a channel leaves the bridge */
static void softmix_bridge_leave(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
{
struct softmix_channel *sc;
struct softmix_bridge_data *softmix_data;
softmix_data = bridge->tech_pvt;
sc = bridge_channel->tech_pvt;
if (!sc) {
return;
}
if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_SFU) {
sfu_topologies_on_leave(bridge_channel, &bridge->channels);
}
if (bridge->softmix.binaural_active) {
if (sc->binaural) {
set_binaural_data_leave(&softmix_data->convolve, sc->binaural_pos,
softmix_data->default_sample_size);
}
}
bridge_channel->tech_pvt = NULL;
ast_stream_topology_free(sc->topology);
ao2_cleanup(sc->remb_collector);
AST_VECTOR_FREE(&sc->video_sources);
/* Drop mutex lock */
ast_mutex_destroy(&sc->lock);
/* Drop the factory */
ast_slinfactory_destroy(&sc->factory);
/* Drop any formats on the frames */
ao2_cleanup(sc->write_frame.subclass.format);
/* Drop the DSP */
ast_dsp_free(sc->dsp);
/* Eep! drop ourselves */
ast_free(sc);
}
static void softmix_pass_video_top_priority(struct ast_bridge *bridge, struct ast_frame *frame)
{
struct ast_bridge_channel *cur;
AST_LIST_TRAVERSE(&bridge->channels, cur, entry) {
if (cur->suspended) {
continue;
}
if (ast_bridge_is_video_src(bridge, cur->chan) == 1) {
ast_bridge_channel_queue_frame(cur, frame);
break;
}
}
}
/*!
* \internal
* \brief Determine what to do with a video frame.
* \since 12.0.0
*
* \param bridge Which bridge is getting the frame
* \param bridge_channel Which channel is writing the frame.
* \param frame What is being written.
*/
static void softmix_bridge_write_video(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
{
struct softmix_channel *sc;
int video_src_priority;
/* Determine if the video frame should be distributed or not */
switch (bridge->softmix.video_mode.mode) {
case AST_BRIDGE_VIDEO_MODE_NONE:
break;
case AST_BRIDGE_VIDEO_MODE_SINGLE_SRC:
video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
if (video_src_priority == 1) {
/* Pass to me and everyone else. */
ast_bridge_queue_everyone_else(bridge, NULL, frame);
}
break;
case AST_BRIDGE_VIDEO_MODE_TALKER_SRC:
sc = bridge_channel->tech_pvt;
ast_mutex_lock(&sc->lock);
ast_bridge_update_talker_src_video_mode(bridge, bridge_channel->chan,
sc->video_talker.energy_average,
frame->subclass.frame_ending);
ast_mutex_unlock(&sc->lock);
video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
if (video_src_priority == 1) {
int num_src = ast_bridge_number_video_src(bridge);
int echo = num_src > 1 ? 0 : 1;
ast_bridge_queue_everyone_else(bridge, echo ? NULL : bridge_channel, frame);
} else if (video_src_priority == 2) {
softmix_pass_video_top_priority(bridge, frame);
}
break;
case AST_BRIDGE_VIDEO_MODE_SFU:
/* Nothing special to do here, the bridge channel stream map will ensure the
* video goes everywhere it needs to
*/
ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
break;
}
}
/*!
* \internal
* \brief Determine what to do with a voice frame.
* \since 12.0.0
*
* \param bridge Which bridge is getting the frame
* \param bridge_channel Which channel is writing the frame.
* \param frame What is being written.
*/
static void softmix_bridge_write_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
{
struct softmix_channel *sc = bridge_channel->tech_pvt;
struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
int silent = 0;
int totalsilence = 0;
int cur_energy = 0;
int silence_threshold = bridge_channel->tech_args.silence_threshold ?
bridge_channel->tech_args.silence_threshold :
DEFAULT_SOFTMIX_SILENCE_THRESHOLD;
/*
* If update_talking is set to 0 or 1, tell the bridge that the channel
* has started or stopped talking.
*/
char update_talking = -1;
/* Write the frame into the conference */
ast_mutex_lock(&sc->lock);
if (ast_format_cmp(frame->subclass.format, sc->read_slin_format) != AST_FORMAT_CMP_EQUAL) {
/*
* The incoming frame is not the expected format. Update
* the channel's translation path to get us slinear from
* the new format for the next frame.
*
* There is the possibility that this frame is an old slinear
* rate frame that was in flight when the softmix bridge
* changed rates. If so it will self correct on subsequent
* frames.
*/
ast_channel_lock(bridge_channel->chan);
ast_debug(1, "Channel %s wrote unexpected format into bridge. Got %s, expected %s.\n",
ast_channel_name(bridge_channel->chan),
ast_format_get_name(frame->subclass.format),
ast_format_get_name(sc->read_slin_format));
ast_set_read_format_path(bridge_channel->chan, frame->subclass.format,
sc->read_slin_format);
ast_channel_unlock(bridge_channel->chan);
}
/* The channel will be leaving soon if there is no dsp. */
if (sc->dsp) {
silent = ast_dsp_silence_with_energy(sc->dsp, frame, &totalsilence, &cur_energy);
}
if (bridge->softmix.video_mode.mode == AST_BRIDGE_VIDEO_MODE_TALKER_SRC) {
int cur_slot = sc->video_talker.energy_history_cur_slot;
sc->video_talker.energy_accum -= sc->video_talker.energy_history[cur_slot];
sc->video_talker.energy_accum += cur_energy;
sc->video_talker.energy_history[cur_slot] = cur_energy;
sc->video_talker.energy_average = sc->video_talker.energy_accum / DEFAULT_ENERGY_HISTORY_LEN;
sc->video_talker.energy_history_cur_slot++;
if (sc->video_talker.energy_history_cur_slot == DEFAULT_ENERGY_HISTORY_LEN) {
sc->video_talker.energy_history_cur_slot = 0; /* wrap around */
}
}
if (totalsilence < silence_threshold) {
if (!sc->talking && !silent) {
/* Tell the write process we have audio to be mixed out */
sc->talking = 1;
update_talking = 1;
}