forked from apple/darwin-xnu
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathdlil.c
10701 lines (9248 loc) · 285 KB
/
dlil.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*
* Copyright (c) 1999-2021 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
* compliance with the License. The rights granted to you under the License
* may not be used to create, or enable the creation or redistribution of,
* unlawful or unlicensed copies of an Apple operating system, or to
* circumvent, violate, or enable the circumvention or violation of, any
* terms of an Apple operating system software license agreement.
*
* Please obtain a copy of the License at
* http://www.opensource.apple.com/apsl/ and read it before using this file.
*
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
* Please see the License for the specific language governing rights and
* limitations under the License.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
/*
* NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
* support for mandatory and extensible security protections. This notice
* is included in support of clause 2.2 (b) of the Apple Public License,
* Version 2.0.
*/
#include <stddef.h>
#include <ptrauth.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/socket.h>
#include <sys/domain.h>
#include <sys/user.h>
#include <sys/random.h>
#include <sys/socketvar.h>
#include <net/if_dl.h>
#include <net/if.h>
#include <net/route.h>
#include <net/if_var.h>
#include <net/dlil.h>
#include <net/if_arp.h>
#include <net/iptap.h>
#include <net/pktap.h>
#include <sys/kern_event.h>
#include <sys/kdebug.h>
#include <sys/mcache.h>
#include <sys/syslog.h>
#include <sys/protosw.h>
#include <sys/priv.h>
#include <kern/assert.h>
#include <kern/task.h>
#include <kern/thread.h>
#include <kern/sched_prim.h>
#include <kern/locks.h>
#include <kern/zalloc.h>
#include <net/kpi_protocol.h>
#include <net/if_types.h>
#include <net/if_ipsec.h>
#include <net/if_llreach.h>
#include <net/if_utun.h>
#include <net/kpi_interfacefilter.h>
#include <net/classq/classq.h>
#include <net/classq/classq_sfb.h>
#include <net/flowhash.h>
#include <net/ntstat.h>
#include <net/if_llatbl.h>
#include <net/net_api_stats.h>
#include <net/if_ports_used.h>
#include <net/if_vlan_var.h>
#include <netinet/in.h>
#if INET
#include <netinet/in_var.h>
#include <netinet/igmp_var.h>
#include <netinet/ip_var.h>
#include <netinet/tcp.h>
#include <netinet/tcp_var.h>
#include <netinet/udp.h>
#include <netinet/udp_var.h>
#include <netinet/if_ether.h>
#include <netinet/in_pcb.h>
#include <netinet/in_tclass.h>
#include <netinet/ip.h>
#include <netinet/ip_icmp.h>
#include <netinet/icmp_var.h>
#endif /* INET */
#include <net/nat464_utils.h>
#include <netinet6/in6_var.h>
#include <netinet6/nd6.h>
#include <netinet6/mld6_var.h>
#include <netinet6/scope6_var.h>
#include <netinet/ip6.h>
#include <netinet/icmp6.h>
#include <net/pf_pbuf.h>
#include <libkern/OSAtomic.h>
#include <libkern/tree.h>
#include <dev/random/randomdev.h>
#include <machine/machine_routines.h>
#include <mach/thread_act.h>
#include <mach/sdt.h>
#if CONFIG_MACF
#include <sys/kauth.h>
#include <security/mac_framework.h>
#include <net/ethernet.h>
#include <net/firewire.h>
#endif
#if PF
#include <net/pfvar.h>
#endif /* PF */
#include <net/pktsched/pktsched.h>
#include <net/pktsched/pktsched_netem.h>
#if NECP
#include <net/necp.h>
#endif /* NECP */
#include <os/log.h>
#define DBG_LAYER_BEG DLILDBG_CODE(DBG_DLIL_STATIC, 0)
#define DBG_LAYER_END DLILDBG_CODE(DBG_DLIL_STATIC, 2)
#define DBG_FNC_DLIL_INPUT DLILDBG_CODE(DBG_DLIL_STATIC, (1 << 8))
#define DBG_FNC_DLIL_OUTPUT DLILDBG_CODE(DBG_DLIL_STATIC, (2 << 8))
#define DBG_FNC_DLIL_IFOUT DLILDBG_CODE(DBG_DLIL_STATIC, (3 << 8))
#define MAX_FRAME_TYPE_SIZE 4 /* LONGWORDS */
#define MAX_LINKADDR 4 /* LONGWORDS */
#define M_NKE M_IFADDR
#if 1
#define DLIL_PRINTF printf
#else
#define DLIL_PRINTF kprintf
#endif
#define IF_DATA_REQUIRE_ALIGNED_64(f) \
_CASSERT(!(offsetof(struct if_data_internal, f) % sizeof (u_int64_t)))
#define IFNET_IF_DATA_REQUIRE_ALIGNED_64(f) \
_CASSERT(!(offsetof(struct ifnet, if_data.f) % sizeof (u_int64_t)))
enum {
kProtoKPI_v1 = 1,
kProtoKPI_v2 = 2
};
/*
* List of if_proto structures in if_proto_hash[] is protected by
* the ifnet lock. The rest of the fields are initialized at protocol
* attach time and never change, thus no lock required as long as
* a reference to it is valid, via if_proto_ref().
*/
struct if_proto {
SLIST_ENTRY(if_proto) next_hash;
u_int32_t refcount;
u_int32_t detached;
struct ifnet *ifp;
protocol_family_t protocol_family;
int proto_kpi;
union {
struct {
proto_media_input input;
proto_media_preout pre_output;
proto_media_event event;
proto_media_ioctl ioctl;
proto_media_detached detached;
proto_media_resolve_multi resolve_multi;
proto_media_send_arp send_arp;
} v1;
struct {
proto_media_input_v2 input;
proto_media_preout pre_output;
proto_media_event event;
proto_media_ioctl ioctl;
proto_media_detached detached;
proto_media_resolve_multi resolve_multi;
proto_media_send_arp send_arp;
} v2;
} kpi;
};
SLIST_HEAD(proto_hash_entry, if_proto);
#define DLIL_SDLDATALEN \
(DLIL_SDLMAXLEN - offsetof(struct sockaddr_dl, sdl_data[0]))
struct dlil_ifnet {
struct ifnet dl_if; /* public ifnet */
/*
* DLIL private fields, protected by dl_if_lock
*/
decl_lck_mtx_data(, dl_if_lock);
TAILQ_ENTRY(dlil_ifnet) dl_if_link; /* dlil_ifnet link */
u_int32_t dl_if_flags; /* flags (below) */
u_int32_t dl_if_refcnt; /* refcnt */
void (*dl_if_trace)(struct dlil_ifnet *, int); /* ref trace callback */
void *dl_if_uniqueid; /* unique interface id */
size_t dl_if_uniqueid_len; /* length of the unique id */
char dl_if_namestorage[IFNAMSIZ]; /* interface name storage */
char dl_if_xnamestorage[IFXNAMSIZ]; /* external name storage */
struct {
struct ifaddr ifa; /* lladdr ifa */
u_int8_t asdl[DLIL_SDLMAXLEN]; /* addr storage */
u_int8_t msdl[DLIL_SDLMAXLEN]; /* mask storage */
} dl_if_lladdr;
u_int8_t dl_if_descstorage[IF_DESCSIZE]; /* desc storage */
u_int8_t dl_if_permanent_ether[ETHER_ADDR_LEN]; /* permanent address */
u_int8_t dl_if_permanent_ether_is_set;
u_int8_t dl_if_unused;
struct dlil_threading_info dl_if_inpstorage; /* input thread storage */
ctrace_t dl_if_attach; /* attach PC stacktrace */
ctrace_t dl_if_detach; /* detach PC stacktrace */
};
/* Values for dl_if_flags (private to DLIL) */
#define DLIF_INUSE 0x1 /* DLIL ifnet recycler, ifnet in use */
#define DLIF_REUSE 0x2 /* DLIL ifnet recycles, ifnet is not new */
#define DLIF_DEBUG 0x4 /* has debugging info */
#define IF_REF_TRACE_HIST_SIZE 8 /* size of ref trace history */
/* For gdb */
__private_extern__ unsigned int if_ref_trace_hist_size = IF_REF_TRACE_HIST_SIZE;
struct dlil_ifnet_dbg {
struct dlil_ifnet dldbg_dlif; /* dlil_ifnet */
u_int16_t dldbg_if_refhold_cnt; /* # ifnet references */
u_int16_t dldbg_if_refrele_cnt; /* # ifnet releases */
/*
* Circular lists of ifnet_{reference,release} callers.
*/
ctrace_t dldbg_if_refhold[IF_REF_TRACE_HIST_SIZE];
ctrace_t dldbg_if_refrele[IF_REF_TRACE_HIST_SIZE];
};
#define DLIL_TO_IFP(s) (&s->dl_if)
#define IFP_TO_DLIL(s) ((struct dlil_ifnet *)s)
struct ifnet_filter {
TAILQ_ENTRY(ifnet_filter) filt_next;
u_int32_t filt_skip;
u_int32_t filt_flags;
ifnet_t filt_ifp;
const char *filt_name;
void *filt_cookie;
protocol_family_t filt_protocol;
iff_input_func filt_input;
iff_output_func filt_output;
iff_event_func filt_event;
iff_ioctl_func filt_ioctl;
iff_detached_func filt_detached;
};
struct proto_input_entry;
static TAILQ_HEAD(, dlil_ifnet) dlil_ifnet_head;
static lck_grp_t *dlil_lock_group;
lck_grp_t *ifnet_lock_group;
static lck_grp_t *ifnet_head_lock_group;
static lck_grp_t *ifnet_snd_lock_group;
static lck_grp_t *ifnet_rcv_lock_group;
lck_attr_t *ifnet_lock_attr;
decl_lck_rw_data(static, ifnet_head_lock);
decl_lck_mtx_data(static, dlil_ifnet_lock);
u_int32_t dlil_filter_disable_tso_count = 0;
#if DEBUG
static unsigned int ifnet_debug = 1; /* debugging (enabled) */
#else
static unsigned int ifnet_debug; /* debugging (disabled) */
#endif /* !DEBUG */
static unsigned int dlif_size; /* size of dlil_ifnet to allocate */
static unsigned int dlif_bufsize; /* size of dlif_size + headroom */
static struct zone *dlif_zone; /* zone for dlil_ifnet */
#define DLIF_ZONE_NAME "ifnet" /* zone name */
static ZONE_DECLARE(dlif_filt_zone, "ifnet_filter",
sizeof(struct ifnet_filter), ZC_ZFREE_CLEARMEM);
static ZONE_DECLARE(dlif_phash_zone, "ifnet_proto_hash",
sizeof(struct proto_hash_entry) * PROTO_HASH_SLOTS, ZC_ZFREE_CLEARMEM);
static ZONE_DECLARE(dlif_proto_zone, "ifnet_proto",
sizeof(struct if_proto), ZC_ZFREE_CLEARMEM);
static unsigned int dlif_tcpstat_size; /* size of tcpstat_local to allocate */
static unsigned int dlif_tcpstat_bufsize; /* size of dlif_tcpstat_size + headroom */
static struct zone *dlif_tcpstat_zone; /* zone for tcpstat_local */
#define DLIF_TCPSTAT_ZONE_NAME "ifnet_tcpstat" /* zone name */
static unsigned int dlif_udpstat_size; /* size of udpstat_local to allocate */
static unsigned int dlif_udpstat_bufsize; /* size of dlif_udpstat_size + headroom */
static struct zone *dlif_udpstat_zone; /* zone for udpstat_local */
#define DLIF_UDPSTAT_ZONE_NAME "ifnet_udpstat" /* zone name */
static u_int32_t net_rtref;
static struct dlil_main_threading_info dlil_main_input_thread_info;
__private_extern__ struct dlil_threading_info *dlil_main_input_thread =
(struct dlil_threading_info *)&dlil_main_input_thread_info;
static int dlil_event_internal(struct ifnet *ifp, struct kev_msg *msg, bool update_generation);
static int dlil_detach_filter_internal(interface_filter_t filter, int detached);
static void dlil_if_trace(struct dlil_ifnet *, int);
static void if_proto_ref(struct if_proto *);
static void if_proto_free(struct if_proto *);
static struct if_proto *find_attached_proto(struct ifnet *, u_int32_t);
static u_int32_t dlil_ifp_protolist(struct ifnet *ifp, protocol_family_t *list,
u_int32_t list_count);
static void if_flt_monitor_busy(struct ifnet *);
static void if_flt_monitor_unbusy(struct ifnet *);
static void if_flt_monitor_enter(struct ifnet *);
static void if_flt_monitor_leave(struct ifnet *);
static int dlil_interface_filters_input(struct ifnet *, struct mbuf **,
char **, protocol_family_t);
static int dlil_interface_filters_output(struct ifnet *, struct mbuf **,
protocol_family_t);
static struct ifaddr *dlil_alloc_lladdr(struct ifnet *,
const struct sockaddr_dl *);
static int ifnet_lookup(struct ifnet *);
static void if_purgeaddrs(struct ifnet *);
static errno_t ifproto_media_input_v1(struct ifnet *, protocol_family_t,
struct mbuf *, char *);
static errno_t ifproto_media_input_v2(struct ifnet *, protocol_family_t,
struct mbuf *);
static errno_t ifproto_media_preout(struct ifnet *, protocol_family_t,
mbuf_t *, const struct sockaddr *, void *, char *, char *);
static void ifproto_media_event(struct ifnet *, protocol_family_t,
const struct kev_msg *);
static errno_t ifproto_media_ioctl(struct ifnet *, protocol_family_t,
unsigned long, void *);
static errno_t ifproto_media_resolve_multi(ifnet_t, const struct sockaddr *,
struct sockaddr_dl *, size_t);
static errno_t ifproto_media_send_arp(struct ifnet *, u_short,
const struct sockaddr_dl *, const struct sockaddr *,
const struct sockaddr_dl *, const struct sockaddr *);
static errno_t ifp_if_input(struct ifnet *ifp, struct mbuf *m_head,
struct mbuf *m_tail, const struct ifnet_stat_increment_param *s,
boolean_t poll, struct thread *tp);
static void ifp_if_input_poll(struct ifnet *, u_int32_t, u_int32_t,
struct mbuf **, struct mbuf **, u_int32_t *, u_int32_t *);
static errno_t ifp_if_ctl(struct ifnet *, ifnet_ctl_cmd_t, u_int32_t, void *);
static errno_t ifp_if_demux(struct ifnet *, struct mbuf *, char *,
protocol_family_t *);
static errno_t ifp_if_add_proto(struct ifnet *, protocol_family_t,
const struct ifnet_demux_desc *, u_int32_t);
static errno_t ifp_if_del_proto(struct ifnet *, protocol_family_t);
static errno_t ifp_if_check_multi(struct ifnet *, const struct sockaddr *);
#if !XNU_TARGET_OS_OSX
static errno_t ifp_if_framer(struct ifnet *, struct mbuf **,
const struct sockaddr *, const char *, const char *,
u_int32_t *, u_int32_t *);
#else /* XNU_TARGET_OS_OSX */
static errno_t ifp_if_framer(struct ifnet *, struct mbuf **,
const struct sockaddr *, const char *, const char *);
#endif /* XNU_TARGET_OS_OSX */
static errno_t ifp_if_framer_extended(struct ifnet *, struct mbuf **,
const struct sockaddr *, const char *, const char *,
u_int32_t *, u_int32_t *);
static errno_t ifp_if_set_bpf_tap(struct ifnet *, bpf_tap_mode, bpf_packet_func);
static void ifp_if_free(struct ifnet *);
static void ifp_if_event(struct ifnet *, const struct kev_msg *);
static __inline void ifp_inc_traffic_class_in(struct ifnet *, struct mbuf *);
static __inline void ifp_inc_traffic_class_out(struct ifnet *, struct mbuf *);
static errno_t dlil_input_async(struct dlil_threading_info *, struct ifnet *,
struct mbuf *, struct mbuf *, const struct ifnet_stat_increment_param *,
boolean_t, struct thread *);
static errno_t dlil_input_sync(struct dlil_threading_info *, struct ifnet *,
struct mbuf *, struct mbuf *, const struct ifnet_stat_increment_param *,
boolean_t, struct thread *);
static void dlil_main_input_thread_func(void *, wait_result_t);
static void dlil_main_input_thread_cont(void *, wait_result_t);
static void dlil_input_thread_func(void *, wait_result_t);
static void dlil_input_thread_cont(void *, wait_result_t);
static void dlil_rxpoll_input_thread_func(void *, wait_result_t);
static void dlil_rxpoll_input_thread_cont(void *, wait_result_t);
static int dlil_create_input_thread(ifnet_t, struct dlil_threading_info *,
thread_continue_t *);
static void dlil_terminate_input_thread(struct dlil_threading_info *);
static void dlil_input_stats_add(const struct ifnet_stat_increment_param *,
struct dlil_threading_info *, struct ifnet *, boolean_t);
static boolean_t dlil_input_stats_sync(struct ifnet *,
struct dlil_threading_info *);
static void dlil_input_packet_list_common(struct ifnet *, struct mbuf *,
u_int32_t, ifnet_model_t, boolean_t);
static errno_t ifnet_input_common(struct ifnet *, struct mbuf *, struct mbuf *,
const struct ifnet_stat_increment_param *, boolean_t, boolean_t);
static int dlil_is_clat_needed(protocol_family_t, mbuf_t );
static errno_t dlil_clat46(ifnet_t, protocol_family_t *, mbuf_t *);
static errno_t dlil_clat64(ifnet_t, protocol_family_t *, mbuf_t *);
#if DEBUG || DEVELOPMENT
static void dlil_verify_sum16(void);
#endif /* DEBUG || DEVELOPMENT */
static void dlil_output_cksum_dbg(struct ifnet *, struct mbuf *, uint32_t,
protocol_family_t);
static void dlil_input_cksum_dbg(struct ifnet *, struct mbuf *, char *,
protocol_family_t);
static void dlil_incr_pending_thread_count(void);
static void dlil_decr_pending_thread_count(void);
static void ifnet_detacher_thread_func(void *, wait_result_t);
static void ifnet_detacher_thread_cont(void *, wait_result_t);
static void ifnet_detach_final(struct ifnet *);
static void ifnet_detaching_enqueue(struct ifnet *);
static struct ifnet *ifnet_detaching_dequeue(void);
static void ifnet_start_thread_func(void *, wait_result_t);
static void ifnet_start_thread_cont(void *, wait_result_t);
static void ifnet_poll_thread_func(void *, wait_result_t);
static void ifnet_poll_thread_cont(void *, wait_result_t);
static errno_t ifnet_enqueue_common(struct ifnet *, classq_pkt_t *,
boolean_t, boolean_t *);
static void ifp_src_route_copyout(struct ifnet *, struct route *);
static void ifp_src_route_copyin(struct ifnet *, struct route *);
static void ifp_src_route6_copyout(struct ifnet *, struct route_in6 *);
static void ifp_src_route6_copyin(struct ifnet *, struct route_in6 *);
static int sysctl_rxpoll SYSCTL_HANDLER_ARGS;
static int sysctl_rxpoll_mode_holdtime SYSCTL_HANDLER_ARGS;
static int sysctl_rxpoll_sample_holdtime SYSCTL_HANDLER_ARGS;
static int sysctl_rxpoll_interval_time SYSCTL_HANDLER_ARGS;
static int sysctl_rxpoll_wlowat SYSCTL_HANDLER_ARGS;
static int sysctl_rxpoll_whiwat SYSCTL_HANDLER_ARGS;
static int sysctl_sndq_maxlen SYSCTL_HANDLER_ARGS;
static int sysctl_rcvq_maxlen SYSCTL_HANDLER_ARGS;
static int sysctl_hwcksum_dbg_mode SYSCTL_HANDLER_ARGS;
static int sysctl_hwcksum_dbg_partial_rxoff_forced SYSCTL_HANDLER_ARGS;
static int sysctl_hwcksum_dbg_partial_rxoff_adj SYSCTL_HANDLER_ARGS;
struct chain_len_stats tx_chain_len_stats;
static int sysctl_tx_chain_len_stats SYSCTL_HANDLER_ARGS;
#if TEST_INPUT_THREAD_TERMINATION
static int sysctl_input_thread_termination_spin SYSCTL_HANDLER_ARGS;
#endif /* TEST_INPUT_THREAD_TERMINATION */
/* The following are protected by dlil_ifnet_lock */
static TAILQ_HEAD(, ifnet) ifnet_detaching_head;
static u_int32_t ifnet_detaching_cnt;
static boolean_t ifnet_detaching_embryonic;
static void *ifnet_delayed_run; /* wait channel for detaching thread */
decl_lck_mtx_data(static, ifnet_fc_lock);
static uint32_t ifnet_flowhash_seed;
struct ifnet_flowhash_key {
char ifk_name[IFNAMSIZ];
uint32_t ifk_unit;
uint32_t ifk_flags;
uint32_t ifk_eflags;
uint32_t ifk_capabilities;
uint32_t ifk_capenable;
uint32_t ifk_output_sched_model;
uint32_t ifk_rand1;
uint32_t ifk_rand2;
};
/* Flow control entry per interface */
struct ifnet_fc_entry {
RB_ENTRY(ifnet_fc_entry) ifce_entry;
u_int32_t ifce_flowhash;
struct ifnet *ifce_ifp;
};
static uint32_t ifnet_calc_flowhash(struct ifnet *);
static int ifce_cmp(const struct ifnet_fc_entry *,
const struct ifnet_fc_entry *);
static int ifnet_fc_add(struct ifnet *);
static struct ifnet_fc_entry *ifnet_fc_get(u_int32_t);
static void ifnet_fc_entry_free(struct ifnet_fc_entry *);
/* protected by ifnet_fc_lock */
RB_HEAD(ifnet_fc_tree, ifnet_fc_entry) ifnet_fc_tree;
RB_PROTOTYPE(ifnet_fc_tree, ifnet_fc_entry, ifce_entry, ifce_cmp);
RB_GENERATE(ifnet_fc_tree, ifnet_fc_entry, ifce_entry, ifce_cmp);
static ZONE_DECLARE(ifnet_fc_zone, "ifnet_fc_zone",
sizeof(struct ifnet_fc_entry), ZC_ZFREE_CLEARMEM);
extern void bpfdetach(struct ifnet *);
extern void proto_input_run(void);
extern uint32_t udp_count_opportunistic(unsigned int ifindex,
u_int32_t flags);
extern uint32_t tcp_count_opportunistic(unsigned int ifindex,
u_int32_t flags);
__private_extern__ void link_rtrequest(int, struct rtentry *, struct sockaddr *);
#if CONFIG_MACF
#if !XNU_TARGET_OS_OSX
int dlil_lladdr_ckreq = 1;
#else /* XNU_TARGET_OS_OSX */
int dlil_lladdr_ckreq = 0;
#endif /* XNU_TARGET_OS_OSX */
#endif /* CONFIG_MACF */
#if DEBUG
int dlil_verbose = 1;
#else
int dlil_verbose = 0;
#endif /* DEBUG */
#if IFNET_INPUT_SANITY_CHK
/* sanity checking of input packet lists received */
static u_int32_t dlil_input_sanity_check = 0;
#endif /* IFNET_INPUT_SANITY_CHK */
/* rate limit debug messages */
struct timespec dlil_dbgrate = { .tv_sec = 1, .tv_nsec = 0 };
SYSCTL_DECL(_net_link_generic_system);
SYSCTL_INT(_net_link_generic_system, OID_AUTO, dlil_verbose,
CTLFLAG_RW | CTLFLAG_LOCKED, &dlil_verbose, 0, "Log DLIL error messages");
#define IF_SNDQ_MINLEN 32
u_int32_t if_sndq_maxlen = IFQ_MAXLEN;
SYSCTL_PROC(_net_link_generic_system, OID_AUTO, sndq_maxlen,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &if_sndq_maxlen, IFQ_MAXLEN,
sysctl_sndq_maxlen, "I", "Default transmit queue max length");
#define IF_RCVQ_MINLEN 32
#define IF_RCVQ_MAXLEN 256
u_int32_t if_rcvq_maxlen = IF_RCVQ_MAXLEN;
SYSCTL_PROC(_net_link_generic_system, OID_AUTO, rcvq_maxlen,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &if_rcvq_maxlen, IFQ_MAXLEN,
sysctl_rcvq_maxlen, "I", "Default receive queue max length");
#define IF_RXPOLL_DECAY 2 /* ilog2 of EWMA decay rate (4) */
u_int32_t if_rxpoll_decay = IF_RXPOLL_DECAY;
SYSCTL_UINT(_net_link_generic_system, OID_AUTO, rxpoll_decay,
CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll_decay, IF_RXPOLL_DECAY,
"ilog2 of EWMA decay rate of avg inbound packets");
#define IF_RXPOLL_MODE_HOLDTIME_MIN (10ULL * 1000 * 1000) /* 10 ms */
#define IF_RXPOLL_MODE_HOLDTIME (1000ULL * 1000 * 1000) /* 1 sec */
static u_int64_t if_rxpoll_mode_holdtime = IF_RXPOLL_MODE_HOLDTIME;
SYSCTL_PROC(_net_link_generic_system, OID_AUTO, rxpoll_freeze_time,
CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll_mode_holdtime,
IF_RXPOLL_MODE_HOLDTIME, sysctl_rxpoll_mode_holdtime,
"Q", "input poll mode freeze time");
#define IF_RXPOLL_SAMPLETIME_MIN (1ULL * 1000 * 1000) /* 1 ms */
#define IF_RXPOLL_SAMPLETIME (10ULL * 1000 * 1000) /* 10 ms */
static u_int64_t if_rxpoll_sample_holdtime = IF_RXPOLL_SAMPLETIME;
SYSCTL_PROC(_net_link_generic_system, OID_AUTO, rxpoll_sample_time,
CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll_sample_holdtime,
IF_RXPOLL_SAMPLETIME, sysctl_rxpoll_sample_holdtime,
"Q", "input poll sampling time");
static u_int64_t if_rxpoll_interval_time = IF_RXPOLL_INTERVALTIME;
SYSCTL_PROC(_net_link_generic_system, OID_AUTO, rxpoll_interval_time,
CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll_interval_time,
IF_RXPOLL_INTERVALTIME, sysctl_rxpoll_interval_time,
"Q", "input poll interval (time)");
#define IF_RXPOLL_INTERVAL_PKTS 0 /* 0 (disabled) */
u_int32_t if_rxpoll_interval_pkts = IF_RXPOLL_INTERVAL_PKTS;
SYSCTL_UINT(_net_link_generic_system, OID_AUTO, rxpoll_interval_pkts,
CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll_interval_pkts,
IF_RXPOLL_INTERVAL_PKTS, "input poll interval (packets)");
#define IF_RXPOLL_WLOWAT 10
static u_int32_t if_sysctl_rxpoll_wlowat = IF_RXPOLL_WLOWAT;
SYSCTL_PROC(_net_link_generic_system, OID_AUTO, rxpoll_wakeups_lowat,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &if_sysctl_rxpoll_wlowat,
IF_RXPOLL_WLOWAT, sysctl_rxpoll_wlowat,
"I", "input poll wakeup low watermark");
#define IF_RXPOLL_WHIWAT 100
static u_int32_t if_sysctl_rxpoll_whiwat = IF_RXPOLL_WHIWAT;
SYSCTL_PROC(_net_link_generic_system, OID_AUTO, rxpoll_wakeups_hiwat,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &if_sysctl_rxpoll_whiwat,
IF_RXPOLL_WHIWAT, sysctl_rxpoll_whiwat,
"I", "input poll wakeup high watermark");
static u_int32_t if_rxpoll_max = 0; /* 0 (automatic) */
SYSCTL_UINT(_net_link_generic_system, OID_AUTO, rxpoll_max,
CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll_max, 0,
"max packets per poll call");
u_int32_t if_rxpoll = 1;
SYSCTL_PROC(_net_link_generic_system, OID_AUTO, rxpoll,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll, 0,
sysctl_rxpoll, "I", "enable opportunistic input polling");
#if TEST_INPUT_THREAD_TERMINATION
static u_int32_t if_input_thread_termination_spin = 0;
SYSCTL_PROC(_net_link_generic_system, OID_AUTO, input_thread_termination_spin,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
&if_input_thread_termination_spin, 0,
sysctl_input_thread_termination_spin,
"I", "input thread termination spin limit");
#endif /* TEST_INPUT_THREAD_TERMINATION */
static u_int32_t cur_dlil_input_threads = 0;
SYSCTL_UINT(_net_link_generic_system, OID_AUTO, dlil_input_threads,
CTLFLAG_RD | CTLFLAG_LOCKED, &cur_dlil_input_threads, 0,
"Current number of DLIL input threads");
#if IFNET_INPUT_SANITY_CHK
SYSCTL_UINT(_net_link_generic_system, OID_AUTO, dlil_input_sanity_check,
CTLFLAG_RW | CTLFLAG_LOCKED, &dlil_input_sanity_check, 0,
"Turn on sanity checking in DLIL input");
#endif /* IFNET_INPUT_SANITY_CHK */
static u_int32_t if_flowadv = 1;
SYSCTL_UINT(_net_link_generic_system, OID_AUTO, flow_advisory,
CTLFLAG_RW | CTLFLAG_LOCKED, &if_flowadv, 1,
"enable flow-advisory mechanism");
static u_int32_t if_delaybased_queue = 1;
SYSCTL_UINT(_net_link_generic_system, OID_AUTO, delaybased_queue,
CTLFLAG_RW | CTLFLAG_LOCKED, &if_delaybased_queue, 1,
"enable delay based dynamic queue sizing");
static uint64_t hwcksum_in_invalidated = 0;
SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
hwcksum_in_invalidated, CTLFLAG_RD | CTLFLAG_LOCKED,
&hwcksum_in_invalidated, "inbound packets with invalidated hardware cksum");
uint32_t hwcksum_dbg = 0;
SYSCTL_UINT(_net_link_generic_system, OID_AUTO, hwcksum_dbg,
CTLFLAG_RW | CTLFLAG_LOCKED, &hwcksum_dbg, 0,
"enable hardware cksum debugging");
u_int32_t ifnet_start_delayed = 0;
SYSCTL_UINT(_net_link_generic_system, OID_AUTO, start_delayed,
CTLFLAG_RW | CTLFLAG_LOCKED, &ifnet_start_delayed, 0,
"number of times start was delayed");
u_int32_t ifnet_delay_start_disabled = 0;
SYSCTL_UINT(_net_link_generic_system, OID_AUTO, start_delay_disabled,
CTLFLAG_RW | CTLFLAG_LOCKED, &ifnet_delay_start_disabled, 0,
"number of times start was delayed");
static inline void
ifnet_delay_start_disabled_increment(void)
{
OSIncrementAtomic(&ifnet_delay_start_disabled);
}
#define HWCKSUM_DBG_PARTIAL_FORCED 0x1 /* forced partial checksum */
#define HWCKSUM_DBG_PARTIAL_RXOFF_ADJ 0x2 /* adjust start offset */
#define HWCKSUM_DBG_FINALIZE_FORCED 0x10 /* forced finalize */
#define HWCKSUM_DBG_MASK \
(HWCKSUM_DBG_PARTIAL_FORCED | HWCKSUM_DBG_PARTIAL_RXOFF_ADJ | \
HWCKSUM_DBG_FINALIZE_FORCED)
static uint32_t hwcksum_dbg_mode = 0;
SYSCTL_PROC(_net_link_generic_system, OID_AUTO, hwcksum_dbg_mode,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &hwcksum_dbg_mode,
0, sysctl_hwcksum_dbg_mode, "I", "hardware cksum debugging mode");
static uint64_t hwcksum_dbg_partial_forced = 0;
SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
hwcksum_dbg_partial_forced, CTLFLAG_RD | CTLFLAG_LOCKED,
&hwcksum_dbg_partial_forced, "packets forced using partial cksum");
static uint64_t hwcksum_dbg_partial_forced_bytes = 0;
SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
hwcksum_dbg_partial_forced_bytes, CTLFLAG_RD | CTLFLAG_LOCKED,
&hwcksum_dbg_partial_forced_bytes, "bytes forced using partial cksum");
static uint32_t hwcksum_dbg_partial_rxoff_forced = 0;
SYSCTL_PROC(_net_link_generic_system, OID_AUTO,
hwcksum_dbg_partial_rxoff_forced, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
&hwcksum_dbg_partial_rxoff_forced, 0,
sysctl_hwcksum_dbg_partial_rxoff_forced, "I",
"forced partial cksum rx offset");
static uint32_t hwcksum_dbg_partial_rxoff_adj = 0;
SYSCTL_PROC(_net_link_generic_system, OID_AUTO, hwcksum_dbg_partial_rxoff_adj,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &hwcksum_dbg_partial_rxoff_adj,
0, sysctl_hwcksum_dbg_partial_rxoff_adj, "I",
"adjusted partial cksum rx offset");
static uint64_t hwcksum_dbg_verified = 0;
SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
hwcksum_dbg_verified, CTLFLAG_RD | CTLFLAG_LOCKED,
&hwcksum_dbg_verified, "packets verified for having good checksum");
static uint64_t hwcksum_dbg_bad_cksum = 0;
SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
hwcksum_dbg_bad_cksum, CTLFLAG_RD | CTLFLAG_LOCKED,
&hwcksum_dbg_bad_cksum, "packets with bad hardware calculated checksum");
static uint64_t hwcksum_dbg_bad_rxoff = 0;
SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
hwcksum_dbg_bad_rxoff, CTLFLAG_RD | CTLFLAG_LOCKED,
&hwcksum_dbg_bad_rxoff, "packets with invalid rxoff");
static uint64_t hwcksum_dbg_adjusted = 0;
SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
hwcksum_dbg_adjusted, CTLFLAG_RD | CTLFLAG_LOCKED,
&hwcksum_dbg_adjusted, "packets with rxoff adjusted");
static uint64_t hwcksum_dbg_finalized_hdr = 0;
SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
hwcksum_dbg_finalized_hdr, CTLFLAG_RD | CTLFLAG_LOCKED,
&hwcksum_dbg_finalized_hdr, "finalized headers");
static uint64_t hwcksum_dbg_finalized_data = 0;
SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
hwcksum_dbg_finalized_data, CTLFLAG_RD | CTLFLAG_LOCKED,
&hwcksum_dbg_finalized_data, "finalized payloads");
uint32_t hwcksum_tx = 1;
SYSCTL_UINT(_net_link_generic_system, OID_AUTO, hwcksum_tx,
CTLFLAG_RW | CTLFLAG_LOCKED, &hwcksum_tx, 0,
"enable transmit hardware checksum offload");
uint32_t hwcksum_rx = 1;
SYSCTL_UINT(_net_link_generic_system, OID_AUTO, hwcksum_rx,
CTLFLAG_RW | CTLFLAG_LOCKED, &hwcksum_rx, 0,
"enable receive hardware checksum offload");
SYSCTL_PROC(_net_link_generic_system, OID_AUTO, tx_chain_len_stats,
CTLFLAG_RD | CTLFLAG_LOCKED, 0, 9,
sysctl_tx_chain_len_stats, "S", "");
uint32_t tx_chain_len_count = 0;
SYSCTL_UINT(_net_link_generic_system, OID_AUTO, tx_chain_len_count,
CTLFLAG_RW | CTLFLAG_LOCKED, &tx_chain_len_count, 0, "");
static uint32_t threshold_notify = 1; /* enable/disable */
SYSCTL_UINT(_net_link_generic_system, OID_AUTO, threshold_notify,
CTLFLAG_RW | CTLFLAG_LOCKED, &threshold_notify, 0, "");
static uint32_t threshold_interval = 2; /* in seconds */
SYSCTL_UINT(_net_link_generic_system, OID_AUTO, threshold_interval,
CTLFLAG_RW | CTLFLAG_LOCKED, &threshold_interval, 0, "");
#if (DEVELOPMENT || DEBUG)
static int sysctl_get_kao_frames SYSCTL_HANDLER_ARGS;
SYSCTL_NODE(_net_link_generic_system, OID_AUTO, get_kao_frames,
CTLFLAG_RD | CTLFLAG_LOCKED, sysctl_get_kao_frames, "");
#endif /* DEVELOPMENT || DEBUG */
struct net_api_stats net_api_stats;
SYSCTL_STRUCT(_net, OID_AUTO, api_stats, CTLFLAG_RD | CTLFLAG_LOCKED,
&net_api_stats, net_api_stats, "");
unsigned int net_rxpoll = 1;
unsigned int net_affinity = 1;
unsigned int net_async = 1; /* 0: synchronous, 1: asynchronous */
static kern_return_t dlil_affinity_set(struct thread *, u_int32_t);
extern u_int32_t inject_buckets;
static lck_grp_attr_t *dlil_grp_attributes = NULL;
static lck_attr_t *dlil_lck_attributes = NULL;
/* DLIL data threshold thread call */
static void dlil_dt_tcall_fn(thread_call_param_t, thread_call_param_t);
void
ifnet_filter_update_tso(boolean_t filter_enable)
{
/*
* update filter count and route_generation ID to let TCP
* know it should reevalute doing TSO or not
*/
OSAddAtomic(filter_enable ? 1 : -1, &dlil_filter_disable_tso_count);
routegenid_update();
}
#define DLIL_INPUT_CHECK(m, ifp) { \
struct ifnet *_rcvif = mbuf_pkthdr_rcvif(m); \
if (_rcvif == NULL || (ifp != lo_ifp && _rcvif != ifp) || \
!(mbuf_flags(m) & MBUF_PKTHDR)) { \
panic_plain("%s: invalid mbuf %p\n", __func__, m); \
/* NOTREACHED */ \
} \
}
#define DLIL_EWMA(old, new, decay) do { \
u_int32_t _avg; \
if ((_avg = (old)) > 0) \
_avg = (((_avg << (decay)) - _avg) + (new)) >> (decay); \
else \
_avg = (new); \
(old) = _avg; \
} while (0)
#define MBPS (1ULL * 1000 * 1000)
#define GBPS (MBPS * 1000)
struct rxpoll_time_tbl {
u_int64_t speed; /* downlink speed */
u_int32_t plowat; /* packets low watermark */
u_int32_t phiwat; /* packets high watermark */
u_int32_t blowat; /* bytes low watermark */
u_int32_t bhiwat; /* bytes high watermark */
};
static struct rxpoll_time_tbl rxpoll_tbl[] = {
{ .speed = 10 * MBPS, .plowat = 2, .phiwat = 8, .blowat = (1 * 1024), .bhiwat = (6 * 1024) },
{ .speed = 100 * MBPS, .plowat = 10, .phiwat = 40, .blowat = (4 * 1024), .bhiwat = (64 * 1024) },
{ .speed = 1 * GBPS, .plowat = 10, .phiwat = 40, .blowat = (4 * 1024), .bhiwat = (64 * 1024) },
{ .speed = 10 * GBPS, .plowat = 10, .phiwat = 40, .blowat = (4 * 1024), .bhiwat = (64 * 1024) },
{ .speed = 100 * GBPS, .plowat = 10, .phiwat = 40, .blowat = (4 * 1024), .bhiwat = (64 * 1024) },
{ .speed = 0, .plowat = 0, .phiwat = 0, .blowat = 0, .bhiwat = 0 }
};
decl_lck_mtx_data(static, dlil_thread_sync_lock);
static uint32_t dlil_pending_thread_cnt = 0;
static void
dlil_incr_pending_thread_count(void)
{
LCK_MTX_ASSERT(&dlil_thread_sync_lock, LCK_MTX_ASSERT_NOTOWNED);
lck_mtx_lock(&dlil_thread_sync_lock);
dlil_pending_thread_cnt++;
lck_mtx_unlock(&dlil_thread_sync_lock);
}
static void
dlil_decr_pending_thread_count(void)
{
LCK_MTX_ASSERT(&dlil_thread_sync_lock, LCK_MTX_ASSERT_NOTOWNED);
lck_mtx_lock(&dlil_thread_sync_lock);
VERIFY(dlil_pending_thread_cnt > 0);
dlil_pending_thread_cnt--;
if (dlil_pending_thread_cnt == 0) {
wakeup(&dlil_pending_thread_cnt);
}
lck_mtx_unlock(&dlil_thread_sync_lock);
}
int
proto_hash_value(u_int32_t protocol_family)
{
/*
* dlil_proto_unplumb_all() depends on the mapping between
* the hash bucket index and the protocol family defined
* here; future changes must be applied there as well.
*/
switch (protocol_family) {
case PF_INET:
return 0;
case PF_INET6:
return 1;
case PF_VLAN:
return 2;
case PF_802154:
return 3;
case PF_UNSPEC:
default:
return 4;
}
}
/*
* Caller must already be holding ifnet lock.
*/
static struct if_proto *
find_attached_proto(struct ifnet *ifp, u_int32_t protocol_family)
{
struct if_proto *proto = NULL;
u_int32_t i = proto_hash_value(protocol_family);
ifnet_lock_assert(ifp, IFNET_LCK_ASSERT_OWNED);
if (ifp->if_proto_hash != NULL) {
proto = SLIST_FIRST(&ifp->if_proto_hash[i]);
}
while (proto != NULL && proto->protocol_family != protocol_family) {
proto = SLIST_NEXT(proto, next_hash);
}
if (proto != NULL) {
if_proto_ref(proto);
}
return proto;
}
static void
if_proto_ref(struct if_proto *proto)
{
atomic_add_32(&proto->refcount, 1);
}
extern void if_rtproto_del(struct ifnet *ifp, int protocol);
static void
if_proto_free(struct if_proto *proto)
{
u_int32_t oldval;
struct ifnet *ifp = proto->ifp;
u_int32_t proto_family = proto->protocol_family;
struct kev_dl_proto_data ev_pr_data;
oldval = atomic_add_32_ov(&proto->refcount, -1);
if (oldval > 1) {
return;
}
if (proto->proto_kpi == kProtoKPI_v1) {
if (proto->kpi.v1.detached) {
proto->kpi.v1.detached(ifp, proto->protocol_family);
}
}
if (proto->proto_kpi == kProtoKPI_v2) {
if (proto->kpi.v2.detached) {
proto->kpi.v2.detached(ifp, proto->protocol_family);
}
}
/*
* Cleanup routes that may still be in the routing table for that
* interface/protocol pair.
*/
if_rtproto_del(ifp, proto_family);
ifnet_lock_shared(ifp);
/* No more reference on this, protocol must have been detached */
VERIFY(proto->detached);
/*
* The reserved field carries the number of protocol still attached
* (subject to change)
*/
ev_pr_data.proto_family = proto_family;
ev_pr_data.proto_remaining_count = dlil_ifp_protolist(ifp, NULL, 0);
ifnet_lock_done(ifp);
dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_PROTO_DETACHED,
(struct net_event_data *)&ev_pr_data,
sizeof(struct kev_dl_proto_data));
if (ev_pr_data.proto_remaining_count == 0) {
/*
* The protocol count has gone to zero, mark the interface down.
* This used to be done by configd.KernelEventMonitor, but that
* is inherently prone to races (rdar://problem/30810208).
*/
(void) ifnet_set_flags(ifp, 0, IFF_UP);
(void) ifnet_ioctl(ifp, 0, SIOCSIFFLAGS, NULL);
dlil_post_sifflags_msg(ifp);
}
zfree(dlif_proto_zone, proto);
}
__private_extern__ void
ifnet_lock_assert(struct ifnet *ifp, ifnet_lock_assert_t what)
{
#if !MACH_ASSERT
#pragma unused(ifp)
#endif
unsigned int type = 0;
int ass = 1;
switch (what) {
case IFNET_LCK_ASSERT_EXCLUSIVE:
type = LCK_RW_ASSERT_EXCLUSIVE;
break;
case IFNET_LCK_ASSERT_SHARED:
type = LCK_RW_ASSERT_SHARED;
break;
case IFNET_LCK_ASSERT_OWNED:
type = LCK_RW_ASSERT_HELD;
break;