forked from phaag/nfdump
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathipfix.c
1764 lines (1517 loc) · 63.9 KB
/
ipfix.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*
* Copyright (c) 2014, Peter Haag
* Copyright (c) 2012, Peter Haag
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* * Neither the name of the author nor the names of its contributors may be
* used to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
* $Author:$
*
* $Id:$
*
* $LastChangedRevision:$
*
*/
#include "config.h"
#include <stdio.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <time.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#ifdef HAVE_STDINT_H
#include <stdint.h>
#endif
#include "nffile.h"
#include "nfx.h"
#include "nfnet.h"
#include "nf_common.h"
#include "util.h"
#include "bookkeeper.h"
#include "nfxstat.h"
#include "collector.h"
#include "exporter.h"
#include "ipfix.h"
#ifndef DEVEL
# define dbg_printf(...) /* printf(__VA_ARGS__) */
#else
# define dbg_printf(...) printf(__VA_ARGS__)
#endif
// a few handy macros
#define GET_FLOWSET_ID(p) (Get_val16(p))
#define GET_FLOWSET_LENGTH(p) (Get_val16((void *)((p) + 2)))
#define GET_TEMPLATE_ID(p) (Get_val16(p))
#define GET_TEMPLATE_COUNT(p) (Get_val16((void *)((p) + 2)))
#define GET_OPTION_TEMPLATE_ID(p) (Get_val16(p))
#define GET_OPTION_TEMPLATE_FIELD_COUNT(p) (Get_val16((void *)((p) + 2)))
#define GET_OPTION_TEMPLATE_SCOPE_FIELD_COUNT(p) (Get_val16((void *)((p) + 4)))
/* module limited globals */
/*
* sequence element to move data from data input to output
* a sequence exists for each IPFIX element
*/
typedef struct sequence_map_s {
/* sequence definition:
just move a certain number of bytes -> moveXX
set a certain number of output bytes to zero -> zeroXX
process input data into appropriate output -> AnyName
*/
#define nop 0
#define move8 1
#define move16 2
#define move32 3
#define move40 4
#define move48 5
#define move56 6
#define move64 7
#define move128 8
#define move32_sampling 9
#define move64_sampling 10
#define move_mac 11
#define move_mpls 12
#define Time64Mili 13
#define saveICMP 14
#define zero8 15
#define zero16 16
#define zero32 17
#define zero64 18
#define zero128 19
uint32_t id; // sequence ID as defined above
uint16_t input_offset; // copy/process data at this input offset
uint16_t output_offset; // copy final data to this output offset
void *stack; // optionally copy data onto this stack
} sequence_map_t;
/*
* the IPFIX template records are processed and
* for each template we create a a translation table, which contains
* all information required, to transform the data records from
* the exporter into nfdump internal data structurs.
* All templates are chained in a linked list
*/
typedef struct input_translation_s {
struct input_translation_s *next; // linked list
uint32_t flags; // flags for output record
time_t updated; // timestamp of last update/refresh
uint32_t id; // template ID of exporter domains
uint32_t input_record_size; // size of the input record
uint32_t output_record_size; // required size in nfdump format
// tmp vars needed while processing the data record
uint64_t flow_start; // start time in msec
uint64_t flow_end; // end time in msec
uint32_t ICMP_offset; // offset of ICMP type/code in data stream
uint64_t packets; // total (in)packets - sampling corrected
uint64_t bytes; // total (in)bytes - sampling corrected
uint64_t out_packets; // total out packets - sampling corrected
uint64_t out_bytes; // total out bytes - sampling corrected
// uint32_t src_as_offset;
// uint32_t dst_as_offset;
// uint32_t sampler_offset;
// uint32_t sampler_size;
// uint32_t engine_offset;
uint32_t router_ip_offset;
uint32_t received_offset;
// etension map infos
uint32_t extension_map_changed; // map changed while refreshing?
extension_info_t extension_info; // the extension map reflecting this template
// sequence map information
uint32_t number_of_sequences; // number of sequences for the translate
sequence_map_t *sequence; // sequence map
} input_translation_t;
/*
* All Obervation Domains from all exporter are stored in a linked list
* which uniquely can identify each exporter/Observation Domain
*/
typedef struct exporter_ipfix_domain_s {
struct exporter_ipfix_domain_s *next; // linkes list to next exporter
// generic exporter information
exporter_info_record_t info;
uint64_t packets; // number of packets sent by this exporter
uint64_t flows; // number of flow records sent by this exporter
uint32_t sequence_failure; // number of sequence failues
// generic sampler
generic_sampler_t *sampler;
// exporter parameters
uint32_t ExportTime;
// Current sequence number
uint32_t PacketSequence;
// statistics
uint64_t TemplateRecords; // stat counter
uint64_t DataRecords; // stat counter
// linked list of all templates sent by this exporter
input_translation_t *input_translation_table;
// in order to prevent search through all lists keep
// the last template we processed as a cache
input_translation_t *current_table;
} exporter_ipfix_domain_t;
static struct ipfix_element_map_s {
uint16_t id; // IPFIX element id
uint16_t length; // type of this element ( input length )
uint16_t out_length; // type of this element ( output length )
uint32_t sequence; //
uint32_t zero_sequence; //
uint16_t extension; // maps into nfdump extension ID
} ipfix_element_map[] = {
{0, 0, 0},
{ IPFIX_octetDeltaCount, _8bytes, _8bytes, move64_sampling, zero64, COMMON_BLOCK },
{ IPFIX_octetDeltaCount, _4bytes, _8bytes, move32_sampling, zero64, COMMON_BLOCK },
{ IPFIX_packetDeltaCount, _8bytes, _8bytes, move64_sampling, zero64, COMMON_BLOCK },
{ IPFIX_packetDeltaCount, _4bytes, _8bytes, move32_sampling, zero64, COMMON_BLOCK },
{ IPFIX_octetTotalCount, _8bytes, _8bytes, move64_sampling, zero64, COMMON_BLOCK },
{ IPFIX_octetTotalCount, _4bytes, _8bytes, move32_sampling, zero64, COMMON_BLOCK },
{ IPFIX_packetTotalCount, _8bytes, _8bytes, move64_sampling, zero64, COMMON_BLOCK },
{ IPFIX_packetTotalCount, _4bytes, _8bytes, move32_sampling, zero64, COMMON_BLOCK },
{ IPFIX_protocolIdentifier, _1byte, _1byte, move8, zero8, COMMON_BLOCK },
{ IPFIX_ipClassOfService, _1byte, _1byte, move8, zero8, COMMON_BLOCK },
{ IPFIX_tcpControlBits, _1byte, _1byte, move8, zero8, COMMON_BLOCK },
{ IPFIX_SourceTransportPort, _2bytes, _2bytes, move16, zero16, COMMON_BLOCK },
{ IPFIX_SourceIPv4Address, _4bytes, _4bytes, move32, zero32, COMMON_BLOCK },
{ IPFIX_SourceIPv4PrefixLength, _1byte, _1byte, move8, zero8, EX_MULIPLE },
{ IPFIX_ingressInterface, _4bytes, _4bytes, move32, zero32, EX_IO_SNMP_4 },
{ IPFIX_ingressInterface, _2bytes, _2bytes, move16, zero16, EX_IO_SNMP_2 },
{ IPFIX_DestinationTransportPort, _2bytes, _2bytes, move16, zero16, COMMON_BLOCK },
{ IPFIX_DestinationIPv4Address, _4bytes, _4bytes, move32, zero32, COMMON_BLOCK },
{ IPFIX_DestinationIPv4PrefixLength, _1byte, _1byte, move8, zero8, EX_MULIPLE },
{ IPFIX_egressInterface, _4bytes, _4bytes, move32, zero32, EX_IO_SNMP_4 },
{ IPFIX_egressInterface, _2bytes, _2bytes, move16, zero16, EX_IO_SNMP_2 },
{ IPFIX_ipNextHopIPv4Address, _4bytes, _4bytes, move32, zero32, EX_NEXT_HOP_v4 },
{ IPFIX_bgpSourceAsNumber, _4bytes, _4bytes, move32, zero32, EX_AS_4 },
{ IPFIX_bgpSourceAsNumber, _2bytes, _2bytes, move16, zero16, EX_AS_2 },
{ IPFIX_bgpDestinationAsNumber, _4bytes, _4bytes, move32, zero32, EX_AS_4 },
{ IPFIX_bgpDestinationAsNumber, _2bytes, _2bytes, move16, zero16, EX_AS_2 },
{ IPFIX_bgpNextHopIPv4Address, _4bytes, _4bytes, move32, zero32, EX_NEXT_HOP_BGP_v4},
{ IPFIX_flowEndSysUpTime, _4bytes, _4bytes, nop, nop, COMMON_BLOCK },
{ IPFIX_flowStartSysUpTime, _4bytes, _4bytes, nop, nop, COMMON_BLOCK },
{ IPFIX_postOctetDeltaCount, _8bytes, _8bytes, move64, zero64, EX_OUT_BYTES_8 },
{ IPFIX_postOctetDeltaCount, _4bytes, _4bytes, move32, zero32, EX_OUT_BYTES_4 },
{ IPFIX_postPacketDeltaCount, _8bytes, _8bytes, move64, zero64, EX_OUT_PKG_8 },
{ IPFIX_postPacketDeltaCount, _4bytes, _4bytes, move32, zero32, EX_OUT_PKG_4 },
{ IPFIX_SourceIPv6Address, _16bytes, _16bytes, move128, zero128, COMMON_BLOCK },
{ IPFIX_DestinationIPv6Address, _16bytes, _16bytes, move128, zero128, COMMON_BLOCK },
{ IPFIX_SourceIPv6PrefixLength, _1byte, _1byte, move8, zero8, EX_MULIPLE },
{ IPFIX_DestinationIPv6PrefixLength, _1byte, _1byte, move8, zero8, EX_MULIPLE },
{ IPFIX_flowLabelIPv6, _4bytes, _4bytes, nop, nop, COMMON_BLOCK },
{ IPFIX_icmpTypeCodeIPv4, _2bytes, _2bytes, nop, nop, COMMON_BLOCK },
{ IPFIX_postIpClassOfService, _1byte, _1byte, move8, zero8, EX_MULIPLE },
{ IPFIX_SourceMacAddress, _6bytes, _8bytes, move_mac, zero64, EX_MAC_1},
{ IPFIX_postDestinationMacAddress, _6bytes, _8bytes, move_mac, zero64, EX_MAC_1},
{ IPFIX_vlanId, _2bytes, _2bytes, move16, zero16, EX_VLAN},
{ IPFIX_postVlanId, _2bytes, _2bytes, move16, zero16, EX_VLAN},
{ IPFIX_flowDirection, _1byte, _1byte, move8, zero8, EX_MULIPLE },
{ IPFIX_ipNextHopIPv6Address, _16bytes, _16bytes, move128, zero128, EX_NEXT_HOP_v6},
{ IPFIX_bgpNextHopIPv6Address, _16bytes, _16bytes, move128, zero128, EX_NEXT_HOP_BGP_v6},
{ IPFIX_mplsTopLabelStackSection, _3bytes, _4bytes, move_mpls, zero32, EX_MPLS},
{ IPFIX_mplsLabelStackSection2, _3bytes, _4bytes, move_mpls, zero32, EX_MPLS},
{ IPFIX_mplsLabelStackSection3, _3bytes, _4bytes, move_mpls, zero32, EX_MPLS},
{ IPFIX_mplsLabelStackSection4, _3bytes, _4bytes, move_mpls, zero32, EX_MPLS},
{ IPFIX_mplsLabelStackSection5, _3bytes, _4bytes, move_mpls, zero32, EX_MPLS},
{ IPFIX_mplsLabelStackSection6, _3bytes, _4bytes, move_mpls, zero32, EX_MPLS},
{ IPFIX_mplsLabelStackSection7, _3bytes, _4bytes, move_mpls, zero32, EX_MPLS},
{ IPFIX_mplsLabelStackSection8, _3bytes, _4bytes, move_mpls, zero32, EX_MPLS},
{ IPFIX_mplsLabelStackSection9, _3bytes, _4bytes, move_mpls, zero32, EX_MPLS},
{ IPFIX_mplsLabelStackSection10, _3bytes, _4bytes, move_mpls, zero32, EX_MPLS},
{ IPFIX_DestinationMacAddress, _6bytes, _8bytes, move_mac, zero64, EX_MAC_2},
{ IPFIX_postSourceMacAddress, _6bytes, _8bytes, move_mac, zero64, EX_MAC_2},
{ IPFIX_flowStartMilliseconds, _8bytes, _8bytes, Time64Mili, zero32, COMMON_BLOCK},
{ IPFIX_flowEndMilliseconds, _8bytes, _8bytes, Time64Mili, zero32, COMMON_BLOCK},
{0, 0, 0}
};
// cache to be used while parsing a template
static struct cache_s {
struct element_param_s {
uint16_t index;
uint16_t found;
uint16_t offset;
uint16_t length;
} *lookup_info;
uint32_t max_ipfix_elements;
uint32_t *common_extensions;
} cache;
// module limited globals
static uint32_t processed_records;
// externals
extern int verbose;
extern uint32_t Max_num_extensions;
extern extension_descriptor_t extension_descriptor[];
extern uint32_t overwrite_sampling;
extern uint32_t exporter_sysid;
// prototypes
static input_translation_t *add_translation_table(exporter_ipfix_domain_t *exporter, uint16_t id);
static void remove_translation_table(FlowSource_t *fs, exporter_ipfix_domain_t *exporter, uint16_t id);
static void remove_all_translation_tables(exporter_ipfix_domain_t *exporter);
static inline exporter_ipfix_domain_t *GetExporter(FlowSource_t *fs, ipfix_header_t *ipfix_header);
static inline uint32_t MapElement(uint16_t Type, uint16_t Length, uint16_t Offset);
static inline void PushSequence(input_translation_t *table, uint16_t Type, uint32_t *offset, void *stack);
static inline void Process_ipfix_templates(exporter_ipfix_domain_t *exporter, void *flowset_header, uint32_t size_left, FlowSource_t *fs);
static inline void Process_ipfix_template_add(exporter_ipfix_domain_t *exporter, void *DataPtr, uint32_t size_left, FlowSource_t *fs);
static inline void Process_ipfix_template_withdraw(exporter_ipfix_domain_t *exporter, void *DataPtr, uint32_t size_left, FlowSource_t *fs);
#include "inline.c"
#include "nffile_inline.c"
int Init_IPFIX(void) {
int i;
cache.lookup_info = (struct element_param_s *)calloc(65536, sizeof(struct element_param_s));
cache.common_extensions = (uint32_t *)malloc((Max_num_extensions+1)*sizeof(uint32_t));
if ( !cache.common_extensions || !cache.lookup_info ) {
LogError("Process_ipfix: Panic! malloc(): %s line %d: %s", __FILE__, __LINE__, strerror (errno));
return 0;
}
// init the helper element table
for (i=1; ipfix_element_map[i].id != 0; i++ ) {
uint32_t Type = ipfix_element_map[i].id;
// multiple same type - save first index only
// iterate through same Types afterwards
if ( cache.lookup_info[Type].index == 0 )
cache.lookup_info[Type].index = i;
}
cache.max_ipfix_elements = i;
LogError("Init IPFIX: Max number of IPFIX tags: %u", cache.max_ipfix_elements);
return 1;
} // End of Init_IPFIX
static inline exporter_ipfix_domain_t *GetExporter(FlowSource_t *fs, ipfix_header_t *ipfix_header) {
#define IP_STRING_LEN 40
char ipstr[IP_STRING_LEN];
exporter_ipfix_domain_t **e = (exporter_ipfix_domain_t **)&(fs->exporter_data);
uint32_t ObservationDomain = ntohl(ipfix_header->ObservationDomain);
while ( *e ) {
if ( (*e)->info.id == ObservationDomain && (*e)->info.version == 10 &&
(*e)->info.ip.v6[0] == fs->ip.v6[0] && (*e)->info.ip.v6[1] == fs->ip.v6[1])
return *e;
e = &((*e)->next);
}
if ( fs->sa_family == AF_INET ) {
uint32_t _ip = htonl(fs->ip.v4);
inet_ntop(AF_INET, &_ip, ipstr, sizeof(ipstr));
} else if ( fs->sa_family == AF_INET6 ) {
uint64_t _ip[2];
_ip[0] = htonll(fs->ip.v6[0]);
_ip[1] = htonll(fs->ip.v6[1]);
inet_ntop(AF_INET6, &_ip, ipstr, sizeof(ipstr));
} else {
strncpy(ipstr, "<unknown>", IP_STRING_LEN);
}
// nothing found
*e = (exporter_ipfix_domain_t *)malloc(sizeof(exporter_ipfix_domain_t));
if ( !(*e)) {
LogError("Process_ipfix: Panic! malloc() %s line %d: %s", __FILE__, __LINE__, strerror (errno));
return NULL;
}
memset((void *)(*e), 0, sizeof(exporter_ipfix_domain_t));
(*e)->info.header.type = ExporterInfoRecordType;
(*e)->info.header.size = sizeof(exporter_info_record_t);
(*e)->info.id = ObservationDomain;
(*e)->info.ip = fs->ip;
(*e)->info.sa_family = fs->sa_family;
(*e)->info.version = 10;
(*e)->info.sysid = 0;
(*e)->TemplateRecords = 0;
(*e)->DataRecords = 0;
(*e)->sequence_failure = 0;
(*e)->next = NULL;
(*e)->sampler = NULL;
FlushInfoExporter(fs, &((*e)->info));
dbg_printf("[%u] New exporter: SysID: %u, Observation domain %u from: %s\n",
ObservationDomain, (*e)->info.sysid, ObservationDomain, ipstr);
LogInfo("Process_ipfix: New exporter: SysID: %u, Observation domain %u from: %s\n",
(*e)->info.sysid, ObservationDomain, ipstr);
return (*e);
} // End of GetExporter
static inline uint32_t MapElement(uint16_t Type, uint16_t Length, uint16_t Offset) {
int index;
index = cache.lookup_info[Type].index;
if ( index ) {
while ( index && ipfix_element_map[index].id == Type ) {
if ( Length == ipfix_element_map[index].length ) {
cache.lookup_info[Type].found = 1;
cache.lookup_info[Type].offset = Offset;
cache.lookup_info[Type].length = Length;
cache.lookup_info[Type].index = index;
dbg_printf("found extension %u for type: %u, input length: %u output length: %u Extension: %u\n",
ipfix_element_map[index].extension, ipfix_element_map[index].id,
ipfix_element_map[index].length, ipfix_element_map[index].out_length, ipfix_element_map[index].extension);
return ipfix_element_map[index].extension;
}
index++;
}
}
dbg_printf("Skip unknown element type: %u, Length: %u\n", Type, Length);
return 0;
} // End of MapElement
static inline input_translation_t *GetTranslationTable(exporter_ipfix_domain_t *exporter, uint16_t id) {
input_translation_t *table;
if ( exporter->current_table && ( exporter->current_table->id == id ) )
return exporter->current_table;
table = exporter->input_translation_table;
while ( table ) {
if ( table->id == id ) {
exporter->current_table = table;
return table;
}
table = table->next;
}
dbg_printf("[%u] Get translation table %u: %s\n", exporter->info.id, id, table == NULL ? "not found" : "found");
exporter->current_table = table;
return table;
} // End of GetTranslationTable
static input_translation_t *add_translation_table(exporter_ipfix_domain_t *exporter, uint16_t id) {
input_translation_t **table;
table = &(exporter->input_translation_table);
while ( *table ) {
table = &((*table)->next);
}
// Allocate enough space for all potential ipfix tags, which we support
// so template refreshing may change the table size without danger of overflowing
*table = calloc(1, sizeof(input_translation_t));
if ( !(*table) ) {
LogError("Process_ipfix: Panic! calloc() %s line %d: %s", __FILE__, __LINE__, strerror (errno));
return NULL;
}
(*table)->sequence = calloc(cache.max_ipfix_elements, sizeof(sequence_map_t));
if ( !(*table)->sequence ) {
LogError("Process_ipfix: Panic! malloc() %s line %d: %s", __FILE__, __LINE__, strerror (errno));
return NULL;
}
(*table)->id = id;
(*table)->next = NULL;
dbg_printf("[%u] Get new translation table %u\n", exporter->info.id, id);
return *table;
} // End of add_translation_table
static void remove_translation_table(FlowSource_t *fs, exporter_ipfix_domain_t *exporter, uint16_t id) {
input_translation_t *table, *parent;
LogInfo("Process_ipfix: [%u] Withdraw template id: %i",
exporter->info.id, id);
parent = NULL;
table = exporter->input_translation_table;
while ( table && ( table->id != id ) ) {
parent = table;
table = table->next;
}
if ( table == NULL ) {
LogError("Process_ipfix: [%u] Withdraw template id: %i. translation table not found",
exporter->info.id, id);
return;
}
dbg_printf("\n[%u] Withdraw template ID: %u\n", exporter->info.id, table->id);
// clear table cache, if this is the table to delete
if (exporter->current_table == table)
exporter->current_table = NULL;
if ( parent ) {
// remove table from list
parent->next = table->next;
} else {
// last table removed
exporter->input_translation_table = NULL;
}
RemoveExtensionMap(fs, table->extension_info.map);
free(table->sequence);
free(table->extension_info.map);
free(table);
} // End of remove_translation_table
static void remove_all_translation_tables(exporter_ipfix_domain_t *exporter) {
input_translation_t *table, *next;
LogInfo("Process_ipfix: Withdraw all templates from observation domain %u\n",
exporter->info.id);
table = exporter->input_translation_table;
while ( table ) {
next = table->next;
dbg_printf("\n[%u] Withdraw template ID: %u\n", exporter->info.id, table->id);
free(table->sequence);
free(table->extension_info.map);
free(table);
table = next;
}
// clear references
exporter->input_translation_table = NULL;
exporter->current_table = NULL;
} // End of remove_all_translation_tables
static inline void PushSequence(input_translation_t *table, uint16_t Type, uint32_t *offset, void *stack) {
uint32_t i = table->number_of_sequences;
uint32_t index = cache.lookup_info[Type].index;
if ( table->number_of_sequences >= cache.max_ipfix_elements ) {
LogError("Process_ipfix: Software bug! Sequence table full. at %s line %d",
__FILE__, __LINE__);
dbg_printf("Software bug! Sequence table full. at %s line %d",
__FILE__, __LINE__);
return;
}
if ( cache.lookup_info[Type].found ) {
table->sequence[i].id = ipfix_element_map[index].sequence;
table->sequence[i].input_offset = cache.lookup_info[Type].offset;
table->sequence[i].output_offset = *offset;
table->sequence[i].stack = stack;
} else {
table->sequence[i].id = ipfix_element_map[index].zero_sequence;
table->sequence[i].input_offset = 0;
table->sequence[i].output_offset = *offset;
table->sequence[i].stack = NULL;
}
dbg_printf("Push: sequence: %u, Type: %u, length: %u, out length: %u, id: %u, in offset: %u, out offset: %u\n",
i, Type, ipfix_element_map[index].length, ipfix_element_map[index].out_length, table->sequence[i].id,
table->sequence[i].input_offset, table->sequence[i].output_offset);
table->number_of_sequences++;
(*offset) += ipfix_element_map[index].out_length;
} // End of PushSequence
static input_translation_t *setup_translation_table (exporter_ipfix_domain_t *exporter, uint16_t id, uint16_t input_record_size) {
input_translation_t *table;
extension_map_t *extension_map;
uint32_t i, ipv6, offset, next_extension;
size_t size_required;
ipv6 = 0;
table = GetTranslationTable(exporter, id);
if ( !table ) {
LogInfo("Process_ipfix: [%u] Add template %u", exporter->info.id, id);
table = add_translation_table(exporter, id);
if ( !table ) {
return NULL;
}
// Add an extension map
// The number of extensions for this template is currently unknown
// Allocate enough space for all configured extensions - some may be unused later
// make sure memory is 4byte alligned
size_required = Max_num_extensions * sizeof(uint16_t) + sizeof(extension_map_t);
size_required = (size_required + 3) &~(size_t)3;
extension_map = malloc(size_required);
if ( !extension_map ) {
LogError("Process_ipfix: Panic! malloc() error in %s line %d: %s", __FILE__, __LINE__, strerror (errno));
return NULL;
}
extension_map->type = ExtensionMapType;
// Set size to an empty table - will be adapted later
extension_map->size = sizeof(extension_map_t);
extension_map->map_id = INIT_ID;
// packed record size still unknown at this point - will be added later
extension_map->extension_size = 0;
table->extension_info.map = extension_map;
table->extension_map_changed = 1;
table->number_of_sequences = 0;
} else {
extension_map = table->extension_info.map;
// reset size/extension size - it's refreshed automatically
extension_map->size = sizeof(extension_map_t);
extension_map->extension_size = 0;
dbg_printf("[%u] Refresh template %u\n", exporter->info.id, id);
// very noisy with somee exporters
dbg_printf("[%u] Refresh template %u\n", exporter->info.id, id);
}
// clear current table
memset((void *)table->sequence, 0, cache.max_ipfix_elements * sizeof(sequence_map_t));
table->number_of_sequences = 0;
table->updated = time(NULL);
// IPFIX only has 64bit counters
table->flags = 0;
SetFlag(table->flags, FLAG_PKG_64);
SetFlag(table->flags, FLAG_BYTES_64);
table->ICMP_offset = 0;
// table->sampler_offset = 0;
// table->sampler_size = 0;
// table->engine_offset = 0;
table->router_ip_offset = 0;
table->received_offset = 0;
dbg_printf("[%u] Build sequence table %u\n", exporter->info.id, id);
// fill table
table->id = id;
/*
* common data block: The common record is expected in the output stream. If not available
* in the template, fill values with 0
*/
// All required extensions
// The order we Push all ipfix elements, must corresponde to the structure of the common record
// followed by all available extension in the extension map
offset = BYTE_OFFSET_first;
PushSequence( table, IPFIX_flowStartMilliseconds, &offset, &table->flow_start);
offset = BYTE_OFFSET_first + 4;
PushSequence( table, IPFIX_flowEndMilliseconds, &offset, &table->flow_end);
offset = BYTE_OFFSET_first + 8;
offset +=1; // Skip netflow v9 fwd status
PushSequence( table, IPFIX_tcpControlBits, &offset, NULL);
PushSequence( table, IPFIX_protocolIdentifier, &offset, NULL);
PushSequence( table, IPFIX_ipClassOfService, &offset, NULL);
PushSequence( table, IPFIX_SourceTransportPort, &offset, NULL);
PushSequence( table, IPFIX_DestinationTransportPort, &offset, NULL);
// skip exporter_sysid and reserved
offset += 4;
/* IP addresss record
* This record is expected in the output stream. If not available
* in the template, assume empty v4 address.
*/
if ( cache.lookup_info[IPFIX_SourceIPv4Address].found ) {
// IPv4 addresses
PushSequence( table, IPFIX_SourceIPv4Address, &offset, NULL);
PushSequence( table, IPFIX_DestinationIPv4Address, &offset, NULL);
} else if ( cache.lookup_info[IPFIX_SourceIPv6Address].found ) {
// IPv6 addresses
PushSequence( table, IPFIX_SourceIPv6Address, &offset, NULL);
PushSequence( table, IPFIX_DestinationIPv6Address, &offset, NULL);
// mark IPv6
SetFlag(table->flags, FLAG_IPV6_ADDR);
ipv6 = 1;
} else {
// should not happen, assume empty IPv4 addresses, zero
PushSequence( table, IPFIX_SourceIPv4Address, &offset, NULL);
PushSequence( table, IPFIX_DestinationIPv4Address, &offset, NULL);
}
// decide between Delta or Total counters - prefer Total if available
if ( cache.lookup_info[IPFIX_packetTotalCount].found )
PushSequence( table, IPFIX_packetTotalCount, &offset, &table->packets);
else
PushSequence( table, IPFIX_packetDeltaCount, &offset, &table->packets);
SetFlag(table->flags, FLAG_PKG_64);
if ( cache.lookup_info[IPFIX_octetTotalCount].found )
PushSequence( table, IPFIX_octetTotalCount, &offset, &table->bytes);
else
PushSequence( table, IPFIX_octetDeltaCount, &offset, &table->bytes);
SetFlag(table->flags, FLAG_BYTES_64);
// Optional extensions
next_extension = 0;
for (i=4; extension_descriptor[i].id; i++ ) {
uint32_t map_index = i;
if ( cache.common_extensions[i] == 0 )
continue;
switch(i) {
case EX_IO_SNMP_2:
PushSequence( table, IPFIX_ingressInterface, &offset, NULL);
PushSequence( table, IPFIX_egressInterface, &offset, NULL);
break;
case EX_IO_SNMP_4:
PushSequence( table, IPFIX_ingressInterface, &offset, NULL);
PushSequence( table, IPFIX_egressInterface, &offset, NULL);
break;
case EX_AS_2:
PushSequence( table, IPFIX_bgpSourceAsNumber, &offset, NULL);
PushSequence( table, IPFIX_bgpDestinationAsNumber, &offset, NULL);
break;
case EX_AS_4:
PushSequence( table, IPFIX_bgpSourceAsNumber, &offset, NULL);
PushSequence( table, IPFIX_bgpDestinationAsNumber, &offset, NULL);
break;
case EX_MULIPLE:
PushSequence( table, IPFIX_postIpClassOfService, &offset, NULL);
PushSequence( table, IPFIX_flowDirection, &offset, NULL);
if ( ipv6 ) {
// IPv6
PushSequence( table, IPFIX_SourceIPv6PrefixLength, &offset, NULL);
PushSequence( table, IPFIX_DestinationIPv6PrefixLength, &offset, NULL);
} else {
// IPv4
PushSequence( table, IPFIX_SourceIPv4PrefixLength, &offset, NULL);
PushSequence( table, IPFIX_DestinationIPv4PrefixLength, &offset, NULL);
}
break;
case EX_NEXT_HOP_v4:
PushSequence( table, IPFIX_ipNextHopIPv4Address, &offset, NULL);
break;
case EX_NEXT_HOP_v6:
PushSequence( table, IPFIX_ipNextHopIPv6Address, &offset, NULL);
SetFlag(table->flags, FLAG_IPV6_NH);
break;
case EX_NEXT_HOP_BGP_v4:
PushSequence( table, IPFIX_bgpNextHopIPv4Address, &offset, NULL);
break;
case EX_NEXT_HOP_BGP_v6:
PushSequence( table, IPFIX_bgpNextHopIPv6Address, &offset, NULL);
SetFlag(table->flags, FLAG_IPV6_NHB);
break;
case EX_VLAN:
PushSequence( table, IPFIX_vlanId, &offset, NULL);
PushSequence( table, IPFIX_postVlanId, &offset, NULL);
break;
case EX_OUT_PKG_4:
PushSequence( table, IPFIX_postPacketDeltaCount, &offset, NULL);
break;
case EX_OUT_PKG_8:
PushSequence( table, IPFIX_postPacketDeltaCount, &offset, NULL);
break;
case EX_OUT_BYTES_4:
PushSequence( table, IPFIX_postOctetDeltaCount, &offset, NULL);
break;
case EX_OUT_BYTES_8:
PushSequence( table, IPFIX_postOctetDeltaCount, &offset, NULL);
break;
case EX_AGGR_FLOWS_8:
break;
case EX_MAC_1:
PushSequence( table, IPFIX_SourceMacAddress, &offset, NULL);
PushSequence( table, IPFIX_postDestinationMacAddress, &offset, NULL);
break;
case EX_MAC_2:
PushSequence( table, IPFIX_DestinationMacAddress, &offset, NULL);
PushSequence( table, IPFIX_postSourceMacAddress, &offset, NULL);
break;
case EX_MPLS:
PushSequence( table, IPFIX_mplsTopLabelStackSection, &offset, NULL);
PushSequence( table, IPFIX_mplsLabelStackSection2, &offset, NULL);
PushSequence( table, IPFIX_mplsLabelStackSection3, &offset, NULL);
PushSequence( table, IPFIX_mplsLabelStackSection4, &offset, NULL);
PushSequence( table, IPFIX_mplsLabelStackSection5, &offset, NULL);
PushSequence( table, IPFIX_mplsLabelStackSection6, &offset, NULL);
PushSequence( table, IPFIX_mplsLabelStackSection7, &offset, NULL);
PushSequence( table, IPFIX_mplsLabelStackSection8, &offset, NULL);
PushSequence( table, IPFIX_mplsLabelStackSection9, &offset, NULL);
PushSequence( table, IPFIX_mplsLabelStackSection10, &offset, NULL);
break;
case EX_ROUTER_IP_v4:
case EX_ROUTER_IP_v6:
if ( exporter->info.sa_family == PF_INET6 ) {
table->router_ip_offset = offset;
dbg_printf("Router IPv6: offset: %u, olen: %u\n", offset, 16 );
// not an entry for the translateion table.
// but reserve space in the output record for IPv6
offset += 16;
SetFlag(table->flags, FLAG_IPV6_EXP);
map_index = EX_ROUTER_IP_v6;
} else {
table->router_ip_offset = offset;
dbg_printf("Router IPv4: offset: %u, olen: %u\n", offset, 4 );
// not an entry for the translateion table.
// but reserve space in the output record for IPv4
offset += 4;
ClearFlag(table->flags, FLAG_IPV6_EXP);
map_index = EX_ROUTER_IP_v4;
}
break;
case EX_ROUTER_ID:
// no value in ipfix
break;
case EX_RECEIVED:
table->received_offset = offset;
dbg_printf("Received offset: %u\n", offset);
offset += 8;
break;
}
extension_map->size += sizeof(uint16_t);
extension_map->extension_size += extension_descriptor[map_index].size;
// found extension in map_index must be the same as in map - otherwise map is dirty
if ( extension_map->ex_id[next_extension] != map_index ) {
// dirty map - needs to be refreshed in output stream
extension_map->ex_id[next_extension] = map_index;
table->extension_map_changed = 1;
}
next_extension++;
}
extension_map->ex_id[next_extension++] = 0;
// make sure map is aligned
if ( extension_map->size & 0x3 ) {
extension_map->ex_id[next_extension] = 0;
extension_map->size = ( extension_map->size + 3 ) &~ 0x3;
}
table->output_record_size = offset;
table->input_record_size = input_record_size;
// for netflow historical reason, ICMP type/code goes into dst port field
// remember offset, for decoding
if ( cache.lookup_info[IPFIX_icmpTypeCodeIPv4].found && cache.lookup_info[IPFIX_icmpTypeCodeIPv4].length == 2 ) {
table->ICMP_offset = cache.lookup_info[IPFIX_icmpTypeCodeIPv4].offset;
}
#ifdef DEVEL
if ( table->extension_map_changed ) {
printf("Extension Map id=%u changed!\n", extension_map->map_id);
} else {
printf("[%u] template %u unchanged\n", exporter->info.id, id);
}
printf("Process_ipfix: Check extension map: id: %d, size: %u, extension_size: %u\n",
extension_map->map_id, extension_map->size, extension_map->extension_size);
{ int i;
for (i=0; i<table->number_of_sequences; i++ ) {
printf("Sequence %i: id: %u, in offset: %u, out offset: %u, stack: %llu\n",
i, table->sequence[i].id, table->sequence[i].input_offset, table->sequence[i].output_offset,
(unsigned long long)table->sequence[i].stack);
}
printf("Flags: 0x%x\n", table->flags);
printf("Input record size: %u, output record size: %u\n",
table->input_record_size, table->output_record_size);
}
PrintExtensionMap(extension_map);
#endif
return table;
} // End of setup_translation_table
static inline void Process_ipfix_templates(exporter_ipfix_domain_t *exporter, void *flowset_header, uint32_t size_left, FlowSource_t *fs) {
ipfix_template_record_t *ipfix_template_record;
void *DataPtr;
uint32_t id, count;
size_left -= 4; // subtract message header
DataPtr = flowset_header + 4;
ipfix_template_record = (ipfix_template_record_t *)DataPtr;
id = ntohs(ipfix_template_record->TemplateID);
count = ntohs(ipfix_template_record->FieldCount);
if ( count == 0 ) {
// withdraw template
Process_ipfix_template_withdraw(exporter, DataPtr, size_left, fs);
} else {
// refresh/add templates
Process_ipfix_template_add(exporter, DataPtr, size_left, fs);
}
} // End of Process_ipfix_templates
static inline void Process_ipfix_template_add(exporter_ipfix_domain_t *exporter, void *DataPtr, uint32_t size_left, FlowSource_t *fs) {
input_translation_t *translation_table;
ipfix_template_record_t *ipfix_template_record;
ipfix_template_elements_std_t *NextElement;
int i;
uint16_t Offset = 0;
// a template flowset can contain multiple records ( templates )
while ( size_left ) {
if ( size_left && size_left < 4 ) {
LogError("Process_ipfix [%u] Template size error at %s line %u" ,
exporter->info.id, __FILE__, __LINE__, strerror (errno));
size_left = 0;
continue;
}
// clear helper tables
memset((void *)cache.common_extensions, 0, (Max_num_extensions+1)*sizeof(uint32_t));
memset((void *)cache.lookup_info, 0, 65536 * sizeof(struct element_param_s));
for (i=1; ipfix_element_map[i].id != 0; i++ ) {
uint32_t Type = ipfix_element_map[i].id;
if ( ipfix_element_map[i].id == ipfix_element_map[i-1].id )
continue;
cache.lookup_info[Type].index = i;
// other elements cleard be memset
}
uint32_t id, count, size_required;
uint32_t num_extensions = 0;
// map next record.
ipfix_template_record = (ipfix_template_record_t *)DataPtr;
size_left -= 4;
id = ntohs(ipfix_template_record->TemplateID);
count = ntohs(ipfix_template_record->FieldCount);
dbg_printf("\n[%u] Template ID: %u\n", exporter->info.id, id);
dbg_printf("FieldCount: %u buffersize: %u\n", count, size_left);
// assume all elements in template are std elements. correct this value, if we find an enterprise element
size_required = 4*count;
if ( size_left < size_required ) {
// if we fail this check, this flowset must be skipped.
LogError("Process_ipfix: [%u] Not enough data for template elements! required: %i, left: %u",
exporter->info.id, size_required, size_left);
dbg_printf("ERROR: Not enough data for template elements! required: %i, left: %u", size_required, size_left);
return;
}
Offset = 0;
// process all elements in this record
NextElement = (ipfix_template_elements_std_t *)ipfix_template_record->elements;
for ( i=0; i<count; i++ ) {
uint16_t Type, Length;
uint32_t ext_id;
int Enterprise;
Type = ntohs(NextElement->Type);
Length = ntohs(NextElement->Length);
Enterprise = Type & 0x8000 ? 1 : 0;
ext_id = MapElement(Type, Length, Offset);
// do we store this extension? enabled != 0
// more than 1 v9 tag may map to an extension - so count this extension once only
if ( ext_id && extension_descriptor[ext_id].enabled ) {
if ( cache.common_extensions[ext_id] == 0 ) {
cache.common_extensions[ext_id] = 1;
num_extensions++;
}
}
Offset += Length;
if ( Enterprise ) {
ipfix_template_elements_e_t *e = (ipfix_template_elements_e_t *)NextElement;
size_required += 4; // ad 4 for enterprise value
if ( size_left < size_required ) {
LogError("Process_ipfix: [%u] Not enough data for template elements! required: %i, left: %u",
exporter->info.id, size_required, size_left);
dbg_printf("ERROR: Not enough data for template elements! required: %i, left: %u", size_required, size_left);
return;
}
dbg_printf(" [%i] Enterprise: 1, Type: %u, Length %u EnterpriseNumber: %u\n", i, Type, Length, ntohl(e->EnterpriseNumber));
e++;
NextElement = (ipfix_template_elements_std_t *)e;
} else {
dbg_printf(" [%i] Enterprise: 0, Type: %u, Length %u\n", i, Type, Length);
NextElement++;
}
}
dbg_printf("Processed: %u\n", size_required);
// as the router IP address extension is not part announced in a template, we need to deal with it here
if ( extension_descriptor[EX_ROUTER_IP_v4].enabled ) {