forked from mozilla/gecko-dev
-
Notifications
You must be signed in to change notification settings - Fork 1
/
Encoding.h
1316 lines (1235 loc) · 54.4 KB
/
Encoding.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// Copyright 2015-2016 Mozilla Foundation. See the COPYRIGHT
// file at the top-level directory of this distribution.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
// Adapted from third_party/rust/encoding_c/include/encoding_rs_cpp.h, so the
// "top-level directory" in the above notice refers to
// third_party/rust/encoding_c/.
#ifndef mozilla_Encoding_h
#define mozilla_Encoding_h
#include "mozilla/CheckedInt.h"
#include "mozilla/NotNull.h"
#include "mozilla/Span.h"
#include "mozilla/Tuple.h"
#include "nsString.h"
namespace mozilla {
class Encoding;
class Decoder;
class Encoder;
}; // namespace mozilla
#define ENCODING_RS_ENCODING mozilla::Encoding
#define ENCODING_RS_NOT_NULL_CONST_ENCODING_PTR \
mozilla::NotNull<const mozilla::Encoding*>
#define ENCODING_RS_ENCODER mozilla::Encoder
#define ENCODING_RS_DECODER mozilla::Decoder
#include "encoding_rs.h"
extern "C" {
nsresult mozilla_encoding_decode_to_nsstring(mozilla::Encoding const** encoding,
uint8_t const* src, size_t src_len,
nsAString* dst);
nsresult mozilla_encoding_decode_to_nsstring_with_bom_removal(
mozilla::Encoding const* encoding, uint8_t const* src, size_t src_len,
nsAString* dst);
nsresult mozilla_encoding_decode_to_nsstring_without_bom_handling(
mozilla::Encoding const* encoding, uint8_t const* src, size_t src_len,
nsAString* dst);
nsresult
mozilla_encoding_decode_to_nsstring_without_bom_handling_and_without_replacement(
mozilla::Encoding const* encoding, uint8_t const* src, size_t src_len,
nsAString* dst);
nsresult mozilla_encoding_encode_from_utf16(mozilla::Encoding const** encoding,
char16_t const* src, size_t src_len,
nsACString* dst);
nsresult mozilla_encoding_decode_to_nscstring(
mozilla::Encoding const** encoding, nsACString const* src, nsACString* dst);
nsresult mozilla_encoding_decode_to_nscstring_with_bom_removal(
mozilla::Encoding const* encoding, nsACString const* src, nsACString* dst);
nsresult mozilla_encoding_decode_to_nscstring_without_bom_handling(
mozilla::Encoding const* encoding, nsACString const* src, nsACString* dst);
nsresult mozilla_encoding_decode_from_slice_to_nscstring_without_bom_handling(
mozilla::Encoding const* encoding, uint8_t const* src, size_t src_len,
nsACString* dst, size_t already_validated);
nsresult
mozilla_encoding_decode_to_nscstring_without_bom_handling_and_without_replacement(
mozilla::Encoding const* encoding, nsACString const* src, nsACString* dst);
nsresult mozilla_encoding_encode_from_nscstring(
mozilla::Encoding const** encoding, nsACString const* src, nsACString* dst);
} // extern "C"
namespace mozilla {
/**
* Return value from `Decoder`/`Encoder` to indicate that input
* was exhausted.
*/
const uint32_t kInputEmpty = INPUT_EMPTY;
/**
* Return value from `Decoder`/`Encoder` to indicate that output
* space was insufficient.
*/
const uint32_t kOutputFull = OUTPUT_FULL;
/**
* An encoding as defined in the Encoding Standard
* (https://encoding.spec.whatwg.org/).
*
* See https://docs.rs/encoding_rs/ for the Rust API docs.
*
* An _encoding_ defines a mapping from a byte sequence to a Unicode code point
* sequence and, in most cases, vice versa. Each encoding has a name, an output
* encoding, and one or more labels.
*
* _Labels_ are ASCII-case-insensitive strings that are used to identify an
* encoding in formats and protocols. The _name_ of the encoding is the
* preferred label in the case appropriate for returning from the
* `characterSet` property of the `Document` DOM interface, except for
* the replacement encoding whose name is not one of its labels.
*
* The _output encoding_ is the encoding used for form submission and URL
* parsing on Web pages in the encoding. This is UTF-8 for the replacement,
* UTF-16LE and UTF-16BE encodings and the encoding itself for other
* encodings.
*
* # Streaming vs. Non-Streaming
*
* When you have the entire input in a single buffer, you can use the
* methods `Decode()`, `DecodeWithBOMRemoval()`,
* `DecodeWithoutBOMHandling()`,
* `DecodeWithoutBOMHandlingAndWithoutReplacement()` and
* `Encode()`. Unlike the rest of the API (apart from the `NewDecoder()` and
* NewEncoder()` methods), these methods perform heap allocations. You should
* the `Decoder` and `Encoder` objects when your input is split into multiple
* buffers or when you want to control the allocation of the output buffers.
*
* # Instances
*
* All instances of `Encoding` are statically allocated and have the process's
* lifetime. There is precisely one unique `Encoding` instance for each
* encoding defined in the Encoding Standard.
*
* To obtain a reference to a particular encoding whose identity you know at
* compile time, use a `static` that refers to encoding. There is a `static`
* for each encoding. The `static`s are named in all caps with hyphens
* replaced with underscores and with `_ENCODING` appended to the
* name. For example, if you know at compile time that you will want to
* decode using the UTF-8 encoding, use the `UTF_8_ENCODING` `static`.
*
* If you don't know what encoding you need at compile time and need to
* dynamically get an encoding by label, use `Encoding::for_label()`.
*
* Pointers to `Encoding` can be compared with `==` to check for the sameness
* of two encodings.
*
* A pointer to a `mozilla::Encoding` in C++ is the same thing as a pointer
* to an `encoding_rs::Encoding` in Rust. When writing FFI code, use
* `const mozilla::Encoding*` in the C signature and
* `*const encoding_rs::Encoding` is the corresponding Rust signature.
*/
class Encoding final {
public:
/**
* Implements the _get an encoding_ algorithm
* (https://encoding.spec.whatwg.org/#concept-encoding-get).
*
* If, after ASCII-lowercasing and removing leading and trailing
* whitespace, the argument matches a label defined in the Encoding
* Standard, `const Encoding*` representing the corresponding
* encoding is returned. If there is no match, `nullptr` is returned.
*
* This is the right method to use if the action upon the method returning
* `nullptr` is to use a fallback encoding (e.g. `WINDOWS_1252_ENCODING`)
* instead. When the action upon the method returning `nullptr` is not to
* proceed with a fallback but to refuse processing,
* `ForLabelNoReplacement()` is more appropriate.
*/
static inline const Encoding* ForLabel(Span<const char> aLabel) {
return encoding_for_label(
reinterpret_cast<const uint8_t*>(aLabel.Elements()), aLabel.Length());
}
/**
* `nsAString` argument version. See above for docs.
*/
static inline const Encoding* ForLabel(const nsAString& aLabel) {
return Encoding::ForLabel(NS_ConvertUTF16toUTF8(aLabel));
}
/**
* This method behaves the same as `ForLabel()`, except when `ForLabel()`
* would return `REPLACEMENT_ENCODING`, this method returns `nullptr` instead.
*
* This method is useful in scenarios where a fatal error is required
* upon invalid label, because in those cases the caller typically wishes
* to treat the labels that map to the replacement encoding as fatal
* errors, too.
*
* It is not OK to use this method when the action upon the method returning
* `nullptr` is to use a fallback encoding (e.g. `WINDOWS_1252_ENCODING`). In
* such a case, the `ForLabel()` method should be used instead in order to
* avoid unsafe fallback for labels that `ForLabel()` maps to
* `REPLACEMENT_ENCODING`.
*/
static inline const Encoding* ForLabelNoReplacement(Span<const char> aLabel) {
return encoding_for_label_no_replacement(
reinterpret_cast<const uint8_t*>(aLabel.Elements()), aLabel.Length());
}
/**
* `nsAString` argument version. See above for docs.
*/
static inline const Encoding* ForLabelNoReplacement(const nsAString& aLabel) {
return Encoding::ForLabelNoReplacement(NS_ConvertUTF16toUTF8(aLabel));
}
/**
* Performs non-incremental BOM sniffing.
*
* The argument must either be a buffer representing the entire input
* stream (non-streaming case) or a buffer representing at least the first
* three bytes of the input stream (streaming case).
*
* Returns `MakeTuple(UTF_8_ENCODING, 3)`, `MakeTuple(UTF_16LE_ENCODING, 2)`
* or `MakeTuple(UTF_16BE_ENCODING, 3)` if the argument starts with the
* UTF-8, UTF-16LE or UTF-16BE BOM or `MakeTuple(nullptr, 0)` otherwise.
*/
static inline Tuple<const Encoding*, size_t> ForBOM(
Span<const uint8_t> aBuffer) {
size_t len = aBuffer.Length();
const Encoding* encoding = encoding_for_bom(aBuffer.Elements(), &len);
return MakeTuple(encoding, len);
}
/**
* Writes the name of this encoding into `aName`.
*
* This name is appropriate to return as-is from the DOM
* `document.characterSet` property.
*/
inline void Name(nsACString& aName) const {
aName.SetLength(ENCODING_NAME_MAX_LENGTH);
size_t length =
encoding_name(this, reinterpret_cast<uint8_t*>(aName.BeginWriting()));
aName.SetLength(length); // truncation is the 64-bit case is OK
}
/**
* Checks whether the _output encoding_ of this encoding can encode every
* Unicode code point. (Only true if the output encoding is UTF-8.)
*/
inline bool CanEncodeEverything() const {
return encoding_can_encode_everything(this);
}
/**
* Checks whether the bytes 0x00...0x7F map exclusively to the characters
* U+0000...U+007F and vice versa.
*/
inline bool IsAsciiCompatible() const {
return encoding_is_ascii_compatible(this);
}
/**
* Checks whether this is a Japanese legacy encoding.
*/
inline bool IsJapaneseLegacy() const {
return this == SHIFT_JIS_ENCODING || this == EUC_JP_ENCODING ||
this == ISO_2022_JP_ENCODING;
}
/**
* Returns the _output encoding_ of this encoding. This is UTF-8 for
* UTF-16BE, UTF-16LE and replacement and the encoding itself otherwise.
*/
inline NotNull<const mozilla::Encoding*> OutputEncoding() const {
return WrapNotNull(encoding_output_encoding(this));
}
/**
* Decode complete input to `nsACString` _with BOM sniffing_ and with
* malformed sequences replaced with the REPLACEMENT CHARACTER when the
* entire input is available as a single buffer (i.e. the end of the
* buffer marks the end of the stream).
*
* This method implements the (non-streaming version of) the
* _decode_ (https://encoding.spec.whatwg.org/#decode) spec concept.
*
* The second item in the returned tuple is the encoding that was actually
* used (which may differ from this encoding thanks to BOM sniffing).
*
* Returns `NS_ERROR_OUT_OF_MEMORY` upon OOM, `NS_OK_HAD_REPLACEMENTS`
* if there were malformed sequences (that were replaced with the
* REPLACEMENT CHARACTER) and `NS_OK` otherwise as the first item of the
* tuple.
*
* The backing buffer of the string isn't copied if the input buffer
* is heap-allocated and decoding from UTF-8 and the input is valid
* BOMless UTF-8, decoding from an ASCII-compatible encoding and
* the input is valid ASCII or decoding from ISO-2022-JP and the
* input stays in the ASCII state of ISO-2022-JP. It is OK to pass
* the same string as both arguments.
*
* _Note:_ It is wrong to use this when the input buffer represents only
* a segment of the input instead of the whole input. Use `NewDecoder()`
* when decoding segmented input.
*/
inline Tuple<nsresult, NotNull<const mozilla::Encoding*>> Decode(
const nsACString& aBytes, nsACString& aOut) const {
const Encoding* encoding = this;
const nsACString* bytes = &aBytes;
nsACString* out = &aOut;
nsresult rv;
if (bytes == out) {
nsAutoCString temp(aBytes);
rv = mozilla_encoding_decode_to_nscstring(&encoding, &temp, out);
} else {
rv = mozilla_encoding_decode_to_nscstring(&encoding, bytes, out);
}
return MakeTuple(rv, WrapNotNull(encoding));
}
/**
* Decode complete input to `nsAString` _with BOM sniffing_ and with
* malformed sequences replaced with the REPLACEMENT CHARACTER when the
* entire input is available as a single buffer (i.e. the end of the
* buffer marks the end of the stream).
*
* This method implements the (non-streaming version of) the
* _decode_ (https://encoding.spec.whatwg.org/#decode) spec concept.
*
* The second item in the returned tuple is the encoding that was actually
* used (which may differ from this encoding thanks to BOM sniffing).
*
* Returns `NS_ERROR_OUT_OF_MEMORY` upon OOM, `NS_OK_HAD_REPLACEMENTS`
* if there were malformed sequences (that were replaced with the
* REPLACEMENT CHARACTER) and `NS_OK` otherwise as the first item of the
* tuple.
*
* _Note:_ It is wrong to use this when the input buffer represents only
* a segment of the input instead of the whole input. Use `NewDecoder()`
* when decoding segmented input.
*/
inline Tuple<nsresult, NotNull<const mozilla::Encoding*>> Decode(
Span<const uint8_t> aBytes, nsAString& aOut) const {
const Encoding* encoding = this;
nsresult rv = mozilla_encoding_decode_to_nsstring(
&encoding, aBytes.Elements(), aBytes.Length(), &aOut);
return MakeTuple(rv, WrapNotNull(encoding));
}
/**
* Decode complete input to `nsACString` _with BOM removal_ and with
* malformed sequences replaced with the REPLACEMENT CHARACTER when the
* entire input is available as a single buffer (i.e. the end of the
* buffer marks the end of the stream).
*
* When invoked on `UTF_8`, this method implements the (non-streaming
* version of) the _UTF-8 decode_
* (https://encoding.spec.whatwg.org/#utf-8-decode) spec concept.
*
* Returns `NS_ERROR_OUT_OF_MEMORY` upon OOM, `NS_OK_HAD_REPLACEMENTS`
* if there were malformed sequences (that were replaced with the
* REPLACEMENT CHARACTER) and `NS_OK` otherwise.
*
* The backing buffer of the string isn't copied if the input buffer
* is heap-allocated and decoding from UTF-8 and the input is valid
* BOMless UTF-8, decoding from an ASCII-compatible encoding and
* the input is valid ASCII or decoding from ISO-2022-JP and the
* input stays in the ASCII state of ISO-2022-JP. It is OK to pass
* the same string as both arguments.
*
* _Note:_ It is wrong to use this when the input buffer represents only
* a segment of the input instead of the whole input. Use
* `NewDecoderWithBOMRemoval()` when decoding segmented input.
*/
inline nsresult DecodeWithBOMRemoval(const nsACString& aBytes,
nsACString& aOut) const {
const nsACString* bytes = &aBytes;
nsACString* out = &aOut;
if (bytes == out) {
nsAutoCString temp(aBytes);
return mozilla_encoding_decode_to_nscstring_with_bom_removal(this, &temp,
out);
}
return mozilla_encoding_decode_to_nscstring_with_bom_removal(this, bytes,
out);
}
/**
* Decode complete input to `nsAString` _with BOM removal_ and with
* malformed sequences replaced with the REPLACEMENT CHARACTER when the
* entire input is available as a single buffer (i.e. the end of the
* buffer marks the end of the stream).
*
* When invoked on `UTF_8`, this method implements the (non-streaming
* version of) the _UTF-8 decode_
* (https://encoding.spec.whatwg.org/#utf-8-decode) spec concept.
*
* Returns `NS_ERROR_OUT_OF_MEMORY` upon OOM, `NS_OK_HAD_REPLACEMENTS`
* if there were malformed sequences (that were replaced with the
* REPLACEMENT CHARACTER) and `NS_OK` otherwise.
*
* _Note:_ It is wrong to use this when the input buffer represents only
* a segment of the input instead of the whole input. Use
* `NewDecoderWithBOMRemoval()` when decoding segmented input.
*/
inline nsresult DecodeWithBOMRemoval(Span<const uint8_t> aBytes,
nsAString& aOut) const {
return mozilla_encoding_decode_to_nsstring_with_bom_removal(
this, aBytes.Elements(), aBytes.Length(), &aOut);
}
/**
* Decode complete input to `nsACString` _without BOM handling_ and
* with malformed sequences replaced with the REPLACEMENT CHARACTER when
* the entire input is available as a single buffer (i.e. the end of the
* buffer marks the end of the stream).
*
* When invoked on `UTF_8`, this method implements the (non-streaming
* version of) the _UTF-8 decode without BOM_
* (https://encoding.spec.whatwg.org/#utf-8-decode-without-bom) spec concept.
*
* Returns `NS_ERROR_OUT_OF_MEMORY` upon OOM, `NS_OK_HAD_REPLACEMENTS`
* if there were malformed sequences (that were replaced with the
* REPLACEMENT CHARACTER) and `NS_OK` otherwise.
*
* The backing buffer of the string isn't copied if the input buffer
* is heap-allocated and decoding from UTF-8 and the input is valid
* UTF-8, decoding from an ASCII-compatible encoding and the input
* is valid ASCII or decoding from ISO-2022-JP and the input stays
* in the ASCII state of ISO-2022-JP. It is OK to pass the same string
* as both arguments.
*
* _Note:_ It is wrong to use this when the input buffer represents only
* a segment of the input instead of the whole input. Use
* `NewDecoderWithoutBOMHandling()` when decoding segmented input.
*/
inline nsresult DecodeWithoutBOMHandling(const nsACString& aBytes,
nsACString& aOut) const {
const nsACString* bytes = &aBytes;
nsACString* out = &aOut;
if (bytes == out) {
nsAutoCString temp(aBytes);
return mozilla_encoding_decode_to_nscstring_without_bom_handling(
this, &temp, out);
}
return mozilla_encoding_decode_to_nscstring_without_bom_handling(
this, bytes, out);
}
/**
* Decode complete input to `nsAString` _without BOM handling_ and
* with malformed sequences replaced with the REPLACEMENT CHARACTER when
* the entire input is available as a single buffer (i.e. the end of the
* buffer marks the end of the stream).
*
* When invoked on `UTF_8`, this method implements the (non-streaming
* version of) the _UTF-8 decode without BOM_
* (https://encoding.spec.whatwg.org/#utf-8-decode-without-bom) spec concept.
*
* Returns `NS_ERROR_OUT_OF_MEMORY` upon OOM, `NS_OK_HAD_REPLACEMENTS`
* if there were malformed sequences (that were replaced with the
* REPLACEMENT CHARACTER) and `NS_OK` otherwise.
*
* _Note:_ It is wrong to use this when the input buffer represents only
* a segment of the input instead of the whole input. Use
* `NewDecoderWithoutBOMHandling()` when decoding segmented input.
*/
inline nsresult DecodeWithoutBOMHandling(Span<const uint8_t> aBytes,
nsAString& aOut) const {
return mozilla_encoding_decode_to_nsstring_without_bom_handling(
this, aBytes.Elements(), aBytes.Length(), &aOut);
}
/**
* Decode complete input to `nsACString` _without BOM handling_ and
* _with malformed sequences treated as fatal_ when the entire input is
* available as a single buffer (i.e. the end of the buffer marks the end
* of the stream).
*
* When invoked on `UTF_8`, this method implements the (non-streaming
* version of) the _UTF-8 decode without BOM or fail_
* (https://encoding.spec.whatwg.org/#utf-8-decode-without-bom-or-fail)
* spec concept.
*
* Returns `NS_ERROR_OUT_OF_MEMORY` upon OOM, `NS_ERROR_UDEC_ILLEGALINPUT`
* if a malformed sequence was encountered and `NS_OK` otherwise.
*
* The backing buffer of the string isn't copied if the input buffer
* is heap-allocated and decoding from UTF-8 and the input is valid
* UTF-8, decoding from an ASCII-compatible encoding and the input
* is valid ASCII or decoding from ISO-2022-JP and the input stays
* in the ASCII state of ISO-2022-JP. It is OK to pass the same string
* as both arguments.
*
* _Note:_ It is wrong to use this when the input buffer represents only
* a segment of the input instead of the whole input. Use
* `NewDecoderWithoutBOMHandling()` when decoding segmented input.
*/
inline nsresult DecodeWithoutBOMHandlingAndWithoutReplacement(
const nsACString& aBytes, nsACString& aOut) const {
const nsACString* bytes = &aBytes;
nsACString* out = &aOut;
if (bytes == out) {
nsAutoCString temp(aBytes);
return mozilla_encoding_decode_to_nscstring_without_bom_handling_and_without_replacement(
this, &temp, out);
}
return mozilla_encoding_decode_to_nscstring_without_bom_handling_and_without_replacement(
this, bytes, out);
}
/**
* Decode complete input to `nsACString` _without BOM handling_ and
* with malformed sequences replaced with the REPLACEMENT CHARACTER when
* the entire input is available as a single buffer (i.e. the end of the
* buffer marks the end of the stream) _asserting that a number of bytes
* from the start are already known to be valid UTF-8_.
*
* The use case for this method is avoiding copying when dealing with
* input that has a UTF-8 BOM. _When in doubt, do not use this method._
*
* When invoked on `UTF_8`, this method implements the (non-streaming
* version of) the _UTF-8 decode without BOM_
* (https://encoding.spec.whatwg.org/#utf-8-decode-without-bom) spec concept.
*
* Returns `NS_ERROR_OUT_OF_MEMORY` upon OOM, `NS_OK_HAD_REPLACEMENTS`
* if there were malformed sequences (that were replaced with the
* REPLACEMENT CHARACTER) and `NS_OK` otherwise.
*
* _Note:_ It is wrong to use this when the input buffer represents only
* a segment of the input instead of the whole input. Use
* `NewDecoderWithoutBOMHandling()` when decoding segmented input.
*
* # Safety
*
* The first `aAlreadyValidated` bytes of `aBytes` _must_ be valid UTF-8.
* `aBytes` _must not_ alias the buffer (if any) of `aOut`.
*/
inline nsresult DecodeWithoutBOMHandling(Span<const uint8_t> aBytes,
nsACString& aOut,
size_t aAlreadyValidated) const {
return mozilla_encoding_decode_from_slice_to_nscstring_without_bom_handling(
this, aBytes.Elements(), aBytes.Length(), &aOut, aAlreadyValidated);
}
/**
* Decode complete input to `nsAString` _without BOM handling_ and
* _with malformed sequences treated as fatal_ when the entire input is
* available as a single buffer (i.e. the end of the buffer marks the end
* of the stream).
*
* When invoked on `UTF_8`, this method implements the (non-streaming
* version of) the _UTF-8 decode without BOM or fail_
* (https://encoding.spec.whatwg.org/#utf-8-decode-without-bom-or-fail)
* spec concept.
*
* Returns `NS_ERROR_OUT_OF_MEMORY` upon OOM, `NS_ERROR_UDEC_ILLEGALINPUT`
* if a malformed sequence was encountered and `NS_OK` otherwise.
*
* _Note:_ It is wrong to use this when the input buffer represents only
* a segment of the input instead of the whole input. Use
* `NewDecoderWithoutBOMHandling()` when decoding segmented input.
*/
inline nsresult DecodeWithoutBOMHandlingAndWithoutReplacement(
Span<const uint8_t> aBytes, nsAString& aOut) const {
return mozilla_encoding_decode_to_nsstring_without_bom_handling_and_without_replacement(
this, aBytes.Elements(), aBytes.Length(), &aOut);
}
/**
* Encode complete input to `nsACString` with unmappable characters
* replaced with decimal numeric character references when the entire input
* is available as a single buffer (i.e. the end of the buffer marks the
* end of the stream).
*
* This method implements the (non-streaming version of) the
* _encode_ (https://encoding.spec.whatwg.org/#encode) spec concept.
*
* The second item in the returned tuple is the encoding that was actually
* used (which may differ from this encoding thanks to some encodings
* having UTF-8 as their output encoding).
*
* The first item of the returned tuple is `NS_ERROR_UDEC_ILLEGALINPUT` if
* the input is not valid UTF-8, `NS_ERROR_OUT_OF_MEMORY` upon OOM,
* `NS_OK_HAD_REPLACEMENTS` if there were unmappable code points (that were
* replaced with numeric character references) and `NS_OK` otherwise.
*
* The backing buffer of the string isn't copied if the input buffer
* is heap-allocated and encoding to UTF-8 and the input is valid
* UTF-8, encoding to an ASCII-compatible encoding and the input
* is valid ASCII or encoding from ISO-2022-JP and the input stays
* in the ASCII state of ISO-2022-JP. It is OK to pass the same string
* as both arguments.
*
* _Note:_ It is wrong to use this when the input buffer represents only
* a segment of the input instead of the whole input. Use `NewEncoder()`
* when encoding segmented output.
*/
inline Tuple<nsresult, NotNull<const mozilla::Encoding*>> Encode(
const nsACString& aString, nsACString& aOut) const {
const Encoding* encoding = this;
const nsACString* string = &aString;
nsACString* out = &aOut;
nsresult rv;
if (string == out) {
nsAutoCString temp(aString);
rv = mozilla_encoding_encode_from_nscstring(&encoding, &temp, out);
} else {
rv = mozilla_encoding_encode_from_nscstring(&encoding, string, out);
}
return MakeTuple(rv, WrapNotNull(encoding));
}
/**
* Encode complete input to `nsACString` with unmappable characters
* replaced with decimal numeric character references when the entire input
* is available as a single buffer (i.e. the end of the buffer marks the
* end of the stream).
*
* This method implements the (non-streaming version of) the
* _encode_ (https://encoding.spec.whatwg.org/#encode) spec concept.
*
* The second item in the returned tuple is the encoding that was actually
* used (which may differ from this encoding thanks to some encodings
* having UTF-8 as their output encoding).
*
* The first item of the returned tuple is `NS_ERROR_OUT_OF_MEMORY` upon
* OOM, `NS_OK_HAD_REPLACEMENTS` if there were unmappable code points (that
* were replaced with numeric character references) and `NS_OK` otherwise.
* _Note:_ It is wrong to use this when the input buffer represents only
* a segment of the input instead of the whole input. Use `NewEncoder()`
* when encoding segmented output.
*/
inline Tuple<nsresult, NotNull<const mozilla::Encoding*>> Encode(
Span<const char16_t> aString, nsACString& aOut) const {
const Encoding* encoding = this;
nsresult rv = mozilla_encoding_encode_from_utf16(
&encoding, aString.Elements(), aString.Length(), &aOut);
return MakeTuple(rv, WrapNotNull(encoding));
}
/**
* Instantiates a new decoder for this encoding with BOM sniffing enabled.
*
* BOM sniffing may cause the returned decoder to morph into a decoder
* for UTF-8, UTF-16LE or UTF-16BE instead of this encoding.
*/
inline UniquePtr<Decoder> NewDecoder() const {
UniquePtr<Decoder> decoder(encoding_new_decoder(this));
return decoder;
}
/**
* Instantiates a new decoder for this encoding with BOM sniffing enabled
* into memory occupied by a previously-instantiated decoder.
*
* BOM sniffing may cause the returned decoder to morph into a decoder
* for UTF-8, UTF-16LE or UTF-16BE instead of this encoding.
*/
inline void NewDecoderInto(Decoder& aDecoder) const {
encoding_new_decoder_into(this, &aDecoder);
}
/**
* Instantiates a new decoder for this encoding with BOM removal.
*
* If the input starts with bytes that are the BOM for this encoding,
* those bytes are removed. However, the decoder never morphs into a
* decoder for another encoding: A BOM for another encoding is treated as
* (potentially malformed) input to the decoding algorithm for this
* encoding.
*/
inline UniquePtr<Decoder> NewDecoderWithBOMRemoval() const {
UniquePtr<Decoder> decoder(encoding_new_decoder_with_bom_removal(this));
return decoder;
}
/**
* Instantiates a new decoder for this encoding with BOM removal
* into memory occupied by a previously-instantiated decoder.
*
* If the input starts with bytes that are the BOM for this encoding,
* those bytes are removed. However, the decoder never morphs into a
* decoder for another encoding: A BOM for another encoding is treated as
* (potentially malformed) input to the decoding algorithm for this
* encoding.
*/
inline void NewDecoderWithBOMRemovalInto(Decoder& aDecoder) const {
encoding_new_decoder_with_bom_removal_into(this, &aDecoder);
}
/**
* Instantiates a new decoder for this encoding with BOM handling disabled.
*
* If the input starts with bytes that look like a BOM, those bytes are
* not treated as a BOM. (Hence, the decoder never morphs into a decoder
* for another encoding.)
*
* _Note:_ If the caller has performed BOM sniffing on its own but has not
* removed the BOM, the caller should use `NewDecoderWithBOMRemoval()`
* instead of this method to cause the BOM to be removed.
*/
inline UniquePtr<Decoder> NewDecoderWithoutBOMHandling() const {
UniquePtr<Decoder> decoder(encoding_new_decoder_without_bom_handling(this));
return decoder;
}
/**
* Instantiates a new decoder for this encoding with BOM handling disabled
* into memory occupied by a previously-instantiated decoder.
*
* If the input starts with bytes that look like a BOM, those bytes are
* not treated as a BOM. (Hence, the decoder never morphs into a decoder
* for another encoding.)
*
* _Note:_ If the caller has performed BOM sniffing on its own but has not
* removed the BOM, the caller should use `NewDecoderWithBOMRemovalInto()`
* instead of this method to cause the BOM to be removed.
*/
inline void NewDecoderWithoutBOMHandlingInto(Decoder& aDecoder) const {
encoding_new_decoder_without_bom_handling_into(this, &aDecoder);
}
/**
* Instantiates a new encoder for the output encoding of this encoding.
*/
inline UniquePtr<Encoder> NewEncoder() const {
UniquePtr<Encoder> encoder(encoding_new_encoder(this));
return encoder;
}
/**
* Instantiates a new encoder for the output encoding of this encoding
* into memory occupied by a previously-instantiated encoder.
*/
inline void NewEncoderInto(Encoder& aEncoder) const {
encoding_new_encoder_into(this, &aEncoder);
}
/**
* Validates UTF-8.
*
* Returns the index of the first byte that makes the input malformed as
* UTF-8 or the length of the input if the input is entirely valid.
*/
static inline size_t UTF8ValidUpTo(Span<const uint8_t> aBuffer) {
return encoding_utf8_valid_up_to(aBuffer.Elements(), aBuffer.Length());
}
/**
* Validates ASCII.
*
* Returns the index of the first byte that makes the input malformed as
* ASCII or the length of the input if the input is entirely valid.
*/
static inline size_t ASCIIValidUpTo(Span<const uint8_t> aBuffer) {
return encoding_ascii_valid_up_to(aBuffer.Elements(), aBuffer.Length());
}
/**
* Validates ISO-2022-JP ASCII-state data.
*
* Returns the index of the first byte that makes the input not
* representable in the ASCII state of ISO-2022-JP or the length of the
* input if the input is entirely representable in the ASCII state of
* ISO-2022-JP.
*/
static inline size_t ISO2022JPASCIIValidUpTo(Span<const uint8_t> aBuffer) {
return encoding_iso_2022_jp_ascii_valid_up_to(aBuffer.Elements(),
aBuffer.Length());
}
private:
Encoding() = delete;
Encoding(const Encoding&) = delete;
Encoding& operator=(const Encoding&) = delete;
~Encoding() = delete;
};
/**
* A converter that decodes a byte stream into Unicode according to a
* character encoding in a streaming (incremental) manner.
*
* The various `Decode*` methods take an input buffer (`aSrc`) and an output
* buffer `aDst` both of which are caller-allocated. There are variants for
* both UTF-8 and UTF-16 output buffers.
*
* A `Decode*` method decodes bytes from `aSrc` into Unicode characters stored
* into `aDst` until one of the following three things happens:
*
* 1. A malformed byte sequence is encountered (`*WithoutReplacement`
* variants only).
*
* 2. The output buffer has been filled so near capacity that the decoder
* cannot be sure that processing an additional byte of input wouldn't
* cause so much output that the output buffer would overflow.
*
* 3. All the input bytes have been processed.
*
* The `Decode*` method then returns tuple of a status indicating which one
* of the three reasons to return happened, how many input bytes were read,
* how many output code units (`uint8_t` when decoding into UTF-8 and `char16_t`
* when decoding to UTF-16) were written, and in the case of the
* variants performing replacement, a boolean indicating whether an error was
* replaced with the REPLACEMENT CHARACTER during the call.
*
* The number of bytes "written" is what's logically written. Garbage may be
* written in the output buffer beyond the point logically written to.
*
* In the case of the `*WithoutReplacement` variants, the status is a
* `uint32_t` whose possible values are packed info about a malformed byte
* sequence, `kOutputFull` and `kInputEmpty` corresponding to the three cases
* listed above).
*
* Packed info about malformed sequences has the following format:
* The lowest 8 bits, which can have the decimal value 0, 1, 2 or 3,
* indicate the number of bytes that were consumed after the malformed
* sequence and whose next-lowest 8 bits, when shifted right by 8 indicate
* the length of the malformed byte sequence (possible decimal values 1, 2,
* 3 or 4). The maximum possible sum of the two is 6.
*
* In the case of methods whose name does not end with
* `*WithoutReplacement`, malformed sequences are automatically replaced
* with the REPLACEMENT CHARACTER and errors do not cause the methods to
* return early.
*
* When decoding to UTF-8, the output buffer must have at least 4 bytes of
* space. When decoding to UTF-16, the output buffer must have at least two
* UTF-16 code units (`char16_t`) of space.
*
* When decoding to UTF-8 without replacement, the methods are guaranteed
* not to return indicating that more output space is needed if the length
* of the output buffer is at least the length returned by
* `MaxUTF8BufferLengthWithoutReplacement()`. When decoding to UTF-8
* with replacement, the length of the output buffer that guarantees the
* methods not to return indicating that more output space is needed is given
* by `MaxUTF8BufferLength()`. When decoding to UTF-16 with
* or without replacement, the length of the output buffer that guarantees
* the methods not to return indicating that more output space is needed is
* given by `MaxUTF16BufferLength()`.
*
* The output written into `aDst` is guaranteed to be valid UTF-8 or UTF-16,
* and the output after each `Decode*` call is guaranteed to consist of
* complete characters. (I.e. the code unit sequence for the last character is
* guaranteed not to be split across output buffers.)
*
* The boolean argument `aLast` indicates that the end of the stream is reached
* when all the bytes in `aSrc` have been consumed.
*
* A `Decoder` object can be used to incrementally decode a byte stream.
*
* During the processing of a single stream, the caller must call `Decode*`
* zero or more times with `aLast` set to `false` and then call `Decode*` at
* least once with `aLast` set to `true`. If `Decode*` returns `kInputEmpty`,
* the processing of the stream has ended. Otherwise, the caller must call
* `Decode*` again with `aLast` set to `true` (or treat a malformed result,
* i.e. neither `kInputEmpty` nor `kOutputFull`, as a fatal error).
*
* Once the stream has ended, the `Decoder` object must not be used anymore.
* That is, you need to create another one to process another stream.
*
* When the decoder returns `kOutputFull` or the decoder returns a malformed
* result and the caller does not wish to treat it as a fatal error, the input
* buffer `aSrc` may not have been completely consumed. In that case, the caller
* must pass the unconsumed contents of `aSrc` to `Decode*` again upon the next
* call.
*
* # Infinite loops
*
* When converting with a fixed-size output buffer whose size is too small to
* accommodate one character of output, an infinite loop ensues. When
* converting with a fixed-size output buffer, it generally makes sense to
* make the buffer fairly large (e.g. couple of kilobytes).
*/
class Decoder final {
public:
~Decoder() {}
static void operator delete(void* aDecoder) {
decoder_free(reinterpret_cast<Decoder*>(aDecoder));
}
/**
* The `Encoding` this `Decoder` is for.
*
* BOM sniffing can change the return value of this method during the life
* of the decoder.
*/
inline NotNull<const mozilla::Encoding*> Encoding() const {
return WrapNotNull(decoder_encoding(this));
}
/**
* Query the worst-case UTF-8 output size _with replacement_.
*
* Returns the size of the output buffer in UTF-8 code units (`uint8_t`)
* that will not overflow given the current state of the decoder and
* `aByteLength` number of additional input bytes when decoding with
* errors handled by outputting a REPLACEMENT CHARACTER for each malformed
* sequence.
*/
inline CheckedInt<size_t> MaxUTF8BufferLength(size_t aByteLength) const {
CheckedInt<size_t> max(decoder_max_utf8_buffer_length(this, aByteLength));
if (max.value() == MaxValue<size_t>::value) {
// Mark invalid by overflowing
max++;
MOZ_ASSERT(!max.isValid());
}
return max;
}
/**
* Query the worst-case UTF-8 output size _without replacement_.
*
* Returns the size of the output buffer in UTF-8 code units (`uint8_t`)
* that will not overflow given the current state of the decoder and
* `aByteLength` number of additional input bytes when decoding without
* replacement error handling.
*
* Note that this value may be too small for the `WithReplacement` case.
* Use `MaxUTF8BufferLength()` for that case.
*/
inline CheckedInt<size_t> MaxUTF8BufferLengthWithoutReplacement(
size_t aByteLength) const {
CheckedInt<size_t> max(
decoder_max_utf8_buffer_length_without_replacement(this, aByteLength));
if (max.value() == MaxValue<size_t>::value) {
// Mark invalid by overflowing
max++;
MOZ_ASSERT(!max.isValid());
}
return max;
}
/**
* Incrementally decode a byte stream into UTF-8 with malformed sequences
* replaced with the REPLACEMENT CHARACTER.
*
* See the documentation of the class for documentation for `Decode*`
* methods collectively.
*/
inline Tuple<uint32_t, size_t, size_t, bool> DecodeToUTF8(
Span<const uint8_t> aSrc, Span<uint8_t> aDst, bool aLast) {
size_t srcRead = aSrc.Length();
size_t dstWritten = aDst.Length();
bool hadReplacements;
uint32_t result =
decoder_decode_to_utf8(this, aSrc.Elements(), &srcRead, aDst.Elements(),
&dstWritten, aLast, &hadReplacements);
return MakeTuple(result, srcRead, dstWritten, hadReplacements);
}
/**
* Incrementally decode a byte stream into UTF-8 _without replacement_.
*
* See the documentation of the class for documentation for `Decode*`
* methods collectively.
*/
inline Tuple<uint32_t, size_t, size_t> DecodeToUTF8WithoutReplacement(
Span<const uint8_t> aSrc, Span<uint8_t> aDst, bool aLast) {
size_t srcRead = aSrc.Length();
size_t dstWritten = aDst.Length();
uint32_t result = decoder_decode_to_utf8_without_replacement(
this, aSrc.Elements(), &srcRead, aDst.Elements(), &dstWritten, aLast);
return MakeTuple(result, srcRead, dstWritten);
}
/**
* Query the worst-case UTF-16 output size (with or without replacement).
*
* Returns the size of the output buffer in UTF-16 code units (`char16_t`)
* that will not overflow given the current state of the decoder and
* `aByteLength` number of additional input bytes.
*
* Since the REPLACEMENT CHARACTER fits into one UTF-16 code unit, the
* return value of this method applies also in the
* `_without_replacement` case.
*/
inline CheckedInt<size_t> MaxUTF16BufferLength(size_t aU16Length) const {
CheckedInt<size_t> max(decoder_max_utf16_buffer_length(this, aU16Length));
if (max.value() == MaxValue<size_t>::value) {
// Mark invalid by overflowing
max++;
MOZ_ASSERT(!max.isValid());
}
return max;
}
/**
* Incrementally decode a byte stream into UTF-16 with malformed sequences
* replaced with the REPLACEMENT CHARACTER.
*
* See the documentation of the class for documentation for `Decode*`
* methods collectively.
*/
inline Tuple<uint32_t, size_t, size_t, bool> DecodeToUTF16(
Span<const uint8_t> aSrc, Span<char16_t> aDst, bool aLast) {
size_t srcRead = aSrc.Length();
size_t dstWritten = aDst.Length();
bool hadReplacements;
uint32_t result = decoder_decode_to_utf16(this, aSrc.Elements(), &srcRead,
aDst.Elements(), &dstWritten,
aLast, &hadReplacements);
return MakeTuple(result, srcRead, dstWritten, hadReplacements);
}
/**