Skip to content

Commit

Permalink
[std/encodings]move to tests (nim-lang#17866)
Browse files Browse the repository at this point in the history
  • Loading branch information
ringabout authored Apr 29, 2021
1 parent 927ae26 commit 016a8cc
Show file tree
Hide file tree
Showing 2 changed files with 73 additions and 72 deletions.
72 changes: 0 additions & 72 deletions lib/pure/encodings.nim
Original file line number Diff line number Diff line change
Expand Up @@ -476,75 +476,3 @@ proc convert*(s: string, destEncoding = "UTF-8",
result = convert(c, s)
finally:
close(c)

when not defined(testing) and isMainModule:
let
orig = "öäüß"
cp1252 = convert(orig, "CP1252", "UTF-8")
ibm850 = convert(cp1252, "ibm850", "CP1252")
current = getCurrentEncoding()
echo "Original string from source code: ", orig
echo "Forced ibm850 encoding: ", ibm850
echo "Current encoding: ", current
echo "From ibm850 to current: ", convert(ibm850, current, "ibm850")

when not defined(testing) and isMainModule and defined(windows):
block should_throw_on_unsupported_conversions:
let original = "some string"

doAssertRaises(EncodingError):
discard convert(original, "utf-8", "utf-32")

doAssertRaises(EncodingError):
discard convert(original, "utf-8", "unicodeFFFE")

doAssertRaises(EncodingError):
discard convert(original, "utf-8", "utf-32BE")

doAssertRaises(EncodingError):
discard convert(original, "unicodeFFFE", "utf-8")

doAssertRaises(EncodingError):
discard convert(original, "utf-32", "utf-8")

doAssertRaises(EncodingError):
discard convert(original, "utf-32BE", "utf-8")

block should_convert_from_utf16_to_utf8:
let original = "\x42\x04\x35\x04\x41\x04\x42\x04" # utf-16 little endian test string "тест"
let result = convert(original, "utf-8", "utf-16")
doAssert(result == "\xd1\x82\xd0\xb5\xd1\x81\xd1\x82")

block should_convert_from_utf16_to_win1251:
let original = "\x42\x04\x35\x04\x41\x04\x42\x04" # utf-16 little endian test string "тест"
let result = convert(original, "windows-1251", "utf-16")
doAssert(result == "\xf2\xe5\xf1\xf2")

block should_convert_from_win1251_to_koi8r:
let original = "\xf2\xe5\xf1\xf2" # win1251 test string "тест"
let result = convert(original, "koi8-r", "windows-1251")
doAssert(result == "\xd4\xc5\xd3\xd4")

block should_convert_from_koi8r_to_win1251:
let original = "\xd4\xc5\xd3\xd4" # koi8r test string "тест"
let result = convert(original, "windows-1251", "koi8-r")
doAssert(result == "\xf2\xe5\xf1\xf2")

block should_convert_from_utf8_to_win1251:
let original = "\xd1\x82\xd0\xb5\xd1\x81\xd1\x82" # utf-8 test string "тест"
let result = convert(original, "windows-1251", "utf-8")
doAssert(result == "\xf2\xe5\xf1\xf2")

block should_convert_from_utf8_to_utf16:
let original = "\xd1\x82\xd0\xb5\xd1\x81\xd1\x82" # utf-8 test string "тест"
let result = convert(original, "utf-16", "utf-8")
doAssert(result == "\x42\x04\x35\x04\x41\x04\x42\x04")

block should_handle_empty_string_for_any_conversion:
let original = ""
var result = convert(original, "utf-16", "utf-8")
doAssert(result == "")
result = convert(original, "utf-8", "utf-16")
doAssert(result == "")
result = convert(original, "windows-1251", "koi8-r")
doAssert(result == "")
73 changes: 73 additions & 0 deletions tests/stdlib/tencodings.nim
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,76 @@ block:
block:
let data = "谁怕?一蓑烟雨任平生"
doAssert toGB2312.convert(data) == "\203\173\197\194\163\191\210\187\203\242\209\204\211\234\200\206\198\189\201\250"


when defined(windows):
block should_throw_on_unsupported_conversions:
let original = "some string"

doAssertRaises(EncodingError):
discard convert(original, "utf-8", "utf-32")

doAssertRaises(EncodingError):
discard convert(original, "utf-8", "unicodeFFFE")

doAssertRaises(EncodingError):
discard convert(original, "utf-8", "utf-32BE")

doAssertRaises(EncodingError):
discard convert(original, "unicodeFFFE", "utf-8")

doAssertRaises(EncodingError):
discard convert(original, "utf-32", "utf-8")

doAssertRaises(EncodingError):
discard convert(original, "utf-32BE", "utf-8")

block should_convert_from_utf16_to_utf8:
let original = "\x42\x04\x35\x04\x41\x04\x42\x04" # utf-16 little endian test string "тест"
let result = convert(original, "utf-8", "utf-16")
doAssert(result == "\xd1\x82\xd0\xb5\xd1\x81\xd1\x82")

block should_convert_from_utf16_to_win1251:
let original = "\x42\x04\x35\x04\x41\x04\x42\x04" # utf-16 little endian test string "тест"
let result = convert(original, "windows-1251", "utf-16")
doAssert(result == "\xf2\xe5\xf1\xf2")

block should_convert_from_win1251_to_koi8r:
let original = "\xf2\xe5\xf1\xf2" # win1251 test string "тест"
let result = convert(original, "koi8-r", "windows-1251")
doAssert(result == "\xd4\xc5\xd3\xd4")

block should_convert_from_koi8r_to_win1251:
let original = "\xd4\xc5\xd3\xd4" # koi8r test string "тест"
let result = convert(original, "windows-1251", "koi8-r")
doAssert(result == "\xf2\xe5\xf1\xf2")

block should_convert_from_utf8_to_win1251:
let original = "\xd1\x82\xd0\xb5\xd1\x81\xd1\x82" # utf-8 test string "тест"
let result = convert(original, "windows-1251", "utf-8")
doAssert(result == "\xf2\xe5\xf1\xf2")

block should_convert_from_utf8_to_utf16:
let original = "\xd1\x82\xd0\xb5\xd1\x81\xd1\x82" # utf-8 test string "тест"
let result = convert(original, "utf-16", "utf-8")
doAssert(result == "\x42\x04\x35\x04\x41\x04\x42\x04")

block should_handle_empty_string_for_any_conversion:
let original = ""
var result = convert(original, "utf-16", "utf-8")
doAssert(result == "")
result = convert(original, "utf-8", "utf-16")
doAssert(result == "")
result = convert(original, "windows-1251", "koi8-r")
doAssert(result == "")


block:
let
orig = "öäüß"
cp1252 = convert(orig, "CP1252", "UTF-8")
ibm850 = convert(cp1252, "ibm850", "CP1252")
current = getCurrentEncoding()
doAssert orig == "\195\182\195\164\195\188\195\159"
doAssert ibm850 == "\148\132\129\225"
doAssert convert(ibm850, current, "ibm850") == orig

0 comments on commit 016a8cc

Please sign in to comment.