Fix quoting in error messages for Unicode

Author: mpeterv

.gitignore

@@ -4,3 +4,4 @@ luacov.report.out
 doc
 build
 package
+scripts/UnicodeData-*

.luacheckrc

@@ -1,6 +1,7 @@
 std = "min"
 cache = true
-include_files = {"src", "spec/*.lua", "install.lua"}
+include_files = {"src", "spec/*.lua", "scripts/*.lua"}
 
 files["spec/*_spec.lua"].std = "+busted"
 files["src/luacheck/argparse.lua"].max_line_length = 140
+files["src/luacheck/unicode_printability_boundaries.lua"].max_line_length = false

luacheck-dev-1.rockspec

@@ -56,6 +56,8 @@ build = {
       ["luacheck.stages.resolve_locals"] = "src/luacheck/stages/resolve_locals.lua",
       ["luacheck.stages.unwrap_parens"] = "src/luacheck/stages/unwrap_parens.lua",
       ["luacheck.standards"] = "src/luacheck/standards.lua",
+      ["luacheck.unicode"] = "src/luacheck/unicode.lua",
+      ["luacheck.unicode_printability_boundaries"] = "src/luacheck/unicode_printability_boundaries.lua",
       ["luacheck.utils"] = "src/luacheck/utils.lua",
       ["luacheck.version"] = "src/luacheck/version.lua"
    },

scripts/gen_unicode_printability_module.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -eu
+set -o pipefail
+
+# Generates luacheck.unicode_printability_boundaries module given Unicode version.
+# Should be executed from root Luacheck directory.
+
+url="https://www.unicode.org/Public/$1/ucd/UnicodeData.txt"
+cache="scripts/UnicodeData-$1.txt"
+
+if [ ! -e "$cache" ]; then
+    wget -O "$cache" "$url"
+fi
+
+(
+    echo "-- Autogenerated using data from $url";
+    lua scripts/unicode_data_to_printability_module.lua < "$cache"
+) > src/luacheck/unicode_printability_boundaries.lua

scripts/unicode_data_to_printability_module.lua

@@ -0,0 +1,86 @@
+-- Reads Unicode character data in UnicodeData.txt format from stdin.
+-- Prints a Lua module retuning an array of first codepoints of
+-- each continuous block of codepoints that are all printable or all not printable.
+-- See https://unicode.org/reports/tr44/
+
+local category_printabilities = {
+   Lu = true,
+   Ll = true,
+   Lt = true,
+   Lm = true,
+   Lo = true,
+   Mn = true,
+   Mc = true,
+   Me = true,
+   Nd = true,
+   Nl = true,
+   No = true,
+   Pc = true,
+   Pd = true,
+   Ps = true,
+   Pe = true,
+   Pi = true,
+   Pf = true,
+   Po = true,
+   Sm = true,
+   Sc = true,
+   Sk = true,
+   So = true,
+   Zs = true,
+   Zl = false,
+   Zp = false,
+   Cc = false,
+   Cf = false,
+   Cs = false,
+   Co = false,
+   Cn = false
+}
+
+local codepoint_printabilities = {}
+local max_codepoint = 0
+
+local range_start_codepoint
+
+for line in io.lines() do
+   local codepoint_hex, name, category = assert(line:match("^([^;]+);([^;]+);([^;]+)"))
+   local codepoint = assert(tonumber("0x" .. codepoint_hex))
+   local printability = category_printabilities[category]
+   assert(printability ~= nil)
+
+   if name:find(", First>$") then
+      assert(not range_start_codepoint)
+      range_start_codepoint = codepoint
+   elseif name:find(", Last>$") then
+      assert(range_start_codepoint and range_start_codepoint >= range_start_codepoint)
+
+      for range_codepoint = range_start_codepoint, codepoint do
+         codepoint_printabilities[range_codepoint] = printability
+      end
+
+      range_start_codepoint = nil
+   else
+      codepoint_printabilities[codepoint] = printability
+   end
+
+   max_codepoint = math.max(max_codepoint, codepoint)
+end
+
+assert(not range_start_codepoint)
+
+local parts = {"return {"}
+local prev_printability = true
+
+-- Iterate up to a non-existent codepoint to ensure that the last required codepoint is printed.
+for codepoint = 0, max_codepoint + 1 do
+   local printability = codepoint_printabilities[codepoint] or false
+
+   if printability ~= prev_printability then
+      table.insert(parts, ("%d,"):format(codepoint))
+   end
+
+   prev_printability = printability
+end
+
+table.insert(parts, "}")
+print(table.concat(parts))
+

src/luacheck/decoder.lua

@@ -1,3 +1,4 @@
+local unicode = require "luacheck.unicode"
 local utils = require "luacheck.utils"
 
 local decoder = {}
@@ -11,9 +12,9 @@ local ssub = string.sub
 -- and provide Unicode-aware access to them with a common interface.
 -- Source bytes should not be accessed directly.
 -- Provided methods are:
--- `Chars:get_first_byte(index)`: returns the first byte of the character at given index or nil.
+-- `Chars:get_codepoint(index)`: returns codepoint at given index as integer or nil if index is out of range.
 -- `Chars:get_substring(from, to)`: returns substring of original bytes corresponding to characters from `from` to `to`.
--- `Chars:get_quoted_substring(from. to)`: same but quotes and escapes characters to make them printable.
+-- `Chars:get_quoted_substring(from. to)`: like get_substring but quotes and escapes characters to make them printable.
 -- `Chars:get_length()`: returns total number of characters.
 -- `Chars:find(pattern, from)`: `string.find` but `from` is in characters. Return values are still in bytes.
 
@@ -24,20 +25,20 @@ function LatinChars:__init(bytes)
    self._bytes = bytes
 end
 
-function LatinChars:get_first_byte(index)
+function LatinChars:get_codepoint(index)
    return sbyte(self._bytes, index)
 end
 
 function LatinChars:get_substring(from, to)
    return ssub(self._bytes, from, to)
 end
 
-local function decimal_escaper(byte)
-   return "\\" .. tostring(sbyte(byte))
+local function hexadecimal_escaper(byte)
+   return ("\\x%02X"):format(sbyte(byte))
 end
 
 function LatinChars:get_quoted_substring(from, to)
-   return "'" .. sgsub(ssub(self._bytes, from, to), "[^\32-\126]", decimal_escaper) .. "'"
+   return '"' .. sgsub(ssub(self._bytes, from, to), "[^\32-\126]", hexadecimal_escaper) .. '"'
 end
 
 function LatinChars:get_length()
@@ -48,32 +49,28 @@ function LatinChars:find(pattern, from)
    return sfind(self._bytes, pattern, from)
 end
 
--- Decodes `bytes` as UTF8. Returns arrays of first bytes of characters and their byte offsets.
--- Byte offset has one extra item pointing to one byte past the end of `bytes`.
+-- Decodes `bytes` as UTF8. Returns arrays of codepoints as integers and their byte offsets.
+-- Byte offsets have one extra item pointing to one byte past the end of `bytes`.
 -- On decoding error returns nothing.
-local function get_first_bytes_and_byte_offsets(bytes)
-   -- TODO: group codepoints into grapheme clusters.
-   local first_bytes = {}
+local function get_codepoints_and_byte_offsets(bytes)
+   local codepoints = {}
    local byte_offsets = {}
 
    local byte_index = 1
-   local char_index = 1
+   local codepoint_index = 1
 
    while true do
-      byte_offsets[char_index] = byte_index
+      byte_offsets[codepoint_index] = byte_index
 
       -- Attempt to decode the next codepoint from UTF8.
       local codepoint = sbyte(bytes, byte_index)
 
       if not codepoint then
-         return first_bytes, byte_offsets
+         return codepoints, byte_offsets
       end
 
-      first_bytes[char_index] = codepoint
-
       byte_index = byte_index + 1
 
-      -- luacheck: ignore 311/codepoint (TODO: use codepoints for grapheme clustering)
       if codepoint >= 0x80 then
          -- Not ASCII.
 
@@ -133,22 +130,23 @@ local function get_first_bytes_and_byte_offsets(bytes)
          end
       end
 
-      char_index = char_index + 1
+      codepoints[codepoint_index] = codepoint
+      codepoint_index = codepoint_index + 1
    end
 end
 
 -- `UnicodeChars` is the general case for non-latin1 strings.
 -- Assumes UTF8, on decoding error falls back to latin1.
 local UnicodeChars = utils.class()
 
-function UnicodeChars:__init(bytes, first_bytes, byte_offsets)
+function UnicodeChars:__init(bytes, codepoints, byte_offsets)
    self._bytes = bytes
-   self._first_bytes = first_bytes
+   self._codepoints = codepoints
    self._byte_offsets = byte_offsets
 end
 
-function UnicodeChars:get_first_byte(index)
-   return self._first_bytes[index]
+function UnicodeChars:get_codepoint(index)
+   return self._codepoints[index]
 end
 
 function UnicodeChars:get_substring(from, to)
@@ -157,12 +155,26 @@ function UnicodeChars:get_substring(from, to)
 end
 
 function UnicodeChars:get_quoted_substring(from, to)
-   -- TODO: fix for Unicode.
-   return "'" .. sgsub(self:get_substring(from, to), "[^\32-\126]", decimal_escaper) .. "'"
+   -- This is only called on syntax error, it's okay to be slow.
+   local parts = {"'"}
+
+   for index = from, to do
+      local codepoint = self._codepoints[index]
+
+      if unicode.is_printable(codepoint) then
+         table.insert(parts, self:get_substring(index, index))
+      else
+         table.insert(parts, (codepoint > 255 and "\\u{%X}" or "\\x%02X"):format(codepoint))
+      end
+   end
+
+
+   table.insert(parts, "'")
+   return table.concat(parts)
 end
 
 function UnicodeChars:get_length()
-   return #self._first_bytes
+   return #self._codepoints
 end
 
 function UnicodeChars:find(pattern, from)
@@ -172,10 +184,10 @@ end
 function decoder.decode(bytes)
    -- TODO: check if this optimization actually helps.
    if sfind(bytes, "[\128-\255]") then
-      local first_bytes, byte_offsets = get_first_bytes_and_byte_offsets(bytes)
+      local codepoints, byte_offsets = get_codepoints_and_byte_offsets(bytes)
 
-      if first_bytes then
-         return UnicodeChars(bytes, first_bytes, byte_offsets)
+      if codepoints then
+         return UnicodeChars(bytes, codepoints, byte_offsets)
       end
    end
 

src/luacheck/lexer.lua

@@ -98,7 +98,7 @@ local simple_escapes = {
 local function next_byte(state)
    local offset = state.offset + 1
    state.offset = offset
-   return state.src:get_first_byte(offset)
+   return state.src:get_codepoint(offset)
 end
 
 -- Skipping helpers.
@@ -454,10 +454,10 @@ local function lex_number(state, b)
       if not is_float then
          if b == BYTE_u or b == BYTE_U then
             -- It may be uint64_t literal.
-            local b1 = state.src:get_first_byte(state.offset+1)
+            local b1 = state.src:get_codepoint(state.offset+1)
 
             if b1 == BYTE_l or b1 == BYTE_L then
-               local b2 = state.src:get_first_byte(state.offset+2)
+               local b2 = state.src:get_codepoint(state.offset+2)
 
                if b2 == BYTE_l or b2 == BYTE_L then
                   -- It is uint64_t literal.
@@ -466,10 +466,10 @@ local function lex_number(state, b)
             end
          elseif b == BYTE_l or b == BYTE_L then
             -- It may be uint64_t or int64_t literal.
-            local b1 = state.src:get_first_byte(state.offset+1)
+            local b1 = state.src:get_codepoint(state.offset+1)
 
             if b1 == BYTE_l or b1 == BYTE_L then
-               local b2 = state.src:get_first_byte(state.offset+2)
+               local b2 = state.src:get_codepoint(state.offset+2)
 
                if b2 == BYTE_u or b2 == BYTE_U then
                   -- It is uint64_t literal.
@@ -641,6 +641,11 @@ end
 
 local function lex_any(state, b)
    state.offset = state.offset + 1
+
+   if b > 255 then
+      b = 255
+   end
+
    return schar(b)
 end
 
@@ -718,7 +723,7 @@ end
 -- On error returns nil, error message, error location (line, offset), error end offset.
 function lexer.next_token(state)
    local line_offsets = state.line_offsets
-   local b = skip_space(state, state.src:get_first_byte(state.offset))
+   local b = skip_space(state, state.src:get_codepoint(state.offset))
 
    -- Save location of token start.
    local token_line = state.line

src/luacheck/unicode.lua

@@ -0,0 +1,36 @@
+local unicode_printability_boundaries = require "luacheck.unicode_printability_boundaries"
+
+local unicode = {}
+
+-- unicode_printability_boundaries is an array of first codepoints of
+-- each continuous block of codepoints that are all printable or all not printable.
+
+function unicode.is_printable(codepoint)
+   -- Binary search for index of the first boundary less than or equal to given codepoint.
+   local floor_boundary_index
+
+   -- Target index is always in [begin_index..end_index).
+   local begin_index = 1
+   local end_index = #unicode_printability_boundaries + 1
+
+   while end_index - begin_index > 1 do
+      local mid_index = math.floor((begin_index + end_index) / 2)
+      local mid_codepoint = unicode_printability_boundaries[mid_index]
+
+      if codepoint < mid_codepoint then
+         end_index = mid_index
+      elseif codepoint > mid_codepoint then
+         begin_index = mid_index
+      else
+         floor_boundary_index = mid_index
+         break
+      end
+   end
+
+   floor_boundary_index = floor_boundary_index or begin_index
+   -- floor_boundary_index is the number of the block containing codepoint.
+   -- Printable and not printable blocks alternate and the first one is not printable (zero is not printable).
+   return floor_boundary_index % 2 == 0
+end
+
+return unicode

src/luacheck/unicode_printability_boundaries.lua

@@ -0,0 +1,2 @@
+-- Autogenerated using data from https://www.unicode.org/Public/11.0.0/ucd/UnicodeData.txt
+return {0,32,127,160,173,174,888,890,896,900,907,908,909,910,930,931,1328,1329,1367,1369,1419,1421,1424,1425,1480,1488,1515,1519,1525,1542,1564,1566,1757,1758,1806,1808,1867,1869,1970,1984,2043,2045,2094,2096,2111,2112,2140,2142,2143,2144,2155,2208,2229,2230,2238,2259,2274,2275,2436,2437,2445,2447,2449,2451,2473,2474,2481,2482,2483,2486,2490,2492,2501,2503,2505,2507,2511,2519,2520,2524,2526,2527,2532,2534,2559,2561,2564,2565,2571,2575,2577,2579,2601,2602,2609,2610,2612,2613,2615,2616,2618,2620,2621,2622,2627,2631,2633,2635,2638,2641,2642,2649,2653,2654,2655,2662,2679,2689,2692,2693,2702,2703,2706,2707,2729,2730,2737,2738,2740,2741,2746,2748,2758,2759,2762,2763,2766,2768,2769,2784,2788,2790,2802,2809,2816,2817,2820,2821,2829,2831,2833,2835,2857,2858,2865,2866,2868,2869,2874,2876,2885,2887,2889,2891,2894,2902,2904,2908,2910,2911,2916,2918,2936,2946,2948,2949,2955,2958,2961,2962,2966,2969,2971,2972,2973,2974,2976,2979,2981,2984,2987,2990,3002,3006,3011,3014,3017,3018,3022,3024,3025,3031,3032,3046,3067,3072,3085,3086,3089,3090,3113,3114,3130,3133,3141,3142,3145,3146,3150,3157,3159,3160,3163,3168,3172,3174,3184,3192,3213,3214,3217,3218,3241,3242,3252,3253,3258,3260,3269,3270,3273,3274,3278,3285,3287,3294,3295,3296,3300,3302,3312,3313,3315,3328,3332,3333,3341,3342,3345,3346,3397,3398,3401,3402,3408,3412,3428,3430,3456,3458,3460,3461,3479,3482,3506,3507,3516,3517,3518,3520,3527,3530,3531,3535,3541,3542,3543,3544,3552,3558,3568,3570,3573,3585,3643,3647,3676,3713,3715,3716,3717,3719,3721,3722,3723,3725,3726,3732,3736,3737,3744,3745,3748,3749,3750,3751,3752,3754,3756,3757,3770,3771,3774,3776,3781,3782,3783,3784,3790,3792,3802,3804,3808,3840,3912,3913,3949,3953,3992,3993,4029,4030,4045,4046,4059,4096,4294,4295,4296,4301,4302,4304,4681,4682,4686,4688,4695,4696,4697,4698,4702,4704,4745,4746,4750,4752,4785,4786,4790,4792,4799,4800,4801,4802,4806,4808,4823,4824,4881,4882,4886,4888,4955,4957,4989,4992,5018,5024,5110,5112,5118,5120,5789,5792,5881,5888,5901,5902,5909,5920,5943,5952,5972,5984,5997,5998,6001,6002,6004,6016,6110,6112,6122,6128,6138,6144,6158,6160,6170,6176,6265,6272,6315,6320,6390,6400,6431,6432,6444,6448,6460,6464,6465,6468,6510,6512,6517,6528,6572,6576,6602,6608,6619,6622,6684,6686,6751,6752,6781,6783,6794,6800,6810,6816,6830,6832,6847,6912,6988,6992,7037,7040,7156,7164,7224,7227,7242,7245,7305,7312,7355,7357,7368,7376,7418,7424,7674,7675,7958,7960,7966,7968,8006,8008,8014,8016,8024,8025,8026,8027,8028,8029,8030,8031,8062,8064,8117,8118,8133,8134,8148,8150,8156,8157,8176,8178,8181,8182,8191,8192,8203,8208,8232,8239,8288,8304,8306,8308,8335,8336,8349,8352,8384,8400,8433,8448,8588,8592,9255,9280,9291,9312,11124,11126,11158,11160,11209,11210,11263,11264,11311,11312,11359,11360,11508,11513,11558,11559,11560,11565,11566,11568,11624,11631,11633,11647,11671,11680,11687,11688,11695,11696,11703,11704,11711,11712,11719,11720,11727,11728,11735,11736,11743,11744,11855,11904,11930,11931,12020,12032,12246,12272,12284,12288,12352,12353,12439,12441,12544,12549,12592,12593,12687,12688,12731,12736,12772,12784,12831,12832,13055,13056,19894,19904,40944,40960,42125,42128,42183,42192,42540,42560,42744,42752,42938,42999,43052,43056,43066,43072,43128,43136,43206,43214,43226,43232,43348,43359,43389,43392,43470,43471,43482,43486,43519,43520,43575,43584,43598,43600,43610,43612,43715,43739,43767,43777,43783,43785,43791,43793,43799,43808,43815,43816,43823,43824,43878,43888,44014,44016,44026,44032,55204,55216,55239,55243,55292,63744,64110,64112,64218,64256,64263,64275,64280,64285,64311,64312,64317,64318,64319,64320,64322,64323,64325,64326,64450,64467,64832,64848,64912,64914,64968,65008,65022,65024,65050,65056,65107,65108,65127,65128,65132,65136,65141,65142,65277,65281,65471,65474,65480,65482,65488,65490,65496,65498,65501,65504,65511,65512,65519,65532,65534,65536,65548,65549,65575,65576,65595,65596,65598,65599,65614,65616,65630,65664,65787,65792,65795,65799,65844,65847,65935,65936,65948,65952,65953,66000,66046,66176,66205,66208,66257,66272,66300,66304,66340,66349,66379,66384,66427,66432,66462,66463,66500,66504,66518,66560,66718,66720,66730,66736,66772,66776,66812,66816,66856,66864,66916,66927,66928,67072,67383,67392,67414,67424,67432,67584,67590,67592,67593,67594,67638,67639,67641,67644,67645,67647,67670,67671,67743,67751,67760,67808,67827,67828,67830,67835,67868,67871,67898,67903,67904,67968,68024,68028,68048,68050,68100,68101,68103,68108,68116,68117,68120,68121,68150,68152,68155,68159,68169,68176,68185,68192,68256,68288,68327,68331,68343,68352,68406,68409,68438,68440,68467,68472,68498,68505,68509,68521,68528,68608,68681,68736,68787,68800,68851,68858,68904,68912,68922,69216,69247,69376,69416,69424,69466,69632,69710,69714,69744,69759,69821,69822,69826,69840,69865,69872,69882,69888,69941,69942,69959,69968,70007,70016,70094,70096,70112,70113,70133,70144,70162,70163,70207,70272,70279,70280,70281,70282,70286,70287,70302,70303,70314,70320,70379,70384,70394,70400,70404,70405,70413,70415,70417,70419,70441,70442,70449,70450,70452,70453,70458,70459,70469,70471,70473,70475,70478,70480,70481,70487,70488,70493,70500,70502,70509,70512,70517,70656,70746,70747,70748,70749,70751,70784,70856,70864,70874,71040,71094,71096,71134,71168,71237,71248,71258,71264,71277,71296,71352,71360,71370,71424,71451,71453,71468,71472,71488,71680,71740,71840,71923,71935,71936,72192,72264,72272,72324,72326,72355,72384,72441,72704,72713,72714,72759,72760,72774,72784,72813,72816,72848,72850,72872,72873,72887,72960,72967,72968,72970,72971,73015,73018,73019,73020,73022,73023,73032,73040,73050,73056,73062,73063,73065,73066,73103,73104,73106,73107,73113,73120,73130,73440,73465,73728,74650,74752,74863,74864,74869,74880,75076,77824,78895,82944,83527,92160,92729,92736,92767,92768,92778,92782,92784,92880,92910,92912,92918,92928,92998,93008,93018,93019,93026,93027,93048,93053,93072,93760,93851,93952,94021,94032,94079,94095,94112,94176,94178,94208,100338,100352,101107,110592,110879,110960,111356,113664,113771,113776,113789,113792,113801,113808,113818,113820,113824,118784,119030,119040,119079,119081,119155,119163,119273,119296,119366,119520,119540,119552,119639,119648,119673,119808,119893,119894,119965,119966,119968,119970,119971,119973,119975,119977,119981,119982,119994,119995,119996,119997,120004,120005,120070,120071,120075,120077,120085,120086,120093,120094,120122,120123,120127,120128,120133,120134,120135,120138,120145,120146,120486,120488,120780,120782,121484,121499,121504,121505,121520,122880,122887,122888,122905,122907,122914,122915,122917,122918,122923,124928,125125,125127,125143,125184,125259,125264,125274,125278,125280,126065,126133,126464,126468,126469,126496,126497,126499,126500,126501,126503,126504,126505,126515,126516,126520,126521,126522,126523,126524,126530,126531,126535,126536,126537,126538,126539,126540,126541,126544,126545,126547,126548,126549,126551,126552,126553,126554,126555,126556,126557,126558,126559,126560,126561,126563,126564,126565,126567,126571,126572,126579,126580,126584,126585,126589,126590,126591,126592,126602,126603,126620,126625,126628,126629,126634,126635,126652,126704,126706,126976,127020,127024,127124,127136,127151,127153,127168,127169,127184,127185,127222,127232,127245,127248,127340,127344,127405,127462,127491,127504,127548,127552,127561,127568,127570,127584,127590,127744,128725,128736,128749,128752,128762,128768,128884,128896,128985,129024,129036,129040,129096,129104,129114,129120,129160,129168,129198,129280,129292,129296,129343,129344,129393,129395,129399,129402,129403,129404,129443,129456,129466,129472,129475,129488,129536,129632,129646,131072,173783,173824,177973,177984,178206,178208,183970,183984,191457,194560,195102,917760,918000,}