Forward-port 9.18.1.7 changes into master

- 854b80b
- 9cafee6
- 05996fc

.gitignore

@@ -7,3 +7,4 @@ vendor/
 composer.lock
 test/**/*.js
 .php_cs.cache
+.php-cs-fixer.cache

.php_cs.dist → .php-cs-fixer.dist.php

@@ -9,7 +9,8 @@
     ->exclude('lib_highlight')
 ;
 
-return PhpCsFixer\Config::create()
+$config = new PhpCsFixer\Config();
+return $config
     ->setRules([
         '@PSR1' => true,
         '@PSR2' => true,
@@ -24,11 +25,11 @@
         'no_useless_return' => true,
         'phpdoc_align' => true,
         'phpdoc_order' => true,
-        'phpdoc_inline_tag' => false,
         'phpdoc_trim_consecutive_blank_line_separation' => true,
         'single_quote' => false,
         'ternary_to_null_coalescing' => false,
-        'trailing_comma_in_multiline_array' => true,
+        'trailing_comma_in_multiline' => true,
+        'visibility_required' => false,
         'yoda_style' => [
             'equal' => false,
             'identical' => false,

CONTRIBUTING.md

@@ -104,3 +104,9 @@ The process of bringing in languages from highlight.js has been put into a singl
 cd tools
 bash process.sh
 ```
+
+## Why don't we generally use `mb_*` functions?
+
+PHP offers `mb_` prefixed string functions to support multi-byte strings, this means supporting unicode characters. However, the PREG functions in PHP calculate [string lengths and positions in _bytes_, and not character lengths](https://www.php.net/manual/en/function.preg-match.php#refsect1-function.preg-match-parameters). For that reason, we will generally **not** use the multi-byte variants of string functions; there are exceptions to this policy.
+
+An exception to this policy is the use of functions that aren't used for calculating string lengths or positions; e.g. `mb_strtolower`.

src/Highlight/Highlighter.php

@@ -328,7 +328,7 @@ private function endOfMode($mode, $lexeme)
      */
     private function keywordMatch($mode, $match)
     {
-        $kwd = $this->language->case_insensitive ? mb_strtolower($match[0], "UTF-8") : $match[0];
+        $kwd = $this->language->case_insensitive ? mb_strtolower($match[0]) : $match[0];
 
         return isset($mode->keywords[$kwd]) ? $mode->keywords[$kwd] : null;
     }

src/Highlight/Language.php

@@ -42,7 +42,7 @@
  *
  * // Backward compatibility properties
  *
- * @property Mode $mode (DEPRECATED) All properties traditionally inside of $mode are now available directly from this class.
+ * @property Mode $mode            (DEPRECATED) All properties traditionally inside of $mode are now available directly from this class.
  * @property bool $caseInsensitive (DEPRECATED) Due to compatibility requirements with highlight.js, use `case_insensitive` instead.
  */
 class Language extends Mode

src/Highlight/Mode.php

@@ -38,41 +38,41 @@
  *
  * Language definition set via language definition JSON files
  *
- * @property bool $case_insensitive = false
- * @property string[] $aliases = array()
- * @property string|null $className = null
- * @property string|null $begin = null
- * @property RegEx|null $beginRe = null
- * @property string|null $end = null
- * @property RegEx|null $endRe = null
- * @property string|null $beginKeywords = null
- * @property bool $endsWithParent = false
- * @property bool $endsParent = false
- * @property bool $endSameAsBegin = false
- * @property string|null $lexemes = null
- * @property RegEx|null $lexemesRe = null
- * @property array<string, array<int, string|int>> $keywords = array()
- * @property string|null $illegal = null
- * @property RegEx|null $illegalRe = null
- * @property bool $excludeBegin = false
- * @property bool $excludeEnd = false
- * @property bool $returnBegin = false
- * @property bool $returnEnd = false
- * @property Mode[] $contains = array()
- * @property Mode|null $starts = null
- * @property Mode[] $variants = array()
- * @property int|null $relevance = null
- * @property string|string[]|null $subLanguage = null
- * @property bool $skip = false
- * @property bool $disableAutodetect = false
+ * @property bool                                  $case_insensitive  = false
+ * @property string[]                              $aliases           = array()
+ * @property string|null                           $className         = null
+ * @property string|null                           $begin             = null
+ * @property RegEx|null                            $beginRe           = null
+ * @property string|null                           $end               = null
+ * @property RegEx|null                            $endRe             = null
+ * @property string|null                           $beginKeywords     = null
+ * @property bool                                  $endsWithParent    = false
+ * @property bool                                  $endsParent        = false
+ * @property bool                                  $endSameAsBegin    = false
+ * @property string|null                           $lexemes           = null
+ * @property RegEx|null                            $lexemesRe         = null
+ * @property array<string, array<int, string|int>> $keywords          = array()
+ * @property string|null                           $illegal           = null
+ * @property RegEx|null                            $illegalRe         = null
+ * @property bool                                  $excludeBegin      = false
+ * @property bool                                  $excludeEnd        = false
+ * @property bool                                  $returnBegin       = false
+ * @property bool                                  $returnEnd         = false
+ * @property Mode[]                                $contains          = array()
+ * @property Mode|null                             $starts            = null
+ * @property Mode[]                                $variants          = array()
+ * @property int|null                              $relevance         = null
+ * @property string|string[]|null                  $subLanguage       = null
+ * @property bool                                  $skip              = false
+ * @property bool                                  $disableAutodetect = false
  *
  * Properties set at runtime by the language compilation process
  * @property array<int, Mode> $cachedVariants = array()
- * @property Terminators|null $terminators = null
- * @property string $terminator_end = ""
- * @property bool $compiled = false
- * @property Mode|null $parent = null
- * @property string $type = ''
+ * @property Terminators|null $terminators    = null
+ * @property string           $terminator_end = ""
+ * @property bool             $compiled       = false
+ * @property Mode|null        $parent         = null
+ * @property string           $type           = ''
  *
  * @see https://highlightjs.readthedocs.io/en/latest/reference.html
  */

src/Highlight/RegEx.php

@@ -97,7 +97,7 @@ public function exec($str)
 
         unset($result);
 
-        $this->lastIndex += mb_strlen($results[0]) + ($index - $this->lastIndex);
+        $this->lastIndex += strlen($results[0]) + ($index - $this->lastIndex);
 
         $matches = new RegExMatch($results);
         $matches->index = isset($index) ? $index : 0;

test/CasesFromIssuesTest.php

@@ -0,0 +1,94 @@
+<?php
+
+/* Copyright (c) 2013-2019 Geert Bergman ([email protected]), highlight.php
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ * 3. Neither the name of "highlight.js", "highlight.php", nor the names of its
+ *    contributors may be used to endorse or promote products derived from this
+ *    software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+use Highlight\Highlighter;
+use Symfony\Component\Finder\Finder;
+
+class CasesFromIssuesTest extends PHPUnit_Framework_TestCase
+{
+    private static function parseFileName($filename)
+    {
+        $re = '/(\d+)(?:-\[(\w+)])?(\.expected)?\.txt/';
+        $matches = array();
+
+        preg_match($re, $filename, $matches);
+
+        return array(
+            'issueNumber' => $matches[1],
+            'languages' => isset($matches[2]) ? $matches[2] : null,
+            'expected' => isset($matches[3]),
+        );
+    }
+
+    public static function projectSpecificIssueProvider()
+    {
+        $specialCases = array();
+
+        $cases = new Finder();
+        $cases
+            ->in(__DIR__ . '/issues/')
+            ->sortByName()
+            ->files()
+        ;
+
+        foreach ($cases as $case) {
+            $config = self::parseFileName($case->getFilename());
+
+            if (!isset($specialCases[$config['issueNumber']])) {
+                $specialCases[$config['issueNumber']] = array(
+                    'issueNumber' => $config['issueNumber'],
+                    'source' => '',
+                    'expected' => '',
+                    'languages' => $config['languages'],
+                );
+            }
+
+            if ($config['expected']) {
+                $specialCases[$config['issueNumber']]['expected'] = trim($case->getContents());
+            } else {
+                $specialCases[$config['issueNumber']]['source'] = trim($case->getContents());
+            }
+        }
+
+        return $specialCases;
+    }
+
+    /**
+     * @dataProvider projectSpecificIssueProvider
+     */
+    public function testIssueSpecificCases($issueNumber, $source, $expected, $languages)
+    {
+        $hl = new Highlighter();
+        $actual = $hl->highlight($languages, $source);
+
+        $errMessage = sprintf("Unit test added for Issue #%d failed", $issueNumber);
+
+        $this->assertEquals($expected, $actual->value, $errMessage);
+    }
+}

test/issues/87-[php].expected.txt

@@ -0,0 +1,17 @@
+<span class="hljs-meta">&lt;?php</span>
+<span class="hljs-keyword">use</span> <span class="hljs-title">yii</span>\<span class="hljs-title">helpers</span>\<span class="hljs-title">Html</span>;
+<span class="hljs-keyword">use</span> <span class="hljs-title">yii</span>\<span class="hljs-title">widgets</span>\<span class="hljs-title">ActiveForm</span>;
+
+$form = ActiveForm::begin([
+    <span class="hljs-string">'id'</span> =&gt; <span class="hljs-string">'login-form'</span>,
+    <span class="hljs-string">'options'</span> =&gt; [<span class="hljs-string">'class'</span> =&gt; <span class="hljs-string">'form-horizontal'</span>],
+]) <span class="hljs-meta">?&gt;</span>
+    <span class="hljs-meta">&lt;?</span>= $form-&gt;field($model, <span class="hljs-string">'username'</span>) <span class="hljs-meta">?&gt;</span>
+    <span class="hljs-meta">&lt;?</span>= $form-&gt;field($model, <span class="hljs-string">'password'</span>)-&gt;passwordInput() <span class="hljs-meta">?&gt;</span>
+
+    &lt;div <span class="hljs-class"><span class="hljs-keyword">class</span>="<span class="hljs-title">form</span>-<span class="hljs-title">group</span>"&gt;
+        &lt;<span class="hljs-title">div</span> <span class="hljs-title">class</span>="<span class="hljs-title">col</span>-<span class="hljs-title">lg</span>-<span class="hljs-title">offset</span>-1 <span class="hljs-title">col</span>-<span class="hljs-title">lg</span>-11"&gt;
+            &lt;?= <span class="hljs-title">Html</span>::<span class="hljs-title">submitButton</span>('Вход', ['<span class="hljs-title">class</span>' =&gt; '<span class="hljs-title">btn</span> <span class="hljs-title">btn</span>-<span class="hljs-title">primary</span>']) ?&gt;
+        &lt;/<span class="hljs-title">div</span>&gt;
+    &lt;/<span class="hljs-title">div</span>&gt;
+&lt;?<span class="hljs-title">php</span> <span class="hljs-title">ActiveForm</span>::<span class="hljs-title">end</span>() ?&gt;</span>

test/issues/87-[php].txt

@@ -0,0 +1,17 @@
+<?php
+use yii\helpers\Html;
+use yii\widgets\ActiveForm;
+
+$form = ActiveForm::begin([
+    'id' => 'login-form',
+    'options' => ['class' => 'form-horizontal'],
+]) ?>
+    <?= $form->field($model, 'username') ?>
+    <?= $form->field($model, 'password')->passwordInput() ?>
+
+    <div class="form-group">
+        <div class="col-lg-offset-1 col-lg-11">
+            <?= Html::submitButton('Вход', ['class' => 'btn btn-primary']) ?>
+        </div>
+    </div>
+<?php ActiveForm::end() ?>

test/issues/90-[c].expected.txt

@@ -0,0 +1,8 @@
+<span class="hljs-keyword">if</span> (FELTÉTEL)
+{
+  IGAZ ÁG
+}
+<span class="hljs-keyword">else</span>
+{
+  HAMIS ÁG
+}

test/issues/90-[c].txt

@@ -0,0 +1,8 @@
+if (FELTÉTEL)
+{
+  IGAZ ÁG
+}
+else
+{
+  HAMIS ÁG
+}