add RST highlighting for command line / shells (also fixes nim-lang#1…

…6858) (nim-lang#17789)

config/nimdoc.tex.cfg

@@ -137,6 +137,8 @@ bottomline=false}
 \newcommand{\spanReference}[1]{#1}
 \newcommand{\spanOther}[1]{#1}
 \newcommand{\spantok}[1]{\frame{#1}}
+\newcommand{\spanprogram}[1]{\textbf{\underline{#1}}}
+\newcommand{\spanoption}[1]{\textbf{#1}}
 
 $content
 \end{document}

doc/docstyle.rst

@@ -6,17 +6,17 @@ General Guidelines
 
 * See also `nep1<https://nim-lang.github.io/Nim/nep1.html>`_ which should probably be merged here.
 * Authors should document anything that is exported; documentation for private
-  procs can be useful too (visible via `nim doc --docInternal foo.nim`).
+  procs can be useful too (visible via `nim doc --docInternal foo.nim`:cmd:).
 * Within documentation, a period (`.`) should follow each sentence (or sentence fragment) in a comment block.
   The documentation may be limited to one sentence fragment, but if multiple sentences are within the documentation,
   each sentence after the first should be complete and in present tense.
 * Documentation is parsed as a custom ReStructuredText (RST) with partial markdown support.
 * In nim sources, prefer single backticks to double backticks since it's simpler
-  and `nim doc` supports it. Likewise with rst files: `nim rst2html` will render those as monospace, and
-  adding `.. default-role:: code` to an rst file will also make those render as monospace when rendered directly
+  and `nim doc`:cmd: supports it. Likewise with ``rst`` files: `nim rst2html`:cmd: will render those as monospace, and
+  adding ``.. default-role:: code`` to an ``rst`` file will also make those render as monospace when rendered directly
   in tools such as github.
-* In nim sources, for links, prefer `[link text](link.html)` to `` `link text<link.html>`_ ``
-  since the syntax is simpler and markdown is more common (likewise, `nim rst2html` also supports it in rst files).
+* (debatable) In nim sources, for links, prefer ``[link text](link.html)`` to `\`link text<link.html>\`_`:code:
+  since the syntax is simpler and markdown is more common (likewise, `nim rst2html`:cmd: also supports it in ``rst`` files).
 
 .. code-block:: nim
 
@@ -29,7 +29,7 @@ Module-level documentation
 --------------------------
 
 Documentation of a module is placed at the top of the module itself. Each line of documentation begins with double hashes (`##`).
-Sometimes `##[ multiline docs containing code ]##` is preferable, see `lib/pure/times.nim`.
+Sometimes `##[ multiline docs containing code ]##` is preferable, see ``lib/pure/times.nim``.
 Code samples are encouraged, and should follow the general RST syntax:
 
 .. code-block:: Nim
@@ -76,11 +76,11 @@ Whenever an example of usage would be helpful to the user, you should include on
     ##  echo execCmdEx("git pull")
     ##  drawOnScreen()
     runnableExamples:
-      # `runnableExamples` is usually preferred to `code-block`, when possible.
+      # `runnableExamples` is usually preferred to ``code-block``, when possible.
       doAssert addThree(3, 125, 6) == -122
     result = x +% y +% z
 
-The command `nim doc` will then correctly syntax highlight the Nim code within the documentation.
+The command `nim doc`:cmd: will then correctly syntax highlight the Nim code within the documentation.
 
 Types
 -----

doc/nimdoc.css

@@ -35,6 +35,8 @@ Modified by Boyd Greenfield and narimiran
   --escapeSequence: #c4891b;
   --number: #252dbe;
   --literal: #a4255b;
+  --program: #6060c0;
+  --option: #508000;
   --raw-data: #a4255b;
 }
 
@@ -63,6 +65,8 @@ Modified by Boyd Greenfield and narimiran
   --escapeSequence: #bd93f9;
   --number: #bd93f9;
   --literal: #f1fa8c;
+  --program: #9090c0;
+  --option: #90b010;
   --raw-data: #8be9fd;
 }
 
@@ -527,7 +531,6 @@ div.option-list-label {
   margin-left: -11.5em;
   margin-right: 0em;
   min-width: 11.5em;
-  font-weight: bolder;
   display: inline-block;
   vertical-align: top;
 }
@@ -546,7 +549,7 @@ blockquote {
   border-left: 5px solid #bbc;
 }
 
-.pre {
+.pre, span.tok {
   font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace;
   font-weight: 500;
   font-size: 0.85em;
@@ -557,6 +560,12 @@ blockquote {
   border-radius: 4px;
 }
 
+span.tok {
+  border: 1px solid #808080;
+  padding-bottom: 0.1em;
+  margin-right: 0.2em;
+}
+
 pre {
   font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace;
   color: var(--text);
@@ -844,9 +853,6 @@ span.classifier {
 span.classifier-delimiter {
   font-weight: bold; }
 
-span.option {
-  white-space: nowrap; }
-
 span.problematic {
   color: #b30000; }
 
@@ -926,6 +932,21 @@ span.Preprocessor {
 span.Directive {
   color: #252dbe; }
 
+span.option {
+  font-weight: bold;
+  font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace;
+  color: var(--option);
+}
+
+span.program {
+  font-weight: bold;
+  color: var(--program);
+  text-decoration: underline;
+  text-decoration-color: var(--hint);
+  text-decoration-thickness: 0.05em;
+  text-underline-offset: 0.15em;
+}
+
 span.Command, span.Rule, span.Hyperlink, span.Label, span.Reference,
 span.Other {
   color: var(--other); }

lib/packages/docutils/highlite.nim

@@ -37,6 +37,18 @@
 ## .. code:: Nim
 ##   for l in ["C", "c++", "jAvA", "Nim", "c#"]: echo getSourceLanguage(l)
 ##
+## There is also a `Cmd` pseudo-language supported, which is a simple generic
+## shell/cmdline tokenizer (UNIX shell/Powershell/Windows Command):
+## no escaping, no programming language constructs besides variable definition
+## at the beginning of line. It supports these operators:
+##
+## .. code:: Cmd
+##    &  &&  |  ||  (  )  ''  ""  ;  # for comments
+##
+## Instead of escaping always use quotes like here
+## `nimgrep --ext:'nim|nims' file.name`:cmd: shows how to input ``|``.
+## Any argument that contains ``.`` or ``/`` or ``\`` will be treated
+## as a file or directory.
 
 import
   strutils
@@ -45,15 +57,15 @@ from algorithm import binarySearch
 type
   SourceLanguage* = enum
     langNone, langNim, langCpp, langCsharp, langC, langJava,
-    langYaml, langPython
+    langYaml, langPython, langCmd
   TokenClass* = enum
     gtEof, gtNone, gtWhitespace, gtDecNumber, gtBinNumber, gtHexNumber,
     gtOctNumber, gtFloatNumber, gtIdentifier, gtKeyword, gtStringLit,
     gtLongStringLit, gtCharLit, gtEscapeSequence, # escape sequence like \xff
     gtOperator, gtPunctuation, gtComment, gtLongComment, gtRegularExpression,
     gtTagStart, gtTagEnd, gtKey, gtValue, gtRawData, gtAssembler,
     gtPreprocessor, gtDirective, gtCommand, gtRule, gtHyperlink, gtLabel,
-    gtReference, gtOther
+    gtReference, gtProgram, gtOption, gtOther
   GeneralTokenizer* = object of RootObj
     kind*: TokenClass
     start*, length*: int
@@ -64,14 +76,17 @@ type
 
 const
   sourceLanguageToStr*: array[SourceLanguage, string] = ["none",
-    "Nim", "C++", "C#", "C", "Java", "Yaml", "Python"]
+    "Nim", "C++", "C#", "C", "Java", "Yaml", "Python", "Cmd"]
   tokenClassToStr*: array[TokenClass, string] = ["Eof", "None", "Whitespace",
     "DecNumber", "BinNumber", "HexNumber", "OctNumber", "FloatNumber",
     "Identifier", "Keyword", "StringLit", "LongStringLit", "CharLit",
     "EscapeSequence", "Operator", "Punctuation", "Comment", "LongComment",
     "RegularExpression", "TagStart", "TagEnd", "Key", "Value", "RawData",
     "Assembler", "Preprocessor", "Directive", "Command", "Rule", "Hyperlink",
-    "Label", "Reference", "Other"]
+    "Label", "Reference",
+    # start from lower-case if there is a corresponding RST role (see rst.nim)
+    "program", "option",
+    "Other"]
 
   # The following list comes from doc/keywords.txt, make sure it is
   # synchronized with this array by running the module itself as a test case.
@@ -898,6 +913,65 @@ proc pythonNextToken(g: var GeneralTokenizer) =
       "with", "yield"]
   nimNextToken(g, keywords)
 
+proc cmdNextToken(g: var GeneralTokenizer) =
+  var pos = g.pos
+  g.start = g.pos
+  if g.state == low(TokenClass):
+    g.state = gtProgram
+  case g.buf[pos]
+  of ' ', '\t'..'\r':
+    g.kind = gtWhitespace
+    while g.buf[pos] in {' ', '\t'..'\r'}:
+      if g.buf[pos] == '\n':
+        g.state = gtProgram
+      inc(pos)
+  of '\'', '"':
+    g.kind = gtOption
+    let q = g.buf[pos]
+    inc(pos)
+    while g.buf[pos] notin {q, '\0'}:
+      inc(pos)
+    if g.buf[pos] == q: inc(pos)
+  of '#':
+    g.kind = gtComment
+    while g.buf[pos] notin {'\n', '\0'}:
+      inc(pos)
+  of '&', '|':
+    g.kind = gtOperator
+    inc(pos)
+    if g.buf[pos] == g.buf[pos-1]: inc(pos)
+    g.state = gtProgram
+  of '(':
+    g.kind = gtOperator
+    g.state = gtProgram
+    inc(pos)
+  of ')':
+    g.kind = gtOperator
+    inc(pos)
+  of ';':
+    g.state = gtProgram
+    g.kind = gtOperator
+    inc(pos)
+  of '\0': g.kind = gtEof
+  else:
+    if g.state == gtProgram:
+      g.kind = gtProgram
+      g.state = gtOption
+    else:
+      g.kind = gtOption
+    while g.buf[pos] notin {' ', '\t'..'\r', '&', '|', '(', ')', '\'', '"', '\0'}:
+      if g.buf[pos] == ';' and g.buf[pos+1] == ' ':
+        # (check space because ';' can be used inside arguments in Win bat)
+        break
+      if g.kind == gtOption and g.buf[pos] in {'/', '\\', '.'}:
+        g.kind = gtIdentifier  # for file/dir name
+      elif g.kind == gtProgram and g.buf[pos] == '=':
+        g.kind = gtIdentifier  # for env variable setting at beginning of line
+        g.state = gtProgram
+      inc(pos)
+  g.length = pos - g.pos
+  g.pos = pos
+
 proc getNextToken*(g: var GeneralTokenizer, lang: SourceLanguage) =
   g.lang = lang
   case lang
@@ -909,6 +983,7 @@ proc getNextToken*(g: var GeneralTokenizer, lang: SourceLanguage) =
   of langJava: javaNextToken(g)
   of langYaml: yamlNextToken(g)
   of langPython: pythonNextToken(g)
+  of langCmd: cmdNextToken(g)
 
 when isMainModule:
   var keywords: seq[string]

lib/packages/docutils/rst.nim

@@ -23,10 +23,10 @@
 ##
 ## Nim can output the result to HTML [#html]_ or Latex [#latex]_.
 ##
-## .. [#html] commands ``nim doc`` for ``*.nim`` files and
-##    ``nim rst2html`` for ``*.rst`` files
+## .. [#html] commands `nim doc`:cmd: for ``*.nim`` files and
+##    `nim rst2html`:cmd: for ``*.rst`` files
 ##
-## .. [#latex] command ``nim rst2tex`` for ``*.rst``.
+## .. [#latex] command `nim rst2tex`:cmd: for ``*.rst``.
 ##
 ## If you are new to RST please consider reading the following:
 ##
@@ -78,14 +78,21 @@
 ##
 ## * directives: ``code-block`` [cmp:Sphinx]_, ``title``,
 ##   ``index`` [cmp:Sphinx]_
-## * predefined roles ``:nim:`` (default), ``:c:`` (C programming language),
-##   ``:python:``, ``:yaml:``, ``:java:``, ``:cpp:`` (C++), ``:csharp`` (C#).
-##   That is every language that `highlite <highlite.html>`_ supports.
-##   They turn on appropriate syntax highlighting in inline code.
+## * predefined roles
+##   - ``:nim:`` (default), ``:c:`` (C programming language),
+##     ``:python:``, ``:yaml:``, ``:java:``, ``:cpp:`` (C++), ``:csharp`` (C#).
+##     That is every language that `highlite <highlite.html>`_ supports.
+##     They turn on appropriate syntax highlighting in inline code.
 ##
-##   .. Note:: default role for Nim files is ``:nim:``,
-##             for ``*.rst`` it's currently ``:literal:``.
+##     .. Note:: default role for Nim files is ``:nim:``,
+##               for ``*.rst`` it's currently ``:literal:``.
 ##
+##   - generic command line highlighting roles:
+##     - ``:cmd:`` for commands and common shells syntax
+##     - ``:program:`` for executable names [cmp:Sphinx]_
+##       (one can just use ``:cmd:`` on single word)
+##     - ``:option:`` for command line options [cmp:Sphinx]_
+##   - ``:tok:``, a role for highlighting of programming language tokens
 ## * ***triple emphasis*** (bold and italic) using \*\*\*
 ## * ``:idx:`` role for \`interpreted text\` to include the link to this
 ##   text into an index (example: `Nim index`_).
@@ -95,11 +102,11 @@
 ##     //compile   compile the project
 ##     //doc       generate documentation
 ##
-##   Here the dummy `//` will disappear, while options ``compile``
-##   and ``doc`` will be left in the final document.
+##   Here the dummy `//` will disappear, while options `compile`:option:
+##   and `doc`:option: will be left in the final document.
 ##
 ## .. [cmp:Sphinx] similar but different from the directives of
-##    Python `Sphinx directives`_ extensions
+##    Python `Sphinx directives`_ and `Sphinx roles`_ extensions
 ##
 ## .. _`extra features`:
 ##
@@ -144,7 +151,7 @@
 ## -----
 ##
 ## See `Nim DocGen Tools Guide <docgen.html>`_ for the details about
-## ``nim doc``, ``nim rst2html`` and ``nim rst2tex`` commands.
+## `nim doc`:cmd:, `nim rst2html`:cmd: and `nim rst2tex`:cmd: commands.
 ##
 ## See `packages/docutils/rstgen module <rstgen.html>`_ to know how to
 ## generate HTML or Latex strings to embed them into your documents.
@@ -156,6 +163,7 @@
 ## .. _RST roles list: https://docutils.sourceforge.io/docs/ref/rst/roles.html
 ## .. _Nim index: https://nim-lang.org/docs/theindex.html
 ## .. _Sphinx directives: https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html
+## .. _Sphinx roles: https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html
 
 import
   os, strutils, rstast, std/enumutils, algorithm, lists, sequtils,
@@ -530,7 +538,7 @@ proc defaultRole(options: RstParseOptions): string =
 
 # mirror highlite.nim sourceLanguageToStr with substitutions c++ cpp, c# csharp
 const supportedLanguages = ["nim", "yaml", "python", "java", "c",
-                            "cpp", "csharp"]
+                            "cpp", "csharp", "cmd"]
 
 proc whichRoleAux(sym: string): RstNodeKind =
   let r = sym.toLowerAscii
@@ -543,6 +551,7 @@ proc whichRoleAux(sym: string): RstNodeKind =
   of "sup", "superscript": result = rnSup
   # literal and code are the same in our implementation
   of "code": result = rnInlineLiteral
+  of "program", "option", "tok": result = rnCodeFragment
   # c++ currently can be spelled only as cpp, c# only as csharp
   elif r in supportedLanguages:
     result = rnInlineCode
@@ -1113,10 +1122,10 @@ proc toInlineCode(n: PRstNode, language: string): PRstNode =
   lb.add newLeaf(s)
   result.add lb
 
-proc toUnknownRole(n: PRstNode, roleName: string): PRstNode =
+proc toOtherRole(n: PRstNode, kind: RstNodeKind, roleName: string): PRstNode =
   let newN = newRstNode(rnInner, n.sons)
   let newSons = @[newN, newLeaf(roleName)]
-  result = newRstNode(rnUnknownRole, newSons)
+  result = newRstNode(kind, newSons)
 
 proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
   var newKind = n.kind
@@ -1144,8 +1153,8 @@ proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
     # a role:
     let (roleName, lastIdx) = getRefname(p, p.idx+1)
     newKind = whichRole(p, roleName)
-    if newKind == rnUnknownRole:
-      result = n.toUnknownRole(roleName)
+    if newKind in {rnUnknownRole, rnCodeFragment}:
+      result = n.toOtherRole(newKind, roleName)
     elif newKind == rnInlineCode:
       result = n.toInlineCode(language=roleName)
     else:
@@ -1417,8 +1426,8 @@ proc parseInline(p: var RstParser, father: PRstNode) =
       if k == rnInlineCode:
         n = n.toInlineCode(language=roleName)
       parseUntil(p, n, "`", false) # bug #17260
-      if k == rnUnknownRole:
-        n = n.toUnknownRole(roleName)
+      if k in {rnUnknownRole, rnCodeFragment}:
+        n = n.toOtherRole(k, roleName)
       father.add(n)
     elif isInlineMarkupStart(p, "`"):
       var n = newRstNode(rnInterpretedText)