completion: autogenerate (iterative#3921)

* cli: split main_parser

* cli: basic parser tree printing

* cli: major completion tidy

* completion: more missing root commands

* completion: much automagic such beauty

- fixes iterative#2985 automatically generate bash completion

* tests: bash completion

* tests: completion: tidy script a little

* minor tidy

* command: positional name consistency

* completion: bash: add logging

* double-check renames and consistency

* cli: add completion subcommand

* ci: completion: update test

* minor logging

* more completion logging verbosity

* fix completion of files

* add shtab

* purge unneeded completion => shtab

* completion: neaten UI

* completion: use `shtab>=0.0.2`

* completion: purge hardcoded completions!

* revert unneeded renames for now

* fix isort

* completion: tidy CLI API

* fix snap build

* fix snap build again

* snap: internally build completion

* allow repo-less completion

* command: tidy headless logic

* use CmdBaseNoRepo

* better DVC-file completion

* merge command.choices -> command.completion

* drop choices class

* remove completion optionals

* tidy docs

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,6 +1,6 @@
 * [ ] ❗ I have followed the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) checklist.
 
-* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here. If the CLI API is changed, I have updated [tab completion scripts](https://github.com/iterative/dvc/tree/master/scripts/completion).
+* [ ] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here.
 
 * [ ] ❌ I will check DeepSource, CodeClimate, and other sanity checks below. (We consider them recommendatory and don't expect everything to be addressed. Please fix things that actually improve code or fix bugs.)
 

dvc/cli.py

@@ -8,6 +8,7 @@
     cache,
     checkout,
     commit,
+    completion,
     config,
     daemon,
     dag,
@@ -70,6 +71,7 @@
     dag,
     daemon,
     commit,
+    completion,
     diff,
     version,
     update,
@@ -140,15 +142,7 @@ def get_parent_parser():
     return parent_parser
 
 
-def parse_args(argv=None):
-    """Parses CLI arguments.
-
-    Args:
-        argv: optional list of arguments to parse. sys.argv is used by default.
-
-    Raises:
-        dvc.exceptions.DvcParserError: raised for argument parsing errors.
-    """
+def get_main_parser():
     parent_parser = get_parent_parser()
 
     # Main parser
@@ -196,6 +190,18 @@ def parse_args(argv=None):
     for cmd in COMMANDS:
         cmd.add_parser(subparsers, parent_parser)
 
-    args = parser.parse_args(argv)
+    return parser
+
 
+def parse_args(argv=None):
+    """Parses CLI arguments.
+
+    Args:
+        argv: optional list of arguments to parse. sys.argv is used by default.
+
+    Raises:
+        dvc.exceptions.DvcParserError: raised for argument parsing errors.
+    """
+    parser = get_main_parser()
+    args = parser.parse_args(argv)
     return args

dvc/command/add.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename
 
@@ -62,6 +63,9 @@ def add_parser(subparsers, parent_parser):
         metavar="<filename>",
     )
     parser.add_argument(
-        "targets", nargs="+", help="Input files/directories to add."
+        "targets",
+        nargs="+",
+        help="Input files/directories to add.",
+        choices=completion.Required.FILE,
     )
     parser.set_defaults(func=CmdAdd)

dvc/command/cache.py

@@ -1,5 +1,6 @@
 import argparse
 
+from dvc.command import completion
 from dvc.command.base import append_doc_link, fix_subparsers
 from dvc.command.config import CmdConfig
 
@@ -55,5 +56,6 @@ def add_parser(subparsers, parent_parser):
         help="Path to cache directory. Relative paths are resolved relative "
         "to the current directory and saved to config relative to the "
         "config file location.",
+        choices=completion.Required.DIR,
     )
     cache_dir_parser.set_defaults(func=CmdCacheDir)

dvc/command/checkout.py

@@ -4,6 +4,7 @@
 
 import colorama
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import CheckoutError
 from dvc.utils.humanize import get_summary
@@ -113,5 +114,6 @@ def add_parser(subparsers, parent_parser):
         nargs="*",
         help="DVC-files to checkout. Optional. "
         "(Finds all DVC-files in the workspace by default.)",
+        choices=completion.Optional.DVC_FILE,
     )
     checkout_parser.set_defaults(func=CmdCheckout)

dvc/command/commit.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
@@ -66,5 +67,6 @@ def add_parser(subparsers, parent_parser):
         nargs="*",
         help="DVC-files to commit. Optional. "
         "(Finds all DVC-files in the workspace by default.)",
+        choices=completion.Optional.DVC_FILE,
     )
     commit_parser.set_defaults(func=CmdCommit)

dvc/command/completion.py

@@ -0,0 +1,68 @@
+import argparse
+import logging
+
+import shtab
+
+from dvc.command.base import CmdBaseNoRepo, append_doc_link
+
+logger = logging.getLogger(__name__)
+CHOICE_FUNCTIONS = {
+    "bash": {"DVCFile": "_dvc_compgen_DVCFiles"},
+    "zsh": {"DVCFile": "_files -g '(*?.dvc|Dvcfile|dvc.yaml)'"},
+}
+PREAMBLE = {
+    "bash": """
+# $1=COMP_WORDS[1]
+_dvc_compgen_DVCFiles() {
+  compgen -d -S '/' -- $1  # recurse into subdirs
+  compgen -f -X '!*?.dvc' -- $1
+  compgen -f -X '!*Dvcfile' -- $1
+  compgen -f -X '!*dvc.yaml' -- $1
+}
+""",
+    "zsh": "",
+}
+
+
+class Optional(shtab.Optional):
+    DVC_FILE = [shtab.Choice("DVCFile", required=False)]
+
+
+class Required(shtab.Required):
+    DVC_FILE = [shtab.Choice("DVCFile", required=True)]
+
+
+class CmdCompletion(CmdBaseNoRepo):
+    def run(self):
+        from dvc.cli import get_main_parser
+
+        parser = get_main_parser()
+        shell = self.args.shell
+        script = shtab.complete(
+            parser,
+            shell=shell,
+            preamble=PREAMBLE[shell],
+            choice_functions=CHOICE_FUNCTIONS[shell],
+        )
+        print(script)
+        return 0
+
+
+def add_parser(subparsers, parent_parser):
+    COMPLETION_HELP = "Generate shell tab completion."
+    COMPLETION_DESCRIPTION = "Prints out shell tab completion scripts."
+    completion_parser = subparsers.add_parser(
+        "completion",
+        parents=[parent_parser],
+        description=append_doc_link(COMPLETION_DESCRIPTION, "completion"),
+        help=COMPLETION_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    completion_parser.add_argument(
+        "-s",
+        "--shell",
+        help="Shell syntax for completions.",
+        default="bash",
+        choices=["bash", "zsh"],
+    )
+    completion_parser.set_defaults(func=CmdCompletion)

dvc/command/daemon.py

@@ -1,3 +1,4 @@
+from dvc.command import completion
 from dvc.command.base import CmdBaseNoRepo, fix_subparsers
 
 
@@ -64,5 +65,7 @@ def add_parser(subparsers, parent_parser):
         description=DAEMON_ANALYTICS_HELP,
         help=DAEMON_ANALYTICS_HELP,
     )
-    daemon_analytics_parser.add_argument("target", help="Analytics file.")
+    daemon_analytics_parser.add_argument(
+        "target", help="Analytics file.", choices=completion.Required.FILE
+    )
     daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)

dvc/command/data_sync.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.command.checkout import log_changes
 from dvc.exceptions import CheckoutError, DvcException
@@ -104,6 +105,7 @@ def shared_parent_parser():
         nargs="*",
         help="Limit command scope to these DVC-files. "
         "Using -R, directories to search DVC-files in can also be given.",
+        choices=completion.Optional.DVC_FILE,
     )
 
     return parent_parser

dvc/command/freeze.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
@@ -39,7 +40,10 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     freeze_parser.add_argument(
-        "targets", nargs="+", help="DVC-files to freeze."
+        "targets",
+        nargs="+",
+        help="DVC-files to freeze.",
+        choices=completion.Required.DVC_FILE,
     )
     freeze_parser.set_defaults(func=CmdFreeze)
 
@@ -52,6 +56,9 @@ def add_parser(subparsers, parent_parser):
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     unfreeze_parser.add_argument(
-        "targets", nargs="+", help="DVC-files to unfreeze."
+        "targets",
+        nargs="+",
+        help="DVC-files to unfreeze.",
+        choices=completion.Required.DVC_FILE,
     )
     unfreeze_parser.set_defaults(func=CmdUnfreeze)

dvc/command/get.py

@@ -3,6 +3,7 @@
 
 from dvc.exceptions import DvcException
 
+from . import completion
 from .base import CmdBaseNoRepo, append_doc_link
 
 logger = logging.getLogger(__name__)
@@ -62,14 +63,17 @@ def add_parser(subparsers, parent_parser):
         "url", help="Location of DVC or Git repository to download from"
     )
     get_parser.add_argument(
-        "path", help="Path to a file or directory within the repository"
+        "path",
+        help="Path to a file or directory within the repository",
+        choices=completion.Required.FILE,
     )
     get_parser.add_argument(
         "-o",
         "--out",
         nargs="?",
         help="Destination path to download files to",
         metavar="<path>",
+        choices=completion.Optional.DIR,
     )
     get_parser.add_argument(
         "--rev",

dvc/command/get_url.py

@@ -3,6 +3,7 @@
 
 from dvc.exceptions import DvcException
 
+from . import completion
 from .base import CmdBaseNoRepo, append_doc_link
 
 logger = logging.getLogger(__name__)
@@ -33,6 +34,9 @@ def add_parser(subparsers, parent_parser):
         "url", help="See `dvc import-url -h` for full list of supported URLs."
     )
     get_parser.add_argument(
-        "out", nargs="?", help="Destination path to put data to."
+        "out",
+        nargs="?",
+        help="Destination path to put data to.",
+        choices=completion.Optional.DIR,
     )
     get_parser.set_defaults(func=CmdGetUrl)

dvc/command/imp.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
@@ -43,14 +44,17 @@ def add_parser(subparsers, parent_parser):
         "url", help="Location of DVC or Git repository to download from"
     )
     import_parser.add_argument(
-        "path", help="Path to a file or directory within the repository"
+        "path",
+        help="Path to a file or directory within the repository",
+        choices=completion.Required.FILE,
     )
     import_parser.add_argument(
         "-o",
         "--out",
         nargs="?",
         help="Destination path to download files to",
         metavar="<path>",
+        choices=completion.Optional.DIR,
     )
     import_parser.add_argument(
         "--rev",

dvc/command/imp_url.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link
 from dvc.exceptions import DvcException
 
@@ -54,11 +55,15 @@ def add_parser(subparsers, parent_parser):
         "remote://myremote/path/to/file (see `dvc remote`)",
     )
     import_parser.add_argument(
-        "out", nargs="?", help="Destination path to put files to."
+        "out",
+        nargs="?",
+        help="Destination path to put files to.",
+        choices=completion.Optional.DIR,
     )
     import_parser.add_argument(
         "--file",
         help="Specify name of the DVC-file this command will generate.",
         metavar="<filename>",
+        choices=completion.Optional.DIR,
     )
     import_parser.set_defaults(func=CmdImportUrl)

dvc/command/ls/__init__.py

@@ -2,6 +2,7 @@
 import logging
 import sys
 
+from dvc.command import completion
 from dvc.command.base import CmdBaseNoRepo, append_doc_link
 from dvc.command.ls.ls_colors import LsColors
 from dvc.exceptions import DvcException
@@ -74,5 +75,6 @@ def add_parser(subparsers, parent_parser):
         "path",
         nargs="?",
         help="Path to directory within the repository to list outputs for",
+        choices=completion.Optional.DIR,
     )
     list_parser.set_defaults(func=CmdList)

dvc/command/metrics.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 
+from dvc.command import completion
 from dvc.command.base import CmdBase, append_doc_link, fix_subparsers
 from dvc.exceptions import BadMetricError, DvcException
 
@@ -169,6 +170,7 @@ def add_parser(subparsers, parent_parser):
             "Limit command scope to these metric files. Using -R, "
             "directories to search metric files in can also be given."
         ),
+        choices=completion.Optional.FILE,
     )
     metrics_show_parser.add_argument(
         "-a",
@@ -235,6 +237,7 @@ def add_parser(subparsers, parent_parser):
             "directories to search metric files in can also be given."
         ),
         metavar="<paths>",
+        choices=completion.Optional.FILE,
     )
     metrics_diff_parser.add_argument(
         "-R",