Merge branch 'master' into suite-to-workflow

CHANGES.md

@@ -52,6 +52,24 @@ creating a new release entry be sure to copy & paste the span tag with the
 `actions:bind` attribute, which is used by a regex to find the text to be
 updated. Only the first match gets replaced, so it's fine to leave the old
 ones in. -->
+-------------------------------------------------------------------------------
+## __cylc-8.0b2 (<span actions:bind='release-date'>Released 2021-??-??</span>)__
+
+### Enhancements
+
+[#4177](https://github.com/cylc/cylc-flow/pull/4177) - Remove obsolete
+configuration items from `global.cylc[platforms][<platform name>]`:
+`run directory`, `work directory` and `suite definition directory`. This
+functionality is now provided by `[symlink dirs]`.
+
+### Fixes
+
+[#4180](https://github.com/cylc/cylc-flow/pull/4180) - Fix bug where installing
+a workflow that uses the deprecated `suite.rc` filename would symlink `flow.cylc`
+to the `suite.rc` in the source dir instead of the run dir. Also fixes a couple
+of other, small bugs.
+
+
 -------------------------------------------------------------------------------
 ## __cylc-8.0b1 (<span actions:bind='release-date'>Released 2021-04-21</span>)__
 
@@ -70,7 +88,7 @@ Replace the job "host" field with "platform" in the GraphQL schema.
 Fix a host ⇒ platform upgrade bug where host names were being popped from task
 configs causing subsequent tasks to run on localhost.
 
-[#4173](https://github.com/cylc/cylc-flow/pull/4173)
+[#4173](https://github.com/cylc/cylc-flow/pull/4173) -
 Fix the state totals shown in both the UI and TUI, including incorrect counts
 during workflow run and post pause.
 

cylc/flow/cfgspec/globalcfg.py

@@ -424,22 +424,6 @@
             ''')
             Conf('job runner command template', VDR.V_STRING)
             Conf('shell', VDR.V_STRING, '/bin/bash')
-            Conf('run directory', VDR.V_STRING, '$HOME/cylc-run', desc='''
-                The directory in which to install workflows.
-            ''')
-            Conf('work directory', VDR.V_STRING, '$HOME/cylc-run', desc='''
-                The top level for work and share directories. Can contain
-                ``$HOME`` or ``$USER`` but not other environment variables (the
-                item cannot actually be evaluated by the shell on HOST before
-                use, but the remote home directory is where ``rsync`` and
-                ``ssh`` naturally land, and the remote username is known by the
-                scheduler).
-
-                Example::
-
-                   /nfs/data/$USER/cylc-run
-            ''')
-            Conf('workflow definition directory', VDR.V_STRING)
             Conf('communication method',
                  VDR.V_STRING, 'zmq',
                  options=[meth.value for meth in CommsMeth], desc='''
@@ -839,30 +823,25 @@ class GlobalConfig(ParsecConfig):
     CONF_BASENAME: str = "global.cylc"
     DEFAULT_SITE_CONF_PATH: str = os.path.join(os.sep, 'etc', 'cylc')
     USER_CONF_PATH: str = os.path.join(
-        os.getenv('HOME') or get_user_home(),
-        '.cylc',
-        'flow'
+        os.getenv('HOME') or get_user_home(), '.cylc', 'flow'
     )
     VERSION_HIERARCHY: List[str] = get_version_hierarchy(CYLC_VERSION)
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args, **kwargs) -> None:
+        site_conf_root = (
+            os.getenv('CYLC_SITE_CONF_PATH') or self.DEFAULT_SITE_CONF_PATH
+        )
         self.conf_dir_hierarchy: List[Tuple[str, str]] = [
             *[
-                (
-                    upgrader.SITE_CONFIG,
-                    os.path.join(
-                        (
-                            os.getenv('CYLC_SITE_CONF_PATH')
-                            or self.DEFAULT_SITE_CONF_PATH
-                        ),
-                        'flow',
-                        ver
-                    )
-                )
+                (upgrader.SITE_CONFIG,
+                 os.path.join(site_conf_root, 'flow', ver))
                 for ver in self.VERSION_HIERARCHY
             ],
-            *[(upgrader.USER_CONFIG, os.path.join(self.USER_CONF_PATH, ver))
-              for ver in self.VERSION_HIERARCHY]
+            *[
+                (upgrader.USER_CONFIG,
+                 os.path.join(self.USER_CONF_PATH, ver))
+                for ver in self.VERSION_HIERARCHY
+            ]
         ]
         super().__init__(*args, **kwargs)
 

cylc/flow/cfgspec/workflow.py

@@ -18,6 +18,7 @@
 import re
 
 from itertools import product
+from typing import Any, Dict, Set
 
 from metomi.isodatetime.data import Calendar
 
@@ -848,8 +849,7 @@
 
                 The top level share and work directory location can be changed
                 (e.g. to a large data area) by a global config setting (see
-                :cylc:conf:`global.cylc
-                [platforms][<platform name>]work directory`).
+                :cylc:conf:`global.cylc[symlink dirs]`).
 
                 .. note::
 
@@ -1534,7 +1534,7 @@ def upg(cfg, descr):
                 )
 
 
-def upgrade_graph_section(cfg, descr):
+def upgrade_graph_section(cfg: Dict[str, Any], descr: str) -> None:
     """Upgrade Cylc 7 `[scheduling][dependencies][X]graph` format to
     `[scheduling][graph]X`."""
     # Parsec upgrader cannot do this type of move
@@ -1548,23 +1548,25 @@ def upgrade_graph_section(cfg, descr):
                     f'because {msg_new[:-1]} already exists.'
                 )
             else:
-                keys = set()
+                keys: Set[str] = set()
                 cfg['scheduling'].setdefault('graph', {})
                 cfg['scheduling']['graph'].update(
                     cfg['scheduling'].pop('dependencies')
                 )
-                graphdict = cfg['scheduling']['graph']
+                graphdict: Dict[str, Any] = cfg['scheduling']['graph']
                 for key, value in graphdict.copy().items():
                     if isinstance(value, dict) and 'graph' in value:
                         graphdict[key] = value['graph']
                         keys.add(key)
+                    elif key == 'graph' and isinstance(value, str):
+                        graphdict[key] = value
+                        keys.add(key)
                 if keys:
-                    keys = ', '.join(sorted(keys))
                     LOG.warning(
                         'deprecated graph items were automatically upgraded '
                         f'in "{descr}":\n'
                         f' * (8.0.0) {msg_old} -> {msg_new} - for X in:\n'
-                        f'       {keys}'
+                        f"       {', '.join(sorted(keys))}"
                     )
     except KeyError:
         pass

cylc/flow/dbstatecheck.py

@@ -19,6 +19,8 @@
 import os
 import sqlite3
 import sys
+
+from cylc.flow.pathutil import expand_path
 from cylc.flow.rundb import CylcWorkflowDAO
 from cylc.flow.task_state import (
     TASK_STATUS_SUBMITTED,
@@ -55,9 +57,9 @@ class CylcWorkflowDBChecker:
     }
 
     def __init__(self, rund, workflow):
-        db_path = os.path.join(
-            os.path.expanduser(rund), workflow, "log",
-            CylcWorkflowDAO.DB_FILE_BASE_NAME)
+        db_path = expand_path(
+            rund, workflow, "log", CylcWorkflowDAO.DB_FILE_BASE_NAME
+        )
         if not os.path.exists(db_path):
             raise OSError(errno.ENOENT, os.strerror(errno.ENOENT), db_path)
         self.conn = sqlite3.connect(db_path, timeout=10.0)

cylc/flow/etc/job.sh

@@ -71,9 +71,8 @@ cylc__job__main() {
     echo
     # Derived environment variables
     export CYLC_WORKFLOW_LOG_DIR="${CYLC_WORKFLOW_RUN_DIR}/log/workflow"
-    CYLC_WORKFLOW_WORK_DIR_ROOT="${CYLC_WORKFLOW_WORK_DIR_ROOT:-${CYLC_WORKFLOW_RUN_DIR}}"
-    export CYLC_WORKFLOW_SHARE_DIR="${CYLC_WORKFLOW_WORK_DIR_ROOT}/share"
-    export CYLC_WORKFLOW_WORK_DIR="${CYLC_WORKFLOW_WORK_DIR_ROOT}/work"
+    export CYLC_WORKFLOW_SHARE_DIR="${CYLC_WORKFLOW_RUN_DIR}/share"
+    export CYLC_WORKFLOW_WORK_DIR="${CYLC_WORKFLOW_RUN_DIR}/work"
     CYLC_TASK_CYCLE_POINT="$(cut -d '/' -f 1 <<<"${CYLC_TASK_JOB}")"
     CYLC_TASK_NAME="$(cut -d '/' -f 2 <<<"${CYLC_TASK_JOB}")"
     export CYLC_TASK_NAME CYLC_TASK_CYCLE_POINT ISODATETIMEREF

cylc/flow/etc/syntax/cylc-mode.el

@@ -1,7 +1,7 @@
 ;; ____________________________________________________________________________
 ;;
 ;; = cylc-mode.el =
-;;    Emacs syntax highlighting mode for Cylc suite definition (flow.cylc)
+;;    Emacs syntax highlighting mode for Cylc workflow definition (flow.cylc)
 ;;    files
 ;; ____________________________________________________________________________
 ;;
@@ -37,7 +37,7 @@
 
 ;; Define the mode and the syntax highlighting for it
 (define-derived-mode cylc-mode fundamental-mode
-  "flow.cylc" "Major mode for editing Cylc suite definition files"
+  "flow.cylc" "Major mode for editing Cylc workflow definition files"
 
   ;; Note: ordered according to reverse application precedence, where
   ;; specification order for faces changes resultant highlighting

cylc/flow/etc/syntax/cylc.lang

@@ -106,7 +106,6 @@
         <keyword>task event batch interval</keyword>
         <keyword>suite state polling</keyword>
         <keyword>workflow state polling</keyword>
-        <keyword>suite definition directory</keyword>
         <keyword>succeeded handler</keyword>
         <keyword>submitted handler</keyword>
         <keyword>submission timeout handler</keyword>

cylc/flow/etc/syntax/cylc.xml

@@ -32,7 +32,6 @@
         <RegExpr attribute='Keyword' String=' task event batch interval '/>
         <RegExpr attribute='Keyword' String=' suite state polling '/>
         <RegExpr attribute='Keyword' String=' workflow state polling '/>
-        <RegExpr attribute='Keyword' String=' suite definition directory '/>
         <RegExpr attribute='Keyword' String=' succeeded handler '/>
         <RegExpr attribute='Keyword' String=' submitted handler '/>
         <RegExpr attribute='Keyword' String=' submission timeout handler '/>

cylc/flow/job_file.py

@@ -24,9 +24,7 @@
 from cylc.flow import __version__ as CYLC_VERSION
 from cylc.flow.job_runner_mgr import JobRunnerManager
 import cylc.flow.flags
-from cylc.flow.pathutil import (
-    get_remote_workflow_run_dir,
-    get_remote_workflow_work_dir)
+from cylc.flow.pathutil import get_remote_workflow_run_dir
 from cylc.flow.config import interpolate_template, ParamExpandError
 
 
@@ -56,11 +54,8 @@ def write(self, local_job_file_path, job_conf, check_syntax=True):
         # Access to cylc must be configured before user environment so
         # that cylc commands can be used in defining user environment
         # variables: NEXT_CYCLE=$( cylc cycle-point --offset-hours=6 )
-        platform = job_conf['platform']
         tmp_name = os.path.expandvars(local_job_file_path + '.tmp')
-        run_d = get_remote_workflow_run_dir(
-            platform, job_conf['workflow_name']
-        )
+        run_d = get_remote_workflow_run_dir(job_conf['workflow_name'])
         try:
             with open(tmp_name, 'w') as handle:
                 self._write_header(handle, job_conf)
@@ -203,14 +198,7 @@ def _write_workflow_environment(self, handle, job_conf, run_d):
 
         handle.write('\n')
         # override and write task-host-specific workflow variables
-        work_d = get_remote_workflow_work_dir(
-            job_conf["platform"], job_conf['workflow_name'])
         handle.write('\n    export CYLC_WORKFLOW_RUN_DIR="%s"' % run_d)
-        if work_d != run_d:
-            # Note: not an environment variable, but used by job.sh
-            handle.write(
-                '\n    export CYLC_WORKFLOW_WORK_DIR_ROOT="%s"' % work_d
-            )
         handle.write(
             '\n    export CYLC_WORKFLOW_UUID="%s"' % job_conf['uuid_str'])
 

cylc/flow/parsec/exceptions.py

@@ -52,13 +52,14 @@ class FileParseError(ParsecError):
     """Error raised when attempting to read in the config file(s)."""
 
     def __init__(self, reason, index=None, line=None, lines=None,
-                 err_type=None, fpath=None):
+                 err_type=None, fpath=None, help_lines=None):
         self.reason = reason
         self.line_num = index + 1 if index is not None else None
         self.line = line
         self.lines = lines
         self.err_type = err_type
         self.fpath = fpath
+        self.help_lines = help_lines or []
 
     def __str__(self):
         msg = ''
@@ -78,9 +79,12 @@ def __str__(self):
             msg += "\t<--"
             if self.err_type:
                 msg += ' %s' % self.err_type
+        help_lines = list(self.help_lines)
         if self.line_num:
             # TODO - make 'view' function independent of cylc:
-            msg += "\n(line numbers match 'cylc view -p')"
+            help_lines.append("line numbers match 'cylc view -p'")
+        for help_line in help_lines:
+            msg += f'\n({help_line})'
         return msg
 
 

cylc/flow/parsec/fileparse.py

@@ -95,6 +95,10 @@
     '"""': (_SINGLE_LINE_DOUBLE, _MULTI_LINE_DOUBLE),
 }
 
+_UNCLOSED_MULTILINE = re.compile(
+    r'(?<![\w>])\[.*\]'
+)
+
 
 def _concatenate(lines):
     """concatenate continuation lines"""
@@ -460,7 +464,19 @@ def parse(fpath, output_fname=None, template_vars=None):
                 addict(config, key, val, parents, index)
             else:
                 # no match
+                help_lines = None
+                if 'val' in locals() and _UNCLOSED_MULTILINE.search(val):
+                    # this might be an unclosed multiline string
+                    # provide a helpful error message
+                    key_name = ''.join(
+                        [f'[{parent}]' for parent in parents]
+                    ) + key
+                    help_lines = [f'Did you forget to close {key_name}?']
                 raise FileParseError(
-                    'Invalid line', index=index, line=line)
+                    'Invalid line',
+                    index=index,
+                    line=line,
+                    help_lines=help_lines
+                )
 
     return config