From 262c10d4a971a01082287f3dc73d731fd2dc218e Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Tue, 23 Jul 2024 15:08:16 -0600 Subject: [PATCH 1/9] minor changes after deployment --- qiita_db/support_files/patches/{test_db_sql => }/91.sql | 0 qiita_db/util.py | 3 ++- 2 files changed, 2 insertions(+), 1 deletion(-) rename qiita_db/support_files/patches/{test_db_sql => }/91.sql (100%) diff --git a/qiita_db/support_files/patches/test_db_sql/91.sql b/qiita_db/support_files/patches/91.sql similarity index 100% rename from qiita_db/support_files/patches/test_db_sql/91.sql rename to qiita_db/support_files/patches/91.sql diff --git a/qiita_db/util.py b/qiita_db/util.py index e77648cb0..66e4faac5 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -2775,7 +2775,7 @@ def update_resource_allocation_table(weeks=1, test=None): sacct = [ 'sacct', '-p', '--format=JobID,ElapsedRaw,MaxRSS,Submit,Start,End,CPUTimeRAW,' - 'ReqMem,AllocCPUs,AveVMSize', '--starttime', + 'ReqMem,AllocCPUs,AveVMSize,MaxVMSizeNode', '--starttime', dates[0].strftime('%Y-%m-%d'), '--endtime', dates[1].strftime('%Y-%m-%d'), '--user', 'qiita', '--state', 'CD'] @@ -2894,6 +2894,7 @@ def merge_rows(rows): df['MaxRSSRaw'] = df.MaxRSS.apply(lambda x: MaxRSS_helper(str(x))) df['ElapsedRawTime'] = df.ElapsedRaw.apply( lambda x: timedelta(seconds=float(x))) + df.replace({nan: None}, inplace=True) for index, row in df.iterrows(): with qdb.sql_connection.TRN: From a30f56805182ee528dcdb669f32c5953100e6403 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Tue, 23 Jul 2024 15:16:53 -0600 Subject: [PATCH 2/9] update qiime 2 --- .github/workflows/qiita-ci.yml | 6 +++--- qiita_db/util.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index 2960caf6e..dca188a0d 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -104,9 +104,9 @@ jobs: - name: Install plugins shell: bash -l {0} run: | - wget https://data.qiime2.org/distro/core/qiime2-2022.11-py38-linux-conda.yml - conda env create --quiet -n qtp-biom --file qiime2-2022.11-py38-linux-conda.yml - rm qiime2-2022.11-py38-linux-conda.yml + https://data.qiime2.org/distro/amplicon/qiime2-amplicon-2024.5-py39-linux-conda.yml + conda env create --quiet -n qtp-biom --file qiime2-amplicon-2024.5-py39-linux-conda.yml + rm qiime2-amplicon-2024.5-py39-linux-conda.yml export QIITA_ROOTCA_CERT=`pwd`/qiita_core/support_files/ci_rootca.crt export QIITA_CONFIG_FP=`pwd`/qiita_core/support_files/config_test.cfg export REDBIOM_HOST="http://localhost:7379" diff --git a/qiita_db/util.py b/qiita_db/util.py index 66e4faac5..839e4ff40 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -2894,7 +2894,7 @@ def merge_rows(rows): df['MaxRSSRaw'] = df.MaxRSS.apply(lambda x: MaxRSS_helper(str(x))) df['ElapsedRawTime'] = df.ElapsedRaw.apply( lambda x: timedelta(seconds=float(x))) - df.replace({nan: None}, inplace=True) + df.replace({np.nan: None}, inplace=True) for index, row in df.iterrows(): with qdb.sql_connection.TRN: From b6b57e8a0911187a100a1848adbf3b859c82e633 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Mon, 12 Aug 2024 07:55:34 -0600 Subject: [PATCH 3/9] update conda qiime2 to 3.10 --- .github/workflows/qiita-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index dca188a0d..60c8fada3 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -45,7 +45,7 @@ jobs: uses: conda-incubator/setup-miniconda@v2 with: auto-update-conda: true - python-version: '3.9' + python-version: '3.10' - name: Basic dependencies install env: From 3f16c46d3fd431420437bdc72047a43641e88859 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Mon, 12 Aug 2024 08:03:56 -0600 Subject: [PATCH 4/9] install tiny Q2 --- .github/workflows/qiita-ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index 60c8fada3..354d0144c 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -45,7 +45,7 @@ jobs: uses: conda-incubator/setup-miniconda@v2 with: auto-update-conda: true - python-version: '3.10' + python-version: '3.9' - name: Basic dependencies install env: @@ -104,9 +104,10 @@ jobs: - name: Install plugins shell: bash -l {0} run: | - https://data.qiime2.org/distro/amplicon/qiime2-amplicon-2024.5-py39-linux-conda.yml - conda env create --quiet -n qtp-biom --file qiime2-amplicon-2024.5-py39-linux-conda.yml - rm qiime2-amplicon-2024.5-py39-linux-conda.yml + # making sure we have the latesets conda version before + # installing Q2 + conda update conda --yes + conda env create -n qiime2-tiny-2024.5 --file https://data.qiime2.org/distro/tiny/qiime2-tiny-2024.5-py39-linux-conda.yml export QIITA_ROOTCA_CERT=`pwd`/qiita_core/support_files/ci_rootca.crt export QIITA_CONFIG_FP=`pwd`/qiita_core/support_files/config_test.cfg export REDBIOM_HOST="http://localhost:7379" From 7a147d8a616f5da505ee6e7baf6848bb89fb6cda Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Mon, 12 Aug 2024 08:14:08 -0600 Subject: [PATCH 5/9] rm conda update --- .github/workflows/qiita-ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index 354d0144c..edf818478 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -104,9 +104,6 @@ jobs: - name: Install plugins shell: bash -l {0} run: | - # making sure we have the latesets conda version before - # installing Q2 - conda update conda --yes conda env create -n qiime2-tiny-2024.5 --file https://data.qiime2.org/distro/tiny/qiime2-tiny-2024.5-py39-linux-conda.yml export QIITA_ROOTCA_CERT=`pwd`/qiita_core/support_files/ci_rootca.crt export QIITA_CONFIG_FP=`pwd`/qiita_core/support_files/config_test.cfg From 7eb0038a72f6483ebd4a1b0e05b0693a2a774586 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Mon, 12 Aug 2024 08:19:19 -0600 Subject: [PATCH 6/9] qtp-biom --- .github/workflows/qiita-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index edf818478..4e455429d 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -104,7 +104,7 @@ jobs: - name: Install plugins shell: bash -l {0} run: | - conda env create -n qiime2-tiny-2024.5 --file https://data.qiime2.org/distro/tiny/qiime2-tiny-2024.5-py39-linux-conda.yml + conda env create -n qtp-biom --file https://data.qiime2.org/distro/tiny/qiime2-tiny-2024.5-py39-linux-conda.yml export QIITA_ROOTCA_CERT=`pwd`/qiita_core/support_files/ci_rootca.crt export QIITA_CONFIG_FP=`pwd`/qiita_core/support_files/config_test.cfg export REDBIOM_HOST="http://localhost:7379" From 8612459154e1daf2a570c368ece4ed5e19a6ad29 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Mon, 12 Aug 2024 08:24:46 -0600 Subject: [PATCH 7/9] tiny -> amplicon --- .github/workflows/qiita-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index 4e455429d..51bae2a9b 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -104,7 +104,7 @@ jobs: - name: Install plugins shell: bash -l {0} run: | - conda env create -n qtp-biom --file https://data.qiime2.org/distro/tiny/qiime2-tiny-2024.5-py39-linux-conda.yml + conda env create -n qtp-biom --file https://data.qiime2.org/distro/amplicon/qiime2-amplicon-2024.5-py39-linux-conda.yml export QIITA_ROOTCA_CERT=`pwd`/qiita_core/support_files/ci_rootca.crt export QIITA_CONFIG_FP=`pwd`/qiita_core/support_files/config_test.cfg export REDBIOM_HOST="http://localhost:7379" From 00a1fae40eba7be4449f02a9bd65dbbfe8d02e97 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Wed, 14 Aug 2024 10:45:53 -0600 Subject: [PATCH 8/9] fix workflow GUI --- qiita_db/artifact.py | 45 +++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/qiita_db/artifact.py b/qiita_db/artifact.py index c19648276..f116236f7 100644 --- a/qiita_db/artifact.py +++ b/qiita_db/artifact.py @@ -1342,23 +1342,6 @@ def _helper(sql_edges, edges, nodes): # If the job is in success we don't need to do anything # else since it would've been added by the code above if jstatus != 'success': - # Connect the job with his input artifacts, the - # input artifacts may or may not exist yet, so we - # need to check both the input_artifacts and the - # pending properties - for in_art in n_obj.input_artifacts: - iid = in_art.id - if iid not in nodes and iid in extra_nodes: - nodes[iid] = extra_nodes[iid] - _add_edge(edges, nodes[iid], nodes[n_obj.id]) - - pending = n_obj.pending - for pred_id in pending: - for pname in pending[pred_id]: - in_node_id = '%s:%s' % ( - pred_id, pending[pred_id][pname]) - _add_edge(edges, nodes[in_node_id], - nodes[n_obj.id]) if jstatus != 'error': # If the job is not errored, we can add the @@ -1380,6 +1363,34 @@ def _helper(sql_edges, edges, nodes): queue.append(cjob.id) if cjob.id not in nodes: nodes[cjob.id] = ('job', cjob) + + # including the outputs + for o_name, o_type in cjob.command.outputs: + node_id = '%s:%s' % (cjob.id, o_name) + node = TypeNode( + id=node_id, job_id=cjob.id, + name=o_name, type=o_type) + if node_id not in nodes: + nodes[node_id] = ('type', node) + + # Connect the job with his input artifacts, the + # input artifacts may or may not exist yet, so we + # need to check both the input_artifacts and the + # pending properties + for in_art in n_obj.input_artifacts: + iid = in_art.id + if iid not in nodes and iid in extra_nodes: + nodes[iid] = extra_nodes[iid] + _add_edge(edges, nodes[iid], nodes[n_obj.id]) + + pending = n_obj.pending + for pred_id in pending: + for pname in pending[pred_id]: + in_node_id = '%s:%s' % ( + pred_id, pending[pred_id][pname]) + _add_edge(edges, nodes[in_node_id], + nodes[n_obj.id]) + elif n_type == 'type': # Connect this 'future artifact' with the job that will # generate it From ef5b2e7cf1aab570e2734760e08c4cea3fd743fc Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Tue, 20 Aug 2024 01:14:06 -0600 Subject: [PATCH 9/9] nltk<=3.8.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index caa72a46f..6f641a609 100644 --- a/setup.py +++ b/setup.py @@ -105,7 +105,7 @@ install_requires=['psycopg2', 'click', 'bcrypt', 'pandas<2.0', 'biom-format', 'tornado<6.0', 'toredis', 'redis', 'scp', 'pyparsing', 'h5py', 'natsort', 'nose', 'pep8', - 'networkx', 'humanize', 'wtforms<3.0.0', 'nltk', + 'networkx', 'humanize', 'wtforms<3.0.0', 'nltk<=3.8.1', 'openpyxl', 'sphinx-bootstrap-theme', 'Sphinx<3.0', 'gitpython', 'redbiom', 'pyzmq', 'sphinx_rtd_theme', 'paramiko', 'seaborn', 'matplotlib', 'scipy<=1.10.1',