Added Python unit tests to confirm that generated code performs as ex…

…pected (finos#2236) * integrate python unit tests into codefresh * included python unit tests * clean up - removal of ".project" file * codefresh.yml dir for python revised to "target" * Excluding vulnerability CVE-2023-35116 * typo * Tidy up * Narrow CVE exclusion * Updated process to generate deployment POM (finos#2238) * Add Simon --------- Co-authored-by: dan <[email protected]>
dackattack218 · Jul 3, 2023 · aee1b4f · aee1b4f
1 parent b1198aa
commit aee1b4f
Show file tree

Hide file tree

Showing 12 changed files with 431 additions and 7 deletions.
diff --git a/.gitignore b/.gitignore
@@ -53,6 +53,7 @@ venv/
 **/src/test/generated
 /tests/cdm-sample-files/
 /.metadata/
+*.pyc
 
 # Docusaurus website
 website/.docusaurus

diff --git a/codefresh.yml b/codefresh.yml
@@ -24,6 +24,7 @@ steps:
       - cf_export MVN_BUILD_FLAGS="-Dmaven.repo.local=${{CF_VOLUME_PATH}}/.m2"
       - cf_export GPG_IMPORT_COMMAND="cat <(echo -e '${{GPG_PRIVATE_KEY}}') | gpg --batch --import"
       - cf_export GEN_DEPLOY_POM_SCRIPT="${{CF_VOLUME_PATH}}/${{CF_REPO_NAME}}/rosetta-source/src/main/resources/build-resources/create-deploy-pom.sh"
+      - cf_export GEN_DEPLOY_POM_PY="${{CF_VOLUME_PATH}}/${{CF_REPO_NAME}}/rosetta-source/src/main/resources/build-resources/create-deploy-pom.py"
 
   ReleaseProperties:
     stage: 'setup'
@@ -114,14 +115,18 @@ steps:
         title: Python build
         fail_fast: false
         image: python:3.10-alpine
-        working_directory: ./rosetta-source/target/classes/cdm/python
+        working_directory: ./rosetta-source
         shell: sh
         commands:
-          - python3 -m pip install pydantic
+          - python3 -m pip install "pydantic==1.*"
           - python3 -m pip install jsonpickle
-          - python3 -m pip install rosetta_runtime-1.0.0-py3-none-any.whl
+          - python3 -m pip install ./target/classes/cdm/python/runtime/rosetta_runtime-1.0.0-py3-none-any.whl
           - |-
-            python3 -m pip wheel --no-deps --only-binary :all: .
+            python3 -m pip wheel --no-deps --only-binary :all: --wheel-dir ./target/classes/cdm/python ./target/classes/cdm/python
+          - python3 -m pip install ./target/classes/cdm/python/python_cdm-*-py3-none-any.whl
+          - python3 -m pip install pytest
+          - pytest ./src/test/python/
+          - rm -rf ./src/test/python/serialization/__pycache__ ./src/test/python/semantics/__pycache__
 
   DeployParallelTasks:
     stage: 'build'
@@ -229,8 +234,8 @@ steps:
         commands:
           - bash -c "${{GPG_IMPORT_COMMAND}}"
           - cd target/classes/cdm/python
-          - tar -cvzf cdm-python-${{RELEASE_NAME}}.tar.gz python_cdm-*-py3-none-any.whl rosetta_runtime-*-py3-none-any.whl
-          - ${{GEN_DEPLOY_POM_SCRIPT}} cdm-python ${{RELEASE_NAME}} tar.gz
+          - tar -cvzf cdm-python-${{RELEASE_NAME}}.tar.gz python_cdm-*-py3-none-any.whl runtime/rosetta_runtime-*-py3-none-any.whl
+          - python3 ${{GEN_DEPLOY_POM_PY}} cdm-python ${{RELEASE_NAME}} tar.gz ${{CF_VOLUME_PATH}}/${{CF_REPO_NAME}}/rosetta-source/src/main/resources/build-resources/python/python-developer.txt
           - mvn gpg:sign-and-deploy-file ${{MVN_DEPLOY_FILE_FLAGS}} -Dfile=cdm-python-${{RELEASE_NAME}}.tar.gz -DgroupId=org.finos.cdm -DartifactId=cdm-python -Dversion=${{RELEASE_NAME}} -Dpackaging=tar.gz -DpomFile=cdm-python-${{RELEASE_NAME}}.pom
 
   NotifySlackOnFail:

diff --git a/rosetta-source/pom.xml b/rosetta-source/pom.xml
@@ -795,9 +795,10 @@
                                             <artifactId>python</artifactId>
                                             <version>${rosetta.code-gen.version}</version>
                                             <type>jar</type>
+                                            <outputDirectory>${project.build.directory}/classes/cdm/python</outputDirectory>
+                                            <includes>runtime/rosetta_runtime-1.0.0-py3-none-any.whl</includes>
                                         </artifactItem>
                                     </artifactItems>
-                                    <includes>rosetta_runtime-1.0.0-py3-none-any.whl</includes>
                                     <overWriteReleases>false</overWriteReleases>
                                     <overWriteSnapshots>false</overWriteSnapshots>
                                     <overWriteIfNewer>true</overWriteIfNewer>

diff --git a/rosetta-source/src/main/resources/build-resources/create-deploy-pom.py b/rosetta-source/src/main/resources/build-resources/create-deploy-pom.py
@@ -0,0 +1,88 @@
+import sys
+from string import Template
+import os.path
+
+def main():
+    # build parameters from the command line arguments
+    params_dict = {'ARTIFACT_ID': sys.argv[1],
+                   'RELEASE_NAME': sys.argv[2],
+                   'PACKAGING':sys.argv[3],
+                   'GROUP_ID': 'org.finos.cdm'}
+    # add in additional developers if any
+    if (len (sys.argv) > 4 and os.path.isfile(sys.argv[4])) :
+        developers_file = open(sys.argv[4], 'r') 
+        params_dict['DEVELOPERS'] = developers_file.read ()
+        developers_file.close()
+    else:
+        params_dict['DEVELOPERS'] = '''        <developer>
+            <id>minesh-s-patel</id>
+            <name>Minesh Patel</name>
+            <email>[email protected]</email>
+            <url>http://github.com/minesh-s-patel</url>
+            <organization>REGnosys</organization>
+            <organizationUrl>https://regnosys.com</organizationUrl>
+            <timezone>+1</timezone>
+            <roles>
+                <role>Maintainer</role>
+                <role>Developer</role>
+            </roles>
+        </developer>
+        <developer>
+            <id>hugohills-regnosys</id>
+            <name>Hugo Hills</name>
+            <email>[email protected]</email>
+            <url>http://github.com/hugohills-regnosys</url>
+            <organization>REGnosys</organization>
+            <organizationUrl>https://regnosys.com</organizationUrl>
+            <timezone>+1</timezone>
+            <roles>
+                <role>Maintainer</role>
+                <role>Developer</role>
+            </roles>
+        </developer>'''
+
+    deploy_pom_text = '''<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>$GROUP_ID</groupId>
+    <artifactId>$ARTIFACT_ID</artifactId>
+    <version>$RELEASE_NAME</version>
+    <packaging>$PACKAGING</packaging>
+
+    <name>$ARTIFACT_ID</name>
+
+    <url>https://www.finos.org/common-domain-model</url>
+
+    <scm>
+        <developerConnection>scm:git:https://github.com/finos/common-domain-model</developerConnection>
+        <connection>scm:git:git://github.com/finos/common-domain-model.git</connection>
+        <tag>HEAD</tag>
+        <url>https://github.com/finos/common-domain-model</url>
+    </scm>
+
+    <description>The FINOS Common Domain Model (CDM) is a standardised, machine-readable and machine-executable blueprint for how financial products are traded and managed across the transaction lifecycle. It is represented as a domain model and distributed in open source.</description>
+
+    <organization>
+        <name>FINOS</name>
+        <url>https://finos.org</url>
+    </organization>
+
+    <licenses>
+        <license>
+            <name>Community Specification License 1.0</name>
+            <url>https://github.com/finos/common-domain-model/blob/master/LICENSE.md</url>
+        </license>
+    </licenses>
+
+    <developers>
+$DEVELOPERS
+    </developers>
+</project>
+'''
+    deploy_pom_text = Template(deploy_pom_text).safe_substitute(params_dict)
+    deploy_pom_file = open(params_dict['ARTIFACT_ID'] + '-' + params_dict['RELEASE_NAME']+ '.pom', "w")
+    deploy_pom_file.write(deploy_pom_text)
+    deploy_pom_file.close()
+
+if __name__ == "__main__":
+    main()
diff --git a/rosetta-source/src/main/resources/build-resources/python/python-developer.txt b/rosetta-source/src/main/resources/build-resources/python/python-developer.txt
@@ -0,0 +1,74 @@
+        <developer>
+            <id>CloudRisk</id>
+            <name>CloudRisk</name>
+            <email>[email protected]</email>
+            <url>https://www.cloudrisk.uk</url>
+            <organization>CloudRisk</organization>
+            <organizationUrl>https://www.cloudrisk.uk</organizationUrl>
+            <timezone>+1</timezone>
+            <roles>
+                <role>Developer</role>
+            </roles>
+        </developer>
+        <developer>
+            <id>FT-Advisory</id>
+            <name>FT Advisory</name>
+            <email>[email protected]</email>
+            <url>https://www.ftadvisory.co</url>
+            <organization>FT Advisory</organization>
+            <organizationUrl>https://www.ftadvisory.co</organizationUrl>
+            <timezone>-4</timezone>
+            <roles>
+                <role>Developer</role>
+            </roles>
+        </developer>
+        <developer>
+            <id>minesh-s-patel</id>
+            <name>Minesh Patel</name>
+            <email>[email protected]</email>
+            <url>http://github.com/minesh-s-patel</url>
+            <organization>REGnosys</organization>
+            <organizationUrl>https://regnosys.com</organizationUrl>
+            <timezone>+1</timezone>
+            <roles>
+                <role>Maintainer</role>
+                <role>Developer</role>
+            </roles>
+        </developer>
+        <developer>
+            <id>hugohills-regnosys</id>
+            <name>Hugo Hills</name>
+            <email>[email protected]</email>
+            <url>http://github.com/hugohills-regnosys</url>
+            <organization>REGnosys</organization>
+            <organizationUrl>https://regnosys.com</organizationUrl>
+            <timezone>+1</timezone>
+            <roles>
+                <role>Maintainer</role>
+                <role>Developer</role>
+            </roles>
+        </developer>
+        <developer>
+            <id>SimonCockx</id>
+            <name>Simon Cockx</name>
+            <email>[email protected]</email>
+            <url>http://github.com/SimonCockx</url>
+            <organization>REGnosys</organization>
+            <organizationUrl>https://regnosys.com</organizationUrl>
+            <timezone>+1</timezone>
+            <roles>
+                <role>Developer</role>
+            </roles>
+        </developer>
+        <developer>
+            <id>TradeHeader</id>
+            <name>TradeHeader</name>
+            <email>[email protected]</email>
+            <url>https://www.tradeheader.com</url>
+            <organization>TradeHeader</organization>
+            <organizationUrl>https://www.tradeheader.com</organizationUrl>
+            <timezone>+1</timezone>
+            <roles>
+                <role>Developer</role>
+            </roles>
+        </developer>
diff --git a/rosetta-source/src/test/python/run_tests.sh b/rosetta-source/src/test/python/run_tests.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+type -P python > /dev/null && PYEXE=python || PYEXE=python3
+if ! $PYEXE -c 'import sys; assert sys.version_info >= (3,10)' > /dev/null 2>&1; then
+        echo "Found $($PYEXE -V)"
+        echo "Expecting at least python 3.10 - exiting!"
+        exit 1
+fi
+
+MYPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+ROSETTARUNTIMEDIR="../../../target/classes/cdm/python/runtime"
+PYTHONCDMDIR="../../generated/python"
+cd $MYPATH
+
+ACDIR=$(python -c "import sys;print('Scripts' if sys.platform.startswith('win') else 'bin')")
+
+rm -rf testenv
+$PYEXE -m venv --clear testenv
+source testenv/$ACDIR/activate
+
+$PYEXE -m pip install pytest
+$PYEXE -m pip install $MYPATH/$ROSETTARUNTIMEDIR/rosetta_runtime-1.0.0-py3-none-any.whl 
+$PYEXE -m pip install $MYPATH/$PYTHONCDMDIR/python_cdm-*-py3-none-any.whl
+pytest
+rm -rf testenv .pytest_cache serialization/__pycache__ semantics/__pycache__
diff --git a/rosetta-source/src/test/python/semantics/test_cardinality.py b/rosetta-source/src/test/python/semantics/test_cardinality.py
@@ -0,0 +1,35 @@
+import pytest
+import datetime
+from cdm.base.datetime.DateList import DateList
+from rosetta.runtime.utils import ConditionViolationError
+
+
+def test_1_many_fail():
+    dl = DateList(date=[])
+    with pytest.raises(ConditionViolationError):
+        dl.validate_conditions()
+
+
+def test_1_many_fail_nopar():
+    dl = DateList()
+    with pytest.raises(ConditionViolationError):
+        dl.validate_conditions()
+
+
+def test_1_many_pass():
+    dl = DateList(date=[datetime.date(2020, 1, 1)])
+    dl.validate_conditions()
+
+
+if __name__ == "__main__":
+	print("first one")
+	test_1_many_pass()
+	print("second one")
+	test_1_many_fail()
+	print("third one")
+	test_1_many_fail_nopar()
+
+
+# EOF
+
+
diff --git a/rosetta-source/src/test/python/semantics/test_conditions.py b/rosetta-source/src/test/python/semantics/test_conditions.py
@@ -0,0 +1,40 @@
+import pytest
+from pydantic import ValidationError
+from rosetta.runtime.utils import ConditionViolationError
+from cdm.base.math.NonNegativeQuantity import NonNegativeQuantity
+from cdm.base.math.UnitType import UnitType
+
+'''
+def test_recursive_conds():
+    unit = UnitType(currency='EUR')
+    mq = NonNegativeQuantity(value=10, unit=unit)
+    mq.validate_model()
+
+'''
+
+def test_recursive_conds_base_fail():
+    unit = UnitType(currency='EUR')
+    mq = NonNegativeQuantity(unit=unit)
+    with pytest.raises(ConditionViolationError):
+        mq.validate_model()
+
+def test_recursive_conds_direct_fail():
+    unit = UnitType(currency='EUR')
+    mq = NonNegativeQuantity(value=-10, unit=unit)
+    with pytest.raises(ConditionViolationError):
+        mq.validate_model()
+
+
+def test_attrib_validity():
+    unit = UnitType(currency='EUR')
+    mq = NonNegativeQuantity(value=10, unit=unit)
+    mq.frequency = 'Blah'
+    with pytest.raises(ValidationError):
+        mq.validate_model()
+
+if __name__ == "__main__":
+	test_recursive_conds_base_fail()
+	test_recursive_conds_direct_fail()
+	test_attrib_validity()
+
+# EOF