diff --git a/.gitignore b/.gitignore index 8f70452d15..962398b600 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,7 @@ venv/ **/src/test/generated /tests/cdm-sample-files/ /.metadata/ +*.pyc # Docusaurus website website/.docusaurus diff --git a/codefresh.yml b/codefresh.yml index 622eeb7967..c0eb9ace70 100644 --- a/codefresh.yml +++ b/codefresh.yml @@ -24,6 +24,7 @@ steps: - cf_export MVN_BUILD_FLAGS="-Dmaven.repo.local=${{CF_VOLUME_PATH}}/.m2" - cf_export GPG_IMPORT_COMMAND="cat <(echo -e '${{GPG_PRIVATE_KEY}}') | gpg --batch --import" - cf_export GEN_DEPLOY_POM_SCRIPT="${{CF_VOLUME_PATH}}/${{CF_REPO_NAME}}/rosetta-source/src/main/resources/build-resources/create-deploy-pom.sh" + - cf_export GEN_DEPLOY_POM_PY="${{CF_VOLUME_PATH}}/${{CF_REPO_NAME}}/rosetta-source/src/main/resources/build-resources/create-deploy-pom.py" ReleaseProperties: stage: 'setup' @@ -114,14 +115,18 @@ steps: title: Python build fail_fast: false image: python:3.10-alpine - working_directory: ./rosetta-source/target/classes/cdm/python + working_directory: ./rosetta-source shell: sh commands: - - python3 -m pip install pydantic + - python3 -m pip install "pydantic==1.*" - python3 -m pip install jsonpickle - - python3 -m pip install rosetta_runtime-1.0.0-py3-none-any.whl + - python3 -m pip install ./target/classes/cdm/python/runtime/rosetta_runtime-1.0.0-py3-none-any.whl - |- - python3 -m pip wheel --no-deps --only-binary :all: . + python3 -m pip wheel --no-deps --only-binary :all: --wheel-dir ./target/classes/cdm/python ./target/classes/cdm/python + - python3 -m pip install ./target/classes/cdm/python/python_cdm-*-py3-none-any.whl + - python3 -m pip install pytest + - pytest ./src/test/python/ + - rm -rf ./src/test/python/serialization/__pycache__ ./src/test/python/semantics/__pycache__ DeployParallelTasks: stage: 'build' @@ -229,8 +234,8 @@ steps: commands: - bash -c "${{GPG_IMPORT_COMMAND}}" - cd target/classes/cdm/python - - tar -cvzf cdm-python-${{RELEASE_NAME}}.tar.gz python_cdm-*-py3-none-any.whl rosetta_runtime-*-py3-none-any.whl - - ${{GEN_DEPLOY_POM_SCRIPT}} cdm-python ${{RELEASE_NAME}} tar.gz + - tar -cvzf cdm-python-${{RELEASE_NAME}}.tar.gz python_cdm-*-py3-none-any.whl runtime/rosetta_runtime-*-py3-none-any.whl + - python3 ${{GEN_DEPLOY_POM_PY}} cdm-python ${{RELEASE_NAME}} tar.gz ${{CF_VOLUME_PATH}}/${{CF_REPO_NAME}}/rosetta-source/src/main/resources/build-resources/python/python-developer.txt - mvn gpg:sign-and-deploy-file ${{MVN_DEPLOY_FILE_FLAGS}} -Dfile=cdm-python-${{RELEASE_NAME}}.tar.gz -DgroupId=org.finos.cdm -DartifactId=cdm-python -Dversion=${{RELEASE_NAME}} -Dpackaging=tar.gz -DpomFile=cdm-python-${{RELEASE_NAME}}.pom NotifySlackOnFail: diff --git a/rosetta-source/pom.xml b/rosetta-source/pom.xml index 0a749dd5eb..eb90fac821 100644 --- a/rosetta-source/pom.xml +++ b/rosetta-source/pom.xml @@ -795,9 +795,10 @@ python ${rosetta.code-gen.version} jar + ${project.build.directory}/classes/cdm/python + runtime/rosetta_runtime-1.0.0-py3-none-any.whl - rosetta_runtime-1.0.0-py3-none-any.whl false false true diff --git a/rosetta-source/src/main/resources/build-resources/create-deploy-pom.py b/rosetta-source/src/main/resources/build-resources/create-deploy-pom.py new file mode 100755 index 0000000000..ea5a75c85f --- /dev/null +++ b/rosetta-source/src/main/resources/build-resources/create-deploy-pom.py @@ -0,0 +1,88 @@ +import sys +from string import Template +import os.path + +def main(): + # build parameters from the command line arguments + params_dict = {'ARTIFACT_ID': sys.argv[1], + 'RELEASE_NAME': sys.argv[2], + 'PACKAGING':sys.argv[3], + 'GROUP_ID': 'org.finos.cdm'} + # add in additional developers if any + if (len (sys.argv) > 4 and os.path.isfile(sys.argv[4])) : + developers_file = open(sys.argv[4], 'r') + params_dict['DEVELOPERS'] = developers_file.read () + developers_file.close() + else: + params_dict['DEVELOPERS'] = ''' + minesh-s-patel + Minesh Patel + infra@regnosys.com + http://github.com/minesh-s-patel + REGnosys + https://regnosys.com + +1 + + Maintainer + Developer + + + + hugohills-regnosys + Hugo Hills + infra@regnosys.com + http://github.com/hugohills-regnosys + REGnosys + https://regnosys.com + +1 + + Maintainer + Developer + + ''' + + deploy_pom_text = ''' + 4.0.0 + $GROUP_ID + $ARTIFACT_ID + $RELEASE_NAME + $PACKAGING + + $ARTIFACT_ID + + https://www.finos.org/common-domain-model + + + scm:git:https://github.com/finos/common-domain-model + scm:git:git://github.com/finos/common-domain-model.git + HEAD + https://github.com/finos/common-domain-model + + + The FINOS Common Domain Model (CDM) is a standardised, machine-readable and machine-executable blueprint for how financial products are traded and managed across the transaction lifecycle. It is represented as a domain model and distributed in open source. + + + FINOS + https://finos.org + + + + + Community Specification License 1.0 + https://github.com/finos/common-domain-model/blob/master/LICENSE.md + + + + +$DEVELOPERS + + +''' + deploy_pom_text = Template(deploy_pom_text).safe_substitute(params_dict) + deploy_pom_file = open(params_dict['ARTIFACT_ID'] + '-' + params_dict['RELEASE_NAME']+ '.pom', "w") + deploy_pom_file.write(deploy_pom_text) + deploy_pom_file.close() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/rosetta-source/src/main/resources/build-resources/python/python-developer.txt b/rosetta-source/src/main/resources/build-resources/python/python-developer.txt new file mode 100644 index 0000000000..b446124a4a --- /dev/null +++ b/rosetta-source/src/main/resources/build-resources/python/python-developer.txt @@ -0,0 +1,74 @@ + + CloudRisk + CloudRisk + info@cloudrisk.co.uk + https://www.cloudrisk.uk + CloudRisk + https://www.cloudrisk.uk + +1 + + Developer + + + + FT-Advisory + FT Advisory + info@ftadvisory.co + https://www.ftadvisory.co + FT Advisory + https://www.ftadvisory.co + -4 + + Developer + + + + minesh-s-patel + Minesh Patel + infra@regnosys.com + http://github.com/minesh-s-patel + REGnosys + https://regnosys.com + +1 + + Maintainer + Developer + + + + hugohills-regnosys + Hugo Hills + infra@regnosys.com + http://github.com/hugohills-regnosys + REGnosys + https://regnosys.com + +1 + + Maintainer + Developer + + + + SimonCockx + Simon Cockx + infra@regnosys.com + http://github.com/SimonCockx + REGnosys + https://regnosys.com + +1 + + Developer + + + + TradeHeader + TradeHeader + info@tradeheader.com + https://www.tradeheader.com + TradeHeader + https://www.tradeheader.com + +1 + + Developer + + \ No newline at end of file diff --git a/rosetta-source/src/test/python/run_tests.sh b/rosetta-source/src/test/python/run_tests.sh new file mode 100755 index 0000000000..06ce2638c0 --- /dev/null +++ b/rosetta-source/src/test/python/run_tests.sh @@ -0,0 +1,24 @@ +#!/bin/bash +type -P python > /dev/null && PYEXE=python || PYEXE=python3 +if ! $PYEXE -c 'import sys; assert sys.version_info >= (3,10)' > /dev/null 2>&1; then + echo "Found $($PYEXE -V)" + echo "Expecting at least python 3.10 - exiting!" + exit 1 +fi + +MYPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +ROSETTARUNTIMEDIR="../../../target/classes/cdm/python/runtime" +PYTHONCDMDIR="../../generated/python" +cd $MYPATH + +ACDIR=$(python -c "import sys;print('Scripts' if sys.platform.startswith('win') else 'bin')") + +rm -rf testenv +$PYEXE -m venv --clear testenv +source testenv/$ACDIR/activate + +$PYEXE -m pip install pytest +$PYEXE -m pip install $MYPATH/$ROSETTARUNTIMEDIR/rosetta_runtime-1.0.0-py3-none-any.whl +$PYEXE -m pip install $MYPATH/$PYTHONCDMDIR/python_cdm-*-py3-none-any.whl +pytest +rm -rf testenv .pytest_cache serialization/__pycache__ semantics/__pycache__ \ No newline at end of file diff --git a/rosetta-source/src/test/python/semantics/test_cardinality.py b/rosetta-source/src/test/python/semantics/test_cardinality.py new file mode 100644 index 0000000000..f015443c18 --- /dev/null +++ b/rosetta-source/src/test/python/semantics/test_cardinality.py @@ -0,0 +1,35 @@ +import pytest +import datetime +from cdm.base.datetime.DateList import DateList +from rosetta.runtime.utils import ConditionViolationError + + +def test_1_many_fail(): + dl = DateList(date=[]) + with pytest.raises(ConditionViolationError): + dl.validate_conditions() + + +def test_1_many_fail_nopar(): + dl = DateList() + with pytest.raises(ConditionViolationError): + dl.validate_conditions() + + +def test_1_many_pass(): + dl = DateList(date=[datetime.date(2020, 1, 1)]) + dl.validate_conditions() + + +if __name__ == "__main__": + print("first one") + test_1_many_pass() + print("second one") + test_1_many_fail() + print("third one") + test_1_many_fail_nopar() + + +# EOF + + diff --git a/rosetta-source/src/test/python/semantics/test_conditions.py b/rosetta-source/src/test/python/semantics/test_conditions.py new file mode 100644 index 0000000000..11d1af34da --- /dev/null +++ b/rosetta-source/src/test/python/semantics/test_conditions.py @@ -0,0 +1,40 @@ +import pytest +from pydantic import ValidationError +from rosetta.runtime.utils import ConditionViolationError +from cdm.base.math.NonNegativeQuantity import NonNegativeQuantity +from cdm.base.math.UnitType import UnitType + +''' +def test_recursive_conds(): + unit = UnitType(currency='EUR') + mq = NonNegativeQuantity(value=10, unit=unit) + mq.validate_model() + +''' + +def test_recursive_conds_base_fail(): + unit = UnitType(currency='EUR') + mq = NonNegativeQuantity(unit=unit) + with pytest.raises(ConditionViolationError): + mq.validate_model() + +def test_recursive_conds_direct_fail(): + unit = UnitType(currency='EUR') + mq = NonNegativeQuantity(value=-10, unit=unit) + with pytest.raises(ConditionViolationError): + mq.validate_model() + + +def test_attrib_validity(): + unit = UnitType(currency='EUR') + mq = NonNegativeQuantity(value=10, unit=unit) + mq.frequency = 'Blah' + with pytest.raises(ValidationError): + mq.validate_model() + +if __name__ == "__main__": + test_recursive_conds_base_fail() + test_recursive_conds_direct_fail() + test_attrib_validity() + +# EOF diff --git a/rosetta-source/src/test/python/semantics/test_if_cond.py b/rosetta-source/src/test/python/semantics/test_if_cond.py new file mode 100644 index 0000000000..500ec945ca --- /dev/null +++ b/rosetta-source/src/test/python/semantics/test_if_cond.py @@ -0,0 +1,81 @@ +import pytest +from rosetta.runtime.utils import ConditionViolationError +from rosetta.runtime.utils import if_cond +##from cdm.base.math.Measure import Measure +from cdm.base.math.QuantitySchedule import QuantitySchedule +from cdm.base.math.UnitType import UnitType +##from drr.regulation.cftc.rewrite.CFTCPart43TransactionReport import CFTCPart43TransactionReport + + +''' +def test_if_cond_pass(): + unit = UnitType(currency='EUR') + multiplier = Measure(value=1) + qs = QuantitySchedule(value=1, unit=unit, multiplier=multiplier) + qs.validate_conditions() + + +def test_if_cond_fail(): + unit = UnitType(currency='EUR') + multiplier = Measure(value=-1) + qs = QuantitySchedule(unit=unit, multiplier=multiplier) + with pytest.raises(ConditionViolationError): + qs.validate_conditions() + +''' +def test_if_cond_literals(): + class T: + def __init__(self): + self.cleared = 'Y' + self.counterparty1FinancialEntityIndicator = None + self.counterparty1FinancialEntityIndicator = None + self.actionType = "NEWT" + self.eventType = "CLRG" + self.originalSwapUTI = 1 + self.originalSwapUSI = 'OKI' + self = T() + + res = if_cond( + ((self.cleared == "N") or (self.cleared == "I")), + '((self.counterparty1FinancialEntityIndicator) is not None)', + 'if_cond((self.cleared == "Y"), \'((self.counterparty1FinancialEntityIndicator) == "NO")\', \'True\', self)', + self) + assert not res + + res = if_cond((((self.cleared == "Y") and (self.actionType == "NEWT")) and (self.eventType == "CLRG")), + 'if_cond(((self.originalSwapUTI) is None), \'((self.originalSwapUSI) is not None)\', \'if_cond(((self.originalSwapUTI) is not None), \\\'((self.originalSwapUSI) == "OKI")\\\', \\\'True\\\', self)\', self)', + '((self.originalSwapUTI) is None)', + self) + assert res + +''' +def test_if_cond_any(): + class T: + def __init__(self): + self.actionType = "TERM" + self.eventType = 'CORP' + self = T() + fnc = CFTCPart43TransactionReport.condition_0_EventTypeCondition + res = fnc(self) + assert res + +''' +def test_if_direct(): + class T: + def __init__(self): + self.x, self.y = 1, 2 + + assert not if_cond(True, + 'if_cond(True, \'self.x > self.y\', \'True\', self)', 'True', T()) + assert not if_cond(True, + 'if_cond(True, \'if_cond(True, \\\'self.x > self.y\\\', \\\'True\\\', self)\', \'True\', self)', + 'True', T()) + assert not if_cond(True, + 'if_cond(True, \'if_cond(True, \\\'if_cond(True, \\\\\\\'self.x > self.y\\\\\\\', \\\\\\\'True\\\\\\\', self)\\\', \\\'True\\\', self)\', \'True\', self)', + 'True', T()) + +if __name__ == "__main__": + test_if_cond_literals() + test_if_direct() + +# EOF diff --git a/rosetta-source/src/test/python/serialization/cdm_comparison_test.py b/rosetta-source/src/test/python/serialization/cdm_comparison_test.py new file mode 100644 index 0000000000..d590fedf10 --- /dev/null +++ b/rosetta-source/src/test/python/serialization/cdm_comparison_test.py @@ -0,0 +1,29 @@ +''' Jsonn test utilities''' +import json +from pathlib import Path +import sys +import os +dirPath = os.path.dirname(__file__) +sys.path.append(os.path.join(dirPath)) + +from dict_comp import dict_comp +from cdm.version import __build_time__ + + +def cdm_comparison_test_from_file(path, class_name): + '''loads the json from a file and runs the comparison''' + print('testing: ' + path + ' with className: ' + class_name.__name__ + ' using CDM built at: ' + __build_time__) + json_str = Path(path).read_text() + cdm_comparison_test_from_string(json_str, class_name) + + +def cdm_comparison_test_from_string(json_str, class_name): + '''loads the json from a string and runs the comparison''' + cdm_object = class_name.parse_raw(json_str) + json_data_out = cdm_object.json(exclude_defaults=True, indent=4) + orig_dict = json.loads(json_str) + generated_dict = json.loads(json_data_out) + assert dict_comp(orig_dict, generated_dict), "Failed corrected dict comparison" + print('passed: dicts matched') + +# EOF diff --git a/rosetta-source/src/test/python/serialization/dict_comp.py b/rosetta-source/src/test/python/serialization/dict_comp.py new file mode 100644 index 0000000000..284223f0b5 --- /dev/null +++ b/rosetta-source/src/test/python/serialization/dict_comp.py @@ -0,0 +1,27 @@ +def dict_comp(d1, d2, prefix=''): + if d1 == d2: + return True + if type(d1) != type(d2): + print(f'Types differ for path {prefix} - d1: {type(d1)}, d2: {type(d2)}') + return False + if isinstance(d1, dict) and isinstance(d2, dict): + if d1.keys() != d2.keys(): + print(f'Keys for path {prefix} differ: d1: {d1.keys()}, d2: {d2.keys()}') + return False + for k, v in d1.items(): + if not dict_comp(v, d2[k], f'{prefix}.{k}'): + return False + elif isinstance(d1, list) and isinstance(d2, list): + for i, (e1, e2) in enumerate(zip(d1, d2)): + if not dict_comp(e1, e2, f'{prefix}[{i}]'): + return False + if len(d1) != len(d2): + print(f'Lists at {prefix} are of diffrent lenght d1: {len(d1)}, d2: {len(d2)}') + else: + print(f'Lists at {prefix} differ for unknown reason?!?') + return False + else: + print(f'Elements for ptah {prefix} differ: d1: {d1}, d2 {d2}') + return False + +# EOF diff --git a/rosetta-source/src/test/python/serialization/test_trade_state_product.py b/rosetta-source/src/test/python/serialization/test_trade_state_product.py new file mode 100644 index 0000000000..6a309ccabc --- /dev/null +++ b/rosetta-source/src/test/python/serialization/test_trade_state_product.py @@ -0,0 +1,19 @@ +import sys +import pytest +import argparse + +from cdm.event.common.TradeState import TradeState +import sys,os + +from cdm_comparison_test import cdm_comparison_test_from_file + +def test_trade_state (cdm_sample=None): + if cdm_sample == None: + dirPath = os.path.dirname(__file__) + sys.path.append(os.path.join(dirPath)) + cdm_sample = dirPath + '/../../../main/resources/result-json-files/fpml-5-10/products/rates/EUR-Vanilla-account.json' + cdm_comparison_test_from_file (cdm_sample, TradeState) + +if __name__ == "__main__": + cdm_sample = sys.argv[1] if len(sys.argv) > 1 else None + test_trade_state(cdm_sample) \ No newline at end of file