From a8d3ac16e887053d09844b793485cb5516b20b20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Kolarovszki?= Date: Fri, 19 Mar 2021 22:25:24 +0100 Subject: [PATCH] feat(benchmarks): Running with `pytest-benchmark` The existing `homodyne.py` benchmark under `benchmarks` has been refactored into two parts: one for the state preparation, and one for the measurement. Some differences were notices between the SF and PQ programs mostly due to the different definitions for the beamsplitter gates. Instead of an executable, the `pytest-benchmark` plugin is used. The `benchmark` CI/CD stage has been deleted, since benchmarks are now the responsibility of `pytest`. --- .gitlab-ci.yml | 10 -- benchmarks/conftest.py | 78 +++++++++++ benchmarks/gaussian_homodyne_benchmark.py | 46 +++++++ benchmarks/gaussian_preparation_benchmark.py | 22 +++ benchmarks/homodyne.py | 134 ------------------- poetry.lock | 54 ++++++-- pyproject.toml | 1 + pytest.ini | 3 + 8 files changed, 193 insertions(+), 155 deletions(-) create mode 100644 benchmarks/conftest.py create mode 100644 benchmarks/gaussian_homodyne_benchmark.py create mode 100644 benchmarks/gaussian_preparation_benchmark.py delete mode 100755 benchmarks/homodyne.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 007b33d4..e1de94de 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,6 @@ before_script: stages: - test - - benchmark - publish - docs @@ -38,15 +37,6 @@ test-py38: only: - web -benchmark: - stage: benchmark - script: - - apt update - - apt-get install libeigen3-dev - - poetry install - - echo "Piquasso vs. StrawberryFields vacuum homodyne benchmark" - - poetry run python benchmarks/homodyne.py - publish: stage: publish script: diff --git a/benchmarks/conftest.py b/benchmarks/conftest.py new file mode 100644 index 00000000..eac1e650 --- /dev/null +++ b/benchmarks/conftest.py @@ -0,0 +1,78 @@ +# +# Copyright (C) 2020 by TODO - All rights reserved. +# + +import pytest +import numpy as np + +import strawberryfields as sf +import piquasso as pq + + +@pytest.fixture +def d(): + return 5 + + +@pytest.fixture +def example_gaussian_pq_program(d): + with pq.Program() as program: + pq.Q() | pq.GaussianState.create_vacuum(d=d) + + pq.Q(0) | pq.S(amp=0.1) | pq.D(alpha=1) + pq.Q(1) | pq.S(amp=0.1) | pq.D(alpha=1) + pq.Q(2) | pq.S(amp=0.1) | pq.D(alpha=1) + pq.Q(3) | pq.S(amp=0.1) | pq.D(alpha=1) + pq.Q(4) | pq.S(amp=0.1) | pq.D(alpha=1) + + # NOTE: we need to tweak the parameters here a bit, because we use a different + # definition for the beamsplitter. + pq.Q(0, 1) | pq.B(0.0959408065906761, np.pi - 0.06786053071484363) + pq.Q(2, 3) | pq.B(0.7730047654405018, np.pi - 1.453770233324797) + pq.Q(1, 2) | pq.B(1.0152680371119776, np.pi - 1.2863559998816205) + pq.Q(3, 4) | pq.B(1.3205517879465705, np.pi - 0.5236836466492961) + pq.Q(0, 1) | pq.B(4.394480318177715, np.pi - 4.481575657714487) + pq.Q(2, 3) | pq.B(2.2300919706807534, np.pi - 1.5073556513699888) + pq.Q(1, 2) | pq.B(2.2679037068773673, np.pi - 1.9550229282085838) + pq.Q(3, 4) | pq.B(3.340269832485504, np.pi - 3.289367083610399) + + yield program + + # TODO: The state has to be reset, because the setup runs only once at the beginning + # of the calculations, therefore the same `GaussianState` instance will be used. + program.state.reset() + + +@pytest.fixture +def example_gaussian_sf_program_and_engine(d): + """ + NOTE: the covariance matrix SF is returning is half of ours... + It seems that our implementation is OK, however. + """ + + program = sf.Program(d) + engine = sf.Engine(backend="gaussian") + + with program.context as q: + sf.ops.Sgate(0.1) | q[0] + sf.ops.Sgate(0.1) | q[1] + sf.ops.Sgate(0.1) | q[2] + sf.ops.Sgate(0.1) | q[3] + sf.ops.Sgate(0.1) | q[4] + + sf.ops.Dgate(1) | q[0] + sf.ops.Dgate(1) | q[1] + sf.ops.Dgate(1) | q[2] + sf.ops.Dgate(1) | q[3] + sf.ops.Dgate(1) | q[4] + + sf.ops.BSgate(0.0959408065906761, 0.06786053071484363) | (q[0], q[1]) + sf.ops.BSgate(0.7730047654405018, 1.453770233324797) | (q[2], q[3]) + sf.ops.BSgate(1.0152680371119776, 1.2863559998816205) | (q[1], q[2]) + sf.ops.BSgate(1.3205517879465705, 0.5236836466492961) | (q[3], q[4]) + sf.ops.BSgate(4.394480318177715, 4.481575657714487) | (q[0], q[1]) + sf.ops.BSgate(2.2300919706807534, 1.5073556513699888) | (q[2], q[3]) + sf.ops.BSgate(2.2679037068773673, 1.9550229282085838) | (q[1], q[2]) + sf.ops.BSgate(3.340269832485504, 3.289367083610399) | (q[3], q[4]) + + return program, engine diff --git a/benchmarks/gaussian_homodyne_benchmark.py b/benchmarks/gaussian_homodyne_benchmark.py new file mode 100644 index 00000000..f153d379 --- /dev/null +++ b/benchmarks/gaussian_homodyne_benchmark.py @@ -0,0 +1,46 @@ +# +# Copyright (C) 2020 by TODO - All rights reserved. +# + +import pytest + +import piquasso as pq +import strawberryfields as sf + + +pytestmark = pytest.mark.benchmark( + group="gaussian-homodyne-measurement", +) + + +def piquasso_benchmark( + benchmark, example_gaussian_pq_program +): + example_gaussian_pq_program.execute() + + with pq.Program() as new_program: + pq.Q() | example_gaussian_pq_program.state + + # TODO: Support rotation by an angle, too. + pq.Q(0) | pq.MeasureHomodyne() + + results = benchmark(new_program.execute) + + assert results + + +def strawberryfields_benchmark( + benchmark, example_gaussian_sf_program_and_engine, d +): + program, engine = example_gaussian_sf_program_and_engine + + results = engine.run(program) + + new_program = sf.Program(d) + + new_program.state = results.state + + with new_program.context as q: + sf.ops.MeasureHomodyne(phi=0) | q[0] + + benchmark(engine.run, new_program) diff --git a/benchmarks/gaussian_preparation_benchmark.py b/benchmarks/gaussian_preparation_benchmark.py new file mode 100644 index 00000000..73fd16c1 --- /dev/null +++ b/benchmarks/gaussian_preparation_benchmark.py @@ -0,0 +1,22 @@ +# +# Copyright (C) 2020 by TODO - All rights reserved. +# + +import pytest + + +pytestmark = pytest.mark.benchmark( + group="gaussian-preparation", +) + + +def simple_piquasso_benchmark(benchmark, example_gaussian_pq_program): + benchmark(example_gaussian_pq_program.execute) + + +def simple_strawberryfields_benchmark( + benchmark, example_gaussian_sf_program_and_engine +): + program, engine = example_gaussian_sf_program_and_engine + + benchmark(engine.run, program) diff --git a/benchmarks/homodyne.py b/benchmarks/homodyne.py deleted file mode 100755 index 8d799063..00000000 --- a/benchmarks/homodyne.py +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env python -"""Benchmark and profiling for computing homodyne expectation values on vacuum. - -Both frameworks are initialized with the same circuits, and the same outputs are -calculated, which is the quadrature mean and the covariance after a rotation with angle -$`\pi/3`$ on the second mode. - -To run this script: - -.. code-block:: bash - - ./benchmarks/homodyne.py [NUMBER_OF_ITERATIONS] - -""" - -import timeit - -import numpy as np -import strawberryfields as sf - -import sys -import os - -sys.path.append(os.getcwd()) - -import piquasso as pq # noqa: E402 - - -NO_OF_MODES = 5 - - -def piquasso_setup(): - program = pq.Program( - state=pq.GaussianState.create_vacuum(d=NO_OF_MODES), - ) - - with program: - pq.Q(0) | pq.S(amp=0.1) | pq.D(alpha=1) - pq.Q(1) | pq.S(amp=0.1) | pq.D(alpha=1) - pq.Q(2) | pq.S(amp=0.1) | pq.D(alpha=1) - pq.Q(3) | pq.S(amp=0.1) | pq.D(alpha=1) - pq.Q(4) | pq.S(amp=0.1) | pq.D(alpha=1) - - pq.Q(0, 1) | pq.B(0.0959408065906761, 0.06786053071484363) - pq.Q(2, 3) | pq.B(0.7730047654405018, 1.453770233324797) - pq.Q(1, 2) | pq.B(1.0152680371119776, 1.2863559998816205) - pq.Q(3, 4) | pq.B(1.3205517879465705, 0.5236836466492961) - pq.Q(0, 1) | pq.B(4.394480318177715, 4.481575657714487) - pq.Q(2, 3) | pq.B(2.2300919706807534, 1.5073556513699888) - pq.Q(1, 2) | pq.B(2.2679037068773673, 1.9550229282085838) - pq.Q(3, 4) | pq.B(3.340269832485504, 3.289367083610399) - - return program - - -def piquasso_benchmark(program): - program.execute() - - result = program.state.reduced_rotated_mean_and_cov( - modes=(1,), phi=np.pi/3 - ) - - # TODO: The state has to be reset, because the setup runs only once at the beginning - # of the calculations, therefore the same `GaussianState` instance will be used. - program.state.reset() - - return result - - -def strawberryfields_setup(): - program = sf.Program(NO_OF_MODES) - engine = sf.Engine(backend="gaussian") - - with program.context as q: - sf.ops.Sgate(0.1) | q[0] - sf.ops.Sgate(0.1) | q[1] - sf.ops.Sgate(0.1) | q[2] - sf.ops.Sgate(0.1) | q[3] - sf.ops.Sgate(0.1) | q[4] - - sf.ops.Dgate(1) | q[0] - sf.ops.Dgate(1) | q[1] - sf.ops.Dgate(1) | q[2] - sf.ops.Dgate(1) | q[3] - sf.ops.Dgate(1) | q[4] - - sf.ops.BSgate(0.0959408065906761, 0.06786053071484363) | (q[0], q[1]) - sf.ops.BSgate(0.7730047654405018, 1.453770233324797) | (q[2], q[3]) - sf.ops.BSgate(1.0152680371119776, 1.2863559998816205) | (q[1], q[2]) - sf.ops.BSgate(1.3205517879465705, 0.5236836466492961) | (q[3], q[4]) - sf.ops.BSgate(4.394480318177715, 4.481575657714487) | (q[0], q[1]) - sf.ops.BSgate(2.2300919706807534, 1.5073556513699888) | (q[2], q[3]) - sf.ops.BSgate(2.2679037068773673, 1.9550229282085838) | (q[1], q[2]) - sf.ops.BSgate(3.340269832485504, 3.289367083610399) | (q[3], q[4]) - - return program, engine - - -def strawberryfields_benchmark(program, engine): - results = engine.run(program) - - return results.state.quad_expectation(mode=1, phi=np.pi/3) - - -def main(iterations): - pq_time = timeit.timeit( - "homodyne.piquasso_benchmark(program)", - setup=( - "import homodyne;" - "program = homodyne.piquasso_setup()" - ), - number=iterations, - ) - print("Piquasso (PQ): {}".format(pq_time)) - - sf_time = timeit.timeit( - "homodyne.strawberryfields_benchmark(program, engine)", - setup=( - "import homodyne;" - "program, engine = homodyne.strawberryfields_setup()" - ), - number=iterations, - ) - print("Strawberry Fields (SF): {}".format(sf_time)) - - print() - - print("Ratio (SF/PQ): {}".format(sf_time / pq_time)) - - -if __name__ == "__main__": - iterations = int(sys.argv[1]) if len(sys.argv) > 1 else 1000 - - main(iterations) diff --git a/poetry.lock b/poetry.lock index d80acab8..2dd15ea6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -26,7 +26,7 @@ python-versions = "*" name = "atomicwrites" version = "1.4.0" description = "Atomic file writes." -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" @@ -34,7 +34,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" name = "attrs" version = "20.3.0" description = "Classes Without Boilerplate" -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" @@ -123,7 +123,7 @@ python-versions = ">=3.5" name = "colorama" version = "0.4.4" description = "Cross-platform colored terminal text." -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" @@ -307,7 +307,7 @@ python-versions = "*" name = "more-itertools" version = "8.7.0" description = "More routines for operating on iterables, beyond itertools" -category = "dev" +category = "main" optional = false python-versions = ">=3.5" @@ -371,7 +371,7 @@ python-versions = ">=3.6" name = "packaging" version = "20.9" description = "Core utilities for Python packages" -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" @@ -411,7 +411,7 @@ strawberryfields = ">=0.11.2" name = "pluggy" version = "0.13.1" description = "plugin and hook calling mechanisms for python" -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" @@ -425,10 +425,18 @@ dev = ["pre-commit", "tox"] name = "py" version = "1.10.0" description = "library with cross-python path, ini-parsing, io, code, log facilities" -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +[[package]] +name = "py-cpuinfo" +version = "7.0.0" +description = "Get CPU info with pure Python 2 & 3" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "pycodestyle" version = "2.6.0" @@ -457,7 +465,7 @@ python-versions = ">=3.5" name = "pyparsing" version = "2.4.7" description = "Python parsing module" -category = "dev" +category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" @@ -465,7 +473,7 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" name = "pytest" version = "5.4.3" description = "pytest: simple powerful testing with Python" -category = "dev" +category = "main" optional = false python-versions = ">=3.5" @@ -484,6 +492,23 @@ wcwidth = "*" checkqa-mypy = ["mypy (==v0.761)"] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] +[[package]] +name = "pytest-benchmark" +version = "3.2.3" +description = "A ``pytest`` fixture for benchmarking code. It will group the tests into rounds that are calibrated to the chosen timer. See calibration_ and FAQ_." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.dependencies] +py-cpuinfo = "*" +pytest = ">=3.8" + +[package.extras] +aspect = ["aspectlib"] +elasticsearch = ["elasticsearch"] +histogram = ["pygal", "pygaljs"] + [[package]] name = "python-dateutil" version = "2.8.1" @@ -817,7 +842,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] name = "wcwidth" version = "0.2.5" description = "Measures the displayed width of unicode strings in a terminal" -category = "dev" +category = "main" optional = false python-versions = "*" @@ -836,7 +861,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.6" -content-hash = "e21930e6e208c68ba0e3a28fc3e403ba5092d2780fac53985dbc04a47773cde9" +content-hash = "9d4cac582591a109399043dfd1eb3ae69b18a6296af40dcb6f1af468ca76c887" [metadata.files] alabaster = [ @@ -1134,6 +1159,9 @@ py = [ {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"}, {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"}, ] +py-cpuinfo = [ + {file = "py-cpuinfo-7.0.0.tar.gz", hash = "sha256:9aa2e49675114959697d25cf57fec41c29b55887bff3bc4809b44ac6f5730097"}, +] pycodestyle = [ {file = "pycodestyle-2.6.0-py2.py3-none-any.whl", hash = "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367"}, {file = "pycodestyle-2.6.0.tar.gz", hash = "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e"}, @@ -1154,6 +1182,10 @@ pytest = [ {file = "pytest-5.4.3-py3-none-any.whl", hash = "sha256:5c0db86b698e8f170ba4582a492248919255fcd4c79b1ee64ace34301fb589a1"}, {file = "pytest-5.4.3.tar.gz", hash = "sha256:7979331bfcba207414f5e1263b5a0f8f521d0f457318836a7355531ed1a4c7d8"}, ] +pytest-benchmark = [ + {file = "pytest-benchmark-3.2.3.tar.gz", hash = "sha256:ad4314d093a3089701b24c80a05121994c7765ce373478c8f4ba8d23c9ba9528"}, + {file = "pytest_benchmark-3.2.3-py2.py3-none-any.whl", hash = "sha256:01f79d38d506f5a3a0a9ada22ded714537bbdfc8147a881a35c1655db07289d9"}, +] python-dateutil = [ {file = "python-dateutil-2.8.1.tar.gz", hash = "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c"}, {file = "python_dateutil-2.8.1-py2.py3-none-any.whl", hash = "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"}, diff --git a/pyproject.toml b/pyproject.toml index 85db12b7..cd74841f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ flake8 = "^3.8.3" quantum-blackbird = "^0.2.3" BoSS-Tomev = "^0.0.6" coverage = "^5.3" +pytest-benchmark = "^3.2.3" [tool.poetry.dev-dependencies] pytest = "^5.2" diff --git a/pytest.ini b/pytest.ini index 24563aa7..dfb902b3 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,6 @@ [pytest] markers = monkey: marks tests with stochastic inputs. + +python_functions = test_* *_benchmark +python_files = test_*.py *_benchmark.py