plots: tests: integration tests for studio

Related: iterative#7549

dvc/render/convert.py

@@ -24,6 +24,7 @@ def _get_converter(
 def to_datapoints(renderer_class, data: Dict, props: Dict):
     converter = _get_converter(renderer_class, props)
     datapoints: List[Dict] = []
+    final_props: Dict = {}
     for revision, rev_data in data.items():
         for filename, file_data in rev_data.get("data", {}).items():
             if "data" in file_data:

pyproject.toml

@@ -33,6 +33,8 @@ log_level = "debug"
 addopts = "-ra"
 markers = [
     "needs_internet: Might need network access for the tests",
+"vscode: Tests verifying contract between DVC and VSCode plugin",
+"studio: Tests verifying contract between DVC and Studio"
 ]
 xfail_strict = true
 

tests/integration/plots/conftest.py

@@ -0,0 +1,81 @@
+import pytest
+
+
+@pytest.fixture
+def repo_with_plots(tmp_dir, scm, dvc, run_copy_metrics):
+    def make():
+        linear_v1 = [
+            {"x": 1, "y": 0.1},
+            {"x": 2, "y": 0.2},
+            {"x": 3, "y": 0.3},
+        ]
+
+        confusion_v1 = [
+            {"actual": 0, "predicted": 1},
+            {"actual": 0, "predicted": 1},
+            {"actual": 1, "predicted": 0},
+            {"actual": 1, "predicted": 0},
+        ]
+
+        image_v1 = b"content"
+
+        (tmp_dir / "linear_src.json").dump_json(linear_v1)
+        (tmp_dir / "confusion_src.json").dump_json(confusion_v1)
+        (tmp_dir / "image_src.png").write_bytes(image_v1)
+
+        scm.add(["linear_src.json", "confusion_src.json", "image_src.png"])
+        scm.commit("add data sources")
+
+        run_copy_metrics(
+            "linear_src.json",
+            "linear.json",
+            name="linear",
+            plots=["linear.json"],
+            commit="linear",
+        )
+        run_copy_metrics(
+            "confusion_src.json",
+            "confusion.json",
+            name="confusion",
+            plots=["confusion.json"],
+            commit="confusion",
+        )
+        confusion_props = {
+            "title": "confusion matrix",
+            "x": "predicted",
+            "y": "actual",
+            "template": "confusion",
+        }
+        dvc.plots.modify("confusion.json", confusion_props)
+        run_copy_metrics(
+            "image_src.png",
+            "image.png",
+            name="image",
+            plots=["image.png"],
+            commit="image",
+        )
+
+        scm.add(["dvc.yaml", "dvc.lock"])
+        scm.commit("commit dvc files")
+        yield image_v1, linear_v1, confusion_v1, confusion_props
+        linear_v2 = [
+            {"x": 1, "y": 0.2},
+            {"x": 2, "y": 0.3},
+            {"x": 3, "y": 0.4},
+        ]
+        confusion_v2 = [
+            {"actual": 0, "predicted": 0},
+            {"actual": 0, "predicted": 0},
+            {"actual": 1, "predicted": 1},
+            {"actual": 1, "predicted": 1},
+        ]
+        image_v2 = b"content2"
+
+        (tmp_dir / "linear_src.json").dump_json(linear_v2)
+        (tmp_dir / "confusion_src.json").dump_json(confusion_v2)
+        (tmp_dir / "image_src.png").write_bytes(image_v2)
+
+        dvc.reproduce()
+        yield image_v2, linear_v2, confusion_v2, confusion_props
+
+    return make

tests/integration/plots/test_json.py

@@ -6,6 +6,7 @@
 from urllib.request import url2pathname
 
 import dpath.util
+import pytest
 from funcy import first
 
 from dvc.cli import main
@@ -36,13 +37,6 @@ def call(capsys, subcommand="show"):
     return abspath, json_result, split_json_result
 
 
-def test_no_plots(tmp_dir, scm, dvc, capsys):
-    html_path, json_result, split_json_result = call(capsys)
-    assert os.path.exists(html_path)
-    assert json_result == {}
-    assert split_json_result == {}
-
-
 def filter_fields(datapoints: List[Dict], fields: List[str]):
 
     tmp = deepcopy(datapoints)
@@ -77,6 +71,9 @@ def verify_image(tmp_dir, version, filename, content, html_path, json_result):
         os.path.join("dvc_plots", "static", f"{version}_{filename}")
         in html_content
     )
+
+    # there should be no absolute paths in produced HTML
+    assert str(tmp_dir) not in html_content
     assert (
         tmp_dir / "dvc_plots" / "static" / f"{version}_{filename}"
     ).read_bytes() == content
@@ -147,7 +144,7 @@ def verify_vega(
     assert json_content == split_json_content
 
 
-def verify_vega_props(filename, json_result, title, x, y):
+def verify_vega_props(filename, json_result, title, x, y, **kwargs):
     data = json_result[filename]
     assert len(data) == 1
     data = first(data)
@@ -164,85 +161,26 @@ def verify_vega_props(filename, json_result, title, x, y):
     )
 
 
-def repo_with_plots(tmp_dir, scm, dvc, run_copy_metrics):
-    linear_v1 = [{"x": 1, "y": 0.1}, {"x": 2, "y": 0.2}, {"x": 3, "y": 0.3}]
-
-    confusion_v1 = [
-        {"actual": 0, "predicted": 1},
-        {"actual": 0, "predicted": 1},
-        {"actual": 1, "predicted": 0},
-        {"actual": 1, "predicted": 0},
-    ]
-
-    image_v1 = b"content"
-
-    (tmp_dir / "linear_src.json").dump_json(linear_v1)
-    (tmp_dir / "confusion_src.json").dump_json(confusion_v1)
-    (tmp_dir / "image_src.png").write_bytes(image_v1)
-
-    scm.add(["linear_src.json", "confusion_src.json", "image_src.png"])
-    scm.commit("add data sources")
-
-    run_copy_metrics(
-        "linear_src.json",
-        "linear.json",
-        name="linear",
-        plots=["linear.json"],
-        commit="linear",
-    )
-    run_copy_metrics(
-        "confusion_src.json",
-        "confusion.json",
-        name="confusion",
-        plots=["confusion.json"],
-        commit="confusion",
-    )
-    confusion_props = {
-        "title": "confusion matrix",
-        "x": "predicted",
-        "y": "actual",
-    }
-    dvc.plots.modify(
-        "confusion.json",
-        {**confusion_props, **{"template": "confusion"}},
-    )
-    run_copy_metrics(
-        "image_src.png",
-        "image.png",
-        name="image",
-        plots=["image.png"],
-        commit="image",
-    )
-
-    scm.add(["dvc.yaml", "dvc.lock"])
-    scm.commit("commit dvc files")
-    yield image_v1, linear_v1, confusion_v1, confusion_props
-    linear_v2 = [{"x": 1, "y": 0.2}, {"x": 2, "y": 0.3}, {"x": 3, "y": 0.4}]
-    confusion_v2 = [
-        {"actual": 0, "predicted": 0},
-        {"actual": 0, "predicted": 0},
-        {"actual": 1, "predicted": 1},
-        {"actual": 1, "predicted": 1},
-    ]
-    image_v2 = b"content2"
-
-    (tmp_dir / "linear_src.json").dump_json(linear_v2)
-    (tmp_dir / "confusion_src.json").dump_json(confusion_v2)
-    (tmp_dir / "image_src.png").write_bytes(image_v2)
-
-    dvc.reproduce()
-    yield image_v2, linear_v2, confusion_v2, confusion_props
+@pytest.mark.vscode
+def test_no_plots(tmp_dir, scm, dvc, capsys):
+    html_path, json_result, split_json_result = call(capsys)
+    assert os.path.exists(html_path)
+    assert json_result == {}
+    assert split_json_result == {}
 
 
-def test_repo_with_plots(tmp_dir, scm, dvc, capsys, run_copy_metrics):
-    repo_state = repo_with_plots(tmp_dir, scm, dvc, run_copy_metrics)
+@pytest.mark.vscode
+def test_repo_with_plots(
+    tmp_dir, scm, dvc, capsys, run_copy_metrics, repo_with_plots
+):
+    repo_state = repo_with_plots()
 
     image_v1, linear_v1, confusion_v1, confusion_props = next(repo_state)
 
     html_path, json_result, split_json_result = call(capsys)
 
     verify_image(
-        tmp_dir, "workspace", "image.png", b"content", html_path, json_result
+        tmp_dir, "workspace", "image.png", image_v1, html_path, json_result
     )
 
     verify_vega(
@@ -297,3 +235,12 @@ def test_repo_with_plots(tmp_dir, scm, dvc, capsys, run_copy_metrics):
         split_json_result,
     )
     verify_vega_props("confusion.json", json_result, **confusion_props)
+    from dvc.utils.fs import remove
+
+    # even if there is no data, call should be successful
+    remove(tmp_dir / ".dvc" / "cache")
+    remove("linear.json")
+    remove("confusion.json")
+    remove("image.png")
+    call(capsys)
+    call(capsys, subcommand="diff")

tests/integration/plots/test_repo_plots_api.py

@@ -0,0 +1,118 @@
+import dpath.util
+import pytest
+
+
+@pytest.mark.studio
+def test_api(tmp_dir, dvc, repo_with_plots):
+    repo_state = repo_with_plots()
+    image_v1, linear_v1, confusion_v1, confusion_params = next(repo_state)
+
+    workspace_data = next(dvc.plots.collect())
+
+    assert (
+        dpath.util.get(
+            workspace_data, ["workspace", "data", "image.png", "props"]
+        )
+        == {}
+    )
+    image_source = dpath.util.get(
+        workspace_data, ["workspace", "data", "image.png", "data_source"]
+    )
+    assert callable(image_source)
+    assert image_source() == {"data": image_v1}
+
+    assert (
+        dpath.util.get(
+            workspace_data, ["workspace", "data", "linear.json", "props"]
+        )
+        == {}
+    )
+    linear_source = dpath.util.get(
+        workspace_data, ["workspace", "data", "linear.json", "data_source"]
+    )
+    assert callable(linear_source)
+    assert linear_source() == {"data": linear_v1}
+
+    assert (
+        dpath.util.get(
+            workspace_data, ["workspace", "data", "confusion.json", "props"]
+        )
+        == confusion_params
+    )
+    confusion_source = dpath.util.get(
+        workspace_data, ["workspace", "data", "confusion.json", "data_source"]
+    )
+    assert callable(confusion_source)
+    assert confusion_source() == {"data": confusion_v1}
+
+    image_v2, linear_v2, confusion_v2, _ = next(repo_state)
+    data_generator = dvc.plots.collect(revs=["workspace", "HEAD"])
+
+    workspace_data = next(data_generator)
+
+    assert (
+        dpath.util.get(
+            workspace_data, ["workspace", "data", "image.png", "props"]
+        )
+        == {}
+    )
+    image_source = dpath.util.get(
+        workspace_data, ["workspace", "data", "image.png", "data_source"]
+    )
+    assert callable(image_source)
+    assert image_source() == {"data": image_v2}
+
+    assert (
+        dpath.util.get(
+            workspace_data, ["workspace", "data", "linear.json", "props"]
+        )
+        == {}
+    )
+    linear_source = dpath.util.get(
+        workspace_data, ["workspace", "data", "linear.json", "data_source"]
+    )
+    assert callable(linear_source)
+    assert linear_source() == {"data": linear_v2}
+
+    assert (
+        dpath.util.get(
+            workspace_data, ["workspace", "data", "confusion.json", "props"]
+        )
+        == confusion_params
+    )
+    confusion_source = dpath.util.get(
+        workspace_data, ["workspace", "data", "confusion.json", "data_source"]
+    )
+    assert callable(confusion_source)
+    assert confusion_source() == {"data": confusion_v2}
+
+    head_data = next(data_generator)
+
+    assert (
+        dpath.util.get(head_data, ["HEAD", "data", "image.png", "props"]) == {}
+    )
+    image_source = dpath.util.get(
+        head_data, ["HEAD", "data", "image.png", "data_source"]
+    )
+    assert callable(image_source)
+    assert image_source() == {"data": image_v1}
+
+    assert (
+        dpath.util.get(head_data, ["HEAD", "data", "linear.json", "props"])
+        == {}
+    )
+    linear_source = dpath.util.get(
+        head_data, ["HEAD", "data", "linear.json", "data_source"]
+    )
+    assert callable(linear_source)
+    assert linear_source() == {"data": linear_v1}
+
+    assert (
+        dpath.util.get(head_data, ["HEAD", "data", "confusion.json", "props"])
+        == confusion_params
+    )
+    confusion_source = dpath.util.get(
+        head_data, ["HEAD", "data", "confusion.json", "data_source"]
+    )
+    assert callable(confusion_source)
+    assert confusion_source() == {"data": confusion_v1}