index: load meta from outs.files

avrtt · Dec 22, 2023 · 6a44783 · 6a44783
1 parent 98a286d
commit 6a44783
Show file tree

Hide file tree

Showing 4 changed files with 49 additions and 11 deletions.
diff --git a/dvc/repo/index.py b/dvc/repo/index.py
@@ -129,10 +129,25 @@ def _load_data_from_outs(index, prefix, outs):
         for key_len in range(1, len(key)):
             parents.add((ws, key[:key_len]))
 
+        loaded = None
+        if out.files:
+            loaded = True
+            for okey, ometa, ohi in out.get_obj():
+                for key_len in range(1, len(okey)):
+                    parents.add((ws, (*key, *okey[:key_len])))
+
+                fkey = (*key, *okey)
+                index[(*prefix, ws, *fkey)] = DataIndexEntry(
+                    key=fkey,
+                    meta=ometa,
+                    hash_info=ohi,
+                )
+
         entry = DataIndexEntry(
             key=key,
             meta=out.meta,
             hash_info=out.hash_info,
+            loaded=loaded,
         )
 
         if (

diff --git a/dvc/testing/cloud.py b/dvc/testing/cloud.py
@@ -32,6 +32,14 @@ def write_text(self, contents, encoding=None, errors=None):
     def write_bytes(self, contents):
         raise NotImplementedError
 
+    @abstractmethod
+    def unlink(self, missing_ok: bool = False) -> None:
+        pass
+
+    @abstractmethod
+    def rmdir(self, recursive: bool = True) -> None:
+        pass
+
     def read_text(self, encoding=None, errors=None):
         if not encoding:
             encoding = locale.getpreferredencoding(False)

diff --git a/dvc/testing/remote_tests.py b/dvc/testing/remote_tests.py
@@ -171,7 +171,7 @@ def test_file(self, tmp_dir, dvc, run_copy, remote_version_aware):
         (stage,) = tmp_dir.dvc_gen("foo", "foo")
         run_copy("foo", "foo_copy", name="copy")
 
-        dvc.push()
+        assert dvc.push()
         assert (remote_version_aware / "foo").read_text() == "foo"
         assert (remote_version_aware / "foo_copy").read_text() == "foo"
         foo_dvc = (tmp_dir / "foo.dvc").read_text()
@@ -185,26 +185,26 @@ def test_file(self, tmp_dir, dvc, run_copy, remote_version_aware):
         remove(tmp_dir / "foo")
         remove(tmp_dir / "foo_copy")
 
-        dvc.pull()
+        assert dvc.pull()
         assert (tmp_dir / "foo").read_text() == "foo"
         assert (tmp_dir / "foo_copy").read_text() == "foo"
         assert (tmp_dir / "foo.dvc").read_text() == foo_dvc
         assert (tmp_dir / "dvc.lock").read_text() == dvc_lock
 
-        dvc.push()
+        assert not dvc.push()
         assert (remote_version_aware / "foo").read_text() == "foo"
         assert (remote_version_aware / "foo_copy").read_text() == "foo"
         assert (tmp_dir / "foo.dvc").read_text() == foo_dvc
         assert (tmp_dir / "dvc.lock").read_text() == dvc_lock
 
         dvc.reproduce()
-        dvc.push()
+        assert not dvc.push()
         assert (remote_version_aware / "foo").read_text() == "foo"
         assert (remote_version_aware / "foo_copy").read_text() == "foo"
         assert (tmp_dir / "foo.dvc").read_text() == foo_dvc
         assert (tmp_dir / "dvc.lock").read_text() == dvc_lock
 
-    def test_dir(self, tmp_dir, dvc, run_copy, remote_version_aware):
+    def test_dir(self, tmp_dir, dvc, run_copy, remote_version_aware):  # noqa: PLR0915
         (stage,) = tmp_dir.dvc_gen(
             {
                 "data_dir": {
@@ -215,7 +215,7 @@ def test_dir(self, tmp_dir, dvc, run_copy, remote_version_aware):
             }
         )
 
-        dvc.push()
+        assert dvc.push()
 
         data_dir_dvc = (tmp_dir / "data_dir.dvc").read_text()
         assert "files" in data_dir_dvc
@@ -230,7 +230,7 @@ def test_dir(self, tmp_dir, dvc, run_copy, remote_version_aware):
         remove(dvc.cache.local.path)
         remove(tmp_dir / "data_dir")
 
-        dvc.pull()
+        assert dvc.pull()
         assert (tmp_dir / "data_dir" / "data").read_text() == "data"
         assert (
             tmp_dir / "data_dir" / "data_sub_dir" / "data_sub"
@@ -240,7 +240,7 @@ def test_dir(self, tmp_dir, dvc, run_copy, remote_version_aware):
         run_copy("data_dir", "data_dir_copy", name="copy")
         dvc_lock = (tmp_dir / "dvc.lock").read_text()
 
-        dvc.push()
+        assert dvc.push()
         assert (remote_version_aware / "data_dir").exists()
         assert (remote_version_aware / "data_dir" / "data").exists()
         assert (remote_version_aware / "data_dir_copy").exists()
@@ -249,7 +249,7 @@ def test_dir(self, tmp_dir, dvc, run_copy, remote_version_aware):
         assert (tmp_dir / "dvc.lock").read_text() != dvc_lock
         dvc_lock = (tmp_dir / "dvc.lock").read_text()
 
-        dvc.push()
+        assert not dvc.push()
         assert (remote_version_aware / "data_dir").exists()
         assert (remote_version_aware / "data_dir" / "data").exists()
         assert (remote_version_aware / "data_dir_copy").exists()
@@ -259,14 +259,29 @@ def test_dir(self, tmp_dir, dvc, run_copy, remote_version_aware):
 
         dvc.cache.local.clear()
         remove(tmp_dir / "data_dir")
-        dvc.push()
+        remove(tmp_dir / "data_dir_copy")
+        assert not dvc.push()
         assert (remote_version_aware / "data_dir").exists()
         assert (remote_version_aware / "data_dir" / "data").exists()
         assert (remote_version_aware / "data_dir_copy").exists()
         assert (remote_version_aware / "data_dir_copy" / "data").exists()
         assert (tmp_dir / "data_dir.dvc").read_text() == data_dir_dvc
         assert (tmp_dir / "dvc.lock").read_text() == dvc_lock
 
+        (remote_version_aware / "data_dir").rmdir()
+        (remote_version_aware / "data_dir_copy").rmdir()
+        assert not (remote_version_aware / "data_dir").exists()
+        assert not (remote_version_aware / "data_dir_copy").exists()
+        assert dvc.pull()
+        assert (tmp_dir / "data_dir" / "data").read_text() == "data"
+        assert (
+            tmp_dir / "data_dir" / "data_sub_dir" / "data_sub"
+        ).read_text() == "data_sub"
+        assert (tmp_dir / "data_dir_copy" / "data").read_text() == "data"
+        assert (
+            tmp_dir / "data_dir_copy" / "data_sub_dir" / "data_sub"
+        ).read_text() == "data_sub"
+
 
 class TestRemoteWorktree:
     def test_file(self, tmp_dir, dvc, remote_worktree):

diff --git a/pyproject.toml b/pyproject.toml
@@ -35,7 +35,7 @@ dependencies = [
     "configobj>=5.0.6",
     "distro>=1.3",
     "dpath<3,>=2.1.0",
-    "dvc-data>=3.1.1,<3.2",
+    "dvc-data>=3.2,<3.3",
     "dvc-http>=2.29.0",
     "dvc-render>=1.0.0,<2",
     "dvc-studio-client>=0.17.1,<1",