[core][cluster] handle file deleted case when trying to re-open file (r…

…ay-project#31125) Signed-off-by: Clarence Ng <[email protected]> We are getting errors in log monitor when the file it is monitoring is deleted, which could happen during rotation
Ransnomy · Dec 15, 2022 · 982d4ae · 982d4ae
1 parent 50d436e
commit 982d4ae
Show file tree

Hide file tree

Showing 2 changed files with 33 additions and 7 deletions.
diff --git a/python/ray/_private/log_monitor.py b/python/ray/_private/log_monitor.py
@@ -73,13 +73,17 @@ def reopen_if_necessary(self):
         would have different inodes, such as log rotation or file syncing
         semantics.
         """
-        open_inode = None
-        if self.file_handle and not self.file_handle.closed:
-            open_inode = os.fstat(self.file_handle.fileno()).st_ino
-        new_inode = os.stat(self.filename).st_ino
-        if open_inode != new_inode:
-            self.file_handle = open(self.filename, "rb")
-            self.file_handle.seek(self.file_position)
+        try:
+            open_inode = None
+            if self.file_handle and not self.file_handle.closed:
+                open_inode = os.fstat(self.file_handle.fileno()).st_ino
+
+            new_inode = os.stat(self.filename).st_ino
+            if open_inode != new_inode:
+                self.file_handle = open(self.filename, "rb")
+                self.file_handle.seek(self.file_position)
+        except Exception:
+            logger.debug(f"file no longer exists, skip re-opening of {self.filename}")
 
     def __repr__(self):
         return (

diff --git a/python/ray/tests/test_logging.py b/python/ray/tests/test_logging.py
@@ -79,6 +79,28 @@ def test_reopen_changed_inode(tmp_path):
     assert file_info.file_handle.tell() == orig_file_pos
 
 
+def test_deleted_file_does_not_throw_error(tmp_path):
+    filename = tmp_path / "file"
+
+    Path(filename).touch()
+
+    file_info = LogFileInfo(
+        filename=filename,
+        size_when_last_opened=0,
+        file_position=0,
+        file_handle=None,
+        is_err_file=False,
+        job_id=None,
+        worker_pid=None,
+    )
+
+    file_info.reopen_if_necessary()
+
+    os.remove(filename)
+
+    file_info.reopen_if_necessary()
+
+
 def test_log_rotation_config(ray_start_cluster, monkeypatch):
     cluster = ray_start_cluster
     max_bytes = 100