Skip to content

Commit

Permalink
The get_info API returns run_statuses and end_statuses.
Browse files Browse the repository at this point in the history
  • Loading branch information
wei committed Aug 23, 2024
1 parent 89e8491 commit 80aeb0d
Show file tree
Hide file tree
Showing 6 changed files with 268 additions and 11 deletions.
57 changes: 57 additions & 0 deletions tests/metadate.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
import subprocess

import h5py


def remove_metadata(input_file, output_file):
with h5py.File(input_file, "r") as src, h5py.File(output_file, "w") as dst:

def copy_attrs(src_obj, dst_obj):
# 不复制属性(元数据)
pass

def copy_group(src_group, dst_group):
copy_attrs(src_group, dst_group)
for name, item in src_group.items():
if isinstance(item, h5py.Group):
new_group = dst_group.create_group(name)
copy_group(item, new_group)
elif isinstance(item, h5py.Dataset):
dst_group.create_dataset(name, data=item[...])

copy_group(src, dst)


def get_file_md5(file) -> str:
"""
Get the file md5
Args:
file (str): The path of the gzip input file.
"""
try:
result = subprocess.run(["md5sum", file], capture_output=True, text=True)
return result.stdout.split(" ")[0]
except Exception as e:
return ""


if __name__ == "__main__":
f1 = "11.hdf5.gz"
f2 = "22.hdf5.gz"
f3 = "33.hdf5" # 输出去除元数据后的 HDF5 文件路径
f4 = "44.hdf5" # 输出去除元数据后的 HDF5 文件路径

md5_f1 = get_file_md5("11.hdf5.gz")
print(md5_f1)
md5_f2 = get_file_md5("22.hdf5.gz")
print(md5_f2)

# 移除元数据
remove_metadata(f1, f3)
remove_metadata(f2, f4)

md5_f3 = get_file_md5(f3)
print(md5_f3)
md5_f4 = get_file_md5(f4)
print(md5_f4)
77 changes: 74 additions & 3 deletions tests/test_web/test_webapi.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
upload,
)
from tidy3d.web.core.environment import Env
from tidy3d.web.core.system_config import SystemConfig
from tidy3d.web.core.types import TaskType

TASK_NAME = "task_name_test"
Expand Down Expand Up @@ -174,6 +175,42 @@ def mock_get_info(monkeypatch, set_api_key):
)


@pytest.fixture
def mock_get_system_config(monkeypatch, set_api_key):
"""Mocks webapi.get_system_config."""

responses.add(
responses.GET,
f"{Env.current.web_api_endpoint}/tidy3d/system/py/config",
json={
"data": {
"runStatuses": [
"draft",
"queued",
"preprocess",
"queued_solver",
"running",
"postprocess",
"visualize",
"success",
],
"endStatuses": [
"success",
"error",
"errored",
"diverged",
"diverge",
"deleted",
"draft",
"abort",
"aborted",
],
}
},
status=200,
)


@pytest.fixture
def mock_start(monkeypatch, set_api_key, mock_get_info):
"""Mocks webapi.start."""
Expand Down Expand Up @@ -225,10 +262,38 @@ def mock_get_run_info(task_id):
run_count[0] += 1
return perc_done, 1

def mock_get_system_info():
return SystemConfig(
**{
"runStatuses": [
"draft",
"queued",
"preprocess",
"queued_solver",
"running",
"postprocess",
"visualize",
"success",
],
"endStatuses": [
"success",
"error",
"errored",
"diverged",
"diverge",
"deleted",
"draft",
"abort",
"aborted",
],
}
)

monkeypatch.setattr("tidy3d.web.api.connect_util.REFRESH_TIME", 0.00001)
monkeypatch.setattr(f"{api_path}.RUN_REFRESH_TIME", 0.00001)
monkeypatch.setattr(f"{api_path}.get_status", mock_get_status)
monkeypatch.setattr(f"{api_path}.get_run_info", mock_get_run_info)
monkeypatch.setattr(f"{api_path}.get_system_config", mock_get_system_info)


@pytest.fixture
Expand Down Expand Up @@ -290,7 +355,13 @@ def mock_get_run_info(monkeypatch, set_api_key):

@pytest.fixture
def mock_webapi(
mock_upload, mock_metadata, mock_get_info, mock_start, mock_monitor, mock_download, mock_load
mock_upload,
mock_metadata,
mock_get_info,
mock_start,
mock_monitor,
mock_download,
mock_load,
):
"""Mocks all webapi operation."""

Expand Down Expand Up @@ -545,7 +616,7 @@ def mock_job_status(monkeypatch):


@responses.activate
def test_batch(mock_webapi, mock_job_status, mock_load, tmp_path):
def test_batch(mock_webapi, mock_job_status, mock_load, mock_get_system_config, tmp_path):
# monkeypatch.setattr("tidy3d.web.api.container.Batch.monitor", lambda self: time.sleep(0.1))
# monkeypatch.setattr("tidy3d.web.api.container.Job.status", property(lambda self: "success"))

Expand All @@ -569,7 +640,7 @@ def test_batch(mock_webapi, mock_job_status, mock_load, tmp_path):


@responses.activate
def test_async(mock_webapi, mock_job_status):
def test_async(mock_webapi, mock_get_system_config, mock_job_status):
# monkeypatch.setattr("tidy3d.web.api.container.Job.status", property(lambda self: "success"))
sims = {TASK_NAME: make_sim()}
_ = run_async(sims, folder_name=PROJECT_NAME)
Expand Down
46 changes: 46 additions & 0 deletions tests/test_web/test_webapi_system.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
# Tests webapi and things that depend on it

import pytest
import responses
import tidy3d as td
from tidy3d.web.api.webapi import (
get_system_config,
)
from tidy3d.web.core.environment import Env

task_core_path = "tidy3d.web.core.task_core"
api_path = "tidy3d.web.api.webapi"

Env.dev.active()


@pytest.fixture
def set_api_key(monkeypatch):
"""Set the api key."""
import tidy3d.web.core.http_util as http_module

monkeypatch.setattr(http_module, "api_key", lambda: "apikey")
monkeypatch.setattr(http_module, "get_version", lambda: td.version.__version__)


@pytest.fixture
def mock_get_system_config(monkeypatch, set_api_key):
"""Mocks webapi.get_info."""

responses.add(
responses.GET,
f"{Env.current.web_api_endpoint}/tidy3d/system/py/config",
json={
"data": {
"runStatuses": [],
"endStatuses": [],
}
},
status=200,
)


@responses.activate
def test_system_config(mock_get_system_config):
info = get_system_config()
assert info is not None
27 changes: 24 additions & 3 deletions tidy3d/web/api/container.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from ...exceptions import DataError
from ...log import get_logging_console, log
from ..api import webapi as web
from ..api.webapi import get_system_config
from ..core.constants import TaskId, TaskName
from ..core.task_info import RunInfo, TaskInfo
from .tidy3d_stub import SimulationDataType, SimulationType
Expand Down Expand Up @@ -692,7 +693,7 @@ def pbar_description(task_name: str, status: str) -> str:

return description

run_statuses = [
run_statuses = (
"draft",
"queued",
"preprocess",
Expand All @@ -701,8 +702,28 @@ def pbar_description(task_name: str, status: str) -> str:
"postprocess",
"visualize",
"success",
]
end_statuses = ("success", "error", "errored", "diverged", "diverge", "deleted", "draft")
)

end_statuses = (
"success",
"error",
"errored",
"diverged",
"diverge",
"deleted",
"draft",
"abort",
"aborted",
)

config = get_system_config()
if config is not None:
config_end_statuses = config.end_statuses
config_run_statuses = config.run_statuses
if config_end_statuses is not None:
end_statuses = config_end_statuses
if config_run_statuses is not None:
run_statuses = config_run_statuses

if self.verbose:
console = get_logging_console()
Expand Down
27 changes: 22 additions & 5 deletions tidy3d/web/api/webapi.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from ..core.account import Account
from ..core.constants import SIM_FILE_HDF5, TaskId
from ..core.environment import Env
from ..core.system_config import SystemConfig
from ..core.task_core import Folder, SimulationTask
from ..core.task_info import ChargeType, TaskInfo
from .connect_util import (
Expand Down Expand Up @@ -359,16 +360,17 @@ def monitor(task_id: TaskId, verbose: bool = True) -> None:
----
To load results when finished, may call :meth:`load`.
"""
break_statuses = ("success", "error", "diverged", "deleted", "draft", "abort", "aborted")

console = get_logging_console() if verbose else None

task_info = get_info(task_id)

task_name = task_info.taskName

task_type = task_info.taskType

break_statuses = ("success", "error", "diverged", "deleted", "draft", "abort", "aborted")
config = get_system_config()
if config is not None:
config_end_statuses = config.end_statuses
if config_end_statuses is not None:
break_statuses = config_end_statuses

def get_estimated_cost() -> float:
"""Get estimated cost, if None, is not ready."""
Expand Down Expand Up @@ -959,6 +961,21 @@ def account(verbose=True) -> Account:
return account_info


@wait_for_connection
def get_system_config() -> SystemConfig:
"""Return system config.
Parameters
----------
Returns
-------
:class:`SystemConfig`
Object containing information about system configuration.
"""
return SystemConfig.get()


@wait_for_connection
def test() -> None:
"""
Expand Down
45 changes: 45 additions & 0 deletions tidy3d/web/core/system_config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
"""Tidy3d system config."""

from __future__ import annotations

from typing import Optional

from pydantic.v1 import Extra, Field

from .http_util import http
from .types import Tidy3DResource


class SystemConfig(Tidy3DResource, extra=Extra.allow):
"""Tidy3D system config."""

end_statuses: Optional[tuple] = Field(
None,
title="End Statuses",
description="Tuple of status keys that signify that the task has completed.",
alias="endStatuses",
)
run_statuses: Optional[tuple] = Field(
None,
title="Run Statuses",
description="Tuple of ordered status keys that signify that the task is in progress.",
alias="runStatuses",
)

@classmethod
def get(cls):
"""Get user SystemConfig information.
Parameters
----------
Returns
-------
systemConfig : SystemConfig
"""
resp = http.get("tidy3d/system/py/config")
if resp:
config = SystemConfig(**resp)
return config
else:
return None

0 comments on commit 80aeb0d

Please sign in to comment.