Skip to content

Commit

Permalink
Misc. imports lazy on utils,stage.cache and dvc.* (iterative#5228)
Browse files Browse the repository at this point in the history
  • Loading branch information
skshetry authored Jan 7, 2021
1 parent 66fc988 commit 9331ad2
Showing 8 changed files with 50 additions and 36 deletions.
28 changes: 12 additions & 16 deletions dvc/_debug.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
import argparse
from contextlib import ExitStack, contextmanager
from typing import TYPE_CHECKING

if TYPE_CHECKING:
from argparse import Namespace


@contextmanager
@@ -56,7 +59,7 @@ def debug():


@contextmanager
def debugtools(args: "argparse.Namespace" = None, **kwargs):
def debugtools(args: "Namespace" = None, **kwargs):
kw = vars(args) if args else {}
kw.update(kwargs)

@@ -71,25 +74,18 @@ def debugtools(args: "argparse.Namespace" = None, **kwargs):


def add_debugging_flags(parser):
from argparse import SUPPRESS

parser.add_argument(
"--cprofile",
action="store_true",
default=False,
help=argparse.SUPPRESS,
"--cprofile", action="store_true", default=False, help=SUPPRESS,
)
parser.add_argument("--cprofile-dump", help=argparse.SUPPRESS)
parser.add_argument("--cprofile-dump", help=SUPPRESS)
parser.add_argument(
"--pdb", action="store_true", default=False, help=argparse.SUPPRESS,
"--pdb", action="store_true", default=False, help=SUPPRESS,
)
parser.add_argument(
"--instrument",
action="store_true",
default=False,
help=argparse.SUPPRESS,
"--instrument", action="store_true", default=False, help=SUPPRESS,
)
parser.add_argument(
"--instrument-open",
action="store_true",
default=False,
help=argparse.SUPPRESS,
"--instrument-open", action="store_true", default=False, help=SUPPRESS,
)
7 changes: 4 additions & 3 deletions dvc/data_cloud.py
Original file line number Diff line number Diff line change
@@ -2,9 +2,6 @@

import logging

from dvc.config import NoRemoteError
from dvc.remote import get_remote

logger = logging.getLogger(__name__)


@@ -23,6 +20,8 @@ def __init__(self, repo):
self.repo = repo

def get_remote(self, name=None, command="<command>"):
from dvc.config import NoRemoteError

if not name:
name = self.repo.config["core"].get("remote")

@@ -45,6 +44,8 @@ def get_remote(self, name=None, command="<command>"):
raise NoRemoteError(error_msg)

def _init_remote(self, name):
from dvc.remote import get_remote

return get_remote(self.repo, name=name)

def push(
3 changes: 2 additions & 1 deletion dvc/rwlock.py
Original file line number Diff line number Diff line change
@@ -6,7 +6,6 @@
from voluptuous import Invalid, Optional, Required, Schema

from .exceptions import DvcException
from .lock import LockError
from .utils import relpath

INFO_SCHEMA = {Required("pid"): int, Required("cmd"): str}
@@ -60,6 +59,8 @@ def _infos_to_str(infos):


def _check_blockers(lock, info, *, mode, waiters):
from .lock import LockError

for path_info in waiters:
blockers = [
blocker
26 changes: 19 additions & 7 deletions dvc/stage/cache.py
Original file line number Diff line number Diff line change
@@ -4,17 +4,10 @@
from contextlib import contextmanager

from funcy import cached_property, first
from voluptuous import Invalid

from dvc.exceptions import DvcException
from dvc.path_info import PathInfo
from dvc.remote.base import _log_exceptions
from dvc.schema import COMPILED_LOCK_FILE_STAGE_SCHEMA
from dvc.utils import dict_sha256, relpath
from dvc.utils.serialize import YAMLFileCorruptedError, dump_yaml, load_yaml

from .loader import StageLoader
from .serialize import to_single_stage_lockfile

logger = logging.getLogger(__name__)

@@ -53,6 +46,8 @@ def _can_hash(stage):


def _get_stage_hash(stage):
from .serialize import to_single_stage_lockfile

assert _can_hash(stage)
return _get_cache_hash(to_single_stage_lockfile(stage), key=True)

@@ -76,6 +71,11 @@ def _get_cache_path(self, key, value):
return os.path.join(self._get_cache_dir(key), value)

def _load_cache(self, key, value):
from voluptuous import Invalid

from dvc.schema import COMPILED_LOCK_FILE_STAGE_SCHEMA
from dvc.utils.serialize import YAMLFileCorruptedError, load_yaml

path = self._get_cache_path(key, value)

try:
@@ -105,6 +105,7 @@ def _load(self, stage):

def _create_stage(self, cache, wdir=None):
from . import PipelineStage, create_stage
from .loader import StageLoader

stage = create_stage(
PipelineStage,
@@ -144,6 +145,8 @@ def _uncached_outs(self, stage, cache):
yield out

def save(self, stage):
from .serialize import to_single_stage_lockfile

if not _can_hash(stage):
return

@@ -160,6 +163,9 @@ def save(self, stage):
if existing_cache:
return

from dvc.schema import COMPILED_LOCK_FILE_STAGE_SCHEMA
from dvc.utils.serialize import dump_yaml

# sanity check
COMPILED_LOCK_FILE_STAGE_SCHEMA(cache)

@@ -172,6 +178,8 @@ def save(self, stage):
self.tree.move(PathInfo(tmp), path)

def restore(self, stage, run_cache=True, pull=False):
from .serialize import to_single_stage_lockfile

if not _can_hash(stage):
raise RunCacheNotFoundError(stage)

@@ -224,6 +232,8 @@ def _transfer(func, from_remote, to_remote):
return ret

def push(self, remote):
from dvc.remote.base import _log_exceptions

remote = self.repo.cloud.get_remote(remote)
return self._transfer(
_log_exceptions(remote.tree.upload, "upload"),
@@ -232,6 +242,8 @@ def push(self, remote):
)

def pull(self, remote):
from dvc.remote.base import _log_exceptions

remote = self.repo.cloud.get_remote(remote)
return self._transfer(
_log_exceptions(remote.tree.download, "download"),
7 changes: 5 additions & 2 deletions dvc/state.py
Original file line number Diff line number Diff line change
@@ -3,7 +3,6 @@
import logging
import os
import re
import sqlite3
from abc import ABC, abstractmethod
from urllib.parse import urlencode, urlunparse

@@ -259,12 +258,14 @@ def load(self):
self.database = _connect_sqlite(self.state_file, {"nolock": 1})
self.cursor = self.database.cursor()

from sqlite3 import DatabaseError

# Try loading once to check that the file is indeed a database
# and reformat it if it is not.
try:
self._prepare_db(empty=empty)
return
except sqlite3.DatabaseError:
except DatabaseError:
self.cursor.close()
self.database.close()
self.database = None
@@ -520,6 +521,8 @@ def remove_links(self, unused):
def _connect_sqlite(filename, options):
# Connect by URI was added in Python 3.4 and sqlite 3.7.7,
# we ignore options, which should be fine unless repo is on old NFS/CIFS
import sqlite3

if sqlite3.sqlite_version_info < (3, 7, 7):
return sqlite3.connect(filename)

6 changes: 4 additions & 2 deletions dvc/utils/__init__.py
Original file line number Diff line number Diff line change
@@ -12,8 +12,6 @@
from typing import Optional, Tuple

import colorama
import nanotime
from shortuuid import uuid

logger = logging.getLogger(__name__)

@@ -230,10 +228,14 @@ def fix_env(env=None):

def tmp_fname(fname):
""" Temporary name for a partial download """
from shortuuid import uuid

return os.fspath(fname) + "." + uuid() + ".tmp"


def current_timestamp():
import nanotime

return int(nanotime.timestamp(time.time()))


5 changes: 2 additions & 3 deletions dvc/utils/fs.py
Original file line number Diff line number Diff line change
@@ -5,9 +5,6 @@
import stat
import sys

import nanotime
from shortuuid import uuid

from dvc.exceptions import DvcException, FileOwnershipError
from dvc.system import System
from dvc.utils import dict_md5
@@ -31,6 +28,7 @@ def get_inode(path):


def get_mtime_and_size(path, tree):
import nanotime

if tree.isdir(path):
size = 0
@@ -91,6 +89,7 @@ def move(src, dst, mode=None):
case src and dst are on different filesystems and actual physical copying
of data is happening.
"""
from shortuuid import uuid

dst = os.path.abspath(dst)
tmp = f"{dst}.{uuid()}"
4 changes: 2 additions & 2 deletions dvc/utils/html.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
from typing import Dict, List, Optional

import tabulate

PAGE_HTML = """<!DOCTYPE html>
<html>
<head>
@@ -27,6 +25,8 @@ def __init__(self):
self.elements = []

def with_metrics(self, metrics: Dict[str, Dict]) -> "HTML":
import tabulate

header: List[str] = []
rows: List[List[str]] = []

0 comments on commit 9331ad2

Please sign in to comment.