Skip to content

Commit

Permalink
Merge pull request iterative#506 from dataversioncontrol/misc_fixes
Browse files Browse the repository at this point in the history 
Misc fixes: Not expected files in dir cache and message formatting
  • Loading branch information
@dmpetrov
dmpetrov authored Mar 3, 2018
2 parents d0263aa + e3b63be commit 5bc7ff9
Show file tree
Hide file tree
Showing 2 changed files with 32 additions and 22 deletions.
48 changes: 29 additions & 19 deletions dvc/output.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,8 +100,8 @@ def save(self):
state = self.project.state.get(self.path)
if state and state.mtime == self.mtime() and state.inode == self.inode():
md5 = state.md5
msg = '{} using md5 {} from state file'
self.project.logger.debug(msg.format(self.path, md5))
msg = u'\'{}\' using md5 {} from state file'
self.project.logger.debug(msg.format(self.dvc_path, md5))
self.md5 = md5
else:
self.md5 = self.compute_md5()
Expand Down Expand Up @@ -150,6 +150,10 @@ def __init__(self, project, path, md5=None, use_cache=True):
def cache(self):
return self.project.cache.get(self.md5)

@property
def dvc_cache(self):
return os.path.relpath(self.cache, self.project.root_dir)

def dumpd(self, cwd):
ret = super(Output, self).dumpd(cwd)
ret[Output.PARAM_CACHE] = self.use_cache
Expand Down Expand Up @@ -214,17 +218,17 @@ def changed(self):
os.path.isdir(self.cache):
ret = self._changed_dir()

msg = "Data {} with cache {} "
msg = u'Data file or dir \'{}\' with cache \'{}\' '
if ret:
msg += "changed"
msg += 'changed'
else:
msg += "didn't change"
self.project.logger.debug(msg.format(self.path, self.cache))
msg += 'didn\'t change'
self.project.logger.debug(msg.format(self.dvc_path, self.dvc_cache))

return ret

def hardlink(self, src, link):
self.project.logger.debug("creating hardlink {} -> {}".format(src, link))
self.project.logger.debug(u'creating hardlink {} -> {}'.format(src, link))
System.hardlink(src, link)
os.chmod(src, stat.S_IREAD)

Expand All @@ -236,19 +240,25 @@ def dir_cache(self):
relpath = os.path.relpath(path, self.cache)
with open(path, 'r') as fd:
d = yaml.safe_load(fd)
md5 = d[Output.PARAM_MD5]
res[relpath] = self.project.cache.get(md5)

if not isinstance(d, dict):
msg = u'Dir cache file format error \'{}\': skipping the file'
self.project.logger.error(msg.format(relpath))
else:
md5 = d[Output.PARAM_MD5]
res[relpath] = self.project.cache.get(md5)
return res

def checkout(self):
if not self.use_cache:
return

self.project.logger.debug("Checking out {} with cache {}".format(self.path, self.cache))
msg = u'Checking out \'{}\' with cache \'{}\''
self.project.logger.debug(msg.format(self.dvc_path, self.dvc_cache))

if not self.changed():
msg = "Data {} with cache {} didn't change, skipping checkout."
self.project.logger.debug(msg.format(self.path, self.cache))
msg = u'Data file \'{}\' with cache \'{}\' didn\'t change, skipping checkout.'
self.project.logger.debug(msg.format(self.dvc_path, self.dvc_cache))
return

if not os.path.exists(self.cache):
Expand All @@ -257,8 +267,8 @@ def checkout(self):
return

if os.path.exists(self.path):
msg = "Data {} exists. Removing before checkout"
self.project.logger.debug(msg.format(self.path))
msg = u'Data file \'{}\' exists. Removing before checkout'
self.project.logger.debug(msg.format(self.dvc_path))
self.remove()

if os.path.isfile(self.cache):
Expand All @@ -280,7 +290,7 @@ def save(self):
if not self.use_cache:
return

self.project.logger.debug("Saving {} to {}".format(self.path, self.cache))
self.project.logger.debug(u'Saving \'{}\' to \'{}\''.format(self.dvc_path, self.dvc_cache))

if self.project.scm.is_tracked(self.path):
raise CmdOutputAlreadyTrackedError(self.rel_path)
Expand All @@ -292,8 +302,8 @@ def save(self):
# This means that we already have cache for this data.
# We remove data and link it to existing cache to save
# some space.
msg = "Cache {} already exists, performing checkout for {}"
self.project.logger.debug(msg.format(self.cache, self.path))
msg = u'Cache \'{}\' already exists, performing checkout for \'{}\''
self.project.logger.debug(msg.format(self.dvc_cache, self.dvc_path))
self.checkout()
return

Expand Down Expand Up @@ -323,10 +333,10 @@ def save(self):
yaml.safe_dump({self.PARAM_MD5: md5}, fd, default_flow_style=False)

def _remove(self, path, cache):
self.project.logger.debug("Removing '{}'".format(path))
self.project.logger.debug(u'Removing \'{}\''.format(path))
os.chmod(path, stat.S_IWUSR)
os.unlink(path)
if cache != None and os.path.exists(cache):
if cache is not None and os.path.exists(cache):
os.chmod(cache, stat.S_IREAD)

def remove(self):
Expand Down
6 changes: 3 additions & 3 deletions dvc/stage.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

class StageCmdFailedError(DvcException):
def __init__(self, stage):
msg = 'Stage {} cmd {} failed'.format(stage.relpath, stage.cmd)
msg = u'Stage \'{}\' cmd {} failed'.format(stage.relpath, stage.cmd)
super(StageCmdFailedError, self).__init__(msg)


Expand Down Expand Up @@ -66,10 +66,10 @@ def is_stage_file(path):
def changed(self):
for entry in itertools.chain(self.outs, self.deps):
if entry.changed():
self.project.logger.debug("{} changed".format(self.path))
self.project.logger.debug(u'Dvc file \'{}\' changed'.format(self.dvc_path))
return True
else:
self.project.logger.debug("{} didn't change".format(self.path))
self.project.logger.debug(u'Dvc file \'{}\' didn\'t change'.format(self.dvc_path))
return False

def remove_outs(self):
Expand Down

0 comments on commit 5bc7ff9

Please sign in to comment.