Skip to content

Commit

Permalink
[FIX,AUTOTVM] Print warning when all autotvm tasks fail with errors (a…
Browse files Browse the repository at this point in the history
…pache#6612)

* [FIX,AUTOTVM] Print warning when all autotvm tasks fail with errors.

* formatting

* write errors to tempfile

* wording

* wording

* don't duplicate errors

* Ensure we have a string for an error
  • Loading branch information
tkonolige authored Oct 5, 2020
1 parent 21002cd commit 86122d1
Showing 1 changed file with 17 additions and 0 deletions.
17 changes: 17 additions & 0 deletions python/tvm/autotvm/tuner/tuner.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
# pylint: disable=unused-argument, no-self-use, invalid-name
"""Base class of tuner"""
import logging
import tempfile

import numpy as np

Expand Down Expand Up @@ -121,6 +122,7 @@ def tune(self, n_trial, measure_option, early_stopping=None, callbacks=(), si_pr

GLOBAL_SCOPE.in_tuning = True
i = error_ct = 0
errors = []
while i < n_trial:
if not self.has_next():
break
Expand All @@ -139,6 +141,11 @@ def tune(self, n_trial, measure_option, early_stopping=None, callbacks=(), si_pr
else:
flops = 0
error_ct += 1
error = res.costs[0]
if isinstance(error, str):
errors.append(error)
else:
errors.append(str(error))

if flops > self.best_flops:
self.best_flops = flops
Expand Down Expand Up @@ -174,6 +181,16 @@ def tune(self, n_trial, measure_option, early_stopping=None, callbacks=(), si_pr
else:
logger.setLevel(old_level)

if error_ct == i:
_, f = tempfile.mkstemp(prefix="tvm_tuning_errors_", suffix=".log", text=True)
with open(f, "w") as file:
file.write("\n".join(errors))
logging.warning(
"Could not find any valid schedule for task %s. "
"A file containing the errors has been written to %s.",
self.task,
f,
)
GLOBAL_SCOPE.in_tuning = False
del measure_batch

Expand Down

0 comments on commit 86122d1

Please sign in to comment.