forked from pytorch/pytorch
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
U/kostmo/gen circle conf (pytorch#17189)
Summary: Diagram preview: ![binarysmoketests-config-dimensions](https://user-images.githubusercontent.com/261693/53040977-a0f88d00-3437-11e9-9190-796cc243e0f9.png) Pull Request resolved: pytorch#17189 Differential Revision: D14141362 Pulled By: kostmo fbshipit-source-id: 0625a1234d0307c6be79f17e756ddb1cc445b374
- Loading branch information
1 parent
f827f9f
commit 09c9af9
Showing
32 changed files
with
1,669 additions
and
2,567 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
*.svg | ||
*.png |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,187 @@ | ||
from collections import OrderedDict | ||
|
||
import conf_tree | ||
import miniutils | ||
import make_build_configs | ||
|
||
|
||
class Conf: | ||
def __init__(self, os, cuda_version, pydistro, parms, smoke=False, libtorch_variant=None): | ||
|
||
self.os = os | ||
self.cuda_version = cuda_version | ||
self.pydistro = pydistro | ||
self.parms = parms | ||
self.smoke = smoke | ||
self.libtorch_variant = libtorch_variant | ||
|
||
def genBuildEnvParms(self): | ||
return [self.pydistro] + self.parms + [make_build_configs.get_processor_arch_name(self.cuda_version)] | ||
|
||
def genDockerImage(self): | ||
|
||
docker_word_substitution = { | ||
"manywheel": "manylinux", | ||
"libtorch": "manylinux", | ||
} | ||
|
||
docker_distro_prefix = miniutils.override(self.pydistro, docker_word_substitution) | ||
|
||
alt_docker_suffix = self.cuda_version or "80" | ||
docker_distro_suffix = "" if self.pydistro == "conda" else alt_docker_suffix | ||
return miniutils.quote("soumith/" + docker_distro_prefix + "-cuda" + docker_distro_suffix) | ||
|
||
def getNamePrefix(self): | ||
return "smoke" if self.smoke else "binary" | ||
|
||
def genBuildName(self, build_or_test): | ||
parts = [self.getNamePrefix(), self.os] + self.genBuildEnvParms() | ||
|
||
if self.smoke: | ||
if self.libtorch_variant: | ||
parts.append(self.libtorch_variant) | ||
else: | ||
parts.append(build_or_test) | ||
|
||
return "_".join(parts) | ||
|
||
def genYamlTree(self, build_or_test): | ||
|
||
env_dict = OrderedDict({ | ||
"BUILD_ENVIRONMENT": miniutils.quote(" ".join(self.genBuildEnvParms())), | ||
}) | ||
|
||
if self.libtorch_variant: | ||
env_dict["LIBTORCH_VARIANT"] = miniutils.quote(self.libtorch_variant) | ||
|
||
os_word_substitution = { | ||
"macos": "mac", | ||
} | ||
|
||
os_name = miniutils.override(self.os, os_word_substitution) | ||
|
||
d = { | ||
"environment": env_dict, | ||
"<<": "*" + "_".join([self.getNamePrefix(), os_name, build_or_test]), | ||
} | ||
|
||
if build_or_test == "test": | ||
tuples = [] | ||
if self.cuda_version: | ||
tuples.append(("USE_CUDA_DOCKER_RUNTIME", miniutils.quote("1"))) | ||
|
||
if not (self.smoke and self.os == "macos"): | ||
tuples.append(("DOCKER_IMAGE", self.genDockerImage())) | ||
|
||
if self.smoke: | ||
# TODO: Fix this discrepancy upstream | ||
tuples.reverse() | ||
|
||
for (k, v) in tuples: | ||
env_dict[k] = v | ||
|
||
else: | ||
if self.os == "linux" and build_or_test != "upload": | ||
d["docker"] = [{"image": self.genDockerImage()}] | ||
|
||
if build_or_test == "test": | ||
if self.cuda_version: | ||
d["resource_class"] = "gpu.medium" | ||
|
||
return d | ||
|
||
|
||
def gen_build_env_list(smoke): | ||
|
||
root = make_build_configs.TopLevelNode( | ||
"Builds", | ||
make_build_configs.CONFIG_TREE_DATA, | ||
smoke, | ||
) | ||
|
||
config_list, dot = conf_tree.dfs(root) | ||
|
||
newlist = [] | ||
for c in config_list: | ||
conf = Conf( | ||
c.find_prop("os_name"), | ||
c.find_prop("cu"), | ||
c.find_prop("package_format"), | ||
[c.find_prop("pyver")], | ||
c.find_prop("smoke"), | ||
c.find_prop("libtorch_variant") | ||
) | ||
newlist.append(conf) | ||
|
||
return newlist, dot | ||
|
||
|
||
def add_build_entries(jobs_dict, phase, smoke): | ||
|
||
configs, _ = gen_build_env_list(smoke) | ||
for conf_options in configs: | ||
jobs_dict[conf_options.genBuildName(phase)] = conf_options.genYamlTree(phase) | ||
|
||
|
||
def add_binary_build_specs(jobs_dict): | ||
add_build_entries(jobs_dict, "build", False) | ||
|
||
|
||
def add_binary_build_uploads(jobs_dict): | ||
add_build_entries(jobs_dict, "upload", False) | ||
|
||
|
||
def add_smoke_test_specs(jobs_dict): | ||
add_build_entries(jobs_dict, "test", True) | ||
|
||
|
||
def add_binary_build_tests(jobs_dict): | ||
|
||
def testable_binary_predicate(x): | ||
return x.os == "linux" and (x.smoke or x.pydistro != "libtorch") | ||
|
||
configs, _ = gen_build_env_list(False) | ||
filtered_configs = filter(testable_binary_predicate, configs) | ||
|
||
for conf_options in filtered_configs: | ||
jobs_dict[conf_options.genBuildName("test")] = conf_options.genYamlTree("test") | ||
|
||
|
||
def gen_schedule_tree(cron_timing): | ||
return [{ | ||
"schedule": { | ||
"cron": miniutils.quote(cron_timing), | ||
"filters": { | ||
"branches": { | ||
"only": ["master"], | ||
}, | ||
}, | ||
}, | ||
}] | ||
|
||
|
||
def add_jobs_and_render(jobs_dict, toplevel_key, smoke, cron_schedule): | ||
|
||
jobs_list = [] | ||
|
||
configs, graph = gen_build_env_list(smoke) | ||
for build_config in configs: | ||
build_name = build_config.genBuildName("build") | ||
jobs_list.append(build_name) | ||
|
||
d = OrderedDict( | ||
triggers=gen_schedule_tree(cron_schedule), | ||
jobs=jobs_list, | ||
) | ||
|
||
jobs_dict[toplevel_key] = d | ||
|
||
graph.draw(toplevel_key + "-config-dimensions.png", prog="twopi") | ||
|
||
|
||
def add_binary_build_jobs(jobs_dict): | ||
add_jobs_and_render(jobs_dict, "binarybuilds", False, "5 5 * * *") | ||
|
||
|
||
def add_binary_smoke_test_jobs(jobs_dict): | ||
add_jobs_and_render(jobs_dict, "binarysmoketests", True, "15 16 * * *") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,195 @@ | ||
from collections import OrderedDict | ||
|
||
import miniutils | ||
|
||
|
||
DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/" | ||
|
||
DEFAULT_DOCKER_VERSION = 282 | ||
|
||
|
||
class DockerHide: | ||
"""Hides element for construction of docker path""" | ||
def __init__(self, val): | ||
self.val = val | ||
|
||
def __str__(self): | ||
return self.val | ||
|
||
|
||
class Conf: | ||
def __init__(self, | ||
distro, | ||
parms, | ||
pyver=None, | ||
cuda_version=None, | ||
is_xla=False, | ||
restrict_phases=None, | ||
cuda_docker_phases=None, | ||
gpu_resource=None, | ||
docker_version_override=None): | ||
|
||
self.distro = distro | ||
self.pyver = pyver | ||
self.parms = parms | ||
|
||
self.cuda_version = cuda_version | ||
self.is_xla = is_xla | ||
self.restrict_phases = restrict_phases | ||
|
||
# FIXME does the build phase ever need CUDA runtime? | ||
self.cuda_docker_phases = cuda_docker_phases or [] | ||
|
||
self.gpu_resource = gpu_resource | ||
|
||
# FIXME is this different docker version intentional? | ||
self.docker_version_override = docker_version_override | ||
|
||
def getParms(self): | ||
leading = ["pytorch"] | ||
if self.is_xla: | ||
leading.append(DockerHide("xla")) | ||
|
||
cuda_parms = [] | ||
if self.cuda_version: | ||
cuda_parms.extend(["cuda" + self.cuda_version, "cudnn7"]) | ||
return leading + ["linux", self.distro] + cuda_parms + self.parms | ||
|
||
# TODO: Eliminate this special casing in docker paths | ||
def genDockerImagePath(self, build_or_test): | ||
|
||
build_env_pieces = self.getParms() | ||
build_env_pieces = list(filter(lambda x: type(x) is not DockerHide, build_env_pieces)) | ||
|
||
build_job_name_pieces = build_env_pieces + [build_or_test] | ||
|
||
base_build_env_name = "-".join(build_env_pieces) | ||
|
||
docker_version = self.docker_version_override or DEFAULT_DOCKER_VERSION | ||
|
||
return miniutils.quote(DOCKER_IMAGE_PATH_BASE + base_build_env_name + ":" + str(docker_version)) | ||
|
||
def getBuildJobNamePieces(self, build_or_test): | ||
return self.getParms() + [build_or_test] | ||
|
||
def genBuildName(self, build_or_test): | ||
return ("_".join(map(str, self.getBuildJobNamePieces(build_or_test)))).replace(".", "_") | ||
|
||
def genYamlTree(self, build_or_test): | ||
|
||
build_job_name_pieces = self.getBuildJobNamePieces(build_or_test) | ||
|
||
base_build_env_name = "-".join(map(str, self.getParms())) | ||
build_env_name = "-".join(map(str, build_job_name_pieces)) | ||
|
||
env_dict = { | ||
"BUILD_ENVIRONMENT": build_env_name, | ||
"DOCKER_IMAGE": self.genDockerImagePath(build_or_test), | ||
} | ||
|
||
if self.pyver: | ||
env_dict["PYTHON_VERSION"] = miniutils.quote(self.pyver) | ||
|
||
if build_or_test in self.cuda_docker_phases: | ||
env_dict["USE_CUDA_DOCKER_RUNTIME"] = miniutils.quote("1") | ||
|
||
d = { | ||
"environment": env_dict, | ||
"<<": "*" + "_".join(["pytorch", "linux", build_or_test, "defaults"]), | ||
} | ||
|
||
if build_or_test == "test": | ||
resource_class = "large" | ||
if self.gpu_resource: | ||
resource_class = "gpu." + self.gpu_resource | ||
|
||
if self.gpu_resource == "large": | ||
env_dict["MULTI_GPU"] = miniutils.quote("1") | ||
|
||
d["resource_class"] = resource_class | ||
|
||
return d | ||
|
||
|
||
BUILD_ENV_LIST = [ | ||
Conf("trusty", ["py2.7.9"]), | ||
Conf("trusty", ["py2.7"]), | ||
Conf("trusty", ["py3.5"]), | ||
Conf("trusty", ["py3.5"]), | ||
Conf("trusty", ["py3.6", "gcc4.8"]), | ||
Conf("trusty", ["py3.6", "gcc5.4"]), | ||
Conf("trusty", ["py3.6", "gcc5.4"], is_xla=True, docker_version_override=278), | ||
Conf("trusty", ["py3.6", "gcc7"]), | ||
Conf("trusty", ["pynightly"]), | ||
Conf("xenial", ["py3", "clang5", "asan"], pyver="3.6"), | ||
Conf("xenial", | ||
["py3"], | ||
pyver="3.6", | ||
cuda_version="8", | ||
gpu_resource="medium", | ||
cuda_docker_phases=["test"]), | ||
Conf("xenial", | ||
["py3", DockerHide("multigpu")], | ||
pyver="3.6", | ||
cuda_version="8", | ||
restrict_phases=["test"], | ||
cuda_docker_phases=["build", "test"], | ||
gpu_resource="large"), | ||
Conf("xenial", | ||
["py3", DockerHide("NO_AVX2")], | ||
pyver="3.6", | ||
cuda_version="8", | ||
restrict_phases=["test"], | ||
cuda_docker_phases=["build", "test"], | ||
gpu_resource="medium"), | ||
Conf("xenial", | ||
["py3", DockerHide("NO_AVX"), DockerHide("NO_AVX2")], | ||
pyver="3.6", | ||
cuda_version="8", | ||
restrict_phases=["test"], | ||
cuda_docker_phases=["build", "test"], | ||
gpu_resource="medium"), | ||
Conf("xenial", | ||
["py2"], | ||
pyver="2.7", | ||
cuda_version="9", | ||
cuda_docker_phases=["test"], | ||
gpu_resource="medium"), | ||
Conf("xenial", | ||
["py3"], | ||
pyver="3.6", | ||
cuda_version="9", | ||
gpu_resource="medium", | ||
cuda_docker_phases=["test"]), | ||
Conf("xenial", | ||
["py3", "gcc7"], | ||
pyver="3.6", | ||
cuda_version="9.2", | ||
gpu_resource="medium", | ||
cuda_docker_phases=["test"]), | ||
Conf("xenial", | ||
["py3", "gcc7"], | ||
pyver="3.6", | ||
cuda_version="10", | ||
restrict_phases=["build"]), | ||
] | ||
|
||
|
||
def add_build_env_defs(jobs_dict): | ||
|
||
mydict = OrderedDict() | ||
for conf_options in BUILD_ENV_LIST: | ||
|
||
def append_environment_dict(build_or_test): | ||
d = conf_options.genYamlTree(build_or_test) | ||
mydict[conf_options.genBuildName(build_or_test)] = d | ||
|
||
phases = ["build", "test"] | ||
if conf_options.restrict_phases: | ||
phases = conf_options.restrict_phases | ||
|
||
for phase in phases: | ||
append_environment_dict(phase) | ||
|
||
jobs_dict["version"] = 2 | ||
jobs_dict["jobs"] = mydict |
Oops, something went wrong.