Merge remote-tracking branch 'upstream/master' into pr-chainer-transf…

…ormer
jennifer1995 · Apr 17, 2019 · 871e4e9 · 871e4e9
2 parents fb0e699 + 3bb0a25
commit 871e4e9
Show file tree

Hide file tree

Showing 3 changed files with 149 additions and 112 deletions.
diff --git a/README.md b/README.md
@@ -33,18 +33,21 @@ and also follows [Kaldi](http://kaldi-asr.org/) style data processing, feature e
 ## Requirements
 
 - Python 2.7+, 3.7+ (mainly support Python3.7+)
-- Cuda 8.0, 9.0, 9.1, 10.0 depending on each DNN library (for the use of GPU)
-- Cudnn 6+ (for the use of GPU)
-- NCCL 2.0+ (for the use of multi-GPUs)
 - protocol buffer (for the sentencepiece, you need to install via package manager e.g. `sudo apt-get install libprotobuf9v5 protobuf-compiler libprotobuf-dev`. See details `Installation` of https://github.com/google/sentencepiece/blob/master/README.md)
 
 - PyTorch 0.4.1, 1.0.0
 - gcc>=4.9 for PyTorch1.0.0
 - Chainer 5.0.0
 
+Optionally, GPU environment requires the following libraries:
+
+- Cuda 8.0, 9.0, 9.1, 10.0 depending on each DNN library
+- Cudnn 6+
+- NCCL 2.0+ (for the use of multi-GPUs)
+
 ## Installation
 
-### Step 1) setting of the environment
+### Step 1) setting of the environment for GPU support
 
 To use cuda (and cudnn), make sure to set paths in your `.bashrc` or `.bash_profile` appropriately.
 ```
@@ -111,13 +114,25 @@ $ cd tools
 $ make PYTHON_VERSION=3.6
 ```
 
+### Step 2-C) installation for CPU-only
+
+To install in a terminal that does not have a GPU installed, just clear the version of `CUPY` as follows:
+
+```sh
+$ cd tools
+$ make CUPY_VERSION='' -j 10 
+```
+
+This option is enabled for any of the install configuration. 
+
 ### Step 3) installation check
 
 You can check whether the install is succeeded via the following commands
 ```sh
 $ cd tools
 $ make check_install
 ```
+or `make check_install --no-cupy` if you do not have a GPU on your terminal. 
 If you have no warning, ready to run the recipe!
 
 If there are some problems in python libraries, you can re-setup only python environment via following commands

diff --git a/tools/Makefile b/tools/Makefile
@@ -18,7 +18,7 @@ GCC_VERSION := $(shell gcc -dumpversion)
 
 all: kaldi.done python check_install
 
-python: venv chainer_patch.done warp-ctc.done chainer_ctc.done
+python: venv cupy.done chainer_patch.done warp-ctc.done chainer_ctc.done
 
 extra: nkf.done sentencepiece.done mecab.done moses.done
 
@@ -48,7 +48,7 @@ espnet.done: venv
 	else \
 		. venv/bin/activate; pip install matplotlib; \
 	fi
-	. venv/bin/activate; pip install cupy==$(CUPY_VERSION) torch==$(TH_VERSION)
+	. venv/bin/activate; pip install torch==$(TH_VERSION)
 	touch espnet.done
 else
 miniconda.sh:
@@ -62,10 +62,15 @@ espnet.done: venv
 	. venv/bin/activate && conda install -y pytorch=$(TH_VERSION) -c pytorch
 	. venv/bin/activate && conda install -y matplotlib
 	. venv/bin/activate && pip install -e ..
-	. venv/bin/activate && pip install cupy==$(CUPY_VERSION)
 	touch espnet.done
 endif
 
+cupy.done: espnet.done
+ifneq ($(strip $(CUPY_VERSION)),)
+	. venv/bin/activate && pip install cupy==$(CUPY_VERSION)
+	touch cupy.done
+endif
+
 chainer_patch.done: espnet.done
 	$(eval FILENAME := $(shell find venv/lib -name "multiprocess_iterator.py"))
 	patch $(FILENAME) < prefetch.patch
@@ -139,7 +144,11 @@ moses.done:
 	touch moses.done
 
 check_install: kaldi.done python
+ifneq ($(strip $(CUPY_VERSION)),)
 	. venv/bin/activate; python check_install.py
+else
+	. venv/bin/activate; python check_install.py --no-cupy
+endif
 
 clean: clean_extra
 	rm -rf kaldi venv warp-ctc chainer_ctc

diff --git a/tools/check_install.py b/tools/check_install.py
@@ -8,110 +8,123 @@
 import logging
 import sys
 
-# you should add the libraries which are not included in setup.py
-MANUALLY_INSTALLED_LIBRARIES = [
-    ('espnet', None),
-    ('kaldiio', None),
-    ('matplotlib', None),
-    ('torch', ("0.4.1", "1.0.0")),
-    ('chainer', ("5.0.0")),
-    ('cupy', ("5.0.0")),
-    ('chainer_ctc', None),
-    ('warpctc_pytorch', ("0.1.1"))
-]
-
-logging.basicConfig(
-    level=logging.INFO,
-    format="%(levelname)s: %(message)s")
-
-logging.info("python version = " + sys.version)
-
-library_list = []
-library_list.extend(MANUALLY_INSTALLED_LIBRARIES)
-
-# check library availableness
-logging.info("library availableness check start.")
-logging.info("# libraries to be checked = %d" % len(library_list))
-is_correct_installed_list = []
-for idx, (name, version) in enumerate(library_list):
-    try:
-        importlib.import_module(name)
-        logging.info("--> %s is installed." % name)
-        is_correct_installed_list.append(True)
-    except ImportError:
-        logging.warning("--> %s is not installed." % name)
-        is_correct_installed_list.append(False)
-logging.info("library availableness check done.")
-logging.info("%d / %d libraries are correctly installed." % (
-    sum(is_correct_installed_list), len(library_list)))
-
-if len(library_list) != sum(is_correct_installed_list):
-    logging.info("please try to setup again and then re-run this script.")
-    sys.exit(1)
-
-# check library version
-num_version_specified = sum([True if v is not None else False for n, v in library_list])
-logging.info("library version check start.")
-logging.info("# libraries to be checked = %d" % num_version_specified)
-is_correct_version_list = []
-for idx, (name, version) in enumerate(library_list):
-    if version is not None:
-        lib = importlib.import_module(name)
-        if hasattr(lib, "__version__"):
-            is_correct = lib.__version__ in version
-            if is_correct:
-                logging.info("--> %s version is matched." % name)
-                is_correct_version_list.append(True)
+
+def main(args):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--no-cupy', action='store_true', default=False,
+                        help='Disable CUPY tests')
+    args = parser.parse_args(args)                
+
+    # you should add the libraries which are not included in setup.py
+    MANUALLY_INSTALLED_LIBRARIES = [
+        ('espnet', None),
+        ('kaldiio', None),
+        ('matplotlib', None),
+        ('torch', ("0.4.1", "1.0.0")),
+        ('chainer', ("5.0.0")),
+        ('chainer_ctc', None),
+        ('warpctc_pytorch', ("0.1.1"))
+    ]
+
+    if not args.no_cupy:
+        MANUALLY_INSTALLED_LIBRARIES.append(('cupy', ("5.0.0")))
+
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(levelname)s: %(message)s")
+
+    logging.info("python version = " + sys.version)
+
+    library_list = []
+    library_list.extend(MANUALLY_INSTALLED_LIBRARIES)
+
+    # check library availableness
+    logging.info("library availableness check start.")
+    logging.info("# libraries to be checked = %d" % len(library_list))
+    is_correct_installed_list = []
+    for idx, (name, version) in enumerate(library_list):
+        try:
+            importlib.import_module(name)
+            logging.info("--> %s is installed." % name)
+            is_correct_installed_list.append(True)
+        except ImportError:
+            logging.warning("--> %s is not installed." % name)
+            is_correct_installed_list.append(False)
+    logging.info("library availableness check done.")
+    logging.info("%d / %d libraries are correctly installed." % (
+        sum(is_correct_installed_list), len(library_list)))
+
+    if len(library_list) != sum(is_correct_installed_list):
+        logging.info("please try to setup again and then re-run this script.")
+        sys.exit(1)
+
+    # check library version
+    num_version_specified = sum([True if v is not None else False for n, v in library_list])
+    logging.info("library version check start.")
+    logging.info("# libraries to be checked = %d" % num_version_specified)
+    is_correct_version_list = []
+    for idx, (name, version) in enumerate(library_list):
+        if version is not None:
+            lib = importlib.import_module(name)
+            if hasattr(lib, "__version__"):
+                is_correct = lib.__version__ in version
+                if is_correct:
+                    logging.info("--> %s version is matched." % name)
+                    is_correct_version_list.append(True)
+                else:
+                    logging.warning("--> %s version is not matched (%s is not in %s)." % (
+                        name, lib.__version__, str(version)))
+                    is_correct_version_list.append(False)
             else:
-                logging.warning("--> %s version is not matched (%s is not in %s)." % (
-                    name, lib.__version__, str(version)))
+                logging.info("--> %s has no version info, but version is specified." % name)
+                logging.info("--> maybe it is better to reinstall the latest version.")
                 is_correct_version_list.append(False)
-        else:
-            logging.info("--> %s has no version info, but version is specified." % name)
-            logging.info("--> maybe it is better to reinstall the latest version.")
-            is_correct_version_list.append(False)
-logging.info("library version check done.")
-logging.info("%d / %d libraries are correct version." % (
-    sum(is_correct_version_list), num_version_specified))
-
-if sum(is_correct_version_list) != num_version_specified:
-    logging.info("please try to setup again and then re-run this script.")
-    sys.exit(1)
-
-# check cuda availableness
-logging.info("cuda availableness check start.")
-import chainer
-import torch
-try:
-    assert torch.cuda.is_available()
-    logging.info("--> cuda is available in torch.")
-except AssertionError:
-    logging.warning("--> it seems that cuda is not available in torch.")
-try:
-    assert torch.backends.cudnn.is_available()
-    logging.info("--> cudnn is available in torch.")
-except AssertionError:
-    logging.warning("--> it seems that cudnn is not available in torch.")
-try:
-    assert chainer.backends.cuda.available
-    logging.info("--> cuda is available in chainer.")
-except AssertionError:
-    logging.warning("--> it seems that cuda is not available in chainer.")
-try:
-    assert chainer.backends.cuda.cudnn_enabled
-    logging.info("--> cudnn is available in chainer.")
-except AssertionError:
-    logging.warning("--> it seems that cudnn is not available in chainer.")
-try:
-    from cupy.cuda import nccl  # NOQA
-    logging.info("--> nccl is installed.")
-except ImportError:
-    logging.warning("--> it seems that nccl is not installed. multi-gpu is not enabled.")
-    logging.warning("--> if you want to use multi-gpu, please install it and then re-setup.")
-try:
-    assert torch.cuda.device_count() > 1
-    logging.info("--> multi-gpu is available (#gpus = %d)." % torch.cuda.device_count())
-except AssertionError:
-    logging.warning("--> it seems that only single gpu is available.")
-    logging.warning('--> maybe your machine has only one gpu.')
-logging.info("cuda availableness check done.")
+    logging.info("library version check done.")
+    logging.info("%d / %d libraries are correct version." % (
+        sum(is_correct_version_list), num_version_specified))
+
+    if sum(is_correct_version_list) != num_version_specified:
+        logging.info("please try to setup again and then re-run this script.")
+        sys.exit(1)
+
+    # check cuda availableness
+    logging.info("cuda availableness check start.")
+    import chainer
+    import torch
+    try:
+        assert torch.cuda.is_available()
+        logging.info("--> cuda is available in torch.")
+    except AssertionError:
+        logging.warning("--> it seems that cuda is not available in torch.")
+    try:
+        assert torch.backends.cudnn.is_available()
+        logging.info("--> cudnn is available in torch.")
+    except AssertionError:
+        logging.warning("--> it seems that cudnn is not available in torch.")
+    try:
+        assert chainer.backends.cuda.available
+        logging.info("--> cuda is available in chainer.")
+    except AssertionError:
+        logging.warning("--> it seems that cuda is not available in chainer.")
+    try:
+        assert chainer.backends.cuda.cudnn_enabled
+        logging.info("--> cudnn is available in chainer.")
+    except AssertionError:
+        logging.warning("--> it seems that cudnn is not available in chainer.")
+    try:
+        from cupy.cuda import nccl  # NOQA
+        logging.info("--> nccl is installed.")
+    except ImportError:
+        logging.warning("--> it seems that nccl is not installed. multi-gpu is not enabled.")
+        logging.warning("--> if you want to use multi-gpu, please install it and then re-setup.")
+    try:
+        assert torch.cuda.device_count() > 1
+        logging.info("--> multi-gpu is available (#gpus = %d)." % torch.cuda.device_count())
+    except AssertionError:
+        logging.warning("--> it seems that only single gpu is available.")
+        logging.warning('--> maybe your machine has only one gpu.')
+    logging.info("cuda availableness check done.")
+
+
+if __name__ == '__main__':
+    main(sys.argv[1:])