8.2-Parallelization-on-multiple-GPUs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "!pip install -r https://raw.githubusercontent.com/datamllab/automl-in-action-notebooks/master/requirements.txt\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## 8.2.1 Data parallelism\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.datasets import mnist\n",
    "import autokeras as ak\n",
    "\n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "clf = ak.ImageClassifier(\n",
    "    overwrite=True, max_trials=1, distribution_strategy=tf.distribute.MirroredStrategy()\n",
    ")\n",
    "clf.fit(x_train, y_train, epochs=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import keras_tuner as kt\n",
    "\n",
    "\n",
    "def build_model(hp):\n",
    "    model = tf.keras.Sequential()\n",
    "    model.add(tf.keras.layers.Flatten())\n",
    "    model.add(\n",
    "        tf.keras.layers.Dense(\n",
    "            units=hp.Int(\"units\", min_value=32, max_value=512, step=32),\n",
    "            activation=\"relu\",\n",
    "        )\n",
    "    )\n",
    "    model.add(tf.keras.layers.Dense(10, activation=\"softmax\"))\n",
    "    model.compile(optimizer=\"adam\", loss=\"sparse_categorical_crossentropy\")\n",
    "    return model\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "tuner = kt.RandomSearch(\n",
    "    build_model,\n",
    "    objective=\"val_loss\",\n",
    "    max_trials=1,\n",
    "    directory=\"my_dir\",\n",
    "    distribution_strategy=tf.distribute.MirroredStrategy(),\n",
    "    project_name=\"helloworld\",\n",
    ")\n",
    "\n",
    "tuner.search(x_train, y_train, epochs=1, validation_data=(x_test, y_test))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## 8.2.3 Parallel tuning\n",
    "\n",
    "Note: The code below are only for the convenience of copying. It should be setup on your\n",
    "machines and run locally.\n",
    "\n",
    "```shell\n",
    "export KERASTUNER_TUNER_ID=\"chief\"\n",
    "export KERASTUNER_ORACLE_IP=\"127.0.0.1\"\n",
    "export KERASTUNER_ORACLE_PORT=\"8000\"\n",
    "python run_tuning.py\n",
    "```\n",
    "\n",
    "```shell\n",
    "export KERASTUNER_TUNER_ID=\"tuner0\"\n",
    "export KERASTUNER_ORACLE_IP=\"127.0.0.1\"\n",
    "export KERASTUNER_ORACLE_PORT=\"8000\"\n",
    "python run_tuning.py\n",
    "```\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.datasets import mnist\n",
    "import autokeras as ak\n",
    "import keras_tuner as kt\n",
    "\n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "\n",
    "\n",
    "def build_model(hp):\n",
    "    model = tf.keras.Sequential()\n",
    "    model.add(tf.keras.layers.Flatten())\n",
    "    model.add(\n",
    "        tf.keras.layers.Dense(\n",
    "            units=hp.Int(\"units\", min_value=32, max_value=512, step=32),\n",
    "            activation=\"relu\",\n",
    "        )\n",
    "    )\n",
    "    model.add(tf.keras.layers.Dense(10, activation=\"softmax\"))\n",
    "    model.compile(optimizer=\"adam\", loss=\"sparse_categorical_crossentropy\")\n",
    "    return model\n",
    "\n",
    "\n",
    "tuner = kt.RandomSearch(\n",
    "    build_model,\n",
    "    objective=\"val_loss\",\n",
    "    max_trials=1,\n",
    "    directory=\"result_dir\",\n",
    "    project_name=\"helloworld\",\n",
    ")\n",
    "\n",
    "tuner.search(x_train, y_train, epochs=1, validation_data=(x_test, y_test))\n"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "8.2-Parallelization-on-multiple-GPUs",
   "private_outputs": false,
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}