notebooks/train_local.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Yd5gaL2lI0u_",
    "outputId": "529295e3-1529-4218-85e7-3c3248c0564b"
   },
   "outputs": [],
   "source": [
    "# Import libraries and modules\n",
    "import datetime\n",
    "import math\n",
    "import os\n",
    "import sys\n",
    "import tensorflow as tf\n",
    "print(tf.__version__)\n",
    "\n",
    "module_path = os.path.abspath(os.path.join(\"..\"))\n",
    "if module_path not in sys.path:\n",
    "    sys.path.append(module_path)\n",
    "\n",
    "from proganomaly_modules.training_module.trainer import gan_layer_architecture_shapes\n",
    "from proganomaly_modules.training_module.trainer import image_masks\n",
    "from proganomaly_modules.training_module.trainer import model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "U_673BVaI0vF"
   },
   "source": [
    "# Local Development"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "0V91zzvfI0vG"
   },
   "source": [
    "## Arguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generator hyperparameters.\n",
    "generator_dict = dict()\n",
    "# Which paper to use for generator architecture: \"berg\", \"GANomaly\".\n",
    "generator_dict[\"architecture\"] = \"GANomaly\"\n",
    "\n",
    "# Whether generator will be trained or not.\n",
    "generator_dict[\"train\"] = True\n",
    "# Number of steps to train generator for per cycle.\n",
    "generator_dict[\"train_steps\"] = 1\n",
    "\n",
    "# The latent size of the berg input noise vector or the GANomaly generator's\n",
    "# encoder logits vector.\n",
    "generator_dict[\"latent_size\"] = 512\n",
    "# Whether to normalize latent vector before projection.\n",
    "generator_dict[\"normalize_latents\"] = True\n",
    "# Whether to use pixel norm op after each convolution.\n",
    "generator_dict[\"use_pixel_norm\"] = True\n",
    "# Small value to add to denominator for numerical stability.\n",
    "generator_dict[\"pixel_norm_epsilon\"] = 1e-8\n",
    "# The 3D dimensions to project latent noise vector into.\n",
    "generator_dict[\"projection_dims\"] = [4, 4, 512]\n",
    "# The amount of leakyness of generator's leaky relus.\n",
    "generator_dict[\"leaky_relu_alpha\"] = 0.2\n",
    "# The final activation function of generator: None, sigmoid, tanh, relu.\n",
    "generator_dict[\"final_activation\"] = \"None\"\n",
    "# Whether to add uniform noise to fake images.\n",
    "generator_dict[\"add_uniform_noise_to_fake_images\"] = True\n",
    "# Scale factor for L1 regularization for generator.\n",
    "generator_dict[\"l1_regularization_scale\"] = 0.\n",
    "# Scale factor for L2 regularization for generator.\n",
    "generator_dict[\"l2_regularization_scale\"] = 0.\n",
    "# Name of optimizer to use for generator.\n",
    "generator_dict[\"optimizer\"] = \"Adam\"\n",
    "# How quickly we train model by scaling the gradient for generator.\n",
    "generator_dict[\"learning_rate\"] = 0.001\n",
    "# Adam optimizer's beta1 hyperparameter for first moment.\n",
    "generator_dict[\"adam_beta1\"] = 0.0\n",
    "# Adam optimizer's beta2 hyperparameter for second moment.\n",
    "generator_dict[\"adam_beta2\"] = 0.99\n",
    "# Adam optimizer's epsilon hyperparameter for numerical stability.\n",
    "generator_dict[\"adam_epsilon\"] = 1e-8\n",
    "# Global clipping to prevent gradient norm to exceed this value for generator.\n",
    "generator_dict[\"clip_gradients\"] = None\n",
    "\n",
    "generator_berg_dict = dict()\n",
    "generator_ganomaly_dict = dict()\n",
    "\n",
    "generator_berg_losses_dict = dict()\n",
    "generator_ganomaly_losses_dict = dict()\n",
    "if generator_dict[\"architecture\"] == \"berg\":\n",
    "    # The latent vector's random normal mean.\n",
    "    generator_berg_dict[\"latent_mean\"] = 0.0\n",
    "    # The latent vector's random normal standard deviation.\n",
    "    generator_berg_dict[\"latent_stddev\"] = 1.0\n",
    "\n",
    "    # These are just example values, yours will vary.\n",
    "    # Weights to multiply loss of D(G(z))\n",
    "    generator_berg_losses_dict[\"D_of_G_of_z_loss_weight\"] = 1.0\n",
    "\n",
    "    # Weights to multiply loss of D(G(E(x)))\n",
    "    generator_berg_losses_dict[\"D_of_G_of_E_of_x_loss_weight\"] = 0.0\n",
    "    # Weights to multiply loss of D(G(E(G(z)))\n",
    "    generator_berg_losses_dict[\"D_of_G_of_E_of_G_of_z_loss_weight\"] = 0.0\n",
    "\n",
    "    # Weights to multiply loss of z - E(G(z))\n",
    "    generator_berg_losses_dict[\"z_minus_E_of_G_of_z_l1_loss_weight\"] = 0.0\n",
    "    generator_berg_losses_dict[\"z_minus_E_of_G_of_z_l2_loss_weight\"] = 0.0\n",
    "    # Weights to multiply loss of G(z) - G(E(G(z))\n",
    "    generator_berg_losses_dict[\"G_of_z_minus_G_of_E_of_G_of_z_l1_loss_weight\"] = 0.0\n",
    "    generator_berg_losses_dict[\"G_of_z_minus_G_of_E_of_G_of_z_l2_loss_weight\"] = 0.0\n",
    "    # Weights to multiply loss of E(x) - E(G(E(x)))\n",
    "    generator_berg_losses_dict[\"E_of_x_minus_E_of_G_of_E_of_x_l1_loss_weight\"] = 1.0\n",
    "    generator_berg_losses_dict[\"E_of_x_minus_E_of_G_of_E_of_x_l2_loss_weight\"] = 0.0\n",
    "    # Weights to multiply loss of x - G(E(x))\n",
    "    generator_berg_losses_dict[\"x_minus_G_of_E_of_x_l1_loss_weight\"] = 0.0\n",
    "    generator_berg_losses_dict[\"x_minus_G_of_E_of_x_l2_loss_weight\"] = 0.0\n",
    "\n",
    "    # GANomaly parameters to zero.\n",
    "    # Weights to multiply loss of D(G(x))\n",
    "    generator_ganomaly_losses_dict[\"D_of_G_of_x_loss_weight\"] = 0.0\n",
    "\n",
    "    # Weights to multiply loss of x - G(x)\n",
    "    generator_ganomaly_losses_dict[\"x_minus_G_of_x_l1_loss_weight\"] = 0.0\n",
    "    generator_ganomaly_losses_dict[\"x_minus_G_of_x_l2_loss_weight\"] = 0.0\n",
    "\n",
    "    # Weights to multiply loss of Ge(x) - E(G(x))\n",
    "    generator_ganomaly_losses_dict[\"Ge_of_x_minus_E_of_G_of_x_l1_loss_weight\"] = 0.0\n",
    "    generator_ganomaly_losses_dict[\"Ge_of_x_minus_E_of_G_of_x_l2_loss_weight\"] = 0.0\n",
    "else:  # GANomaly\n",
    "    # Whether generator GANomaly architecture uses U-net skip connection for each block.\n",
    "    generator_ganomaly_dict[\"use_unet_skip_connections\"] = [True] * 9\n",
    "\n",
    "    # Percent of masking image inputs to generator.\n",
    "    generator_ganomaly_dict[\"mask_generator_input_images_percent\"] = 0.2\n",
    "    # Integer amount to randomly shift image mask block sizes.\n",
    "    generator_ganomaly_dict[\"image_mask_block_random_shift_amount\"] = 0\n",
    "    # Whether to use shuffle or dead image block masking.\n",
    "    generator_ganomaly_dict[\"use_shuffle_image_masks\"] = True\n",
    "    # Whether to add uniform noise to GANomaly Z vector.\n",
    "    generator_ganomaly_dict[\"add_uniform_noise_to_z\"] = True\n",
    "\n",
    "    # These are just example values, yours will vary.\n",
    "    # Weights to multiply loss of D(G(x))\n",
    "    generator_ganomaly_losses_dict[\"D_of_G_of_x_loss_weight\"] = 1.0\n",
    "\n",
    "    # Weights to multiply loss of x - G(x)\n",
    "    generator_ganomaly_losses_dict[\"x_minus_G_of_x_l1_loss_weight\"] = 0.0\n",
    "    generator_ganomaly_losses_dict[\"x_minus_G_of_x_l2_loss_weight\"] = 1.0\n",
    "\n",
    "    # Weights to multiply loss of Ge(x) - E(G(x))\n",
    "    generator_ganomaly_losses_dict[\"Ge_of_x_minus_E_of_G_of_x_l1_loss_weight\"] = 0.0\n",
    "    generator_ganomaly_losses_dict[\"Ge_of_x_minus_E_of_G_of_x_l2_loss_weight\"] = 0.0\n",
    "\n",
    "    # Berg parameters to zero.\n",
    "    # Weights to multiply loss of D(G(z))\n",
    "    generator_berg_losses_dict[\"D_of_G_of_z_loss_weight\"] = 0.0\n",
    "\n",
    "    # Weights to multiply loss of D(G(E(x)))\n",
    "    generator_berg_losses_dict[\"D_of_G_of_E_of_x_loss_weight\"] = 0.0\n",
    "    # Weights to multiply loss of D(G(E(G(z)))\n",
    "    generator_berg_losses_dict[\"D_of_G_of_E_of_G_of_z_loss_weight\"] = 0.0\n",
    "\n",
    "    # Weights to multiply loss of z - E(G(z))\n",
    "    generator_berg_losses_dict[\"z_minus_E_of_G_of_z_l1_loss_weight\"] = 0.0\n",
    "    generator_berg_losses_dict[\"z_minus_E_of_G_of_z_l2_loss_weight\"] = 0.0\n",
    "    # Weights to multiply loss of G(z) - G(E(G(z))\n",
    "    generator_berg_losses_dict[\"G_of_z_minus_G_of_E_of_G_of_z_l1_loss_weight\"] = 0.0\n",
    "    generator_berg_losses_dict[\"G_of_z_minus_G_of_E_of_G_of_z_l2_loss_weight\"] = 0.0\n",
    "    # Weights to multiply loss of E(x) - E(G(E(x)))\n",
    "    generator_berg_losses_dict[\"E_of_x_minus_E_of_G_of_E_of_x_l1_loss_weight\"] = 0.0\n",
    "    generator_berg_losses_dict[\"E_of_x_minus_E_of_G_of_E_of_x_l2_loss_weight\"] = 0.0\n",
    "    # Weights to multiply loss of x - G(E(x))\n",
    "    generator_berg_losses_dict[\"x_minus_G_of_E_of_x_l1_loss_weight\"] = 0.0\n",
    "    generator_berg_losses_dict[\"x_minus_G_of_E_of_x_l2_loss_weight\"] = 0.0\n",
    "\n",
    "generator_dict[\"berg\"] = generator_berg_dict\n",
    "generator_dict[\"GANomaly\"] = generator_ganomaly_dict\n",
    "\n",
    "generator_dict[\"losses\"] = {}\n",
    "generator_dict[\"losses\"][\"berg\"] = generator_berg_losses_dict\n",
    "generator_dict[\"losses\"][\"GANomaly\"] = generator_ganomaly_losses_dict\n",
    "generator_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Encoder hyperparameters.\n",
    "encoder_dict = dict()\n",
    "# These are optional if using GANomaly architecture, required for berg.\n",
    "# Whether encoder will be created or not.\n",
    "encoder_dict[\"create\"] = False\n",
    "# Whether encoder will be trained or not.\n",
    "encoder_dict[\"train\"] = False\n",
    "\n",
    "# Whether to use minibatch stddev op before first base conv layer.\n",
    "encoder_dict[\"use_minibatch_stddev\"] = True\n",
    "# The size of groups to split minibatch examples into.\n",
    "encoder_dict[\"minibatch_stddev_group_size\"] = 4\n",
    "# Whether to average across feature maps and pixels for minibatch stddev.\n",
    "encoder_dict[\"minibatch_stddev_use_averaging\"] = True\n",
    "# The amount of leakyness of encoder's leaky relus.\n",
    "encoder_dict[\"leaky_relu_alpha\"] = 0.2\n",
    "# Scale factor for L1 regularization for encoder.\n",
    "encoder_dict[\"l1_regularization_scale\"] = 0.\n",
    "# Scale factor for L2 regularization for encoder.\n",
    "encoder_dict[\"l2_regularization_scale\"] = 0.\n",
    "# Name of optimizer to use for encoder.\n",
    "encoder_dict[\"optimizer\"] = \"Adam\"\n",
    "# How quickly we train model by scaling the gradient for encoder.\n",
    "encoder_dict[\"learning_rate\"] = 0.001\n",
    "# Adam optimizer's beta1 hyperparameter for first moment.\n",
    "encoder_dict[\"adam_beta1\"] = 0.0\n",
    "# Adam optimizer's beta2 hyperparameter for second moment.\n",
    "encoder_dict[\"adam_beta2\"] = 0.99\n",
    "# Adam optimizer's epsilon hyperparameter for numerical stability.\n",
    "encoder_dict[\"adam_epsilon\"] = 1e-8\n",
    "# Global clipping to prevent gradient norm to exceed this value for encoder.\n",
    "encoder_dict[\"clip_gradients\"] = None\n",
    "\n",
    "encoder_losses_dict = dict()\n",
    "# Berg Losses\n",
    "encoder_losses_berg_dict = dict()\n",
    "# Weights to multiply loss of D(G(E(x)))\n",
    "encoder_losses_berg_dict[\"D_of_G_of_E_of_x_loss_weight\"] = 0.0\n",
    "# Weights to multiply loss of D(G(E(G(z)))\n",
    "encoder_losses_berg_dict[\"D_of_G_of_E_of_G_of_z_loss_weight\"] = 0.0\n",
    "\n",
    "# Weights to multiply loss of z - E(G(z))\n",
    "encoder_losses_berg_dict[\"z_minus_E_of_G_of_z_l1_loss_weight\"] = 0.0\n",
    "encoder_losses_berg_dict[\"z_minus_E_of_G_of_z_l2_loss_weight\"] = 0.0\n",
    "# Weights to multiply loss of G(z) - G(E(G(z))\n",
    "encoder_losses_berg_dict[\"G_of_z_minus_G_of_E_of_G_of_z_l1_loss_weight\"] = 0.0\n",
    "encoder_losses_berg_dict[\"G_of_z_minus_G_of_E_of_G_of_z_l2_loss_weight\"] = 0.0\n",
    "# Weights to multiply loss of E(x) - E(G(E(x)))\n",
    "encoder_losses_berg_dict[\"E_of_x_minus_E_of_G_of_E_of_x_l1_loss_weight\"] = 0.0\n",
    "encoder_losses_berg_dict[\"E_of_x_minus_E_of_G_of_E_of_x_l2_loss_weight\"] = 0.0\n",
    "# Weights to multiply loss of x - G(E(x))\n",
    "encoder_losses_berg_dict[\"x_minus_G_of_E_of_x_l1_loss_weight\"] = 0.0\n",
    "encoder_losses_berg_dict[\"x_minus_G_of_E_of_x_l2_loss_weight\"] = 0.0\n",
    "\n",
    "# GANomaly Losses\n",
    "encoder_losses_ganomaly_dict = dict()\n",
    "# Weights to multiply loss of Ge(x) - E(G(x))\n",
    "encoder_losses_ganomaly_dict[\"Ge_of_x_minus_E_of_G_of_x_l1_loss_weight\"] = 0.0\n",
    "encoder_losses_ganomaly_dict[\"Ge_of_x_minus_E_of_G_of_x_l2_loss_weight\"] = 1.0\n",
    "\n",
    "encoder_losses_dict[\"berg\"] = encoder_losses_berg_dict\n",
    "encoder_losses_dict[\"GANomaly\"] = encoder_losses_ganomaly_dict\n",
    "encoder_dict[\"losses\"] = encoder_losses_dict\n",
    "\n",
    "encoder_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Discriminator hyperparameters.\n",
    "discriminator_dict = dict()\n",
    "# Whether discriminator will be created or not.\n",
    "discriminator_dict[\"create\"] = True\n",
    "# Whether discriminator will be trained or not.\n",
    "discriminator_dict[\"train\"] = True\n",
    "# Number of steps to train discriminator for per cycle.\n",
    "discriminator_dict[\"train_steps\"] = 1\n",
    "\n",
    "# Whether to use minibatch stddev op before first base conv layer.\n",
    "discriminator_dict[\"use_minibatch_stddev\"] = True\n",
    "# The size of groups to split minibatch examples into.\n",
    "discriminator_dict[\"minibatch_stddev_group_size\"] = 4\n",
    "# Whether to average across feature maps and pixels for minibatch stddev.\n",
    "discriminator_dict[\"minibatch_stddev_use_averaging\"] = True\n",
    "# The amount of leakyness of discriminator's leaky relus.\n",
    "discriminator_dict[\"leaky_relu_alpha\"] = 0.2\n",
    "# Scale factor for L1 regularization for discriminator.\n",
    "discriminator_dict[\"l1_regularization_scale\"] = 0.\n",
    "# Scale factor for L2 regularization for discriminator.\n",
    "discriminator_dict[\"l2_regularization_scale\"] = 0.\n",
    "# Name of optimizer to use for discriminator.\n",
    "discriminator_dict[\"optimizer\"] = \"Adam\"\n",
    "# How quickly we train model by scaling the gradient for discriminator.\n",
    "discriminator_dict[\"learning_rate\"] = 0.001\n",
    "# Adam optimizer's beta1 hyperparameter for first moment.\n",
    "discriminator_dict[\"adam_beta1\"] = 0.0\n",
    "# Adam optimizer's beta2 hyperparameter for second moment.\n",
    "discriminator_dict[\"adam_beta2\"] = 0.99\n",
    "# Adam optimizer's epsilon hyperparameter for numerical stability.\n",
    "discriminator_dict[\"adam_epsilon\"] = 1e-8\n",
    "# Global clipping to prevent gradient norm to exceed this value for discriminator.\n",
    "discriminator_dict[\"clip_gradients\"] = None\n",
    "# Coefficient of gradient penalty for discriminator.\n",
    "discriminator_dict[\"gradient_penalty_coefficient\"] = 10.0\n",
    "# Target value of gradient magnitudes for gradient penalty for discriminator.\n",
    "discriminator_dict[\"gradient_penalty_target\"] = 1.0\n",
    "# Coefficient of epsilon drift penalty for discriminator.\n",
    "discriminator_dict[\"epsilon_drift\"] = 0.001\n",
    "\n",
    "# Losses\n",
    "discriminator_losses_dict = dict()\n",
    "# Weight to multiply loss of D(x)\n",
    "discriminator_losses_dict[\"D_of_x_loss_weight\"] = 1.0\n",
    "\n",
    "# Berg Losses\n",
    "discriminator_losses_berg_dict = dict()\n",
    "# Weight to multiply loss of D(G(z))\n",
    "discriminator_losses_berg_dict[\"D_of_G_of_z_loss_weight\"] = 0.0\n",
    "# Weight to multiply loss of D(G(E(x)))\n",
    "discriminator_losses_berg_dict[\"D_of_G_of_E_of_x_loss_weight\"] = 0.0\n",
    "# Weight to multiply loss of D(G(E(G(z)))\n",
    "discriminator_losses_berg_dict[\"D_of_G_of_E_of_G_of_z_loss_weight\"] = 0.0\n",
    "\n",
    "# GANomaly Losses\n",
    "discriminator_losses_ganomaly_dict = dict()\n",
    "# Weight to multiply loss of D(G(x))\n",
    "discriminator_losses_ganomaly_dict[\"D_of_G_of_x_loss_weight\"] = 1.0\n",
    "\n",
    "discriminator_losses_dict[\"berg\"] = discriminator_losses_berg_dict\n",
    "discriminator_losses_dict[\"GANomaly\"] = discriminator_losses_ganomaly_dict\n",
    "discriminator_dict[\"losses\"] = discriminator_losses_dict\n",
    "\n",
    "discriminator_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reconstruction training parameters.\n",
    "reconstruction_dict = dict()\n",
    "# Whether using multiple resolutions across a list of TF Records.\n",
    "reconstruction_dict[\"use_multiple_resolution_records\"] = True\n",
    "# GCS locations to read reconstruction training data.\n",
    "reconstruction_dict[\"train_file_patterns\"] = [\n",
    "    tf.io.gfile.glob(\n",
    "        pattern=\"gs://.../data/*/*.svs.{}.*.tfrecords\".format(i)\n",
    "    )[0:150]\n",
    "    for i in range(9 - 1, -1, -1)\n",
    "]\n",
    "# GCS locations to read reconstruction evaluation data.\n",
    "reconstruction_dict[\"eval_file_patterns\"] = [\n",
    "    tf.io.gfile.glob(\n",
    "        pattern=\"gs://.../data/*/*.svs.{}.*.tfrecords\".format(i)\n",
    "    )[0:150]\n",
    "    for i in range(9 - 1, -1, -1)\n",
    "]\n",
    "# Which dataset to use for reconstruction training:\n",
    "# \"mnist\", \"cifar10\", \"cifar10_car\", \"tf_record\"\n",
    "reconstruction_dict[\"dataset\"] = \"tf_record\"\n",
    "# TF Record Example feature schema for reconstruction.\n",
    "reconstruction_dict[\"tf_record_example_schema\"] = [\n",
    "    {\n",
    "        \"name\": \"image/encoded\",\n",
    "        \"type\": \"FixedLen\",\n",
    "        \"shape\": [],\n",
    "        \"dtype\": \"str\"\n",
    "    },\n",
    "    {\n",
    "        \"name\": \"image/name\",\n",
    "        \"type\": \"FixedLen\",\n",
    "        \"shape\": [],\n",
    "        \"dtype\": \"str\"\n",
    "    }\n",
    "]\n",
    "# Name of image feature within schema dictionary.\n",
    "reconstruction_dict[\"image_feature_name\"] = \"image/encoded\"\n",
    "# Encoding of image: raw, png, or jpeg.\n",
    "reconstruction_dict[\"image_encoding\"] = \"png\"\n",
    "# Height of predownscaled image if NOT using multiple resolution records.\n",
    "reconstruction_dict[\"image_predownscaled_height\"] = 1024\n",
    "# Width of predownscaled image if NOT using multiple resolution records.\n",
    "reconstruction_dict[\"image_predownscaled_width\"] = 1024\n",
    "# Depth of image, number of channels.\n",
    "reconstruction_dict[\"image_depth\"] = 3\n",
    "# Name of label feature within schema dictionary.\n",
    "reconstruction_dict[\"label_feature_name\"] = \"\"\n",
    "# Schedule list of number of epochs to train for reconstruction.\n",
    "reconstruction_dict[\"num_epochs_schedule\"] = [1] * 9\n",
    "# Number of examples in one epoch of reconstruction training set.\n",
    "reconstruction_dict[\"train_dataset_length\"] = 330415\n",
    "# Schedule list of number of examples in reconstruction training batch for each resolution block.\n",
    "reconstruction_dict[\"train_batch_size_schedule\"] = [32] + [16] * 4 + [4] + [2] * 2 + [1]\n",
    "# Schedule list of number of examples in reconstruction evaluation batch for each resolution block.\n",
    "reconstruction_dict[\"eval_batch_size_schedule\"] = [32] + [16] * 4 + [4] + [2] * 2 + [1]\n",
    "# Number of steps/batches to evaluate for reconstruction.\n",
    "reconstruction_dict[\"eval_steps\"] = 1\n",
    "# List of number of examples until block added to networks.\n",
    "reconstruction_dict[\"num_examples_until_growth_schedule\"] = [\n",
    "    epochs * reconstruction_dict[\"train_dataset_length\"]\n",
    "    for epochs in reconstruction_dict[\"num_epochs_schedule\"]\n",
    "]\n",
    "# List of number of steps/batches until block added to networks.\n",
    "reconstruction_dict[\"num_steps_until_growth_schedule\"] = [\n",
    "    ex // bs\n",
    "    for ex, bs in zip(\n",
    "        reconstruction_dict[\"num_examples_until_growth_schedule\"],\n",
    "        reconstruction_dict[\"train_batch_size_schedule\"]\n",
    "    )\n",
    "]\n",
    "# Whether to autotune input function performance for reconstruction datasets.\n",
    "reconstruction_dict[\"input_fn_autotune\"] = True\n",
    "# How many steps to train before writing steps and loss to log.\n",
    "reconstruction_dict[\"log_step_count_steps\"] = 100\n",
    "# How many steps to train before saving a summary.\n",
    "reconstruction_dict[\"save_summary_steps\"] = 100\n",
    "# Whether to write loss summaries for TensorBoard.\n",
    "reconstruction_dict[\"write_loss_summaries\"] = False\n",
    "# Whether to write generator image summaries for TensorBoard.\n",
    "reconstruction_dict[\"write_generator_image_summaries\"] = False\n",
    "# Whether to write encoder image summaries for TensorBoard.\n",
    "reconstruction_dict[\"write_encoder_image_summaries\"] = False\n",
    "# Whether to write variable histogram summaries for TensorBoard.\n",
    "reconstruction_dict[\"write_variable_histogram_summaries\"] = False\n",
    "# Whether to write gradient histogram summaries for TensorBoard.\n",
    "reconstruction_dict[\"write_gradient_histogram_summaries\"] = False\n",
    "# How many steps to train reconstruction before saving a checkpoint.\n",
    "reconstruction_dict[\"save_checkpoints_steps\"] = 10000\n",
    "# Max number of reconstruction checkpoints to keep.\n",
    "reconstruction_dict[\"keep_checkpoint_max\"] = 100\n",
    "# Whether to save checkpoint every growth phase.\n",
    "reconstruction_dict[\"checkpoint_every_growth_phase\"] = True\n",
    "# Whether to save checkpoint every epoch.\n",
    "reconstruction_dict[\"checkpoint_every_epoch\"] = True\n",
    "# Checkpoint growth index to restore checkpoint.\n",
    "reconstruction_dict[\"checkpoint_growth_idx\"] = 0\n",
    "# Checkpoint epoch index to restore checkpoint.\n",
    "reconstruction_dict[\"checkpoint_epoch_idx\"] = 0\n",
    "# The checkpoint save path for saving and restoring.\n",
    "reconstruction_dict[\"checkpoint_save_path\"] = \"\"\n",
    "# Whether to store loss logs.\n",
    "reconstruction_dict[\"store_loss_logs\"] = True\n",
    "# Whether to normalize loss logs.\n",
    "reconstruction_dict[\"normalized_loss_logs\"] = True\n",
    "# Whether to print model summaries.\n",
    "reconstruction_dict[\"print_training_model_summaries\"] = False\n",
    "# Initial growth index to resume training midway.\n",
    "reconstruction_dict[\"initial_growth_idx\"] = 0\n",
    "# Initial epoch index to resume training midway.\n",
    "reconstruction_dict[\"initial_epoch_idx\"] = 0\n",
    "# Max number of times training loop can be restarted such as for NaN losses.\n",
    "reconstruction_dict[\"max_training_loop_restarts\"] = 20\n",
    "\n",
    "# Whether to scale layer weights to equalize learning rate each forward pass.\n",
    "reconstruction_dict[\"use_equalized_learning_rate\"] = True\n",
    "# Whether to normalize reconstruction losses by number of pixels.\n",
    "reconstruction_dict[\"normalize_reconstruction_losses\"] = True\n",
    "\n",
    "reconstruction_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Error distribution training parameters.\n",
    "error_distribution_dict = dict()\n",
    "# Whether using multiple resolutions across a list of TF Records.\n",
    "error_distribution_dict[\"use_multiple_resolution_records\"] = False\n",
    "# GCS locations to read error distribution training data.\n",
    "error_distribution_dict[\"train_file_pattern\"] = tf.io.gfile.glob(\n",
    "    pattern=\"gs://.../data/*/*.svs.{}.*.tfrecords\".format(0)\n",
    ")[150:175]\n",
    "# GCS locations to read error distribution training data.\n",
    "error_distribution_dict[\"eval_file_pattern\"] = tf.io.gfile.glob(\n",
    "    pattern=\"gs://.../data/*/*.svs.{}.*.tfrecords\".format(0)\n",
    ")[150:175]\n",
    "# Which dataset to use for error distribution training:\n",
    "# \"mnist\", \"cifar10\", \"cifar10_car\", \"tf_record\"\n",
    "error_distribution_dict[\"dataset\"] = \"tf_record\"\n",
    "# TF Record Example feature schema for error distribution.\n",
    "error_distribution_dict[\"tf_record_example_schema\"] = [\n",
    "    {\n",
    "        \"name\": \"image/encoded\",\n",
    "        \"type\": \"FixedLen\",\n",
    "        \"shape\": [],\n",
    "        \"dtype\": \"str\"\n",
    "    },\n",
    "    {\n",
    "        \"name\": \"image/name\",\n",
    "        \"type\": \"FixedLen\",\n",
    "        \"shape\": [],\n",
    "        \"dtype\": \"str\"\n",
    "    }\n",
    "]\n",
    "# Name of image feature within schema dictionary.\n",
    "error_distribution_dict[\"image_feature_name\"] = \"image/encoded\"\n",
    "# Encoding of image: raw, png, or jpeg.\n",
    "error_distribution_dict[\"image_encoding\"] = \"png\"\n",
    "# Height of predownscaled image if NOT using multiple resolution records.\n",
    "error_distribution_dict[\"image_predownscaled_height\"] = 1024\n",
    "# Width of predownscaled image if NOT using multiple resolution records.\n",
    "error_distribution_dict[\"image_predownscaled_width\"] = 1024\n",
    "# Depth of image, number of channels.\n",
    "error_distribution_dict[\"image_depth\"] = 3\n",
    "# Name of label feature within schema dictionary.\n",
    "error_distribution_dict[\"label_feature_name\"] = \"\"\n",
    "# Number of examples in one epoch of error distribution training set.\n",
    "error_distribution_dict[\"train_dataset_length\"] = 44693\n",
    "# Number of examples in error distribution training batch.\n",
    "error_distribution_dict[\"train_batch_size\"] = 16\n",
    "# Number of steps/batches to evaluate for error distribution.\n",
    "error_distribution_dict[\"eval_steps\"] = 1\n",
    "# Whether to autotune input function performance for error distribution datasets.\n",
    "error_distribution_dict[\"input_fn_autotune\"] = True\n",
    "# How many steps to train error distribution before saving a checkpoint.\n",
    "error_distribution_dict[\"save_checkpoints_steps\"] = 10000\n",
    "# Max number of error distribution checkpoints to keep.\n",
    "error_distribution_dict[\"keep_checkpoint_max\"] = 100\n",
    "# The checkpoint save path for saving and restoring.\n",
    "error_distribution_dict[\"checkpoint_save_path\"] = \"\"\n",
    "# Max number of times training loop can be restarted.\n",
    "error_distribution_dict[\"max_training_loop_restarts\"] = 20\n",
    "\n",
    "# Whether using sample or population covariance for error distribution.\n",
    "error_distribution_dict[\"use_sample_covariance\"] = True\n",
    "\n",
    "error_distribution_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dynamic threshold training parameters.\n",
    "dynamic_threshold_dict = dict()\n",
    "# Whether using multiple resolutions across a list of TF Records.\n",
    "dynamic_threshold_dict[\"use_multiple_resolution_records\"] = False\n",
    "# GCS locations to read dynamic threshold training data.\n",
    "dynamic_threshold_dict[\"train_file_pattern\"] = tf.io.gfile.glob(\n",
    "    pattern=\"gs://.../data/*/*.svs.{}.*.tfrecords\".format(0)\n",
    ")[175:200]\n",
    "# GCS locations to read dynamic threshold evaluation data.\n",
    "dynamic_threshold_dict[\"eval_file_pattern\"] = tf.io.gfile.glob(\n",
    "    pattern=\"gs://.../data/*/*.svs.{}.*.tfrecords\".format(0)\n",
    ")[175:200]\n",
    "# Which dataset to use for dynamic threshold training:\n",
    "# \"mnist\", \"cifar10\", \"cifar10_car\", \"tf_record\"\n",
    "dynamic_threshold_dict[\"dataset\"] = \"tf_record\"\n",
    "# TF Record Example feature schema for dynamic threshold.\n",
    "dynamic_threshold_dict[\"tf_record_example_schema\"] = [\n",
    "    {\n",
    "        \"name\": \"image/encoded\",\n",
    "        \"type\": \"FixedLen\",\n",
    "        \"shape\": [],\n",
    "        \"dtype\": \"str\"\n",
    "    },\n",
    "    {\n",
    "        \"name\": \"image/name\",\n",
    "        \"type\": \"FixedLen\",\n",
    "        \"shape\": [],\n",
    "        \"dtype\": \"str\"\n",
    "    }\n",
    "]\n",
    "# Name of image feature within schema dictionary.\n",
    "dynamic_threshold_dict[\"image_feature_name\"] = \"image/encoded\"\n",
    "# Encoding of image: raw, png, or jpeg.\n",
    "dynamic_threshold_dict[\"image_encoding\"] = \"png\"\n",
    "# Height of predownscaled image if NOT using multiple resolution records.\n",
    "dynamic_threshold_dict[\"image_predownscaled_height\"] = 1024\n",
    "# Width of predownscaled image if NOT using multiple resolution records.\n",
    "dynamic_threshold_dict[\"image_predownscaled_width\"] = 1024\n",
    "# Depth of image, number of channels.\n",
    "dynamic_threshold_dict[\"image_depth\"] = 3\n",
    "# Name of label feature within schema dictionary.\n",
    "dynamic_threshold_dict[\"label_feature_name\"] = \"\"\n",
    "# Number of examples in one epoch of dynamic threshold training set.\n",
    "dynamic_threshold_dict[\"train_dataset_length\"] = 52517\n",
    "# Number of examples in dynamic threshold training batch.\n",
    "dynamic_threshold_dict[\"train_batch_size\"] = 16\n",
    "# Number of steps/batches to evaluate for dynamic threshold.\n",
    "dynamic_threshold_dict[\"eval_steps\"] = 1\n",
    "# Whether to autotune input function performance for dynamic threshold datasets.\n",
    "dynamic_threshold_dict[\"input_fn_autotune\"] = True\n",
    "# How many steps to train dynamic threshold before saving a checkpoint.\n",
    "dynamic_threshold_dict[\"save_checkpoints_steps\"] = 10000\n",
    "# Max number of dynamic threshold checkpoints to keep.\n",
    "dynamic_threshold_dict[\"keep_checkpoint_max\"] = 100\n",
    "# The checkpoint save path for saving and restoring.\n",
    "dynamic_threshold_dict[\"checkpoint_save_path\"] = \"\"\n",
    "# Max number of times training loop can be restarted.\n",
    "dynamic_threshold_dict[\"max_training_loop_restarts\"] = 20\n",
    "\n",
    "# Whether using supervised dynamic thresholding or unsupervised.\n",
    "dynamic_threshold_dict[\"use_supervised\"] = False\n",
    "\n",
    "supervised_dict = dict()\n",
    "# Beta value for supervised F-beta score.\n",
    "supervised_dict[\"f_score_beta\"] = 0.05\n",
    "\n",
    "unsupervised_dict = dict()\n",
    "# Whether using sample or population covariance for dynamic threshold.\n",
    "unsupervised_dict[\"use_sample_covariance\"] = True\n",
    "# Max standard deviations of Mahalanobis distance to flag as outlier.\n",
    "unsupervised_dict[\"max_mahalanobis_stddevs\"] = 3.0\n",
    "\n",
    "dynamic_threshold_dict[\"supervised\"] = supervised_dict\n",
    "dynamic_threshold_dict[\"unsupervised\"] = unsupervised_dict\n",
    "\n",
    "dynamic_threshold_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training parameters.\n",
    "training_dict = dict()\n",
    "# GCS location to write checkpoints, loss logs, and export models.\n",
    "training_dict[\"output_dir\"] = \"gs://my-bucket/trained_models/experiment\"\n",
    "# Version of TensorFlow.\n",
    "training_dict[\"tf_version\"] = 2.3\n",
    "# Whether to use graph mode or not (eager).\n",
    "training_dict[\"use_graph_mode\"] = True\n",
    "# Which distribution strategy to use, if any.\n",
    "training_dict[\"distribution_strategy\"] = \"Mirrored\"\n",
    "# Whether we subclass models or use Functional API.\n",
    "training_dict[\"subclass_models\"] = True\n",
    "# Whether performing training phase 1 or not.\n",
    "training_dict[\"train_reconstruction\"] = True\n",
    "# Whether performing training phase 2 or not.\n",
    "training_dict[\"train_error_distribution\"] = True\n",
    "# Whether performing training phase 3 or not.\n",
    "training_dict[\"train_dynamic_threshold\"] = True\n",
    "\n",
    "training_dict[\"reconstruction\"] = reconstruction_dict\n",
    "training_dict[\"error_distribution\"] = error_distribution_dict\n",
    "training_dict[\"dynamic_threshold\"] = dynamic_threshold_dict\n",
    "\n",
    "training_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export parameters.\n",
    "export_dict = dict()\n",
    "\n",
    "# Most recent export's growth index so that there are no repeat exports.\n",
    "export_dict[\"most_recent_export_growth_idx\"] = -1\n",
    "# Most recent export's epoch index so that there are no repeat exports.\n",
    "export_dict[\"most_recent_export_epoch_idx\"] = -1\n",
    "# Whether to export SavedModel every growth phase.\n",
    "export_dict[\"export_every_growth_phase\"] = True\n",
    "# Whether to export SavedModel every epoch.\n",
    "export_dict[\"export_every_epoch\"] = True\n",
    "# Whether to export all growth phases or just current.\n",
    "export_dict[\"export_all_growth_phases\"] = True\n",
    "\n",
    "# Using a random noise vector Z with shape (batch_size, generator_latent_size) for berg.\n",
    "# Whether to export Z.\n",
    "export_dict[\"export_Z\"] = True\n",
    "# Whether to export generated images, G(z).\n",
    "export_dict[\"export_generated_images\"] = True\n",
    "# Whether to export encoded generated logits, E(G(z)).\n",
    "export_dict[\"export_encoded_generated_logits\"] = True\n",
    "# Whether to export encoded generated images, G(E(G(z))).\n",
    "export_dict[\"export_encoded_generated_images\"] = True\n",
    "# Whether to export Z generated images, Gd(z).\n",
    "export_dict[\"export_Z_generated_images\"] = True\n",
    "\n",
    "# Using a query image with shape (batch_size, height, width, depth)\n",
    "# Whether to export query images.\n",
    "export_dict[\"export_query_images\"] = True\n",
    "\n",
    "# Berg encoded exports.\n",
    "# Whether to export encoded query logits, E(x).\n",
    "export_dict[\"export_query_encoded_logits\"] = True\n",
    "# Whether to export encoded query images, G(E(x)).\n",
    "export_dict[\"export_query_encoded_images\"] = True\n",
    "\n",
    "# GANomaly encoded exports.\n",
    "# Whether to export generator encoded query logits, Ge(x).\n",
    "export_dict[\"export_query_gen_encoded_logits\"] = True\n",
    "# Whether to export generator encoded query images, G(x) = Gd(Ge(x)).\n",
    "export_dict[\"export_query_gen_encoded_images\"] = True\n",
    "# Whether to export encoder encoded query logits, E(G(x)).\n",
    "export_dict[\"export_query_enc_encoded_logits\"] = True\n",
    "# Whether to export encoder encoded query images, Gd(E(G(x))).\n",
    "export_dict[\"export_query_enc_encoded_images\"] = True\n",
    "\n",
    "# Anomaly exports.\n",
    "# Whether to export query anomaly images using sigmoid scaling.\n",
    "export_dict[\"export_query_anomaly_images_sigmoid\"] = True\n",
    "# Whether to export query anomaly images using linear scaling.\n",
    "export_dict[\"export_query_anomaly_images_linear\"] = True\n",
    "# Whether to export query Mahalanobis distances.\n",
    "export_dict[\"export_query_mahalanobis_distances\"] = True\n",
    "# Whether to export query Mahalanobis distance images using sigmoid scaling.\n",
    "export_dict[\"export_query_mahalanobis_distance_images_sigmoid\"] = True\n",
    "# Whether to export query Mahalanobis distance images using linear scaling.\n",
    "export_dict[\"export_query_mahalanobis_distance_images_linear\"] = True\n",
    "# Whether to export query pixel anomaly flag binary images.\n",
    "export_dict[\"export_query_pixel_anomaly_flag_images\"] = True\n",
    "# Whether to export query pixel anomaly flag binary images.\n",
    "export_dict[\"export_query_pixel_anomaly_flag_counts\"] = True\n",
    "# Whether to export query pixel anomaly flag binary images.\n",
    "export_dict[\"export_query_pixel_anomaly_flag_percentages\"] = True\n",
    "# Whether to export query anomaly scores, only for Berg.\n",
    "export_dict[\"export_query_anomaly_scores\"] = False\n",
    "# Whether to export query anomaly flags, only for Berg.\n",
    "export_dict[\"export_query_anomaly_flags\"] = False\n",
    "\n",
    "# Anomaly parameters.\n",
    "# The threshold value at which above flags scores images as anomalous.\n",
    "export_dict[\"anomaly_threshold\"] = 5.0\n",
    "# The anomaly convex combination factor for weighting the two anomaly losses.\n",
    "export_dict[\"anom_convex_combo_factor\"] = 0.05\n",
    "\n",
    "# Whether to print model summaries.\n",
    "export_dict[\"print_serving_model_summaries\"] = False\n",
    "\n",
    "export_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "JuMAd2M9I0vK"
   },
   "outputs": [],
   "source": [
    "# Full parameters.\n",
    "arguments = dict()\n",
    "\n",
    "arguments[\"generator\"] = generator_dict\n",
    "arguments[\"encoder\"] = encoder_dict\n",
    "arguments[\"discriminator\"] = discriminator_dict\n",
    "arguments[\"training\"] = training_dict\n",
    "arguments[\"export\"] = export_dict\n",
    "\n",
    "# Full lists for full 1024x1024 network growth.\n",
    "full_conv_num_filters = [[512, 512], [512, 512], [512, 512], [512, 512], [256, 256], [128, 128], [64, 64], [32, 32], [16, 16]]\n",
    "full_conv_kernel_sizes = [[4, 3], [3, 3], [3, 3], [3, 3], [3, 3], [3, 3], [3, 3], [3, 3], [3, 3]]\n",
    "full_conv_strides = [[1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1], [1, 1]]\n",
    "\n",
    "# Set final image size as a multiple of 2, starting at 4.\n",
    "image_size = 1024\n",
    "num_conv_blocks = max(\n",
    "    min(int(math.log(image_size, 2) - 1), len(full_conv_num_filters)), 1\n",
    ")\n",
    "\n",
    "arguments[\"conv_num_filters\"] = full_conv_num_filters[0:num_conv_blocks]\n",
    "arguments[\"conv_kernel_sizes\"] = full_conv_kernel_sizes[0:num_conv_blocks]\n",
    "arguments[\"conv_strides\"] = full_conv_strides[0:num_conv_blocks]\n",
    "\n",
    "# Get conv layer properties for generator and discriminator.\n",
    "(generator,\n",
    " discriminator) = (\n",
    "    gan_layer_architecture_shapes.calc_generator_discriminator_conv_layer_properties(\n",
    "        arguments[\"conv_num_filters\"],\n",
    "        arguments[\"conv_kernel_sizes\"],\n",
    "        arguments[\"conv_strides\"],\n",
    "        arguments[\"training\"][\"reconstruction\"][\"image_depth\"]\n",
    "    )\n",
    ")\n",
    "\n",
    "# Split up generator properties into separate lists.\n",
    "(generator_base_conv_blocks,\n",
    " generator_growth_conv_blocks,\n",
    " generator_to_rgb_layers) = (\n",
    "    gan_layer_architecture_shapes.split_up_generator_conv_layer_properties(\n",
    "        generator,\n",
    "        arguments[\"conv_num_filters\"],\n",
    "        arguments[\"conv_strides\"],\n",
    "        arguments[\"training\"][\"reconstruction\"][\"image_depth\"]\n",
    "    )\n",
    ")\n",
    "\n",
    "# Generator list of list of lists of base conv block layer shapes.\n",
    "arguments[\"generator\"][\"base_conv_blocks\"] = generator_base_conv_blocks\n",
    "# Generator list of list of lists of growth conv block layer shapes.\n",
    "arguments[\"generator\"][\"growth_conv_blocks\"] = generator_growth_conv_blocks\n",
    "# Generator list of list of lists of to_RGB layer shapes.\n",
    "arguments[\"generator\"][\"to_rgb_layers\"] = generator_to_rgb_layers\n",
    "\n",
    "# Split up discriminator properties into separate lists.\n",
    "(discriminator_from_rgb_layers,\n",
    " discriminator_base_conv_blocks,\n",
    " discriminator_growth_conv_blocks) = (\n",
    "    gan_layer_architecture_shapes.split_up_discriminator_conv_layer_properties(\n",
    "        discriminator,\n",
    "        arguments[\"conv_num_filters\"],\n",
    "        arguments[\"conv_strides\"],\n",
    "        arguments[\"training\"][\"reconstruction\"][\"image_depth\"]\n",
    "    )\n",
    ")\n",
    "\n",
    "# Discriminator list of list of lists of from_RGB layer shapes.\n",
    "arguments[\"discriminator\"][\"from_rgb_layers\"] = discriminator_from_rgb_layers\n",
    "# Discriminator list of list of lists of base conv block layer shapes.\n",
    "arguments[\"discriminator\"][\"base_conv_blocks\"] = (\n",
    "    discriminator_base_conv_blocks\n",
    ")\n",
    "# Discriminator list of list of lists of growth conv block layer shapes.\n",
    "arguments[\"discriminator\"][\"growth_conv_blocks\"] = (\n",
    "    discriminator_growth_conv_blocks\n",
    ")\n",
    "\n",
    "if (arguments[\"generator\"][\"architecture\"] == \"GANomaly\" and\n",
    "    arguments[\"generator\"][\"GANomaly\"][\"mask_generator_input_images_percent\"] > 0.):\n",
    "    # Image mask block pixel sizes list of lists.\n",
    "    arguments[\"generator\"][\"GANomaly\"][\"image_mask_block_sizes\"] = (\n",
    "        image_masks.calculate_image_mask_block_sizes_per_resolution(\n",
    "            num_resolutions=num_conv_blocks,\n",
    "            min_height=arguments[\"generator\"][\"projection_dims\"][0],\n",
    "            min_width=arguments[\"generator\"][\"projection_dims\"][1],\n",
    "            pixel_mask_percent=(\n",
    "                arguments[\"generator\"][\"GANomaly\"][\n",
    "                    \"mask_generator_input_images_percent\"]\n",
    "            )\n",
    "        )\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "sF26eGxbI0vN",
    "outputId": "546407cc-f894-47dd-90d3-38f6c4876f1c"
   },
   "outputs": [],
   "source": [
    "arguments"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "qTPKj0CtI0wr"
   },
   "source": [
    "## Run model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_VBhI_fmI0wu"
   },
   "outputs": [],
   "source": [
    "os.environ[\"OUTPUT_DIR\"] = arguments[\"training\"][\"output_dir\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "kZw3aFOvI0ww",
    "outputId": "9085e53f-7fe0-4894-af41-ddd10e4680fa"
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "echo ${OUTPUT_DIR}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9PrpInXLI0wy",
    "outputId": "23a37051-03b0-4dd4-826f-5d6c58285a8b"
   },
   "outputs": [],
   "source": [
    "# %%bash\n",
    "# gsutil -m rm -rf ${OUTPUT_DIR}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1Dhzw4lHI0w2"
   },
   "outputs": [],
   "source": [
    "train_and_evaluate_model = model.TrainAndEvaluateModel(params=arguments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hrkMgOj9I0w5",
    "outputId": "ba6d1c26-ae41-4bdd-9d69-427abfa9368e"
   },
   "outputs": [],
   "source": [
    "train_and_evaluate_model.train_and_evaluate()"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [
    "0WOfCOLuI0v5",
    "XlYsY-iYI0v9",
    "547YZ57pI0wH",
    "5amZD9R-I0wR",
    "4XOSoRQdI0wU",
    "pAwYlzUcI0wW",
    "gCq9BjhWI0wa",
    "ZFR0jrEBI0wd",
    "-3FT7oPeI0wh",
    "8RAnzQn1I0wk",
    "plRFnmDzI0wo",
    "qTPKj0CtI0wr",
    "V4AwQzK8I0xT",
    "mxch4X7NI0xV"
   ],
   "name": "tf2_progranomaly_local.ipynb",
   "provenance": []
  },
  "environment": {
   "name": "tf2-2-3-gpu.2-3.m56",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/tf2-2-3-gpu.2-3:m56"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}