network in progress

Author: carpedm20

README.md

@@ -13,6 +13,8 @@ TensorFlow implementation of [Learning from Simulated and Unsupervised Images th
 
 ## Usage
 
+First generate synthetic gaze dataset with [UnityEyes](http://www.cl.cam.ac.uk/research/rainbow/projects/unityeyes/). There is no details in the paper but I changed `Camera parameters` to `0, 0, 20, 40` before generating images.
+
 To train a model:
 
     $ python main.py --data_set gaze

config.py

@@ -22,10 +22,17 @@ def add_argument_group(name):
 data_arg = add_argument_group('Data')
 data_arg.add_argument('--data_set', type=str, default='gaze')
 data_arg.add_argument('--data_dir', type=str, default='data')
+data_arg.add_argument('--input_height', type=int, default=35)
+data_arg.add_argument('--input_width', type=int, default=55)
+data_arg.add_argument('--input_channel', type=int, default=1)
 
 # Training / test parameters
 train_arg = add_argument_group('Training')
-train_arg.add_argument('--optimizer', default='rmsprop', help='')
+train_arg.add_argument('--optimizer', type=str, default='rmsprop', help='')
+train_arg.add_argument('--max_step', type=int, default=200, help='')
+train_arg.add_argument('--lambda', type=float, default=1, help='')
+train_arg.add_argument('--K_d', type=int, default=1, help='')
+train_arg.add_argument('--K_g', type=int, default=5, help='')
 train_arg.add_argument('--batch_size', type=int, default=512, help='')
 train_arg.add_argument('--num_epochs', type=int, default=12, help='')
 train_arg.add_argument('--random_seed', type=int, default=123, help='')

data/gaze_data.py

@@ -16,8 +16,8 @@
 
 def maybe_download_and_extract(
     data_path,
-    url='http://datasets.d2.mpi-inf.mpg.de/MPIIGAZE_PATH/MPIIGAZE_PATH.tar.gz'):
-  if not os.path.exists(os.path.join(data_path, 'MPIIGAZE_PATH')):
+    url='http://datasets.d2.mpi-inf.mpg.de/MPIIGaze/MPIIGaze.tar.gz'):
+  if not os.path.exists(os.path.join(data_path, MPIIGAZE_PATH)):
     if not os.path.exists(data_path):
       os.makedirs(data_path)
 
@@ -32,11 +32,11 @@ def _progress(count, block_size, total_size):
 
       filepath, _ = urllib.request.urlretrieve(url, filepath, _progress)
       statinfo = os.stat(filepath)
-      print('\nSuccessfully downloaded', filename, statinfo.st_size, 'bytes.')
+      print('\nSuccessfully downloaded {} {} bytes.'.format(filename, statinfo.st_size))
       tarfile.open(filepath, 'r:gz').extractall(data_path)
 
 def maybe_preprocess(data_path):
-  base_path = os.path.join(data_path, 'MPIIGAZE_PATH/Data/Normalized')
+  base_path = os.path.join(data_path, '{}/Data/Normalized'.format(MPIIGAZE_PATH))
   npz_path = os.path.join(data_path, DATA_FNAME)
 
   if os.path.exists(npz_path):
@@ -48,6 +48,8 @@ def maybe_preprocess(data_path):
     for filename in fnmatch.filter(filenames, '*.mat'):
       mat_paths.append(os.path.join(root, filename))
 
+  print("[*] Preprocessing `gaze` data...")
+
   images =[]
   for mat_path in tqdm(mat_paths):
     mat = loadmat(mat_path)
@@ -57,14 +59,7 @@ def maybe_preprocess(data_path):
     images.extend(mat['data'][0][0][1][0][0][1])
 
   real_data = np.stack(images, axis=0)
-
-  # UnityEyes dataset
-  synthetic_data = None
-
-  #raise Exception("[!] Not implemented yet")
-
-  np.savez(npz_path, real=real_data, synthetic=synthetic_data)
-  print("[*] Preprocessing of `gaze` data is finished.")
+  np.savez(npz_path, real=real_data)
 
 def load(data_path, debug=False):
   if not os.path.exists(data_path):
@@ -75,28 +70,25 @@ def load(data_path, debug=False):
   maybe_preprocess(data_path)
 
   gaze_data = np.load(os.path.join(data_path, DATA_FNAME))
+  real_data = gaze_data['real']
 
-  real_data, synthetic_data = gaze_data['real'], gaze_data['synthetic']
   if debug:
     print("[*] Save sample images in {}".format(data_path))
-    for idx in range(10):
-      image_path = os.path.join(synthetic_images,
-                                "sample_real_{}".format(idx))
+    for idx in range(100):
+      image_path = os.path.join(data_path, "sample_real_{}.png".format(idx))
       imwrite(image_path, real_data[idx])
-  return real_data, synthetic_data
+  return real_data
 
 class DataLoader(object):
   def __init__(self, data_dir, batch_size, debug=False, rng=None):
     self.data_path = os.path.join(data_dir, 'gaze')
     self.batch_size = batch_size
 
-    self.data, self.labels = load(self.data_path, conf.debug)
-    self.data = np.transpose(self.data, (0,2,3,1)) # (N,3,32,32) -> (N,32,32,3)
+    self.data = load(self.data_path, debug)
 
     self.p = 0 # pointer to where we are in iteration
     self.rng = np.random.RandomState(1) if rng is None else rng
 
-
   def get_observation_size(self):
     return self.data.shape[1:]
 

layers.py

@@ -1,5 +1,40 @@
+import tensorflow as tf
+import tensorflow.contrib.slim as slim
 from tensorflow.contrib.framework import add_arg_scope
 
+def _update_dict(layer_dict, scope, layer):
+  name = "{}/{}".format(tf.get_variable_scope().name, scope)
+  layer_dict[name] = layer
+
+@add_arg_scope
+def resnet_block(
+    inputs, scope, num_outputs=64, kernel_size=[3, 3],
+    stride=[1, 1], padding="SAME", layer_dict={}):
+  with tf.variable_scope(scope):
+    layer = slim.conv2d(
+        inputs, num_outputs, kernel_size, stride,
+        padding=padding, activation_fn=tf.nn.relu, scope="conv1")
+    layer = slim.conv2d(
+        inputs, num_outputs, kernel_size, stride,
+        padding=padding, scope="conv2")
+    outputs = tf.nn.relu(tf.add(inputs, layer))
+  _update_dict(layer_dict, scope, outputs)
+  return outputs
+
+@add_arg_scope
+def repeat(inputs, repetitions, layer, layer_dict={}, **kargv):
+  outputs = slim.repeat(inputs, repetitions, layer, **kargv)
+  _update_dict(layer_dict, kargv['scope'], outputs)
+  return outputs
+
+@add_arg_scope
+def conv2d(inputs, num_outputs, kernel_size, stride, layer_dict={}, **kargv):
+  outputs = slim.conv2d(inputs, num_outputs, kernel_size, stride, **kargv)
+  _update_dict(layer_dict, kargv['scope'], outputs)
+  return outputs
+
 @add_arg_scope
-def resnet_block():
-  pass
+def max_pool2d(inputs, kernel_size=[3, 3], stride=[1, 1], layer_dict={}, **kargv):
+  outputs = slim.max_pool2d(inputs, kernel_size, stride, **kargv)
+  _update_dict(layer_dict, kargv['scope'], outputs)
+  return outputs

main.py

@@ -24,9 +24,9 @@ def main(_):
       'hand': hand_data.DataLoader,
   }[config.data_set]
 
+  model = Model(config)
   data_loader = DataLoader(config.data_dir, config.batch_size,
                            config.debug, rng=rng)
-  model = Model()
 
 if __name__ == "__main__":
   config, unparsed = get_config()

model.py

@@ -1,9 +1,21 @@
+from tqdm import tqdm
+
 import tensorflow as tf
 from tensorflow.contrib.framework.python.ops import arg_scope
 
 from network import Network
 
 class Model(object):
   def __init__(self, config):
+    self.K_d = config.K_d
+    self.K_g = config.K_g
+
     self.network = Network(config)
-    pass
+
+  def train(self):
+    for step in range(self.max_step):
+      for k in range(self.K_g):
+        pass
+
+      for k in range(self.K_d):
+        pass

network.py

@@ -1,5 +1,60 @@
 import tensorflow as tf
+from tensorflow.contrib.framework import arg_scope
+
+from layers import *
 
 class Network(object):
   def __init__(self, config):
-    pass
+    input_dims = [
+        None, config.input_height,
+        config.input_width, config.input_channel,
+    ]
+
+    def to_float(layer):
+      return tf.image.convert_image_dtype(layer, tf.float32)
+
+    self.inputs = to_float(tf.placeholder(tf.uint8, input_dims, 'inputs'))
+    #self.input_real = to_float(tf.placeholder(tf.uint8, input_dims, 'input_real'))
+    #self.input_synthetic = to_float(tf.placeholder(tf.uint8, input_dims, 'input_synthetic'))
+    self.targets = to_float(tf.placeholder(tf.uint8, input_dims, 'targets'))
+
+    self.layer_dict = {}
+
+    with arg_scope([resnet_block, conv2d, max_pool2d], layer_dict=self.layer_dict):
+      self.refiner_outputs = self._build_refiner_network()
+
+      self.discrim_inputs = self._build_discriminator_network(self.inputs)
+      self.discrim_refiner = self._build_discriminator_network(self.refiner_outputs)
+      import ipdb; ipdb.set_trace() 
+
+      #self.estimate_outputs = self._build_estimation_network()
+
+    self.refiner_loss = tf.reduce_sum(self.refiner_outputs - self.inputs, [1, 2, 3])
+
+  def _build_refiner_network(self):
+    layer = self.inputs
+    with tf.variable_scope("refiner"):
+      layer = repeat(layer, 5, resnet_block, scope="resnet")
+      layer = conv2d(layer, 1, 1, 1, scope="conv_1")
+    return layer
+
+  def _build_discriminator_network(self, layer):
+    with tf.variable_scope("discriminator"):
+      layer = conv2d(layer, 96, 3, 2, scope="conv_1")
+      layer = conv2d(layer, 64, 3, 2, scope="conv_2")
+      layer = max_pool2d(layer, 3, 1, scope="max_1")
+      layer = conv2d(layer, 32, 3, 1, scope="conv_3")
+      layer = conv2d(layer, 32, 1, 1, scope="conv_4")
+      layer = conv2d(layer, 2, 1, 1, activation_fn=tf.nn.softmax, scope="conv_5")
+    return layer
+
+  def _build_estimation_network(self):
+    layer = self.inputs
+    with tf.variable_scope("estimation"):
+      layer = conv2d(layer, 96, 3, 2, scope="conv_1")
+      layer = conv2d(layer, 64, 3, 2, scope="conv_2")
+      layer = max_pool2d(layer, 64, 3, scope="max_1")
+      layer = conv2d(layer, 32, 3, 1, scope="conv_3")
+      layer = conv2d(layer, 32, 1, 1, scope="conv_4")
+      layer = conv2d(layer, 2, 1, 1, activation_fn=slim.softmax)
+    return layer