Fully deployed pipeline. Gru training works.

examples/gru_function_classifier.json

@@ -0,0 +1,18 @@
+{
+    "train": {
+        "learning_rate": 0.0001,
+        "decay": 0.9,
+        "batch_size": 200,
+        "iters": 50000,
+        "summary_iters": 50,
+        "checkpoint_path": "models/checkpoints/function_classifier"
+    },
+    "arch": {
+        "sequence_length": 100,
+        "input_dimension": 1,
+        "output_dimension": 2,
+        "hidden_layer_size": 30,
+        "hidden_layer_depth": 2,
+        "pkeep": 0.5
+    }
+}

examples/gru_function_classifier_example.py

@@ -0,0 +1,56 @@
+import math
+
+import tensorflow as tf
+
+from datasets.classification.function_generator import function_generator
+from utils.prepare_training import write_tf_records, read_tf_records
+from models.gru_function_classifier import FunctionClassifier
+
+
+GENERATE_DATA = False
+
+
+def main():
+    # Define "constants".
+    hyper_params_filepath = "examples/gru_function_classifier.json"
+    data_tmp_folder = "data/.records/gru_function_classifier"
+    training_examples_number = 10000
+    validation_examples_number = 1000
+
+    if GENERATE_DATA:
+        # Create training data.
+        print("Generating data")
+        train_data = function_generator([lambda x, off: math.sin(x / 50.0 + off), lambda x, off: x / 50.0 + off], 100, training_examples_number)
+        validation_data  = function_generator([lambda x, off: math.sin(x / 50.0 + off), lambda x, off: x / 50.0 + off], 100, validation_examples_number)
+
+        # Write tf records
+        print("Writing data")
+        write_tf_records(data_tmp_folder, 4, 2, train_data, validation_data)
+
+    # Create model.
+    print("Creating Model")
+    model = FunctionClassifier(hyper_params_filepath)
+
+    # Load data with tf records.
+    print("Loading data")
+    train_features, train_labels = read_tf_records(data_tmp_folder, "train", model.hyper_params.train.batch_size, (100,), (2,), tf.float32, tf.uint8, 4)
+    validation_features, validation_labels = read_tf_records(data_tmp_folder, "validation", model.hyper_params.train.batch_size, (100,), (2,), tf.float32, tf.uint8, 2)
+
+    # Limit used gpu memory.
+    config = tf.ConfigProto()
+    config.gpu_options.per_process_gpu_memory_fraction = 0.75
+
+    # train model.
+    with tf.Session(config=config) as sess:
+        print("Setup")
+        model.setup(sess)
+
+        print("Training")
+        model.fit(train_features, train_labels, validation_examples_number, validation_features, validation_labels, verbose=True)
+
+        print("Exporting")
+        model.export()
+
+
+if __name__ == "__main__":
+    main()

models/gru_function_classifier.py

@@ -0,0 +1,63 @@
+import numpy as np
+
+import tensorflow as tf
+from tensorflow.contrib import layers
+from tensorflow.contrib import rnn
+
+from models.model import Model
+
+
+class FunctionClassifier(Model):
+    def __init__(self, hyper_params_filepath):
+        super(FunctionClassifier, self).__init__(hyper_params_filepath)
+
+    def _create_model(self, input_tensor, reuse_weights):        
+        with tf.variable_scope('NeuralNet') as scope:
+            if reuse_weights:
+                scope.reuse_variables()
+
+            input_tensor = tf.reshape(input_tensor, (self.hyper_params.train.batch_size, self.hyper_params.arch.sequence_length, 1))
+
+            Hin = tf.placeholder(tf.float32, [None, self.hyper_params.arch.hidden_layer_size * self.hyper_params.arch.hidden_layer_depth], name='Hin')  # [ BATCHSIZE, INTERNALSIZE * NLAYERS]
+            self.feed_dict[Hin] = np.zeros([self.hyper_params.train.batch_size, self.hyper_params.arch.hidden_layer_size * self.hyper_params.arch.hidden_layer_depth])
+
+            # using a NLAYERS=3 layers of GRU cells, unrolled SEQLEN=30 times
+            # dynamic_rnn infers SEQLEN from the size of the inputs Xo
+
+            # How to properly apply dropout in RNNs: see README.md
+            cells = [rnn.GRUCell(self.hyper_params.arch.hidden_layer_size) for _ in range(self.hyper_params.arch.hidden_layer_depth)]
+            # "naive dropout" implementation
+            dropcells = [rnn.DropoutWrapper(cell,input_keep_prob=self.hyper_params.arch.pkeep) for cell in cells]
+            multicell = rnn.MultiRNNCell(dropcells, state_is_tuple=False)
+            multicell = rnn.DropoutWrapper(multicell, output_keep_prob=self.hyper_params.arch.pkeep)  # dropout for the softmax layer
+
+            Yr, H = tf.nn.dynamic_rnn(multicell, input_tensor, dtype=tf.float32, initial_state=Hin)
+            # Yr: [ BATCHSIZE, SEQLEN, INTERNALSIZE ]
+            # H:  [ BATCHSIZE, INTERNALSIZE*NLAYERS ] # this is the last state in the sequence
+
+            H = tf.identity(H, name='H')  # just to give it a name
+
+            # Softmax layer implementation:
+            # Flatten the first two dimension of the output [ BATCHSIZE, SEQLEN, self.hyper_params.arch.output_dim ] => [ BATCHSIZE x SEQLEN, self.hyper_params.arch.output_dim ]
+            # then apply softmax readout layer. This way, the weights and biases are shared across unrolled time steps.
+            # From the readout point of view, a value coming from a sequence time step or a minibatch item is the same thing.
+
+            # Select last output.
+            output = tf.transpose(Yr, [1, 0, 2])
+            last = tf.gather(output, int(output.get_shape()[0]) - 1)
+            #Yflat = tf.reshape(Yr, [-1, self.hyper_params.arch.hidden_layer_size])    # [ BATCHSIZE x SEQLEN, INTERNALSIZE ]
+            self.outputs["logits"] = layers.linear(last, self.hyper_params.arch.output_dimension)     # [ BATCHSIZE x SEQLEN, self.hyper_params.arch.output_dim ]
+
+
+    def _create_loss(self, labels, validation_labels=None):
+        labels = tf.reshape(labels, [-1, self.hyper_params.arch.output_dimension])
+        loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=self.outputs["logits"], labels=labels))
+        train_op = tf.train.RMSPropOptimizer(learning_rate=self.hyper_params.train.learning_rate, decay=self.hyper_params.train.decay).minimize(loss_op)
+
+        # Create a validation loss if possible.
+        validation_loss_op = None
+        if validation_labels is not None:
+            validation_labels = tf.reshape(validation_labels, [-1, self.hyper_params.arch.output_dimension])
+            validation_loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=self.outputs["logits"], labels=validation_labels))
+
+        return train_op, loss_op, validation_loss_op

models/lfw.py

@@ -1,45 +1,31 @@
+import numpy as np
+
+import tensorflow as tf
+
 from models.model import Model
 
 class LFWNetwork(Model):
     def __init__(self, hyper_params_filepath):
-        pass
-
-
-    def setup(self, session):
-        """
-        Initialize everything for the model that needs a session.
-        This includes loading checkpoints if provided in the hyperparameters.
-
-        :param session: The tensorflow session to live inside.
-        """
-        pass
-
-    def predict(self, features):
-        """
-        Predict the output of the network given only the feature input.
-        This is handy for deployment of the network.
-
-        :param features: The input features of the network. For a cnn this is an image.
-        """
-        pass
+        super(LFWNetwork, self).__init__(hyper_params_filepath)
 
-    def fit(self, training_data, iters, validation_data=None, summary_iters=1000, verbose=True):
-        """
-        Fit the model to given training data.
+    def _create_model(self, input_tensor, reuse_weights):        
+        with tf.variable_scope('NeuralNet') as scope:
+            if reuse_weights:
+                scope.reuse_variables()
 
-        :param training_data: training_data TODO
-        :param validation_data: validation_data TODO (This data is optional, if not provided no validation is done.)
-        :param iters: iters The number of epochs to train in total.
-        :param summary_iters: summary_iters How many epochs to do between two summaries.
-        :param verbose: verbose If you want debug outputs or not.
-        """
-        pass
+            # TODO define net architecture.
+
+            self.outputs["logits"] = input_tensor
 
+    def _create_loss(self, labels, validation_labels=None):
+        labels = tf.reshape(labels, [-1, self.hyper_params.arch.output_dimension])
+        loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=self.outputs["logits"], labels=labels))
+        train_op = tf.train.RMSPropOptimizer(learning_rate=self.hyper_params.train.learning_rate, decay=self.hyper_params.train.decay).minimize(loss_op)
 
-    def export(self):
-        """
-        Export the model for deployment.
+        # Create a validation loss if possible.
+        validation_loss_op = None
+        if validation_labels is not None:
+            validation_labels = tf.reshape(validation_labels, [-1, self.hyper_params.arch.output_dimension])
+            validation_loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=self.outputs["logits"], labels=validation_labels))
 
-        The exported models can be used in an android app or a rosnode.
-        """
-        pass
+        return train_op, loss_op, validation_loss_op

models/model.py

@@ -2,6 +2,7 @@
 import abc
 
 from utils.dict2obj import Dict2Obj
+import tensorflow as tf
 
 
 class Model(object):
@@ -16,18 +17,35 @@ def __init__(self, hyper_params_filepath):
         :param hyper_params_filepath: The path to the hyper parameters file
         """
         self.hyper_params = Dict2Obj(json.load(open(hyper_params_filepath)))
+        self.model_train = None
+        self.model_validation = None
+        self.sess = None
+        self.feed_dict = {}
+        self.outputs = {}
 
-    @abc.abstractmethod
     def setup(self, session):
         """
         Initialize everything for the model that needs a session.
         This includes loading checkpoints if provided in the hyperparameters.
 
         :param session: The tensorflow session to live inside.
         """
+        self.sess = session
+
+    @abc.abstractmethod
+    def _create_model(self, input_tensor, reuse_weights):
+        """
+            Create a model.
+        """
         pass
 
     @abc.abstractmethod
+    def _create_loss(self, labels, validation_labels=None):
+        """
+            Create a loss.
+        """
+        pass
+
     def predict(self, features):
         """
         Predict the output of the network given only the feature input.
@@ -37,21 +55,62 @@ def predict(self, features):
         """
         pass
 
-    @abc.abstractmethod
-    def fit(self, training_data, iters, validation_data=None, summary_iters=1000, verbose=True):
+    def fit(self, features, labels, validation_examples_num=0, validation_features=None, validation_labels=None, verbose=True):
         """
         Fit the model to given training data.
 
-        :param training_data: training_data TODO
-        :param validation_data: validation_data TODO (This data is optional, if not provided no validation is done.)
+        :param features: features An input queue tensor as provided by prepare_training.read_tf_records(...).
+        :param labels: labels An input queue tensor as provided by prepare_training.read_tf_records(...).
+        :param validation_features: validation_features An input queue tensor. (This data is optional, if not provided no validation is done.)
+        :param validation_labels: validation_labels An input queue tensor. (This data is optional, if not provided no validation is done.)
         :param iters: iters The number of epochs to train in total.
         :param summary_iters: summary_iters How many epochs to do between two summaries.
         :param verbose: verbose If you want debug outputs or not.
         """
-        pass
+        self.model_train      = self._create_model(features, reuse_weights=False)
+        self.model_validation = self._create_model(validation_features, reuse_weights=True)
 
+        # Create loss and training op.
+        train_op, loss_op, validation_loss_op = self._create_loss(labels, validation_labels)
+
+        # Init vars.
+        init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
+        self.sess.run(init_op)
+
+        # Prepare training.
+        coord = tf.train.Coordinator()
+        threads = tf.train.start_queue_runners(coord=coord, sess=self.sess)
+        saver = tf.train.Saver()
+
+        # Train
+        acc_loss = 0.0
+        for i_step in range(self.hyper_params.train.iters):
+
+            # Train step.
+            _, loss = self.sess.run([train_op, loss_op], feed_dict=self.feed_dict)
+            acc_loss += loss
+
+            # Do validation and summary.
+            if i_step % self.hyper_params.train.summary_iters == 0:
+                loss_val = 0.0
+                if validation_examples_num > 0:
+                    for i_val in range(int(validation_examples_num / self.hyper_params.train.batch_size)):
+                        loss_val_loc = self.sess.run([validation_loss_op], feed_dict=self.feed_dict)[0]
+                        loss_val += loss_val_loc / float(self.hyper_params.train.batch_size)
+                    loss_val /= float(validation_examples_num)
+
+                if verbose:
+                    print("Iter: %d, Loss: %.4f, Validation Loss: %.4f" % (i_step, acc_loss / float(self.hyper_params.train.batch_size) / float(self.hyper_params.train.summary_iters), loss_val))
+                    acc_loss = 0.0
+
+                saver.save(self.sess, self.hyper_params.train.checkpoint_path, global_step=i_step)
+
+        if verbose:
+            print("Training stopped.")
+
+        coord.request_stop()
+        coord.join(threads)
 
-    @abc.abstractmethod
     def export(self):
         """
         Export the model for deployment.