(#42) Update InceptionTime model for classification task.

hyperts/framework/dl/__init__.py

@@ -1,2 +1,2 @@
 from ._base import BaseDeepEstimator
-from .models import DeepAR, HybirdRNN, LSTNet, NBeats
+from .models import DeepAR, HybirdRNN, LSTNet, NBeats, InceptionTime

hyperts/framework/dl/layers/__init__.py

@@ -1,7 +1,7 @@
 from tensorflow.keras.layers import *
 
 from ._layers import MultiColEmbedding, WeightedAttention, FeedForwardAttention, \
- AutoRegressive, Highway, Time2Vec, RevInstanceNormalization
+ AutoRegressive, Highway, Time2Vec, RevInstanceNormalization, Identity, Shortcut, InceptionBlock
 
 from ._layers import build_input_head, build_denses, build_embeddings, build_output_tail, rnn_forward
 

hyperts/framework/dl/layers/_layers.py

@@ -309,6 +309,113 @@ def get_config(self):
  return dict(list(base_config.items()) + list(config.items()))
 
 
+class Identity(layers.Layer):
+ """Identity Layer.
+
+ """
+ def __init__(self):
+ super(Identity, self).__init__()
+
+ def call(self, inputs, **kwargs):
+ return inputs
+
+
+class Shortcut(layers.Layer):
+ """ Shortcut Layer.
+
+ Parampers
+ ----------
+ filters: the dimensionality of the output space for Conv1D.
+ """
+ def __init__(self, filters, activation='relu', **kwargs):
+ super(Shortcut, self).__init__(**kwargs)
+ self.filters = filters
+ self.activation = activation
+ self.conv = layers.Conv1D(filters, kernel_size=1, padding='same', use_bias=False)
+ self.bn = layers.BatchNormalization()
+
+ def call(self, inputs, **kwargs):
+ x = self.conv(inputs)
+ x = self.bn(x)
+ return x
+
+ def get_config(self):
+ config = {'filters': self.filters}
+ base_config = super(Shortcut, self).get_config()
+ return dict(list(base_config.items()) + list(config.items()))
+
+
+class InceptionBlock(layers.Layer):
+ """InceptionBlock for time series.
+
+ Parampers
+ ----------
+ filters: the dimensionality of the output space for Conv1D.
+ kernel_size_list: list or tuple, a list of kernel size for Conv1D.
+ strides: int or tuple, default 1.
+ use_bottleneck: bool, whether to use bottleneck, default True.
+ bottleneck_size: int, if use bottleneck, bottleneck_size is 32(default).
+ activation: str, activation function, default 'relu'.
+ """
+ def __init__(self,
+ filters=32,
+ kernel_size_list=(1, 3, 5, 8, 12),
+ strides=1,
+ use_bottleneck=True,
+ bottleneck_size=32,
+ activation='linear',
+ **kwargs):
+ super(InceptionBlock, self).__init__(**kwargs)
+ self.filters = filters
+ self.kernel_size_list = kernel_size_list
+ self.strides = strides
+ self.use_bottleneck = use_bottleneck
+ self.bottleneck_size = bottleneck_size
+ self.activation = activation
+
+ if use_bottleneck:
+ self.head = layers.Conv1D(bottleneck_size, 1, padding='same', activation=activation, use_bias=False)
+ else:
+ self.head = Identity()
+
+ self.conv_list = []
+ for kernel_size in kernel_size_list:
+ self.conv_list.append(layers.Conv1D(filters=filters,
+ kernel_size=kernel_size,
+ padding='same',
+ activation=activation,
+ use_bias=False))
+
+ self.max_pool = layers.MaxPool1D(pool_size=3, strides=1, padding='same')
+ self.pool_conv = layers.Conv1D(filters, kernel_size=1, padding='same', activation=activation, use_bias=False)
+
+ self.concat = layers.Concatenate(axis=2)
+ self.bn = layers.BatchNormalization()
+ self.relu = layers.Activation(activation='relu')
+
+ def call(self, inputs, **kwargs):
+ x = self.head(inputs)
+ convs = [conv(x) for conv in self.conv_list]
+ pool = self.max_pool(x)
+ pool_conv = self.pool_conv(pool)
+ convs.append(pool_conv)
+ x = self.concat(convs)
+ x = self.bn(x)
+ x = self.relu(x)
+
+ return x
+
+ def get_config(self):
+ config = {'filters': self.filters,
+ 'kernel_size_list': self.kernel_size_list,
+ 'strides': self.strides,
+ 'use_bottleneck': self.use_bottleneck,
+ 'bottleneck_size': self.bottleneck_size,
+ 'activation': self.activation}
+ base_config = super(InceptionBlock, self).get_config()
+ return dict(list(base_config.items()) + list(config.items()))
+
+
 def build_input_head(window, continuous_columns, categorical_columns):
  """Build the input head. An input variable may have two parts: continuous variables
  and categorical variables.
@@ -445,5 +552,8 @@ def rnn_forward(x, nb_units, nb_layers, rnn_type, name, drop_rate=0., i=0, activ
  'AutoRegressive': AutoRegressive,
  'Highway': Highway,
  'Time2Vec': Time2Vec,
- 'RevInstanceNormalization': RevInstanceNormalization
+ 'RevInstanceNormalization': RevInstanceNormalization,
+ 'Identity': Identity,
+ 'Shortcut': Shortcut,
+ 'InceptionBlock': InceptionBlock,
 }

hyperts/framework/dl/models/__init__.py

@@ -1,4 +1,5 @@
 from .deepar import DeepAR
 from .hybird_rnn import HybirdRNN
 from .lstnet import LSTNet
-from .nbeats import NBeats
+from .nbeats import NBeats
+from .inceptiontime import InceptionTime

hyperts/framework/dl/models/inceptiontime.py

@@ -1,6 +1,5 @@
 # -*- coding:utf-8 -*-
 
-import numpy as np
 import tensorflow as tf
 import tensorflow.keras.backend as K
 
@@ -12,25 +11,202 @@
 logger = logging.get_logger(__name__)
 
 
-def InceptionTimeModel():
- """
+def InceptionTimeModel(task, window, continuous_columns, categorical_columns, blocks=3,
+ cnn_filters=32, bottleneck_size=32, kernel_size_list=(1, 3, 5, 8, 12),
+ shortcut=True, short_filters=64, nb_outputs=1, **kwargs):
+ """Inception Time Model (InceptionTime).
 
+ Parameters
+ ----------
+ task : Str - Only 'classification' is supported.
+ window : Positive Int - Length of the time series sequences for a sample.
+ continuous_columns: CategoricalColumn class.
+ Contains some information(name, column_names, input_dim, dtype,
+ input_name) about continuous variables.
+ categorical_columns: CategoricalColumn class.
+ Contains some information(name, vocabulary_size, embedding_dim,
+ dtype, input_name) about categorical variables.
+ blocks : Int - The depth of the net architecture.
+ cnn_filters: Int - The number of cnn filters.
+ bottleneck_size: Int - The number of bottleneck (a cnn layer).
+ kernel_size_list: Tuple - The kernel size of cnn for a inceptionblock.
+ shortcut : Bool - Whether to use shortcut opration.
+ short_filters: Int - The number of filters of shortcut conv1d layer.
+ nb_outputs : Int - The number of classes, default 1.
  """
+ if task not in consts.TASK_LIST_CLASSIFICATION:
+ raise ValueError(f'Unsupported task type {task}.')
+
+ kernel_size_list = list(filter(lambda x: x < window, kernel_size_list))
+
+ K.clear_session()
+ continuous_inputs, categorical_inputs = layers.build_input_head(window, continuous_columns, categorical_columns)
+ denses = layers.build_denses(continuous_columns, continuous_inputs)
+ embeddings = layers.build_embeddings(categorical_columns, categorical_inputs)
+ if embeddings is not None:
+ denses = layers.LayerNormalization(name='layer_norm')(denses)
+ x = layers.Concatenate(axis=-1, name='concat_embeddings_dense_inputs')([denses, embeddings])
+ else:
+ x = denses
+
+ s = x
+ for i in range(blocks):
+ x = layers.InceptionBlock(filters=cnn_filters,
+ bottleneck_size=bottleneck_size,
+ kernel_size_list=kernel_size_list,
+ name=f'inceptionblock_{i}')(x)
+ if shortcut and i % 3 == 2:
+ x = layers.Conv1D(short_filters, 1, padding='same', use_bias=False, name=f'shortconv_{i}')(x)
+ s = layers.Shortcut(filters=short_filters, name=f'shortcut_{i}')(s)
+ x = layers.Add(name=f'add_x_s_{i}')([x, s])
+ x = layers.Activation('relu', name=f'relu_x_s_{i}')(x)
+ s = x
+
+ x = layers.GlobalAveragePooling1D(name='globeal_avg_pool')(x)
+ outputs = layers.build_output_tail(x, task, nb_outputs)
+
+ all_inputs = list(continuous_inputs.values()) + list(categorical_inputs.values())
+ model = tf.keras.models.Model(inputs=all_inputs, outputs=[outputs], name=f'InceptionTime')
+
+ return model
 
 
 class InceptionTime(BaseDeepEstimator):
- """
+ """InceptionTime Estimator.
 
+ Parameters
+ ----------
+ task : Str - Support forecast, classification, and regression.
+ See hyperts.utils.consts for details.
+ blocks : Int - The depth of the net architecture.
+ default = 3.
+ cnn_filters: Int - The number of cnn filters.
+ default = 32.
+ bottleneck_size: Int - The number of bottleneck (a cnn layer).
+ default = 32.
+ kernel_size_list: Tuple - The kernel size of cnn for a inceptionblock.
+ default = (1, 3, 5, 8, 12).
+ shortcut : Bool - Whether to use shortcut opration.
+ default = True.
+ short_filters: Int - The number of filters of shortcut conv1d layer.
+ default = 64.
+ timestamp : Str or None - Timestamp name, the forecast task must be given,
+ default None.
+ window : Positive Int - Length of the time series sequences for a sample,
+ default = 3.
+ horizon : Positive Int - Length of the prediction horizon,
+ default = 1.
+ forecast_length : Positive Int - Step of the forecast outputs,
+ default = 1.
+ metrics : Str - List of metrics to be evaluated by the model during training and testing,
+ default = 'auto'.
+ monitor_metric : Str - Quality indicators monitored during neural network training.
+ default = 'val_loss'.
+ optimizer : Str or keras Instance - for example, 'adam', 'sgd', and so on.
+ default = 'auto'.
+ learning_rate : Positive Float - The optimizer's learning rate,
+ default = 0.001.
+ loss : Str - Loss function, for forecsting or regression, optional {'auto', 'mae', 'mse', 'huber_loss',
+ 'mape'}, for classification, optional {'auto', 'categorical_crossentropy', 'binary_crossentropy},
+ default = 'auto'.
+ reducelr_patience : Positive Int - The number of epochs with no improvement after which learning rate
+ will be reduced, default = 5.
+ earlystop_patience : Positive Int - The number of epochs with no improvement after which training
+ will be stopped, default = 5.
+ summary : Bool - Whether to output network structure information,
+ default = True.
+ continuous_columns: CategoricalColumn class.
+ Contains some information(name, column_names, input_dim, dtype,
+ input_name) about continuous variables.
+ categorical_columns: CategoricalColumn class.
+ Contains some information(name, vocabulary_size, embedding_dim,
+ dtype, input_name) about categorical variables.
  """
- def __init__(self, **kwargs):
- super(InceptionTime, self).__init__(**kwargs)
+ def __init__(self,
+ task,
+ blocks=3,
+ cnn_filters=32,
+ bottleneck_size=32,
+ kernel_size_list=(1, 3, 5, 8, 12),
+ shortcut=True,
+ short_filters=64,
+ timestamp=None,
+ window=3,
+ horizon=1,
+ forecast_length=1,
+ metrics='auto',
+ monitor_metric='val_loss',
+ optimizer='auto',
+ learning_rate=0.001,
+ loss='auto',
+ reducelr_patience=5,
+ earlystop_patience=10,
+ summary=True,
+ continuous_columns=None,
+ categorical_columns=None,
+ **kwargs):
+ self.blocks = blocks
+ self.cnn_filters = cnn_filters
+ self.bottleneck_size = bottleneck_size
+ self.kernel_size_list = kernel_size_list
+ self.shortcut = shortcut
+ self.short_filters = short_filters
+ self.metrics = metrics
+ self.optimizer = optimizer
+ self.learning_rate = learning_rate
+ self.loss = loss
+ self.summary = summary
+ self.model_kwargs = kwargs.copy()
 
+ super(InceptionTime, self).__init__(task=task,
+ timestamp=timestamp,
+ window=window,
+ horizon=horizon,
+ forecast_length=forecast_length,
+ monitor_metric=monitor_metric,
+ reducelr_patience=reducelr_patience,
+ earlystop_patience=earlystop_patience,
+ continuous_columns=continuous_columns,
+ categorical_columns=categorical_columns)
 
  def _build_estimator(self, **kwargs):
- raise NotImplementedError('Return inception model.')
+ model_params = {
+ 'task': self.task,
+ 'window': self.window,
+ 'blocks': self.blocks,
+ 'continuous_columns': self.continuous_columns,
+ 'categorical_columns': self.categorical_columns,
+ 'cnn_filters': self.cnn_filters,
+ 'bottleneck_size': self.bottleneck_size,
+ 'kernel_size_list': self.kernel_size_list,
+ 'nb_outputs': self.meta.classes_,
+ 'shortcut': self.shortcut,
+ 'short_filters': self.short_filters,
+ }
+ model_params = {**model_params, **self.model_kwargs, **kwargs}
+ return InceptionTimeModel(**model_params)
 
  def _fit(self, train_X, train_y, valid_X, valid_y, **kwargs):
- raise NotImplementedError('Return estimator.')
+ train_ds = self._from_tensor_slices(X=train_X, y=train_y,
+ batch_size=kwargs['batch_size'],
+ epochs=kwargs['epochs'],
+ shuffle=True)
+ valid_ds = self._from_tensor_slices(X=valid_X, y=valid_y,
+ batch_size=kwargs.pop('batch_size'),
+ epochs=kwargs['epochs'],
+ shuffle=False)
+ model = self._build_estimator()
+
+ if self.summary and kwargs['verbose'] != 0:
+ model.summary()
+ else:
+ logger.info(f'Number of current InceptionTime params: {model.count_params()}')
+
+ model = self._compile_model(model, self.optimizer, self.learning_rate)
+
+ history = model.fit(train_ds, validation_data=valid_ds, **kwargs)
+
+ return model, history
 
  @tf.function(experimental_relax_shapes=True)
  def _predict(self, X):