From 4cab97d42b2a8a8eb8388bba0fe365a52c871909 Mon Sep 17 00:00:00 2001 From: csxeba Date: Mon, 24 Jun 2019 16:51:23 +0200 Subject: [PATCH] Buggy metrics --- brainforge/atomic/core_op.py | 4 +- brainforge/atomic/tensor_op.py | 25 ++-- brainforge/cost/__init__.py | 5 - brainforge/layers/tensor.py | 29 +++-- brainforge/learner/abstract_learner.py | 116 +++++++++--------- brainforge/learner/backpropagation.py | 10 +- brainforge/metrics/__init__.py | 3 + .../{cost/_costs.py => metrics/costs.py} | 33 +++-- brainforge/metrics/metrics.py | 31 +++++ brainforge/{cost => metrics}/regularizers.py | 0 .../{optimization => optimizers}/__init__.py | 0 .../_coroutines.py | 0 .../abstract_optimizer.py | 0 .../adaptive_gd.py | 2 +- .../gradient_descent.py | 0 brainforge/util/__init__.py | 2 +- brainforge/util/logging.py | 49 ++++++++ brainforge/util/persistance.py | 4 +- brainforge/util/typing.py | 4 + xperiments/xp_3conv.py | 2 +- xperiments/xp_conv.py | 13 +- xperiments/xp_decoupled.py | 2 +- xperiments/xp_elm.py | 2 +- xperiments/xp_feedbackalignment.py | 13 +- xperiments/xp_mnist.py | 19 +-- xperiments/xp_pggymin.py | 2 +- xperiments/xp_qcartpole.py | 2 +- xperiments/xp_sin.py | 4 +- xperiments/xp_vae.py | 9 +- 29 files changed, 242 insertions(+), 143 deletions(-) delete mode 100644 brainforge/cost/__init__.py create mode 100644 brainforge/metrics/__init__.py rename brainforge/{cost/_costs.py => metrics/costs.py} (67%) create mode 100644 brainforge/metrics/metrics.py rename brainforge/{cost => metrics}/regularizers.py (100%) rename brainforge/{optimization => optimizers}/__init__.py (100%) rename brainforge/{optimization => optimizers}/_coroutines.py (100%) rename brainforge/{optimization => optimizers}/abstract_optimizer.py (100%) rename brainforge/{optimization => optimizers}/adaptive_gd.py (99%) rename brainforge/{optimization => optimizers}/gradient_descent.py (100%) create mode 100644 brainforge/util/logging.py diff --git a/brainforge/atomic/core_op.py b/brainforge/atomic/core_op.py index 3dbd0d0..7962015 100644 --- a/brainforge/atomic/core_op.py +++ b/brainforge/atomic/core_op.py @@ -25,8 +25,8 @@ class ReshapeOp: type = "Reshape" @staticmethod - def forward(X, outshape): - return X.reshape(X.shape[0], *outshape) + def forward(X: np.ndarray, outshape: tuple): + return X.reshape(-1, *outshape) @staticmethod def backward(E, inshape): diff --git a/brainforge/atomic/tensor_op.py b/brainforge/atomic/tensor_op.py index 585f1ca..f4567f3 100644 --- a/brainforge/atomic/tensor_op.py +++ b/brainforge/atomic/tensor_op.py @@ -20,17 +20,15 @@ def valid(A, F): err += "input depth: {} != {} :filter depth".format(ic, fc) raise ValueError(err) - for i, pic in enumerate(A): - for sy in range(oy): - for sx in range(ox): - rfields[i][sy*ox + sx] = pic[:, sy:sy+fy, sx:sx+fx].ravel() + for i, sy, sx in ((idx, shy, shx) for shx in range(ox) for shy in range(oy) for idx in range(im)): + rfields[i][sy*ox + sx] = A[i, :, sy:sy+fy, sx:sx+fx].ravel() - output = np.zeros((im, oy*ox, nf)) - for m in range(im): - output[m] = np.dot(rfields[m], Frsh.T) + # output = np.zeros((im, oy*ox, nf)) + # for m in range(im): + # output[m] = np.dot(rfields[m], Frsh.T) - # output = np.matmul(rfields, F.reshape(nf, recfield_size).T) - output = output.transpose((0, 2, 1)).reshape(im, nf, oy, ox) + output = np.matmul(rfields, Frsh.T) + output = output.transpose((0, 2, 1)).reshape((im, nf, oy, ox)) return output @staticmethod @@ -82,6 +80,15 @@ class MaxPoolOp: def __str__(self): return "MaxPool" + @staticmethod + def predict(A): + return np.max([ + A[:, :, 0::2, 0::2], + A[:, :, 0::2, 1::2], + A[:, :, 1::2, 0::2], + A[:, :, 1::2, 1::2], + ], axis=0) + @staticmethod def forward(A, fdim): im, ic, iy, ix = A.shape diff --git a/brainforge/cost/__init__.py b/brainforge/cost/__init__.py deleted file mode 100644 index fb3312d..0000000 --- a/brainforge/cost/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from ._costs import mean_squared_error, categorical_crossentropy, binary_crossentropy, hinge -from ._costs import mse, cxent, bxent -from ._costs import CostFunction - -costs = {k: v for k, v in locals().items() if k[0] != "_" and k != "CostFunction"} diff --git a/brainforge/layers/tensor.py b/brainforge/layers/tensor.py index b7f68d7..00c357d 100644 --- a/brainforge/layers/tensor.py +++ b/brainforge/layers/tensor.py @@ -1,3 +1,4 @@ +from brainforge.util import emptyX from .abstract_layer import LayerBase, NoParamMixin from ..util import zX, zX_like, white @@ -25,21 +26,10 @@ def connect(self, brain): self.output = zX(ic, iy // self.fdim, ix // self.fdim) def feedforward(self, questions): - """ - Implementation of a max pooling layer. - - :param questions: numpy.ndarray, a batch of outsize from the previous layer - :return: numpy.ndarray, max pooled batch - """ self.output, self.filter = self.op.forward(questions, self.fdim) return self.output def backpropagate(self, delta): - """ - Calculates the error of the previous layer. - :param delta: - :return: numpy.ndarray, the errors of the previous layer - """ return self.op.backward(delta, self.filter) @property @@ -101,3 +91,20 @@ def outshape(self): def __str__(self): return "Conv({}x{}x{})-{}".format(self.nfilters, self.fy, self.fx, str(self.activation)[:4]) + + +class GlobalAveragePooling(NoParamMixin, LayerBase): + + def __init__(self): + super().__init__() + self.dynamic_input_shape = None + + def feedforward(self, X): + self.dynamic_input_shape = X.shape + return X.mean(axis=(2, 3)) + + def backpropagate(self, delta): + canvas = emptyX(*self.inputs.shape) + nxy = self.dynamic_input_shape[-2] * self.dynamic_input_shape[-1] + for mm, cc in ((m, c) for c in range(delta.shape[1]) for m in range(delta.shape[0])): + canvas.flat[mm, cc] = delta[mm, cc] / nxy diff --git a/brainforge/learner/abstract_learner.py b/brainforge/learner/abstract_learner.py index b64f44e..6f1ef8f 100644 --- a/brainforge/learner/abstract_learner.py +++ b/brainforge/learner/abstract_learner.py @@ -1,10 +1,6 @@ -import abc - -import numpy as np - from ..model.layerstack import LayerStack -from ..cost import costs, CostFunction -from ..util import batch_stream +from ..metrics import costs as _costs, metrics as _metrics +from ..util import batch_stream, logging class Learner: @@ -17,83 +13,81 @@ def __init__(self, layerstack, cost="mse", name="", **kw): self.layers = layerstack self.name = name self.age = 0 - self.cost = cost if isinstance(cost, CostFunction) else costs[cost] + self.cost = _costs.get(cost) - def fit_generator(self, generator, lessons_per_epoch, epochs=30, classify=True, validation=(), verbose=1, **kw): - epcosts = [] + def fit_generator(self, generator, lessons_per_epoch, epochs=30, metrics=(), validation=(), verbose=1, **kw): + metrics = [_metrics.get(metric) for metric in metrics] + history = logging.MetricLogs.from_metric_list(lessons_per_epoch, ("cost",), metrics) lstr = len(str(epochs)) for epoch in range(1, epochs+1): if verbose: print("Epoch {:>{w}}/{}".format(epoch, epochs, w=lstr)) - epcosts += self.epoch(generator, no_lessons=lessons_per_epoch, classify=classify, - validation=validation, verbose=verbose, **kw) - return epcosts + epoch_history = self.epoch(generator, updates_per_epoch=lessons_per_epoch, metrics=metrics, + validation=validation, verbose=verbose, **kw) + history.update(epoch_history) + + return history - def fit(self, X, Y, batch_size=20, epochs=30, classify=True, validation=(), verbose=1, shuffle=True, **kw): + def fit(self, X, Y, batch_size=20, epochs=30, metrics=(), validation=(), verbose=1, shuffle=True, **kw): + metrics = [_metrics.get(metric) for metric in metrics] datastream = batch_stream(X, Y, m=batch_size, shuffle=shuffle) - return self.fit_generator(datastream, len(X), epochs, classify, validation, verbose, **kw) + return self.fit_generator(datastream, len(X) // batch_size, epochs, metrics, validation, verbose, **kw) - def epoch(self, generator, no_lessons, classify=True, validation=None, verbose=1, **kw): - losses = [] + def epoch(self, generator, updates_per_epoch, metrics=(), validation=None, verbose=1, **kw): + metrics = [_metrics.get(metric) for metric in metrics] + history = logging.MetricLogs.from_metric_list(updates_per_epoch, ["cost"], metrics) done = 0 self.layers.learning = True - while done < no_lessons: + batch_size = 0 + for i in range(updates_per_epoch): batch = next(generator) - cost = self.learn_batch(*batch, **kw) - losses.append(cost) - - done += len(batch[0]) + batch_size = len(batch[0]) + epoch_metrics = self.learn_batch(*batch, metrics=metrics, **kw) + history.record(epoch_metrics) if verbose: - print("\rDone: {0:>6.1%} Cost: {1: .5f}\t " - .format(done/no_lessons, np.mean(losses)), end="") + history.log(prefix="\r", end="") + self.layers.learning = False - if verbose: - print("\rDone: {0:>6.1%} Cost: {1: .5f}\t ".format(1., np.mean(losses)), end="") - if validation: - self._print_progress(validation, classify) + if verbose and validation: + history = self.evaluate(*validation, batch_size=batch_size, metrics=metrics) + history.log(prefix=" ", suffix="") print() - self.age += no_lessons - return losses - - def _print_progress(self, validation, classify): - results = self.evaluate(*validation, classify=classify) - - chain = "Testing cost: {0:.5f}" - if classify: - tcost, tacc = results - accchain = " accuracy: {0:.2%}".format(tacc) - else: - tcost = results - accchain = "" - print(chain.format(tcost) + accchain, end="") + self.age += updates_per_epoch + return history def predict(self, X): return self.layers.feedforward(X) - def evaluate(self, X, Y, batch_size=32, classify=True, shuffle=False, verbose=False): + def evaluate_batch(self, x, y, metrics=()): + m = len(x) + preds = self.predict(x) + eval_metrics = {"cost": self.cost(self.output, y) / m} + if metrics: + for metric in metrics: + eval_metrics[str(metric).lower()] = metric(preds, y) / m + return eval_metrics + + def evaluate(self, X, Y, batch_size=32, metrics=(), verbose=False): + metrics = [_metrics.get(metric) for metric in metrics] N = X.shape[0] - batches = batch_stream(X, Y, m=batch_size, shuffle=shuffle, infinite=False) + batch_size = min(batch_size, N) + steps = int(round(N / batch_size)) + history = logging.MetricLogs.from_metric_list(steps, ["cost"], metrics) - cost, acc = [], [] - for bno, (x, y) in enumerate(batches, start=1): + for x, y in batch_stream(X, Y, m=batch_size, shuffle=False, infinite=False): + eval_metrics = self.evaluate_batch(x, y, metrics) + history.record(eval_metrics) if verbose: - print("\rEvaluating: {:>7.2%}".format((bno*batch_size) / N), end="") - pred = self.predict(x) - cost.append(self.cost(pred, y) / len(x)) - if classify: - pred_classes = np.argmax(pred, axis=1) - trgt_classes = np.argmax(y, axis=1) - eq = np.equal(pred_classes, trgt_classes) - acc.append(eq.mean()) - results = np.mean(cost) - if classify: - results = (results, np.mean(acc)) - return results - - @abc.abstractmethod - def learn_batch(self, X, Y, **kw): + history.log("\r", end="") + + if verbose: + print() + history.reduce_mean() + return history + + def learn_batch(self, X, Y, metrics=(), **kw) -> dict: raise NotImplementedError @property @@ -106,7 +100,7 @@ def outshape(self): @property def num_params(self): - return self.layers.nparams + return self.layers.num_params @property def trainable_layers(self): diff --git a/brainforge/learner/backpropagation.py b/brainforge/learner/backpropagation.py index 16ebaaf..86a07da 100644 --- a/brainforge/learner/backpropagation.py +++ b/brainforge/learner/backpropagation.py @@ -1,7 +1,7 @@ import numpy as np from .abstract_learner import Learner -from brainforge.optimization import optimizers, GradientDescent +from ..optimizers import optimizers, GradientDescent class BackpropNetwork(Learner): @@ -13,7 +13,7 @@ def __init__(self, layerstack, cost="mse", optimizer="sgd", name="", **kw): ) self.optimizer.initialize(nparams=self.layers.num_params) - def learn_batch(self, X, Y, w=None): + def learn_batch(self, X, Y, w=None, metrics=()): m = len(X) preds = self.predict(X) delta = self.cost.derivative(preds, Y) @@ -21,7 +21,11 @@ def learn_batch(self, X, Y, w=None): delta *= w[:, None] self.backpropagate(delta) self.update(m) - return self.cost(self.output, Y) / m + train_metrics = {"cost": self.cost(self.output, Y) / m} + if metrics: + for metric in metrics: + train_metrics[str(metric).lower()] = metric(preds, Y) / m + return train_metrics def backpropagate(self, error): for layer in self.layers[-1:0:-1]: diff --git a/brainforge/metrics/__init__.py b/brainforge/metrics/__init__.py new file mode 100644 index 0000000..279aae9 --- /dev/null +++ b/brainforge/metrics/__init__.py @@ -0,0 +1,3 @@ +from .costs import mean_squared_error, categorical_crossentropy, binary_crossentropy, hinge +from .costs import mse, cxent, bxent +from .metrics import accuracy diff --git a/brainforge/cost/_costs.py b/brainforge/metrics/costs.py similarity index 67% rename from brainforge/cost/_costs.py rename to brainforge/metrics/costs.py index 6aace28..9f99cf2 100644 --- a/brainforge/cost/_costs.py +++ b/brainforge/metrics/costs.py @@ -1,5 +1,3 @@ -import abc - import numpy as np from ..util.typing import scalX @@ -10,7 +8,7 @@ s2 = scalX(2.) -class CostFunction(abc.ABC): +class CostFunction: def __call__(self, outputs, targets): pass @@ -22,7 +20,7 @@ def derivative(outputs, targets): return outputs - targets -class MeanSquaredError(CostFunction): +class _MeanSquaredError(CostFunction): def __call__(self, outputs, targets): return s05 * np.linalg.norm(outputs - targets) ** s2 @@ -32,7 +30,7 @@ def true_derivative(outputs, targets): return outputs - targets -class CategoricalCrossEntropy(CostFunction): +class _CategoricalCrossEntropy(CostFunction): def __call__(self, outputs: np.ndarray, targets: np.ndarray): return -(targets * np.log(outputs)).sum() @@ -44,7 +42,7 @@ def true_derivative(outputs, targets): return enum / denom -class BinaryCrossEntropy(CostFunction): +class _BinaryCrossEntropy(CostFunction): def __call__(self, outputs: np.ndarray, targets: np.ndarray): return -(targets * np.log(outputs) + (s1 - targets) * np.log(s1 - outputs)).sum() @@ -54,7 +52,7 @@ def true_derivative(outputs, targets): raise NotImplementedError -class Hinge(CostFunction): +class _Hinge(CostFunction): def __call__(self, outputs, targets): return (np.maximum(s0, s1 - targets * outputs)).sum() @@ -70,11 +68,22 @@ def derivative(outputs, targets): return out -mean_squared_error = MeanSquaredError() -categorical_crossentropy = CategoricalCrossEntropy() -binary_crossentropy = BinaryCrossEntropy() -hinge = Hinge() +mean_squared_error = _MeanSquaredError() +categorical_crossentropy = _CategoricalCrossEntropy() +binary_crossentropy = _BinaryCrossEntropy() +hinge = _Hinge() mse = mean_squared_error cxent = categorical_crossentropy -bxent = binary_crossentropy \ No newline at end of file +bxent = binary_crossentropy + +_costs = {k: v for k, v in locals().items() if k[0] != "_" and k != "CostFunction"} + + +def get(cost_function): + if isinstance(cost_function, CostFunction): + return cost_function + cost = _costs.get(cost_function) + if cost is None: + raise ValueError("No such cost function: {}".format(cost)) + return cost diff --git a/brainforge/metrics/metrics.py b/brainforge/metrics/metrics.py new file mode 100644 index 0000000..444eff4 --- /dev/null +++ b/brainforge/metrics/metrics.py @@ -0,0 +1,31 @@ +import numpy as np + + +class Metric: + + def __call__(self, outputs, targets): + raise NotImplementedError + + def __str__(self): + return self.__class__.__name__.lower().replace("_", "") + + +class _Accuracy(Metric): + + def __call__(self, outputs, targets): + return np.sum(outputs.argmax(axis=1) == targets.argmax(axis=1)) + + +accuracy = _Accuracy() +acc = accuracy + +_metrics = {k: v for k, v in locals().items() if k[0] != "_" and k != "Metric"} + + +def get(metric): + if isinstance(metric, Metric): + return metric + metric = _metrics.get(metric) + if metric is None: + raise ValueError("No such metric: {}".format(metric)) + return metric diff --git a/brainforge/cost/regularizers.py b/brainforge/metrics/regularizers.py similarity index 100% rename from brainforge/cost/regularizers.py rename to brainforge/metrics/regularizers.py diff --git a/brainforge/optimization/__init__.py b/brainforge/optimizers/__init__.py similarity index 100% rename from brainforge/optimization/__init__.py rename to brainforge/optimizers/__init__.py diff --git a/brainforge/optimization/_coroutines.py b/brainforge/optimizers/_coroutines.py similarity index 100% rename from brainforge/optimization/_coroutines.py rename to brainforge/optimizers/_coroutines.py diff --git a/brainforge/optimization/abstract_optimizer.py b/brainforge/optimizers/abstract_optimizer.py similarity index 100% rename from brainforge/optimization/abstract_optimizer.py rename to brainforge/optimizers/abstract_optimizer.py diff --git a/brainforge/optimization/adaptive_gd.py b/brainforge/optimizers/adaptive_gd.py similarity index 99% rename from brainforge/optimization/adaptive_gd.py rename to brainforge/optimizers/adaptive_gd.py index f24e6da..ccc6f7b 100644 --- a/brainforge/optimization/adaptive_gd.py +++ b/brainforge/optimizers/adaptive_gd.py @@ -80,7 +80,7 @@ def __str__(self): class Adam(_SGD): - def __init__(self, eta=0.1, decay_memory=0.999, decay_velocity=0.9, epsilon=1e-8): + def __init__(self, eta=0.1, decay_memory=0.999, decay_velocity=0.9, epsilon=1e-7): super().__init__(eta) self.decay_velocity = decay_velocity self.decay_memory = decay_memory diff --git a/brainforge/optimization/gradient_descent.py b/brainforge/optimizers/gradient_descent.py similarity index 100% rename from brainforge/optimization/gradient_descent.py rename to brainforge/optimizers/gradient_descent.py diff --git a/brainforge/util/__init__.py b/brainforge/util/__init__.py index 0c2ac46..c613047 100644 --- a/brainforge/util/__init__.py +++ b/brainforge/util/__init__.py @@ -2,5 +2,5 @@ from ._etalon import etalon from .typing import ( - scalX, ctx1, zX, zX_like, white, white_like + scalX, ctx1, zX, zX_like, white, white_like, emptyX ) diff --git a/brainforge/util/logging.py b/brainforge/util/logging.py new file mode 100644 index 0000000..3c6b49e --- /dev/null +++ b/brainforge/util/logging.py @@ -0,0 +1,49 @@ +from collections import defaultdict +from statistics import mean + + +class MetricLogs: + + def __init__(self, fields, max_steps=-1): + self._metrics = defaultdict(list) + self._max_steps = max_steps + self._step = 0 + self.reduced = False + + @classmethod + def from_metric_list(cls, max_steps, fields=(), metric_list=()): + fields = list(fields) + [str(metric).lower() for metric in metric_list] + return cls(fields, max_steps) + + def record(self, data: dict): + step = 1 + for key, value in data.items(): + if isinstance(value, list): + self._metrics[key].extend(value) + step = len(value) + else: + self._metrics[key].append(value) + self._step += step + + def update(self, data: "MetricLogs"): + self.record(data._metrics) + + def __getitem__(self, item): + return self._metrics[item] + + def log(self, prefix="", suffix="", **print_kwargs): + log_str = [] + if self._max_steps > 0: + log_str.append("Progress: {:>6.1%} ".format(self._step / self._max_steps)) + means = self.mean() + log_str += ["{}: {:.4f}".format(metric, metric_values) for metric, metric_values in means.items()] + print(prefix + " ".join(log_str) + suffix, **print_kwargs) + + def mean(self): + if self.reduced: + return self._metrics + return {k: mean(v) for k, v in self._metrics.items()} + + def reduce_mean(self): + self._metrics = {k: mean(v) for k, v in self._metrics.items()} + self.reduced = True diff --git a/brainforge/util/persistance.py b/brainforge/util/persistance.py index cb05f57..8cabf34 100644 --- a/brainforge/util/persistance.py +++ b/brainforge/util/persistance.py @@ -19,7 +19,7 @@ def dump(self, path): def encapsulate(cls, network, dumppath=None): capsule = cls(**{ "name": network.name, - "cost": network.cost, + "metrics": network.cost, "optimizer": network.optimizer, "architecture": network.layers.architecture[:], "layers": [layer.capsule() for layer in network.layers.layers]}) @@ -51,7 +51,7 @@ def __getitem__(self, item): def load(capsule): from ..learner import BackpropNetwork - from ..optimization import optimizers + from ..optimizers import optimizers from ..util.shame import translate_architecture as trsl if not isinstance(capsule, Capsule): diff --git a/brainforge/util/typing.py b/brainforge/util/typing.py index fa82710..4ef525d 100644 --- a/brainforge/util/typing.py +++ b/brainforge/util/typing.py @@ -35,3 +35,7 @@ def white(*dims, dtype=floatX) -> np.ndarray: def white_like(array, dtype=floatX): return white(*array.shape, dtype=dtype) + + +def emptyX(*dims): + return np.empty(dims, dtype=floatX) diff --git a/xperiments/xp_3conv.py b/xperiments/xp_3conv.py index fec8cbe..266cfb2 100644 --- a/xperiments/xp_3conv.py +++ b/xperiments/xp_3conv.py @@ -4,7 +4,7 @@ from brainforge.layers import ( ConvLayer, PoolLayer, DenseLayer, Activation, Flatten ) -from brainforge.optimization import Momentum +from brainforge.optimizers import Momentum from brainforge.util import batch_stream config.floatX = "float64" diff --git a/xperiments/xp_conv.py b/xperiments/xp_conv.py index 01fd7d0..15ba397 100644 --- a/xperiments/xp_conv.py +++ b/xperiments/xp_conv.py @@ -1,11 +1,12 @@ -import numpy as np +from verres.data import MNIST from brainforge.learner import BackpropNetwork from brainforge.layers import ConvLayer, PoolLayer, Flatten, DenseLayer, Activation -from brainforge.optimization import RMSprop +from brainforge.optimizers import RMSprop from brainforge.gradientcheck import GradientCheck -X, Y = np.random.randn(5, 3, 12, 12), np.ones((5, 2)) +X, Y = MNIST().table("train") +X = X[..., 0][:, None, ...] ins, ous = X.shape[1:], Y.shape[1:] net = BackpropNetwork(input_shape=ins, layerstack=[ ConvLayer(3, 3, 3, compiled=1), @@ -14,10 +15,10 @@ PoolLayer(2, compiled=1), Activation("tanh"), Flatten(), - DenseLayer(ous[0], activation="linear", trainable=False) + DenseLayer(ous[0], activation="linear", trainable=True) ], cost="mse", optimizer=RMSprop(eta=0.01)) -net.learn_batch(X, Y) +net.learn_batch(X[-5:], Y[-5:]) net.age += 1 -GradientCheck(net, epsilon=1e-5).run(X, Y, throw=True) +GradientCheck(net, epsilon=1e-5).run(X[:5], Y[:5], throw=True) diff --git a/xperiments/xp_decoupled.py b/xperiments/xp_decoupled.py index f30a8d1..cc21612 100644 --- a/xperiments/xp_decoupled.py +++ b/xperiments/xp_decoupled.py @@ -1,6 +1,6 @@ from brainforge.learner import BackpropNetwork from brainforge.layers import DenseLayer -from brainforge.optimization import Momentum +from brainforge.optimizers import Momentum from brainforge.util import etalon diff --git a/xperiments/xp_elm.py b/xperiments/xp_elm.py index 831e743..c273175 100644 --- a/xperiments/xp_elm.py +++ b/xperiments/xp_elm.py @@ -35,6 +35,6 @@ def pull_mnist(split=0.1): elm = ExtremeLearningMachine(layers, cost="mse") elm.learn_batch(tX, tY) pred = elm.predict(tX) -print("ELM cost:", elm.cost(pred, tY)) +print("ELM metrics:", elm.cost(pred, tY)) print("ELM acc :", np.mean(tY.argmax(axis=0) == pred.argmax(axis=0))) pass diff --git a/xperiments/xp_feedbackalignment.py b/xperiments/xp_feedbackalignment.py index 23e3365..b7ff58d 100644 --- a/xperiments/xp_feedbackalignment.py +++ b/xperiments/xp_feedbackalignment.py @@ -1,12 +1,13 @@ from brainforge.model import LayerStack from brainforge.learner import DirectFeedbackAlignment -from brainforge.layers import DenseLayer +from brainforge.layers import DenseLayer, Flatten -from csxdata.utilities.loader import pull_mnist_data +from verres.data import inmemory def build_layerstack(input_shape, output_dim): return LayerStack(input_shape, [ + Flatten(), DenseLayer(64, activation="tanh", compiled=True), DenseLayer(output_dim, activation="softmax")]) @@ -17,9 +18,13 @@ def build_dfa_net(input_shape, output_dim): def run_xperiment(): - lX, lY, tX, tY = pull_mnist_data() + mnist = inmemory.MNIST() + + lX, lY = mnist.table("train", shuffle=True) + tX, tY = mnist.table("val", shuffle=False) + ann = build_dfa_net(lX.shape[1:], lY.shape[1]) - ann.fit(lX, lY, batch_size=32, epochs=10, validation=(tX, tY)) + ann.fit(lX, lY, batch_size=32, epochs=10, validation=(tX, tY), metrics=["acc"]) if __name__ == '__main__': diff --git a/xperiments/xp_mnist.py b/xperiments/xp_mnist.py index 26c9758..7062ee7 100644 --- a/xperiments/xp_mnist.py +++ b/xperiments/xp_mnist.py @@ -1,22 +1,15 @@ -from verres.data import load_mnist +from verres.data import inmemory from brainforge import BackpropNetwork -from brainforge.layers import DenseLayer, ConvLayer, Activation, PoolLayer, Flatten +from brainforge.layers import DenseLayer, Flatten - -lX, lY, tX, tY = load_mnist() -lX = lX.transpose((0, 3, 1, 2)) -tX = tX.transpose((0, 3, 1, 2)) +mnist = inmemory.MNIST() +(lX, lY), (tX, tY) = mnist.table("train", shuffle=True), mnist.table("val", shuffle=False) ann = BackpropNetwork(input_shape=lX.shape[1:], layerstack=[ - ConvLayer(8), Activation("relu"), # 30 - ConvLayer(8), PoolLayer(2), Activation("relu"), # 28->14 - ConvLayer(16), Activation("relu"), # 12 - ConvLayer(16), PoolLayer(2), Activation("relu"), # 10->5 - ConvLayer(32), Activation("relu"), # 3 Flatten(), - DenseLayer(32, activation="relu"), + DenseLayer(64, activation="tanh"), DenseLayer(10, activation="softmax") ], cost="cxent", optimizer="adam") -ann.fit(lX, lY, 32, 10, validation=(tX, tY)) +ann.fit(lX, lY, 32, 10, metrics=["acc"], validation=(tX, tY)) diff --git a/xperiments/xp_pggymin.py b/xperiments/xp_pggymin.py index 7a81303..003d919 100644 --- a/xperiments/xp_pggymin.py +++ b/xperiments/xp_pggymin.py @@ -7,7 +7,7 @@ from brainforge.learner import BackpropNetwork from brainforge.layers import DenseLayer -from brainforge.optimization import Momentum +from brainforge.optimizers import Momentum from brainforge.reinforcement import PG, AgentConfig env = gym.make("CartPole-v0") diff --git a/xperiments/xp_qcartpole.py b/xperiments/xp_qcartpole.py index be39c6b..17687b0 100644 --- a/xperiments/xp_qcartpole.py +++ b/xperiments/xp_qcartpole.py @@ -5,7 +5,7 @@ from brainforge.learner import BackpropNetwork from brainforge.layers import DenseLayer, ClockworkLayer -from brainforge.optimization import RMSprop +from brainforge.optimizers import RMSprop from brainforge.reinforcement import AgentConfig, DDQN as AgentType from matplotlib import pyplot diff --git a/xperiments/xp_sin.py b/xperiments/xp_sin.py index 7fc25d6..2ea463d 100644 --- a/xperiments/xp_sin.py +++ b/xperiments/xp_sin.py @@ -42,12 +42,12 @@ plt.legend() batchno = 1 while 1: - tcost = net.learn_batch(tX, tY) + tmetrics = net.learn_batch(tX, tY) tpred = net.predict(tX) vpred = net.predict(vX) vcost = net.cost(vpred, vY) / len(vpred) tobj.set_data(tX, tpred) vobj.set_data(vX, vpred) plt.pause(0.1) - t.set_text(templ.format(batchno, tcost, vcost)) + t.set_text(templ.format(batchno, tmetrics["cost"], vcost)) batchno += 1 diff --git a/xperiments/xp_vae.py b/xperiments/xp_vae.py index 9d15070..1f2bb06 100644 --- a/xperiments/xp_vae.py +++ b/xperiments/xp_vae.py @@ -1,7 +1,5 @@ import numpy as np -from csxdata.utilities.loader import pull_mnist_data - -from homyd import FlatDataset +from verres.data import generators from brainforge import LayerStack, BackpropNetwork from brainforge.layers.abstract_layer import LayerBase, NoParamMixin @@ -52,7 +50,7 @@ def backpropagate(self, delta): DenseLayer(784, activation="linear")], cost="mse", optimizer="momentum") -ds = FlatDataset.from_multiarrays(*pull_mnist_data(split=0)) + for epoch in range(10): print("\n\nEpoch", epoch) for i, (x, y) in enumerate(ds.batch_stream(batchsize=32)): @@ -74,5 +72,4 @@ def backpropagate(self, delta): stdev_z.update(m) decoder.update(m) - print("\rDone: {:.2%} Cost: {:4f}".format(i*32 / len(ds), - decoder.cost(dcd, x)), end="") + print("\rDone: {:.2%} Cost: {:4f}".format(i*32 / len(ds), decoder.cost(dcd, x)), end="")