diff --git a/CITATION.cff b/CITATION.cff index db9de09..a3cf1e8 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -20,5 +20,5 @@ keywords: - Neural network - Deep learning license: AGPL-3.0 -version: 2.4.1 -date-released: '2024-10-02' +version: 2.4.2 +date-released: '2024-10-07' diff --git a/Cargo.toml b/Cargo.toml index 0e30d2f..8174ea6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,10 +57,30 @@ path = "examples/ftir/mlp/looping.rs" name = "ftir-mlp-feedback" path = "examples/ftir/mlp/feedback.rs" +[[example]] +name = "ftir-cnn-plain" +path = "examples/ftir/cnn/plain.rs" + +[[example]] +name = "ftir-cnn-skip" +path = "examples/ftir/cnn/skip.rs" + +[[example]] +name = "ftir-cnn-loop" +path = "examples/ftir/cnn/looping.rs" + +[[example]] +name = "ftir-cnn-feedback" +path = "examples/ftir/cnn/feedback.rs" + [[example]] name = "mnist-plain" path = "examples/mnist/plain.rs" +[[example]] +name = "mnist-deconvolution" +path = "examples/mnist/deconvolution.rs" + [[example]] name = "mnist-skip" path = "examples/mnist/skip.rs" diff --git a/README.md b/README.md index bd2cbe0..8d3310b 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,10 @@ The package is divided into separate modules, each containing different parts of > > Describes the convolutional layer and its operations.
> > If the input is a tensor of shape `Single`, the layer will automatically reshape it into a `Triple` tensor. > +> #### deconvolution.rs +> > Describes the deconvolutional layer and its operations.
+> > If the input is a tensor of shape `Single`, the layer will automatically reshape it into a `Triple` tensor. +> > #### maxpool.rs > > Describes the maxpool layer and its operations.
> > If the input is a tensor of shape `Single`, the layer will automatically reshape it into a `Triple` tensor. @@ -168,6 +172,15 @@ fn main() { ## Releases +
+ v2.4.2 – Initial deconvolution layer. + + Initial implementation of the deconvolution layer. + Created with the good help of the GitHub Copilot. + Validated against corresponding PyTorch implementation; + * `documentation/validation/deconvolution.py` +
+
v2.4.1 – Bug-fixes. @@ -375,7 +388,16 @@ fn main() { Layer types - [x] Dense - - [x] Convolutional + - [x] Convolution + - [x] Forward pass + - [x] Padding + - [x] Stride + - [ ] Dilation + - [x] Backward pass + - [x] Padding + - [x] Stride + - [ ] Dilation + - [x] Deconvolution (#22) - [x] Forward pass - [x] Padding - [x] Stride @@ -384,7 +406,6 @@ fn main() { - [x] Padding - [x] Stride - [ ] Dilation - - [ ] Transposed convolution (#24) - [x] Max pooling - [x] Feedback
diff --git a/documentation/validation/deconvolution.py b/documentation/validation/deconvolution.py new file mode 100644 index 0000000..ba00f85 --- /dev/null +++ b/documentation/validation/deconvolution.py @@ -0,0 +1,61 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +# Define the deconvolution layer +class DeconvolutionLayer(nn.Module): + def __init__(self, in_channels, out_channels, kernel_size, stride, padding): + super(DeconvolutionLayer, self).__init__() + self.deconv = nn.ConvTranspose2d(in_channels, out_channels, kernel_size, stride, padding) + + def forward(self, x): + return self.deconv(x) + +# Test the backward pass +def test_backward_pass(): + # Define the input parameters + in_channels = 1 + out_channels = 1 + kernel_size = (3, 3) + stride = (2, 2) + padding = (1, 1) + + # Create the deconvolution layer + layer = DeconvolutionLayer(in_channels, out_channels, kernel_size, stride, padding) + + # Define the input tensor + input_tensor = torch.tensor([[[[1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + [13.0, 14.0, 15.0, 16.0]]]], requires_grad=True) + + # Define the gradient tensor + grad_tensor = torch.tensor([[[[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7], + [0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5], + [1.7, 1.8, 1.9, 2.0, 2.1, 2.2, 2.3], + [2.5, 2.6, 2.7, 2.8, 2.9, 3.0, 3.1], + [3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9], + [4.1, 4.2, 4.3, 4.4, 4.5, 4.6, 4.7], + [4.9, 5.0, 5.1, 5.2, 5.3, 5.4, 5.5]]]]) + + # Forward pass + output = layer(input_tensor) + + # Backward pass + output.backward(grad_tensor) + + # Get the gradients + input_grad = input_tensor.grad + weight_grad = layer.deconv.weight.grad + + # Check the shapes + assert input_grad.shape == input_tensor.shape + assert weight_grad.shape == layer.deconv.weight.shape + + print("Input gradient shape:", input_grad.shape) + print("Weight gradient shape:", weight_grad.shape) + + print("Kernel gradient:\n", weight_grad) + +# Run the test +test_backward_pass() diff --git a/examples/ftir/cnn/feedback.rs b/examples/ftir/cnn/feedback.rs new file mode 100644 index 0000000..bfe83d3 --- /dev/null +++ b/examples/ftir/cnn/feedback.rs @@ -0,0 +1,177 @@ +// Copyright (C) 2024 Hallvard Høyland Lavik + +use neurons::{activation, feedback, network, objective, optimizer, plot, tensor}; + +use std::{ + fs::File, + io::{BufRead, BufReader}, +}; + +fn data( + path: &str, +) -> ( + ( + Vec, + Vec, + Vec, + ), + ( + Vec, + Vec, + Vec, + ), + ( + Vec, + Vec, + Vec, + ), +) { + let reader = BufReader::new(File::open(&path).unwrap()); + + let mut x_train: Vec = Vec::new(); + let mut y_train: Vec = Vec::new(); + let mut class_train: Vec = Vec::new(); + + let mut x_test: Vec = Vec::new(); + let mut y_test: Vec = Vec::new(); + let mut class_test: Vec = Vec::new(); + + let mut x_val: Vec = Vec::new(); + let mut y_val: Vec = Vec::new(); + let mut class_val: Vec = Vec::new(); + + for line in reader.lines().skip(1) { + let line = line.unwrap(); + let record: Vec<&str> = line.split(',').collect(); + + let mut data: Vec = Vec::new(); + for i in 0..571 { + data.push(record.get(i).unwrap().parse::().unwrap()); + } + match record.get(573).unwrap() { + &"Train" => { + x_train.push(tensor::Tensor::single(data)); + y_train.push(tensor::Tensor::single(vec![record + .get(571) + .unwrap() + .parse::() + .unwrap()])); + class_train.push(tensor::Tensor::one_hot( + record.get(572).unwrap().parse::().unwrap() - 1, // For zero-indexed. + 28, + )); + } + &"Test" => { + x_test.push(tensor::Tensor::single(data)); + y_test.push(tensor::Tensor::single(vec![record + .get(571) + .unwrap() + .parse::() + .unwrap()])); + class_test.push(tensor::Tensor::one_hot( + record.get(572).unwrap().parse::().unwrap() - 1, // For zero-indexed. + 28, + )); + } + &"Val" => { + x_val.push(tensor::Tensor::single(data)); + y_val.push(tensor::Tensor::single(vec![record + .get(571) + .unwrap() + .parse::() + .unwrap()])); + class_val.push(tensor::Tensor::one_hot( + record.get(572).unwrap().parse::().unwrap() - 1, // For zero-indexed. + 28, + )); + } + _ => panic!("> Unknown class."), + } + } + + // let mut generator = random::Generator::create(12345); + // let mut indices: Vec = (0..x.len()).collect(); + // generator.shuffle(&mut indices); + + ( + (x_train, y_train, class_train), + (x_test, y_test, class_test), + (x_val, y_val, class_val), + ) +} + +fn main() { + // Load the ftir dataset + let ((x_train, y_train, class_train), (x_test, y_test, class_test), (x_val, y_val, class_val)) = + data("./examples/datasets/ftir.csv"); + + let x_train: Vec<&tensor::Tensor> = x_train.iter().collect(); + let _y_train: Vec<&tensor::Tensor> = y_train.iter().collect(); + let class_train: Vec<&tensor::Tensor> = class_train.iter().collect(); + + let x_test: Vec<&tensor::Tensor> = x_test.iter().collect(); + let y_test: Vec<&tensor::Tensor> = y_test.iter().collect(); + let class_test: Vec<&tensor::Tensor> = class_test.iter().collect(); + + let x_val: Vec<&tensor::Tensor> = x_val.iter().collect(); + let _y_val: Vec<&tensor::Tensor> = y_val.iter().collect(); + let class_val: Vec<&tensor::Tensor> = class_val.iter().collect(); + + println!("Train data {}x{}", x_train.len(), x_train[0].shape,); + println!("Test data {}x{}", x_test.len(), x_test[0].shape,); + println!("Validation data {}x{}", x_val.len(), x_val[0].shape,); + + // Create the network + let mut network = network::Network::new(tensor::Shape::Single(571)); + + network.dense(100, activation::Activation::ReLU, false, None); + network.convolution( + 1, + (3, 3), + (1, 1), + (1, 1), + activation::Activation::ReLU, + None, + ); + network.dense(28, activation::Activation::Softmax, false, None); + + network.set_accumulation(feedback::Accumulation::Mean); + + network.set_optimizer(optimizer::Adam::create(0.001, 0.9, 0.999, 1e-8, None)); + network.set_objective(objective::Objective::CrossEntropy, None); + + println!("{}", network); + + // Train the network + let (train_loss, val_loss, val_acc) = network.learn( + &x_train, + &class_train, + Some((&x_val, &class_val, 50)), + 16, + 500, + Some(100), + ); + plot::loss( + &train_loss, + &val_loss, + &val_acc, + "FEEDBACK : FTIR", + "./static/ftir-cnn-feedback.png", + ); + + // Validate the network + let (val_loss, val_acc) = network.validate(&x_test, &class_test, 1e-6); + println!( + "Final validation accuracy: {:.2} % and loss: {:.5}", + val_acc * 100.0, + val_loss + ); + + // Use the network + let prediction = network.predict(x_test.get(0).unwrap()); + println!( + "Prediction. Target: {}. Output: {}.", + class_test[0].argmax(), + prediction.argmax() + ); +} diff --git a/examples/ftir/cnn/looping.rs b/examples/ftir/cnn/looping.rs new file mode 100644 index 0000000..5efc42a --- /dev/null +++ b/examples/ftir/cnn/looping.rs @@ -0,0 +1,179 @@ +// Copyright (C) 2024 Hallvard Høyland Lavik + +use neurons::{activation, feedback, network, objective, optimizer, plot, tensor}; + +use std::sync::Arc; +use std::{ + fs::File, + io::{BufRead, BufReader}, +}; + +fn data( + path: &str, +) -> ( + ( + Vec, + Vec, + Vec, + ), + ( + Vec, + Vec, + Vec, + ), + ( + Vec, + Vec, + Vec, + ), +) { + let reader = BufReader::new(File::open(&path).unwrap()); + + let mut x_train: Vec = Vec::new(); + let mut y_train: Vec = Vec::new(); + let mut class_train: Vec = Vec::new(); + + let mut x_test: Vec = Vec::new(); + let mut y_test: Vec = Vec::new(); + let mut class_test: Vec = Vec::new(); + + let mut x_val: Vec = Vec::new(); + let mut y_val: Vec = Vec::new(); + let mut class_val: Vec = Vec::new(); + + for line in reader.lines().skip(1) { + let line = line.unwrap(); + let record: Vec<&str> = line.split(',').collect(); + + let mut data: Vec = Vec::new(); + for i in 0..571 { + data.push(record.get(i).unwrap().parse::().unwrap()); + } + match record.get(573).unwrap() { + &"Train" => { + x_train.push(tensor::Tensor::single(data)); + y_train.push(tensor::Tensor::single(vec![record + .get(571) + .unwrap() + .parse::() + .unwrap()])); + class_train.push(tensor::Tensor::one_hot( + record.get(572).unwrap().parse::().unwrap() - 1, // For zero-indexed. + 28, + )); + } + &"Test" => { + x_test.push(tensor::Tensor::single(data)); + y_test.push(tensor::Tensor::single(vec![record + .get(571) + .unwrap() + .parse::() + .unwrap()])); + class_test.push(tensor::Tensor::one_hot( + record.get(572).unwrap().parse::().unwrap() - 1, // For zero-indexed. + 28, + )); + } + &"Val" => { + x_val.push(tensor::Tensor::single(data)); + y_val.push(tensor::Tensor::single(vec![record + .get(571) + .unwrap() + .parse::() + .unwrap()])); + class_val.push(tensor::Tensor::one_hot( + record.get(572).unwrap().parse::().unwrap() - 1, // For zero-indexed. + 28, + )); + } + _ => panic!("> Unknown class."), + } + } + + // let mut generator = random::Generator::create(12345); + // let mut indices: Vec = (0..x.len()).collect(); + // generator.shuffle(&mut indices); + + ( + (x_train, y_train, class_train), + (x_test, y_test, class_test), + (x_val, y_val, class_val), + ) +} + +fn main() { + // Load the ftir dataset + let ((x_train, y_train, class_train), (x_test, y_test, class_test), (x_val, y_val, class_val)) = + data("./examples/datasets/ftir.csv"); + + let x_train: Vec<&tensor::Tensor> = x_train.iter().collect(); + let _y_train: Vec<&tensor::Tensor> = y_train.iter().collect(); + let class_train: Vec<&tensor::Tensor> = class_train.iter().collect(); + + let x_test: Vec<&tensor::Tensor> = x_test.iter().collect(); + let y_test: Vec<&tensor::Tensor> = y_test.iter().collect(); + let class_test: Vec<&tensor::Tensor> = class_test.iter().collect(); + + let x_val: Vec<&tensor::Tensor> = x_val.iter().collect(); + let _y_val: Vec<&tensor::Tensor> = y_val.iter().collect(); + let class_val: Vec<&tensor::Tensor> = class_val.iter().collect(); + + println!("Train data {}x{}", x_train.len(), x_train[0].shape,); + println!("Test data {}x{}", x_test.len(), x_test[0].shape,); + println!("Validation data {}x{}", x_val.len(), x_val[0].shape,); + + // Create the network + let mut network = network::Network::new(tensor::Shape::Single(571)); + + network.dense(100, activation::Activation::ReLU, false, None); + network.convolution( + 1, + (3, 3), + (1, 1), + (1, 1), + activation::Activation::ReLU, + None, + ); + network.dense(28, activation::Activation::Softmax, false, None); + + network.set_optimizer(optimizer::Adam::create(0.001, 0.9, 0.999, 1e-8, None)); + network.set_objective(objective::Objective::CrossEntropy, None); + + network.loopback(2, 1, Arc::new(|_| 1.0)); + network.set_accumulation(feedback::Accumulation::Mean); + + println!("{}", network); + + // Train the network + let (train_loss, val_loss, val_acc) = network.learn( + &x_train, + &class_train, + Some((&x_val, &class_val, 50)), + 40, + 500, + Some(100), + ); + plot::loss( + &train_loss, + &val_loss, + &val_acc, + "LOOP : FTIR", + "./static/ftir-cnn-loop.png", + ); + + // Validate the network + let (val_loss, val_acc) = network.validate(&x_test, &class_test, 1e-6); + println!( + "Final validation accuracy: {:.2} % and loss: {:.5}", + val_acc * 100.0, + val_loss + ); + + // Use the network + let prediction = network.predict(x_test.get(0).unwrap()); + println!( + "Prediction. Target: {}. Output: {}.", + class_test[0].argmax(), + prediction.argmax() + ); +} diff --git a/examples/ftir/cnn/plain.rs b/examples/ftir/cnn/plain.rs new file mode 100644 index 0000000..f4c4a75 --- /dev/null +++ b/examples/ftir/cnn/plain.rs @@ -0,0 +1,191 @@ +// Copyright (C) 2024 Hallvard Høyland Lavik + +use neurons::{activation, feedback, network, objective, optimizer, plot, tensor}; + +use std::{ + fs::File, + io::{BufRead, BufReader}, +}; + +fn data( + path: &str, +) -> ( + ( + Vec, + Vec, + Vec, + ), + ( + Vec, + Vec, + Vec, + ), + ( + Vec, + Vec, + Vec, + ), +) { + let reader = BufReader::new(File::open(&path).unwrap()); + + let mut x_train: Vec = Vec::new(); + let mut y_train: Vec = Vec::new(); + let mut class_train: Vec = Vec::new(); + + let mut x_test: Vec = Vec::new(); + let mut y_test: Vec = Vec::new(); + let mut class_test: Vec = Vec::new(); + + let mut x_val: Vec = Vec::new(); + let mut y_val: Vec = Vec::new(); + let mut class_val: Vec = Vec::new(); + + for line in reader.lines().skip(1) { + let line = line.unwrap(); + let record: Vec<&str> = line.split(',').collect(); + + let mut data: Vec = Vec::new(); + for i in 0..571 { + data.push(record.get(i).unwrap().parse::().unwrap()); + } + match record.get(573).unwrap() { + &"Train" => { + x_train.push(tensor::Tensor::single(data)); + y_train.push(tensor::Tensor::single(vec![record + .get(571) + .unwrap() + .parse::() + .unwrap()])); + class_train.push(tensor::Tensor::one_hot( + record.get(572).unwrap().parse::().unwrap() - 1, // For zero-indexed. + 28, + )); + } + &"Test" => { + x_test.push(tensor::Tensor::single(data)); + y_test.push(tensor::Tensor::single(vec![record + .get(571) + .unwrap() + .parse::() + .unwrap()])); + class_test.push(tensor::Tensor::one_hot( + record.get(572).unwrap().parse::().unwrap() - 1, // For zero-indexed. + 28, + )); + } + &"Val" => { + x_val.push(tensor::Tensor::single(data)); + y_val.push(tensor::Tensor::single(vec![record + .get(571) + .unwrap() + .parse::() + .unwrap()])); + class_val.push(tensor::Tensor::one_hot( + record.get(572).unwrap().parse::().unwrap() - 1, // For zero-indexed. + 28, + )); + } + _ => panic!("> Unknown class."), + } + } + + // let mut generator = random::Generator::create(12345); + // let mut indices: Vec = (0..x.len()).collect(); + // generator.shuffle(&mut indices); + + ( + (x_train, y_train, class_train), + (x_test, y_test, class_test), + (x_val, y_val, class_val), + ) +} + +fn main() { + // Load the ftir dataset + let ((x_train, y_train, class_train), (x_test, y_test, class_test), (x_val, y_val, class_val)) = + data("./examples/datasets/ftir.csv"); + + let x_train: Vec<&tensor::Tensor> = x_train.iter().collect(); + let _y_train: Vec<&tensor::Tensor> = y_train.iter().collect(); + let class_train: Vec<&tensor::Tensor> = class_train.iter().collect(); + + let x_test: Vec<&tensor::Tensor> = x_test.iter().collect(); + let _y_test: Vec<&tensor::Tensor> = y_test.iter().collect(); + let class_test: Vec<&tensor::Tensor> = class_test.iter().collect(); + + let x_val: Vec<&tensor::Tensor> = x_val.iter().collect(); + let _y_val: Vec<&tensor::Tensor> = y_val.iter().collect(); + let class_val: Vec<&tensor::Tensor> = class_val.iter().collect(); + + println!("Train data {}x{}", x_train.len(), x_train[0].shape,); + println!("Test data {}x{}", x_test.len(), x_test[0].shape,); + println!("Validation data {}x{}", x_val.len(), x_val[0].shape,); + + // Create the network + let mut network = network::Network::new(tensor::Shape::Single(571)); + network.dense(100, activation::Activation::ReLU, false, None); + network.convolution( + 1, + (3, 3), + (1, 1), + (0, 0), + activation::Activation::ReLU, + None, + ); + network.convolution( + 1, + (3, 3), + (1, 1), + (0, 0), + activation::Activation::ReLU, + None, + ); + network.convolution( + 1, + (3, 3), + (1, 1), + (0, 0), + activation::Activation::ReLU, + None, + ); + network.dense(100, activation::Activation::ReLU, false, None); + network.dense(28, activation::Activation::Softmax, false, None); + + network.set_optimizer(optimizer::Adam::create(0.001, 0.9, 0.999, 1e-8, None)); + network.set_objective(objective::Objective::CrossEntropy, None); + + println!("{}", network); + + // Train the network + let (train_loss, val_loss, val_acc) = network.learn( + &x_train, + &class_train, + Some((&x_val, &class_val, 50)), + 16, + 500, + Some(50), + ); + plot::loss( + &train_loss, + &val_loss, + &val_acc, + "PLAIN : FTIR", + "./static/ftir-cnn.png", + ); + + // Validate the network + let (val_loss, val_acc) = network.validate(&x_test, &class_test, 1e-6); + println!( + "Final validation accuracy: {:.2} % and loss: {:.5}", + val_acc * 100.0, + val_loss + ); + + // Use the network + let prediction = network.predict(x_test.get(0).unwrap()); + println!( + "Prediction. Target: {}. Output: {}.", + class_test[0].argmax(), + prediction.argmax() + ); +} diff --git a/examples/ftir/cnn/skip.rs b/examples/ftir/cnn/skip.rs new file mode 100644 index 0000000..21592f6 --- /dev/null +++ b/examples/ftir/cnn/skip.rs @@ -0,0 +1,177 @@ +// Copyright (C) 2024 Hallvard Høyland Lavik + +use neurons::{activation, feedback, network, objective, optimizer, plot, tensor}; + +use std::{ + fs::File, + io::{BufRead, BufReader}, +}; + +fn data( + path: &str, +) -> ( + ( + Vec, + Vec, + Vec, + ), + ( + Vec, + Vec, + Vec, + ), + ( + Vec, + Vec, + Vec, + ), +) { + let reader = BufReader::new(File::open(&path).unwrap()); + + let mut x_train: Vec = Vec::new(); + let mut y_train: Vec = Vec::new(); + let mut class_train: Vec = Vec::new(); + + let mut x_test: Vec = Vec::new(); + let mut y_test: Vec = Vec::new(); + let mut class_test: Vec = Vec::new(); + + let mut x_val: Vec = Vec::new(); + let mut y_val: Vec = Vec::new(); + let mut class_val: Vec = Vec::new(); + + for line in reader.lines().skip(1) { + let line = line.unwrap(); + let record: Vec<&str> = line.split(',').collect(); + + let mut data: Vec = Vec::new(); + for i in 0..571 { + data.push(record.get(i).unwrap().parse::().unwrap()); + } + match record.get(573).unwrap() { + &"Train" => { + x_train.push(tensor::Tensor::single(data)); + y_train.push(tensor::Tensor::single(vec![record + .get(571) + .unwrap() + .parse::() + .unwrap()])); + class_train.push(tensor::Tensor::one_hot( + record.get(572).unwrap().parse::().unwrap() - 1, // For zero-indexed. + 28, + )); + } + &"Test" => { + x_test.push(tensor::Tensor::single(data)); + y_test.push(tensor::Tensor::single(vec![record + .get(571) + .unwrap() + .parse::() + .unwrap()])); + class_test.push(tensor::Tensor::one_hot( + record.get(572).unwrap().parse::().unwrap() - 1, // For zero-indexed. + 28, + )); + } + &"Val" => { + x_val.push(tensor::Tensor::single(data)); + y_val.push(tensor::Tensor::single(vec![record + .get(571) + .unwrap() + .parse::() + .unwrap()])); + class_val.push(tensor::Tensor::one_hot( + record.get(572).unwrap().parse::().unwrap() - 1, // For zero-indexed. + 28, + )); + } + _ => panic!("> Unknown class."), + } + } + + // let mut generator = random::Generator::create(12345); + // let mut indices: Vec = (0..x.len()).collect(); + // generator.shuffle(&mut indices); + + ( + (x_train, y_train, class_train), + (x_test, y_test, class_test), + (x_val, y_val, class_val), + ) +} + +fn main() { + // Load the ftir dataset + let ((x_train, y_train, class_train), (x_test, y_test, class_test), (x_val, y_val, class_val)) = + data("./examples/datasets/ftir.csv"); + + let x_train: Vec<&tensor::Tensor> = x_train.iter().collect(); + let _y_train: Vec<&tensor::Tensor> = y_train.iter().collect(); + let class_train: Vec<&tensor::Tensor> = class_train.iter().collect(); + + let x_test: Vec<&tensor::Tensor> = x_test.iter().collect(); + let y_test: Vec<&tensor::Tensor> = y_test.iter().collect(); + let class_test: Vec<&tensor::Tensor> = class_test.iter().collect(); + + let x_val: Vec<&tensor::Tensor> = x_val.iter().collect(); + let _y_val: Vec<&tensor::Tensor> = y_val.iter().collect(); + let class_val: Vec<&tensor::Tensor> = class_val.iter().collect(); + + println!("Train data {}x{}", x_train.len(), x_train[0].shape,); + println!("Test data {}x{}", x_test.len(), x_test[0].shape,); + println!("Validation data {}x{}", x_val.len(), x_val[0].shape,); + + // Create the network + let mut network = network::Network::new(tensor::Shape::Single(571)); + + network.dense(100, activation::Activation::ReLU, false, None); + network.convolution( + 1, + (3, 3), + (1, 1), + (1, 1), + activation::Activation::ReLU, + None, + ); + network.dense(28, activation::Activation::Softmax, false, None); + + network.connect(1, 3); + + network.set_optimizer(optimizer::Adam::create(0.001, 0.9, 0.999, 1e-8, None)); + network.set_objective(objective::Objective::CrossEntropy, None); + + println!("{}", network); + + // Train the network + let (train_loss, val_loss, val_acc) = network.learn( + &x_train, + &class_train, + Some((&x_val, &class_val, 50)), + 40, + 500, + Some(100), + ); + plot::loss( + &train_loss, + &val_loss, + &val_acc, + "SKIP : FTIR", + "./static/ftir-cnn-skip.png", + ); + + // Validate the network + let (val_loss, val_acc) = network.validate(&x_test, &class_test, 1e-6); + println!( + "Final validation accuracy: {:.2} % and loss: {:.5}", + val_acc * 100.0, + val_loss + ); + + // Use the network + let prediction = network.predict(x_test.get(0).unwrap()); + println!( + "Prediction. Target: {}. Output: {}.", + class_test[0].argmax(), + prediction.argmax() + ); +} diff --git a/examples/ftir/mlp/feedback.rs b/examples/ftir/mlp/feedback.rs index 07670db..e813223 100644 --- a/examples/ftir/mlp/feedback.rs +++ b/examples/ftir/mlp/feedback.rs @@ -158,7 +158,7 @@ fn main() { &val_loss, &val_acc, "FEEDBACK : FTIR", - "./static/ftir-feedback.png", + "./static/ftir-mlp-feedback.png", ); // Validate the network diff --git a/examples/ftir/mlp/looping.rs b/examples/ftir/mlp/looping.rs index 68bb35f..522cc16 100644 --- a/examples/ftir/mlp/looping.rs +++ b/examples/ftir/mlp/looping.rs @@ -152,7 +152,7 @@ fn main() { &val_loss, &val_acc, "LOOP : FTIR", - "./static/ftir-loop.png", + "./static/ftir-mlp-loop.png", ); // Validate the network diff --git a/examples/ftir/mlp/plain.rs b/examples/ftir/mlp/plain.rs index fe7a3ad..764878f 100644 --- a/examples/ftir/mlp/plain.rs +++ b/examples/ftir/mlp/plain.rs @@ -148,7 +148,7 @@ fn main() { &val_loss, &val_acc, "PLAIN : FTIR", - "./static/ftir.png", + "./static/ftir-mlp.png", ); // Validate the network diff --git a/examples/ftir/mlp/skip.rs b/examples/ftir/mlp/skip.rs index 3e6d280..8656f9c 100644 --- a/examples/ftir/mlp/skip.rs +++ b/examples/ftir/mlp/skip.rs @@ -150,7 +150,7 @@ fn main() { &val_loss, &val_acc, "SKIP : FTIR", - "./static/ftir-skip.png", + "./static/ftir-mlp-skip.png", ); // Validate the network diff --git a/examples/mnist-fashion/feedback.rs b/examples/mnist-fashion/feedback.rs index 5960dcb..1a3d606 100644 --- a/examples/mnist-fashion/feedback.rs +++ b/examples/mnist-fashion/feedback.rs @@ -94,7 +94,7 @@ fn main() { (1, 1), None, )], - 3, + 4, true, ); network.convolution( diff --git a/examples/mnist-fashion/looping.rs b/examples/mnist-fashion/looping.rs index 7b3c531..ec88791 100644 --- a/examples/mnist-fashion/looping.rs +++ b/examples/mnist-fashion/looping.rs @@ -111,7 +111,7 @@ fn main() { Arc::new(|loops| 1.0 / loops), // Gradient scaling. ); network.set_accumulation( - feedback::Accumulation::Add, // How the pre- and post-activations are accumulated. + feedback::Accumulation::Mean, // How the pre- and post-activations are accumulated. ); network.set_optimizer(optimizer::SGD::create( diff --git a/examples/mnist/deconvolution.rs b/examples/mnist/deconvolution.rs new file mode 100644 index 0000000..faa2b51 --- /dev/null +++ b/examples/mnist/deconvolution.rs @@ -0,0 +1,158 @@ +// Copyright (C) 2024 Hallvard Høyland Lavik + +use neurons::{activation, network, objective, optimizer, plot, tensor}; + +use std::fs::File; +use std::io::{BufReader, Read, Result}; + +fn read(reader: &mut dyn Read) -> Result { + let mut buffer = [0; 4]; + reader.read_exact(&mut buffer)?; + Ok(u32::from_be_bytes(buffer)) +} + +fn load_mnist(path: &str) -> Result> { + let mut reader = BufReader::new(File::open(path)?); + let mut images: Vec = Vec::new(); + + let _magic_number = read(&mut reader)?; + let num_images = read(&mut reader)?; + let num_rows = read(&mut reader)?; + let num_cols = read(&mut reader)?; + + for _ in 0..num_images { + let mut image: Vec> = Vec::new(); + for _ in 0..num_rows { + let mut row: Vec = Vec::new(); + for _ in 0..num_cols { + let mut pixel = [0]; + reader.read_exact(&mut pixel)?; + row.push(pixel[0] as f32 / 255.0); + } + image.push(row); + } + images.push(tensor::Tensor::triple(vec![image]).resize(tensor::Shape::Triple(1, 14, 14))); + } + + Ok(images) +} + +fn load_labels(file_path: &str, numbers: usize) -> Result> { + let mut reader = BufReader::new(File::open(file_path)?); + let _magic_number = read(&mut reader)?; + let num_labels = read(&mut reader)?; + + let mut _labels = vec![0; num_labels as usize]; + reader.read_exact(&mut _labels)?; + + Ok(_labels + .iter() + .map(|&x| tensor::Tensor::one_hot(x as usize, numbers)) + .collect()) +} + +fn main() { + let x_train = load_mnist("./examples/datasets/mnist/train-images-idx3-ubyte").unwrap(); + let y_train = load_labels("./examples/datasets/mnist/train-labels-idx1-ubyte", 10).unwrap(); + let x_test = load_mnist("./examples/datasets/mnist/t10k-images-idx3-ubyte").unwrap(); + let y_test = load_labels("./examples/datasets/mnist/t10k-labels-idx1-ubyte", 10).unwrap(); + println!( + "Train: {} images, Test: {} images", + x_train.len(), + x_test.len() + ); + + let x_train: Vec<&tensor::Tensor> = x_train.iter().collect(); + let y_train: Vec<&tensor::Tensor> = y_train.iter().collect(); + let x_test: Vec<&tensor::Tensor> = x_test.iter().collect(); + let y_test: Vec<&tensor::Tensor> = y_test.iter().collect(); + + let mut network = network::Network::new(tensor::Shape::Triple(1, 14, 14)); + + network.convolution( + 1, + (3, 3), + (1, 1), + (0, 0), + activation::Activation::ReLU, + None, + ); + network.maxpool((2, 2), (2, 2)); + network.convolution( + 4, + (3, 3), + (1, 1), + (0, 0), + activation::Activation::ReLU, + None, + ); + network.deconvolution( + 4, + (3, 3), + (1, 1), + (0, 0), + activation::Activation::ReLU, + None, + ); + network.maxpool((2, 2), (2, 2)); + network.dense(10, activation::Activation::Softmax, true, None); + + network.set_optimizer(optimizer::SGD::create( + 0.0001, // Learning rate + None, // Decay + )); + network.set_objective( + objective::Objective::CrossEntropy, // Objective function + None, // Gradient clipping + ); + + println!("{}", network); + + // Train the network + let (train_loss, val_loss, val_acc) = network.learn( + &x_train, + &y_train, + Some((&x_test, &y_test, 10)), + 32, + 25, + Some(5), + ); + plot::loss( + &train_loss, + &val_loss, + &val_acc, + "PLAIN : MNIST", + "./static/mnist-deconvolution.png", + ); + + // Validate the network + let (val_loss, val_acc) = network.validate(&x_test, &y_test, 1e-6); + println!( + "Final validation accuracy: {:.2} % and loss: {:.5}", + val_acc * 100.0, + val_loss + ); + + // Use the network + let prediction = network.predict(x_test.get(0).unwrap()); + println!( + "Prediction on input: Target: {}. Output: {}.", + y_test[0].argmax(), + prediction.argmax() + ); + + let x = x_test.get(5).unwrap(); + let y = y_test.get(5).unwrap(); + plot::heatmap(&x, &format!("Target: {}", y.argmax()), "./static/input.png"); + + // Plot the pre- and post-activation heatmaps for each (image) layer. + // let (pre, post, _) = network.forward(x); + // for (i, (i_pre, i_post)) in pre.iter().zip(post.iter()).enumerate() { + // let pre_title = format!("layer_{}_pre", i); + // let post_title = format!("layer_{}_post", i); + // let pre_file = format!("layer_{}_pre.png", i); + // let post_file = format!("layer_{}_post.png", i); + // plot::heatmap(&i_pre, &pre_title, &pre_file); + // plot::heatmap(&i_post, &post_title, &post_file); + // } +} diff --git a/src/convolution.rs b/src/convolution.rs index 30ad8ed..501ca09 100644 --- a/src/convolution.rs +++ b/src/convolution.rs @@ -209,7 +209,9 @@ impl Convolution { for w in 0..kw { let _h = height * self.stride.0 + h; let _w = width * self.stride.1 + w; - sum += kernels[filter][c][h][w] * x[c][_h][_w]; + if _h < ih && _w < iw { + sum += kernels[filter][c][h][w] * x[c][_h][_w]; + } } } } @@ -265,7 +267,9 @@ impl Convolution { for n in 0..bw { let _h = m * self.stride.0 + k; let _w = n * self.stride.1 + l; - sum += a[j][_h][_w] * b[i][m][n]; + if _h < ah && _w < aw { + sum += a[j][_h][_w] * b[i][m][n]; + } } } y[i][j][k][l] = sum; diff --git a/src/deconvolution.rs b/src/deconvolution.rs new file mode 100644 index 0000000..9eef0e5 --- /dev/null +++ b/src/deconvolution.rs @@ -0,0 +1,497 @@ +// Copyright (C) 2024 Hallvard Høyland Lavik + +use crate::{activation, tensor}; + +use std::sync::Arc; + +/// A deconvolutional layer. +/// +/// # Attributes +/// +/// * `inputs` - The `tensor::Shape` of the input to the layer. +/// * `outputs` - The `tensor::Shape` of the output from the layer. +/// * `loops` - The number of loops to run the layer. +/// * `scale` - The scaling function of the loops. Default is `1.0 / x`. +/// * `kernels` - The kernels of the layer. +/// * `stride` - The stride of the filter. +/// * `padding` - The padding applied to the input before deconvolving. +/// * `activation` - The `activation::Function` of the layer. +/// * `dropout` - The dropout rate of the layer (when training). +/// * `flatten` - Whether the output should be flattened. +/// * `training` - Whether the layer is training. +#[derive(Clone)] +pub struct Deconvolution { + pub(crate) inputs: tensor::Shape, + pub(crate) outputs: tensor::Shape, + + pub(crate) loops: f32, + pub(crate) scale: tensor::Scale, + + pub(crate) kernels: Vec, + stride: (usize, usize), + padding: (usize, usize), + + pub(crate) activation: activation::Function, + + dropout: Option, + pub(crate) flatten: bool, + pub(crate) training: bool, +} + +impl std::fmt::Display for Deconvolution { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "Deconvolution{} (\n", self.activation)?; + write!(f, "\t\t\t{} -> {}\n", self.inputs, self.outputs)?; + write!( + f, + "\t\t\tkernel: {}x({})\n", + self.kernels.len(), + self.kernels[0].shape + )?; + write!(f, "\t\t\tstride: {:?}\n", self.stride)?; + write!(f, "\t\t\tpadding: {:?}\n", self.padding)?; + if self.dropout.is_some() { + write!(f, "\t\t\tdropout: {}\n", self.dropout.unwrap().to_string())?; + } + if self.loops > 1.0 { + write!( + f, + "\t\t\tloops: {} (scaling factor: {})\n", + self.loops, + (self.scale)(self.loops) + )?; + } + write!(f, "\t\t)")?; + Ok(()) + } +} + +impl Deconvolution { + /// Calculates the output size of the deconvolutional layer. + /// + /// # Arguments + /// + /// * `input` - The `tensor::Shape` of the input to the layer. + /// * `channels` - The number of output channels from the layer (i.e., number of filters). + /// * `kernel` - The size of each filter. + /// * `stride` - The stride of the filter. + /// * `padding` - The padding applied to the input before deconvolving. + /// + /// # Returns + /// + /// The `tensor::Shape` of the output from the layer. + fn calculate_output_size( + input: &tensor::Shape, + channels: &usize, + kernel: &(usize, usize), + stride: &(usize, usize), + padding: &(usize, usize), + ) -> tensor::Shape { + let input: &(usize, usize, usize) = match input { + tensor::Shape::Single(size) => { + let root = (*size as f32).sqrt() as usize; + &(1, root, root) + } + tensor::Shape::Triple(ch, he, wi) => &(*ch, *he, *wi), + _ => panic!("Incorrect input shape."), + }; + + let height = (input.1 - 1) * stride.0 + kernel.0 - 2 * padding.0; + let width = (input.2 - 1) * stride.1 + kernel.1 - 2 * padding.1; + + tensor::Shape::Triple(*channels, height, width) + } + + /// Creates a new deconvolutional layer with randomized kernel weights. + /// + /// # Arguments + /// + /// * `input` - The `tensor::Shape` of the input to the layer. + /// * `filters` - The number of output channels from the layer. + /// * `activation` - The `activation::Activation` function of the layer. + /// * `kernel` - The size of each filter. + /// * `stride` - The stride of the filter. + /// * `padding` - The padding applied to the input before deconvolving. + /// * `dropout` - The dropout rate of the layer (when training). + /// + /// # Returns + /// + /// A new layer with random weights with the given dimensions. + pub fn create( + inputs: tensor::Shape, + filters: usize, + activation: &activation::Activation, + kernel: (usize, usize), + stride: (usize, usize), + padding: (usize, usize), + dropout: Option, + ) -> Self { + let (inputs, ic) = match &inputs { + tensor::Shape::Single(size) => { + let root = (*size as f32).sqrt() as usize; + if size % root == 0 { + (tensor::Shape::Triple(1, root, root), 1) + } else { + panic!("> When adding a deconvolutional layer after a dense layer, the dense layer must have a square output.\n> Currently, the layer has {} outputs, which cannot cannot be reshaped to a (1, root[{}], root[{}]) tensor.\n> Try using {} or {} outputs for the preceding dense layer.", size, size, size, root*root, (root+1)*(root+1)); + } + } + tensor::Shape::Triple(ic, _, _) => (inputs.clone(), *ic), + _ => unimplemented!("Expected a `tensor::Tensor` input shape."), + }; + let outputs = Self::calculate_output_size(&inputs, &filters, &kernel, &stride, &padding); + Self { + inputs, + outputs, + kernels: (0..filters) + .map(|_| { + tensor::Tensor::random(tensor::Shape::Triple(ic, kernel.0, kernel.1), -1.0, 1.0) + }) + .collect(), + activation: activation::Function::create(&activation), + dropout, + stride, + padding, + training: false, + flatten: false, + loops: 1.0, + scale: Arc::new(|x| 1.0 / x), + } + } + + /// Extract the number of parameters in the layer. + pub fn parameters(&self) -> usize { + self.kernels.len() + * match self.kernels[0].data { + tensor::Data::Triple(ref tensor) => { + tensor.len() * tensor[0].len() * tensor[0][0].len() + } + _ => 0, + } + } + + /// Applies the forward pass (deconvolution) to the input `tensor::Tensor`. + /// Assumes `x` to match `self.inputs`, and for performance reasons does not check. + /// + /// # Arguments + /// + /// * `x` - The input `tensor::Tensor` to the layer. + /// + /// # Returns + /// + /// The pre- and post-activation `tensor::Tensor`s of the deconvolved input wrt. the kernels. + pub fn forward(&self, x: &tensor::Tensor) -> (tensor::Tensor, tensor::Tensor) { + // Extracting the data from the input `tensor::Tensor`. + let x = match &x.data { + tensor::Data::Single(vector) => { + let (h, w) = match &self.inputs { + tensor::Shape::Triple(_, h, w) => (*h, *w), + _ => panic!("Convolutional layers should have `tensor::Shape::Triple` input."), + }; + vector + .chunks_exact(h * w) + .map(|channel| channel.chunks_exact(w).map(|row| row.to_vec()).collect()) + .collect() + } + tensor::Data::Triple(tensor) => tensor.clone(), + _ => panic!("Unexpected input data type."), + }; + + // Extracting the weights from the kernels. + let kernels: Vec<&Vec>>> = self + .kernels + .iter() + .map(|ref k| match k.data { + tensor::Data::Triple(ref kernel) => kernel, + _ => panic!("Expected `tensor::Shape::Triple` kernel shape."), + }) + .collect(); + + let (ih, iw) = (x[0].len(), x[0][0].len()); + let (kf, kc, kh, kw) = ( + kernels.len(), + kernels[0].len(), + kernels[0][0].len(), + kernels[0][0][0].len(), + ); + + // Defining the output dimensions and vector. + let oh = (ih - 1) * self.stride.0 - 2 * self.padding.0 + kh; + let ow = (iw - 1) * self.stride.1 - 2 * self.padding.1 + kw; + let mut y = vec![vec![vec![0.0; ow]; oh]; kf]; + + // Deconvolving the input with the kernels. + for k in 0..kf { + for c in 0..kc { + for i in 0..ih { + for j in 0..iw { + for ki in 0..kh { + for kj in 0..kw { + let oi = i * self.stride.0 + ki; + let oi = match oi.checked_sub(self.padding.0) { + Some(value) => value, + None => { + continue; + } + }; + + let oj = j * self.stride.1 + kj; + let oj = match oj.checked_sub(self.padding.1) { + Some(value) => value, + None => { + continue; + } + }; + + if oi < oh && oj < ow { + y[k][oi][oj] += x[c][i][j] * kernels[k][c][ki][kj]; + } + } + } + } + } + } + } + + let pre = tensor::Tensor::triple(y); + let mut post = self.activation.forward(&pre); + + // Apply dropout if the network is training. + if self.training { + if let Some(dropout) = self.dropout { + post.dropout(dropout); + } + } + + if self.flatten { + post = post.flatten(); + } + + (pre, post) + } + + /// Applies the backward pass of the layer to the gradient `tensor::Tensor`. + /// + /// # Arguments + /// + /// * `gradient` - The gradient `tensor::Tensor` to the layer. + /// * `input` - The input `tensor::Tensor` to the layer. + /// * `output` - The output `tensor::Tensor` of the layer. + /// + /// # Returns + /// + /// The input-, weight- and bias gradient of the layer. + pub fn backward( + &self, + gradient: &tensor::Tensor, + input: &tensor::Tensor, + output: &tensor::Tensor, + ) -> (tensor::Tensor, tensor::Tensor, Option) { + let gradient = gradient.get_triple(&self.outputs); + let derivative = self.activation.backward(&output).get_triple(&self.outputs); + let delta = tensor::hadamard3d(&gradient, &derivative, (self.scale)(self.loops)); + + // Extracting the input and its dimensions. + let input = input.get_triple(&self.inputs); + let (ih, iw) = (input[0].len(), input[0][0].len()); + let (oh, ow) = (delta[0].len(), delta[0][0].len()); + + // Extracting the kernel and its dimensions. + let kernels: Vec>>> = self + .kernels + .iter() + .map(|k| match &k.data { + tensor::Data::Triple(ref kernel) => kernel.clone(), + _ => panic!("Expected `Tensor` kernel data."), + }) + .collect(); + + let (kf, kc, kh, kw) = ( + kernels.len(), + kernels[0].len(), + kernels[0][0].len(), + kernels[0][0][0].len(), + ); + + let mut kgradient = vec![vec![vec![vec![0.0; kw]; kh]; kc]; kf]; + let mut igradient = vec![vec![vec![0.0; iw]; ih]; kc]; + + for f in 0..kf { + for c in 0..kc { + for h in 0..ih { + for w in 0..iw { + for i in 0..kh { + for j in 0..kw { + let oi = h * self.stride.0 + i; + let oi = match oi.checked_sub(self.padding.0) { + Some(value) => value, + None => { + continue; + } + }; + + let oj = w * self.stride.1 + j; + let oj = match oj.checked_sub(self.padding.1) { + Some(value) => value, + None => { + continue; + } + }; + + if oi < oh && oj < ow { + igradient[c][h][w] += delta[f][oi][oj] * kernels[f][c][i][j]; + kgradient[f][c][i][j] += delta[f][oi][oj] * input[c][h][w]; + } + } + } + } + } + } + } + + ( + tensor::Tensor::triple(igradient), + tensor::Tensor::quadruple(kgradient), + None, + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::activation::Activation; + use crate::tensor::{Shape, Tensor}; + use crate::{assert_eq_data, assert_eq_shape}; + + #[test] + fn test_create_deconvolution_layer() { + let inputs = Shape::Triple(1, 2, 2); + let filters = 2; + let activation = Activation::ReLU; + let kernel = (2, 2); + let stride = (2, 2); + let padding = (0, 0); + let dropout = Some(0.5); + + let layer = Deconvolution::create( + inputs.clone(), + filters, + &activation, + kernel, + stride, + padding, + dropout, + ); + + assert_eq!(layer.inputs, inputs); + assert_eq!(layer.outputs, Shape::Triple(filters, 4, 4)); + assert_eq!(layer.kernels.len(), filters); + assert_eq!(layer.kernels[0].shape, Shape::Triple(1, 2, 2)); + assert_eq!(layer.dropout, dropout); + assert_eq!(layer.stride, stride); + assert_eq!(layer.padding, padding); + } + + #[test] + fn test_forward_pass() { + let inputs = Shape::Triple(1, 2, 2); + let filters = 1; + let activation = Activation::ReLU; + let kernel = (2, 2); + let stride = (2, 2); + let padding = (0, 0); + let dropout = Some(0.5); + + let mut layer = Deconvolution::create( + inputs.clone(), + filters, + &activation, + kernel, + stride, + padding, + dropout, + ); + + layer.kernels = vec![Tensor::triple(vec![vec![vec![1.0, 4.0], vec![2.0, 3.0]]])]; + + let input = Tensor::triple(vec![vec![vec![0.0, 1.0], vec![2.0, 3.0]]]); + + let (pre, post) = layer.forward(&input); + + assert_eq_shape!(pre.shape, layer.outputs); + + assert_eq!(pre.shape, Shape::Triple(1, 4, 4)); + assert_eq!(post.shape, Shape::Triple(1, 4, 4)); + + let output = Tensor::triple(vec![vec![ + vec![0.0, 0.0, 1.0, 4.0], + vec![0.0, 0.0, 2.0, 3.0], + vec![2.0, 8.0, 3.0, 12.0], + vec![4.0, 6.0, 6.0, 9.0], + ]]); + + assert_eq_data!(post.data, output.data); + } + + #[test] + fn test_backward_pass() { + let input_shape = Shape::Triple(1, 4, 4); + let filters = 1; + let activation = Activation::Linear; + let kernel = (3, 3); + let stride = (2, 2); + let padding = (1, 1); + let dropout = None; + + let layer = Deconvolution::create( + input_shape.clone(), + filters, + &activation, + kernel, + stride, + padding, + dropout, + ); + + let input = Tensor::triple(vec![vec![ + vec![1.0, 2.0, 3.0, 4.0], + vec![5.0, 6.0, 7.0, 8.0], + vec![9.0, 10.0, 11.0, 12.0], + vec![13.0, 14.0, 15.0, 16.0], + ]]); + + let grad = Tensor::triple(vec![vec![ + vec![0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7], + vec![0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5], + vec![1.7, 1.8, 1.9, 2.0, 2.1, 2.2, 2.3], + vec![2.5, 2.6, 2.7, 2.8, 2.9, 3.0, 3.1], + vec![3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9], + vec![4.1, 4.2, 4.3, 4.4, 4.5, 4.6, 4.7], + vec![4.9, 5.0, 5.1, 5.2, 5.3, 5.4, 5.5], + ]]); + + let (pre, post) = layer.forward(&input); + let (igrad, wgrad, _) = layer.backward(&grad, &input, &post); + + assert_eq!(igrad.shape, input_shape); + assert_eq!(wgrad.shape, Shape::Quadruple(1, 1, 3, 3)); + + let kgrad: Vec>> = vec![vec![ + vec![316.8000, 407.0000, 291.6000], + vec![400.0000, 512.8000, 366.4000], + vec![216.0000, 272.6000, 190.8000], + ]]; + let wdata = match wgrad.data { + crate::tensor::Data::Quadruple(data) => data[0].clone(), + _ => panic!("Invalid data type"), + }; + + for (k, w) in kgrad.iter().zip(wdata.iter()) { + for (k, w) in k.iter().zip(w.iter()) { + for (k, w) in k.iter().zip(w.iter()) { + assert!((k - w).abs() < 1e-4); + } + } + } + } +} diff --git a/src/feedback.rs b/src/feedback.rs index bb1c139..52a0b03 100644 --- a/src/feedback.rs +++ b/src/feedback.rs @@ -60,6 +60,14 @@ pub enum Layer { (usize, usize), Option, ), + Deconvolution( + usize, + activation::Activation, + (usize, usize), + (usize, usize), + (usize, usize), + Option, + ), Maxpool((usize, usize), (usize, usize)), } @@ -122,6 +130,13 @@ impl std::fmt::Display for Feedback { i, layer.activation, layer.inputs, layer.outputs )?; } + network::Layer::Deconvolution(layer) => { + write!( + f, + "\t\t\t\t{}: Decovolution{} ({} -> {})\n", + i, layer.activation, layer.inputs, layer.outputs + )?; + } network::Layer::Maxpool(layer) => { write!( f, @@ -176,12 +191,14 @@ impl Feedback { let inputs = match layers.first().unwrap() { network::Layer::Dense(dense) => dense.inputs.clone(), network::Layer::Convolution(convolution) => convolution.inputs.clone(), + network::Layer::Deconvolution(deconvolution) => deconvolution.inputs.clone(), network::Layer::Maxpool(maxpool) => maxpool.inputs.clone(), network::Layer::Feedback(_) => panic!("Nested feedback blocks are not supported."), }; let outputs = match layers.last().unwrap() { network::Layer::Dense(dense) => dense.outputs.clone(), network::Layer::Convolution(convolution) => convolution.outputs.clone(), + network::Layer::Deconvolution(deconvolution) => deconvolution.outputs.clone(), network::Layer::Maxpool(maxpool) => maxpool.outputs.clone(), network::Layer::Feedback(_) => panic!("Nested feedback blocks are not supported."), }; @@ -262,6 +279,19 @@ impl Feedback { layer.kernels.len() ]); } + network::Layer::Deconvolution(layer) => { + let (ch, kh, kw) = match layer.kernels[0].shape { + tensor::Shape::Triple(ch, he, wi) => (ch, he, wi), + _ => panic!("Expected Convolution shape"), + }; + vectors.push(vec![ + vec![ + tensor::Tensor::triple(vec![vec![vec![0.0; kw]; kh]; ch]), + // TODO: Add bias term here. + ]; + layer.kernels.len() + ]); + } network::Layer::Maxpool(_) => { vectors.push(vec![vec![tensor::Tensor::single(vec![0.0; 0])]]) } @@ -284,6 +314,7 @@ impl Feedback { parameters += match &self.layers[idx] { network::Layer::Dense(dense) => dense.parameters(), network::Layer::Convolution(convolution) => convolution.parameters(), + network::Layer::Deconvolution(deconvolution) => deconvolution.parameters(), network::Layer::Maxpool(_) => 0, network::Layer::Feedback(_) => panic!("Nested feedback blocks are not supported."), }; @@ -295,6 +326,7 @@ impl Feedback { self.layers.iter_mut().for_each(|layer| match layer { network::Layer::Dense(layer) => layer.training = train, network::Layer::Convolution(layer) => layer.training = train, + network::Layer::Deconvolution(layer) => layer.training = train, network::Layer::Maxpool(_) => {} network::Layer::Feedback(_) => panic!("Nested feedback blocks are not supported."), }); @@ -371,6 +403,13 @@ impl Feedback { activated.push(post); maxpools.push(None); } + network::Layer::Deconvolution(layer) => { + assert_eq_shape!(layer.inputs, x.shape); + let (pre, post) = layer.forward(&x); + unactivated.push(pre); + activated.push(post); + maxpools.push(None); + } network::Layer::Maxpool(layer) => { assert_eq_shape!(layer.inputs, x.shape); let (pre, post, max) = layer.forward(&x); @@ -453,6 +492,9 @@ impl Feedback { network::Layer::Convolution(layer) => { layer.backward(&gradients.last().unwrap(), input, output) } + network::Layer::Deconvolution(layer) => { + layer.backward(&gradients.last().unwrap(), input, output) + } _ => panic!("Unsupported layer type."), }; @@ -515,6 +557,17 @@ impl Feedback { // TODO: Add bias term here. } } + network::Layer::Deconvolution(layer) => { + for (f, (filter, gradient)) in layer + .kernels + .iter_mut() + .zip(weight_gradients[i].quadruple_to_vec_triple().iter_mut()) + .enumerate() + { + self.optimizer.update(i, f, false, stepnr, filter, gradient); + // TODO: Add bias term here. + } + } network::Layer::Maxpool(_) => {} network::Layer::Feedback(_) => panic!("Feedback layers are not supported."), }); @@ -538,6 +591,9 @@ impl Feedback { network::Layer::Convolution(layer) => { weights.push(tensor::Tensor::nested(layer.kernels.clone())); } + network::Layer::Deconvolution(layer) => { + weights.push(tensor::Tensor::nested(layer.kernels.clone())); + } _ => continue, } count += 1.0; @@ -596,6 +652,7 @@ impl Feedback { } Accumulation::Overwrite => { // Do nothing? + unimplemented!("Overwrite accumulation is not implemented.") } } @@ -611,6 +668,9 @@ impl Feedback { network::Layer::Convolution(layer) => { layer.kernels = weight.unnested(); } + network::Layer::Deconvolution(layer) => { + layer.kernels = weight.unnested(); + } _ => continue, } } diff --git a/src/lib.rs b/src/lib.rs index 3fd97e4..0d1cfda 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,6 +15,7 @@ pub mod objective; pub mod optimizer; pub mod convolution; +pub mod deconvolution; pub mod dense; pub mod feedback; pub mod maxpool; diff --git a/src/network.rs b/src/network.rs index f622b10..a413dd1 100644 --- a/src/network.rs +++ b/src/network.rs @@ -1,8 +1,8 @@ // Copyright (C) 2024 Hallvard Høyland Lavik use crate::{ - activation, assert_eq_shape, convolution, dense, feedback, maxpool, objective, optimizer, - tensor, + activation, assert_eq_shape, convolution, deconvolution, dense, feedback, maxpool, objective, + optimizer, tensor, }; use rayon::prelude::*; @@ -15,6 +15,7 @@ use std::sync::Arc; pub enum Layer { Dense(dense::Dense), Convolution(convolution::Convolution), + Deconvolution(deconvolution::Deconvolution), Maxpool(maxpool::Maxpool), Feedback(feedback::Feedback), } @@ -24,6 +25,7 @@ impl std::fmt::Display for Layer { match self { Layer::Dense(layer) => write!(f, "{}", layer), Layer::Convolution(layer) => write!(f, "{}", layer), + Layer::Deconvolution(layer) => write!(f, "{}", layer), Layer::Maxpool(layer) => write!(f, "{}", layer), Layer::Feedback(layer) => write!(f, "{}", layer), } @@ -36,6 +38,7 @@ impl Layer { match self { Layer::Dense(layer) => layer.parameters(), Layer::Convolution(layer) => layer.parameters(), + Layer::Deconvolution(layer) => layer.parameters(), Layer::Feedback(layer) => layer.parameters(), Layer::Maxpool(_) => 0, } @@ -186,6 +189,13 @@ impl Network { _ => panic!("Expected `tensor::Tensor` shape."), } } + Layer::Deconvolution(layer) => { + layer.flatten = true; + match layer.outputs { + tensor::Shape::Triple(ch, he, wi) => tensor::Shape::Single(ch * he * wi), + _ => panic!("Expected `tensor::Tensor` shape."), + } + } Layer::Maxpool(layer) => { layer.flatten = true; match layer.outputs { @@ -259,6 +269,67 @@ impl Network { match self.layers.last().unwrap() { Layer::Dense(layer) => layer.outputs.clone(), Layer::Convolution(layer) => layer.outputs.clone(), + Layer::Deconvolution(layer) => layer.outputs.clone(), + Layer::Maxpool(layer) => layer.outputs.clone(), + Layer::Feedback(layer) => layer.outputs.clone(), + }, + filters, + &activation, + kernel, + stride, + padding, + dropout, + ))); + } + + /// Adds a new deconvolutional layer to the network. + /// + /// The layer is added to the end of the network, and the number of inputs to the layer must + /// be equal to the number of outputs from the previous layer. The activation function of the + /// layer is set to the given activation function, and the layer may have a bias if specified. + /// + /// # Arguments + /// + /// * `filters` - The number of filters of the layer. + /// * `kernel` - The size of the kernel. + /// * `stride` - The stride of the kernel. + /// * `padding` - The padding of the input. + /// * `activation` - The `activation::Activation` function of the layer. + /// * `dropout` - The dropout rate of the layer (applied during training). + pub fn deconvolution( + &mut self, + filters: usize, + kernel: (usize, usize), + stride: (usize, usize), + padding: (usize, usize), + activation: activation::Activation, + dropout: Option, + ) { + if self.layers.is_empty() { + match self.input { + tensor::Shape::Triple(_, _, _) => (), + _ => panic!( + "Network is configured for dense inputs; the first layer cannot be convolutional. Modify the input shape to `tensor::Shape::Triple` or add a dense layer first." + ), + }; + self.layers + .push(Layer::Deconvolution(deconvolution::Deconvolution::create( + self.input.clone(), + filters, + &activation, + kernel, + stride, + padding, + dropout, + ))); + return; + } + self.layers + .push(Layer::Deconvolution(deconvolution::Deconvolution::create( + match self.layers.last().unwrap() { + Layer::Dense(layer) => layer.outputs.clone(), + Layer::Convolution(layer) => layer.outputs.clone(), + Layer::Deconvolution(layer) => layer.outputs.clone(), Layer::Maxpool(layer) => layer.outputs.clone(), Layer::Feedback(layer) => layer.outputs.clone(), }, @@ -296,6 +367,7 @@ impl Network { match self.layers.last().unwrap() { Layer::Dense(layer) => layer.outputs.clone(), Layer::Convolution(layer) => layer.outputs.clone(), + Layer::Deconvolution(layer) => layer.outputs.clone(), Layer::Maxpool(layer) => layer.outputs.clone(), Layer::Feedback(layer) => layer.outputs.clone(), }, @@ -332,6 +404,7 @@ impl Network { match self.layers.last().unwrap() { Layer::Dense(layer) => layer.outputs.clone(), Layer::Convolution(layer) => layer.outputs.clone(), + Layer::Deconvolution(layer) => layer.outputs.clone(), Layer::Maxpool(layer) => layer.outputs.clone(), Layer::Feedback(layer) => layer.outputs.clone(), } @@ -364,6 +437,22 @@ impl Network { *padding, *dropout, )), + feedback::Layer::Deconvolution( + filters, + activation, + kernel, + stride, + padding, + dropout, + ) => Layer::Deconvolution(deconvolution::Deconvolution::create( + input.clone(), + *filters, + activation, + *kernel, + *stride, + *padding, + *dropout, + )), feedback::Layer::Maxpool(kernel, stride) => { Layer::Maxpool(maxpool::Maxpool::create(input.clone(), *kernel, *stride)) } @@ -371,6 +460,7 @@ impl Network { input = match _layers.last().unwrap() { Layer::Dense(layer) => layer.outputs.clone(), Layer::Convolution(layer) => layer.outputs.clone(), + Layer::Deconvolution(layer) => layer.outputs.clone(), Layer::Maxpool(layer) => layer.outputs.clone(), Layer::Feedback(layer) => layer.outputs.clone(), }; @@ -403,12 +493,14 @@ impl Network { let inputs = match &self.layers[to] { Layer::Dense(layer) => &layer.inputs, Layer::Convolution(layer) => &layer.inputs, + Layer::Deconvolution(layer) => &layer.inputs, Layer::Maxpool(layer) => &layer.inputs, Layer::Feedback(feedback) => &feedback.inputs, }; let outputs = match &self.layers[from] { Layer::Dense(layer) => &layer.outputs, Layer::Convolution(layer) => &layer.outputs, + Layer::Deconvolution(layer) => &layer.outputs, Layer::Maxpool(layer) => &layer.outputs, Layer::Feedback(feedback) => &feedback.outputs, }; @@ -425,6 +517,10 @@ impl Network { layer.scale = Arc::clone(&scale); layer.loops += 1.0 } + Layer::Deconvolution(layer) => { + layer.scale = Arc::clone(&scale); + layer.loops += 1.0 + } Layer::Maxpool(layer) => layer.loops += 1.0, Layer::Feedback(_) => panic!("Loop connection includes feedback block."), } @@ -454,12 +550,14 @@ impl Network { let outof = match &self.layers[from] { Layer::Dense(layer) => &layer.inputs, Layer::Convolution(layer) => &layer.inputs, + Layer::Deconvolution(layer) => &layer.inputs, Layer::Maxpool(_) => panic!("Skip connection from Maxpool layer not supported."), Layer::Feedback(feedback) => &feedback.inputs, }; let into = match &self.layers[to] { Layer::Dense(layer) => &layer.inputs, Layer::Convolution(layer) => &layer.inputs, + Layer::Deconvolution(layer) => &layer.inputs, Layer::Maxpool(layer) => &layer.inputs, Layer::Feedback(feedback) => &feedback.inputs, }; @@ -508,6 +606,9 @@ impl Network { Layer::Convolution(ref mut layer) => { layer.activation = activation::Function::create(&activation) } + Layer::Deconvolution(ref mut layer) => { + layer.activation = activation::Function::create(&activation) + } _ => panic!("Maxpool layers do not use activation functions!"), } } @@ -549,6 +650,19 @@ impl Network { layer.kernels.len() ]); } + Layer::Deconvolution(layer) => { + let (ch, kh, kw) = match layer.kernels[0].shape { + tensor::Shape::Triple(ch, he, wi) => (ch, he, wi), + _ => panic!("Expected Deconvolution shape"), + }; + vectors.push(vec![ + vec![ + tensor::Tensor::triple(vec![vec![vec![0.0; kw]; kh]; ch]), + // TODO: Add bias term here. + ]; + layer.kernels.len() + ]); + } Layer::Maxpool(_) => vectors.push(vec![vec![tensor::Tensor::single(vec![0.0; 0])]]), Layer::Feedback(_) => { vectors.push(vec![vec![tensor::Tensor::single(vec![0.0; 0])]]) @@ -641,6 +755,7 @@ impl Network { self.layers.iter_mut().for_each(|layer| match layer { Layer::Dense(layer) => layer.training = true, Layer::Convolution(layer) => layer.training = true, + Layer::Deconvolution(layer) => layer.training = true, Layer::Feedback(feedback) => feedback.training(true), _ => (), }); @@ -663,13 +778,13 @@ impl Network { let results: Vec<_> = batch .into_par_iter() .map(|(input, target)| { - let (unactivated, activated, maxpools, feedbacks) = self.forward(input); + let (preactivated, activated, maxpools, feedbacks) = self.forward(input); let (loss, gradient) = self.objective.loss(&activated.last().unwrap(), target); let (wg, bg) = self.backward( gradient, - &unactivated, + &preactivated, &activated, &maxpools, &feedbacks, @@ -763,6 +878,7 @@ impl Network { match layer { Layer::Dense(layer) => layer.training = false, Layer::Convolution(layer) => layer.training = false, + Layer::Deconvolution(layer) => layer.training = false, Layer::Feedback(feedback) => feedback.training(false), _ => (), } @@ -780,7 +896,7 @@ impl Network { /// /// # Returns /// - /// * A vector of unactivated tensors. + /// * A vector of preactivated tensors. /// * A vector of activated tensors. /// * A vector of maxpool tensors. /// * A nested vector of intermediate feedback block tensors. @@ -793,7 +909,7 @@ impl Network { Vec>, Vec>, ) { - let mut unactivated: Vec = Vec::new(); + let mut preactivated: Vec = Vec::new(); let mut activated: Vec = vec![input.clone()]; let mut maxpools: Vec> = Vec::new(); let mut feedbacks: Vec> = Vec::new(); @@ -830,7 +946,7 @@ impl Network { let (mut pre, mut post, mut max, fbs) = self._forward(&x, i, i + 1); // Store the outputs of the current layer. - unactivated.append(&mut pre); + preactivated.append(&mut pre); activated.append(&mut post); maxpools.append(&mut max); feedbacks.extend(fbs); @@ -843,6 +959,7 @@ impl Network { current = current.reshape(match self.layers[self.loopbacks[&i]] { Layer::Dense(ref layer) => layer.inputs.clone(), Layer::Convolution(ref layer) => layer.inputs.clone(), + Layer::Deconvolution(ref layer) => layer.inputs.clone(), Layer::Maxpool(ref layer) => layer.inputs.clone(), _ => panic!("Feedback not implemented for this layer type."), }); @@ -874,7 +991,7 @@ impl Network { for (idx, j) in (self.loopbacks[&i]..i + 1).enumerate() { match self.accumulation { feedback::Accumulation::Add => { - unactivated[j].add_inplace(&fpre[idx]); + preactivated[j].add_inplace(&fpre[idx]); activated[j + 1].add_inplace(&fpost[idx]); // Extend the maxpool indices. @@ -887,7 +1004,7 @@ impl Network { } } feedback::Accumulation::Subtract => { - unactivated[j].sub_inplace(&fpre[idx]); + preactivated[j].sub_inplace(&fpre[idx]); activated[j + 1].sub_inplace(&fpost[idx]); // Extend the maxpool indices. @@ -900,7 +1017,7 @@ impl Network { } } feedback::Accumulation::Multiply => { - unactivated[j].mul_inplace(&fpre[idx]); + preactivated[j].mul_inplace(&fpre[idx]); activated[j + 1].mul_inplace(&fpost[idx]); // Extend the maxpool indices. @@ -913,7 +1030,7 @@ impl Network { } } feedback::Accumulation::Overwrite => { - unactivated[j] = fpre[idx].to_owned(); + preactivated[j] = fpre[idx].to_owned(); activated[j + 1] = fpost[idx].to_owned(); // Overwrite the maxpool indices. @@ -926,7 +1043,7 @@ impl Network { } } feedback::Accumulation::Mean => { - unactivated[j].mean_inplace(&fpre[idx]); + preactivated[j].mean_inplace(&fpre[idx]); activated[j + 1].mean_inplace(&fpost[idx]); // Extend the maxpool indices. @@ -945,7 +1062,7 @@ impl Network { } } - (unactivated, activated, maxpools, feedbacks) + (preactivated, activated, maxpools, feedbacks) } /// Compute the forward pass for the specified range of layers. @@ -958,7 +1075,7 @@ impl Network { /// /// # Returns /// - /// * A vector of unactivated tensors. + /// * A vector of preactivated tensors. /// * A vector of activated tensors. /// * A vector of maxpool tensors. /// * A nested vector of intermediate feedback block tensors. @@ -973,7 +1090,7 @@ impl Network { Vec>, Vec>, ) { - let mut unactivated: Vec = Vec::new(); + let mut preactivated: Vec = Vec::new(); let mut activated: Vec = vec![input.clone()]; let mut maxpools: Vec> = Vec::new(); let mut feedbacks: Vec> = Vec::new(); @@ -984,28 +1101,33 @@ impl Network { Layer::Dense(layer) => { assert_eq_shape!(layer.inputs, x.shape); let (pre, post) = layer.forward(x); - unactivated.push(pre); + preactivated.push(pre); activated.push(post); maxpools.push(None); } Layer::Convolution(layer) => { - assert_eq_shape!(layer.inputs, x.shape); let (pre, post) = layer.forward(x); - unactivated.push(pre); + preactivated.push(pre); + activated.push(post); + maxpools.push(None); + } + Layer::Deconvolution(layer) => { + let (pre, post) = layer.forward(x); + preactivated.push(pre); activated.push(post); maxpools.push(None); } Layer::Maxpool(layer) => { assert_eq_shape!(layer.inputs, x.shape); let (pre, post, max) = layer.forward(x); - unactivated.push(pre); + preactivated.push(pre); activated.push(post); maxpools.push(Some(max)); } Layer::Feedback(block) => { assert_eq_shape!(block.inputs, x.shape); let (pre, post, max, fbpre, fbpost) = block.forward(x); - unactivated.push(pre); + preactivated.push(pre); activated.push(post); maxpools.push(Some(max)); feedbacks.push(vec![fbpre, fbpost]); @@ -1017,7 +1139,7 @@ impl Network { // As this is present in the `forward` function. activated.remove(0); - (unactivated, activated, maxpools, feedbacks) + (preactivated, activated, maxpools, feedbacks) } /// Compute the backward pass of the network for the given output gradient. @@ -1025,7 +1147,7 @@ impl Network { /// # Arguments /// /// * `gradient` - The gradient of the output. - /// * `unactivated` - The pre-activation values of each layer. + /// * `preactivated` - The pre-activation values of each layer. /// * `activated` - The post-activation values of each layer. /// * `maxpools` - The maxpool indices of each maxpool-layer. /// @@ -1035,7 +1157,7 @@ impl Network { fn backward( &self, gradient: tensor::Tensor, - unactivated: &Vec, + preactivated: &Vec, activated: &Vec, maxpools: &Vec>, feedbacks: &Vec>, @@ -1054,7 +1176,7 @@ impl Network { let idx = self.layers.len() - i - 1; let input: &tensor::Tensor = &activated[idx]; - let output: &tensor::Tensor = &unactivated[idx]; + let output: &tensor::Tensor = &preactivated[idx]; // Check for skip connections. // Add the gradient of the skip connection to the current gradient. @@ -1069,6 +1191,9 @@ impl Network { Layer::Convolution(layer) => { layer.backward(&gradients.last().unwrap(), input, output) } + Layer::Deconvolution(layer) => { + layer.backward(&gradients.last().unwrap(), input, output) + } Layer::Maxpool(layer) => ( layer.backward( &gradients.last().unwrap(), @@ -1144,6 +1269,17 @@ impl Network { // TODO: Add bias term here. } } + Layer::Deconvolution(layer) => { + for (f, (filter, gradient)) in layer + .kernels + .iter_mut() + .zip(weight_gradients[i].quadruple_to_vec_triple().iter_mut()) + .enumerate() + { + self.optimizer.update(i, f, false, stepnr, filter, gradient); + // TODO: Add bias term here. + } + } Layer::Maxpool(_) => {} Layer::Feedback(block) => block.update( stepnr, @@ -1186,6 +1322,7 @@ impl Network { layer.training = false } Layer::Convolution(layer) => layer.training = false, + Layer::Deconvolution(layer) => layer.training = false, Layer::Feedback(feedback) => feedback.training(false), _ => (), } @@ -1230,6 +1367,9 @@ impl Network { Layer::Convolution(_) => { unimplemented!("Image output (target) not supported.") } + Layer::Deconvolution(_) => { + unimplemented!("Image output (target) not supported.") + } Layer::Maxpool(_) => { unimplemented!("Image output (target) not supported.") } @@ -1245,6 +1385,7 @@ impl Network { match layer { Layer::Dense(layer) => layer.training = true, Layer::Convolution(layer) => layer.training = true, + Layer::Deconvolution(layer) => layer.training = true, Layer::Feedback(feedback) => feedback.training(true), _ => (), } @@ -1288,6 +1429,10 @@ impl Network { let (_, out) = layer.forward(&output); output = out; } + Layer::Deconvolution(layer) => { + let (_, out) = layer.forward(&output); + output = out; + } Layer::Maxpool(layer) => { let (_, out, _) = layer.forward(&output); output = out; diff --git a/src/tensor.rs b/src/tensor.rs index c62f06d..a451c2c 100644 --- a/src/tensor.rs +++ b/src/tensor.rs @@ -1324,6 +1324,28 @@ pub fn pad3d(data: &Vec>>, into: (usize, usize)) -> Vec>>, + into: (usize, usize), + stride: (usize, usize), +) -> Vec>> { + let mut upsampled = vec![vec![vec![0.0; into.1]; into.0]; input.len()]; + + for (c, channel) in input.iter().enumerate() { + for (i, row) in channel.iter().enumerate() { + for (j, &val) in row.iter().enumerate() { + let h = i * stride.0; + let w = j * stride.1; + if h < into.0 && w < into.1 { + upsampled[c][h][w] = val; + } + } + } + } + + upsampled +} + /// Element-wise multiplication of two tensors. /// For performance reasons, this function does not validate the length of the tensors. /// It is assumed that the tensors have the same length.