Merge pull request #37 from rozukke/perf/alloc-infer

Improve perf by using less heap allocs

src/main.c

@@ -25,101 +25,51 @@ char letters[52] = {'A', 'a', 'B', 'b', 'C', 'c', 'D', 'd', 'E', 'e', 'F', 'f',
  'J', 'j', 'K', 'k', 'L', 'l', 'M', 'm', 'N', 'n', 'O', 'o', 'P', 'p', 'Q', 'q', 'R', 'r',
  'S', 's', 'T', 't', 'U', 'u', 'V', 'v', 'W', 'w', 'X', 'x', 'Y', 'y', 'Z', 'z'};
 
-void propagate_fwd(const matrix* weights, const vector* inputs, vector* results, const vector* biases) {
- sgemv_t_tuned(weights->data, inputs->data, results->data, weights->cols, weights->rows);
+void propagate_fwd(const matrix* weights, const float* inputs, float* results, const vector* biases) {
+ sgemv_t_tuned(weights->data, inputs, results, weights->cols, weights->rows);
  // Add biases onto results
- vector_add_inplace(biases->len, biases->data, results->data);
+ vector_add_inplace(biases->len, biases->data, results);
 }
 
-// Basic version, too many aligned_alloc
-u8 infer(vector* input) {
- vector* outputs[NUM_LAYERS];
- outputs[0] = new_vec_aligned(98);
- outputs[1] = new_vec_aligned(65);
- outputs[2] = new_vec_aligned(50);
- outputs[3] = new_vec_aligned(30);
- outputs[4] = new_vec_aligned(25);
- outputs[5] = new_vec_aligned(40);
- outputs[6] = new_vec_aligned(52);
-
- propagate_fwd(weights[0], input, outputs[0], biases[0]);
- relu_inplace(outputs[0]->data, 98);
- propagate_fwd(weights[1], outputs[0], outputs[1], biases[1]);
- relu_inplace(outputs[1]->data, 65);
- propagate_fwd(weights[2], outputs[1], outputs[2], biases[2]);
- relu_inplace(outputs[2]->data, 50);
- propagate_fwd(weights[3], outputs[2], outputs[3], biases[3]);
- relu_inplace(outputs[3]->data, 30);
- propagate_fwd(weights[4], outputs[3], outputs[4], biases[4]);
- relu_inplace(outputs[4]->data, 25);
- propagate_fwd(weights[5], outputs[4], outputs[5], biases[5]);
- relu_inplace(outputs[5]->data, 40);
- propagate_fwd(weights[6], outputs[5], outputs[6], biases[6]);
- softmax_inplace(outputs[6]->data, 52);
-
- u8 pred = argmax(outputs[6]->data, 52);
-
- free(outputs[0]->data);
- free(outputs[0]);
- free(outputs[1]->data);
- free(outputs[1]);
- free(outputs[2]->data);
- free(outputs[2]);
- free(outputs[3]->data);
- free(outputs[3]);
- free(outputs[4]->data);
- free(outputs[4]);
- free(outputs[5]->data);
- free(outputs[5]);
- free(outputs[6]->data);
- free(outputs[6]);
-
- return pred;
-}
-
-// Somewhat experimental, minumum number of alligned_alloc without breaking things.
-// This code fucking sucks but its fast so uhhhh
+// Minumum number of alligned_alloc without breaking things.
+// This code f***ing sucks but its fast so uhhhh
 u8 infer_reuse_layers_thread(vector* input, matrix** weights, vector** biases) {
- vector* outputs[2];
- outputs[0] = new_vec_aligned(98);
- outputs[1] = new_vec_aligned(65);
+ // Slightly larger than required for padding
+ float out0[104] __attribute__((aligned(SIMD_ALIGN))) = {0};
+ float out1[72] __attribute__((aligned(SIMD_ALIGN))) = {0};
 
- propagate_fwd(weights[0], input, outputs[0], biases[0]);
- relu_inplace(outputs[0]->data, 98);
- propagate_fwd(weights[1], outputs[0], outputs[1], biases[1]);
- relu_inplace(outputs[1]->data, 65);
+ propagate_fwd(weights[0], input->data, out0, biases[0]);
+ relu_inplace(out0, 98);
 
- memset(outputs[0]->data, 0, 50 * sizeof(f32));
+ propagate_fwd(weights[1], out0, out1, biases[1]);
+ relu_inplace(out1, 65);
 
- propagate_fwd(weights[2], outputs[1], outputs[0], biases[2]);
- relu_inplace(outputs[0]->data, 50);
+ memset(out0, 0, 50 * sizeof(f32));
 
- memset(outputs[1]->data, 0, 30 * sizeof(f32));
+ propagate_fwd(weights[2], out1, out0, biases[2]);
+ relu_inplace(out0, 50);
 
- propagate_fwd(weights[3], outputs[0], outputs[1], biases[3]);
- relu_inplace(outputs[1]->data, 30);
+ memset(out1, 0, 30 * sizeof(f32));
 
- memset(outputs[0]->data, 0, 25 * sizeof(f32));
+ propagate_fwd(weights[3], out0, out1, biases[3]);
+ relu_inplace(out1, 30);
 
- propagate_fwd(weights[4], outputs[1], outputs[0], biases[4]);
- relu_inplace(outputs[0]->data, 25);
+ memset(out0, 0, 25 * sizeof(f32));
 
- memset(outputs[1]->data, 0, 40 * sizeof(f32));
+ propagate_fwd(weights[4], out1, out0, biases[4]);
+ relu_inplace(out0, 25);
 
- propagate_fwd(weights[5], outputs[0], outputs[1], biases[5]);
- relu_inplace(outputs[1]->data, 40);
+ memset(out1, 0, 40 * sizeof(f32));
 
- memset(outputs[0]->data, 0, 52 * sizeof(f32));
+ propagate_fwd(weights[5], out0, out1, biases[5]);
+ relu_inplace(out1, 40);
 
- propagate_fwd(weights[6], outputs[1], outputs[0], biases[6]);
- softmax_inplace(outputs[0]->data, 52);
+ memset(out0, 0, 52 * sizeof(f32));
 
- u8 prediction = argmax(outputs[0]->data, 52);
+ propagate_fwd(weights[6], out1, out0, biases[6]);
+ softmax_inplace(out0, 52);
 
- free(outputs[0]->data);
- free(outputs[0]);
- free(outputs[1]->data);
- free(outputs[1]);
+ u8 prediction = argmax(out0, 52);
 
  return prediction;
 }