AEGIS: improve performance of AD absorption on x86_64

No apparent regression on other platforms. Adapted from libaegis.
jedisct1 · May 24, 2024 · 9d05bae · 9d05bae
1 parent 601eaea
commit 9d05bae
Show file tree

Hide file tree

Showing 2 changed files with 40 additions and 4 deletions.
diff --git a/src/libsodium/crypto_aead/aegis128l/aegis128l_common.h b/src/libsodium/crypto_aead/aegis128l/aegis128l_common.h
@@ -72,6 +72,19 @@ aegis128l_absorb(const uint8_t *const src, aes_block_t *const state)
     aegis128l_update(state, msg0, msg1);
 }
 
+static inline void
+aegis128l_absorb2(const uint8_t *const src, aes_block_t *const state)
+{
+    aes_block_t msg0, msg1, msg2, msg3;
+
+    msg0 = AES_BLOCK_LOAD(src + 0 * AES_BLOCK_LENGTH);
+    msg1 = AES_BLOCK_LOAD(src + 1 * AES_BLOCK_LENGTH);
+    msg2 = AES_BLOCK_LOAD(src + 2 * AES_BLOCK_LENGTH);
+    msg3 = AES_BLOCK_LOAD(src + 3 * AES_BLOCK_LENGTH);
+    aegis128l_update(state, msg0, msg1);
+    aegis128l_update(state, msg2, msg3);
+}
+
 static void
 aegis128l_enc(uint8_t *const dst, const uint8_t *const src, aes_block_t *const state)
 {
@@ -152,7 +165,10 @@ encrypt_detached(uint8_t *c, uint8_t *mac, size_t maclen, const uint8_t *m, size
 
     aegis128l_init(k, npub, state);
 
-    for (i = 0; i + RATE <= adlen; i += RATE) {
+    for (i = 0; i + RATE * 2 <= adlen; i += RATE * 2) {
+        aegis128l_absorb2(ad + i, state);
+    }
+    for (; i + RATE <= adlen; i += RATE) {
         aegis128l_absorb(ad + i, state);
     }
     if (adlen % RATE) {
@@ -189,7 +205,10 @@ decrypt_detached(uint8_t *m, const uint8_t *c, size_t clen, const uint8_t *mac,
 
     aegis128l_init(k, npub, state);
 
-    for (i = 0; i + RATE <= adlen; i += RATE) {
+    for (i = 0; i + RATE * 2 <= adlen; i += RATE * 2) {
+        aegis128l_absorb2(ad + i, state);
+    }
+    for (; i + RATE <= adlen; i += RATE) {
         aegis128l_absorb(ad + i, state);
     }
     if (adlen % RATE) {

diff --git a/src/libsodium/crypto_aead/aegis256/aegis256_common.h b/src/libsodium/crypto_aead/aegis256/aegis256_common.h
@@ -71,6 +71,17 @@ aegis256_absorb(const uint8_t *const src, aes_block_t *const state)
     aegis256_update(state, msg);
 }
 
+static inline void
+aegis256_absorb2(const uint8_t *const src, aes_block_t *const state)
+{
+    aes_block_t msg, msg2;
+
+    msg  = AES_BLOCK_LOAD(src + 0 * AES_BLOCK_LENGTH);
+    msg2 = AES_BLOCK_LOAD(src + 1 * AES_BLOCK_LENGTH);
+    aegis256_update(state, msg);
+    aegis256_update(state, msg2);
+}
+
 static void
 aegis256_enc(uint8_t *const dst, const uint8_t *const src, aes_block_t *const state)
 {
@@ -137,7 +148,10 @@ encrypt_detached(uint8_t *c, uint8_t *mac, size_t maclen, const uint8_t *m, size
 
     aegis256_init(k, npub, state);
 
-    for (i = 0; i + RATE <= adlen; i += RATE) {
+    for (i = 0; i + 2 * RATE <= adlen; i += 2 * RATE) {
+        aegis256_absorb2(ad + i, state);
+    }
+    for (; i + RATE <= adlen; i += RATE) {
         aegis256_absorb(ad + i, state);
     }
     if (adlen % RATE) {
@@ -174,7 +188,10 @@ decrypt_detached(uint8_t *m, const uint8_t *c, size_t clen, const uint8_t *mac,
 
     aegis256_init(k, npub, state);
 
-    for (i = 0; i + RATE <= adlen; i += RATE) {
+    for (i = 0; i + 2 * RATE <= adlen; i += 2 * RATE) {
+        aegis256_absorb2(ad + i, state);
+    }
+    for (; i + RATE <= adlen; i += RATE) {
         aegis256_absorb(ad + i, state);
     }
     if (adlen % RATE) {