Source-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[src/trunk]: src/sys/crypto/aes/arch/x86 Implement AES-CCM with SSE2.



details:   https://anonhg.NetBSD.org/src/rev/7978431a9819
branches:  trunk
changeset: 974147:7978431a9819
user:      riastradh <riastradh%NetBSD.org@localhost>
date:      Sat Jul 25 22:29:56 2020 +0000

description:
Implement AES-CCM with SSE2.

diffstat:

 sys/crypto/aes/arch/x86/aes_sse2.h      |    8 +-
 sys/crypto/aes/arch/x86/aes_sse2_impl.c |   40 ++++++-
 sys/crypto/aes/arch/x86/aes_sse2_subr.c |  178 +++++++++++++++++++++++++++++++-
 3 files changed, 221 insertions(+), 5 deletions(-)

diffs (287 lines):

diff -r cac45f434f2c -r 7978431a9819 sys/crypto/aes/arch/x86/aes_sse2.h
--- a/sys/crypto/aes/arch/x86/aes_sse2.h        Sat Jul 25 22:29:06 2020 +0000
+++ b/sys/crypto/aes/arch/x86/aes_sse2.h        Sat Jul 25 22:29:56 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: aes_sse2.h,v 1.3 2020/07/25 22:12:57 riastradh Exp $   */
+/*     $NetBSD: aes_sse2.h,v 1.4 2020/07/25 22:29:56 riastradh Exp $   */
 
 /*-
  * Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -58,6 +58,12 @@
     uint8_t[static 16], size_t nbytes, uint8_t[static 16], uint32_t);
 void aes_sse2_xts_dec(const struct aesdec *, const uint8_t[static 16],
     uint8_t[static 16], size_t nbytes, uint8_t[static 16], uint32_t);
+void aes_sse2_cbcmac_update1(const struct aesenc *, const uint8_t[static 16],
+    size_t, uint8_t[static 16], uint32_t);
+void aes_sse2_ccm_enc1(const struct aesenc *, const uint8_t[static 16],
+    uint8_t[static 16], size_t, uint8_t[static 32], uint32_t);
+void aes_sse2_ccm_dec1(const struct aesenc *, const uint8_t[static 16],
+    uint8_t[static 16], size_t, uint8_t[static 32], uint32_t);
 
 int aes_sse2_selftest(void);
 
diff -r cac45f434f2c -r 7978431a9819 sys/crypto/aes/arch/x86/aes_sse2_impl.c
--- a/sys/crypto/aes/arch/x86/aes_sse2_impl.c   Sat Jul 25 22:29:06 2020 +0000
+++ b/sys/crypto/aes/arch/x86/aes_sse2_impl.c   Sat Jul 25 22:29:56 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: aes_sse2_impl.c,v 1.4 2020/07/25 22:12:57 riastradh Exp $      */
+/*     $NetBSD: aes_sse2_impl.c,v 1.5 2020/07/25 22:29:56 riastradh Exp $      */
 
 /*-
  * Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -27,7 +27,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(1, "$NetBSD: aes_sse2_impl.c,v 1.4 2020/07/25 22:12:57 riastradh Exp $");
+__KERNEL_RCSID(1, "$NetBSD: aes_sse2_impl.c,v 1.5 2020/07/25 22:29:56 riastradh Exp $");
 
 #include <sys/types.h>
 #include <sys/endian.h>
@@ -143,6 +143,39 @@
        fpu_kern_leave();
 }
 
+static void
+aes_sse2_cbcmac_update1_impl(const struct aesenc *enc,
+    const uint8_t in[static 16], size_t nbytes, uint8_t auth[static 16],
+    uint32_t nrounds)
+{
+
+       fpu_kern_enter();
+       aes_sse2_cbcmac_update1(enc, in, nbytes, auth, nrounds);
+       fpu_kern_leave();
+}
+
+static void
+aes_sse2_ccm_enc1_impl(const struct aesenc *enc, const uint8_t in[static 16],
+    uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32],
+    uint32_t nrounds)
+{
+
+       fpu_kern_enter();
+       aes_sse2_ccm_enc1(enc, in, out, nbytes, authctr, nrounds);
+       fpu_kern_leave();
+}
+
+static void
+aes_sse2_ccm_dec1_impl(const struct aesenc *enc, const uint8_t in[static 16],
+    uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32],
+    uint32_t nrounds)
+{
+
+       fpu_kern_enter();
+       aes_sse2_ccm_dec1(enc, in, out, nbytes, authctr, nrounds);
+       fpu_kern_leave();
+}
+
 static int
 aes_sse2_probe(void)
 {
@@ -182,4 +215,7 @@
        .ai_cbc_dec = aes_sse2_cbc_dec_impl,
        .ai_xts_enc = aes_sse2_xts_enc_impl,
        .ai_xts_dec = aes_sse2_xts_dec_impl,
+       .ai_cbcmac_update1 = aes_sse2_cbcmac_update1_impl,
+       .ai_ccm_enc1 = aes_sse2_ccm_enc1_impl,
+       .ai_ccm_dec1 = aes_sse2_ccm_dec1_impl,
 };
diff -r cac45f434f2c -r 7978431a9819 sys/crypto/aes/arch/x86/aes_sse2_subr.c
--- a/sys/crypto/aes/arch/x86/aes_sse2_subr.c   Sat Jul 25 22:29:06 2020 +0000
+++ b/sys/crypto/aes/arch/x86/aes_sse2_subr.c   Sat Jul 25 22:29:56 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: aes_sse2_subr.c,v 1.2 2020/06/30 20:32:11 riastradh Exp $      */
+/*     $NetBSD: aes_sse2_subr.c,v 1.3 2020/07/25 22:29:56 riastradh Exp $      */
 
 /*-
  * Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -27,7 +27,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(1, "$NetBSD: aes_sse2_subr.c,v 1.2 2020/06/30 20:32:11 riastradh Exp $");
+__KERNEL_RCSID(1, "$NetBSD: aes_sse2_subr.c,v 1.3 2020/07/25 22:29:56 riastradh Exp $");
 
 #ifdef _KERNEL
 #include <sys/systm.h>
@@ -518,6 +518,180 @@
        explicit_memset(t, 0, sizeof t);
 }
 
+void
+aes_sse2_cbcmac_update1(const struct aesenc *enc, const uint8_t in[static 16],
+    size_t nbytes, uint8_t auth[static 16], uint32_t nrounds)
+{
+       uint64_t sk_exp[120];
+       __m128i q[4];
+
+       KASSERT(nbytes);
+       KASSERT(nbytes % 16 == 0);
+
+       /* Expand round keys for bitslicing.  */
+       aes_sse2_skey_expand(sk_exp, nrounds, enc->aese_aes.aes_rk64);
+
+       /* Load initial authenticator.  */
+       q[0] = aes_sse2_interleave_in(_mm_loadu_epi8(auth));
+
+       for (; nbytes; nbytes -= 16, in += 16) {
+               q[0] ^= aes_sse2_interleave_in(_mm_loadu_epi8(in));
+               aes_sse2_ortho(q);
+               aes_sse2_bitslice_encrypt(nrounds, sk_exp, q);
+               aes_sse2_ortho(q);
+       }
+
+       /* Store updated authenticator.  */
+       _mm_storeu_epi8(auth, aes_sse2_interleave_out(q[0]));
+
+       /* Paranoia: Zero temporary buffers.  */
+       explicit_memset(sk_exp, 0, sizeof sk_exp);
+       explicit_memset(q, 0, sizeof q);
+}
+
+void
+aes_sse2_ccm_enc1(const struct aesenc *enc, const uint8_t in[static 16],
+    uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32],
+    uint32_t nrounds)
+{
+       uint64_t sk_exp[120];
+       __m128i q[4];
+       __m128i ctr;
+       uint32_t c0, c1, c2, c3;
+
+       KASSERT(nbytes);
+       KASSERT(nbytes % 16 == 0);
+
+       /* Expand round keys for bitslicing.  */
+       aes_sse2_skey_expand(sk_exp, nrounds, enc->aese_aes.aes_rk64);
+
+       /* Set first block to authenticator.  */
+       q[0] = aes_sse2_interleave_in(_mm_loadu_epi8(authctr));
+
+       /* Load initial counter block, big-endian so we can increment it.  */
+       c0 = le32dec(authctr + 16 + 4*0);
+       c1 = le32dec(authctr + 16 + 4*1);
+       c2 = le32dec(authctr + 16 + 4*2);
+       c3 = be32dec(authctr + 16 + 4*3);
+
+       /* Set other blocks to garbage -- can't take advantage.  */
+       q[2] = q[3] = _mm_setzero_si128();
+
+       for (; nbytes; nbytes -= 16, in += 16, out += 16) {
+               /* Update authenticator.  */
+               q[0] ^= aes_sse2_interleave_in(_mm_loadu_epi8(in));
+
+               /* Increment 32-bit counter.  */
+               ctr = _mm_set_epi32(bswap32(++c3), c2, c1, c0);
+               q[1] = aes_sse2_interleave_in(ctr);
+
+               /* Encrypt authenticator and counter.  */
+               aes_sse2_ortho(q);
+               aes_sse2_bitslice_encrypt(nrounds, sk_exp, q);
+               aes_sse2_ortho(q);
+
+               /* Encrypt with CTR output.  */
+               _mm_storeu_epi8(out,
+                   _mm_loadu_epi8(in) ^ aes_sse2_interleave_out(q[1]));
+       }
+
+       /* Update authenticator.  */
+       _mm_storeu_epi8(authctr, aes_sse2_interleave_out(q[0]));
+
+       /* Update counter.  */
+       be32enc(authctr + 16 + 4*3, c3);
+
+       /* Paranoia: Zero temporary buffers.  */
+       explicit_memset(sk_exp, 0, sizeof sk_exp);
+       explicit_memset(q, 0, sizeof q);
+}
+
+void
+aes_sse2_ccm_dec1(const struct aesenc *enc, const uint8_t in[static 16],
+    uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32],
+    uint32_t nrounds)
+{
+       uint64_t sk_exp[120];
+       __m128i q[4];
+       __m128i ctr, block;
+       uint32_t c0, c1, c2, c3;
+
+       KASSERT(nbytes);
+       KASSERT(nbytes % 16 == 0);
+
+       /* Expand round keys for bitslicing.  */
+       aes_sse2_skey_expand(sk_exp, nrounds, enc->aese_aes.aes_rk64);
+
+       /* Load initial counter block, big-endian so we can increment it.  */
+       c0 = le32dec(authctr + 16 + 4*0);
+       c1 = le32dec(authctr + 16 + 4*1);
+       c2 = le32dec(authctr + 16 + 4*2);
+       c3 = be32dec(authctr + 16 + 4*3);
+
+       /* Increment 32-bit counter.  */
+       ctr = _mm_set_epi32(bswap32(++c3), c2, c1, c0);
+       q[0] = aes_sse2_interleave_in(ctr);
+
+       /*
+        * Set the other blocks to garbage -- we don't have any
+        * plaintext to authenticate yet.
+        */
+       q[1] = q[2] = q[3] = _mm_setzero_si128();
+
+       /* Encrypt first CTR.  */
+       aes_sse2_ortho(q);
+       aes_sse2_bitslice_encrypt(nrounds, sk_exp, q);
+       aes_sse2_ortho(q);
+
+       /* Load the initial authenticator.  */
+       q[1] = aes_sse2_interleave_in(_mm_loadu_epi8(authctr));
+
+       for (;; in += 16, out += 16) {
+               /* Decrypt the block.  */
+               block = _mm_loadu_epi8(in) ^ aes_sse2_interleave_out(q[0]);
+
+               /* Update authenticator.  */
+               q[1] ^= aes_sse2_interleave_in(block);
+
+               /* Store plaintext.  */
+               _mm_storeu_epi8(out, block);
+
+               /* If this is the last block, stop.  */
+               if ((nbytes -= 16) == 0)
+                       break;
+
+               /* Increment 32-bit counter.  */
+               ctr = _mm_set_epi32(bswap32(++c3), c2, c1, c0);
+               q[0] = aes_sse2_interleave_in(ctr);
+
+               /* Authenticate previous plaintext, encrypt next CTR.  */
+               aes_sse2_ortho(q);
+               aes_sse2_bitslice_encrypt(nrounds, sk_exp, q);
+               aes_sse2_ortho(q);
+       }
+
+       /*
+        * Authenticate last plaintext.  We're only doing this for the
+        * authenticator, not for the counter, so don't bother to
+        * initialize q[0], q[2], q[3].  (Even for the sake of
+        * sanitizers, they're already initialized to something by
+        * now.)
+        */
+       aes_sse2_ortho(q);
+       aes_sse2_bitslice_encrypt(nrounds, sk_exp, q);
+       aes_sse2_ortho(q);
+
+       /* Update authenticator.  */
+       _mm_storeu_epi8(authctr, aes_sse2_interleave_out(q[1]));
+
+       /* Update counter.  */
+       be32enc(authctr + 16 + 4*3, c3);
+
+       /* Paranoia: Zero temporary buffers.  */
+       explicit_memset(sk_exp, 0, sizeof sk_exp);
+       explicit_memset(q, 0, sizeof q);
+}
+
 int
 aes_sse2_selftest(void)
 {



Home | Main Index | Thread Index | Old Index