Source-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[src/trunk]: src/sys/crypto/aes/arch/x86 Implement AES-CCM with x86 AES-NI.



details:   https://anonhg.NetBSD.org/src/rev/c363bc85578b
branches:  trunk
changeset: 1012181:c363bc85578b
user:      riastradh <riastradh%NetBSD.org@localhost>
date:      Sat Jul 25 22:29:06 2020 +0000

description:
Implement AES-CCM with x86 AES-NI.

diffstat:

 sys/crypto/aes/arch/x86/aes_ni.c    |   49 ++++++++++-
 sys/crypto/aes/arch/x86/aes_ni.h    |    9 +-
 sys/crypto/aes/arch/x86/aes_ni_64.S |  167 +++++++++++++++++++++++++++++++++++-
 3 files changed, 221 insertions(+), 4 deletions(-)

diffs (286 lines):

diff -r 059c21e508c4 -r c363bc85578b sys/crypto/aes/arch/x86/aes_ni.c
--- a/sys/crypto/aes/arch/x86/aes_ni.c  Sat Jul 25 22:28:27 2020 +0000
+++ b/sys/crypto/aes/arch/x86/aes_ni.c  Sat Jul 25 22:29:06 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: aes_ni.c,v 1.3 2020/07/25 22:12:57 riastradh Exp $     */
+/*     $NetBSD: aes_ni.c,v 1.4 2020/07/25 22:29:06 riastradh Exp $     */
 
 /*-
  * Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -27,7 +27,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(1, "$NetBSD: aes_ni.c,v 1.3 2020/07/25 22:12:57 riastradh Exp $");
+__KERNEL_RCSID(1, "$NetBSD: aes_ni.c,v 1.4 2020/07/25 22:29:06 riastradh Exp $");
 
 #ifdef _KERNEL
 #include <sys/types.h>
@@ -204,6 +204,48 @@
        fpu_kern_leave();
 }
 
+static void
+aesni_cbcmac_update1_impl(const struct aesenc *enc,
+    const uint8_t in[static 16], size_t nbytes, uint8_t auth[static 16],
+    uint32_t nrounds)
+{
+
+       KASSERT(nbytes);
+       KASSERT(nbytes % 16 == 0);
+
+       fpu_kern_enter();
+       aesni_cbcmac_update1(enc, in, nbytes, auth, nrounds);
+       fpu_kern_leave();
+}
+
+static void
+aesni_ccm_enc1_impl(const struct aesenc *enc, const uint8_t in[static 16],
+    uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32],
+    uint32_t nrounds)
+{
+
+       KASSERT(nbytes);
+       KASSERT(nbytes % 16 == 0);
+
+       fpu_kern_enter();
+       aesni_ccm_enc1(enc, in, out, nbytes, authctr, nrounds);
+       fpu_kern_leave();
+}
+
+static void
+aesni_ccm_dec1_impl(const struct aesenc *enc, const uint8_t in[static 16],
+    uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32],
+    uint32_t nrounds)
+{
+
+       KASSERT(nbytes);
+       KASSERT(nbytes % 16 == 0);
+
+       fpu_kern_enter();
+       aesni_ccm_dec1(enc, in, out, nbytes, authctr, nrounds);
+       fpu_kern_leave();
+}
+
 static int
 aesni_xts_update_selftest(void)
 {
@@ -273,4 +315,7 @@
        .ai_cbc_dec = aesni_cbc_dec_impl,
        .ai_xts_enc = aesni_xts_enc_impl,
        .ai_xts_dec = aesni_xts_dec_impl,
+       .ai_cbcmac_update1 = aesni_cbcmac_update1_impl,
+       .ai_ccm_enc1 = aesni_ccm_enc1_impl,
+       .ai_ccm_dec1 = aesni_ccm_dec1_impl,
 };
diff -r 059c21e508c4 -r c363bc85578b sys/crypto/aes/arch/x86/aes_ni.h
--- a/sys/crypto/aes/arch/x86/aes_ni.h  Sat Jul 25 22:28:27 2020 +0000
+++ b/sys/crypto/aes/arch/x86/aes_ni.h  Sat Jul 25 22:29:06 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: aes_ni.h,v 1.2 2020/07/25 22:12:57 riastradh Exp $     */
+/*     $NetBSD: aes_ni.h,v 1.3 2020/07/25 22:29:06 riastradh Exp $     */
 
 /*-
  * Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -66,6 +66,13 @@
            uint8_t[static 128], size_t, uint8_t[static 16], uint32_t);
 void   aesni_xts_update(const uint8_t[static 16], uint8_t[static 16]);
 
+void   aesni_cbcmac_update1(const struct aesenc *, const uint8_t[static 16],
+           size_t, uint8_t[static 16], uint32_t);
+void   aesni_ccm_enc1(const struct aesenc *, const uint8_t[static 16],
+           uint8_t[static 16], size_t, uint8_t[static 32], uint32_t);
+void   aesni_ccm_dec1(const struct aesenc *, const uint8_t[static 16],
+           uint8_t[static 16], size_t, uint8_t[static 32], uint32_t);
+
 extern struct aes_impl aes_ni_impl;
 
 #endif /* _CRYPTO_AES_ARCH_X86_AES_NI_H */
diff -r 059c21e508c4 -r c363bc85578b sys/crypto/aes/arch/x86/aes_ni_64.S
--- a/sys/crypto/aes/arch/x86/aes_ni_64.S       Sat Jul 25 22:28:27 2020 +0000
+++ b/sys/crypto/aes/arch/x86/aes_ni_64.S       Sat Jul 25 22:29:06 2020 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: aes_ni_64.S,v 1.3 2020/07/25 22:11:05 riastradh Exp $  */
+/*     $NetBSD: aes_ni_64.S,v 1.4 2020/07/25 22:29:06 riastradh Exp $  */
 
 /*-
  * Copyright (c) 2020 The NetBSD Foundation, Inc.
@@ -951,6 +951,142 @@
 END(aesni_xts_update)
 
 /*
+ * aesni_cbcmac_update1(const struct aesenc *enckey@rdi, const uint8_t *in@rsi,
+ *     size_t nbytes@rdx, uint8_t auth[16] @rcx, uint32_t nrounds@r8d)
+ *
+ *     Update CBC-MAC.
+ *
+ *     nbytes must be a positive integral multiple of 16.
+ *
+ *     Standard ABI calling convention.
+ */
+ENTRY(aesni_cbcmac_update1)
+       movdqu  (%rcx),%xmm0            /* xmm0 := auth */
+       mov     %rdx,%r10               /* r10 := nbytes */
+       mov     %rcx,%rdx               /* rdx := &auth */
+1:     pxor    (%rsi),%xmm0            /* xmm0 ^= plaintext block */
+       lea     0x10(%rsi),%rsi
+       mov     %r8d,%ecx               /* ecx := nrounds */
+       call    aesni_enc1              /* xmm0 := auth'; trash rax,rcx,xmm8 */
+       sub     $0x10,%r10
+       jnz     1b
+       movdqu  %xmm0,(%rdx)            /* store auth' */
+       ret
+END(aesni_cbcmac_update1)
+
+/*
+ * aesni_ccm_enc1(const struct aesenc *enckey@rdi, const uint8_t *in@rsi,
+ *     uint8_t *out@rdx, size_t nbytes@rcx,
+ *     uint8_t authctr[32] @r8, uint32_t nrounds@r9d)
+ *
+ *     Update CCM encryption.
+ *
+ *     nbytes must be a positive integral multiple of 16.
+ *
+ *     Standard ABI calling convention.
+ */
+ENTRY(aesni_ccm_enc1)
+       mov     %rcx,%r10               /* r10 := nbytes */
+       movdqu  0x10(%r8),%xmm2         /* xmm2 := ctr (be) */
+       movdqa  bswap32(%rip),%xmm4     /* xmm4 := bswap32 table */
+       movdqa  ctr32_inc(%rip),%xmm5   /* xmm5 := (0,0,0,1) (le) */
+       movdqu  (%r8),%xmm0             /* xmm0 := auth */
+       pshufb  %xmm4,%xmm2             /* xmm2 := ctr (le) */
+1:     movdqu  (%rsi),%xmm3            /* xmm3 := plaintext block */
+       paddd   %xmm5,%xmm2             /* increment ctr (32-bit) */
+       lea     0x10(%rsi),%rsi
+       movdqa  %xmm2,%xmm1             /* xmm1 := ctr (le) */
+       mov     %r9d,%ecx               /* ecx := nrounds */
+       pshufb  %xmm4,%xmm1             /* xmm1 := ctr (be) */
+       pxor    %xmm3,%xmm0             /* xmm0 := auth ^ ptxt */
+       call    aesni_enc2              /* trash rax/rcx/xmm8 */
+       pxor    %xmm1,%xmm3             /* xmm3 := ciphertext block */
+       sub     $0x10,%r10              /* count down bytes */
+       movdqu  %xmm3,(%rdx)            /* store ciphertext block */
+       lea     0x10(%rdx),%rdx
+       jnz     1b                      /* repeat if more blocks */
+       pshufb  %xmm4,%xmm2             /* xmm2 := ctr (be) */
+       movdqu  %xmm0,(%r8)             /* store updated auth */
+       movdqu  %xmm2,0x10(%r8)         /* store updated ctr */
+       ret
+END(aesni_ccm_enc1)
+
+/*
+ * aesni_ccm_dec1(const struct aesenc *enckey@rdi, const uint8_t *in@rsi,
+ *     uint8_t *out@rdx, size_t nbytes@rcx,
+ *     uint8_t authctr[32] @r8, uint32_t nrounds@r9d)
+ *
+ *     Update CCM decryption.
+ *
+ *     nbytes must be a positive integral multiple of 16.
+ *
+ *     Standard ABI calling convention.
+ */
+ENTRY(aesni_ccm_dec1)
+       movdqu  0x10(%r8),%xmm2         /* xmm2 := ctr (be) */
+       movdqa  bswap32(%rip),%xmm4     /* xmm4 := bswap32 table */
+       movdqa  ctr32_inc(%rip),%xmm5   /* xmm5 := (0,0,0,1) (le) */
+       movdqu  (%r8),%xmm1             /* xmm1 := auth */
+       pshufb  %xmm4,%xmm2             /* xmm2 := ctr (le) */
+       mov     %rcx,%r10               /* r10 := nbytes */
+
+       /* Decrypt the first block.  */
+       paddd   %xmm5,%xmm2             /* increment ctr (32-bit) */
+       mov     %r9d,%ecx               /* ecx := nrounds */
+       movdqa  %xmm2,%xmm0             /* xmm0 := ctr (le) */
+       movdqu  (%rsi),%xmm3            /* xmm3 := ctxt */
+       pshufb  %xmm4,%xmm0             /* xmm0 := ctr (be) */
+       lea     0x10(%rsi),%rsi
+       call    aesni_enc1              /* xmm0 := pad; trash rax/rcx/xmm8 */
+       jmp     2f
+
+1:     /*
+        * Authenticate the last block and decrypt the next block
+        * simultaneously.
+        *
+        *      xmm1 = auth ^ ptxt[-1]
+        *      xmm2 = ctr[-1] (le)
+        */
+       paddd   %xmm5,%xmm2             /* increment ctr (32-bit) */
+       mov     %r9d,%ecx               /* ecx := nrounds */
+       movdqa  %xmm2,%xmm0             /* xmm0 := ctr (le) */
+       movdqu  (%rsi),%xmm3            /* xmm3 := ctxt */
+       pshufb  %xmm4,%xmm0             /* xmm0 := ctr (be) */
+       lea     0x10(%rsi),%rsi
+       call    aesni_enc2              /* xmm0 := pad, xmm1 := auth';
+                                        * trash rax/rcx/xmm8 */
+2:     pxor    %xmm0,%xmm3             /* xmm3 := ptxt */
+       sub     $0x10,%r10
+       movdqu  %xmm3,(%rdx)            /* store plaintext */
+       lea     0x10(%rdx),%rdx
+       pxor    %xmm3,%xmm1             /* xmm1 := auth ^ ptxt */
+       jnz     1b
+
+       /* Authenticate the last block.  */
+       movdqa  %xmm1,%xmm0             /* xmm0 := auth ^ ptxt */
+       mov     %r9d,%ecx               /* ecx := nrounds */
+       call    aesni_enc1              /* xmm0 := auth' */
+       pshufb  %xmm4,%xmm2             /* xmm2 := ctr (be) */
+       movdqu  %xmm0,(%r8)             /* store updated auth */
+       movdqu  %xmm2,0x10(%r8)         /* store updated ctr */
+       ret
+END(aesni_ccm_dec1)
+
+       .section .rodata
+       .p2align 4
+       .type   bswap32,@object
+bswap32:
+       .byte   3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12
+END(bswap32)
+
+       .section .rodata
+       .p2align 4
+       .type   ctr32_inc,@object
+ctr32_inc:
+       .byte   0,0,0,0, 0,0,0,0, 0,0,0,0, 1,0,0,0
+END(ctr32_inc)
+
+/*
  * aesni_enc1(const struct aesenc *enckey@rdi, uint128_t block@xmm0,
  *     uint32_t nrounds@ecx)
  *
@@ -976,6 +1112,35 @@
 END(aesni_enc1)
 
 /*
+ * aesni_enc2(const struct aesenc *enckey@rdi, uint128_t block0@xmm0,
+ *     uint128_t block1@xmm1, uint32_t nrounds@ecx)
+ *
+ *     Encrypt two AES blocks in %xmm0 and %xmm1.
+ *
+ *     Internal ABI.  Uses %rax and %xmm8 as temporaries.  Destroys %ecx.
+ */
+       .text
+       _ALIGN_TEXT
+       .type   aesni_enc2,@function
+aesni_enc2:
+       movdqa  (%rdi),%xmm8    /* xmm8 := first round key */
+       shl     $4,%ecx         /* ecx := total byte size of round keys */
+       lea     0x10(%rdi,%rcx),%rax    /* rax := end of round key array */
+       neg     %rcx            /* rcx := byte offset of round key from end */
+       pxor    %xmm8,%xmm0     /* xor in first round key */
+       pxor    %xmm8,%xmm1
+       jmp     2f
+1:     aesenc  %xmm8,%xmm0
+       aesenc  %xmm8,%xmm1
+2:     movdqa  (%rax,%rcx),%xmm8       /* load round key */
+       add     $0x10,%rcx
+       jnz     1b              /* repeat if there's more */
+       aesenclast %xmm8,%xmm0
+       aesenclast %xmm8,%xmm1
+       ret
+END(aesni_enc2)
+
+/*
  * aesni_enc8(const struct aesenc *enckey@rdi, uint128_t block0@xmm0, ...,
  *     block7@xmm7, uint32_t nrounds@ecx)
  *



Home | Main Index | Thread Index | Old Index