Source-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[src-draft/trunk]: src/sys New SSE2-based bitsliced AES implementation.



details:   https://anonhg.NetBSD.org/src-all/rev/2b94962b2c03
branches:  trunk
changeset: 935060:2b94962b2c03
user:      Taylor R Campbell <riastradh%NetBSD.org@localhost>
date:      Sat Jun 20 02:02:41 2020 +0000

description:
New SSE2-based bitsliced AES implementation.

This should work on essentially all x86 CPUs of the last two decades,
and may improve throughput over the portable C aes_ct implementation
from BearSSL by

(a) reducing the number of vector operations in sequence, and
(b) batching four rather than two blocks in parallel.

Derived from BearSSL'S aes_ct64 implementation adjusted so that where
aes_ct64 uses 64-bit q[0],...,q[7], aes_sse2 uses (q[0], q[4]), ...,
(q[3], q[7]), each tuple representing a pair of 64-bit quantities
stacked in a single 128-bit register.  This translation was done very
naively, and mostly reduces the cost of ShiftRows and data movement
without doing anything to address the S-box or (Inv)MixColumns, which
spread all 64-bit quantities across separate registers and ignore the
upper halves.

Unfortunately, SSE2 -- which is all that is guaranteed on all amd64
CPUs -- doesn't have PSHUFB, which would help out a lot more.  For
example, vpaes relies on that.  Perhaps there are enough CPUs out
there with PSHUFB but not AES-NI to make it worthwhile to import or
adapt vpaes too.

diffstat:

 sys/arch/x86/conf/files.x86             |    3 +
 sys/arch/x86/x86/identcpu.c             |    3 +
 sys/crypto/aes/aes.h                    |    7 +-
 sys/crypto/aes/arch/x86/aes_sse2.c      |  398 ++++++++++++++++++++
 sys/crypto/aes/arch/x86/aes_sse2.h      |   36 +
 sys/crypto/aes/arch/x86/aes_sse2_dec.c  |  178 +++++++++
 sys/crypto/aes/arch/x86/aes_sse2_enc.c  |  136 +++++++
 sys/crypto/aes/arch/x86/aes_sse2_impl.c |  611 ++++++++++++++++++++++++++++++++
 sys/crypto/aes/arch/x86/aes_sse2_impl.h |   47 ++
 sys/crypto/aes/arch/x86/files.aessse2   |   11 +
 sys/crypto/aes/arch/x86/immintrin.h     |  228 +++++++++++
 11 files changed, 1655 insertions(+), 3 deletions(-)

diffs (truncated from 1735 to 300 lines):

diff -r 9dec7559737d -r 2b94962b2c03 sys/arch/x86/conf/files.x86
--- a/sys/arch/x86/conf/files.x86       Wed Jun 17 02:47:43 2020 +0000
+++ b/sys/arch/x86/conf/files.x86       Sat Jun 20 02:02:41 2020 +0000
@@ -171,3 +171,6 @@
 
 # VIA ACE
 include "crypto/aes/arch/x86/files.aesvia"
+
+# Bitsliced AES with SSE2
+include "crypto/aes/arch/x86/files.aessse2"
diff -r 9dec7559737d -r 2b94962b2c03 sys/arch/x86/x86/identcpu.c
--- a/sys/arch/x86/x86/identcpu.c       Wed Jun 17 02:47:43 2020 +0000
+++ b/sys/arch/x86/x86/identcpu.c       Sat Jun 20 02:02:41 2020 +0000
@@ -40,6 +40,7 @@
 #include <sys/cpu.h>
 
 #include <crypto/aes/arch/x86/aes_ni.h>
+#include <crypto/aes/arch/x86/aes_sse2.h>
 #include <crypto/aes/arch/x86/aes_via.h>
 
 #include <uvm/uvm_extern.h>
@@ -1005,6 +1006,8 @@
 #endif
                if (cpu_feature[4] & CPUID_VIA_HAS_ACE)
                        aes_md_init(&aes_via_impl);
+               else if (i386_has_sse && i386_has_sse2)
+                       aes_md_init(&aes_sse2_impl);
        } else {
                /*
                 * If not first. Warn about cpu_feature mismatch for
diff -r 9dec7559737d -r 2b94962b2c03 sys/crypto/aes/aes.h
--- a/sys/crypto/aes/aes.h      Wed Jun 17 02:47:43 2020 +0000
+++ b/sys/crypto/aes/aes.h      Sat Jun 20 02:02:41 2020 +0000
@@ -37,8 +37,9 @@
  *
  *     Expanded round keys.
  */
-struct aes {
+union aes {
        uint32_t        aes_rk[60];
+       uint64_t        aes_rk64[30];
 } __aligned(16);
 
 #define        AES_128_NROUNDS 10
@@ -46,11 +47,11 @@
 #define        AES_256_NROUNDS 14
 
 struct aesenc {
-       struct aes      aese_aes;
+       union aes       aese_aes;
 };
 
 struct aesdec {
-       struct aes      aesd_aes;
+       union aes       aesd_aes;
 };
 
 struct aes_impl {
diff -r 9dec7559737d -r 2b94962b2c03 sys/crypto/aes/arch/x86/aes_sse2.c
--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
+++ b/sys/crypto/aes/arch/x86/aes_sse2.c        Sat Jun 20 02:02:41 2020 +0000
@@ -0,0 +1,398 @@
+/*
+ * Copyright (c) 2016 Thomas Pornin <pornin%bolet.org@localhost>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining 
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be 
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <sys/cdefs.h>
+__KERNEL_RCSID(1, "$NetBSD$");
+
+#include <sys/types.h>
+
+#include <lib/libkern/libkern.h>
+
+#include "aes_sse2_impl.h"
+
+static void
+br_range_dec32le(uint32_t *p32, size_t nwords, const void *v)
+{
+       const uint8_t *p8 = v;
+
+       while (nwords --> 0) {
+               uint32_t x0 = *p8++;
+               uint32_t x1 = *p8++;
+               uint32_t x2 = *p8++;
+               uint32_t x3 = *p8++;
+
+               *p32++ = x0 | (x1 << 8) | (x2 << 16) | (x3 << 24);
+       }
+}
+
+void
+aes_sse2_bitslice_Sbox(__m128i q[static 4])
+{
+       __m128i x0, x1, x2, x3, x4, x5, x6, x7;
+       __m128i y1, y2, y3, y4, y5, y6, y7, y8, y9;
+       __m128i y10, y11, y12, y13, y14, y15, y16, y17, y18, y19;
+       __m128i y20, y21;
+       __m128i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9;
+       __m128i z10, z11, z12, z13, z14, z15, z16, z17;
+       __m128i t0, t1, t2, t3, t4, t5, t6, t7, t8, t9;
+       __m128i t10, t11, t12, t13, t14, t15, t16, t17, t18, t19;
+       __m128i t20, t21, t22, t23, t24, t25, t26, t27, t28, t29;
+       __m128i t30, t31, t32, t33, t34, t35, t36, t37, t38, t39;
+       __m128i t40, t41, t42, t43, t44, t45, t46, t47, t48, t49;
+       __m128i t50, t51, t52, t53, t54, t55, t56, t57, t58, t59;
+       __m128i t60, t61, t62, t63, t64, t65, t66, t67;
+       __m128i s0, s1, s2, s3, s4, s5, s6, s7;
+
+       x0 = _mm_shuffle_epi32(q[3], 0x0e);
+       x1 = _mm_shuffle_epi32(q[2], 0x0e);
+       x2 = _mm_shuffle_epi32(q[1], 0x0e);
+       x3 = _mm_shuffle_epi32(q[0], 0x0e);
+       x4 = q[3];
+       x5 = q[2];
+       x6 = q[1];
+       x7 = q[0];
+
+       /*
+        * Top linear transformation.
+        */
+       y14 = x3 ^ x5;
+       y13 = x0 ^ x6;
+       y9 = x0 ^ x3;
+       y8 = x0 ^ x5;
+       t0 = x1 ^ x2;
+       y1 = t0 ^ x7;
+       y4 = y1 ^ x3;
+       y12 = y13 ^ y14;
+       y2 = y1 ^ x0;
+       y5 = y1 ^ x6;
+       y3 = y5 ^ y8;
+       t1 = x4 ^ y12;
+       y15 = t1 ^ x5;
+       y20 = t1 ^ x1;
+       y6 = y15 ^ x7;
+       y10 = y15 ^ t0;
+       y11 = y20 ^ y9;
+       y7 = x7 ^ y11;
+       y17 = y10 ^ y11;
+       y19 = y10 ^ y8;
+       y16 = t0 ^ y11;
+       y21 = y13 ^ y16;
+       y18 = x0 ^ y16;
+
+       /*
+        * Non-linear section.
+        */
+       t2 = y12 & y15;
+       t3 = y3 & y6;
+       t4 = t3 ^ t2;
+       t5 = y4 & x7;
+       t6 = t5 ^ t2;
+       t7 = y13 & y16;
+       t8 = y5 & y1;
+       t9 = t8 ^ t7;
+       t10 = y2 & y7;
+       t11 = t10 ^ t7;
+       t12 = y9 & y11;
+       t13 = y14 & y17;
+       t14 = t13 ^ t12;
+       t15 = y8 & y10;
+       t16 = t15 ^ t12;
+       t17 = t4 ^ t14;
+       t18 = t6 ^ t16;
+       t19 = t9 ^ t14;
+       t20 = t11 ^ t16;
+       t21 = t17 ^ y20;
+       t22 = t18 ^ y19;
+       t23 = t19 ^ y21;
+       t24 = t20 ^ y18;
+
+       t25 = t21 ^ t22;
+       t26 = t21 & t23;
+       t27 = t24 ^ t26;
+       t28 = t25 & t27;
+       t29 = t28 ^ t22;
+       t30 = t23 ^ t24;
+       t31 = t22 ^ t26;
+       t32 = t31 & t30;
+       t33 = t32 ^ t24;
+       t34 = t23 ^ t33;
+       t35 = t27 ^ t33;
+       t36 = t24 & t35;
+       t37 = t36 ^ t34;
+       t38 = t27 ^ t36;
+       t39 = t29 & t38;
+       t40 = t25 ^ t39;
+
+       t41 = t40 ^ t37;
+       t42 = t29 ^ t33;
+       t43 = t29 ^ t40;
+       t44 = t33 ^ t37;
+       t45 = t42 ^ t41;
+       z0 = t44 & y15;
+       z1 = t37 & y6;
+       z2 = t33 & x7;
+       z3 = t43 & y16;
+       z4 = t40 & y1;
+       z5 = t29 & y7;
+       z6 = t42 & y11;
+       z7 = t45 & y17;
+       z8 = t41 & y10;
+       z9 = t44 & y12;
+       z10 = t37 & y3;
+       z11 = t33 & y4;
+       z12 = t43 & y13;
+       z13 = t40 & y5;
+       z14 = t29 & y2;
+       z15 = t42 & y9;
+       z16 = t45 & y14;
+       z17 = t41 & y8;
+
+       /*
+        * Bottom linear transformation.
+        */
+       t46 = z15 ^ z16;
+       t47 = z10 ^ z11;
+       t48 = z5 ^ z13;
+       t49 = z9 ^ z10;
+       t50 = z2 ^ z12;
+       t51 = z2 ^ z5;
+       t52 = z7 ^ z8;
+       t53 = z0 ^ z3;
+       t54 = z6 ^ z7;
+       t55 = z16 ^ z17;
+       t56 = z12 ^ t48;
+       t57 = t50 ^ t53;
+       t58 = z4 ^ t46;
+       t59 = z3 ^ t54;
+       t60 = t46 ^ t57;
+       t61 = z14 ^ t57;
+       t62 = t52 ^ t58;
+       t63 = t49 ^ t58;
+       t64 = z4 ^ t59;
+       t65 = t61 ^ t62;
+       t66 = z1 ^ t63;
+       s0 = t59 ^ t63;
+       s6 = t56 ^ ~t62;
+       s7 = t48 ^ ~t60;
+       t67 = t64 ^ t65;
+       s3 = t53 ^ t66;
+       s4 = t51 ^ t66;
+       s5 = t47 ^ t65;
+       s1 = t64 ^ ~s3;
+       s2 = t55 ^ ~t67;
+
+       q[3] = _mm_unpacklo_epi64(s4, s0);
+       q[2] = _mm_unpacklo_epi64(s5, s1);
+       q[1] = _mm_unpacklo_epi64(s6, s2);
+       q[0] = _mm_unpacklo_epi64(s7, s3);
+}
+
+void
+aes_sse2_ortho(__m128i q[static 4])
+{
+#define SWAPN(cl, ch, s, x, y)   do { \
+               __m128i a, b; \
+               a = (x); \
+               b = (y); \
+               (x) = (a & _mm_set1_epi64x(cl)) | \
+                   _mm_slli_epi64(b & _mm_set1_epi64x(cl), (s)); \
+               (y) = _mm_srli_epi64(a & _mm_set1_epi64x(ch), (s)) | \
+                   (b & _mm_set1_epi64x(ch)); \
+       } while (0)
+
+#define SWAP2(x, y)    SWAPN(0x5555555555555555, 0xAAAAAAAAAAAAAAAA,  1, x, y)
+#define SWAP4(x, y)    SWAPN(0x3333333333333333, 0xCCCCCCCCCCCCCCCC,  2, x, y)
+#define SWAP8(x, y)    SWAPN(0x0F0F0F0F0F0F0F0F, 0xF0F0F0F0F0F0F0F0,  4, x, y)
+
+       SWAP2(q[0], q[1]);
+       SWAP2(q[2], q[3]);
+
+       SWAP4(q[0], q[2]);
+       SWAP4(q[1], q[3]);
+
+       __m128i q0 = q[0];
+       __m128i q1 = q[1];
+       __m128i q2 = q[2];
+       __m128i q3 = q[3];
+       __m128i q4 = _mm_shuffle_epi32(q[0], 0x0e);



Home | Main Index | Thread Index | Old Index