Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src-draft/trunk]: src/sys Add AES implementation with VIA ACE.
details: https://anonhg.NetBSD.org/src-all/rev/23c4f014b23f
branches: trunk
changeset: 934944:23c4f014b23f
user: Taylor R Campbell <riastradh%NetBSD.org@localhost>
date: Mon Jun 15 16:27:33 2020 +0000
description:
Add AES implementation with VIA ACE.
diffstat:
sys/arch/x86/conf/files.x86 | 3 +
sys/arch/x86/x86/identcpu.c | 4 +
sys/crypto/aes/arch/x86/aes_via.c | 626 +++++++++++++++++++++++++++++++++++
sys/crypto/aes/arch/x86/aes_via.h | 36 ++
sys/crypto/aes/arch/x86/files.aesvia | 3 +
5 files changed, 672 insertions(+), 0 deletions(-)
diffs (truncated from 709 to 300 lines):
diff -r 9341203b8fa4 -r 23c4f014b23f sys/arch/x86/conf/files.x86
--- a/sys/arch/x86/conf/files.x86 Mon Jun 15 16:19:29 2020 +0000
+++ b/sys/arch/x86/conf/files.x86 Mon Jun 15 16:27:33 2020 +0000
@@ -168,3 +168,6 @@
# AES-NI
include "crypto/aes/arch/x86/files.aesni"
+
+# VIA ACE
+include "crypto/aes/arch/x86/files.aesvia"
diff -r 9341203b8fa4 -r 23c4f014b23f sys/arch/x86/x86/identcpu.c
--- a/sys/arch/x86/x86/identcpu.c Mon Jun 15 16:19:29 2020 +0000
+++ b/sys/arch/x86/x86/identcpu.c Mon Jun 15 16:27:33 2020 +0000
@@ -40,6 +40,7 @@
#include <sys/cpu.h>
#include <crypto/aes/arch/x86/aes_ni.h>
+#include <crypto/aes/arch/x86/aes_via.h>
#include <uvm/uvm_extern.h>
@@ -1000,7 +1001,10 @@
#ifdef __x86_64__ /* not yet implemented on i386 */
if (cpu_feature[1] & CPUID2_AES)
aes_md_init(&aes_ni_impl);
+ else
#endif
+ if (cpu_feature[4] & CPUID_VIA_HAS_ACE)
+ aes_md_init(&aes_via_impl);
} else {
/*
* If not first. Warn about cpu_feature mismatch for
diff -r 9341203b8fa4 -r 23c4f014b23f sys/crypto/aes/arch/x86/aes_via.c
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sys/crypto/aes/arch/x86/aes_via.c Mon Jun 15 16:27:33 2020 +0000
@@ -0,0 +1,626 @@
+/* $NetBSD$ */
+
+/*-
+ * Copyright (c) 2020 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__KERNEL_RCSID(1, "$NetBSD$");
+
+#include <sys/types.h>
+#include <sys/evcnt.h>
+#include <sys/systm.h>
+
+#include <crypto/aes/aes.h>
+#include <crypto/aes/aes_bear.h>
+
+#include <x86/cpufunc.h>
+#include <x86/cpuvar.h>
+#include <x86/fpu.h>
+#include <x86/specialreg.h>
+#include <x86/via_padlock.h>
+
+static void
+aesvia_reload_keys(void)
+{
+
+ asm volatile("pushf; popf");
+}
+
+static uint32_t
+aesvia_keylen_cw0(unsigned nrounds)
+{
+
+ /*
+ * Determine the control word bits for the key size / number of
+ * rounds. For AES-128, the hardware can do key expansion on
+ * the fly; for AES-192 and AES-256, software must do it.
+ */
+ switch (nrounds) {
+ case AES_128_NROUNDS:
+ return C3_CRYPT_CWLO_KEY128;
+ case AES_192_NROUNDS:
+ return C3_CRYPT_CWLO_KEY192 | C3_CRYPT_CWLO_KEYGEN_SW;
+ case AES_256_NROUNDS:
+ return C3_CRYPT_CWLO_KEY256 | C3_CRYPT_CWLO_KEYGEN_SW;
+ default:
+ panic("invalid AES nrounds: %u", nrounds);
+ }
+}
+
+static void
+aesvia_setenckey(struct aesenc *enc, const uint8_t *key, uint32_t nrounds)
+{
+ size_t key_len;
+
+ switch (nrounds) {
+ case AES_128_NROUNDS:
+ enc->aese_aes.aes_rk[0] = le32dec(key + 4*0);
+ enc->aese_aes.aes_rk[1] = le32dec(key + 4*1);
+ enc->aese_aes.aes_rk[2] = le32dec(key + 4*2);
+ enc->aese_aes.aes_rk[3] = le32dec(key + 4*3);
+ return;
+ case AES_192_NROUNDS:
+ key_len = 24;
+ break;
+ case AES_256_NROUNDS:
+ key_len = 32;
+ break;
+ default:
+ panic("invalid AES nrounds: %u", nrounds);
+ }
+ br_aes_ct_keysched_stdenc(enc->aese_aes.aes_rk, key, key_len);
+}
+
+static void
+aesvia_setdeckey(struct aesdec *dec, const uint8_t *key, uint32_t nrounds)
+{
+ size_t key_len;
+
+ switch (nrounds) {
+ case AES_128_NROUNDS:
+ dec->aesd_aes.aes_rk[0] = le32dec(key + 4*0);
+ dec->aesd_aes.aes_rk[1] = le32dec(key + 4*1);
+ dec->aesd_aes.aes_rk[2] = le32dec(key + 4*2);
+ dec->aesd_aes.aes_rk[3] = le32dec(key + 4*3);
+ return;
+ case AES_192_NROUNDS:
+ key_len = 24;
+ break;
+ case AES_256_NROUNDS:
+ key_len = 32;
+ break;
+ default:
+ panic("invalid AES nrounds: %u", nrounds);
+ }
+ br_aes_ct_keysched_stddec(dec->aesd_aes.aes_rk, key, key_len);
+}
+
+static inline void
+aesvia_enc1(const struct aesenc *enc, const uint8_t in[static 16],
+ uint8_t out[static 16], uint32_t cw0)
+{
+ const uint32_t cw[4] __aligned(16) = {
+ [0] = (cw0
+ | C3_CRYPT_CWLO_ALG_AES
+ | C3_CRYPT_CWLO_ENCRYPT
+ | C3_CRYPT_CWLO_NORMAL),
+ };
+ size_t nblocks = 1;
+
+ KASSERT(((uintptr_t)enc & 0xf) == 0);
+ KASSERT(((uintptr_t)in & 0xf) == 0);
+ KASSERT(((uintptr_t)out & 0xf) == 0);
+
+ asm volatile("rep xcrypt-ecb"
+ : "+c"(nblocks), "+S"(in), "+D"(out)
+ : "b"(enc), "d"(cw)
+ : "memory", "cc");
+}
+
+static inline void
+aesvia_dec1(const struct aesdec *dec, const uint8_t in[static 16],
+ uint8_t out[static 16], uint32_t cw0)
+{
+ const uint32_t cw[4] __aligned(16) = {
+ [0] = (cw0
+ | C3_CRYPT_CWLO_ALG_AES
+ | C3_CRYPT_CWLO_DECRYPT
+ | C3_CRYPT_CWLO_NORMAL),
+ };
+ size_t nblocks = 1;
+
+ KASSERT(((uintptr_t)dec & 0xf) == 0);
+ KASSERT(((uintptr_t)in & 0xf) == 0);
+ KASSERT(((uintptr_t)out & 0xf) == 0);
+
+ asm volatile("rep xcrypt-ecb"
+ : "+c"(nblocks), "+S"(in), "+D"(out)
+ : "b"(dec), "d"(cw)
+ : "memory", "cc");
+}
+
+static struct evcnt enc_aligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+ NULL, "aesvia", "enc aligned");
+EVCNT_ATTACH_STATIC(enc_aligned_evcnt);
+static struct evcnt enc_unaligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+ NULL, "aesvia", "dec unaligned");
+EVCNT_ATTACH_STATIC(enc_unaligned_evcnt);
+
+static void
+aesvia_enc(const struct aesenc *enc, const uint8_t in[static 16],
+ uint8_t out[static 16], uint32_t nrounds)
+{
+ const uint32_t cw0 = aesvia_keylen_cw0(nrounds);
+
+ fpu_kern_enter();
+ aesvia_reload_keys();
+ if ((((uintptr_t)in | (uintptr_t)out) & 0xf) == 0 &&
+ ((uintptr_t)in & 0xff0) != 0xff0) {
+ enc_aligned_evcnt.ev_count++;
+ aesvia_enc1(enc, in, out, cw0);
+ } else {
+ enc_unaligned_evcnt.ev_count++;
+ /*
+ * VIA requires 16-byte/128-bit alignment, and
+ * xcrypt-ecb reads one block past the one we're
+ * working on -- which may go past the end of the page
+ * into unmapped territory. Use a bounce buffer if
+ * either constraint is violated.
+ */
+ uint8_t inbuf[16] __aligned(16);
+ uint8_t outbuf[16] __aligned(16);
+
+ memcpy(inbuf, in, 16);
+ aesvia_enc1(enc, inbuf, outbuf, cw0);
+ memcpy(out, outbuf, 16);
+
+ explicit_memset(inbuf, 0, sizeof inbuf);
+ explicit_memset(outbuf, 0, sizeof outbuf);
+ }
+ fpu_kern_leave();
+}
+
+static struct evcnt dec_aligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+ NULL, "aesvia", "dec aligned");
+EVCNT_ATTACH_STATIC(dec_aligned_evcnt);
+static struct evcnt dec_unaligned_evcnt = EVCNT_INITIALIZER(EVCNT_TYPE_MISC,
+ NULL, "aesvia", "dec unaligned");
+EVCNT_ATTACH_STATIC(dec_unaligned_evcnt);
+
+static void
+aesvia_dec(const struct aesdec *dec, const uint8_t in[static 16],
+ uint8_t out[static 16], uint32_t nrounds)
+{
+ const uint32_t cw0 = aesvia_keylen_cw0(nrounds);
+
+ fpu_kern_enter();
+ aesvia_reload_keys();
+ if ((((uintptr_t)in | (uintptr_t)out) & 0xf) == 0 &&
+ ((uintptr_t)in & 0xff0) != 0xff0) {
+ dec_aligned_evcnt.ev_count++;
+ aesvia_dec1(dec, in, out, cw0);
+ } else {
+ dec_unaligned_evcnt.ev_count++;
+ /*
+ * VIA requires 16-byte/128-bit alignment, and
+ * xcrypt-ecb reads one block past the one we're
+ * working on -- which may go past the end of the page
+ * into unmapped territory. Use a bounce buffer if
+ * either constraint is violated.
+ */
+ uint8_t inbuf[16] __aligned(16);
+ uint8_t outbuf[16] __aligned(16);
+
+ memcpy(inbuf, in, 16);
+ aesvia_dec1(dec, inbuf, outbuf, cw0);
+ memcpy(out, outbuf, 16);
+
+ explicit_memset(inbuf, 0, sizeof inbuf);
+ explicit_memset(outbuf, 0, sizeof outbuf);
+ }
+ fpu_kern_leave();
+}
+
+static inline void
+aesvia_cbc_enc1(const struct aesenc *enc, const uint8_t in[static 16],
+ uint8_t out[static 16], size_t nblocks, uint8_t **ivp, uint32_t cw0)
+{
+ const uint32_t cw[4] __aligned(16) = {
+ [0] = (cw0
+ | C3_CRYPT_CWLO_ALG_AES
+ | C3_CRYPT_CWLO_ENCRYPT
+ | C3_CRYPT_CWLO_NORMAL),
+ };
+
+ KASSERT(((uintptr_t)enc & 0xf) == 0);
+ KASSERT(((uintptr_t)in & 0xf) == 0);
+ KASSERT(((uintptr_t)out & 0xf) == 0);
+ KASSERT(((uintptr_t)*ivp & 0xf) == 0);
+
+ /*
+ * Register effects:
Home |
Main Index |
Thread Index |
Old Index