|
@@ -1,401 +0,0 @@
|
|
|
-From dfb6015ca79a9fee28f7fcb0af7e350a83574b83 Mon Sep 17 00:00:00 2001
|
|
|
-From: "Markku-Juhani O. Saarinen" <[email protected]>
|
|
|
-Date: Mon, 20 Nov 2017 14:58:41 +0000
|
|
|
-Subject: Implements AES and GCM with ARMv8 Crypto Extensions
|
|
|
-
|
|
|
-A compact patch that provides AES and GCM implementations that utilize the
|
|
|
-ARMv8 Crypto Extensions. The config flag is MBEDTLS_ARMV8CE_AES_C, which
|
|
|
-is disabled by default as we don't do runtime checking for the feature.
|
|
|
-The new implementation lives in armv8ce_aes.c.
|
|
|
-
|
|
|
-Provides similar functionality to https://github.com/ARMmbed/mbedtls/pull/432
|
|
|
-Thanks to Barry O'Rourke and others for that contribtion.
|
|
|
-
|
|
|
-Tested on a Cortex A53 device and QEMU. On a midrange phone the real AES-GCM
|
|
|
-throughput increases about 4x, while raw AES speed is up to 10x faster.
|
|
|
-
|
|
|
-When cross-compiling, you want to set something like:
|
|
|
-
|
|
|
- export CC='aarch64-linux-gnu-gcc'
|
|
|
- export CFLAGS='-Ofast -march=armv8-a+crypto'
|
|
|
- scripts/config.pl set MBEDTLS_ARMV8CE_AES_C
|
|
|
-
|
|
|
-QEMU seems to also need
|
|
|
-
|
|
|
- export LDFLAGS='-static'
|
|
|
-
|
|
|
-Then run normal make or cmake etc.
|
|
|
----
|
|
|
-
|
|
|
-diff -ruNa --binary a/ChangeLog.d/armv8_crypto_extensions.txt b/ChangeLog.d/armv8_crypto_extensions.txt
|
|
|
---- a/ChangeLog.d/armv8_crypto_extensions.txt 1970-01-01 08:00:00.000000000 +0800
|
|
|
-+++ b/ChangeLog.d/armv8_crypto_extensions.txt 2021-03-07 15:07:17.781911791 +0800
|
|
|
-@@ -0,0 +1,2 @@
|
|
|
-+Features
|
|
|
-+ * Support ARMv8 Cryptography Extensions for AES and GCM.
|
|
|
-diff -ruNa --binary a/include/mbedtls/armv8ce_aes.h b/include/mbedtls/armv8ce_aes.h
|
|
|
---- a/include/mbedtls/armv8ce_aes.h 1970-01-01 08:00:00.000000000 +0800
|
|
|
-+++ b/include/mbedtls/armv8ce_aes.h 2021-03-07 15:07:17.781911791 +0800
|
|
|
-@@ -0,0 +1,63 @@
|
|
|
-+/**
|
|
|
-+ * \file armv8ce_aes.h
|
|
|
-+ *
|
|
|
-+ * \brief ARMv8 Cryptography Extensions -- Optimized code for AES and GCM
|
|
|
-+ */
|
|
|
-+
|
|
|
-+/*
|
|
|
-+ *
|
|
|
-+ * Copyright (C) 2006-2017, ARM Limited, All Rights Reserved
|
|
|
-+ * SPDX-License-Identifier: Apache-2.0
|
|
|
-+ *
|
|
|
-+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
|
-+ * not use this file except in compliance with the License.
|
|
|
-+ * You may obtain a copy of the License at
|
|
|
-+ *
|
|
|
-+ * http://www.apache.org/licenses/LICENSE-2.0
|
|
|
-+ *
|
|
|
-+ * Unless required by applicable law or agreed to in writing, software
|
|
|
-+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
|
-+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
-+ * See the License for the specific language governing permissions and
|
|
|
-+ * limitations under the License.
|
|
|
-+ *
|
|
|
-+ * This file is part of mbed TLS (https://tls.mbed.org)
|
|
|
-+ */
|
|
|
-+
|
|
|
-+#ifndef MBEDTLS_ARMV8CE_AES_H
|
|
|
-+#define MBEDTLS_ARMV8CE_AES_H
|
|
|
-+
|
|
|
-+#include "aes.h"
|
|
|
-+
|
|
|
-+/**
|
|
|
-+ * \brief [ARMv8 Crypto Extensions] AES-ECB block en(de)cryption
|
|
|
-+ *
|
|
|
-+ * \param ctx AES context
|
|
|
-+ * \param mode MBEDTLS_AES_ENCRYPT or MBEDTLS_AES_DECRYPT
|
|
|
-+ * \param input 16-byte input block
|
|
|
-+ * \param output 16-byte output block
|
|
|
-+ *
|
|
|
-+ * \return 0 on success (cannot fail)
|
|
|
-+ */
|
|
|
-+
|
|
|
-+int mbedtls_armv8ce_aes_crypt_ecb( mbedtls_aes_context *ctx,
|
|
|
-+ int mode,
|
|
|
-+ const unsigned char input[16],
|
|
|
-+ unsigned char output[16] );
|
|
|
-+
|
|
|
-+/**
|
|
|
-+ * \brief [ARMv8 Crypto Extensions] Multiply in GF(2^128) for GCM
|
|
|
-+ *
|
|
|
-+ * \param c Result
|
|
|
-+ * \param a First operand
|
|
|
-+ * \param b Second operand
|
|
|
-+ *
|
|
|
-+ * \note Both operands and result are bit strings interpreted as
|
|
|
-+ * elements of GF(2^128) as per the GCM spec.
|
|
|
-+ */
|
|
|
-+
|
|
|
-+void mbedtls_armv8ce_gcm_mult( unsigned char c[16],
|
|
|
-+ const unsigned char a[16],
|
|
|
-+ const unsigned char b[16] );
|
|
|
-+
|
|
|
-+#endif /* MBEDTLS_ARMV8CE_AES_H */
|
|
|
-diff -ruNa --binary a/include/mbedtls/check_config.h b/include/mbedtls/check_config.h
|
|
|
---- a/include/mbedtls/check_config.h 2020-12-10 20:54:15.000000000 +0800
|
|
|
-+++ b/include/mbedtls/check_config.h 2021-03-07 15:06:45.625543309 +0800
|
|
|
-@@ -95,6 +95,10 @@
|
|
|
- #error "MBEDTLS_AESNI_C defined, but not all prerequisites"
|
|
|
- #endif
|
|
|
-
|
|
|
-+#if defined(MBEDTLS_ARMV8CE_AES_C) && !defined(MBEDTLS_HAVE_ASM)
|
|
|
-+#error "MBEDTLS_ARMV8CE_AES_C defined, but not all prerequisites"
|
|
|
-+#endif
|
|
|
-+
|
|
|
- #if defined(MBEDTLS_CTR_DRBG_C) && !defined(MBEDTLS_AES_C)
|
|
|
- #error "MBEDTLS_CTR_DRBG_C defined, but not all prerequisites"
|
|
|
- #endif
|
|
|
-@@ -772,3 +776,4 @@
|
|
|
- typedef int mbedtls_iso_c_forbids_empty_translation_units;
|
|
|
-
|
|
|
- #endif /* MBEDTLS_CHECK_CONFIG_H */
|
|
|
-+
|
|
|
-diff -ruNa --binary a/include/mbedtls/config.h b/include/mbedtls/config.h
|
|
|
---- a/include/mbedtls/config.h 2020-12-10 20:54:15.000000000 +0800
|
|
|
-+++ b/include/mbedtls/config.h 2021-03-07 15:14:27.957855484 +0800
|
|
|
-@@ -73,6 +73,7 @@
|
|
|
- * Requires support for asm() in compiler.
|
|
|
- *
|
|
|
- * Used in:
|
|
|
-+ * library/armv8ce_aes.c
|
|
|
- * library/aria.c
|
|
|
- * library/timing.c
|
|
|
- * include/mbedtls/bn_mul.h
|
|
|
-@@ -1888,6 +1889,21 @@
|
|
|
- #define MBEDTLS_AESNI_C
|
|
|
-
|
|
|
- /**
|
|
|
-+ * \def MBEDTLS_ARMV8CE_AES_C
|
|
|
-+ *
|
|
|
-+ * Enable ARMv8 Crypto Extensions for AES and GCM
|
|
|
-+ *
|
|
|
-+ * Module: library/armv8ce_aes.c
|
|
|
-+ * Caller: library/aes.c
|
|
|
-+ * library/gcm.c
|
|
|
-+ *
|
|
|
-+ * Requires: MBEDTLS_HAVE_ASM
|
|
|
-+ *
|
|
|
-+ * This module adds support for Armv8 Cryptography Extensions for AES and GCM.
|
|
|
-+ */
|
|
|
-+//#define MBEDTLS_ARMV8CE_AES_C
|
|
|
-+
|
|
|
-+/**
|
|
|
- * \def MBEDTLS_AES_C
|
|
|
- *
|
|
|
- * Enable the AES block cipher.
|
|
|
-diff -ruNa --binary a/library/aes.c b/library/aes.c
|
|
|
---- a/library/aes.c 2020-12-10 20:54:15.000000000 +0800
|
|
|
-+++ b/library/aes.c 2021-03-07 15:06:45.625543309 +0800
|
|
|
-@@ -69,7 +69,9 @@
|
|
|
- #if defined(MBEDTLS_AESNI_C)
|
|
|
- #include "mbedtls/aesni.h"
|
|
|
- #endif
|
|
|
--
|
|
|
-+#if defined(MBEDTLS_ARMV8CE_AES_C)
|
|
|
-+#include "mbedtls/armv8ce_aes.h"
|
|
|
-+#endif
|
|
|
- #if defined(MBEDTLS_SELF_TEST)
|
|
|
- #if defined(MBEDTLS_PLATFORM_C)
|
|
|
- #include "mbedtls/platform.h"
|
|
|
-@@ -1052,6 +1054,11 @@
|
|
|
- return( mbedtls_aesni_crypt_ecb( ctx, mode, input, output ) );
|
|
|
- #endif
|
|
|
-
|
|
|
-+#if defined(MBEDTLS_ARMV8CE_AES_C)
|
|
|
-+ // We don't do runtime checking for ARMv8 Crypto Extensions
|
|
|
-+ return mbedtls_armv8ce_aes_crypt_ecb( ctx, mode, input, output );
|
|
|
-+#endif
|
|
|
-+
|
|
|
- #if defined(MBEDTLS_PADLOCK_C) && defined(MBEDTLS_HAVE_X86)
|
|
|
- if( aes_padlock_ace )
|
|
|
- {
|
|
|
-diff -ruNa --binary a/library/armv8ce_aes.c b/library/armv8ce_aes.c
|
|
|
---- a/library/armv8ce_aes.c 1970-01-01 08:00:00.000000000 +0800
|
|
|
-+++ b/library/armv8ce_aes.c 2021-03-07 15:07:17.781911791 +0800
|
|
|
-@@ -0,0 +1,142 @@
|
|
|
-+/*
|
|
|
-+ * ARMv8 Cryptography Extensions -- Optimized code for AES and GCM
|
|
|
-+ *
|
|
|
-+ * Copyright (C) 2006-2017, ARM Limited, All Rights Reserved
|
|
|
-+ * SPDX-License-Identifier: Apache-2.0
|
|
|
-+ *
|
|
|
-+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
|
-+ * not use this file except in compliance with the License.
|
|
|
-+ * You may obtain a copy of the License at
|
|
|
-+ *
|
|
|
-+ * http://www.apache.org/licenses/LICENSE-2.0
|
|
|
-+ *
|
|
|
-+ * Unless required by applicable law or agreed to in writing, software
|
|
|
-+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
|
-+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
-+ * See the License for the specific language governing permissions and
|
|
|
-+ * limitations under the License.
|
|
|
-+ *
|
|
|
-+ * This file is part of mbed TLS (https://tls.mbed.org)
|
|
|
-+ */
|
|
|
-+
|
|
|
-+#if !defined(MBEDTLS_CONFIG_FILE)
|
|
|
-+#include "mbedtls/config.h"
|
|
|
-+#else
|
|
|
-+#include MBEDTLS_CONFIG_FILE
|
|
|
-+#endif
|
|
|
-+
|
|
|
-+#if defined(MBEDTLS_ARMV8CE_AES_C)
|
|
|
-+
|
|
|
-+#include <arm_neon.h>
|
|
|
-+#include "mbedtls/armv8ce_aes.h"
|
|
|
-+
|
|
|
-+#ifndef asm
|
|
|
-+#define asm __asm
|
|
|
-+#endif
|
|
|
-+
|
|
|
-+/*
|
|
|
-+ * [Armv8 Cryptography Extensions] AES-ECB block en(de)cryption
|
|
|
-+ */
|
|
|
-+
|
|
|
-+#if defined(MBEDTLS_AES_C)
|
|
|
-+
|
|
|
-+int mbedtls_armv8ce_aes_crypt_ecb( mbedtls_aes_context *ctx,
|
|
|
-+ int mode,
|
|
|
-+ const unsigned char input[16],
|
|
|
-+ unsigned char output[16] )
|
|
|
-+{
|
|
|
-+ unsigned int i;
|
|
|
-+ const uint8_t *rk;
|
|
|
-+ uint8x16_t x, k;
|
|
|
-+
|
|
|
-+ x = vld1q_u8( input ); /* input block */
|
|
|
-+ rk = (const uint8_t *) ctx->rk; /* round keys */
|
|
|
-+
|
|
|
-+ if( mode == MBEDTLS_AES_ENCRYPT )
|
|
|
-+ {
|
|
|
-+ for( i = ctx->nr - 1; i != 0; i-- ) /* encryption loop */
|
|
|
-+ {
|
|
|
-+ k = vld1q_u8( rk );
|
|
|
-+ rk += 16;
|
|
|
-+ x = vaeseq_u8( x, k );
|
|
|
-+ x = vaesmcq_u8( x );
|
|
|
-+ }
|
|
|
-+ k = vld1q_u8( rk );
|
|
|
-+ rk += 16;
|
|
|
-+ x = vaeseq_u8( x, k );
|
|
|
-+ }
|
|
|
-+ else
|
|
|
-+ {
|
|
|
-+ for( i = ctx->nr - 1; i != 0 ; i-- ) /* decryption loop */
|
|
|
-+ {
|
|
|
-+ k = vld1q_u8( rk );
|
|
|
-+ rk += 16;
|
|
|
-+ x = vaesdq_u8( x, k );
|
|
|
-+ x = vaesimcq_u8( x );
|
|
|
-+ }
|
|
|
-+ k = vld1q_u8( rk );
|
|
|
-+ rk += 16;
|
|
|
-+ x = vaesdq_u8( x, k );
|
|
|
-+ }
|
|
|
-+
|
|
|
-+ k = vld1q_u8( rk ); /* final key just XORed */
|
|
|
-+ x = veorq_u8( x, k );
|
|
|
-+ vst1q_u8( output, x ); /* write out */
|
|
|
-+
|
|
|
-+ return ( 0 );
|
|
|
-+}
|
|
|
-+
|
|
|
-+#endif /* MBEDTLS_AES_C */
|
|
|
-+
|
|
|
-+
|
|
|
-+/*
|
|
|
-+ * [Armv8 Cryptography Extensions] Multiply in GF(2^128) for GCM
|
|
|
-+ */
|
|
|
-+
|
|
|
-+#if defined(MBEDTLS_GCM_C)
|
|
|
-+
|
|
|
-+void mbedtls_armv8ce_gcm_mult( unsigned char c[16],
|
|
|
-+ const unsigned char a[16],
|
|
|
-+ const unsigned char b[16] )
|
|
|
-+{
|
|
|
-+ /* GCM's GF(2^128) polynomial basis is x^128 + x^7 + x^2 + x + 1 */
|
|
|
-+ const uint64x2_t base = { 0, 0x86 }; /* note missing LS bit */
|
|
|
-+
|
|
|
-+ register uint8x16_t vc asm( "v0" ); /* named registers */
|
|
|
-+ register uint8x16_t va asm( "v1" ); /* (to avoid conflict) */
|
|
|
-+ register uint8x16_t vb asm( "v2" );
|
|
|
-+ register uint64x2_t vp asm( "v3" );
|
|
|
-+
|
|
|
-+ va = vld1q_u8( a ); /* load inputs */
|
|
|
-+ vb = vld1q_u8( b );
|
|
|
-+ vp = base;
|
|
|
-+
|
|
|
-+ asm (
|
|
|
-+ "rbit %1.16b, %1.16b \n\t" /* reverse bit order */
|
|
|
-+ "rbit %2.16b, %2.16b \n\t"
|
|
|
-+ "pmull2 %0.1q, %1.2d, %2.2d \n\t" /* v0 = a.hi * b.hi */
|
|
|
-+ "pmull2 v4.1q, %0.2d, %3.2d \n\t" /* mul v0 by x^64, reduce */
|
|
|
-+ "ext %0.16b, %0.16b, %0.16b, #8 \n\t"
|
|
|
-+ "eor %0.16b, %0.16b, v4.16b \n\t"
|
|
|
-+ "ext v5.16b, %2.16b, %2.16b, #8 \n\t" /* (swap hi and lo in b) */
|
|
|
-+ "pmull v4.1q, %1.1d, v5.1d \n\t" /* v0 ^= a.lo * b.hi */
|
|
|
-+ "eor %0.16b, %0.16b, v4.16b \n\t"
|
|
|
-+ "pmull2 v4.1q, %1.2d, v5.2d \n\t" /* v0 ^= a.hi * b.lo */
|
|
|
-+ "eor %0.16b, %0.16b, v4.16b \n\t"
|
|
|
-+ "pmull2 v4.1q, %0.2d, %3.2d \n\t" /* mul v0 by x^64, reduce */
|
|
|
-+ "ext %0.16b, %0.16b, %0.16b, #8 \n\t"
|
|
|
-+ "eor %0.16b, %0.16b, v4.16b \n\t"
|
|
|
-+ "pmull v4.1q, %1.1d, %2.1d \n\t" /* v0 ^= a.lo * b.lo */
|
|
|
-+ "eor %0.16b, %0.16b, v4.16b \n\t"
|
|
|
-+ "rbit %0.16b, %0.16b \n\t" /* reverse bits for output */
|
|
|
-+ : "=w" (vc) /* q0: output */
|
|
|
-+ : "w" (va), "w" (vb), "w" (vp) /* q1, q2: input */
|
|
|
-+ : "v4", "v5" /* q4, q5: clobbered */
|
|
|
-+ );
|
|
|
-+
|
|
|
-+ vst1q_u8( c, vc ); /* write out */
|
|
|
-+}
|
|
|
-+
|
|
|
-+#endif /* MBEDTLS_GCM_C */
|
|
|
-+
|
|
|
-+#endif /* MBEDTLS_ARMV8CE_AES_C */
|
|
|
-diff -ruNa --binary a/library/CMakeLists.txt b/library/CMakeLists.txt
|
|
|
---- a/library/CMakeLists.txt 2020-12-10 20:54:15.000000000 +0800
|
|
|
-+++ b/library/CMakeLists.txt 2021-03-07 15:06:45.625543309 +0800
|
|
|
-@@ -7,6 +7,7 @@
|
|
|
- aesni.c
|
|
|
- arc4.c
|
|
|
- aria.c
|
|
|
-+ armv8ce_aes.c
|
|
|
- asn1parse.c
|
|
|
- asn1write.c
|
|
|
- base64.c
|
|
|
-diff -ruNa --binary a/library/gcm.c b/library/gcm.c
|
|
|
---- a/library/gcm.c 2020-12-10 20:54:15.000000000 +0800
|
|
|
-+++ b/library/gcm.c 2021-03-07 15:06:45.625543309 +0800
|
|
|
-@@ -71,6 +71,10 @@
|
|
|
- #include "mbedtls/aesni.h"
|
|
|
- #endif
|
|
|
-
|
|
|
-+#if defined(MBEDTLS_ARMV8CE_AES_C)
|
|
|
-+#include "mbedtls/armv8ce_aes.h"
|
|
|
-+#endif
|
|
|
-+
|
|
|
- #if defined(MBEDTLS_SELF_TEST) && defined(MBEDTLS_AES_C)
|
|
|
- #include "mbedtls/aes.h"
|
|
|
- #include "mbedtls/platform.h"
|
|
|
-@@ -140,6 +144,12 @@
|
|
|
- if( ( ret = mbedtls_cipher_update( &ctx->cipher_ctx, h, 16, h, &olen ) ) != 0 )
|
|
|
- return( ret );
|
|
|
-
|
|
|
-+#if defined(MBEDTLS_ARMV8CE_AES_C)
|
|
|
-+ // we don't do feature testing with ARMv8 cryptography extensions
|
|
|
-+ memcpy( ctx ->HL, h, 16 ); // put H at the beginning of buffer
|
|
|
-+ return( 0 ); // that's all we need
|
|
|
-+#endif
|
|
|
-+
|
|
|
- /* pack h as two 64-bits ints, big-endian */
|
|
|
- GET_UINT32_BE( hi, h, 0 );
|
|
|
- GET_UINT32_BE( lo, h, 4 );
|
|
|
-@@ -248,6 +258,11 @@
|
|
|
- unsigned char lo, hi, rem;
|
|
|
- uint64_t zh, zl;
|
|
|
-
|
|
|
-+#if defined(MBEDTLS_ARMV8CE_AES_C)
|
|
|
-+ mbedtls_armv8ce_gcm_mult( output, x, (const unsigned char *) ctx->HL );
|
|
|
-+ return;
|
|
|
-+#endif
|
|
|
-+
|
|
|
- #if defined(MBEDTLS_AESNI_C) && defined(MBEDTLS_HAVE_X86_64)
|
|
|
- if( mbedtls_aesni_has_support( MBEDTLS_AESNI_CLMUL ) ) {
|
|
|
- unsigned char h[16];
|
|
|
-diff -ruNa --binary a/library/Makefile b/library/Makefile
|
|
|
---- a/library/Makefile 2020-12-10 20:54:15.000000000 +0800
|
|
|
-+++ b/library/Makefile 2021-03-07 15:12:49.277078224 +0800
|
|
|
-@@ -65,6 +65,7 @@
|
|
|
-
|
|
|
- OBJS_CRYPTO= aes.o aesni.o arc4.o \
|
|
|
- aria.o asn1parse.o asn1write.o \
|
|
|
-+ armv8ce_aes.o \
|
|
|
- base64.o bignum.o blowfish.o \
|
|
|
- camellia.o ccm.o chacha20.o \
|
|
|
- chachapoly.o cipher.o cipher_wrap.o \
|
|
|
-diff -ruNa --binary a/library/version_features.c b/library/version_features.c
|
|
|
---- a/library/version_features.c 2020-12-10 20:54:15.000000000 +0800
|
|
|
-+++ b/library/version_features.c 2021-03-07 15:06:45.625543309 +0800
|
|
|
-@@ -583,6 +583,9 @@
|
|
|
- #if defined(MBEDTLS_AESNI_C)
|
|
|
- "MBEDTLS_AESNI_C",
|
|
|
- #endif /* MBEDTLS_AESNI_C */
|
|
|
-+#if defined(MBEDTLS_ARMV8CE_AES_C)
|
|
|
-+ "MBEDTLS_ARMV8CE_AES_C",
|
|
|
-+#endif /* MBEDTLS_ARMV8CE_AES_C */
|
|
|
- #if defined(MBEDTLS_AES_C)
|
|
|
- "MBEDTLS_AES_C",
|
|
|
- #endif /* MBEDTLS_AES_C */
|
|
|
-
|