OpenWrt_x86-r2s-r4s-r5s-N1/devices/common/diy/package/libs/mbedtls/patches/100-Implements-AES-and-GCM-with-ARMv8-Crypto-Extensions.patch

From dfb6015ca79a9fee28f7fcb0af7e350a83574b83 Mon Sep 17 00:00:00 2001
From: "Markku-Juhani O. Saarinen" <[email protected]>
Date: Mon, 20 Nov 2017 14:58:41 +0000
Subject: Implements AES and GCM with ARMv8 Crypto Extensions

A compact patch that provides AES and GCM implementations that utilize the
ARMv8 Crypto Extensions. The config flag is MBEDTLS_ARMV8CE_AES_C, which
is disabled by default as we don't do runtime checking for the feature.
The new implementation lives in armv8ce_aes.c.

Provides similar functionality to https://github.com/ARMmbed/mbedtls/pull/432
Thanks to Barry O'Rourke and others for that contribtion.

Tested on a Cortex A53 device and QEMU. On a midrange phone the real AES-GCM
throughput increases about 4x, while raw AES speed is up to 10x faster.

When cross-compiling, you want to set something like:

  export CC='aarch64-linux-gnu-gcc'
  export CFLAGS='-Ofast -march=armv8-a+crypto'
  scripts/config.pl set MBEDTLS_ARMV8CE_AES_C

QEMU seems to also need

  export LDFLAGS='-static'

Then run normal make or cmake etc.
---

diff -ruNa --binary a/ChangeLog.d/armv8_crypto_extensions.txt b/ChangeLog.d/armv8_crypto_extensions.txt
--- a/ChangeLog.d/armv8_crypto_extensions.txt	1970-01-01 08:00:00.000000000 +0800
+++ b/ChangeLog.d/armv8_crypto_extensions.txt	2021-03-07 15:07:17.781911791 +0800
@@ -0,0 +1,2 @@
+Features
+    * Support ARMv8 Cryptography Extensions for AES and GCM.
diff -ruNa --binary a/include/mbedtls/armv8ce_aes.h b/include/mbedtls/armv8ce_aes.h
--- a/include/mbedtls/armv8ce_aes.h	1970-01-01 08:00:00.000000000 +0800
+++ b/include/mbedtls/armv8ce_aes.h	2021-03-07 15:07:17.781911791 +0800
@@ -0,0 +1,63 @@
+/**
+ * \file armv8ce_aes.h
+ *
+ * \brief ARMv8 Cryptography Extensions -- Optimized code for AES and GCM
+ */
+
+/*
+ *
+ *  Copyright (C) 2006-2017, ARM Limited, All Rights Reserved
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License"); you may
+ *  not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  This file is part of mbed TLS (https://tls.mbed.org)
+ */
+
+#ifndef MBEDTLS_ARMV8CE_AES_H
+#define MBEDTLS_ARMV8CE_AES_H
+
+#include "aes.h"
+
+/**
+ * \brief          [ARMv8 Crypto Extensions] AES-ECB block en(de)cryption
+ *
+ * \param ctx      AES context
+ * \param mode     MBEDTLS_AES_ENCRYPT or MBEDTLS_AES_DECRYPT
+ * \param input    16-byte input block
+ * \param output   16-byte output block
+ *
+ * \return         0 on success (cannot fail)
+ */
+
+int mbedtls_armv8ce_aes_crypt_ecb( mbedtls_aes_context *ctx,
+                                   int mode,
+                                   const unsigned char input[16],
+                                   unsigned char output[16] );
+
+/**
+ * \brief          [ARMv8 Crypto Extensions]  Multiply in GF(2^128) for GCM
+ *
+ * \param c        Result
+ * \param a        First operand
+ * \param b        Second operand
+ *
+ * \note           Both operands and result are bit strings interpreted as
+ *                 elements of GF(2^128) as per the GCM spec.
+ */
+
+void mbedtls_armv8ce_gcm_mult( unsigned char c[16],
+                               const unsigned char a[16],
+                               const unsigned char b[16] );
+
+#endif /* MBEDTLS_ARMV8CE_AES_H */
diff -ruNa --binary a/include/mbedtls/check_config.h b/include/mbedtls/check_config.h
--- a/include/mbedtls/check_config.h	2020-12-10 20:54:15.000000000 +0800
+++ b/include/mbedtls/check_config.h	2021-03-07 15:06:45.625543309 +0800
@@ -95,6 +95,10 @@
 #error "MBEDTLS_AESNI_C defined, but not all prerequisites"
 #endif
 
+#if defined(MBEDTLS_ARMV8CE_AES_C) && !defined(MBEDTLS_HAVE_ASM)
+#error "MBEDTLS_ARMV8CE_AES_C defined, but not all prerequisites"
+#endif
+
 #if defined(MBEDTLS_CTR_DRBG_C) && !defined(MBEDTLS_AES_C)
 #error "MBEDTLS_CTR_DRBG_C defined, but not all prerequisites"
 #endif
@@ -772,3 +776,4 @@
 typedef int mbedtls_iso_c_forbids_empty_translation_units;
 
 #endif /* MBEDTLS_CHECK_CONFIG_H */
+
diff -ruNa --binary a/include/mbedtls/config.h b/include/mbedtls/config.h
--- a/include/mbedtls/config.h	2020-12-10 20:54:15.000000000 +0800
+++ b/include/mbedtls/config.h	2021-03-07 15:14:27.957855484 +0800
@@ -73,6 +73,7 @@
  * Requires support for asm() in compiler.
  *
  * Used in:
+ *      library/armv8ce_aes.c
  *      library/aria.c
  *      library/timing.c
  *      include/mbedtls/bn_mul.h
@@ -1888,6 +1889,21 @@
 #define MBEDTLS_AESNI_C
 
 /**
+ * \def MBEDTLS_ARMV8CE_AES_C
+ *
+ * Enable ARMv8 Crypto Extensions for AES and GCM
+ *
+ * Module:  library/armv8ce_aes.c
+ * Caller:  library/aes.c
+ *          library/gcm.c
+ *
+ * Requires: MBEDTLS_HAVE_ASM
+ *
+ * This module adds support for Armv8 Cryptography Extensions for AES and GCM.
+ */
+//#define MBEDTLS_ARMV8CE_AES_C
+
+/**
  * \def MBEDTLS_AES_C
  *
  * Enable the AES block cipher.
diff -ruNa --binary a/library/aes.c b/library/aes.c
--- a/library/aes.c	2020-12-10 20:54:15.000000000 +0800
+++ b/library/aes.c	2021-03-07 15:06:45.625543309 +0800
@@ -69,7 +69,9 @@
 #if defined(MBEDTLS_AESNI_C)
 #include "mbedtls/aesni.h"
 #endif
-
+#if defined(MBEDTLS_ARMV8CE_AES_C)
+#include "mbedtls/armv8ce_aes.h"
+#endif
 #if defined(MBEDTLS_SELF_TEST)
 #if defined(MBEDTLS_PLATFORM_C)
 #include "mbedtls/platform.h"
@@ -1052,6 +1054,11 @@
         return( mbedtls_aesni_crypt_ecb( ctx, mode, input, output ) );
 #endif
 
+#if defined(MBEDTLS_ARMV8CE_AES_C)
+	// We don't do runtime checking for ARMv8 Crypto Extensions
+	return mbedtls_armv8ce_aes_crypt_ecb( ctx, mode, input, output );
+#endif
+
 #if defined(MBEDTLS_PADLOCK_C) && defined(MBEDTLS_HAVE_X86)
     if( aes_padlock_ace )
     {
diff -ruNa --binary a/library/armv8ce_aes.c b/library/armv8ce_aes.c
--- a/library/armv8ce_aes.c	1970-01-01 08:00:00.000000000 +0800
+++ b/library/armv8ce_aes.c	2021-03-07 15:07:17.781911791 +0800
@@ -0,0 +1,142 @@
+/*
+ *  ARMv8 Cryptography Extensions -- Optimized code for AES and GCM
+ *
+ *  Copyright (C) 2006-2017, ARM Limited, All Rights Reserved
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License"); you may
+ *  not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  This file is part of mbed TLS (https://tls.mbed.org)
+ */
+
+#if !defined(MBEDTLS_CONFIG_FILE)
+#include "mbedtls/config.h"
+#else
+#include MBEDTLS_CONFIG_FILE
+#endif
+
+#if defined(MBEDTLS_ARMV8CE_AES_C)
+
+#include <arm_neon.h>
+#include "mbedtls/armv8ce_aes.h"
+
+#ifndef asm
+#define asm __asm
+#endif
+
+/*
+ *  [Armv8 Cryptography Extensions]  AES-ECB block en(de)cryption
+ */
+
+#if defined(MBEDTLS_AES_C)
+
+int mbedtls_armv8ce_aes_crypt_ecb( mbedtls_aes_context *ctx,
+                                   int mode,
+                                   const unsigned char input[16],
+                                   unsigned char output[16] )
+{
+    unsigned int i;
+    const uint8_t *rk;
+    uint8x16_t x, k;
+
+    x = vld1q_u8( input );                          /* input block */
+    rk = (const uint8_t *) ctx->rk;                 /* round keys  */
+
+    if( mode == MBEDTLS_AES_ENCRYPT )
+    {
+        for( i = ctx->nr - 1; i != 0; i-- )         /* encryption loop */
+        {
+            k = vld1q_u8( rk );
+            rk += 16;
+            x = vaeseq_u8( x, k );
+            x = vaesmcq_u8( x );
+        }
+        k = vld1q_u8( rk );
+        rk += 16;
+        x = vaeseq_u8( x, k );
+    }
+    else
+    {
+        for( i = ctx->nr - 1; i != 0 ; i-- )         /* decryption loop */
+        {
+            k = vld1q_u8( rk );
+            rk += 16;
+            x = vaesdq_u8( x, k );
+            x = vaesimcq_u8( x );
+        }
+        k = vld1q_u8( rk );
+        rk += 16;
+        x = vaesdq_u8( x, k );
+    }
+
+    k = vld1q_u8( rk );                             /* final key just XORed */
+    x = veorq_u8( x, k );
+    vst1q_u8( output, x );                          /* write out */
+
+    return ( 0 );
+}
+
+#endif /* MBEDTLS_AES_C */
+
+
+/*
+ *  [Armv8 Cryptography Extensions]  Multiply in GF(2^128) for GCM
+ */
+
+#if defined(MBEDTLS_GCM_C)
+
+void mbedtls_armv8ce_gcm_mult( unsigned char c[16],
+                               const unsigned char a[16],
+                               const unsigned char b[16] )
+{
+    /* GCM's GF(2^128) polynomial basis is x^128 + x^7 + x^2 + x + 1 */
+    const uint64x2_t base = { 0, 0x86 };            /* note missing LS bit */
+
+    register uint8x16_t vc asm( "v0" );             /* named registers */
+    register uint8x16_t va asm( "v1" );             /* (to avoid conflict) */
+    register uint8x16_t vb asm( "v2" );
+    register uint64x2_t vp asm( "v3" );
+
+    va = vld1q_u8( a );                             /* load inputs */
+    vb = vld1q_u8( b );
+    vp = base;
+
+    asm (
+        "rbit    %1.16b, %1.16b             \n\t"   /* reverse bit order */
+        "rbit    %2.16b, %2.16b             \n\t"
+        "pmull2  %0.1q,  %1.2d,  %2.2d      \n\t"   /* v0 = a.hi * b.hi */
+        "pmull2  v4.1q,  %0.2d,  %3.2d      \n\t"   /* mul v0 by x^64, reduce */
+        "ext     %0.16b, %0.16b, %0.16b, #8 \n\t"
+        "eor     %0.16b, %0.16b, v4.16b     \n\t"
+        "ext     v5.16b, %2.16b, %2.16b, #8 \n\t"   /* (swap hi and lo in b) */
+        "pmull   v4.1q,  %1.1d,  v5.1d      \n\t"   /* v0 ^= a.lo * b.hi */
+        "eor     %0.16b, %0.16b, v4.16b     \n\t"
+        "pmull2  v4.1q,  %1.2d,  v5.2d      \n\t"   /* v0 ^= a.hi * b.lo */
+        "eor     %0.16b, %0.16b, v4.16b     \n\t"
+        "pmull2  v4.1q,  %0.2d,  %3.2d      \n\t"   /* mul v0 by x^64, reduce */
+        "ext     %0.16b, %0.16b, %0.16b, #8 \n\t"
+        "eor     %0.16b, %0.16b, v4.16b     \n\t"
+        "pmull   v4.1q,  %1.1d,  %2.1d      \n\t"   /* v0 ^= a.lo * b.lo */
+        "eor     %0.16b, %0.16b, v4.16b     \n\t"
+        "rbit    %0.16b, %0.16b             \n\t"   /* reverse bits for output */
+        : "=w" (vc)                                 /* q0:      output */
+        : "w" (va), "w" (vb), "w" (vp)              /* q1, q2:  input */
+        : "v4", "v5"                                /* q4, q5:  clobbered */
+    );
+
+    vst1q_u8( c, vc );                              /* write out */
+}
+
+#endif /* MBEDTLS_GCM_C */
+
+#endif /* MBEDTLS_ARMV8CE_AES_C */
diff -ruNa --binary a/library/CMakeLists.txt b/library/CMakeLists.txt
--- a/library/CMakeLists.txt	2020-12-10 20:54:15.000000000 +0800
+++ b/library/CMakeLists.txt	2021-03-07 15:06:45.625543309 +0800
@@ -7,6 +7,7 @@
     aesni.c
     arc4.c
     aria.c
+    armv8ce_aes.c
     asn1parse.c
     asn1write.c
     base64.c
diff -ruNa --binary a/library/gcm.c b/library/gcm.c
--- a/library/gcm.c	2020-12-10 20:54:15.000000000 +0800
+++ b/library/gcm.c	2021-03-07 15:06:45.625543309 +0800
@@ -71,6 +71,10 @@
 #include "mbedtls/aesni.h"
 #endif
 
+#if defined(MBEDTLS_ARMV8CE_AES_C)
+#include "mbedtls/armv8ce_aes.h"
+#endif
+
 #if defined(MBEDTLS_SELF_TEST) && defined(MBEDTLS_AES_C)
 #include "mbedtls/aes.h"
 #include "mbedtls/platform.h"
@@ -140,6 +144,12 @@
     if( ( ret = mbedtls_cipher_update( &ctx->cipher_ctx, h, 16, h, &olen ) ) != 0 )
         return( ret );
 
+#if defined(MBEDTLS_ARMV8CE_AES_C)
+	// we don't do feature testing with ARMv8 cryptography extensions
+    memcpy( ctx ->HL, h, 16 );          // put H at the beginning of buffer
+    return( 0 );                        // that's all we need
+#endif
+
     /* pack h as two 64-bits ints, big-endian */
     GET_UINT32_BE( hi, h,  0  );
     GET_UINT32_BE( lo, h,  4  );
@@ -248,6 +258,11 @@
     unsigned char lo, hi, rem;
     uint64_t zh, zl;
 
+#if defined(MBEDTLS_ARMV8CE_AES_C)
+	mbedtls_armv8ce_gcm_mult( output, x, (const unsigned char *) ctx->HL );
+	return;
+#endif
+
 #if defined(MBEDTLS_AESNI_C) && defined(MBEDTLS_HAVE_X86_64)
     if( mbedtls_aesni_has_support( MBEDTLS_AESNI_CLMUL ) ) {
         unsigned char h[16];
diff -ruNa --binary a/library/Makefile b/library/Makefile
--- a/library/Makefile	2020-12-10 20:54:15.000000000 +0800
+++ b/library/Makefile	2021-03-07 15:12:49.277078224 +0800
@@ -65,6 +65,7 @@
 
 OBJS_CRYPTO=	aes.o		aesni.o		arc4.o		\
 		aria.o		asn1parse.o	asn1write.o	\
+		armv8ce_aes.o					\
 		base64.o	bignum.o	blowfish.o	\
 		camellia.o	ccm.o		chacha20.o	\
 		chachapoly.o	cipher.o	cipher_wrap.o	\
diff -ruNa --binary a/library/version_features.c b/library/version_features.c
--- a/library/version_features.c	2020-12-10 20:54:15.000000000 +0800
+++ b/library/version_features.c	2021-03-07 15:06:45.625543309 +0800
@@ -583,6 +583,9 @@
 #if defined(MBEDTLS_AESNI_C)
     "MBEDTLS_AESNI_C",
 #endif /* MBEDTLS_AESNI_C */
+#if defined(MBEDTLS_ARMV8CE_AES_C)
+    "MBEDTLS_ARMV8CE_AES_C",
+#endif /* MBEDTLS_ARMV8CE_AES_C */
 #if defined(MBEDTLS_AES_C)
     "MBEDTLS_AES_C",
 #endif /* MBEDTLS_AES_C */