From dfb6015ca79a9fee28f7fcb0af7e350a83574b83 Mon Sep 17 00:00:00 2001
From: "Markku-Juhani O. Saarinen" <mjos@mjos.fi>
Date: Mon, 20 Nov 2017 14:58:41 +0000
Subject: Implements AES and GCM with ARMv8 Crypto Extensions
A compact patch that provides AES and GCM implementations that utilize the
ARMv8 Crypto Extensions. The config flag is MBEDTLS_ARMV8CE_AES_C, which
is disabled by default as we don't do runtime checking for the feature.
The new implementation lives in armv8ce_aes.c.
Provides similar functionality to https://github.com/ARMmbed/mbedtls/pull/432
Thanks to Barry O'Rourke and others for that contribtion.
Tested on a Cortex A53 device and QEMU. On a midrange phone the real AES-GCM
throughput increases about 4x, while raw AES speed is up to 10x faster.
When cross-compiling, you want to set something like:
export CC='aarch64-linux-gnu-gcc'
export CFLAGS='-Ofast -march=armv8-a+crypto'
scripts/config.pl set MBEDTLS_ARMV8CE_AES_C
QEMU seems to also need
export LDFLAGS='-static'
Then run normal make or cmake etc.
---
--- a/ChangeLog.d/armv8_crypto_extensions.txt
+++ b/ChangeLog.d/armv8_crypto_extensions.txt
@@ -0,0 +1,2 @@
+Features
+ * Support ARMv8 Cryptography Extensions for AES and GCM.
--- a/include/mbedtls/armv8ce_aes.h
+++ b/include/mbedtls/armv8ce_aes.h
@@ -0,0 +1,63 @@
+/**
+ * \file armv8ce_aes.h
+ *
+ * \brief ARMv8 Cryptography Extensions -- Optimized code for AES and GCM
+ */
+
+/*
+ *
+ * Copyright (C) 2006-2017, ARM Limited, All Rights Reserved
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * This file is part of mbed TLS (https://tls.mbed.org)
+ */
+
+#ifndef MBEDTLS_ARMV8CE_AES_H
+#define MBEDTLS_ARMV8CE_AES_H
+
+#include "aes.h"
+
+/**
+ * \brief [ARMv8 Crypto Extensions] AES-ECB block en(de)cryption
+ *
+ * \param ctx AES context
+ * \param mode MBEDTLS_AES_ENCRYPT or MBEDTLS_AES_DECRYPT
+ * \param input 16-byte input block
+ * \param output 16-byte output block
+ *
+ * \return 0 on success (cannot fail)
+ */
+
+int mbedtls_armv8ce_aes_crypt_ecb( mbedtls_aes_context *ctx,
+ int mode,
+ const unsigned char input[16],
+ unsigned char output[16] );
+
+/**
+ * \brief [ARMv8 Crypto Extensions] Multiply in GF(2^128) for GCM
+ *
+ * \param c Result
+ * \param a First operand
+ * \param b Second operand
+ *
+ * \note Both operands and result are bit strings interpreted as
+ * elements of GF(2^128) as per the GCM spec.
+ */
+
+void mbedtls_armv8ce_gcm_mult( unsigned char c[16],
+ const unsigned char a[16],
+ const unsigned char b[16] );
+
+#endif /* MBEDTLS_ARMV8CE_AES_H */
--- a/include/mbedtls/check_config.h
+++ b/include/mbedtls/check_config.h
@@ -95,6 +95,10 @@
#error "MBEDTLS_AESNI_C defined, but not all prerequisites"
#endif
+#if defined(MBEDTLS_ARMV8CE_AES_C) && !defined(MBEDTLS_HAVE_ASM)
+#error "MBEDTLS_ARMV8CE_AES_C defined, but not all prerequisites"
+#endif
+
#if defined(MBEDTLS_CTR_DRBG_C) && !defined(MBEDTLS_AES_C)
#error "MBEDTLS_CTR_DRBG_C defined, but not all prerequisites"
#endif
@@ -772,3 +776,4 @@
typedef int mbedtls_iso_c_forbids_empty_translation_units;
#endif /* MBEDTLS_CHECK_CONFIG_H */
+
--- a/include/mbedtls/config.h
+++ b/include/mbedtls/config.h
@@ -73,6 +73,7 @@
* Requires support for asm() in compiler.
*
* Used in:
+ * library/armv8ce_aes.c
* library/aria.c
* library/timing.c
* include/mbedtls/bn_mul.h
@@ -1888,6 +1889,21 @@
#define MBEDTLS_AESNI_C
/**
+ * \def MBEDTLS_ARMV8CE_AES_C
+ *
+ * Enable ARMv8 Crypto Extensions for AES and GCM
+ *
+ * Module: library/armv8ce_aes.c
+ * Caller: library/aes.c
+ * library/gcm.c
+ *
+ * Requires: MBEDTLS_HAVE_ASM
+ *
+ * This module adds support for Armv8 Cryptography Extensions for AES and GCM.
+ */
+//#define MBEDTLS_ARMV8CE_AES_C
+
+/**
* \def MBEDTLS_AES_C
*
* Enable the AES block cipher.
--- a/library/aes.c
+++ b/library/aes.c
@@ -69,7 +69,9 @@
#if defined(MBEDTLS_AESNI_C)
#include "mbedtls/aesni.h"
#endif
-
+#if defined(MBEDTLS_ARMV8CE_AES_C)
+#include "mbedtls/armv8ce_aes.h"
+#endif
#if defined(MBEDTLS_SELF_TEST)
#if defined(MBEDTLS_PLATFORM_C)
#include "mbedtls/platform.h"
@@ -1052,6 +1054,11 @@
return( mbedtls_aesni_crypt_ecb( ctx, mode, input, output ) );
#endif
+#if defined(MBEDTLS_ARMV8CE_AES_C)
+ // We don't do runtime checking for ARMv8 Crypto Extensions
+ return mbedtls_armv8ce_aes_crypt_ecb( ctx, mode, input, output );
+#endif
+
#if defined(MBEDTLS_PADLOCK_C) && defined(MBEDTLS_HAVE_X86)
if( aes_padlock_ace )
{
--- a/library/armv8ce_aes.c
+++ b/library/armv8ce_aes.c
@@ -0,0 +1,142 @@
+/*
+ * ARMv8 Cryptography Extensions -- Optimized code for AES and GCM
+ *
+ * Copyright (C) 2006-2017, ARM Limited, All Rights Reserved
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * This file is part of mbed TLS (https://tls.mbed.org)
+ */
+
+#if !defined(MBEDTLS_CONFIG_FILE)
+#include "mbedtls/config.h"
+#else
+#include MBEDTLS_CONFIG_FILE
+#endif
+
+#if defined(MBEDTLS_ARMV8CE_AES_C)
+
+#include <arm_neon.h>
+#include "mbedtls/armv8ce_aes.h"
+
+#ifndef asm
+#define asm __asm
+#endif
+
+/*
+ * [Armv8 Cryptography Extensions] AES-ECB block en(de)cryption
+ */
+
+#if defined(MBEDTLS_AES_C)
+
+int mbedtls_armv8ce_aes_crypt_ecb( mbedtls_aes_context *ctx,
+ int mode,
+ const unsigned char input[16],
+ unsigned char output[16] )
+{
+ unsigned int i;
+ const uint8_t *rk;
+ uint8x16_t x, k;
+
+ x = vld1q_u8( input ); /* input block */
+ rk = (const uint8_t *) ctx->rk; /* round keys */
+
+ if( mode == MBEDTLS_AES_ENCRYPT )
+ {
+ for( i = ctx->nr - 1; i != 0; i-- ) /* encryption loop */
+ {
+ k = vld1q_u8( rk );
+ rk += 16;
+ x = vaeseq_u8( x, k );
+ x = vaesmcq_u8( x );
+ }
+ k = vld1q_u8( rk );
+ rk += 16;
+ x = vaeseq_u8( x, k );
+ }
+ else
+ {
+ for( i = ctx->nr - 1; i != 0 ; i-- ) /* decryption loop */
+ {
+ k = vld1q_u8( rk );
+ rk += 16;
+ x = vaesdq_u8( x, k );
+ x = vaesimcq_u8( x );
+ }
+ k = vld1q_u8( rk );
+ rk += 16;
+ x = vaesdq_u8( x, k );
+ }
+
+ k = vld1q_u8( rk ); /* final key just XORed */
+ x = veorq_u8( x, k );
+ vst1q_u8( output, x ); /* write out */
+
+ return ( 0 );
+}
+
+#endif /* MBEDTLS_AES_C */
+
+
+/*
+ * [Armv8 Cryptography Extensions] Multiply in GF(2^128) for GCM
+ */
+
+#if defined(MBEDTLS_GCM_C)
+
+void mbedtls_armv8ce_gcm_mult( unsigned char c[16],
+ const unsigned char a[16],
+ const unsigned char b[16] )
+{
+ /* GCM's GF(2^128) polynomial basis is x^128 + x^7 + x^2 + x + 1 */
+ const uint64x2_t base = { 0, 0x86 }; /* note missing LS bit */
+
+ register uint8x16_t vc asm( "v0" ); /* named registers */
+ register uint8x16_t va asm( "v1" ); /* (to avoid conflict) */
+ register uint8x16_t vb asm( "v2" );
+ register uint64x2_t vp asm( "v3" );
+
+ va = vld1q_u8( a ); /* load inputs */
+ vb = vld1q_u8( b );
+ vp = base;
+
+ asm (
+ "rbit %1.16b, %1.16b \n\t" /* reverse bit order */
+ "rbit %2.16b, %2.16b \n\t"
+ "pmull2 %0.1q, %1.2d, %2.2d \n\t" /* v0 = a.hi * b.hi */
+ "pmull2 v4.1q, %0.2d, %3.2d \n\t" /* mul v0 by x^64, reduce */
+ "ext %0.16b, %0.16b, %0.16b, #8 \n\t"
+ "eor %0.16b, %0.16b, v4.16b \n\t"
+ "ext v5.16b, %2.16b, %2.16b, #8 \n\t" /* (swap hi and lo in b) */
+ "pmull v4.1q, %1.1d, v5.1d \n\t" /* v0 ^= a.lo * b.hi */
+ "eor %0.16b, %0.16b, v4.16b \n\t"
+ "pmull2 v4.1q, %1.2d, v5.2d \n\t" /* v0 ^= a.hi * b.lo */
+ "eor %0.16b, %0.16b, v4.16b \n\t"
+ "pmull2 v4.1q, %0.2d, %3.2d \n\t" /* mul v0 by x^64, reduce */
+ "ext %0.16b, %0.16b, %0.16b, #8 \n\t"
+ "eor %0.16b, %0.16b, v4.16b \n\t"
+ "pmull v4.1q, %1.1d, %2.1d \n\t" /* v0 ^= a.lo * b.lo */
+ "eor %0.16b, %0.16b, v4.16b \n\t"
+ "rbit %0.16b, %0.16b \n\t" /* reverse bits for output */
+ : "=w" (vc) /* q0: output */
+ : "w" (va), "w" (vb), "w" (vp) /* q1, q2: input */
+ : "v4", "v5" /* q4, q5: clobbered */
+ );
+
+ vst1q_u8( c, vc ); /* write out */
+}
+
+#endif /* MBEDTLS_GCM_C */
+
+#endif /* MBEDTLS_ARMV8CE_AES_C */
--- a/library/CMakeLists.txt
+++ b/library/CMakeLists.txt
@@ -7,6 +7,7 @@
aesni.c
arc4.c
aria.c
+ armv8ce_aes.c
asn1parse.c
asn1write.c
base64.c
--- a/library/gcm.c
+++ b/library/gcm.c
@@ -71,6 +71,10 @@
#include "mbedtls/aesni.h"
#endif
+#if defined(MBEDTLS_ARMV8CE_AES_C)
+#include "mbedtls/armv8ce_aes.h"
+#endif
+
#if defined(MBEDTLS_SELF_TEST) && defined(MBEDTLS_AES_C)
#include "mbedtls/aes.h"
#include "mbedtls/platform.h"
@@ -140,6 +144,12 @@
if( ( ret = mbedtls_cipher_update( &ctx->cipher_ctx, h, 16, h, &olen ) ) != 0 )
return( ret );
+#if defined(MBEDTLS_ARMV8CE_AES_C)
+ // we don't do feature testing with ARMv8 cryptography extensions
+ memcpy( ctx ->HL, h, 16 ); // put H at the beginning of buffer
+ return( 0 ); // that's all we need
+#endif
+
/* pack h as two 64-bits ints, big-endian */
GET_UINT32_BE( hi, h, 0 );
GET_UINT32_BE( lo, h, 4 );
@@ -248,6 +258,11 @@
unsigned char lo, hi, rem;
uint64_t zh, zl;
+#if defined(MBEDTLS_ARMV8CE_AES_C)
+ mbedtls_armv8ce_gcm_mult( output, x, (const unsigned char *) ctx->HL );
+ return;
+#endif
+
#if defined(MBEDTLS_AESNI_C) && defined(MBEDTLS_HAVE_X86_64)
if( mbedtls_aesni_has_support( MBEDTLS_AESNI_CLMUL ) ) {
unsigned char h[16];
--- a/library/Makefile
+++ b/library/Makefile
@@ -65,6 +65,7 @@
OBJS_CRYPTO= aes.o aesni.o arc4.o \
aria.o asn1parse.o asn1write.o \
+ armv8ce_aes.o \
base64.o bignum.o blowfish.o \
camellia.o ccm.o chacha20.o \
chachapoly.o cipher.o cipher_wrap.o \
--- a/library/version_features.c
+++ b/library/version_features.c
@@ -583,6 +583,9 @@
#if defined(MBEDTLS_AESNI_C)
"MBEDTLS_AESNI_C",
#endif /* MBEDTLS_AESNI_C */
+#if defined(MBEDTLS_ARMV8CE_AES_C)
+ "MBEDTLS_ARMV8CE_AES_C",
+#endif /* MBEDTLS_ARMV8CE_AES_C */
#if defined(MBEDTLS_AES_C)
"MBEDTLS_AES_C",
#endif /* MBEDTLS_AES_C */