diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/Build/Include/Makefile.inc | 12 | ||||
-rw-r--r-- | src/Common/Crypto.c | 28 | ||||
-rw-r--r-- | src/Common/Dlgcode.c | 4 | ||||
-rw-r--r-- | src/Common/Dlgcode.h | 2 | ||||
-rw-r--r-- | src/Common/Random.c | 4 | ||||
-rw-r--r-- | src/Common/Tests.c | 2 | ||||
-rw-r--r-- | src/Crypto/Aes_hw_armv8.c | 316 | ||||
-rw-r--r-- | src/Crypto/Aes_hw_cpu.h | 2 | ||||
-rw-r--r-- | src/Crypto/Crypto.vcxproj | 4 | ||||
-rw-r--r-- | src/Crypto/Crypto.vcxproj.filters | 15 | ||||
-rw-r--r-- | src/Crypto/config.h | 39 | ||||
-rw-r--r-- | src/Crypto/cpu.c | 38 | ||||
-rw-r--r-- | src/Crypto/cpu.h | 18 | ||||
-rw-r--r-- | src/Driver/Driver.vcxproj | 4 | ||||
-rw-r--r-- | src/Driver/Driver.vcxproj.filters | 3 | ||||
-rw-r--r-- | src/Driver/Ntdriver.c | 11 | ||||
-rw-r--r-- | src/ExpandVolume/WinMain.cpp | 6 | ||||
-rw-r--r-- | src/Format/Tcformat.c | 6 | ||||
-rw-r--r-- | src/Main/UserInterface.cpp | 5 | ||||
-rw-r--r-- | src/Makefile | 6 | ||||
-rw-r--r-- | src/Mount/Mount.c | 6 | ||||
-rw-r--r-- | src/Volume/Volume.make | 10 |
22 files changed, 492 insertions, 49 deletions
diff --git a/src/Build/Include/Makefile.inc b/src/Build/Include/Makefile.inc index 0f68df36..281d206a 100644 --- a/src/Build/Include/Makefile.inc +++ b/src/Build/Include/Makefile.inc @@ -13,9 +13,9 @@ $(NAME): $(NAME).a clean: @echo Cleaning $(NAME) - rm -f $(APPNAME) $(NAME).a $(OBJS) $(OBJSEX) $(OBJSNOOPT) $(OBJSHANI) $(OBJSSSE41) $(OBJSSSSE3) $(OBJS:.o=.d) $(OBJSEX:.oo=.d) $(OBJSNOOPT:.o0=.d) $(OBJSHANI:.oshani=.d) $(OBJSSSE41:.osse41=.d) $(OBJSSSSE3:.ossse3=.d) *.gch + rm -f $(APPNAME) $(NAME).a $(OBJS) $(OBJSEX) $(OBJSNOOPT) $(OBJSHANI) $(OBJSSSE41) $(OBJSSSSE3) $(OBJARMV8CRYPTO) $(OBJS:.o=.d) $(OBJSEX:.oo=.d) $(OBJSNOOPT:.o0=.d) $(OBJSHANI:.oshani=.d) $(OBJSSSE41:.osse41=.d) $(OBJSSSSE3:.ossse3=.d) $(OBJARMV8CRYPTO:.oarmv8crypto=.d) *.gch %.o: %.c @echo Compiling $(<F) $(CC) $(CFLAGS) -c $< -o $@ @@ -35,8 +35,12 @@ clean: %.ossse3: %.c @echo Compiling $(<F) $(CC) $(CFLAGS) -mssse3 -c $< -o $@ +%.oarmv8crypto: %.c + @echo Compiling $(<F) + $(CC) $(CFLAGS) -march=armv8-a+crypto -c $< -o $@ + %.o: %.cpp @echo Compiling $(<F) $(CXX) $(CXXFLAGS) -c $< -o $@ @@ -95,11 +99,11 @@ TR_SED_BIN := tr '\n' ' ' | tr -s ' ' ',' | sed -e 's/^,//g' -e 's/,$$/n/' | tr $(OD_BIN) $< | $(TR_SED_BIN) >$@ # Dependencies --include $(OBJS:.o=.d) $(OBJSEX:.oo=.d) $(OBJSNOOPT:.o0=.d) $(OBJSHANI:.oshani=.d) $(OBJSSSE41:.osse41=.d) $(OBJSSSSE3:.ossse3=.d) +-include $(OBJS:.o=.d) $(OBJSEX:.oo=.d) $(OBJSNOOPT:.o0=.d) $(OBJSHANI:.oshani=.d) $(OBJSSSE41:.osse41=.d) $(OBJSSSSE3:.ossse3=.d) $(OBJARMV8CRYPTO:.oarmv8crypto=.d) -$(NAME).a: $(OBJS) $(OBJSEX) $(OBJSNOOPT) $(OBJSHANI) $(OBJSSSE41) $(OBJSSSSE3) +$(NAME).a: $(OBJS) $(OBJSEX) $(OBJSNOOPT) $(OBJSHANI) $(OBJSSSE41) $(OBJSSSSE3) $(OBJARMV8CRYPTO) @echo Updating library $@ - $(AR) $(AFLAGS) -rc $@ $(OBJS) $(OBJSEX) $(OBJSNOOPT) $(OBJSHANI) $(OBJSSSE41) $(OBJSSSSE3) + $(AR) $(AFLAGS) -rc $@ $(OBJS) $(OBJSEX) $(OBJSNOOPT) $(OBJSHANI) $(OBJSSSE41) $(OBJSSSSE3) $(OBJARMV8CRYPTO) $(RANLIB) $@ diff --git a/src/Common/Crypto.c b/src/Common/Crypto.c index 32ef4b56..2f4e447f 100644 --- a/src/Common/Crypto.c +++ b/src/Common/Crypto.c @@ -1163,10 +1163,8 @@ BOOL IsAesHwCpuSupported () stateValid = TRUE; } return state && !HwEncryptionDisabled; -#elif defined (_M_ARM64) || defined(__arm__) || defined (__arm64__) || defined (__aarch64__) - return 0; #else return (HasAESNI() && !HwEncryptionDisabled)? TRUE : FALSE; #endif } @@ -1482,30 +1480,4 @@ void VcUnprotectKeys (PCRYPTO_INFO pCryptoInfo, uint64 encID) #endif #endif -#if defined(_M_ARM64) || defined(__arm__) || defined (__arm64__) || defined (__aarch64__) -/* dummy implementation that should never be called */ -void aes_hw_cpu_decrypt(const uint8* ks, uint8* data) -{ - ks = ks; - data = data; -} - -void aes_hw_cpu_decrypt_32_blocks(const uint8* ks, uint8* data) -{ - ks = ks; - data = data; -} - -void aes_hw_cpu_encrypt(const uint8* ks, uint8* data) -{ - ks = ks; - data = data; -} - -void aes_hw_cpu_encrypt_32_blocks(const uint8* ks, uint8* data) -{ - ks = ks; - data = data; -} -#endif diff --git a/src/Common/Dlgcode.c b/src/Common/Dlgcode.c index 045def76..e471fc46 100644 --- a/src/Common/Dlgcode.c +++ b/src/Common/Dlgcode.c @@ -1045,9 +1045,9 @@ BOOL IsOSVersionAtLeast (OSVersionEnum reqMinOS, int reqMinServicePack) return ((CurrentOSMajor << 16 | CurrentOSMinor << 8 | CurrentOSServicePack) >= (major << 16 | minor << 8 | reqMinServicePack)); } -BOOL IsWin10BuildAtLeast(DWORD minBuild) +BOOL IsWin10BuildAtLeast(int minBuild) { // Must first be recognized as Windows 10 or higher if (nCurrentOS < WIN_10) return FALSE; @@ -14882,16 +14882,18 @@ void GetAppRandomSeed (unsigned char* pbRandSeed, size_t cbRandSeed) jent_entropy_collector_free (ec); } } +#ifndef _M_ARM64 // use RDSEED or RDRAND from CPU as source of entropy if enabled if ( IsCpuRngEnabled() && ( (HasRDSEED() && RDSEED_getBytes (digest, sizeof (digest))) || (HasRDRAND() && RDRAND_getBytes (digest, sizeof (digest))) )) { WHIRLPOOL_add (digest, sizeof(digest), &tctx); } +#endif WHIRLPOOL_finalize (&tctx, digest); count = VC_MIN (cbRandSeed, sizeof (digest)); diff --git a/src/Common/Dlgcode.h b/src/Common/Dlgcode.h index 8148bd09..aaaad97e 100644 --- a/src/Common/Dlgcode.h +++ b/src/Common/Dlgcode.h @@ -502,9 +502,9 @@ BOOL LoadDefaultKeyFilesParam (void); void Debug (char *format, ...); void DebugMsgBox (char *format, ...); BOOL IsOSAtLeast (OSVersionEnum reqMinOS); BOOL IsOSVersionAtLeast (OSVersionEnum reqMinOS, int reqMinServicePack); -BOOL IsWin10BuildAtLeast(DWORD minBuild); +BOOL IsWin10BuildAtLeast(int minBuild); BOOL IsSupportedOS (); BOOL Is64BitOs (); BOOL IsARM(); BOOL IsServerOS (); diff --git a/src/Common/Random.c b/src/Common/Random.c index 0aab0cff..10995f74 100644 --- a/src/Common/Random.c +++ b/src/Common/Random.c @@ -877,16 +877,18 @@ BOOL SlowPoll (void) jent_entropy_collector_free (ec); } } +#ifndef _M_ARM64 // use RDSEED or RDRAND from CPU as source of entropy if present if ( IsCpuRngEnabled() && ( (HasRDSEED() && RDSEED_getBytes (buffer, sizeof (buffer))) || (HasRDRAND() && RDRAND_getBytes (buffer, sizeof (buffer))) )) { RandaddBuf (buffer, sizeof (buffer)); } +#endif burn(buffer, sizeof (buffer)); /* Mix the pool */ @@ -1010,16 +1012,18 @@ BOOL FastPoll (void) CryptoAPILastError = pRtlNtStatusToDosError (bStatus); return FALSE; } +#ifndef _M_ARM64 // use RDSEED or RDRAND from CPU as source of entropy if enabled if ( IsCpuRngEnabled() && ( (HasRDSEED() && RDSEED_getBytes (buffer, sizeof (buffer))) || (HasRDRAND() && RDRAND_getBytes (buffer, sizeof (buffer))) )) { RandaddBuf (buffer, sizeof (buffer)); } +#endif burn (buffer, sizeof(buffer)); /* Apply the pool mixing function */ diff --git a/src/Common/Tests.c b/src/Common/Tests.c index f6a9183d..89af24f1 100644 --- a/src/Common/Tests.c +++ b/src/Common/Tests.c @@ -1489,9 +1489,11 @@ BOOL AutoTestAlgorithms (void) if (exceptionCatched) { /* unexepected exception raised. Disable all CPU extended feature and try again */ EnableHwEncryption (hwEncryptionEnabled); +#ifndef _M_ARM64 DisableCPUExtendedFeatures (); +#endif __try { result = DoAutoTestAlgorithms(); } diff --git a/src/Crypto/Aes_hw_armv8.c b/src/Crypto/Aes_hw_armv8.c new file mode 100644 index 00000000..b67ed1a5 --- /dev/null +++ b/src/Crypto/Aes_hw_armv8.c @@ -0,0 +1,316 @@ +/* +* AES using ARMv8 +* Contributed by Jeffrey Walton +* +* Further changes +* (C) 2017,2018 Jack Lloyd +* +* Botan is released under the Simplified BSD License (see license.txt) +*/ + +/* Modified and adapted for VeraCrypt */ + +#include "Common/Tcdefs.h" +#include "Aes_hw_cpu.h" +#if !defined(_UEFI) +#include <memory.h> +#include <stdlib.h> +#endif +#include "cpu.h" +#include "misc.h" + +#if CRYPTOPP_ARM_AES_AVAILABLE + +#include <arm_neon.h> + +// Single block encryption operations +VC_INLINE void aes_enc_block(uint8x16_t* B, uint8x16_t K) +{ + *B = vaesmcq_u8(vaeseq_u8(*B, K)); +} + +VC_INLINE void aes_enc_block_last(uint8x16_t* B, uint8x16_t K, uint8x16_t K2) +{ + *B = veorq_u8(vaeseq_u8(*B, K), K2); +} + +// 4-block parallel encryption operations +VC_INLINE void aes_enc_4_blocks(uint8x16_t* B0, uint8x16_t* B1, + uint8x16_t* B2, uint8x16_t* B3, uint8x16_t K) +{ + *B0 = vaesmcq_u8(vaeseq_u8(*B0, K)); + *B1 = vaesmcq_u8(vaeseq_u8(*B1, K)); + *B2 = vaesmcq_u8(vaeseq_u8(*B2, K)); + *B3 = vaesmcq_u8(vaeseq_u8(*B3, K)); +} + +VC_INLINE void aes_enc_4_blocks_last(uint8x16_t* B0, uint8x16_t* B1, + uint8x16_t* B2, uint8x16_t* B3, + uint8x16_t K, uint8x16_t K2) +{ + *B0 = veorq_u8(vaeseq_u8(*B0, K), K2); + *B1 = veorq_u8(vaeseq_u8(*B1, K), K2); + *B2 = veorq_u8(vaeseq_u8(*B2, K), K2); + *B3 = veorq_u8(vaeseq_u8(*B3, K), K2); +} + +// Single block decryption operations +VC_INLINE void aes_dec_block(uint8x16_t* B, uint8x16_t K) +{ + *B = vaesimcq_u8(vaesdq_u8(*B, K)); +} + +VC_INLINE void aes_dec_block_last(uint8x16_t* B, uint8x16_t K, uint8x16_t K2) +{ + *B = veorq_u8(vaesdq_u8(*B, K), K2); +} + +// 4-block parallel decryption operations +VC_INLINE void aes_dec_4_blocks(uint8x16_t* B0, uint8x16_t* B1, + uint8x16_t* B2, uint8x16_t* B3, uint8x16_t K) +{ + *B0 = vaesimcq_u8(vaesdq_u8(*B0, K)); + *B1 = vaesimcq_u8(vaesdq_u8(*B1, K)); + *B2 = vaesimcq_u8(vaesdq_u8(*B2, K)); + *B3 = vaesimcq_u8(vaesdq_u8(*B3, K)); +} + +VC_INLINE void aes_dec_4_blocks_last(uint8x16_t* B0, uint8x16_t* B1, + uint8x16_t* B2, uint8x16_t* B3, + uint8x16_t K, uint8x16_t K2) +{ + *B0 = veorq_u8(vaesdq_u8(*B0, K), K2); + *B1 = veorq_u8(vaesdq_u8(*B1, K), K2); + *B2 = veorq_u8(vaesdq_u8(*B2, K), K2); + *B3 = veorq_u8(vaesdq_u8(*B3, K), K2); +} + +VC_INLINE void aes256_hw_encrypt_blocks(uint8 buffer[], size_t blocks, const uint8* ks) +{ + const uint8x16_t K0 = vld1q_u8(ks + 0 * 16); + const uint8x16_t K1 = vld1q_u8(ks + 1 * 16); + const uint8x16_t K2 = vld1q_u8(ks + 2 * 16); + const uint8x16_t K3 = vld1q_u8(ks + 3 * 16); + const uint8x16_t K4 = vld1q_u8(ks + 4 * 16); + const uint8x16_t K5 = vld1q_u8(ks + 5 * 16); + const uint8x16_t K6 = vld1q_u8(ks + 6 * 16); + const uint8x16_t K7 = vld1q_u8(ks + 7 * 16); + const uint8x16_t K8 = vld1q_u8(ks + 8 * 16); + const uint8x16_t K9 = vld1q_u8(ks + 9 * 16); + const uint8x16_t K10 = vld1q_u8(ks + 10 * 16); + const uint8x16_t K11 = vld1q_u8(ks + 11 * 16); + const uint8x16_t K12 = vld1q_u8(ks + 12 * 16); + const uint8x16_t K13 = vld1q_u8(ks + 13 * 16); + const uint8x16_t K14 = vld1q_u8(ks + 14 * 16); + + while(blocks >= 4) { + uint8x16_t B0 = vld1q_u8(buffer); + uint8x16_t B1 = vld1q_u8(buffer + 16); + uint8x16_t B2 = vld1q_u8(buffer + 32); + uint8x16_t B3 = vld1q_u8(buffer + 48); + + aes_enc_4_blocks(&B0, &B1, &B2, &B3, K0); + aes_enc_4_blocks(&B0, &B1, &B2, &B3, K1); + aes_enc_4_blocks(&B0, &B1, &B2, &B3, K2); + aes_enc_4_blocks(&B0, &B1, &B2, &B3, K3); + aes_enc_4_blocks(&B0, &B1, &B2, &B3, K4); + aes_enc_4_blocks(&B0, &B1, &B2, &B3, K5); + aes_enc_4_blocks(&B0, &B1, &B2, &B3, K6); + aes_enc_4_blocks(&B0, &B1, &B2, &B3, K7); + aes_enc_4_blocks(&B0, &B1, &B2, &B3, K8); + aes_enc_4_blocks(&B0, &B1, &B2, &B3, K9); + aes_enc_4_blocks(&B0, &B1, &B2, &B3, K10); + aes_enc_4_blocks(&B0, &B1, &B2, &B3, K11); + aes_enc_4_blocks(&B0, &B1, &B2, &B3, K12); + aes_enc_4_blocks_last(&B0, &B1, &B2, &B3, K13, K14); + + vst1q_u8(buffer, B0); + vst1q_u8(buffer + 16, B1); + vst1q_u8(buffer + 32, B2); + vst1q_u8(buffer + 48, B3); + + buffer += 16 * 4; + blocks -= 4; + } + + for(size_t i = 0; i != blocks; ++i) { + uint8x16_t B = vld1q_u8(buffer + 16 * i); + aes_enc_block(&B, K0); + aes_enc_block(&B, K1); + aes_enc_block(&B, K2); + aes_enc_block(&B, K3); + aes_enc_block(&B, K4); + aes_enc_block(&B, K5); + aes_enc_block(&B, K6); + aes_enc_block(&B, K7); + aes_enc_block(&B, K8); + aes_enc_block(&B, K9); + aes_enc_block(&B, K10); + aes_enc_block(&B, K11); + aes_enc_block(&B, K12); + aes_enc_block_last(&B, K13, K14); + vst1q_u8(buffer + 16 * i, B); + } +} + +VC_INLINE void aes256_hw_encrypt_block(uint8 buffer[], const uint8* ks) +{ + const uint8x16_t K0 = vld1q_u8(ks + 0 * 16); + const uint8x16_t K1 = vld1q_u8(ks + 1 * 16); + const uint8x16_t K2 = vld1q_u8(ks + 2 * 16); + const uint8x16_t K3 = vld1q_u8(ks + 3 * 16); + const uint8x16_t K4 = vld1q_u8(ks + 4 * 16); + const uint8x16_t K5 = vld1q_u8(ks + 5 * 16); + const uint8x16_t K6 = vld1q_u8(ks + 6 * 16); + const uint8x16_t K7 = vld1q_u8(ks + 7 * 16); + const uint8x16_t K8 = vld1q_u8(ks + 8 * 16); + const uint8x16_t K9 = vld1q_u8(ks + 9 * 16); + const uint8x16_t K10 = vld1q_u8(ks + 10 * 16); + const uint8x16_t K11 = vld1q_u8(ks + 11 * 16); + const uint8x16_t K12 = vld1q_u8(ks + 12 * 16); + const uint8x16_t K13 = vld1q_u8(ks + 13 * 16); + const uint8x16_t K14 = vld1q_u8(ks + 14 * 16); + + uint8x16_t B = vld1q_u8(buffer); + aes_enc_block(&B, K0); + aes_enc_block(&B, K1); + aes_enc_block(&B, K2); + aes_enc_block(&B, K3); + aes_enc_block(&B, K4); + aes_enc_block(&B, K5); + aes_enc_block(&B, K6); + aes_enc_block(&B, K7); + aes_enc_block(&B, K8); + aes_enc_block(&B, K9); + aes_enc_block(&B, K10); + aes_enc_block(&B, K11); + aes_enc_block(&B, K12); + aes_enc_block_last(&B, K13, K14); + vst1q_u8(buffer, B); +} + +VC_INLINE void aes256_hw_decrypt_blocks(uint8 buffer[], size_t blocks, const uint8* ks) +{ + const uint8x16_t K0 = vld1q_u8(ks + 0 * 16); + const uint8x16_t K1 = vld1q_u8(ks + 1 * 16); + const uint8x16_t K2 = vld1q_u8(ks + 2 * 16); + const uint8x16_t K3 = vld1q_u8(ks + 3 * 16); + const uint8x16_t K4 = vld1q_u8(ks + 4 * 16); + const uint8x16_t K5 = vld1q_u8(ks + 5 * 16); + const uint8x16_t K6 = vld1q_u8(ks + 6 * 16); + const uint8x16_t K7 = vld1q_u8(ks + 7 * 16); + const uint8x16_t K8 = vld1q_u8(ks + 8 * 16); + const uint8x16_t K9 = vld1q_u8(ks + 9 * 16); + const uint8x16_t K10 = vld1q_u8(ks + 10 * 16); + const uint8x16_t K11 = vld1q_u8(ks + 11 * 16); + const uint8x16_t K12 = vld1q_u8(ks + 12 * 16); + const uint8x16_t K13 = vld1q_u8(ks + 13 * 16); + const uint8x16_t K14 = vld1q_u8(ks + 14 * 16); + + while(blocks >= 4) { + uint8x16_t B0 = vld1q_u8(buffer); + uint8x16_t B1 = vld1q_u8(buffer + 16); + uint8x16_t B2 = vld1q_u8(buffer + 32); + uint8x16_t B3 = vld1q_u8(buffer + 48); + + aes_dec_4_blocks(&B0, &B1, &B2, &B3, K0); + aes_dec_4_blocks(&B0, &B1, &B2, &B3, K1); + aes_dec_4_blocks(&B0, &B1, &B2, &B3, K2); + aes_dec_4_blocks(&B0, &B1, &B2, &B3, K3); + aes_dec_4_blocks(&B0, &B1, &B2, &B3, K4); + aes_dec_4_blocks(&B0, &B1, &B2, &B3, K5); + aes_dec_4_blocks(&B0, &B1, &B2, &B3, K6); + aes_dec_4_blocks(&B0, &B1, &B2, &B3, K7); + aes_dec_4_blocks(&B0, &B1, &B2, &B3, K8); + aes_dec_4_blocks(&B0, &B1, &B2, &B3, K9); + aes_dec_4_blocks(&B0, &B1, &B2, &B3, K10); + aes_dec_4_blocks(&B0, &B1, &B2, &B3, K11); + aes_dec_4_blocks(&B0, &B1, &B2, &B3, K12); + aes_dec_4_blocks_last(&B0, &B1, &B2, &B3, K13, K14); + + vst1q_u8(buffer, B0); + vst1q_u8(buffer + 16, B1); + vst1q_u8(buffer + 32, B2); + vst1q_u8(buffer + 48, B3); + + buffer += 16 * 4; + blocks -= 4; + } + + for(size_t i = 0; i != blocks; ++i) { + uint8x16_t B = vld1q_u8(buffer + 16 * i); + aes_dec_block(&B, K0); + aes_dec_block(&B, K1); + aes_dec_block(&B, K2); + aes_dec_block(&B, K3); + aes_dec_block(&B, K4); + aes_dec_block(&B, K5); + aes_dec_block(&B, K6); + aes_dec_block(&B, K7); + aes_dec_block(&B, K8); + aes_dec_block(&B, K9); + aes_dec_block(&B, K10); + aes_dec_block(&B, K11); + aes_dec_block(&B, K12); + aes_dec_block_last(&B, K13, K14); + vst1q_u8(buffer + 16 * i, B); + } +} + +VC_INLINE void aes256_hw_decrypt_block(uint8 buffer[], const uint8* ks) +{ + const uint8x16_t K0 = vld1q_u8(ks + 0 * 16); + const uint8x16_t K1 = vld1q_u8(ks + 1 * 16); + const uint8x16_t K2 = vld1q_u8(ks + 2 * 16); + const uint8x16_t K3 = vld1q_u8(ks + 3 * 16); + const uint8x16_t K4 = vld1q_u8(ks + 4 * 16); + const uint8x16_t K5 = vld1q_u8(ks + 5 * 16); + const uint8x16_t K6 = vld1q_u8(ks + 6 * 16); + const uint8x16_t K7 = vld1q_u8(ks + 7 * 16); + const uint8x16_t K8 = vld1q_u8(ks + 8 * 16); + const uint8x16_t K9 = vld1q_u8(ks + 9 * 16); + const uint8x16_t K10 = vld1q_u8(ks + 10 * 16); + const uint8x16_t K11 = vld1q_u8(ks + 11 * 16); + const uint8x16_t K12 = vld1q_u8(ks + 12 * 16); + const uint8x16_t K13 = vld1q_u8(ks + 13 * 16); + const uint8x16_t K14 = vld1q_u8(ks + 14 * 16); + + uint8x16_t B = vld1q_u8(buffer); + aes_dec_block(&B, K0); + aes_dec_block(&B, K1); + aes_dec_block(&B, K2); + aes_dec_block(&B, K3); + aes_dec_block(&B, K4); + aes_dec_block(&B, K5); + aes_dec_block(&B, K6); + aes_dec_block(&B, K7); + aes_dec_block(&B, K8); + aes_dec_block(&B, K9); + aes_dec_block(&B, K10); + aes_dec_block(&B, K11); + aes_dec_block(&B, K12); + aes_dec_block_last(&B, K13, K14); + vst1q_u8(buffer, B); +} + +void aes_hw_cpu_decrypt (const uint8 *ks, uint8 *data) +{ + aes256_hw_decrypt_block(data, ks); +} + +void aes_hw_cpu_decrypt_32_blocks (const uint8 *ks, uint8 *data) +{ + aes256_hw_decrypt_blocks(data, 32, ks); +} + +void aes_hw_cpu_encrypt (const uint8 *ks, uint8 *data) +{ + aes256_hw_encrypt_block(data, ks); +} + +void aes_hw_cpu_encrypt_32_blocks (const uint8 *ks, uint8 *data) +{ + aes256_hw_encrypt_blocks(data, 32, ks); +} + +#endif diff --git a/src/Crypto/Aes_hw_cpu.h b/src/Crypto/Aes_hw_cpu.h index 685a1dff..d9dda1af 100644 --- a/src/Crypto/Aes_hw_cpu.h +++ b/src/Crypto/Aes_hw_cpu.h @@ -21,10 +21,10 @@ extern "C" #endif #if defined (TC_WINDOWS_BOOT) uint8 is_aes_hw_cpu_supported (); -#endif void aes_hw_cpu_enable_sse (); +#endif void aes_hw_cpu_decrypt (const uint8 *ks, uint8 *data); void VC_CDECL aes_hw_cpu_decrypt_32_blocks (const uint8 *ks, uint8 *data); void aes_hw_cpu_encrypt (const uint8 *ks, uint8 *data); void VC_CDECL aes_hw_cpu_encrypt_32_blocks (const uint8 *ks, uint8 *data); diff --git a/src/Crypto/Crypto.vcxproj b/src/Crypto/Crypto.vcxproj index 4aebc084..c17bd607 100644 --- a/src/Crypto/Crypto.vcxproj +++ b/src/Crypto/Crypto.vcxproj @@ -225,8 +225,12 @@ <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild> </ClCompile> <ClCompile Include="Aeskey.c" /> <ClCompile Include="Aestab.c" /> + <ClCompile Include="Aes_hw_armv8.c"> + <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild> + <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild> + </ClCompile> <ClCompile Include="blake2s.c" /> <ClCompile Include="blake2s_SSE2.c" /> <ClCompile Include="blake2s_SSE41.c" /> <ClCompile Include="blake2s_SSSE3.c" /> diff --git a/src/Crypto/Crypto.vcxproj.filters b/src/Crypto/Crypto.vcxproj.filters index 3d384f97..f2b1b54d 100644 --- a/src/Crypto/Crypto.vcxproj.filters +++ b/src/Crypto/Crypto.vcxproj.filters @@ -89,8 +89,14 @@ </ClCompile> <ClCompile Include="Sha2Intel.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="Aescrypt.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="Aes_hw_armv8.c"> + <Filter>Source Files</Filter> + </ClCompile> </ItemGroup> <ItemGroup> <ClInclude Include="Aes.h"> <Filter>Header Files</Filter> @@ -166,17 +172,8 @@ </ClInclude> <ClInclude Include="t1ha_selfcheck.h"> <Filter>Header Files</Filter> </ClInclude> - <ClInclude Include="blake2s-load-sse2.h"> - <Filter>Header Files</Filter> - </ClInclude> - <ClInclude Include="blake2s-load-sse41.h"> - <Filter>Header Files</Filter> - </ClInclude> - <ClInclude Include="blake2s-round.h"> - <Filter>Header Files</Filter> - </ClInclude> </ItemGroup> <ItemGroup> <CustomBuild Include="Aes_hw_cpu.asm"> <Filter>Source Files</Filter> diff --git a/src/Crypto/config.h b/src/Crypto/config.h index 1c2aff72..dd8e3f06 100644 --- a/src/Crypto/config.h +++ b/src/Crypto/config.h @@ -28,8 +28,13 @@ #define CRYPTOPP_APPLE_CLANG_VERSION (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) #define CRYPTOPP_CLANG_INTEGRATED_ASSEMBLER 1 #endif +#if defined(_MSC_VER) && !defined(__clang__) +# undef CRYPTOPP_LLVM_CLANG_VERSION +# define CRYPTOPP_MSC_VERSION (_MSC_VER) +#endif + // Clang due to "Inline assembly operands don't work with .intel_syntax", http://llvm.org/bugs/show_bug.cgi?id=24232 // TODO: supply the upper version when LLVM fixes it. We set it to 20.0 for compilation purposes. #if (defined(CRYPTOPP_LLVM_CLANG_VERSION) && CRYPTOPP_LLVM_CLANG_VERSION <= 200000) || (defined(CRYPTOPP_APPLE_CLANG_VERSION) && CRYPTOPP_APPLE_CLANG_VERSION <= 200000) || defined(CRYPTOPP_CLANG_INTEGRATED_ASSEMBLER) #define CRYPTOPP_DISABLE_INTEL_ASM 1 @@ -200,8 +205,42 @@ #else #define CRYPTOPP_BOOL_X64 0 #endif +#if defined(__arm64__) || defined(__aarch64__) || defined(_M_ARM64) + #define CRYPTOPP_BOOL_ARMV8 1 + #define CRYPTOPP_BOOL_ARM64 1 +#else + #define CRYPTOPP_BOOL_ARMV8 0 + #define CRYPTOPP_BOOL_ARM64 0 +#endif + +// ARMv8 and ASIMD. -march=armv8-a or above must be present +// Requires GCC 4.8, Clang 3.3 or Visual Studio 2017 +// Do not use APPLE_CLANG_VERSION; use __ARM_FEATURE_XXX instead. +#if !defined(CRYPTOPP_ARM_ASIMD_AVAILABLE) && !defined(CRYPTOPP_DISABLE_ARM_ASIMD) +# if defined(__aarch32__) || defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) +# if defined(__ARM_NEON) || defined(__ARM_ASIMD) || defined(__ARM_FEATURE_NEON) || defined(__ARM_FEATURE_ASIMD) || \ + (CRYPTOPP_GCC_VERSION >= 40800) || (CRYPTOPP_LLVM_CLANG_VERSION >= 30300) || \ + (CRYPTOPP_APPLE_CLANG_VERSION >= 40000) || (CRYPTOPP_MSC_VERSION >= 1916) +# define CRYPTOPP_ARM_NEON_AVAILABLE 1 +# define CRYPTOPP_ARM_ASIMD_AVAILABLE 1 +# endif // Compilers +# endif // Platforms +#endif + +// ARMv8 and AES. -march=armv8-a+crypto or above must be present +// Requires GCC 4.8, Clang 3.3 or Visual Studio 2017 +#if !defined(CRYPTOPP_ARM_AES_AVAILABLE) && !defined(CRYPTOPP_DISABLE_ARM_AES) +# if defined(__aarch32__) || defined(__aarch64__) || defined(_M_ARM64) +# if defined(__ARM_FEATURE_CRYPTO) || (CRYPTOPP_GCC_VERSION >= 40800) || \ + (CRYPTOPP_LLVM_CLANG_VERSION >= 30300) || (CRYPTOPP_APPLE_CLANG_VERSION >= 40300) || \ + (CRYPTOPP_MSC_VERSION >= 1916) +# define CRYPTOPP_ARM_AES_AVAILABLE 1 +# endif // Compilers +# endif // Platforms +#endif + // Undo the ASM and Intrinsic related defines due to X32. #if CRYPTOPP_BOOL_X32 # undef CRYPTOPP_BOOL_X64 # undef CRYPTOPP_X64_ASM_AVAILABLE diff --git a/src/Crypto/cpu.c b/src/Crypto/cpu.c index e611e9bb..85278a92 100644 --- a/src/Crypto/cpu.c +++ b/src/Crypto/cpu.c @@ -468,4 +468,42 @@ void DisableCPUExtendedFeatures () } #endif +#if CRYPTOPP_BOOL_ARMV8 + +volatile int g_hasAESARM = 0; + +#ifndef HWCAP_AES +# define HWCAP_AES (1 << 3) +#endif + +inline int CPU_QueryAES() +{ +#if defined(CRYPTOPP_ARM_AES_AVAILABLE) +#if defined(__linux__) && defined(__aarch64__) + if ((getauxval(AT_HWCAP) & HWCAP_AES) != 0) + return 1; +#elif defined(__APPLE__) && defined(__aarch64__) + // Apple Sillcon (M1) and later + return 1; +#elif defined(_WIN32) && defined(_M_ARM64) +#ifdef TC_WINDOWS_DRIVER + if (ExIsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) != 0) + return 1; +#else + if (IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) != 0) + return 1; +#endif +#endif + return 0; +#else + return 0; +#endif +} + +void DetectArmFeatures() +{ + g_hasAESARM = CPU_QueryAES(); +} + +#endif
\ No newline at end of file diff --git a/src/Crypto/cpu.h b/src/Crypto/cpu.h index b0df6462..761258f2 100644 --- a/src/Crypto/cpu.h +++ b/src/Crypto/cpu.h @@ -287,8 +287,26 @@ void DisableCPUExtendedFeatures (); #if defined(__cplusplus) } #endif +#elif CRYPTOPP_BOOL_ARMV8 +#if defined(__cplusplus) +extern "C" { +#endif + +#if !defined(CRYPTOPP_DISABLE_AESNI) && !defined(WOLFCRYPT_BACKEND) +#define TC_AES_HW_CPU +#endif + +extern volatile int g_hasAESARM; +void DetectArmFeatures(); + +#define HasAESNI() g_hasAESARM + +#if defined(__cplusplus) +} +#endif + #else #define HasSSE2() 0 #define HasISSE() 0 diff --git a/src/Driver/Driver.vcxproj b/src/Driver/Driver.vcxproj index aa920225..628e24a4 100644 --- a/src/Driver/Driver.vcxproj +++ b/src/Driver/Driver.vcxproj @@ -227,8 +227,12 @@ copy $(OutDir)veracrypt.inf "$(SolutionDir)Debug\Setup Files\veracrypt.inf"</Com <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild> </ClCompile> <ClCompile Include="..\Crypto\Aeskey.c" /> <ClCompile Include="..\Crypto\Aestab.c" /> + <ClCompile Include="..\Crypto\Aes_hw_armv8.c"> + <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild> + <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild> + </ClCompile> <ClCompile Include="..\Crypto\blake2s.c" /> <ClCompile Include="..\Crypto\blake2s_SSE2.c"> <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">true</ExcludedFromBuild> <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">true</ExcludedFromBuild> diff --git a/src/Driver/Driver.vcxproj.filters b/src/Driver/Driver.vcxproj.filters index 478432fa..6f43b0e8 100644 --- a/src/Driver/Driver.vcxproj.filters +++ b/src/Driver/Driver.vcxproj.filters @@ -164,8 +164,11 @@ </ClCompile> <ClCompile Include="..\Driver\VolumeFilter.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Crypto\Aes_hw_armv8.c"> + <Filter>Crypto\Source Files</Filter> + </ClCompile> </ItemGroup> <ItemGroup> <ClInclude Include="..\Common\Tcdefs.h"> <Filter>Common</Filter> diff --git a/src/Driver/Ntdriver.c b/src/Driver/Ntdriver.c index ab208019..6d218517 100644 --- a/src/Driver/Ntdriver.c +++ b/src/Driver/Ntdriver.c @@ -231,17 +231,18 @@ void GetDriverRandomSeed (unsigned char* pbRandSeed, size_t cbRandSeed) WHIRLPOOL_add (digest, (unsigned int) rndLen, &tctx); jent_entropy_collector_free (ec); } } - +#ifndef _M_ARM64 // use RDSEED or RDRAND from CPU as source of entropy if enabled if ( IsCpuRngEnabled() && ( (HasRDSEED() && RDSEED_getBytes (digest, sizeof (digest))) || (HasRDRAND() && RDRAND_getBytes (digest, sizeof (digest))) )) { WHIRLPOOL_add (digest, sizeof(digest), &tctx); } +#endif WHIRLPOOL_finalize (&tctx, digest); count = VC_MIN (cbRandSeed, sizeof (digest)); @@ -265,9 +266,13 @@ NTSTATUS DriverEntry(PDRIVER_OBJECT DriverObject, PUNICODE_STRING RegistryPath) int i; Dump("DriverEntry " TC_APP_NAME " " VERSION_STRING VERSION_STRING_SUFFIX "\n"); +#ifndef _M_ARM64 DetectX86Features(); +#else + DetectArmFeatures(); +#endif PsGetVersion(&OsMajorVersion, &OsMinorVersion, NULL, NULL); Dump("OsMajorVersion=%d OsMinorVersion=%d\n", OsMajorVersion, OsMinorVersion); @@ -292,9 +297,13 @@ NTSTATUS DriverEntry(PDRIVER_OBJECT DriverObject, PUNICODE_STRING RegistryPath) if (!SelfTestsPassed) { // in case of system encryption, if self-tests fail, disable all extended CPU // features and try again in order to workaround faulty configurations +#ifndef _M_ARM64 DisableCPUExtendedFeatures(); +#else + EnableHwEncryption(FALSE); +#endif SelfTestsPassed = AutoTestAlgorithms(); // BUG CHECK if the self-tests still fail if (!SelfTestsPassed) diff --git a/src/ExpandVolume/WinMain.cpp b/src/ExpandVolume/WinMain.cpp index b7a14662..64fccd55 100644 --- a/src/ExpandVolume/WinMain.cpp +++ b/src/ExpandVolume/WinMain.cpp @@ -1092,9 +1092,13 @@ int WINAPI wWinMain (HINSTANCE hInstance, HINSTANCE hPrevInstance, wchar_t *lpsz /* application title */ lpszTitle = L"VeraCrypt Expander"; - DetectX86Features (); +#ifndef _M_ARM64 + DetectX86Features(); +#else + DetectArmFeatures(); +#endif status = DriverAttach (); if (status != 0) { diff --git a/src/Format/Tcformat.c b/src/Format/Tcformat.c index 3c2422a5..4afeb174 100644 --- a/src/Format/Tcformat.c +++ b/src/Format/Tcformat.c @@ -10587,9 +10587,13 @@ int WINAPI wWinMain (HINSTANCE hInstance, HINSTANCE hPrevInstance, wchar_t *lpsz VirtualLock (&szFileName, sizeof(szFileName)); VirtualLock (&szDiskFile, sizeof(szDiskFile)); - DetectX86Features (); +#ifndef _M_ARM64 + DetectX86Features(); +#else + DetectArmFeatures(); +#endif try { BootEncObj = new BootEncryption (NULL); diff --git a/src/Main/UserInterface.cpp b/src/Main/UserInterface.cpp index 8da77f5b..5f82db49 100644 --- a/src/Main/UserInterface.cpp +++ b/src/Main/UserInterface.cpp @@ -555,8 +555,11 @@ namespace VeraCrypt #ifdef CRYPTOPP_CPUID_AVAILABLE DetectX86Features (); #endif +#if CRYPTOPP_BOOL_ARMV8 + DetectArmFeatures(); +#endif LangString.Init(); Core->Init(); CmdLine.reset (new CommandLineInterface (argc, argv, InterfaceType)); @@ -971,9 +974,9 @@ const FileManager fileManagers[] = { } } ShowWarning(wxT("Unable to find a file manager to open the mounted volume.\n" - "Please install xdg-utils or set a default file manager.")); + "Please install xdg-utils or set a default file manager.")); #endif } bool UserInterface::ProcessCommandLine () diff --git a/src/Makefile b/src/Makefile index b176975e..4f282e5a 100644 --- a/src/Makefile +++ b/src/Makefile @@ -161,8 +161,11 @@ else ifneq (,$(filter x86_64 x86-64 amd64 x64,$(ARCH))) ASFLAGS += -f elf64 -D __BITS__=64 else ifneq (,$(filter armv7l,$(ARCH))) PLATFORM_ARCH := armv7 CPU_ARCH = armv7 +else ifneq (,$(filter aarch64 arm64 armv8l,$(ARCH))) + PLATFORM_ARCH := arm64 + CPU_ARCH = arm64 endif ifeq "$(origin NOASM)" "command line" CPU_ARCH = unknown @@ -337,8 +340,11 @@ $(error Specified SDK version was not found, ensure your active developer direct C_CXX_FLAGS += -DVC_MACOSX_FUSET VC_FUSE_PACKAGE := fuse-t endif + export CFLAGS_ARM64 := $(CFLAGS) $(C_CXX_FLAGS) -arch arm64 -march=armv8-a+crypto + export CFLAGS_X64 := $(CFLAGS) $(C_CXX_FLAGS) -arch x86_64 + # Set x86 assembly flags (-msse2, -mssse3, -msse4.1) # Apply flags if SIMD_SUPPORTED is 1 or if not in local development build (we are creating universal binary in this case) ifneq "$(LOCAL_DEVELOPMENT_BUILD)" "true" SIMD_SUPPORTED = 1 diff --git a/src/Mount/Mount.c b/src/Mount/Mount.c index c90e2bb3..f543f371 100644 --- a/src/Mount/Mount.c +++ b/src/Mount/Mount.c @@ -10131,9 +10131,13 @@ int WINAPI wWinMain (HINSTANCE hInstance, HINSTANCE hPrevInstance, wchar_t *lpsz VirtualLock (&mountOptions, sizeof (mountOptions)); VirtualLock (&defaultMountOptions, sizeof (defaultMountOptions)); VirtualLock (&szFileName, sizeof(szFileName)); - DetectX86Features (); +#ifndef _M_ARM64 + DetectX86Features(); +#else + DetectArmFeatures(); +#endif try { BootEncObj = new BootEncryption (NULL); diff --git a/src/Volume/Volume.make b/src/Volume/Volume.make index 52d212eb..e38542bb 100644 --- a/src/Volume/Volume.make +++ b/src/Volume/Volume.make @@ -42,8 +42,9 @@ ifeq "$(ENABLE_WOLFCRYPT)" "0" ifeq "$(PLATFORM)" "MacOSX" ifneq "$(COMPILE_ASM)" "false" OBJSEX += ../Crypto/Aes_asm.oo OBJS += ../Crypto/Aes_hw_cpu.o + OBJSEX += ../Crypto/Aes_hw_armv8.oo OBJS += ../Crypto/Aescrypt.o OBJSEX += ../Crypto/Twofish_asm.oo OBJSEX += ../Crypto/Camellia_asm.oo OBJSEX += ../Crypto/Camellia_aesni_asm.oo @@ -77,8 +78,11 @@ else ifeq "$(CPU_ARCH)" "x64" OBJS += ../Crypto/sha256_sse4_x64.o OBJS += ../Crypto/sha512_avx1_x64.o OBJS += ../Crypto/sha512_avx2_x64.o OBJS += ../Crypto/sha512_sse4_x64.o +else ifeq "$(CPU_ARCH)" "arm64" + OBJARMV8CRYPTO += ../Crypto/Aes_hw_armv8.oarmv8crypto + OBJS += ../Crypto/Aescrypt.o else OBJS += ../Crypto/Aescrypt.o endif @@ -139,8 +143,14 @@ VolumeLibrary: Volume.a ifeq "$(ENABLE_WOLFCRYPT)" "0" ifeq "$(PLATFORM)" "MacOSX" ifneq "$(COMPILE_ASM)" "false" +../Crypto/Aes_hw_armv8.oo: ../Crypto/Aes_hw_armv8.c + @echo Compiling $(<F) + $(CC) $(CFLAGS_ARM64) -c ../Crypto/Aes_hw_armv8.c -o ../Crypto/Aes_hw_armv8_arm64.o + $(CC) $(CFLAGS_X64) -c ../Crypto/Aes_hw_armv8.c -o ../Crypto/Aes_hw_armv8_x64.o + lipo -create ../Crypto/Aes_hw_armv8_arm64.o ../Crypto/Aes_hw_armv8_x64.o -output ../Crypto/Aes_hw_armv8.oo + rm -fr ../Crypto/Aes_hw_armv8_arm64.o ../Crypto/Aes_hw_armv8_x64.o ../Crypto/Aes_asm.oo: ../Crypto/Aes_x86.asm ../Crypto/Aes_x64.asm @echo Assembling $(<F) $(AS) $(ASFLAGS32) -o ../Crypto/Aes_x86.o ../Crypto/Aes_x86.asm $(AS) $(ASFLAGS64) -o ../Crypto/Aes_x64.o ../Crypto/Aes_x64.asm |