diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/Volume/EncryptionModeXTS.cpp | 139 |
1 files changed, 90 insertions, 49 deletions
diff --git a/src/Volume/EncryptionModeXTS.cpp b/src/Volume/EncryptionModeXTS.cpp index e99b977c..66f0ff62 100644 --- a/src/Volume/EncryptionModeXTS.cpp +++ b/src/Volume/EncryptionModeXTS.cpp @@ -10,9 +10,41 @@ code distribution packages. */ +#include "Crypto/cpu.h" +#include "Crypto/misc.h" #include "EncryptionModeXTS.h" #include "Common/Crypto.h" +#if (CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE && CRYPTOPP_BOOL_X64) + +#define XorBlocks(result,ptr,len,start,end) \ + while (len >= 2) \ + { \ + __m128i xmm1 = _mm_loadu_si128((const __m128i*) ptr); \ + __m128i xmm2 = _mm_loadu_si128((__m128i*)result); \ + __m128i xmm3 = _mm_loadu_si128((const __m128i*) (ptr + 2)); \ + __m128i xmm4 = _mm_loadu_si128((__m128i*)(result + 2)); \ + \ + _mm_storeu_si128((__m128i*)result, _mm_xor_si128(xmm1, xmm2)); \ + _mm_storeu_si128((__m128i*)(result + 2), _mm_xor_si128(xmm3, xmm4)); \ + ptr+= 4; \ + result+= 4; \ + len -= 2; \ + } \ + \ + if (len) \ + { \ + __m128i xmm1 = _mm_loadu_si128((const __m128i*)ptr); \ + __m128i xmm2 = _mm_loadu_si128((__m128i*)result); \ + \ + _mm_storeu_si128((__m128i*)result, _mm_xor_si128(xmm1, xmm2)); \ + ptr+= 2; \ + result+= 2; \ + } \ + len = end - start; + +#endif + namespace VeraCrypt { void EncryptionModeXTS::Encrypt (byte *data, uint64 length) const @@ -45,9 +77,8 @@ namespace VeraCrypt uint64 *whiteningValuePtr64 = (uint64 *) whiteningValue; uint64 *bufPtr = (uint64 *) buffer; uint64 *dataUnitBufPtr; - unsigned int startBlock = startCipherBlockNo, endBlock, block; - uint64 *const finalInt64WhiteningValuesPtr = whiteningValuesPtr64 + sizeof (whiteningValues) / sizeof (*whiteningValuesPtr64) - 1; - uint64 blockCount, dataUnitNo; + unsigned int startBlock = startCipherBlockNo, endBlock, block, countBlock; + uint64 remainingBlocks, dataUnitNo; startDataUnitNo += SectorOffset; @@ -67,17 +98,18 @@ namespace VeraCrypt if (length % BYTES_PER_XTS_BLOCK) TC_THROW_FATAL_EXCEPTION; - blockCount = length / BYTES_PER_XTS_BLOCK; + remainingBlocks = length / BYTES_PER_XTS_BLOCK; // Process all blocks in the buffer - while (blockCount > 0) + while (remainingBlocks > 0) { - if (blockCount < BLOCKS_PER_XTS_DATA_UNIT) - endBlock = startBlock + (unsigned int) blockCount; + if (remainingBlocks < BLOCKS_PER_XTS_DATA_UNIT) + endBlock = startBlock + (unsigned int) remainingBlocks; else endBlock = BLOCKS_PER_XTS_DATA_UNIT; + countBlock = endBlock - startBlock; - whiteningValuesPtr64 = finalInt64WhiteningValuesPtr; + whiteningValuesPtr64 = (uint64 *) whiteningValues; whiteningValuePtr64 = (uint64 *) whiteningValue; // Encrypt the data unit number using the secondary key (in order to generate the first @@ -87,13 +119,13 @@ namespace VeraCrypt secondaryCipher.EncryptBlock (whiteningValue); // Generate subsequent whitening values for blocks in this data unit. Note that all generated 128-bit - // whitening values are stored in memory as a sequence of 64-bit integers in reverse order. + // whitening values are stored in memory as a sequence of 64-bit integers. for (block = 0; block < endBlock; block++) { if (block >= startBlock) { - *whiteningValuesPtr64-- = *whiteningValuePtr64++; - *whiteningValuesPtr64-- = *whiteningValuePtr64; + *whiteningValuesPtr64++ = *whiteningValuePtr64++; + *whiteningValuesPtr64++ = *whiteningValuePtr64; } else whiteningValuePtr64++; @@ -136,31 +168,36 @@ namespace VeraCrypt } dataUnitBufPtr = bufPtr; - whiteningValuesPtr64 = finalInt64WhiteningValuesPtr; + whiteningValuesPtr64 = (uint64 *) whiteningValues; // Encrypt all blocks in this data unit - - for (block = startBlock; block < endBlock; block++) +#if (CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE && CRYPTOPP_BOOL_X64) + XorBlocks (bufPtr, whiteningValuesPtr64, countBlock, startBlock, endBlock); +#else + for (block = 0; block < countBlock; block++) { // Pre-whitening - *bufPtr++ ^= *whiteningValuesPtr64--; - *bufPtr++ ^= *whiteningValuesPtr64--; + *bufPtr++ ^= *whiteningValuesPtr64++; + *bufPtr++ ^= *whiteningValuesPtr64++; } - +#endif // Actual encryption - cipher.EncryptBlocks ((byte *) dataUnitBufPtr, endBlock - startBlock); + cipher.EncryptBlocks ((byte *) dataUnitBufPtr, countBlock); bufPtr = dataUnitBufPtr; - whiteningValuesPtr64 = finalInt64WhiteningValuesPtr; + whiteningValuesPtr64 = (uint64 *) whiteningValues; - for (block = startBlock; block < endBlock; block++) +#if (CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE && CRYPTOPP_BOOL_X64) + XorBlocks (bufPtr, whiteningValuesPtr64, countBlock, startBlock, endBlock); +#else + for (block = 0; block < countBlock; block++) { // Post-whitening - *bufPtr++ ^= *whiteningValuesPtr64--; - *bufPtr++ ^= *whiteningValuesPtr64--; + *bufPtr++ ^= *whiteningValuesPtr64++; + *bufPtr++ ^= *whiteningValuesPtr64++; } - - blockCount -= endBlock - startBlock; +#endif + remainingBlocks -= countBlock; startBlock = 0; dataUnitNo++; *((uint64 *) byteBufUnitNo) = Endian::Little (dataUnitNo); @@ -219,9 +256,8 @@ namespace VeraCrypt uint64 *whiteningValuePtr64 = (uint64 *) whiteningValue; uint64 *bufPtr = (uint64 *) buffer; uint64 *dataUnitBufPtr; - unsigned int startBlock = startCipherBlockNo, endBlock, block; - uint64 *const finalInt64WhiteningValuesPtr = whiteningValuesPtr64 + sizeof (whiteningValues) / sizeof (*whiteningValuesPtr64) - 1; - uint64 blockCount, dataUnitNo; + unsigned int startBlock = startCipherBlockNo, endBlock, block, countBlock; + uint64 remainingBlocks, dataUnitNo; startDataUnitNo += SectorOffset; @@ -234,17 +270,18 @@ namespace VeraCrypt if (length % BYTES_PER_XTS_BLOCK) TC_THROW_FATAL_EXCEPTION; - blockCount = length / BYTES_PER_XTS_BLOCK; + remainingBlocks = length / BYTES_PER_XTS_BLOCK; // Process all blocks in the buffer - while (blockCount > 0) + while (remainingBlocks > 0) { - if (blockCount < BLOCKS_PER_XTS_DATA_UNIT) - endBlock = startBlock + (unsigned int) blockCount; + if (remainingBlocks < BLOCKS_PER_XTS_DATA_UNIT) + endBlock = startBlock + (unsigned int) remainingBlocks; else endBlock = BLOCKS_PER_XTS_DATA_UNIT; + countBlock = endBlock - startBlock; - whiteningValuesPtr64 = finalInt64WhiteningValuesPtr; + whiteningValuesPtr64 = (uint64 *) whiteningValues; whiteningValuePtr64 = (uint64 *) whiteningValue; // Encrypt the data unit number using the secondary key (in order to generate the first @@ -254,13 +291,13 @@ namespace VeraCrypt secondaryCipher.EncryptBlock (whiteningValue); // Generate subsequent whitening values for blocks in this data unit. Note that all generated 128-bit - // whitening values are stored in memory as a sequence of 64-bit integers in reverse order. + // whitening values are stored in memory as a sequence of 64-bit integers. for (block = 0; block < endBlock; block++) { if (block >= startBlock) { - *whiteningValuesPtr64-- = *whiteningValuePtr64++; - *whiteningValuesPtr64-- = *whiteningValuePtr64; + *whiteningValuesPtr64++ = *whiteningValuePtr64++; + *whiteningValuesPtr64++ = *whiteningValuePtr64; } else whiteningValuePtr64++; @@ -303,28 +340,32 @@ namespace VeraCrypt } dataUnitBufPtr = bufPtr; - whiteningValuesPtr64 = finalInt64WhiteningValuesPtr; + whiteningValuesPtr64 = (uint64 *) whiteningValues; // Decrypt blocks in this data unit - - for (block = startBlock; block < endBlock; block++) +#if (CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE && CRYPTOPP_BOOL_X64) + XorBlocks (bufPtr, whiteningValuesPtr64, countBlock, startBlock, endBlock); +#else + for (block = 0; block < countBlock; block++) { - *bufPtr++ ^= *whiteningValuesPtr64--; - *bufPtr++ ^= *whiteningValuesPtr64--; + *bufPtr++ ^= *whiteningValuesPtr64++; + *bufPtr++ ^= *whiteningValuesPtr64++; } - - cipher.DecryptBlocks ((byte *) dataUnitBufPtr, endBlock - startBlock); +#endif + cipher.DecryptBlocks ((byte *) dataUnitBufPtr, countBlock); bufPtr = dataUnitBufPtr; - whiteningValuesPtr64 = finalInt64WhiteningValuesPtr; - - for (block = startBlock; block < endBlock; block++) + whiteningValuesPtr64 = (uint64 *) whiteningValues; +#if (CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE && CRYPTOPP_BOOL_X64) + XorBlocks (bufPtr, whiteningValuesPtr64, countBlock, startBlock, endBlock); +#else + for (block = 0; block < countBlock; block++) { - *bufPtr++ ^= *whiteningValuesPtr64--; - *bufPtr++ ^= *whiteningValuesPtr64--; + *bufPtr++ ^= *whiteningValuesPtr64++; + *bufPtr++ ^= *whiteningValuesPtr64++; } - - blockCount -= endBlock - startBlock; +#endif + remainingBlocks -= countBlock; startBlock = 0; dataUnitNo++; |