diff options
Diffstat (limited to 'src/Common/EncryptionThreadPool.c')
-rw-r--r-- | src/Common/EncryptionThreadPool.c | 270 |
1 files changed, 238 insertions, 32 deletions
diff --git a/src/Common/EncryptionThreadPool.c b/src/Common/EncryptionThreadPool.c index f1207113..8a0c6e78 100644 --- a/src/Common/EncryptionThreadPool.c +++ b/src/Common/EncryptionThreadPool.c @@ -3,9 +3,9 @@ Copyright (c) 2008-2012 TrueCrypt Developers Association and which is governed by the TrueCrypt License 3.0. Modifications and additions to the original source code (contained in this file) - and all other portions of this file are Copyright (c) 2013-2016 IDRIX + and all other portions of this file are Copyright (c) 2013-2017 IDRIX and are governed by the Apache License 2.0 the full text of which is contained in the file License.txt included in VeraCrypt binary and source code distribution packages. */ @@ -15,11 +15,18 @@ #ifdef DEVICE_DRIVER #include "Driver/Ntdriver.h" #endif -#define TC_ENC_THREAD_POOL_MAX_THREAD_COUNT 64 +//Increasing the maximum number of threads +#define TC_ENC_THREAD_POOL_MAX_THREAD_COUNT 256 //64 #define TC_ENC_THREAD_POOL_QUEUE_SIZE (TC_ENC_THREAD_POOL_MAX_THREAD_COUNT * 2) +#define TC_ENC_THREAD_POOL_LEGACY_MAX_THREAD_COUNT 64 +#define TC_ENC_THREAD_POOL_LEGACY_QUEUE_SIZE (TC_ENC_THREAD_POOL_LEGACY_MAX_THREAD_COUNT * 2) + +static volatile size_t ThreadPoolCount = TC_ENC_THREAD_POOL_LEGACY_MAX_THREAD_COUNT; +static volatile int ThreadQueueSize = TC_ENC_THREAD_POOL_LEGACY_QUEUE_SIZE; + #ifdef DEVICE_DRIVER #define TC_THREAD_HANDLE PKTHREAD #define TC_THREAD_PROC VOID @@ -42,8 +49,20 @@ #define TC_MUTEX HANDLE #define TC_ACQUIRE_MUTEX(MUTEX) WaitForSingleObject (*(MUTEX), INFINITE) #define TC_RELEASE_MUTEX(MUTEX) ReleaseMutex (*(MUTEX)) +typedef BOOL (WINAPI *SetThreadGroupAffinityFn)( + HANDLE hThread, + const GROUP_AFFINITY *GroupAffinity, + PGROUP_AFFINITY PreviousGroupAffinity +); + +typedef WORD (WINAPI* GetActiveProcessorGroupCountFn)(); + +typedef DWORD (WINAPI *GetActiveProcessorCountFn)( + WORD GroupNumber +); + #endif // !DEVICE_DRIVER typedef enum @@ -68,9 +87,9 @@ typedef struct EncryptionThreadPoolWorkItemStruct { struct { PCRYPTO_INFO CryptoInfo; - byte *Data; + uint8 *Data; UINT64_STRUCT StartUnitNo; uint32 UnitCount; } Encryption; @@ -78,18 +97,30 @@ typedef struct EncryptionThreadPoolWorkItemStruct struct { TC_EVENT *CompletionEvent; LONG *CompletionFlag; - char *DerivedKey; + unsigned char *DerivedKey; int IterationCount; TC_EVENT *NoOutstandingWorkItemEvent; LONG *OutstandingWorkItemCount; - char *Password; + unsigned char *Password; int PasswordLength; int Pkcs5Prf; - char *Salt; + unsigned char *Salt; } KeyDerivation; + + struct + { + TC_EVENT *KeyDerivationCompletedEvent; + TC_EVENT *NoOutstandingWorkItemEvent; + LONG *outstandingWorkItemCount; + void* keyInfoBuffer; + int keyInfoBufferSize; + void* keyDerivationWorkItems; + int keyDerivationWorkItemsSize; + + } ReadVolumeHeaderFinalization; }; } EncryptionThreadPoolWorkItem; @@ -98,8 +129,9 @@ static volatile BOOL ThreadPoolRunning = FALSE; static volatile BOOL StopPending = FALSE; static uint32 ThreadCount; static TC_THREAD_HANDLE ThreadHandles[TC_ENC_THREAD_POOL_MAX_THREAD_COUNT]; +static WORD ThreadProcessorGroups[TC_ENC_THREAD_POOL_MAX_THREAD_COUNT]; static EncryptionThreadPoolWorkItem WorkItemQueue[TC_ENC_THREAD_POOL_QUEUE_SIZE]; static volatile int EnqueuePosition; @@ -110,8 +142,39 @@ static TC_MUTEX DequeueMutex; static TC_EVENT WorkItemReadyEvent; static TC_EVENT WorkItemCompletedEvent; +void EncryptDataUnitsCurrentThreadEx (unsigned __int8 *buf, const UINT64_STRUCT *structUnitNo, TC_LARGEST_COMPILER_UINT nbrUnits, PCRYPTO_INFO ci) +{ + if (IsRamEncryptionEnabled()) + { + CRYPTO_INFO tmpCI; + memcpy (&tmpCI, ci, sizeof (CRYPTO_INFO)); + VcUnprotectKeys (&tmpCI, VcGetEncryptionID (ci)); + + EncryptDataUnitsCurrentThread (buf, structUnitNo, nbrUnits, &tmpCI); + + burn (&tmpCI, sizeof(CRYPTO_INFO)); + } + else + EncryptDataUnitsCurrentThread (buf, structUnitNo, nbrUnits, ci); +} + +void DecryptDataUnitsCurrentThreadEx (unsigned __int8 *buf, const UINT64_STRUCT *structUnitNo, TC_LARGEST_COMPILER_UINT nbrUnits, PCRYPTO_INFO ci) +{ + if (IsRamEncryptionEnabled()) + { + CRYPTO_INFO tmpCI; + memcpy (&tmpCI, ci, sizeof (CRYPTO_INFO)); + VcUnprotectKeys (&tmpCI, VcGetEncryptionID (ci)); + + DecryptDataUnitsCurrentThread (buf, structUnitNo, nbrUnits, &tmpCI); + + burn (&tmpCI, sizeof(CRYPTO_INFO)); + } + else + DecryptDataUnitsCurrentThread (buf, structUnitNo, nbrUnits, ci); +} static WorkItemState GetWorkItemState (EncryptionThreadPoolWorkItem *workItem) { return InterlockedExchangeAdd ((LONG *) &workItem->State, 0); @@ -126,16 +189,33 @@ static void SetWorkItemState (EncryptionThreadPoolWorkItem *workItem, WorkItemSt static TC_THREAD_PROC EncryptionThreadProc (void *threadArg) { EncryptionThreadPoolWorkItem *workItem; + if (threadArg) + { +#ifdef DEVICE_DRIVER + SetThreadCpuGroupAffinity ((USHORT) *(WORD*)(threadArg)); +#else + SetThreadGroupAffinityFn SetThreadGroupAffinityPtr = (SetThreadGroupAffinityFn) GetProcAddress (GetModuleHandle (L"kernel32.dll"), "SetThreadGroupAffinity"); + if (SetThreadGroupAffinityPtr && threadArg) + { + GROUP_AFFINITY groupAffinity = {0}; + groupAffinity.Mask = ~0ULL; + groupAffinity.Group = *(WORD*)(threadArg); + SetThreadGroupAffinityPtr(GetCurrentThread(), &groupAffinity, NULL); + } + +#endif + } + while (!StopPending) { TC_ACQUIRE_MUTEX (&DequeueMutex); workItem = &WorkItemQueue[DequeuePosition++]; - if (DequeuePosition >= TC_ENC_THREAD_POOL_QUEUE_SIZE) + if (DequeuePosition >= ThreadQueueSize) DequeuePosition = 0; while (!StopPending && GetWorkItemState (workItem) != WorkItemReady) { @@ -151,20 +231,20 @@ static TC_THREAD_PROC EncryptionThreadProc (void *threadArg) switch (workItem->Type) { case DecryptDataUnitsWork: - DecryptDataUnitsCurrentThread (workItem->Encryption.Data, &workItem->Encryption.StartUnitNo, workItem->Encryption.UnitCount, workItem->Encryption.CryptoInfo); + DecryptDataUnitsCurrentThreadEx (workItem->Encryption.Data, &workItem->Encryption.StartUnitNo, workItem->Encryption.UnitCount, workItem->Encryption.CryptoInfo); break; case EncryptDataUnitsWork: - EncryptDataUnitsCurrentThread (workItem->Encryption.Data, &workItem->Encryption.StartUnitNo, workItem->Encryption.UnitCount, workItem->Encryption.CryptoInfo); + EncryptDataUnitsCurrentThreadEx (workItem->Encryption.Data, &workItem->Encryption.StartUnitNo, workItem->Encryption.UnitCount, workItem->Encryption.CryptoInfo); break; case DeriveKeyWork: switch (workItem->KeyDerivation.Pkcs5Prf) { - case RIPEMD160: - derive_key_ripemd160 (workItem->KeyDerivation.Password, workItem->KeyDerivation.PasswordLength, workItem->KeyDerivation.Salt, PKCS5_SALT_SIZE, + case BLAKE2S: + derive_key_blake2s (workItem->KeyDerivation.Password, workItem->KeyDerivation.PasswordLength, workItem->KeyDerivation.Salt, PKCS5_SALT_SIZE, workItem->KeyDerivation.IterationCount, workItem->KeyDerivation.DerivedKey, GetMaxPkcs5OutSize()); break; case SHA512: @@ -200,8 +280,39 @@ static TC_THREAD_PROC EncryptionThreadProc (void *threadArg) SetWorkItemState (workItem, WorkItemFree); TC_SET_EVENT (WorkItemCompletedEvent); continue; + case ReadVolumeHeaderFinalizationWork: + TC_WAIT_EVENT (*(workItem->ReadVolumeHeaderFinalization.NoOutstandingWorkItemEvent)); + + if (workItem->ReadVolumeHeaderFinalization.keyDerivationWorkItems) + { + burn (workItem->ReadVolumeHeaderFinalization.keyDerivationWorkItems, workItem->ReadVolumeHeaderFinalization.keyDerivationWorkItemsSize); +#if !defined(DEVICE_DRIVER) + VirtualUnlock (workItem->ReadVolumeHeaderFinalization.keyDerivationWorkItems, workItem->ReadVolumeHeaderFinalization.keyDerivationWorkItemsSize); +#endif + TCfree (workItem->ReadVolumeHeaderFinalization.keyDerivationWorkItems); + } + + if (workItem->ReadVolumeHeaderFinalization.keyInfoBuffer) + { + burn (workItem->ReadVolumeHeaderFinalization.keyInfoBuffer, workItem->ReadVolumeHeaderFinalization.keyInfoBufferSize); +#if !defined(DEVICE_DRIVER) + VirtualUnlock (workItem->ReadVolumeHeaderFinalization.keyInfoBuffer, workItem->ReadVolumeHeaderFinalization.keyInfoBufferSize); +#endif + TCfree (workItem->ReadVolumeHeaderFinalization.keyInfoBuffer); + } + +#if !defined(DEVICE_DRIVER) + CloseHandle (*(workItem->ReadVolumeHeaderFinalization.KeyDerivationCompletedEvent)); + CloseHandle (*(workItem->ReadVolumeHeaderFinalization.NoOutstandingWorkItemEvent)); +#endif + TCfree (workItem->ReadVolumeHeaderFinalization.KeyDerivationCompletedEvent); + TCfree (workItem->ReadVolumeHeaderFinalization.NoOutstandingWorkItemEvent); + TCfree (workItem->ReadVolumeHeaderFinalization.outstandingWorkItemCount); + SetWorkItemState (workItem, WorkItemFree); + TC_SET_EVENT (WorkItemCompletedEvent); + continue; default: TC_THROW_FATAL_EXCEPTION; } @@ -222,34 +333,66 @@ static TC_THREAD_PROC EncryptionThreadProc (void *threadArg) return 0; #endif } +#ifndef DEVICE_DRIVER + +size_t GetCpuCount (WORD* pGroupCount) +{ + size_t cpuCount = 0; + SYSTEM_INFO sysInfo; + GetActiveProcessorGroupCountFn GetActiveProcessorGroupCountPtr = (GetActiveProcessorGroupCountFn) GetProcAddress (GetModuleHandle (L"Kernel32.dll"), "GetActiveProcessorGroupCount"); + GetActiveProcessorCountFn GetActiveProcessorCountPtr = (GetActiveProcessorCountFn) GetProcAddress (GetModuleHandle (L"Kernel32.dll"), "GetActiveProcessorCount"); + if (GetActiveProcessorGroupCountPtr && GetActiveProcessorCountPtr) + { + WORD j, groupCount = GetActiveProcessorGroupCountPtr(); + size_t totalProcessors = 0; + for (j = 0; j < groupCount; ++j) + { + totalProcessors += (size_t) GetActiveProcessorCountPtr(j); + } + cpuCount = totalProcessors; + if (pGroupCount) + *pGroupCount = groupCount; + } + else + { + GetSystemInfo(&sysInfo); + cpuCount = (size_t) sysInfo.dwNumberOfProcessors; + if (pGroupCount) + *pGroupCount = 1; + } + + return cpuCount; +} + +#endif + BOOL EncryptionThreadPoolStart (size_t encryptionFreeCpuCount) { - size_t cpuCount, i; + size_t cpuCount = 0, i = 0; + WORD groupCount = 1; + + cpuCount = GetCpuCount(&groupCount); if (ThreadPoolRunning) return TRUE; -#ifdef DEVICE_DRIVER - cpuCount = GetCpuCount(); -#else + if (groupCount > 1) { - SYSTEM_INFO sysInfo; - GetSystemInfo (&sysInfo); - cpuCount = sysInfo.dwNumberOfProcessors; + ThreadPoolCount = TC_ENC_THREAD_POOL_MAX_THREAD_COUNT; + ThreadQueueSize = TC_ENC_THREAD_POOL_QUEUE_SIZE; } -#endif if (cpuCount > encryptionFreeCpuCount) cpuCount -= encryptionFreeCpuCount; if (cpuCount < 2) return TRUE; - if (cpuCount > TC_ENC_THREAD_POOL_MAX_THREAD_COUNT) - cpuCount = TC_ENC_THREAD_POOL_MAX_THREAD_COUNT; + if (cpuCount > ThreadPoolCount) + cpuCount = ThreadPoolCount; StopPending = FALSE; DequeuePosition = 0; EnqueuePosition = 0; @@ -299,13 +442,42 @@ BOOL EncryptionThreadPoolStart (size_t encryptionFreeCpuCount) } for (ThreadCount = 0; ThreadCount < cpuCount; ++ThreadCount) { + WORD* pThreadArg = NULL; + if (groupCount > 1) + { +#ifdef DEVICE_DRIVER + ThreadProcessorGroups[ThreadCount] = GetCpuGroup ((size_t) ThreadCount); +#else + GetActiveProcessorCountFn GetActiveProcessorCountPtr = (GetActiveProcessorCountFn) GetProcAddress (GetModuleHandle (L"Kernel32.dll"), "GetActiveProcessorCount"); + // Determine which processor group to bind the thread to. + if (GetActiveProcessorCountPtr) + { + WORD j; + uint32 totalProcessors = 0U; + for (j = 0U; j < groupCount; j++) + { + totalProcessors += (uint32) GetActiveProcessorCountPtr(j); + if (totalProcessors >= ThreadCount) + { + ThreadProcessorGroups[ThreadCount] = j; + break; + } + } + } + else + ThreadProcessorGroups[ThreadCount] = 0; +#endif + pThreadArg = &ThreadProcessorGroups[ThreadCount]; + } + #ifdef DEVICE_DRIVER - if (!NT_SUCCESS (TCStartThread (EncryptionThreadProc, NULL, &ThreadHandles[ThreadCount]))) + if (!NT_SUCCESS(TCStartThread(EncryptionThreadProc, (void*) pThreadArg, &ThreadHandles[ThreadCount]))) #else - if (!(ThreadHandles[ThreadCount] = (HANDLE) _beginthreadex (NULL, 0, EncryptionThreadProc, NULL, 0, NULL))) + if (!(ThreadHandles[ThreadCount] = (HANDLE)_beginthreadex(NULL, 0, EncryptionThreadProc, (void*) pThreadArg, 0, NULL))) #endif + { EncryptionThreadPoolStop(); return FALSE; } @@ -354,9 +526,9 @@ void EncryptionThreadPoolStop () ThreadPoolRunning = FALSE; } -void EncryptionThreadPoolBeginKeyDerivation (TC_EVENT *completionEvent, TC_EVENT *noOutstandingWorkItemEvent, LONG *completionFlag, LONG *outstandingWorkItemCount, int pkcs5Prf, char *password, int passwordLength, char *salt, int iterationCount, char *derivedKey) +void EncryptionThreadPoolBeginKeyDerivation (TC_EVENT *completionEvent, TC_EVENT *noOutstandingWorkItemEvent, LONG *completionFlag, LONG *outstandingWorkItemCount, int pkcs5Prf, unsigned char *password, int passwordLength, unsigned char *salt, int iterationCount, unsigned char *derivedKey) { EncryptionThreadPoolWorkItem *workItem; if (!ThreadPoolRunning) @@ -364,9 +536,9 @@ void EncryptionThreadPoolBeginKeyDerivation (TC_EVENT *completionEvent, TC_EVENT TC_ACQUIRE_MUTEX (&EnqueueMutex); workItem = &WorkItemQueue[EnqueuePosition++]; - if (EnqueuePosition >= TC_ENC_THREAD_POOL_QUEUE_SIZE) + if (EnqueuePosition >= ThreadQueueSize) EnqueuePosition = 0; while (GetWorkItemState (workItem) != WorkItemFree) { @@ -392,16 +564,50 @@ void EncryptionThreadPoolBeginKeyDerivation (TC_EVENT *completionEvent, TC_EVENT TC_SET_EVENT (WorkItemReadyEvent); TC_RELEASE_MUTEX (&EnqueueMutex); } +void EncryptionThreadPoolBeginReadVolumeHeaderFinalization (TC_EVENT *keyDerivationCompletedEvent, TC_EVENT *noOutstandingWorkItemEvent, LONG* outstandingWorkItemCount, + void* keyInfoBuffer, int keyInfoBufferSize, + void* keyDerivationWorkItems, int keyDerivationWorkItemsSize) +{ + EncryptionThreadPoolWorkItem *workItem; + + if (!ThreadPoolRunning) + TC_THROW_FATAL_EXCEPTION; + + TC_ACQUIRE_MUTEX (&EnqueueMutex); + + workItem = &WorkItemQueue[EnqueuePosition++]; + if (EnqueuePosition >= ThreadQueueSize) + EnqueuePosition = 0; + + while (GetWorkItemState (workItem) != WorkItemFree) + { + TC_WAIT_EVENT (WorkItemCompletedEvent); + } + + workItem->Type = ReadVolumeHeaderFinalizationWork; + workItem->ReadVolumeHeaderFinalization.NoOutstandingWorkItemEvent = noOutstandingWorkItemEvent; + workItem->ReadVolumeHeaderFinalization.KeyDerivationCompletedEvent = keyDerivationCompletedEvent; + workItem->ReadVolumeHeaderFinalization.keyInfoBuffer = keyInfoBuffer; + workItem->ReadVolumeHeaderFinalization.keyInfoBufferSize = keyInfoBufferSize; + workItem->ReadVolumeHeaderFinalization.keyDerivationWorkItems = keyDerivationWorkItems; + workItem->ReadVolumeHeaderFinalization.keyDerivationWorkItemsSize = keyDerivationWorkItemsSize; + workItem->ReadVolumeHeaderFinalization.outstandingWorkItemCount = outstandingWorkItemCount; + + SetWorkItemState (workItem, WorkItemReady); + TC_SET_EVENT (WorkItemReadyEvent); + TC_RELEASE_MUTEX (&EnqueueMutex); +} + -void EncryptionThreadPoolDoWork (EncryptionThreadPoolWorkType type, byte *data, const UINT64_STRUCT *startUnitNo, uint32 unitCount, PCRYPTO_INFO cryptoInfo) +void EncryptionThreadPoolDoWork (EncryptionThreadPoolWorkType type, uint8 *data, const UINT64_STRUCT *startUnitNo, uint32 unitCount, PCRYPTO_INFO cryptoInfo) { uint32 fragmentCount; uint32 unitsPerFragment; uint32 remainder; - byte *fragmentData; + uint8 *fragmentData; uint64 fragmentStartUnitNo; EncryptionThreadPoolWorkItem *workItem; EncryptionThreadPoolWorkItem *firstFragmentWorkItem; @@ -413,13 +619,13 @@ void EncryptionThreadPoolDoWork (EncryptionThreadPoolWorkType type, byte *data, { switch (type) { case DecryptDataUnitsWork: - DecryptDataUnitsCurrentThread (data, startUnitNo, unitCount, cryptoInfo); + DecryptDataUnitsCurrentThreadEx (data, startUnitNo, unitCount, cryptoInfo); break; case EncryptDataUnitsWork: - EncryptDataUnitsCurrentThread (data, startUnitNo, unitCount, cryptoInfo); + EncryptDataUnitsCurrentThreadEx (data, startUnitNo, unitCount, cryptoInfo); break; default: TC_THROW_FATAL_EXCEPTION; @@ -463,9 +669,9 @@ void EncryptionThreadPoolDoWork (EncryptionThreadPoolWorkType type, byte *data, while (fragmentCount-- > 0) { workItem = &WorkItemQueue[EnqueuePosition++]; - if (EnqueuePosition >= TC_ENC_THREAD_POOL_QUEUE_SIZE) + if (EnqueuePosition >= ThreadQueueSize) EnqueuePosition = 0; while (GetWorkItemState (workItem) != WorkItemFree) { @@ -479,9 +685,9 @@ void EncryptionThreadPoolDoWork (EncryptionThreadPoolWorkType type, byte *data, workItem->Encryption.Data = fragmentData; workItem->Encryption.UnitCount = unitsPerFragment; workItem->Encryption.StartUnitNo.Value = fragmentStartUnitNo; - fragmentData += unitsPerFragment * ENCRYPTION_DATA_UNIT_SIZE; + fragmentData += ((uint64)unitsPerFragment) * ENCRYPTION_DATA_UNIT_SIZE; fragmentStartUnitNo += unitsPerFragment; if (remainder > 0 && --remainder == 0) --unitsPerFragment; @@ -505,9 +711,9 @@ size_t GetEncryptionThreadCount () size_t GetMaxEncryptionThreadCount () { - return TC_ENC_THREAD_POOL_MAX_THREAD_COUNT; + return ThreadPoolCount; } BOOL IsEncryptionThreadPoolRunning () |