VeraCrypt
aboutsummaryrefslogtreecommitdiff
path: root/src/Common/EncryptionThreadPool.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/Common/EncryptionThreadPool.c')
-rw-r--r--src/Common/EncryptionThreadPool.c270
1 files changed, 238 insertions, 32 deletions
diff --git a/src/Common/EncryptionThreadPool.c b/src/Common/EncryptionThreadPool.c
index f1207113..8a0c6e78 100644
--- a/src/Common/EncryptionThreadPool.c
+++ b/src/Common/EncryptionThreadPool.c
@@ -3,9 +3,9 @@
Copyright (c) 2008-2012 TrueCrypt Developers Association and which is governed
by the TrueCrypt License 3.0.
Modifications and additions to the original source code (contained in this file)
- and all other portions of this file are Copyright (c) 2013-2016 IDRIX
+ and all other portions of this file are Copyright (c) 2013-2017 IDRIX
and are governed by the Apache License 2.0 the full text of which is
contained in the file License.txt included in VeraCrypt binary and source
code distribution packages.
*/
@@ -15,11 +15,18 @@
#ifdef DEVICE_DRIVER
#include "Driver/Ntdriver.h"
#endif
-#define TC_ENC_THREAD_POOL_MAX_THREAD_COUNT 64
+//Increasing the maximum number of threads
+#define TC_ENC_THREAD_POOL_MAX_THREAD_COUNT 256 //64
#define TC_ENC_THREAD_POOL_QUEUE_SIZE (TC_ENC_THREAD_POOL_MAX_THREAD_COUNT * 2)
+#define TC_ENC_THREAD_POOL_LEGACY_MAX_THREAD_COUNT 64
+#define TC_ENC_THREAD_POOL_LEGACY_QUEUE_SIZE (TC_ENC_THREAD_POOL_LEGACY_MAX_THREAD_COUNT * 2)
+
+static volatile size_t ThreadPoolCount = TC_ENC_THREAD_POOL_LEGACY_MAX_THREAD_COUNT;
+static volatile int ThreadQueueSize = TC_ENC_THREAD_POOL_LEGACY_QUEUE_SIZE;
+
#ifdef DEVICE_DRIVER
#define TC_THREAD_HANDLE PKTHREAD
#define TC_THREAD_PROC VOID
@@ -42,8 +49,20 @@
#define TC_MUTEX HANDLE
#define TC_ACQUIRE_MUTEX(MUTEX) WaitForSingleObject (*(MUTEX), INFINITE)
#define TC_RELEASE_MUTEX(MUTEX) ReleaseMutex (*(MUTEX))
+typedef BOOL (WINAPI *SetThreadGroupAffinityFn)(
+ HANDLE hThread,
+ const GROUP_AFFINITY *GroupAffinity,
+ PGROUP_AFFINITY PreviousGroupAffinity
+);
+
+typedef WORD (WINAPI* GetActiveProcessorGroupCountFn)();
+
+typedef DWORD (WINAPI *GetActiveProcessorCountFn)(
+ WORD GroupNumber
+);
+
#endif // !DEVICE_DRIVER
typedef enum
@@ -68,9 +87,9 @@ typedef struct EncryptionThreadPoolWorkItemStruct
{
struct
{
PCRYPTO_INFO CryptoInfo;
- byte *Data;
+ uint8 *Data;
UINT64_STRUCT StartUnitNo;
uint32 UnitCount;
} Encryption;
@@ -78,18 +97,30 @@ typedef struct EncryptionThreadPoolWorkItemStruct
struct
{
TC_EVENT *CompletionEvent;
LONG *CompletionFlag;
- char *DerivedKey;
+ unsigned char *DerivedKey;
int IterationCount;
TC_EVENT *NoOutstandingWorkItemEvent;
LONG *OutstandingWorkItemCount;
- char *Password;
+ unsigned char *Password;
int PasswordLength;
int Pkcs5Prf;
- char *Salt;
+ unsigned char *Salt;
} KeyDerivation;
+
+ struct
+ {
+ TC_EVENT *KeyDerivationCompletedEvent;
+ TC_EVENT *NoOutstandingWorkItemEvent;
+ LONG *outstandingWorkItemCount;
+ void* keyInfoBuffer;
+ int keyInfoBufferSize;
+ void* keyDerivationWorkItems;
+ int keyDerivationWorkItemsSize;
+
+ } ReadVolumeHeaderFinalization;
};
} EncryptionThreadPoolWorkItem;
@@ -98,8 +129,9 @@ static volatile BOOL ThreadPoolRunning = FALSE;
static volatile BOOL StopPending = FALSE;
static uint32 ThreadCount;
static TC_THREAD_HANDLE ThreadHandles[TC_ENC_THREAD_POOL_MAX_THREAD_COUNT];
+static WORD ThreadProcessorGroups[TC_ENC_THREAD_POOL_MAX_THREAD_COUNT];
static EncryptionThreadPoolWorkItem WorkItemQueue[TC_ENC_THREAD_POOL_QUEUE_SIZE];
static volatile int EnqueuePosition;
@@ -110,8 +142,39 @@ static TC_MUTEX DequeueMutex;
static TC_EVENT WorkItemReadyEvent;
static TC_EVENT WorkItemCompletedEvent;
+void EncryptDataUnitsCurrentThreadEx (unsigned __int8 *buf, const UINT64_STRUCT *structUnitNo, TC_LARGEST_COMPILER_UINT nbrUnits, PCRYPTO_INFO ci)
+{
+ if (IsRamEncryptionEnabled())
+ {
+ CRYPTO_INFO tmpCI;
+ memcpy (&tmpCI, ci, sizeof (CRYPTO_INFO));
+ VcUnprotectKeys (&tmpCI, VcGetEncryptionID (ci));
+
+ EncryptDataUnitsCurrentThread (buf, structUnitNo, nbrUnits, &tmpCI);
+
+ burn (&tmpCI, sizeof(CRYPTO_INFO));
+ }
+ else
+ EncryptDataUnitsCurrentThread (buf, structUnitNo, nbrUnits, ci);
+}
+
+void DecryptDataUnitsCurrentThreadEx (unsigned __int8 *buf, const UINT64_STRUCT *structUnitNo, TC_LARGEST_COMPILER_UINT nbrUnits, PCRYPTO_INFO ci)
+{
+ if (IsRamEncryptionEnabled())
+ {
+ CRYPTO_INFO tmpCI;
+ memcpy (&tmpCI, ci, sizeof (CRYPTO_INFO));
+ VcUnprotectKeys (&tmpCI, VcGetEncryptionID (ci));
+
+ DecryptDataUnitsCurrentThread (buf, structUnitNo, nbrUnits, &tmpCI);
+
+ burn (&tmpCI, sizeof(CRYPTO_INFO));
+ }
+ else
+ DecryptDataUnitsCurrentThread (buf, structUnitNo, nbrUnits, ci);
+}
static WorkItemState GetWorkItemState (EncryptionThreadPoolWorkItem *workItem)
{
return InterlockedExchangeAdd ((LONG *) &workItem->State, 0);
@@ -126,16 +189,33 @@ static void SetWorkItemState (EncryptionThreadPoolWorkItem *workItem, WorkItemSt
static TC_THREAD_PROC EncryptionThreadProc (void *threadArg)
{
EncryptionThreadPoolWorkItem *workItem;
+ if (threadArg)
+ {
+#ifdef DEVICE_DRIVER
+ SetThreadCpuGroupAffinity ((USHORT) *(WORD*)(threadArg));
+#else
+ SetThreadGroupAffinityFn SetThreadGroupAffinityPtr = (SetThreadGroupAffinityFn) GetProcAddress (GetModuleHandle (L"kernel32.dll"), "SetThreadGroupAffinity");
+ if (SetThreadGroupAffinityPtr && threadArg)
+ {
+ GROUP_AFFINITY groupAffinity = {0};
+ groupAffinity.Mask = ~0ULL;
+ groupAffinity.Group = *(WORD*)(threadArg);
+ SetThreadGroupAffinityPtr(GetCurrentThread(), &groupAffinity, NULL);
+ }
+
+#endif
+ }
+
while (!StopPending)
{
TC_ACQUIRE_MUTEX (&DequeueMutex);
workItem = &WorkItemQueue[DequeuePosition++];
- if (DequeuePosition >= TC_ENC_THREAD_POOL_QUEUE_SIZE)
+ if (DequeuePosition >= ThreadQueueSize)
DequeuePosition = 0;
while (!StopPending && GetWorkItemState (workItem) != WorkItemReady)
{
@@ -151,20 +231,20 @@ static TC_THREAD_PROC EncryptionThreadProc (void *threadArg)
switch (workItem->Type)
{
case DecryptDataUnitsWork:
- DecryptDataUnitsCurrentThread (workItem->Encryption.Data, &workItem->Encryption.StartUnitNo, workItem->Encryption.UnitCount, workItem->Encryption.CryptoInfo);
+ DecryptDataUnitsCurrentThreadEx (workItem->Encryption.Data, &workItem->Encryption.StartUnitNo, workItem->Encryption.UnitCount, workItem->Encryption.CryptoInfo);
break;
case EncryptDataUnitsWork:
- EncryptDataUnitsCurrentThread (workItem->Encryption.Data, &workItem->Encryption.StartUnitNo, workItem->Encryption.UnitCount, workItem->Encryption.CryptoInfo);
+ EncryptDataUnitsCurrentThreadEx (workItem->Encryption.Data, &workItem->Encryption.StartUnitNo, workItem->Encryption.UnitCount, workItem->Encryption.CryptoInfo);
break;
case DeriveKeyWork:
switch (workItem->KeyDerivation.Pkcs5Prf)
{
- case RIPEMD160:
- derive_key_ripemd160 (workItem->KeyDerivation.Password, workItem->KeyDerivation.PasswordLength, workItem->KeyDerivation.Salt, PKCS5_SALT_SIZE,
+ case BLAKE2S:
+ derive_key_blake2s (workItem->KeyDerivation.Password, workItem->KeyDerivation.PasswordLength, workItem->KeyDerivation.Salt, PKCS5_SALT_SIZE,
workItem->KeyDerivation.IterationCount, workItem->KeyDerivation.DerivedKey, GetMaxPkcs5OutSize());
break;
case SHA512:
@@ -200,8 +280,39 @@ static TC_THREAD_PROC EncryptionThreadProc (void *threadArg)
SetWorkItemState (workItem, WorkItemFree);
TC_SET_EVENT (WorkItemCompletedEvent);
continue;
+ case ReadVolumeHeaderFinalizationWork:
+ TC_WAIT_EVENT (*(workItem->ReadVolumeHeaderFinalization.NoOutstandingWorkItemEvent));
+
+ if (workItem->ReadVolumeHeaderFinalization.keyDerivationWorkItems)
+ {
+ burn (workItem->ReadVolumeHeaderFinalization.keyDerivationWorkItems, workItem->ReadVolumeHeaderFinalization.keyDerivationWorkItemsSize);
+#if !defined(DEVICE_DRIVER)
+ VirtualUnlock (workItem->ReadVolumeHeaderFinalization.keyDerivationWorkItems, workItem->ReadVolumeHeaderFinalization.keyDerivationWorkItemsSize);
+#endif
+ TCfree (workItem->ReadVolumeHeaderFinalization.keyDerivationWorkItems);
+ }
+
+ if (workItem->ReadVolumeHeaderFinalization.keyInfoBuffer)
+ {
+ burn (workItem->ReadVolumeHeaderFinalization.keyInfoBuffer, workItem->ReadVolumeHeaderFinalization.keyInfoBufferSize);
+#if !defined(DEVICE_DRIVER)
+ VirtualUnlock (workItem->ReadVolumeHeaderFinalization.keyInfoBuffer, workItem->ReadVolumeHeaderFinalization.keyInfoBufferSize);
+#endif
+ TCfree (workItem->ReadVolumeHeaderFinalization.keyInfoBuffer);
+ }
+
+#if !defined(DEVICE_DRIVER)
+ CloseHandle (*(workItem->ReadVolumeHeaderFinalization.KeyDerivationCompletedEvent));
+ CloseHandle (*(workItem->ReadVolumeHeaderFinalization.NoOutstandingWorkItemEvent));
+#endif
+ TCfree (workItem->ReadVolumeHeaderFinalization.KeyDerivationCompletedEvent);
+ TCfree (workItem->ReadVolumeHeaderFinalization.NoOutstandingWorkItemEvent);
+ TCfree (workItem->ReadVolumeHeaderFinalization.outstandingWorkItemCount);
+ SetWorkItemState (workItem, WorkItemFree);
+ TC_SET_EVENT (WorkItemCompletedEvent);
+ continue;
default:
TC_THROW_FATAL_EXCEPTION;
}
@@ -222,34 +333,66 @@ static TC_THREAD_PROC EncryptionThreadProc (void *threadArg)
return 0;
#endif
}
+#ifndef DEVICE_DRIVER
+
+size_t GetCpuCount (WORD* pGroupCount)
+{
+ size_t cpuCount = 0;
+ SYSTEM_INFO sysInfo;
+ GetActiveProcessorGroupCountFn GetActiveProcessorGroupCountPtr = (GetActiveProcessorGroupCountFn) GetProcAddress (GetModuleHandle (L"Kernel32.dll"), "GetActiveProcessorGroupCount");
+ GetActiveProcessorCountFn GetActiveProcessorCountPtr = (GetActiveProcessorCountFn) GetProcAddress (GetModuleHandle (L"Kernel32.dll"), "GetActiveProcessorCount");
+ if (GetActiveProcessorGroupCountPtr && GetActiveProcessorCountPtr)
+ {
+ WORD j, groupCount = GetActiveProcessorGroupCountPtr();
+ size_t totalProcessors = 0;
+ for (j = 0; j < groupCount; ++j)
+ {
+ totalProcessors += (size_t) GetActiveProcessorCountPtr(j);
+ }
+ cpuCount = totalProcessors;
+ if (pGroupCount)
+ *pGroupCount = groupCount;
+ }
+ else
+ {
+ GetSystemInfo(&sysInfo);
+ cpuCount = (size_t) sysInfo.dwNumberOfProcessors;
+ if (pGroupCount)
+ *pGroupCount = 1;
+ }
+
+ return cpuCount;
+}
+
+#endif
+
BOOL EncryptionThreadPoolStart (size_t encryptionFreeCpuCount)
{
- size_t cpuCount, i;
+ size_t cpuCount = 0, i = 0;
+ WORD groupCount = 1;
+
+ cpuCount = GetCpuCount(&groupCount);
if (ThreadPoolRunning)
return TRUE;
-#ifdef DEVICE_DRIVER
- cpuCount = GetCpuCount();
-#else
+ if (groupCount > 1)
{
- SYSTEM_INFO sysInfo;
- GetSystemInfo (&sysInfo);
- cpuCount = sysInfo.dwNumberOfProcessors;
+ ThreadPoolCount = TC_ENC_THREAD_POOL_MAX_THREAD_COUNT;
+ ThreadQueueSize = TC_ENC_THREAD_POOL_QUEUE_SIZE;
}
-#endif
if (cpuCount > encryptionFreeCpuCount)
cpuCount -= encryptionFreeCpuCount;
if (cpuCount < 2)
return TRUE;
- if (cpuCount > TC_ENC_THREAD_POOL_MAX_THREAD_COUNT)
- cpuCount = TC_ENC_THREAD_POOL_MAX_THREAD_COUNT;
+ if (cpuCount > ThreadPoolCount)
+ cpuCount = ThreadPoolCount;
StopPending = FALSE;
DequeuePosition = 0;
EnqueuePosition = 0;
@@ -299,13 +442,42 @@ BOOL EncryptionThreadPoolStart (size_t encryptionFreeCpuCount)
}
for (ThreadCount = 0; ThreadCount < cpuCount; ++ThreadCount)
{
+ WORD* pThreadArg = NULL;
+ if (groupCount > 1)
+ {
+#ifdef DEVICE_DRIVER
+ ThreadProcessorGroups[ThreadCount] = GetCpuGroup ((size_t) ThreadCount);
+#else
+ GetActiveProcessorCountFn GetActiveProcessorCountPtr = (GetActiveProcessorCountFn) GetProcAddress (GetModuleHandle (L"Kernel32.dll"), "GetActiveProcessorCount");
+ // Determine which processor group to bind the thread to.
+ if (GetActiveProcessorCountPtr)
+ {
+ WORD j;
+ uint32 totalProcessors = 0U;
+ for (j = 0U; j < groupCount; j++)
+ {
+ totalProcessors += (uint32) GetActiveProcessorCountPtr(j);
+ if (totalProcessors >= ThreadCount)
+ {
+ ThreadProcessorGroups[ThreadCount] = j;
+ break;
+ }
+ }
+ }
+ else
+ ThreadProcessorGroups[ThreadCount] = 0;
+#endif
+ pThreadArg = &ThreadProcessorGroups[ThreadCount];
+ }
+
#ifdef DEVICE_DRIVER
- if (!NT_SUCCESS (TCStartThread (EncryptionThreadProc, NULL, &ThreadHandles[ThreadCount])))
+ if (!NT_SUCCESS(TCStartThread(EncryptionThreadProc, (void*) pThreadArg, &ThreadHandles[ThreadCount])))
#else
- if (!(ThreadHandles[ThreadCount] = (HANDLE) _beginthreadex (NULL, 0, EncryptionThreadProc, NULL, 0, NULL)))
+ if (!(ThreadHandles[ThreadCount] = (HANDLE)_beginthreadex(NULL, 0, EncryptionThreadProc, (void*) pThreadArg, 0, NULL)))
#endif
+
{
EncryptionThreadPoolStop();
return FALSE;
}
@@ -354,9 +526,9 @@ void EncryptionThreadPoolStop ()
ThreadPoolRunning = FALSE;
}
-void EncryptionThreadPoolBeginKeyDerivation (TC_EVENT *completionEvent, TC_EVENT *noOutstandingWorkItemEvent, LONG *completionFlag, LONG *outstandingWorkItemCount, int pkcs5Prf, char *password, int passwordLength, char *salt, int iterationCount, char *derivedKey)
+void EncryptionThreadPoolBeginKeyDerivation (TC_EVENT *completionEvent, TC_EVENT *noOutstandingWorkItemEvent, LONG *completionFlag, LONG *outstandingWorkItemCount, int pkcs5Prf, unsigned char *password, int passwordLength, unsigned char *salt, int iterationCount, unsigned char *derivedKey)
{
EncryptionThreadPoolWorkItem *workItem;
if (!ThreadPoolRunning)
@@ -364,9 +536,9 @@ void EncryptionThreadPoolBeginKeyDerivation (TC_EVENT *completionEvent, TC_EVENT
TC_ACQUIRE_MUTEX (&EnqueueMutex);
workItem = &WorkItemQueue[EnqueuePosition++];
- if (EnqueuePosition >= TC_ENC_THREAD_POOL_QUEUE_SIZE)
+ if (EnqueuePosition >= ThreadQueueSize)
EnqueuePosition = 0;
while (GetWorkItemState (workItem) != WorkItemFree)
{
@@ -392,16 +564,50 @@ void EncryptionThreadPoolBeginKeyDerivation (TC_EVENT *completionEvent, TC_EVENT
TC_SET_EVENT (WorkItemReadyEvent);
TC_RELEASE_MUTEX (&EnqueueMutex);
}
+void EncryptionThreadPoolBeginReadVolumeHeaderFinalization (TC_EVENT *keyDerivationCompletedEvent, TC_EVENT *noOutstandingWorkItemEvent, LONG* outstandingWorkItemCount,
+ void* keyInfoBuffer, int keyInfoBufferSize,
+ void* keyDerivationWorkItems, int keyDerivationWorkItemsSize)
+{
+ EncryptionThreadPoolWorkItem *workItem;
+
+ if (!ThreadPoolRunning)
+ TC_THROW_FATAL_EXCEPTION;
+
+ TC_ACQUIRE_MUTEX (&EnqueueMutex);
+
+ workItem = &WorkItemQueue[EnqueuePosition++];
+ if (EnqueuePosition >= ThreadQueueSize)
+ EnqueuePosition = 0;
+
+ while (GetWorkItemState (workItem) != WorkItemFree)
+ {
+ TC_WAIT_EVENT (WorkItemCompletedEvent);
+ }
+
+ workItem->Type = ReadVolumeHeaderFinalizationWork;
+ workItem->ReadVolumeHeaderFinalization.NoOutstandingWorkItemEvent = noOutstandingWorkItemEvent;
+ workItem->ReadVolumeHeaderFinalization.KeyDerivationCompletedEvent = keyDerivationCompletedEvent;
+ workItem->ReadVolumeHeaderFinalization.keyInfoBuffer = keyInfoBuffer;
+ workItem->ReadVolumeHeaderFinalization.keyInfoBufferSize = keyInfoBufferSize;
+ workItem->ReadVolumeHeaderFinalization.keyDerivationWorkItems = keyDerivationWorkItems;
+ workItem->ReadVolumeHeaderFinalization.keyDerivationWorkItemsSize = keyDerivationWorkItemsSize;
+ workItem->ReadVolumeHeaderFinalization.outstandingWorkItemCount = outstandingWorkItemCount;
+
+ SetWorkItemState (workItem, WorkItemReady);
+ TC_SET_EVENT (WorkItemReadyEvent);
+ TC_RELEASE_MUTEX (&EnqueueMutex);
+}
+
-void EncryptionThreadPoolDoWork (EncryptionThreadPoolWorkType type, byte *data, const UINT64_STRUCT *startUnitNo, uint32 unitCount, PCRYPTO_INFO cryptoInfo)
+void EncryptionThreadPoolDoWork (EncryptionThreadPoolWorkType type, uint8 *data, const UINT64_STRUCT *startUnitNo, uint32 unitCount, PCRYPTO_INFO cryptoInfo)
{
uint32 fragmentCount;
uint32 unitsPerFragment;
uint32 remainder;
- byte *fragmentData;
+ uint8 *fragmentData;
uint64 fragmentStartUnitNo;
EncryptionThreadPoolWorkItem *workItem;
EncryptionThreadPoolWorkItem *firstFragmentWorkItem;
@@ -413,13 +619,13 @@ void EncryptionThreadPoolDoWork (EncryptionThreadPoolWorkType type, byte *data,
{
switch (type)
{
case DecryptDataUnitsWork:
- DecryptDataUnitsCurrentThread (data, startUnitNo, unitCount, cryptoInfo);
+ DecryptDataUnitsCurrentThreadEx (data, startUnitNo, unitCount, cryptoInfo);
break;
case EncryptDataUnitsWork:
- EncryptDataUnitsCurrentThread (data, startUnitNo, unitCount, cryptoInfo);
+ EncryptDataUnitsCurrentThreadEx (data, startUnitNo, unitCount, cryptoInfo);
break;
default:
TC_THROW_FATAL_EXCEPTION;
@@ -463,9 +669,9 @@ void EncryptionThreadPoolDoWork (EncryptionThreadPoolWorkType type, byte *data,
while (fragmentCount-- > 0)
{
workItem = &WorkItemQueue[EnqueuePosition++];
- if (EnqueuePosition >= TC_ENC_THREAD_POOL_QUEUE_SIZE)
+ if (EnqueuePosition >= ThreadQueueSize)
EnqueuePosition = 0;
while (GetWorkItemState (workItem) != WorkItemFree)
{
@@ -479,9 +685,9 @@ void EncryptionThreadPoolDoWork (EncryptionThreadPoolWorkType type, byte *data,
workItem->Encryption.Data = fragmentData;
workItem->Encryption.UnitCount = unitsPerFragment;
workItem->Encryption.StartUnitNo.Value = fragmentStartUnitNo;
- fragmentData += unitsPerFragment * ENCRYPTION_DATA_UNIT_SIZE;
+ fragmentData += ((uint64)unitsPerFragment) * ENCRYPTION_DATA_UNIT_SIZE;
fragmentStartUnitNo += unitsPerFragment;
if (remainder > 0 && --remainder == 0)
--unitsPerFragment;
@@ -505,9 +711,9 @@ size_t GetEncryptionThreadCount ()
size_t GetMaxEncryptionThreadCount ()
{
- return TC_ENC_THREAD_POOL_MAX_THREAD_COUNT;
+ return ThreadPoolCount;
}
BOOL IsEncryptionThreadPoolRunning ()