diff options
Diffstat (limited to 'src/Driver/EncryptedIoQueue.c')
| -rw-r--r-- | src/Driver/EncryptedIoQueue.c | 1008 |
1 files changed, 1008 insertions, 0 deletions
diff --git a/src/Driver/EncryptedIoQueue.c b/src/Driver/EncryptedIoQueue.c new file mode 100644 index 00000000..7cf48cd6 --- /dev/null +++ b/src/Driver/EncryptedIoQueue.c @@ -0,0 +1,1008 @@ +/*
+ Copyright (c) 2008-2009 TrueCrypt Developers Association. All rights reserved.
+
+ Governed by the TrueCrypt License 3.0 the full text of which is contained in
+ the file License.txt included in TrueCrypt binary and source code distribution
+ packages.
+*/
+
+#include "TCdefs.h"
+#include "Apidrvr.h"
+#include "Ntdriver.h"
+#include "DriveFilter.h"
+#include "EncryptedIoQueue.h"
+#include "EncryptionThreadPool.h"
+#include "Volumes.h"
+
+
+static void AcquireBufferPoolMutex (EncryptedIoQueue *queue)
+{
+ NTSTATUS status;
+
+ status = KeWaitForMutexObject (&queue->BufferPoolMutex, Executive, KernelMode, FALSE, NULL);
+ if (!NT_SUCCESS (status))
+ TC_BUG_CHECK (status);
+}
+
+
+static void ReleaseBufferPoolMutex (EncryptedIoQueue *queue)
+{
+ KeReleaseMutex (&queue->BufferPoolMutex, FALSE);
+}
+
+
+static void *GetPoolBuffer (EncryptedIoQueue *queue, ULONG requestedSize)
+{
+ EncryptedIoQueueBuffer *buffer;
+ void *bufferAddress = NULL;
+ BOOL requestedSizePresentInPool = FALSE;
+
+ while (TRUE)
+ {
+ AcquireBufferPoolMutex (queue);
+
+ for (buffer = queue->FirstPoolBuffer; ; buffer = buffer->NextBuffer)
+ {
+ if (buffer && buffer->Size == requestedSize)
+ {
+ requestedSizePresentInPool = TRUE;
+
+ if (!buffer->InUse)
+ {
+ // Reuse a free buffer
+ buffer->InUse = TRUE;
+ bufferAddress = buffer->Address;
+ break;
+ }
+ }
+
+ if (!buffer || !buffer->NextBuffer)
+ {
+ EncryptedIoQueueBuffer *newBuffer;
+
+ if (requestedSizePresentInPool && !queue->StartPending)
+ break;
+
+ // Allocate a new buffer
+ newBuffer = TCalloc (sizeof (EncryptedIoQueueBuffer));
+ if (!newBuffer)
+ {
+ bufferAddress = NULL;
+ break;
+ }
+
+ bufferAddress = TCalloc (requestedSize);
+ if (bufferAddress)
+ {
+ newBuffer->NextBuffer = NULL;
+ newBuffer->Address = bufferAddress;
+ newBuffer->Size = requestedSize;
+ newBuffer->InUse = TRUE;
+
+ if (!buffer)
+ queue->FirstPoolBuffer = newBuffer;
+ else
+ buffer->NextBuffer = newBuffer;
+ }
+ else
+ TCfree (newBuffer);
+
+ break;
+ }
+ }
+
+ ReleaseBufferPoolMutex (queue);
+
+ if (bufferAddress || !requestedSizePresentInPool || queue->StartPending)
+ break;
+
+ KeWaitForSingleObject (&queue->PoolBufferFreeEvent, Executive, KernelMode, FALSE, NULL);
+ }
+
+ return bufferAddress;
+}
+
+
+static void ReleasePoolBuffer (EncryptedIoQueue *queue, void *address)
+{
+ EncryptedIoQueueBuffer *buffer;
+ AcquireBufferPoolMutex (queue);
+
+ for (buffer = queue->FirstPoolBuffer; buffer != NULL; buffer = buffer->NextBuffer)
+ {
+ if (buffer->Address == address)
+ {
+ ASSERT (buffer->InUse);
+
+ buffer->InUse = FALSE;
+ break;
+ }
+ }
+
+ ReleaseBufferPoolMutex (queue);
+ KeSetEvent (&queue->PoolBufferFreeEvent, IO_DISK_INCREMENT, FALSE);
+}
+
+
+static void FreePoolBuffers (EncryptedIoQueue *queue)
+{
+ EncryptedIoQueueBuffer *buffer;
+ AcquireBufferPoolMutex (queue);
+
+ for (buffer = queue->FirstPoolBuffer; buffer != NULL; )
+ {
+ EncryptedIoQueueBuffer *nextBuffer = buffer->NextBuffer;
+
+ ASSERT (!buffer->InUse || queue->StartPending);
+
+ TCfree (buffer->Address);
+ TCfree (buffer);
+
+ buffer = nextBuffer;
+ }
+
+ queue->FirstPoolBuffer = NULL;
+ ReleaseBufferPoolMutex (queue);
+}
+
+
+static void DecrementOutstandingIoCount (EncryptedIoQueue *queue)
+{
+ if (InterlockedDecrement (&queue->OutstandingIoCount) == 0 && (queue->SuspendPending || queue->StopPending))
+ KeSetEvent (&queue->NoOutstandingIoEvent, IO_DISK_INCREMENT, FALSE);
+}
+
+
+static void OnItemCompleted (EncryptedIoQueueItem *item, BOOL freeItem)
+{
+ DecrementOutstandingIoCount (item->Queue);
+ IoReleaseRemoveLock (&item->Queue->RemoveLock, item->OriginalIrp);
+
+ if (NT_SUCCESS (item->Status))
+ {
+ if (item->Write)
+ item->Queue->TotalBytesWritten += item->OriginalLength;
+ else
+ item->Queue->TotalBytesRead += item->OriginalLength;
+ }
+
+ if (freeItem)
+ ReleasePoolBuffer (item->Queue, item);
+}
+
+
+static NTSTATUS CompleteOriginalIrp (EncryptedIoQueueItem *item, NTSTATUS status, ULONG_PTR information)
+{
+#ifdef TC_TRACE_IO_QUEUE
+ Dump ("< %I64d [%I64d] %c status=%x info=%I64d\n", item->OriginalIrpOffset, GetElapsedTime (&item->Queue->LastPerformanceCounter), item->Write ? 'W' : 'R', status, (int64) information);
+#endif
+
+ TCCompleteDiskIrp (item->OriginalIrp, status, information);
+
+ item->Status = status;
+ OnItemCompleted (item, TRUE);
+
+ return status;
+}
+
+
+static void AcquireFragmentBuffer (EncryptedIoQueue *queue, byte *buffer)
+{
+ NTSTATUS status = STATUS_INVALID_PARAMETER;
+
+ if (buffer == queue->FragmentBufferA)
+ {
+ status = KeWaitForSingleObject (&queue->FragmentBufferAFreeEvent, Executive, KernelMode, FALSE, NULL);
+ }
+ else if (buffer == queue->FragmentBufferB)
+ {
+ status = KeWaitForSingleObject (&queue->FragmentBufferBFreeEvent, Executive, KernelMode, FALSE, NULL);
+ }
+
+ if (!NT_SUCCESS (status))
+ TC_BUG_CHECK (status);
+}
+
+
+static void ReleaseFragmentBuffer (EncryptedIoQueue *queue, byte *buffer)
+{
+ if (buffer == queue->FragmentBufferA)
+ {
+ KeSetEvent (&queue->FragmentBufferAFreeEvent, IO_DISK_INCREMENT, FALSE);
+ }
+ else if (buffer == queue->FragmentBufferB)
+ {
+ KeSetEvent (&queue->FragmentBufferBFreeEvent, IO_DISK_INCREMENT, FALSE);
+ }
+ else
+ {
+ TC_BUG_CHECK (STATUS_INVALID_PARAMETER);
+ }
+}
+
+
+static VOID CompletionThreadProc (PVOID threadArg)
+{
+ EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg;
+ PLIST_ENTRY listEntry;
+ EncryptedIoRequest *request;
+ UINT64_STRUCT dataUnit;
+
+ if (IsEncryptionThreadPoolRunning())
+ KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY);
+
+ while (!queue->ThreadExitRequested)
+ {
+ if (!NT_SUCCESS (KeWaitForSingleObject (&queue->CompletionThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL)))
+ continue;
+
+ if (queue->ThreadExitRequested)
+ break;
+
+ while ((listEntry = ExInterlockedRemoveHeadList (&queue->CompletionThreadQueue, &queue->CompletionThreadQueueLock)))
+ {
+ request = CONTAINING_RECORD (listEntry, EncryptedIoRequest, CompletionListEntry);
+
+ if (request->EncryptedLength > 0 && NT_SUCCESS (request->Item->Status))
+ {
+ ASSERT (request->EncryptedOffset + request->EncryptedLength <= request->Offset.QuadPart + request->Length);
+ dataUnit.Value = (request->Offset.QuadPart + request->EncryptedOffset) / ENCRYPTION_DATA_UNIT_SIZE;
+
+ if (queue->CryptoInfo->bPartitionInInactiveSysEncScope)
+ dataUnit.Value += queue->CryptoInfo->FirstDataUnitNo.Value;
+ else if (queue->RemapEncryptedArea)
+ dataUnit.Value += queue->RemappedAreaDataUnitOffset;
+
+ DecryptDataUnits (request->Data + request->EncryptedOffset, &dataUnit, request->EncryptedLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo);
+ }
+
+ if (request->CompleteOriginalIrp)
+ {
+ CompleteOriginalIrp (request->Item, request->Item->Status,
+ NT_SUCCESS (request->Item->Status) ? request->Item->OriginalLength : 0);
+ }
+
+ ReleasePoolBuffer (queue, request);
+ }
+ }
+
+ PsTerminateSystemThread (STATUS_SUCCESS);
+}
+
+
+static NTSTATUS TCCachedRead (EncryptedIoQueue *queue, IO_STATUS_BLOCK *ioStatus, PVOID buffer, LARGE_INTEGER offset, ULONG length)
+{
+ queue->LastReadOffset = offset;
+ queue->LastReadLength = length;
+
+ if (queue->ReadAheadBufferValid && queue->ReadAheadOffset.QuadPart == offset.QuadPart && queue->ReadAheadLength >= length)
+ {
+ memcpy (buffer, queue->ReadAheadBuffer, length);
+
+ if (!queue->IsFilterDevice)
+ {
+ ioStatus->Information = length;
+ ioStatus->Status = STATUS_SUCCESS;
+ }
+
+ return STATUS_SUCCESS;
+ }
+
+ if (queue->IsFilterDevice)
+ return TCReadDevice (queue->LowerDeviceObject, buffer, offset, length);
+
+ return ZwReadFile (queue->HostFileHandle, NULL, NULL, NULL, ioStatus, buffer, length, &offset, NULL);
+}
+
+
+static VOID IoThreadProc (PVOID threadArg)
+{
+ EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg;
+ PLIST_ENTRY listEntry;
+ EncryptedIoRequest *request;
+
+ KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY);
+
+ if (!queue->IsFilterDevice && queue->SecurityClientContext)
+ {
+#ifdef DEBUG
+ NTSTATUS status =
+#endif
+ SeImpersonateClientEx (queue->SecurityClientContext, NULL);
+ ASSERT (NT_SUCCESS (status));
+ }
+
+ while (!queue->ThreadExitRequested)
+ {
+ if (!NT_SUCCESS (KeWaitForSingleObject (&queue->IoThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL)))
+ continue;
+
+ if (queue->ThreadExitRequested)
+ break;
+
+ while ((listEntry = ExInterlockedRemoveHeadList (&queue->IoThreadQueue, &queue->IoThreadQueueLock)))
+ {
+ InterlockedDecrement (&queue->IoThreadPendingRequestCount);
+ request = CONTAINING_RECORD (listEntry, EncryptedIoRequest, ListEntry);
+
+#ifdef TC_TRACE_IO_QUEUE
+ Dump ("%c %I64d [%I64d] roff=%I64d rlen=%d\n", request->Item->Write ? 'W' : 'R', request->Item->OriginalIrpOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), request->Offset.QuadPart, request->Length);
+#endif
+
+ // Perform IO request if no preceding request of the item failed
+ if (NT_SUCCESS (request->Item->Status))
+ {
+ if (queue->IsFilterDevice)
+ {
+ if (queue->RemapEncryptedArea && request->EncryptedLength > 0)
+ {
+ if (request->EncryptedLength != request->Length)
+ {
+ // Up to three subfragments may be required to handle a partially remapped fragment
+ int subFragment;
+ byte *subFragmentData = request->Data;
+
+ for (subFragment = 0 ; subFragment < 3; ++subFragment)
+ {
+ LARGE_INTEGER subFragmentOffset;
+ ULONG subFragmentLength;
+ subFragmentOffset.QuadPart = request->Offset.QuadPart;
+
+ switch (subFragment)
+ {
+ case 0:
+ subFragmentLength = (ULONG) request->EncryptedOffset;
+ break;
+
+ case 1:
+ subFragmentOffset.QuadPart += request->EncryptedOffset + queue->RemappedAreaOffset;
+ subFragmentLength = request->EncryptedLength;
+ break;
+
+ case 2:
+ subFragmentOffset.QuadPart += request->EncryptedOffset + request->EncryptedLength;
+ subFragmentLength = (ULONG) (request->Length - (request->EncryptedOffset + request->EncryptedLength));
+ break;
+ }
+
+ if (subFragmentLength > 0)
+ {
+ if (request->Item->Write)
+ request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, subFragmentData, subFragmentOffset, subFragmentLength);
+ else
+ request->Item->Status = TCCachedRead (queue, NULL, subFragmentData, subFragmentOffset, subFragmentLength);
+
+ subFragmentData += subFragmentLength;
+ }
+ }
+ }
+ else
+ {
+ // Remap the fragment
+ LARGE_INTEGER remappedOffset;
+ remappedOffset.QuadPart = request->Offset.QuadPart + queue->RemappedAreaOffset;
+
+ if (request->Item->Write)
+ request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, request->Data, remappedOffset, request->Length);
+ else
+ request->Item->Status = TCCachedRead (queue, NULL, request->Data, remappedOffset, request->Length);
+ }
+ }
+ else
+ {
+ if (request->Item->Write)
+ request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, request->Data, request->Offset, request->Length);
+ else
+ request->Item->Status = TCCachedRead (queue, NULL, request->Data, request->Offset, request->Length);
+ }
+ }
+ else
+ {
+ IO_STATUS_BLOCK ioStatus;
+
+ if (request->Item->Write)
+ request->Item->Status = ZwWriteFile (queue->HostFileHandle, NULL, NULL, NULL, &ioStatus, request->Data, request->Length, &request->Offset, NULL);
+ else
+ request->Item->Status = TCCachedRead (queue, &ioStatus, request->Data, request->Offset, request->Length);
+
+ if (NT_SUCCESS (request->Item->Status) && ioStatus.Information != request->Length)
+ request->Item->Status = STATUS_END_OF_FILE;
+ }
+ }
+
+ if (request->Item->Write)
+ {
+ queue->ReadAheadBufferValid = FALSE;
+
+ ReleaseFragmentBuffer (queue, request->Data);
+
+ if (request->CompleteOriginalIrp)
+ {
+ CompleteOriginalIrp (request->Item, request->Item->Status,
+ NT_SUCCESS (request->Item->Status) ? request->Item->OriginalLength : 0);
+ }
+
+ ReleasePoolBuffer (queue, request);
+ }
+ else
+ {
+ BOOL readAhead = FALSE;
+
+ if (NT_SUCCESS (request->Item->Status))
+ memcpy (request->OrigDataBufferFragment, request->Data, request->Length);
+
+ ReleaseFragmentBuffer (queue, request->Data);
+ request->Data = request->OrigDataBufferFragment;
+
+ if (request->CompleteOriginalIrp
+ && queue->LastReadLength > 0
+ && NT_SUCCESS (request->Item->Status)
+ && InterlockedExchangeAdd (&queue->IoThreadPendingRequestCount, 0) == 0)
+ {
+ readAhead = TRUE;
+ InterlockedIncrement (&queue->OutstandingIoCount);
+ }
+
+ ExInterlockedInsertTailList (&queue->CompletionThreadQueue, &request->CompletionListEntry, &queue->CompletionThreadQueueLock);
+ KeSetEvent (&queue->CompletionThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE);
+
+ if (readAhead)
+ {
+ queue->ReadAheadBufferValid = FALSE;
+ queue->ReadAheadOffset.QuadPart = queue->LastReadOffset.QuadPart + queue->LastReadLength;
+ queue->ReadAheadLength = queue->LastReadLength;
+
+ if (queue->ReadAheadOffset.QuadPart + queue->ReadAheadLength <= queue->MaxReadAheadOffset.QuadPart)
+ {
+#ifdef TC_TRACE_IO_QUEUE
+ Dump ("A %I64d [%I64d] roff=%I64d rlen=%d\n", request->Item->OriginalIrpOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), queue->ReadAheadOffset, queue->ReadAheadLength);
+#endif
+ if (queue->IsFilterDevice)
+ {
+ queue->ReadAheadBufferValid = NT_SUCCESS (TCReadDevice (queue->LowerDeviceObject, queue->ReadAheadBuffer, queue->ReadAheadOffset, queue->ReadAheadLength));
+ }
+ else
+ {
+ IO_STATUS_BLOCK ioStatus;
+ queue->ReadAheadBufferValid = NT_SUCCESS (ZwReadFile (queue->HostFileHandle, NULL, NULL, NULL, &ioStatus, queue->ReadAheadBuffer, queue->ReadAheadLength, &queue->ReadAheadOffset, NULL));
+ queue->ReadAheadLength = (ULONG) ioStatus.Information;
+ }
+ }
+
+ DecrementOutstandingIoCount (queue);
+ }
+ }
+ }
+ }
+
+ PsTerminateSystemThread (STATUS_SUCCESS);
+}
+
+
+static VOID MainThreadProc (PVOID threadArg)
+{
+ EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg;
+ PLIST_ENTRY listEntry;
+ EncryptedIoQueueItem *item;
+
+ LARGE_INTEGER fragmentOffset;
+ ULONG dataRemaining;
+ PUCHAR activeFragmentBuffer = queue->FragmentBufferA;
+ PUCHAR dataBuffer;
+ EncryptedIoRequest *request;
+ uint64 intersectStart;
+ uint32 intersectLength;
+
+ if (IsEncryptionThreadPoolRunning())
+ KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY);
+
+ while (!queue->ThreadExitRequested)
+ {
+ if (!NT_SUCCESS (KeWaitForSingleObject (&queue->MainThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL)))
+ continue;
+
+ while ((listEntry = ExInterlockedRemoveHeadList (&queue->MainThreadQueue, &queue->MainThreadQueueLock)))
+ {
+ PIRP irp = CONTAINING_RECORD (listEntry, IRP, Tail.Overlay.ListEntry);
+ PIO_STACK_LOCATION irpSp = IoGetCurrentIrpStackLocation (irp);
+
+ if (queue->Suspended)
+ KeWaitForSingleObject (&queue->QueueResumedEvent, Executive, KernelMode, FALSE, NULL);
+
+ item = GetPoolBuffer (queue, sizeof (EncryptedIoQueueItem));
+ item->Queue = queue;
+ item->OriginalIrp = irp;
+ item->Status = STATUS_SUCCESS;
+
+ IoSetCancelRoutine (irp, NULL);
+ if (irp->Cancel)
+ {
+ CompleteOriginalIrp (item, STATUS_CANCELLED, 0);
+ continue;
+ }
+
+ switch (irpSp->MajorFunction)
+ {
+ case IRP_MJ_READ:
+ item->Write = FALSE;
+ item->OriginalOffset = irpSp->Parameters.Read.ByteOffset;
+ item->OriginalLength = irpSp->Parameters.Read.Length;
+ break;
+
+ case IRP_MJ_WRITE:
+ item->Write = TRUE;
+ item->OriginalOffset = irpSp->Parameters.Write.ByteOffset;
+ item->OriginalLength = irpSp->Parameters.Write.Length;
+ break;
+
+ default:
+ CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0);
+ continue;
+ }
+
+#ifdef TC_TRACE_IO_QUEUE
+ item->OriginalIrpOffset = item->OriginalOffset;
+#endif
+
+ // Handle misaligned read operations to work around a bug in Windows System Assessment Tool which does not follow FILE_FLAG_NO_BUFFERING requirements when benchmarking disk devices
+ if (queue->IsFilterDevice
+ && !item->Write
+ && item->OriginalLength > 0
+ && (item->OriginalLength & (ENCRYPTION_DATA_UNIT_SIZE - 1)) == 0
+ && (item->OriginalOffset.QuadPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0)
+ {
+ byte *buffer;
+ ULONG alignedLength = item->OriginalLength + ENCRYPTION_DATA_UNIT_SIZE;
+ LARGE_INTEGER alignedOffset;
+ alignedOffset.QuadPart = item->OriginalOffset.QuadPart & ~((LONGLONG) ENCRYPTION_DATA_UNIT_SIZE - 1);
+
+ buffer = TCalloc (alignedLength);
+ if (!buffer)
+ {
+ CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0);
+ continue;
+ }
+
+ item->Status = TCReadDevice (queue->LowerDeviceObject, buffer, alignedOffset, alignedLength);
+
+ if (NT_SUCCESS (item->Status))
+ {
+ UINT64_STRUCT dataUnit;
+
+ dataBuffer = (PUCHAR) MmGetSystemAddressForMdlSafe (irp->MdlAddress, HighPagePriority);
+ if (!dataBuffer)
+ {
+ TCfree (buffer);
+ CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0);
+ continue;
+ }
+
+ if (queue->EncryptedAreaStart != -1 && queue->EncryptedAreaEnd != -1)
+ {
+ GetIntersection (alignedOffset.QuadPart, alignedLength, queue->EncryptedAreaStart, queue->EncryptedAreaEnd, &intersectStart, &intersectLength);
+ if (intersectLength > 0)
+ {
+ dataUnit.Value = intersectStart / ENCRYPTION_DATA_UNIT_SIZE;
+ DecryptDataUnits (buffer + (intersectStart - alignedOffset.QuadPart), &dataUnit, intersectLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo);
+ }
+ }
+
+ memcpy (dataBuffer, buffer + (item->OriginalOffset.LowPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)), item->OriginalLength);
+ }
+
+ TCfree (buffer);
+ CompleteOriginalIrp (item, item->Status, NT_SUCCESS (item->Status) ? item->OriginalLength : 0);
+ continue;
+ }
+
+ // Validate offset and length
+ if (item->OriginalLength == 0
+ || (item->OriginalLength & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0
+ || (item->OriginalOffset.QuadPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0
+ || (!queue->IsFilterDevice && item->OriginalOffset.QuadPart + item->OriginalLength > queue->VirtualDeviceLength))
+ {
+ CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0);
+ continue;
+ }
+
+#ifdef TC_TRACE_IO_QUEUE
+ Dump ("Q %I64d [%I64d] %c len=%d\n", item->OriginalOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), item->Write ? 'W' : 'R', item->OriginalLength);
+#endif
+
+ if (!queue->IsFilterDevice)
+ {
+ // Adjust the offset for host file or device
+ if (queue->CryptoInfo->hiddenVolume)
+ item->OriginalOffset.QuadPart += queue->CryptoInfo->hiddenVolumeOffset;
+ else
+ item->OriginalOffset.QuadPart += queue->CryptoInfo->volDataAreaOffset;
+
+ // Hidden volume protection
+ if (item->Write && queue->CryptoInfo->bProtectHiddenVolume)
+ {
+ // If there has already been a write operation denied in order to protect the
+ // hidden volume (since the volume mount time)
+ if (queue->CryptoInfo->bHiddenVolProtectionAction)
+ {
+ // Do not allow writing to this volume anymore. This is to fake a complete volume
+ // or system failure (otherwise certain kinds of inconsistency within the file
+ // system could indicate that this volume has used hidden volume protection).
+ CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0);
+ continue;
+ }
+
+ // Verify that no byte is going to be written to the hidden volume area
+ if (RegionsOverlap ((unsigned __int64) item->OriginalOffset.QuadPart,
+ (unsigned __int64) item->OriginalOffset.QuadPart + item->OriginalLength - 1,
+ queue->CryptoInfo->hiddenVolumeOffset,
+ (unsigned __int64) queue->CryptoInfo->hiddenVolumeOffset + queue->CryptoInfo->hiddenVolumeProtectedSize - 1))
+ {
+ Dump ("Hidden volume protection triggered: write %I64d-%I64d (protected %I64d-%I64d)\n", item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, queue->CryptoInfo->hiddenVolumeOffset, queue->CryptoInfo->hiddenVolumeOffset + queue->CryptoInfo->hiddenVolumeProtectedSize - 1);
+ queue->CryptoInfo->bHiddenVolProtectionAction = TRUE;
+
+ // Deny this write operation to prevent the hidden volume from being overwritten
+ CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0);
+ continue;
+ }
+ }
+ }
+ else if (item->Write
+ && RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, TC_BOOT_VOLUME_HEADER_SECTOR_OFFSET, TC_BOOT_VOLUME_HEADER_SECTOR_OFFSET + TC_BOOT_ENCRYPTION_VOLUME_HEADER_SIZE - 1))
+ {
+ // Prevent inappropriately designed software from damaging important data that may be out of sync with the backup on the Rescue Disk (such as the end of the encrypted area).
+ Dump ("Preventing write to the system encryption key data area\n");
+ CompleteOriginalIrp (item, STATUS_MEDIA_WRITE_PROTECTED, 0);
+ continue;
+ }
+ else if (item->Write && IsHiddenSystemRunning()
+ && (RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, TC_SECTOR_SIZE_BIOS, TC_BOOT_LOADER_AREA_SECTOR_COUNT * TC_SECTOR_SIZE_BIOS - 1)
+ || RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, GetBootDriveLength(), _I64_MAX)))
+ {
+ Dump ("Preventing write to boot loader or host protected area\n");
+ CompleteOriginalIrp (item, STATUS_MEDIA_WRITE_PROTECTED, 0);
+ continue;
+ }
+
+ dataBuffer = (PUCHAR) MmGetSystemAddressForMdlSafe (irp->MdlAddress, HighPagePriority);
+
+ if (dataBuffer == NULL)
+ {
+ CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0);
+ continue;
+ }
+
+ // Divide data block to fragments to enable efficient overlapping of encryption and IO operations
+
+ dataRemaining = item->OriginalLength;
+ fragmentOffset = item->OriginalOffset;
+
+ while (dataRemaining > 0)
+ {
+ BOOL isLastFragment = dataRemaining <= TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE;
+
+ ULONG dataFragmentLength = isLastFragment ? dataRemaining : TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE;
+ activeFragmentBuffer = (activeFragmentBuffer == queue->FragmentBufferA ? queue->FragmentBufferB : queue->FragmentBufferA);
+
+ InterlockedIncrement (&queue->IoThreadPendingRequestCount);
+
+ // Create IO request
+ request = GetPoolBuffer (queue, sizeof (EncryptedIoRequest));
+ request->Item = item;
+ request->CompleteOriginalIrp = isLastFragment;
+ request->Offset = fragmentOffset;
+ request->Data = activeFragmentBuffer;
+ request->OrigDataBufferFragment = dataBuffer;
+ request->Length = dataFragmentLength;
+
+ if (queue->IsFilterDevice)
+ {
+ if (queue->EncryptedAreaStart == -1 || queue->EncryptedAreaEnd == -1)
+ {
+ request->EncryptedLength = 0;
+ }
+ else
+ {
+ // Get intersection of data fragment with encrypted area
+ GetIntersection (fragmentOffset.QuadPart, dataFragmentLength, queue->EncryptedAreaStart, queue->EncryptedAreaEnd, &intersectStart, &intersectLength);
+
+ request->EncryptedOffset = intersectStart - fragmentOffset.QuadPart;
+ request->EncryptedLength = intersectLength;
+ }
+ }
+ else
+ {
+ request->EncryptedOffset = 0;
+ request->EncryptedLength = dataFragmentLength;
+ }
+
+ AcquireFragmentBuffer (queue, activeFragmentBuffer);
+
+ if (item->Write)
+ {
+ // Encrypt data
+ memcpy (activeFragmentBuffer, dataBuffer, dataFragmentLength);
+
+ if (request->EncryptedLength > 0)
+ {
+ UINT64_STRUCT dataUnit;
+ ASSERT (request->EncryptedOffset + request->EncryptedLength <= request->Offset.QuadPart + request->Length);
+
+ dataUnit.Value = (request->Offset.QuadPart + request->EncryptedOffset) / ENCRYPTION_DATA_UNIT_SIZE;
+
+ if (queue->CryptoInfo->bPartitionInInactiveSysEncScope)
+ dataUnit.Value += queue->CryptoInfo->FirstDataUnitNo.Value;
+ else if (queue->RemapEncryptedArea)
+ dataUnit.Value += queue->RemappedAreaDataUnitOffset;
+
+ EncryptDataUnits (activeFragmentBuffer + request->EncryptedOffset, &dataUnit, request->EncryptedLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo);
+ }
+ }
+
+ // Queue IO request
+ ExInterlockedInsertTailList (&queue->IoThreadQueue, &request->ListEntry, &queue->IoThreadQueueLock);
+ KeSetEvent (&queue->IoThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE);
+
+ if (isLastFragment)
+ break;
+
+ dataRemaining -= TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE;
+ dataBuffer += TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE;
+ fragmentOffset.QuadPart += TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE;
+ }
+ }
+ }
+
+ PsTerminateSystemThread (STATUS_SUCCESS);
+}
+
+
+NTSTATUS EncryptedIoQueueAddIrp (EncryptedIoQueue *queue, PIRP irp)
+{
+ NTSTATUS status;
+
+ InterlockedIncrement (&queue->OutstandingIoCount);
+ if (queue->StopPending)
+ {
+ Dump ("STATUS_DEVICE_NOT_READY out=%d\n", queue->OutstandingIoCount);
+ status = STATUS_DEVICE_NOT_READY;
+ goto err;
+ }
+
+ status = IoAcquireRemoveLock (&queue->RemoveLock, irp);
+ if (!NT_SUCCESS (status))
+ goto err;
+
+#ifdef TC_TRACE_IO_QUEUE
+ {
+ PIO_STACK_LOCATION irpSp = IoGetCurrentIrpStackLocation (irp);
+ Dump ("* %I64d [%I64d] %c len=%d out=%d\n", irpSp->MajorFunction == IRP_MJ_WRITE ? irpSp->Parameters.Write.ByteOffset : irpSp->Parameters.Read.ByteOffset, GetElapsedTime (&queue->LastPerformanceCounter), irpSp->MajorFunction == IRP_MJ_WRITE ? 'W' : 'R', irpSp->MajorFunction == IRP_MJ_WRITE ? irpSp->Parameters.Write.Length : irpSp->Parameters.Read.Length, queue->OutstandingIoCount);
+ }
+#endif
+
+ IoMarkIrpPending (irp);
+
+ ExInterlockedInsertTailList (&queue->MainThreadQueue, &irp->Tail.Overlay.ListEntry, &queue->MainThreadQueueLock);
+ KeSetEvent (&queue->MainThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE);
+
+ return STATUS_PENDING;
+
+err:
+ DecrementOutstandingIoCount (queue);
+ return status;
+}
+
+
+NTSTATUS EncryptedIoQueueHoldWhenIdle (EncryptedIoQueue *queue, int64 timeout)
+{
+ NTSTATUS status;
+ ASSERT (!queue->Suspended);
+
+ queue->SuspendPending = TRUE;
+
+ while (TRUE)
+ {
+ while (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0)
+ {
+ LARGE_INTEGER waitTimeout;
+
+ waitTimeout.QuadPart = timeout * -10000;
+ status = KeWaitForSingleObject (&queue->NoOutstandingIoEvent, Executive, KernelMode, FALSE, timeout != 0 ? &waitTimeout : NULL);
+
+ if (status == STATUS_TIMEOUT)
+ status = STATUS_UNSUCCESSFUL;
+
+ if (!NT_SUCCESS (status))
+ {
+ queue->SuspendPending = FALSE;
+ return status;
+ }
+
+ TCSleep (1);
+ if (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0)
+ {
+ queue->SuspendPending = FALSE;
+ return STATUS_UNSUCCESSFUL;
+ }
+ }
+
+ KeClearEvent (&queue->QueueResumedEvent);
+ queue->Suspended = TRUE;
+
+ if (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) == 0)
+ break;
+
+ queue->Suspended = FALSE;
+ KeSetEvent (&queue->QueueResumedEvent, IO_DISK_INCREMENT, FALSE);
+ }
+
+ queue->ReadAheadBufferValid = FALSE;
+
+ queue->SuspendPending = FALSE;
+ return STATUS_SUCCESS;
+}
+
+
+BOOL EncryptedIoQueueIsSuspended (EncryptedIoQueue *queue)
+{
+ return queue->Suspended;
+}
+
+
+BOOL EncryptedIoQueueIsRunning (EncryptedIoQueue *queue)
+{
+ return !queue->StopPending;
+}
+
+
+NTSTATUS EncryptedIoQueueResumeFromHold (EncryptedIoQueue *queue)
+{
+ ASSERT (queue->Suspended);
+
+ queue->Suspended = FALSE;
+ KeSetEvent (&queue->QueueResumedEvent, IO_DISK_INCREMENT, FALSE);
+
+ return STATUS_SUCCESS;
+}
+
+
+NTSTATUS EncryptedIoQueueStart (EncryptedIoQueue *queue)
+{
+ NTSTATUS status;
+ EncryptedIoQueueBuffer *buffer;
+ int i;
+
+ queue->StartPending = TRUE;
+ queue->ThreadExitRequested = FALSE;
+
+ queue->OutstandingIoCount = 0;
+ queue->IoThreadPendingRequestCount = 0;
+
+ queue->FirstPoolBuffer = NULL;
+ KeInitializeMutex (&queue->BufferPoolMutex, 0);
+
+ KeInitializeEvent (&queue->NoOutstandingIoEvent, SynchronizationEvent, FALSE);
+ KeInitializeEvent (&queue->PoolBufferFreeEvent, SynchronizationEvent, FALSE);
+ KeInitializeEvent (&queue->QueueResumedEvent, SynchronizationEvent, FALSE);
+
+ queue->FragmentBufferA = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE);
+ if (!queue->FragmentBufferA)
+ goto noMemory;
+
+ queue->FragmentBufferB = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE);
+ if (!queue->FragmentBufferB)
+ goto noMemory;
+
+ KeInitializeEvent (&queue->FragmentBufferAFreeEvent, SynchronizationEvent, TRUE);
+ KeInitializeEvent (&queue->FragmentBufferBFreeEvent, SynchronizationEvent, TRUE);
+
+ queue->ReadAheadBufferValid = FALSE;
+ queue->ReadAheadBuffer = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE);
+ if (!queue->ReadAheadBuffer)
+ goto noMemory;
+
+ // Preallocate buffers
+ for (i = 0; i < TC_ENC_IO_QUEUE_PREALLOCATED_IO_REQUEST_COUNT; ++i)
+ {
+ if (i < TC_ENC_IO_QUEUE_PREALLOCATED_ITEM_COUNT && !GetPoolBuffer (queue, sizeof (EncryptedIoQueueItem)))
+ goto noMemory;
+
+ if (!GetPoolBuffer (queue, sizeof (EncryptedIoRequest)))
+ goto noMemory;
+ }
+
+ for (buffer = queue->FirstPoolBuffer; buffer != NULL; buffer = buffer->NextBuffer)
+ {
+ buffer->InUse = FALSE;
+ }
+
+ // Main thread
+ InitializeListHead (&queue->MainThreadQueue);
+ KeInitializeSpinLock (&queue->MainThreadQueueLock);
+ KeInitializeEvent (&queue->MainThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE);
+
+ status = TCStartThread (MainThreadProc, queue, &queue->MainThread);
+ if (!NT_SUCCESS (status))
+ goto err;
+
+ // IO thread
+ InitializeListHead (&queue->IoThreadQueue);
+ KeInitializeSpinLock (&queue->IoThreadQueueLock);
+ KeInitializeEvent (&queue->IoThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE);
+
+ status = TCStartThread (IoThreadProc, queue, &queue->IoThread);
+ if (!NT_SUCCESS (status))
+ {
+ queue->ThreadExitRequested = TRUE;
+ TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent);
+ goto err;
+ }
+
+ // Completion thread
+ InitializeListHead (&queue->CompletionThreadQueue);
+ KeInitializeSpinLock (&queue->CompletionThreadQueueLock);
+ KeInitializeEvent (&queue->CompletionThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE);
+
+ status = TCStartThread (CompletionThreadProc, queue, &queue->CompletionThread);
+ if (!NT_SUCCESS (status))
+ {
+ queue->ThreadExitRequested = TRUE;
+ TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent);
+ TCStopThread (queue->IoThread, &queue->IoThreadQueueNotEmptyEvent);
+ goto err;
+ }
+
+#ifdef TC_TRACE_IO_QUEUE
+ GetElapsedTimeInit (&queue->LastPerformanceCounter);
+#endif
+
+ queue->StopPending = FALSE;
+ queue->StartPending = FALSE;
+
+ Dump ("Queue started\n");
+ return STATUS_SUCCESS;
+
+noMemory:
+ status = STATUS_INSUFFICIENT_RESOURCES;
+
+err:
+ if (queue->FragmentBufferA)
+ TCfree (queue->FragmentBufferA);
+ if (queue->FragmentBufferB)
+ TCfree (queue->FragmentBufferB);
+ if (queue->ReadAheadBuffer)
+ TCfree (queue->ReadAheadBuffer);
+
+ FreePoolBuffers (queue);
+
+ queue->StartPending = FALSE;
+ return status;
+}
+
+
+NTSTATUS EncryptedIoQueueStop (EncryptedIoQueue *queue)
+{
+ ASSERT (!queue->StopPending);
+ queue->StopPending = TRUE;
+
+ while (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0)
+ {
+ KeWaitForSingleObject (&queue->NoOutstandingIoEvent, Executive, KernelMode, FALSE, NULL);
+ }
+
+ Dump ("Queue stopping out=%d\n", queue->OutstandingIoCount);
+
+ queue->ThreadExitRequested = TRUE;
+
+ TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent);
+ TCStopThread (queue->IoThread, &queue->IoThreadQueueNotEmptyEvent);
+ TCStopThread (queue->CompletionThread, &queue->CompletionThreadQueueNotEmptyEvent);
+
+ TCfree (queue->FragmentBufferA);
+ TCfree (queue->FragmentBufferB);
+ TCfree (queue->ReadAheadBuffer);
+
+ FreePoolBuffers (queue);
+
+ Dump ("Queue stopped out=%d\n", queue->OutstandingIoCount);
+ return STATUS_SUCCESS;
+}
|