Gradienthealth/file streaming of tar files

packages/core/src/enums/RequestType.ts

@@ -10,6 +10,8 @@ enum RequestType {
   Prefetch = 'prefetch',
   /** Lower priority, often used for background computations in the worker */
   Compute = 'compute',
+  /** Lowest Priority, used to cache adjascent studies using GoogleSheetService */
+  PreCache = 'precache',
 }
 
 export default RequestType;

packages/core/src/requestPool/imageLoadPoolManager.ts

@@ -39,5 +39,6 @@ imageLoadPoolManager.grabDelay = 0;
 imageLoadPoolManager.setMaxSimultaneousRequests(RequestType.Interaction, 1000);
 imageLoadPoolManager.setMaxSimultaneousRequests(RequestType.Thumbnail, 1000);
 imageLoadPoolManager.setMaxSimultaneousRequests(RequestType.Prefetch, 1000);
+imageLoadPoolManager.setMaxSimultaneousRequests(RequestType.PreCache, 1000);
 
 export default imageLoadPoolManager;

packages/core/src/requestPool/imageRetrievalPoolManager.ts

@@ -20,6 +20,7 @@ imageRetrievalPoolManager.setMaxSimultaneousRequests(
   200
 );
 imageRetrievalPoolManager.setMaxSimultaneousRequests(RequestType.Prefetch, 200);
+imageRetrievalPoolManager.setMaxSimultaneousRequests(RequestType.PreCache, 200);
 imageRetrievalPoolManager.grabDelay = 0;
 
 export default imageRetrievalPoolManager;

packages/core/src/requestPool/requestPoolManager.ts

@@ -79,13 +79,15 @@ class RequestPoolManager {
     thumbnail: 0,
     prefetch: 0,
     compute: 0,
+    precache: 0,
   };
   /* maximum number of requests of each type. */
   public maxNumRequests: {
     interaction: number;
     thumbnail: number;
     prefetch: number;
     compute: number;
+    precache: number;
   };
   /* A public property that is used to set the delay between requests. */
   public grabDelay: number;
@@ -104,6 +106,7 @@ class RequestPoolManager {
       thumbnail: { 0: [] },
       prefetch: { 0: [] },
       compute: { 0: [] },
+      precache: { 0: [] },
     };
 
     this.grabDelay = 5;
@@ -114,13 +117,15 @@ class RequestPoolManager {
       thumbnail: 0,
       prefetch: 0,
       compute: 0,
+      precache: 0,
     };
 
     this.maxNumRequests = {
       interaction: 6,
       thumbnail: 6,
       prefetch: 5,
       compute: 15,
+      precache: 5,
     };
   }
 
@@ -271,12 +276,16 @@ class RequestPoolManager {
       RequestType.Prefetch
     );
     const hasRemainingComputeRequests = this.sendRequests(RequestType.Compute);
+    const hasRemainingPreCacheRequests = this.sendRequests(
+      RequestType.PreCache
+    );
 
     if (
       !hasRemainingInteractionRequests &&
       !hasRemainingThumbnailRequests &&
       !hasRemainingPrefetchRequests &&
-      !hasRemainingComputeRequests
+      !hasRemainingComputeRequests &&
+      !hasRemainingPreCacheRequests
     ) {
       this.awake = false;
     }

packages/core/src/webWorkerManager/webWorkerManager.js

@@ -213,6 +213,42 @@ class CentralizedWorkerManager {
       worker.terminate();
     });
   }
+
+  postMessage(workerName, message) {
+    const workerProperties = this.workerRegistry[workerName];
+    if (!workerProperties) {
+      console.error(`Worker type '${workerName}' is not registered.`);
+      return;
+    }
+
+    workerProperties.nativeWorkers.forEach((worker) =>
+      worker.postMessage(message)
+    );
+  }
+
+  addEventListener(workerName, eventType = 'message', listener) {
+    const workerProperties = this.workerRegistry[workerName];
+    if (!workerProperties) {
+      console.error(`Worker type '${workerName}' is not registered.`);
+      return;
+    }
+
+    workerProperties.nativeWorkers.forEach((worker) =>
+      worker.addEventListener(eventType, listener)
+    );
+  }
+
+  removeEventListener(workerName, eventType = 'message', listener) {
+    const workerProperties = this.workerRegistry[workerName];
+    if (!workerProperties) {
+      console.error(`Worker type '${workerName}' is not registered.`);
+      return;
+    }
+
+    workerProperties.nativeWorkers.forEach((worker) =>
+      worker.removeEventListener(eventType, listener)
+    );
+  }
 }
 
 export default CentralizedWorkerManager;

packages/dicomImageLoader/package.json

@@ -67,6 +67,7 @@
     "@cornerstonejs/codec-openjpeg": "^1.2.2",
     "@cornerstonejs/codec-openjph": "^2.4.5",
     "@cornerstonejs/core": "^1.41.0",
+    "cod-dicomweb-server": "^1.3.4",
     "dicom-parser": "^1.8.9",
     "jszip": "^3.10.1",
     "pako": "^2.0.4",

packages/dicomImageLoader/src/imageLoader/internal/codWebServer.ts

@@ -0,0 +1,11 @@
+import type { CodDicomWebServer } from 'cod-dicomweb-server';
+
+let codWebServer: CodDicomWebServer;
+
+export function setWadoRsWebServer(webServer: CodDicomWebServer) {
+  codWebServer = webServer;
+}
+
+export function getWadoRsWebServer(): CodDicomWebServer {
+  return codWebServer;
+}

packages/dicomImageLoader/src/imageLoader/wadors/codRequest.ts

@@ -0,0 +1,54 @@
+import { Enums, metaData } from '@cornerstonejs/core';
+import { FetchType } from 'cod-dicomweb-server';
+
+import { getWadoRsWebServer } from '../internal/codWebServer';
+import { getOptions } from '../internal/options';
+import type { CornerstoneWadoRsLoaderOptions } from '../wadors/loadImage';
+
+export default function codRequest(
+  url: string,
+  imageId: string,
+  defaultHeaders: Record<string, string> = {},
+  options: CornerstoneWadoRsLoaderOptions = {}
+): Promise<{
+  contentType: string;
+  pixelData: Uint8Array;
+  imageQualityStatus: Enums.ImageQualityStatus;
+  percentComplete: number;
+}> {
+  const instance = metaData.get('instance', imageId) || {};
+  const {
+    DeidStudyInstanceUID,
+    DeidSeriesInstanceUID,
+    DeidSopInstanceUID,
+    TransferSyntaxUID,
+  } = instance;
+
+  const deidReplacedUrl = url.replace(
+    /\/studies\/[^/]+\/series\/[^/]+\/instances\/[^/]+\/frames\//,
+    `/studies/${DeidStudyInstanceUID}/series/${DeidSeriesInstanceUID}/instances/${DeidSopInstanceUID}/frames/`
+  );
+
+  // @ts-ignore
+  const headers = getOptions()?.beforeSend() || {};
+  const webServer = getWadoRsWebServer();
+
+  return webServer
+    .fetchCod(
+      deidReplacedUrl,
+      { ...defaultHeaders, ...headers },
+      {
+        useSharedArrayBuffer: false,
+        fetchType: FetchType.API_OPTIMIZED,
+      }
+    )
+    .then((result) => ({
+      contentType: `transfer-syntax=${TransferSyntaxUID}`,
+      imageQualityStatus: Enums.ImageQualityStatus.FULL_RESOLUTION,
+      pixelData: new Uint8Array(result as ArrayBufferLike),
+      percentComplete: 100,
+    }))
+    .catch((error) => {
+      throw error;
+    });
+}

packages/dicomImageLoader/src/imageLoader/wadors/getPixelData.ts

@@ -6,6 +6,7 @@ import extractMultipart from './extractMultipart';
 import { getImageQualityStatus } from './getImageQualityStatus';
 import { CornerstoneWadoRsLoaderOptions } from './loadImage';
 import { RangeRetrieveOptions } from 'core/dist/types/types';
+import codRequest from './codRequest';
 
 function getPixelData(
   uri: string,
@@ -37,6 +38,11 @@ function getPixelData(
     options.streamingData = { url };
   }
 
+  // Use the cod loader only when image scheme is cod:
+  if (imageId.startsWith('cod:')) {
+    return codRequest(url, imageId, headers);
+  }
+
   if ((retrieveOptions as RangeRetrieveOptions).rangeIndex !== undefined) {
     return rangeRequest(url, imageId, headers, options);
   }

packages/dicomImageLoader/src/imageLoader/wadors/loadImage.ts

@@ -198,7 +198,7 @@ function loadImage(
   const additionalDetails = options.additionalDetails || { imageId };
   const priority = options.priority === undefined ? 5 : options.priority;
   const addToBeginning = options.addToBeginning || false;
-  const uri = imageId.substring(7);
+  const uri = imageId.substring(imageId.indexOf(':') + 1);
 
   imageRetrievalPool.addRequest(
     sendXHR.bind(this, uri, imageId, mediaType),

packages/dicomImageLoader/src/imageLoader/wadors/register.ts

@@ -1,8 +1,23 @@
+import { registerImageLoader } from '@cornerstonejs/core';
+import { CodDicomWebServer } from 'cod-dicomweb-server';
 import loadImage from './loadImage';
-import { metaDataProvider } from './metaData/index';
+import { metaDataProvider } from './metaData';
+import { registerFileStreamingWebWorker } from './registerFileStreaming';
+import { setWadoRsWebServer } from '../internal/codWebServer';
 
 export default function (cornerstone) {
   // register wadors scheme and metadata provider
   cornerstone.registerImageLoader('wadors', loadImage);
   cornerstone.metaData.addProvider(metaDataProvider);
+
+  // register file streaming web worker
+  registerFileStreamingWebWorker();
+
+  // initialize the CodDicomWebServer
+  registerImageLoader('cod', loadImage);
+  const MAXIMUM_WORKER_FETCH_SIZE = 2 * 1_073_741_824; // 2 x 1 GB
+  const codDicomWebServer = new CodDicomWebServer({
+    maxWorkerFetchSize: MAXIMUM_WORKER_FETCH_SIZE,
+  });
+  setWadoRsWebServer(codDicomWebServer);
 }

packages/dicomImageLoader/src/imageLoader/wadors/registerFileStreaming.ts

@@ -0,0 +1,27 @@
+import { getWebWorkerManager } from '@cornerstonejs/core';
+
+export const FILE_STREAMING_WORKER_NAME = 'file-streaming-test';
+export const MAXIMUM_WORKER_FETCH_SIZE = 2 * 1_073_741_824; // 2 x 1 GB
+
+export function registerFileStreamingWebWorker() {
+  const workerFn = () => {
+    return new Worker(
+      new URL('../../workers/fileStreaming.ts?v=12', import.meta.url),
+      { name: FILE_STREAMING_WORKER_NAME }
+    );
+  };
+
+  const workerManager = getWebWorkerManager();
+
+  const options = {
+    maxWorkerInstances: 1,
+  };
+
+  workerManager.registerWorker(FILE_STREAMING_WORKER_NAME, workerFn, options);
+
+  workerManager.executeTask(
+    FILE_STREAMING_WORKER_NAME,
+    'setMaxFetchSize',
+    MAXIMUM_WORKER_FETCH_SIZE
+  );
+}

packages/dicomImageLoader/src/imageLoader/wadouri/loadImage.ts

@@ -180,7 +180,7 @@ function loadImageWithRange(
 ): Types.IImageLoadObject {
   const start = new Date().getTime();
   const instance = metaData.get('instance', imageId);
-  const { ExtendedOffsetTable, ExtendedOffsetTableLengths } = instance;
+  const { CustomOffsetTable, CustomOffsetTableLengths } = instance;
 
   const headerPromise: Promise<{ dataSet; headerArrayBuffer }> = new Promise(
     (resolve) => {
@@ -189,11 +189,11 @@ function loadImageWithRange(
           dataSet: loadedDataSets[sharedCacheKey]?.dataSet,
           headerArrayBuffer: loadedDataSets[
             sharedCacheKey
-          ].dataSet.byteArray.slice(0, ExtendedOffsetTable[0] - 1),
+          ].dataSet.byteArray.slice(0, CustomOffsetTable[0] - 1),
         });
       } else {
         loader(sharedCacheKey, imageId, {
-          Range: `bytes=0-${ExtendedOffsetTable[0] - 1}`,
+          Range: `bytes=0-${CustomOffsetTable[0] - 1}`,
         }).then((arraybuffer) => {
           const dataSet = external.dicomParser.parseDicom(
             new Uint8Array(arraybuffer),
@@ -206,8 +206,8 @@ function loadImageWithRange(
     }
   );
 
-  const startByte = ExtendedOffsetTable[frameIndex];
-  const endByte = startByte + ExtendedOffsetTableLengths[frameIndex];
+  const startByte = CustomOffsetTable[frameIndex];
+  const endByte = startByte + CustomOffsetTableLengths[frameIndex];
   const pixelDataPromise = loader(sharedCacheKey, imageId, {
     Range: `bytes=${startByte}-${endByte}`,
   }).then((arraybuffer) => ({ pixelDataArrayBuffer: arraybuffer }));
@@ -223,7 +223,7 @@ function loadImageWithRange(
           values;
         const loadEnd = new Date().getTime();
         const pixelData = new Uint8Array(pixelDataArrayBuffer);
-        const transferSyntax = instance._meta.TransferSyntaxUID;
+        const transferSyntax = dataSet.string('x00020010');
 
         if (!dataSetCacheManager.isLoaded(sharedCacheKey)) {
           dataSet.elements.x7fe00010 = {};
@@ -244,8 +244,8 @@ function loadImageWithRange(
           const completeByteArray = new Uint8Array(
             dataSet.byteArray.byteLength + pixelDataArrayBuffer.byteLength
           );
-          completeByteArray.set(dataSet.byteArray.byteLength);
-          completeByteArray.set(pixelData, headerArrayBuffer.byteLength);
+          completeByteArray.set(dataSet.byteArray);
+          completeByteArray.set(pixelData, dataSet.byteArray.byteLength);
           dataSet.byteArray = completeByteArray;
 
           loadedDataSets[sharedCacheKey].cacheCount++;
@@ -360,7 +360,7 @@ function loadImage(
   }
 
   const instance = metaData.get('instance', imageId);
-  if (instance?.ExtendedOffsetTable && instance?.ExtendedOffsetTableLengths) {
+  if (instance?.CustomOffsetTable && instance?.CustomOffsetTableLengths) {
     // Fetch only a single frame pixeldata of a multiframe dicom file.
     return loadImageWithRange(
       imageId,
@@ -381,7 +381,7 @@ function loadImage(
   return loadImageFromPromise(
     dataSetPromise,
     imageId,
-    parsedImageId.frame,
+    parsedImageId.pixelDataFrame,
     parsedImageId.url,
     options
   );