From bc79a9613e572578e48ba8a1ad755f435427c117 Mon Sep 17 00:00:00 2001 From: Patrick Edson Date: Sat, 28 Oct 2023 08:25:02 -0400 Subject: [PATCH 1/4] Update shader for single channel. --- .../material/shader/VolumeMipFrag.glsl | 9 +++-- .../horta/volume/VolumeMipMaterial.java | 36 ++++++++++--------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/modules/GLViewerTools/src/main/resources/org/janelia/gltools/material/shader/VolumeMipFrag.glsl b/modules/GLViewerTools/src/main/resources/org/janelia/gltools/material/shader/VolumeMipFrag.glsl index 7226b0d77..1e5a2f20f 100644 --- a/modules/GLViewerTools/src/main/resources/org/janelia/gltools/material/shader/VolumeMipFrag.glsl +++ b/modules/GLViewerTools/src/main/resources/org/janelia/gltools/material/shader/VolumeMipFrag.glsl @@ -38,10 +38,9 @@ uniform int pickIndex = 3; // default value for pick buffer uniform COLOR_VEC opacityFunctionMin = COLOR_VEC(0); uniform COLOR_VEC opacityFunctionMax = COLOR_VEC(1); -uniform vec3 colorChannel1 = vec3(0, 1, 0); -uniform vec3 colorChannel2 = vec3(1, 0, 1); +uniform vec3 colorChannel = vec3(0, 1, 0); -uniform vec3 channelVisibilityMask = vec3(1); +uniform int channelVisibilityMask = 1; uniform vec3 camPosInTc; // camera position, in texture coordinate frame uniform sampler3D volumeTexture; // the confocal image stack @@ -678,9 +677,9 @@ void save_color(in IntegratedIntensity i, in ViewSlab slab) const float knot = 0.5; // where to kink linear ramp between three colors float intensity = rescaled.r; // First channel only at the moment if (intensity <= knot) - hotColor = mix(vec3(0), colorChannel1 * channelVisibilityMask.r, intensity / knot); + hotColor = mix(vec3(0), colorChannel * channelVisibilityMask, intensity / knot); else - hotColor = mix(colorChannel1 * channelVisibilityMask.r, vec3(0.95,1,0.9), (intensity - knot)/(1.0 - knot)); + hotColor = mix(colorChannel * channelVisibilityMask, vec3(0.95,1,0.9), (intensity - knot)/(1.0 - knot)); colorOut = vec4(hotColor, i.opacity); // colorOut = vec4(i.intensity.g, i.intensity.r, i.intensity.g, i.opacity); diff --git a/modules/HortaTracer/src/main/java/org/janelia/horta/volume/VolumeMipMaterial.java b/modules/HortaTracer/src/main/java/org/janelia/horta/volume/VolumeMipMaterial.java index 51c7429f0..8199183b3 100644 --- a/modules/HortaTracer/src/main/java/org/janelia/horta/volume/VolumeMipMaterial.java +++ b/modules/HortaTracer/src/main/java/org/janelia/horta/volume/VolumeMipMaterial.java @@ -41,17 +41,17 @@ public class VolumeMipMaterial extends BasicMaterial private int volumeMicrometersIndex = -1; private int tcToCameraIndex = -1; private int opaqueZNearFarIndex = -1; + + private int filteringOrderIndex = -1; private int colorChannelIndex1 = -1; private int channelVisibilityMaskIndex = -1; + + private int colorChannelId; private float[] opaqueZNearFar = {1e-2f, 1e4f}; // absolute clip in camera space private final ImageColorModel colorMap; - private int filteringOrderIndex = -1; - - // private int projectionModeIndex = -1; - protected final ShaderProgram mipShader = new VolumeMipShader(0); protected final ShaderProgram occShader = new VolumeMipShader(1); protected final ShaderProgram isoShader = new VolumeMipShader(2); @@ -67,8 +67,12 @@ public class VolumeMipMaterial extends BasicMaterial // Relative clip in camera space private float relativeZNear = 0.92f; private float relativeZFar = 1.08f; - - public VolumeMipMaterial(Texture3d volumeTexture, ImageColorModel colorMap) + + public VolumeMipMaterial(Texture3d volumeTexture, ImageColorModel colorMap) { + this(volumeTexture, colorMap, 0); + } + + public VolumeMipMaterial(Texture3d volumeTexture, ImageColorModel colorMap, int colorChannelId) { this.colorMap = colorMap; this.volumeTexture = volumeTexture; @@ -76,6 +80,7 @@ public VolumeMipMaterial(Texture3d volumeTexture, ImageColorModel colorMap) // this.volumeTexture.setMinFilter(GL3.GL_NEAREST_MIPMAP_NEAREST); this.volumeTexture.setMinFilter(GL3.GL_LINEAR_MIPMAP_NEAREST); this.volumeTexture.setMagFilter(GL3.GL_LINEAR); + this.colorChannelId = colorChannelId; shaderProgram = mipShader; @@ -196,19 +201,18 @@ protected void displayMesh(GL3 gl, MeshActor mesh, AbstractCamera camera, Matrix float [] opMin = new float[] {0, 0}; float [] opMax = new float[] {1, 1}; if (colorMap != null) { - ChannelColorModel m = colorMap.getChannel(0); + int channelIdx = Math.min(colorChannelId, colorMap.getChannelCount() - 1); + + ChannelColorModel m = colorMap.getChannel(channelIdx); + Vector3 color = new Vector3(m.getColor().getRed()/255.0f, m.getColor().getGreen()/255.0f, m.getColor().getBlue()/255.0f); - gl.glUniform3fv(colorChannelIndex1, 1, color.toArray(), 0); - Vector3 channelVisibilityMask = new Vector3(colorMap.getChannel(0).isVisible() ? 1 : 0, - colorMap.getChannel(1).isVisible() ? 1 : 0, - 1); - gl.glUniform3fv(channelVisibilityMaskIndex, 1, channelVisibilityMask.toArray(), 0); + gl.glUniform3fv(colorChannelIndex1, 1, color.toArray(), 0); + gl.glUniform1i(channelVisibilityMaskIndex, m.isVisible() ? 1 : 0); for (int c = 0; c < 2; ++c) { - ChannelColorModel chan = colorMap.getChannel(c); - opMin[c] = chan.getNormalizedMinimum(); - opMax[c] = chan.getNormalizedMaximum(); + opMin[c] = m.getNormalizedMinimum(); + opMax[c] = m.getNormalizedMaximum(); } } gl.glUniform2fv(opacityFunctionMinIndex, 1, opMin, 0); @@ -342,7 +346,7 @@ private void updateUniformIndices(GL3 gl) { projectionIndex = gl.glGetUniformLocation(s, "projectionMatrix"); tcToCameraIndex = gl.glGetUniformLocation(s, "tcToCamera"); opaqueZNearFarIndex = gl.glGetUniformLocation(s, "opaqueZNearFar"); - colorChannelIndex1 = gl.glGetUniformLocation(s, "colorChannel1"); + colorChannelIndex1 = gl.glGetUniformLocation(s, "colorChannel"); channelVisibilityMaskIndex = gl.glGetUniformLocation(s, "channelVisibilityMask"); uniformIndicesAreDirty = false; From 674db3cd1e22ba2c43ec13797393a5a1d14197e4 Mon Sep 17 00:00:00 2001 From: Patrick Edson Date: Sat, 28 Oct 2023 08:35:44 -0400 Subject: [PATCH 2/4] Simplify initial load. --- .../org/janelia/horta/NeuronTraceLoader.java | 40 +++++++------------ .../horta/actors/OmeZarrVolumeActor.java | 4 ++ 2 files changed, 18 insertions(+), 26 deletions(-) diff --git a/modules/HortaTracer/src/main/java/org/janelia/horta/NeuronTraceLoader.java b/modules/HortaTracer/src/main/java/org/janelia/horta/NeuronTraceLoader.java index 259f76826..9a89bb137 100644 --- a/modules/HortaTracer/src/main/java/org/janelia/horta/NeuronTraceLoader.java +++ b/modules/HortaTracer/src/main/java/org/janelia/horta/NeuronTraceLoader.java @@ -204,39 +204,27 @@ void loadOmeZarrTileAtLocation(OmeZarrBlockTileSource omeZarrSource, Vector3 loc if (omeZarrSource == null) { return; } - OmeZarrBlockTileKey centerKey = omeZarrSource.getBlockKeyAt(location, omeZarrSource.getMaximumResolution()); - if (centerKey == null) { - return; + + OmeZarrVolumeActor parentActor = OmeZarrVolumeActor.getInstance(); + + if (!neuronMPRenderer.containsVolumeActor(parentActor)) { // just add singleton actor once... + parentActor.setBrightnessModel(neuronMPRenderer.getBrightnessModel()); + neuronMPRenderer.addVolumeActor(parentActor); } if (loadPersistent) { - final OmeZarrBlockLoadRunner loader = new OmeZarrBlockLoadRunner(omeZarrSource, centerKey); - loader.addObserver(new Observer() { - @Override - public void update(Observable o, Object arg) { - if (loader.state != OmeZarrBlockLoadRunner.State.LOADED) { - return; - } - OmeZarrVolumeActor parentActor = OmeZarrVolumeActor.getInstance(); - parentActor.addPersistentBlock(loader.blockActor); - parentActor.addPersistentBlock(loader.blockActor); - if (!neuronMPRenderer.containsVolumeActor(parentActor)) { // just add singleton actor once... - parentActor.setBrightnessModel(neuronMPRenderer.getBrightnessModel()); - neuronMPRenderer.addVolumeActor(parentActor); - } - neuronMPRenderer.setIntensityBufferDirty(); - nttc.redrawNow(); - } - }); - loader.run(); + parentActor.refreshBlocks(location); } else { - OmeZarrVolumeActor parentActor = OmeZarrVolumeActor.getInstance(); - if (!neuronMPRenderer.containsVolumeActor(parentActor)) { // just add singleton actor once... - parentActor.setBrightnessModel(neuronMPRenderer.getBrightnessModel()); - neuronMPRenderer.addVolumeActor(parentActor); + OmeZarrBlockTileKey centerKey = omeZarrSource.getBlockKeyAt(location, omeZarrSource.getMaximumResolution()); + if (centerKey == null) { + return; } + parentActor.addTransientBlock(centerKey); } + + neuronMPRenderer.setIntensityBufferDirty(); + nttc.redrawNow(); } void loadPersistentKtxTileAtCurrentFocus(KtxOctreeBlockTileSource ktxSource) diff --git a/modules/HortaTracer/src/main/java/org/janelia/horta/actors/OmeZarrVolumeActor.java b/modules/HortaTracer/src/main/java/org/janelia/horta/actors/OmeZarrVolumeActor.java index 31bec004f..0e31328a9 100644 --- a/modules/HortaTracer/src/main/java/org/janelia/horta/actors/OmeZarrVolumeActor.java +++ b/modules/HortaTracer/src/main/java/org/janelia/horta/actors/OmeZarrVolumeActor.java @@ -281,6 +281,10 @@ public void setBrightnessModel(ImageColorModel brightnessModel) { this.brightnessModel = brightnessModel; } + public void refreshBlocks(ConstVector3 location) { + blockDisplayUpdater.refreshBlocks(location); + } + public void clearAllBlocks() { dynamicTiles.clearAllTiles(); getChildren().clear(); From 1c2e3b4d408f08df50fed5655349ff50560c0e02 Mon Sep 17 00:00:00 2001 From: Patrick Edson Date: Sat, 28 Oct 2023 08:46:31 -0400 Subject: [PATCH 3/4] Remove unused auto-contrast --- .../horta/NeuronTracerTopComponent.java | 2 +- .../horta/blocks/OmeZarrBlockTileKey.java | 4 +- .../horta/blocks/OmeZarrBlockTileSource.java | 52 +------------------ 3 files changed, 4 insertions(+), 54 deletions(-) diff --git a/modules/HortaTracer/src/main/java/org/janelia/horta/NeuronTracerTopComponent.java b/modules/HortaTracer/src/main/java/org/janelia/horta/NeuronTracerTopComponent.java index c2b1147eb..5c93705bb 100644 --- a/modules/HortaTracer/src/main/java/org/janelia/horta/NeuronTracerTopComponent.java +++ b/modules/HortaTracer/src/main/java/org/janelia/horta/NeuronTracerTopComponent.java @@ -1049,7 +1049,7 @@ public boolean loadDroppedOmeZarr(String sourceName) { final boolean[] haveSetBoundingBox = {false}; - setOmeZarrSource(new OmeZarrBlockTileSource(null, getImageColorModel(), false).init(sourceName, + setOmeZarrSource(new OmeZarrBlockTileSource(null, getImageColorModel()).init(sourceName, (source, update) -> SwingUtilities.invokeLater(() -> { progress.setDisplayName(update); diff --git a/modules/HortaTracer/src/main/java/org/janelia/horta/blocks/OmeZarrBlockTileKey.java b/modules/HortaTracer/src/main/java/org/janelia/horta/blocks/OmeZarrBlockTileKey.java index d7839e4fa..80ed9a96d 100644 --- a/modules/HortaTracer/src/main/java/org/janelia/horta/blocks/OmeZarrBlockTileKey.java +++ b/modules/HortaTracer/src/main/java/org/janelia/horta/blocks/OmeZarrBlockTileKey.java @@ -166,11 +166,11 @@ public double getResolutionMicrometers() { private static final ColorModel colorModel = new ComponentColorModel(ColorSpace.getInstance(ColorSpace.CS_GRAY), false, true, Transparency.OPAQUE, DataBuffer.TYPE_USHORT); - public Texture3d loadBrick(AutoContrastParameters parameters) { + public Texture3d loadBrick() { Texture3d texture = new Texture3d(); try { - WritableRaster[] slices = TCZYXRasterZStack.fromDataset(dataset, readShape, readOffset, 1, parameters != null, parameters, null); + WritableRaster[] slices = TCZYXRasterZStack.fromDataset(dataset, readShape, readOffset, 1, false, null, null); texture.loadRasterSlices(slices, colorModel); diff --git a/modules/HortaTracer/src/main/java/org/janelia/horta/blocks/OmeZarrBlockTileSource.java b/modules/HortaTracer/src/main/java/org/janelia/horta/blocks/OmeZarrBlockTileSource.java index 9cf297ec1..0f89a8b0d 100644 --- a/modules/HortaTracer/src/main/java/org/janelia/horta/blocks/OmeZarrBlockTileSource.java +++ b/modules/HortaTracer/src/main/java/org/janelia/horta/blocks/OmeZarrBlockTileSource.java @@ -35,28 +35,19 @@ public class OmeZarrBlockTileSource implements BlockTileSource resolutions = new ArrayList<>(); private OmeZarrBlockResolution maximumResolution = null; private final ImageColorModel imageColorModel; - private final boolean useAutoContrast; - private BoundingBox3d boundingBox3d = new BoundingBox3d(); private Vec3 voxelCenter = new Vec3(0, 0, 0); private static final ExecutorService cachedThreadPool = Executors.newCachedThreadPool(); public OmeZarrBlockTileSource(URL originatingSampleURL, ImageColorModel imageColorModel) { - this(originatingSampleURL, imageColorModel, false); - } - - public OmeZarrBlockTileSource(URL originatingSampleURL, ImageColorModel imageColorModel, boolean useAutoContrast) { this.originatingSampleURL = originatingSampleURL; this.imageColorModel = imageColorModel; - this.useAutoContrast = useAutoContrast; } public OmeZarrBlockTileSource init(String localPath) throws IOException { @@ -134,20 +125,6 @@ private OmeZarrBlockTileSource init(OmeZarrReaderProgressObserver progressObserv progressObserver.update(this, "Loading dataset " + dataset.getPath()); } - if (useAutoContrast && autoContrastParameters == null) { - int[] autoContrastShape = {1, 1, 256, 256, 128}; - - AutoContrastParameters parameters = TCZYXRasterZStack.computeAutoContrast(dataset, autoContrastShape); - - double existingMax = parameters.min + (65535.0 / parameters.slope); - - double min = Math.max(100, parameters.min * 0.1); - double max = Math.min(65535.0, Math.max(min + 100, existingMax * 4)); - double slope = 65535.0 / (max - min); - - autoContrastParameters = new AutoContrastParameters(min, slope); - } - log.info("finished loading path " + dataset.getPath()); resolutions.add(resolution); @@ -242,33 +219,6 @@ public Texture3d loadBrick(OmeZarrBlockTileKey tile, int colorChannel) { } public Texture3d loadBrick(OmeZarrBlockTileKey tile) { - return tile.loadBrick(useAutoContrast ? autoContrastParameters : null); + return tile.loadBrick(); } -/* - private void createTileKeysForDataset(OmeZarrDataset dataset, OmeZarrReaderProgressObserver progressReceiver) { - try { - if (progressReceiver != null) { - progressReceiver.update(this, "Loading dataset " + dataset.getPath()); - } - - if (useAutoContrast && autoContrastParameters == null) { - int[] autoContrastShape = {1, 1, 256, 256, 128}; - - AutoContrastParameters parameters = TCZYXRasterZStack.computeAutoContrast(dataset, autoContrastShape); - - double existingMax = parameters.min + (65535.0 / parameters.slope); - - double min = Math.max(100, parameters.min * 0.1); - double max = Math.min(65535.0, Math.max(min + 100, existingMax * 4)); - double slope = 65535.0 / (max - min); - - autoContrastParameters = new AutoContrastParameters(min, slope); - } - - log.info("finished loading path " + dataset.getPath()); - } catch (Exception ex) { - log.error("failed to load path " + dataset.getPath()); - log.error(ex.getMessage()); - } - }*/ } From 6ec11833d93163ff2dfa5f2ca7d1f0955157cb70 Mon Sep 17 00:00:00 2001 From: Patrick Edson Date: Sun, 29 Oct 2023 09:48:11 -0400 Subject: [PATCH 4/4] OME-Zarr Multichannel --- .../material/shader/VolumeMipFrag.glsl | 58 +-- .../horta/NeuronTracerTopComponent.java | 9 +- .../horta/actors/OmeZarrVolumeActor.java | 385 +++++------------- .../actors/OmeZarrVolumeChannelActor.java | 353 ++++++++++++++++ .../horta/actors/OmeZarrVolumeMeshActor.java | 2 +- .../horta/blocks/OmeZarrBlockLoadRunner.java | 6 +- .../horta/blocks/OmeZarrBlockTileKey.java | 8 +- .../horta/blocks/OmeZarrBlockTileSource.java | 19 +- .../horta/blocks/OmeZarrTileCache.java | 6 +- 9 files changed, 502 insertions(+), 344 deletions(-) create mode 100644 modules/HortaTracer/src/main/java/org/janelia/horta/actors/OmeZarrVolumeChannelActor.java diff --git a/modules/GLViewerTools/src/main/resources/org/janelia/gltools/material/shader/VolumeMipFrag.glsl b/modules/GLViewerTools/src/main/resources/org/janelia/gltools/material/shader/VolumeMipFrag.glsl index 1e5a2f20f..43840a1c0 100644 --- a/modules/GLViewerTools/src/main/resources/org/janelia/gltools/material/shader/VolumeMipFrag.glsl +++ b/modules/GLViewerTools/src/main/resources/org/janelia/gltools/material/shader/VolumeMipFrag.glsl @@ -15,7 +15,7 @@ // output 1) maximum intensity along ray in red channel // 2) relative depth of brightest voxel in green channel -layout(location = 0) out vec4 colorOut; +layout(location = 0) out vec4 colorOut; // put surface normals in eye space for isosurface projection uniform mat4 tcToCamera = mat4(1); @@ -170,7 +170,7 @@ vec3 hsv2rgb(vec3 c) vec2 clipRayToUnitVolume(vec3 x0, vec3 x1) { // Compute extreme texture coordinates // Texture coordinates are restricted to [0,1] - // The texMax/texMin variables represent texture coordinates corresponding to + // The texMax/texMin variables represent texture coordinates corresponding to // large/small values of ray parameter "t". vec3 texMax = clamp(ceil(x1 * 100), vec3(0,0,0), vec3(1,1,1)); // each component is now 0 or 1 vec3 texMin = vec3(1,1,1) - texMax; // complement texMax @@ -178,7 +178,7 @@ vec2 clipRayToUnitVolume(vec3 x0, vec3 x1) { // colorOut = vec4(texMin, 1); return; // for debugging // colorOut = vec4(texMax, 1); return; // for debugging - // Compute parameter t limits in all three directions at once, + // Compute parameter t limits in all three directions at once, // using plane ray tracing equation. vec3 vtMin = -(x0 - texMin)/x1; float tMin = max(max(vtMin.x, vtMin.y), vtMin.z); // looks OK @@ -232,7 +232,7 @@ float minElement(vec4 v) { return min( min(v.r, v.g), min(v.b, v.a) ); } -float sampleScaledIntensity(sampler3D volume, vec3 uvw, vec3 textureScale) +float sampleScaledIntensity(sampler3D volume, vec3 uvw, vec3 textureScale) { COLOR_VEC unscaled; if (filteringOrder == 3) @@ -242,7 +242,7 @@ float sampleScaledIntensity(sampler3D volume, vec3 uvw, vec3 textureScale) return maxElement( unscaled ); } -vec3 calculateNormalInScreenSpace(sampler3D volume, vec3 uvw, vec3 voxelMicrometers, vec3 textureScale) +vec3 calculateNormalInScreenSpace(sampler3D volume, vec3 uvw, vec3 voxelMicrometers, vec3 textureScale) { vec3 v[2]; const float delta = 0.75; // sample 1/2 pixel away @@ -340,7 +340,7 @@ struct VoxelIntensity // Delegated Methods for compartmentalized volume rendering algorithm -float advance_to_voxel_edge(in float previousEdge, in RayParameters rayParameters) +float advance_to_voxel_edge(in float previousEdge, in RayParameters rayParameters) { // Units of ray parameter, t, are roughly texels const float minStep = 0.01; @@ -348,13 +348,13 @@ float advance_to_voxel_edge(in float previousEdge, in RayParameters rayParameter // Advance ray by at least minStep, to avoid getting stuck in tiny corners float t = previousEdge + minStep; vec3 x0 = rayParameters.rayOriginInTexels; - vec3 x1 = rayParameters.rayDirectionInTexels; + vec3 x1 = rayParameters.rayDirectionInTexels; vec3 currentTexelPos = (x0 + t*x1); // apply ray equation to find new voxel // Advance ray to next voxel edge. // For NEAREST filter, advance to midplanes between voxel centers. // For TRILINEAR and TRICUBIC filters, advance to planes connecing voxel centers. - vec3 currentTexel = floor(currentTexelPos + rayParameters.rayBoxCorner) + vec3 currentTexel = floor(currentTexelPos + rayParameters.rayBoxCorner) - rayParameters.rayBoxCorner; // Three out of six total voxel edges represent forward progress @@ -378,7 +378,7 @@ VoxelRayState find_first_voxel(in RayBounds rayBounds, in RayParameters rayParam } // Compute begin and end points of ray, once and for all -RayBounds initialize_ray_bounds(in RayParameters rayParameters, in ViewSlab viewSlab) +RayBounds initialize_ray_bounds(in RayParameters rayParameters, in ViewSlab viewSlab) { // bound ray within view slab thickness float slabTMin = viewSlab.minRayParam; @@ -394,7 +394,7 @@ RayBounds initialize_ray_bounds(in RayParameters rayParameters, in ViewSlab view vec3 texelMin = reverseMask / rayParameters.textureScale; vec3 x0 = rayParameters.rayOriginInTexels; vec3 x1 = rayParameters.rayDirectionInTexels; - // Compute parameter t limits in all three directions at once, + // Compute parameter t limits in all three directions at once, // using plane ray tracing equation. vec3 vtMin = -(x0 - texelMin)/x1; float texCoordTMin = max(max(vtMin.x, vtMin.y), vtMin.z); // looks OK @@ -436,10 +436,10 @@ RayParameters initialize_ray_parameters() { if (filteringOrder == 0) // nearest neighbor { rayBoxCorner = vec3(0, 0, 0); // intersect ray at voxel edge planes, optimal for nearest-neighbor - } + } else // trilinear and tricubic { - rayBoxCorner = + rayBoxCorner = vec3(0.5, 0.5, 0.5); // intersect ray at voxel center planes } @@ -450,7 +450,7 @@ RayParameters initialize_ray_parameters() { return RayParameters(originInTexels, directionInTexels, rayBoxCorner, forwardMask, textureScale); } -ViewSlab initialize_view_slab(RayParameters rayParams) +ViewSlab initialize_view_slab(RayParameters rayParams) { vec3 x0 = rayParams.rayOriginInTexels; vec3 x1 = rayParams.rayDirectionInTexels; @@ -465,7 +465,7 @@ ViewSlab initialize_view_slab(RayParameters rayParams) // Integrate the latest sample into the accumulated color void integrate_intensity( - in VoxelIntensity localIntensity, + in VoxelIntensity localIntensity, inout IntegratedIntensity integratedIntensity, inout CoreStatus core, in ViewSlab viewSlab, @@ -561,8 +561,8 @@ void integrate_intensity( // * it also uses the OpacityFunction, which is otherwise not strictly // needed in this shader at all. So, for example, without the fade // effect, this entire render pass could be cached and skipped, as - // the user drags the opacity parameters. - float rd = (rayParameter - viewSlab.minRayParam) + // the user drags the opacity parameters. + float rd = (rayParameter - viewSlab.minRayParam) / (viewSlab.maxRayParam - viewSlab.minRayParam); // relative depth float fadeInterval = fadeThicknessInTexels / (viewSlab.maxRayParam - viewSlab.minRayParam); // But don't fade much in extremely thin slabs @@ -614,14 +614,14 @@ void integrate_intensity( // Are we done casting the current view ray? bool ray_complete( - in VoxelRayState state, - in RayBounds bounds, + in VoxelRayState state, + in RayBounds bounds, in IntegratedIntensity intensity, in CoreStatus core) { - if (state.exitRayParameter >= bounds.maxRayParameter) + if (state.exitRayParameter >= bounds.maxRayParameter) return true; // exited back of block - if ((intensity.opacity >= 0.99) && (! core.inLocalBody)) + if ((intensity.opacity >= 0.99) && (! core.inLocalBody)) return true; // stop at full occlusion return false; } @@ -629,9 +629,9 @@ bool ray_complete( // Fetch the color of the current voxel // AND track brightest point in tracing channel void sample_intensity( - in VoxelRayState rayState, - in RayParameters rayParams, - inout VoxelIntensity intensity) + in VoxelRayState rayState, + in RayParameters rayParams, + inout VoxelIntensity intensity) { // Cache previous intensity value, for later lookbehind. intensity.previousVoxelMiddleIntensity = intensity.voxelMiddleIntensity; @@ -668,7 +668,7 @@ void sample_intensity( } // Write out the final color/data after volume ray casting -void save_color(in IntegratedIntensity i, in ViewSlab slab) +void save_color(in IntegratedIntensity i, in ViewSlab slab) { // primary render target contains final color RGBA as of September 2016 // hot color map, green only @@ -677,9 +677,9 @@ void save_color(in IntegratedIntensity i, in ViewSlab slab) const float knot = 0.5; // where to kink linear ramp between three colors float intensity = rescaled.r; // First channel only at the moment if (intensity <= knot) - hotColor = mix(vec3(0), colorChannel * channelVisibilityMask, intensity / knot); + hotColor = mix(vec3(0) * channelVisibilityMask, colorChannel * channelVisibilityMask, (intensity / knot) * channelVisibilityMask); else - hotColor = mix(colorChannel * channelVisibilityMask, vec3(0.95,1,0.9), (intensity - knot)/(1.0 - knot)); + hotColor = mix(colorChannel * channelVisibilityMask, vec3(0.95,1,0.9) * channelVisibilityMask, (intensity - knot)/(1.0 - knot) * channelVisibilityMask); colorOut = vec4(hotColor, i.opacity); // colorOut = vec4(i.intensity.g, i.intensity.r, i.intensity.g, i.opacity); @@ -718,8 +718,8 @@ IntegratedIntensity cast_volume_ray(in RayParameters rayParameters, in ViewSlab return integratedIntensity; // DONE, we made it to the far side of the volume sample_intensity(voxelRayState, rayParameters, voxelIntensity); // this is an expensive step, due to texture fetch(es) integrate_intensity( - voxelIntensity, - integratedIntensity, + voxelIntensity, + integratedIntensity, coreStatus, viewSlab, voxelRayState, rayParameters); step_ray(rayParameters, voxelRayState); stepCount += 1; @@ -733,7 +733,7 @@ void main() { RayParameters rayParams = initialize_ray_parameters(); ViewSlab viewSlab = initialize_view_slab(rayParams); IntegratedIntensity integratedIntensity = cast_volume_ray(rayParams, viewSlab); - if (integratedIntensity.opacity <= 0.005) + if (integratedIntensity.opacity <= 0.005) discard; // This ray is invisible, so try to save resources by not painting it. save_color(integratedIntensity, viewSlab); } diff --git a/modules/HortaTracer/src/main/java/org/janelia/horta/NeuronTracerTopComponent.java b/modules/HortaTracer/src/main/java/org/janelia/horta/NeuronTracerTopComponent.java index 5c93705bb..2d1f8e178 100644 --- a/modules/HortaTracer/src/main/java/org/janelia/horta/NeuronTracerTopComponent.java +++ b/modules/HortaTracer/src/main/java/org/janelia/horta/NeuronTracerTopComponent.java @@ -335,12 +335,9 @@ public void update(Observable o, Object arg) { }); OmeZarrVolumeActor.getInstance().setVolumeState(volumeState); - OmeZarrVolumeActor.getInstance().getDynamicTileUpdateObservable().addObserver(new Observer() { - @Override - public void update(Observable o, Object arg) { - getNeuronMPRenderer().setIntensityBufferDirty(); - redrawNow(); - } + OmeZarrVolumeActor.getInstance().addDynamicTileUpdateObservable((o, arg) -> { + getNeuronMPRenderer().setIntensityBufferDirty(); + redrawNow(); }); neuronMPRenderer = setUpActors(); diff --git a/modules/HortaTracer/src/main/java/org/janelia/horta/actors/OmeZarrVolumeActor.java b/modules/HortaTracer/src/main/java/org/janelia/horta/actors/OmeZarrVolumeActor.java index 0e31328a9..f0fc5958a 100644 --- a/modules/HortaTracer/src/main/java/org/janelia/horta/actors/OmeZarrVolumeActor.java +++ b/modules/HortaTracer/src/main/java/org/janelia/horta/actors/OmeZarrVolumeActor.java @@ -2,27 +2,19 @@ import org.janelia.geometry3d.*; import org.janelia.gltools.BasicGL3Actor; -import org.janelia.gltools.GL3Actor; import org.janelia.gltools.material.DepthSlabClipper; import org.janelia.gltools.texture.Texture2d; -import org.janelia.horta.blocks.*; +import org.janelia.horta.blocks.BlockTileKey; +import org.janelia.horta.blocks.OmeZarrBlockTileSource; import org.janelia.horta.volume.VolumeMipMaterial; -import org.janelia.horta.volume.VolumeMipMaterial.VolumeState; import org.janelia.workstation.controller.model.color.ImageColorModel; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import javax.media.opengl.GL3; -import javax.media.opengl.GL4; - import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; import java.util.List; +import java.util.Observer; public class OmeZarrVolumeActor extends BasicGL3Actor implements DepthSlabClipper { - private static final Logger LOG = LoggerFactory.getLogger(OmeZarrVolumeActor.class); - private static OmeZarrVolumeActor singletonInstance; // Singleton access @@ -36,328 +28,133 @@ static public OmeZarrVolumeActor getInstance() { private float zNearRelative = 0.50f; private float zFarRelative = 150.0f; // relative z clip planes private ImageColorModel brightnessModel; + private VolumeMipMaterial.VolumeState volumeState; + private OmeZarrBlockTileSource source; - private VolumeMipMaterial.VolumeState volumeState = new VolumeMipMaterial.VolumeState(); + private Observer dynamicTilesObserver; + private Vantage vantage; + private boolean autoUpdateCache; - private OmeZarrBlockTileSource source; - private final OmeZarrTileCache dynamicTiles = new OmeZarrTileCache(null); - private BlockChooser chooser; - private BlockDisplayUpdater blockDisplayUpdater; - - private final BlockSorter blockSorter = new BlockSorter(); + private final List channelActors; private OmeZarrVolumeActor() { super(null); - chooser = new OmeZarrBlockChooser(); - blockDisplayUpdater = new BlockDisplayUpdater<>(chooser); - initBlockStrategy(chooser); - blockDisplayUpdater.getDisplayChangeObservable().addObserver((o, arg) -> dynamicTiles.updateDesiredTiles(blockDisplayUpdater.getDesiredBlocks())); - dynamicTiles.getDisplayChangeObservable().addObserver(((o, arg) -> { - List list = new ArrayList<>(); - getChildren().forEach(c -> { - if (!dynamicTiles.getDisplayedActors().contains(c)) { - list.add(c); - } - }); - getChildren().removeAll(list); - dynamicTiles.getDisplayedActors().forEach(a -> { - if (!getChildren().contains(a)) { - addPersistentBlock(a); - } - }); - })); - } - - //Calculate a distance from a camera to a block. - //Centroids cannot be used to calculate distances between camera and blocks in zarr. - //Centroids only work when all blocks in the same resolution level have the same dimensions. - //But blocks in zarr can have arbitrary dimensions. - private float calcDistanceFromCamera(Vector3 minp, Vector3 maxp, Matrix4 viewMatrix) - { - final Vector4[] corner = { - viewMatrix.multiply(new Vector4(minp.getX(), minp.getY(), minp.getZ(), 1.0f)), - viewMatrix.multiply(new Vector4(minp.getX(), minp.getY(), maxp.getZ(), 1.0f)), - viewMatrix.multiply(new Vector4(minp.getX(), maxp.getY(), minp.getZ(), 1.0f)), - viewMatrix.multiply(new Vector4(maxp.getX(), minp.getY(), minp.getZ(), 1.0f)), - viewMatrix.multiply(new Vector4(minp.getX(), maxp.getY(), maxp.getZ(), 1.0f)), - viewMatrix.multiply(new Vector4(maxp.getX(), minp.getY(), maxp.getZ(), 1.0f)), - viewMatrix.multiply(new Vector4(maxp.getX(), maxp.getY(), minp.getZ(), 1.0f)), - viewMatrix.multiply(new Vector4(maxp.getX(), maxp.getY(), maxp.getZ(), 1.0f)) - }; - - //distances from a camera to faces - final int[][] faces = { - {0, 1, 2}, - {0, 2, 3}, - {0, 3, 1}, - {7, 6, 5}, - {7, 5, 4}, - {7, 4, 6} - }; - float dmin = Float.MAX_VALUE; - for (int f = 0; f < 6; f++) - { - Vector4[] p = {corner[faces[f][0]], corner[faces[f][1]], corner[faces[f][2]]}; - Vector3 v0 = new Vector3(p[0].get(0), p[0].get(1), p[0].get(2)); - Vector3 v1 = new Vector3(p[1].get(0)-p[0].get(0), p[1].get(1)-p[0].get(1), p[1].get(2)-p[0].get(2)); - Vector3 v2 = new Vector3(p[2].get(0)-p[0].get(0), p[2].get(1)-p[0].get(1), p[2].get(2)-p[0].get(2)); - Vector3 n = v1.cross(v2).normalize(); - float t = n.dot(v0); - - Vector3 vf = new Vector3(n.get(0)*t - v0.get(0), n.get(1)*t - v0.get(1), n.get(2)*t - v0.get(2)); - float v1len = v1.length(); - float v2len = v2.length(); - Vector3 n1 = v1.normalize(); - Vector3 n2 = v2.normalize(); - float t1 = n1.dot(vf) / v1len; - float t2 = n2.dot(vf) / v2len; - - if (t1 > 0 && t1 < 1 && t2 > 0 && t2 < 1) - { - float dd = t * t; - if (dd < dmin) - dmin = dd; - } - } - - //distances from a camera to edges - final int[][] edges = { - {0, 1, 2, 4}, //min x -> max x - {0, 1, 3, 5}, //min y -> max y - {0, 2, 3, 6} //min z -> max z - }; - Vector3[] ev = { //ev[0]: x axis, ev[1]: y axis, ev[2]: z axis (of the block) - new Vector3(corner[3].get(0) - corner[0].get(0), corner[3].get(1) - corner[0].get(1), corner[3].get(2) - corner[0].get(2)), - new Vector3(corner[2].get(0) - corner[0].get(0), corner[2].get(1) - corner[0].get(1), corner[2].get(2) - corner[0].get(2)), - new Vector3(corner[1].get(0) - corner[0].get(0), corner[1].get(1) - corner[0].get(1), corner[1].get(2) - corner[0].get(2)), - }; - float[] dimensions = new float[3]; //dimensions[0]: width, dimensions[1]: hight, dimensions[2]: depth - for (int i = 0; i < 3; i++) - { - dimensions[i] = ev[i].length(); - ev[i].normalize(); - } - for (int i = 0; i < 3; i++) - { - for (int j = 0; j < 4; j++) - { - Vector3 v = new Vector3(corner[edges[i][j]].get(0), corner[edges[i][j]].get(1), corner[edges[i][j]].get(2)); - float t = -(ev[i].dot(v)); - if (t > 0 && t < dimensions[i]) - { - Vector3 ep1 = new Vector3(ev[i].get(0)*t + v.get(0), ev[i].get(1)*t + v.get(1), ev[i].get(2)*t + v.get(2)); - float dd = ep1.lengthSquared(); - if (dd < dmin) - dmin = dd; - } - } - } - - //distances from a camera to corners - for (int i = 0; i < 8; i++) - { - float dd = corner[i].get(0)*corner[i].get(0) + corner[i].get(1)*corner[i].get(1) + corner[i].get(2)*corner[i].get(2); - if (dd < dmin) - dmin = dd; - } - - //return the shortest distance - return dmin; - } - - @Override - public void display(GL3 gl, AbstractCamera camera, Matrix4 parentModelViewMatrix) { - if (! isVisible()) - return; - if (! isInitialized) init(gl); - if (getChildren() == null) - return; - if (getChildren().size() < 1) - return; - - gl.glEnable(GL3.GL_BLEND); - if (volumeState.projectionMode == VolumeState.PROJECTION_OCCLUDING) { - // Occluding - ((GL4)gl).glBlendEquationi(0, GL4.GL_FUNC_ADD); // RGBA color target - ((GL4)gl).glBlendFuncSeparatei(0, - GL4.GL_SRC_ALPHA, GL4.GL_ONE_MINUS_SRC_ALPHA, - GL4.GL_ONE, GL4.GL_ONE_MINUS_SRC_ALPHA); - } - else { - ((GL4)gl).glBlendEquationi(0, GL4.GL_MAX); // RGBA color target - } - // Always use Maximum for blending secondary render target - ((GL4)gl).glBlendEquationi(1, GL4.GL_MAX); // core intensity/depth target - - // Display Front faces only. - gl.glEnable(GL3.GL_CULL_FACE); - gl.glCullFace(GL3.GL_BACK); - - if (opaqueDepthTexture != null) { - final int depthTextureUnit = 2; - opaqueDepthTexture.bind(gl, depthTextureUnit); - } - - Matrix4 modelViewMatrix = parentModelViewMatrix; - if (modelViewMatrix == null) - modelViewMatrix = camera.getViewMatrix(); - Matrix4 localMatrix = getTransformInParent(); - if (localMatrix != null) - modelViewMatrix = new Matrix4(modelViewMatrix).multiply(localMatrix); - - List blockList = new ArrayList<>(); - List otherActorList = new ArrayList<>(); - List otherList = new ArrayList<>(); - for (SortableBlockActor actor : dynamicTiles.getDisplayedActors()) { - blockList.add(actor); - } - for (Object3d child : getChildren()) { - if (child instanceof SortableBlockActorSource) { - SortableBlockActorSource source = (SortableBlockActorSource)child; - for (SortableBlockActor block : source.getSortableBlockActors()) { - if (volumeState.projectionMode != VolumeState.PROJECTION_MAXIMUM) { - Vector3 minp = ((OmeZarrVolumeMeshActor)block).getBBoxMin(); - Vector3 maxp = ((OmeZarrVolumeMeshActor)block).getBBoxMax(); - ((OmeZarrVolumeMeshActor)block).setDistance(calcDistanceFromCamera(minp, maxp, modelViewMatrix)); - } - blockList.add(block); - } - } - else if (child instanceof GL3Actor) { - otherActorList.add((GL3Actor)child); - } - else { - otherList.add(child); - } - } - assert otherActorList.isEmpty(); - assert otherList.isEmpty(); - - // Order does not matter in MIP mode - if (volumeState.projectionMode != VolumeState.PROJECTION_MAXIMUM) { - blockSorter.setViewMatrix(modelViewMatrix); - Collections.sort(blockList, blockSorter); - } - for (SortableBlockActor actor : blockList) { - actor.display(gl, camera, modelViewMatrix); - } - } - private void initBlockStrategy(BlockChooser chooser) { - dynamicTiles.setBlockStrategy(chooser); - blockDisplayUpdater.setBlockChooser(chooser); + channelActors = new ArrayList<>(); + channelActors.add(new OmeZarrVolumeChannelActor(this, 0)); + channelActors.add(new OmeZarrVolumeChannelActor(this, 1)); + channelActors.add(new OmeZarrVolumeChannelActor(this, 2)); } - public void setAutoUpdate(boolean updateCache) { - blockDisplayUpdater.setAutoUpdate(updateCache); - } - - public VolumeMipMaterial.VolumeState getVolumeState() { - return volumeState; + public void addTransientBlock(BlockTileKey key) { + for (OmeZarrVolumeChannelActor channelActor : channelActors) { + channelActor.addTransientBlock(key); + } } - public Object3d addPersistentBlock(Object3d child) { - return addChild(child); + public void setAutoUpdate(boolean updateCache) { + this.autoUpdateCache = updateCache; + for (OmeZarrVolumeChannelActor channelActor : channelActors) { + channelActor.setAutoUpdate(updateCache); + } } - @Override - public Object3d addChild(Object3d child) { - if (child instanceof DepthSlabClipper) { - ((DepthSlabClipper) child).setOpaqueDepthTexture(opaqueDepthTexture); - ((DepthSlabClipper) child).setRelativeSlabThickness(zNearRelative, zFarRelative); + public void setVolumeState(VolumeMipMaterial.VolumeState volumeState) { + this.volumeState = volumeState; + for (OmeZarrVolumeChannelActor channelActor : channelActors) { + channelActor.setVolumeState(volumeState); } - return super.addChild(child); } - public void addTransientBlock(BlockTileKey key) { - dynamicTiles.addDesiredTile((OmeZarrBlockTileKey) key); + public void setHortaVantage(Vantage vantage) { + this.vantage = vantage; + for (OmeZarrVolumeChannelActor channelActor : channelActors) { + channelActor.setHortaVantage(vantage); + } } - public void setVolumeState(VolumeMipMaterial.VolumeState volumeState) { - this.volumeState = volumeState; + public void addDynamicTileUpdateObservable(Observer observer) { + this.dynamicTilesObserver = observer; + for (OmeZarrVolumeChannelActor channelActor : channelActors) { + channelActor.getDynamicTileUpdateObservable().addObserver(observer); + } } public void setBrightnessModel(ImageColorModel brightnessModel) { this.brightnessModel = brightnessModel; + for (OmeZarrVolumeChannelActor channelActor : channelActors) { + channelActor.setBrightnessModel(brightnessModel); + } } - public void refreshBlocks(ConstVector3 location) { - blockDisplayUpdater.refreshBlocks(location); + public void setOmeZarrTileSource(OmeZarrBlockTileSource source) { + this.source = source; + + /* + if (source != null) { + int numChannels = source.getNumColorChannels(); + + if (numChannels > channelActors.size()) { + for (int idx = channelActors.size(); idx < numChannels; idx++) { + OmeZarrVolumeChannelActor actor = new OmeZarrVolumeChannelActor(this, idx); + actor.setOmeZarrTileSource(source); + actor.setBrightnessModel(brightnessModel); + actor.setAutoUpdate(autoUpdateCache); + actor.setHortaVantage(vantage); + actor.setVolumeState(volumeState); + actor.setOpaqueDepthTexture(opaqueDepthTexture); + actor.setRelativeSlabThickness(zNearRelative, zFarRelative); + actor.getDynamicTileUpdateObservable().addObserver(dynamicTilesObserver); + channelActors.add(actor); + } + } else if (numChannels < channelActors.size()){ + channelActors.subList(numChannels, channelActors.size()).clear(); + } + } + */ + + for (OmeZarrVolumeChannelActor channelActor : channelActors) { + channelActor.setOmeZarrTileSource(source); + } } - public void clearAllBlocks() { - dynamicTiles.clearAllTiles(); - getChildren().clear(); + public VolumeMipMaterial.VolumeState getVolumeState() { + return this.volumeState; } - public void setHortaVantage(Vantage vantage) { - blockDisplayUpdater.setVantage(vantage); + public void refreshBlocks(ConstVector3 focus) { + for (OmeZarrVolumeChannelActor channelActor : channelActors) { + channelActor.refreshBlocks(focus); + } } - public void setOmeZarrTileSource(OmeZarrBlockTileSource source) { - dynamicTiles.setSource(source); - blockDisplayUpdater.setBlockTileSource(source); - this.source = source; + public void clearAllBlocks() { + for (OmeZarrVolumeChannelActor channelActor : channelActors) { + channelActor.clearAllBlocks(); + } } - public ObservableInterface getDynamicTileUpdateObservable() { - return dynamicTiles.getDisplayChangeObservable(); + @Override + public void display(GL3 gl, AbstractCamera camera, Matrix4 parentModelViewMatrix) { + for (OmeZarrVolumeChannelActor channelActor : channelActors) { + channelActor.display(gl, camera, parentModelViewMatrix); + } } @Override public void setOpaqueDepthTexture(Texture2d opaqueDepthTexture) { this.opaqueDepthTexture = opaqueDepthTexture; - - getChildren().forEach(c -> { - if (c instanceof DepthSlabClipper) { - ((DepthSlabClipper) c).setOpaqueDepthTexture(opaqueDepthTexture); - } - }); + for (OmeZarrVolumeChannelActor channelActor : channelActors) { + channelActor.setOpaqueDepthTexture(opaqueDepthTexture); + } } @Override public void setRelativeSlabThickness(float zNear, float zFar) { - if (zNear > 0.5 || zFar < 150) { - return; - } - - zNearRelative = zNear; - zFarRelative = zFar; - - getChildren().forEach(c -> { - if (c instanceof DepthSlabClipper) { - ((DepthSlabClipper) c).setRelativeSlabThickness(zNear, zFar); - } - }); - } - - private static class BlockSorter implements Comparator - { - private Matrix4 viewMatrix; - - public BlockSorter() { - } - - @Override - public int compare(SortableBlockActor o1, SortableBlockActor o2) { - // 1) If blocks are not the same resolution, sort on resolution - BlockTileResolution res1 = o1.getResolution(); - BlockTileResolution res2 = o2.getResolution(); - if (! res1.equals(res2)) { - return res1.compareTo(res2); - } - // 2) sort based on distance from centroid to camera - if (viewMatrix == null) - throw new UnsupportedOperationException("View Matrix is Null"); - - float d1 = (float)((OmeZarrVolumeMeshActor)o1).getDistance(); - float d2 = (float)((OmeZarrVolumeMeshActor)o2).getDistance(); - - return d1 > d2 ? -1 : d1 == d2 ? 0 : 1; - } - - public void setViewMatrix(Matrix4 viewMatrix) { - this.viewMatrix = viewMatrix; + this.zNearRelative = zNear; + this.zFarRelative = zFar; + for (OmeZarrVolumeChannelActor channelActor : channelActors) { + channelActor.setRelativeSlabThickness(zNear, zFar); } } -} \ No newline at end of file +} diff --git a/modules/HortaTracer/src/main/java/org/janelia/horta/actors/OmeZarrVolumeChannelActor.java b/modules/HortaTracer/src/main/java/org/janelia/horta/actors/OmeZarrVolumeChannelActor.java new file mode 100644 index 000000000..c6578afa7 --- /dev/null +++ b/modules/HortaTracer/src/main/java/org/janelia/horta/actors/OmeZarrVolumeChannelActor.java @@ -0,0 +1,353 @@ +package org.janelia.horta.actors; + +import org.janelia.geometry3d.*; +import org.janelia.gltools.BasicGL3Actor; +import org.janelia.gltools.GL3Actor; +import org.janelia.gltools.material.DepthSlabClipper; +import org.janelia.gltools.texture.Texture2d; +import org.janelia.horta.blocks.*; +import org.janelia.horta.volume.VolumeMipMaterial; +import org.janelia.horta.volume.VolumeMipMaterial.VolumeState; +import org.janelia.workstation.controller.model.color.ImageColorModel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.media.opengl.GL3; +import javax.media.opengl.GL4; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +public class OmeZarrVolumeChannelActor extends BasicGL3Actor implements DepthSlabClipper { + private Texture2d opaqueDepthTexture = null; + private float zNearRelative = 0.50f; + private float zFarRelative = 150.0f; // relative z clip planes + private ImageColorModel brightnessModel; + + private VolumeMipMaterial.VolumeState volumeState = new VolumeMipMaterial.VolumeState(); + + private OmeZarrBlockTileSource source; + private final OmeZarrTileCache dynamicTiles; + private BlockChooser chooser; + private BlockDisplayUpdater blockDisplayUpdater; + + private final BlockSorter blockSorter = new BlockSorter(); + + public OmeZarrVolumeChannelActor(OmeZarrVolumeActor parent, int colorChannel) { + super(parent); + chooser = new OmeZarrBlockChooser(); + blockDisplayUpdater = new BlockDisplayUpdater<>(chooser); + dynamicTiles = new OmeZarrTileCache(null, colorChannel); + initBlockStrategy(chooser); + blockDisplayUpdater.getDisplayChangeObservable().addObserver((o, arg) -> dynamicTiles.updateDesiredTiles(blockDisplayUpdater.getDesiredBlocks())); + dynamicTiles.getDisplayChangeObservable().addObserver(((o, arg) -> { + List list = new ArrayList<>(); + getChildren().forEach(c -> { + if (!dynamicTiles.getDisplayedActors().contains(c)) { + list.add(c); + } + }); + getChildren().removeAll(list); + dynamicTiles.getDisplayedActors().forEach(a -> { + if (!getChildren().contains(a)) { + addPersistentBlock(a); + } + }); + })); + } + + //Calculate a distance from a camera to a block. + //Centroids cannot be used to calculate distances between camera and blocks in zarr. + //Centroids only work when all blocks in the same resolution level have the same dimensions. + //But blocks in zarr can have arbitrary dimensions. + private float calcDistanceFromCamera(Vector3 minp, Vector3 maxp, Matrix4 viewMatrix) + { + final Vector4[] corner = { + viewMatrix.multiply(new Vector4(minp.getX(), minp.getY(), minp.getZ(), 1.0f)), + viewMatrix.multiply(new Vector4(minp.getX(), minp.getY(), maxp.getZ(), 1.0f)), + viewMatrix.multiply(new Vector4(minp.getX(), maxp.getY(), minp.getZ(), 1.0f)), + viewMatrix.multiply(new Vector4(maxp.getX(), minp.getY(), minp.getZ(), 1.0f)), + viewMatrix.multiply(new Vector4(minp.getX(), maxp.getY(), maxp.getZ(), 1.0f)), + viewMatrix.multiply(new Vector4(maxp.getX(), minp.getY(), maxp.getZ(), 1.0f)), + viewMatrix.multiply(new Vector4(maxp.getX(), maxp.getY(), minp.getZ(), 1.0f)), + viewMatrix.multiply(new Vector4(maxp.getX(), maxp.getY(), maxp.getZ(), 1.0f)) + }; + + //distances from a camera to faces + final int[][] faces = { + {0, 1, 2}, + {0, 2, 3}, + {0, 3, 1}, + {7, 6, 5}, + {7, 5, 4}, + {7, 4, 6} + }; + float dmin = Float.MAX_VALUE; + for (int f = 0; f < 6; f++) + { + Vector4[] p = {corner[faces[f][0]], corner[faces[f][1]], corner[faces[f][2]]}; + Vector3 v0 = new Vector3(p[0].get(0), p[0].get(1), p[0].get(2)); + Vector3 v1 = new Vector3(p[1].get(0)-p[0].get(0), p[1].get(1)-p[0].get(1), p[1].get(2)-p[0].get(2)); + Vector3 v2 = new Vector3(p[2].get(0)-p[0].get(0), p[2].get(1)-p[0].get(1), p[2].get(2)-p[0].get(2)); + Vector3 n = v1.cross(v2).normalize(); + float t = n.dot(v0); + + Vector3 vf = new Vector3(n.get(0)*t - v0.get(0), n.get(1)*t - v0.get(1), n.get(2)*t - v0.get(2)); + float v1len = v1.length(); + float v2len = v2.length(); + Vector3 n1 = v1.normalize(); + Vector3 n2 = v2.normalize(); + float t1 = n1.dot(vf) / v1len; + float t2 = n2.dot(vf) / v2len; + + if (t1 > 0 && t1 < 1 && t2 > 0 && t2 < 1) + { + float dd = t * t; + if (dd < dmin) + dmin = dd; + } + } + + //distances from a camera to edges + final int[][] edges = { + {0, 1, 2, 4}, //min x -> max x + {0, 1, 3, 5}, //min y -> max y + {0, 2, 3, 6} //min z -> max z + }; + Vector3[] ev = { //ev[0]: x axis, ev[1]: y axis, ev[2]: z axis (of the block) + new Vector3(corner[3].get(0) - corner[0].get(0), corner[3].get(1) - corner[0].get(1), corner[3].get(2) - corner[0].get(2)), + new Vector3(corner[2].get(0) - corner[0].get(0), corner[2].get(1) - corner[0].get(1), corner[2].get(2) - corner[0].get(2)), + new Vector3(corner[1].get(0) - corner[0].get(0), corner[1].get(1) - corner[0].get(1), corner[1].get(2) - corner[0].get(2)), + }; + float[] dimensions = new float[3]; //dimensions[0]: width, dimensions[1]: hight, dimensions[2]: depth + for (int i = 0; i < 3; i++) + { + dimensions[i] = ev[i].length(); + ev[i].normalize(); + } + for (int i = 0; i < 3; i++) + { + for (int j = 0; j < 4; j++) + { + Vector3 v = new Vector3(corner[edges[i][j]].get(0), corner[edges[i][j]].get(1), corner[edges[i][j]].get(2)); + float t = -(ev[i].dot(v)); + if (t > 0 && t < dimensions[i]) + { + Vector3 ep1 = new Vector3(ev[i].get(0)*t + v.get(0), ev[i].get(1)*t + v.get(1), ev[i].get(2)*t + v.get(2)); + float dd = ep1.lengthSquared(); + if (dd < dmin) + dmin = dd; + } + } + } + + //distances from a camera to corners + for (int i = 0; i < 8; i++) + { + float dd = corner[i].get(0)*corner[i].get(0) + corner[i].get(1)*corner[i].get(1) + corner[i].get(2)*corner[i].get(2); + if (dd < dmin) + dmin = dd; + } + + //return the shortest distance + return dmin; + } + + @Override + public void display(GL3 gl, AbstractCamera camera, Matrix4 parentModelViewMatrix) { + if (! isVisible()) + return; + if (! isInitialized) init(gl); + if (getChildren() == null) + return; + if (getChildren().size() < 1) + return; + + gl.glEnable(GL3.GL_BLEND); + if (volumeState.projectionMode == VolumeState.PROJECTION_OCCLUDING) { + // Occluding + ((GL4)gl).glBlendEquationi(0, GL4.GL_FUNC_ADD); // RGBA color target + ((GL4)gl).glBlendFuncSeparatei(0, + GL4.GL_SRC_ALPHA, GL4.GL_ONE_MINUS_SRC_ALPHA, + GL4.GL_ONE, GL4.GL_ONE_MINUS_SRC_ALPHA); + } + else { + ((GL4)gl).glBlendEquationi(0, GL4.GL_MAX); // RGBA color target + } + // Always use Maximum for blending secondary render target + ((GL4)gl).glBlendEquationi(1, GL4.GL_MAX); // core intensity/depth target + + // Display Front faces only. + gl.glEnable(GL3.GL_CULL_FACE); + gl.glCullFace(GL3.GL_BACK); + + if (opaqueDepthTexture != null) { + final int depthTextureUnit = 2; + opaqueDepthTexture.bind(gl, depthTextureUnit); + } + + Matrix4 modelViewMatrix = parentModelViewMatrix; + if (modelViewMatrix == null) + modelViewMatrix = camera.getViewMatrix(); + Matrix4 localMatrix = getTransformInParent(); + if (localMatrix != null) + modelViewMatrix = new Matrix4(modelViewMatrix).multiply(localMatrix); + + List blockList = new ArrayList<>(); + List otherActorList = new ArrayList<>(); + List otherList = new ArrayList<>(); + for (SortableBlockActor actor : dynamicTiles.getDisplayedActors()) { + blockList.add(actor); + } + for (Object3d child : getChildren()) { + if (child instanceof SortableBlockActorSource) { + SortableBlockActorSource source = (SortableBlockActorSource)child; + for (SortableBlockActor block : source.getSortableBlockActors()) { + if (volumeState.projectionMode != VolumeState.PROJECTION_MAXIMUM) { + Vector3 minp = ((OmeZarrVolumeMeshActor)block).getBBoxMin(); + Vector3 maxp = ((OmeZarrVolumeMeshActor)block).getBBoxMax(); + ((OmeZarrVolumeMeshActor)block).setDistance(calcDistanceFromCamera(minp, maxp, modelViewMatrix)); + } + blockList.add(block); + } + } + else if (child instanceof GL3Actor) { + otherActorList.add((GL3Actor)child); + } + else { + otherList.add(child); + } + } + assert otherActorList.isEmpty(); + assert otherList.isEmpty(); + + // Order does not matter in MIP mode + if (volumeState.projectionMode != VolumeState.PROJECTION_MAXIMUM) { + blockSorter.setViewMatrix(modelViewMatrix); + Collections.sort(blockList, blockSorter); + } + for (SortableBlockActor actor : blockList) { + actor.display(gl, camera, modelViewMatrix); + } + } + + private void initBlockStrategy(BlockChooser chooser) { + dynamicTiles.setBlockStrategy(chooser); + blockDisplayUpdater.setBlockChooser(chooser); + } + + public void setAutoUpdate(boolean updateCache) { + blockDisplayUpdater.setAutoUpdate(updateCache); + } + + public VolumeMipMaterial.VolumeState getVolumeState() { + return volumeState; + } + + public Object3d addPersistentBlock(Object3d child) { + return addChild(child); + } + + @Override + public Object3d addChild(Object3d child) { + if (child instanceof DepthSlabClipper) { + ((DepthSlabClipper) child).setOpaqueDepthTexture(opaqueDepthTexture); + ((DepthSlabClipper) child).setRelativeSlabThickness(zNearRelative, zFarRelative); + } + return super.addChild(child); + } + + public void addTransientBlock(BlockTileKey key) { + dynamicTiles.addDesiredTile((OmeZarrBlockTileKey) key); + } + + public void setVolumeState(VolumeMipMaterial.VolumeState volumeState) { + this.volumeState = volumeState; + } + + public void setBrightnessModel(ImageColorModel brightnessModel) { + this.brightnessModel = brightnessModel; + } + + public void refreshBlocks(ConstVector3 location) { + blockDisplayUpdater.refreshBlocks(location); + } + + public void clearAllBlocks() { + dynamicTiles.clearAllTiles(); + getChildren().clear(); + } + + public void setHortaVantage(Vantage vantage) { + blockDisplayUpdater.setVantage(vantage); + } + + public void setOmeZarrTileSource(OmeZarrBlockTileSource source) { + dynamicTiles.setSource(source); + blockDisplayUpdater.setBlockTileSource(source); + this.source = source; + } + + public ObservableInterface getDynamicTileUpdateObservable() { + return dynamicTiles.getDisplayChangeObservable(); + } + + @Override + public void setOpaqueDepthTexture(Texture2d opaqueDepthTexture) { + this.opaqueDepthTexture = opaqueDepthTexture; + + getChildren().forEach(c -> { + if (c instanceof DepthSlabClipper) { + ((DepthSlabClipper) c).setOpaqueDepthTexture(opaqueDepthTexture); + } + }); + } + + @Override + public void setRelativeSlabThickness(float zNear, float zFar) { + if (zNear > 0.5 || zFar < 150) { + return; + } + + zNearRelative = zNear; + zFarRelative = zFar; + + getChildren().forEach(c -> { + if (c instanceof DepthSlabClipper) { + ((DepthSlabClipper) c).setRelativeSlabThickness(zNear, zFar); + } + }); + } + + private static class BlockSorter implements Comparator + { + private Matrix4 viewMatrix; + + public BlockSorter() { + } + + @Override + public int compare(SortableBlockActor o1, SortableBlockActor o2) { + // 1) If blocks are not the same resolution, sort on resolution + BlockTileResolution res1 = o1.getResolution(); + BlockTileResolution res2 = o2.getResolution(); + if (! res1.equals(res2)) { + return res1.compareTo(res2); + } + // 2) sort based on distance from centroid to camera + if (viewMatrix == null) + throw new UnsupportedOperationException("View Matrix is Null"); + + float d1 = (float)((OmeZarrVolumeMeshActor)o1).getDistance(); + float d2 = (float)((OmeZarrVolumeMeshActor)o2).getDistance(); + + return d1 > d2 ? -1 : d1 == d2 ? 0 : 1; + } + + public void setViewMatrix(Matrix4 viewMatrix) { + this.viewMatrix = viewMatrix; + } + } +} \ No newline at end of file diff --git a/modules/HortaTracer/src/main/java/org/janelia/horta/actors/OmeZarrVolumeMeshActor.java b/modules/HortaTracer/src/main/java/org/janelia/horta/actors/OmeZarrVolumeMeshActor.java index 2ee1fd194..04b2cb451 100644 --- a/modules/HortaTracer/src/main/java/org/janelia/horta/actors/OmeZarrVolumeMeshActor.java +++ b/modules/HortaTracer/src/main/java/org/janelia/horta/actors/OmeZarrVolumeMeshActor.java @@ -101,7 +101,7 @@ public BlockTileResolution getResolution() { private static class MeshMaterial extends VolumeMipMaterial { private MeshMaterial(OmeZarrBlockTileSource source, OmeZarrBlockTileKey tile, ImageColorModel imageColorModel, VolumeState volumeState, int colorChannel) throws IOException { - super(safeLoadData(source, tile, colorChannel), imageColorModel); + super(safeLoadData(source, tile, colorChannel), imageColorModel, colorChannel); setVolumeState(volumeState); } diff --git a/modules/HortaTracer/src/main/java/org/janelia/horta/blocks/OmeZarrBlockLoadRunner.java b/modules/HortaTracer/src/main/java/org/janelia/horta/blocks/OmeZarrBlockLoadRunner.java index dacaa75d4..a774c63d3 100644 --- a/modules/HortaTracer/src/main/java/org/janelia/horta/blocks/OmeZarrBlockLoadRunner.java +++ b/modules/HortaTracer/src/main/java/org/janelia/horta/blocks/OmeZarrBlockLoadRunner.java @@ -21,14 +21,16 @@ public enum State { private final OmeZarrBlockTileSource omeZarrBlockTileSource; private final OmeZarrBlockTileKey omeZarrOctreeBlockTileKey; + private final int colorChannel; public OmeZarrBlockLoadRunner.State state = OmeZarrBlockLoadRunner.State.INITIAL; public OmeZarrVolumeMeshActor blockActor; - public OmeZarrBlockLoadRunner(OmeZarrBlockTileSource source, OmeZarrBlockTileKey key) { + public OmeZarrBlockLoadRunner(OmeZarrBlockTileSource source, OmeZarrBlockTileKey key, int colorChannel) { this.omeZarrBlockTileSource = source; this.omeZarrOctreeBlockTileKey = key; + this.colorChannel = colorChannel; } @Override @@ -42,7 +44,7 @@ private void loadFromBlockSource() { try { state = OmeZarrBlockLoadRunner.State.LOADING; OmeZarrVolumeActor parentActor = OmeZarrVolumeActor.getInstance(); - blockActor = new OmeZarrVolumeMeshActor(omeZarrBlockTileSource, omeZarrOctreeBlockTileKey, parentActor.getVolumeState(), 0); + blockActor = new OmeZarrVolumeMeshActor(omeZarrBlockTileSource, omeZarrOctreeBlockTileKey, parentActor.getVolumeState(), colorChannel); state = OmeZarrBlockLoadRunner.State.LOADED; setChanged(); long endTime = System.currentTimeMillis(); diff --git a/modules/HortaTracer/src/main/java/org/janelia/horta/blocks/OmeZarrBlockTileKey.java b/modules/HortaTracer/src/main/java/org/janelia/horta/blocks/OmeZarrBlockTileKey.java index 80ed9a96d..bec074dcf 100644 --- a/modules/HortaTracer/src/main/java/org/janelia/horta/blocks/OmeZarrBlockTileKey.java +++ b/modules/HortaTracer/src/main/java/org/janelia/horta/blocks/OmeZarrBlockTileKey.java @@ -166,11 +166,15 @@ public double getResolutionMicrometers() { private static final ColorModel colorModel = new ComponentColorModel(ColorSpace.getInstance(ColorSpace.CS_GRAY), false, true, Transparency.OPAQUE, DataBuffer.TYPE_USHORT); - public Texture3d loadBrick() { + public Texture3d loadBrick(int colorChannel) { Texture3d texture = new Texture3d(); try { - WritableRaster[] slices = TCZYXRasterZStack.fromDataset(dataset, readShape, readOffset, 1, false, null, null); + int[] offset = Arrays.copyOf(readOffset, readOffset.length); + + offset[1] = colorChannel; + + WritableRaster[] slices = TCZYXRasterZStack.fromDataset(dataset, readShape, offset, 1, false, null, null); texture.loadRasterSlices(slices, colorModel); diff --git a/modules/HortaTracer/src/main/java/org/janelia/horta/blocks/OmeZarrBlockTileSource.java b/modules/HortaTracer/src/main/java/org/janelia/horta/blocks/OmeZarrBlockTileSource.java index 0f89a8b0d..7492baf4a 100644 --- a/modules/HortaTracer/src/main/java/org/janelia/horta/blocks/OmeZarrBlockTileSource.java +++ b/modules/HortaTracer/src/main/java/org/janelia/horta/blocks/OmeZarrBlockTileSource.java @@ -43,6 +43,8 @@ public class OmeZarrBlockTileSource implements BlockTileSource { + // cachedThreadPool.submit(() -> { int datasetCount = omeZarrGroup.getAttributes().getMultiscales()[0].getDatasets().size(); boolean haveExtents = false; @@ -117,6 +119,8 @@ private OmeZarrBlockTileSource init(OmeZarrReaderProgressObserver progressObserv voxelCenter = boundingBox3d.getCenter(); haveExtents = true; + + numColorChannels = shapeIndex.getC(); } OmeZarrBlockResolution resolution = new OmeZarrBlockResolution(dataset, idx, chunkSizeXYZ, voxelSize, resolutionMicrometers); @@ -139,11 +143,15 @@ private OmeZarrBlockTileSource init(OmeZarrReaderProgressObserver progressObserv } completionObserver.complete(this); - }); + // }); return this; } + public int getNumColorChannels() { + return numColorChannels; + } + public BoundingBox3d getBoundingBox3d() { return boundingBox3d; } @@ -214,11 +222,6 @@ ConstVector3 getBlockSize(OmeZarrBlockResolution resolution) { } public Texture3d loadBrick(OmeZarrBlockTileKey tile, int colorChannel) { - // setColorChannelIndex(colorChannel); - return loadBrick(tile); - } - - public Texture3d loadBrick(OmeZarrBlockTileKey tile) { - return tile.loadBrick(); + return tile.loadBrick(colorChannel); } } diff --git a/modules/HortaTracer/src/main/java/org/janelia/horta/blocks/OmeZarrTileCache.java b/modules/HortaTracer/src/main/java/org/janelia/horta/blocks/OmeZarrTileCache.java index 88321253c..1ba03b262 100644 --- a/modules/HortaTracer/src/main/java/org/janelia/horta/blocks/OmeZarrTileCache.java +++ b/modules/HortaTracer/src/main/java/org/janelia/horta/blocks/OmeZarrTileCache.java @@ -8,9 +8,11 @@ public class OmeZarrTileCache extends BasicTileCache { private OmeZarrBlockTileSource source; + private int colorChannel; - public OmeZarrTileCache(OmeZarrBlockTileSource source) { + public OmeZarrTileCache(OmeZarrBlockTileSource source, int colorChannel) { this.source = source; + this.colorChannel = colorChannel; } public void setSource(OmeZarrBlockTileSource source) { @@ -20,7 +22,7 @@ public void setSource(OmeZarrBlockTileSource source) { @Override LoadRunner getLoadRunner() { return key -> { - final OmeZarrBlockLoadRunner loader = new OmeZarrBlockLoadRunner(source, key); + final OmeZarrBlockLoadRunner loader = new OmeZarrBlockLoadRunner(source, key, colorChannel); loader.run(); return loader.blockActor; };