Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
47 commits
Select commit Hold shift + click to select a range
eb9e7c5
zstd debug
TodicaIonut Sep 29, 2025
55878c0
Merge branch 'AcademySoftwareFoundation:main' into New-Zstd-compressor
TodicaIonut Oct 13, 2025
fcc78aa
c-blosc2
TodicaIonut Oct 13, 2025
a3c984e
Zstd Compressor
TodicaIonut Oct 13, 2025
ca0ddb7
Add zstd
TodicaIonut Oct 14, 2025
5b22213
install Blosc2
TodicaIonut Oct 14, 2025
0004967
error Add zstd
TodicaIonut Oct 14, 2025
5760525
Fix
TodicaIonut Oct 14, 2025
049c635
openexr error code
TodicaIonut Oct 14, 2025
4f907e8
ZSTD part
TodicaIonut Oct 14, 2025
0ee5700
Zstd build
TodicaIonut Oct 14, 2025
5a10de3
OpenEXR blosc2
TodicaIonut Oct 14, 2025
400de68
CMakeLists blosc2
TodicaIonut Oct 14, 2025
e9d09ee
CMakeLists blosc2
TodicaIonut Oct 14, 2025
7e49cd1
zstd compression level
TodicaIonut Oct 14, 2025
de5d0ae
Add ZSTD
TodicaIonut Oct 14, 2025
7f481ff
Compression ZSTD
TodicaIonut Oct 14, 2025
9eb9418
blosc zstd
TodicaIonut Oct 14, 2025
ea09170
zstd Compression Level
TodicaIonut Oct 14, 2025
bca4954
zstd Compression Level
TodicaIonut Oct 14, 2025
9259457
zstd Compression Level
TodicaIonut Oct 15, 2025
29e33e8
set Zstd Level
TodicaIonut Oct 15, 2025
2eecde0
zstd level
TodicaIonut Oct 15, 2025
a4b8174
zstd level
TodicaIonut Oct 15, 2025
500321c
context zstd level
TodicaIonut Oct 16, 2025
ad2165d
zstd compression level
TodicaIonut Oct 16, 2025
c4a5f20
get zstd compression level test
TodicaIonut Oct 16, 2025
d5c8494
zstd level test
TodicaIonut Oct 16, 2025
9f63ad4
Deep ScanLine test zstd
TodicaIonut Oct 16, 2025
6c31833
test Large ZSTD
TodicaIonut Oct 16, 2025
91db8f2
zstd Compression Level
TodicaIonut Oct 16, 2025
661cda2
exrmetrics zstd
TodicaIonut Oct 16, 2025
f9b2efc
zstd
TodicaIonut Oct 17, 2025
465df39
zstd
TodicaIonut Oct 17, 2025
8d12714
zstd
TodicaIonut Oct 17, 2025
c376554
zstd compression level
TodicaIonut Oct 17, 2025
23226f8
test ZSTD
TodicaIonut Oct 17, 2025
9471de7
test Deep Zstd
TodicaIonut Oct 17, 2025
84aafd7
test Deep Zstd
TodicaIonut Oct 17, 2025
9a4d8a6
test ZSTD
TodicaIonut Oct 17, 2025
edfa8d2
blosc2
TodicaIonut Oct 17, 2025
b9feaa3
c-blosc2
TodicaIonut Oct 17, 2025
b3906bf
Fix Blosc2
TodicaIonut Oct 20, 2025
057703a
Fix bulid
TodicaIonut Oct 23, 2025
ea6fdef
Fix c-blosc2
TodicaIonut Oct 26, 2025
f6a1097
thread blosc2
TodicaIonut Oct 27, 2025
fd76d60
c-blosc2 2.22
TodicaIonut Nov 13, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,7 @@ cc_library(
"src/lib/OpenEXRCore/internal_win32_file_impl.h",
"src/lib/OpenEXRCore/internal_xdr.h",
"src/lib/OpenEXRCore/internal_zip.c",
"src/lib/OpenEXRCore/internal_zstd.c",
"src/lib/OpenEXRCore/memory.c",
"src/lib/OpenEXRCore/opaque.c",
"src/lib/OpenEXRCore/openexr_version.h",
Expand Down Expand Up @@ -264,12 +265,14 @@ cc_library(
":windows": [],
"//conditions:default": [
"-pthread",
"-ldl",
],
}),
visibility = ["//visibility:public"],
deps = [
"@imath",
"@openjph",
"@c-blosc2",
"@libdeflate//:deflate",
],
)
Expand Down Expand Up @@ -373,6 +376,7 @@ cc_library(
"src/lib/OpenEXR/ImfWav.cpp",
"src/lib/OpenEXR/ImfZip.cpp",
"src/lib/OpenEXR/ImfZipCompressor.cpp",
"src/lib/OpenEXR/ImfZstdCompressor.cpp",
],
hdrs = [
"src/lib/Iex/IexConfig.h",
Expand Down Expand Up @@ -493,6 +497,7 @@ cc_library(
"src/lib/OpenEXR/ImfXdr.h",
"src/lib/OpenEXR/ImfZip.h",
"src/lib/OpenEXR/ImfZipCompressor.h",
"src/lib/OpenEXR/ImfZstdCompressor.h",
"src/lib/OpenEXR/OpenEXRConfig.h",
"src/lib/OpenEXR/OpenEXRConfigInternal.h",
],
Expand All @@ -509,6 +514,7 @@ cc_library(
":windows": [],
"//conditions:default": [
"-pthread",
"-ldl",
],
}),
visibility = ["//visibility:public"],
Expand All @@ -517,6 +523,7 @@ cc_library(
":OpenEXRCore",
"@imath",
"@openjph"
"@c-blosc2",
],
)

Expand Down
1 change: 1 addition & 0 deletions MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,6 @@ bazel_dep(name = "bazel_skylib", version = "1.8.2")
bazel_dep(name = "imath", version = "3.2.1")
bazel_dep(name = "libdeflate", version = "1.24")
bazel_dep(name = "openjph", version = "0.24.1")
bazel_dep(name = "c-blosc2", version = "2.22.0")
bazel_dep(name = "platforms", version = "1.0.0")
bazel_dep(name = "rules_cc", version = "0.2.9")
1 change: 1 addition & 0 deletions cmake/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,7 @@ if(OPENEXR_INSTALL_PKG_CONFIG)
function(openexr_pkg_config_help pcinfile)
string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE)
set(LIB_SUFFIX_DASH ${OPENEXR_LIB_SUFFIX}${CMAKE_${uppercase_CMAKE_BUILD_TYPE}_POSTFIX})
set(LIB_BUILD_SUFFIX ${CMAKE_${uppercase_CMAKE_BUILD_TYPE}_POSTFIX})
if(OPENEXR_ENABLE_THREADING AND TARGET Threads::Threads)
# hrm, can't use properties as they end up as generator expressions
# which don't seem to evaluate
Expand Down
3 changes: 2 additions & 1 deletion cmake/OpenEXR.pc.in
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,13 @@ libdir=@PKG_CONFIG_INSTALL_LIBDIR@
includedir=@PKG_CONFIG_INSTALL_INCLUDEDIR@
OpenEXR_includedir=${includedir}/OpenEXR
libsuffix=@LIB_SUFFIX_DASH@
libbuildsuffix=@LIB_BUILD_SUFFIX@

Name: OpenEXR
Description: OpenEXR image library
Version: @OPENEXR_VERSION@

Libs: @exr_pthread_libs@ -L${libdir} -lOpenEXR${libsuffix} -lOpenEXRUtil${libsuffix} -lOpenEXRCore${libsuffix} -lIex${libsuffix} -lIlmThread${libsuffix}
Libs: @exr_pthread_libs@ -L${libdir} -lOpenEXR${libsuffix} -lOpenEXRUtil${libsuffix} -lOpenEXRCore${libsuffix} -lIex${libsuffix} -lIlmThread${libsuffix} -lblosc2${libbuildsuffix} -ldl
Cflags: -I${includedir} -I${OpenEXR_includedir} @exr_pthread_cflags@
Requires: Imath
Requires.private: @EXR_DEFLATE_PKGCONFIG_REQUIRES@ @EXR_OPENJPH_PKGCONFIG_REQUIRES@
Expand Down
14 changes: 8 additions & 6 deletions cmake/OpenEXRConfig.cmake.in
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,10 @@

include(CMakeFindDependencyMacro)

set(openexr_needthreads @OPENEXR_ENABLE_THREADING@)
if (openexr_needthreads)
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_dependency(Threads)
endif()
unset(openexr_needthreads)
# blosc2 needs threads, so we set it irrespective of OPENEXR_ENABLE_THREADING
# which enables threaded processing of requests.
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_dependency(Threads)

find_dependency(Imath)

Expand All @@ -22,5 +20,9 @@ if (@openjph_FOUND@)
find_dependency(openjph)
endif()

if (@c-blosc2_FOUND@)
find_dependency(c-blosc2)
endif()

include("${CMAKE_CURRENT_LIST_DIR}/@[email protected]")
check_required_components("@PROJECT_NAME@")
91 changes: 91 additions & 0 deletions cmake/OpenEXRSetup.cmake
Original file line number Diff line number Diff line change
@@ -1,6 +1,20 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright (c) Contributors to the OpenEXR Project.

function(_error_if_not_found prop var fallback)
message(STATUS "Blosc2: ${prop} ${var} '${fallback}'")
string(FIND "${var}" "-NOTFOUND" pos)
if(NOT pos EQUAL -1)
if(fallback STREQUAL "")
message(FATAL_ERROR "Blosc2: Property ${prop} not found: ${var}")
else()
string(SUBSTRING "${var}" 0 ${pos} var_name)
message(STATUS "Blosc2: Property ${prop} not found: ${var_name} falling back to '${fallback}'")
set(${var_name} "${fallback}" PARENT_SCOPE)
endif()
endif()
endfunction(_error_if_not_found)

include(GNUInstallDirs)

if(NOT "${CMAKE_PROJECT_NAME}" STREQUAL "${PROJECT_NAME}")
Expand Down Expand Up @@ -251,6 +265,83 @@ else()
endif()


#######################################
# Find or install Blosc2
#######################################

set(MINIMUM_BLOSC2_VERSION 2.11.0)
option(OPENEXR_FORCE_INTERNAL_BLOSC2 [=[Force using installed Blosc2.]=] OFF)

set(BLOSC2_REPO "https://github.com/Blosc/c-blosc2.git" CACHE STRING "Repo path for blosc2 source")
set(BLOSC2_TAG "v${MINIMUM_BLOSC2_VERSION}" CACHE STRING "Tag to use for blosc2 source repo")

# Try to find a local bloc2 install if allowed to.
if(NOT OPENEXR_FORCE_INTERNAL_BLOSC2)
message(STATUS "Blosc2: Looking for local install...")
set(CMAKE_IGNORE_PATH "${CMAKE_CURRENT_BINARY_DIR}/_deps/blosc2-src/config;${CMAKE_CURRENT_BINARY_DIR}/_deps/blosc2-build/config")
find_package(Blosc2 ${MINIMUM_BLOSC2_VERSION})
set(CMAKE_IGNORE_PATH)
endif()

if(NOT TARGET Blosc2::blosc2_static AND NOT Blosc2_FOUND)
# we didn't find a local install: let's get it from its repository.
if(OPENEXR_FORCE_INTERNAL_BLOSC2)
message(STATUS "Blosc2: forced internal, installing from ${BLOSC2_REPO} (${BLOSC2_TAG})")
else()
message(STATUS "Blosc2: no local blosc2 found, installing from ${BLOSC2_REPO} (${BLOSC2_TAG})")
endif()

# configure the blosc2 build
set(BUILD_BENCHMARKS OFF CACHE INTERNAL "no benchmarks")
set(BUILD_EXAMPLES OFF CACHE INTERNAL "no examples")
set(BUILD_FUZZERS OFF CACHE INTERNAL "no fuzzer")
set(BUILD_SHARED OFF CACHE INTERNAL "no shared library")
set(BUILD_TESTS OFF CACHE INTERNAL "no tests")

include(FetchContent)
FetchContent_Declare(Blosc2
GIT_REPOSITORY "${BLOSC2_REPO}"
GIT_TAG "${BLOSC2_TAG}"
GIT_SHALLOW ON
GIT_PROGRESS ON)

FetchContent_GetProperties(Blosc2)
if(NOT Blosc2_POPULATED)
message(STATUS "Blosc2: Downloading ${BLOSC2_TAG} from ${BLOSC2_REPO}...")
FetchContent_Populate(Blosc2)
add_subdirectory(${blosc2_SOURCE_DIR} ${blosc2_BINARY_DIR})
else()
message(STATUS "Blosc2: repo code has already been downloaded.")
endif()

# the install creates this but if we're using the library locally we
# haven't installed the header files yet, so need to extract those
# and make a variable for header only usage
if(TARGET Blosc2::blosc2_static)
message(STATUS "Blosc2: Setting up blosc directories")

get_target_property(blosc2inc Blosc2::blosc2_static INCLUDE_DIRECTORIES)
set(BLOSC2_INCLUDE_DIRS ${blosc2inc})

get_target_property(blosc2libdir Blosc2::blosc2_static BINARY_DIR)
set(BLOSC2_LIB_DIR ${blosc2libdir})
endif()
else()
message(STATUS "Blosc2: Using installed Blosc2 ${Blosc2_VERSION} from ${Blosc2_DIR}")
# local build
if(TARGET Blosc2::blosc2_static)
message(STATUS "Blosc2: Setting up installed blosc directories")

get_target_property(blosc2inc Blosc2::blosc2_static INTERFACE_INCLUDE_DIRECTORIES)
_error_if_not_found("INTERFACE_INCLUDE_DIRECTORIES" ${blosc2inc} "")
set(BLOSC2_INCLUDE_DIRS ${blosc2inc})

get_target_property(blosc2libdir Blosc2::blosc2_static BINARY_DIR)
_error_if_not_found("BINARY_DIR" ${blosc2libdir} "")
set(BLOSC2_LIB_DIR ${blosc2libdir})
endif()
endif()

#######################################
# Find or download OpenJPH
#######################################
Expand Down
7 changes: 4 additions & 3 deletions src/bin/exrmetrics/exrmetrics.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1022,9 +1022,10 @@ exrmetrics (
outHeaders[p].zipCompressionLevel () = level;
compressionSet = true;
break;
// case ZSTD_COMPRESSION :
// outHeader.zstdCompressionLevel()=level;
// break;
case ZSTD_COMPRESSION :
outHeader[p].zstdCompressionLevel () = level;
compressionSet = true;
break;
default: break;
}
}
Expand Down
2 changes: 1 addition & 1 deletion src/bin/exrmetrics/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ usageMessage (ostream& stream, const char* program_name, bool verbose = false)
" -t n Use a pool of n worker threads for processing files.\n"
" Default is single threaded (no thread pool)\n"
"\n"
" -l level set DWA or ZIP compression level\n"
" -l level set DWA, ZIP or ZSTD compression level\n"
"\n"
" -z,--compression list list of compression methods to test\n"
" ("
Expand Down
3 changes: 3 additions & 0 deletions src/lib/OpenEXR/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,8 @@ openexr_define_library(OpenEXR
ImfZip.h
ImfZipCompressor.cpp
ImfZipCompressor.h
ImfZstdCompressor.cpp
ImfZstdCompressor.h
HEADERS
ImfAcesFile.h
ImfArray.h
Expand Down Expand Up @@ -222,6 +224,7 @@ openexr_define_library(OpenEXR
ImfXdr.h
DEPENDENCIES
Imath::Imath
Blosc2::blosc2_static
OpenEXR::Config
OpenEXR::Iex
OpenEXR::IlmThread
Expand Down
3 changes: 2 additions & 1 deletion src/lib/OpenEXR/ImfCRgbaFile.h
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,8 @@ typedef struct ImfRgba ImfRgba;
#define IMF_DWAB_COMPRESSION 9
#define IMF_HTJ2K256_COMPRESSION 10
#define IMF_HTJ2K32_COMPRESSION 11
#define IMF_NUM_COMPRESSION_METHODS 12
#define IMF_ZSTD_COMPRESSION 12
#define IMF_NUM_COMPRESSION_METHODS 13

/*
** Channels; values must be the same as in Imf::RgbaChannels.
Expand Down
7 changes: 7 additions & 0 deletions src/lib/OpenEXR/ImfCompression.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,12 @@ static const CompressionDesc IdToDesc[] = {
32,
true,
false),
CompressionDesc (
"zstd",
"blosc zstd lossless compression, one scan line at a time.",
1,
false,
true),
};
// clang-format on

Expand All @@ -206,6 +212,7 @@ static const std::map<std::string, Compression> CompressionNameToId = {
{"dwab", Compression::DWAB_COMPRESSION},
{"htj2k256", Compression::HTJ2K256_COMPRESSION},
{"htj2k32", Compression::HTJ2K32_COMPRESSION},
{"zstd", Compression::ZSTD_COMPRESSION},
};

#define UNKNOWN_COMPRESSION_ID_MSG "INVALID COMPRESSION ID"
Expand Down
6 changes: 6 additions & 0 deletions src/lib/OpenEXR/ImfCompression.h
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@ enum IMF_EXPORT_ENUM Compression

HTJ2K32_COMPRESSION = 11, // High-Throughput JPEG2000 (HTJ2K), 32 scanlines

ZSTD_COMPRESSION = 12, // blosc zstd lossless compression, one scan line
// at a time.

NUM_COMPRESSION_METHODS // number of different compression methods
};

Expand Down Expand Up @@ -92,6 +95,9 @@ IMF_EXPORT void setDefaultZipCompressionLevel (int level);
/// Controls the default quality level for the DWA lossy compression
IMF_EXPORT void setDefaultDwaCompressionLevel (float level);

/// Controls the default zstd compression level used. Zstd is used for
IMF_EXPORT void setDefaultZstdCompressionLevel (int level);

OPENEXR_IMF_INTERNAL_NAMESPACE_HEADER_EXIT

#endif
2 changes: 2 additions & 0 deletions src/lib/OpenEXR/ImfCompressor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
#include "ImfPizCompressor.h"
#include "ImfPxr24Compressor.h"
#include "ImfRleCompressor.h"
#include "ImfZstdCompressor.h"
#include "ImfZipCompressor.h"
#include "ImfZip.h"

Expand Down Expand Up @@ -54,6 +55,7 @@ Compressor::Compressor (

exr_set_zip_compression_level (_ctxt, 0, hdr.zipCompressionLevel ());
exr_set_dwa_compression_level (_ctxt, 0, hdr.dwaCompressionLevel ());
exr_set_zstd_compression_level (_ctxt, 0, hdr.zstdCompressionLevel ());

exr_compression_t hdrcomp;
if (EXR_ERR_SUCCESS != exr_get_compression (_ctxt, 0, &hdrcomp))
Expand Down
6 changes: 6 additions & 0 deletions src/lib/OpenEXR/ImfContextInit.h
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,12 @@ class IMF_EXPORT_TYPE ContextInitializer
return *this;
}

ContextInitializer& setZstdLevel (int zl) noexcept
{
_initializer.zstd_level = zl;
return *this;
}

ContextInitializer& strictHeaderValidation (bool onoff) noexcept
{
setFlag (EXR_CONTEXT_FLAG_STRICT_HEADER, onoff);
Expand Down
14 changes: 14 additions & 0 deletions src/lib/OpenEXR/ImfHeader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,11 @@ struct CompressionRecord
{
exr_get_default_zip_compression_level (&zip_level);
exr_get_default_dwa_compression_quality (&dwa_level);
exr_get_default_zstd_compression_level (&zstd_level);
}
int zip_level;
float dwa_level;
int zstd_level;
};
// NB: This is extra complicated than one would normally write to
// handle scenario that seems to happen on MacOS/Windows (probably
Expand Down Expand Up @@ -693,6 +695,18 @@ Header::zipCompressionLevel () const
return retrieveCompressionRecord (this).zip_level;
}

int&
Header::zstdCompressionLevel ()
{
return retrieveCompressionRecord (this).zstd_level;
}

int
Header::zstdCompressionLevel () const
{
return retrieveCompressionRecord (this).zstd_level;
}

float&
Header::dwaCompressionLevel ()
{
Expand Down
4 changes: 4 additions & 0 deletions src/lib/OpenEXR/ImfHeader.h
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,10 @@ class IMF_EXPORT_TYPE Header
float& dwaCompressionLevel ();
IMF_EXPORT
float dwaCompressionLevel () const;
IMF_EXPORT
int& zstdCompressionLevel ();
IMF_EXPORT
int zstdCompressionLevel () const;

//-----------------------------------------------------
// Access to required attributes for multipart files
Expand Down
17 changes: 17 additions & 0 deletions src/lib/OpenEXR/ImfZstdCompressor.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
//
// SPDX-License-Identifier: BSD-3-Clause
// Copyright (c) Contributors to the OpenEXR Project.
//

#include "ImfZstdCompressor.h"

OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_ENTER

ZstdCompressor::ZstdCompressor (const Header& hdr, size_t maxScanLineSize, int scanLines)
: Compressor (hdr, EXR_COMPRESSION_ZSTD, maxScanLineSize, scanLines)
{}

ZstdCompressor::~ZstdCompressor ()
{}

OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_EXIT
Loading