Update Tensorstore driver to support structured dtype and add support for Zarr v2 and v3 backend

.devcontainer/Dockerfile

@@ -1,5 +1,6 @@
 FROM mcr.microsoft.com/devcontainers/cpp:dev-ubuntu24.04
 
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 
 ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
 
@@ -12,7 +13,6 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \
 
 ENV DEBIAN_FRONTEND=noninteractive 
 
-ARG POETRY_VERSION=1.2.2
 ARG CMAKE_MAJOR=3.24
 ARG CMAKE_VERSION=3.24.2
 ARG HYPERFINE=1.15.0
@@ -45,27 +45,22 @@ RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | \
   apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
 RUN apt-get update && apt-get install -y  google-cloud-cli
 
-RUN apt update -y \
-  && apt install software-properties-common -y \
-  && add-apt-repository ppa:deadsnakes/ppa
-
-# Install apt dependencies
+# Install apt dependencies (Python 3.12 is native on Ubuntu 24.04)
 RUN apt update -y \
   && apt install -y --no-install-recommends \
-  python3.10-full \
+  python3-full \
+  python3-pip \
   && rm -rf /var/lib/apt/lists/* \
   && apt clean
 
-RUN mkdir -p /venv
-RUN python3.10 -m venv /venv
+# Create virtual environment and sync dependencies
+RUN uv venv /venv
+ENV UV_PROJECT_ENVIRONMENT=/venv
 ENV PATH="/venv/bin:$PATH"
 
-RUN pip install wheel yapf cpplint==${CPPLINT_VERSION} zarr xarray
-
-RUN pip install \
-  "poetry==$POETRY_VERSION" \
-  && poetry config virtualenvs.create false
-# if poetry doesn't pick up a deps this for some reason pip install here
+RUN mkdir -p /tmp/devcontainer
+COPY .devcontainer/pyproject.toml .devcontainer/uv.lock /tmp/devcontainer/
+RUN uv sync --project /tmp/devcontainer --frozen --all-groups --all-extras
 
 # if we want the user to be able to do pip install etc.
 RUN chmod -R 777 /venv

.devcontainer/pyproject.toml

@@ -0,0 +1,78 @@
+# Copyright 2024 TGS
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[project]
+name = "mdio-cpp-dev"
+version = "0.1.0"
+description = "MDIO C++ library development environment"
+requires-python = ">=3.12,<3.13"
+
+# multidimio[cloud] pulls in zarr, xarray, numpy, and cloud storage backends
+dependencies = [
+    "multidimio[cloud]==1.1.4.dev1781540584",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0",
+    "black>=24.3",
+    "mypy>=1.5",
+    "isort>=5.0",
+    "ruff>=0.8",
+    "yapf>=0.40",
+    "cpplint==1.6.1",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[dependency-groups]
+dev = [
+    "pytest>=7.0",
+    "black>=24.3",
+    "mypy>=1.5",
+    "isort>=5.0",
+    "ruff>=0.8",
+    "yapf>=0.40",
+    "cpplint==1.6.1",
+]
+
+[tool.uv]
+package = false
+prerelease = "allow"
+
+[[tool.uv.index]]
+name = "testpypi"
+url = "https://test.pypi.org/simple/"
+explicit = true
+
+[tool.uv.sources]
+multidimio = { index = "testpypi" }
+
+[tool.isort]
+profile = "black"
+
+[tool.black]
+line-length = 79
+
+[tool.ruff]
+line-length = 79
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP"]
+
+[tool.mypy]
+python_version = "3.12"
+plugins = []

.github/workflows/cmake_build.yaml

@@ -30,18 +30,17 @@ jobs:
           cxx-compiler: ${{ matrix.compiler }}
       - name: Build tests
         run: cd build && pwd && make -j
-      - name: Install Python 3.10
+      - name: Install UV
         run: |
-          sudo add-apt-repository -y ppa:deadsnakes/ppa
-          sudo apt update -y
-          sudo apt install -y python3.10 python3.10-venv python3.10-dev
-          sudo update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
       - name: Install test dependencies
         run: |
-          python3.10 -m pip install --upgrade pip setuptools wheel --no-input
-          python3.10 -m pip install yapf cpplint zarr xarray --no-input
+          uv sync --project .devcontainer --frozen
+        # No system python environment bypass! uv will automatically handle creating a local .venv in .devcontainer folder.
       - name: Run tests
         run: |
+          source .devcontainer/.venv/bin/activate
           cd build/mdio/ \
           && ./mdio_acceptance_test \
           && ./mdio_variable_test \
@@ -52,4 +51,8 @@ jobs:
           && ./mdio_utils_trim_test \
           && ./mdio_utils_delete_test \
           && ./mdio_variable_collection_test \
-          && ./mdio_coordinate_selector_test
+          && ./mdio_coordinate_selector_test \
+          && ./mdio_header_variable_test \
+          && ./mdio_zarr_test \
+          && ./mdio_gcs_test \
+          && ./mdio_s3_test

CMakeLists.txt

@@ -75,6 +75,8 @@ include(CMakeDependentOption)
 # provides a method to download dependencies:
 include(FetchContent)
 include(CMakeHelpers/MdioHelpers)
+# optional code coverage support:
+include(CodeCoverage)
 
 list(APPEND mdio_DEFAULT_COPTS
     "-Wno-deprecated-declarations"
@@ -108,6 +110,7 @@ if(NOT CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
   set(mdio_INTERNAL_DEPS
     tensorstore::driver_array
     tensorstore::driver_zarr
+    tensorstore::driver_zarr3
     tensorstore::driver_json
     tensorstore::kvstore_file
     tensorstore::stack

cmake/CMakeHelpers/MdioHelpers.cmake

@@ -297,6 +297,12 @@ function(mdio_cc_test)
     PRIVATE ${mdio_CC_TEST_COPTS}
   )
 
+  # Add coverage flags only to mdio test targets (not dependencies)
+  if(MDIO_ENABLE_COVERAGE)
+    target_compile_options(${_NAME} PRIVATE ${MDIO_COVERAGE_COMPILE_FLAGS})
+    target_link_options(${_NAME} PRIVATE ${MDIO_COVERAGE_LINK_FLAGS})
+  endif()
+
   target_link_libraries(${_NAME}
     PUBLIC ${mdio_CC_TEST_DEPS}
     PRIVATE ${mdio_CC_TEST_LINKOPTS}

cmake/CodeCoverage.cmake

@@ -0,0 +1,166 @@
+# Copyright 2024 TGS
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# ==============================================================================
+# Code Coverage Support (gcov + lcov)
+# ==============================================================================
+#
+# Coverage instrumentation is applied ONLY to mdio test targets, not to
+# dependencies like Tensorstore. This keeps build times and test execution fast.
+#
+# REQUIRED CMAKE FLAGS:
+#   -DMDIO_ENABLE_COVERAGE=ON    Enable coverage instrumentation
+#   -DCMAKE_BUILD_TYPE=Debug     Recommended for meaningful line coverage
+#
+# REQUIRED SYSTEM TOOLS:
+#   - gcov   (usually bundled with GCC)
+#   - lcov   (install via: apt install lcov / brew install lcov)
+#   - genhtml (included with lcov)
+#
+# USAGE:
+#   1. Configure and build with coverage enabled:
+#        cd build
+#        cmake .. -DMDIO_ENABLE_COVERAGE=ON -DCMAKE_BUILD_TYPE=Debug
+#        make
+#
+#   2. Run tests to generate coverage data (coverage accumulates across runs):
+#        ./mdio/mdio_variable_test                              # single test
+#        ./mdio/mdio_variable_test && ./mdio/mdio_dataset_test  # multiple tests
+#        ctest                                                  # all registered tests
+#
+#   3. Generate HTML coverage report:
+#        make coverage-capture
+#
+#   4. View the report:
+#        Open build/coverage_report/index.html in a browser
+#
+# AVAILABLE TARGETS:
+#   make coverage         - Reset counters, capture data, generate report
+#   make coverage-capture - Capture current data and generate report (no reset)
+#   make coverage-reset   - Zero out all coverage counters
+#
+# ==============================================================================
+
+option(MDIO_ENABLE_COVERAGE "Enable code coverage instrumentation (requires GCC or Clang)" OFF)
+
+if(MDIO_ENABLE_COVERAGE)
+  message(STATUS "Code coverage enabled")
+
+  # Check for supported compiler
+  if(NOT CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
+    message(FATAL_ERROR "Code coverage requires GCC or Clang compiler")
+  endif()
+
+  # Coverage compiler/linker flags - exported for use in MdioHelpers.cmake
+  # These are applied only to mdio targets, not to dependencies like Tensorstore
+  set(MDIO_COVERAGE_COMPILE_FLAGS "-fprofile-arcs;-ftest-coverage" CACHE INTERNAL "Coverage compile flags")
+  set(MDIO_COVERAGE_LINK_FLAGS "--coverage" CACHE INTERNAL "Coverage link flags")
+
+  # Find required tools
+  find_program(LCOV_PATH lcov)
+  find_program(GENHTML_PATH genhtml)
+
+  if(NOT LCOV_PATH)
+    message(WARNING "lcov not found - coverage report generation will not be available")
+  endif()
+
+  if(NOT GENHTML_PATH)
+    message(WARNING "genhtml not found - coverage report generation will not be available")
+  endif()
+
+  # Create coverage report target if tools are available
+  if(LCOV_PATH AND GENHTML_PATH)
+    # Custom target to generate coverage report
+    add_custom_target(coverage
+      COMMENT "Generating code coverage report..."
+
+      # Clear previous coverage data
+      COMMAND ${LCOV_PATH} --directory ${CMAKE_BINARY_DIR} --zerocounters
+
+      # Run all tests (ctest must be run separately or you can uncomment below)
+      # COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure
+
+      # Capture coverage data
+      COMMAND ${LCOV_PATH} 
+        --directory ${CMAKE_BINARY_DIR}
+        --capture
+        --output-file ${CMAKE_BINARY_DIR}/coverage.info
+        --ignore-errors mismatch,negative
+
+      # Remove coverage data for external dependencies
+      COMMAND ${LCOV_PATH}
+        --remove ${CMAKE_BINARY_DIR}/coverage.info
+        '/usr/*'
+        '${CMAKE_BINARY_DIR}/_deps/*'
+        '*/googletest/*'
+        '*/gtest/*'
+        '*/gmock/*'
+        '*_test.cc'
+        --output-file ${CMAKE_BINARY_DIR}/coverage.info
+        --ignore-errors unused,negative
+
+      # Generate HTML report
+      COMMAND ${GENHTML_PATH}
+        ${CMAKE_BINARY_DIR}/coverage.info
+        --output-directory ${CMAKE_BINARY_DIR}/coverage_report
+        --title "MDIO Code Coverage"
+        --legend
+        --show-details
+
+      WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+    )
+
+    # Target to just capture coverage (without zeroing first)
+    add_custom_target(coverage-capture
+      COMMENT "Capturing coverage data..."
+
+      COMMAND ${LCOV_PATH} 
+        --directory ${CMAKE_BINARY_DIR}
+        --capture
+        --output-file ${CMAKE_BINARY_DIR}/coverage.info
+        --ignore-errors mismatch,negative
+
+      COMMAND ${LCOV_PATH}
+        --remove ${CMAKE_BINARY_DIR}/coverage.info
+        '/usr/*'
+        '${CMAKE_BINARY_DIR}/_deps/*'
+        '*/googletest/*'
+        '*/gtest/*'
+        '*/gmock/*'
+        '*_test.cc'
+        --output-file ${CMAKE_BINARY_DIR}/coverage.info
+        --ignore-errors unused,negative
+
+      COMMAND ${GENHTML_PATH}
+        ${CMAKE_BINARY_DIR}/coverage.info
+        --output-directory ${CMAKE_BINARY_DIR}/coverage_report
+        --title "MDIO Code Coverage"
+        --legend
+        --show-details
+
+      WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+    )
+
+    # Target to reset coverage counters
+    add_custom_target(coverage-reset
+      COMMENT "Resetting coverage counters..."
+      COMMAND ${LCOV_PATH} --directory ${CMAKE_BINARY_DIR} --zerocounters
+      WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+    )
+
+    message(STATUS "Coverage targets available: 'coverage', 'coverage-capture', 'coverage-reset'")
+    message(STATUS "Coverage report will be generated at: ${CMAKE_BINARY_DIR}/coverage_report/index.html")
+  endif()
+endif()
+

cmake/FindEXT_TENSORSTORE.cmake

@@ -3,11 +3,11 @@ IF ( NOT TARGET tensorstore )
 
  include(FetchContent)
 
- FetchContent_Declare(
-  tensorstore
-  GIT_REPOSITORY
-  https://github.com/brian-michell/tensorstore.git
-  GIT_TAG v0.1.63_latest
+FetchContent_Declare(
+ tensorstore
+ GIT_REPOSITORY
+ https://github.com/google/tensorstore.git
+ GIT_TAG 917edaf341217f750b7bd3b8db6e75e6db64eab8
 )
 
 FetchContent_MakeAvailable(tensorstore)

cmake/Findnlohmann_json_schema_validator.cmake

@@ -10,7 +10,7 @@ if (NOT TARGET nlohmann_json_schema_validator)
   FetchContent_Declare(
     nlohmann_json_schema_validator
     GIT_REPOSITORY https://github.com/pboettch/json-schema-validator.git
-    GIT_TAG 2.2.0
+    GIT_TAG 2.4.0
   )
 
   if(NOT BUILD_VALIDATOR)