Compression Framework

This example shows the intended lifecycle:

• write a model with the fast/default compression path

• inspect the persisted compression metadata

• promote the same model in place with compress()

from pathlib import Path
import json
import tempfile

import pyarrow.parquet as pq

from parq_blockmodel import ParquetBlockModel

# sphinx_gallery_thumbnail_path = "../docs/_static/branding/parq-blockmodel-gallery-thumbnail.svg"

Create a small demo block model.

temp_dir = Path(tempfile.gettempdir()) / "compression_framework_example"
temp_dir.mkdir(parents=True, exist_ok=True)

pbm = ParquetBlockModel.create_demo_block_model(temp_dir / "compression_demo.parquet")
pbm

ParquetBlockModel(name=compression_demo, path=/tmp/compression_framework_example/compression_demo.pbm)

Inspect the active-write compression metadata.

before = pq.read_metadata(pbm.blockmodel_path).metadata or {}
json.loads(before[b"parq-blockmodel"].decode("utf-8"))

{'geometry': {'schema_version': '1.1', 'corner': [0.0, 0.0, 0.0], 'block_size': [1.0, 1.0, 1.0], 'shape': [3, 3, 3], 'origin': [0.0, 0.0, 0.0], 'axis_u': [1.0, 0.0, 0.0], 'axis_v': [0.0, 1.0, 0.0], 'axis_w': [0.0, 0.0, 1.0], 'srs': None, 'world_id_encoding': {'enabled': True, 'column': 'world_id', 'frame': 'world_xyz', 'axis_order': ['x', 'y', 'z'], 'encoding': {'type': 'morton_zorder', 'version': '1.0', 'bits_per_axis': 24, 'axis_bits': {'x': 24, 'y': 24, 'z': 16}, 'signed': False}, 'quantization': {'scale': 10.0, 'rounding': 'nearest', 'precision_decimals': 1}, 'offset': {'x': 0.0, 'y': 0.0, 'z': 0.0, 'units': 'world'}, 'range_after_offset': {'x_min': 0.0, 'x_max': 1677721.5, 'y_min': 0.0, 'y_max': 1677721.5, 'z_min': 0.0, 'z_max': 6553.5}, 'overflow_policy': 'error', 'null_policy': 'no_nulls'}}, 'compression': {'mode': 'active', 'default': {'codec': 'snappy', 'level': None}, 'columns': {}}}

Rewrite the model in place using archive compression.

pbm.compress(level=7)

ParquetBlockModel(name=compression_demo, path=/tmp/compression_framework_example/compression_demo.pbm)

Check the archived policy after the rewrite.

after = pq.read_metadata(pbm.blockmodel_path).metadata or {}
json.loads(after[b"parq-blockmodel"].decode("utf-8"))

{'geometry': {'schema_version': '1.1', 'corner': [0.0, 0.0, 0.0], 'block_size': [1.0, 1.0, 1.0], 'shape': [3, 3, 3], 'origin': [0.0, 0.0, 0.0], 'axis_u': [1.0, 0.0, 0.0], 'axis_v': [0.0, 1.0, 0.0], 'axis_w': [0.0, 0.0, 1.0], 'srs': None, 'world_id_encoding': {'enabled': True, 'column': 'world_id', 'frame': 'world_xyz', 'axis_order': ['x', 'y', 'z'], 'encoding': {'type': 'morton_zorder', 'version': '1.0', 'bits_per_axis': 24, 'axis_bits': {'x': 24, 'y': 24, 'z': 16}, 'signed': False}, 'quantization': {'scale': 10.0, 'rounding': 'nearest', 'precision_decimals': 1}, 'offset': {'x': 0.0, 'y': 0.0, 'z': 0.0, 'units': 'world'}, 'range_after_offset': {'x_min': 0.0, 'x_max': 1677721.5, 'y_min': 0.0, 'y_max': 1677721.5, 'z_min': 0.0, 'z_max': 6553.5}, 'overflow_policy': 'error', 'null_policy': 'no_nulls'}}, 'compression': {'mode': 'archive', 'default': {'codec': 'zstd', 'level': 7}, 'columns': {}}}