Skip to content

Commit

Permalink
Merge pull request #4 from maxrjones/object-store-update
Browse files Browse the repository at this point in the history 
Use new ByteRequest syntax and raise NotImplementedError on pickling
  • Loading branch information
@kylebarron
kylebarron authored Jan 13, 2025
2 parents 3aa3578 + 315e22e commit 264eac6
Show file tree
Hide file tree
Showing 121 changed files with 8,591 additions and 6,669 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/gpu_test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ jobs:
cache: 'pip'
- name: Install Hatch and CuPy
run: |
python -m pip install --upgrade pip
python -m pip install --upgrade pip
pip install hatch
- name: Set Up Hatch Env
run: |
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/releases.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ jobs:

- name: Install PyBuild
run: |
python -m pip install --upgrade pip
python -m pip install --upgrade pip
pip install hatch
- name: Build wheel and sdist
run: hatch build
Expand Down Expand Up @@ -55,7 +55,7 @@ jobs:
with:
name: releases
path: dist
- uses: pypa/gh-action-pypi-publish@v1.12.2
- uses: pypa/gh-action-pypi-publish@v1.12.3
with:
user: __token__
password: ${{ secrets.pypi_password }}
Expand Down
55 changes: 52 additions & 3 deletions .github/workflows/test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,15 +52,20 @@ jobs:
cache: 'pip'
- name: Install Hatch
run: |
python -m pip install --upgrade pip
python -m pip install --upgrade pip
pip install hatch
- name: Set Up Hatch Env
run: |
hatch env create test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }}
hatch env run -e test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} list-env
- name: Run Tests
run: |
hatch env run --env test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run
hatch env run --env test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run-coverage
- name: Upload coverage
uses: codecov/codecov-action@v5
with:
token: ${{ secrets.CODECOV_TOKEN }}
verbose: true # optional (default = false)

test-upstream-and-min-deps:
name: py=${{ matrix.python-version }}-${{ matrix.dependency-set }}
Expand All @@ -84,7 +89,7 @@ jobs:
cache: 'pip'
- name: Install Hatch
run: |
python -m pip install --upgrade pip
python -m pip install --upgrade pip
pip install hatch
- name: Set Up Hatch Env
run: |
Expand All @@ -93,3 +98,47 @@ jobs:
- name: Run Tests
run: |
hatch env run --env ${{ matrix.dependency-set }} run
doctests:
name: doctests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # required for hatch version discovery, which is needed for numcodecs.zarr3
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.13'
cache: 'pip'
- name: Install Hatch
run: |
python -m pip install --upgrade pip
pip install hatch
- name: Set Up Hatch Env
run: |
hatch env create doctest
hatch env run -e doctest list-env
- name: Run Tests
run: |
hatch env run --env doctest run
test-complete:
name: Test complete

needs:
[
test,
test-upstream-and-min-deps,
doctests
]
if: always()
runs-on: ubuntu-latest
steps:
- name: Check failure
if: |
contains(needs.*.result, 'failure') ||
contains(needs.*.result, 'cancelled')
run: exit 1
- name: Success
run: echo Success!
5 changes: 4 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,10 @@ coverage.xml

# Sphinx documentation
docs/_build/
docs/_autoapi
docs/api
docs/data
data
data.zip

# PyBuilder
target/
Expand Down
19 changes: 9 additions & 10 deletions .pre-commit-config.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
ci:
autoupdate_commit_msg: "chore: update pre-commit hooks"
autoupdate_schedule: "monthly"
autofix_commit_msg: "style: pre-commit fixes"
autofix_prs: false
default_stages: [pre-commit, pre-push]
default_language_version:
python: python3
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.7.3
rev: v0.8.6
hooks:
- id: ruff
args: ["--fix", "--show-fixes"]
Expand All @@ -16,23 +15,23 @@ repos:
rev: v2.3.0
hooks:
- id: codespell
args: ["-L", "ba,ihs,kake,nd,noe,nwo,te,fo,zar", "-S", "fixture"]
args: ["-L", "fo,ihs,kake,te", "-S", "fixture"]
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v5.0.0
hooks:
- id: check-yaml
- id: check-yaml
- id: trailing-whitespace
- repo: https://github.com/pre-commit/mirrors-mypy
rev: v1.13.0
rev: v1.14.1
hooks:
- id: mypy
files: src|tests
additional_dependencies:
# Package dependencies
- asciitree
- crc32c
- packaging
- donfig
- numcodecs
- numpy
- numcodecs[crc32c]
- numpy==2.1 # until https://github.com/numpy/numpy/issues/28034 is resolved
- typing_extensions
- universal-pathlib
- obstore==0.3.0-beta.8
Expand Down
49 changes: 0 additions & 49 deletions README-v3.md
View file

This file was deleted.

40 changes: 20 additions & 20 deletions bench/compress_normal.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ Line # Hits Time Per Hit % Time Line Contents
==============================================================
137 def compress(source, char* cname, int clevel, int shuffle):
138 """Compress data in a numpy array.
139
139
140 Parameters
141 ----------
142 source : array-like
Expand All @@ -30,33 +30,33 @@ Line # Hits Time Per Hit % Time Line Contents
147 Compression level.
148 shuffle : int
149 Shuffle filter.
150
150
151 Returns
152 -------
153 dest : bytes-like
154 Compressed data.
155
155
156 """
157
157
158 cdef:
159 char *source_ptr
160 char *dest_ptr
161 Py_buffer source_buffer
162 size_t nbytes, cbytes, itemsize
163 200 506 2.5 0.2 array.array char_array_template = array.array('b', [])
164 array.array dest
165
165
166 # setup source buffer
167 200 458 2.3 0.2 PyObject_GetBuffer(source, &source_buffer, PyBUF_ANY_CONTIGUOUS)
168 200 119 0.6 0.0 source_ptr = <char *> source_buffer.buf
169
169
170 # setup destination
171 200 239 1.2 0.1 nbytes = source_buffer.len
172 200 103 0.5 0.0 itemsize = source_buffer.itemsize
173 200 2286 11.4 0.8 dest = array.clone(char_array_template, nbytes + BLOSC_MAX_OVERHEAD,
174 zero=False)
175 200 129 0.6 0.0 dest_ptr = <char *> dest.data.as_voidptr
176
176
177 # perform compression
178 200 1734 8.7 0.6 if _get_use_threads():
179 # allow blosc to use threads internally
Expand All @@ -67,24 +67,24 @@ Line # Hits Time Per Hit % Time Line Contents
184 cbytes = blosc_compress(clevel, shuffle, itemsize, nbytes,
185 source_ptr, dest_ptr,
186 nbytes + BLOSC_MAX_OVERHEAD)
187
187
188 else:
189 with nogil:
190 cbytes = blosc_compress_ctx(clevel, shuffle, itemsize, nbytes,
191 source_ptr, dest_ptr,
192 nbytes + BLOSC_MAX_OVERHEAD, cname,
193 0, 1)
194
194
195 # release source buffer
196 200 616 3.1 0.2 PyBuffer_Release(&source_buffer)
197
197
198 # check compression was successful
199 200 120 0.6 0.0 if cbytes <= 0:
200 raise RuntimeError('error during blosc compression: %d' % cbytes)
201
201
202 # resize after compression
203 200 1896 9.5 0.6 array.resize(dest, cbytes)
204
204
205 200 186 0.9 0.1 return dest

*******************************************************************************
Expand All @@ -100,19 +100,19 @@ Line # Hits Time Per Hit % Time Line Contents
==============================================================
75 def decompress(source, dest):
76 """Decompress data.
77
77
78 Parameters
79 ----------
80 source : bytes-like
81 Compressed data, including blosc header.
82 dest : array-like
83 Object to decompress into.
84
84
85 Notes
86 -----
87 Assumes that the size of the destination buffer is correct for the size of
88 the uncompressed data.
89
89
90 """
91 cdef:
92 int ret
Expand All @@ -122,7 +122,7 @@ Line # Hits Time Per Hit % Time Line Contents
96 array.array source_array
97 Py_buffer dest_buffer
98 size_t nbytes
99
99
100 # setup source buffer
101 200 573 2.9 0.2 if PY2 and isinstance(source, array.array):
102 # workaround fact that array.array does not support new-style buffer
Expand All @@ -134,13 +134,13 @@ Line # Hits Time Per Hit % Time Line Contents
108 200 112 0.6 0.0 release_source_buffer = True
109 200 144 0.7 0.1 PyObject_GetBuffer(source, &source_buffer, PyBUF_ANY_CONTIGUOUS)
110 200 98 0.5 0.0 source_ptr = <char *> source_buffer.buf
111
111
112 # setup destination buffer
113 200 552 2.8 0.2 PyObject_GetBuffer(dest, &dest_buffer,
114 PyBUF_ANY_CONTIGUOUS | PyBUF_WRITEABLE)
115 200 100 0.5 0.0 dest_ptr = <char *> dest_buffer.buf
116 200 84 0.4 0.0 nbytes = dest_buffer.len
117
117
118 # perform decompression
119 200 1856 9.3 0.8 if _get_use_threads():
120 # allow blosc to use threads internally
Expand All @@ -149,12 +149,12 @@ Line # Hits Time Per Hit % Time Line Contents
123 else:
124 with nogil:
125 ret = blosc_decompress_ctx(source_ptr, dest_ptr, nbytes, 1)
126
126
127 # release buffers
128 200 754 3.8 0.3 if release_source_buffer:
129 200 326 1.6 0.1 PyBuffer_Release(&source_buffer)
130 200 165 0.8 0.1 PyBuffer_Release(&dest_buffer)
131
131
132 # handle errors
133 200 128 0.6 0.1 if ret <= 0:
134 raise RuntimeError('error during blosc decompression: %d' % ret)
10 changes: 10 additions & 0 deletions codecov.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
coverage:
status:
patch:
default:
target: auto
project:
default:
target: auto
threshold: 0.1
comment: false
1 change: 0 additions & 1 deletion data/donotdelete
View file

This file was deleted.

2 changes: 1 addition & 1 deletion docs/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ help:
.PHONY: clean
clean:
rm -rf $(BUILDDIR)/*
rm -rf $(BUILDDIR)/../_autoapi
rm -rf $(BUILDDIR)/../api

.PHONY: html
html:
Expand Down
Loading

0 comments on commit 264eac6

Please sign in to comment.