Skip to content

Commit

Permalink
fix style
Browse files Browse the repository at this point in the history
  • Loading branch information
xyb committed Nov 5, 2022
1 parent b69df24 commit 5d5c0d6
Show file tree
Hide file tree
Showing 3 changed files with 25 additions and 6 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -127,3 +127,6 @@ dmypy.json

# Pyre type checker
.pyre/

# flake8
*.flac
2 changes: 1 addition & 1 deletion chunksum/cdc.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ def tail(self) -> bytes:
def reset(self) -> bytes:
"""
Reset the chunker.
>>> # not reset
>>> c = Chunker()
>>> list(c.update(b'12345').chunks)
Expand Down
26 changes: 21 additions & 5 deletions chunksum/chunksum.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
import os
import sys

from fastcdc import fastcdc
from fastcdc.const import AVERAGE_MIN
from tqdm.auto import tqdm
from tqdm.utils import CallbackIOWrapper
Expand All @@ -27,6 +26,7 @@ def iter_file_content(file, size=1024):
break
yield content


def iter_file_content_progress(file, path, size=1024):
with tqdm(total=getsize(path),
desc=path,
Expand All @@ -37,26 +37,32 @@ def iter_file_content_progress(file, path, size=1024):
fobj = CallbackIOWrapper(t.update, file, 'read')
yield from iter_file_content(fobj, size)


class ChunkSizeError(Exception):
...


class ChunkSizeTooSmall(ChunkSizeError):
...


class ChunkSizeAlign(ChunkSizeError):
...


class ChunkSize:
"""
>>> ChunkSize(1024)
ChunkSize<1024>
"""

def __init__(self, avg_bytes=AVERAGE_MIN):
if (avg_bytes) < AVERAGE_MIN:
raise ChunkSizeError('chunk size too small: {}'.format(avg_bytes))
if avg_bytes % 4 != 0:
raise ChunkSizeError('chunk size should be a multiple of 4, '
'but {} % 5 = {}'.format(avg_bytes, avg_bytes % 4))
'but {} % 5 = {}'.format(avg_bytes,
avg_bytes % 4))
self.avg = avg_bytes
self.min = avg_bytes / 4
self.max = avg_bytes * 4
Expand All @@ -82,6 +88,7 @@ def __rmul__(self, x):
"""
return ChunkSize(self.avg * x)


KILO = ChunkSize(1024) # 1KB
MEGA = KILO * 1024 # 1MB
GIGA = KILO * 1024 # 1MB
Expand All @@ -100,6 +107,7 @@ def get_chunker(size_name='', avg=1024, min=256, max=4096):
else:
return Chunker(avg, min, max)


def get_hasher(name):
name = name.lower()
if name == 'sha2':
Expand All @@ -120,14 +128,18 @@ def get_hasher(name):
else:
raise Exception('unsupported hash: {}'.format(name))


def compute_file(file, alg_name='fck4sha2', avg=0, min=0, max=0):
"""
>>> import io
>>> stream = io.BytesIO(b'abcdefgh' * 20000)
>>> result = compute_file(stream, alg_name='fck4sha2')
>>> result
[(b'\\xfb...\\xd3', 65536), (b'\\xfb...\\xd3', 65536), (b'tG...\\xfe', 28928)]
>>> for i in result:
... print(i)
(b'\\xfb...\\xd3', 65536)
(b'\\xfb...\\xd3', 65536)
(b'tG...\\xfe', 28928)
"""
if alg_name:
chunk_size_name = alg_name[2:4]
Expand Down Expand Up @@ -156,6 +168,7 @@ def gen_item(data, hasher_name):
result.append(gen_item(chunker.tail, hasher_name))
return result


def list_hash(digests, hasher_name='sha2'):
"""
>>> list_hash([b'abc', b'def'])
Expand All @@ -166,6 +179,7 @@ def list_hash(digests, hasher_name='sha2'):
h.update(plain)
return h.digest()


def format_a_result(path, result, alg_name):
"""
>>> import io
Expand All @@ -178,7 +192,7 @@ def format_a_result(path, result, alg_name):
for digest, size in result])
hasher_name = alg_name[len('fck0'):]
digest = list_hash([d for d, _ in result], hasher_name)
#alg_name = 'fastcdc-{}-{}-{}-sha256'.format(AVG, MIN, MAX)
# alg_name = 'fastcdc-{}-{}-{}-sha256'.format(AVG, MIN, MAX)
return '{} {} {}!{}'.format(digest.hex(), path, alg_name, chunks)


Expand All @@ -192,6 +206,7 @@ def walk(target, output_file, alg_name='fck4sha2'):
flush=True)
dirs.sort()


def help():
print('''Print FastCDC rolling hash chunks and checksums.
Expand Down Expand Up @@ -231,5 +246,6 @@ def main():
path, alg_name = sys.argv[1], 'fck4sha2'
walk(path, sys.stdout, alg_name)


if __name__ == '__main__':
main()

0 comments on commit 5d5c0d6

Please sign in to comment.