Skip to content

Commit

Permalink
add tests for upload/download
Browse files Browse the repository at this point in the history
  • Loading branch information
isabelizimm committed Dec 13, 2024
1 parent c5a731f commit d0ad777
Show file tree
Hide file tree
Showing 4 changed files with 19 additions and 10 deletions.
9 changes: 6 additions & 3 deletions pins/boards.py
Original file line number Diff line number Diff line change
Expand Up @@ -246,7 +246,11 @@ def _pin_store(
object_name = _p.name[:_base_len]
else:
# multifile upload, keep list of filenames
object_name = x
object_name = []
for file in x:
_p = Path(file)
# _base_len = len(_p.name) - len("".join(_p.suffixes))
object_name.append(_p.name) # [:_base_len])
else:
object_name = None

Expand Down Expand Up @@ -375,12 +379,12 @@ def pin_download(self, name, version=None, hash=None) -> Sequence[str]:
if hash is not None:
raise NotImplementedError("TODO: validate hash")

# Check that only a single file name was given
fnames = [meta.file] if isinstance(meta.file, str) else meta.file
pin_type = meta.type

if len(fnames) > 1 and pin_type in REQUIRES_SINGLE_FILE:
raise ValueError("Cannot load data when more than 1 file")

pin_name = self.path_to_pin(name)
files = []

Expand Down Expand Up @@ -698,7 +702,6 @@ def _create_meta(
p_obj = str(Path(pin_dir_path) / name)
else:
p_obj = str(Path(pin_dir_path) / object_name)

# file is saved locally in order to hash, calc size
file_names = save_data(x, p_obj, type, apply_suffix)

Expand Down
4 changes: 2 additions & 2 deletions pins/drivers.py
Original file line number Diff line number Diff line change
Expand Up @@ -187,9 +187,9 @@ def save_data(
import shutil

if isinstance(obj, list):
for f, j in zip(obj, final_name):
for file, final in zip(obj, final_name):
with contextlib.suppress(shutil.SameFileError):
shutil.copyfile(str(f), j)
shutil.copyfile(str(file), final)
return obj
# ignore the case where the source is the same as the target
else:
Expand Down
9 changes: 6 additions & 3 deletions pins/tests/test_boards.py
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,7 @@ def test_board_pin_upload_path_list(board_with_cache, tmp_path):
def test_board_pin_download_filename_multifile(board_with_cache, tmp_path):
# create and save data
df = pd.DataFrame({"x": [1, 2, 3]})
print(tmp_path)

path1, path2 = tmp_path / "data1.csv", tmp_path / "data2.csv"
df.to_csv(path1, index=False)
df.to_csv(path2, index=False)
Expand All @@ -245,8 +245,11 @@ def test_board_pin_download_filename_multifile(board_with_cache, tmp_path):
assert meta.type == "file"
assert meta.file == ["data1.csv", "data2.csv"]

# (pin_path,) = board_with_cache.pin_download("cool_pin")
# assert Path(pin_path).name == "data.csv"
pin_path = board_with_cache.pin_download("cool_pin")

assert len(pin_path) == 2
assert Path(pin_path[0]).name == "data1.csv"
assert Path(pin_path[1]).name == "data2.csv"


def test_board_pin_write_rsc_index_html(board, tmp_path: Path, snapshot):
Expand Down
7 changes: 5 additions & 2 deletions pins/versions.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,8 +105,11 @@ def from_files(
created = datetime.now()

if len(hashes) > 1:
hashes = [str(hash(tuple(hashes)))]
# raise NotImplementedError("Only 1 file may be currently be hashed")
# Combine the hashes into a single string
combined_hashes = "".join(hashes)

# Create an xxh64 hash of the combined string
hashes = xxh64(combined_hashes).hexdigest()

return cls(created, hashes[0])

Expand Down

0 comments on commit d0ad777

Please sign in to comment.