Skip to content

Commit

Permalink
Update document for GDS usage
Browse files Browse the repository at this point in the history 
Signed-off-by: Gigon Bae <gbae@nvidia.com>
  • Loading branch information
@gigony
gigony committed Mar 9, 2022
1 parent d59d9c9 commit 8e80b5b
Showing 1 changed file with 39 additions and 41 deletions.
80 changes: 39 additions & 41 deletions notebooks/Accessing_File_with_GDS.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -75,7 +75,7 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 2,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -103,14 +103,16 @@
"from cucim.clara.filesystem import CuFileDriver\n",
"\n",
"fno = os.open( \"input/image.tif\", os.O_RDONLY | os.O_DIRECT)\n",
"fno2 = os.dup(fno) \n",
"fno2 = os.dup(fno)\n",
"\n",
"fd = CuFileDriver(fno, False)\n",
"fd.close()\n",
"os.close(fno)\n",
"\n",
"# Do not use GDS even when GDS can be supported for the file.\n",
"fd2 = CuFileDriver(fno2, True)\n",
"fd2.close()\n",
"os.close(fno2)\n",
"\n",
"help(CuFileDriver.__init__)"
]
Expand Down Expand Up @@ -153,7 +155,7 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 3,
"metadata": {},
"outputs": [
{
Expand All @@ -162,7 +164,7 @@
"True"
]
},
"execution_count": 2,
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -171,11 +173,11 @@
"import cucim.clara.filesystem as fs\n",
"\n",
"fd = fs.open(\"input/image.tif\", \"r\")\n",
"fs.close(fd)\n",
"fs.close(fd) # same with fd.close()\n",
"\n",
"# Open file without using GDS\n",
"fd2 = fs.open(\"input/image.tif\", \"rp\")\n",
"fs.close(fd2)\n"
"fs.close(fd2) # same with fd2.close()\n"
]
},
{
Expand Down Expand Up @@ -203,7 +205,7 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 4,
"metadata": {},
"outputs": [
{
Expand All @@ -215,18 +217,9 @@
"torch_arr cnt: 7 content: tensor([104, 105, 106, 107, 108, 109, 110, 108, 109, 110], dtype=torch.uint8)\n",
"output.raw cnt: 10 content: [0, 0, 0, 0, 0, 104, 105, 106, 107, 108, 109, 110, 108, 109, 110]\n",
"\n",
"np_arr cnt: 10 content: [ 0 0 0 0 0 104 105 106 107 108]\n",
"np_arr cnt: 10 content: [ 0 0 0 0 0 104 105 106 107 108]\n"
]
},
{
"data": {
"text/plain": [
"True"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
Expand All @@ -253,11 +246,13 @@
"read_count = fd.pread(torch_arr.data_ptr(), 10, 3) # read 10 bytes starting from file offset 3\n",
"print(\"{:10} cnt: {} content: {}\".format(\"torch_arr\", read_count, torch_arr))\n",
"fd.close()\n",
"os.close(fno)\n",
"\n",
"fno = os.open(\"output.raw\", os.O_RDWR | os.O_CREAT | os.O_TRUNC)\n",
"fd = CuFileDriver(fno)\n",
"write_count = fd.pwrite(np_arr, 10, 5) # write 10 bytes from np_array to file starting from offset 5\n",
"fd.close()\n",
"os.close(fno)\n",
"print(\"{:10} cnt: {} content: {}\".format(\"output.raw\", write_count, list(open(\"output.raw\", \"rb\").read())))\n",
"\n",
"\n",
Expand All @@ -266,7 +261,12 @@
"fd = fs.open(\"output.raw\", \"r\")\n",
"read_count = fs.pread(fd, np_arr, 10, 0) # read 10 bytes starting from offset 0\n",
"print(\"{:10} cnt: {} content: {}\".format(\"np_arr\", read_count, np_arr))\n",
"fs.close(fd) # same with fd.close()"
"fs.close(fd) # same with fd.close()\n",
"\n",
"# Using 'with' statement\n",
"with fs.open(\"output.raw\", \"r\") as fd:\n",
" read_count = fd.pread(np_arr, 10, 0) # read 10 bytes starting from offset 0\n",
" print(\"{:10} cnt: {} content: {}\".format(\"np_arr\", read_count, np_arr))\n"
]
},
{
Expand All @@ -281,7 +281,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 5,
"metadata": {},
"outputs": [
{
Expand All @@ -290,22 +290,13 @@
"text": [
"np_arr cnt: 8 content: [ 1 2 101 102 103 104 105 106 107 108]\n",
"cp_arr cnt: 10 content: [ 0 0 0 0 0 104 105 106 107 108]\n",
"torch_arr cnt: 7 content: tensor([104, 105, 106, 107, 108, 109, 110, 0, 0, 0], device='cuda:0',\n",
"torch_arr cnt: 7 content: tensor([104, 105, 106, 107, 108, 109, 110, 108, 109, 110], device='cuda:0',\n",
" dtype=torch.uint8)\n",
"output.raw cnt: 10 content: [0, 0, 0, 0, 0, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110]\n",
"\n",
"cp_arr cnt: 10 content: [ 0 0 0 0 0 104 105 106 107 108]\n"
"cp_arr cnt: 10 content: [ 0 0 0 0 0 101 102 103 104 105]\n",
"np_arr cnt: 10 content: [ 0 0 0 0 0 101 102 103 104 105]\n"
]
},
{
"data": {
"text/plain": [
"True"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
Expand Down Expand Up @@ -337,24 +328,31 @@
"read_count = fd.pread(torch_arr, 10, 3) # read 10 bytes starting from offset 3\n",
"print(\"{:20} cnt: {} content: {}\".format(\"torch_arr\", read_count, torch_arr))\n",
"fd.close()\n",
"os.close(fno)\n",
"\n",
"fno = os.open(\"output.raw\", os.O_RDWR | os.O_CREAT | os.O_TRUNC)\n",
"fd = CuFileDriver(fno)\n",
"write_count = fd.pwrite(cp_arr, 10, 5) # write 10 bytes from np_array to file starting from offset 5\n",
"fd.close()\n",
"os.close(fno)\n",
"print(\"{:20} cnt: {} content: {}\".format(\"output.raw\", write_count, list(open(\"output.raw\", \"rb\").read())))\n",
"\n",
"print()\n",
"# Using filesystem package\n",
"fd = fs.open(\"output.raw\", \"r\")\n",
"read_count = fs.pread(fd, cp_arr, 10, 0) # read 10 bytes starting from offset 0\n",
"print(\"{:20} cnt: {} content: {}\".format(\"cp_arr\", read_count, np_arr))\n",
"fs.close(fd) # same with fd.close()"
"print(\"{:20} cnt: {} content: {}\".format(\"cp_arr\", read_count, cp_arr))\n",
"fs.close(fd) # same with fd.close()\n",
"\n",
"# Using 'with' statement\n",
"with fs.open(\"output.raw\", \"r\") as fd:\n",
" read_count = fd.pread(cp_arr, 10, 0) # read 10 bytes starting from offset 0\n",
" print(\"{:10} cnt: {} content: {}\".format(\"np_arr\", read_count, cp_arr))\n"
]
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 6,
"metadata": {},
"outputs": [
{
Expand All @@ -366,10 +364,10 @@
" 'stream': 1,\n",
" 'version': 3,\n",
" 'strides': None,\n",
" 'data': (140035445751808, False)}"
" 'data': (139779203137536, False)}"
]
},
"execution_count": 5,
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -380,7 +378,7 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 7,
"metadata": {},
"outputs": [
{
Expand All @@ -389,11 +387,11 @@
"{'typestr': '|u1',\n",
" 'shape': (10,),\n",
" 'strides': None,\n",
" 'data': (140035106013184, False),\n",
" 'data': (139776277610496, False),\n",
" 'version': 2}"
]
},
"execution_count": 6,
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
Expand Down Expand Up @@ -490,7 +488,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.10"
"version": "3.9.10"
}
},
"nbformat": 4,
Expand Down

0 comments on commit 8e80b5b

Please sign in to comment.