From 8e80b5b2deae8b5cc6eef66fc76cc81c7a4ac355 Mon Sep 17 00:00:00 2001 From: Gigon Bae Date: Tue, 8 Mar 2022 18:00:11 -0800 Subject: [PATCH] Update document for GDS usage Signed-off-by: Gigon Bae --- notebooks/Accessing_File_with_GDS.ipynb | 80 ++++++++++++------------- 1 file changed, 39 insertions(+), 41 deletions(-) diff --git a/notebooks/Accessing_File_with_GDS.ipynb b/notebooks/Accessing_File_with_GDS.ipynb index d5a5587cd..cb49f6b4b 100644 --- a/notebooks/Accessing_File_with_GDS.ipynb +++ b/notebooks/Accessing_File_with_GDS.ipynb @@ -39,7 +39,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -75,7 +75,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -103,14 +103,16 @@ "from cucim.clara.filesystem import CuFileDriver\n", "\n", "fno = os.open( \"input/image.tif\", os.O_RDONLY | os.O_DIRECT)\n", - "fno2 = os.dup(fno) \n", + "fno2 = os.dup(fno)\n", "\n", "fd = CuFileDriver(fno, False)\n", "fd.close()\n", + "os.close(fno)\n", "\n", "# Do not use GDS even when GDS can be supported for the file.\n", "fd2 = CuFileDriver(fno2, True)\n", "fd2.close()\n", + "os.close(fno2)\n", "\n", "help(CuFileDriver.__init__)" ] @@ -153,7 +155,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -162,7 +164,7 @@ "True" ] }, - "execution_count": 2, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -171,11 +173,11 @@ "import cucim.clara.filesystem as fs\n", "\n", "fd = fs.open(\"input/image.tif\", \"r\")\n", - "fs.close(fd)\n", + "fs.close(fd) # same with fd.close()\n", "\n", "# Open file without using GDS\n", "fd2 = fs.open(\"input/image.tif\", \"rp\")\n", - "fs.close(fd2)\n" + "fs.close(fd2) # same with fd2.close()\n" ] }, { @@ -203,7 +205,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -215,18 +217,9 @@ "torch_arr cnt: 7 content: tensor([104, 105, 106, 107, 108, 109, 110, 108, 109, 110], dtype=torch.uint8)\n", "output.raw cnt: 10 content: [0, 0, 0, 0, 0, 104, 105, 106, 107, 108, 109, 110, 108, 109, 110]\n", "\n", + "np_arr cnt: 10 content: [ 0 0 0 0 0 104 105 106 107 108]\n", "np_arr cnt: 10 content: [ 0 0 0 0 0 104 105 106 107 108]\n" ] - }, - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" } ], "source": [ @@ -253,11 +246,13 @@ "read_count = fd.pread(torch_arr.data_ptr(), 10, 3) # read 10 bytes starting from file offset 3\n", "print(\"{:10} cnt: {} content: {}\".format(\"torch_arr\", read_count, torch_arr))\n", "fd.close()\n", + "os.close(fno)\n", "\n", "fno = os.open(\"output.raw\", os.O_RDWR | os.O_CREAT | os.O_TRUNC)\n", "fd = CuFileDriver(fno)\n", "write_count = fd.pwrite(np_arr, 10, 5) # write 10 bytes from np_array to file starting from offset 5\n", "fd.close()\n", + "os.close(fno)\n", "print(\"{:10} cnt: {} content: {}\".format(\"output.raw\", write_count, list(open(\"output.raw\", \"rb\").read())))\n", "\n", "\n", @@ -266,7 +261,12 @@ "fd = fs.open(\"output.raw\", \"r\")\n", "read_count = fs.pread(fd, np_arr, 10, 0) # read 10 bytes starting from offset 0\n", "print(\"{:10} cnt: {} content: {}\".format(\"np_arr\", read_count, np_arr))\n", - "fs.close(fd) # same with fd.close()" + "fs.close(fd) # same with fd.close()\n", + "\n", + "# Using 'with' statement\n", + "with fs.open(\"output.raw\", \"r\") as fd:\n", + " read_count = fd.pread(np_arr, 10, 0) # read 10 bytes starting from offset 0\n", + " print(\"{:10} cnt: {} content: {}\".format(\"np_arr\", read_count, np_arr))\n" ] }, { @@ -281,7 +281,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -290,22 +290,13 @@ "text": [ "np_arr cnt: 8 content: [ 1 2 101 102 103 104 105 106 107 108]\n", "cp_arr cnt: 10 content: [ 0 0 0 0 0 104 105 106 107 108]\n", - "torch_arr cnt: 7 content: tensor([104, 105, 106, 107, 108, 109, 110, 0, 0, 0], device='cuda:0',\n", + "torch_arr cnt: 7 content: tensor([104, 105, 106, 107, 108, 109, 110, 108, 109, 110], device='cuda:0',\n", " dtype=torch.uint8)\n", "output.raw cnt: 10 content: [0, 0, 0, 0, 0, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110]\n", "\n", - "cp_arr cnt: 10 content: [ 0 0 0 0 0 104 105 106 107 108]\n" + "cp_arr cnt: 10 content: [ 0 0 0 0 0 101 102 103 104 105]\n", + "np_arr cnt: 10 content: [ 0 0 0 0 0 101 102 103 104 105]\n" ] - }, - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" } ], "source": [ @@ -337,24 +328,31 @@ "read_count = fd.pread(torch_arr, 10, 3) # read 10 bytes starting from offset 3\n", "print(\"{:20} cnt: {} content: {}\".format(\"torch_arr\", read_count, torch_arr))\n", "fd.close()\n", + "os.close(fno)\n", "\n", "fno = os.open(\"output.raw\", os.O_RDWR | os.O_CREAT | os.O_TRUNC)\n", "fd = CuFileDriver(fno)\n", "write_count = fd.pwrite(cp_arr, 10, 5) # write 10 bytes from np_array to file starting from offset 5\n", "fd.close()\n", + "os.close(fno)\n", "print(\"{:20} cnt: {} content: {}\".format(\"output.raw\", write_count, list(open(\"output.raw\", \"rb\").read())))\n", "\n", "print()\n", "# Using filesystem package\n", "fd = fs.open(\"output.raw\", \"r\")\n", "read_count = fs.pread(fd, cp_arr, 10, 0) # read 10 bytes starting from offset 0\n", - "print(\"{:20} cnt: {} content: {}\".format(\"cp_arr\", read_count, np_arr))\n", - "fs.close(fd) # same with fd.close()" + "print(\"{:20} cnt: {} content: {}\".format(\"cp_arr\", read_count, cp_arr))\n", + "fs.close(fd) # same with fd.close()\n", + "\n", + "# Using 'with' statement\n", + "with fs.open(\"output.raw\", \"r\") as fd:\n", + " read_count = fd.pread(cp_arr, 10, 0) # read 10 bytes starting from offset 0\n", + " print(\"{:10} cnt: {} content: {}\".format(\"np_arr\", read_count, cp_arr))\n" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -366,10 +364,10 @@ " 'stream': 1,\n", " 'version': 3,\n", " 'strides': None,\n", - " 'data': (140035445751808, False)}" + " 'data': (139779203137536, False)}" ] }, - "execution_count": 5, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -380,7 +378,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -389,11 +387,11 @@ "{'typestr': '|u1',\n", " 'shape': (10,),\n", " 'strides': None,\n", - " 'data': (140035106013184, False),\n", + " 'data': (139776277610496, False),\n", " 'version': 2}" ] }, - "execution_count": 6, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -490,7 +488,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.10" + "version": "3.9.10" } }, "nbformat": 4,