[Tutorial] Cache the test data in tutorial (apache#2923)

nnvm/tests/python/frontend/coreml/model_zoo/__init__.py

@@ -18,7 +18,7 @@ def get_resnet50():
 def get_cat_image():
     url = 'https://gist.githubusercontent.com/zhreshold/bcda4716699ac97ea44f791c24310193/raw/fa7ef0e9c9a5daea686d6473a62aacd1a5885849/cat.png'
     dst = 'cat.png'
-    real_dst = download_testdata(url, dst, module='coreml')
+    real_dst = download_testdata(url, dst, module='data')
     img = Image.open(real_dst).resize((224, 224))
     img = np.transpose(img, (2, 0, 1))[np.newaxis, :]
     return np.asarray(img)

nnvm/tests/python/frontend/darknet/test_forward.py

@@ -103,7 +103,7 @@ def get_darknet_output(net, img):
 
     test_image = 'dog.jpg'
     img_url = 'https://github.com/siju-samuel/darknet/blob/master/data/' + test_image   +'?raw=true'
-    img_path = download_testdata(img_url, test_image, module='darknet')
+    img_path = download_testdata(img_url, test_image, module='data')
     img = LIB.letterbox_image(LIB.load_image_color(img_path.encode('utf-8'), 0, 0), net.w, net.h)
     darknet_output = get_darknet_output(net, img)
     batch_size = 1

nnvm/tutorials/deploy_model_on_mali_gpu.py

@@ -15,6 +15,7 @@
 import nnvm.testing
 from tvm import rpc
 from tvm.contrib import util, graph_runtime as runtime
+from tvm.contrib.download import download_testdata
 
 ######################################################################
 # Build TVM Runtime on Device
@@ -81,7 +82,6 @@
 # You can found more details about this part at tutorial :ref:`tutorial-from-mxnet`.
 
 from mxnet.gluon.model_zoo.vision import get_model
-from mxnet.gluon.utils import download
 from PIL import Image
 import numpy as np
 
@@ -92,8 +92,9 @@
 # In order to test our model, here we download an image of cat and
 # transform its format.
 img_name = 'cat.png'
-download('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true', img_name)
-image = Image.open(img_name).resize((224, 224))
+img_path = download_testdata('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true',
+                             img_name, module='data')
+image = Image.open(img_path).resize((224, 224))
 
 def transform_image(image):
     image = np.array(image) - np.array([123., 117., 104.])
@@ -112,9 +113,9 @@ def transform_image(image):
                       '596b27d23537e5a1b5751d2b0481ef172f58b539/',
                       'imagenet1000_clsid_to_human.txt'])
 
-synset_name = 'synset.txt'
-download(synset_url, synset_name)
-with open(synset_name) as f:
+synset_name = 'imagenet1000_clsid_to_human.txt'
+synset_path = download_testdata(synset_url, synset_name, module='data')
+with open(synset_path) as f:
     synset = eval(f.read())
 
 ######################################################################

nnvm/tutorials/deploy_model_on_rasp.py

@@ -14,6 +14,7 @@
 import nnvm.testing
 from tvm import rpc
 from tvm.contrib import util, graph_runtime as runtime
+from tvm.contrib.download import download_testdata
 
 ######################################################################
 # .. _build-tvm-runtime-on-device:
@@ -78,7 +79,6 @@
 # You can found more details about this part at tutorial :ref:`tutorial-from-mxnet`.
 
 from mxnet.gluon.model_zoo.vision import get_model
-from mxnet.gluon.utils import download
 from PIL import Image
 import numpy as np
 
@@ -89,8 +89,9 @@
 # In order to test our model, here we download an image of cat and
 # transform its format.
 img_name = 'cat.png'
-download('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true', img_name)
-image = Image.open(img_name).resize((224, 224))
+img_path = download_testdata('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true',
+                             img_name, module='data')
+image = Image.open(img_path).resize((224, 224))
 
 def transform_image(image):
     image = np.array(image) - np.array([123., 117., 104.])
@@ -108,9 +109,9 @@ def transform_image(image):
                       '4d0b62f3d01426887599d4f7ede23ee5/raw/',
                       '596b27d23537e5a1b5751d2b0481ef172f58b539/',
                       'imagenet1000_clsid_to_human.txt'])
-synset_name = 'synset.txt'
-download(synset_url, synset_name)
-with open(synset_name) as f:
+synset_name = 'imagenet1000_clsid_to_human.txt'
+synset_path = download_testdata(synset_url, synset_name, module='data')
+with open(synset_path) as f:
     synset = eval(f.read())
 
 ######################################################################

nnvm/tutorials/deploy_ssd_mxnet.py

@@ -18,7 +18,7 @@
 from nnvm import compiler
 from nnvm.frontend import from_mxnet
 from tvm import relay
-from tvm.contrib.download import download
+from tvm.contrib.download import download_testdata
 from tvm.contrib import graph_runtime
 from mxnet.model import load_checkpoint
 
@@ -65,28 +65,24 @@
 inference_symbol_url = "https://gist.github.com/kevinthesun/c1904e900848df4548ce5dfb18c719c7/" \
                        "archive/a28c4856c827fe766aa3da0e35bad41d44f0fb26.zip"
 
-dir = "ssd_model"
-if not os.path.exists(dir):
-    os.makedirs(dir)
-model_file_path = "%s/%s" % (dir, model_file)
-test_image_path = "%s/%s" % (dir, test_image)
-inference_symbol_path = "%s/inference_model.zip" % dir
-download(model_url, model_file_path)
-download(image_url, test_image_path)
-download(inference_symbol_url, inference_symbol_path)
+model_file_path = download_testdata(model_url, model_file, module=["mxnet", "ssd_model"])
+inference_symbol_path = download_testdata(inference_symbol_url, "inference_model.zip",
+                                          module=["mxnet", "ssd_model"])
+test_image_path = download_testdata(image_url, test_image, module="data")
+model_dir = os.path.dirname(model_file_path)
 
 zip_ref = zipfile.ZipFile(model_file_path, 'r')
-zip_ref.extractall(dir)
+zip_ref.extractall(model_dir)
 zip_ref.close()
 zip_ref = zipfile.ZipFile(inference_symbol_path)
-zip_ref.extractall(dir)
+zip_ref.extractall(model_dir)
 zip_ref.close()
 
 ######################################################################
 # Convert and compile model with NNVM or Relay for CPU.
 
-sym = mx.sym.load("%s/%s/ssd_resnet50_inference.json" % (dir, inference_symbol_folder))
-_, arg_params, aux_params = load_checkpoint("%s/%s" % (dir, model_name), 0)
+sym = mx.sym.load("%s/%s/ssd_resnet50_inference.json" % (model_dir, inference_symbol_folder))
+_, arg_params, aux_params = load_checkpoint("%s/%s" % (model_dir, model_name), 0)
 
 import argparse
 parser = argparse.ArgumentParser()

nnvm/tutorials/from_coreml.py

@@ -21,19 +21,7 @@
 import coremltools as cm
 import numpy as np
 from PIL import Image
-
-def download(url, path, overwrite=False):
-    import os
-    if os.path.isfile(path) and not overwrite:
-        print('File {} existed, skip.'.format(path))
-        return
-    print('Downloading from url {} to {}'.format(url, path))
-    try:
-        import urllib.request
-        urllib.request.urlretrieve(url, path)
-    except:
-        import urllib
-        urllib.urlretrieve(url, path)
+from tvm.contrib.download import download_testdata
 
 ######################################################################
 # Load pretrained CoreML model
@@ -42,9 +30,9 @@ def download(url, path, overwrite=False):
 # provided by apple in this example
 model_url = 'https://docs-assets.developer.apple.com/coreml/models/MobileNet.mlmodel'
 model_file = 'mobilenet.mlmodel'
-download(model_url, model_file)
+model_path = download_testdata(model_url, model_file, module='coreml')
 # now you mobilenet.mlmodel on disk
-mlmodel = cm.models.MLModel(model_file)
+mlmodel = cm.models.MLModel(model_path)
 # we can load the graph as NNVM compatible model
 sym, params = nnvm.frontend.from_coreml(mlmodel)
 
@@ -54,8 +42,8 @@ def download(url, path, overwrite=False):
 # A single cat dominates the examples!
 from PIL import Image
 img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
-download(img_url, 'cat.png')
-img = Image.open('cat.png').resize((224, 224))
+img_path = download_testdata(img_url, 'cat.png', module='data')
+img = Image.open(img_path).resize((224, 224))
 #x = np.transpose(img, (2, 0, 1))[np.newaxis, :]
 image = np.asarray(img)
 image = image.transpose((2, 0, 1))
@@ -95,8 +83,8 @@ def download(url, path, overwrite=False):
                       '4d0b62f3d01426887599d4f7ede23ee5/raw/',
                       '596b27d23537e5a1b5751d2b0481ef172f58b539/',
                       'imagenet1000_clsid_to_human.txt'])
-synset_name = 'synset.txt'
-download(synset_url, synset_name)
-with open(synset_name) as f:
+synset_name = 'imagenet1000_clsid_to_human.txt'
+synset_path = download_testdata(synset_url, synset_name, module='data')
+with open(synset_path) as f:
     synset = eval(f.read())
 print('Top-1 id', top1, 'class name', synset[top1])

nnvm/tutorials/from_darknet.py

@@ -25,7 +25,7 @@
 import sys
 
 from ctypes import *
-from tvm.contrib.download import download
+from tvm.contrib.download import download_testdata
 from nnvm.testing.darknet import __darknetffi__
 
 # Model name
@@ -41,8 +41,8 @@
 CFG_URL = REPO_URL + 'cfg/' + CFG_NAME + '?raw=true'
 WEIGHTS_URL = 'https://pjreddie.com/media/files/' + WEIGHTS_NAME
 
-download(CFG_URL, CFG_NAME)
-download(WEIGHTS_URL, WEIGHTS_NAME)
+cfg_path = download_testdata(CFG_URL, CFG_NAME, module="darknet")
+weights_path = download_testdata(WEIGHTS_URL, WEIGHTS_NAME, module="darknet")
 
 # Download and Load darknet library
 if sys.platform in ['linux', 'linux2']:
@@ -55,12 +55,10 @@
     err = "Darknet lib is not supported on {} platform".format(sys.platform)
     raise NotImplementedError(err)
 
-download(DARKNET_URL, DARKNET_LIB)
+lib_path = download_testdata(DARKNET_URL, DARKNET_LIB, module="darknet")
 
-DARKNET_LIB = __darknetffi__.dlopen('./' + DARKNET_LIB)
-cfg = "./" + str(CFG_NAME)
-weights = "./" + str(WEIGHTS_NAME)
-net = DARKNET_LIB.load_network(cfg.encode('utf-8'), weights.encode('utf-8'), 0)
+DARKNET_LIB = __darknetffi__.dlopen(lib_path)
+net = DARKNET_LIB.load_network(cfg_path.encode('utf-8'), weights_path.encode('utf-8'), 0)
 dtype = 'float32'
 batch_size = 1
 
@@ -88,9 +86,9 @@
 print("Loading the test image...")
 img_url = 'https://github.com/siju-samuel/darknet/blob/master/data/' + \
           test_image + '?raw=true'
-download(img_url, test_image)
+img_path = download_testdata(img_url, test_image, "data")
 
-data = nnvm.testing.darknet.load_image(test_image, netw, neth)
+data = nnvm.testing.darknet.load_image(img_path, netw, neth)
 ######################################################################
 # Execute on TVM Runtime
 # ----------------------
@@ -150,10 +148,10 @@
 coco_url = 'https://github.com/siju-samuel/darknet/blob/master/data/' + coco_name + '?raw=true'
 font_name = 'arial.ttf'
 font_url = 'https://github.com/siju-samuel/darknet/blob/master/data/' + font_name + '?raw=true'
-download(coco_url, coco_name)
-download(font_url, font_name)
+coco_path = download_testdata(coco_url, coco_name, module='data')
+font_path = download_testdata(font_url, font_name, module='data')
 
-with open(coco_name) as f:
+with open(coco_path) as f:
     content = f.readlines()
 
 names = [x.strip() for x in content]

nnvm/tutorials/from_mxnet.py

@@ -20,30 +20,31 @@
 """
 # some standard imports
 import mxnet as mx
+import numpy as np
 import nnvm
 import tvm
-import numpy as np
+from tvm.contrib.download import download_testdata
 
 ######################################################################
 # Download Resnet18 model from Gluon Model Zoo
 # ---------------------------------------------
 # In this section, we download a pretrained imagenet model and classify an image.
 from mxnet.gluon.model_zoo.vision import get_model
-from mxnet.gluon.utils import download
 from PIL import Image
 from matplotlib import pyplot as plt
 block = get_model('resnet18_v1', pretrained=True)
+img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
 img_name = 'cat.png'
 synset_url = ''.join(['https://gist.githubusercontent.com/zhreshold/',
                       '4d0b62f3d01426887599d4f7ede23ee5/raw/',
                       '596b27d23537e5a1b5751d2b0481ef172f58b539/',
                       'imagenet1000_clsid_to_human.txt'])
-synset_name = 'synset.txt'
-download('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true', img_name)
-download(synset_url, synset_name)
-with open(synset_name) as f:
+synset_name = 'imagenet1000_clsid_to_human.txt'
+img_path = download_testdata(img_url, img_name, module='data')
+synset_path = download_testdata(synset_url, synset_name, module='data')
+with open(synset_path) as f:
     synset = eval(f.read())
-image = Image.open(img_name).resize((224, 224))
+image = Image.open(img_path).resize((224, 224))
 plt.imshow(image)
 plt.show()
 

nnvm/tutorials/from_mxnet_to_webgl.py

@@ -33,6 +33,7 @@
 
 import numpy as np
 import tvm
+from tvm.contrib.download import download_testdata
 import nnvm.compiler
 import nnvm.testing
 
@@ -106,16 +107,14 @@ def download_synset():
 
     print("Downloading synset...")
 
-    from mxnet import gluon
-
     url = "https://gist.githubusercontent.com/zhreshold/" + \
           "4d0b62f3d01426887599d4f7ede23ee5/raw/" + \
           "596b27d23537e5a1b5751d2b0481ef172f58b539/" + \
           "imagenet1000_clsid_to_human.txt"
-    file_name = "synset.txt"
+    file_name = "imagenet1000_clsid_to_human.txt"
 
-    gluon.utils.download(url, file_name)
-    with open(file_name) as f:
+    file_path = download_testdata(url, file_name, module='data')
+    with open(file_path) as f:
         synset = eval(f.read())
 
     print("- Synset downloaded!")
@@ -144,14 +143,13 @@ def download_image():
     print("Downloading cat image...")
 
     from matplotlib import pyplot as plt
-    from mxnet import gluon
     from PIL import Image
 
     url = "https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true"
     img_name = "cat.png"
 
-    gluon.utils.download(url, img_name)
-    image = Image.open(img_name).resize((224, 224))
+    img_path = download_testdata(url, img_name, module='data')
+    image = Image.open(img_path).resize((224, 224))
 
     print("- Cat image downloaded!")
 

nnvm/tutorials/from_onnx.py

@@ -18,22 +18,10 @@
 """
 import nnvm
 import tvm
+from tvm.contrib.download import download_testdata
 import onnx
 import numpy as np
 
-def download(url, path, overwrite=False):
-    import os
-    if os.path.isfile(path) and not overwrite:
-        print('File {} existed, skip.'.format(path))
-        return
-    print('Downloading from url {} to {}'.format(url, path))
-    try:
-        import urllib.request
-        urllib.request.urlretrieve(url, path)
-    except:
-        import urllib
-        urllib.urlretrieve(url, path)
-
 ######################################################################
 # Load pretrained ONNX model
 # ---------------------------------------------
@@ -44,9 +32,9 @@ def download(url, path, overwrite=False):
                      'bcda4716699ac97ea44f791c24310193/raw/',
                      '93672b029103648953c4e5ad3ac3aadf346a4cdc/',
                      'super_resolution_0.2.onnx'])
-download(model_url, 'super_resolution.onnx', True)
+model_path = download_testdata(model_url, 'super_resolution.onnx', module='onnx')
 # now you have super_resolution.onnx on disk
-onnx_model = onnx.load_model('super_resolution.onnx')
+onnx_model = onnx.load_model(model_path)
 # we can load the graph as NNVM compatible model
 sym, params = nnvm.frontend.from_onnx(onnx_model)
 
@@ -56,8 +44,8 @@ def download(url, path, overwrite=False):
 # A single cat dominates the examples!
 from PIL import Image
 img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
-download(img_url, 'cat.png')
-img = Image.open('cat.png').resize((224, 224))
+img_path = download_testdata(img_url, 'cat.png', module='data')
+img = Image.open(img_path).resize((224, 224))
 img_ycbcr = img.convert("YCbCr")  # convert to YCbCr
 img_y, img_cb, img_cr = img_ycbcr.split()
 x = np.array(img_y)[np.newaxis, np.newaxis, :, :]