diff --git a/nnvm/tests/python/frontend/coreml/model_zoo/__init__.py b/nnvm/tests/python/frontend/coreml/model_zoo/__init__.py index d43768f1e376..aa28475776ef 100644 --- a/nnvm/tests/python/frontend/coreml/model_zoo/__init__.py +++ b/nnvm/tests/python/frontend/coreml/model_zoo/__init__.py @@ -18,7 +18,7 @@ def get_resnet50(): def get_cat_image(): url = 'https://gist.githubusercontent.com/zhreshold/bcda4716699ac97ea44f791c24310193/raw/fa7ef0e9c9a5daea686d6473a62aacd1a5885849/cat.png' dst = 'cat.png' - real_dst = download_testdata(url, dst, module='coreml') + real_dst = download_testdata(url, dst, module='data') img = Image.open(real_dst).resize((224, 224)) img = np.transpose(img, (2, 0, 1))[np.newaxis, :] return np.asarray(img) diff --git a/nnvm/tests/python/frontend/darknet/test_forward.py b/nnvm/tests/python/frontend/darknet/test_forward.py index 7cd47d382d22..db7d58b1b60a 100644 --- a/nnvm/tests/python/frontend/darknet/test_forward.py +++ b/nnvm/tests/python/frontend/darknet/test_forward.py @@ -103,7 +103,7 @@ def get_darknet_output(net, img): test_image = 'dog.jpg' img_url = 'https://github.com/siju-samuel/darknet/blob/master/data/' + test_image +'?raw=true' - img_path = download_testdata(img_url, test_image, module='darknet') + img_path = download_testdata(img_url, test_image, module='data') img = LIB.letterbox_image(LIB.load_image_color(img_path.encode('utf-8'), 0, 0), net.w, net.h) darknet_output = get_darknet_output(net, img) batch_size = 1 diff --git a/nnvm/tutorials/deploy_model_on_mali_gpu.py b/nnvm/tutorials/deploy_model_on_mali_gpu.py index 6e3962a6609f..8a495961e435 100644 --- a/nnvm/tutorials/deploy_model_on_mali_gpu.py +++ b/nnvm/tutorials/deploy_model_on_mali_gpu.py @@ -15,6 +15,7 @@ import nnvm.testing from tvm import rpc from tvm.contrib import util, graph_runtime as runtime +from tvm.contrib.download import download_testdata ###################################################################### # Build TVM Runtime on Device @@ -81,7 +82,6 @@ # You can found more details about this part at tutorial :ref:`tutorial-from-mxnet`. from mxnet.gluon.model_zoo.vision import get_model -from mxnet.gluon.utils import download from PIL import Image import numpy as np @@ -92,8 +92,9 @@ # In order to test our model, here we download an image of cat and # transform its format. img_name = 'cat.png' -download('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true', img_name) -image = Image.open(img_name).resize((224, 224)) +img_path = download_testdata('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true', + img_name, module='data') +image = Image.open(img_path).resize((224, 224)) def transform_image(image): image = np.array(image) - np.array([123., 117., 104.]) @@ -112,9 +113,9 @@ def transform_image(image): '596b27d23537e5a1b5751d2b0481ef172f58b539/', 'imagenet1000_clsid_to_human.txt']) -synset_name = 'synset.txt' -download(synset_url, synset_name) -with open(synset_name) as f: +synset_name = 'imagenet1000_clsid_to_human.txt' +synset_path = download_testdata(synset_url, synset_name, module='data') +with open(synset_path) as f: synset = eval(f.read()) ###################################################################### diff --git a/nnvm/tutorials/deploy_model_on_rasp.py b/nnvm/tutorials/deploy_model_on_rasp.py index c110d7ffdc5c..b2fad1fd9e0b 100644 --- a/nnvm/tutorials/deploy_model_on_rasp.py +++ b/nnvm/tutorials/deploy_model_on_rasp.py @@ -14,6 +14,7 @@ import nnvm.testing from tvm import rpc from tvm.contrib import util, graph_runtime as runtime +from tvm.contrib.download import download_testdata ###################################################################### # .. _build-tvm-runtime-on-device: @@ -78,7 +79,6 @@ # You can found more details about this part at tutorial :ref:`tutorial-from-mxnet`. from mxnet.gluon.model_zoo.vision import get_model -from mxnet.gluon.utils import download from PIL import Image import numpy as np @@ -89,8 +89,9 @@ # In order to test our model, here we download an image of cat and # transform its format. img_name = 'cat.png' -download('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true', img_name) -image = Image.open(img_name).resize((224, 224)) +img_path = download_testdata('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true', + img_name, module='data') +image = Image.open(img_path).resize((224, 224)) def transform_image(image): image = np.array(image) - np.array([123., 117., 104.]) @@ -108,9 +109,9 @@ def transform_image(image): '4d0b62f3d01426887599d4f7ede23ee5/raw/', '596b27d23537e5a1b5751d2b0481ef172f58b539/', 'imagenet1000_clsid_to_human.txt']) -synset_name = 'synset.txt' -download(synset_url, synset_name) -with open(synset_name) as f: +synset_name = 'imagenet1000_clsid_to_human.txt' +synset_path = download_testdata(synset_url, synset_name, module='data') +with open(synset_path) as f: synset = eval(f.read()) ###################################################################### diff --git a/nnvm/tutorials/deploy_ssd_mxnet.py b/nnvm/tutorials/deploy_ssd_mxnet.py index 1a71c96eaa0c..ccccd35f04a6 100644 --- a/nnvm/tutorials/deploy_ssd_mxnet.py +++ b/nnvm/tutorials/deploy_ssd_mxnet.py @@ -18,7 +18,7 @@ from nnvm import compiler from nnvm.frontend import from_mxnet from tvm import relay -from tvm.contrib.download import download +from tvm.contrib.download import download_testdata from tvm.contrib import graph_runtime from mxnet.model import load_checkpoint @@ -65,28 +65,24 @@ inference_symbol_url = "https://gist.github.com/kevinthesun/c1904e900848df4548ce5dfb18c719c7/" \ "archive/a28c4856c827fe766aa3da0e35bad41d44f0fb26.zip" -dir = "ssd_model" -if not os.path.exists(dir): - os.makedirs(dir) -model_file_path = "%s/%s" % (dir, model_file) -test_image_path = "%s/%s" % (dir, test_image) -inference_symbol_path = "%s/inference_model.zip" % dir -download(model_url, model_file_path) -download(image_url, test_image_path) -download(inference_symbol_url, inference_symbol_path) +model_file_path = download_testdata(model_url, model_file, module=["mxnet", "ssd_model"]) +inference_symbol_path = download_testdata(inference_symbol_url, "inference_model.zip", + module=["mxnet", "ssd_model"]) +test_image_path = download_testdata(image_url, test_image, module="data") +model_dir = os.path.dirname(model_file_path) zip_ref = zipfile.ZipFile(model_file_path, 'r') -zip_ref.extractall(dir) +zip_ref.extractall(model_dir) zip_ref.close() zip_ref = zipfile.ZipFile(inference_symbol_path) -zip_ref.extractall(dir) +zip_ref.extractall(model_dir) zip_ref.close() ###################################################################### # Convert and compile model with NNVM or Relay for CPU. -sym = mx.sym.load("%s/%s/ssd_resnet50_inference.json" % (dir, inference_symbol_folder)) -_, arg_params, aux_params = load_checkpoint("%s/%s" % (dir, model_name), 0) +sym = mx.sym.load("%s/%s/ssd_resnet50_inference.json" % (model_dir, inference_symbol_folder)) +_, arg_params, aux_params = load_checkpoint("%s/%s" % (model_dir, model_name), 0) import argparse parser = argparse.ArgumentParser() diff --git a/nnvm/tutorials/from_coreml.py b/nnvm/tutorials/from_coreml.py index 1c958746247b..b25645a4d6a1 100644 --- a/nnvm/tutorials/from_coreml.py +++ b/nnvm/tutorials/from_coreml.py @@ -21,19 +21,7 @@ import coremltools as cm import numpy as np from PIL import Image - -def download(url, path, overwrite=False): - import os - if os.path.isfile(path) and not overwrite: - print('File {} existed, skip.'.format(path)) - return - print('Downloading from url {} to {}'.format(url, path)) - try: - import urllib.request - urllib.request.urlretrieve(url, path) - except: - import urllib - urllib.urlretrieve(url, path) +from tvm.contrib.download import download_testdata ###################################################################### # Load pretrained CoreML model @@ -42,9 +30,9 @@ def download(url, path, overwrite=False): # provided by apple in this example model_url = 'https://docs-assets.developer.apple.com/coreml/models/MobileNet.mlmodel' model_file = 'mobilenet.mlmodel' -download(model_url, model_file) +model_path = download_testdata(model_url, model_file, module='coreml') # now you mobilenet.mlmodel on disk -mlmodel = cm.models.MLModel(model_file) +mlmodel = cm.models.MLModel(model_path) # we can load the graph as NNVM compatible model sym, params = nnvm.frontend.from_coreml(mlmodel) @@ -54,8 +42,8 @@ def download(url, path, overwrite=False): # A single cat dominates the examples! from PIL import Image img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true' -download(img_url, 'cat.png') -img = Image.open('cat.png').resize((224, 224)) +img_path = download_testdata(img_url, 'cat.png', module='data') +img = Image.open(img_path).resize((224, 224)) #x = np.transpose(img, (2, 0, 1))[np.newaxis, :] image = np.asarray(img) image = image.transpose((2, 0, 1)) @@ -95,8 +83,8 @@ def download(url, path, overwrite=False): '4d0b62f3d01426887599d4f7ede23ee5/raw/', '596b27d23537e5a1b5751d2b0481ef172f58b539/', 'imagenet1000_clsid_to_human.txt']) -synset_name = 'synset.txt' -download(synset_url, synset_name) -with open(synset_name) as f: +synset_name = 'imagenet1000_clsid_to_human.txt' +synset_path = download_testdata(synset_url, synset_name, module='data') +with open(synset_path) as f: synset = eval(f.read()) print('Top-1 id', top1, 'class name', synset[top1]) diff --git a/nnvm/tutorials/from_darknet.py b/nnvm/tutorials/from_darknet.py index f0eec98c00ea..29053ea793c7 100644 --- a/nnvm/tutorials/from_darknet.py +++ b/nnvm/tutorials/from_darknet.py @@ -25,7 +25,7 @@ import sys from ctypes import * -from tvm.contrib.download import download +from tvm.contrib.download import download_testdata from nnvm.testing.darknet import __darknetffi__ # Model name @@ -41,8 +41,8 @@ CFG_URL = REPO_URL + 'cfg/' + CFG_NAME + '?raw=true' WEIGHTS_URL = 'https://pjreddie.com/media/files/' + WEIGHTS_NAME -download(CFG_URL, CFG_NAME) -download(WEIGHTS_URL, WEIGHTS_NAME) +cfg_path = download_testdata(CFG_URL, CFG_NAME, module="darknet") +weights_path = download_testdata(WEIGHTS_URL, WEIGHTS_NAME, module="darknet") # Download and Load darknet library if sys.platform in ['linux', 'linux2']: @@ -55,12 +55,10 @@ err = "Darknet lib is not supported on {} platform".format(sys.platform) raise NotImplementedError(err) -download(DARKNET_URL, DARKNET_LIB) +lib_path = download_testdata(DARKNET_URL, DARKNET_LIB, module="darknet") -DARKNET_LIB = __darknetffi__.dlopen('./' + DARKNET_LIB) -cfg = "./" + str(CFG_NAME) -weights = "./" + str(WEIGHTS_NAME) -net = DARKNET_LIB.load_network(cfg.encode('utf-8'), weights.encode('utf-8'), 0) +DARKNET_LIB = __darknetffi__.dlopen(lib_path) +net = DARKNET_LIB.load_network(cfg_path.encode('utf-8'), weights_path.encode('utf-8'), 0) dtype = 'float32' batch_size = 1 @@ -88,9 +86,9 @@ print("Loading the test image...") img_url = 'https://github.com/siju-samuel/darknet/blob/master/data/' + \ test_image + '?raw=true' -download(img_url, test_image) +img_path = download_testdata(img_url, test_image, "data") -data = nnvm.testing.darknet.load_image(test_image, netw, neth) +data = nnvm.testing.darknet.load_image(img_path, netw, neth) ###################################################################### # Execute on TVM Runtime # ---------------------- @@ -150,10 +148,10 @@ coco_url = 'https://github.com/siju-samuel/darknet/blob/master/data/' + coco_name + '?raw=true' font_name = 'arial.ttf' font_url = 'https://github.com/siju-samuel/darknet/blob/master/data/' + font_name + '?raw=true' -download(coco_url, coco_name) -download(font_url, font_name) +coco_path = download_testdata(coco_url, coco_name, module='data') +font_path = download_testdata(font_url, font_name, module='data') -with open(coco_name) as f: +with open(coco_path) as f: content = f.readlines() names = [x.strip() for x in content] diff --git a/nnvm/tutorials/from_mxnet.py b/nnvm/tutorials/from_mxnet.py index b4c2c5b7dfbd..bc8a23e3e898 100644 --- a/nnvm/tutorials/from_mxnet.py +++ b/nnvm/tutorials/from_mxnet.py @@ -20,30 +20,31 @@ """ # some standard imports import mxnet as mx +import numpy as np import nnvm import tvm -import numpy as np +from tvm.contrib.download import download_testdata ###################################################################### # Download Resnet18 model from Gluon Model Zoo # --------------------------------------------- # In this section, we download a pretrained imagenet model and classify an image. from mxnet.gluon.model_zoo.vision import get_model -from mxnet.gluon.utils import download from PIL import Image from matplotlib import pyplot as plt block = get_model('resnet18_v1', pretrained=True) +img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true' img_name = 'cat.png' synset_url = ''.join(['https://gist.githubusercontent.com/zhreshold/', '4d0b62f3d01426887599d4f7ede23ee5/raw/', '596b27d23537e5a1b5751d2b0481ef172f58b539/', 'imagenet1000_clsid_to_human.txt']) -synset_name = 'synset.txt' -download('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true', img_name) -download(synset_url, synset_name) -with open(synset_name) as f: +synset_name = 'imagenet1000_clsid_to_human.txt' +img_path = download_testdata(img_url, img_name, module='data') +synset_path = download_testdata(synset_url, synset_name, module='data') +with open(synset_path) as f: synset = eval(f.read()) -image = Image.open(img_name).resize((224, 224)) +image = Image.open(img_path).resize((224, 224)) plt.imshow(image) plt.show() diff --git a/nnvm/tutorials/from_mxnet_to_webgl.py b/nnvm/tutorials/from_mxnet_to_webgl.py index 4e7b57706de6..540585d3c7c6 100644 --- a/nnvm/tutorials/from_mxnet_to_webgl.py +++ b/nnvm/tutorials/from_mxnet_to_webgl.py @@ -33,6 +33,7 @@ import numpy as np import tvm +from tvm.contrib.download import download_testdata import nnvm.compiler import nnvm.testing @@ -106,16 +107,14 @@ def download_synset(): print("Downloading synset...") - from mxnet import gluon - url = "https://gist.githubusercontent.com/zhreshold/" + \ "4d0b62f3d01426887599d4f7ede23ee5/raw/" + \ "596b27d23537e5a1b5751d2b0481ef172f58b539/" + \ "imagenet1000_clsid_to_human.txt" - file_name = "synset.txt" + file_name = "imagenet1000_clsid_to_human.txt" - gluon.utils.download(url, file_name) - with open(file_name) as f: + file_path = download_testdata(url, file_name, module='data') + with open(file_path) as f: synset = eval(f.read()) print("- Synset downloaded!") @@ -144,14 +143,13 @@ def download_image(): print("Downloading cat image...") from matplotlib import pyplot as plt - from mxnet import gluon from PIL import Image url = "https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true" img_name = "cat.png" - gluon.utils.download(url, img_name) - image = Image.open(img_name).resize((224, 224)) + img_path = download_testdata(url, img_name, module='data') + image = Image.open(img_path).resize((224, 224)) print("- Cat image downloaded!") diff --git a/nnvm/tutorials/from_onnx.py b/nnvm/tutorials/from_onnx.py index 0fdef8afa98c..c5fe03fd1c96 100644 --- a/nnvm/tutorials/from_onnx.py +++ b/nnvm/tutorials/from_onnx.py @@ -18,22 +18,10 @@ """ import nnvm import tvm +from tvm.contrib.download import download_testdata import onnx import numpy as np -def download(url, path, overwrite=False): - import os - if os.path.isfile(path) and not overwrite: - print('File {} existed, skip.'.format(path)) - return - print('Downloading from url {} to {}'.format(url, path)) - try: - import urllib.request - urllib.request.urlretrieve(url, path) - except: - import urllib - urllib.urlretrieve(url, path) - ###################################################################### # Load pretrained ONNX model # --------------------------------------------- @@ -44,9 +32,9 @@ def download(url, path, overwrite=False): 'bcda4716699ac97ea44f791c24310193/raw/', '93672b029103648953c4e5ad3ac3aadf346a4cdc/', 'super_resolution_0.2.onnx']) -download(model_url, 'super_resolution.onnx', True) +model_path = download_testdata(model_url, 'super_resolution.onnx', module='onnx') # now you have super_resolution.onnx on disk -onnx_model = onnx.load_model('super_resolution.onnx') +onnx_model = onnx.load_model(model_path) # we can load the graph as NNVM compatible model sym, params = nnvm.frontend.from_onnx(onnx_model) @@ -56,8 +44,8 @@ def download(url, path, overwrite=False): # A single cat dominates the examples! from PIL import Image img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true' -download(img_url, 'cat.png') -img = Image.open('cat.png').resize((224, 224)) +img_path = download_testdata(img_url, 'cat.png', module='data') +img = Image.open(img_path).resize((224, 224)) img_ycbcr = img.convert("YCbCr") # convert to YCbCr img_y, img_cb, img_cr = img_ycbcr.split() x = np.array(img_y)[np.newaxis, np.newaxis, :, :] diff --git a/nnvm/tutorials/from_tensorflow.py b/nnvm/tutorials/from_tensorflow.py index ac632b122e76..ab9ba03232a4 100644 --- a/nnvm/tutorials/from_tensorflow.py +++ b/nnvm/tutorials/from_tensorflow.py @@ -52,8 +52,8 @@ map_proto_url = os.path.join(repo_base, map_proto) # Human readable text for labels -lable_map = 'imagenet_synset_to_human_label_map.txt' -lable_map_url = os.path.join(repo_base, lable_map) +label_map = 'imagenet_synset_to_human_label_map.txt' +label_map_url = os.path.join(repo_base, label_map) # Target settings # Use these commented settings to build for cuda. @@ -70,19 +70,19 @@ # Download required files # ----------------------- # Download files listed above. -from mxnet.gluon.utils import download +from tvm.contrib.download import download_testdata -download(image_url, img_name) -download(model_url, model_name) -download(map_proto_url, map_proto) -download(lable_map_url, lable_map) +img_path = download_testdata(image_url, img_name, module='data') +model_path = download_testdata(model_url, model_name, module=['tf', 'InceptionV1']) +map_proto_path = download_testdata(map_proto_url, map_proto, module='data') +label_path = download_testdata(label_map_url, label_map, module='data') ###################################################################### # Import model # ------------ # Creates tensorflow graph definition from protobuf file. -with tf.gfile.FastGFile(os.path.join("./", model_name), 'rb') as f: +with tf.gfile.FastGFile(model_path, 'rb') as f: graph_def = tf.GraphDef() graph_def.ParseFromString(f.read()) graph = tf.import_graph_def(graph_def, name='') @@ -103,7 +103,7 @@ # from PIL import Image -image = Image.open(img_name).resize((299, 299)) +image = Image.open(img_path).resize((299, 299)) x = np.array(image) @@ -117,7 +117,7 @@ # params: params converted from tensorflow params (tensor protobuf). sym, params = nnvm.frontend.from_tensorflow(graph_def, layout=layout) -print ("Tensorflow protobuf imported as nnvm graph") +print("Tensorflow protobuf imported as nnvm graph") ###################################################################### # NNVM Compilation # ---------------- @@ -157,8 +157,8 @@ predictions = np.squeeze(predictions) # Creates node ID --> English string lookup. -node_lookup = tf_testing.NodeLookup(label_lookup_path=os.path.join("./", map_proto), - uid_lookup_path=os.path.join("./", lable_map)) +node_lookup = tf_testing.NodeLookup(label_lookup_path=map_proto_path, + uid_lookup_path=label_path) # Print top 5 predictions from TVM output. top_k = predictions.argsort()[-5:][::-1] @@ -175,7 +175,7 @@ def create_graph(): """Creates a graph from saved GraphDef file and returns a saver.""" # Creates graph from saved graph_def.pb. - with tf.gfile.FastGFile(model_name, 'rb') as f: + with tf.gfile.FastGFile(model_path, 'rb') as f: graph_def = tf.GraphDef() graph_def.ParseFromString(f.read()) graph = tf.import_graph_def(graph_def, name='') @@ -209,8 +209,8 @@ def run_inference_on_image(image): predictions = np.squeeze(predictions) # Creates node ID --> English string lookup. - node_lookup = tf_testing.NodeLookup(label_lookup_path=os.path.join("./", map_proto), - uid_lookup_path=os.path.join("./", lable_map)) + node_lookup = tf_testing.NodeLookup(label_lookup_path=map_proto_path, + uid_lookup_path=label_path) # Print top 5 predictions from tensorflow. top_k = predictions.argsort()[-5:][::-1] @@ -220,4 +220,4 @@ def run_inference_on_image(image): score = predictions[node_id] print('%s (score = %.5f)' % (human_string, score)) -run_inference_on_image (img_name) +run_inference_on_image(img_path) diff --git a/nnvm/tutorials/nlp/from_darknet_rnn.py b/nnvm/tutorials/nlp/from_darknet_rnn.py index 54013f04fca6..bbf70c724bbe 100644 --- a/nnvm/tutorials/nlp/from_darknet_rnn.py +++ b/nnvm/tutorials/nlp/from_darknet_rnn.py @@ -22,9 +22,9 @@ """ import random import numpy as np -from mxnet.gluon.utils import download import tvm from tvm.contrib import graph_runtime +from tvm.contrib.download import download_testdata from nnvm.testing.darknet import __darknetffi__ import nnvm import nnvm.frontend.darknet @@ -49,17 +49,15 @@ CFG_URL = REPO_URL + 'cfg/' + CFG_NAME + '?raw=true' WEIGHTS_URL = REPO_URL + 'weights/' + WEIGHTS_NAME + '?raw=true' -download(CFG_URL, CFG_NAME) -download(WEIGHTS_URL, WEIGHTS_NAME) +cfg_path = download_testdata(CFG_URL, CFG_NAME, module='darknet') +weights_path = download_testdata(WEIGHTS_URL, WEIGHTS_NAME, module='darknet') # Download and Load darknet library DARKNET_LIB = 'libdarknet.so' DARKNET_URL = REPO_URL + 'lib/' + DARKNET_LIB + '?raw=true' -download(DARKNET_URL, DARKNET_LIB) -DARKNET_LIB = __darknetffi__.dlopen('./' + DARKNET_LIB) -cfg = "./" + str(CFG_NAME) -weights = "./" + str(WEIGHTS_NAME) -net = DARKNET_LIB.load_network(cfg.encode('utf-8'), weights.encode('utf-8'), 0) +lib_path = download_testdata(DARKNET_URL, DARKNET_LIB, module='darknet') +DARKNET_LIB = __darknetffi__.dlopen(lib_path) +net = DARKNET_LIB.load_network(cfg_path.encode('utf-8'), weights_path.encode('utf-8'), 0) dtype = 'float32' batch_size = 1 diff --git a/nnvm/tutorials/nlp/keras_s2s_translate.py b/nnvm/tutorials/nlp/keras_s2s_translate.py index 77c7f23902f4..97c7a706a80f 100644 --- a/nnvm/tutorials/nlp/keras_s2s_translate.py +++ b/nnvm/tutorials/nlp/keras_s2s_translate.py @@ -59,9 +59,9 @@ data_url = os.path.join(repo_base, data_file) # Download files listed below. -from mxnet.gluon.utils import download -download(model_url, model_file) -download(data_url, model_file) +from tvm.contrib.download import download_testdata +model_path = download_testdata(model_url, model_file, module='keras') +data_path = download_testdata(data_url, data_file, module='data') latent_dim = 256 # Latent dimensionality of the encoding space. test_samples = 10000 # Number of samples used for testing. @@ -76,7 +76,7 @@ target_texts = [] input_characters = set() target_characters = set() -with open(data_file, 'r', encoding='utf-8') as f: +with open(data_path, 'r', encoding='utf-8') as f: lines = f.read().split('\n') test_samples = min(test_samples, len(lines)) max_encoder_seq_length = 0 @@ -112,7 +112,7 @@ # Load Keras Model # ---------------- # Restore the model and construct the encoder and decoder. -model = load_model(model_file) +model = load_model(model_path) encoder_inputs = model.input[0] # input_1 encoder_outputs, state_h_enc, state_c_enc = model.layers[2].output # lstm_1 diff --git a/python/tvm/contrib/download.py b/python/tvm/contrib/download.py index 26f285a0b29a..ed1740eb92e8 100644 --- a/python/tvm/contrib/download.py +++ b/python/tvm/contrib/download.py @@ -28,6 +28,9 @@ def download(url, path, overwrite=False, size_compare=False, verbose=1, retries= verbose: int, optional Verbose level + + retries: int, optional + Number of time to retry download, default at 3. """ if sys.version_info >= (3,): import urllib.request as urllib2 diff --git a/python/tvm/relay/testing/tf.py b/python/tvm/relay/testing/tf.py index 0f22939768c5..2bbbb46fe233 100644 --- a/python/tvm/relay/testing/tf.py +++ b/python/tvm/relay/testing/tf.py @@ -331,7 +331,7 @@ def get_workload_ptb(): ptb_model_file = 'RNN/ptb/ptb_model_with_lstmblockcell.pb' import tarfile - file_path = download_testdata(sample_url, sample_data_file, module=['tf', 'ptb_data']) + file_path = download_testdata(sample_url, sample_data_file, module=['data', 'ptb_data']) dir_path = os.path.dirname(file_path) t = tarfile.open(file_path, 'r') t.extractall(dir_path) diff --git a/tests/python/frontend/coreml/model_zoo/__init__.py b/tests/python/frontend/coreml/model_zoo/__init__.py index ca10a96ff649..b89075c7a04c 100644 --- a/tests/python/frontend/coreml/model_zoo/__init__.py +++ b/tests/python/frontend/coreml/model_zoo/__init__.py @@ -18,7 +18,7 @@ def get_resnet50(): def get_cat_image(): url = 'https://gist.githubusercontent.com/zhreshold/bcda4716699ac97ea44f791c24310193/raw/fa7ef0e9c9a5daea686d6473a62aacd1a5885849/cat.png' dst = 'cat.png' - real_dst = download_testdata(url, dst, module='coreml') + real_dst = download_testdata(url, dst, module='data') img = Image.open(real_dst).resize((224, 224)) img = np.transpose(img, (2, 0, 1))[np.newaxis, :] return np.asarray(img) \ No newline at end of file diff --git a/tutorials/frontend/deploy_model_on_rasp.py b/tutorials/frontend/deploy_model_on_rasp.py index b90127b3858e..c96962a52c1b 100644 --- a/tutorials/frontend/deploy_model_on_rasp.py +++ b/tutorials/frontend/deploy_model_on_rasp.py @@ -14,6 +14,7 @@ import tvm.relay as relay from tvm import rpc from tvm.contrib import util, graph_runtime as runtime +from tvm.contrib.download import download_testdata ###################################################################### # .. _build-tvm-runtime-on-device: @@ -82,7 +83,6 @@ # You can found more details about this part at tutorial :ref:`tutorial-from-mxnet`. from mxnet.gluon.model_zoo.vision import get_model -from mxnet.gluon.utils import download from PIL import Image import numpy as np @@ -92,9 +92,10 @@ ###################################################################### # In order to test our model, here we download an image of cat and # transform its format. +img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true' img_name = 'cat.png' -download('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true', img_name) -image = Image.open(img_name).resize((224, 224)) +img_path = download_testdata(img_url, img_name, module='data') +image = Image.open(img_path).resize((224, 224)) def transform_image(image): image = np.array(image) - np.array([123., 117., 104.]) @@ -112,9 +113,9 @@ def transform_image(image): '4d0b62f3d01426887599d4f7ede23ee5/raw/', '596b27d23537e5a1b5751d2b0481ef172f58b539/', 'imagenet1000_clsid_to_human.txt']) -synset_name = 'synset.txt' -download(synset_url, synset_name) -with open(synset_name) as f: +synset_name = 'imagenet1000_clsid_to_human.txt' +synset_path = download_testdata(synset_url, synset_name, module='data') +with open(synset_path) as f: synset = eval(f.read()) ###################################################################### diff --git a/tutorials/frontend/deploy_ssd_gluoncv.py b/tutorials/frontend/deploy_ssd_gluoncv.py index f0063ead190b..38a84f5cbbe0 100644 --- a/tutorials/frontend/deploy_ssd_gluoncv.py +++ b/tutorials/frontend/deploy_ssd_gluoncv.py @@ -12,6 +12,7 @@ from tvm.relay.testing.config import ctx_list from tvm import relay from tvm.contrib import graph_runtime +from tvm.contrib.download import download_testdata from gluoncv import model_zoo, data, utils @@ -49,9 +50,9 @@ ###################################################################### # Download and pre-process demo image -im_fname = utils.download('https://github.com/dmlc/web-data/blob/master/' + - 'gluoncv/detection/street_small.jpg?raw=true', - path='street_small.jpg') +im_fname = download_testdata('https://github.com/dmlc/web-data/blob/master/' + + 'gluoncv/detection/street_small.jpg?raw=true', + 'street_small.jpg', module='data') x, img = data.transforms.presets.ssd.load_test(im_fname, short=512) ###################################################################### diff --git a/tutorials/frontend/from_caffe2.py b/tutorials/frontend/from_caffe2.py index fce7f30d865d..fbb54bd89809 100644 --- a/tutorials/frontend/from_caffe2.py +++ b/tutorials/frontend/from_caffe2.py @@ -19,21 +19,6 @@ or please refer to official site https://caffe2.ai/docs/getting-started.html """ -###################################################################### -# Utils for downloading files -# ---------------------------- -def download(url, path, overwrite=False): - import os - if os.path.isfile(path) and not overwrite: - print('File {} exists, skip.'.format(path)) - return - print('Downloading from url {} to {}'.format(url, path)) - try: - import urllib.request - urllib.request.urlretrieve(url, path) - except: - import urllib - urllib.urlretrieve(url, path) ###################################################################### # Load pretrained Caffe2 model @@ -52,12 +37,13 @@ def __init__(self, model_name): # Load a test image # ------------------ # A single cat dominates the examples! +from tvm.contrib.download import download_testdata from PIL import Image from matplotlib import pyplot as plt import numpy as np img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true' -download(img_url, 'cat.png') -img = Image.open('cat.png').resize((224, 224)) +img_path = download_testdata(img_url, 'cat.png', module='data') +img = Image.open(img_path).resize((224, 224)) plt.imshow(img) plt.show() # input preprocess @@ -118,9 +104,9 @@ def transform_image(image): '4d0b62f3d01426887599d4f7ede23ee5/raw/', '596b27d23537e5a1b5751d2b0481ef172f58b539/', 'imagenet1000_clsid_to_human.txt']) -synset_name = 'synset.txt' -download(synset_url, synset_name) -with open(synset_name) as f: +synset_name = 'imagenet1000_clsid_to_human.txt' +synset_path = download_testdata(synset_url, synset_name, module='data') +with open(synset_path) as f: synset = eval(f.read()) print('Relay top-1 id: {}, class name: {}'.format(top1_tvm, synset[top1_tvm])) # confirm correctness with caffe2 output diff --git a/tutorials/frontend/from_coreml.py b/tutorials/frontend/from_coreml.py index a79e21921068..cefac979317a 100644 --- a/tutorials/frontend/from_coreml.py +++ b/tutorials/frontend/from_coreml.py @@ -19,23 +19,11 @@ """ import tvm import tvm.relay as relay +from tvm.contrib.download import download_testdata import coremltools as cm import numpy as np from PIL import Image -def download(url, path, overwrite=False): - import os - if os.path.isfile(path) and not overwrite: - print('File {} existed, skip.'.format(path)) - return - print('Downloading from url {} to {}'.format(url, path)) - try: - import urllib.request - urllib.request.urlretrieve(url, path) - except: - import urllib - urllib.urlretrieve(url, path) - ###################################################################### # Load pretrained CoreML model # ---------------------------- @@ -43,17 +31,17 @@ def download(url, path, overwrite=False): # provided by apple in this example model_url = 'https://docs-assets.developer.apple.com/coreml/models/MobileNet.mlmodel' model_file = 'mobilenet.mlmodel' -download(model_url, model_file) +model_path = download_testdata(model_url, model_file, module='coreml') # Now you have mobilenet.mlmodel on disk -mlmodel = cm.models.MLModel(model_file) +mlmodel = cm.models.MLModel(model_path) ###################################################################### # Load a test image # ------------------ # A single cat dominates the examples! img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true' -download(img_url, 'cat.png') -img = Image.open('cat.png').resize((224, 224)) +img_path = download_testdata(img_url, 'cat.png', module='data') +img = Image.open(img_path).resize((224, 224)) x = np.transpose(img, (2, 0, 1))[np.newaxis, :] ###################################################################### @@ -94,8 +82,8 @@ def download(url, path, overwrite=False): '4d0b62f3d01426887599d4f7ede23ee5/raw/', '596b27d23537e5a1b5751d2b0481ef172f58b539/', 'imagenet1000_clsid_to_human.txt']) -synset_name = 'synset.txt' -download(synset_url, synset_name) -with open(synset_name) as f: +synset_name = 'imagenet1000_clsid_to_human.txt' +synset_path = download_testdata(synset_url, synset_name, module='data') +with open(synset_path) as f: synset = eval(f.read()) print('Top-1 id', top1, 'class name', synset[top1]) diff --git a/tutorials/frontend/from_keras.py b/tutorials/frontend/from_keras.py index 0e30a8d012a8..e8a972cfbb1c 100644 --- a/tutorials/frontend/from_keras.py +++ b/tutorials/frontend/from_keras.py @@ -20,22 +20,10 @@ """ import tvm import tvm.relay as relay +from tvm.contrib.download import download_testdata import keras import numpy as np -def download(url, path, overwrite=False): - import os - if os.path.isfile(path) and not overwrite: - print('File {} exists, skip.'.format(path)) - return - print('Downloading from url {} to {}'.format(url, path)) - try: - import urllib.request - urllib.request.urlretrieve(url, path) - except: - import urllib - urllib.urlretrieve(url, path) - ###################################################################### # Load pretrained keras model # ---------------------------- @@ -43,10 +31,10 @@ def download(url, path, overwrite=False): weights_url = ''.join(['https://github.com/fchollet/deep-learning-models/releases/', 'download/v0.2/resnet50_weights_tf_dim_ordering_tf_kernels.h5']) weights_file = 'resnet50_weights.h5' -download(weights_url, weights_file) +weights_path = download_testdata(weights_url, weights_file, module='keras') keras_resnet50 = keras.applications.resnet50.ResNet50(include_top=True, weights=None, input_shape=(224, 224, 3), classes=1000) -keras_resnet50.load_weights('resnet50_weights.h5') +keras_resnet50.load_weights(weights_path) ###################################################################### # Load a test image @@ -56,8 +44,8 @@ def download(url, path, overwrite=False): from matplotlib import pyplot as plt from keras.applications.resnet50 import preprocess_input img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true' -download(img_url, 'cat.png') -img = Image.open('cat.png').resize((224, 224)) +img_path = download_testdata(img_url, 'cat.png', module='data') +img = Image.open(img_path).resize((224, 224)) plt.imshow(img) plt.show() # input preprocess @@ -92,9 +80,9 @@ def download(url, path, overwrite=False): '4d0b62f3d01426887599d4f7ede23ee5/raw/', '596b27d23537e5a1b5751d2b0481ef172f58b539/', 'imagenet1000_clsid_to_human.txt']) -synset_name = 'synset.txt' -download(synset_url, synset_name) -with open(synset_name) as f: +synset_name = 'imagenet1000_clsid_to_human.txt' +synset_path = download_testdata(synset_url, synset_name, module='data') +with open(synset_path) as f: synset = eval(f.read()) print('Relay top-1 id: {}, class name: {}'.format(top1_tvm, synset[top1_tvm])) # confirm correctness with keras output diff --git a/tutorials/frontend/from_mxnet.py b/tutorials/frontend/from_mxnet.py index a465350a0df8..e7e6db1fe9a4 100644 --- a/tutorials/frontend/from_mxnet.py +++ b/tutorials/frontend/from_mxnet.py @@ -29,22 +29,23 @@ # Download Resnet18 model from Gluon Model Zoo # --------------------------------------------- # In this section, we download a pretrained imagenet model and classify an image. +from tvm.contrib.download import download_testdata from mxnet.gluon.model_zoo.vision import get_model -from mxnet.gluon.utils import download from PIL import Image from matplotlib import pyplot as plt block = get_model('resnet18_v1', pretrained=True) +img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true' img_name = 'cat.png' synset_url = ''.join(['https://gist.githubusercontent.com/zhreshold/', '4d0b62f3d01426887599d4f7ede23ee5/raw/', '596b27d23537e5a1b5751d2b0481ef172f58b539/', 'imagenet1000_clsid_to_human.txt']) -synset_name = 'synset.txt' -download('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true', img_name) -download(synset_url, synset_name) -with open(synset_name) as f: +synset_name = 'imagenet1000_clsid_to_human.txt' +img_path = download_testdata(img_url, 'cat.png', module='data') +synset_path = download_testdata(synset_url, synset_name, module='data') +with open(synset_path) as f: synset = eval(f.read()) -image = Image.open(img_name).resize((224, 224)) +image = Image.open(img_path).resize((224, 224)) plt.imshow(image) plt.show() diff --git a/tutorials/frontend/from_onnx.py b/tutorials/frontend/from_onnx.py index 90b0c1c6645c..19bee0a0319b 100644 --- a/tutorials/frontend/from_onnx.py +++ b/tutorials/frontend/from_onnx.py @@ -20,19 +20,7 @@ import numpy as np import tvm import tvm.relay as relay - -def download(url, path, overwrite=False): - import os - if os.path.isfile(path) and not overwrite: - print('File {} existed, skip.'.format(path)) - return - print('Downloading from url {} to {}'.format(url, path)) - try: - import urllib.request - urllib.request.urlretrieve(url, path) - except: - import urllib - urllib.urlretrieve(url, path) +from tvm.contrib.download import download_testdata ###################################################################### # Load pretrained ONNX model @@ -44,9 +32,9 @@ def download(url, path, overwrite=False): 'bcda4716699ac97ea44f791c24310193/raw/', '93672b029103648953c4e5ad3ac3aadf346a4cdc/', 'super_resolution_0.2.onnx']) -download(model_url, 'super_resolution.onnx', False) +model_path = download_testdata(model_url, 'super_resolution.onnx', module='onnx') # now you have super_resolution.onnx on disk -onnx_model = onnx.load('super_resolution.onnx') +onnx_model = onnx.load(model_path) ###################################################################### # Load a test image @@ -54,8 +42,8 @@ def download(url, path, overwrite=False): # A single cat dominates the examples! from PIL import Image img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true' -download(img_url, 'cat.png') -img = Image.open('cat.png').resize((224, 224)) +img_path = download_testdata(img_url, 'cat.png', module='data') +img = Image.open(img_path).resize((224, 224)) img_ycbcr = img.convert("YCbCr") # convert to YCbCr img_y, img_cb, img_cr = img_ycbcr.split() x = np.array(img_y)[np.newaxis, np.newaxis, :, :] diff --git a/tutorials/frontend/from_tensorflow.py b/tutorials/frontend/from_tensorflow.py index 1f76db890ade..1b01ef8a6b8b 100644 --- a/tutorials/frontend/from_tensorflow.py +++ b/tutorials/frontend/from_tensorflow.py @@ -43,8 +43,8 @@ map_proto_url = os.path.join(repo_base, map_proto) # Human readable text for labels -lable_map = 'imagenet_synset_to_human_label_map.txt' -lable_map_url = os.path.join(repo_base, lable_map) +label_map = 'imagenet_synset_to_human_label_map.txt' +label_map_url = os.path.join(repo_base, label_map) # Target settings # Use these commented settings to build for cuda. @@ -61,19 +61,19 @@ # Download required files # ----------------------- # Download files listed above. -from mxnet.gluon.utils import download +from tvm.contrib.download import download_testdata -download(image_url, img_name) -download(model_url, model_name) -download(map_proto_url, map_proto) -download(lable_map_url, lable_map) +img_path = download_testdata(image_url, img_name, module='data') +model_path = download_testdata(model_url, model_name, module=['tf', 'InceptionV1']) +map_proto_path = download_testdata(map_proto_url, map_proto, module='data') +label_path = download_testdata(label_map_url, label_map, module='data') ###################################################################### # Import model # ------------ # Creates tensorflow graph definition from protobuf file. -with tf.gfile.FastGFile(os.path.join("./", model_name), 'rb') as f: +with tf.gfile.FastGFile(model_path, 'rb') as f: graph_def = tf.GraphDef() graph_def.ParseFromString(f.read()) graph = tf.import_graph_def(graph_def, name='') @@ -94,7 +94,7 @@ # from PIL import Image -image = Image.open(img_name).resize((299, 299)) +image = Image.open(img_path).resize((299, 299)) x = np.array(image) @@ -110,7 +110,7 @@ dtype_dict = {'DecodeJpeg/contents': 'uint8'} sym, params = relay.frontend.from_tensorflow(graph_def, layout=layout, shape=shape_dict) -print ("Tensorflow protobuf imported to relay frontend.") +print("Tensorflow protobuf imported to relay frontend.") ###################################################################### # Relay Build # ----------- @@ -148,8 +148,8 @@ predictions = np.squeeze(predictions) # Creates node ID --> English string lookup. -node_lookup = tf_testing.NodeLookup(label_lookup_path=os.path.join("./", map_proto), - uid_lookup_path=os.path.join("./", lable_map)) +node_lookup = tf_testing.NodeLookup(label_lookup_path=map_proto_path, + uid_lookup_path=label_path) # Print top 5 predictions from TVM output. top_k = predictions.argsort()[-5:][::-1] @@ -166,7 +166,7 @@ def create_graph(): """Creates a graph from saved GraphDef file and returns a saver.""" # Creates graph from saved graph_def.pb. - with tf.gfile.FastGFile(model_name, 'rb') as f: + with tf.gfile.FastGFile(model_path, 'rb') as f: graph_def = tf.GraphDef() graph_def.ParseFromString(f.read()) graph = tf.import_graph_def(graph_def, name='') @@ -200,8 +200,8 @@ def run_inference_on_image(image): predictions = np.squeeze(predictions) # Creates node ID --> English string lookup. - node_lookup = tf_testing.NodeLookup(label_lookup_path=os.path.join("./", map_proto), - uid_lookup_path=os.path.join("./", lable_map)) + node_lookup = tf_testing.NodeLookup(label_lookup_path=map_proto_path, + uid_lookup_path=label_path) # Print top 5 predictions from tensorflow. top_k = predictions.argsort()[-5:][::-1] @@ -211,4 +211,4 @@ def run_inference_on_image(image): score = predictions[node_id] print('%s (score = %.5f)' % (human_string, score)) -run_inference_on_image (img_name) +run_inference_on_image(img_path) diff --git a/tutorials/frontend/from_tflite.py b/tutorials/frontend/from_tflite.py index cfa18af291b1..fbd4a6d2837b 100644 --- a/tutorials/frontend/from_tflite.py +++ b/tutorials/frontend/from_tflite.py @@ -52,25 +52,14 @@ ###################################################################### # Utils for downloading and extracting zip files # --------------------------------------------- - -def download(url, path, overwrite=False): - import os - if os.path.isfile(path) and not overwrite: - print('File {} existed, skip.'.format(path)) - return - print('Downloading from url {} to {}'.format(url, path)) - try: - import urllib.request - urllib.request.urlretrieve(url, path) - except: - import urllib - urllib.urlretrieve(url, path) +import os def extract(path): import tarfile if path.endswith("tgz") or path.endswith("gz"): + dir_path = os.path.dirname(path) tar = tarfile.open(path) - tar.extractall() + tar.extractall(path=dir_path) tar.close() else: raise RuntimeError('Could not decompress the file: ' + path) @@ -80,14 +69,17 @@ def extract(path): # Load pretrained TFLite model # --------------------------------------------- # we load mobilenet V1 TFLite model provided by Google +from tvm.contrib.download import download_testdata + model_url = "http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_224.tgz" # we download model tar file and extract, finally get mobilenet_v1_1.0_224.tflite -download(model_url, "mobilenet_v1_1.0_224.tgz", False) -extract("mobilenet_v1_1.0_224.tgz") +model_path = download_testdata(model_url, "mobilenet_v1_1.0_224.tgz", module=['tf', 'official']) +model_dir = os.path.dirname(model_path) +extract(model_path) # now we have mobilenet_v1_1.0_224.tflite on disk and open it -tflite_model_file = "mobilenet_v1_1.0_224.tflite" +tflite_model_file = os.path.join(model_dir, "mobilenet_v1_1.0_224.tflite") tflite_model_buf = open(tflite_model_file, "rb").read() # get TFLite model from buffer @@ -103,8 +95,8 @@ def extract(path): import numpy as np image_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true' -download(image_url, 'cat.png') -resized_image = Image.open('cat.png').resize((224, 224)) +image_path = download_testdata(image_url, 'cat.png', module='data') +resized_image = Image.open(image_path).resize((224, 224)) plt.imshow(resized_image) plt.show() image_data = np.asarray(resized_image).astype("float32") @@ -179,11 +171,11 @@ def extract(path): 'app/src/main/assets/', 'labels_mobilenet_quant_v1_224.txt']) label_file = "labels_mobilenet_quant_v1_224.txt" -download(label_file_url, label_file) +label_path = download_testdata(label_file_url, label_file, module='data') # map id to 1001 classes labels = dict() -with open(label_file) as f: +with open(label_path) as f: for id, line in enumerate(f): labels[id] = line