Merge pull request #3 from oci-labs/develop

Develop
oci-labs · Nov 5, 2018 · bb93cbc · bb93cbc
2 parents 9b3a8fe + 27d82b7
commit bb93cbc
Show file tree

Hide file tree

Showing 2,213 changed files with 611 additions and 3,469 deletions.
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
diff --git a/.ipynb_checkpoints/gaugeReader-checkpoint.ipynb b/.ipynb_checkpoints/gaugeReader-checkpoint.ipynb
diff --git a/CustomEstimator/CLI_commands/copy_to_bucket.sh b/CustomEstimator/CLI_commands/copy_to_bucket.sh
@@ -0,0 +1,5 @@
+gsutil -m cp -r ./data/ImageEveryUnit/. gs://deep_gauge/data/
+gsutil -m cp -r ./trainer/. gs://deep_gauge
+
+## make a package
+tar -czvf ensemble_package.tar.gz /ensemble_package
diff --git a/CustomEstimator/CLI_commands/ens.sh b/CustomEstimator/CLI_commands/ens.sh
@@ -0,0 +1,51 @@
+python customEstimator.py --dev=True --retrain_primary_models=False --hidden_units='[100, 200, 300, 500]' --verbosity=DEBUG
+
+
+#######
+gcloud ml-engine local train \
+          --module-name trainer_from_storage.task \
+          --package-path /home/khodayarim/PycharmProjects/DeepGauge-ML-Demo/CustomEstimator/modules/ensemble_modules/trainer_from_storage \
+          -- \
+          --primary_models_directory=/home/khodayarim/PycharmProjects/DeepGauge-ML-Demo/CustomEstimator/modules/ensemble_modules/trainer_from_storage/logs/primary_models \
+          --ensemble_architecture_path=/home/khodayarim/PycharmProjects/DeepGauge-ML-Demo/CustomEstimator/modules/ensemble_modules/trainer_from_storage/logs/temporary_models \
+          --path_to_images=/home/khodayarim/PycharmProjects/DeepGauge-ML-Demo/CustomEstimator/modules/ensemble_modules/trainer_from_storage/data/ImageEveryUnit \
+          --bin_path=/home/khodayarim/PycharmProjects/DeepGauge-ML-Demo/CustomEstimator/modules/ensemble_modules/trainer_from_storage/logs/dumps/ \
+          --export_dir=/home/khodayarim/PycharmProjects/DeepGauge-ML-Demo/CustomEstimator/modules/ensemble_modules/trainer_from_storage/logs/exported_model
+
+
+
+
+####### on GCP
+gcloud ml-engine jobs submit training ensemble_training \
+          --scale-tier basic \
+          --package-path /home/khodayarim/PycharmProjects/ensamble_package.tar.gz  \
+          --module-name ensemble_package.task \
+          --job-dir gs://deep_gauge \
+          --region "us-east1"
+          -- \
+          --primary_models_directory=gs://deep_gauge/ensemble_package/misc/primary_models \
+
+
+
+
+
+
+          --staging-bucket gs://deep_gauge/ensemble_package \
+
+          --packages additional-dep1.tar.gz,dep2.whl
+          -- \
+          --primary_models_directory=gs://deep_gauge/ensemble_package/misc/primary_models \
+
+          --module-name trainer_from_storage.task \
+          --package-path gs://deep_gauge/ensemble_package \
+          -- \
+          --primary_models_directory=/home/khodayarim/PycharmProjects/DeepGauge-ML-Demo/CustomEstimator/modules/ensemble_modules/trainer_from_storage/logs/primary_models \
+          --ensemble_architecture_path=/home/khodayarim/PycharmProjects/DeepGauge-ML-Demo/CustomEstimator/modules/ensemble_modules/trainer_from_storage/logs/temporary_models \
+          --path_to_images=/home/khodayarim/PycharmProjects/DeepGauge-ML-Demo/CustomEstimator/modules/ensemble_modules/trainer_from_storage/data/ImageEveryUnit \
+          --bin_path=/home/khodayarim/PycharmProjects/DeepGauge-ML-Demo/CustomEstimator/modules/ensemble_modules/trainer_from_storage/logs/dumps/ \
+          --export_dir=/home/khodayarim/PycharmProjects/DeepGauge-ML-Demo/CustomEstimator/modules/ensemble_modules/trainer_from_storage/logs/exported_model
+
+
+
+
+
diff --git a/DeepGauge_ML-Demo.ipynb → CustomEstimator/DeepGauge_ML-Demo.ipynb b/DeepGauge_ML-Demo.ipynb → CustomEstimator/DeepGauge_ML-Demo.ipynb
diff --git a/README.md → CustomEstimator/README.md b/README.md → CustomEstimator/README.md
diff --git a/modules/__init__.py → CustomEstimator/__init__.py b/modules/__init__.py → CustomEstimator/__init__.py
diff --git a/customEstimator.py → CustomEstimator/customEstimator.py b/customEstimator.py → CustomEstimator/customEstimator.py
diff --git a/gaugeReader.ipynb → CustomEstimator/gaugeReader.ipynb b/gaugeReader.ipynb → CustomEstimator/gaugeReader.ipynb
diff --git a/modules/ensemble_modules/__init__.py → CustomEstimator/modules/__init__.py b/modules/ensemble_modules/__init__.py → CustomEstimator/modules/__init__.py
diff --git a/modules/__init__.pyc → CustomEstimator/modules/__init__.pyc b/modules/__init__.pyc → CustomEstimator/modules/__init__.pyc
diff --git a/modules/__pycache__/__init__.cpython-36.pyc → ...dules/__pycache__/__init__.cpython-36.pyc b/modules/__pycache__/__init__.cpython-36.pyc → ...dules/__pycache__/__init__.cpython-36.pyc
diff --git a/modules/ensemble_modules/trainer/__init__.py → ...ator/modules/ensemble_modules/__init__.py b/modules/ensemble_modules/trainer/__init__.py → ...ator/modules/ensemble_modules/__init__.py
diff --git a/...__/CustomEnsembleEstimator.cpython-36.pyc → ...__/CustomEnsembleEstimator.cpython-36.pyc b/...__/CustomEnsembleEstimator.cpython-36.pyc → ...__/CustomEnsembleEstimator.cpython-36.pyc
diff --git a/...semble_modules/CustomEnsembleEstimator.py → ...dules/ensemble/CustomEnsembleEstimator.py b/...semble_modules/CustomEnsembleEstimator.py → ...dules/ensemble/CustomEnsembleEstimator.py
diff --git a/..._modules/trainer_from_storage/__init__.py → ...les/ensemble_modules/ensemble/__init__.py b/..._modules/trainer_from_storage/__init__.py → ...les/ensemble_modules/ensemble/__init__.py
diff --git a/modules/ensemble_modules/ensemble_build.py → ...semble_modules/ensemble/ensemble_build.py b/modules/ensemble_modules/ensemble_build.py → ...semble_modules/ensemble/ensemble_build.py
@@ -1,15 +1,18 @@
 import glob
+import numpy as np
 import os
-from tensorflow.python.framework import meta_graph
 import tensorflow as tf
-from modules import LoadImg
 import time
-import numpy as np
+from tensorflow.python.framework import meta_graph
+
+from CustomEstimator.modules.primary_models_modules import LoadImg
 
 X_train, X_test, y_train, y_test, cls_indices = LoadImg.Dataset.prep_datasets(
-    ver_ratio=0.2, container_path='data/ImageEveryUnit',
+    ver_ratio=0.2, container_path='./CustomEstimator/data/ImageEveryUnit',
     final_img_width=224, final_img_height=224,
-    color_mode="grayscale", random_state=1911, is_trial=True)
+    color_mode="grayscale", random_state=1911,
+    is_trial=True,
+    bin_path='./CustomEstimator/modules/ensemble_modules/ensemble/bin/')
 
 
 class Ensemble():
@@ -189,9 +192,9 @@ def new_fc_layer(inp,
         return raw_imgs, logits_fc
 
 
-def build_model_and_train(primary_models_directory='./logs/primary_models/',
-                          writer_path='./trial/writer',
-                          save_model_path='./trial/best_model_main',
+def build_model_and_train(primary_models_directory='./CustomEstimator/logs/primary_models/',
+                          writer_path='./CustomEstimator/modules/ensemble_modules/ensemble/trial/writer',
+                          save_model_path='./CustomEstimator/modules/ensemble_modules/ensemble/trial/best_model_main',
                           images_shape=[None, 224, 224, 3],
                           hidden_units=[500, 100],
                           X_train=X_train, X_test=X_test,
@@ -203,77 +206,101 @@ def build_model_and_train(primary_models_directory='./logs/primary_models/',
         n_output=y_train.shape[1],
         primary_models_directory=primary_models_directory,
         images_shape=images_shape)
-    tf.reset_default_graph()
-    graph = tf.get_default_graph()
-
-    ## y
-    y_true_tf = tf.placeholder(tf.float64, shape=[None, y_train.shape[1]],
-                               name='y_true_tf')
-
-    y_pred = tf.nn.softmax(logits_fc)
-    y_pred_cls = tf.argmax(y_pred, axis=1)
-    y_true_cls = tf.argmax(y_true_tf, axis=1)
-    correct_prediction = tf.equal(y_pred_cls, y_true_cls)
-    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
-
-    ## dataset
-    dataset = tf.data.Dataset.from_tensor_slices((raw_imgs, y_true_tf))
-    dataset = dataset.shuffle(buffer_size=100)
-    dataset = dataset.batch(batch_size)
-    dataset = dataset.repeat(epochs)
-    iter = dataset.make_initializable_iterator()
-    # iter = dataset.make_one_shot_iterator()
-    get_batch = iter.get_next()
-
-    ## cost
-    cost = tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits_fc,
-                                                      labels=y_true_tf,
-                                                      name='cost_fc')
-
-    trainable_variables = [v for v in tf.trainable_variables() if 'ensemble' in v.name]
-    optimizer = tf.train.AdamOptimizer(learning_rate=3e-4, name='adam_fc'). \
-        minimize(cost, var_list=trainable_variables)
-    # variables_list = [n.name for n in tf.get_default_graph().as_graph_def().node if 'raw_imgs' in n.name]
-
-    feed_dict_train = {raw_imgs: X_train,
-                       y_true_tf: y_train}
-
-    feed_dict_test = {raw_imgs: X_test,
-                      y_true_tf: y_test}
+    # tf.reset_default_graph()
+    # graph = tf.get_default_graph()
+    #
+    # ## y
+    # y_true_tf = tf.placeholder(tf.float64, shape=[None, y_train.shape[1]],
+    #                            name='y_true_tf')
+    #
+    # y_pred = tf.nn.softmax(logits_fc)
+    # y_pred_cls = tf.argmax(y_pred, axis=1)
+    # y_true_cls = tf.argmax(y_true_tf, axis=1)
+    # correct_prediction = tf.equal(y_pred_cls, y_true_cls)
+    # accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
+    #
+    # ## dataset
+    # dataset = tf.data.Dataset.from_tensor_slices((raw_imgs, y_true_tf))
+    # dataset = dataset.shuffle(buffer_size=100)
+    # dataset = dataset.batch(batch_size)
+    # dataset = dataset.repeat(epochs)
+    # iter = dataset.make_initializable_iterator()
+    # # iter = dataset.make_one_shot_iterator()
+    # get_batch = iter.get_next()
+    #
+    # ## cost
+    # cost = tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits_fc,
+    #                                                   labels=y_true_tf,
+    #                                                   name='cost_fc')
+    #
+    # trainable_variables = [v for v in tf.trainable_variables() if 'ensemble' in v.name]
+    # optimizer = tf.train.AdamOptimizer(learning_rate=3e-4, name='adam_fc'). \
+    #     minimize(cost, var_list=trainable_variables)
+    # # variables_list = [n.name for n in tf.get_default_graph().as_graph_def().node if 'raw_imgs' in n.name]
+    #
+    # feed_dict_train = {raw_imgs: X_train,
+    #                    y_true_tf: y_train}
+    #
+    # feed_dict_test = {raw_imgs: X_test,
+    #                   y_true_tf: y_test}
 
     # Start-time used for printing time-usage below.
-    start_time = time.time()
-
-    with tf.Session(graph=graph) as session:
-        # initialize weights and biases variables
-        session.run(tf.global_variables_initializer())
-        session.run(iter.initializer, feed_dict=feed_dict_train)
-        writer = tf.summary.FileWriter(writer_path, session.graph)
-
-        best_validation_accuracy=0
-
-        for i in range(200):
-            try:
-                print('batch {} ... '.format(i))
-                Xydata = session.run(get_batch)
-                session.run(optimizer, feed_dict={raw_imgs: Xydata[0],
-                                                  y_true_tf: Xydata[1]})
-                accuracy_train = session.run(accuracy, feed_dict=feed_dict_train)
-                print('train_acc = {} ... '.format(accuracy_train))
-                accuracy_validation = session.run(accuracy, feed_dict=feed_dict_test)
-                print('validation_acc = {} ... '.format(accuracy_validation))
-                if accuracy_validation > best_validation_accuracy:
-                    saver = tf.train.Saver(max_to_keep=1)
-                    saver.save(session, save_model_path)
-            except:
-                break
-
-        writer.close()
-
-    end_time = time.time()
-
-    print("Training took {}.".format(end_time-start_time))
+    # start_time = time.time()
+
+    # with tf.Session() as session:
+    #     # initialize weights and biases variables
+    #     session.run(tf.global_variables_initializer())
+        # session.run(iter.initializer, feed_dict=feed_dict_train)
+        # writer = tf.summary.FileWriter(writer_path, session.graph)
+
+        # best_validation_accuracy=0
+    #
+    #     for i in range(200):
+    #         try:
+    #             print('batch {} ... '.format(i))
+    #             Xydata = session.run(get_batch)
+    #             session.run(optimizer, feed_dict={raw_imgs: Xydata[0],
+    #                                               y_true_tf: Xydata[1]})
+    #             accuracy_train = session.run(accuracy, feed_dict=feed_dict_train)
+    #             print('train_acc = {} ... '.format(accuracy_train))
+    #             accuracy_validation = session.run(accuracy, feed_dict=feed_dict_test)
+    #             print('validation_acc = {} ... '.format(accuracy_validation))
+    #             if accuracy_validation > best_validation_accuracy:
+    #                 saver = tf.train.Saver(max_to_keep=1)
+    #                 saver.save(session, save_model_path)
+    #         except:
+    #             break
+    #
+    #     writer.close()
+    #
+    # end_time = time.time()
+    #
+    # print("Training took {}.".format(end_time-start_time))
 
     return
 
-build_model_and_train()
+build_model_and_train()
+
+graph_pred = tf.get_default_graph()
+a=tf.trainable_variables()
+trainable_variables = [v for v in tf.trainable_variables() if 'logits_tf' in v.name]
+
+
+
+with tf.Session() as session:
+    saver_3 = tf.train.import_meta_graph('./CustomEstimator/modules/ensemble_modules/ensemble/logs/primary_models/1st/best_model_main.meta',
+                                         clear_devices=True)
+    saver_3.restore(tf.get_default_session(),
+                    tf.train.latest_checkpoint('./CustomEstimator/modules/ensemble_modules/ensemble/logs/primary_models/1st/'))
+    graph_pred = tf.get_default_graph()
+    raw_imgs = graph_pred.get_tensor_by_name("raw_imgs:0")
+    logits_tf = graph_pred.get_tensor_by_name("logits_tf:0")
+    b = tf.trainable_variables()
+
+    c=b[0].eval()
+
+    ##
+    feed_dict_pred = {raw_imgs: X_train}
+
+    ##
+    # logits = session.run(b[0])
diff --git a/modules/ensemble_modules/funcs_backup.py → ...ensemble_modules/ensemble/funcs_backup.py b/modules/ensemble_modules/funcs_backup.py → ...ensemble_modules/ensemble/funcs_backup.py
diff --git a/...nsemble_modules/obsolete_EnsenmbleData.py → ...odules/ensemble/obsolete_EnsenmbleData.py b/...nsemble_modules/obsolete_EnsenmbleData.py → ...odules/ensemble/obsolete_EnsenmbleData.py
diff --git a/...odules/trainer/CustomEnsembleEstimator.py → ...er_from_memory/CustomEnsembleEstimator.py b/...odules/trainer/CustomEnsembleEstimator.py → ...er_from_memory/CustomEnsembleEstimator.py
diff --git a/modules/LoadImg.py → ...le_modules/trainer_from_memory/LoadImg.py b/modules/LoadImg.py → ...le_modules/trainer_from_memory/LoadImg.py
diff --git a/hyperParamOptimizer.py → ...e_modules/trainer_from_memory/__init__.py b/hyperParamOptimizer.py → ...e_modules/trainer_from_memory/__init__.py
diff --git a/CustomEstimator/modules/ensemble_modules/trainer_from_storage/__init__.py b/CustomEstimator/modules/ensemble_modules/trainer_from_storage/__init__.py
diff --git a/...orage/__pycache__/__init__.cpython-36.pyc → ...orage/__pycache__/__init__.cpython-36.pyc b/...orage/__pycache__/__init__.cpython-36.pyc → ...orage/__pycache__/__init__.cpython-36.pyc
diff --git a/CustomEstimator/modules/ensemble_modules/trainer_from_storage/bucket_data.sh b/CustomEstimator/modules/ensemble_modules/trainer_from_storage/bucket_data.sh
@@ -0,0 +1,7 @@
+BUCKET_NAME="tt_ttt"
+echo $BUCKET_NAME
+REGION=us-central1
+gsutil mb -l $REGION gs://$BUCKET_NAME/
+
+## to copy the data folder to your Cloud Storage bucket.
+gsutil cp -r data gs://$BUCKET_NAME/data
diff --git a/CustomEstimator/modules/ensemble_modules/trainer_from_storage/misc/__init__.py b/CustomEstimator/modules/ensemble_modules/trainer_from_storage/misc/__init__.py
diff --git a/...omEstimator/modules/ensemble_modules/trainer_from_storage/misc/exported_model/__init__.py b/...omEstimator/modules/ensemble_modules/trainer_from_storage/misc/exported_model/__init__.py
diff --git a/CustomEstimator/modules/ensemble_modules/trainer_from_storage/misc/logs/__init__.py b/CustomEstimator/modules/ensemble_modules/trainer_from_storage/misc/logs/__init__.py
diff --git a/CustomEstimator/modules/ensemble_modules/trainer_from_storage/misc/logs/dumps/__init__.py b/CustomEstimator/modules/ensemble_modules/trainer_from_storage/misc/logs/dumps/__init__.py
diff --git a/...timator/modules/ensemble_modules/trainer_from_storage/misc/primary_models/1st/__init__.py b/...timator/modules/ensemble_modules/trainer_from_storage/misc/primary_models/1st/__init__.py
diff --git a/..._good/best_model_main.data-00000-of-00001 → ...s/1st/best_model_main.data-00000-of-00001 b/..._good/best_model_main.data-00000-of-00001 → ...s/1st/best_model_main.data-00000-of-00001
diff --git a/...s_original/1st_good/best_model_main.index → .../primary_models/1st/best_model_main.index b/...s_original/1st_good/best_model_main.index → .../primary_models/1st/best_model_main.index
diff --git a/...ls_original/1st_good/best_model_main.meta → ...c/primary_models/1st/best_model_main.meta b/...ls_original/1st_good/best_model_main.meta → ...c/primary_models/1st/best_model_main.meta
diff --git a/logs/models/main/checkpoint → ...torage/misc/primary_models/1st/checkpoint b/logs/models/main/checkpoint → ...torage/misc/primary_models/1st/checkpoint
diff --git a/...timator/modules/ensemble_modules/trainer_from_storage/misc/primary_models/2nd/__init__.py b/...timator/modules/ensemble_modules/trainer_from_storage/misc/primary_models/2nd/__init__.py
diff --git a/..._good/best_model_main.data-00000-of-00001 → ...s/2nd/best_model_main.data-00000-of-00001 b/..._good/best_model_main.data-00000-of-00001 → ...s/2nd/best_model_main.data-00000-of-00001
diff --git a/...ary_models/2nd_good/best_model_main.index → .../primary_models/2nd/best_model_main.index b/...ary_models/2nd_good/best_model_main.index → .../primary_models/2nd/best_model_main.index
diff --git a/...mary_models/2nd_good/best_model_main.meta → ...c/primary_models/2nd/best_model_main.meta b/...mary_models/2nd_good/best_model_main.meta → ...c/primary_models/2nd/best_model_main.meta
diff --git a/logs/primary_models/2nd_good/checkpoint → ...torage/misc/primary_models/2nd/checkpoint b/logs/primary_models/2nd_good/checkpoint → ...torage/misc/primary_models/2nd/checkpoint
diff --git a/...timator/modules/ensemble_modules/trainer_from_storage/misc/primary_models/3rd/__init__.py b/...timator/modules/ensemble_modules/trainer_from_storage/misc/primary_models/3rd/__init__.py
diff --git a/..._good/best_model_main.data-00000-of-00001 → ...s/3rd/best_model_main.data-00000-of-00001 b/..._good/best_model_main.data-00000-of-00001 → ...s/3rd/best_model_main.data-00000-of-00001
diff --git a/...s_original/3rd_good/best_model_main.index → .../primary_models/3rd/best_model_main.index b/...s_original/3rd_good/best_model_main.index → .../primary_models/3rd/best_model_main.index
diff --git a/...ls_original/3rd_good/best_model_main.meta → ...c/primary_models/3rd/best_model_main.meta b/...ls_original/3rd_good/best_model_main.meta → ...c/primary_models/3rd/best_model_main.meta
diff --git a/...imary_models_original/3rd_good/checkpoint → ...torage/misc/primary_models/3rd/checkpoint b/...imary_models_original/3rd_good/checkpoint → ...torage/misc/primary_models/3rd/checkpoint
diff --git a/..._original/3rd_good/hyper_params_main.json → ...primary_models/3rd/hyper_params_main.json b/..._original/3rd_good/hyper_params_main.json → ...primary_models/3rd/hyper_params_main.json
diff --git a/...timator/modules/ensemble_modules/trainer_from_storage/misc/primary_models/4th/__init__.py b/...timator/modules/ensemble_modules/trainer_from_storage/misc/primary_models/4th/__init__.py
diff --git a/..._good/best_model_main.data-00000-of-00001 → ...s/4th/best_model_main.data-00000-of-00001 b/..._good/best_model_main.data-00000-of-00001 → ...s/4th/best_model_main.data-00000-of-00001
diff --git a/...s_original/4th_good/best_model_main.index → .../primary_models/4th/best_model_main.index b/...s_original/4th_good/best_model_main.index → .../primary_models/4th/best_model_main.index
diff --git a/...ls_original/4th_good/best_model_main.meta → ...c/primary_models/4th/best_model_main.meta b/...ls_original/4th_good/best_model_main.meta → ...c/primary_models/4th/best_model_main.meta
diff --git a/logs/primary_models/8th_good/checkpoint → ...torage/misc/primary_models/4th/checkpoint b/logs/primary_models/8th_good/checkpoint → ...torage/misc/primary_models/4th/checkpoint
diff --git a/..._original/4th_good/hyper_params_main.json → ...primary_models/4th/hyper_params_main.json b/..._original/4th_good/hyper_params_main.json → ...primary_models/4th/hyper_params_main.json
diff --git a/...timator/modules/ensemble_modules/trainer_from_storage/misc/primary_models/5th/__init__.py b/...timator/modules/ensemble_modules/trainer_from_storage/misc/primary_models/5th/__init__.py
diff --git a/..._good/best_model_main.data-00000-of-00001 → ...s/5th/best_model_main.data-00000-of-00001 b/..._good/best_model_main.data-00000-of-00001 → ...s/5th/best_model_main.data-00000-of-00001
diff --git a/...s_original/5th_good/best_model_main.index → .../primary_models/5th/best_model_main.index b/...s_original/5th_good/best_model_main.index → .../primary_models/5th/best_model_main.index
diff --git a/...ls_original/5th_good/best_model_main.meta → ...c/primary_models/5th/best_model_main.meta b/...ls_original/5th_good/best_model_main.meta → ...c/primary_models/5th/best_model_main.meta
diff --git a/...imary_models_original/1st_good/checkpoint → ...torage/misc/primary_models/5th/checkpoint b/...imary_models_original/1st_good/checkpoint → ...torage/misc/primary_models/5th/checkpoint
diff --git a/..._original/5th_good/hyper_params_main.json → ...primary_models/5th/hyper_params_main.json b/..._original/5th_good/hyper_params_main.json → ...primary_models/5th/hyper_params_main.json
diff --git a/...timator/modules/ensemble_modules/trainer_from_storage/misc/primary_models/6th/__init__.py b/...timator/modules/ensemble_modules/trainer_from_storage/misc/primary_models/6th/__init__.py
diff --git a/..._good/best_model_main.data-00000-of-00001 → ...s/6th/best_model_main.data-00000-of-00001 b/..._good/best_model_main.data-00000-of-00001 → ...s/6th/best_model_main.data-00000-of-00001
diff --git a/...s_original/6th_good/best_model_main.index → .../primary_models/6th/best_model_main.index b/...s_original/6th_good/best_model_main.index → .../primary_models/6th/best_model_main.index
diff --git a/...ls_original/6th_good/best_model_main.meta → ...c/primary_models/6th/best_model_main.meta b/...ls_original/6th_good/best_model_main.meta → ...c/primary_models/6th/best_model_main.meta
diff --git a/...imary_models_original/2nd_good/checkpoint → ...torage/misc/primary_models/6th/checkpoint b/...imary_models_original/2nd_good/checkpoint → ...torage/misc/primary_models/6th/checkpoint
diff --git a/..._original/6th_good/hyper_params_main.json → ...primary_models/6th/hyper_params_main.json b/..._original/6th_good/hyper_params_main.json → ...primary_models/6th/hyper_params_main.json
diff --git a/...timator/modules/ensemble_modules/trainer_from_storage/misc/primary_models/7th/__init__.py b/...timator/modules/ensemble_modules/trainer_from_storage/misc/primary_models/7th/__init__.py
diff --git a/..._good/best_model_main.data-00000-of-00001 → ...s/7th/best_model_main.data-00000-of-00001 b/..._good/best_model_main.data-00000-of-00001 → ...s/7th/best_model_main.data-00000-of-00001
diff --git a/...s_original/7th_good/best_model_main.index → .../primary_models/7th/best_model_main.index b/...s_original/7th_good/best_model_main.index → .../primary_models/7th/best_model_main.index
diff --git a/...ls_original/7th_good/best_model_main.meta → ...c/primary_models/7th/best_model_main.meta b/...ls_original/7th_good/best_model_main.meta → ...c/primary_models/7th/best_model_main.meta
diff --git a/...imary_models_original/4th_good/checkpoint → ...torage/misc/primary_models/7th/checkpoint b/...imary_models_original/4th_good/checkpoint → ...torage/misc/primary_models/7th/checkpoint
diff --git a/..._original/7th_good/hyper_params_main.json → ...primary_models/7th/hyper_params_main.json b/..._original/7th_good/hyper_params_main.json → ...primary_models/7th/hyper_params_main.json
diff --git a/...timator/modules/ensemble_modules/trainer_from_storage/misc/primary_models/8th/__init__.py b/...timator/modules/ensemble_modules/trainer_from_storage/misc/primary_models/8th/__init__.py
diff --git a/.../main/best_model_main.data-00000-of-00001 → ...s/8th/best_model_main.data-00000-of-00001 b/.../main/best_model_main.data-00000-of-00001 → ...s/8th/best_model_main.data-00000-of-00001
diff --git a/logs/models/main/best_model_main.index → .../primary_models/8th/best_model_main.index b/logs/models/main/best_model_main.index → .../primary_models/8th/best_model_main.index
diff --git a/logs/models/main/best_model_main.meta → ...c/primary_models/8th/best_model_main.meta b/logs/models/main/best_model_main.meta → ...c/primary_models/8th/best_model_main.meta
diff --git a/...imary_models_original/5th_good/checkpoint → ...torage/misc/primary_models/8th/checkpoint b/...imary_models_original/5th_good/checkpoint → ...torage/misc/primary_models/8th/checkpoint
diff --git a/logs/models/main/hyper_params_main.json → ...primary_models/8th/hyper_params_main.json b/logs/models/main/hyper_params_main.json → ...primary_models/8th/hyper_params_main.json
diff --git a/...omEstimator/modules/ensemble_modules/trainer_from_storage/misc/primary_models/__init__.py b/...omEstimator/modules/ensemble_modules/trainer_from_storage/misc/primary_models/__init__.py
diff --git a/...Estimator/modules/ensemble_modules/trainer_from_storage/misc/temporary_models/__init__.py b/...Estimator/modules/ensemble_modules/trainer_from_storage/misc/temporary_models/__init__.py
diff --git a/CustomEstimator/modules/ensemble_modules/trainer_from_storage/setup.py b/CustomEstimator/modules/ensemble_modules/trainer_from_storage/setup.py
@@ -0,0 +1,24 @@
+from setuptools import find_packages
+from setuptools import setup
+
+REQUIRED_PACKAGES = ['argparse',
+                     'tensorflow==1.10.0',
+                     'glob',
+                     'os',
+                     'sklearn==1.19.2',
+                     'numpy==1.14.5',
+                     'pandas==0.23.4',
+                     'MultiColProcessor',
+                     'pickle',
+                     'json',
+                     'collections',
+                     'multiprocessing']
+
+setup(
+    name='deepGauge_custom_estimator',
+    version='0.1',
+    install_requires=REQUIRED_PACKAGES,
+    packages=find_packages(),
+    include_package_data=True,
+    description=''
+)
diff --git a/CustomEstimator/modules/ensemble_modules/trainer_from_storage/train.sh b/CustomEstimator/modules/ensemble_modules/trainer_from_storage/train.sh
@@ -0,0 +1,28 @@
+gcloud ml-engine jobs submit training "$JOB_ID" \
+  --stream-logs \
+  --module-name trainer.task \
+  --package-path trainer \
+  --staging-bucket "$BUCKET" \
+  --region us-central1 \
+  --runtime-version=1.4 \
+  -- \
+  --output_path "${GCS_PATH}/training" \
+  --eval_data_paths "${GCS_PATH}/preproc/eval*" \
+  --train_data_paths "${GCS_PATH}/preproc/train*"
+
+
+##
+REGION=us-central1
+JOB_NAME=deep_gauge_1
+BUCKET=gs://deep_gauge
+PACKAGE_PATH=CustomEstimator/modules/ensemble_modules/trainer_from_storage/trainer
+
+
+gcloud ml-engine jobs submit training $JOB_NAME \
+    --runtime-version 1.8 \
+    --module-name trainer.task \
+    --package-path "$PACKAGE_PATH" \
+    --staging-bucket "$BUCKET" \
+    --region $REGION \
+    -- \
+    --verbosity DEBUG
diff --git a/CustomEstimator/modules/ensemble_modules/trainer_from_storage/trainer/__init__.py b/CustomEstimator/modules/ensemble_modules/trainer_from_storage/trainer/__init__.py