[Deployment] Choose services for different cluster type (#3528)

* Choose services for different cluster type Choose services for different cluster type in deployment. * Update cluster type in service.yaml Update cluster type in service.yaml. * Change the initial cluster type to None Change the initial cluster type to None.
microsoft · Sep 3, 2019 · a1332d3 · a1332d3
1 parent 18414d9
commit a1332d3
Show file tree

Hide file tree

Showing 28 changed files with 106 additions and 29 deletions.
diff --git a/deployment/paiLibrary/paiService/service_management_configuration.py b/deployment/paiLibrary/paiService/service_management_configuration.py
@@ -14,6 +14,7 @@
 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
 import time
 import os
 import tempfile
@@ -41,12 +42,14 @@ def get_cluster_object_model_from_k8s(kube_config_path):
     return cluster_object_service
 
 
-def get_service_list():
+def get_service_list(cluster_type="yarn"):
     service_list = list()
     subdir_list = directory_handler.get_subdirectory_list("src/")
     for subdir in subdir_list:
         service_deploy_dir = "src/{0}/deploy".format(subdir)
         service_deploy_conf_path = "src/{0}/deploy/service.yaml".format(subdir)
         if file_handler.directory_exits(service_deploy_dir) and file_handler.file_exist_or_not(service_deploy_conf_path):
-            service_list.append(subdir)
+            service_conf = file_handler.load_yaml_config(service_deploy_conf_path)
+            if ("cluster-type" not in service_conf) or ("cluster-type" in service_conf and cluster_type in service_conf["cluster-type"]):
+                service_list.append(subdir)
     return service_list
diff --git a/deployment/paiLibrary/paiService/service_management_delete.py b/deployment/paiLibrary/paiService/service_management_delete.py
@@ -34,12 +34,14 @@ class service_management_delete:
     def __init__(self, kube_config_path=None, service_list=None, **kwargs):
         self.logger = logging.getLogger(__name__)
 
-        self.cluster_object_model = None
-
+        self.cluster_object_model = service_management_configuration.get_cluster_object_model_from_k8s(kube_config_path)
         self.kube_config_path = kube_config_path
+        self.cluster_type = None
 
         if service_list is None:
-            self.service_list = service_management_configuration.get_service_list()
+            if "cluster-type" in self.cluster_object_model["cluster"]["common"]:
+                self.cluster_type = self.cluster_object_model["cluster"]["common"]["cluster-type"]
+            self.service_list = service_management_configuration.get_service_list(self.cluster_type)
         else:
             self.service_list = service_list
         self.logger.info("Get the service-list to manage : {0}".format(str(self.service_list)))
@@ -75,9 +77,6 @@ def start(self, serv):
         self.logger.info("----------------------------------------------------------------------")
 
     def run(self):
-
-        self.cluster_object_model = service_management_configuration.get_cluster_object_model_from_k8s(kube_config_path=self.kube_config_path)
-
         for serv in self.service_list:
             if serv == "cluster-configuration":
                 continue

diff --git a/deployment/paiLibrary/paiService/service_management_refresh.py b/deployment/paiLibrary/paiService/service_management_refresh.py
@@ -33,14 +33,14 @@ class service_management_refresh:
     def __init__(self, kube_config_path=None, service_list=None, **kwargs):
         self.logger = logging.getLogger(__name__)
 
-        self.cluster_object_model = None
-
-        self.kube_config_path = None
-        if kube_config_path != None:
-            self.kube_config_path = kube_config_path
+        self.cluster_object_model = service_management_configuration.get_cluster_object_model_from_k8s(kube_config_path)
+        self.kube_config_path = kube_config_path
+        self.cluster_type = None
 
         if service_list is None:
-            self.service_list = service_management_configuration.get_service_list()
+            if "cluster-type" in self.cluster_object_model["cluster"]["common"]:
+                self.cluster_type = self.cluster_object_model["cluster"]["common"]["cluster-type"]
+            self.service_list = service_management_configuration.get_service_list(self.cluster_type)
         else:
             self.service_list = service_list
         self.logger.info("Get the service-list to manage : {0}".format(str(self.service_list)))
@@ -156,9 +156,6 @@ def start(self, serv):
         self.done_dict[serv] = True
 
     def run(self):
-
-        self.cluster_object_model = service_management_configuration.get_cluster_object_model_from_k8s(kube_config_path=self.kube_config_path)
-
         self.done_dict = dict()
         self.refresh_all_label()
         for serv in self.service_list:

diff --git a/deployment/paiLibrary/paiService/service_management_start.py b/deployment/paiLibrary/paiService/service_management_start.py
@@ -34,12 +34,14 @@ class serivce_management_start:
     def __init__(self, kube_config_path=None, service_list=None, **kwargs):
         self.logger = logging.getLogger(__name__)
 
-        self.cluster_object_model = None
-
+        self.cluster_object_model = service_management_configuration.get_cluster_object_model_from_k8s(kube_config_path)
         self.kube_config_path = kube_config_path
+        self.cluster_type = None
 
         if service_list is None:
-            self.service_list = service_management_configuration.get_service_list()
+            if "cluster-type" in self.cluster_object_model["cluster"]["common"]:
+                self.cluster_type = self.cluster_object_model["cluster"]["common"]["cluster-type"]
+            self.service_list = service_management_configuration.get_service_list(self.cluster_type)
         else:
             self.service_list = service_list
         self.logger.info("Get the service-list to manage : {0}".format(str(self.service_list)))
@@ -111,9 +113,6 @@ def start(self, serv):
         self.done_dict[serv] = True
 
     def run(self):
-
-        self.cluster_object_model = service_management_configuration.get_cluster_object_model_from_k8s(kube_config_path=self.kube_config_path)
-
         self.done_dict = dict()
 
         for serv in self.service_list:

diff --git a/deployment/paiLibrary/paiService/service_management_stop.py b/deployment/paiLibrary/paiService/service_management_stop.py
@@ -33,12 +33,14 @@ class service_management_stop:
     def __init__(self, kube_config_path=None, service_list=None, **kwargs):
         self.logger = logging.getLogger(__name__)
 
-        self.cluster_object_model = None
-
+        self.cluster_object_model = service_management_configuration.get_cluster_object_model_from_k8s(kube_config_path)
         self.kube_config_path = kube_config_path
+        self.cluster_type = None
 
         if service_list is None:
-            self.service_list = service_management_configuration.get_service_list()
+            if "cluster-type" in self.cluster_object_model["cluster"]["common"]:
+                self.cluster_type = self.cluster_object_model["cluster"]["common"]["cluster-type"]
+            self.service_list = service_management_configuration.get_service_list(self.cluster_type)
         else:
             self.service_list = service_list
         self.logger.info("Get the service-list to manage : {0}".format(str(self.service_list)))
@@ -75,9 +77,6 @@ def start(self, serv):
         self.logger.info("----------------------------------------------------------------------")
 
     def run(self):
-
-        self.cluster_object_model = service_management_configuration.get_cluster_object_model_from_k8s(kube_config_path=self.kube_config_path)
-
         for serv in self.service_list:
             if serv == "cluster-configuration":
                 continue

diff --git a/src/alert-manager/deploy/service.yaml b/src/alert-manager/deploy/service.yaml
@@ -15,6 +15,10 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+cluster-type:
+  - yarn
+  - k8s
+
 prerequisite:
   - cluster-configuration
   - drivers

diff --git a/src/cleaner/deploy/service.yaml b/src/cleaner/deploy/service.yaml
@@ -15,6 +15,9 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+cluster-type:
+  - yarn
+
 # to avoid possible race condition, start cleaner after all services are ready
 prerequisite:
   - cluster-configuration

diff --git a/src/cluster-configuration/deploy/service.yaml b/src/cluster-configuration/deploy/service.yaml
@@ -15,6 +15,10 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+cluster-type:
+  - yarn
+  - k8s
+
 template-list:
   - secret.yaml
   - host-configuration/host-configuration.yaml

diff --git a/src/drivers/deploy/service.yaml b/src/drivers/deploy/service.yaml
@@ -15,6 +15,10 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+cluster-type:
+  - yarn
+  - k8s
+
 prerequisite:
   - cluster-configuration
 

diff --git a/src/end-to-end-test/deploy/service.yaml b/src/end-to-end-test/deploy/service.yaml
@@ -15,6 +15,9 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+cluster-type:
+  - yarn
+
 prerequisite:
   - cluster-configuration
   - yarn-frameworklauncher

diff --git a/src/grafana/deploy/service.yaml b/src/grafana/deploy/service.yaml
@@ -15,6 +15,10 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+cluster-type:
+  - yarn
+  - k8s
+
 prerequisite:
   - cluster-configuration
   - prometheus

diff --git a/src/hadoop-batch-job/deploy/service.yaml b/src/hadoop-batch-job/deploy/service.yaml
@@ -15,6 +15,9 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+cluster-type:
+  - yarn
+
 prerequisite:
   - cluster-configuration
   - drivers

diff --git a/src/hadoop-data-node/deploy/service.yaml b/src/hadoop-data-node/deploy/service.yaml
@@ -15,6 +15,9 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+cluster-type:
+  - yarn
+
 prerequisite:
   - cluster-configuration
   - drivers

diff --git a/src/hadoop-jobhistory/deploy/service.yaml b/src/hadoop-jobhistory/deploy/service.yaml
@@ -15,6 +15,9 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+cluster-type:
+  - yarn
+
 prerequisite:
   - cluster-configuration
   - drivers

diff --git a/src/hadoop-name-node/deploy/service.yaml b/src/hadoop-name-node/deploy/service.yaml
@@ -15,6 +15,9 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+cluster-type:
+  - yarn
+
 prerequisite:
   - cluster-configuration
   - drivers

diff --git a/src/hadoop-node-manager/deploy/service.yaml b/src/hadoop-node-manager/deploy/service.yaml
@@ -15,6 +15,9 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+cluster-type:
+  - yarn
+
 prerequisite:
   - cluster-configuration
   - drivers

diff --git a/src/hadoop-resource-manager/deploy/service.yaml b/src/hadoop-resource-manager/deploy/service.yaml
@@ -15,6 +15,9 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+cluster-type:
+  - yarn
+
 prerequisite:
   - cluster-configuration
   - drivers

diff --git a/src/job-exporter/deploy/service.yaml b/src/job-exporter/deploy/service.yaml
@@ -15,6 +15,10 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+cluster-type:
+  - yarn
+  - k8s
+
 prerequisite:
   - cluster-configuration
   - drivers

diff --git a/src/k8s-dashboard/deploy/service.yaml b/src/k8s-dashboard/deploy/service.yaml
@@ -15,6 +15,10 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+cluster-type:
+  - yarn
+  - k8s
+
 template-list:
   - k8s-dashboard.yaml
   - start.sh

diff --git a/src/log-manager/deploy/service.yaml b/src/log-manager/deploy/service.yaml
@@ -15,6 +15,9 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+cluster-type:
+  - k8s
+
 prerequisite:
   - cluster-configuration
 

diff --git a/src/node-exporter/deploy/service.yaml b/src/node-exporter/deploy/service.yaml
@@ -15,6 +15,10 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+cluster-type:
+  - yarn
+  - k8s
+
 prerequisite:
   - cluster-configuration
 

diff --git a/src/prometheus/deploy/service.yaml b/src/prometheus/deploy/service.yaml
@@ -15,6 +15,10 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+cluster-type:
+  - yarn
+  - k8s
+
 prerequisite:
   - cluster-configuration
 

diff --git a/src/pylon/deploy/service.yaml b/src/pylon/deploy/service.yaml
@@ -15,6 +15,10 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+cluster-type:
+  - yarn
+  - k8s
+
 prerequisite:
   - cluster-configuration
 

diff --git a/src/rest-server/deploy/service.yaml b/src/rest-server/deploy/service.yaml
@@ -15,6 +15,10 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+cluster-type:
+  - yarn
+  - k8s
+
 prerequisite:
   - cluster-configuration
   - yarn-frameworklauncher

diff --git a/src/watchdog/deploy/service.yaml b/src/watchdog/deploy/service.yaml
@@ -15,6 +15,10 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+cluster-type:
+  - yarn
+  - k8s
+
 prerequisite:
   - cluster-configuration
 

diff --git a/src/webportal/deploy/service.yaml b/src/webportal/deploy/service.yaml
@@ -15,6 +15,10 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+cluster-type:
+  - yarn
+  - k8s
+
 prerequisite:
   - cluster-configuration
   - rest-server

diff --git a/src/yarn-frameworklauncher/deploy/service.yaml b/src/yarn-frameworklauncher/deploy/service.yaml
@@ -15,6 +15,8 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+cluster-type:
+  - yarn
 
 prerequisite:
   - cluster-configuration

diff --git a/src/zookeeper/deploy/service.yaml b/src/zookeeper/deploy/service.yaml
@@ -15,6 +15,9 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+cluster-type:
+  - yarn
+
 prerequisite:
   - cluster-configuration