Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

ACM-6842: SDK changes for creating a cluster with hs2 on master (is_hs2) #320

Merged
merged 5 commits into from
May 6, 2020
Merged

ACM-6842: SDK changes for creating a cluster with hs2 on master (is_hs2) #320

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
1e59c2d
Initial
surajg-q May 5, 2020
b5b3df0
changing help argument name for hive settings
surajg-q May 5, 2020
f07496e
Merge branch 'unreleased' into ACM-6842-1
chattarajoy May 6, 2020
3bc427a
adding test case
surajg-q May 6, 2020
8dbdc9b
Merge branch 'ACM-6842-1' of github.com:surajg-q/qds-sdk-py into ACM-…
surajg-q May 6, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 23 additions & 3 deletions qds_sdk/engine.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ def set_engine_config(self,
spark_version=None,
custom_spark_config=None,
hive_version=None,
is_hs2=None,
hs2_thrift_port=None,
dbtap_id=None,
fernet_key=None,
overrides=None,
Expand Down Expand Up @@ -60,6 +62,10 @@ def set_engine_config(self,

hive_version: Version of hive to be used in cluster

is_hs2: Enable HS2 on master

hs2_thrift_port: Thrift port HS2 on master will run on

dbtap_id: ID of the data store inside QDS

fernet_key: Encryption key for sensitive information inside airflow database.
Expand All @@ -82,7 +88,7 @@ def set_engine_config(self,
self.set_hadoop_settings(custom_hadoop_config, use_qubole_placement_policy,
is_ha, fairscheduler_config_xml,
default_pool, enable_rubix)
self.set_hive_settings(hive_version)
self.set_hive_settings(hive_version, is_hs2, hs2_thrift_port)
self.set_presto_settings(presto_version, custom_presto_config)
self.set_spark_settings(spark_version, custom_spark_config)
self.set_airflow_settings(dbtap_id, fernet_key, overrides, airflow_version, airflow_python_version)
Expand Down Expand Up @@ -110,8 +116,12 @@ def set_hadoop_settings(self,
self.hadoop_settings['enable_rubix'] = enable_rubix

def set_hive_settings(self,
hive_version=None):
hive_version=None,
is_hs2=None,
hs2_thrift_port=None):
self.hive_settings['hive_version'] = hive_version
self.hive_settings['is_hs2'] = is_hs2
self.hive_settings['hs2_thrift_port'] = hs2_thrift_port

def set_presto_settings(self,
presto_version=None,
Expand Down Expand Up @@ -158,6 +168,8 @@ def set_engine_config_settings(self, arguments):
spark_version=arguments.spark_version,
custom_spark_config=arguments.custom_spark_config,
hive_version=arguments.hive_version,
is_hs2=arguments.is_hs2,
hs2_thrift_port=arguments.hs2_thrift_port,
dbtap_id=arguments.dbtap_id,
fernet_key=arguments.fernet_key,
overrides=arguments.overrides,
Expand Down Expand Up @@ -229,11 +241,19 @@ def engine_parser(argparser):
dest="presto_custom_config_file",
help="location of file containg custom" +
" presto configuration overrides")
hive_settings_group = argparser.add_argument_group("hive version settings")
hive_settings_group = argparser.add_argument_group("hive settings")
hive_settings_group.add_argument("--hive_version",
dest="hive_version",
default=None,
help="Version of hive for the cluster",)
hive_settings_group.add_argument("--is_hs2",
dest="is_hs2",
default=None,
help="Enable hs2 on master", )
hive_settings_group.add_argument("--hs2_thrift_port",
dest="hs2_thrift_port",
default=None,
help="thrift port hs2 master will run on", )

spark_settings_group = argparser.add_argument_group("spark settings")
spark_settings_group.add_argument("--spark-version",
Expand Down
19 changes: 19 additions & 0 deletions tests/test_clusterv22.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,3 +211,22 @@ def test_image_version_v22(self):
'slave_instance_type': 'c1.xlarge',
'cluster_image_version': '1.latest',
'composition': {'min_nodes': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'master': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'autoscaling_nodes': {'nodes': [{'percentage': 50, 'type': 'ondemand'}, {'timeout_for_request': 1, 'percentage': 50, 'type': 'spot', 'fallback': 'ondemand', 'maximum_bid_price_percentage': 100, 'allocation_strategy': None}]}}, 'label': ['test_label']}})

def test_hive_settings(self):
sys.argv = ['qds.py', '--version', 'v2.2', 'cluster', 'create', '--label', 'test_label',
'--is_hs2', 'true', '--hs2_thrift_port', '10001']
print_command()
Connection._api_call = Mock(return_value={})
qds.main()
print("Suraj")
print(Connection._api_call)
Connection._api_call.assert_called_with('POST', 'clusters',
{'engine_config': {'hive_settings': {'is_hs2': 'true', 'hs2_thrift_port': '10001'}},
'cluster_info': {'label': ['test_label'],
'composition': {'master': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]},
'min_nodes': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]},
'autoscaling_nodes': {'nodes': [{'percentage': 50, 'type': 'ondemand'},
{'percentage': 50, 'type': 'spot', 'maximum_bid_price_percentage': 100, 'timeout_for_request': 1, 'allocation_strategy': None, 'fallback': 'ondemand'}]}}}})

if __name__ == '__main__':
unittest.main()