Merge branch 'master' into restapi_auto_authorization/resource-manager

.github/CODEOWNERS

@@ -18,6 +18,7 @@
 /azure-mgmt-datalake-store/ @ro-joowan
 /azure-mgmt-datamigration/ @vchske
 /azure-mgmt-eventgrid/ @kalyanaj
+/azure-mgmt-hdinsight/ @wawon-msft
 /azure-mgmt-keyvault/ @schaabs
 /azure-mgmt-machinelearningcompute/ @shutchings
 /azure-mgmt-recoveryservices/ @DheerendraRathor

.gitignore

@@ -2,6 +2,8 @@
 __pycache__/
 *.pyc
 .pytest_cache
+.mypy_cache
+.cache
 
 # Virtual environment
 env*/
@@ -26,7 +28,7 @@ build/
 # Test results
 TestResults/
 
-# Credentials 
+# Credentials
 credentials_real.json
 testsettings_local.json
 testsettings_local.cfg
@@ -66,4 +68,6 @@ src/build
 *.pubxml
 
 # [begoldsm] ignore virtual env if it exists.
-adlEnv/
+adlEnv/
+
+code_reports

.travis.yml

@@ -23,7 +23,7 @@ matrix:
       python: "pypy3.5-5.8.0"
       dist: trusty
     - os: osx
-      osx_image: xcode7.3
+      osx_image: xcode9.4
       language: generic
   allow_failures:
     - os: osx
@@ -34,7 +34,6 @@ matrix:
 # Perform the manual steps on osx to install python3 and activate venv
 before_install:
   - if [[ -n "$TRAVIS_TAG" && "$TRAVIS_PYTHON_VERSION" != "3.6" ]]; then travis_terminate 0; fi; # Deploy on 3.6
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; fi
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew upgrade python; fi
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then python3 -m venv venv; fi
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then source venv/bin/activate; fi

.vscode/settings.json

@@ -2,4 +2,4 @@
     "git.ignoreLimitWarning": true,
     "python.unitTest.pyTestArgs": [],
     "python.unitTest.pyTestEnabled": true
-}
+}

azure-applicationinsights/MANIFEST.in

@@ -1,2 +1 @@
 include *.rst
-include azure_bdist_wheel.py

azure-applicationinsights/README.rst

@@ -6,7 +6,7 @@ This is the Microsoft Azure Application Insights Client Library.
 Azure Resource Manager (ARM) is the next generation of management APIs that
 replace the old Azure Service Management (ASM).
 
-This package has been tested with Python 2.7, 3.4, 3.5 and 3.6.
+This package has been tested with Python 2.7, 3.4, 3.5, 3.6 and 3.7.
 
 For the older Azure Service Management (ASM) libraries, see
 `azure-servicemanagement-legacy <https://pypi.python.org/pypi/azure-servicemanagement-legacy>`__ library.

azure-applicationinsights/azure/__init__.py

@@ -1 +1 @@
-__import__('pkg_resources').declare_namespace(__name__)
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)

azure-applicationinsights/setup.cfg

@@ -1,3 +1,2 @@
 [bdist_wheel]
 universal=1
-azure-namespace-package=azure-nspkg

azure-applicationinsights/setup.py

@@ -10,12 +10,6 @@
 import os.path
 from io import open
 from setuptools import find_packages, setup
-try:
-    from azure_bdist_wheel import cmdclass
-except ImportError:
-    from distutils import log as logger
-    logger.warn("Wheel is not available, disabling bdist_wheel hook")
-    cmdclass = {}
 
 # Change the PACKAGE_NAME only to change folder and different name
 PACKAGE_NAME = "azure-applicationinsights"
@@ -76,10 +70,16 @@
         'License :: OSI Approved :: MIT License',
     ],
     zip_safe=False,
-    packages=find_packages(exclude=["tests"]),
+    packages=find_packages(exclude=[
+        'tests',
+        # Exclude packages that will be covered by PEP420 or nspkg
+        'azure',
+    ]),
     install_requires=[
-        'msrest>=0.5.4,<2.0.0',
+        'msrest>=0.5.0',
         'azure-common~=1.1',
     ],
-    cmdclass=cmdclass
+    extras_require={
+        ":python_version<'3.0'": ['azure-nspkg'],
+    }
 )

azure-batch/HISTORY.rst

@@ -3,6 +3,44 @@
 Release History
 ===============
 
+5.1.1 (2018-10-16)
+++++++++++++++++++
+
+**Bugfixes**
+
+- Fix authentication class to allow HTTP session to be re-used
+
+**Note**
+
+- azure-nspkg is not installed anymore on Python 3 (PEP420-based namespace package)
+
+5.1.0 (2018-08-28)
+++++++++++++++++++
+
+- Update operation TaskOperations.add_collection with the following added functionality:
+    - Retry server side errors.
+    - Automatically chunk lists of more than 100 tasks to multiple requests.
+    - If tasks are too large to be submitted in chunks of 100, reduces number of tasks per request.
+    - Add a parameter to specify number of threads to use when submitting tasks.
+
+5.0.0 (2018-08-24)
+++++++++++++++++++
+
+- Using REST API version 2018-08-01.7.0.
+  - Added `node_agent_info` in ComputeNode to return the node agent information
+  - **Breaking** Removed the `validation_status` property from `TaskCounts`.
+  - **Breaking** The default caching type for `DataDisk` and `OSDisk` is now `read_write` instead of `none`.
+- `BatchServiceClient` can be used as a context manager to keep the underlying HTTP session open for performance.
+- **Breaking** Model signatures are now using only keywords-arguments syntax. Each positional argument must be rewritten as a keyword argument.
+- **Breaking** The following operations signatures are changed:
+   - Operation PoolOperations.enable_auto_scale
+   - Operation TaskOperations.update
+   - Operation ComputeNodeOperations.reimage
+   - Operation ComputeNodeOperations.disable_scheduling
+   - Operation ComputeNodeOperations.reboot
+   - Operation JobOperations.terminate
+- Enum types now use the "str" mixin (class AzureEnum(str, Enum)) to improve the behavior when unrecognized enum values are encountered.
+
 4.1.3 (2018-04-24)
 ++++++++++++++++++
 
@@ -36,10 +74,10 @@ Release History
 - Using REST API version 2017-09-01.6.0.
 - Added the ability to get a discount on Windows VM pricing if you have on-premises licenses for the OS SKUs you are deploying, via `license_type` on `VirtualMachineConfiguration`.
 - Added support for attaching empty data drives to `VirtualMachineConfiguration` based pools, via the new `data_disks` attribute on `VirtualMachineConfiguration`.
-- [Breaking] Custom images must now be deployed using a reference to an ARM Image, instead of pointing to .vhd files in blobs directly.
+- **Breaking** Custom images must now be deployed using a reference to an ARM Image, instead of pointing to .vhd files in blobs directly.
   - The new `virtual_machine_image_id` property on `ImageReference` contains the reference to the ARM Image, and `OSDisk.image_uris` no longer exists.
   - Because of this, `image_reference` is now a required attribute of `VirtualMachineConfiguration`.
-- [Breaking] Multi-instance tasks (created using `MultiInstanceSettings`) must now specify a `coordination_commandLine`, and `number_of_instances` is now optional and defaults to 1.
+- **Breaking** Multi-instance tasks (created using `MultiInstanceSettings`) must now specify a `coordination_commandLine`, and `number_of_instances` is now optional and defaults to 1.
 - Added support for tasks run using Docker containers. To run a task using a Docker container you must specify a `container_configuration` on the `VirtualMachineConfiguration` for a pool, and then add `container_settings` on the Task.
 
 3.1.0 (2017-07-24)
@@ -60,16 +98,16 @@ Release History
 - Added a new `allow_low_priority_node` property to `JobManagerTask`, which if `true` allows the `JobManagerTask` to run on a low-priority compute node.
 - `PoolResizeParameter` now takes two optional parameters, `target_dedicated_nodes` and `target_low_priority_nodes`, instead of one required parameter `target_dedicated`.
   At least one of these two parameters must be specified.
-- Added support for uploading task output files to persistent storage, via the `OutputFiles` property on `CloudTask` and `JobManagerTask`. 
-- Added support for specifying actions to take based on a task's output file upload status, via the `file_upload_error` property on `ExitConditions`. 
+- Added support for uploading task output files to persistent storage, via the `OutputFiles` property on `CloudTask` and `JobManagerTask`.
+- Added support for specifying actions to take based on a task's output file upload status, via the `file_upload_error` property on `ExitConditions`.
 - Added support for determining if a task was a success or a failure via the new `result` property on all task execution information objects.
 - Renamed `scheduling_error` on all task execution information objects to `failure_information`. `TaskFailureInformation` replaces `TaskSchedulingError` and is returned any
-  time there is a task failure. This includes all previous scheduling error cases, as well as nonzero task exit codes, and file upload failures from the new output files feature. 
+  time there is a task failure. This includes all previous scheduling error cases, as well as nonzero task exit codes, and file upload failures from the new output files feature.
 - Renamed `SchedulingErrorCategory` enum to `ErrorCategory`.
 - Renamed `scheduling_error` on `ExitConditions` to `pre_processing_error` to more clearly clarify when the error took place in the task life-cycle.
 - Added support for provisioning application licenses to your pool, via a new `application_licenses` property on `PoolAddParameter`, `CloudPool` and `PoolSpecification`.
   Please note that this feature is in gated public preview, and you must request access to it via a support ticket.
-- The `ssh_private_key` attribute of a `UserAccount` object has been replaced with an expanded `LinuxUserConfiguration` object with additional settings for a user ID and group ID of the 
+- The `ssh_private_key` attribute of a `UserAccount` object has been replaced with an expanded `LinuxUserConfiguration` object with additional settings for a user ID and group ID of the
   user account.
 - Removed `unmapped` enum state from `AddTaskStatus`, `CertificateFormat`, `CertificateVisibility`, `CertStoreLocation`, `ComputeNodeFillType`, `OSType`, and `PoolLifetimeOption` as they were not ever used.
 - Improved and clarified documentation.
@@ -106,7 +144,7 @@ Release History
 
 - Added support for joining a CloudPool to a virtual network on using the network_configuration property.
 - Added support for application package references on CloudTask and JobManagerTask.
-- Added support for automatically terminating jobs when all tasks complete or when a task fails, via the on_all_tasks_complete property and 
+- Added support for automatically terminating jobs when all tasks complete or when a task fails, via the on_all_tasks_complete property and
   the CloudTask exit_conditions property.
 
 0.30.0rc5

azure-batch/MANIFEST.in

@@ -1,2 +1,3 @@
 include *.rst
-include azure_bdist_wheel.py
+include azure/__init__.py
+

azure-batch/README.rst

@@ -3,7 +3,9 @@ Microsoft Azure SDK for Python
 
 This is the Microsoft Azure Batch Client Library.
 
-This package has been tested with Python 2.7, 3.4, 3.5 and 3.6.
+This package has been tested with Python 2.7, 3.4, 3.5, 3.6 and 3.7.
+
+For a more complete set of Azure libraries, see the `azure <https://pypi.python.org/pypi/azure>`__ bundle package.
 
 
 Compatibility
@@ -28,14 +30,13 @@ If you see azure==0.11.0 (or any version below 1.0), uninstall it first:
 Usage
 =====
 
-For code examples, see `the Batch samples repo  
+For code examples, see `the Batch samples repo
 <https://github.com/Azure/azure-batch-samples/tree/master/Python>`__
 on GitHub or see `Batch
 <https://docs.microsoft.com/python/api/overview/azure/batch>`__
 on docs.microsoft.com.
 
 
-
 Provide Feedback
 ================
 

azure-batch/azure/__init__.py

@@ -1 +1 @@
-__import__('pkg_resources').declare_namespace(__name__)
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)

azure-batch/azure/batch/batch_auth.py

@@ -53,8 +53,6 @@ def __call__(self, request):
 
         url = urlparse(request.url)
         uri_path = url.path
-        uri_path = uri_path.replace('%5C', '/')
-        uri_path = uri_path.replace('%2F', '/')
 
         # method to sign
         string_to_sign = request.method + '\n'
@@ -118,10 +116,10 @@ def __init__(self, account_name, key):
         super(SharedKeyCredentials, self).__init__()
         self.auth = SharedKeyAuth(self.header, account_name, key)
 
-    def signed_session(self):
+    def signed_session(self, session=None):
 
-        session = super(SharedKeyCredentials, self).signed_session()
+        session = super(SharedKeyCredentials, self).signed_session(session=session)
         session.auth = self.auth
 
         return session
-
+

azure-batch/azure/batch/batch_service_client.py

@@ -9,7 +9,7 @@
 # regenerated.
 # --------------------------------------------------------------------------
 
-from msrest.service_client import ServiceClient
+from msrest.service_client import SDKClient
 from msrest import Serializer, Deserializer
 from msrestazure import AzureConfiguration
 from .version import VERSION
@@ -23,6 +23,7 @@
 from .operations.task_operations import TaskOperations
 from .operations.compute_node_operations import ComputeNodeOperations
 from . import models
+from .custom.patch import patch_client
 
 
 class BatchServiceClientConfiguration(AzureConfiguration):
@@ -46,13 +47,13 @@ def __init__(
 
         super(BatchServiceClientConfiguration, self).__init__(base_url)
 
-        self.add_user_agent('batchserviceclient/{}'.format(VERSION))
+        self.add_user_agent('azure-batch/{}'.format(VERSION))
         self.add_user_agent('Azure-SDK-For-Python')
 
         self.credentials = credentials
 
 
-class BatchServiceClient(object):
+class BatchServiceClient(SDKClient):
     """A client for issuing REST requests to the Azure Batch service.
 
     :ivar config: Configuration for client.
@@ -87,10 +88,10 @@ def __init__(
             self, credentials, base_url=None):
 
         self.config = BatchServiceClientConfiguration(credentials, base_url)
-        self._client = ServiceClient(self.config.credentials, self.config)
+        super(BatchServiceClient, self).__init__(self.config.credentials, self.config)
 
         client_models = {k: v for k, v in models.__dict__.items() if isinstance(v, type)}
-        self.api_version = '2018-03-01.6.1'
+        self.api_version = '2018-08-01.7.0'
         self._serialize = Serializer(client_models)
         self._deserialize = Deserializer(client_models)
 
@@ -112,3 +113,6 @@ def __init__(
             self._client, self.config, self._serialize, self._deserialize)
         self.compute_node = ComputeNodeOperations(
             self._client, self.config, self._serialize, self._deserialize)
+
+
+patch_client()

azure-batch/azure/batch/custom/custom_errors.py

@@ -0,0 +1,19 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+
+class CreateTasksErrorException(Exception):
+    """ Aggregate Exception containing details for any failures from a task add operation.
+
+    :param str message: Error message describing exit reason
+    :param [~TaskAddParameter] pending_task_list: List of tasks remaining to be submitted.
+    :param [~TaskAddResult] failure_tasks: List of tasks which failed to add
+    :param [~Exception] errors: List of unknown errors forcing early termination
+    """
+    def __init__(self, message, pending_task_list=None, failure_tasks=None, errors=None):
+        self.message = message
+        self.pending_tasks = list(pending_task_list)
+        self.failure_tasks = list(failure_tasks)
+        self.errors = list(errors)