-
Notifications
You must be signed in to change notification settings - Fork 5.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
.NET (v3): Adding SageMaker workflow example (#5022)
- Loading branch information
Showing
24 changed files
with
2,280 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| // snippet-start:[SageMaker.dotnetv3.HelloSageMaker] | ||
|
|
||
| using Amazon.SageMaker; | ||
| using Amazon.SageMaker.Model; | ||
|
|
||
| namespace SageMakerActions; | ||
|
|
||
| public static class HelloSageMaker | ||
| { | ||
| static async Task Main(string[] args) | ||
| { | ||
| var sageMakerClient = new AmazonSageMakerClient(); | ||
|
|
||
| Console.WriteLine($"Hello Amazon SageMaker! Let's list some of your notebook instances:"); | ||
| Console.WriteLine(); | ||
|
|
||
| // You can use await and any of the async methods to get a response. | ||
| // Let's get the first five notebook instances. | ||
| var response = await sageMakerClient.ListNotebookInstancesAsync( | ||
| new ListNotebookInstancesRequest() | ||
| { | ||
| MaxResults = 5 | ||
| }); | ||
|
|
||
| if (!response.NotebookInstances.Any()) | ||
| { | ||
| Console.WriteLine($"No notebook instances found."); | ||
| Console.WriteLine("See https://docs.aws.amazon.com/sagemaker/latest/dg/howitworks-create-ws.html to create one."); | ||
| } | ||
|
|
||
| foreach (var notebookInstance in response.NotebookInstances) | ||
| { | ||
| Console.WriteLine($"\tInstance: {notebookInstance.NotebookInstanceName}"); | ||
| Console.WriteLine($"\tArn: {notebookInstance.NotebookInstanceArn}"); | ||
| Console.WriteLine($"\tCreation Date: {notebookInstance.CreationTime.ToShortDateString()}"); | ||
| Console.WriteLine(); | ||
| } | ||
| } | ||
| } | ||
| // snippet-end:[SageMaker.dotnetv3.HelloSageMaker] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <OutputType>Exe</OutputType> | ||
| <TargetFramework>net6.0</TargetFramework> | ||
| <ImplicitUsings>enable</ImplicitUsings> | ||
| <Nullable>enable</Nullable> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="AWSSDK.EC2" Version="3.7.136.1" /> | ||
| <PackageReference Include="AWSSDK.Extensions.NETCore.Setup" Version="3.7.2" /> | ||
| <PackageReference Include="AWSSDK.IdentityManagement" Version="3.7.101" /> | ||
| <PackageReference Include="AWSSDK.SageMaker" Version="3.7.134.2" /> | ||
| <PackageReference Include="AWSSDK.SageMakerGeospatial" Version="3.7.102.11" /> | ||
| <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="6.0.0" /> | ||
| <PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.1" /> | ||
| </ItemGroup> | ||
|
|
||
| </Project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,162 @@ | ||
| // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| // snippet-start:[SageMaker.dotnetv3.SagemakerWrapper] | ||
|
|
||
| using System.Text.Json; | ||
| using Amazon.SageMaker; | ||
| using Amazon.SageMaker.Model; | ||
| using Amazon.SageMakerGeospatial.Model; | ||
| using Amazon.SageMakerGeospatial; | ||
|
|
||
| namespace SageMakerActions; | ||
|
|
||
| /// <summary> | ||
| /// Wrapper class for Amazon SageMaker actions and logic. | ||
| /// </summary> | ||
| public class SageMakerWrapper | ||
| { | ||
| private readonly IAmazonSageMaker _amazonSageMaker; | ||
| public SageMakerWrapper(IAmazonSageMaker amazonSageMaker) | ||
| { | ||
| _amazonSageMaker = amazonSageMaker; | ||
| } | ||
|
|
||
| // snippet-start:[SageMaker.dotnetv3.CreatePipeline] | ||
| /// <summary> | ||
| /// Create a pipeline from a JSON definition, or update it if the pipeline already exists. | ||
| /// </summary> | ||
| /// <returns>The Amazon Resource Name (ARN) of the pipeline.</returns> | ||
| public async Task<string> SetupPipeline(string pipelineJson, string roleArn, string name, string description, string displayName) | ||
| { | ||
| try | ||
| { | ||
| var updateResponse = await _amazonSageMaker.UpdatePipelineAsync( | ||
| new UpdatePipelineRequest() | ||
| { | ||
| PipelineDefinition = pipelineJson, | ||
| PipelineDescription = description, | ||
| PipelineDisplayName = displayName, | ||
| PipelineName = name, | ||
| RoleArn = roleArn | ||
| }); | ||
| return updateResponse.PipelineArn; | ||
| } | ||
| catch (Amazon.SageMaker.Model.ResourceNotFoundException) | ||
| { | ||
| var createResponse = await _amazonSageMaker.CreatePipelineAsync( | ||
| new CreatePipelineRequest() | ||
| { | ||
| PipelineDefinition = pipelineJson, | ||
| PipelineDescription = description, | ||
| PipelineDisplayName = displayName, | ||
| PipelineName = name, | ||
| RoleArn = roleArn | ||
| }); | ||
|
|
||
| return createResponse.PipelineArn; | ||
| } | ||
| } | ||
| // snippet-end:[SageMaker.dotnetv3.CreatePipeline] | ||
|
|
||
| // snippet-start:[SageMaker.dotnetv3.ExecutePipeline] | ||
| /// <summary> | ||
| /// Run a pipeline with input and output file locations. | ||
| /// </summary> | ||
| /// <param name="queueUrl">The URL for the queue to use for pipeline callbacks.</param> | ||
| /// <param name="inputLocationUrl">The input location in Amazon Simple Storage Service (Amazon S3).</param> | ||
| /// <param name="outputLocationUrl">The output location in Amazon S3.</param> | ||
| /// <param name="pipelineName">The name of the pipeline.</param> | ||
| /// <param name="executionRoleArn">The ARN of the role.</param> | ||
| /// <returns>The ARN of the pipeline run.</returns> | ||
| public async Task<string> ExecutePipeline( | ||
| string queueUrl, | ||
| string inputLocationUrl, | ||
| string outputLocationUrl, | ||
| string pipelineName, | ||
| string executionRoleArn) | ||
| { | ||
| var inputConfig = new VectorEnrichmentJobInputConfig() | ||
| { | ||
| DataSourceConfig = new() | ||
| { | ||
| S3Data = new VectorEnrichmentJobS3Data() | ||
| { | ||
| S3Uri = inputLocationUrl | ||
| } | ||
| }, | ||
| DocumentType = VectorEnrichmentJobDocumentType.CSV | ||
| }; | ||
|
|
||
| var exportConfig = new ExportVectorEnrichmentJobOutputConfig() | ||
| { | ||
| S3Data = new VectorEnrichmentJobS3Data() | ||
| { | ||
| S3Uri = outputLocationUrl | ||
| } | ||
| }; | ||
|
|
||
| var jobConfig = new VectorEnrichmentJobConfig() | ||
| { | ||
| ReverseGeocodingConfig = new ReverseGeocodingConfig() | ||
| { | ||
| XAttributeName = "Longitude", | ||
| YAttributeName = "Latitude" | ||
| } | ||
| }; | ||
|
|
||
| var startExecutionResponse = await _amazonSageMaker.StartPipelineExecutionAsync( | ||
| new StartPipelineExecutionRequest() | ||
| { | ||
| PipelineName = pipelineName, | ||
| PipelineExecutionDisplayName = pipelineName + "-example-execution", | ||
| PipelineParameters = new List<Parameter>() | ||
| { | ||
| new Parameter() { Name = "parameter_execution_role", Value = executionRoleArn }, | ||
| new Parameter() { Name = "parameter_queue_url", Value = queueUrl }, | ||
| new Parameter() { Name = "parameter_vej_input_config", Value = JsonSerializer.Serialize(inputConfig) }, | ||
| new Parameter() { Name = "parameter_vej_export_config", Value = JsonSerializer.Serialize(exportConfig) }, | ||
| new Parameter() { Name = "parameter_step_1_vej_config", Value = JsonSerializer.Serialize(jobConfig) } | ||
| } | ||
| }); | ||
| return startExecutionResponse.PipelineExecutionArn; | ||
| } | ||
| // snippet-end:[SageMaker.dotnetv3.ExecutePipeline] | ||
|
|
||
| // snippet-start:[SageMaker.dotnetv3.DescribePipelineExecution] | ||
| /// <summary> | ||
| /// Check the status of a run. | ||
| /// </summary> | ||
| /// <param name="pipelineExecutionArn">The ARN.</param> | ||
| /// <returns>The status of the pipeline.</returns> | ||
| public async Task<PipelineExecutionStatus> CheckPipelineExecutionStatus(string pipelineExecutionArn) | ||
| { | ||
| var describeResponse = await _amazonSageMaker.DescribePipelineExecutionAsync( | ||
| new DescribePipelineExecutionRequest() | ||
| { | ||
| PipelineExecutionArn = pipelineExecutionArn | ||
| }); | ||
|
|
||
| return describeResponse.PipelineExecutionStatus; | ||
| } | ||
| // snippet-end:[SageMaker.dotnetv3.DescribePipelineExecution] | ||
|
|
||
| // snippet-start:[SageMaker.dotnetv3.DeletePipeline] | ||
| /// <summary> | ||
| /// Delete a SageMaker pipeline by name. | ||
| /// </summary> | ||
| /// <param name="pipelineName">The name of the pipeline to delete.</param> | ||
| /// <returns>The ARN of the pipeline.</returns> | ||
| public async Task<string> DeletePipelineByName(string pipelineName) | ||
| { | ||
| var deleteResponse = await _amazonSageMaker.DeletePipelineAsync( | ||
| new DeletePipelineRequest() | ||
| { | ||
| PipelineName = pipelineName | ||
| }); | ||
|
|
||
| return deleteResponse.PipelineArn; | ||
| } | ||
| // snippet-end:[SageMaker.dotnetv3.DeletePipeline] | ||
| } | ||
| // snippet-end:[SageMaker.dotnetv3.SagemakerWrapper] |
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Oops, something went wrong.