Skip to content

Commit

Permalink
SDK/Components - Added Json Schema spec for the component format (#669)
Browse files Browse the repository at this point in the history 
* SDK/Components - Added JsonSchema spec for the component format
Added the schema outline

* Fixed missing "required"

* Replaced PrimitiveTypes with just string

* Renamed CommandlineArgumentType to StringOrPlaceholder
Made more ContainerSpec properties support placeholders

* Removed support for type inheritance and generic types as requested by Ning and Ajay

* Some people are scared of graphs/pipelines - removing them

* Some people do not like optional inputs and conditionals - removing them
Sorry, Yasser and Bradley.

* Some people might be scared of predicates or conditional execution - removing them
Sure, Argo and DSL supports it, but some people care more about the spec file size even when that means dropping already supported features.
Sorry, Bradley and Riley.

* Reverting the last 4 commits

Making those big compromises did not have any noticeable effect on people asking for them.

* Removed list-style type specifications

We've standardized on the map-style specification.

* Renamed TypeType to TypeSpecType

* Updated the structure of graphInput

* Added the type attribute to taskOutput and graphInput

* Updated the execution options structure

* Using the official Kubernetes PodSpec schema instead of Argo's subset
  • Loading branch information
@Ark-kun @k8s-ci-robot
Ark-kun authored and k8s-ci-robot committed Nov 12, 2019
1 parent fa6859d commit da5cbb8
Show file tree
Hide file tree
Showing 2 changed files with 463 additions and 0 deletions.
389 changes: 389 additions & 0 deletions sdk/python/kfp/components/structures/components.json_schema.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,389 @@
{
"$schema": "http://json-schema.org/draft-06/schema#",
"$id": "http://kubeflow.org/pipelines/components.json_schema.json",

"$ref": "#/definitions/PipelineRunSpec",

"definitions": {
"TypeSpecType": {
"oneOf": [
{"type": "string"},
{"type": "object", "additionalProperties": {"$ref": "#/definitions/TypeSpecType"}}
]
},

"InputSpec": {
"description": "Describes the component input specification",
"type": "object",
"required": ["name"],
"properties": {
"name": {"type": "string"},
"type": {"$ref": "#/definitions/TypeSpecType"},
"description": {"type": "string"},
"default": {"type": "string"},
"optional": {"type": "boolean", "default": false}
},
"additionalProperties": false
},

"OutputSpec": {
"description": "Describes the component output specification",
"type": "object",
"required": ["name"],
"properties": {
"name": {"type": "string"},
"type": {"$ref": "#/definitions/TypeSpecType"},
"description": {"type": "string"}
},
"additionalProperties": false
},

"InputValuePlaceholder": {
"description": "Represents the command-line argument placeholder that will be replaced at run-time by the input argument value.",
"type": "object",
"required": ["inputValue"],
"properties": {
"inputValue" : {
"description": "Name of the input.",
"type": "string"
}
},
"additionalProperties": false
},

"InputPathPlaceholder": {
"description": "Represents the command-line argument placeholder that will be replaced at run-time by a local file path pointing to a file containing the input argument value.",
"type": "object",
"required": ["inputPath"],
"properties": {
"inputPath" : {
"description": "Name of the input.",
"type": "string"
}
},
"additionalProperties": false
},

"OutputPathPlaceholder": {
"description": "Represents the command-line argument placeholder that will be replaced at run-time by a local file path pointing to a file where the program should write its output data.",
"type": "object",
"required": ["outputPath"],
"properties": {
"outputPath" : {
"description": "Name of the output.",
"type": "string"
}
},
"additionalProperties": false
},

"StringOrPlaceholder": {
"oneOf": [
{"type": "string"},
{"$ref": "#/definitions/InputValuePlaceholder"},
{"$ref": "#/definitions/InputPathPlaceholder"},
{"$ref": "#/definitions/OutputPathPlaceholder"},
{"$ref": "#/definitions/ConcatPlaceholder"},
{"$ref": "#/definitions/IfPlaceholder"}
]
},

"ConcatPlaceholder": {
"description": "Represents the command-line argument placeholder that will be replaced at run-time by the concatenated values of its items.",
"type": "object",
"required": ["concat"],
"properties": {
"concat" : {
"description": "Items to concatenate",
"type": "array",
"items": {"$ref": "#/definitions/StringOrPlaceholder"}
}
},
"additionalProperties": false
},

"IsPresentPlaceholder": {
"description": "Represents the command-line argument placeholder that will be replaced at run-time by a boolean value specifying whether the caller has passed an argument for the specified optional input.",
"type": "object",
"properties": {
"isPresent": {
"description": "Name of the input.",
"type": "string"
}
},
"additionalProperties": false
},

"IfConditionArgumentType": {
"oneOf": [
{"$ref": "#/definitions/IsPresentPlaceholder"},
{"type": "boolean"},
{"type": "string"},
{"$ref": "#/definitions/InputValuePlaceholder"}
]
},

"IfPlaceholder": {
"description": "Represents the command-line argument placeholder that will be replaced at run-time by a boolean value specifying whether the caller has passed an argument for the specified optional input.",
"type": "object",
"required": ["if"],
"properties": {
"if" : {
"type": "object",
"required": ["cond", "then"],
"properties": {
"cond": {"$ref": "#/definitions/IfConditionArgumentType"},
"then": {"$ref": "#/definitions/CommandlineArgumentOrArrayType"},
"else": {"$ref": "#/definitions/CommandlineArgumentOrArrayType"}
}
}
}
},

"ContainerSpec": {
"type": "object",
"required": ["image"],
"properties": {
"image": {
"description": "Docker image name.",
"$ref": "#/definitions/StringOrPlaceholder"
},
"command": {
"description": "Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided.",
"type": "array",
"items": {"$ref": "#/definitions/StringOrPlaceholder"}
},
"args": {
"description": "Arguments to the entrypoint. The docker image's CMD is used if this is not provided.",
"type": "array",
"items": {"$ref": "#/definitions/StringOrPlaceholder"}
},
"env": {
"description": "List of environment variables to set in the container.",
"type": "object",
"additionalProperties": {"$ref": "#/definitions/StringOrPlaceholder"}
},
"unconfigurableOutputPaths": {
"description": "Legacy. Deprecated. Can be used to specify local output file paths for containers that do not allow setting the path of some output using command-line.",
"type": "object",
"additionalProperties": {"type": "string"}
}
},
"additionalProperties": false
},

"ContainerImplementation": {
"description": "Represents the container component implementation.",
"type": "object",
"required": ["container"],
"properties": {
"container": {"$ref": "#/definitions/ContainerSpec"}
}
},

"ImplementationType": {
"oneOf": [
{"$ref": "#/definitions/ContainerImplementation"},
{"$ref": "#/definitions/GraphImplementation"}
]
},

"SourceSpec": {
"description": "Specifies the location of the component source code.",
"type": "object",
"properties": {
"url": {"type": "string"}
},
"additionalProperties": false
},

"ComponentSpec": {
"description": "Component specification. Describes the metadata (name, description, source), the interface (inputs and outputs) and the implementation of the component.",
"type": "object",
"required": ["implementation"],
"properties": {
"name": {"type": "string"},
"description": {"type": "string"},
"source": {"$ref": "#/definitions/SourceSpec"},
"inputs": {"type": "array", "items": {"$ref": "#/definitions/InputSpec"}},
"outputs": {"type": "array", "items": {"$ref": "#/definitions/OutputSpec"}},
"implementation": {"$ref": "#/definitions/ImplementationType"},
"schemaVersion": {"type": "string", "default": "kubeflow.org/pipelines/component/v1"}
},
"additionalProperties": false
},

"ComponentReference": {
"description": "Component reference. Contains information that can be used to locate and load a component by name, digest or URL",
"type": "object",
"properties": {
"name": {"type": "string"},
"digest": {"type": "string"},
"tag": {"type": "string"},
"url": {"type": "string"},
"spec": {"$ref": "#/definitions/ComponentSpec"}
},
"additionalProperties": false
},

"GraphInputArgument": {
"description": "Represents the component argument value that comes from the graph component input.",
"type": "object",
"required": ["graphInput"],
"properties": {
"graphInput": {
"description": "References the input of the graph/pipeline.",
"type": "object",
"required": ["inputName"],
"properties": {
"inputName": {"type": "string"},
"type": {"$ref": "#/definitions/TypeSpecType"}
},
"additionalProperties": false
}
},
"additionalProperties": false
},

"TaskOutputArgument": {
"description": "Represents the component argument value that comes from the output of a sibling task.",
"type": "object",
"required": ["taskOutput"],
"properties": {
"taskOutput": {
"description": "References the output of a sibling task.",
"type": "object",
"required": ["taskId", "outputName"],
"properties": {
"taskId": {"type": "string"},
"outputName": {"type": "string"},
"type": {"$ref": "#/definitions/TypeSpecType"}
},
"additionalProperties": false
}
},
"additionalProperties": false
},

"ArgumentType": {
"oneOf": [
{"type": "string"},
{"$ref": "#/definitions/GraphInputArgument"},
{"$ref": "#/definitions/TaskOutputArgument"}
]
},

"TwoArgumentOperands": {
"description": "Pair of operands for a binary operation.",
"type": "object",
"required": ["op1", "op2"],
"properties": {
"op1": {"$ref": "#/definitions/ArgumentType"},
"op2": {"$ref": "#/definitions/ArgumentType"}
},
"additionalProperties": false
},

"TwoLogicalOperands": {
"description": "Pair of operands for a binary logical operation.",
"type": "object",
"required": ["op1", "op2"],
"properties": {
"op1": {"$ref": "#/definitions/PredicateType"},
"op2": {"$ref": "#/definitions/PredicateType"}
},
"additionalProperties": false
},

"PredicateType": {
"oneOf": [
{"type": "object", "required": ["=="], "properties": {"==": {"$ref": "#/definitions/TwoArgumentOperands"}}},
{"type": "object", "required": ["!="], "properties": {"!=": {"$ref": "#/definitions/TwoArgumentOperands"}}},
{"type": "object", "required": [">"], "properties": {">": {"$ref": "#/definitions/TwoArgumentOperands"}}},
{"type": "object", "required": [">="], "properties": {">=": {"$ref": "#/definitions/TwoArgumentOperands"}}},
{"type": "object", "required": ["<"], "properties": {"<": {"$ref": "#/definitions/TwoArgumentOperands"}}},
{"type": "object", "required": ["<="], "properties": {"<=": {"$ref": "#/definitions/TwoArgumentOperands"}}},

{"type": "object", "required": ["and"], "properties": {"and": {"$ref": "#/definitions/TwoLogicalOperands"}}},
{"type": "object", "required": ["or"], "properties": {"or": {"$ref": "#/definitions/TwoLogicalOperands"}}},

{"type": "object", "required": ["not"], "properties": {"not": {"$ref": "#/definitions/PredicateType"}}}
]
},

"RetryStrategySpec": {
"description": "Optional configuration that specifies how the task should be retried if it fails.",
"type": "object",
"properties": {
"maxRetries": {"type": "integer"}
},
"additionalProperties": false
},

"KubernetesExecutionOptionsSpec": {
"description": "When running on Kubernetes, KubernetesExecutionOptionsSpec describes changes to the configuration of a Kubernetes Pod that will execute the task.",
"type": "object",
"properties": {
"metadata": {"$ref": "#https://kubernetesjsonschema.dev/v1.14.0/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"},
"mainContainer": {"$ref": "https://kubernetesjsonschema.dev/v1.14.0/_definitions.json#/definitions/io.k8s.api.core.v1.Container"},
"podSpec": {"$ref": "https://kubernetesjsonschema.dev/v1.14.0/_definitions.json#/definitions/io.k8s.api.core.v1.PodSpec"}
},
"additionalProperties": false
},

"ExecutionOptionsSpec": {
"description": "Optional configuration that specifies how the task should be executed. Can be used to set some platform-specific options.",
"type": "object",
"properties": {
"retryStrategy": {"$ref": "#/definitions/RetryStrategySpec"},
"kubernetesOptions": {"$ref": "#/definitions/KubernetesExecutionOptionsSpec"}
},
"additionalProperties": false
},

"TaskSpec": {
"description": "'Task specification. Task is a configured component - a component supplied with arguments and other applied configuration changes.",
"type": "object",
"required": ["componentRef"],
"properties": {
"componentRef": {"$ref": "#/definitions/ComponentReference"},
"arguments": {"type": "object", "additionalProperties": {"$ref": "#/definitions/ArgumentType"}},
"isEnabled": {"$ref": "#/definitions/PredicateType"},
"executionOptions": {"$ref": "#/definitions/ExecutionOptionsSpec"}
},
"additionalProperties": false
},

"GraphSpec": {
"description": "Describes the graph component implementation. It represents a graph of component tasks connected to the upstream sources of data using the argument specifications. It also describes the sources of graph output values.",
"type": "object",
"required": ["tasks"],
"properties": {
"tasks": {"type": "object", "additionalProperties": {"$ref": "#/definitions/TaskSpec"}},
"outputValues": {"type": "object", "additionalProperties": {"$ref": "#/definitions/TaskOutputArgument"}}
},
"additionalProperties": false
},

"GraphImplementation": {
"description": "Represents the graph component implementation.",
"type": "object",
"required": ["graph"],
"properties": {
"graph": {"$ref": "#/definitions/GraphSpec"}
},
"additionalProperties": false
},

"PipelineRunSpec": {
"description": "The object that can be sent to the backend to start a new Run.",
"type": "object",
"required": ["rootTask"],
"properties": {
"rootTask": {"$ref": "#/definitions/TaskSpec"},
"onExitTask": {"$ref": "#/definitions/TaskSpec"}
},
"additionalProperties": false
}
}
}
Loading

0 comments on commit da5cbb8

Please sign in to comment.