Skip to content

Commit

Permalink
feat: Include inference resources in metadata
Browse files Browse the repository at this point in the history
  • Loading branch information
alvarolopez committed Nov 13, 2024
1 parent 06242a0 commit 47ae46c
Show file tree
Hide file tree
Showing 4 changed files with 303 additions and 2 deletions.
7 changes: 7 additions & 0 deletions instances/sample-v2.mods.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,13 @@
"links": {
"source_code": "https://github.com/deephdc/DEEP-OC-mods",
"docker_image": "https://hub.docker.com/r/deephdc/deep-oc-mods"
},
"resources": {
"inference": {
"gpu": 1,
"memory": "16GB",
"cpu": 4
}
}
}

5 changes: 4 additions & 1 deletion src/ai4_metadata/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,11 @@ def extract_version() -> str:
class MetadataVersions(str, enum.Enum):
"""Available versions of the AI4 metadata schema."""

V2 = "2.1.0"
V2_1_0 = "2.0.0"
V2_0_0 = "2.0.0"

V1 = "1.0.0"
V2 = "2.0.0"


_metadata_version_files = {
Expand Down
50 changes: 49 additions & 1 deletion src/ai4_metadata/schemata/ai4-apps-v2.0.0.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
"metadata_version": {
"description": "Version of the metadata schema.",
"pattern": "^[0-9]{1,}.[0-9]{1,}.[0-9]{1,}$",
"example": "2.0.0",
"example": "2.1.0",
"type": "string"
},
"title": {
Expand Down Expand Up @@ -216,6 +216,54 @@
},
"user_friendly": "Data Type",
"example": ["Text"]
},
"resources": {
"description": "Resources (CPU, GPU, storage) needed for the application.",
"type": "object",
"properties": {
"inference": {
"description": "Minimum / reccomended resources needed for inference.",
"type": "object",
"properties": {
"cpu": {
"description": "Number of CPUs needed for inference.",
"type": "number",
"user_friendly": "Number of CPUs",
"example": 1
},
"gpu": {
"description": "Number of GPUs needed for inference.",
"type": "number",
"user_friendly": "Number of GPUs",
"example": 1
},
"memory": {
"description": "Memory needed for inference.",
"type": "string",
"user_friendly": "Memory",
"example": "8GB"
},
"gpu_memory": {
"description": "GPU memory needed for inference.",
"type": "string",
"user_friendly": "GPU Memory",
"example": "8GB"
},
"gpu_compute_capability": {
"description": "GPU compute capability needed for inference. Check https://developer.nvidia.com/cuda-gpus",
"type": "string",
"user_friendly": "GPU Compute Capability",
"example": "7.0"
},
"storage": {
"description": "Storage needed for inference.",
"type": "string",
"user_friendly": "Storage",
"example": "10GB"
}
}
}
}
}
},
"required": [
Expand Down
243 changes: 243 additions & 0 deletions src/ai4_metadata/schemata/ai4-apps-v2.1.0.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,243 @@
{
"$schema": "http://json-schema.org/draft/2020-12/schema",
"$id": "https://github.com/ai4os/ai4-metadata/blob/master/src/ai4_metadata/schemata/ai4-apps-v2.0.0.json",
"type": "object",
"properties": {
"metadata_version": {
"description": "Version of the metadata schema.",
"pattern": "^[0-9]{1,}.[0-9]{1,}.[0-9]{1,}$",
"example": "2.0.0",
"type": "string"
},
"title": {
"description": "Name of the Data Science application.",
"type": "string",
"user_friendly": "Title",
"example": "Deep Learning Model for Metadata Classification"
},
"summary": {
"description": "Short text describing the value of the Data Science application.",
"type": "string",
"user_friendly": "Summary",
"example": "This model classifies metadata using deep learning techniques."
},
"description": {
"description": "Elaborated description of the Data Science application.",
"type": "string",
"user_friendly": "Description",
"example": "This model uses a deep learning architecture to classify metadata into different categories. The model is trained on a dataset of 10,000 metadata entries and achieves an accuracy of 90%."
},
"doi" : {
"description": "DOI of the Data Science application.",
"type": "string",
"format": "doi",
"user_friendly": "Digital Object Identifier",
"example": "10.5281/zenodo.1234567"
},
"links": {
"title": "links",
"type": "object",
"properties": {
"documentation": {
"description": "URL pointing to the relevant documentation.",
"type": "string",
"format": "uri",
"user_friendly": "Documentation URL",
"example": "https://example.org/docs"
},
"source_code": {
"description": "URL pointing to the relevant source code.",
"type": "string",
"format": "uri",
"user_friendly": "Source Code URL",
"example": "https://example.org/code"
},
"docker_image": {
"description": "URL pointing to the relevant Docker image.",
"type": "string",
"format": "uri",
"user_friendly": "Docker Image URL",
"example": "https://example.org/docker"
},
"zenodo_doi": {
"description": "DOI of the Zenodo record.",
"type": "string",
"format": "uri",
"user_friendly": "Link to Zenodo DOI",
"example": "https://doi.org/10.5281/zenodo.1234567"
},
"dataset": {
"description": "URL pointing to the relevant training dataset.",
"type": "string",
"format": "uri",
"user_friendly": "Traning Dataset URL",
"example": "https://example.org/dataset"
},
"weights": {
"description": "URL of file containing weights from training",
"type": "string",
"format": "uri",
"user_friendly": "Weights URL",
"example": "https://example.org/weights"
},
"citation": {
"description": "URL pointing to the relevant citation. Please use a Citation File Format (CFF) if possible.",
"type": "string",
"format": "uri",
"user_friendly": "Citation URL",
"example": "https://example.org/citation.cff"
},
"base_model": {
"description": "URL pointing to a base model if this model is a fine-tune, adaptation, etc. of another one.",
"type": "string",
"format": "uri",
"user_friendly": "Base Model URL",
"example": "https://example.org/base_model"
}
},
"required": [
"source_code"
]
},
"dates": {
"title": "dates",
"type": "object",
"properties": {
"created": {
"description": "Creation date of the Data Science application.",
"type": "string",
"format": "date",
"user_friendly": "Creation Date",
"example": "2022-01-01"
},
"updated": {
"description": "Last update date of the Data Science application.",
"type": "string",
"format": "date",
"user_friendly": "Last Update Date",
"example": "2024-12-31"
}
},
"required": [
"created",
"updated"
]
},
"libraries": {
"description": "Libraries used in the Data Science application.",
"type": "array",
"uniqueItems": true,
"items": {
"type": "string",
"enum": [
"TensorFlow",
"PyTorch",
"Keras",
"Scikit-learn",
"XGBoost",
"LightGBM",
"CatBoost",
"Other"
]
},
"user_friendly": "Libraries used",
"example": ["TensorFlow", "PyTorch"]
},
"tasks": {
"description": "Topic/s for the tasks that categorize the Data Science application.",
"type": "array",
"uniqueItems": true,
"items": {
"type": "string",
"enum": [
"Computer Vision",
"Natural Language Processing",
"Time Series",
"Recommender Systems",
"Anomaly Detection",
"Regression",
"Classification",
"Clustering",
"Dimensionality Reduction",
"Generative Models",
"Graph Neural Networks",
"Optimization",
"Reinforcement Learning",
"Transfer Learning",
"Uncertainty Estimation",
"Other"
]
},
"user_friendly": "Model Tasks",
"example": ["Computer Vision", "Natural Language Processing"]
},
"categories": {
"description": "Platform categories for the tasks that categorize the Data Science application.",
"type": "array",
"uniqueItems": true,
"items": {
"type": "string",
"enum": [
"AI4 pre trained",
"AI4 trainable",
"AI4 inference",
"AI4 tools"
]
},
"user_friendly": "AI4 Platform Categories",
"example": ["AI4 pre trained", "AI4 trainable"]
},
"tags": {
"description": "User provided tags to best describe the Data Science application.",
"type": "array",
"uniqueItems": true,
"items": {
"type": "string"
},
"user_friendly": "Tags",
"example": ["Deep Learning", "Metadata Classification"]
},
"data-type": {
"description": "Data type used in the Data Science application.",
"type": "array",
"uniqueItems": true,
"items": {
"type": "string",
"enum": [
"Image",
"Text",
"Time Series",
"Tabular",
"Graph",
"Audio",
"Video",
"Other"
]
},
"user_friendly": "Data Type",
"example": ["Text"]
}
},
"required": [
"metadata_version",
"title",
"summary",
"description",
"links",
"libraries",
"tasks",
"categories",
"tags"
],
"not": {
"anyOf": [
{ "required": ["tosca"] },
{ "required": ["sources"] },
{ "required": ["dataset_url"] },
{ "required": ["cite_url"] },
{ "required": ["date_creation"] },
{ "required": ["keywords"] },
{ "required": ["license"] }
]
}
}

0 comments on commit 47ae46c

Please sign in to comment.