Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: DIA-1325: update response for inference-runs #297

Open
wants to merge 9 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 51 additions & 3 deletions .mock/definition/__package__.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2385,15 +2385,63 @@ types:
project_subset: InferenceRunProjectSubset
status: optional<InferenceRunStatus>
job_id: optional<string>
total_predictions: optional<integer>
total_correct_predictions: optional<integer>
total_tasks: optional<integer>
created_at: optional<datetime>
triggered_at: optional<datetime>
predictions_updated_at: optional<datetime>
completed_at: optional<datetime>
source:
openapi: openapi/openapi.yaml
KeyIndicatorsItemAdditionalKpisItem:
properties:
key:
type: optional<string>
docs: >-
The key for this KPI, where you can find the value from inside
main_kpi
label:
type: optional<string>
docs: The label for this KPI, to be displayed to the user
source:
openapi: openapi/openapi.yaml
KeyIndicatorsItemExtraKpisItem:
properties:
key:
type: optional<string>
docs: >-
The key for this KPI, where you can find the value from inside
main_kpi
label:
type: optional<string>
docs: The label for this KPI, to be displayed to the user
source:
openapi: openapi/openapi.yaml
KeyIndicatorsItem:
properties:
key:
type: string
docs: >-
The key for this KPI, where you can find the value from inside
main_kpi
title:
type: string
docs: The title for this metric, to be displayed to the user
main_kpi: string
secondary_kpi: optional<string>
additional_kpis:
type: optional<list<KeyIndicatorsItemAdditionalKpisItem>>
docs: Additional KPIs to be displayed at the bottom of the box
extra_kpis:
type: optional<list<KeyIndicatorsItemExtraKpisItem>>
docs: Extra KPIs to be displayed in the hover-tootip for that indicator
source:
openapi: openapi/openapi.yaml
KeyIndicators: list<KeyIndicatorsItem>
KeyIndicatorValue:
properties:
title: optional<string>
values: optional<map<string, unknown>>
source:
openapi: openapi/openapi.yaml
ModelProviderConnectionProvider:
enum:
- OpenAI
Expand Down
65 changes: 65 additions & 0 deletions .mock/definition/prompts/indicators.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
imports:
root: ../__package__.yml
service:
auth: false
base-path: ''
endpoints:
list:
path: /api/inference-runs/{pk}/indicators
method: GET
auth: true
docs: |
Get key indicators for the Prompt dashboard.
path-parameters:
pk:
type: integer
docs: Inference run ID
display-name: Get key indicators
response:
docs: ''
type: root.KeyIndicators
examples:
- path-parameters:
pk: 1
response:
body:
- key: key
title: title
main_kpi: main_kpi
secondary_kpi: secondary_kpi
additional_kpis:
- {}
extra_kpis:
- {}
audiences:
- public
get:
path: /api/inference-runs/{pk}/indicators/{indicator_key}
method: GET
auth: true
docs: |
Get a specific key indicator for the Prompt dashboard.
path-parameters:
indicator_key:
type: string
docs: Key of the indicator
pk:
type: integer
docs: Inference run ID
display-name: Get key indicator
response:
docs: ''
type: root.KeyIndicatorValue
examples:
- path-parameters:
indicator_key: indicator_key
pk: 1
response:
body:
title: title
values:
key: value
audiences:
- public
source:
openapi: openapi/openapi.yaml
107 changes: 107 additions & 0 deletions .mock/definition/prompts/runs.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
types:
RunsListRequestProjectSubset:
enum:
- All
- HasGT
- Sample
source:
openapi: openapi/openapi.yaml
imports:
root: ../__package__.yml
service:
auth: false
base-path: ''
endpoints:
list:
path: /api/prompts/{id}/versions/{version_id}/inference-runs
method: GET
auth: true
docs: Get information (status, etadata, etc) about an existing inference run
path-parameters:
id:
type: integer
docs: Prompt ID
version_id:
type: integer
docs: Prompt Version ID
display-name: Get inference run info
request:
name: RunsListRequest
query-parameters:
project:
type: integer
docs: The ID of the project that this Interence Run makes predictions on
project_subset:
type: RunsListRequestProjectSubset
docs: >
Defines which tasks are operated on (e.g. HasGT will only operate
on tasks with a ground truth annotation, but All will operate on
all records)
response:
docs: Success
type: root.InferenceRun
examples:
- path-parameters:
id: 1
version_id: 1
query-parameters:
project: 1
project_subset: All
response:
body:
organization: 1
project: 1
model_version: model_version
created_by: 1
project_subset: All
status: Pending
job_id: job_id
created_at: '2024-01-15T09:30:00Z'
triggered_at: '2024-01-15T09:30:00Z'
predictions_updated_at: '2024-01-15T09:30:00Z'
completed_at: '2024-01-15T09:30:00Z'
audiences:
- public
create:
path: /api/prompts/{id}/versions/{version_id}/inference-runs
method: POST
auth: true
docs: |
Run a prompt inference.
path-parameters:
id:
type: integer
docs: Prompt ID
version_id:
type: integer
docs: Prompt Version ID
display-name: Run prompt inference
request:
body: root.InferenceRun
response:
docs: ''
type: root.InferenceRun
examples:
- path-parameters:
id: 1
version_id: 1
request:
project: 1
project_subset: All
response:
body:
organization: 1
project: 1
model_version: model_version
created_by: 1
project_subset: All
status: Pending
job_id: job_id
created_at: '2024-01-15T09:30:00Z'
triggered_at: '2024-01-15T09:30:00Z'
predictions_updated_at: '2024-01-15T09:30:00Z'
completed_at: '2024-01-15T09:30:00Z'
audiences:
- public
source:
openapi: openapi/openapi.yaml
44 changes: 0 additions & 44 deletions .mock/definition/prompts/versions.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,49 +41,5 @@ service:
organization: 1
audiences:
- public
create_run:
path: /api/prompts/{id}/versions/{version_id}/inference-runs
method: POST
auth: true
docs: |
Run a prompt inference.
path-parameters:
id:
type: integer
docs: Prompt ID
version_id:
type: integer
docs: Prompt Version ID
display-name: Run prompt inference
request:
body: root.InferenceRun
response:
docs: ''
type: root.InferenceRun
examples:
- path-parameters:
id: 1
version_id: 1
request:
project: 1
project_subset: All
response:
body:
organization: 1
project: 1
model_version: model_version
created_by: 1
project_subset: All
status: Pending
job_id: job_id
total_predictions: 1
total_correct_predictions: 1
total_tasks: 1
created_at: '2024-01-15T09:30:00Z'
triggered_at: '2024-01-15T09:30:00Z'
predictions_updated_at: '2024-01-15T09:30:00Z'
completed_at: '2024-01-15T09:30:00Z'
audiences:
- public
source:
openapi: openapi/openapi.yaml
Loading
Loading