Inferences

Use to retrieve metrics for inference workloads and pods.

Create an inference. [Experimental]

Create an inference using container related fields.

SecuritybearerAuth

Request

Request Body schema: application/json

name required	string (WorkloadName) non-empty The name of the workload.
useGivenNameAsPrefix	boolean Default: false When true, the requested name will be treated as a prefix. The final name of the workload will be composed of the name followed by a random set of characters.
projectId required	string (ProjectId2) The id of the project.
clusterId required	string <uuid> (ClusterId) The id of the cluster.
	object or null (CommonFlatFields) Container overrideable fields. In the context of assets,these are environment asset fields that can be overriden in the submit workload request.

Responses

200

Request completed successfully.

400

Bad request.

401

Unauthorized

403

Forbidden

503

unexpected error

post/api/v1/workloads/inferences

Request samples

Payload

application/json

{"name": "my-workload-name",
"useGivenNameAsPrefix": true,
"projectId": 1,
"clusterId": "71f69d83-ba66-4822-adf5-55ce55efd210",
"spec": {"command": "python",
"args": "-x my-script.py",
"image": "python:3.8",
"imagePullPolicy": "Always",
"workingDir": "/home/myfolder",
"createHomeDir": true,
"probes": {"readiness": {"initialDelaySeconds": 0,
"periodSeconds": 1,
"timeoutSeconds": 1,
"successThreshold": 1,
"failureThreshold": 1,
"handler": {"httpGet": {"path": "string",
"port": 1,
"host": "example.com",
"scheme": "HTTP"
}
}
}
},
"nodeType": "my-node-type",
"nodePools": ["my-node-pool-a",
"my-node-pool-b"
],
"podAffinity": {"type": "Required",
"key": "string"
},
"environmentVariables": [{"name": "HOME",
"value": "/home/my-folder",
"secret": {"name": "postgress_secret",
"key": "POSTGRES_PASSWORD"
},
"exclude": false
}
],
"annotations": [{"name": "billing",
"value": "my-billing-unit",
"exclude": false
}
],
"labels": [{"name": "stage",
"value": "initial-research",
"exclude": false
}
],
"tolerations": [{"name": "string",
"operator": "Equal",
"key": "string",
"value": "string",
"effect": "NoSchedule",
"seconds": 1
}
],
"ports": [{"container": 8080,
"serviceType": "LoadBalancer",
"external": 30080,
"toolType": "pytorch",
"toolName": "my-pytorch",
"name": "port-instance-a"
}
],
"exposedUrls": [{"container": 8080,
"url": "https://my-url.com",
"authorizedUsers": ["user-a",
"user-b"
],
"authorizedGroups": ["group-a",
"group-b"
],
"toolType": "jupyter",
"toolName": "my-pytorch",
"name": "url-instance-a"
}
],
"relatedUrls": [{"url": "https://my-url.com",
"type": "wandb",
"name": "url-instance-a"
}
],
"compute": {"gpuDevicesRequest": 1,
"gpuRequestType": "portion",
"gpuPortionRequest": 0.5,
"gpuPortionLimit": 0.5,
"gpuMemoryRequest": "10M",
"gpuMemoryLimit": "10M",
"migProfile": "1g.5gb",
"cpuCoreRequest": 0.5,
"cpuCoreLimit": 2,
"cpuMemoryRequest": "20M",
"cpuMemoryLimit": "30M",
"largeShmRequest": false,
"extendedResources": [{"resource": "hardware-vendor.example/foo",
"quantity": 2,
"exclude": false
}
]
},
"storage": {"dataVolume": [{"id": "123e4567-e89b-12d3-a456-426614174000",
"mountPath": "/mnt/data"
}
],
"pvc": [{"name": "storage-instance-a",
"existingPvc": false,
"claimName": "my-claim",
"path": "/container/my-claim",
"readOnly": false,
"ephemeral": false,
"claimInfo": {"size": "1G",
"storageClass": "my-storage-class",
"accessModes": {"readWriteOnce": true,
"readOnlyMany": false,
"readWriteMany": false
},
"volumeMode": "Filesystem"
}
}
],
"git": [{"name": "storage-instance-a",
"repository": "https://github.com/my-git/my-repo",
"branch": "main",
"revision": "string",
"path": "/container/my-repository",
"passwordSecret": "my-password-secret",
"secretKeyOfUser": "User",
"secretKeyOfPassword": "Password"
}
],
"configMapVolume": [{"name": "storage-instance-a",
"configMap": "string",
"mountPath": "string"
}
],
"secretVolume": [{"name": "storage-instance-a",
"secret": "string",
"mountPath": "string"
}
]
},
"security": {"uidGidSource": "fromTheImage",
"capabilities": ["CHOWN",
"KILL"
],
"seccompProfileType": "RuntimeDefault",
"runAsNonRoot": true,
"readOnlyRootFilesystem": false,
"runAsUid": 500,
"runAsGid": 30,
"supplementalGroups": "2,3,5,8"
},
"servingPort": {"container": 8080,
"protocol": "http"
},
"autoscaling": {"minReplicas": 0,
"maxReplicas": 1,
"scaleToZeroRetentionSeconds": 3600,
"metric": "throughput",
"metricThreshold": 0
}
}
}

Response samples

application/json

{"name": "my-workload-name",
"requestedName": "string",
"workloadId": "06d16c5d-4728-42fa-b573-3b11820d999f",
"projectId": 1,
"departmentId": 2,
"clusterId": "71f69d83-ba66-4822-adf5-55ce55efd210",
"createdBy": "test@lab.com",
"createdAt": "2022-01-01T03:49:52.531Z",
"desiredPhase": "Running",
"actualPhase": "Creating",
"spec": {"command": "python",
"args": "-x my-script.py",
"image": "python:3.8",
"imagePullPolicy": "Always",
"workingDir": "/home/myfolder",
"createHomeDir": true,
"probes": {"readiness": {"initialDelaySeconds": 0,
"periodSeconds": 1,
"timeoutSeconds": 1,
"successThreshold": 1,
"failureThreshold": 1,
"handler": {"httpGet": {"path": "string",
"port": 1,
"host": "example.com",
"scheme": "HTTP"
}
}
}
},
"nodeType": "my-node-type",
"nodePools": ["my-node-pool-a",
"my-node-pool-b"
],
"podAffinity": {"type": "Required",
"key": "string"
},
"environmentVariables": [{"name": "HOME",
"value": "/home/my-folder",
"secret": {"name": "postgress_secret",
"key": "POSTGRES_PASSWORD"
},
"exclude": false
}
],
"annotations": [{"name": "billing",
"value": "my-billing-unit",
"exclude": false
}
],
"labels": [{"name": "stage",
"value": "initial-research",
"exclude": false
}
],
"tolerations": [{"name": "string",
"operator": "Equal",
"key": "string",
"value": "string",
"effect": "NoSchedule",
"seconds": 1
}
],
"ports": [{"container": 8080,
"serviceType": "LoadBalancer",
"external": 30080,
"toolType": "pytorch",
"toolName": "my-pytorch",
"name": "port-instance-a"
}
],
"exposedUrls": [{"container": 8080,
"url": "https://my-url.com",
"authorizedUsers": ["user-a",
"user-b"
],
"authorizedGroups": ["group-a",
"group-b"
],
"toolType": "jupyter",
"toolName": "my-pytorch",
"name": "url-instance-a"
}
],
"relatedUrls": [{"url": "https://my-url.com",
"type": "wandb",
"name": "url-instance-a"
}
],
"compute": {"gpuDevicesRequest": 1,
"gpuRequestType": "portion",
"gpuPortionRequest": 0.5,
"gpuPortionLimit": 0.5,
"gpuMemoryRequest": "10M",
"gpuMemoryLimit": "10M",
"migProfile": "1g.5gb",
"cpuCoreRequest": 0.5,
"cpuCoreLimit": 2,
"cpuMemoryRequest": "20M",
"cpuMemoryLimit": "30M",
"largeShmRequest": false,
"extendedResources": [{"resource": "hardware-vendor.example/foo",
"quantity": 2,
"exclude": false
}
]
},
"storage": {"dataVolume": [{"id": "123e4567-e89b-12d3-a456-426614174000",
"mountPath": "/mnt/data"
}
],
"pvc": [{"name": "storage-instance-a",
"existingPvc": false,
"claimName": "my-claim",
"path": "/container/my-claim",
"readOnly": false,
"ephemeral": false,
"claimInfo": {"size": "1G",
"storageClass": "my-storage-class",
"accessModes": {"readWriteOnce": true,
"readOnlyMany": false,
"readWriteMany": false
},
"volumeMode": "Filesystem"
}
}
],
"git": [{"name": "storage-instance-a",
"repository": "https://github.com/my-git/my-repo",
"branch": "main",
"revision": "string",
"path": "/container/my-repository",
"passwordSecret": "my-password-secret",
"secretKeyOfUser": "User",
"secretKeyOfPassword": "Password"
}
],
"configMapVolume": [{"name": "storage-instance-a",
"configMap": "string",
"mountPath": "string"
}
],
"secretVolume": [{"name": "storage-instance-a",
"secret": "string",
"mountPath": "string"
}
]
},
"security": {"uidGidSource": "fromTheImage",
"capabilities": ["CHOWN",
"KILL"
],
"seccompProfileType": "RuntimeDefault",
"runAsNonRoot": true,
"readOnlyRootFilesystem": false,
"runAsUid": 500,
"runAsGid": 30,
"supplementalGroups": "2,3,5,8"
},
"servingPort": {"container": 8080,
"protocol": "http"
},
"autoscaling": {"minReplicas": 0,
"maxReplicas": 1,
"scaleToZeroRetentionSeconds": 3600,
"metric": "throughput",
"metricThreshold": 0
}
}
}

Delete an inference. [Experimental]

Delete an inference using a workload id.

SecuritybearerAuth

Request

path Parameters

workloadId

required

string <uuid>

Unique identifier of the workload.

Responses

204

No Content.

401

Unauthorized

403

Forbidden

404

The specified resource was not found

500

unexpected error

503

unexpected error

delete/api/v1/workloads/inferences/{workloadId}

Response samples

application/json

{"code": 401,
"message": "Issuer is not familiar."
}

Get inference data. [Experimental]

Retrieve inference details using a workload id.

SecuritybearerAuth

Request

path Parameters

workloadId

required

string <uuid>

Unique identifier of the workload.

Responses

200

Executed successfully.

401

Unauthorized

403

Forbidden

404

The specified resource was not found

500

unexpected error

503

unexpected error

get/api/v1/workloads/inferences/{workloadId}

Response samples

application/json

{"name": "my-workload-name",
"requestedName": "string",
"workloadId": "06d16c5d-4728-42fa-b573-3b11820d999f",
"projectId": 1,
"departmentId": 2,
"clusterId": "71f69d83-ba66-4822-adf5-55ce55efd210",
"createdBy": "test@lab.com",
"createdAt": "2022-01-01T03:49:52.531Z",
"desiredPhase": "Running",
"actualPhase": "Creating",
"spec": {"command": "python",
"args": "-x my-script.py",
"image": "python:3.8",
"imagePullPolicy": "Always",
"workingDir": "/home/myfolder",
"createHomeDir": true,
"probes": {"readiness": {"initialDelaySeconds": 0,
"periodSeconds": 1,
"timeoutSeconds": 1,
"successThreshold": 1,
"failureThreshold": 1,
"handler": {"httpGet": {"path": "string",
"port": 1,
"host": "example.com",
"scheme": "HTTP"
}
}
}
},
"nodeType": "my-node-type",
"nodePools": ["my-node-pool-a",
"my-node-pool-b"
],
"podAffinity": {"type": "Required",
"key": "string"
},
"environmentVariables": [{"name": "HOME",
"value": "/home/my-folder",
"secret": {"name": "postgress_secret",
"key": "POSTGRES_PASSWORD"
},
"exclude": false
}
],
"annotations": [{"name": "billing",
"value": "my-billing-unit",
"exclude": false
}
],
"labels": [{"name": "stage",
"value": "initial-research",
"exclude": false
}
],
"tolerations": [{"name": "string",
"operator": "Equal",
"key": "string",
"value": "string",
"effect": "NoSchedule",
"seconds": 1
}
],
"ports": [{"container": 8080,
"serviceType": "LoadBalancer",
"external": 30080,
"toolType": "pytorch",
"toolName": "my-pytorch",
"name": "port-instance-a"
}
],
"exposedUrls": [{"container": 8080,
"url": "https://my-url.com",
"authorizedUsers": ["user-a",
"user-b"
],
"authorizedGroups": ["group-a",
"group-b"
],
"toolType": "jupyter",
"toolName": "my-pytorch",
"name": "url-instance-a"
}
],
"relatedUrls": [{"url": "https://my-url.com",
"type": "wandb",
"name": "url-instance-a"
}
],
"compute": {"gpuDevicesRequest": 1,
"gpuRequestType": "portion",
"gpuPortionRequest": 0.5,
"gpuPortionLimit": 0.5,
"gpuMemoryRequest": "10M",
"gpuMemoryLimit": "10M",
"migProfile": "1g.5gb",
"cpuCoreRequest": 0.5,
"cpuCoreLimit": 2,
"cpuMemoryRequest": "20M",
"cpuMemoryLimit": "30M",
"largeShmRequest": false,
"extendedResources": [{"resource": "hardware-vendor.example/foo",
"quantity": 2,
"exclude": false
}
]
},
"storage": {"dataVolume": [{"id": "123e4567-e89b-12d3-a456-426614174000",
"mountPath": "/mnt/data"
}
],
"pvc": [{"name": "storage-instance-a",
"existingPvc": false,
"claimName": "my-claim",
"path": "/container/my-claim",
"readOnly": false,
"ephemeral": false,
"claimInfo": {"size": "1G",
"storageClass": "my-storage-class",
"accessModes": {"readWriteOnce": true,
"readOnlyMany": false,
"readWriteMany": false
},
"volumeMode": "Filesystem"
}
}
],
"git": [{"name": "storage-instance-a",
"repository": "https://github.com/my-git/my-repo",
"branch": "main",
"revision": "string",
"path": "/container/my-repository",
"passwordSecret": "my-password-secret",
"secretKeyOfUser": "User",
"secretKeyOfPassword": "Password"
}
],
"configMapVolume": [{"name": "storage-instance-a",
"configMap": "string",
"mountPath": "string"
}
],
"secretVolume": [{"name": "storage-instance-a",
"secret": "string",
"mountPath": "string"
}
]
},
"security": {"uidGidSource": "fromTheImage",
"capabilities": ["CHOWN",
"KILL"
],
"seccompProfileType": "RuntimeDefault",
"runAsNonRoot": true,
"readOnlyRootFilesystem": false,
"runAsUid": 500,
"runAsGid": 30,
"supplementalGroups": "2,3,5,8"
},
"servingPort": {"container": 8080,
"protocol": "http"
},
"autoscaling": {"minReplicas": 0,
"maxReplicas": 1,
"scaleToZeroRetentionSeconds": 3600,
"metric": "throughput",
"metricThreshold": 0
}
}
}

Get inference metrics data. [Experimental]

Retrieve inference metrics data by id. Supported from control-plane version 2.18 or later.

SecuritybearerAuth

Request

path Parameters

workloadId

required

string <uuid>

Unique identifier of the workload.

query Parameters

metricType required	Array of strings (InferenceWorkloadMetricType) Specify which data to request. Items Enum: "THROUGHPUT" "LATENCY"
start required	string <date-time> Start date of time range to fetch data in ISO 8601 timestamp format. Example: start=2023-06-06T12:09:18.211Z
end required	string <date-time> End date of time range to fetch data in ISO 8601 timestamp format. Example: end=2023-06-07T12:09:18.211Z
numberOfSamples	integer [ 0 .. 1000 ] Default: 20 The number of samples to take in the specified time range. Example: numberOfSamples=20

Responses

200

Executed successfully.

207

Partial success.

400

Bad request.

401

Unauthorized

403

Forbidden

404

The specified resource was not found

500

unexpected error

503

unexpected error

get/api/v1/workloads/inferences/{workloadId}/metrics

Response samples

{"measurements": [{"type": "ALLOCATED_GPU",
"labels": "{'gpu': '3'}",
"values": [{"value": "85",
"timestamp": "2023-06-06 12:09:18.211"
}
]
}
]
}

Get inference pod's metrics data. [Experimental]

Retrieve inference metrics pod's data by workload and pod id. Supported from control-plane version 2.18 or later.

SecuritybearerAuth

Request

path Parameters

workloadId required	string <uuid> Unique identifier of the workload.
podId required	string <uuid> The requested pod id.

query Parameters

metricType required	Array of strings (InferencePodMetricType) Specifies metrics data to request. Inference metrics are only available for inference workloads. Items Enum: "THROUGHPUT" "LATENCY"
start required	string <date-time> Start date of time range to fetch data in ISO 8601 timestamp format. Example: start=2023-06-06T12:09:18.211Z
end required	string <date-time> End date of time range to fetch data in ISO 8601 timestamp format. Example: end=2023-06-07T12:09:18.211Z
numberOfSamples	integer [ 0 .. 1000 ] Default: 20 The number of samples to take in the specified time range. Example: numberOfSamples=20

Responses

200

Executed successfully.

207

Partial success.

400

Bad request.

401

Unauthorized

403

Forbidden

404

The specified resource was not found

500

unexpected error

503

unexpected error

get/api/v1/workloads/inferences/{workloadId}/pods/{podId}/metrics

Response samples

{"measurements": [{"type": "ALLOCATED_GPU",
"labels": "{'gpu': '3'}",
"values": [{"value": "85",
"timestamp": "2023-06-06 12:09:18.211"
}
]
}
]
}

➔ Next to Distributed