Distributed Inferences

Distributed inference enables running inference workloads across multiple pods, typically to scale model serving beyond a single container or node. This approach is useful when a single instance cannot meet resource requirements.NVIDIA Run:ai supports this model using Leader Worker Set (LWS). Each pod plays a specific role, either as a leader or worker, and together they form a coordinated service. NVIDIA Run:ai manages the orchestration and configuration of these pods to ensure efficient and scalable inference execution

Create a distributed inference.

Create a distributed inference using container related fields.

SecuritybearerAuth

Request

Request Body schema: application/json

name required	string (WorkloadName) non-empty .* The name of the workload.
useGivenNameAsPrefix	boolean Default: false When true, the requested name will be treated as a prefix. The final name of the workload will be composed of the name followed by a random set of characters.
projectId required	string (ProjectId) .* The id of the project.
clusterId required	string <uuid> (ClusterId) The id of the cluster.
templateId	string or null <uuid> The unique identifier of the template to use for submitting this workload. The combined values provided in the spec, template and assets will be used to create the workload.
	object or null (DistributedInferenceSpecSpec)

Responses

202

Request completed successfully.

400

Bad request.

401

Unauthorized

403

Forbidden

503

unexpected error

post/api/v1/workloads/distributed-inferences

Request samples

Payload

application/json

{"name": "my-workload-name",
"useGivenNameAsPrefix": true,
"projectId": 1,
"clusterId": "71f69d83-ba66-4822-adf5-55ce55efd210",
"templateId": "550e8400-e29b-41d4-a716-446655440000",
"spec": {"category": "string",
"nodePools": ["my-node-pool-a",
"my-node-pool-b"
],
"preemptibility": "preemptible",
"priorityClass": "string",
"restartPolicy": "RecreateGroupOnPodRestart",
"servingPort": {"port": 8080,
"protocol": "http",
"authorizationType": "public",
"authorizedUsers": ["user.a@example.com",
"user.b@example.com"
],
"authorizedGroups": ["group-a",
"group-b"
],
"exposeExternally": true,
"exposedUrl": "string"
},
"startupPolicy": "LeaderCreated",
"workers": 4,
"replicas": 2,
"leader": {"annotations": [{"name": "billing",
"value": "my-billing-unit",
"exclude": false
}
],
"args": "-x my-script.py",
"command": "python",
"compute": {"cpuCoreLimit": 2,
"cpuCoreRequest": 0.5,
"cpuMemoryLimit": "30M",
"cpuMemoryRequest": "20M",
"extendedResources": [{"resource": "hardware-vendor.example/foo",
"quantity": 2,
"exclude": false
}
],
"gpuDevicesRequest": 1,
"gpuMemoryLimit": "10M",
"gpuMemoryRequest": "10M",
"gpuPortionLimit": 0.5,
"gpuPortionRequest": 0.5,
"gpuRequestType": "portion",
"largeShmRequest": false
},
"createHomeDir": true,
"environmentVariables": [{"name": "HOME",
"value": "/home/my-folder",
"secret": {"name": "postgress_secret",
"key": "POSTGRES_PASSWORD"
},
"configMap": {"name": "my-config-map",
"key": "MY_POSTGRES_SCHEMA"
},
"podFieldRef": {"path": "metadata.name"
},
"userCredential": {"name": "my_postgres_user_and_password",
"key": "POSTGRES_PASSWORD"
},
"exclude": false,
"description": "Home directory of the user."
}
],
"image": "python:3.8",
"imagePullPolicy": "Always",
"imagePullSecrets": [{"name": "string",
"userCredential": true,
"exclude": false
}
],
"labels": [{"name": "stage",
"value": "initial-research",
"exclude": false
}
],
"nodeAffinityRequired": {"nodeSelectorTerms": [{"matchExpressions": [{"key": null,
"operator": null,
"values": [ ]
}
]
}
]
},
"nodeType": "my-node-type",
"podAffinity": {"type": "Required",
"key": "string"
},
"probes": {"readiness": {"initialDelaySeconds": 0,
"periodSeconds": 1,
"timeoutSeconds": 1,
"successThreshold": 1,
"failureThreshold": 1,
"handler": {"httpGet": {"path": "/",
"port": 1,
"host": "example.com",
"scheme": "HTTP"
}
}
}
},
"security": {"capabilities": ["CHOWN",
"KILL"
],
"readOnlyRootFilesystem": false,
"runAsGid": 30,
"runAsNonRoot": true,
"runAsUid": 500,
"seccompProfileType": "RuntimeDefault",
"supplementalGroups": "2,3,5,8",
"uidGidSource": "fromTheImage"
},
"storage": {"configMapVolume": [{"name": "storage-instance-a",
"configMap": "string",
"mountPath": "string",
"subPath": "string",
"defaultMode": "0644",
"exclude": false
}
],
"emptyDirVolume": [{"name": "storage-instance-a",
"path": "/mnt/emptydir",
"medium": "string",
"sizeLimit": "1G",
"exclude": false
}
],
"pvc": [{"name": "storage-instance-a",
"path": "/container/my-claim",
"existingPvc": false,
"claimName": "my-claim",
"readOnly": false,
"ephemeral": false,
"claimInfo": {"size": "1G",
"storageClass": "my-storage-class",
"accessModes": {"readWriteOnce": null,
"readOnlyMany": null,
"readWriteMany": null
},
"volumeMode": "Filesystem",
"addedAttrValues": [null
]
},
"dataSharing": false,
"exclude": false
}
],
"secretVolume": [{"name": "storage-instance-a",
"mountPath": "string",
"defaultMode": "0644",
"secret": "string",
"exclude": false
}
]
},
"tolerations": [{"name": "string",
"operator": "Equal",
"key": "string",
"value": "string",
"effect": "NoSchedule",
"seconds": 1,
"exclude": false
}
],
"workingDir": "/home/myfolder"
},
"worker": {"annotations": [{"name": "billing",
"value": "my-billing-unit",
"exclude": false
}
],
"args": "-x my-script.py",
"command": "python",
"compute": {"cpuCoreLimit": 2,
"cpuCoreRequest": 0.5,
"cpuMemoryLimit": "30M",
"cpuMemoryRequest": "20M",
"extendedResources": [{"resource": "hardware-vendor.example/foo",
"quantity": 2,
"exclude": false
}
],
"gpuDevicesRequest": 1,
"gpuMemoryLimit": "10M",
"gpuMemoryRequest": "10M",
"gpuPortionLimit": 0.5,
"gpuPortionRequest": 0.5,
"gpuRequestType": "portion",
"largeShmRequest": false
},
"createHomeDir": true,
"environmentVariables": [{"name": "HOME",
"value": "/home/my-folder",
"secret": {"name": "postgress_secret",
"key": "POSTGRES_PASSWORD"
},
"configMap": {"name": "my-config-map",
"key": "MY_POSTGRES_SCHEMA"
},
"podFieldRef": {"path": "metadata.name"
},
"userCredential": {"name": "my_postgres_user_and_password",
"key": "POSTGRES_PASSWORD"
},
"exclude": false,
"description": "Home directory of the user."
}
],
"image": "python:3.8",
"imagePullPolicy": "Always",
"imagePullSecrets": [{"name": "string",
"userCredential": true,
"exclude": false
}
],
"labels": [{"name": "stage",
"value": "initial-research",
"exclude": false
}
],
"nodeAffinityRequired": {"nodeSelectorTerms": [{"matchExpressions": [{"key": null,
"operator": null,
"values": [ ]
}
]
}
]
},
"nodeType": "my-node-type",
"podAffinity": {"type": "Required",
"key": "string"
},
"probes": {"readiness": {"initialDelaySeconds": 0,
"periodSeconds": 1,
"timeoutSeconds": 1,
"successThreshold": 1,
"failureThreshold": 1,
"handler": {"httpGet": {"path": "/",
"port": 1,
"host": "example.com",
"scheme": "HTTP"
}
}
}
},
"security": {"capabilities": ["CHOWN",
"KILL"
],
"readOnlyRootFilesystem": false,
"runAsGid": 30,
"runAsNonRoot": true,
"runAsUid": 500,
"seccompProfileType": "RuntimeDefault",
"supplementalGroups": "2,3,5,8",
"uidGidSource": "fromTheImage"
},
"storage": {"configMapVolume": [{"name": "storage-instance-a",
"configMap": "string",
"mountPath": "string",
"subPath": "string",
"defaultMode": "0644",
"exclude": false
}
],
"emptyDirVolume": [{"name": "storage-instance-a",
"path": "/mnt/emptydir",
"medium": "string",
"sizeLimit": "1G",
"exclude": false
}
],
"pvc": [{"name": "storage-instance-a",
"path": "/container/my-claim",
"existingPvc": false,
"claimName": "my-claim",
"readOnly": false,
"ephemeral": false,
"claimInfo": {"size": "1G",
"storageClass": "my-storage-class",
"accessModes": {"readWriteOnce": null,
"readOnlyMany": null,
"readWriteMany": null
},
"volumeMode": "Filesystem",
"addedAttrValues": [null
]
},
"dataSharing": false,
"exclude": false
}
],
"secretVolume": [{"name": "storage-instance-a",
"mountPath": "string",
"defaultMode": "0644",
"secret": "string",
"exclude": false
}
]
},
"tolerations": [{"name": "string",
"operator": "Equal",
"key": "string",
"value": "string",
"effect": "NoSchedule",
"seconds": 1,
"exclude": false
}
],
"workingDir": "/home/myfolder"
}
}
}

Response samples

application/json

{"name": "my-workload-name",
"requestedName": "string",
"workloadId": "06d16c5d-4728-42fa-b573-3b11820d999f",
"projectId": 1,
"departmentId": 2,
"clusterId": "71f69d83-ba66-4822-adf5-55ce55efd210",
"createdBy": "test@lab.com",
"createdAt": "2022-01-01T03:49:52.531Z",
"deletedAt": "2022-01-01T03:49:52.531Z",
"desiredPhase": "Running",
"actualPhase": "Creating",
"spec": {"category": "string",
"nodePools": ["my-node-pool-a",
"my-node-pool-b"
],
"preemptibility": "preemptible",
"priorityClass": "string",
"restartPolicy": "RecreateGroupOnPodRestart",
"servingPort": {"port": 8080,
"protocol": "http",
"authorizationType": "public",
"authorizedUsers": ["user.a@example.com",
"user.b@example.com"
],
"authorizedGroups": ["group-a",
"group-b"
],
"exposeExternally": true,
"exposedUrl": "string"
},
"startupPolicy": "LeaderCreated",
"workers": 4,
"replicas": 2,
"leader": {"annotations": [{"name": "billing",
"value": "my-billing-unit",
"exclude": false
}
],
"args": "-x my-script.py",
"command": "python",
"compute": {"cpuCoreLimit": 2,
"cpuCoreRequest": 0.5,
"cpuMemoryLimit": "30M",
"cpuMemoryRequest": "20M",
"extendedResources": [{"resource": "hardware-vendor.example/foo",
"quantity": 2,
"exclude": false
}
],
"gpuDevicesRequest": 1,
"gpuMemoryLimit": "10M",
"gpuMemoryRequest": "10M",
"gpuPortionLimit": 0.5,
"gpuPortionRequest": 0.5,
"gpuRequestType": "portion",
"largeShmRequest": false
},
"createHomeDir": true,
"environmentVariables": [{"name": "HOME",
"value": "/home/my-folder",
"secret": {"name": "postgress_secret",
"key": "POSTGRES_PASSWORD"
},
"configMap": {"name": "my-config-map",
"key": "MY_POSTGRES_SCHEMA"
},
"podFieldRef": {"path": "metadata.name"
},
"userCredential": {"name": "my_postgres_user_and_password",
"key": "POSTGRES_PASSWORD"
},
"exclude": false,
"description": "Home directory of the user."
}
],
"image": "python:3.8",
"imagePullPolicy": "Always",
"imagePullSecrets": [{"name": "string",
"userCredential": true,
"exclude": false
}
],
"labels": [{"name": "stage",
"value": "initial-research",
"exclude": false
}
],
"nodeAffinityRequired": {"nodeSelectorTerms": [{"matchExpressions": [{"key": null,
"operator": null,
"values": [ ]
}
]
}
]
},
"nodeType": "my-node-type",
"podAffinity": {"type": "Required",
"key": "string"
},
"probes": {"readiness": {"initialDelaySeconds": 0,
"periodSeconds": 1,
"timeoutSeconds": 1,
"successThreshold": 1,
"failureThreshold": 1,
"handler": {"httpGet": {"path": "/",
"port": 1,
"host": "example.com",
"scheme": "HTTP"
}
}
}
},
"security": {"capabilities": ["CHOWN",
"KILL"
],
"readOnlyRootFilesystem": false,
"runAsGid": 30,
"runAsNonRoot": true,
"runAsUid": 500,
"seccompProfileType": "RuntimeDefault",
"supplementalGroups": "2,3,5,8",
"uidGidSource": "fromTheImage"
},
"storage": {"configMapVolume": [{"name": "storage-instance-a",
"configMap": "string",
"mountPath": "string",
"subPath": "string",
"defaultMode": "0644",
"exclude": false
}
],
"emptyDirVolume": [{"name": "storage-instance-a",
"path": "/mnt/emptydir",
"medium": "string",
"sizeLimit": "1G",
"exclude": false
}
],
"pvc": [{"name": "storage-instance-a",
"path": "/container/my-claim",
"existingPvc": false,
"claimName": "my-claim",
"readOnly": false,
"ephemeral": false,
"claimInfo": {"size": "1G",
"storageClass": "my-storage-class",
"accessModes": {"readWriteOnce": null,
"readOnlyMany": null,
"readWriteMany": null
},
"volumeMode": "Filesystem",
"addedAttrValues": [null
]
},
"dataSharing": false,
"exclude": false
}
],
"secretVolume": [{"name": "storage-instance-a",
"mountPath": "string",
"defaultMode": "0644",
"secret": "string",
"exclude": false
}
]
},
"tolerations": [{"name": "string",
"operator": "Equal",
"key": "string",
"value": "string",
"effect": "NoSchedule",
"seconds": 1,
"exclude": false
}
],
"workingDir": "/home/myfolder"
},
"worker": {"annotations": [{"name": "billing",
"value": "my-billing-unit",
"exclude": false
}
],
"args": "-x my-script.py",
"command": "python",
"compute": {"cpuCoreLimit": 2,
"cpuCoreRequest": 0.5,
"cpuMemoryLimit": "30M",
"cpuMemoryRequest": "20M",
"extendedResources": [{"resource": "hardware-vendor.example/foo",
"quantity": 2,
"exclude": false
}
],
"gpuDevicesRequest": 1,
"gpuMemoryLimit": "10M",
"gpuMemoryRequest": "10M",
"gpuPortionLimit": 0.5,
"gpuPortionRequest": 0.5,
"gpuRequestType": "portion",
"largeShmRequest": false
},
"createHomeDir": true,
"environmentVariables": [{"name": "HOME",
"value": "/home/my-folder",
"secret": {"name": "postgress_secret",
"key": "POSTGRES_PASSWORD"
},
"configMap": {"name": "my-config-map",
"key": "MY_POSTGRES_SCHEMA"
},
"podFieldRef": {"path": "metadata.name"
},
"userCredential": {"name": "my_postgres_user_and_password",
"key": "POSTGRES_PASSWORD"
},
"exclude": false,
"description": "Home directory of the user."
}
],
"image": "python:3.8",
"imagePullPolicy": "Always",
"imagePullSecrets": [{"name": "string",
"userCredential": true,
"exclude": false
}
],
"labels": [{"name": "stage",
"value": "initial-research",
"exclude": false
}
],
"nodeAffinityRequired": {"nodeSelectorTerms": [{"matchExpressions": [{"key": null,
"operator": null,
"values": [ ]
}
]
}
]
},
"nodeType": "my-node-type",
"podAffinity": {"type": "Required",
"key": "string"
},
"probes": {"readiness": {"initialDelaySeconds": 0,
"periodSeconds": 1,
"timeoutSeconds": 1,
"successThreshold": 1,
"failureThreshold": 1,
"handler": {"httpGet": {"path": "/",
"port": 1,
"host": "example.com",
"scheme": "HTTP"
}
}
}
},
"security": {"capabilities": ["CHOWN",
"KILL"
],
"readOnlyRootFilesystem": false,
"runAsGid": 30,
"runAsNonRoot": true,
"runAsUid": 500,
"seccompProfileType": "RuntimeDefault",
"supplementalGroups": "2,3,5,8",
"uidGidSource": "fromTheImage"
},
"storage": {"configMapVolume": [{"name": "storage-instance-a",
"configMap": "string",
"mountPath": "string",
"subPath": "string",
"defaultMode": "0644",
"exclude": false
}
],
"emptyDirVolume": [{"name": "storage-instance-a",
"path": "/mnt/emptydir",
"medium": "string",
"sizeLimit": "1G",
"exclude": false
}
],
"pvc": [{"name": "storage-instance-a",
"path": "/container/my-claim",
"existingPvc": false,
"claimName": "my-claim",
"readOnly": false,
"ephemeral": false,
"claimInfo": {"size": "1G",
"storageClass": "my-storage-class",
"accessModes": {"readWriteOnce": null,
"readOnlyMany": null,
"readWriteMany": null
},
"volumeMode": "Filesystem",
"addedAttrValues": [null
]
},
"dataSharing": false,
"exclude": false
}
],
"secretVolume": [{"name": "storage-instance-a",
"mountPath": "string",
"defaultMode": "0644",
"secret": "string",
"exclude": false
}
]
},
"tolerations": [{"name": "string",
"operator": "Equal",
"key": "string",
"value": "string",
"effect": "NoSchedule",
"seconds": 1,
"exclude": false
}
],
"workingDir": "/home/myfolder"
}
}
}

Delete a distributed inference.

Delete a distributed inference using a workload id.

SecuritybearerAuth

Request

path Parameters

workloadId

required

string <uuid>

The Universally Unique Identifier (UUID) of the workload.

Responses

202

Accepted.

401

Unauthorized

403

Forbidden

404

The specified resource was not found

500

unexpected error

503

unexpected error

delete/api/v1/workloads/distributed-inferences/{workloadId}

Response samples

application/json

{"code": 202,
"message": "Request has been accepted."
}

Get a distributed inference data.

Retrieve a distributed inference details using a workload id.

SecuritybearerAuth

Request

path Parameters

workloadId

required

string <uuid>

The Universally Unique Identifier (UUID) of the workload.

Responses

200

Executed successfully.

401

Unauthorized

403

Forbidden

404

The specified resource was not found

500

unexpected error

503

unexpected error

get/api/v1/workloads/distributed-inferences/{workloadId}

Response samples

application/json

{"name": "my-workload-name",
"requestedName": "string",
"workloadId": "06d16c5d-4728-42fa-b573-3b11820d999f",
"projectId": 1,
"departmentId": 2,
"clusterId": "71f69d83-ba66-4822-adf5-55ce55efd210",
"createdBy": "test@lab.com",
"createdAt": "2022-01-01T03:49:52.531Z",
"deletedAt": "2022-01-01T03:49:52.531Z",
"desiredPhase": "Running",
"actualPhase": "Creating",
"spec": {"category": "string",
"nodePools": ["my-node-pool-a",
"my-node-pool-b"
],
"preemptibility": "preemptible",
"priorityClass": "string",
"restartPolicy": "RecreateGroupOnPodRestart",
"servingPort": {"port": 8080,
"protocol": "http",
"authorizationType": "public",
"authorizedUsers": ["user.a@example.com",
"user.b@example.com"
],
"authorizedGroups": ["group-a",
"group-b"
],
"exposeExternally": true,
"exposedUrl": "string"
},
"startupPolicy": "LeaderCreated",
"workers": 4,
"replicas": 2,
"leader": {"annotations": [{"name": "billing",
"value": "my-billing-unit",
"exclude": false
}
],
"args": "-x my-script.py",
"command": "python",
"compute": {"cpuCoreLimit": 2,
"cpuCoreRequest": 0.5,
"cpuMemoryLimit": "30M",
"cpuMemoryRequest": "20M",
"extendedResources": [{"resource": "hardware-vendor.example/foo",
"quantity": 2,
"exclude": false
}
],
"gpuDevicesRequest": 1,
"gpuMemoryLimit": "10M",
"gpuMemoryRequest": "10M",
"gpuPortionLimit": 0.5,
"gpuPortionRequest": 0.5,
"gpuRequestType": "portion",
"largeShmRequest": false
},
"createHomeDir": true,
"environmentVariables": [{"name": "HOME",
"value": "/home/my-folder",
"secret": {"name": "postgress_secret",
"key": "POSTGRES_PASSWORD"
},
"configMap": {"name": "my-config-map",
"key": "MY_POSTGRES_SCHEMA"
},
"podFieldRef": {"path": "metadata.name"
},
"userCredential": {"name": "my_postgres_user_and_password",
"key": "POSTGRES_PASSWORD"
},
"exclude": false,
"description": "Home directory of the user."
}
],
"image": "python:3.8",
"imagePullPolicy": "Always",
"imagePullSecrets": [{"name": "string",
"userCredential": true,
"exclude": false
}
],
"labels": [{"name": "stage",
"value": "initial-research",
"exclude": false
}
],
"nodeAffinityRequired": {"nodeSelectorTerms": [{"matchExpressions": [{"key": null,
"operator": null,
"values": [ ]
}
]
}
]
},
"nodeType": "my-node-type",
"podAffinity": {"type": "Required",
"key": "string"
},
"probes": {"readiness": {"initialDelaySeconds": 0,
"periodSeconds": 1,
"timeoutSeconds": 1,
"successThreshold": 1,
"failureThreshold": 1,
"handler": {"httpGet": {"path": "/",
"port": 1,
"host": "example.com",
"scheme": "HTTP"
}
}
}
},
"security": {"capabilities": ["CHOWN",
"KILL"
],
"readOnlyRootFilesystem": false,
"runAsGid": 30,
"runAsNonRoot": true,
"runAsUid": 500,
"seccompProfileType": "RuntimeDefault",
"supplementalGroups": "2,3,5,8",
"uidGidSource": "fromTheImage"
},
"storage": {"configMapVolume": [{"name": "storage-instance-a",
"configMap": "string",
"mountPath": "string",
"subPath": "string",
"defaultMode": "0644",
"exclude": false
}
],
"emptyDirVolume": [{"name": "storage-instance-a",
"path": "/mnt/emptydir",
"medium": "string",
"sizeLimit": "1G",
"exclude": false
}
],
"pvc": [{"name": "storage-instance-a",
"path": "/container/my-claim",
"existingPvc": false,
"claimName": "my-claim",
"readOnly": false,
"ephemeral": false,
"claimInfo": {"size": "1G",
"storageClass": "my-storage-class",
"accessModes": {"readWriteOnce": null,
"readOnlyMany": null,
"readWriteMany": null
},
"volumeMode": "Filesystem",
"addedAttrValues": [null
]
},
"dataSharing": false,
"exclude": false
}
],
"secretVolume": [{"name": "storage-instance-a",
"mountPath": "string",
"defaultMode": "0644",
"secret": "string",
"exclude": false
}
]
},
"tolerations": [{"name": "string",
"operator": "Equal",
"key": "string",
"value": "string",
"effect": "NoSchedule",
"seconds": 1,
"exclude": false
}
],
"workingDir": "/home/myfolder"
},
"worker": {"annotations": [{"name": "billing",
"value": "my-billing-unit",
"exclude": false
}
],
"args": "-x my-script.py",
"command": "python",
"compute": {"cpuCoreLimit": 2,
"cpuCoreRequest": 0.5,
"cpuMemoryLimit": "30M",
"cpuMemoryRequest": "20M",
"extendedResources": [{"resource": "hardware-vendor.example/foo",
"quantity": 2,
"exclude": false
}
],
"gpuDevicesRequest": 1,
"gpuMemoryLimit": "10M",
"gpuMemoryRequest": "10M",
"gpuPortionLimit": 0.5,
"gpuPortionRequest": 0.5,
"gpuRequestType": "portion",
"largeShmRequest": false
},
"createHomeDir": true,
"environmentVariables": [{"name": "HOME",
"value": "/home/my-folder",
"secret": {"name": "postgress_secret",
"key": "POSTGRES_PASSWORD"
},
"configMap": {"name": "my-config-map",
"key": "MY_POSTGRES_SCHEMA"
},
"podFieldRef": {"path": "metadata.name"
},
"userCredential": {"name": "my_postgres_user_and_password",
"key": "POSTGRES_PASSWORD"
},
"exclude": false,
"description": "Home directory of the user."
}
],
"image": "python:3.8",
"imagePullPolicy": "Always",
"imagePullSecrets": [{"name": "string",
"userCredential": true,
"exclude": false
}
],
"labels": [{"name": "stage",
"value": "initial-research",
"exclude": false
}
],
"nodeAffinityRequired": {"nodeSelectorTerms": [{"matchExpressions": [{"key": null,
"operator": null,
"values": [ ]
}
]
}
]
},
"nodeType": "my-node-type",
"podAffinity": {"type": "Required",
"key": "string"
},
"probes": {"readiness": {"initialDelaySeconds": 0,
"periodSeconds": 1,
"timeoutSeconds": 1,
"successThreshold": 1,
"failureThreshold": 1,
"handler": {"httpGet": {"path": "/",
"port": 1,
"host": "example.com",
"scheme": "HTTP"
}
}
}
},
"security": {"capabilities": ["CHOWN",
"KILL"
],
"readOnlyRootFilesystem": false,
"runAsGid": 30,
"runAsNonRoot": true,
"runAsUid": 500,
"seccompProfileType": "RuntimeDefault",
"supplementalGroups": "2,3,5,8",
"uidGidSource": "fromTheImage"
},
"storage": {"configMapVolume": [{"name": "storage-instance-a",
"configMap": "string",
"mountPath": "string",
"subPath": "string",
"defaultMode": "0644",
"exclude": false
}
],
"emptyDirVolume": [{"name": "storage-instance-a",
"path": "/mnt/emptydir",
"medium": "string",
"sizeLimit": "1G",
"exclude": false
}
],
"pvc": [{"name": "storage-instance-a",
"path": "/container/my-claim",
"existingPvc": false,
"claimName": "my-claim",
"readOnly": false,
"ephemeral": false,
"claimInfo": {"size": "1G",
"storageClass": "my-storage-class",
"accessModes": {"readWriteOnce": null,
"readOnlyMany": null,
"readWriteMany": null
},
"volumeMode": "Filesystem",
"addedAttrValues": [null
]
},
"dataSharing": false,
"exclude": false
}
],
"secretVolume": [{"name": "storage-instance-a",
"mountPath": "string",
"defaultMode": "0644",
"secret": "string",
"exclude": false
}
]
},
"tolerations": [{"name": "string",
"operator": "Equal",
"key": "string",
"value": "string",
"effect": "NoSchedule",
"seconds": 1,
"exclude": false
}
],
"workingDir": "/home/myfolder"
}
}
}

Update distributed inference spec.

Update the specification of an existing distributed inference workload.

SecuritybearerAuth

Request

path Parameters

workloadId

required

string <uuid>

The Universally Unique Identifier (UUID) of the workload.

Request Body schema: application/json

object or null

Responses

202

Executed successfully.

401

Unauthorized

403

Forbidden

404

The specified resource was not found

500

unexpected error

503

unexpected error

patch/api/v1/workloads/distributed-inferences/{workloadId}

Request samples

Payload

application/json

{"spec": {"replicas": 2
}
}

Response samples

application/json

{"name": "my-workload-name",
"requestedName": "string",
"workloadId": "06d16c5d-4728-42fa-b573-3b11820d999f",
"projectId": 1,
"departmentId": 2,
"clusterId": "71f69d83-ba66-4822-adf5-55ce55efd210",
"createdBy": "test@lab.com",
"createdAt": "2022-01-01T03:49:52.531Z",
"deletedAt": "2022-01-01T03:49:52.531Z",
"desiredPhase": "Running",
"actualPhase": "Creating",
"spec": {"category": "string",
"nodePools": ["my-node-pool-a",
"my-node-pool-b"
],
"preemptibility": "preemptible",
"priorityClass": "string",
"restartPolicy": "RecreateGroupOnPodRestart",
"servingPort": {"port": 8080,
"protocol": "http",
"authorizationType": "public",
"authorizedUsers": ["user.a@example.com",
"user.b@example.com"
],
"authorizedGroups": ["group-a",
"group-b"
],
"exposeExternally": true,
"exposedUrl": "string"
},
"startupPolicy": "LeaderCreated",
"workers": 4,
"replicas": 2,
"leader": {"annotations": [{"name": "billing",
"value": "my-billing-unit",
"exclude": false
}
],
"args": "-x my-script.py",
"command": "python",
"compute": {"cpuCoreLimit": 2,
"cpuCoreRequest": 0.5,
"cpuMemoryLimit": "30M",
"cpuMemoryRequest": "20M",
"extendedResources": [{"resource": "hardware-vendor.example/foo",
"quantity": 2,
"exclude": false
}
],
"gpuDevicesRequest": 1,
"gpuMemoryLimit": "10M",
"gpuMemoryRequest": "10M",
"gpuPortionLimit": 0.5,
"gpuPortionRequest": 0.5,
"gpuRequestType": "portion",
"largeShmRequest": false
},
"createHomeDir": true,
"environmentVariables": [{"name": "HOME",
"value": "/home/my-folder",
"secret": {"name": "postgress_secret",
"key": "POSTGRES_PASSWORD"
},
"configMap": {"name": "my-config-map",
"key": "MY_POSTGRES_SCHEMA"
},
"podFieldRef": {"path": "metadata.name"
},
"userCredential": {"name": "my_postgres_user_and_password",
"key": "POSTGRES_PASSWORD"
},
"exclude": false,
"description": "Home directory of the user."
}
],
"image": "python:3.8",
"imagePullPolicy": "Always",
"imagePullSecrets": [{"name": "string",
"userCredential": true,
"exclude": false
}
],
"labels": [{"name": "stage",
"value": "initial-research",
"exclude": false
}
],
"nodeAffinityRequired": {"nodeSelectorTerms": [{"matchExpressions": [{"key": null,
"operator": null,
"values": [ ]
}
]
}
]
},
"nodeType": "my-node-type",
"podAffinity": {"type": "Required",
"key": "string"
},
"probes": {"readiness": {"initialDelaySeconds": 0,
"periodSeconds": 1,
"timeoutSeconds": 1,
"successThreshold": 1,
"failureThreshold": 1,
"handler": {"httpGet": {"path": "/",
"port": 1,
"host": "example.com",
"scheme": "HTTP"
}
}
}
},
"security": {"capabilities": ["CHOWN",
"KILL"
],
"readOnlyRootFilesystem": false,
"runAsGid": 30,
"runAsNonRoot": true,
"runAsUid": 500,
"seccompProfileType": "RuntimeDefault",
"supplementalGroups": "2,3,5,8",
"uidGidSource": "fromTheImage"
},
"storage": {"configMapVolume": [{"name": "storage-instance-a",
"configMap": "string",
"mountPath": "string",
"subPath": "string",
"defaultMode": "0644",
"exclude": false
}
],
"emptyDirVolume": [{"name": "storage-instance-a",
"path": "/mnt/emptydir",
"medium": "string",
"sizeLimit": "1G",
"exclude": false
}
],
"pvc": [{"name": "storage-instance-a",
"path": "/container/my-claim",
"existingPvc": false,
"claimName": "my-claim",
"readOnly": false,
"ephemeral": false,
"claimInfo": {"size": "1G",
"storageClass": "my-storage-class",
"accessModes": {"readWriteOnce": null,
"readOnlyMany": null,
"readWriteMany": null
},
"volumeMode": "Filesystem",
"addedAttrValues": [null
]
},
"dataSharing": false,
"exclude": false
}
],
"secretVolume": [{"name": "storage-instance-a",
"mountPath": "string",
"defaultMode": "0644",
"secret": "string",
"exclude": false
}
]
},
"tolerations": [{"name": "string",
"operator": "Equal",
"key": "string",
"value": "string",
"effect": "NoSchedule",
"seconds": 1,
"exclude": false
}
],
"workingDir": "/home/myfolder"
},
"worker": {"annotations": [{"name": "billing",
"value": "my-billing-unit",
"exclude": false
}
],
"args": "-x my-script.py",
"command": "python",
"compute": {"cpuCoreLimit": 2,
"cpuCoreRequest": 0.5,
"cpuMemoryLimit": "30M",
"cpuMemoryRequest": "20M",
"extendedResources": [{"resource": "hardware-vendor.example/foo",
"quantity": 2,
"exclude": false
}
],
"gpuDevicesRequest": 1,
"gpuMemoryLimit": "10M",
"gpuMemoryRequest": "10M",
"gpuPortionLimit": 0.5,
"gpuPortionRequest": 0.5,
"gpuRequestType": "portion",
"largeShmRequest": false
},
"createHomeDir": true,
"environmentVariables": [{"name": "HOME",
"value": "/home/my-folder",
"secret": {"name": "postgress_secret",
"key": "POSTGRES_PASSWORD"
},
"configMap": {"name": "my-config-map",
"key": "MY_POSTGRES_SCHEMA"
},
"podFieldRef": {"path": "metadata.name"
},
"userCredential": {"name": "my_postgres_user_and_password",
"key": "POSTGRES_PASSWORD"
},
"exclude": false,
"description": "Home directory of the user."
}
],
"image": "python:3.8",
"imagePullPolicy": "Always",
"imagePullSecrets": [{"name": "string",
"userCredential": true,
"exclude": false
}
],
"labels": [{"name": "stage",
"value": "initial-research",
"exclude": false
}
],
"nodeAffinityRequired": {"nodeSelectorTerms": [{"matchExpressions": [{"key": null,
"operator": null,
"values": [ ]
}
]
}
]
},
"nodeType": "my-node-type",
"podAffinity": {"type": "Required",
"key": "string"
},
"probes": {"readiness": {"initialDelaySeconds": 0,
"periodSeconds": 1,
"timeoutSeconds": 1,
"successThreshold": 1,
"failureThreshold": 1,
"handler": {"httpGet": {"path": "/",
"port": 1,
"host": "example.com",
"scheme": "HTTP"
}
}
}
},
"security": {"capabilities": ["CHOWN",
"KILL"
],
"readOnlyRootFilesystem": false,
"runAsGid": 30,
"runAsNonRoot": true,
"runAsUid": 500,
"seccompProfileType": "RuntimeDefault",
"supplementalGroups": "2,3,5,8",
"uidGidSource": "fromTheImage"
},
"storage": {"configMapVolume": [{"name": "storage-instance-a",
"configMap": "string",
"mountPath": "string",
"subPath": "string",
"defaultMode": "0644",
"exclude": false
}
],
"emptyDirVolume": [{"name": "storage-instance-a",
"path": "/mnt/emptydir",
"medium": "string",
"sizeLimit": "1G",
"exclude": false
}
],
"pvc": [{"name": "storage-instance-a",
"path": "/container/my-claim",
"existingPvc": false,
"claimName": "my-claim",
"readOnly": false,
"ephemeral": false,
"claimInfo": {"size": "1G",
"storageClass": "my-storage-class",
"accessModes": {"readWriteOnce": null,
"readOnlyMany": null,
"readWriteMany": null
},
"volumeMode": "Filesystem",
"addedAttrValues": [null
]
},
"dataSharing": false,
"exclude": false
}
],
"secretVolume": [{"name": "storage-instance-a",
"mountPath": "string",
"defaultMode": "0644",
"secret": "string",
"exclude": false
}
]
},
"tolerations": [{"name": "string",
"operator": "Equal",
"key": "string",
"value": "string",
"effect": "NoSchedule",
"seconds": 1,
"exclude": false
}
],
"workingDir": "/home/myfolder"
}
}
}

➔ Next to Compute