From cff9c4d2359f8797b3b1cc5990d1bd3e69a2c7ce Mon Sep 17 00:00:00 2001 From: "susheel.varma@hdruk.ac.uk" Date: Thu, 2 Jul 2020 02:59:13 +0100 Subject: [PATCH 1/7] Added api/task_execution.openapi.yaml --- api/task_execution.openapi.yaml | 632 ++++++++++++++++++++++++++++++++ 1 file changed, 632 insertions(+) create mode 100644 api/task_execution.openapi.yaml diff --git a/api/task_execution.openapi.yaml b/api/task_execution.openapi.yaml new file mode 100644 index 0000000..9e47f56 --- /dev/null +++ b/api/task_execution.openapi.yaml @@ -0,0 +1,632 @@ +swagger: '2.0' +info: + title: Task Execution Service + version: 0.4.0 +schemes: + - http +consumes: + - application/json +produces: + - application/json +basePath: /tes/v1 +paths: + /tasks: + get: + summary: |- + List tasks. + TaskView is requested as such: "v1/tasks?view=BASIC" + operationId: ListTasks + responses: + '200': + description: '' + schema: + $ref: '#/definitions/tesListTasksResponse' + parameters: + - name: name_prefix + description: |- + OPTIONAL. Filter the list to include tasks where the name matches this prefix. + If unspecified, no task name filtering is done. + in: query + required: false + type: string + - name: page_size + description: |- + OPTIONAL. Number of tasks to return in one page. + Must be less than 2048. Defaults to 256. + in: query + required: false + type: integer + format: int64 + - name: page_token + description: |- + OPTIONAL. Page token is used to retrieve the next page of results. + If unspecified, returns the first page of results. + See ListTasksResponse.next_page_token + in: query + required: false + type: string + - name: view + description: |- + OPTIONAL. Affects the fields included in the returned Task messages. + See TaskView below. + + - MINIMAL: Task message will include ONLY the fields: + Task.Id + Task.State + - BASIC: Task message will include all fields EXCEPT: + Task.ExecutorLog.stdout + Task.ExecutorLog.stderr + Input.content + TaskLog.system_logs + - FULL: Task message includes all fields. + in: query + required: false + type: string + enum: + - MINIMAL + - BASIC + - FULL + default: MINIMAL + tags: + - TaskService + post: + summary: Create a new task. + operationId: CreateTask + responses: + '200': + description: '' + schema: + $ref: '#/definitions/tesCreateTaskResponse' + parameters: + - name: body + in: body + required: true + schema: + $ref: '#/definitions/tesTask' + tags: + - TaskService + /tasks/service-info: + get: + summary: |- + GetServiceInfo provides information about the service, + such as storage details, resource availability, and + other documentation. + operationId: GetServiceInfo + responses: + '200': + description: '' + schema: + $ref: '#/definitions/tesServiceInfo' + tags: + - TaskService + '/tasks/{id}': + get: + summary: |- + Get a task. + TaskView is requested as such: "v1/tasks/{id}?view=FULL" + operationId: GetTask + responses: + '200': + description: '' + schema: + $ref: '#/definitions/tesTask' + parameters: + - name: view + description: |- + OPTIONAL. Affects the fields included in the returned Task messages. + See TaskView below. + + - MINIMAL: Task message will include ONLY the fields: + Task.Id + Task.State + - BASIC: Task message will include all fields EXCEPT: + Task.ExecutorLog.stdout + Task.ExecutorLog.stderr + Input.content + TaskLog.system_logs + - FULL: Task message includes all fields. + in: query + required: false + type: string + enum: + - MINIMAL + - BASIC + - FULL + default: MINIMAL + tags: + - TaskService + delete: + summary: '' + operationId: DeleteTask + responses: + '200': + description: OK + schema: + $ref: '#/definitions/tesCancelTaskResponse' + parameters: + - type: string + name: id + in: path + required: true + description: Task ID + /tasks/validate: + post: + summary: Validate Task + tags: + - TaskService + responses: + '200': + description: OK + schema: + $ref: '#/definitions/tesValidateTaskResponse' + '400': + description: Bad Request + schema: + $ref: '#/definitions/tesCancelTaskResponse' + operationId: post-tasks-validate + parameters: + - in: body + name: body + schema: + $ref: '#/definitions/tesTask' +definitions: + tesCancelTaskResponse: + type: object + description: CancelTaskResponse describes a response from the CancelTask endpoint. + readOnly: true + tesCreateTaskResponse: + type: object + properties: + id: + type: string + description: Task identifier assigned by the server. + description: CreateTaskResponse describes a response from the CreateTask endpoint. + readOnly: true + required: + - id + tesExecutor: + type: object + properties: + image: + type: string + description: |- + Name of the container image, for example: + ubuntu + quay.io/aptible/ubuntu + gcr.io/my-org/my-image + etc... + command: + type: array + items: + type: string + description: |- + A sequence of program arguments to execute, where the first argument + is the program to execute (i.e. argv). + workdir: + type: string + description: |- + The working directory that the command will be executed in. + Defaults to the directory set by the container image. + stdin: + type: string + description: |- + Path inside the container to a file which will be piped + to the executor's stdin. Must be an absolute path. + stdout: + type: string + description: |- + Path inside the container to a file where the executor's + stdout will be written to. Must be an absolute path. + stderr: + type: string + description: |- + Path inside the container to a file where the executor's + stderr will be written to. Must be an absolute path. + env: + type: object + additionalProperties: + type: string + description: Enviromental variables to set within the container. + description: 'Executor describes a command to be executed, and its environment.' + required: + - image + - command + tesExecutorLog: + type: object + properties: + start_time: + type: string + description: 'Time the executor started, in RFC 3339 format.' + end_time: + type: string + description: 'Time the executor ended, in RFC 3339 format.' + stdout: + type: string + description: |- + Stdout content. + + This is meant for convenience. No guarantees are made about the content. + Implementations may chose different approaches: only the head, only the tail, + a URL reference only, etc. + + In order to capture the full stdout users should set Executor.stdout + to a container file path, and use Task.outputs to upload that file + to permanent storage. + stderr: + type: string + description: |- + Stderr content. + + This is meant for convenience. No guarantees are made about the content. + Implementations may chose different approaches: only the head, only the tail, + a URL reference only, etc. + + In order to capture the full stderr users should set Executor.stderr + to a container file path, and use Task.outputs to upload that file + to permanent storage. + exit_code: + type: integer + format: int32 + description: Exit code. + description: ExecutorLog describes logging information related to an Executor. + required: + - exit_code + readOnly: true + tesFileType: + type: string + enum: + - FILE + - DIRECTORY + default: FILE + tesInput: + description: Input describes Task input files. + anyOf: + - properties: + name: + type: string + description: + type: string + url: + type: string + description: |- + REQUIRED, unless "content" is set. + + URL in long term storage, for example: + s3://my-object-store/file1 + gs://my-bucket/file2 + file:///path/to/my/file + /path/to/my/file + etc... + path: + type: string + description: |- + Path of the file inside the container. + Must be an absolute path. + type: + $ref: '#/definitions/tesFileType' + content: + type: string + description: |- + File content literal. + Implementations should support a minimum of 128 KiB in this field and may define its own maximum. + UTF-8 encoded + + If content is not empty, "url" must be ignored. + required: + - path + - type + - properties: + selection: + type: string + type: object + tesListTasksResponse: + type: object + properties: + tasks: + type: array + items: + $ref: '#/definitions/tesTask' + description: List of tasks. + next_page_token: + type: string + description: |- + Token used to return the next page of results. + See TaskListRequest.next_page_token + description: ListTasksResponse describes a response from the ListTasks endpoint. + required: + - tasks + readOnly: true + tesOutput: + type: object + properties: + name: + type: string + description: + type: string + url: + type: string + description: |- + URL in long term storage, for example: + s3://my-object-store/file1 + gs://my-bucket/file2 + file:///path/to/my/file + /path/to/my/file + etc... + path: + type: string + description: |- + Path of the file inside the container. + Must be an absolute path. + type: + allOf: + - $ref: '#/definitions/tesFileType' + description: 'Type of the file, FILE or DIRECTORY' + description: Output describes Task output files. + required: + - url + - path + - type + tesOutputFileLog: + type: object + properties: + url: + type: string + description: 'URL of the file in storage, e.g. s3://bucket/file.txt' + path: + type: string + description: Path of the file inside the container. Must be an absolute path. + size_bytes: + type: string + format: int64 + description: Size of the file in bytes. + description: |- + OutputFileLog describes a single output file. This describes + file details after the task has completed successfully, + for logging purposes. + readOnly: true + required: + - url + - path + - size_bytes + tesResources: + type: object + properties: + cpu_cores: + type: integer + format: int64 + description: Requested number of CPUs + preemptible: + type: boolean + format: boolean + description: Is the task allowed to run on preemptible compute instances (e.g. AWS Spot)? + ram_gb: + type: number + format: double + description: Requested RAM required in gigabytes (GB) + disk_gb: + type: number + format: double + description: Requested disk size in gigabytes (GB) + zones: + type: array + items: + type: string + description: Request that the task be run in these compute zones. + description: Resources describes the resources requested by a task. + tesServiceInfo: + type: object + description: |- + ServiceInfo describes information about the service, + such as storage details, resource availability, + and other documentation. + readOnly: true + properties: + name: + type: string + description: 'Returns the name of the service, e.g. "ohsu-compbio-funnel".' + doc: + type: string + description: 'Returns a documentation string, e.g. "Hey, we''re OHSU Comp. Bio!".' + storage: + type: array + description: |- + Lists some, but not necessarily all, storage locations supported by the service. + + Must be in a valid URL format. + e.g. + file:///path/to/local/funnel-storage + s3://ohsu-compbio-funnel/storage + etc. + items: + type: string + registries: + type: array + description: Approved Registries + items: + type: object + properties: + registry_id: + type: string + registry_name: + type: string + registry_host: + type: string + registry url: + type: string + tesState: + type: string + enum: + - UNKNOWN + - QUEUED + - INITIALIZING + - RUNNING + - PAUSED + - COMPLETE + - EXECUTOR_ERROR + - SYSTEM_ERROR + - CANCELED + default: UNKNOWN + description: |- + Task states. + + - UNKNOWN: The state of the task is unknown. + + This provides a safe default for messages where this field is missing, + for example, so that a missing field does not accidentally imply that + the state is QUEUED. + - QUEUED: The task is queued. + - INITIALIZING: The task has been assigned to a worker and is currently preparing to run. + For example, the worker may be turning on, downloading input files, etc. + - RUNNING: The task is running. Input files are downloaded and the first Executor + has been started. + - PAUSED: The task is paused. + + An implementation may have the ability to pause a task, but this is not required. + - COMPLETE: The task has completed running. Executors have exited without error + and output files have been successfully uploaded. + - EXECUTOR_ERROR: The task encountered an error in one of the Executor processes. Generally, + this means that an Executor exited with a non-zero exit code. + - SYSTEM_ERROR: The task was stopped due to a system error, but not from an Executor, + for example an upload failed due to network issues, the worker's ran out + of disk space, etc. + - CANCELED: The task was canceled by the user. + readOnly: true + tesTask: + type: object + properties: + id: + type: string + description: Task identifier assigned by the server. + readOnly: true + state: + allOf: + - $ref: '#/definitions/tesState' + readOnly: true + name: + type: string + description: + type: string + inputs: + type: array + items: + $ref: '#/definitions/tesInput' + description: |- + Input files. + Inputs will be downloaded and mounted into the executor container. + outputs: + type: array + items: + $ref: '#/definitions/tesOutput' + description: |- + Output files. + Outputs will be uploaded from the executor container to long-term storage. + resources: + allOf: + - $ref: '#/definitions/tesResources' + description: Request that the task be run with these resources. + executors: + type: array + items: + $ref: '#/definitions/tesExecutor' + description: |- + A list of executors to be run, sequentially. Execution stops + on the first error. + volumes: + type: array + items: + type: string + description: |- + Volumes are directories which may be used to share data between + Executors. Volumes are initialized as empty directories by the + system when the task starts and are mounted at the same path + in each Executor. + + For example, given a volume defined at "/vol/A", + executor 1 may write a file to "/vol/A/exec1.out.txt", then + executor 2 may read from that file. + + (Essentially, this translates to a `docker run -v` flag where + the container path is the same for each executor). + tags: + type: object + additionalProperties: + type: string + description: A key-value map of arbitrary tags. + logs: + type: array + items: + $ref: '#/definitions/tesTaskLog' + description: |- + Task logging information. + Normally, this will contain only one entry, but in the case where + a task fails and is retried, an entry will be appended to this list. + readOnly: true + creation_time: + type: string + description: |- + Date + time the task was created, in RFC 3339 format. + This is set by the system, not the client. + readOnly: true + description: Task describes an instance of a task. + required: + - executors + tesTaskLog: + type: object + properties: + logs: + type: array + items: + $ref: '#/definitions/tesExecutorLog' + description: Logs for each executor + metadata: + type: object + additionalProperties: + type: string + description: Arbitrary logging metadata included by the implementation. + start_time: + type: string + description: 'When the task started, in RFC 3339 format.' + end_time: + type: string + description: 'When the task ended, in RFC 3339 format.' + outputs: + type: array + items: + $ref: '#/definitions/tesOutputFileLog' + description: |- + Information about all output files. Directory outputs are + flattened into separate items. + system_logs: + type: array + items: + type: string + description: |- + System logs are any logs the system decides are relevant, + which are not tied directly to an Executor process. + Content is implementation specific: format, size, etc. + + System logs may be collected here to provide convenient access. + + For example, the system may include the name of the host + where the task is executing, an error message that caused + a SYSTEM_ERROR state (e.g. disk is full), etc. + + System logs are only included in the FULL task view. + description: TaskLog describes logging information related to a Task. + required: + - logs + - outputs + readOnly: true + tesValidateTaskResponse: + title: tesValidateTaskResponse + type: object + x-tags: + - TaskService + properties: + success: + type: string + enum: + - 'true' + - 'false' +host: example.com +securityDefinitions: {} From c80043a4f579c08f102b8047f724810a9c3ef2e9 Mon Sep 17 00:00:00 2001 From: Susheel Varma Date: Thu, 2 Jul 2020 03:19:15 +0100 Subject: [PATCH 2/7] Update to openapi 3.0.0 --- api/task_execution.openapi.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/task_execution.openapi.yaml b/api/task_execution.openapi.yaml index 9e47f56..634dc66 100644 --- a/api/task_execution.openapi.yaml +++ b/api/task_execution.openapi.yaml @@ -1,4 +1,4 @@ -swagger: '2.0' +openapi: 3.0.0 info: title: Task Execution Service version: 0.4.0 @@ -143,6 +143,7 @@ paths: description: OK schema: $ref: '#/definitions/tesCancelTaskResponse' + description: Delete a Task parameters: - type: string name: id From b3af48a97ad1d91ad671ee89819d68e010664720 Mon Sep 17 00:00:00 2001 From: Susheel Varma Date: Thu, 2 Jul 2020 10:29:39 +0100 Subject: [PATCH 3/7] Fixed Swagger 2.0 to 3.0 migration issues --- api/task_execution.openapi.yaml | 1049 ++++++++++++++++--------------- 1 file changed, 535 insertions(+), 514 deletions(-) diff --git a/api/task_execution.openapi.yaml b/api/task_execution.openapi.yaml index 634dc66..357fd07 100644 --- a/api/task_execution.openapi.yaml +++ b/api/task_execution.openapi.yaml @@ -1,26 +1,28 @@ openapi: 3.0.0 info: title: Task Execution Service - version: 0.4.0 -schemes: - - http -consumes: - - application/json -produces: - - application/json -basePath: /tes/v1 + version: "0.4.0-fds-patch" + contact: + name: Susheel Varma + email: susheel.varma@hdruk.ac.uk + license: + name: MIT + description: GA4GH Task Execution Service (with FDS extenstions) +tags: + - Task Service + - name: TaskService paths: /tasks: get: - summary: |- - List tasks. - TaskView is requested as such: "v1/tasks?view=BASIC" + summary: List Tasks operationId: ListTasks responses: '200': description: '' - schema: - $ref: '#/definitions/tesListTasksResponse' + content: + application/json: + schema: + $ref: '#/components/schemas/tesListTasksResponse' parameters: - name: name_prefix description: |- @@ -28,15 +30,17 @@ paths: If unspecified, no task name filtering is done. in: query required: false - type: string + schema: + type: string - name: page_size description: |- OPTIONAL. Number of tasks to return in one page. Must be less than 2048. Defaults to 256. in: query required: false - type: integer - format: int64 + schema: + type: integer + format: int64 - name: page_token description: |- OPTIONAL. Page token is used to retrieve the next page of results. @@ -44,7 +48,8 @@ paths: See ListTasksResponse.next_page_token in: query required: false - type: string + schema: + type: string - name: view description: |- OPTIONAL. Affects the fields included in the returned Task messages. @@ -61,12 +66,14 @@ paths: - FULL: Task message includes all fields. in: query required: false - type: string - enum: - - MINIMAL - - BASIC - - FULL - default: MINIMAL + schema: + type: string + enum: + - MINIMAL + - BASIC + - FULL + default: MINIMAL + description: 'List tasks.TaskView is requested as such: "v1/tasks?view=BASIC"' tags: - TaskService post: @@ -75,42 +82,50 @@ paths: responses: '200': description: '' - schema: - $ref: '#/definitions/tesCreateTaskResponse' - parameters: - - name: body - in: body - required: true - schema: - $ref: '#/definitions/tesTask' + content: + application/json: + schema: + $ref: '#/components/schemas/tesCreateTaskResponse' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/tesTask' + required: true tags: - TaskService + description: Create a new task. /tasks/service-info: get: - summary: |- - GetServiceInfo provides information about the service, - such as storage details, resource availability, and - other documentation. + summary: Service Info operationId: GetServiceInfo responses: '200': description: '' - schema: - $ref: '#/definitions/tesServiceInfo' + content: + application/json: + schema: + $ref: '#/components/schemas/tesServiceInfo' tags: - TaskService + description: 'GetServiceInfo provides information about the service,such as storage details, resource availability, and other documentation.' '/tasks/{id}': get: - summary: |- - Get a task. - TaskView is requested as such: "v1/tasks/{id}?view=FULL" + summary: Get a task. operationId: GetTask responses: '200': description: '' - schema: - $ref: '#/definitions/tesTask' + content: + application/json: + schema: + $ref: '#/components/schemas/tesTask' parameters: + - name: id + in: path + required: true + schema: + type: string - name: view description: |- OPTIONAL. Affects the fields included in the returned Task messages. @@ -127,507 +142,513 @@ paths: - FULL: Task message includes all fields. in: query required: false - type: string - enum: - - MINIMAL - - BASIC - - FULL - default: MINIMAL + schema: + type: string + enum: + - MINIMAL + - BASIC + - FULL + default: MINIMAL tags: - TaskService - delete: - summary: '' - operationId: DeleteTask + description: 'Get a task. TaskView is requested as such: "v1/tasks/{id}?view=FULL"' + '/tasks/{id}/cancel': + post: + summary: Cancel a task. + operationId: CancelTask responses: '200': - description: OK - schema: - $ref: '#/definitions/tesCancelTaskResponse' - description: Delete a Task + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/tesCancelTaskResponse' + tags: + - TaskService + description: Cancel a task. parameters: - - type: string + - schema: + type: string name: id in: path required: true description: Task ID /tasks/validate: post: - summary: Validate Task - tags: - - TaskService + summary: Validate a Task + tags: [] responses: '200': description: OK - schema: - $ref: '#/definitions/tesValidateTaskResponse' + content: + application/json: + schema: + $ref: '#/components/schemas/tesValidateTaskResponse' '400': description: Bad Request - schema: - $ref: '#/definitions/tesCancelTaskResponse' - operationId: post-tasks-validate - parameters: - - in: body - name: body - schema: - $ref: '#/definitions/tesTask' -definitions: - tesCancelTaskResponse: - type: object - description: CancelTaskResponse describes a response from the CancelTask endpoint. - readOnly: true - tesCreateTaskResponse: - type: object - properties: - id: - type: string - description: Task identifier assigned by the server. - description: CreateTaskResponse describes a response from the CreateTask endpoint. - readOnly: true - required: - - id - tesExecutor: - type: object - properties: - image: - type: string - description: |- - Name of the container image, for example: - ubuntu - quay.io/aptible/ubuntu - gcr.io/my-org/my-image - etc... - command: - type: array - items: - type: string - description: |- - A sequence of program arguments to execute, where the first argument - is the program to execute (i.e. argv). - workdir: - type: string - description: |- - The working directory that the command will be executed in. - Defaults to the directory set by the container image. - stdin: - type: string - description: |- - Path inside the container to a file which will be piped - to the executor's stdin. Must be an absolute path. - stdout: - type: string - description: |- - Path inside the container to a file where the executor's - stdout will be written to. Must be an absolute path. - stderr: - type: string - description: |- - Path inside the container to a file where the executor's - stderr will be written to. Must be an absolute path. - env: - type: object - additionalProperties: - type: string - description: Enviromental variables to set within the container. - description: 'Executor describes a command to be executed, and its environment.' - required: - - image - - command - tesExecutorLog: - type: object - properties: - start_time: - type: string - description: 'Time the executor started, in RFC 3339 format.' - end_time: - type: string - description: 'Time the executor ended, in RFC 3339 format.' - stdout: - type: string - description: |- - Stdout content. + content: + application/json: + schema: + $ref: '#/components/schemas/tesCancelTaskResponse' + operationId: ValidateTask + requestBody: + content: + application/json: + schema: + type: object + properties: {} + application/xml: + schema: + $ref: '#/components/schemas/tesTask' + description: Validate a Task +servers: + - url: /ga4gh/tes/v1 +components: + schemas: + tesCancelTaskResponse: + type: object + description: CancelTaskResponse describes a response from the CancelTask endpoint. + readOnly: true + tesCreateTaskResponse: + type: object + properties: + id: + type: string + description: Task identifier assigned by the server. + description: CreateTaskResponse describes a response from the CreateTask endpoint. + readOnly: true + required: + - id + tesExecutor: + type: object + properties: + image: + type: string + description: |- + Name of the container image, for example: + ubuntu + quay.io/aptible/ubuntu + gcr.io/my-org/my-image + etc... + command: + type: array + items: + type: string + description: |- + A sequence of program arguments to execute, where the first argument + is the program to execute (i.e. argv). + workdir: + type: string + description: |- + The working directory that the command will be executed in. + Defaults to the directory set by the container image. + stdin: + type: string + description: |- + Path inside the container to a file which will be piped + to the executor's stdin. Must be an absolute path. + stdout: + type: string + description: |- + Path inside the container to a file where the executor's + stdout will be written to. Must be an absolute path. + stderr: + type: string + description: |- + Path inside the container to a file where the executor's + stderr will be written to. Must be an absolute path. + env: + type: object + additionalProperties: + type: string + description: Enviromental variables to set within the container. + description: 'Executor describes a command to be executed, and its environment.' + required: + - image + - command + tesExecutorLog: + type: object + properties: + start_time: + type: string + description: 'Time the executor started, in RFC 3339 format.' + end_time: + type: string + description: 'Time the executor ended, in RFC 3339 format.' + stdout: + type: string + description: |- + Stdout content. - This is meant for convenience. No guarantees are made about the content. - Implementations may chose different approaches: only the head, only the tail, - a URL reference only, etc. + This is meant for convenience. No guarantees are made about the content. + Implementations may chose different approaches: only the head, only the tail, + a URL reference only, etc. - In order to capture the full stdout users should set Executor.stdout - to a container file path, and use Task.outputs to upload that file - to permanent storage. - stderr: - type: string - description: |- - Stderr content. + In order to capture the full stdout users should set Executor.stdout + to a container file path, and use Task.outputs to upload that file + to permanent storage. + stderr: + type: string + description: |- + Stderr content. - This is meant for convenience. No guarantees are made about the content. - Implementations may chose different approaches: only the head, only the tail, - a URL reference only, etc. + This is meant for convenience. No guarantees are made about the content. + Implementations may chose different approaches: only the head, only the tail, + a URL reference only, etc. - In order to capture the full stderr users should set Executor.stderr - to a container file path, and use Task.outputs to upload that file - to permanent storage. - exit_code: - type: integer - format: int32 - description: Exit code. - description: ExecutorLog describes logging information related to an Executor. - required: - - exit_code - readOnly: true - tesFileType: - type: string - enum: - - FILE - - DIRECTORY - default: FILE - tesInput: - description: Input describes Task input files. - anyOf: - - properties: - name: - type: string - description: - type: string - url: - type: string - description: |- - REQUIRED, unless "content" is set. + In order to capture the full stderr users should set Executor.stderr + to a container file path, and use Task.outputs to upload that file + to permanent storage. + exit_code: + type: integer + format: int32 + description: Exit code. + description: ExecutorLog describes logging information related to an Executor. + required: + - exit_code + readOnly: true + tesFileType: + type: string + enum: + - FILE + - DIRECTORY + default: FILE + tesInput: + type: object + properties: + name: + type: string + description: + type: string + url: + type: string + description: |- + REQUIRED, unless "content" is set. - URL in long term storage, for example: - s3://my-object-store/file1 - gs://my-bucket/file2 - file:///path/to/my/file - /path/to/my/file - etc... - path: - type: string - description: |- - Path of the file inside the container. - Must be an absolute path. - type: - $ref: '#/definitions/tesFileType' - content: - type: string - description: |- - File content literal. - Implementations should support a minimum of 128 KiB in this field and may define its own maximum. - UTF-8 encoded + URL in long term storage, for example: + s3://my-object-store/file1 + gs://my-bucket/file2 + file:///path/to/my/file + /path/to/my/file + etc... + path: + type: string + description: |- + Path of the file inside the container. + Must be an absolute path. + type: + $ref: '#/components/schemas/tesFileType' + content: + type: string + description: |- + File content literal. + Implementations should support a minimum of 128 KiB in this field and may define its own maximum. + UTF-8 encoded - If content is not empty, "url" must be ignored. - required: - - path - - type - - properties: - selection: + If content is not empty, "url" must be ignored. + description: Input describes Task input files. + required: + - type + - path + tesListTasksResponse: + type: object + properties: + tasks: + type: array + items: + $ref: '#/components/schemas/tesTask' + description: List of tasks. + next_page_token: + type: string + description: |- + Token used to return the next page of results. + See TaskListRequest.next_page_token + description: ListTasksResponse describes a response from the ListTasks endpoint. + required: + - tasks + readOnly: true + tesOutput: + type: object + properties: + name: + type: string + description: + type: string + url: + type: string + description: |- + URL in long term storage, for example: + s3://my-object-store/file1 + gs://my-bucket/file2 + file:///path/to/my/file + /path/to/my/file + etc... + path: + type: string + description: |- + Path of the file inside the container. + Must be an absolute path. + type: + $ref: '#/components/schemas/tesFileType' + description: Output describes Task output files. + required: + - url + - path + - type + tesOutputFileLog: + type: object + properties: + url: + type: string + description: 'URL of the file in storage, e.g. s3://bucket/file.txt' + path: + type: string + description: Path of the file inside the container. Must be an absolute path. + size_bytes: + type: string + format: int64 + description: Size of the file in bytes. + description: |- + OutputFileLog describes a single output file. This describes + file details after the task has completed successfully, + for logging purposes. + readOnly: true + required: + - url + - path + - size_bytes + tesResources: + type: object + properties: + cpu_cores: + type: integer + format: int64 + description: Requested number of CPUs + preemptible: + type: boolean + format: boolean + description: Is the task allowed to run on preemptible compute instances (e.g. AWS Spot)? + ram_gb: + type: number + format: double + description: Requested RAM required in gigabytes (GB) + disk_gb: + type: number + format: double + description: Requested disk size in gigabytes (GB) + zones: + type: array + items: type: string - type: object - tesListTasksResponse: - type: object - properties: - tasks: - type: array - items: - $ref: '#/definitions/tesTask' - description: List of tasks. - next_page_token: - type: string - description: |- - Token used to return the next page of results. - See TaskListRequest.next_page_token - description: ListTasksResponse describes a response from the ListTasks endpoint. - required: - - tasks - readOnly: true - tesOutput: - type: object - properties: - name: - type: string - description: - type: string - url: - type: string - description: |- - URL in long term storage, for example: - s3://my-object-store/file1 - gs://my-bucket/file2 - file:///path/to/my/file - /path/to/my/file - etc... - path: - type: string - description: |- - Path of the file inside the container. - Must be an absolute path. - type: - allOf: - - $ref: '#/definitions/tesFileType' - description: 'Type of the file, FILE or DIRECTORY' - description: Output describes Task output files. - required: - - url - - path - - type - tesOutputFileLog: - type: object - properties: - url: - type: string - description: 'URL of the file in storage, e.g. s3://bucket/file.txt' - path: - type: string - description: Path of the file inside the container. Must be an absolute path. - size_bytes: - type: string - format: int64 - description: Size of the file in bytes. - description: |- - OutputFileLog describes a single output file. This describes - file details after the task has completed successfully, - for logging purposes. - readOnly: true - required: - - url - - path - - size_bytes - tesResources: - type: object - properties: - cpu_cores: - type: integer - format: int64 - description: Requested number of CPUs - preemptible: - type: boolean - format: boolean - description: Is the task allowed to run on preemptible compute instances (e.g. AWS Spot)? - ram_gb: - type: number - format: double - description: Requested RAM required in gigabytes (GB) - disk_gb: - type: number - format: double - description: Requested disk size in gigabytes (GB) - zones: - type: array - items: - type: string - description: Request that the task be run in these compute zones. - description: Resources describes the resources requested by a task. - tesServiceInfo: - type: object - description: |- - ServiceInfo describes information about the service, - such as storage details, resource availability, - and other documentation. - readOnly: true - properties: - name: - type: string - description: 'Returns the name of the service, e.g. "ohsu-compbio-funnel".' - doc: - type: string - description: 'Returns a documentation string, e.g. "Hey, we''re OHSU Comp. Bio!".' - storage: - type: array - description: |- - Lists some, but not necessarily all, storage locations supported by the service. + description: Request that the task be run in these compute zones. + description: Resources describes the resources requested by a task. + tesServiceInfo: + type: object + description: |- + ServiceInfo describes information about the service, + such as storage details, resource availability, + and other documentation. + readOnly: true + properties: + name: + type: string + description: 'Returns the name of the service, e.g. "ohsu-compbio-funnel".' + doc: + type: string + description: 'Returns a documentation string, e.g. "Hey, we''re OHSU Comp. Bio!".' + storage: + type: array + description: |- + Lists some, but not necessarily all, storage locations supported by the service. - Must be in a valid URL format. - e.g. - file:///path/to/local/funnel-storage - s3://ohsu-compbio-funnel/storage - etc. - items: - type: string - registries: - type: array - description: Approved Registries - items: - type: object - properties: - registry_id: - type: string - registry_name: - type: string - registry_host: - type: string - registry url: - type: string - tesState: - type: string - enum: - - UNKNOWN - - QUEUED - - INITIALIZING - - RUNNING - - PAUSED - - COMPLETE - - EXECUTOR_ERROR - - SYSTEM_ERROR - - CANCELED - default: UNKNOWN - description: |- - Task states. + Must be in a valid URL format. + e.g. + file:///path/to/local/funnel-storage + s3://ohsu-compbio-funnel/storage + etc. + items: + type: string + registries: + type: + - string + - array + items: + type: object + properties: + registry_id: + type: string + registry_name: + type: string + registry_host: + type: string + registry_url: + type: string + tesState: + type: string + enum: + - UNKNOWN + - QUEUED + - INITIALIZING + - RUNNING + - PAUSED + - COMPLETE + - EXECUTOR_ERROR + - SYSTEM_ERROR + - CANCELED + default: UNKNOWN + description: |- + Task states. - - UNKNOWN: The state of the task is unknown. + - UNKNOWN: The state of the task is unknown. - This provides a safe default for messages where this field is missing, - for example, so that a missing field does not accidentally imply that - the state is QUEUED. - - QUEUED: The task is queued. - - INITIALIZING: The task has been assigned to a worker and is currently preparing to run. - For example, the worker may be turning on, downloading input files, etc. - - RUNNING: The task is running. Input files are downloaded and the first Executor - has been started. - - PAUSED: The task is paused. + This provides a safe default for messages where this field is missing, + for example, so that a missing field does not accidentally imply that + the state is QUEUED. + - QUEUED: The task is queued. + - INITIALIZING: The task has been assigned to a worker and is currently preparing to run. + For example, the worker may be turning on, downloading input files, etc. + - RUNNING: The task is running. Input files are downloaded and the first Executor + has been started. + - PAUSED: The task is paused. - An implementation may have the ability to pause a task, but this is not required. - - COMPLETE: The task has completed running. Executors have exited without error - and output files have been successfully uploaded. - - EXECUTOR_ERROR: The task encountered an error in one of the Executor processes. Generally, - this means that an Executor exited with a non-zero exit code. - - SYSTEM_ERROR: The task was stopped due to a system error, but not from an Executor, - for example an upload failed due to network issues, the worker's ran out - of disk space, etc. - - CANCELED: The task was canceled by the user. - readOnly: true - tesTask: - type: object - properties: - id: - type: string - description: Task identifier assigned by the server. - readOnly: true - state: - allOf: - - $ref: '#/definitions/tesState' - readOnly: true - name: - type: string - description: - type: string - inputs: - type: array - items: - $ref: '#/definitions/tesInput' - description: |- - Input files. - Inputs will be downloaded and mounted into the executor container. - outputs: - type: array - items: - $ref: '#/definitions/tesOutput' - description: |- - Output files. - Outputs will be uploaded from the executor container to long-term storage. - resources: - allOf: - - $ref: '#/definitions/tesResources' - description: Request that the task be run with these resources. - executors: - type: array - items: - $ref: '#/definitions/tesExecutor' - description: |- - A list of executors to be run, sequentially. Execution stops - on the first error. - volumes: - type: array - items: - type: string - description: |- - Volumes are directories which may be used to share data between - Executors. Volumes are initialized as empty directories by the - system when the task starts and are mounted at the same path - in each Executor. + An implementation may have the ability to pause a task, but this is not required. + - COMPLETE: The task has completed running. Executors have exited without error + and output files have been successfully uploaded. + - EXECUTOR_ERROR: The task encountered an error in one of the Executor processes. Generally, + this means that an Executor exited with a non-zero exit code. + - SYSTEM_ERROR: The task was stopped due to a system error, but not from an Executor, + for example an upload failed due to network issues, the worker's ran out + of disk space, etc. + - CANCELED: The task was canceled by the user. + readOnly: true + tesTask: + type: object + properties: + id: + type: string + description: Task identifier assigned by the server. + readOnly: true + state: + $ref: '#/components/schemas/tesState' + name: + type: string + description: + type: string + inputs: + type: array + items: + $ref: '#/components/schemas/tesInput' + description: |- + Input files. + Inputs will be downloaded and mounted into the executor container. + outputs: + type: array + items: + $ref: '#/components/schemas/tesOutput' + description: |- + Output files. + Outputs will be uploaded from the executor container to long-term storage. + resources: + $ref: '#/components/schemas/tesResources' + executors: + type: array + items: + $ref: '#/components/schemas/tesExecutor' + description: |- + A list of executors to be run, sequentially. Execution stops + on the first error. + volumes: + type: array + items: + type: string + description: |- + Volumes are directories which may be used to share data between + Executors. Volumes are initialized as empty directories by the + system when the task starts and are mounted at the same path + in each Executor. - For example, given a volume defined at "/vol/A", - executor 1 may write a file to "/vol/A/exec1.out.txt", then - executor 2 may read from that file. + For example, given a volume defined at "/vol/A", + executor 1 may write a file to "/vol/A/exec1.out.txt", then + executor 2 may read from that file. - (Essentially, this translates to a `docker run -v` flag where - the container path is the same for each executor). - tags: - type: object - additionalProperties: - type: string - description: A key-value map of arbitrary tags. - logs: - type: array - items: - $ref: '#/definitions/tesTaskLog' - description: |- - Task logging information. - Normally, this will contain only one entry, but in the case where - a task fails and is retried, an entry will be appended to this list. - readOnly: true - creation_time: - type: string - description: |- - Date + time the task was created, in RFC 3339 format. - This is set by the system, not the client. - readOnly: true - description: Task describes an instance of a task. - required: - - executors - tesTaskLog: - type: object - properties: - logs: - type: array - items: - $ref: '#/definitions/tesExecutorLog' - description: Logs for each executor - metadata: - type: object - additionalProperties: - type: string - description: Arbitrary logging metadata included by the implementation. - start_time: - type: string - description: 'When the task started, in RFC 3339 format.' - end_time: - type: string - description: 'When the task ended, in RFC 3339 format.' - outputs: - type: array - items: - $ref: '#/definitions/tesOutputFileLog' - description: |- - Information about all output files. Directory outputs are - flattened into separate items. - system_logs: - type: array - items: - type: string - description: |- - System logs are any logs the system decides are relevant, - which are not tied directly to an Executor process. - Content is implementation specific: format, size, etc. + (Essentially, this translates to a `docker run -v` flag where + the container path is the same for each executor). + tags: + type: object + additionalProperties: + type: string + description: A key-value map of arbitrary tags. + logs: + type: array + items: + $ref: '#/components/schemas/tesTaskLog' + description: |- + Task logging information. + Normally, this will contain only one entry, but in the case where + a task fails and is retried, an entry will be appended to this list. + readOnly: true + creation_time: + type: string + description: |- + Date + time the task was created, in RFC 3339 format. + This is set by the system, not the client. + readOnly: true + description: Task describes an instance of a task. + required: + - executors + tesTaskLog: + type: object + properties: + logs: + type: array + items: + $ref: '#/components/schemas/tesExecutorLog' + description: Logs for each executor + metadata: + type: object + additionalProperties: + type: string + description: Arbitrary logging metadata included by the implementation. + start_time: + type: string + description: 'When the task started, in RFC 3339 format.' + end_time: + type: string + description: 'When the task ended, in RFC 3339 format.' + outputs: + type: array + items: + $ref: '#/components/schemas/tesOutputFileLog' + description: |- + Information about all output files. Directory outputs are + flattened into separate items. + system_logs: + type: array + items: + type: string + description: |- + System logs are any logs the system decides are relevant, + which are not tied directly to an Executor process. + Content is implementation specific: format, size, etc. - System logs may be collected here to provide convenient access. + System logs may be collected here to provide convenient access. - For example, the system may include the name of the host - where the task is executing, an error message that caused - a SYSTEM_ERROR state (e.g. disk is full), etc. + For example, the system may include the name of the host + where the task is executing, an error message that caused + a SYSTEM_ERROR state (e.g. disk is full), etc. - System logs are only included in the FULL task view. - description: TaskLog describes logging information related to a Task. - required: - - logs - - outputs - readOnly: true - tesValidateTaskResponse: - title: tesValidateTaskResponse - type: object - x-tags: - - TaskService - properties: - success: - type: string - enum: - - 'true' - - 'false' -host: example.com -securityDefinitions: {} + System logs are only included in the FULL task view. + description: TaskLog describes logging information related to a Task. + required: + - logs + - outputs + readOnly: true + tesValidateTaskResponse: + title: tesValidateTaskResponse + type: object + properties: + success: + type: string + enum: + - 'true' + - 'false' From 415e3ef9d3f39381112d987970bab25815ec6d26 Mon Sep 17 00:00:00 2001 From: Susheel Varma Date: Thu, 2 Jul 2020 10:51:15 +0100 Subject: [PATCH 4/7] Applying TaskService section to validate API --- api/task_execution.openapi.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/task_execution.openapi.yaml b/api/task_execution.openapi.yaml index 357fd07..ca746c6 100644 --- a/api/task_execution.openapi.yaml +++ b/api/task_execution.openapi.yaml @@ -201,6 +201,8 @@ paths: schema: $ref: '#/components/schemas/tesTask' description: Validate a Task + tags: + - TaskService servers: - url: /ga4gh/tes/v1 components: From 7fb4606fe21ecf50ea17d58a66307aeabd2b781f Mon Sep 17 00:00:00 2001 From: "susheel.varma@hdruk.ac.uk" Date: Thu, 2 Jul 2020 10:54:58 +0100 Subject: [PATCH 5/7] Minor fixes --- api/task_service.openapi.yaml | 656 ++++++++++++++++++++++++++++++++++ 1 file changed, 656 insertions(+) create mode 100644 api/task_service.openapi.yaml diff --git a/api/task_service.openapi.yaml b/api/task_service.openapi.yaml new file mode 100644 index 0000000..cd75cf6 --- /dev/null +++ b/api/task_service.openapi.yaml @@ -0,0 +1,656 @@ +openapi: 3.0.0 +info: + title: Task Execution Service + version: "0.4.0-fds-patch" + contact: + name: Susheel Varma + email: susheel.varma@hdruk.ac.uk + license: + name: MIT + description: GA4GH Task Execution Service (with FDS extenstions) +tags: + - Task Service + - name: TaskService +paths: + /tasks: + get: + summary: List Tasks + operationId: ListTasks + responses: + '200': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/tesListTasksResponse' + parameters: + - name: name_prefix + description: |- + OPTIONAL. Filter the list to include tasks where the name matches this prefix. + If unspecified, no task name filtering is done. + in: query + required: false + schema: + type: string + - name: page_size + description: |- + OPTIONAL. Number of tasks to return in one page. + Must be less than 2048. Defaults to 256. + in: query + required: false + schema: + type: integer + format: int64 + - name: page_token + description: |- + OPTIONAL. Page token is used to retrieve the next page of results. + If unspecified, returns the first page of results. + See ListTasksResponse.next_page_token + in: query + required: false + schema: + type: string + - name: view + description: |- + OPTIONAL. Affects the fields included in the returned Task messages. + See TaskView below. + + - MINIMAL: Task message will include ONLY the fields: + Task.Id + Task.State + - BASIC: Task message will include all fields EXCEPT: + Task.ExecutorLog.stdout + Task.ExecutorLog.stderr + Input.content + TaskLog.system_logs + - FULL: Task message includes all fields. + in: query + required: false + schema: + type: string + enum: + - MINIMAL + - BASIC + - FULL + default: MINIMAL + description: 'List tasks.TaskView is requested as such: "v1/tasks?view=BASIC"' + tags: + - TaskService + post: + summary: Create a new task. + operationId: CreateTask + responses: + '200': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/tesCreateTaskResponse' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/tesTask' + required: true + tags: + - TaskService + description: Create a new task. + /tasks/service-info: + get: + summary: Service Info + operationId: GetServiceInfo + responses: + '200': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/tesServiceInfo' + tags: + - TaskService + description: 'GetServiceInfo provides information about the service,such as storage details, resource availability, and other documentation.' + '/tasks/{id}': + get: + summary: Get a task. + operationId: GetTask + responses: + '200': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/tesTask' + parameters: + - name: id + in: path + required: true + schema: + type: string + - name: view + description: |- + OPTIONAL. Affects the fields included in the returned Task messages. + See TaskView below. + + - MINIMAL: Task message will include ONLY the fields: + Task.Id + Task.State + - BASIC: Task message will include all fields EXCEPT: + Task.ExecutorLog.stdout + Task.ExecutorLog.stderr + Input.content + TaskLog.system_logs + - FULL: Task message includes all fields. + in: query + required: false + schema: + type: string + enum: + - MINIMAL + - BASIC + - FULL + default: MINIMAL + tags: + - TaskService + description: 'Get a task. TaskView is requested as such: "v1/tasks/{id}?view=FULL"' + '/tasks/{id}/cancel': + post: + summary: Cancel a task. + operationId: CancelTask + responses: + '200': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/tesCancelTaskResponse' + tags: + - TaskService + description: Cancel a task. + parameters: + - schema: + type: string + name: id + in: path + required: true + description: Task ID + /tasks/validate: + post: + summary: Validate a Task + tags: [] + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/tesValidateTaskResponse' + '400': + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/tesCancelTaskResponse' + operationId: ValidateTask + requestBody: + content: + application/json: + schema: + type: object + properties: {} + application/xml: + schema: + $ref: '#/components/schemas/tesTask' + description: Validate a Task + tags: + - TaskService +servers: + - url: /ga4gh/tes/v1 +components: + schemas: + tesCancelTaskResponse: + type: object + description: CancelTaskResponse describes a response from the CancelTask endpoint. + readOnly: true + tesCreateTaskResponse: + type: object + properties: + id: + type: string + description: Task identifier assigned by the server. + description: CreateTaskResponse describes a response from the CreateTask endpoint. + readOnly: true + required: + - id + tesExecutor: + type: object + properties: + image: + type: string + description: |- + Name of the container image, for example: + ubuntu + quay.io/aptible/ubuntu + gcr.io/my-org/my-image + etc... + command: + type: array + items: + type: string + description: |- + A sequence of program arguments to execute, where the first argument + is the program to execute (i.e. argv). + workdir: + type: string + description: |- + The working directory that the command will be executed in. + Defaults to the directory set by the container image. + stdin: + type: string + description: |- + Path inside the container to a file which will be piped + to the executor's stdin. Must be an absolute path. + stdout: + type: string + description: |- + Path inside the container to a file where the executor's + stdout will be written to. Must be an absolute path. + stderr: + type: string + description: |- + Path inside the container to a file where the executor's + stderr will be written to. Must be an absolute path. + env: + type: object + additionalProperties: + type: string + description: Enviromental variables to set within the container. + description: 'Executor describes a command to be executed, and its environment.' + required: + - image + - command + tesExecutorLog: + type: object + properties: + start_time: + type: string + description: 'Time the executor started, in RFC 3339 format.' + end_time: + type: string + description: 'Time the executor ended, in RFC 3339 format.' + stdout: + type: string + description: |- + Stdout content. + + This is meant for convenience. No guarantees are made about the content. + Implementations may chose different approaches: only the head, only the tail, + a URL reference only, etc. + + In order to capture the full stdout users should set Executor.stdout + to a container file path, and use Task.outputs to upload that file + to permanent storage. + stderr: + type: string + description: |- + Stderr content. + + This is meant for convenience. No guarantees are made about the content. + Implementations may chose different approaches: only the head, only the tail, + a URL reference only, etc. + + In order to capture the full stderr users should set Executor.stderr + to a container file path, and use Task.outputs to upload that file + to permanent storage. + exit_code: + type: integer + format: int32 + description: Exit code. + description: ExecutorLog describes logging information related to an Executor. + required: + - exit_code + readOnly: true + tesFileType: + type: string + enum: + - FILE + - DIRECTORY + default: FILE + tesInput: + type: object + properties: + name: + type: string + description: + type: string + url: + type: string + description: |- + REQUIRED, unless "content" is set. + + URL in long term storage, for example: + s3://my-object-store/file1 + gs://my-bucket/file2 + file:///path/to/my/file + /path/to/my/file + etc... + path: + type: string + description: |- + Path of the file inside the container. + Must be an absolute path. + type: + $ref: '#/components/schemas/tesFileType' + content: + type: string + description: |- + File content literal. + Implementations should support a minimum of 128 KiB in this field and may define its own maximum. + UTF-8 encoded + + If content is not empty, "url" must be ignored. + description: Input describes Task input files. + required: + - type + - path + tesListTasksResponse: + type: object + properties: + tasks: + type: array + items: + $ref: '#/components/schemas/tesTask' + description: List of tasks. + next_page_token: + type: string + description: |- + Token used to return the next page of results. + See TaskListRequest.next_page_token + description: ListTasksResponse describes a response from the ListTasks endpoint. + required: + - tasks + readOnly: true + tesOutput: + type: object + properties: + name: + type: string + description: + type: string + url: + type: string + description: |- + URL in long term storage, for example: + s3://my-object-store/file1 + gs://my-bucket/file2 + file:///path/to/my/file + /path/to/my/file + etc... + path: + type: string + description: |- + Path of the file inside the container. + Must be an absolute path. + type: + $ref: '#/components/schemas/tesFileType' + description: Output describes Task output files. + required: + - url + - path + - type + tesOutputFileLog: + type: object + properties: + url: + type: string + description: 'URL of the file in storage, e.g. s3://bucket/file.txt' + path: + type: string + description: Path of the file inside the container. Must be an absolute path. + size_bytes: + type: string + format: int64 + description: Size of the file in bytes. + description: |- + OutputFileLog describes a single output file. This describes + file details after the task has completed successfully, + for logging purposes. + readOnly: true + required: + - url + - path + - size_bytes + tesResources: + type: object + properties: + cpu_cores: + type: integer + format: int64 + description: Requested number of CPUs + preemptible: + type: boolean + format: boolean + description: Is the task allowed to run on preemptible compute instances (e.g. AWS Spot)? + ram_gb: + type: number + format: double + description: Requested RAM required in gigabytes (GB) + disk_gb: + type: number + format: double + description: Requested disk size in gigabytes (GB) + zones: + type: array + items: + type: string + description: Request that the task be run in these compute zones. + description: Resources describes the resources requested by a task. + tesServiceInfo: + type: object + description: |- + ServiceInfo describes information about the service, + such as storage details, resource availability, + and other documentation. + readOnly: true + properties: + name: + type: string + description: 'Returns the name of the service, e.g. "ohsu-compbio-funnel".' + doc: + type: string + description: 'Returns a documentation string, e.g. "Hey, we''re OHSU Comp. Bio!".' + storage: + type: array + description: |- + Lists some, but not necessarily all, storage locations supported by the service. + + Must be in a valid URL format. + e.g. + file:///path/to/local/funnel-storage + s3://ohsu-compbio-funnel/storage + etc. + items: + type: string + registries: + type: + - string + - array + items: + type: object + properties: + registry_id: + type: string + registry_name: + type: string + registry_host: + type: string + registry_url: + type: string + tesState: + type: string + enum: + - UNKNOWN + - QUEUED + - INITIALIZING + - RUNNING + - PAUSED + - COMPLETE + - EXECUTOR_ERROR + - SYSTEM_ERROR + - CANCELED + default: UNKNOWN + description: |- + Task states. + + - UNKNOWN: The state of the task is unknown. + + This provides a safe default for messages where this field is missing, + for example, so that a missing field does not accidentally imply that + the state is QUEUED. + - QUEUED: The task is queued. + - INITIALIZING: The task has been assigned to a worker and is currently preparing to run. + For example, the worker may be turning on, downloading input files, etc. + - RUNNING: The task is running. Input files are downloaded and the first Executor + has been started. + - PAUSED: The task is paused. + + An implementation may have the ability to pause a task, but this is not required. + - COMPLETE: The task has completed running. Executors have exited without error + and output files have been successfully uploaded. + - EXECUTOR_ERROR: The task encountered an error in one of the Executor processes. Generally, + this means that an Executor exited with a non-zero exit code. + - SYSTEM_ERROR: The task was stopped due to a system error, but not from an Executor, + for example an upload failed due to network issues, the worker's ran out + of disk space, etc. + - CANCELED: The task was canceled by the user. + readOnly: true + tesTask: + type: object + properties: + id: + type: string + description: Task identifier assigned by the server. + readOnly: true + state: + $ref: '#/components/schemas/tesState' + name: + type: string + description: + type: string + inputs: + type: array + items: + $ref: '#/components/schemas/tesInput' + description: |- + Input files. + Inputs will be downloaded and mounted into the executor container. + outputs: + type: array + items: + $ref: '#/components/schemas/tesOutput' + description: |- + Output files. + Outputs will be uploaded from the executor container to long-term storage. + resources: + $ref: '#/components/schemas/tesResources' + executors: + type: array + items: + $ref: '#/components/schemas/tesExecutor' + description: |- + A list of executors to be run, sequentially. Execution stops + on the first error. + volumes: + type: array + items: + type: string + description: |- + Volumes are directories which may be used to share data between + Executors. Volumes are initialized as empty directories by the + system when the task starts and are mounted at the same path + in each Executor. + + For example, given a volume defined at "/vol/A", + executor 1 may write a file to "/vol/A/exec1.out.txt", then + executor 2 may read from that file. + + (Essentially, this translates to a `docker run -v` flag where + the container path is the same for each executor). + tags: + type: object + additionalProperties: + type: string + description: A key-value map of arbitrary tags. + logs: + type: array + items: + $ref: '#/components/schemas/tesTaskLog' + description: |- + Task logging information. + Normally, this will contain only one entry, but in the case where + a task fails and is retried, an entry will be appended to this list. + readOnly: true + creation_time: + type: string + description: |- + Date + time the task was created, in RFC 3339 format. + This is set by the system, not the client. + readOnly: true + description: Task describes an instance of a task. + required: + - executors + tesTaskLog: + type: object + properties: + logs: + type: array + items: + $ref: '#/components/schemas/tesExecutorLog' + description: Logs for each executor + metadata: + type: object + additionalProperties: + type: string + description: Arbitrary logging metadata included by the implementation. + start_time: + type: string + description: 'When the task started, in RFC 3339 format.' + end_time: + type: string + description: 'When the task ended, in RFC 3339 format.' + outputs: + type: array + items: + $ref: '#/components/schemas/tesOutputFileLog' + description: |- + Information about all output files. Directory outputs are + flattened into separate items. + system_logs: + type: array + items: + type: string + description: |- + System logs are any logs the system decides are relevant, + which are not tied directly to an Executor process. + Content is implementation specific: format, size, etc. + + System logs may be collected here to provide convenient access. + + For example, the system may include the name of the host + where the task is executing, an error message that caused + a SYSTEM_ERROR state (e.g. disk is full), etc. + + System logs are only included in the FULL task view. + description: TaskLog describes logging information related to a Task. + required: + - logs + - outputs + readOnly: true + tesValidateTaskResponse: + title: tesValidateTaskResponse + type: object + properties: + success: + type: string + enum: + - 'true' + - 'false' From 7ee43d47207db5cf8bf4d94bd93b38491ef9b604 Mon Sep 17 00:00:00 2001 From: Susheel Varma Date: Thu, 2 Jul 2020 11:03:58 +0100 Subject: [PATCH 6/7] Renaming file --- api/{task_service.openapi.yaml => task_execution.openapi.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename api/{task_service.openapi.yaml => task_execution.openapi.yaml} (100%) diff --git a/api/task_service.openapi.yaml b/api/task_execution.openapi.yaml similarity index 100% rename from api/task_service.openapi.yaml rename to api/task_execution.openapi.yaml From ffa445095b0e150fd9862a92f7f6a4a0f60693f9 Mon Sep 17 00:00:00 2001 From: Susheel Varma Date: Thu, 2 Jul 2020 11:11:45 +0100 Subject: [PATCH 7/7] Minor fixes --- api/task_execution.openapi.yaml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/api/task_execution.openapi.yaml b/api/task_execution.openapi.yaml index cd75cf6..eeec731 100644 --- a/api/task_execution.openapi.yaml +++ b/api/task_execution.openapi.yaml @@ -173,10 +173,11 @@ paths: in: path required: true description: Task ID - /tasks/validate: + '/tasks/validate': post: summary: Validate a Task - tags: [] + tags: + - TaskService responses: '200': description: OK @@ -201,8 +202,6 @@ paths: schema: $ref: '#/components/schemas/tesTask' description: Validate a Task - tags: - - TaskService servers: - url: /ga4gh/tes/v1 components: @@ -471,9 +470,7 @@ components: items: type: string registries: - type: - - string - - array + type: array items: type: object properties: