diff --git a/ExecutionBroker.tex b/ExecutionBroker.tex index 71733cb..ce013d7 100644 --- a/ExecutionBroker.tex +++ b/ExecutionBroker.tex @@ -17,6 +17,7 @@ \newcommand{\http} {HTTP} \newcommand{\rest} {REST} +\newcommand{\openapi} {OpenAPI} \newcommand{\datamodel} {data~model} \newcommand{\webservice} {web service} \newcommand{\webbrowser} {web browser} @@ -265,7 +266,7 @@ \subsection{Role within the VO Architecture} In response to the increasing size and complexity of the next generation of science \dataset{s} a number of \ivoa{} members are developing intergrated \scienceplatform{s} which bring together the \dataset{s} co-located with the compute resources needed to analyse -them.\footurl{https://data.lsst.cloud/}\footurl{https://rsp.lsst.io/index.html} +them.\footurl{https://data.lsst.cloud/}\footurl{https://rsp.lsst.io/} These \scienceplatform{s} make extensive use of the \ivoa{} data models and vocabularies to describe their \dataset{s}, and use the \ivoa{} data access @@ -283,10 +284,79 @@ \subsection{Role within the VO Architecture} resource allocation and execution scheduling across a heterogeneous federation of execution platforms. -\ivoa{} \executionbroker{} services may use the -\ivoa{} Single-Sign-On standard \citep{2017ivoa.spec.0524T} -for authentication (see section xx) %\ref{subsec:authentication} -and the +\subsection{Supplementary documents} +\label{supplementary-documents} + +\subsubsection{\openapi{} specification} +\label{openapi-specification} + +This document is designed to read in combination with the \openapi{} +\footurl{https://www.openapis.org/} \footurl{https://swagger.io/specification/} +specification published alongside this document. + +The \openapi{} specification defines the technical details of the +\executionbroker{} \webservice{} interface and the schema for the +\executionbroker{} \datamodel{}. +This document compliments the technical specification by using use cases +and specific examples to describe the intended service behaviour and +explain the reasoning behind some of the design choices. + +The two documents are intended to be read together. +The machine-readable \openapi{} spcification defining the \textit{what}, +and the human-readable text document describing the \textit{why}. + +The \openapi{} specification associated with this document is +published in the following files: +\begin{itemize} + \item \codeword{openapi.yaml} - The main service specification, including + the \executionbroker{} service API and the core data model. + \item \codeword{messages.yaml} - a supplimentary data model for INFO, WARN, + and DEBUG messages embedded in the service responses. + \item \codeword{utils.yaml} - a supplimentary data model for re-usable + components such as ISO date formats, min/max pairs, and name-value maps. +\end{itemize} + +This text document may include small examples of \openapi{} schema to +explain specific points, but these are for information only. Unless otherwise +specified, the \openapi{} specification itself should be assumed to be the +definitive source and this text document should be considered as secondary. + +\subsubsection{IVOA profiles} +\label{ivoa-profiles} + +This specification refers to the following documents +to cover the details of how an \executionbroker{} \webservice{} +should use the following protocols and data formats: +\begin{itemize} + \item The \ivoa{} \rest{} profile, describing a common pattern + for \ivoa{} \webservice{s} that implement + \rest{}\footurl{https://en.wikipedia.org/wiki/REST} + webservices. + \item The \ivoa{} \http{} profile, describing how \ivoa{} \webservice{s} + should use aspects of the \http{} protocol, including \http{} parameters, + message content, content negotiation, and \http{} return codes. + \item The \ivoa{} error messages profile, describing a common format + for \ivoa{} \webservice{s} to format error messages. + \item The \ivoa{} \json{} profile, describing how \ivoa{} \webservice{s} + should serialize message content using the + \json{}\footurl{https://www.json.org/json-en.html} data format. + \item The \ivoa{} \yaml{} profile, describing how \ivoa{} \webservice{s} + should serialize message content using the + \yaml{}\footurl{https://yaml.org/} data format. + \item The \ivoa{} \xml{} profile, describing how \ivoa{} \webservice{s} + should serialize message content using the + \xml{}\footurl{https://www.w3.org/XML/} data format. +\end{itemize} + +Unless otherwise specified, an \executionbroker{} \webservice{} implementation +should follow the guidelines outined in these documents. + +\subsubsection{IVOA Single-Sign-On} +\label{ivoa-sso} + +In addition, an \ivoa{} \executionbroker{} service may use parts of the +\ivoa{} Single-Sign-On standard\citep{2017ivoa.spec.0524T} +for authentication, and the \ivoa{} Credential Delegation Protocol \citep{2010ivoa.spec.0218P} for delegating credentials to other services. diff --git a/openapi/components/components.yaml b/openapi/components/components.yaml new file mode 100644 index 0000000..bd91e58 --- /dev/null +++ b/openapi/components/components.yaml @@ -0,0 +1,39 @@ +# +# +# +# Copyright (c) 2024, Manchester University (http://www.manchester.ac.uk/) +# +# This work is made available under the Creative Commons +# Attribution-ShareAlike 4.0 International licence. +# +# For details of the licence terms see: +# https://creativecommons.org/licenses/by-sa/4.0/ +# +# +# +# AIMetrics: [ +# { +# "name": "ChatGPT", +# "contribution": { +# "value": 5, +# "units": "%" +# } +# } +# ] +# + +openapi: 3.1.0 +info: + title: IVOA ExecutionBroker abstract components + version: "0.8" + summary: IVOA ExecutionBroker abstract components + description: > + Abstract components used in the IVOA ExecutionBroker data model + contact: + name: Zarquan + url: https://github.com/Zarquan + +components: + schemas: + + diff --git a/openapi/components/messages.yaml b/openapi/components/messages.yaml new file mode 100644 index 0000000..5336eaa --- /dev/null +++ b/openapi/components/messages.yaml @@ -0,0 +1,95 @@ +# +# +# +# Copyright (c) 2024, Manchester University (http://www.manchester.ac.uk/) +# +# This work is made available under the Creative Commons +# Attribution-ShareAlike 4.0 International licence. +# +# For details of the licence terms see: +# https://creativecommons.org/licenses/by-sa/4.0/ +# +# +# +# AIMetrics: [ +# { +# "name": "ChatGPT", +# "contribution": { +# "value": 5, +# "units": "%" +# } +# } +# ] +# + +openapi: 3.1.0 +info: + title: IVOA ExecutionBroker log messages + version: "1.0" + summary: IVOA ExecutionBroker log messages + description: > + Log messages used in IVOA ExecutionBroker responses + contact: + name: Zarquan + url: https://github.com/Zarquan + +components: + schemas: + + MessageItem: + description: > + A log message based on the Message Templates standard. + https://messagetemplates.org/ + type: object + title: MessageItem + xml: + name: message + properties: + type: + description: > + The message type identifier. + Typically a URL pointing to a human readable description of the message. + type: string + time: + description: > + The date and time of the message. + type: string + format: date-time + level: + description: > + The message level. + type: string + enum: + - "DEBUG" + - "WARN" + - "ERROR" + - "INFO" + - "OTHER" + template: + description: > + The message template. + type: string + values: + description: > + A map of name->value properties. + xml: + name: values + $ref: './utils.yaml#/components/schemas/NameValueMap' + message: + description: > + The resulting message. + type: string + + MessageList: + description: > + A list of log messages. + xml: + name: messages + type: array + title: MessageList + items: + $ref: '#/components/schemas/MessageItem' + xml: + name: message + + diff --git a/openapi/components/utils.yaml b/openapi/components/utils.yaml new file mode 100644 index 0000000..bc8cddc --- /dev/null +++ b/openapi/components/utils.yaml @@ -0,0 +1,210 @@ +# +# +# +# Copyright (c) 2024, Manchester University (http://www.manchester.ac.uk/) +# +# This work is made available under the Creative Commons +# Attribution-ShareAlike 4.0 International licence. +# +# For details of the licence terms see: +# https://creativecommons.org/licenses/by-sa/4.0/ +# +# +# +# AIMetrics: [ +# { +# "name": "ChatGPT", +# "contribution": { +# "value": 5, +# "units": "%" +# } +# } +# ] +# + +openapi: 3.1.0 +info: + title: IVOA ExecutionBroker utility classes + version: "0.8" + summary: IVOA ExecutionBroker utility classes + description: > + Utility classes used in the IVOA ExecutionBroker data model + contact: + name: Dave Morris + url: https://github.com/Zarquan + +components: + schemas: + + NameValueMap: + description: > + A map of name->value properties. + See https://swagger.io/docs/specification/data-models/dictionaries/ + type: object + title: NameValueMap + additionalProperties: + type: string + + ISO8601Duration: + description: > + A regular expression filter for an ISO 8601 duration. + type: string + title: ISO8601Duration + pattern: '^(-?)P(?=\d|T\d)(?:(\d+)Y)?(?:(\d+)M)?(?:(\d+)([DW]))?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+(?:\.\d+)?)S)?)?$' + + ISO8601Interval: + description: > + A regular expression filter for an ISO 8601 interval. + type: string + title: ISO8601Interval + pattern: '^(\d{4}(?:-\d{2}){2}T\d{2}(?::\d{2}){2}(?>[^\/]+))\/(P(?=.)(?:\d+Y)?(?:\d+M)?(?:\d+D)?(?:T(?=.)(?:\d+H)?(?:\d+M)?(?:\d+S)?)?)$' + + MinMaxInteger: + description: > + A minimum and maximum pair for a 32bit integer value. + type: object + title: MinMaxInteger + properties: + min: + type: integer + format: int32 + max: + type: integer + format: int32 + + MinMaxIntegerUnits: + description: > + A minimum and maximum pair with units for a 32bit integer value. + type: object + title: MinMaxIntegerUnits + allOf: + - $ref: 'MinMaxInteger' + - type: object + properties: + units: + type: string + + MinMaxLong: + description: > + A minimum and maximum pair for a 64bit integer value. + type: object + title: MinMaxLong + properties: + min: + type: integer + format: int64 + max: + type: integer + format: int64 + + MinMaxLongUnits: + description: > + A minimum and maximum pair with units for a 64bit integer value. + type: object + title: MinMaxLongUnits + allOf: + - $ref: 'MinMaxLong' + - type: object + properties: + units: + type: string + + MinMaxFloat: + description: > + A minimum and maximum pair with units for a floating point value. + type: object + title: MinMaxFloat + properties: + min: + type: number + format: float + max: + type: number + format: float + + MinMaxFloatUnits: + description: > + A minimum and maximum pair with units for a floating point value. + type: object + title: MinMaxFloatUnits + allOf: + - $ref: 'MinMaxFloat' + - type: object + properties: + units: + type: string + + MinMaxString: + description: > + A minimum and maximum pair for a string value. + type: object + title: MinMaxString + properties: + min: + type: string + max: + type: string + + MinMaxStringUnits: + description: > + A minimum and maximum pair with units for a string value. + type: object + title: MinMaxStringUnits + allOf: + - $ref: 'MinMaxFloat' + - type: object + properties: + units: + type: string + + ComputeUnitsEnum: + description: > + Units for the size of computing resources, multiples of 1000^n and 1024^n. + https://github.com/metio/storage-units.java/ + https://en.wikipedia.org/wiki/Byte#Multiple-byte_units + type: string + title: ComputeSizeUnitsEnum + enum: + - kB + - kiB + - MB + - MiB + - GB + - GiB + - TB + - TiB + - PB + - PiB + - EB + - EiB + - ZB + - ZiB + - YB + - YiB + + MinMaxComputeInteger: + description: > + A minimum and maximum 32bit integer with computing units. + type: object + title: MinMaxComputeInteger + allOf: + - $ref: 'MinMaxInteger' + - type: object + properties: + units: + $ref: 'ComputeUnitsEnum' + + MinMaxComputeLong: + description: > + A minimum and maximum 64bit integer with computing units. + type: object + title: MinMaxComputeLong + allOf: + - $ref: 'MinMaxLong' + - type: object + properties: + units: + $ref: 'ComputeUnitsEnum' + + + diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml new file mode 100644 index 0000000..8ccfb65 --- /dev/null +++ b/openapi/openapi.yaml @@ -0,0 +1,1309 @@ +# +# +# +# Copyright (c) 2024, Manchester University (http://www.manchester.ac.uk/) +# +# This work is made available under the Creative Commons +# Attribution-ShareAlike 4.0 International licence. +# +# For details of the licence terms see: +# https://creativecommons.org/licenses/by-sa/4.0/ +# +# +# +# AIMetrics: [ +# { +# "name": "ChatGPT", +# "contribution": { +# "value": 5, +# "units": "%" +# } +# } +# ] +# + +openapi: 3.1.0 +info: + title: IVOA ExecutionBroker + version: "1.0" + summary: IVOA ExecutionBroker interface + description: > + IVOA ExecutionBroker interface + contact: + name: Dave Morris + url: https://github.com/Zarquan + license: + name: Creative Commons Attribution Share Alike 4.0 International + identifier: CC-BY-SA-4.0 +servers: + - url: http://localhost:8080 + description: Localhost development +paths: + /offersets: + post: + operationId: OfferSetPost + requestBody: + content: + application/json: + schema: +# $ref: './components/components.yaml#/components/schemas/OfferSetRequest' +# $ref: '#/components/schemas/OfferSetRequest' + $ref: 'OfferSetRequest' + application/xml: + schema: +# $ref: './components/components.yaml#/components/schemas/OfferSetRequest' +# $ref: '#/components/schemas/OfferSetRequest' + $ref: 'OfferSetRequest' + application/yaml: + schema: +# $ref: './components/components.yaml#/components/schemas/OfferSetRequest' +# $ref: '#/components/schemas/OfferSetRequest' + $ref: 'OfferSetRequest' + required: true + # TODO Errors return error codes and ErrorResponse. + responses: + "303": + description: > + A redirect response to the '/offersets/{uuid}' endpoint. + headers: + Location: + required: true + schema: + type: string + format: url + "200": + description: > + A simple OK response with the same contant as the '/offersets/{uuid}' endpoint. + content: + application/json: + schema: +# $ref: './components/components.yaml#/components/schemas/OfferSetResponse' +# $ref: '#/components/schemas/OfferSetResponse' + $ref: 'OfferSetResponse' + application/xml: + schema: +# $ref: './components/components.yaml#/components/schemas/OfferSetResponse' +# $ref: '#/components/schemas/OfferSetResponse' + $ref: 'OfferSetResponse' + application/yaml: + schema: +# $ref: './components/components.yaml#/components/schemas/OfferSetResponse' +# $ref: '#/components/schemas/OfferSetResponse' + $ref: 'OfferSetResponse' + + /offersets/{uuid}: + get: + operationId: OfferSetGet + parameters: + - name: uuid + in: path + description: The offerset identifier + required: true + schema: + type: string + format: uuid + style: simple + # TODO Errors return error codes and ErrorResponse. + responses: + "200": + description: > + Required description ... + content: + application/json: + schema: +# $ref: './components/components.yaml#/components/schemas/OfferSetResponse' +# $ref: '#/components/schemas/OfferSetResponse' + $ref: 'OfferSetResponse' + application/xml: + schema: +# $ref: './components/components.yaml#/components/schemas/OfferSetResponse' +# $ref: '#/components/schemas/OfferSetResponse' + $ref: 'OfferSetResponse' + application/yaml: + schema: +# $ref: './components/components.yaml#/components/schemas/OfferSetResponse' +# $ref: '#/components/schemas/OfferSetResponse' + $ref: 'OfferSetResponse' + + /sessions/{uuid}: + get: + operationId: ExecutionSessionGet + parameters: + - name: uuid + in: path + description: The session identifier + required: true + schema: + type: string + format: uuid + style: simple + # TODO Errors return error codes and ErrorResponse. + responses: + "200": + description: > + Required description ... + content: + application/json: + schema: +# $ref: './components/components.yaml#/components/schemas/ExecutionSessionResponse' +# $ref: '#/components/schemas/ExecutionSessionResponse' + $ref: 'ExecutionSessionResponse' + application/xml: + schema: +# $ref: './components/components.yaml#/components/schemas/ExecutionSessionResponse' +# $ref: '#/components/schemas/ExecutionSessionResponse' + $ref: 'ExecutionSessionResponse' + application/yaml: + schema: +# $ref: './components/components.yaml#/components/schemas/ExecutionSessionResponse' +# $ref: '#/components/schemas/ExecutionSessionResponse' + $ref: 'ExecutionSessionResponse' + + post: + operationId: ExecutionSessionPost + parameters: + - name: uuid + in: path + description: > + The session UUID identifier + required: true + schema: + type: string + format: uuid + style: simple + requestBody: + required: true + content: + application/json: + schema: +# $ref: './components/components.yaml#/components/schemas/UpdateRequest' +# $ref: '#/components/schemas/UpdateRequest' + $ref: 'UpdateRequest' + application/xml: + schema: +# $ref: './components/components.yaml#/components/schemas/UpdateRequest' +# $ref: '#/components/schemas/UpdateRequest' + $ref: 'UpdateRequest' + application/yaml: + schema: +# $ref: './components/components.yaml#/components/schemas/UpdateRequest' +# $ref: '#/components/schemas/UpdateRequest' + $ref: 'UpdateRequest' + # TODO Errors return error codes and ErrorResponse. + responses: + "200": + description: > + Required description ... + content: + application/json: + schema: +# $ref: './components/components.yaml#/components/schemas/ExecutionSessionResponse' +# $ref: '#/components/schemas/ExecutionSessionResponse' + $ref: 'ExecutionSessionResponse' + application/xml: + schema: +# $ref: './components/components.yaml#/components/schemas/ExecutionSessionResponse' +# $ref: '#/components/schemas/ExecutionSessionResponse' + $ref: 'ExecutionSessionResponse' + application/yaml: + schema: +# $ref: './components/components.yaml#/components/schemas/ExecutionSessionResponse' +# $ref: '#/components/schemas/ExecutionSessionResponse' + $ref: 'ExecutionSessionResponse' + +components: + schemas: + +# components.yaml +# Tried moving this to a separate file, but had issues with generating the polymorphic classes. + + AbstractComponent: + description: > + Abstract base class for all our components. + This class provides a name, a UUID identifier, a href URL, + and a list of messages. + type: object + title: AbstractComponent + properties: + uuid: + description: > + A machine readable UUID, assigned by the server. + type: string + format: uuid + href: + description: > + The URL to access this component, assigned by the server. + type: string + name: + description: > + A human readable name, assigned by the client. + type: string + created: + description: > + The date and time that this component was created. + type: string + format: date-time + messages: + description: > + A list of messages about this component. + xml: + name: messages + type: array + items: + $ref: './components/messages.yaml#/components/schemas/MessageItem' + xml: + name: message + +# compute-resources.yaml +# Tried moving this to a separate file, but had issues with generating the polymorphic classes. + + ComputeResourceList: + description: > + A list of compute resources. + type: array + title: ComputeResourceList + xml: + name: compute + items: +# $ref: '#/components/schemas/AbstractComputeResource' + $ref: 'AbstractComputeResource' + + AbstractComputeResource: + description: > + Abstract base class for compute resources. + type: object + title: AbstractComputeResource + required: + - type + discriminator: + propertyName: type + mapping: +# "uri:simple-compute-resource-0.1": '#/components/schemas/SimpleComputeResource' +# "uri:simple-compute-resource-0.1": './components/compute-resources.yaml#/components/schemas/SimpleComputeResource' + "uri:simple-compute-resource-0.1": 'SimpleComputeResource' + allOf: +# - $ref: './components/components.yaml#/components/schemas/AbstractComponent' +# - $ref: '#/components/schemas/AbstractComponent' + - $ref: 'AbstractComponent' + - type: object + properties: + type: + type: string + xml: + attribute: true + + SimpleComputeResource: + description: > + A simple compute resource. + type: object + title: SimpleComputeResource + allOf: +# - $ref: '#/components/schemas/AbstractComputeResource' + - $ref: 'AbstractComputeResource' + - type: object + properties: + cores: + $ref: '#/components/schemas/ComputeResourceCores' + memory: + $ref: '#/components/schemas/ComputeResourceMemory' + volumes: + description: > + A list of filesystem volumes. + type: array + items: + $ref: '#/components/schemas/ComputeResourceVolume' + +# Replaced by an enum in utils.yaml +# ComputeResourceSizeUnit: +# description: > +# A size unit for compute resources. +# Specified in multiples of 1000^n and 1024^n, default units are GiB. +# https://github.com/metio/storage-units.java/ +# https://en.wikipedia.org/wiki/Byte#Multiple-byte_units +# type: string +# title: ComputeResourceSizeUnit +# pattern: '/^([0-9]+(\.[0-9]+){0,1})(kB|KiB|MB|MiB|GB|GiB|TB|TiB|PB|PiB){0,1}$/gm' + + ComputeResourceCores: + description: > + The number of CPU cores requested and offered. + type: object + title: ComputeResourceCores + properties: + requested: + description: > + The number of cpu cores requested by the user. + The min value specifies the minimum that required for the session. + The max value limits the extra the server is allowed to offer. + $ref: './components/utils.yaml#/components/schemas/MinMaxLong' + offered: + description: > + The number of cpu cores offered by the server. + The min value specifies the initial resources available for the session. + The max value specifies the maximun resources available for the session. + $ref: './components/utils.yaml#/components/schemas/MinMaxLong' + + ComputeResourceMemory: + description: > + The amount memory requested and offered. + type: object + title: ComputeResourceMemory + properties: + requested: + description: > + The amount memory requested by the user. + The min value specifies the minimum that required for the session. + The max value limits the extra the server is allowed to offer. + $ref: './components/utils.yaml#/components/schemas/MinMaxComputeLong' + offered: + description: > + The amount memory offered byt the server. + The min value specifies the initial resources available for the session. + The max value specifies the maximun resources available for the session. + $ref: './components/utils.yaml#/components/schemas/MinMaxComputeLong' + + ComputeResourceVolume: + description: > + A class to represent a ComputeResource volume mount. + type: object + title: ComputeResourceVolume + allOf: +# - $ref: './components/components.yaml#/components/schemas/AbstractComponent' +# - $ref: '#/components/schemas/AbstractComponent' + - $ref: 'AbstractComponent' + - type: object + properties: + path: + description: > + The mount point in the target filesystem. + type: string + mode: + description: > + The read-write mode. + type: string + enum: + - READONLY + - READWRITE + resource: + description: > + The name or UUID of the resource to mount. + type: string + +# storage-resources.yaml +# Tried moving this to a separate file, but had issues with generating the polymorphic classes. + + StorageResourceList: + description: > + A list of storage resources. + type: array + title: StorageResourceList + xml: + name: storage + items: + $ref: '#/components/schemas/AbstractStorageResource' + + AbstractStorageResource: + description: > + Abstract base class for storage resources. + type: object + title: AbstractStorageResource + required: + - type + discriminator: + propertyName: type + mapping: +# "uri:simple-storage-resource-0.1": '#/components/schemas/SimpleStorageResource' + "uri:simple-storage-resource-0.1": 'SimpleStorageResource' + allOf: +# - $ref: './components/components.yaml#/components/schemas/AbstractComponent' +# - $ref: '#/components/schemas/AbstractComponent' + - $ref: 'AbstractComponent' + - type: object + properties: + type: + type: string + xml: + attribute: true + + SimpleStorageResource: + description: > + A simple storage resource. + type: object + name: SimpleStorageResource + title: SimpleStorageResource + allOf: + - $ref: '#/components/schemas/AbstractStorageResource' + - type: object + properties: + size: + type: object + properties: + requested: + description: > + The storage size requested by the user. + The min value specifies the minimum that required for the session. + The max value limits the extra the server is allowed to offer. + $ref: './components/utils.yaml#/components/schemas/MinMaxComputeLong' + offered: + description: > + The storage size offered byt the server. + The min value specifies the initial resources available for the session. + The max value specifies the maximun resources available for the session. + $ref: './components/utils.yaml#/components/schemas/MinMaxComputeLong' + +# data-resources.yaml +# Tried moving this to a separate file, but had issues with generating the polymorphic classes. + + DataResourceList: + description: > + A list of data resources. + type: array + title: DataResourceList + xml: + name: storage + items: +# $ref: '#/components/schemas/AbstractDataResource' + $ref: 'AbstractDataResource' + + AbstractDataResource: + description: > + Abstract base class for data resources. + type: object + title: AbstractDataResource + required: + - type + discriminator: + propertyName: type + mapping: +# "uri:simple-data-resource-0.1": '#/components/schemas/SimpleDataResource' +# "uri:Rucio-data-resource-0.1": '#/components/schemas/RucioDataResource' +# "uri:S3-data-resource-0.1": '#/components/schemas/S3DataResource' + "uri:simple-data-resource-0.1": 'SimpleDataResource' + "uri:Rucio-data-resource-0.1": 'RucioDataResource' + "uri:S3-data-resource-0.1": 'S3DataResource' + + allOf: +# - $ref: './components/components.yaml#/components/schemas/AbstractComponent' +# - $ref: '#/components/schemas/AbstractComponent' + - $ref: 'AbstractComponent' + - type: object + properties: + type: + type: string + xml: + attribute: true + + SimpleDataResource: + description: > + A simple downloadable data resource. + This type of resource should always be mounted read-only. + type: object + title: SimpleDataResource + allOf: + - $ref: '#/components/schemas/AbstractDataResource' + - type: object + properties: + location: + description: | + The URL of the data to import. + type: string +# TODO add checksums +# TODO add a size estimate +# TODO add download time estimate +# TODO add status flag ... WAITING|PREPARING|READY|RELEASING|COMPLETED|CANCELLED|FAILED + + S3DataResource: + description: A S3 data resource. + type: object + title: S3DataResource + allOf: + - $ref: '#/components/schemas/AbstractDataResource' + - type: object + properties: + endpoint: + description: | + The endpoint address of the S3 service. + type: string + template: + description: | + The URL template for the S3 service. + type: string + bucket: + description: | + The target bucket name. + type: string + object: + description: | + The target object name. + Leaving this blank will mount the whole bucket as a directory. + type: string +# TODO add checksums +# TODO add a size estimate +# TODO add download time estimate +# TODO add status flag ... WAITING|PREPARING|READY|RELEASING|COMPLETED|CANCELLED|FAILED + + RucioDataResource: + description: A Rucio data resource. + type: object + title: RucioDataResource + allOf: + - $ref: '#/components/schemas/AbstractDataResource' + - type: object + properties: + endpoint: + description: | + The endpoint address of the Rucio service. + type: string + domain: + description: | + The domain name. + type: string + object: + description: | + The target object name. + type: string +# TODO add checksums +# TODO add a size estimate +# TODO add download time estimate +# TODO add status flag ... WAITING|PREPARING|READY|RELEASING|COMPLETED|CANCELLED|FAILED + + AbstractExecutable: + description: Abstract base class for executables. + type: object + title: AbstractExecutable + required: + - type + discriminator: + propertyName: type + mapping: +# "uri:docker-container-0.1": '#/components/schemas/DockerContainer' +# "uri:singular-container-0.1": '#/components/schemas/SingularContainer' +# "uri:jupyter-notebook-0.1": '#/components/schemas/JupyterNotebook' +# "uri:repo2docker-0.1": '#/components/schemas/Repo2DockerContainer' +# "uri:binder-notebook-0.1": '#/components/schemas/BinderNotebook' + "uri:docker-container-0.1": 'DockerContainer' + "uri:singular-container-0.1": 'SingularContainer' + "uri:jupyter-notebook-0.1": 'JupyterNotebook' + "uri:repo2docker-0.1": 'Repo2DockerContainer' + "uri:binder-notebook-0.1": 'BinderNotebook' + allOf: +# - $ref: './components/components.yaml#/components/schemas/AbstractComponent' +# - $ref: '#/components/schemas/AbstractComponent' + - $ref: 'AbstractComponent' + - type: object + properties: + type: + description: > + The component type discriminator + type: string + xml: + attribute: true + + DockerContainer: + description: | + A Docker or OCI container. + See https://opencontainers.org/ + type: object + title: DockerContainer + allOf: + - $ref: '#/components/schemas/AbstractExecutable' + - type: object + properties: + image: + type: string + description: > + The full image identifier, with or without the repository, namespace or tag. + namespace: + type: string + description: > + The namespace within the repository, if not already specified in the image name. + tag: + type: string + description: The image tag, if not already specified in the image name. + repository: + type: string + description: The respository tag, if not already specified in the image name. + platform: + type: string + description: > + The target CPU architecture the container is built for. + The default is `linux/amd64`. + privileged: + type: boolean + default: false + description: > + Set the privileged flag on execution. + The default is `false`. + See https://docs.docker.com/reference/cli/docker/container/run/#privileged. + entrypoint: + type: string + description: Overwrite the default ENTRYPOINT of the image. + environment: + description: A name=>value map of environment variables to pass to the container. + $ref: './components/utils.yaml#/components/schemas/NameValueMap' + network: + description: Details of the network access available to the container. + $ref: '#/components/schemas/DockerNetworkSpec' + + DockerNetworkSpec: + description: | + Details of the network access available to the container. + type: object + title: DockerNetworkSpec + properties: + ports: + description: > + An array of network ports to publish. + type: array + items: + $ref: '#/components/schemas/DockerNetworkPort' + + DockerNetworkPort: + description: | + Details of a network port on the container made available for connection from outside. + type: object + title: DockerNetworkPort + properties: + address: + type: string + description: > + The IP address of the external network interface. + The client should not set this value. + The service will update this with a publicly accessible interface address when it sets up the execution. + external: + type: string + description: > + The host machine's external port number to connect. + The client should not set this value. + The service will update this with the corrent port number when it sets up the execution. + internal: + type: string + description: The port number on the container to publish. + protocol: + type: string + enum: + - UDP + - TCP + - HTTP + - HTTPS + description: > + The network protocol to use, default is `TCP`. + The `HTTP` and `HTTPS` values further specify the Protocol to use on top of `TCP`. + Specifying `HTTPS` may help to meet firewall restrictions at some sites. + + SingularContainer: + description: > + A Singularity container executable. + See https://docs.sylabs.io/guides/latest/user-guide/ + type: object + title: SingularContainer + allOf: + - $ref: '#/components/schemas/AbstractExecutable' + - type: object + properties: + image: + type: string + description: > + The URL to download the container image from. + + Repo2DockerContainer: + description: > + A Repo2Docker executable. + See https://repo2docker.readthedocs.io/en/latest/ + type: object + title: Repo2DockerContainer + allOf: + - $ref: '#/components/schemas/AbstractExecutable' + - type: object + properties: + source: + type: string + description: > + The URL of the source repository. + + JupyterNotebook: + description: > + A Jupyter notebook executable. + See https://jupyter.org/ + type: object + title: JupyterNotebook + allOf: + - $ref: '#/components/schemas/AbstractExecutable' + - type: object + properties: + notebook: + type: string + description: > + The URL of the notebook. + TODO - This needs to take into account different ways of referring to a notebook. + + BinderNotebook: + description: > + A Binder notebook executable. + See https://the-turing-way.netlify.app/communication/binder/zero-to-binder.html + type: object + title: BinderNotebook + allOf: + - $ref: '#/components/schemas/AbstractExecutable' + - type: object + properties: + repository: + type: string + description: > + The URL of the repository to package. + notebook: + type: string + description: > + The relative path of the notebook within the repository. + +# update-options.yaml + + AbstractOption: + description: > + Abstract base class for describing options. + This includes the `type` discriminator and the target `path` to update. + type: object + title: AbstractOption + required: + - type + - path + discriminator: + propertyName: type + mapping: +# "uri:string-value-option": '#/components/schemas/StringValueOption' +# "uri:enum-value-option": '#/components/schemas/EnumValueOption' +# "uri:integer-value-option": '#/components/schemas/IntegerValueOption' +# "uri:integer-delta-option": '#/components/schemas/IntegerDeltaOption' + "uri:string-value-option": 'StringValueOption' + "uri:enum-value-option": 'EnumValueOption' + "uri:integer-value-option": 'IntegerValueOption' + "uri:integer-delta-option": 'IntegerDeltaOption' + properties: + type: + type: string + xml: + attribute: true + path: + description: > + The target path that the option applies to. + type: string + xml: + attribute: false + + StringValueOption: + description: > + A simple string value option. + This option enables the client to set a string value pointed to by the path. + type: object + title: StringValueOption + allOf: + - $ref: '#/components/schemas/AbstractOption' + - type: object + properties: + pattern: + description: > + A regular expression pattern restricting the value. + type: string + + EnumValueOption: + description: > + A simple enum value option. + This option enables the client to set an enum value pointed to by the path. + type: object + title: EnumValueOption + required: + - values + allOf: + - $ref: '#/components/schemas/AbstractOption' + - type: object + properties: + values: + description: > + The list of allowed values to use. + type: array + items: + type: string + + IntegerValueOption: + description: > + A simple integer value option. + This option enables the client to set an integer value pointed to by the path. + type: object + title: IntegerValueOption + allOf: + - $ref: '#/components/schemas/AbstractOption' + - type: object + properties: + min: + description: > + The minimum value that can be set. + type: integer + format: int64 + max: + description: > + The maximum value that can be set. + type: integer + format: int64 + units: + description: > + The units used for the maximum and minimum values + and the default units used for the update. + The client may specify different units in the + update if they need to. + type: string + + IntegerDeltaOption: + description: > + A simple integer delta option. + This option enables the client to increment or decrement an integer value pointed to by the path. + type: object + title: IntegerDeltaOption + allOf: + - $ref: '#/components/schemas/AbstractOption' + - type: object + properties: + min: + description: > + The minimum change that can be applied. + type: integer + format: int64 + max: + description: > + The maximum change that can be applied. + type: integer + format: int64 + units: + description: > + The units used for the maximum and minimum values + and the default units used for the update. + The client may specify different units in the + update if they need to. + type: string + +# update-actions.yaml + + AbstractUpdate: + description: > + Abstract base class for updates. + This includes the `type` discriminator and the target `path` to update. + type: object + title: AbstractUpdate + required: + - type + - path + discriminator: + propertyName: type + mapping: +# "uri:string-value-update": '#/components/schemas/StringValueUpdate' +# "uri:enum-value-update": '#/components/schemas/EnumValueUpdate' +# "uri:integer-value-update": '#/components/schemas/IntegerValueUpdate' +# "uri:integer-delta-update": '#/components/schemas/IntegerDeltaUpdate' + "uri:string-value-update": 'StringValueUpdate' + "uri:enum-value-update": 'EnumValueUpdate' + "uri:integer-value-update": 'IntegerValueUpdate' + "uri:integer-delta-update": 'IntegerDeltaUpdate' + properties: + type: + type: string + xml: + attribute: true + path: + description: > + The target path that the update applies to. + type: string + xml: + attribute: false + + StringValueUpdate: + description: > + A simple string value update. + type: object + title: StringValueUpdate + required: + - value + allOf: + - $ref: '#/components/schemas/AbstractUpdate' + - type: object + properties: + value: + description: > + The string value to use. + type: string + + EnumValueUpdate: + description: > + A simple enum value update. + type: object + title: EnumValueUpdate + required: + - value + allOf: + - $ref: '#/components/schemas/AbstractUpdate' + - type: object + properties: + value: + description: > + The enum value to use. + type: string + + IntegerValueUpdate: + description: > + A simple integer value update. + type: object + title: IntegerValueUpdate + required: + - value + allOf: + - $ref: '#/components/schemas/AbstractUpdate' + - type: object + properties: + value: + description: > + The integer value to use. + type: integer + format: int64 + units: + description: > + The units to use for the value. + type: string + + IntegerDeltaUpdate: + description: > + A simple integer delta update. + This increments or decrements the target value by the specified amount. + type: object + title: IntegerDeltaUpdate + required: + - delta + allOf: + - $ref: '#/components/schemas/AbstractUpdate' + - type: object + properties: + delta: + description: > + The increment or decrement to apply. + type: integer + format: int64 + units: + description: > + The units to use for the change. + type: string + + MinMaxDuration: + description: > + A duration time block describing how long the execution will take. + The max and min values are expressed as ISO 8601 time durations. + See https://en.wikipedia.org/wiki/ISO_8601#Durations + type: object + title: MinMaxDuration + xml: + name: duration + properties: + min: + type: string + max: + type: string + + # TODO This should be is-a AbstractUpdate rather than has-a AbstractUpdate + UpdateRequest: + description: > + A request to update an execution. + type: object + title: UpdateRequest + xml: + name: update-request + properties: + update: + xml: + name: update + $ref: '#/components/schemas/AbstractUpdate' + +# schedule-blocks.yaml + +# StringScheduleBlockValue: +# description: > +# Used for the predicted and observed times. +# type: object +# title: StringScheduleBlockValue +# xml: +# name: date-time +# properties: +# start: +# type: string +# duration: +# type: string +# +# StringScheduleBlockItem: +# description: > +# Used for the predicted and observed times. +# type: object +# title: StringScheduleBlockItem +# xml: +# name: date-time +# properties: +# preparing: +# $ref: '#/components/schemas/StringScheduleBlockValue' +# executing: +# $ref: '#/components/schemas/StringScheduleBlockValue' +# finishing: +# $ref: '#/components/schemas/StringScheduleBlockValue' +# +# StringScheduleBlock: +# description: > +# Schedule ... +# type: object +# title: StringScheduleBlock +# xml: +# name: schedule +# properties: +# requested: +# $ref: '#/components/schemas/StringScheduleBlockItem' +# offered: +# $ref: '#/components/schemas/StringScheduleBlockItem' + + + ScheduleRequestBlock: + description: > + Details of when to execute the session. + type: object + title: ScheduleRequestBlock + properties: + duration: + $ref: './components/utils.yaml#/components/schemas/ISO8601Duration' + start: + type: array + items: + $ref: './components/utils.yaml#/components/schemas/ISO8601Interval' + + ScheduleOfferItem: + description: > + Details of when the session will be executed. + type: object + title: ScheduleOfferItem + properties: + duration: + $ref: './components/utils.yaml#/components/schemas/ISO8601Duration' + start: + $ref: './components/utils.yaml#/components/schemas/ISO8601Interval' + + ScheduleOfferBlock: + description: > + Details of when the session will be executed. + type: object + title: ScheduleOfferBlock + properties: + preparing: + $ref: 'ScheduleOfferItem' + executing: + $ref: 'ScheduleOfferItem' + releasing: + $ref: 'ScheduleOfferItem' + +# sessions.yaml + + ExecutionResourceList: + description: > + Combined compute and storage resource lists. + type: object + title: ExecutionResourceList + xml: + name: resources + properties: + compute: + $ref: 'ComputeResourceList' + storage: + $ref: 'StorageResourceList' + data: + $ref: 'DataResourceList' + + ExecutionSessionStatus: + description: > + Status code for an execution session. + type: string + title: ExecutionSessionStatus + enum: + - PROPOSED + - OFFERED + - ACCEPTED + - REJECTED + - EXPIRED + + - WAITING + - PREPARING + - READY + - RUNNING + - RELEASING + + - COMPLETED + - FAILED + - CANCELLED + + ExecutionSessionRequest: + description: > + The base class for an execution. Used in both requests and responses. + type: object + title: ExecutionSessionRequest + xml: + name: execution + properties: + name: + description: > + A human readable name. + type: string + executable: + xml: + name: executable + $ref: '#/components/schemas/AbstractExecutable' + resources: + xml: + name: resources + $ref: '#/components/schemas/ExecutionResourceList' + schedule: + xml: + name: schedule + type: object + properties: + requested: + xml: + name: requested + $ref: '#/components/schemas/ScheduleRequestBlock' + + ExecutionSessionResponse: + description: > + An Execution session, including uuid, href, and status. + type: object + title: ExecutionSessionResponse + xml: + name: execution + allOf: +# - $ref: './components/components.yaml#/components/schemas/AbstractComponent' +# - $ref: '#/components/schemas/AbstractComponent' + - $ref: 'AbstractComponent' + - type: object + required: + - uuid + - state + - href + properties: + offerset: + description: > + A reference to the parent OfferSet. + xml: + name: offerset + $ref: '#/components/schemas/OfferSetLink' + state: + $ref: '#/components/schemas/ExecutionSessionStatus' + expires: + description: > + The date and time that this execution offer expires. + Only valid while the state is OFFERED. + xml: + name: expires + type: string + format: date-time + executable: + xml: + name: executable + $ref: '#/components/schemas/AbstractExecutable' + resources: + xml: + name: resources + $ref: '#/components/schemas/ExecutionResourceList' + schedule: + xml: + name: schedule + type: object + properties: + requested: + xml: + name: requested + $ref: '#/components/schemas/ScheduleRequestBlock' + preparing: + xml: + name: preparing + $ref: '#/components/schemas/ScheduleOfferItem' + executing: + xml: + name: executing + $ref: '#/components/schemas/ScheduleOfferItem' + releasing: + xml: + name: releasing + $ref: '#/components/schemas/ScheduleOfferItem' + options: + description: > + A List of options the client can apply to this Execution. + type: array + items: + $ref: '#/components/schemas/AbstractOption' + +# offersets.yaml + + OfferSetLink: + description: > + A reference from an offer to it's parent offerset. + type: object + title: OfferSetLink + properties: + uuid: + description: > + The UUID identifier of the parent OfferSet. + type: string + format: uuid + href: + description: > + The URL of the parent OfferSet. + type: string + + OfferSetRequest: + description: > + A request for a set of offers for an execution. + type: object + title: OfferSetRequest + xml: + name: offers-request + allOf: + - $ref: '#/components/schemas/ExecutionSessionRequest' + + OfferSetResponse: + description: > + A set of executions offered in response to a request, + including a uuid, href, and an expiry date for the set. + type: object + title: OfferSetResponse + required: + - uuid + - href + - expires + xml: + name: offers-response + allOf: +# - $ref: './components/components.yaml#/components/schemas/AbstractComponent' +# - $ref: '#/components/schemas/AbstractComponent' + - $ref: 'AbstractComponent' + - type: object + properties: + result: + description: > + A flag to indicate whether the request can be handled by this service. + If service is able to handle the request, then the `result` will be `YES` + and the `offers` block should contain one or more offers. + If service is not able to handle the request, the `result` will be `NO` + and the `messages` block should contain one or more reasons explaining why. + type: string + enum: + - "YES" + - "NO" +# # TODO Add simplified state OFFERED, ACCEPTED, REJECTED, EXPIRED, .. + offers: + type: array + xml: + name: offers + wrapped: true + items: + $ref: '#/components/schemas/ExecutionSessionResponse' + xml: + name: execution + + +