-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathworker-dev-base-job-template.yaml
195 lines (195 loc) · 8.58 KB
/
worker-dev-base-job-template.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
{
"job_configuration": {
"command": "{{ command }}",
"env": "{{ env }}",
"labels": "{{ labels }}",
"name": "{{ name }}",
"image": "{{ image }}",
"registry_credentials": "{{ registry_credentials }}",
"image_pull_policy": "{{ image_pull_policy }}",
"networks": "{{ networks }}",
"network_mode": "{{ network_mode }}",
"auto_remove": "{{ auto_remove }}",
"volumes": "{{ volumes }}",
"stream_output": "{{ stream_output }}",
"mem_limit": "{{ mem_limit }}",
"memswap_limit": "{{ memswap_limit }}",
"privileged": "{{ privileged }}"
},
"variables": {
"description": "Configuration class used by the Docker worker.\n\nAn instance of this class is passed to the Docker worker's `run` method\nfor each flow run. It contains all the information necessary to execute the\nflow run as a Docker container.\n\nAttributes:\n name: The name to give to created Docker containers.\n command: The command executed in created Docker containers to kick off\n flow run execution.\n env: The environment variables to set in created Docker containers.\n labels: The labels to set on created Docker containers.\n image: The image reference of a container image to use for created jobs.\n If not set, the latest Prefect image will be used.\n image_pull_policy: The image pull policy to use when pulling images.\n networks: Docker networks that created containers should be connected to.\n network_mode: The network mode for the created containers (e.g. host, bridge).\n If 'networks' is set, this cannot be set.\n auto_remove: If set, containers will be deleted on completion.\n volumes: Docker volumes that should be mounted in created containers.\n stream_output: If set, the output from created containers will be streamed\n to local standard output.\n mem_limit: Memory limit of created containers. Accepts a value\n with a unit identifier (e.g. 100000b, 1000k, 128m, 1g.) If a value is\n given without a unit, bytes are assumed.\n memswap_limit: Total memory (memory + swap), -1 to disable swap. Should only be\n set if `mem_limit` is also set. If `mem_limit` is set, this defaults to\n allowing the container to use as much swap as memory. For example, if\n `mem_limit` is 300m and `memswap_limit` is not set, containers can use\n 600m in total of memory and swap.\n privileged: Give extended privileges to created containers.",
"type": "object",
"properties": {
"command": {
"title": "Command",
"description": "The command to use when starting a flow run. In most cases, this should be left blank and the command will be automatically generated by the worker.",
"type": "string"
},
"env": {
"title": "Environment Variables",
"description": "Environment variables to set when starting a flow run.",
"type": "object",
"additionalProperties": {
"type": "string"
},
"default": {
"PREFECT_NODE_TYPE": "worker",
"PREFECT_API_URL": "http://REPLACE_AZ_INSTANCE_HOST_IP:4200/api",
"MKL_NUM_THREADS": "REPLACE_MKL_NUM_THREADS",
"NUMBA_NUM_THREADS": "REPLACE_NUMBA_NUM_THREADS",
"NUMBA_THREADING_LAYER": "REPLACE_NUMBA_THREADING_LAYER",
"OMP_NUM_THREADS": "REPLACE_OMP_NUM_THREADS",
"OPENBLAS_NUM_THREADS": "REPLACE_OPENBLAS_NUM_THREADS",
"SEEDOO_DB_IP": "REPLACE_SEEDOO_DB_IP",
"SEEDOO_DB_PORT": "REPLACE_SEEDOO_DB_PORT",
"SEEDOO_PRINT_SQL_DEBUG": "REPLACE_SEEDOO_PRINT_SQL_DEBUG"
}
},
"labels": {
"title": "Labels",
"description": "Labels applied to infrastructure created by the worker using this job configuration.",
"type": "object",
"additionalProperties": {
"type": "string"
}
},
"name": {
"title": "Name",
"description": "Name given to infrastructure created by the worker using this job configuration.",
"type": "string"
},
"image": {
"title": "Image",
"description": "The image reference of a container image to use for created jobs. If not set, the latest Prefect image will be used.",
"example": "docker.io/prefecthq/prefect:2-latest",
"type": "string",
"default": "docker.io/seedooinsights/prefect-worker:latest"
},
"registry_credentials": {
"title": "Registry Credentials",
"description": "Credentials for logging into a Docker registry to pull images from.",
"allOf": [
{
"$ref": "#/definitions/DockerRegistryCredentials"
}
],
"default": {
"$ref": {
"block_document_id": "REPLACE_ME_DOCKER_CREDS_ID"
}
}
},
"image_pull_policy": {
"title": "Image Pull Policy",
"description": "The image pull policy to use when pulling images.",
"enum": [
"IfNotPresent",
"Always",
"Never"
],
"type": "string",
"default": "Always"
},
"networks": {
"title": "Networks",
"description": "Docker networks that created containers should be connected to.",
"type": "array",
"items": {
"type": "string"
}
},
"network_mode": {
"title": "Network Mode",
"description": "The network mode for the created containers (e.g. host, bridge). If 'networks' is set, this cannot be set.",
"type": "string",
"default": "host"
},
"auto_remove": {
"title": "Auto Remove",
"description": "If set, containers will be deleted on completion.",
"default": false,
"type": "boolean"
},
"volumes": {
"title": "Volumes",
"description": "A list of volume to mount into created containers.",
"example": [
"/my/local/path:/path/in/container"
],
"type": "array",
"items": {
"type": "string"
},
"default": [
"/seedoodata:/seedoodata",
"/var/run/docker.sock:/var/run/docker.sock"
]
},
"stream_output": {
"title": "Stream Output",
"description": "If set, the output from created containers will be streamed to local standard output.",
"default": true,
"type": "boolean"
},
"mem_limit": {
"title": "Memory Limit",
"description": "Memory limit of created containers. Accepts a value with a unit identifier (e.g. 100000b, 1000k, 128m, 1g.) If a value is given without a unit, bytes are assumed.",
"type": "string"
},
"memswap_limit": {
"title": "Memory Swap Limit",
"description": "Total memory (memory + swap), -1 to disable swap. Should only be set if `mem_limit` is also set. If `mem_limit` is set, this defaults toallowing the container to use as much swap as memory. For example, if `mem_limit` is 300m and `memswap_limit` is not set, containers can use 600m in total of memory and swap.",
"type": "string"
},
"privileged": {
"title": "Privileged",
"description": "Give extended privileges to created container.",
"default": false,
"type": "boolean"
}
},
"definitions": {
"DockerRegistryCredentials": {
"title": "DockerRegistryCredentials",
"description": "Store credentials for interacting with a Docker Registry.",
"type": "object",
"properties": {
"username": {
"title": "Username",
"description": "The username to log into the registry with.",
"type": "string"
},
"password": {
"title": "Password",
"description": "The password to log into the registry with.",
"type": "string",
"writeOnly": true,
"format": "password"
},
"registry_url": {
"title": "Registry Url",
"description": "The URL to the registry. Generally, \"http\" or \"https\" can be omitted.",
"example": "index.docker.io",
"type": "string"
},
"reauth": {
"title": "Reauth",
"description": "Whether or not to reauthenticate on each interaction.",
"default": true,
"type": "boolean"
}
},
"required": [
"username",
"password",
"registry_url"
],
"block_type_slug": "docker-registry-credentials",
"secret_fields": [
"password"
],
"block_schema_references": {}
}
}
}
}