-
Notifications
You must be signed in to change notification settings - Fork 0
/
azure.sh
216 lines (195 loc) · 7.18 KB
/
azure.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
## Export environment variables
set -a
. ./credentials/env
set +a
## Auth to Azure
az login
az account list --refresh --output table
az account set -s $TENANT_ID
## Create vnet and subnet for cloud resources
az network vnet create \
--resource-group $RESOURCE_GROUP \
--name $VNET_NAME\
--address-prefixes 10.1.0.0/16 \
--subnet-name kubesubnet \
--subnet-prefix 10.1.0.0/24
VNET_ID=$(az network vnet show --resource-group $RESOURCE_GROUP --name $VNET_NAME --query id -o tsv)
SUBNET_ID=$(az network vnet subnet show --resource-group $RESOURCE_GROUP --vnet-name $VNET_NAME --name kubesubnet --query id -o tsv)
az role assignment create --assignee $SERVICE_PRINCIPLE_ID --scope $VNET_ID --role Contributor
## Create Kubernetes cluster
ssh-keygen -f credentials/ssh-key-jhubspark
az aks create --name $AKS_NAME \
--resource-group $RESOURCE_GROUP \
--service-principal $SERVICE_PRINCIPLE_ID \
--client-secret $SERVICE_PRINCIPLE_SECRET \
--ssh-key-value credentials/ssh-key-jhubspark.pub \
--node-count 1 \
--node-vm-size Standard_D2s_v3 \
--enable-vmss \
--kubernetes-version $AKS_VERSION \
--load-balancer-sku standard \
--vm-set-type VirtualMachineScaleSets \
--vnet-subnet-id $SUBNET_ID \
--location eastus \
--output table
# Create system nodepool
az aks nodepool add --name systempool \
--cluster-name $AKS_NAME \
--resource-group $RESOURCE_GROUP \
--kubernetes-version $AKS_VERSION \
--node-taints CriticalAddonsOnly=true:NoSchedule \
--mode System \
--node-count 1 \
--node-vm-size Standard_D2s_v3 \
--vnet-subnet-id $SUBNET_ID \
--output table
# Delete default nodepool nodepool1
az aks nodepool delete --name nodepool1 \
--cluster-name $AKS_NAME \
--resource-group $RESOURCE_GROUP \
--output table
# Create node pool for deploying applications
az aks nodepool add --name apppool \
--cluster-name $AKS_NAME \
--resource-group $RESOURCE_GROUP \
--mode user \
--enable-cluster-autoscaler \
--kubernetes-version $AKS_VERSION \
--node-count 1 \
--max-count 4 \
--min-count 0 \
--labels dedicate.pool=apppool \
--node-vm-size Standard_D2s_v3 \
--vnet-subnet-id $SUBNET_ID \
--output table
# Create jupyterhub user node pool
az aks nodepool add --name jhubuserpool \
--cluster-name $AKS_NAME \
--resource-group $RESOURCE_GROUP \
--mode user \
--enable-cluster-autoscaler \
--kubernetes-version $AKS_VERSION \
--node-count 0 \
--max-count 20 \
--min-count 0 \
--labels hub.jupyter.org/node-purpose=user dedicate.pool=jhubuserpool \
--node-taints hub.jupyter.org/dedicated=user:NoSchedule \
--node-vm-size Standard_D2s_v3 \
--vnet-subnet-id $SUBNET_ID \
--output table
# Create spark node pool
az aks nodepool add --name $SPARK_NDOE_POOL \
--cluster-name $AKS_NAME \
--resource-group $RESOURCE_GROUP \
--mode user \
--enable-cluster-autoscaler \
--kubernetes-version $AKS_VERSION \
--node-count 0 \
--max-count 20 \
--min-count 0 \
--labels dedicate.pool=$SPARK_NDOE_POOL \
--node-vm-size Standard_D8s_v3 \
--vnet-subnet-id $SUBNET_ID \
--output table
## Install kubectl
az aks get-credentials --name $AKS_NAME \
--resource-group $RESOURCE_GROUP \
--output table
kubectl get node
## Set up Helm 3
# Mac OS
curl https://get.helm.sh/helm-v3.8.0-darwin-amd64.tar.gz > helm-v3.8.0.tar.gz
tar -zxvf helm-v3.8.0.tar.gz
mv darwin-amd64/helm /usr/local/bin/helm
## Setup container registery
az acr create \
--name $ACR_NAME \
--resource-group $RESOURCE_GROUP \
--sku Standard \
--admin-enabled true \
--location eastus \
--output table
az acr login --name $ACR_NAME
CLIENT_ID=$(az aks show --resource-group $RESOURCE_GROUP --name $AKS_NAME --query "servicePrincipalProfile.clientId" --output tsv)
ACR_ID=$(az acr show --name $ACR_NAME --resource-group $RESOURCE_GROUP --query "id" --output tsv)
az role assignment create --assignee $CLIENT_ID --role acrpull --scope $ACR_ID
# Setup storage account
az storage account create \
--name $ADLS_ACCOUNT_NAME \
--resource-group $RESOURCE_GROUP \
--allow-shared-key-access true \
--allow-blob-public-access true \
--kind StorageV2 \
--access-tier Hot \
--enable-hierarchical-namespace true \
--bypass AzureServices \
--enable-large-file-share \
--location eastus \
--output table
az storage share-rm create \
--name $USER_FS \
--account-name $ADLS_ACCOUNT_NAME \
--access-tier "TransactionOptimized" \
--quota 1024 \
--output table
az storage share-rm create \
--name $PROJECT_FS \
--account-name $ADLS_ACCOUNT_NAME \
--access-tier "TransactionOptimized" \
--quota 1024 \
--output table
# Deploy PVC
kubectl create namespace $JHUB_NAMESPACE
kubectl create secret generic azure-fileshare-secret --from-literal=azurestorageaccountname=$ADLS_ACCOUNT_NAME --from-literal=azurestorageaccountkey=$ADLS_ACCOUNT_KEY -n $JHUB_NAMESPACE
sed -e "s/<USER_FS>/${USER_FS}/" -e "s/<PROJECT_FS>/${PROJECT_FS}/" pvc-pv-jhub.yaml > customized-pvc-pv.yaml
kubectl apply -f customized-pvc-pv.yaml -n $JHUB_NAMESPACE
# Pull jupyterhub helm chart
helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/
helm repo update
helm fetch jupyterhub/jupyterhub --version 1.2.0
## Create pullsecret
ACR_ID=$(az acr show --name $ACR_NAME --resource-group $RESOURCE_GROUP --query "id" --output tsv)
SP_PASSWD=$(az ad sp create-for-rbac --name http://$SERVICE_PRINCIPLE_ID --scopes $ACR_ID --role acrpull --query password --output tsv)
SP_APP_ID=$(az ad sp show --id http://$SERVICE_PRINCIPLE_ID --query appId --output tsv)
kubectl create secret docker-registry $ACR_PULL_SECRET\
--namespace $JHUB_NAMESPACE \
--docker-server=$ACR_NAME.azurecr.io \
--docker-username=$SP_APP_ID \
--docker-password=$SP_PASSWD
## Create service account for spark
kubectl --namespace $JHUB_NAMESPACE create serviceaccount $SERVICE_ACCOUNT
kubectl create clusterrolebinding spark-role-binding --clusterrole cluster-admin --serviceaccount=$JHUB_NAMESPACE:$SERVICE_ACCOUNT
## Build Images
az acr login --name $ACR_NAME
# build spark worker image
docker build \
--build-arg BASE_IMAGE=$BASE_IMAGE \
-f pyspark-notebook/Dockerfile.spark \
-t $ACR_NAME.azurecr.io/pyspark-worker:v3.2.1 ./pyspark-notebook
docker push $ACR_NAME.azurecr.io/pyspark-worker:v3.2.1
# build pyspark notebook image
docker build \
--build-arg ADLS_ACCOUNT_NAME=$ADLS_ACCOUNT_NAME \
--build-arg ADLS_ACCOUNT_KEY=$ADLS_ACCOUNT_KEY \
--build-arg ACR_NAME=$ACR_NAME \
--build-arg ACR_PULL_SECRET=$ACR_PULL_SECRET \
--build-arg JHUB_NAMESPACE=$JHUB_NAMESPACE \
--build-arg WORK_IMAGE=$ACR_NAME.azurecr.io/pyspark-worker:v3.2.1 \
--build-arg SPARK_NDOE_POOL=$SPARK_NDOE_POOL \
--build-arg SERVICE_ACCOUNT=$SERVICE_ACCOUNT \
--build-arg USER_FS_PVC=pvc-$USER_FS \
--build-arg PROJECT_FS_PVC=pvc-$PROJECT_FS \
-f pyspark-notebook/Dockerfile \
-t $ACR_NAME.azurecr.io/pyspark-notebook:v3.2.1 ./pyspark-notebook
docker push $ACR_NAME.azurecr.io/pyspark-notebook:v3.2.1
# build jupyterhub image
docker build -t $ACR_NAME.azurecr.io/k8s-hub:latest -f jupyter-k8s-hub/Dockerfile ./jupyter-k8s-hub
docker push $ACR_NAME.azurecr.io/k8s-hub:latest
# Build customized jupyterhub chart
sed -e "s/<NOTEBOOK-IMAGE>/${ACR_NAME}.azurecr.io\/pyspark-notebook:v3.2.1/" -e "s/<HUB-IMAGE>/${ACR_NAME}.azurecr.io\/k8s-hub/" config.yaml > customized-config.yaml
## Install jupyterhub
helm upgrade --install spark-jhub jupyterhub/jupyterhub \
--namespace $JHUB_NAMESPACE \
--version=1.2.0 \
--values customized-config.yaml \
--timeout=5000s