-
Notifications
You must be signed in to change notification settings - Fork 29
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #3192 from SwissDataScienceCenter/release/v1.9.0
chore: release v1.9.0
- Loading branch information
Showing
69 changed files
with
2,386 additions
and
1,283 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| config.js |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| # Load testing with k6 | ||
|
|
||
| ## To run | ||
| 1. Download k6: https://k6.io/docs/get-started/installation/ - you can also get a binary from the github page | ||
| 2. Enter the credentials and deployment you wish to test in a file named `config.js`, you can | ||
| use the `example.config.js` to start - just make a copy and rename it. | ||
| 3. Run the tests with `k6 run testFileName.js` | ||
|
|
||
| ## Limitations | ||
|
|
||
| - This can log into Renku only and specifically in the cases where Renku has its own built-in gitlab that does | ||
| not require a separate log in OR when the gitlab deployment is part of another renku deployment | ||
| - The login flow cannot handle giving authorization when prompted in the oauth flow - do this | ||
| for the first time manually then run the tests with the same account | ||
| - The project used to test migrations has to be in a namespace that you control and can create | ||
| other projects in. When forks are created they are always created in the same namespace as the | ||
| original project with the name being unique. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| export const baseUrl = "https://dev.renku.ch" | ||
| // oldGitlabProjectId has to point to a project that resides in a namespace that the user | ||
| // has at least maintainer access to. This is because the load tests will fork this project | ||
| // into the same namespace as where the original project resides and only generate a uuid-like | ||
| // name for the project. So if you point to a project that resides in a namespace to which | ||
| // the test runner has no permissions, the forking part of the tests will fail. | ||
| export const oldGitlabProjectId = 5011 | ||
| // This project is used to test calling api/renku/project.show, the project is not forked | ||
| // and it does not have the same strict requirements as the project mentioned above. Any | ||
| // public project should work here (whether the user has write access to it or not). | ||
| export const sampleGitProjectUrl = "https://dev.renku.ch/gitlab/tasko.olevski/test-project-2.git" | ||
|
|
||
| // Two sets of credentials are needed only if the Renku deployment | ||
| // has a separate Gitlab that requires logging into another Renku | ||
| // instance. So for dev.renku.ch you need one set of credentials | ||
| // for CI deployments you need 2. First the credentials to the | ||
| // CI deployment then the ones for dev.renku.ch. | ||
| export const credentials = [ | ||
| { | ||
| username: "user@email.com", | ||
| password: "secret-password1" | ||
| }, | ||
| { | ||
| username: "user@email.com", | ||
| password: "secret-password1" | ||
| }, | ||
| ] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| // Creator: k6 Browser Recorder 0.6.2 | ||
|
|
||
| import { sleep, check, fail } from 'k6' | ||
| import http from 'k6/http' | ||
| import { uuidv4 } from 'https://jslib.k6.io/k6-utils/1.2.0/index.js'; | ||
| import crypto from 'k6/crypto'; | ||
| import { URL } from 'https://jslib.k6.io/url/1.0.0/index.js'; | ||
|
|
||
| import { renkuLogin } from './oauth.js' | ||
| import { credentials, baseUrl } from './config.js' | ||
|
|
||
| export const options = { | ||
| scenarios: { | ||
| testUploads: { | ||
| executor: 'per-vu-iterations', | ||
| vus: 3, | ||
| iterations: 1, | ||
| }, | ||
| } | ||
| } | ||
|
|
||
| function uploadRandomFile(baseUrl, uuid, fileName, numChunks, chunkSizeBytes) { | ||
| const responses = [] | ||
| for (let i = 0; i < numChunks; i++) { | ||
| const url = new URL(`ui-server/api/renku/cache.files_upload`, baseUrl); | ||
| url.searchParams.append('dzuuid', uuid); | ||
| url.searchParams.append('dzchunkindex', i); | ||
| url.searchParams.append('dztotalfilesize', numChunks * chunkSizeBytes); | ||
| url.searchParams.append('dzchunksize', chunkSizeBytes); | ||
| url.searchParams.append('dztotalchunkcount', numChunks); | ||
| url.searchParams.append('dzchunkbyteoffset', i * chunkSizeBytes); | ||
| url.searchParams.append('chunked_content_type', "application/octet-stream"); | ||
| const res = http.post(url.toString(), { | ||
| file: http.file(crypto.randomBytes(chunkSizeBytes), fileName, "application/octet-stream") | ||
| }) | ||
| responses.push(res) | ||
| } | ||
| if (!check(responses, { | ||
| "file uploads all have 200 repsonses": (responses) => responses.every(res => res.status === 200), | ||
| "file uploads all completed without errors": (responses) => responses.every(res => res.json().error === undefined), | ||
| })) { | ||
| const errResponses = responses.filter(res => res.json().error !== undefined).map(res => res.json()) | ||
| const failedResponsesBody = responses.filter(res => res.status != 200).map(res => res.body) | ||
| const failedResponsesCodes = responses.filter(res => res.status != 200).map(res => res.status) | ||
| fail( | ||
| `some responses failed with errors ${JSON.stringify(errResponses)}\nsome respones ` + | ||
| `failed with non-200 status codes codes: ${JSON.stringify(failedResponsesCodes)} bodies: ${JSON.stringify(failedResponsesBody)}` | ||
| ) | ||
| }; | ||
| return responses[numChunks - 1] | ||
| } | ||
|
|
||
| export default function fileUpload() { | ||
| renkuLogin(baseUrl, credentials) | ||
| const baseUrlResponse = http.get(baseUrl) | ||
| const projects = http.get(`${baseUrl}/ui-server/api/projects?query=last_activity_at&per_page=100&starred=true&page=1`) | ||
| check(baseUrlResponse, { | ||
| "baseUrl responds with status 200": (r) => r.status === 200, | ||
| }); | ||
| check(projects, { | ||
| 'projects list endpoint responds with status 200': (r) => r.status === 200, | ||
| 'project list exists': (r) => r.json().length >= 0, | ||
| }); | ||
| sleep(1) | ||
| const uploads = http.get(`${baseUrl}/ui-server/api/renku/cache.files_list`) | ||
| check(uploads, { | ||
| 'uploads list response does not have errors': (r) => r.json().error === undefined, | ||
| 'uploads list response contains a list of uploads': (r) => r.json().result.files.length >= 0, | ||
| }); | ||
| sleep(1) | ||
| const uuid = uuidv4() | ||
| const fileName = `${uuid}.bin` | ||
| const fileUploadResponse = uploadRandomFile(baseUrl, uuid, fileName, 100, 1e6) | ||
| let uploadedFiles = fileUploadResponse.json().result.files | ||
| if (uploadedFiles === undefined) { | ||
| uploadedFiles = [] | ||
| } | ||
| uploadedFiles = uploadedFiles.map(i => i.file_name) | ||
| if (!check(uploadedFiles, { | ||
| 'file name found in last upload response': (r) => r.includes(fileName), | ||
| })) { | ||
| fail(`could not find file in last upload response, body: ${fileUploadResponse.body}, status code: ${fileUploadResponse.status}`) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| import http from 'k6/http'; | ||
| import { parseHTML } from 'k6/html'; | ||
|
|
||
| function handleRenkuLoginForm(httpResponse, username, password) { | ||
| const doc = parseHTML(httpResponse.body); | ||
| const actionUrl = doc.find('#kc-form-login').attr('action'); | ||
| if (!actionUrl) { | ||
| throw new Error(`Could not locate login form in http response ${httpResponse.body}`) | ||
| } | ||
| const loginData = { | ||
| username, | ||
| password, | ||
| credentialId: '', | ||
| }; | ||
| return http.post(actionUrl, loginData) | ||
| } | ||
|
|
||
| function followRedirectLinkFromHtml(httpResponse) { | ||
| const doc = parseHTML(httpResponse.body); | ||
| let url = doc.find('a').attr("href") | ||
| if (!url) { | ||
| throw new Error(`Could not find <a> element with href attribute in ${httpResponse.body}`) | ||
| } | ||
| if (url.endsWith("/")) { | ||
| // leaving trailing slashes here results in 404 | ||
| url = url.slice(0,-1) | ||
| } | ||
| return http.get(url) | ||
| } | ||
|
|
||
| export function renkuLogin(baseUrl, credentials) { | ||
| // double slashes when composing url causes trouble and 404s | ||
| if (baseUrl.endsWith("/")) { | ||
| baseUrl = baseUrl.slice(0,-1) | ||
| } | ||
| // the trailing slash is needed here keycloak accepts only such and longer callbacks | ||
| const redirectUrl = `${baseUrl}/` | ||
| let finalResponse = null | ||
| const res1 = http.get(`${baseUrl}/ui-server/auth/login?redirect_url=${redirectUrl}"`) | ||
| const res2 = handleRenkuLoginForm(res1, credentials[0].username, credentials[0].password) | ||
| const res3 = followRedirectLinkFromHtml(res2) | ||
| if (res3.body.match(".*redirect.*|.*Redirect.*") && parseHTML(res3.body).find("a").toArray().length > 0) { | ||
| // no more login forms just follow a single last redirect | ||
| finalResponse = followRedirectLinkFromHtml(res3) | ||
| } | ||
| else if (parseHTML(res3.body).find('#kc-form-login').toArray().length > 0) { | ||
| // one more login required, usually happens for ci and similar deployments that do not have their own gitlab | ||
| const res4 = handleRenkuLoginForm(res3, credentials[1].username, credentials[1].password) | ||
| finalResponse = followRedirectLinkFromHtml(res4) | ||
| } | ||
| if (finalResponse.status != 200) { | ||
| throw new Error(`Could not successfully login, expected status code 200 but got ${finalResponse.status}`) | ||
| } | ||
| } |
Oops, something went wrong.