Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(#8551): removes python from cht-deploy and some bug fixes #9074

Merged
merged 50 commits into from
Aug 20, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
50 commits
Select commit Hold shift + click to select a range
b6d3d5d
Remove python from cht-deploy
henokgetachew Apr 29, 2024
0a4f83c
Logs archiving functionality
henokgetachew Apr 29, 2024
f9f7d39
Show URL in status
henokgetachew Apr 29, 2024
744a4e8
Rename command
henokgetachew Apr 29, 2024
a34ace5
Simplify namespace creation
henokgetachew Apr 30, 2024
80af317
Replace axios with node-fetch
henokgetachew Apr 30, 2024
d573940
Apply suggestions from code review
henokgetachew May 13, 2024
33f1a3f
Changes based on feedback
henokgetachew May 13, 2024
df9145b
Take out certs and errors
henokgetachew May 13, 2024
6588b4c
Replace throw with Promise.reject in async func
henokgetachew May 13, 2024
7481209
Print entire error object for better investigation
henokgetachew May 13, 2024
2463fdb
Add timestamp to make dir unique
henokgetachew May 13, 2024
e9149ce
Ensure uniqueness, use mktmp
henokgetachew May 13, 2024
b80b010
Use trap instead
henokgetachew May 13, 2024
0742a51
prep for tests
henokgetachew May 15, 2024
573027e
improvements and some tests
henokgetachew May 15, 2024
ef0aa2d
Check node version
henokgetachew May 15, 2024
631dca5
proper naming
henokgetachew May 15, 2024
d453b7c
Add files for npm
henokgetachew May 15, 2024
1f13203
Add other commands
henokgetachew May 15, 2024
fdcd892
Simplify command name for ease
henokgetachew May 15, 2024
6dc5c98
Add cht-deploy command back too
henokgetachew May 15, 2024
0504550
Check that data.tags has value
henokgetachew May 17, 2024
ae75624
Show error detail
henokgetachew May 17, 2024
3881f89
Bug fix - don't try to create ns if it exists
henokgetachew May 17, 2024
82cc47b
Remove let
henokgetachew May 17, 2024
13322d8
Bug fix - trim isnt removing trailing slash
henokgetachew May 17, 2024
8047f60
Add .gitignore
henokgetachew May 17, 2024
e800666
Add since param based on review
henokgetachew May 17, 2024
6ccd551
Add PII warning to logs download
henokgetachew May 18, 2024
9715f7c
Refactor goes beyond the scope of this PR
henokgetachew May 24, 2024
ace7819
Move NoSONAR
henokgetachew May 24, 2024
5914164
eslint conf
henokgetachew May 25, 2024
1c61aa8
Linting, adding tests to run with ci
henokgetachew May 25, 2024
ebb4e08
NoSONAR
henokgetachew May 25, 2024
74e3667
Update scripts/deploy/cht-deploy
henokgetachew May 28, 2024
f1935b7
lint cht-deploy
henokgetachew Jul 5, 2024
011fe8d
Validation tests
henokgetachew Jul 5, 2024
d27b3bf
Better tests
henokgetachew Jul 5, 2024
b23228d
Remove unused import
henokgetachew Jul 5, 2024
0a41d43
Mock each call and test each call for helm
henokgetachew Jul 5, 2024
40b505a
Use config file, improve certificate code
henokgetachew Jul 8, 2024
18c4b18
Apply suggestions from code review
henokgetachew Aug 15, 2024
4c18f17
Update scripts/deploy/troubleshooting/get-all-logs
henokgetachew Aug 15, 2024
363f50c
Changes based on review
henokgetachew Aug 15, 2024
ae773a3
Bash cleanup
henokgetachew Aug 16, 2024
ddbf188
Ignore Sonar false positives
henokgetachew Aug 16, 2024
eb3ec1c
Check exit code directly
henokgetachew Aug 16, 2024
f6edf08
Fix how we run tests
henokgetachew Aug 16, 2024
81c64a2
Add config.js
henokgetachew Aug 16, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,12 @@
"unit-nginx": "cd nginx/tests && make test",
"unit-haproxy": "cd haproxy/tests && make test",
"unit-haproxy-healthcheck": "cd haproxy-healthcheck && make test",
"unit-cht-deploy": "cd scripts/deploy && npm test",
"wdio-default-mobile-local": "export VERSION=$(node ./scripts/build/get-version.js) && ./scripts/build/build-service-images.sh && wdio run ./tests/e2e/default-mobile/wdio.conf.js --suite=all",
"wdio-local": "export VERSION=$(node ./scripts/build/get-version.js) && ./scripts/build/build-service-images.sh && wdio run ./tests/e2e/default/wdio.conf.js",
"-- CI SCRIPTS ": "-----------------------------------------------------------------------------------------------",
"build": "./scripts/build/build-ci.sh",
"ci-compile": "node scripts/ci/check-versions.js && node scripts/build/cli npmCiModules && npm run lint && npm run unit-nginx && npm run unit-haproxy && npm run unit-haproxy-healthcheck && npm run build && npm run integration-api && npm run unit",
"ci-compile": "node scripts/ci/check-versions.js && node scripts/build/cli npmCiModules && npm run lint && npm run unit-nginx && npm run unit-haproxy && npm run unit-haproxy-healthcheck && npm run unit-cht-deploy && npm run build && npm run integration-api && npm run unit",
"ci-integration-all": "mocha --config tests/integration/.mocharc-all.js",
"ci-integration-all-k3d": "mocha --config tests/integration/.mocharc-k3d.js",
"ci-integration-sentinel": "mocha --config tests/integration/.mocharc-sentinel.js",
Expand Down
8 changes: 8 additions & 0 deletions scripts/deploy/.eslintrc
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
henokgetachew marked this conversation as resolved.
Show resolved Hide resolved
"parserOptions": {
"sourceType": "module"
},
"rules": {
"no-console": "off"
}
}
2 changes: 2 additions & 0 deletions scripts/deploy/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
*.tar.gz
*.tgz
124 changes: 83 additions & 41 deletions scripts/deploy/cht-deploy
Original file line number Diff line number Diff line change
@@ -1,41 +1,83 @@
#!/bin/bash

# Check if Python is installed
if ! command -v python3 &> /dev/null ; then
echo "Python is not installed. Please install Python 3."
exit 1
fi

# Check if pip is installed
if ! command -v pip3 &> /dev/null ; then
echo "pip is not installed. Please install pip for Python 3."
exit 1
fi

# Check if Invoke is installed
if ! python3 -c "import invoke" &> /dev/null ; then
echo "Invoke is not installed. Installing it..."
pip3 install invoke --quiet
fi

# Check if PyYAML is installed
if ! python3 -c "import yaml" &> /dev/null ; then
echo "PyYAML is not installed. Installing it..."
pip3 install PyYAML --quiet
fi

# Check if requests is installed
if ! python3 -c "import requests" &> /dev/null ; then
echo "Requests is not installed. Installing it..."
pip3 install requests --quiet
fi

# Validate that -f argument is provided
if [[ $1 != "-f" || -z $2 ]]; then
echo "No values file provided. Please specify a values file using -f <file>"
exit 1
fi

# Pass command line arguments to invoke script
# shellcheck disable=SC2068 # wontfix script will be replaced "soon"
invoke install $@
#!/usr/bin/env node
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

before the script does anything else, I think we need to check for the node version. The reason is bash is very universal, but who knows what version of node folks have. My desktop defaults to 12.0.0 which pretty ensures it'll break everything, for example. Per this post, looks like we can easily get the SemVer of node back to old versions.

here's my shell session when debugging this:

➜  deploy git:(cht-deploy-fixes) ✗ ./cht-deploy 
/home/mrjones/Documents/MedicMobile/cht-core/node_modules/axios/index.js:1
import axios from './lib/axios.js';
       ^^^^^

SyntaxError: Unexpected identifier
    at Module._compile (internal/modules/cjs/loader.js:703:23)
    at Object.Module._extensions..js (internal/modules/cjs/loader.js:770:10)
    at Module.load (internal/modules/cjs/loader.js:628:32)
    at Function.Module._load (internal/modules/cjs/loader.js:555:12)
    at Module.require (internal/modules/cjs/loader.js:666:19)
    at require (internal/modules/cjs/helpers.js:16:16)
    at Object.<anonymous> (/home/mrjones/Documents/MedicMobile/cht-core/scripts/deploy/src/install.js:3:15)
    at Module._compile (internal/modules/cjs/loader.js:759:30)
    at Object.Module._extensions..js (internal/modules/cjs/loader.js:770:10)
    at Module.load (internal/modules/cjs/loader.js:628:32)

➜  deploy git:(cht-deploy-fixes) ✗ node
Welcome to Node.js v12.0.0.
Type ".help" for more information.
> process.versions.node.split('.').map(Number)
[ 12, 0, 0 ]
> 

➜  deploy git:(cht-deploy-fixes) ✗ nvm use 18
Now using node v18.16.0 (npm v9.5.1)

➜  deploy git:(cht-deploy-fixes) ✗ ./cht-deploy
No values file provided. Please specify a values file using -f <file>

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added.

Copy link
Contributor

@mrjones-plip mrjones-plip May 28, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure if this was fixed and then a regression was caused, but this doesn't work for me, all be it with a different error than when I first ran it:

➜  deploy git:(cht-deploy-fixes) ✗ nvm use 12.0.0
Now using node v12.0.0 (npm v6.9.0)


➜  deploy git:(cht-deploy-fixes) ✗ ./cht-deploy 
/home/mrjones/Documents/MedicMobile/cht-core/scripts/deploy/cht-deploy:3
import { install } from './src/install.js';
       ^

SyntaxError: Unexpected token {
    at Module._compile (internal/modules/cjs/loader.js:703:23)
    at Object.Module._extensions..js (internal/modules/cjs/loader.js:770:10)
    at Module.load (internal/modules/cjs/loader.js:628:32)
    at Function.Module._load (internal/modules/cjs/loader.js:555:12)
    at Function.Module.runMain (internal/modules/cjs/loader.js:826:10)
    at internal/main/run_main_module.js:17:11

➜  deploy git:(cht-deploy-fixes) ✗ nvm use 20
Now using node v20.11.0 (npm v10.2.4)

➜  deploy git:(cht-deploy-fixes) ✗ ./cht-deploy
No values file provided. Please specify a values file using -f <file>


import { install } from './src/install.js';
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This looks like it isn't linted with our rules.
These are our linting rules: https://github.com/medic/cht-core/blob/master/.eslintrc

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This file was skipped. Fixed.
The whole thing is linted by our linting rules with overrides configured here: https://github.com/medic/cht-core/blob/cht-deploy-fixes/scripts/deploy/.eslintrc

import fs from 'fs';
import semver from 'semver';
import path from 'path';

import { dirname } from 'path';
import { fileURLToPath } from 'url';

const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);

const validateNodeVersion = function() {
const packageJsonPath = path.resolve(__dirname, 'package.json');

try {
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
const requiredNodeVersion = packageJson.engines && packageJson.engines.node;

if (requiredNodeVersion && !semver.satisfies(process.version, requiredNodeVersion)) {
console.error(`Invalid Node.js version. Required: ${requiredNodeVersion}. Current: ${process.version}`);
process.exit(1);
}
} catch (err) {
console.error('Error reading package.json:', err.message);
process.exit(1);
}
};

const validateArguments = function() {
const args = process.argv.slice(2);
if (args.length < 2 || args[0] !== '-f' || !args[1]) {
console.error('No values file provided. Please specify a values file using -f <file>');
process.exit(1);
}
if (validateFileExists(args[1])) {
return args;
}
};

const validateFileExists = function(filePath) {
try {
fs.accessSync(filePath);
return true;
} catch (err) {
console.error(`File not found: ${filePath}`);
process.exit(1);
}
};

const runInstallScript = async function(args) {
try {
const valuesFilePath = args[1];
await install(valuesFilePath);
} catch (err) {
console.error('Error executing the install script:', err.message);
console.error(JSON.stringify(err));
process.exit(1);
}
};

const main = async function() {
validateNodeVersion();
const args = validateArguments();
await runInstallScript(args);
};

if (import.meta.url === `file://${process.argv[1]}`) { //Make sure the script is being run directly and not being imported
main().catch((err) => {
console.error('An error occurred:', err.message);
console.error(JSON.stringify(err));
henokgetachew marked this conversation as resolved.
Show resolved Hide resolved
process.exit(1);
});
}

export {
main,
validateNodeVersion,
validateArguments,
validateFileExists,
runInstallScript
};
54 changes: 54 additions & 0 deletions scripts/deploy/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
{
henokgetachew marked this conversation as resolved.
Show resolved Hide resolved
"name": "@medic/cht-deploy",
"version": "1.0.10",
"description": "## Deploy the CHT",
"main": "src/install.js",
"directories": {
"test": "tests"
},
"engines": {
"node": ">=20.11.0",
"npm": ">=10.2.4"
},
"bin": {
"cht-deploy": "./cht-deploy",
"upload": "./cht-deploy",
"get-all-logs": "./troubleshooting/get-all-logs",
"describe-deployment": "./troubleshooting/describe-deployment",
"list-all-resources": "./troubleshooting/list-all-resources",
"list-deployments": "./troubleshooting/list-deployments",
"restart-deployment": "./troubleshooting/restart-deployment",
"view-logs": "./troubleshooting/view-logs"
},
"scripts": {
"test": "mocha 'tests/*.js'"
},
"mocha": {
"timeout": 5000,
"recursive": true
},
"files": [
"src",
"cht-deploy",
"troubleshooting",
"README.md",
"package.json"
],
"keywords": [],
"author": "",
"license": "AGPL-3.0-only",
"type": "module",
"devDependencies": {
"@kubernetes/client-node": "^0.19.0",
"@sinonjs/fake-timers": "^11.2.2",
"chai": "^4.4.1",
"chai-as-promised": "^7.1.2",
"mocha": "^10.4.0",
"sinon": "^17.0.1"
},
"dependencies": {
"js-yaml": "^4.1.0",
"node-fetch": "^3.3.2",
"semver": "^7.6.2"
}
}
122 changes: 122 additions & 0 deletions scripts/deploy/src/certificate.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
import fs from 'fs';
import { execSync } from 'child_process';
import fetch from 'node-fetch';
import { UserRuntimeError, CertificateError } from './error.js';
import config from './config.js';

const CERT_SOURCES = {
FILE: 'specify-file-path',
MY_IP: 'my-ip-co',
EKS: 'eks-medic'
};

const cert = config.CERT_FILE;
const key = config.KEY_FILE;

const obtainCertificateAndKey = async function(values) {
console.log('Obtaining certificate...');
const certSource = values.cert_source || '';

const handlers = {
[CERT_SOURCES.FILE]: () => handleFileSource(values),
[CERT_SOURCES.MY_IP]: () => handleMyIpSource(),
[CERT_SOURCES.EKS]: () => {} // No action needed for 'eks-medic'
};

const handler = handlers[certSource];
if (!handler) {
throw new UserRuntimeError(`cert_source must be one of: ${Object.values(CERT_SOURCES).join(', ')}`);
}

await handler();
};

const handleFileSource = function({ certificate_crt_file_path, certificate_key_file_path }) {
if (!certificate_crt_file_path || !certificate_key_file_path) {
throw new CertificateError('certificate_crt_file_path and certificate_key_file_path must be set for file source');
}

copyFile(certificate_crt_file_path, cert);
copyFile(certificate_key_file_path, key);
};

const handleMyIpSource = async function() {
const [crtData, keyData] = await Promise.all([
fetchData(`${config.CERT_API_URL}/fullchain`),
fetchData(`${config.CERT_API_URL}/key`)
]);

writeFile(cert, crtData);
writeFile(key, keyData);
};

const copyFile = function(src, dest) {
try {
fs.copyFileSync(src, dest);
} catch (error) {
throw new CertificateError(`Failed to copy file from ${src} to ${dest}: ${error.message}`);
}
};

const writeFile = function(filename, data) {
try {
fs.writeFileSync(filename, data);
} catch (error) {
throw new CertificateError(`Failed to write file ${filename}: ${error.message}`);
}
};

const fetchData = async function(url) {
try {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), config.FETCH_TIMEOUT);
const response = await fetch(url, { signal: controller.signal });
clearTimeout(timeoutId);
return await response.text();
} catch (error) {
if (error.name === 'AbortError') {
throw new CertificateError(`Request to ${url} timed out`);
}
throw new CertificateError(`Failed to fetch data from ${url}: ${error.message}`);
}
};

const createSecret = async function (namespace, values) {
console.log('Checking if secret api-tls-secret already exists...');
try {
execSync(`kubectl get secret api-tls-secret -n ${namespace}`, { stdio: 'inherit' }); //NoSONAR
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is Sonar complaining about here?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

About using a user supplied value in a shell script. This is not a problem because the user who runs the script is the user who provides this value under his own user/credentials and access levels.

console.log('TLS secret already exists. Skipping creation.');
return;
} catch (err) {
if (err.message.includes('NotFound')) {
console.log('Secret does not exist. Creating Secret from certificate and key...');
} else {
throw err;
}
}

await obtainCertificateAndKey(values);

execSync( //NoSONAR
`kubectl -n ${namespace} create secret tls api-tls-secret --cert=${cert} --key=${key}`, //NoSONAR
{ stdio: 'inherit' }
);
cleanupFiles();
};

const cleanupFiles = function() {
deleteFile(cert);
deleteFile(key);
};

const deleteFile = function(filename) {
try {
if (fs.existsSync(filename)) {
fs.unlinkSync(filename);
}
} catch (error) {
console.error(`Failed to delete file ${filename}: ${error.message}`);
}
};

export { obtainCertificateAndKey, createSecret };
11 changes: 11 additions & 0 deletions scripts/deploy/src/config.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
export default {
MEDIC_REPO_NAME: process.env.MEDIC_REPO_NAME || 'medic',
MEDIC_REPO_URL: process.env.MEDIC_REPO_URL || 'https://docs.communityhealthtoolkit.org/helm-charts',
CHT_CHART_NAME: process.env.CHT_CHART_NAME || 'medic/cht-chart-4x',
DEFAULT_CHART_VERSION: process.env.DEFAULT_CHART_VERSION || '1.0.*',
IMAGE_TAG_API_URL: process.env.IMAGE_TAG_API_URL || 'https://staging.dev.medicmobile.org/_couch/builds_4',
CERT_FILE: process.env.CERT_FILE || 'certificate.crt',
KEY_FILE: process.env.KEY_FILE || 'private.key',
CERT_API_URL: process.env.CERT_API_URL || 'https://local-ip.medicmobile.org',
FETCH_TIMEOUT: parseInt(process.env.FETCH_TIMEOUT, 10) || 5000,
};
13 changes: 13 additions & 0 deletions scripts/deploy/src/error.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
export class UserRuntimeError extends Error {
constructor(message) {
super(message);
this.name = 'UserRuntimeError';
}
}

export class CertificateError extends Error {
constructor(message) {
super(message);
this.name = 'CertificateError';
}
}
Loading