diff --git a/.travis.yml b/.travis.yml index 6e3ed00..0fa896d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,7 @@ before_script: script: - . ./ci/build.sh + - . ./ci/list_artifacts.sh # note before_deploy will run before each deploy provider before_deploy: diff --git a/ci/create_codewind_index.py b/ci/create_codewind_index.py new file mode 100755 index 0000000..1c2ed25 --- /dev/null +++ b/ci/create_codewind_index.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 + +import yaml +import json +import os +import fnmatch +from collections import OrderedDict +import argparse +from argparse import ArgumentDefaultsHelpFormatter + +parser = argparse.ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) + +parser.add_argument("-n", "--namePrefix", help="Display name prefix.", default="Appsody") +parser.add_argument("-f", "--file", help="Absolute or relative path, to a yaml file or directory of yaml files.", default=os.getcwd()) + +args = parser.parse_args() + +displayNamePrefix = args.namePrefix + +yaml_dir = os.path.normpath(args.file) + +def generate_json(): + with open(inputFile, 'r') as yamlFile, open(inputFile.rsplit('.', 1)[0] + ".json", 'wb') as jsonFile: + try: + doc = yaml.safe_load(yamlFile) + list = [] + + if (doc['stacks'] != None): + for item in doc['stacks']: + + # get template name + for n in range(0, len(item['templates'])): + if len(item['templates'])==1: + template = "" + else: + template = " " + item['templates'][n]['id'] + + # populate stack details + res = (OrderedDict([ + ("displayName", displayNamePrefix + " " + item['name'] + template + " template"), + ("description", item['description']), + ("language", item['language']), + ("projectType", "appsodyExtension"), + ("projectStyle", "Appsody"), + ("location", item['templates'][n]['url']), + ("links", OrderedDict([ + ("self", "/devfiles/" + + item['id'] + "/devfile.yaml") + ])) + ])) + + if ('deprecated' in item): + res.update([("displayName", "[Deprecated] " + displayNamePrefix + " " + item['name'] + template + " template"), + ("deprecated", item['deprecated'])]) + + list.append(res) + + jsonFile.write(json.dumps(list, indent=4, ensure_ascii=False).encode('utf8')) + print("Generated: " + inputFile.rsplit('.', 1)[0] + ".json") + except yaml.YAMLError as exc: + print(exc) + +if os.path.isdir(yaml_dir): + for file in os.listdir(yaml_dir): + if fnmatch.fnmatch(file, '*.yaml'): + inputFile = yaml_dir + "/" + file + generate_json() +else: + inputFile = yaml_dir + generate_json() \ No newline at end of file diff --git a/ci/ext/post_env.d/post_env.sh b/ci/ext/post_env.d/post_env.sh new file mode 100755 index 0000000..5c9b82f --- /dev/null +++ b/ci/ext/post_env.d/post_env.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e + +if [ -z $RELEASE_NAME ]; then + if [ -z $TRAVIS_TAG ]; then + if [ -f $base_dir/VERSION ]; then + export RELEASE_NAME="$(cat $base_dir/VERSION)" + fi + else + export RELEASE_NAME=$TRAVIS_TAG + fi +fi + +export COPYFILE_DISABLE=1 + +mkdir -p $HOME/.appsody/stacks/dev.local \ No newline at end of file diff --git a/ci/ext/post_env.d/utility_check.sh b/ci/ext/post_env.d/utility_check.sh new file mode 100755 index 0000000..a66f74b --- /dev/null +++ b/ci/ext/post_env.d/utility_check.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +exit_script=0 +python_exe="" + +if [ -z $USE_BUILDAH ] || [ "$USE_BUILDAH" != "true" ] +then + if ! docker --version > /dev/null 2>&1 + then + echo "Error: 'docker' command is not installed or not available on the path" + exit_script=1 + fi +fi + +if [ "$CODEWIND_INDEX" == "true" ] +then + if ! python3 --version > /dev/null 2>&1 + then + if ! python --version > /dev/null 2>&1 + then + echo "Error: 'python' command is not installed or not available on the path" + exit_script=1 + else + python_exe="python" + fi + else + python_exe="python3" + fi + + if [ $python_exe != "" ] + then + if ! ${python_exe} -c "import yaml" > /dev/null 2>&1 + then + echo "Error: Python 'yaml' package not installed" + exit_script=1 + fi + fi +fi + +if [ $exit_script != 0 ] +then + echo "Error: Some required dependencies are missing, exiting script" + exit 1 +fi diff --git a/ci/ext/post_package.d/create_index.sh b/ci/ext/post_package.d/create_index.sh new file mode 100755 index 0000000..1ad6991 --- /dev/null +++ b/ci/ext/post_package.d/create_index.sh @@ -0,0 +1,55 @@ +#!/bin/bash +set -e + +# setup environment +. $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../../env.sh + +# directory to store assets for test or release +assets_dir=$base_dir/ci/assets +mkdir -p $assets_dir +mkdir -p $base_dir/ci/build/index-src + +# url for downloading released assets +release_url="https://github.com/$TRAVIS_REPO_SLUG/releases/download" + +# iterate over each repo +# iterate over each stack +for stack in $(ls $base_dir/*/stack.yaml 2>/dev/null | sort) +do + echo $stack + stack_dir=$(dirname $stack) + + if [ -d $stack_dir ] + then + pushd $stack_dir + + stack_id=$(basename $stack_dir) + stack_version=$(awk '/^version *:/ { gsub("version:","",$NF); gsub("\"","",$NF); print $NF}' $stack) + stack_version_major=`echo $stack_version | cut -d. -f1` + stack_version_minor=`echo $stack_version | cut -d. -f2` + stack_version_patch=`echo $stack_version | cut -d. -f3` + + index_name=$stack_id-$stack_version-index + + index_file_v2=$assets_dir/$index_name.yaml + index_file_local_v2=$assets_dir/$index_name-local.yaml + index_file_v2_temp=$assets_dir/$index_name-temp.yaml + nginx_file=$base_dir/ci/build/index-src/$index_name.yaml + + if [ -f $index_file_v2 ]; then + # Copy index file as we will update later + cp $index_file_v2 $index_file_v2_temp + + # Resolve external URL for local / github release + sed -e "s|${RELEASE_URL}/.*/|file://$assets_dir/|" $index_file_v2_temp > $index_file_local_v2 + if [ "${BUILD_RELEASE}" == "true" ]; then + sed -e "s|${RELEASE_URL}/.*/|${RELEASE_URL}/${RELEASE_NAME}/|" $index_file_v2_temp > $index_file_v2 + fi + rm -f $base_dir/ci/build/index-src/*.yaml + sed -e "s|${RELEASE_URL}/.*/|{{EXTERNAL_URL}}/|" $index_file_v2_temp > $nginx_file + rm -f $index_file_v2_temp + fi + else + echo "SKIPPING: $repo_dir" + fi +done diff --git a/ci/ext/pre_env.d/pre_env.sh b/ci/ext/pre_env.d/pre_env.sh new file mode 100755 index 0000000..c0f5f54 --- /dev/null +++ b/ci/ext/pre_env.d/pre_env.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -e + +# Setup the environment variable needed to build Kabanero Collections +if [ -z "${BUILD_RELEASE}" ]; then + export BUILD_RELEASE=false +fi +if [ -z "${CODEWIND_INDEX}" ]; then + export CODEWIND_INDEX=true +fi +if [ -z "${DISPLAY_NAME_PREFIX}" ] +then + export DISPLAY_NAME_PREFIX="ICP4A" +fi +if [ -z "${IMAGE_REGISTRY_ORG}" ] +then + export IMAGE_REGISTRY_ORG="ICP4A" +fi +if [ -z "${LATEST_RELEASE}" ]; then + export LATEST_RELEASE=true +fi +if [ "$TRAVIS" == "true" ] +then + if [ $TRAVIS_TAG ] && [[ $TRAVIS_TAG =~ ^.*-(alpha|beta|rc)\.[0-9]* ]] + then + if [ -z "${BETA_REGISTRY_USERNAME}" ] || [ -z "${BETA_REGISTRY_PASSWORD}" ] || [ -z "${BETA_REGISTRY_ORG}" ] + then + echo "ERROR: Non-production release is being created but not all required BETA environment variables have been setup" + echo " Required variables are: BETA_REGISTRY_USERNAME, BETA_REGISTRY_PASSWORD and BETA_REGISTRY_ORG" + exit 1 + else + export IMAGE_REGISTRY_USERNAME="${BETA_REGISTRY_USERNAME}" + export IMAGE_REGISTRY_PASSWORD="${BETA_REGISTRY_PASSWORD}" + export IMAGE_REGISTRY_ORG="${BETA_REGISTRY_ORG}" + fi + fi +fi + diff --git a/ci/list_artifacts.sh b/ci/list_artifacts.sh new file mode 100755 index 0000000..b744d61 --- /dev/null +++ b/ci/list_artifacts.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# setup environment +. $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/env.sh + +# directory to store assets for test or release +assets_dir=$base_dir/ci/assets + +echo "Assets in /ci/assets are:" +echo "-------------------------------" +ls -al $assets_dir +echo "-------------------------------" + +for index in $(ls $assets_dir/*.yaml 2>/dev/null | sort) +do + echo "Content of index file $(basename $index) is:" + echo "-------------------------------" + cat $index + echo "-------------------------------" +done \ No newline at end of file diff --git a/ci/package.sh b/ci/package.sh index 6def4a2..874187f 100755 --- a/ci/package.sh +++ b/ci/package.sh @@ -130,3 +130,17 @@ done # expose an extension point for running after main 'package' processing exec_hooks $script_dir/ext/post_package.d + +if [ "$CODEWIND_INDEX" == "true" ]; then + python3 $script_dir/create_codewind_index.py -n $DISPLAY_NAME_PREFIX -f $assets_dir + + # iterate over each repo + for codewind_file in $assets_dir/*.json + do + # flat json used by static appsody-index for codewind + index_src=$build_dir/index-src/$(basename "$codewind_file") + + sed -e "s|${RELEASE_URL}/.*/|{{EXTERNAL_URL}}/|" $codewind_file > $index_src + done +fi +