Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding parameters for recipe and GDK config custom file selection and new Push command for local deployment #119

Open
wants to merge 9 commits into
base: main
Choose a base branch
from
Open

Adding parameters for recipe and GDK config custom file selection and new Push command for local deployment #119

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
53f020c
Parametrizing GDK CLI to accept different recipies and configuration …
Jan 25, 2023
29c6419
Increasing minor version: 1.1.0 -> 1.2.0
Feb 15, 2023
5d7fcec
Adding new parameters for build and publish, adding new command push
Feb 15, 2023
bcedee0
Adding new command push
Feb 15, 2023
8ac68a9
Adding new logic for push command
Feb 15, 2023
4d97772
Fixing bug on -r parameter for build command
Feb 15, 2023
0039733
Fixing bug on extention check for recipe parameter
Feb 15, 2023
f2f1c2d
Updating REAMDE.md
Feb 15, 2023
60b1a86
Removing temporary code
iakko Feb 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,3 +88,17 @@ Configure AWS CLI with your credentials as shown here - https://docs.aws.amazon.
5. Creates new version of the component in your AWS account.

`gdk component publish`

#### 4. Local deployment for local tests

1. Build the component specifing also the gdk-config.json file, the recipe.json file and the build folder:

`gdk component build -d -c gdk-config.json -r recipe.json -u greengrass-build`

2. Push the component locally to a custom folder:

`gdk component push -u greengrass-build -e file:///tmp/gg`

1. One you have done local tests, publish component on cloud retrieving the built artifact from local folder:

`gdk component publish -d -c gdk-config.json -u /tmp/gg -n <component_name> -w <component_version>`.
2 changes: 1 addition & 1 deletion gdk/_version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "1.1.0"
__version__ = "1.2.0"
41 changes: 37 additions & 4 deletions gdk/commands/component/BuildCommand.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import json
import logging
import os
import platform
import shutil
import subprocess as sp
Expand All @@ -16,7 +17,28 @@
class BuildCommand(Command):
def __init__(self, command_args) -> None:
super().__init__(command_args, "build")
self.project_config = project_utils.get_project_config_values()

logging.debug("Command '{}' arguments: {}".format(self.name, command_args))

project_config_filename = command_args["gdk_config"] if command_args["gdk_config"] != None else consts.cli_project_config_file
project_build_directory = command_args["build_dir"] if command_args["build_dir"] != None else "{}/{}".format(utils.current_directory, consts.greengrass_build_dir)
project_recipe_filename = project_utils.find_recipe_file_in_path(command_args["recipe"] if command_args["recipe"] != None else "recipe", utils.current_directory)

logging.debug("Project config filename: {}".format(project_config_filename))
logging.debug("Project build directory: {}".format(project_build_directory))
logging.debug("Project recipe filename: {}".format(project_recipe_filename))

logging.debug("Loading project configuration values")
self.project_config = project_utils.get_project_config_values(project_config_filename, project_build_directory)

logging.debug("Loading recipe values")
recipe_values = project_utils.get_project_recipe_values(project_recipe_filename)

logging.debug("Merging recipe into project configuration value")
self.project_config.update(recipe_values)

logging.debug("Project configuration: {}".format(self.project_config))

self.supported_build_sytems = project_utils.get_supported_component_builds()

def run(self):
Expand Down Expand Up @@ -74,7 +96,7 @@ def create_gg_build_directories(self):
# Clean build directory if it exists already.
utils.clean_dir(self.project_config["gg_build_directory"])

logging.debug("Creating '{}' directory with artifacts and recipes.".format(consts.greengrass_build_dir))
logging.debug("Creating '{}' directory with artifacts and recipes.".format(self.project_config["gg_build_directory"]))
# Create build artifacts and recipe directories
Path.mkdir(self.project_config["gg_build_recipes_dir"], parents=True, exist_ok=True)
Path.mkdir(self.project_config["gg_build_component_artifacts_dir"], parents=True, exist_ok=True)
Expand Down Expand Up @@ -143,11 +165,12 @@ def run_build_command(self):
None
"""
try:
project_config_filename = self.project_config["project_config_filename"]
build_system = self.project_config["component_build_config"]["build_system"]
build_command = self.get_build_cmd_from_platform(build_system)
logging.warning(
f"This component is identified as using '{build_system}' build system. If this is incorrect, please exit and"
f" specify custom build command in the '{consts.cli_project_config_file}'."
f" specify custom build command in the '{project_config_filename}'."
)
if build_system == "zip":
logging.info("Zipping source code files of the component.")
Expand Down Expand Up @@ -222,6 +245,9 @@ def _ignore_files_during_zip(self, path, names):
consts.cli_project_config_file,
consts.greengrass_build_dir,
self.project_config["component_recipe_file"].name,
self.project_config["project_config_filename"],
self.project_recipe_filename,
self.project_config["gg_build_directory"].name,
"test*",
".*",
"node_modules",
Expand Down Expand Up @@ -408,7 +434,14 @@ def create_build_recipe_file(self):
)
)
parsed_component_recipe = self.project_config["parsed_component_recipe"]
component_recipe_file_name = self.project_config["component_recipe_file"].name

# Standardize output recipe filename in case of custom input
logging.debug("File name: {}".format(self.project_config["component_recipe_file"].name))
file_ext = os.path.splitext(self.project_config["component_recipe_file"].name)
logging.debug("File split: {}".format(file_ext))
component_recipe_file_name = "recipe{}".format(file_ext[1])
logging.debug("Final standardized recipe filename based on extention: {}".format(component_recipe_file_name))

parsed_component_recipe["ComponentName"] = self.project_config["component_name"]
parsed_component_recipe["ComponentVersion"] = self.project_config["component_version"]
parsed_component_recipe["ComponentPublisher"] = self.project_config["component_author"]
Expand Down
131 changes: 34 additions & 97 deletions gdk/commands/component/PublishCommand.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,44 @@
import yaml
from botocore.exceptions import ClientError
from gdk.commands.Command import Command
from gdk.common import consts


class PublishCommand(Command):
def __init__(self, command_args) -> None:
super().__init__(command_args, "publish")
logging.debug("Arguments: {}".format(command_args))

project_config_filename = command_args["gdk_config"] if command_args["gdk_config"] != None else consts.cli_project_config_file
project_build_directory = command_args["build_dir"] if command_args["build_dir"] != None else "{}/{}".format(utils.current_directory, consts.greengrass_build_dir)

component_name = command_args["component_name"]
component_version = command_args["component_version"]

logging.debug("Loading project configuration values")
self.project_config = project_utils.get_project_config_values(project_config_filename, project_build_directory, component_name, component_version)

if component_name is not None and component_version is not None:
self.project_config["component_name"] = component_name
self.project_config["component_version"] = component_version
project_recipe_filename = project_utils.find_recipe_file_in_path("{}-{}".format(component_name, component_version), "{}/recipes".format(project_build_directory))
elif component_name is None and component_version is None:
project_recipe_filename = project_utils.find_recipe_file_in_path("recipe", "{}/recipes".format(project_build_directory))
else:
raise RuntimeError("Component name and version must be provided together")

logging.debug("Project config filename : {}".format(project_config_filename))
logging.debug("Project build directory : {}".format(project_build_directory))
logging.debug("Project recipe filename : {}".format(project_recipe_filename))

logging.debug("Loading recipe values")
recipe_values = project_utils.get_project_recipe_values(project_recipe_filename)

logging.debug("Merging recipe into project configuration value")
self.project_config.update(recipe_values)

logging.debug("Project configuration: {}".format(self.project_config))

self.project_config = project_utils.get_project_config_values()
self.service_clients = project_utils.get_service_clients(self.project_config["region"])

def run(self):
Expand All @@ -41,7 +72,7 @@ def run(self):
self.upload_artifacts_s3(component_name, component_version)

logging.info(f"Updating the component recipe {component_name}-{component_version}.")
self.update_and_create_recipe_file(component_name, component_version)
project_utils.update_and_create_recipe_file(self.project_config, component_name, component_version)

logging.info(f"Creating a new greengrass component {component_name}-{component_version}")
self.create_gg_component(component_name, component_version)
Expand Down Expand Up @@ -286,98 +317,4 @@ def get_component_version_from_config(self):
logging.error(
"Failed to calculate the version of component '{}' based on the configuration.".format(component_name)
)
raise (e)

def update_and_create_recipe_file(self, component_name, component_version):
"""
Updates recipe with the component version calculated and artifact URIs of the artifacts. This updated recipe is
used to create a new publish recipe file in build recipes directory.

Parameters
----------
component_name(string): Name of the component. This is also used in the name of the recipe file.
component_version(string): Version of the component calculated based on the configuration.

Returns
-------
None
"""
logging.debug("Updating artifact URIs in the recipe...")
build_recipe = Path(self.project_config["gg_build_recipes_dir"]).joinpath(
self.project_config["component_recipe_file"].name
)
parsed_component_recipe = project_utils.parse_recipe_file(build_recipe)
if "ComponentName" in parsed_component_recipe:
if parsed_component_recipe["ComponentName"] != component_name:
logging.error("Component '{}' is not build.".format(parsed_component_recipe["ComponentName"]))
raise Exception(
"Failed to publish the component '{}' as it is not build.\nBuild the component `gdk component"
" build` before publishing it.".format(parsed_component_recipe["ComponentName"])
)
gg_build_component_artifacts = self.project_config["gg_build_component_artifacts_dir"]
bucket = self.project_config["bucket"]
artifact_uri = f"{utils.s3_prefix}{bucket}/{component_name}/{component_version}"

if "Manifests" not in parsed_component_recipe:
logging.debug("No 'Manifests' key in the recipe.")
return
for manifest in parsed_component_recipe["Manifests"]:
if "Artifacts" not in manifest:
logging.debug("No 'Artifacts' key in the recipe manifest.")
continue
for artifact in manifest["Artifacts"]:
if "URI" not in artifact:
logging.debug("No 'URI' found in the recipe artifacts.")
continue
# Skip non-s3 URIs in the recipe. Eg docker URIs
if not artifact["URI"].startswith("s3://"):
continue
artifact_file = Path(artifact["URI"]).name
# For artifact in build component artifacts folder, update its URI
build_artifact_files = list(gg_build_component_artifacts.glob(artifact_file))
if len(build_artifact_files) == 1:
logging.debug("Updating artifact URI of '{}' in the recipe file.".format(artifact_file))
artifact["URI"] = f"{artifact_uri}/{artifact_file}"
else:
raise Exception(
f"Could not find the artifact file specified in the recipe '{artifact_file}' inside the build folder"
f" '{gg_build_component_artifacts}'."
)

# Update the version of the component in the recipe
parsed_component_recipe["ComponentVersion"] = component_version
self.create_publish_recipe_file(component_name, component_version, parsed_component_recipe)

def create_publish_recipe_file(self, component_name, component_version, parsed_component_recipe):
"""
Creates a new recipe file(json or yaml) with anme `<component_name>-<component_version>.extension` in the component
recipes build directory.

This recipe is updated with the component version calculated and artifact URIs of the artifacts.

Parameters
----------
component_name(string): Name of the component. This is also used in the name of the recipe file.
component_version(string): Version of the component calculated based on the configuration.
parsed_component_recipe(dict): Updated publish recipe with component version and s3 artifact uris
Returns
-------
None
"""
ext = self.project_config["component_recipe_file"].name.split(".")[-1] # json or yaml
publish_recipe_file_name = f"{component_name}-{component_version}.{ext}" # Eg. HelloWorld-1.0.0.yaml
publish_recipe_file = Path(self.project_config["gg_build_recipes_dir"]).joinpath(publish_recipe_file_name).resolve()
self.project_config["publish_recipe_file"] = publish_recipe_file
with open(publish_recipe_file, "w") as prf:
try:
logging.debug(
"Creating component recipe '{}' in '{}'.".format(
publish_recipe_file_name, self.project_config["gg_build_recipes_dir"]
)
)
if publish_recipe_file_name.endswith(".json"):
prf.write(json.dumps(parsed_component_recipe, indent=4))
else:
yaml.dump(parsed_component_recipe, prf)
except Exception as e:
raise Exception("""Failed to create publish recipe file at '{}'.\n{}""".format(publish_recipe_file, e))
raise (e)
Loading