Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ci(Makefile): enhance Instill Core launch #377

Merged
merged 1 commit into from
Oct 25, 2023
Merged

ci(Makefile): enhance Instill Core launch #377

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .env
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,13 @@
# docker compose project name
COMPOSE_PROJECT_NAME=instill-vdp

# build from scratch or not at launch, which will build all sources from scrach. Default to false.
BUILD=false

# docker compose profiles to selectively launch components for developing the latest codebase of the specified component.
# the value can be all, api-gateway, mgmt, pipeline, connector, controller-vdp, model, controller-model, or console.
PROFILE=all

# system-wise config path (all core, vdp, and model projects must use the same path)
SYSTEM_CONFIG_PATH=$(HOME)/.config/instill

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/make-all.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ jobs:

- name: Launch Instill VDP (release)
run: |
make all EDITION=local-ce:test
make all BUILD=true EDITION=local-ce:test

- name: List all docker containers
run: |
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/make-latest.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ jobs:

- name: Launch Instill VDP (latest)
run: |
make latest PROFILE=all EDITION=local-ce:test
make latest BUILD=true PROFILE=all EDITION=local-ce:test

- name: List all docker containers
run: |
Expand Down
59 changes: 31 additions & 28 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,45 +20,46 @@ HELM_RELEASE_NAME := vdp

.PHONY: all
all: ## Launch all services with their up-to-date release version
@make build-release
@if ! (docker compose ls -q | grep -q "instill-core"); then \
@if [ "${BUILD}" = "true" ]; then make build-release; fi
@if ! docker compose ls -q | grep -q "instill-core"; then \
export TMP_CONFIG_DIR=$(shell mktemp -d) && \
export SYSTEM_CONFIG_PATH=$(shell eval echo ${SYSTEM_CONFIG_PATH}) && \
docker run --rm \
-v /var/run/docker.sock:/var/run/docker.sock \
-v $${TMP_CONFIG_DIR}:$${TMP_CONFIG_DIR} \
-v $${SYSTEM_CONFIG_PATH}:$${SYSTEM_CONFIG_PATH} \
-e BUILD=${BUILD} \
--name ${CONTAINER_COMPOSE_NAME}-release \
${CONTAINER_COMPOSE_IMAGE_NAME}:release /bin/sh -c " \
cp /instill-ai/core/.env $${TMP_CONFIG_DIR}/.env && \
cp /instill-ai/core/docker-compose.build.yml $${TMP_CONFIG_DIR}/docker-compose.build.yml && \
cp -r /instill-ai/core/configs/influxdb $${TMP_CONFIG_DIR} && \
/bin/sh -c 'cd /instill-ai/core && make all EDITION=$${EDITION:=local-ce} SYSTEM_CONFIG_PATH=$${SYSTEM_CONFIG_PATH} BUILD_CONFIG_DIR_PATH=$${TMP_CONFIG_DIR} OBSERVE_CONFIG_DIR_PATH=$${TMP_CONFIG_DIR}' && \
/bin/sh -c 'cd /instill-ai/core && make all BUILD=${BUILD} PROJECT=core EDITION=$${EDITION:=local-ce} SYSTEM_CONFIG_PATH=$${SYSTEM_CONFIG_PATH} BUILD_CONFIG_DIR_PATH=$${TMP_CONFIG_DIR} OBSERVE_CONFIG_DIR_PATH=$${TMP_CONFIG_DIR}' && \
rm -rf $${TMP_CONFIG_DIR}/* \
" && \
rm -rf $${TMP_CONFIG_DIR}; \
" && rm -rf $${TMP_CONFIG_DIR}; \
fi
@EDITION=$${EDITION:=local-ce} docker compose -f docker-compose.yml up -d --quiet-pull

.PHONY: latest
latest: ## Lunch all dependent services with their latest codebase
@make build-latest
@if ! (docker compose ls -q | grep -q "instill-core"); then \
@if [ "${BUILD}" = "true" ]; then make build-latest; fi
@if ! docker compose ls -q | grep -q "instill-core"; then \
export TMP_CONFIG_DIR=$(shell mktemp -d) && \
export SYSTEM_CONFIG_PATH=$(shell eval echo ${SYSTEM_CONFIG_PATH}) && \
docker run --rm \
-v /var/run/docker.sock:/var/run/docker.sock \
-v $${TMP_CONFIG_DIR}:$${TMP_CONFIG_DIR} \
-v $${SYSTEM_CONFIG_PATH}:$${SYSTEM_CONFIG_PATH} \
-e BUILD=${BUILD} \
-e PROFILE=${PROFILE} \
--name ${CONTAINER_COMPOSE_NAME}-latest \
${CONTAINER_COMPOSE_IMAGE_NAME}:latest /bin/sh -c " \
cp /instill-ai/core/.env $${TMP_CONFIG_DIR}/.env && \
cp /instill-ai/core/docker-compose.build.yml $${TMP_CONFIG_DIR}/docker-compose.build.yml && \
cp -r /instill-ai/core/configs/influxdb $${TMP_CONFIG_DIR} && \
/bin/sh -c 'cd /instill-ai/core && make latest PROFILE=all EDITION=$${EDITION:=local-ce:latest} BUILD_CONFIG_DIR_PATH=$${TMP_CONFIG_DIR} SYSTEM_CONFIG_PATH=$${SYSTEM_CONFIG_PATH} OBSERVE_CONFIG_DIR_PATH=$${TMP_CONFIG_DIR}'&& \
/bin/sh -c 'cd /instill-ai/core && make latest BUILD=${BUILD} PROJECT=core PROFILE=$${PROFILE} EDITION=$${EDITION:=local-ce:latest} BUILD_CONFIG_DIR_PATH=$${TMP_CONFIG_DIR} SYSTEM_CONFIG_PATH=$${SYSTEM_CONFIG_PATH} OBSERVE_CONFIG_DIR_PATH=$${TMP_CONFIG_DIR}'&& \
rm -rf $${TMP_CONFIG_DIR}/* \
" && \
rm -rf $${TMP_CONFIG_DIR}; \
" && rm -rf $${TMP_CONFIG_DIR}; \
fi
@COMPOSE_PROFILES=$(PROFILE) EDITION=$${EDITION:=local-ce:latest} docker compose -f docker-compose.yml -f docker-compose.latest.yml up -d --quiet-pull

Expand Down Expand Up @@ -101,22 +102,24 @@ down: ## Stop all services and remove all service containers and volumes
@docker rm -f ${CONTAINER_COMPOSE_NAME}-latest >/dev/null 2>&1
@docker rm -f ${CONTAINER_COMPOSE_NAME}-release >/dev/null 2>&1
@EDITION= docker compose down -v
@if docker compose ls -q | grep -q "instill-core"; then \
if docker image inspect ${CONTAINER_COMPOSE_IMAGE_NAME}:latest >/dev/null 2>&1; then \
docker run --rm \
-v /var/run/docker.sock:/var/run/docker.sock \
--name ${CONTAINER_COMPOSE_NAME} \
${CONTAINER_COMPOSE_IMAGE_NAME}:latest /bin/sh -c " \
/bin/sh -c 'cd /instill-ai/core && make down' \
"; \
elif docker image inspect ${CONTAINER_COMPOSE_IMAGE_NAME}:release >/dev/null 2>&1; then \
docker run --rm \
-v /var/run/docker.sock:/var/run/docker.sock \
--name ${CONTAINER_COMPOSE_NAME} \
${CONTAINER_COMPOSE_IMAGE_NAME}:release /bin/sh -c " \
/bin/sh -c 'cd /instill-ai/core && make down' \
"; \
fi \
@if [ "$$(docker image inspect ${CONTAINER_COMPOSE_IMAGE_NAME}:latest --format='yes')" = "yes" ]; then \
docker run --rm \
-v /var/run/docker.sock:/var/run/docker.sock \
--name ${CONTAINER_COMPOSE_NAME} \
${CONTAINER_COMPOSE_IMAGE_NAME}:latest /bin/sh -c " \
if [ \"$$( docker container inspect -f '{{.State.Status}}' core-dind 2>/dev/null)\" != \"running\" ]; then \
/bin/sh -c 'cd /instill-ai/core && make down'; \
fi \
"; \
elif [ "$$(docker image inspect ${CONTAINER_COMPOSE_IMAGE_NAME}:release --format='yes')" = "yes" ]; then \
docker run --rm \
-v /var/run/docker.sock:/var/run/docker.sock \
--name ${CONTAINER_COMPOSE_NAME} \
${CONTAINER_COMPOSE_IMAGE_NAME}:release /bin/sh -c " \
if [ \"$$( docker container inspect -f '{{.State.Status}}' core-dind 2>/dev/null)\" != \"running\" ]; then \
/bin/sh -c 'cd /instill-ai/core && make down'; \
fi \
"; \
fi

.PHONY: images
Expand Down Expand Up @@ -180,7 +183,7 @@ build-release: ## Build release images for all VDP components

.PHONY: integration-test-latest
integration-test-latest: ## Run integration test on the latest VDP
@make latest PROFILE=all EDITION=local-ce:test
@make latest BUILD=true PROFILE=all EDITION=local-ce:test
@docker run --rm \
--network instill-network \
--name ${CONTAINER_BACKEND_INTEGRATION_TEST_NAME}-latest \
Expand All @@ -193,7 +196,7 @@ integration-test-latest: ## Run integration test on the latest VDP

.PHONY: integration-test-release
integration-test-release: ## Run integration test on the release VDP
@make all EDITION=local-ce:test
@make all BUILD=true EDITION=local-ce:test
@docker run --rm \
--network instill-network \
--name ${CONTAINER_BACKEND_INTEGRATION_TEST_NAME}-release \
Expand Down
Loading