From 8c38025819141d8d01154f67b1d3612a25f1ab5b Mon Sep 17 00:00:00 2001 From: harlee-x Date: Tue, 26 Mar 2024 16:08:51 +0800 Subject: [PATCH] Fix: add test env Fix: change test CI: update ci-build action workflow --- .github/workflows/ci-build.yml | 36 +++++++++---------- .github/workflows/unit-test.yml | 6 ++-- slim.Dockerfile => Dockerfile | 10 ++---- docker/bin/entrypoint.sh | 25 +++++++++++++ docker/python/pip-linktime.conf | 2 -- docker/python/pip.conf | 2 -- .../markdown_to_html/test_markdownToHtml.py | 3 ++ makefiles/build-image.mk | 4 +-- makefiles/const.mk | 2 +- 9 files changed, 54 insertions(+), 36 deletions(-) rename slim.Dockerfile => Dockerfile (78%) create mode 100755 docker/bin/entrypoint.sh delete mode 100644 docker/python/pip-linktime.conf delete mode 100644 docker/python/pip.conf diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 80b93d8..55230b9 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -4,17 +4,9 @@ name: CI-Build on: - push: - branches: - - main - - release-* - pull_request: - branches: - - main - - release-* - -env: - PYTHON_VERSION: "3.10" + release: + types: + - published jobs: build-docker-images: @@ -25,7 +17,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: $PYTHON_VERSION + python-version: "3.10" # Optional - x64 or x86 architecture, defaults to x64 architecture: 'x64' @@ -38,14 +30,22 @@ jobs: - name: Docker Login uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 with: - registry: ${{ secrets.DOCKER_REGISTRY }} username: ${{ secrets.REG_USER }} password: ${{ secrets.REG_PASSWD }} - + + - name: Get Version + id: pversion + run: | + if [[ "${{ github.ref_name }}" =~ ^[.\|v][0-9]{1,}.[0-9]{1,}[.\|-][0-9]{1,} ]];then + VERSION=${{ github.ref_name }} + else + VERSION="latest" + fi + echo "git_revision=$VERSION" >> $GITHUB_OUTPUT + - name: Build Images - run: make docker-build IMG_REGISTRY=${{ secrets.DOCKER_REGISTRY }} VERSION=${{ github.ref_name }} + id: build + run: make docker-build IMG_REGISTRY=${{ secrets.CONTAINER_REGISTRY }} VERSION=${{ steps.pversion.outputs.git_revision }} - name: Push Images - run: make docker-build IMG_REGISTRY=${{ secrets.DOCKER_REGISTRY }} VERSION=${{ github.ref_name }} - - + run: make docker-push IMG_REGISTRY=${{ secrets.CONTAINER_REGISTRY }} VERSION=${{ steps.pversion.outputs.git_revision }} diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index ee00876..174573b 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -3,8 +3,7 @@ name: Unit-Test on: push: branches: - - main - - release-* + - "*" workflow_dispatch: {} pull_request: branches: @@ -49,4 +48,5 @@ jobs: - name: Test with pytest run: | - pytest --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html \ No newline at end of file + export Test=true + pytest --cov --cov-report term --cov-report xml:coverage.xml \ No newline at end of file diff --git a/slim.Dockerfile b/Dockerfile similarity index 78% rename from slim.Dockerfile rename to Dockerfile index 1a17727..7aa4648 100644 --- a/slim.Dockerfile +++ b/Dockerfile @@ -8,13 +8,10 @@ ENV PATH="${VIRTUAL_ENV}/bin:$PATH" # Install Python dependencies COPY docker/python/requirements.txt . -COPY docker/python/pip-linktime.conf /tmp/pip.conf COPY docker/bin/entrypoint.sh . #COPY version.py . COPY kdp_catalog_manager kdp_catalog_manager/ -RUN mkdir -p /root/.pip \ - && mv /tmp/pip.conf /root/.pip/pip.conf \ - && ${VIRTUAL_ENV}/bin/pip3 install --no-cache-dir --upgrade pip setuptools Cython==3.0.8 \ +RUN ${VIRTUAL_ENV}/bin/pip3 install --no-cache-dir --upgrade pip setuptools Cython==3.0.8 \ && ${VIRTUAL_ENV}/bin/pip3 install --no-cache-dir --upgrade --force-reinstall -r requirements.txt \ && rm -f requirements.txt \ && cd kdp_catalog_manager && python setup.py build_ext --inplace \ @@ -32,9 +29,7 @@ ENV BDOS_USER_HOME=${BDOS_USER_HOME:-/home/${BDOS_USER}} WORKDIR ${RUNTIME_HOME} -RUN echo "deb http://mirrors.aliyun.com/debian/ buster main non-free contrib" > /etc/apt/sources.list \ - && sed -i "s@http://deb.debian.org@http://mirrors.aliyun.com@g" /etc/apt/sources.list \ - && apt-get update \ +RUN apt-get update \ && apt-get -y install sudo \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ @@ -54,5 +49,4 @@ COPY --chown=${BDOS_USER}:${BDOS_USER} --from=builder /workspace . COPY --chown=${BDOS_USER}:${BDOS_USER} --from=builder /opt/venv /opt/venv USER ${BDOS_USER} -ENV SERVER_MODE=${SERVER_MODE:-prod} CMD ["/bin/sh", "-c", "${RUNTIME_HOME}/entrypoint.sh"] \ No newline at end of file diff --git a/docker/bin/entrypoint.sh b/docker/bin/entrypoint.sh new file mode 100755 index 0000000..0ad362d --- /dev/null +++ b/docker/bin/entrypoint.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +declare RUNTIME_HOME=${RUNTIME_HOME:-"/opt/bdos/kdp/bdos-core"} +declare BDOS_USER=${BDOS_USER:-bdos} +declare CHOWN_HOST_PATH=${CHOWN_HOST_PATH:-"False"} +declare LOG_LEVEL=${LOG_LEVEL:-"info"} +declare WORKER_NUM=${WORKER_NUM:-4} +declare PARAMS_LIMIT=${PARAMS_LIMIT:-0} +declare MAX_REQUESTS=${MAX_REQUESTS:-100} +declare MAX_REQUESTS_JITTER=${MAX_REQUESTS:-100} + + +sudo chown ${BDOS_USER}:${BDOS_USER} ${RUNTIME_HOME} >/dev/null 2>&1 + +if [[ "${CHOWN_HOST_PATH}" == "True" ]];then + echo -e "### Change owner of $RUNTIME_HOME ###" + sudo chown -R ${BDOS_USER}:${BDOS_USER} ${RUNTIME_HOME}/kdp_catalog_manager ${RUNTIME_HOME}/logs ${RUNTIME_HOME} >/dev/null 2>&1 + echo -e "### Change owner return code: $? ###" +fi + + +echo -e "### Starting Server... ###" +cd ${RUNTIME_HOME} + +gunicorn -c kdp_catalog_manager/config/gunicorn.py kdp_catalog_manager.main:app --workers ${WORKER_NUM} --log-level ${LOG_LEVEL} --limit-request-line=${PARAMS_LIMIT} --max-requests ${MAX_REQUESTS} --max-requests-jitter ${MAX_REQUESTS_JITTER} --preload diff --git a/docker/python/pip-linktime.conf b/docker/python/pip-linktime.conf deleted file mode 100644 index 659f662..0000000 --- a/docker/python/pip-linktime.conf +++ /dev/null @@ -1,2 +0,0 @@ -[global] -index-url = https://nx.linktimecloud.com/repository/pypi-group/simple/ \ No newline at end of file diff --git a/docker/python/pip.conf b/docker/python/pip.conf deleted file mode 100644 index 407fb07..0000000 --- a/docker/python/pip.conf +++ /dev/null @@ -1,2 +0,0 @@ -[global] -index-url = https://pypi.douban.com/simple/ \ No newline at end of file diff --git a/kdp_catalog_manager/utils/markdown_to_html/test_markdownToHtml.py b/kdp_catalog_manager/utils/markdown_to_html/test_markdownToHtml.py index b5f9f7c..278d517 100644 --- a/kdp_catalog_manager/utils/markdown_to_html/test_markdownToHtml.py +++ b/kdp_catalog_manager/utils/markdown_to_html/test_markdownToHtml.py @@ -35,3 +35,6 @@ def setUp(self): def test_mutate_markdown_data(self): rt = MarkdownToHtlm().mutate_markdown_data(self.markdown_file) self.assertEqual(rt, '

desc

') + + def tearDown(self): + os.remove(self.markdown_file) diff --git a/makefiles/build-image.mk b/makefiles/build-image.mk index aa3d131..0920745 100644 --- a/makefiles/build-image.mk +++ b/makefiles/build-image.mk @@ -1,5 +1,5 @@ # Docker image info -IMG ?= kdp-catalog-manager:$(VERSION) +IMG ?= linktimecloud/kdp-catalog-manager:$(VERSION) IMG_REGISTRY ?= "" .PHONY: set-build-info @@ -12,7 +12,7 @@ docker-build: docker-build-image .PHONY: docker-build-image docker-build-image: - docker build --build-arg ARTIFACTS_SERVER=$(ARTIFACTS_SERVER) -t $(IMG_REGISTRY)/$(IMG) -f slim.Dockerfile . + docker build -t $(IMG_REGISTRY)/$(IMG) -f Dockerfile . .PHONY: docker-push docker-push: docker-push-image diff --git a/makefiles/const.mk b/makefiles/const.mk index 7f4fb92..3b3d911 100644 --- a/makefiles/const.mk +++ b/makefiles/const.mk @@ -13,7 +13,7 @@ GIT_REMOTE := origin GIT_BRANCH := $(shell git rev-parse --symbolic-full-name --verify --quiet --abbrev-ref HEAD) GIT_TAG := $(shell git describe --exact-match --tags --abbrev=0 2> /dev/null || echo untagged) GIT_TREE_STATE := $(shell if [[ -z "`git status --porcelain`" ]]; then echo "clean" ; else echo "dirty"; fi) -RELEASE_TAG := $(shell if [[ "$(GIT_TAG)" =~ ^[.\|v][0-9]{1,}.[0-9]{1,}[.\|-][0-9]{1,} ]]; then echo "true"; else echo "false"; fi) +RELEASE_TAG := $(shell if [[ "$(GIT_TAG)" =~ ^v[0-9]+\.[0-9]+\.[0-9]+.*$$ ]]; then echo "true"; else echo "false"; fi) VERSION := latest ifeq ($(RELEASE_TAG),true)