From 2baf49aefddc7fd13cf554b299a457842c453a4b Mon Sep 17 00:00:00 2001 From: Utpal Paul Date: Fri, 24 Nov 2023 00:27:34 +0600 Subject: [PATCH] Docker & CI/CD Pipeline Added --- .github/workflows/.gitkeep | 0 .github/workflows/main.yaml | 98 +++++++++++++++++++++++++ Dockerfile | 10 +++ README.md | 141 +++++++++++++++++++++++++++++++++++- 4 files changed, 248 insertions(+), 1 deletion(-) delete mode 100644 .github/workflows/.gitkeep create mode 100644 .github/workflows/main.yaml create mode 100644 Dockerfile diff --git a/.github/workflows/.gitkeep b/.github/workflows/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml new file mode 100644 index 0000000..933a244 --- /dev/null +++ b/.github/workflows/main.yaml @@ -0,0 +1,98 @@ +name: workflow + +on: + push: + branches: + - main + paths-ignore: + - 'README.md' + +permissions: + id-token: write + contents: read + +jobs: + integration: + name: Continuous Integration + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v3 + + - name: Lint code + run: echo "Linting repository" + + - name: Run unit tests + run: echo "Running unit tests" + + build-and-push-ecr-image: + name: Continuous Delivery + needs: integration + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v3 + + - name: Install Utilities + run: | + sudo apt-get update + sudo apt-get install -y jq unzip + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Build, tag, and push image to Amazon ECR + id: build-image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY_NAME }} + IMAGE_TAG: latest + run: | + # Build a docker container and + # push it to ECR so that it can + # be deployed to ECS. + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + echo "::set-output name=image::$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" + + + Continuous-Deployment: + needs: build-and-push-ecr-image + runs-on: self-hosted + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + + - name: Pull latest images + run: | + docker pull ${{secrets.AWS_ECR_LOGIN_URI}}/${{ secrets.ECR_REPOSITORY_NAME }}:latest + + # - name: Stop and remove container if running + # run: | + # docker ps -q --filter "name=waste-detection" | grep -q . && docker stop waste-detection && docker rm -fv waste-detection + + - name: Run Docker Image to serve users + run: | + docker run -d -p 8080:8080 --ipc="host" --name=waste-detection -e 'AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}' -e 'AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}' -e 'AWS_REGION=${{ secrets.AWS_REGION }}' ${{secrets.AWS_ECR_LOGIN_URI}}/${{ secrets.ECR_REPOSITORY_NAME }}:latest + - name: Clean previous images and containers + run: | + docker system prune -f \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..60d2ccf --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.10-slim-buster + +WORKDIR /app +COPY . /app + +RUN apt update -y && apt install awscli -y + +RUN apt-get update && apt-get install ffmpeg libsm6 libxext6 unzip -y && pip install -r requirements.txt + +CMD ["python3", "app.py"] \ No newline at end of file diff --git a/README.md b/README.md index 2f542de..056aef0 100644 --- a/README.md +++ b/README.md @@ -1 +1,140 @@ -# waste-detection-using-yoloV5 \ No newline at end of file +# Waste Detection using yoloV5 + +## Workflows + +1. Update config.yaml +2. Update secrets.yaml [Optional] +3. Update params.yaml +4. Update the entity +5. Update the configuration manager in src config +6. Update the components +7. Update the pipeline + + +## Dataset + +* Here, I have used [this](https://drive.google.com/file/d/1YBtZfdYtJ341DJ51JJFA8iOrHC4A9b22/view?usp=sharing) waste detection dataset. You can use your own dataset. Just place the URL of the dataset in `config/config.yaml/data_ingestion/source_URL` + +## Steps to run + +### STEP 00 : Clone the repository + +```bash +https://github.com/utpal108/waste-detection-using-yoloV5 +``` +### STEP 01 : Create a virtial environment after opening the repository + +Using Anaconda Virtual Environments + +```bash +conda create -n venv python=3.10 -y +conda activate venv +``` +Or for Linux operating system, you can use that + +```bash +python3.10 -m venv venv +source venv/bin/activate +``` + + +### STEP 02 : install the requirements +```bash +pip install -r requirements.txt +``` + +Finally, run the following command to run your application: +```bash +python app.py +``` + +### STEP 03 : run the application + +Now,open up your local host with a port like that on your web browser. +```bash +http://localhost:8080 +``` +### STEP 04 : train the model +Before predicting, you have to train your model with your own dataset. +```bash +http://localhost:8080/train +``` +After completing the training, you can now detect waste from any image or live video. + +To detect waste from any live video, you have to follow this URL +```bash +http://localhost:8080/live +``` + +## AWS-CICD-Deployment-with-Github-Actions + +### STEP 00 : Login to AWS console. + +### STEP 01 : Create IAM user for deployment + + #with specific access + + 1. EC2 access : It is virtual machine + + 2. ECR: Elastic Container registry to save your docker image in aws + + + #Description: About the deployment + + 1. Build docker image of the source code + + 2. Push your docker image to ECR + + 3. Launch Your EC2 + + 4. Pull Your image from ECR in EC2 + + 5. Lauch your docker image in EC2 + + #Policy: + + 1. AmazonEC2ContainerRegistryFullAccess + + 2. AmazonEC2FullAccess + + +### STEP 02 : Create ECR repo to store/save docker image + - Save the URI: 681776806933.dkr.ecr.us-east-2.amazonaws.com/waste-detection + + +### STEP 03 : Create EC2 machine (Ubuntu) + +### STEP 04 : Open EC2 and Install docker in EC2 Machine: + + + #optinal + + sudo apt-get update -y + + sudo apt-get upgrade + + #required + + curl -fsSL https://get.docker.com -o get-docker.sh + + sudo sh get-docker.sh + + sudo usermod -aG docker ubuntu + + newgrp docker + +### STEP 05 : Configure EC2 as self-hosted runner: + setting>actions>runner>new self hosted runner> choose os> then run command one by one + + +### STEP 06 : Setup github secrets: + + AWS_ACCESS_KEY_ID= + + AWS_SECRET_ACCESS_KEY= + + AWS_REGION = us-east-2 + + AWS_ECR_LOGIN_URI = demo>> 681776806933.dkr.ecr.us-east-2.amazonaws.com + + ECR_REPOSITORY_NAME = waste-detection \ No newline at end of file