Merge pull request #156 from MOV-AI/dev

Release 1.3.0-1

.env

@@ -1,7 +1,8 @@
-STUDIO_WEB_PORT=5004
+STUDIO_HTTP_PORT=8080
+STUDIO_HTTPS_PORT=8083
 STUDIO_DB_PACKAGES=
 STUDIO_APP_PACKAGES="movai-launcher=2.2.2-2,movai-ide-ce=3.0.0-7"
-STUDIO_SPAWNER_PACKAGES="ros-noetic-movai-ce-demos=1.0.0-8,ros-noetic-movai-ports-and-messages-ce=1.0.0-1"
+STUDIO_SPAWNER_PACKAGES="flow-initiator=1.0.1.9,ros-noetic-movai-ce-demos=1.0.0-8,ros-noetic-movai-ports-and-messages-ce=1.0.0-1"
 STUDIO_SIMU_PLUGIN_PACKAGES="movai-ign-plugin-world-launcher=1.0.1-4"
 STUDIO_SIMU_GUI="WorldLauncher"
 MOVAI_DEVICE_NAME=robot1
@@ -11,9 +12,11 @@ JWT_SECRET_KEY=m0va1f10wl0v3s0p3n50rc3@2022
 FLEET_NAME=ce
 LOCAL_IP=0.0.0.0
 PUBLIC_IP=127.0.0.1
-MANAGER_URI=http://localhost:8080
 SPAWNER_GROUPS=
 EXTERNAL_HOST_IP=
 IGN_PARTITION="movai_ce_flow"
 XAUTH=/tmp/.docker.xauth
 DISPLAY=${DISPLAY:-:1}
+REDIS_LOCAL_PORT=6380
+REDIS_MASTER_PORT=6379
+ROS_IP=${MOVAI_FLOW_ROS_IP:-0.0.0.0}

.github/ISSUE_TEMPLATE.md

@@ -22,16 +22,6 @@ Have you checked our docs before proceeding?
 <!-- Include result of `docks ps | grep mov.ai` on your terminal. -->
 
 
-### Versions
-<!-- Result of `apt list movai-flow` command in your terminal -->
-- movai-flow:
-<!-- Check the settings button on the IDE -->
-- movai-ide:
-
-### Possible workarounds
-<!-- List steps for a possible work around if you found any -->
--
-
 ### Versions
 <!-- Result of `apt list movai-flow` command in your terminal -->
 - movai-flow:

.github/workflows/DeployOnMergeMain.yml

@@ -12,7 +12,7 @@ jobs:
   Deploy-App-Internal-Release:
     runs-on: ubuntu-20.04
     container:
-      image: registry.cloud.mov.ai/devops/ros-buildtools-noetic:main
+      image: registry.cloud.mov.ai/qa/ros-buildtools-noetic:v1.0.14
       options: --user root
       credentials:
         username: ${{secrets.PORTUS_APP_USER}}
@@ -42,6 +42,7 @@ jobs:
 
     - name: Pack debian
       run: |
+        git config --global --add safe.directory $(pwd)
         git config --global user.name ${{ secrets.RAISE_BOT_COMMIT_USER }}
         git config --global user.email ${{ secrets.RAISE_BOT_COMMIT_MAIL }}
 

.github/workflows/TestOnPR.yml

@@ -9,9 +9,9 @@ on:
       - 'debian/changelog'
 jobs:
 
-  Test-Project:
+  Lint-Project:
     timeout-minutes: 30
-    name: "Sanity checks"
+    name: "Linting checks"
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout
@@ -24,13 +24,20 @@ jobs:
     - name: Yamllint
       uses: karancode/yamllint-github-action@master
       with:
-        yamllint_file_or_dir: docker-compose.yml docker-compose-nvidia.yml
+        yamllint_file_or_dir: docker-compose.yml docker-compose-nvidia.yml docker-compose-host.yml docker-compose-nvidia-host.yml
         yamllint_strict: false
         yamllint_comment: true
         yamllint_config_filepath: .yamllint-config.yml
       env:
         GITHUB_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+    - name: run docker-compose lint
+      run: |
+        docker-compose -f docker-compose.yml config
+        docker-compose -f docker-compose-host.yml config
+        docker-compose -f docker-compose-nvidia.yml config
+        docker-compose -f docker-compose-nvidia-host.yml config
+
     # - name: Validate json
     #   uses: anyone-developer/anyone-validate-json@main
     #   with:
@@ -39,25 +46,87 @@ jobs:
     #     ignore-directories: '.git'
     #     read-path: 'robots'
 
-    - name: run docker-compose lint
+  Test-Project:
+    timeout-minutes: 30
+    name: "Cluster sanity checks"
+    runs-on: ubuntu-20.04
+    needs: [ Lint-Project ]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: install requirements
       run: |
-        docker-compose -f docker-compose.yml config
-        docker-compose -f docker-compose-nvidia.yml config
+        python3 -m pip install docker-compose
 
     - name: prepare docker-compose cluster
       run: |
         mkdir -p ./{logs,shared,userspace,models_database} || true
         sudo chown 1000:1000 ./{logs,shared,userspace,models_database} -R
 
     - name: run docker-compose cluster
-      uses: isbang/compose-action@v1.1.0
+      uses: isbang/compose-action@v1.2.0
       with:
         compose-file: './docker-compose.yml'
         down-flags: '--volumes'
 
     - name: test cluster frontend
       run: |
-        export token_url=http://localhost:8080/token-auth/
+        export base_url="http://localhost:8080"
+        export token_url="$base_url/token-auth/"
+        export authenticated_url="$base_url/api/v1/User/movai/?"
+
+        # retry for 1 minute
+        count=10
+        set +e
+        for i in $(seq $count); do
+          status_code=$(curl -k -s -o /dev/null --cookie-jar cookies --header 'Content-Type: application/json' -X POST --data '{"username":"movai","password":"movai","remember":true}' -w "%{http_code}" $token_url)
+          if [ $status_code == 200 ]; then
+            break
+          fi
+          echo "Waiting for MOV.AI Flow™ availability (ret:$status_code)"
+          sleep 10
+        done
+        [ $status_code == 200 ] || exit -1
+
+        err_status_code=$(curl "$authenticated_url" -k -s -o /dev/null -w "%{http_code}")
+        [ $err_status_code == 401 ] || exit -1
+
+        json_token=$(curl -k -s --cookie-jar cookies --header 'Content-Type: application/json' -X POST --data '{"username":"movai","password":"movai","remember":true}' $token_url)
+        token=$( jq -r ".access_token" <<<"$json_token" )
+        ok_status_code=$(curl "$authenticated_url" -k -s -o /dev/null -H "Authorization: Bearer $token" -w "%{http_code}")
+        [ $ok_status_code == 200 ] || exit -1
+
+  Test-Project-HostMode:
+    timeout-minutes: 30
+    name: "Host Mode Cluster checks"
+    runs-on: ubuntu-20.04
+    needs: [ Lint-Project ]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: install requirements
+      run: |
+        python3 -m pip install docker-compose
+
+    - name: prepare docker-compose cluster
+      run: |
+        mkdir -p ./{logs,shared,userspace,models_database} || true
+        sudo chown 1000:1000 ./{logs,shared,userspace,models_database} -R
+
+    - name: run docker-compose cluster in host mode
+      uses: isbang/compose-action@v1.2.0
+      with:
+        compose-file: './docker-compose-host.yml'
+        down-flags: '--volumes'
+
+    - name: test cluster frontend in host mode
+      run: |
+        export base_url="http://localhost:8080"
+        export token_url="$base_url/token-auth/"
+        export authenticated_url="$base_url/api/v1/User/movai/?"
+
         # retry for 1 minute
         count=10
         set +e
@@ -69,20 +138,23 @@ jobs:
           echo "Waiting for MOV.AI Flow™ availability (ret:$status_code)"
           sleep 10
         done
-        [ $status_code == 200 ] && exit 0
+        [ $status_code == 200 ] || exit -1
+
+        err_status_code=$(curl "$authenticated_url" -k -s -o /dev/null -w "%{http_code}")
+        [ $err_status_code == 401 ] || exit -1
 
-        export url=http://localhost:8080/api/v1/apps/mov-fe-app-ide-ce/
-        #export url=https://localhost:8443/api/v1/apps/mov-fe-app-ide-ce/
-        json=$( status_code=$(curl -k -s --cookie-jar cookies --header 'Content-Type: application/json' -X POST --data '{"username":"movai","password":"movai","remember":true}' -w "%{http_code}" $token_url)' )
-        token=$( jq -r ".access_token" <<<"$json" )
-        curl "$url" -k -X POST -H "X-Cookie:token=$token"
+        json_token=$(curl -k -s --cookie-jar cookies --header 'Content-Type: application/json' -X POST --data '{"username":"movai","password":"movai","remember":true}' $token_url)
+        token=$( jq -r ".access_token" <<<"$json_token" )
+        ok_status_code=$(curl "$authenticated_url" -k -s -o /dev/null -H "Authorization: Bearer $token" -w "%{http_code}")
+        [ $ok_status_code == 200 ] || exit -1
 
   Pack-Project:
     timeout-minutes: 30
     name: "Pack Checks"
     runs-on: ubuntu-20.04
+    needs: [ Lint-Project ]
     container:
-      image: registry.cloud.mov.ai/devops/ros-buildtools-noetic:main
+      image: registry.cloud.mov.ai/qa/ros-buildtools-noetic:v1.0.14
       options: --user root
       credentials:
         username: ${{secrets.PORTUS_APP_USER}}
@@ -111,6 +183,7 @@ jobs:
 
     - name: Pack debian
       run: |
+        git config --global --add safe.directory $(pwd)
         git config --global user.name '${{ secrets.RAISE_BOT_COMMIT_USER }}'
         git config --global user.email '${{ secrets.RAISE_BOT_COMMIT_MAIL }}'
 

.gitignore

@@ -11,3 +11,5 @@ debian/files
 debian/movai-flow.debhelper.log
 debian/movai-flow.substvars
 debian/.debhelper/generated/movai-flow/installed-by-dh_install
+*.code-workspace
+config/config.env

.yamllint-config.yml

@@ -3,7 +3,7 @@ extends: default
 rules:
   # 120 chars should be enough, but don't fail if a line is longer
   line-length:
-    max: 120
+    max: 132
     level: warning
 
   # accept both     key:
@@ -12,4 +12,4 @@ rules:
   # and             key:
   #                 - item
   indentation:
-    indent-sequences: whatever
+    indent-sequences: whatever

README.md

@@ -7,6 +7,7 @@
   <a href="https://github.com/MOV-AI/movai-flow/actions/workflows/DeployOnGitRelease.yml"><img alt="Official Release" src="https://github.com/MOV-AI/movai-flow/actions/workflows/DeployOnGitRelease.yml/badge.svg"></a>
   <a href="https://github.com/MOV-AI/movai-flow/actions/workflows/DeployOnMergeMain.yml"><img alt="Pre Release" src="https://github.com/MOV-AI/movai-flow/actions/workflows/DeployOnMergeMain.yml/badge.svg"></a>
   <a href="https://github.com/MOV-AI/movai-flow/actions/workflows/TestOnPR.yml"><img alt="PR Checks" src="https://github.com/MOV-AI/movai-flow/actions/workflows/TestOnPR.yml/badge.svg"></a>
+  <a href="https://sonarcloud.io/summary/new_code?id=MOV-AI_movai-flow"><img alt="Quality Gate Status" src="https://sonarcloud.io/api/project_badges/measure?project=MOV-AI_movai-flow&metric=alert_status"></a>
   <a href="https://twitter.com/MovAIRobots"><img alt="Twitter" src="https://img.shields.io/twitter/url/http/shields.io.svg?style=social"></a>
 
 </p>
@@ -137,14 +138,14 @@ Installing the MOV.AI Flow™ package will provide a few command line tools to c
 
 MOV.AI Flow™ initiates a set of services running as a *docker-compose* cluster, namely:
 
- - redis-master: master DB of the cluster
- - redis-local: local DB of the cluster
- - backend: web service application
- - ros-master: ROS core service
- - spawner: Flow orchestrator of ROS nodes
- - haproxy: web proxy
- - simulator: Ignition Fortress container with graphical capabilities enabled
- - ros-tools: rviz container with graphical capabilities enabled
+ - [redis-master](https://github.com/MOV-AI/containers-redis2): Master DB of the cluster
+ - [redis-local](https://github.com/MOV-AI/containers-redis2): Local DB of the cluster
+ - [backend](https://github.com/MOV-AI/backend): Web service application
+ - [ros-master](https://github.com/MOV-AI/containers-ros-master): ROS core service
+ - [spawner](https://github.com/MOV-AI/containers-spawner-base): Flow orchestrator of ROS and MOV.AI nodes
+ - [haproxy](https://github.com/docker-library/haproxy): Web proxy
+ - [simulator](https://github.com/MOV-AI/containers-ign-simulator): Ignition Fortress container with graphical capabilities enabled
+ - [ros-tools](https://github.com/MOV-AI/containers-ros-tools): RVIZ container with graphical capabilities enabled
 
 > Some directories in the containers are mounted, which means that their contents are synchronized between your computer (host) and the container.
 
@@ -265,3 +266,7 @@ The best way to do it is to:
 To stop and delete containers, delete volumes with database data and downloaded images, run:
 
     docker-compose down --volumes --rmi all
+
+## MOV.AI Developer Tools
+A python package build to facilitate development when using MOV.AI. The tool can be used to, but not limited to, expose the ros topics to the host as well as export/import MOV.AI metadata. Please follow [this link](https://github.com/MOV-AI/movai-developer-tools) for more detailed documentation.
+

debian/changelog

@@ -1,7 +1,9 @@
-movai-flow (1.2.0-7) UNRELEASED; urgency=medium
+movai-flow (1.3.0-0) UNRELEASED; urgency=medium
+
+  [ Alex Fernandes ]
+  * add host_mode feature
 
   [ Alex Fernandes ]
-  *
   * review doc and compose
   * remove cli build
   * add ci

debian/install

@@ -3,11 +3,14 @@ rviz usr/share/movai-flow
 models_database usr/share/movai-flow
 docker-compose.yml usr/share/movai-flow
 docker-compose-nvidia.yml usr/share/movai-flow
+docker-compose-host.yml usr/share/movai-flow
+docker-compose-nvidia-host.yml usr/share/movai-flow
 .env usr/share/movai-flow
 scripts/movai-flow-simulator usr/bin/
 scripts/movai-flow-rviz usr/bin/
 scripts/movai-flow-launch usr/bin/
 scripts/movai-flow-stop usr/bin/
+scripts/movai-flow-reset usr/bin/
 misc/movai-flow.desktop usr/share/applications
 misc/movai-flow-sim.desktop usr/share/applications
 misc/movai-flow-rviz.desktop usr/share/applications

debian/movai-flow.postinst

@@ -1,6 +1,14 @@
 #!/bin/bash
 # post install script
 
+SERVICE_NAME=movai-flow
+APP_PATH=/usr/share/$SERVICE_NAME
+CONFIG_PATH=/etc/$SERVICE_NAME
+DOCKER_COMPOSE_CONF=$APP_PATH/docker-compose.yml
+DOCKER_COMPOSE_CONF_MHOST=$APP_PATH/docker-compose-host.yml
+DOCKER_COMPOSE_ENV=$APP_PATH/.env
+DOCKER_COMPOSE_BIN=$(which docker-compose)
+
 # Function to output to log and console.
 #
 # Usage:
@@ -9,14 +17,6 @@
 #
 #   Note: Blank lines (Null text strings "") are not written to the system log.  There is no need to waste that space.
 
-SERVICE_NAME=movai-flow
-SERVICE_PORTS="8080" #8083
-APP_PATH=/usr/share/$SERVICE_NAME
-CONFIG_PATH=/etc/$SERVICE_NAME
-DOCKER_COMPOSE_CONF=$APP_PATH/docker-compose.yml
-DOCKER_COMPOSE_ENV=$APP_PATH/.env
-DOCKER_COMPOSE_BIN=$(which docker-compose)
-
 Output() {
   # Prepare base message
   Message="MOV.AI Flow™ install: $1"
@@ -76,8 +76,7 @@ fi
 if [ $HaveFreePorts -gt 0 ]; then
   Output "Needed ports are available"
 else
-  Output "Error: Needed ports are not available"
-  exit 1
+  Output "Warning: Needed ports are not available you might face networking issues when starting MOV.AI Flow™"
 fi
 
 if [ "$1" == "configure" ]; then
@@ -92,17 +91,24 @@ if [ "$1" == "configure" ]; then
       Output "Created $SERVICE_NAME fleet token"
     fi
 
-    Output "Setting Nvidia config"
+    Output "Setting NVIDIA config"
     if ! grep -q HAVE_NVIDIA $CONFIG_PATH/config.env ; then
       echo "HAVE_NVIDIA=$HaveNvidia" >> $CONFIG_PATH/config.env
     else
       sed -i -e "/^HAVE_NVIDIA=/s/=.*/=\'$HaveNvidia\'/" $CONFIG_PATH/config.env
     fi
 
-    mkdir -p $APP_PATH/{logs,shared,userspace,models_database}
-    sudo chown 1000:1000 $APP_PATH/{logs,shared,userspace,models_database} -R
+    Output "Setting empty network config"
+    echo "NETWORK_MODE=" > $CONFIG_PATH/config.env
+
+    mkdir -p $APP_PATH/{logs,shared,userspace,models_database,config}
+    chown 1000:1000 $APP_PATH/{logs,shared,userspace,models_database} -R
+    chmod 777 $APP_PATH/config -R
+
+    cp  $CONFIG_PATH/config.env $APP_PATH/config/
 
 elif [ "$1" == "abort-remove" ]; then
     Output "Stopping cluster of $SERVICE_NAME"
     $DOCKER_COMPOSE_BIN -f $DOCKER_COMPOSE_CONF --env-file $DOCKER_COMPOSE_ENV down || true
+    $DOCKER_COMPOSE_BIN -f $DOCKER_COMPOSE_CONF_MHOST --env-file $DOCKER_COMPOSE_ENV down || true
 fi

debian/movai-flow.postrm

@@ -41,7 +41,7 @@ if [ "$1" = "purge" ]; then
         $DOCKER_BIN volume rm $volumes 2>/dev/null
     fi
 
-    images=$($DOCKER_BIN images | awk '$1 ~ "mov.ai" { print $3 }')
+    images=$($DOCKER_BIN images | awk '$1 ~ "mov.ai/ce" { print $3 }')
     if [ -n "$images" ]
     then
         Output "removing docker images"