Development

.github/workflows/deploy-docs.yml

@@ -0,0 +1,33 @@
+name: Docs Deployment - GitHub pages
+on:
+  workflow_dispatch: {}
+  push:
+    branches:
+      - development
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    permissions:
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16
+          cache: npm
+      - run: npm ci
+      - name: Build
+        run: npm run docs:build
+      - uses: actions/configure-pages@v2
+      - uses: actions/upload-pages-artifact@v1
+        with:
+          path: docs/.vitepress/dist
+      - name: Deploy
+        id: deployment
+        uses: actions/deploy-pages@v2.0.2

.github/workflows/build.yml → .github/workflows/dev-build.yml

@@ -1,4 +1,4 @@
-name : Deploy to Dev
+name : Setup, Build and Publish to Dev
 
 on:
   push:
@@ -11,11 +11,16 @@ env:
   DEPLOYMENT_NAME: nmrxiv-nmrxiv-app
   SCHEDULER_DEPLOYMENT_NAME: nmrxiv-scheduler
   WORKER_DEPLOYMENT_NAME: nmrxiv-worker
-  REPOSITORY_NAME: nmrxiv-dev
-  APP_IMAGE: nmrxiv-app
-  NGINX_IMAGE: nmrxiv-nginx
+  DOCKER_HUB_USERNAME : ${{ secrets.DOCKER_HUB_USERNAME  }}
+  DOCKER_HUB_PASSWORD : ${{ secrets.DOCKER_HUB_PASSWORD  }}
+  REPOSITORY_NAME: nmrxiv
+  REPOSITORY_NAMESPACE: nfdi4chem
+  DEV_URL: https://dev.nmrxiv.org
 
 jobs:
+  # php-unit-test:
+  #   uses: NFDI4Chem/nmrxiv/.github/workflows/test.yml@development
+
   php-unit-test:
     name: Run test
     runs-on: ubuntu-latest
@@ -43,7 +48,7 @@ jobs:
         composer install --ignore-platform-reqs
     - name: Prepare Laravel Application
       run: |
-        php -r "file_exists('.env') || copy('.env.ci.dev', '.env');"
+        php -r "file_exists('.env') || copy('.env.ci.test', '.env');"
 
         echo AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID_DEV }} >> .env
         echo AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY_DEV }} >> .env
@@ -63,12 +68,13 @@ jobs:
     - name: Run Test
       run: php artisan test --parallel
 
-
   setup-build-publish-deploy:
     name: Build & deploy to development
-    if: github.ref == 'refs/heads/development'
     runs-on: ubuntu-latest
     needs: php-unit-test
+    environment:
+      name: Dev
+      url: ${{ env.DEV_URL }}
     steps:
     - name: Checkout
       uses: actions/checkout@v2
@@ -79,40 +85,47 @@ jobs:
         service_account_key: ${{ secrets.GKE_SA_KEY }}
         project_id: ${{ secrets.GKE_PROJECT }}
 
-    # Configure docker to use the gcloud command-line tool as a credential helper
-    - run: |-
-        gcloud auth configure-docker europe-west3-docker.pkg.dev
-    # Get the GKE credentials so we can deploy to the cluster
+    # Get the GKE credentials
     - uses: google-github-actions/get-gke-credentials@v0.3.0
       with:
         cluster_name: ${{ env.GKE_CLUSTER }}
         location: ${{ env.GKE_ZONE }}
         credentials: ${{ secrets.GKE_SA_KEY }}
 
-    # Build the Docker image
-    - name: Build App Image
-      run: |-
-        docker build -f ./resources/ops/docker/app/app.dockerfile \
-          --tag "europe-west3-docker.pkg.dev/$PROJECT_ID/$REPOSITORY_NAME/$APP_IMAGE:latest" \
-          --build-arg GITHUB_SHA="$GITHUB_SHA" \
-          --build-arg GITHUB_REF="$GITHUB_REF" \
-          .
-    - name: Build Nginx Image
-      run: |-
-        docker build -f ./resources/ops/docker/nginx/nginx.dockerfile \
-          --tag "europe-west3-docker.pkg.dev/$PROJECT_ID/$REPOSITORY_NAME/$NGINX_IMAGE:latest" \
-          --build-arg GITHUB_SHA="$GITHUB_SHA" \
-          --build-arg GITHUB_REF="$GITHUB_REF" \
-          .
-    # Push the Docker image to Google Container Registry
-    - name: Publish App Image to GCR
-      run: |-
-        docker push "europe-west3-docker.pkg.dev/$PROJECT_ID/$REPOSITORY_NAME/$APP_IMAGE:latest"
+    # Login to Docker
+    - name: Log in to Docker Hub
+      uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
+      with:
+        username: ${{ env.DOCKER_HUB_USERNAME  }}
+        password: ${{ env.DOCKER_HUB_PASSWORD  }}
 
-    - name: Publish Nginx Image to GCR
-      run: |-
-        docker push "europe-west3-docker.pkg.dev/$PROJECT_ID/$REPOSITORY_NAME/$NGINX_IMAGE:latest"
+    # Build and push the app Docker image
+    - name: Build and push App Docker image
+      uses: docker/build-push-action@v4
+      with:
+        context: .
+        file: ./resources/ops/docker/app/app.dockerfile
+        push: true
+        build-args: |
+          RELEASE_VERSION=dev-app-latest
+        tags: ${{ env.REPOSITORY_NAMESPACE }}/${{ env.REPOSITORY_NAME }}:dev-app-latest
+        username: ${{ env.DOCKER_HUB_USERNAME  }}
+        password: ${{ env.DOCKER_HUB_PASSWORD  }}
+
+    # Build and push the nginx Docker image
+    - name: Build and push Nginx Docker image
+      uses: docker/build-push-action@v4
+      with:
+        context: .
+        file: ./resources/ops/docker/nginx/nginx.dockerfile
+        push: true
+        build-args: |
+          RELEASE_VERSION=dev-nginx-latest
+        tags: ${{ env.REPOSITORY_NAMESPACE }}/${{ env.REPOSITORY_NAME }}:dev-nginx-latest
+        username: ${{ env.DOCKER_HUB_USERNAME  }}
+        password: ${{ env.DOCKER_HUB_PASSWORD  }}
 
+    # Deploy the latest Docker image to the GKE cluster
     - name: Deploy
       run: |-
         kubectl rollout restart deployment/$DEPLOYMENT_NAME

.github/workflows/test.yml

@@ -0,0 +1,52 @@
+name : Test
+
+on:
+  workflow_call:
+
+jobs:
+  php-unit-test:
+    name: Run test
+    runs-on: ubuntu-latest
+    container:
+      image: kirschbaumdevelopment/laravel-test-runner:8.2
+
+    services:
+      postgres:
+        image: postgres:13
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: nmrxiv
+        ports:
+        - 5432:5432
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+
+    steps:
+    - uses: actions/checkout@v1
+      with:
+        fetch-depth: 1
+
+    - name: Install composer dependencies
+      run: |
+        composer install --ignore-platform-reqs
+    - name: Prepare Laravel Application
+      run: |
+        php -r "file_exists('.env') || copy('.env.ci.test', '.env');"
+
+        echo AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID_DEV }} >> .env
+        echo AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY_DEV }} >> .env
+        echo MEILISEARCH_KEY=${{ secrets.MEILISEARCH_KEY_DEV }} >> .env
+        echo MEILISEARCH_PUBLICKEY=${{ secrets.MEILISEARCH_PUBLICKEY_DEV }} >> .env
+        echo TWITTER_CLIENT_ID=${{ secrets.TWITTER_CLIENT_ID_DEV }} >> .env
+        echo TWITTER_CLIENT_SECRET=${{ secrets.TWITTER_CLIENT_SECRET_DEV }} >> .env
+        echo GITHUB_CLIENT_ID=${{ secrets.CLIENT_ID_GITHUB_DEV }} >> .env
+        echo GITHUB_CLIENT_SECRET=${{ secrets.CLIENT_SECRET_GITHUB_DEV }} >> .env
+
+        php artisan key:generate
+        php artisan migrate --seed
+    - name: Install front-end dependencies
+      run: |
+        npm install
+        npm run build
+    - name: Run Test
+      run: php artisan test --parallel

app/Console/Kernel.php

@@ -27,7 +27,9 @@ protected function schedule(Schedule $schedule)
         $schedule->command('nmrxiv:delete-projects')->daily();
         $schedule->command('nmrxiv:delete-citations')->weekly();
         $schedule->command('nmrxiv:delete-authors')->weekly();
-        $schedule->command('nmrxiv:backup-postgres-dump')->daily();
+        if(App::environment('production')){
+            $schedule->command('nmrxiv:backup-postgres-dump')->daily();
+        }
     }
 
     /**

app/Http/Controllers/DownloadController.php

@@ -20,6 +20,9 @@ public function downloadSet(Request $request, $username, $project, $study = null
         if (str_contains($dataset, '.zip')) {
             $dataset = str_replace('.zip', '', $dataset);
         }
+        if (str_contains($study, '.zip')) {
+            $study = str_replace('.zip', '', $study);
+        }
         $user = User::where('username', $username)->firstOrFail();
         if ($project) {
             $project = Project::where([['slug', $project], ['owner_id', $user->id]])->firstOrFail();

app/Http/Controllers/DraftController.php

@@ -96,6 +96,14 @@ public function files(Request $request, Draft $draft)
             ]);
     }
 
+    public function update(Request $request, Draft $draft)
+    {
+        $draft->name = $request->get('name');
+        $draft->save();
+
+        return $draft;
+    }
+
     public function deleteFSO(Request $request, Draft $draft, FileSystemObject $filesystemobject)
     {
         $fsoIds = $this->getChildrenIds($filesystemobject, []);
@@ -131,7 +139,7 @@ public function complete(Request $request, Draft $draft)
         // ProcessDraft::dispatch($draft);
 
         return response()->json([
-            'project' => Project::with(['studies.datasets', 'owner'])->where('draft_id', $draft->id)->first(),
+            'project' => Project::with(['studies.datasets', 'owner', 'citations', 'authors'])->where('draft_id', $draft->id)->first(),
             'validation' => $validation,
         ]);
     }
@@ -290,7 +298,7 @@ public function process(Request $request, Draft $draft)
                 $sChildren = $folder->children;
 
                 foreach ($sChildren as $sChild) {
-                    if ($sChild->instrument_type != null) {
+                    if ($sChild->instrument_type != null && $sChild->instrument_type != 'nmredata') {
                         // associate all children with the study_id, project_id, dataset_id
                         // create samples
                         // create assays
@@ -319,6 +327,7 @@ public function process(Request $request, Draft $draft)
                             $ds->save();
 
                             $sChild->dataset_id = $ds->id;
+                            $sChild->is_processed = true;
                             $sChild->save();
                         }
                     }
@@ -359,6 +368,7 @@ public function process(Request $request, Draft $draft)
                     $study->sample()->save($sample);
 
                     $folder->study_id = $study->id;
+                    $folder->is_processed = true;
                     $folder->save();
 
                     $ds = Dataset::where([
@@ -383,6 +393,7 @@ public function process(Request $request, Draft $draft)
                         ]);
 
                         $folder->dataset_id = $ds->id;
+                        $folder->is_processed = true;
                         $folder->save();
                     }
                 }
@@ -398,7 +409,7 @@ public function process(Request $request, Draft $draft)
             }
 
             return response()->json([
-                'project' => $project->load(['owner']),
+                'project' => $project->load(['owner', 'citations', 'authors']),
                 'studies' => $studies,
             ]);
         });
@@ -449,11 +460,24 @@ public function processFolder($folders)
                 } elseif ($this->isJcampDX($folder)) {
                     $this->saveInstrumentType($folder, 'jcamp');
                     $this->saveModelType($folder->parent);
+                } elseif ($this->isNMReData($folder)) {
+                    $this->saveInstrumentType($folder, 'nmredata');
+                    $this->saveAnnotationsDetected($folder->parent);
                 }
             }
         }
     }
 
+    public function saveAnnotationsDetected($folder)
+    {
+        $study = $folder->study;
+
+        if ($study) {
+            $study->has_nmredata = true;
+            $study->save();
+        }
+    }
+
     public function saveModelType($folder)
     {
         if ($folder) {
@@ -517,6 +541,23 @@ public function isJcampDX($folder)
         return false;
     }
 
+    public function isNMReData($folder)
+    {
+        $fileTypes = ['nmredata'];
+        $names = [$folder->name];
+        $extensions = array_map(fn ($s) => substr("$s", (strrpos($s, '.') + 1)), $names);
+        $isNMReData = false;
+        if (array_intersect($fileTypes, $extensions) == $fileTypes) {
+            $isNMReData = true;
+        }
+
+        if ($isNMReData) {
+            return true;
+        }
+
+        return false;
+    }
+
     public function isJOEL($folder)
     {
         $fileTypes = ['jdf'];