YDA-5840 Add monitor thread Yoda portal

Add monitoring thread to the Yoda portal for collecting technical
support information in case of problems.

docker/images/yoda_portal/yoda_portal_init.sh

@@ -198,6 +198,10 @@ UPLOAD_PART_FILES              = True
 
 # Text file extensions configuration
 TEXT_FILE_EXTENSIONS = ['bash', 'csv', 'c', 'cpp', 'csharp', 'css', 'diff', 'fortran', 'gams', 'gauss', 'go', 'graphql', 'ini', 'irpf90', 'java', 'js', 'json', 'julia', 'julia-repl', 'kotlin', 'less', 'lua', 'makefile', 'markdown', 'md', 'mathematica', 'matlab', 'maxima', 'mizar', 'objectivec', 'openscad', 'perl', 'php', 'php-template', 'plaintext', 'txt', 'python', 'py', 'python-repl', 'r', 'ruby', 'rust', 'sas', 'scilab', 'scss', 'shell', 'sh', 'sql', 'stan', 'stata', 'swift', 'typescript', 'ts', 'vbnet', 'wasm', 'xml', 'yaml', 'html']
+
+# Monitor thread configuration
+MONITOR_OUTPUT_DIR = "/tmp"
+MONITOR_SIGNAL_FILE = "/var/www/yoda/show-tech.sig"
 FLASKCFG
 progress_update "Portal configured"
 

docs/administration/configuring-yoda.md

@@ -159,6 +159,8 @@ yoda_theme_path                   | Path where themes for the Yoda Portal are re
 portal_session_cookie_samesite    | Samesite setting for session cookies Yoda Portal. Should be 'Lax' if OIDC is enabled and identity provider is in different domain. Otherwise it should be 'Strict'. Default value: 'Strict'.
 yoda_portal_wsgi_daemon_threads   | The number of threads to be created to handle requests in each daemon process (default: 15)
 yoda_portal_upload_part_files     | Whether the portal uploader function should upload multi-chunk files as .part files initially and rename them to their final name later (boolean value, default: true). It is generally recommended to keep this enabled, so that users can easily see when an upload failed and the result is partial. However, on storage systems where renaming data objects takes much time, such as S3 object storage in consistent mode, it may be necessary to switch use of .part files off.
+yoda_portal_monitor_signal_file   | Path to the signal file for the portal monitoring thread. If this file is present, the monitor thread will start logging technical support information to the directory configured in `yoda_portal_monitor_output_dir`. Set to empty string to disable the monitoring thread. Default value: /var/www/yoda/show-tech.sig
+yoda_portal_monitor_output_dir    | Path for the portal monitoring thread to log technical support information to if the signal file is present. Default value: /tmp. Note that the portal uses a private /tmp directory, rather than the global one.
 
 ### Generic logging configuration
 

docs/administration/index.md

@@ -34,6 +34,7 @@ Specific information:
 
 ## Troubleshooting
 
+- [Collecting portal technical support information](portal-tech-support-info.md)
 - [Restoring a collection](restore-collection.md)
 - [Troubleshooting email](troubleshooting-email.md)
 - [Troubleshooting data package archival in the vault](troubleshooting-copy-to-vault.md)

docs/administration/portal-tech-support-info.md

@@ -0,0 +1,81 @@
+---
+parent: Administration Tasks
+title: Collecting portal technical support information
+nav_order: 17
+---
+# Collecting portal technical support information
+
+In case there is a problem with the Yoda portal and the application logs don't provide enough
+information about what is going wrong, you may need to collect additional information. The portal
+has a monitoring thread to collect information for technical support purposes.
+
+You can start information collection by creating a signal file for the monitoring thread. By default,
+this is `/var/www/yoda/show-tech.sig`:
+
+```bash
+$ sudo touch /var/www/yoda/show-tech.sig
+```
+
+When the signal file is present, the portal monitoring thread will start logging technical support information.
+By default, the support information files are written to the Apache private `tmp` directory within the
+global `/tmp` directory. These information files contain stack traces for each portal thread, as well
+as some other information about the system and the portal.
+
+Remove the signal file after you have collected enough information:
+
+```bash
+$ sudo rm /var/www/yoda/show-tech.sig
+```
+
+## Example output
+
+```
+Portal tech support info for combined.yoda.test, collected on 30/07/2024 at 16:42:46.926656
+Yoda version, as per portal config: development (ae4de73e4fe931423caf27f7537a57b72a606764)
+
+System-wide CPU percent:         2.5%
+Memory: global total:            4.1 GB
+Memory: global available:        2.0 GB
+Memory: global buffers:          136.7 MB
+Memory: global cached:           674.2 MB
+Memory: process RSS:             55.9 MB
+Memory: process VMS:             740.7 MB
+Memory: process shared:          17.3 MB
+
+Thread ID: 140688039364352
+  /usr/lib/python3.8/threading.py:890 [_bootstrap]
+     self._bootstrap_inner()
+  /usr/lib/python3.8/threading.py:932 [_bootstrap_inner]
+     self.run()
+  /usr/lib/python3.8/threading.py:1254 [run]
+     self.function(*self.args, **self.kwargs)
+  /var/www/yoda/monitor.py:47 [record_info_if_needed]
+     tshoot_info: StringIO = self.get_tshoot_info()
+  /var/www/yoda/monitor.py:80 [get_tshoot_info]
+     for filename, line_number, function_name, line in traceback.extract_stack(stack):
+
+Thread ID: 140688058500864
+  /usr/lib/python3.8/threading.py:890 [_bootstrap]
+     self._bootstrap_inner()
+  /usr/lib/python3.8/threading.py:932 [_bootstrap_inner]
+     self.run()
+  /usr/lib/python3.8/threading.py:870 [run]
+     self._target(*self._args, **self._kwargs)
+  /var/www/yoda/research/research.py:48 [irods_writer]
+     chunk = q.get()
+  /usr/lib/python3.8/queue.py:170 [get]
+     self.not_empty.wait()
+  /usr/lib/python3.8/threading.py:302 [wait]
+     waiter.acquire()
+
+Thread ID: 140688071923456
+  /usr/lib/python3.8/threading.py:890 [_bootstrap]
+     self._bootstrap_inner()
+  /usr/lib/python3.8/threading.py:932 [_bootstrap_inner]
+     self.run()
+  /usr/lib/python3.8/threading.py:870 [run]
+     self._target(*self._args, **self._kwargs)
+  /var/www/yoda/connman.py:50 [gc]
+     time.sleep(1)
+```
+

roles/apache/templates/usr.sbin.apache2.j2

@@ -32,7 +32,7 @@
   /etc/gss/mech.d/ r,
   /etc/magic r,
   /etc/mime.types r,
-  /proc/*/fd/ r,
+  /proc/** r,
   /run/apache2/wsgi.*.sock rw,
   /usr/bin/dash ix,
   /usr/bin/uname mrix,

roles/yoda_portal/defaults/main.yml

@@ -85,3 +85,7 @@ token_lifetime: 72                                 # Lifetime of data access tok
 
 # SRAM Configuration
 enable_sram: false                                 # Enable SRAM configuration
+
+# Monitor thread configuration for extracting tech support information
+yoda_portal_monitor_output_dir: /tmp
+yoda_portal_monitor_signal_file: /var/www/yoda/show-tech.sig

roles/yoda_portal/templates/flask.cfg.j2

@@ -113,3 +113,7 @@ UPLOAD_PART_FILES              = {{ yoda_portal_upload_part_files }}
 
 # Text file extensions configuration
 TEXT_FILE_EXTENSIONS = {{ text_file_extensions | default(['txt'])}}
+
+# Monitor thread configuration
+MONITOR_OUTPUT_DIR = "{{ yoda_portal_monitor_output_dir }}"
+MONITOR_SIGNAL_FILE = "{{ yoda_portal_monitor_signal_file }}"