Fixed unexpected deletion of log files of other processes that led to…

… OSError: [Errno 116] Stale file handle error on NFS volumes (cvat-ai#8121) Due the nocleanup option defaults to false and we use AUTO-generated names for backend process log files, this causes NFS4ERR_STALE to appear on NFS-mounted volumes ```python Traceback (most recent call last): File "/usr/lib/python3/dist-packages/supervisor/loggers.py", line 109, in emit self.flush() File "/usr/lib/python3/dist-packages/supervisor/loggers.py", line 68, in flush self.stream.flush() OSError: [Errno 116] Stale file handle ``` http://supervisord.org/configuration.html#:~:text=Introduced%3A%203.0-,nocleanup,-Prevent%20supervisord%20from https://stackoverflow.com/questions/40262823/stale-file-handle-error-when-process-trying-read-the-file-that-other-process Co-authored-by: Roman Donchenko <roman@cvat.ai>
midas-research · Jul 9, 2024 · ad1bcd5 · ad1bcd5
1 parent a68fe01
commit ad1bcd5
Show file tree

Hide file tree

Showing 9 changed files with 12 additions and 9 deletions.
diff --git a/changelog.d/20240708_103153_andrey_fix_logger_error.md b/changelog.d/20240708_103153_andrey_fix_logger_error.md
@@ -0,0 +1,11 @@
+### Fixed
+
+- Fixed unexpected deletion of log files of other processes that led to OSError:
+  \[Errno 116\] Stale file handle error on NFS volumes
+  (<https://github.com/cvat-ai/cvat/pull/8121>)
+
+### Changed
+
+- Log files for individual backend processes are now stored in ephemeral
+  storage of each backend container rather than in the `cvat_logs` volume
+  (<https://github.com/cvat-ai/cvat/pull/8121>)
diff --git a/supervisord/server.conf b/supervisord/server.conf
@@ -15,7 +15,6 @@ logfile_maxbytes=50MB       ; maximum size of logfile before rotation
 logfile_backups=10          ; number of backed up logfiles
 loglevel=debug              ; info, debug, warn, trace
 pidfile=/tmp/supervisord/supervisord.pid ; pidfile location
-childlogdir=%(ENV_HOME)s/logs/            ; where child log files will live
 
 [program:clamav_update]
 startsecs=0

diff --git a/supervisord/utils.conf b/supervisord/utils.conf
@@ -15,7 +15,6 @@ logfile_maxbytes=50MB       ; maximum size of logfile before rotation
 logfile_backups=10          ; number of backed up logfiles
 loglevel=debug              ; info, debug, warn, trace
 pidfile=/tmp/supervisord/supervisord.pid ; pidfile location
-childlogdir=%(ENV_HOME)s/logs/            ; where child log files will live
 
 [program:rqscheduler]
 command=%(ENV_HOME)s/wait_for_deps.sh
@@ -32,6 +31,7 @@ command=%(ENV_HOME)s/wait_for_deps.sh
         --worker-class cvat.rqworker.DefaultWorker
 environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler",CVAT_POSTGRES_APPLICATION_NAME="cvat:worker:notifications"
 numprocs=1
+autorestart=true
 
 [program:rqworker-cleaning]
 command=%(ENV_HOME)s/wait_for_deps.sh

diff --git a/supervisord/worker.analytics_reports.conf b/supervisord/worker.analytics_reports.conf
@@ -15,7 +15,6 @@ logfile_maxbytes=50MB       ; maximum size of logfile before rotation
 logfile_backups=10          ; number of backed up logfiles
 loglevel=debug              ; info, debug, warn, trace
 pidfile=/tmp/supervisord/supervisord.pid ; pidfile location
-childlogdir=%(ENV_HOME)s/logs/            ; where child log files will live
 
 [program:rqworker-analytics-reports]
 command=%(ENV_HOME)s/wait_for_deps.sh

diff --git a/supervisord/worker.annotation.conf b/supervisord/worker.annotation.conf
@@ -15,7 +15,6 @@ logfile_maxbytes=50MB       ; maximum size of logfile before rotation
 logfile_backups=10          ; number of backed up logfiles
 loglevel=debug              ; info, debug, warn, trace
 pidfile=/tmp/supervisord/supervisord.pid ; pidfile location
-childlogdir=%(ENV_HOME)s/logs/            ; where child log files will live
 
 [program:rqworker-annotation]
 command=%(ENV_HOME)s/wait_for_deps.sh

diff --git a/supervisord/worker.export.conf b/supervisord/worker.export.conf
@@ -15,7 +15,6 @@ logfile_maxbytes=50MB       ; maximum size of logfile before rotation
 logfile_backups=10          ; number of backed up logfiles
 loglevel=debug              ; info, debug, warn, trace
 pidfile=/tmp/supervisord/supervisord.pid ; pidfile location
-childlogdir=%(ENV_HOME)s/logs/            ; where child log files will live
 
 [program:rqworker-export]
 command=%(ENV_HOME)s/wait_for_deps.sh

diff --git a/supervisord/worker.import.conf b/supervisord/worker.import.conf
@@ -15,7 +15,6 @@ logfile_maxbytes=50MB       ; maximum size of logfile before rotation
 logfile_backups=10          ; number of backed up logfiles
 loglevel=debug              ; info, debug, warn, trace
 pidfile=/tmp/supervisord/supervisord.pid ; pidfile location
-childlogdir=%(ENV_HOME)s/logs/            ; where child log files will live
 
 [program:rqworker-import]
 command=%(ENV_HOME)s/wait_for_deps.sh
@@ -26,7 +25,6 @@ numprocs=%(ENV_NUMPROCS)s
 process_name=%(program_name)s-%(process_num)d
 autorestart=true
 
-
 [program:clamav-update]
 command=bash -c "if [ \"${CLAM_AV}\" = 'yes' ]; then /usr/bin/freshclam -d \
     -l %(ENV_HOME)s/logs/freshclam.log --foreground=true; fi"

diff --git a/supervisord/worker.quality_reports.conf b/supervisord/worker.quality_reports.conf
@@ -15,7 +15,6 @@ logfile_maxbytes=50MB       ; maximum size of logfile before rotation
 logfile_backups=10          ; number of backed up logfiles
 loglevel=debug              ; info, debug, warn, trace
 pidfile=/tmp/supervisord/supervisord.pid ; pidfile location
-childlogdir=%(ENV_HOME)s/logs/            ; where child log files will live
 
 [program:rqworker-quality-reports]
 command=%(ENV_HOME)s/wait_for_deps.sh

diff --git a/supervisord/worker.webhooks.conf b/supervisord/worker.webhooks.conf
@@ -15,7 +15,6 @@ logfile_maxbytes=50MB       ; maximum size of logfile before rotation
 logfile_backups=10          ; number of backed up logfiles
 loglevel=debug              ; info, debug, warn, trace
 pidfile=/tmp/supervisord/supervisord.pid ; pidfile location
-childlogdir=%(ENV_HOME)s/logs/            ; where child log files will live
 
 [program:rqworker-webhooks]
 command=%(ENV_HOME)s/wait_for_deps.sh