Merge branch 'dev128' into index128

bin/dqm-access

@@ -3,8 +3,12 @@
 from Monitoring.Core.HTTP import RequestManager
 from Monitoring.Core.X509 import SSLOptions
 from optparse import OptionParser
-from time import strptime, time
-import sys, re, json, pycurl, urllib
+from time import time
+import sys
+import re
+import json
+import pycurl
+from urllib import parse
 
 DIR = 0
 FILE = 1
@@ -179,7 +183,7 @@ def fetch_tstreamerinfo(options, dataset):
 # -------------------------------------------------------------------------------
 def request_init(c, options, sample, path, filterspec):
     sample.update(path=path)
-    c.url = options.server + urllib.quote(url_content % sample)
+    c.url = options.server + parse.quote(url_content % sample)
     if options.fetch_root:
         c.url += "?rootcontent=1"
     c.setopt(pycurl.URL, c.url)

bin/dqm-grep

@@ -51,7 +51,11 @@ key rather than a proxy, it will prompt for the key password.
 
 from Monitoring.Core.HTTP import RequestManager
 from Monitoring.Core.X509 import SSLOptions
-import sys, re, pycurl, urllib, json
+import sys
+import re
+import pycurl
+from urllib import parse
+import json
 from optparse import OptionParser
 from time import time
 
@@ -208,7 +212,7 @@ def find_matching_samples(options):
 def request_init(c, options, sample, filters, pos, path):
     """`RequestManager` callback to initialise JSON contents request."""
     sample.update(path=path)
-    c.url = options.server + urllib.quote(url_content % sample)
+    c.url = options.server + parse.quote(url_content % sample)
     c.setopt(pycurl.URL, c.url)
     if False and options.verbose:
         print(c.url)

bin/dqm-ls

@@ -17,7 +17,9 @@ key rather than a proxy, it will prompt for the key password.
 
 from Monitoring.Core.HTTP import RequestManager
 from Monitoring.Core.X509 import SSLOptions
-import sys, re, pycurl, urllib
+import sys
+import re
+import pycurl
 from optparse import OptionParser
 from time import time, strptime
 from calendar import timegm
@@ -46,7 +48,7 @@ found = []
 def request_init(c, options, path):
     """`RequestManager` callback to initialise directory contents request."""
     c.setopt(
-        pycurl.URL, options.server + urllib.quote(path) + ((path != "/" and "/") or "")
+        pycurl.URL, options.server + parse.quote(path) + ((path != "/" and "/") or "")
     )
 
 

bin/visDQMIndexMonitoring

@@ -16,41 +16,46 @@ INSTALLATION_DIR=/data/srv
 TMP_CATALOGUE_FILE=/tmp/dqmgui_catalogue
 EMAIL_ADDRESS_TO_NOTIFY=cms-dqm-coreTeam@cern.ch
 # Alarm threshold value [0.0, 1.0].
-# An alarm is triggered if any of the tree sizes exceed 80% of their capacity.
-ALERT_THRESHOLD="0.8"
+# An alarm is triggered if any of the tree sizes exceed 90% of their capacity.
+ALERT_THRESHOLD="0.9"
 
 preliminary_checks() {
     if [ ! -d "$INSTALLATION_DIR" ] || [ ! -d "$INSTALLATION_DIR/state/dqmgui/$FLAVOR/ix128" ]; then
         echo "Could not find $FLAVOR DQMGUI in $INSTALLATION_DIR"
         exit 1
     fi
-    # Needed to get the index limits
-    if [ ! -f "$INSTALLATION_DIR/current/apps/dqmgui/128/src/cpp/DQM/VisDQMIndex.h" ]; then
-        echo "Could not find required \"VisDQMIndex.h\" file in $INSTALLATION_DIR"
-        exit 1
-    fi
+}
+
+# Activate the DQMGUI environment with the required env variables
+activate_env() {
+    source $INSTALLATION_DIR/current/apps/dqmgui/128/etc/profile.d/env.sh
 }
 
 # Sources the env.sh file needed for activating the DQMGUI environment,
 # then dumps the Index' catalogue to the TMP_CATALOGUE_FILE
 dump_catalogue() {
-    source $INSTALLATION_DIR/current/apps/dqmgui/128/etc/profile.d/env.sh
     visDQMIndex dump $INSTALLATION_DIR/state/dqmgui/$FLAVOR/ix128 catalogue >"${TMP_CATALOGUE_FILE}_${FLAVOR}"
 }
 
+get_index_capacity() {
+    for tree_name_and_size in $(visDQMIndex get_capacity); do
+        tree_name=$(echo $tree_name_and_size | cut -d',' -f1)
+        tree_limit=$(echo $tree_name_and_size | cut -d',' -f2)
+        tree_limits[$tree_name]=$tree_limit
+    done
+}
+
 check_index_limits_and_send_email() {
-    visdqmindex_header_file="$INSTALLATION_DIR/current/apps/dqmgui/128/src/cpp/DQM/VisDQMIndex.h"
     msg=
     # Run over all tree types
-    for i in $(seq 0 $((${#_tree_types[@]} - 1))); do
-        tree_current_size=$(grep "${_tree_types[$i]}" "${TMP_CATALOGUE_FILE}_${FLAVOR}" | wc -l)
-        tree_limit=$(grep -oE "${_tree_types_limit_name[$i]}[[:space:]]+[0-9]+" "$visdqmindex_header_file" | awk '{print $2}')
-        percent_full=$(bc -l <<<"($tree_current_size/$tree_limit)*100")
-        echo "Found $tree_current_size ${_tree_types[$i]} in the catalogue. Limit is $tree_limit ($(printf '%.2f' $percent_full)% full)"
+    for tree_name in "${!tree_limits[@]}"; do
+        tree_current_size=$(grep "$tree_name" "${TMP_CATALOGUE_FILE}_${FLAVOR}" | wc -l)
+        percent_full=$(bc -l <<<"($tree_current_size/${tree_limits[$tree_name]})*100")
+        echo "Found $tree_current_size $tree_name in the catalogue. Limit is ${tree_limits[$tree_name]} ($(printf '%.2f' $percent_full)% full)"
         # Check if alarm threshold is exceeded
-        threshold=$(printf '%.0f' $(bc <<<"$ALERT_THRESHOLD * $tree_limit"))
+        threshold=$(printf '%.0f' $(bc <<<"$ALERT_THRESHOLD * ${tree_limits[$tree_name]}"))
         if [ $tree_current_size -gt $threshold ]; then
-            msg=$(printf "%s" "${msg}WARNING: DQMGUI's index tree ${_tree_types[$i]} has $tree_current_size entries out of the maximum ${tree_limit} ($(printf '%.2f' $percent_full)%% full)\n")
+            msg=$(printf "%s" "${msg}WARNING: DQMGUI's index tree $tree_name has $tree_current_size entries out of the maximum ${tree_limit} ($(printf '%.2f' $percent_full)%% full)\n")
         fi
     done
     if [ -n "$msg" ]; then
@@ -64,9 +69,11 @@ cleanup() {
 }
 
 ### Main script
-
+declare -A tree_limits=()
 declare -a steps=(
     preliminary_checks
+    activate_env
+    get_index_capacity
     dump_catalogue
     check_index_limits_and_send_email
     cleanup
@@ -89,11 +96,6 @@ for ARGUMENT in "$@"; do
     eval "$KEY=$VALUE"
 done
 
-# The different types of trees in the index which you get by running visDQMIndex dump catalogue
-_tree_types=("CMSSW-VERSION" "DATASET-NAME" "OBJECT-NAME" "SOURCE-FILE")
-# The definition names of the limits, in VisDQMIndex.h, in one-to-one mapping to the _tree_types
-_tree_types_limit_name=("CMSSWNAMES" "DATASETNAMES" "OBJECTNAMES" "PATHNAMES")
-
 # For each step, check if the appropriate flag is enabled.
 for step in "${steps[@]}"; do
 

bin/visDQMOnlineSyncDaemon

@@ -14,15 +14,18 @@ be set correctly for authentication.
 
 from Monitoring.Core.HTTP import RequestManager
 from Monitoring.Core.X509 import SSLOptions
-import os, os.path, sys, re, pycurl, urllib
+import os
+import os.path
+import sys
+import re
+import pycurl
+from urllib.parse import quote, urlparse
 from time import time, strptime, sleep
 from optparse import OptionParser
-from urllib import parse
 from calendar import timegm
 from Monitoring.Core.Utils.Common import logme
 from traceback import print_exc
 from tempfile import mkstemp
-from stat import *
 
 
 # Directory where we receive input ("drop box"). Not used.
@@ -86,7 +89,7 @@ def need_to_copy(path, size, date):
         info = os.stat(path)
         if int(info.st_mtime) == date and info.st_size == size:
             return False
-    except:
+    except Exception:
         if os.path.exists("%s.dqminfo" % path):
             return False
 
@@ -103,14 +106,12 @@ def handle_init(c):
 def request_init(c, options, kind, path, size, date):
     """`RequestManager` callback to initialise directory contents request."""
     # Set the download URL.
-    assert c.temp_file == None
-    assert c.temp_path == None
-    assert c.local_path == None
+    assert c.temp_file is None
+    assert c.temp_path is None
+    assert c.local_path is None
     c.setopt(
         pycurl.URL,
-        options.server
-        + urllib.quote(path)
-        + ((kind == DIR and path != "/" and "/") or ""),
+        options.server + quote(path) + ((kind == DIR and path != "/" and "/") or ""),
     )
 
     # If this is file download, prepare temporary destination file
@@ -139,17 +140,17 @@ def cleanup(c):
     if c.temp_file:
         try:
             c.temp_file.close()
-        except:
+        except Exception:
             pass
     if c.temp_path:
         try:
             os.remove(c.temp_path)
-        except:
+        except Exception:
             pass
     if c.local_path:
         try:
             os.remove(c.local_path)
-        except:
+        except Exception:
             pass
     c.temp_file = None
     c.temp_path = None
@@ -226,11 +227,11 @@ def process_task(c):
 
     # If it's a DIR, scan directory contents and maybe fetch files.
     elif kind == DIR:
-        assert c.temp_file == None, "Unexpected temporary file for a directory"
-        assert c.temp_path == None, "Unexpected temporary path for a directory"
-        assert c.local_path == None, "Unexpected local path for a directory"
-        assert size == None, "Unexpected size for a directory"
-        assert date == None, "Unexpected date for a directory"
+        assert c.temp_file is None, "Unexpected temporary file for a directory"
+        assert c.temp_path is None, "Unexpected temporary path for a directory"
+        assert c.local_path is None, "Unexpected local path for a directory"
+        assert size is None, "Unexpected size for a directory"
+        assert date is None, "Unexpected date for a directory"
         items = re.findall(
             r"<tr><td><a href='(.*?)'>(.*?)</a></td><td>(\d+|&nbsp;|-)</td>"
             r"<td>(&nbsp;|\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d UTC)</td>",
@@ -303,7 +304,7 @@ if not options.server:
     sys.exit(1)
 
 UMASK = myumask()
-ROOT_URL = parse.urlparse(options.server).path.rstrip("/")
+ROOT_URL = urlparse(options.server).path.rstrip("/")
 DROPBOX = args[0]
 FILEREPO = args[1]
 NEXT = args[2:]
@@ -343,7 +344,7 @@ while True:
         )
 
     # If anything bad happened, barf but keep going.
-    except KeyboardInterrupt as e:
+    except KeyboardInterrupt:
         logme("INFO: exiting")
         map(cleanup, reqman.handles)
         sys.exit(0)