Adds filtering of log files by date, log message type, and folder

jwql/website/apps/jwql/templates/log_view.html

@@ -24,14 +24,50 @@ <h2>Explore JWQL monitoring logs through the web browser.</h2><hr>
 
             <!--LOG SELECT DROPDOWN-->
             <h4>Select JWQL Monitoring Log</h4>
+
+                <div class="col-6 mx-auto text-left mt-5">
+                    Filter logs by folder: 
+                    <select id="log_folders" onchange="getLogsByFolder()">
+                        <option value="all_logs">All Logs</option>
+                        {% for log_folder in all_logs['log_folders'].keys()|sort %}
+                            <option value="{{ log_folder }}">{{ log_folder }}</option>
+                        {% endfor %}
+                    </select>
+                </div>
+
+                <div class="col-6 mx-auto text-left mt-5">
+                    Filter logs by date: 
+                    <select id="log_dates" onchange="getLogsByDate()">
+                        <option value="all_logs">All Logs</option>
+                        <option value="last_year">Last Year</option>
+                        <option value="last_month">Last Month</option>
+                        <option value="last_week">Last Week</option>
+                        <option value="last_day">Last Day</option>
+                    </select>
+                </div>
+
+                <div class="col-6 mx-auto text-left mt-5">
+                    Filter logs by level:
+                    <select id="log_levels" onchange="getLogsByLevel()">
+                        <option value="all_logs">All Logs</option>
+                        <option value="INFO">Logs containing INFO and above</option>
+                        <option value="WARNING">Logs containing WARNING and above</option>
+                        <option value="ERROR">Logs containing ERROR and above</option>
+                        <option value="CRITICAL">Logs containing CRITICAL and above</option>
+                    </select>
+                </div>
+
                 <div class="col-6 mx-auto text-left mt-5">
                     <input id="log_select" type="text" list="logs" placeholder="Log Name" class="form-control">
                     <datalist id="logs">
-                        {% for log_name in all_logs %}
+                        {% for log_name in all_logs['all']|sort %}
                             <option id="{{ log_name }}" value="{{ log_name }}"></option>
                         {% endfor %}
                     </datalist>
                 </div>
+                <div id="no_items_warning" style="display: none">
+                <p style="font-weight: bold">No logs match the chosen filters</p>
+                </div>
 
             <br>
 
@@ -49,25 +85,95 @@ <h4> {{ log_name|safe }} </h4><hr>
             {% endif %}
 
             <script type="text/javascript">
+                const idInputField = document.getElementById('log_select');
+                const folderInputField = document.getElementById('log_folders');
+                const dateInputField = document.getElementById('log_dates');
+                const levelInputField = document.getElementById('log_levels');
+                const noResultsWarningDiv = document.getElementById('no_items_warning');
+                var allLogFiles = {{ all_logs['all']|safe }};
+                var listOfLogNamesByFolder = {{ all_logs['all']|safe }};
+                var logsByFolder = {{ all_logs['log_folders']|safe }};
+                var listOfLogNamesByDate = {{ all_logs['all']|safe }};
+                var logsByDate = {{ all_logs['log_dates']|safe }};
+                var logLevelsList = document.getElementById('log_levels');
+                var listOfLogNamesByLevel = {{ all_logs['all']|safe }};
+                var logsByLevel = {{ all_logs['log_levels']|safe }};
+                var logsList = document.getElementById("logs");
+
                 function GetSelectedTextValue() {
                     var log_value = document.getElementById("log_select").value;
                     document.getElementById("log_submit").value = log_value;
                 }
-            </script>
 
-            <script type="text/javascript">
-                const idInputField = document.getElementById('log_select');
-                var logsList = document.getElementById("logs");
+                function setLogList() {
+                    var logList = allLogFiles;
+                    console.log("Logs list is: "+logList.length+" items");
+                    logList = logList.filter(value => listOfLogNamesByFolder.includes(value));
+                    console.log("Logs list is: "+logList.length+" items");
+                    logList = logList.filter(value => listOfLogNamesByDate.includes(value));
+                    console.log("Logs list is: "+logList.length+" items");
+                    logList = logList.filter(value => listOfLogNamesByLevel.includes(value));
+                    console.log("Logs list is: "+logList.length+" items");
+                    logsList.replaceChildren();
+                    logList.forEach(function (item, index) {
+                        var option = document.createElement('option');
+                        option.value = item;
+                        option.id = item;
+                        logsList.appendChild(option);
+                    });
+                    if (logList.length === 0) {
+                        noResultsWarningDiv.style.display = "block";
+                    } else {
+                        noResultsWarningDiv.style.display = "none";
+                    }
+                }
+
                 idInputField.addEventListener('input', function () {
-                    if (logsList.options.namedItem(this.value) != null) {
+                    if (logsList.options.namedItem(this.value) !== null) {
                         document.getElementById("log_submit").disabled = false;
                     } else {
                         document.getElementById("log_submit").disabled = true;
                     }
                 });
+
+                function getLogsByFolder() {
+                    var currentFolder = folderInputField.value;
+                    console.log("Current folder is "+currentFolder);
+                    if (currentFolder === "all_logs") {
+                        listOfLogNamesByFolder = {{ all_logs['all']|safe }};
+                    } else {
+                        listOfLogNamesByFolder = logsByFolder[currentFolder];
+                    }
+                    console.log("Available log files are "+listOfLogNamesByFolder);
+                    setLogList();
+                }
+
+                function getLogsByDate() {
+                    var currentDate = dateInputField.value;
+                    console.log("Current date is "+currentDate);
+                    if (currentDate === "all_logs") {
+                        listOfLogNamesByDate = {{ all_logs['all']|safe }};
+                    } else {
+                        listOfLogNamesByDate = logsByDate[currentDate];
+                    }
+                    console.log("Available log files are "+listOfLogNamesByDate);
+                    setLogList();
+                }
+
+                function getLogsByLevel() {
+                    var currentLevel = levelInputField.value;
+                    console.log("Current level is "+currentLevel);
+                    if (currentLevel === "all_logs") {
+                        listOfLogNamesByLevel = {{ all_logs['all']|safe }};
+                    } else {
+                        listOfLogNamesByLevel = logsByLevel[currentLevel];
+                    }
+                    console.log("Available log files are "+listOfLogNamesByLevel);
+                    setLogList();
+                }
             </script>
 
         </form>
     </main>
 
-{% endblock %}
+{% endblock %}

jwql/website/apps/jwql/views.py

@@ -49,7 +49,9 @@
 import logging
 import operator
 import os
+from pathlib import Path
 import socket
+import yaml
 
 from astropy.time import Time
 from bokeh.embed import components
@@ -702,32 +704,102 @@ def log_view(request):
     """
 
     template = 'log_view.html'
-    log_path = get_config()['log_dir']
-    log_name = request.POST.get('log_submit', None)
+    log_path = Path(get_config()['log_dir'])
+    selected_log_name = request.POST.get('log_submit', None)
 
     hostname = socket.gethostname()
 
     if 'dljwql' in hostname:
-        server = 'dev'
+        log_path /= "dev"
     elif 'tljwql' in hostname:
-        server = 'test'
+        log_path /= "test"
     else:
-        server = 'ops'
-
-    full_log_paths = sorted(glob.glob(os.path.join(log_path, server, '*', '*')), reverse=True)
-    full_log_paths = [log for log in full_log_paths if not os.path.basename(log).startswith('.')]
-    log_dictionary = {os.path.basename(path): path for path in full_log_paths}
+        log_path /= "ops"
+
+    log_level_file = log_path / "log_info.yaml"
+    if log_level_file.is_file():
+        with open(log_level_file) as f:
+            log_dictionary = yaml.safe_load(f)
+        log_dictionary['log_dates'] = {
+            'last_year': [],
+            'last_month': [],
+            'last_week': [],
+            'last_day': []
+        }
+    else:
+        log_dictionary = {
+            'all': [],
+            'log_folders': {},
+            'log_dates': {
+                'last_year': [],
+                'last_month': [],
+                'last_week': [],
+                'last_day': []
+            },
+            'log_levels': {'INFO': [], 'WARNING': [], 'ERROR': [], 'CRITICAL': []}
+        }
+
+    # Make a "now" time and day/week/month/year time interval
+    last_day = datetime.datetime.now() - datetime.timedelta(days=1)
+    last_week = datetime.datetime.now() - datetime.timedelta(weeks=1)
+    last_month = datetime.datetime.now() - datetime.timedelta(days=30)
+    last_year = datetime.datetime.now() - datetime.timedelta(days=365)
+
+    for log_file in log_path.rglob("*.log"):
+        log_name = log_file.name
+
+        # Don't include hidden dot-files
+        if log_name[0] == '.':
+            continue
 
-    if log_name:
-        with open(log_dictionary[log_name]) as f:
+        # Always want to update date information
+        log_time = datetime.datetime.strptime(log_name[-20:-4], "%Y-%m-%d-%H-%M")
+        if log_time > last_year:
+            log_dictionary['log_dates']['last_year'].append(log_name)
+        if log_time > last_month:
+            log_dictionary['log_dates']['last_month'].append(log_name)
+        if log_time > last_week:
+            log_dictionary['log_dates']['last_week'].append(log_name)
+        if log_time > last_day:
+            log_dictionary['log_dates']['last_day'].append(log_name)
+
+        if log_name not in log_dictionary['all']:
+            log_dictionary[log_name] = str(log_file)
+            log_dictionary['all'].append(log_name)
+            log_folder = log_file.parent.name
+            if log_folder not in log_dictionary['log_folders']:
+                log_dictionary['log_folders'][log_folder] = []
+            log_dictionary['log_folders'][log_folder].append(log_name)
+            with open(log_file) as f:
+                log_content = f.read()
+                if 'CRITICAL:' in log_content:
+                    log_dictionary['log_levels']['CRITICAL'].append(log_name)
+                    log_dictionary['log_levels']['ERROR'].append(log_name)
+                    log_dictionary['log_levels']['WARNING'].append(log_name)
+                    log_dictionary['log_levels']['INFO'].append(log_name)
+                elif 'ERROR:' in log_content:
+                    log_dictionary['log_levels']['ERROR'].append(log_name)
+                    log_dictionary['log_levels']['WARNING'].append(log_name)
+                    log_dictionary['log_levels']['INFO'].append(log_name)
+                elif 'WARNING:' in log_content:
+                    log_dictionary['log_levels']['WARNING'].append(log_name)
+                    log_dictionary['log_levels']['INFO'].append(log_name)
+                elif 'INFO:' in log_content:
+                    log_dictionary['log_levels']['INFO'].append(log_name)
+
+    with open(log_level_file, 'w') as f:
+        yaml.dump(log_dictionary, f, default_flow_style=False)
+
+    if selected_log_name:
+        with open(log_dictionary[selected_log_name]) as f:
             log_text = f.read()
     else:
         log_text = None
 
     context = {'inst': '',
                'all_logs': log_dictionary,
                'log_text': log_text,
-               'log_name': log_name}
+               'log_name': selected_log_name}
 
     return render(request, template, context)