Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Telemetry2 thread info #9084

Closed
wants to merge 6 commits into from
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 21 additions & 17 deletions src/debug/telemetry/telemetry2_thread_info.c
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ static struct previous_counters { // Cached data from previous round
* k_thread_foreach_current_cpu().
*/
struct user_data {
int core;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Data in commit message looks good, but I would put the busy/idle % on the top row (ranking on % busy stack can come later).

struct thread_info_core *core_data;
int thread_count;
#ifdef CONFIG_THREAD_RUNTIME_STATS
Expand All @@ -88,23 +89,24 @@ static uint32_t get_cycles(void *tid, k_thread_runtime_stats_t *thread_stats,
{
int i;

ud->active_threads[ud->thread_count] = tid; // Mark the thread as active
// look for cached value from previous round for 'tid'-thread
/* Mark the thread as active */
ud->active_threads[ud->thread_count] = tid;
jsarha marked this conversation as resolved.
Show resolved Hide resolved
/* look for cached value from previous round for 'tid'-thread */
for (i = 0; i < ARRAY_SIZE(ud->previous->threads); i++) {
if (ud->previous->threads[i].tid == tid) {
// Calculate number cycles since previous round
/* Calculate number cycles since previous round */
uint32_t cycles = (uint32_t) (thread_stats->execution_cycles -
ud->previous->threads[i].cycles);

LOG_DBG("%p found at %d (%s %llu)", tid, i,
name, thread_stats->execution_cycles);
// updare cached value
/* update cached value */
ud->previous->threads[i].cycles = thread_stats->execution_cycles;
return cycles;
}
}

// If no cached value was found, look for an empty slot to store the recent value
/* If no cached value was found, look for an empty slot to store the recent value */
for (i = 0; i < ARRAY_SIZE(ud->previous->threads); i++) {
if (ud->previous->threads[i].tid == NULL) {
ud->previous->threads[i].tid = tid;
Expand All @@ -115,11 +117,11 @@ static uint32_t get_cycles(void *tid, k_thread_runtime_stats_t *thread_stats,
}
}

// If there is more than THREAD_INFO_MAX_THREADS threads in the system
/* If there is more than THREAD_INFO_MAX_THREADS threads in the system */
if (i == ARRAY_SIZE(ud->previous->threads))
LOG_INF("No place found for %s %p", name, tid);

// If there was no previous counter value to compare, return 0 cycles.
/* If there was no previous counter value to compare, return 0 cycles. */
return 0;
}

Expand All @@ -144,7 +146,7 @@ static uint8_t thread_info_cpu_utilization(struct k_thread *thread,
}
#else
static uint8_t thread_info_cpu_utilization(struct k_thread *thread,
struct user_data *ud)
struct user_data *ud, const char *name)
{
return 0;
}
Expand Down Expand Up @@ -220,18 +222,18 @@ static void cleanup_old_thread_cycles(struct user_data *ud)
for (i = 0; i < ARRAY_SIZE(ud->previous->threads); i++) {
bool found = false;

// This entry is already free, continue
/* This entry is already free, continue */
if (ud->previous->threads[i].tid == NULL)
continue;

// Check if the thread is any more active
/* Check if the thread was seen on previous round */
for (j = 0; j < ud->thread_count; j++) {
if (ud->active_threads[j] == ud->previous->threads[i].tid) {
found = true;
break;
}
}
// If the thead is not any more active, mark the entry free
/* If the thead is not any more active, mark the entry free */
if (!found) {
ud->previous->threads[i].tid = NULL;
ud->previous->threads[i].cycles = 0;
Expand All @@ -242,20 +244,21 @@ static void cleanup_old_thread_cycles(struct user_data *ud)
static void cleanup_old_thread_cycles(struct user_data *ud) { }
#endif

static void thread_info_get(struct thread_info_core *core_data)
static void thread_info_get(int core, struct thread_info_core *core_data)
{
k_thread_runtime_stats_t core_stats;
struct user_data ud = {
.core = core,
.core_data = core_data,
.thread_count = 0,
#ifdef CONFIG_THREAD_RUNTIME_STATS
.previous = &previous[arch_curr_cpu()->id],
.previous = &previous[core],
.active_threads = { NULL },
#endif
};
uint8_t load = 0;
#ifdef CONFIG_THREAD_RUNTIME_STATS
int ret = k_thread_runtime_stats_current_cpu_get(&core_stats);
int ret = k_thread_runtime_stats_cpu_get(core, &core_stats);

if (ret == 0) {
uint32_t active_cycles = (uint32_t) (core_stats.total_cycles -
Expand All @@ -265,7 +268,7 @@ static void thread_info_get(struct thread_info_core *core_data)

ud.stats_valid = true;
load = (uint8_t) ((255LLU * active_cycles) / all_cycles);
LOG_DBG("Core %u load %u / %u total %llu / %llu", arch_curr_cpu()->id,
LOG_DBG("Core %u load %u / %u total %llu / %llu", core,
active_cycles, all_cycles,
core_stats.total_cycles, core_stats.execution_cycles);
ud.previous->active = core_stats.total_cycles;
Expand All @@ -276,7 +279,8 @@ static void thread_info_get(struct thread_info_core *core_data)
core_data->state = THREAD_INFO_STATE_BEING_UPDATED;

core_data->load = load;
k_thread_foreach_current_cpu(thread_info_cb, &ud);
/* This is best effort debug tool. Unlocked version should be fine. */
k_thread_foreach_unlocked_filter_by_cpu(core, thread_info_cb, &ud);

cleanup_old_thread_cycles(&ud);

Expand All @@ -292,7 +296,7 @@ static void thread_info_run(void *data, void *cnum, void *a)
struct thread_info_core *core_data = &chunk->core[core];

for (;;) {
thread_info_get(core_data);
thread_info_get(core, core_data);
k_sleep(K_SECONDS(CONFIG_SOF_TELEMETRY2_THREAD_INFO_INTERVAL));
}
}
Expand Down
217 changes: 217 additions & 0 deletions tools/telemetry2/thread-info.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,217 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: BSD-3-Clause
jsarha marked this conversation as resolved.
Show resolved Hide resolved
#
# Copyright (c) 2024, Intel Corporation.

import argparse
import mmap
import ctypes
import time
import sys

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we comment this file more, i.e. describe what the macros and code blocks do.

TELEMETRY2_PAYLOAD_MAGIC = 0x1ED15EED

TELEMETRY2_CHUNK_ID_EMPTY = 0
TELEMETRY2_ID_THREAD_INFO = 1

TELEMETRY2_PAYLOAD_V0_0 = 0

#telemetry2 payload
class Telemetry2PayloadHdr(ctypes.Structure):
_fields_ = [
("magic", ctypes.c_uint),
("hdr_size", ctypes.c_uint),
("total_size", ctypes.c_uint),
("abi", ctypes.c_uint),
("tstamp", ctypes.c_ulonglong)
]

# telemetry2 header struct
class Telemetry2ChunkHdr(ctypes.Structure):
_fields_ = [("id", ctypes.c_uint), ("size", ctypes.c_uint)]

def get_telemetry2_payload_ok(slot):
payload_hdr = ctypes.cast(slot,
ctypes.POINTER(Telemetry2PayloadHdr))
if payload_hdr.contents.magic != TELEMETRY2_PAYLOAD_MAGIC:
print("Telemetry2 payload header bad magic 0x%x\n" %
payload_hdr.contents.magic)
return False
if payload_hdr.contents.abi != TELEMETRY2_PAYLOAD_V0_0:
print("Unknown Telemetry2 abi version %u\n" %
payload_hdr.contents.abi)
return True

def get_telemetry2_chunk_offset(id, slot):
"""
Generic function to get telmetry2 chunk offset by type
"""
offset = 0
while True:
struct_ptr = ctypes.cast(slot[offset:],
ctypes.POINTER(Telemetry2ChunkHdr))
#print("Checking %u id 0x%x size %u\n" %
# (offset, struct_ptr.contents.id, struct_ptr.contents.size))
if struct_ptr.contents.id == id:
return offset
if struct_ptr.contents.size == 0:
return -1
offset = offset + struct_ptr.contents.size

class ThreadInfo(ctypes.Structure):
_pack_ = 1
_fields_ = [
("name", ctypes.c_char * 14),
("stack_usage", ctypes.c_ubyte),
("cpu_usage", ctypes.c_ubyte),
]

class CPUInfo(ctypes.Structure):
_pack_ = 1
_fields_ = [
("state", ctypes.c_ubyte),
("counter", ctypes.c_ubyte),
("load", ctypes.c_ubyte),
("thread_count", ctypes.c_ubyte),
("thread", ThreadInfo * 16),
]

class ThreadInfoChunk(ctypes.Structure):
_pack_ = 1
_fields_ = [
("hdr", Telemetry2ChunkHdr ),
("core_count", ctypes.c_ushort),
("core_offsets", ctypes.c_ushort * 16),
]

class CoreData:
""" Class for tracking thread analyzer info for one core """
counter = 0
offset = 0
core = 0
STATE_UNINITIALIZED = 0
STATE_BEING_UPDATED = 1
STATE_UPTODATE = 2

def __init__(self, offset, core):
self.counter = 0
self.offset = offset
self.core = core

def print(self, slot):
cpu_info = ctypes.cast(slot[self.offset:],
ctypes.POINTER(CPUInfo))
if self.counter == cpu_info.contents.counter:
return
if cpu_info.contents.state != self.STATE_UPTODATE:
return
self.counter = cpu_info.contents.counter
print("Core %d load %02.1f%%" %
(self.core, cpu_info.contents.load / 2.55))
for i in range(cpu_info.contents.thread_count):
thread = cpu_info.contents.thread[i]
print("\t%-20s cpu %02.1f%% stack %02.1f%%" %
(thread.name.decode('utf-8'), thread.cpu_usage / 2.55,
thread.stack_usage / 2.55));

class Telemetry2ThreadAnalyzerDecoder:
"""
Class for finding thread analyzer chuck and initializing CoreData objects.
"""
file_size = 4096 # ADSP debug slot size
f = None
chunk_offset = -1
chunk = None
core_data = []

def set_file(self, f):
self.f = f

def decode_chunk(self):
if self.chunk != None:
return
self.f.seek(0)
slot = self.f.read(self.file_size)
if not get_telemetry2_payload_ok(slot):
return
self.chunk_offset = get_telemetry2_chunk_offset(TELEMETRY2_ID_THREAD_INFO, slot)
if self.chunk_offset < 0:
print("Thread info chunk not found")
self.chunk = None
return
self.chunk = ctypes.cast(slot[self.chunk_offset:],
ctypes.POINTER(ThreadInfoChunk))
hdr = self.chunk.contents.hdr
#print(f"Chunk at {self.chunk_offset} id: {hdr.id} size: {hdr.size}")
print(f"core_count: {self.chunk.contents.core_count}")

def init_core_data(self):
if self.chunk == None:
return
if len(self.core_data) == self.chunk.contents.core_count:
return
self.core_data = [None] * self.chunk.contents.core_count
for i in range(self.chunk.contents.core_count):
offset = self.chunk_offset + self.chunk.contents.core_offsets[i]
self.core_data[i] = CoreData(offset, i)
#print(f"core_offsets {i}: {self.chunk.contents.core_offsets[i]}")

def loop_cores(self):
if len(self.core_data) == 0:
return
self.f.seek(0)
slot = self.f.read(self.file_size)
for i in range(len(self.core_data)):
self.core_data[i].print(slot)

def reset(self):
self.f = None
self.chunk = None

def print_status(self, tag):
print(f"{tag} f {self.f} offset {self.chunk_offset} chunk {self.chunk} cores {len(self.core_data)}")

def main_f(args):
"""
Map telemetry2 slot into memorey and find the chunk we are interested in
"""
decoder = Telemetry2ThreadAnalyzerDecoder()
prev_error = None
while True:
try:
with open(args.telemetry2_file, "rb") as f:
decoder.set_file(f)
decoder.decode_chunk()
decoder.init_core_data()
while True:
decoder.loop_cores()
time.sleep(args.update_interval)
except FileNotFoundError:
print(f"File {args.telemetry2_file} not found!")
break
except OSError as e:
if str(e) != prev_error:
print(f"Open {args.telemetry2_file} failed '{e}'")
prev_error = str(e)
decoder.reset()
time.sleep(args.update_interval)

def parse_params():
""" Parses parameters
"""
parser = argparse.ArgumentParser(description=
"SOF Telemetry2 thread info client. "+
"Opens telemetry2 slot debugfs file, looks"+
" for thread info chunk, and decodes its "+
"contentes into stdout periodically.")
parser.add_argument('-t', '--update-interval', type=float,
help='Telemetry2 window polling interval in seconds, default 1',
default=1)
parser.add_argument('-f', '--telemetry2-file',
help='File to read the telemetry2 data from, default /sys/kernel/debug/sof/telemetry2',
default="/sys/kernel/debug/sof/telemetry2")
parsed_args = parser.parse_args()
return parsed_args

args = parse_params()
main_f(args)