Skip to content

Commit

Permalink
Merge pull request #1547 from Deixx/block-stats
Browse files Browse the repository at this point in the history 
Update for stats tests
  • Loading branch information
@robertbaldyga
robertbaldyga authored Oct 3, 2024
2 parents 865967e + 663b428 commit 188e42b
Show file tree
Hide file tree
Showing 4 changed files with 350 additions and 474 deletions.
190 changes: 58 additions & 132 deletions test/functional/api/cas/statistics.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,8 @@
#

import csv

from enum import Enum
from datetime import timedelta
from enum import Enum
from typing import List

from api.cas import casadm
Expand Down Expand Up @@ -43,34 +42,10 @@ def __init__(
filter: List[StatsFilter] = None,
percentage_val: bool = False,
):
stats_dict = get_stats_dict(filter=filter, cache_id=cache_id)

if filter is None:
filters = [
StatsFilter.conf,
StatsFilter.usage,
StatsFilter.req,
StatsFilter.blk,
StatsFilter.err,
]
else:
filters = filter

csv_stats = casadm.print_statistics(
cache_id=cache_id,
filter=filter,
output_format=casadm.OutputFormat.csv,
).stdout.splitlines()

stat_keys, stat_values = csv.reader(csv_stats)

# Unify names in block stats for core and cache:
# cache stats: Reads from core(s)
# core stats: Reads from core
stat_keys = [x.replace("(s)", "") for x in stat_keys]
stats_dict = dict(zip(stat_keys, stat_values))

for filter in filters:
match filter:
for section in _get_section_filters(filter):
match section:
case StatsFilter.conf:
self.config_stats = CacheConfigStats(stats_dict)
case StatsFilter.usage:
Expand Down Expand Up @@ -102,30 +77,10 @@ def __init__(
filter: List[StatsFilter] = None,
percentage_val: bool = False,
):
stats_dict = get_stats_dict(filter=filter, cache_id=cache_id, core_id=core_id)

if filter is None:
filters = [
StatsFilter.conf,
StatsFilter.usage,
StatsFilter.req,
StatsFilter.blk,
StatsFilter.err,
]
else:
filters = filter

csv_stats = casadm.print_statistics(
cache_id=cache_id,
core_id=core_id,
filter=filter,
output_format=casadm.OutputFormat.csv,
).stdout.splitlines()

stat_keys, stat_values = csv.reader(csv_stats)
stats_dict = dict(zip(stat_keys, stat_values))

for filter in filters:
match filter:
for section in _get_section_filters(filter):
match section:
case StatsFilter.conf:
self.config_stats = CoreConfigStats(stats_dict)
case StatsFilter.usage:
Expand Down Expand Up @@ -158,34 +113,12 @@ def __init__(
filter: List[StatsFilter] = None,
percentage_val: bool = False,
):
if filter is None:
filters = [
StatsFilter.conf,
StatsFilter.usage,
StatsFilter.req,
StatsFilter.blk,
]
else:
filters = filter

csv_stats = casadm.print_statistics(
cache_id=cache_id,
core_id=core_id,
io_class_id=io_class_id,
filter=filter,
output_format=casadm.OutputFormat.csv,
).stdout.splitlines()

stat_keys, stat_values = csv.reader(csv_stats)

# Unify names in block stats for core and cache:
# cache stats: Reads from core(s)
# core stats: Reads from core
stat_keys = [x.replace("(s)", "") for x in stat_keys]
stats_dict = dict(zip(stat_keys, stat_values))
stats_dict = get_stats_dict(
filter=filter, cache_id=cache_id, core_id=core_id, io_class_id=io_class_id
)

for filter in filters:
match filter:
for section in _get_section_filters(filter):
match section:
case StatsFilter.conf:
self.config_stats = IoClassConfigStats(stats_dict)
case StatsFilter.usage:
Expand Down Expand Up @@ -243,7 +176,7 @@ def __init__(self, stats_dict):
self.metadata_memory_footprint = parse_value(
value=stats_dict["Metadata Memory Footprint [MiB]"], unit_type=UnitType.mebibyte
)
self.dirty_for = parse_value(value=stats_dict["Dirty for [s]"], unit_type="[s]")
self.dirty_for = parse_value(value=stats_dict["Dirty for [s]"], unit_type=UnitType.seconds)
self.status = stats_dict["Status"]

def __str__(self):
Expand Down Expand Up @@ -399,21 +332,6 @@ def __eq__(self, other):
def __ne__(self, other):
return not self == other

def __add__(self, other):
return UsageStats(
self.occupancy + other.occupancy,
self.free + other.free,
self.clean + other.clean,
self.dirty + other.dirty,
)

def __iadd__(self, other):
self.occupancy += other.occupancy
self.free += other.free
self.clean += other.clean
self.dirty += other.dirty
return self


class IoClassUsageStats:
def __init__(self, stats_dict, percentage_val):
Expand Down Expand Up @@ -445,43 +363,6 @@ def __eq__(self, other):
def __ne__(self, other):
return not self == other

def __add__(self, other):
return UsageStats(
self.occupancy + other.occupancy,
self.clean + other.clean,
self.dirty + other.dirty,
)

def __iadd__(self, other):
self.occupancy += other.occupancy
self.clean += other.clean
self.dirty += other.dirty
return self


class InactiveUsageStats:
def __init__(self, inactive_occupancy, inactive_clean, inactive_dirty):
self.inactive_occupancy = inactive_occupancy
self.inactive_clean = inactive_clean
self.inactive_dirty = inactive_dirty

def __str__(self):
return (
f"Inactive usage stats:\n"
f"Inactive occupancy: {self.inactive_occupancy}\n"
f"Inactive clean: {self.inactive_clean}\n"
f"Inactive dirty: {self.inactive_dirty}\n"
)

def __eq__(self, other):
if not other:
return False
return (
self.inactive_occupancy == other.inactive_occupancy
and self.inactive_clean == other.inactive_clean
and self.inactive_dirty == other.inactive_dirty
)


class RequestStats:
def __init__(self, stats_dict, percentage_val):
Expand Down Expand Up @@ -655,6 +536,12 @@ def __eq__(self, other):
)


def get_stat_value(stat_dict: dict, key: str):
idx = key.index("[")
unit = UnitType(key[idx:])
return parse_value(stat_dict[key], unit)


def parse_value(value: str, unit_type: UnitType) -> int | float | Size | timedelta | str:
match unit_type:
case UnitType.requests:
Expand All @@ -674,3 +561,42 @@ def parse_value(value: str, unit_type: UnitType) -> int | float | Size | timedel
case _:
stat_unit = value
return stat_unit


def _get_section_filters(filter: List[StatsFilter], io_class_stats: bool = False):
if filter is None or StatsFilter.all in filter:
filters = [
StatsFilter.conf,
StatsFilter.usage,
StatsFilter.req,
StatsFilter.blk,
StatsFilter.err,
]
else:
filters = filter
if io_class_stats and StatsFilter.err in filters:
filters.remove(StatsFilter.err)
return filters


def get_stats_dict(
filter: List[StatsFilter],
cache_id: int,
core_id: int = None,
io_class_id: int = None
):
csv_stats = casadm.print_statistics(
cache_id=cache_id,
core_id=core_id,
io_class_id=io_class_id,
filter=filter,
output_format=casadm.OutputFormat.csv,
).stdout.splitlines()
stat_keys, stat_values = csv.reader(csv_stats)
# Unify names in block stats for core and cache to easier compare
# cache vs core stats using unified key
# cache stats: Reads from core(s)
# core stats: Reads from core
stat_keys = [x.replace("(s)", "") for x in stat_keys]
stats_dict = dict(zip(stat_keys, stat_values))
return stats_dict
Loading

0 comments on commit 188e42b

Please sign in to comment.