diff --git a/src/gurt/fault_inject.c b/src/gurt/fault_inject.c index a0fcfdfea7f..27fd964fd10 100644 --- a/src/gurt/fault_inject.c +++ b/src/gurt/fault_inject.c @@ -1,34 +1,35 @@ /* - * (C) Copyright 2018-2022 Intel Corporation. + * (C) Copyright 2018-2023 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ -/** - * This file is part of gurt, it implements the fault injection feature. - */ -#define D_LOGFAC DD_FAC(fi) -/** max length of argument string in the yaml config file */ +#define D_LOGFAC DD_FAC(fi) +/* max length of argument string in the yaml config file */ #define FI_CONFIG_ARG_STR_MAX_LEN 4096 /* (1 << D_FA_TABLE_BITS) is the number of buckets of fa hash table */ -#define D_FA_TABLE_BITS (13) +#define D_FA_TABLE_BITS (13) #include #include -#include "fi.h" #include -/** +/* * global switch for fault injection. zero globally turns off fault injection, * non-zero turns on fault injection */ -unsigned int d_fault_inject; -unsigned int d_fault_config_file; -struct d_fault_attr_t *d_fault_attr_mem; +unsigned int d_fault_inject; +unsigned int d_fault_config_file; +struct d_fault_attr_t *d_fault_attr_mem; #if FAULT_INJECTION +struct d_fault_attr { + d_list_t fa_link; + struct d_fault_attr_t fa_attr; +}; + static struct d_fault_attr * fa_link2ptr(d_list_t *rlink) { @@ -37,8 +38,7 @@ fa_link2ptr(d_list_t *rlink) } static bool -fa_op_key_cmp(struct d_hash_table *htab, d_list_t *rlink, const void *key, - unsigned int ksize) +fa_op_key_cmp(struct d_hash_table *htab, d_list_t *rlink, const void *key, unsigned int ksize) { struct d_fault_attr *fa_ptr = fa_link2ptr(rlink); @@ -59,21 +59,18 @@ fa_op_rec_hash(struct d_hash_table *htab, d_list_t *link) static void fa_op_rec_free(struct d_hash_table *htab, d_list_t *rlink) { - struct d_fault_attr *ht_rec = fa_link2ptr(rlink); - int rc; + struct d_fault_attr *ht_rec = fa_link2ptr(rlink); + int rc; D_FREE(ht_rec->fa_attr.fa_argument); rc = D_SPIN_DESTROY(&ht_rec->fa_attr.fa_lock); if (rc != DER_SUCCESS) - D_ERROR("Can't destroy spinlock for fault id: %d\n", - ht_rec->fa_attr.fa_id); + D_ERROR("Can't destroy spinlock for fault id: %d\n", ht_rec->fa_attr.fa_id); D_FREE(ht_rec); } -/** - * abuse hop_rec_decref() so that we can safely use it without a - * hop_rec_addref(). The goal is to have d_hash_table_destroy_inplace() - * destroy all records automatically. +/* abuse hop_rec_decref() so that we can safely use it without a hop_rec_addref(). The goal is to + * have d_hash_table_destroy_inplace() destroy all records automatically. */ static bool fa_op_rec_decref(struct d_hash_table *htab, d_list_t *rlink) @@ -82,45 +79,98 @@ fa_op_rec_decref(struct d_hash_table *htab, d_list_t *rlink) } static d_hash_table_ops_t fa_table_ops = { - .hop_key_cmp = fa_op_key_cmp, - .hop_rec_hash = fa_op_rec_hash, - .hop_rec_decref = fa_op_rec_decref, - .hop_rec_free = fa_op_rec_free, + .hop_key_cmp = fa_op_key_cmp, + .hop_rec_hash = fa_op_rec_hash, + .hop_rec_decref = fa_op_rec_decref, + .hop_rec_free = fa_op_rec_free, }; struct d_fi_gdata_t { - unsigned int dfg_refcount; - unsigned int dfg_inited; - pthread_rwlock_t dfg_rwlock; - struct d_hash_table dfg_fa_table; + unsigned int dfg_refcount; + unsigned int dfg_inited; + pthread_rwlock_t dfg_rwlock; + struct d_hash_table dfg_fa_table; + bool dfg_thread_default; }; -/** - * global switch for fault injection. zero globally turns off fault injection, - * non-zero turns on fault injection +/* global switch for fault injection. zero globally turns off fault injection, non-zero turns on + * fault injection + */ +static uint32_t d_fault_inject_seed; +static struct d_fi_gdata_t d_fi_gdata; +static pthread_once_t d_fi_gdata_init_once = PTHREAD_ONCE_INIT; + +/* Check for and set thread specific enabled flag. This is a single function that returns two + * values so it can access a single _thread local. + * + * 0 - no value set; + * 1 - disabled; + * >2 - enabled; */ -static uint32_t d_fault_inject_seed; -static struct d_fi_gdata_t d_fi_gdata; -static pthread_once_t d_fi_gdata_init_once = PTHREAD_ONCE_INIT; +static int +fault_get_thread_enabled_h(int new_value) +{ + static __thread int thread_value; + + if (new_value > 0) + thread_value = new_value; + return thread_value; +} + +static bool +fault_get_thread_enabled() +{ + int rc; + + rc = fault_get_thread_enabled_h(0); + if (rc == 0) + return d_fi_gdata.dfg_thread_default; + else if (rc == 1) + return false; + return true; +} + +static void +fault_set_thread_enable(bool enabled) +{ + if (enabled) + fault_get_thread_enabled_h(2); + else + fault_get_thread_enabled_h(1); +} + +void +d_fault_inject_thread_enable(bool enabled) +{ + D_RWLOCK_RDLOCK(&d_fi_gdata.dfg_rwlock); + fault_set_thread_enable(enabled); + D_RWLOCK_UNLOCK(&d_fi_gdata.dfg_rwlock); +} + +void +d_fault_inject_thread_default_enable(bool enabled) +{ + D_RWLOCK_WRLOCK(&d_fi_gdata.dfg_rwlock); + d_fi_gdata.dfg_thread_default = enabled; + D_RWLOCK_UNLOCK(&d_fi_gdata.dfg_rwlock); +} static inline int fault_attr_set(uint32_t fault_id, struct d_fault_attr_t fa_in, bool take_lock) { - struct d_fault_attr_t *fault_attr; - char *fa_argument = NULL; - bool should_free = true; - struct d_fault_attr *new_rec = NULL; - struct d_fault_attr *rec = NULL; - d_list_t *rlink = NULL; - int rc = DER_SUCCESS; + struct d_fault_attr_t *fault_attr; + char *fa_argument = NULL; + bool should_free = true; + struct d_fault_attr *new_rec; + d_list_t *rlink; + int rc = DER_SUCCESS; D_ALLOC_PTR(new_rec); if (new_rec == NULL) D_GOTO(out, rc = -DER_NOMEM); if (fa_in.fa_argument) { - D_STRNDUP(fa_argument, fa_in.fa_argument, - FI_CONFIG_ARG_STR_MAX_LEN); + D_STRNDUP(fa_argument, fa_in.fa_argument, FI_CONFIG_ARG_STR_MAX_LEN); if (fa_argument == NULL) D_GOTO(out, rc = -DER_NOMEM); } @@ -128,17 +178,19 @@ fault_attr_set(uint32_t fault_id, struct d_fault_attr_t fa_in, bool take_lock) if (take_lock) D_RWLOCK_WRLOCK(&d_fi_gdata.dfg_rwlock); - rlink = d_hash_rec_find_insert(&d_fi_gdata.dfg_fa_table, &fault_id, - sizeof(fault_id), &new_rec->fa_link); + rlink = d_hash_rec_find_insert(&d_fi_gdata.dfg_fa_table, &fault_id, sizeof(fault_id), + &new_rec->fa_link); if (rlink == &new_rec->fa_link) { fault_attr = &new_rec->fa_attr; - rc = D_SPIN_INIT(&fault_attr->fa_lock, - PTHREAD_PROCESS_PRIVATE); + + rc = D_SPIN_INIT(&fault_attr->fa_lock, PTHREAD_PROCESS_PRIVATE); if (rc != DER_SUCCESS) D_GOTO(out_unlock, rc); D_DEBUG(DB_ALL, "new fault id: %u added.\n", fault_id); should_free = false; } else { + struct d_fault_attr *rec; + rec = fa_link2ptr(rlink); D_ASSERT(rec->fa_attr.fa_id == fault_id); fault_attr = &rec->fa_attr; @@ -147,17 +199,16 @@ fault_attr_set(uint32_t fault_id, struct d_fault_attr_t fa_in, bool take_lock) D_SPIN_LOCK(&fault_attr->fa_lock); /* at this point, global lock is released, per entry lock is held */ - fault_attr->fa_id = fault_id; + fault_attr->fa_id = fault_id; fault_attr->fa_probability_x = fa_in.fa_probability_x; fault_attr->fa_probability_y = fa_in.fa_probability_y; - fault_attr->fa_interval = fa_in.fa_interval; - fault_attr->fa_max_faults = fa_in.fa_max_faults; - fault_attr->fa_err_code = fa_in.fa_err_code; - fault_attr->fa_argument = fa_argument; + fault_attr->fa_interval = fa_in.fa_interval; + fault_attr->fa_max_faults = fa_in.fa_max_faults; + fault_attr->fa_err_code = fa_in.fa_err_code; + fault_attr->fa_argument = fa_argument; /** - * Let's update fa_num_faults here too, so the user can reset num faults - * by fault_attr_set, then it can use the same fault_attr to inject - * other failures. + * Let's update fa_num_faults here too, so the user can reset num faults by fault_attr_set, + * then it can use the same fault_attr to inject other failures. */ fault_attr->fa_num_faults = fa_in.fa_num_faults; /* nrand48() only takes the high order 48 bits for its seed */ @@ -186,17 +237,15 @@ d_fault_attr_set(uint32_t fault_id, struct d_fault_attr_t fa_in) struct d_fault_attr_t * d_fault_attr_lookup(uint32_t fault_id) { - struct d_fault_attr_t *fault_attr; - struct d_fault_attr *ht_rec; - d_list_t *rlink; + struct d_fault_attr_t *fault_attr; + struct d_fault_attr *ht_rec; + d_list_t *rlink; D_RWLOCK_RDLOCK(&d_fi_gdata.dfg_rwlock); - rlink = d_hash_rec_find(&d_fi_gdata.dfg_fa_table, (void *)&fault_id, - sizeof(fault_id)); + rlink = d_hash_rec_find(&d_fi_gdata.dfg_fa_table, (void *)&fault_id, sizeof(fault_id)); D_RWLOCK_UNLOCK(&d_fi_gdata.dfg_rwlock); if (rlink == NULL) { - D_DEBUG(DB_ALL, "fault attr for fault ID %d not set yet.\n", - fault_id); + D_DEBUG(DB_ALL, "fault attr for fault ID %d not set yet.\n", fault_id); fault_attr = NULL; } else { ht_rec = fa_link2ptr(rlink); @@ -210,8 +259,8 @@ d_fault_attr_lookup(uint32_t fault_id) int d_fault_attr_err_code(uint32_t fault_id) { - struct d_fault_attr_t *fault_attr; - int32_t err_code; + struct d_fault_attr_t *fault_attr; + int32_t err_code; fault_attr = d_fault_attr_lookup(fault_id); if (fault_attr == NULL) { @@ -229,31 +278,29 @@ d_fault_attr_err_code(uint32_t fault_id) static int one_fault_attr_parse(yaml_parser_t *parser) { - yaml_event_t first; - yaml_event_t second; - struct d_fault_attr_t attr = { .fa_probability_x = 1, - .fa_probability_y = 1, - .fa_interval = 1 }; - const char *id = "id"; - const char *probability_x = "probability_x"; - const char *probability_y = "probability_y"; - const char *interval = "interval"; - const char *max_faults = "max_faults"; - const char *err_code = "err_code"; - const char *argument = "argument"; - const char *key_str; - const char *val_str; - uint64_t val; - int has_id = 0; - int yaml_rc; - int rc = DER_SUCCESS; + yaml_event_t first; + yaml_event_t second; + struct d_fault_attr_t attr = { + .fa_probability_x = 1, .fa_probability_y = 1, .fa_interval = 1}; + const char *id = "id"; + const char *probability_x = "probability_x"; + const char *probability_y = "probability_y"; + const char *interval = "interval"; + const char *max_faults = "max_faults"; + const char *err_code = "err_code"; + const char *argument = "argument"; + const char *key_str; + const char *val_str; + uint64_t val; + int has_id = 0; + int yaml_rc; + int rc = DER_SUCCESS; do { /* libyaml functions return 1 on success, 0 on error */ yaml_rc = yaml_parser_parse(parser, &first); if (yaml_rc != 1) { - D_ERROR("yaml_parser_parse() failed. rc: %d\n", - yaml_rc); + D_ERROR("yaml_parser_parse() failed. rc: %d\n", yaml_rc); D_GOTO(out, rc = -DER_MISC); } @@ -272,8 +319,7 @@ one_fault_attr_parse(yaml_parser_t *parser) yaml_rc = yaml_parser_parse(parser, &second); if (yaml_rc != 1) { yaml_event_delete(&first); - D_ERROR("yaml_parser_parse() failed. rc: %d\n", - yaml_rc); + D_ERROR("yaml_parser_parse() failed. rc: %d\n", yaml_rc); D_GOTO(out, rc = -DER_MISC); } @@ -284,13 +330,13 @@ one_fault_attr_parse(yaml_parser_t *parser) D_GOTO(out, rc = -DER_MISC); } - key_str = (char *) first.data.scalar.value; - val_str = (const char *) second.data.scalar.value; - val = strtoul(val_str, NULL, 0); + key_str = (char *)first.data.scalar.value; + val_str = (const char *)second.data.scalar.value; + val = strtoul(val_str, NULL, 0); if (!strcmp(key_str, id)) { D_DEBUG(DB_ALL, "id: %lu\n", val); attr.fa_id = val; - has_id = 1; + has_id = 1; } else if (!strcmp(key_str, probability_x)) { attr.fa_probability_x = val; D_DEBUG(DB_ALL, "probability_x: %lu\n", val); @@ -305,11 +351,9 @@ one_fault_attr_parse(yaml_parser_t *parser) D_DEBUG(DB_ALL, "max_faults: %lu\n", val); } else if (!strcmp(key_str, err_code)) { attr.fa_err_code = strtol(val_str, NULL, 0); - D_DEBUG(DB_ALL, "err_code: "DF_RC"\n", - DP_RC(attr.fa_err_code)); + D_DEBUG(DB_ALL, "err_code: " DF_RC "\n", DP_RC(attr.fa_err_code)); } else if (!strcmp(key_str, argument)) { - D_STRNDUP(attr.fa_argument, val_str, - FI_CONFIG_ARG_STR_MAX_LEN); + D_STRNDUP(attr.fa_argument, val_str, FI_CONFIG_ARG_STR_MAX_LEN); if (attr.fa_argument == NULL) rc = -DER_NOMEM; D_DEBUG(DB_ALL, "argument: %s\n", attr.fa_argument); @@ -332,8 +376,7 @@ one_fault_attr_parse(yaml_parser_t *parser) rc = fault_attr_set(attr.fa_id, attr, true); if (rc != DER_SUCCESS) - D_ERROR("d_set_fault_attr(%u) failed, rc %d\n", attr.fa_id, - rc); + D_ERROR("d_set_fault_attr(%u) failed, rc %d\n", attr.fa_id, rc); out: D_FREE(attr.fa_argument); @@ -343,17 +386,16 @@ one_fault_attr_parse(yaml_parser_t *parser) static int fault_attr_parse(yaml_parser_t *parser) { - yaml_event_t event; - yaml_event_type_t event_type; - int yaml_rc; - int rc = -DER_SUCCESS; + yaml_event_t event; + yaml_event_type_t event_type; + int yaml_rc; + int rc = -DER_SUCCESS; do { /* libyaml functions return 1 on success, 0 on error */ yaml_rc = yaml_parser_parse(parser, &event); if (yaml_rc != 1) { - D_ERROR("yaml_parser_parse() failed. rc: %d\n", - yaml_rc); + D_ERROR("yaml_parser_parse() failed. rc: %d\n", yaml_rc); yaml_event_delete(&event); return -DER_MISC; } @@ -364,7 +406,8 @@ fault_attr_parse(yaml_parser_t *parser) rc = one_fault_attr_parse(parser); if (rc != DER_SUCCESS) { D_ERROR("yaml_parser_parse() failed. " - "rc: %d\n", rc); + "rc: %d\n", + rc); } break; default: @@ -381,13 +424,41 @@ fault_attr_parse(yaml_parser_t *parser) return rc; } +static int +thread_default_parse(yaml_parser_t *parser) +{ + yaml_event_t event; + int yaml_rc; + int rc = DER_SUCCESS; + + /* libyaml functions return 1 on success, 0 on error */ + yaml_rc = yaml_parser_parse(parser, &event); + if (yaml_rc != 1) { + D_ERROR("yaml_parser_parse() failed. rc: %d\n", yaml_rc); + return -DER_MISC; + } + + if (event.type != YAML_SCALAR_EVENT) + D_GOTO(out, rc = -DER_INVAL); + + if (strncasecmp((char *)event.data.scalar.value, "true", event.data.scalar.length) == 0) + d_fi_gdata.dfg_thread_default = true; + else + d_fi_gdata.dfg_thread_default = false; + +out: + yaml_event_delete(&event); + + return rc; +} + static int seed_parse(yaml_parser_t *parser) { - yaml_event_t event; - const char *val_str; - int yaml_rc; - int rc = DER_SUCCESS; + yaml_event_t event; + const char *val_str; + int yaml_rc; + int rc = DER_SUCCESS; /* libyaml functions return 1 on success, 0 on error */ yaml_rc = yaml_parser_parse(parser, &event); @@ -399,7 +470,7 @@ seed_parse(yaml_parser_t *parser) if (event.type != YAML_SCALAR_EVENT) D_GOTO(out, rc = -DER_INVAL); - val_str = (const char *) event.data.scalar.value; + val_str = (const char *)event.data.scalar.value; d_fault_inject_seed = strtoul(val_str, NULL, 10); out: @@ -413,12 +484,12 @@ d_fi_gdata_init(void) { int rc; - d_fi_gdata.dfg_refcount = 0; - d_fi_gdata.dfg_inited = 1; + d_fi_gdata.dfg_refcount = 0; + d_fi_gdata.dfg_inited = 1; + d_fi_gdata.dfg_thread_default = true; D_RWLOCK_INIT(&d_fi_gdata.dfg_rwlock, NULL); - rc = d_hash_table_create_inplace(D_HASH_FT_NOLOCK, D_FA_TABLE_BITS, - NULL, &fa_table_ops, - &d_fi_gdata.dfg_fa_table); + rc = d_hash_table_create_inplace(D_HASH_FT_NOLOCK, D_FA_TABLE_BITS, NULL, &fa_table_ops, + &d_fi_gdata.dfg_fa_table); if (rc != 0) D_ERROR("d_hash_table_create_inplace() failed, rc: %d.\n", rc); } @@ -428,16 +499,13 @@ d_fi_gdata_destroy(void) { int rc; - rc = d_hash_table_destroy_inplace(&d_fi_gdata.dfg_fa_table, - true /* force */); + rc = d_hash_table_destroy_inplace(&d_fi_gdata.dfg_fa_table, true /* force */); if (rc != 0) { - D_ERROR("failed to destroy fault attr data. force: %d, " - "d_hash_table_destroy_inplace failed, rc: %d\n", - true, rc); + D_ERROR("d_hash_table_destroy_inplace() failed, rc: %d\n", rc); } D_RWLOCK_DESTROY(&d_fi_gdata.dfg_rwlock); d_fi_gdata.dfg_refcount = 0; - d_fi_gdata.dfg_inited = 0; + d_fi_gdata.dfg_inited = 0; } /** @@ -446,14 +514,14 @@ d_fi_gdata_destroy(void) int d_fault_inject_init(void) { - char *config_file; - FILE *fp = NULL; - yaml_parser_t parser; - yaml_event_t event; - yaml_event_type_t event_type; - int last_errno; - int yaml_rc; - int rc = DER_SUCCESS; + char *config_file; + FILE *fp = NULL; + yaml_parser_t parser; + yaml_event_t event; + yaml_event_type_t event_type; + int last_errno; + int yaml_rc; + int rc = DER_SUCCESS; pthread_once(&d_fi_gdata_init_once, d_fi_gdata_init); D_ASSERT(d_fi_gdata.dfg_inited == 1); @@ -472,11 +540,10 @@ d_fault_inject_init(void) D_GOTO(out, rc); } - fp = fopen(config_file, "r"); + fp = fopen(config_file, "r"); last_errno = errno; if (fp == NULL) { - D_ERROR("Failed to open file %s (%s).\n", - config_file, strerror(last_errno)); + D_ERROR("Failed to open file %s (%s).\n", config_file, strerror(last_errno)); rc = d_errno2der(last_errno); D_GOTO(out, rc); } @@ -492,8 +559,7 @@ d_fault_inject_init(void) /* libyaml functions return 1 on success, 0 on error */ yaml_rc = yaml_parser_parse(&parser, &event); if (yaml_rc != 1) { - D_ERROR("yaml_parser_parse() failed. rc: %d\n", - yaml_rc); + D_ERROR("yaml_parser_parse() failed. rc: %d\n", yaml_rc); D_GOTO(out, rc = -DER_MISC); } @@ -507,17 +573,21 @@ d_fault_inject_init(void) continue; } - if (!strncmp((char *) event.data.scalar.value, - "fault_config", strlen("fault_config") + 1)) { + if (!strncmp((char *)event.data.scalar.value, "fault_config", + event.data.scalar.length)) { rc = fault_attr_parse(&parser); if (rc != DER_SUCCESS) - D_ERROR("fault_attr_parse() failed. rc %d\n", - rc); - } else if (!strncmp((char *) event.data.scalar.value, - "seed", strlen("seed") + 1)) { + D_ERROR("fault_attr_parse() failed. rc %d\n", rc); + } else if (!strncmp((char *)event.data.scalar.value, "seed", + event.data.scalar.length)) { rc = seed_parse(&parser); if (rc != DER_SUCCESS) D_ERROR("seed_parse() failed. rc %d\n", rc); + } else if (!strncmp((char *)event.data.scalar.value, "thread_default", + event.data.scalar.length)) { + rc = thread_default_parse(&parser); + if (rc != DER_SUCCESS) + D_ERROR("thread_default_parse() failed. rc %d\n", rc); } else { D_ERROR("unknown key: %s\n", event.data.scalar.value); rc = -DER_INVAL; @@ -530,10 +600,9 @@ d_fault_inject_init(void) yaml_parser_delete(&parser); if (rc == DER_SUCCESS) { - D_INFO("Config file: %s, fault injection is ON.\n", - config_file); + D_INFO("Config file: %s, fault injection is ON.\n", config_file); d_fault_config_file = 1; - d_fault_inject = 1; + d_fault_inject = 1; } else { D_ERROR("Failed to parse fault config file.\n"); D_GOTO(out, rc); @@ -553,7 +622,7 @@ d_fault_inject_init(void) int d_fault_inject_fini() { - int rc = 0; + int rc = 0; if (d_fi_gdata.dfg_inited == 0) { D_DEBUG(DB_TRACE, "fault injection not initialized.\n"); @@ -570,14 +639,13 @@ d_fault_inject_fini() D_RWLOCK_UNLOCK(&d_fi_gdata.dfg_rwlock); d_fi_gdata_destroy(); d_fi_gdata_init_once = PTHREAD_ONCE_INIT; - d_fault_inject = 0; + d_fault_inject = 0; D_DEBUG(DB_ALL, "Finalized.\n"); return rc; } - int d_fault_inject_enable(void) { @@ -611,20 +679,14 @@ d_fault_inject_is_enabled(void) return false; } -/** - * based on the state of fault_id, decide if a fault should be injected - * - * \param[in] fault_id fault injection configuration id +/* based on the state of fault_attr, decide if a fault should be injected * - * \return true if should inject fault, false if should not - * inject fault - * - * support injecting X faults in Y occurrences + * return true if should inject fault, false if should not inject fault */ bool d_should_fail(struct d_fault_attr_t *fault_attr) { - bool rc = true; + bool rc = true; if (!d_fi_initialized()) { D_ERROR("fault injection not initialized.\n"); @@ -639,6 +701,10 @@ d_should_fail(struct d_fault_attr_t *fault_attr) return false; D_SPIN_LOCK(&fault_attr->fa_lock); + + if (!fault_get_thread_enabled()) + D_GOTO(out, rc = false); + if (fault_attr->fa_probability_x == 0) D_GOTO(out, rc = false); @@ -654,7 +720,7 @@ d_should_fail(struct d_fault_attr_t *fault_attr) if (fault_attr->fa_probability_y != 0 && fault_attr->fa_probability_x <= - nrand48(fault_attr->fa_rand_state) % fault_attr->fa_probability_y) + nrand48(fault_attr->fa_rand_state) % fault_attr->fa_probability_y) D_GOTO(out, rc = false); fault_attr->fa_num_faults++; @@ -664,25 +730,29 @@ d_should_fail(struct d_fault_attr_t *fault_attr) return rc; }; #else /* FAULT_INJECT */ -int d_fault_inject_init(void) +int +d_fault_inject_init(void) { D_INFO("Fault Injection not initialized feature not included in build"); return -DER_NOSYS; } -int d_fault_inject_fini(void) +int +d_fault_inject_fini(void) { D_INFO("Fault Injection not finalized feature not included in build"); return -DER_NOSYS; } -int d_fault_inject_enable(void) +int +d_fault_inject_enable(void) { D_INFO("Fault Injection not enabled feature not included in build"); return -DER_NOSYS; } -int d_fault_inject_disable(void) +int +d_fault_inject_disable(void) { D_INFO("Fault Injection not disabled feature not included in build"); return -DER_NOSYS; @@ -718,4 +788,15 @@ d_fault_attr_err_code(uint32_t fault_id) { return 0; } + +void +d_fault_inject_thread_enable(bool enabled) +{ +} + +void +d_fault_inject_thread_default_enable(bool enabled) +{ +} + #endif /* FAULT_INJECT */ diff --git a/src/gurt/fi.h b/src/gurt/fi.h deleted file mode 100644 index 93dbce81bb5..00000000000 --- a/src/gurt/fi.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * (C) Copyright 2019-2021 Intel Corporation. - * - * SPDX-License-Identifier: BSD-2-Clause-Patent - */ -/** - * \file - * - * This file is part of gurt, it contains internal variables and functions for - * the fault injection feature. - */ - -#ifndef __FI_H__ -#define __FI_H__ - -/** @addtogroup GURT - * @{ - */ - -#if defined(__cplusplus) -extern "C" { -#endif - -struct d_fault_attr { - d_list_t fa_link; - struct d_fault_attr_t fa_attr; -}; - -#if defined(__cplusplus) -} -#endif - -/** @} - */ -#endif /* __FI_H__ */ diff --git a/src/include/gurt/fault_inject.h b/src/include/gurt/fault_inject.h index d0106fcf32c..dd91a4aa275 100644 --- a/src/include/gurt/fault_inject.h +++ b/src/include/gurt/fault_inject.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2018-2022 Intel Corporation. + * (C) Copyright 2018-2023 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -7,8 +7,7 @@ /** * \file * - * This file is part of gurt, it contains variables and functions for the fault - * injection feature. + * This file is part of gurt, it contains variables and functions for the fault injection feature. */ #ifndef __FAULT_INJECT__ @@ -27,60 +26,51 @@ extern "C" { #endif /** Env to specify fault injection config file */ -#define D_FAULT_CONFIG_ENV "D_FI_CONFIG" +#define D_FAULT_CONFIG_ENV "D_FI_CONFIG" /** global on/off switch for fault injection */ -extern unsigned int d_fault_inject; -extern unsigned int d_fault_config_file; +extern unsigned int d_fault_inject; +extern unsigned int d_fault_config_file; -/* Location used for inecting memory allocation failures into D_ALLOC - * uses fault_id 0 - */ +/* Location used for inecting memory allocation failures into D_ALLOC uses fault_id 0 */ extern struct d_fault_attr_t *d_fault_attr_mem; -/* DFuse uses fault id 100 to force shutdown rather than mount after initialization - * is complete. +/* DFuse uses fault id 100 to force shutdown rather than mount after initialization is complete. * - * daos_init uses fault id 101 to disable memory faults for the duration of daos_init - * so that fault injection testing can avoid replicating coverage across multiple tests. + * daos_init uses fault id 101 to disable memory faults for the duration of daos_init so that fault + * injection testing can avoid replicating coverage across multiple tests. * * Other fault ids used by daos_engine are defined in src/include/daos/common.h */ struct d_fault_attr_t { + /** config id, used to select configuration from the fault_inject config file */ + uint32_t fa_id; /** - * config id, used to select configuration from the fault_inject config - * file - */ - uint32_t fa_id; - /** - * inject faults every n-th occurrence. If interval is set to 5 and - * probability is set to 20, fault injection only occurs on every 5-th - * hit of fault_id with a 20% probability. + * inject faults every n-th occurrence. If interval is set to 5 and probability is set to + * 20, fault injection only occurs on every 5-th hit of fault_id with a 20% probability. */ - uint32_t fa_interval; + uint32_t fa_interval; /** - * max number of faults to inject. 0 means unlimited. After max_faults - * is reached, no faults will be injected for fault_id. + * max number of faults to inject. 0 means unlimited. After max_faults is reached, no faults + * will be injected for fault_id. */ - uint64_t fa_max_faults; + uint64_t fa_max_faults; /** counter of injected faults */ - uint64_t fa_num_faults; + uint64_t fa_num_faults; /** number of times this injection point has been evaluated */ - uint64_t fa_num_hits; + uint64_t fa_num_hits; /** argument string. Interpretation of content is up to the user */ - char *fa_argument; + char *fa_argument; /** spin lock to protect this struct */ - pthread_spinlock_t fa_lock; + pthread_spinlock_t fa_lock; + /** the error code to inject. Can be retrieved by d_fault_attr_err_code() */ + int32_t fa_err_code; /** - * the error code to inject. Can be retrieved by d_fault_attr_err_code() + * state for nrand48. this allows each injection point has its own independent random number + * sequence. */ - int32_t fa_err_code; - /** - * state for nrand48. this allows each injection point has its own - * independent random number sequence. - */ - unsigned short fa_rand_state[3]; + unsigned short fa_rand_state[3]; /** * the frequency faults should be injected, calculated by: * @@ -89,55 +79,69 @@ struct d_fault_attr_t { * e.g. fa_probability_x = 123, fa_probability_y = 1000 * means faults will be injected randomly with frequency 12.3% */ - uint32_t fa_probability_x; - uint32_t fa_probability_y; + uint32_t fa_probability_x; + uint32_t fa_probability_y; }; /** - * Initialize the fault injection framework, injection attributes are read from - * the config file + * Initialize the fault injection framework, injection attributes are read from the config file * * \return DER_SUCCESS on success, negative value on error */ -int d_fault_inject_init(void); +int +d_fault_inject_init(void); /** * Finalize the fault injection framework * * \return DER_SUCCESS on success, negative value on error */ -int d_fault_inject_fini(void); +int +d_fault_inject_fini(void); /** * Start injecting faults. * * \return DER_SUCCESS on success, -DER_NOSYS if not supported */ -int d_fault_inject_enable(void); +int +d_fault_inject_enable(void); /** * Stop injecting faults. * * \return DER_SUCCESS on success, -DER_NOSYS if not supported */ -int d_fault_inject_disable(void); +int +d_fault_inject_disable(void); -bool d_fault_inject_is_enabled(void); +bool +d_fault_inject_is_enabled(void); -bool d_should_fail(struct d_fault_attr_t *fault_attr_ptr); +/** + * Enable/disable per thread. Sets if faults are enabled on the calling thread. + */ +void +d_fault_inject_thread_enable(bool enabled); /** - * use this macro to determine if a fault should be injected at a specific call - * site + * Enable/disable per thread for threads which haven't called d_fault_inject_thread_enable(). + * Default value here can be set via 'thread_default' in the input file. */ -#define D_SHOULD_FAIL(fault_attr) \ - ({ \ - bool __rb; \ - __rb = d_fault_inject && d_should_fail(fault_attr); \ - if (__rb) \ - D_WARN("fault_id %d, injecting fault.\n", \ - fault_attr->fa_id); \ - __rb; \ +void +d_fault_inject_thread_default_enable(bool enabled); + +bool +d_should_fail(struct d_fault_attr_t *fault_attr_ptr); + +/** use this macro to determine if a fault should be injected at a specific call site */ +#define D_SHOULD_FAIL(fault_attr) \ + ({ \ + bool __rb; \ + __rb = d_fault_inject && d_should_fail(fault_attr); \ + if (__rb) \ + D_WARN("fault_id %d, injecting fault.\n", fault_attr->fa_id); \ + __rb; \ }) /**