From 35c7ec5b266c784512a09ae0cbaa685de776f667 Mon Sep 17 00:00:00 2001 From: Michael Halkenhaeuser Date: Tue, 24 Oct 2023 10:51:59 -0500 Subject: [PATCH] [OpenMP][OMPT] Add "expected fail" testcases to libomptest Added macro `OMPTTESTCASE_XFAIL` to allow for expected fails Moved macros into separate header file Added `printSummary` function which prints an execution summary Fixed smaller issues Change-Id: Ib19097642afa9e68522290aba4c3dd336a8ab290 --- .../libomptarget/test/ompTest/CMakeLists.txt | 1 + openmp/libomptarget/test/ompTest/README.md | 10 +- .../test/ompTest/include/AssertMacros.h | 78 +++++++++++++ .../test/ompTest/include/OmptAliases.h | 2 +- .../test/ompTest/include/OmptAssertEvent.h | 2 - .../test/ompTest/include/OmptAsserter.h | 8 +- .../test/ompTest/include/OmptTester.h | 107 +++++++----------- .../test/ompTest/src/OmptAsserter.cpp | 18 ++- 8 files changed, 150 insertions(+), 76 deletions(-) create mode 100644 openmp/libomptarget/test/ompTest/include/AssertMacros.h diff --git a/openmp/libomptarget/test/ompTest/CMakeLists.txt b/openmp/libomptarget/test/ompTest/CMakeLists.txt index 377731599cf701..4db5e31ca349d7 100644 --- a/openmp/libomptarget/test/ompTest/CMakeLists.txt +++ b/openmp/libomptarget/test/ompTest/CMakeLists.txt @@ -6,6 +6,7 @@ add_llvm_library(omptest SHARED +./include/AssertMacros.h ./include/InternalEvent.h ./include/InternalEventCommon.h ./include/OmptAsserter.h diff --git a/openmp/libomptarget/test/ompTest/README.md b/openmp/libomptarget/test/ompTest/README.md index 1b5fd9af8ead67..79eec2acf527f3 100644 --- a/openmp/libomptarget/test/ompTest/README.md +++ b/openmp/libomptarget/test/ompTest/README.md @@ -9,13 +9,15 @@ TBD Testing macros ============== -## OMPT_SEQ_ASSERT +## OMPT_ASSERT_SEQUENCE TBD -## OMPT_ASSERT_GROUPED_LIST +## OMPT_ASSERT_NAMED_SEQUENCE +This macro will act like `OMPT_ASSERT_SEQUENCE` but internally the provided name +will be stored within the created event and may be printed in messages. -## OMPT_ASSERT_GROUPED_LIST -This macro will act like `OMPT_SEQ_ASSERT` with the addition of grouping. +## OMPT_ASSERT_GROUPED_SEQUENCE +This macro will act like `OMPT_ASSERT_SEQUENCE` with the addition of grouping. ### Grouping diff --git a/openmp/libomptarget/test/ompTest/include/AssertMacros.h b/openmp/libomptarget/test/ompTest/include/AssertMacros.h new file mode 100644 index 00000000000000..0b4d39f0a59f17 --- /dev/null +++ b/openmp/libomptarget/test/ompTest/include/AssertMacros.h @@ -0,0 +1,78 @@ +#ifndef OPENMP_LIBOMPTARGET_TEST_OMPTEST_ASSERTMACROS_H +#define OPENMP_LIBOMPTARGET_TEST_OMPTEST_ASSERTMACROS_H + +#define XQUOTE(str) QUOTE(str) +#define QUOTE(str) #str + +/// ASSERT MACROS TO BE USED BY THE USER + +// Not implemented yet +#define OMPT_ASSERT_EVENT(Event, ...) +#define OMPT_ASSERT_SEQUENCE_NOT(Event, ...) + +// Handle a minimum unordered set of events +#define OMPT_ASSERT_SET_EVENT(Name, Group, EventTy, ...) \ + this->SetAsserter.insert(OmptAssertEvent::EventTy(Name, Group, __VA_ARGS__)); +#define OMPT_ASSERT_SET(EventTy, ...) \ + OMPT_ASSERT_SET_EVENT("", "", EventTy, __VA_ARGS__) +#define OMPT_ASSERT_SET_GROUP(Group, EventTy, ...) \ + OMPT_ASSERT_SET_EVENT("", Group, EventTy, __VA_ARGS__) +#define OMPT_ASSERT_SET_NAMED(Name, EventTy, ...) \ + OMPT_ASSERT_SET_EVENT(Name, "", EventTy, __VA_ARGS__) + +// Handle an exact sequence of events +#define OMPT_ASSERT_SEQUENCE_EVENT(Name, Group, EventTy, ...) \ + this->SequenceAsserter.insert( \ + OmptAssertEvent::EventTy(Name, Group, __VA_ARGS__)); +#define OMPT_ASSERT_SEQUENCE(EventTy, ...) \ + OMPT_ASSERT_SEQUENCE_EVENT("", "", EventTy, __VA_ARGS__) +#define OMPT_ASSERT_GROUPED_SEQUENCE(Group, EventTy, ...) \ + OMPT_ASSERT_SEQUENCE_EVENT("", Group, EventTy, __VA_ARGS__) +#define OMPT_ASSERT_NAMED_SEQUENCE(Name, EventTy, ...) \ + OMPT_ASSERT_SEQUENCE_EVENT(Name, "", EventTy, __VA_ARGS__) + +// Enable / disable asserters entirely +#define OMPT_ASSERTER_DISABLE(AsserterName) this->AsserterName.setActive(false); +#define OMPT_ASSERTER_ENABLE(AsserterName) this->AsserterName.setActive(true); +#define OMPT_ASSERT_SET_DISABLE() OMPT_ASSERTER_DISABLE(SetAsserter) +#define OMPT_ASSERT_SET_ENABLE() OMPT_ASSERTER_ENABLE(SetAsserter) +#define OMPT_REPORT_EVENT_DISABLE() OMPT_ASSERTER_DISABLE(EventReporter) +#define OMPT_REPORT_EVENT_ENABLE() OMPT_ASSERTER_ENABLE(EventReporter) +#define OMPT_ASSERT_SEQUENCE_DISABLE() OMPT_ASSERTER_DISABLE(SequenceAsserter) +#define OMPT_ASSERT_SEQUENCE_ENABLE() OMPT_ASSERTER_ENABLE(SequenceAsserter) + +// Enable / disable certain event types for asserters +#define OMPT_ASSERTER_PERMIT_EVENT(Asserter, EventTy) \ + this->Asserter.permitEvent(EventTy); +#define OMPT_ASSERTER_SUPPRESS_EVENT(Asserter, EventTy) \ + this->Asserter.suppressEvent(EventTy); +#define OMPT_PERMIT_EVENT(EventTy) \ + OMPT_ASSERTER_PERMIT_EVENT(SetAsserter, EventTy); \ + OMPT_ASSERTER_PERMIT_EVENT(EventReporter, EventTy); \ + OMPT_ASSERTER_PERMIT_EVENT(SequenceAsserter, EventTy); +#define OMPT_SUPPRESS_EVENT(EventTy) \ + OMPT_ASSERTER_SUPPRESS_EVENT(SetAsserter, EventTy); \ + OMPT_ASSERTER_SUPPRESS_EVENT(EventReporter, EventTy); \ + OMPT_ASSERTER_SUPPRESS_EVENT(SequenceAsserter, EventTy); + +/// MACROS TO DEFINE A TESTSUITE + TESTCASE (like GoogleTest does) +#define OMPTTESTCASE(SuiteName, CaseName) \ + struct SuiteName##_##CaseName : public TestCase { \ + SuiteName##_##CaseName() : TestCase(XQUOTE(CaseName)) {} \ + virtual void execImpl() override; \ + }; \ + static Registerer R_##SuiteName##CaseName(new SuiteName##_##CaseName(), \ + #SuiteName); \ + void SuiteName##_##CaseName::execImpl() + +#define OMPTTESTCASE_XFAIL(SuiteName, CaseName) \ + struct SuiteName##_##CaseName : public TestCase { \ + SuiteName##_##CaseName() \ + : TestCase(XQUOTE(CaseName), omptest::AssertState::fail) {} \ + virtual void execImpl() override; \ + }; \ + static Registerer R_##SuiteName##CaseName(new SuiteName##_##CaseName(), \ + #SuiteName); \ + void SuiteName##_##CaseName::execImpl() + +#endif // include guard \ No newline at end of file diff --git a/openmp/libomptarget/test/ompTest/include/OmptAliases.h b/openmp/libomptarget/test/ompTest/include/OmptAliases.h index 7681bf7909d238..5292addb6b7dfa 100644 --- a/openmp/libomptarget/test/ompTest/include/OmptAliases.h +++ b/openmp/libomptarget/test/ompTest/include/OmptAliases.h @@ -22,7 +22,7 @@ constexpr ompt_scope_endpoint_t BEGIN = ompt_scope_begin; constexpr ompt_scope_endpoint_t END = ompt_scope_end; constexpr ompt_scope_endpoint_t BEGINEND = ompt_scope_beginend; -/// Aliases for enum: ompt_scope_endpoint_t +/// Aliases for enum: ompt_target_t constexpr ompt_target_t TARGET = ompt_target; constexpr ompt_target_t ENTER_DATA = ompt_target_enter_data; constexpr ompt_target_t EXIT_DATA = ompt_target_exit_data; diff --git a/openmp/libomptarget/test/ompTest/include/OmptAssertEvent.h b/openmp/libomptarget/test/ompTest/include/OmptAssertEvent.h index ac168d174095f4..9d454210b8e1bd 100644 --- a/openmp/libomptarget/test/ompTest/include/OmptAssertEvent.h +++ b/openmp/libomptarget/test/ompTest/include/OmptAssertEvent.h @@ -166,8 +166,6 @@ struct OmptAssertEvent { internal::EventTy getEventType() const; - // ToDo: Maybe look for another way to get access to or copy the underlying - // event. Note: This can easily result in slicing! internal::InternalEvent *getEvent() const; /// Make events comparable diff --git a/openmp/libomptarget/test/ompTest/include/OmptAsserter.h b/openmp/libomptarget/test/ompTest/include/OmptAsserter.h index b8b648a3ad11a4..f4bb1e1461c730 100644 --- a/openmp/libomptarget/test/ompTest/include/OmptAsserter.h +++ b/openmp/libomptarget/test/ompTest/include/OmptAsserter.h @@ -83,7 +83,7 @@ class OmptAsserter : public OmptListener { std::map EventGroups{}; }; -/// Class that can assert in a sequenced fashion, i.e., events hace to occur in +/// Class that can assert in a sequenced fashion, i.e., events have to occur in /// the order they were registered struct OmptSequencedAsserter : public OmptAsserter { OmptSequencedAsserter() : NextEvent(0), Events() {} @@ -110,6 +110,12 @@ struct OmptEventAsserter : public OmptAsserter { /// Implements the asserter's logic virtual void notifyImpl(omptest::OmptAssertEvent &&AE) override; + omptest::AssertState getState() override; + + int NumAssertSuccesses{0}; + int NumNotifications{0}; + size_t NumEvents{0}; + /// For now use vector (but do set semantics) std::vector Events; // TODO std::unordered_set? }; diff --git a/openmp/libomptarget/test/ompTest/include/OmptTester.h b/openmp/libomptarget/test/ompTest/include/OmptTester.h index 631114b23c61fa..1cd7682b96ea31 100644 --- a/openmp/libomptarget/test/ompTest/include/OmptTester.h +++ b/openmp/libomptarget/test/ompTest/include/OmptTester.h @@ -1,12 +1,14 @@ #ifndef OPENMP_LIBOMPTARGET_TEST_OMPTEST_OMPTTESTER_H #define OPENMP_LIBOMPTARGET_TEST_OMPTEST_OMPTTESTER_H +#include "AssertMacros.h" #include "OmptAliases.h" #include "OmptAssertEvent.h" #include "OmptAsserter.h" #include "OmptCallbackHandler.h" #include +#include #include #include #include @@ -16,10 +18,6 @@ #include -#define XQUOTE(str) QUOTE(str) -#define QUOTE(str) #str -#define BAR_IT(Id) Id##_ - #ifdef __cplusplus extern "C" { #endif @@ -41,9 +39,12 @@ struct Error { /// A pretty crude test case abstraction struct TestCase { TestCase(const std::string &name) : Name(name) {} + TestCase(const std::string &name, const omptest::AssertState &expected) + : Name(name), ExpectedState(expected) {} virtual ~TestCase() {} std::string Name; - omptest::AssertState AS; + omptest::AssertState ExpectedState{omptest::AssertState::pass}; + omptest::AssertState ResultState{omptest::AssertState::pass}; Error exec() { OmptCallbackHandler::get().subscribe(&SequenceAsserter); OmptCallbackHandler::get().subscribe(&SetAsserter); @@ -56,8 +57,20 @@ struct TestCase { // We remove subscribers to not be notified of events after our test case // finished. OmptCallbackHandler::get().clearSubscribers(); - if (SequenceAsserter.getState() == omptest::AssertState::fail) + + bool AnyFail = SequenceAsserter.getState() == omptest::AssertState::fail || + SetAsserter.getState() == omptest::AssertState::fail; + bool AllPass = SequenceAsserter.getState() == omptest::AssertState::pass && + SetAsserter.getState() == omptest::AssertState::pass; + + if (ExpectedState == omptest::AssertState::pass && AnyFail) + E.Fail = true; + else if (ExpectedState == omptest::AssertState::fail && AllPass) E.Fail = true; + + if (AnyFail) + ResultState = omptest::AssertState::fail; + return E; }; @@ -148,74 +161,34 @@ struct Runner { } TS.teardown(); } + + printSummary(); + return 0; } void reportError(const Error &Err) {} void abortOrKeepGoing() {} + // Print an execution summary of all testsuites and their corresponding + // testcases. + void printSummary() { + std::cout << "\n====== SUMMARY\n"; + for (auto &TS : TestSuites) { + std::cout << " - " << TS.Name; + for (auto &TC : TS) { + std::string Result = + (TC->ResultState == TC->ExpectedState) + ? (TC->ResultState == omptest::AssertState::pass) ? "PASS" + : "XFAIL" + : "FAIL"; + std::cout << "\n " << std::setw(5) << Result << " : " << TC->Name; + } + std::cout << std::endl; + } + } + std::vector TestSuites; }; -/// ASSERT MACROS TO BE USED BY THE USER - -// Not implemented yet -#define OMPT_ASSERT_EVENT(Event, ...) -#define OMPT_ASSERT_SEQUENCE_NOT(Event, ...) - -// Handle a minimum unordered set of events -#define OMPT_ASSERT_SET_EVENT(Name, Group, EventTy, ...) \ - this->SetAsserter.insert(OmptAssertEvent::EventTy(Name, Group, __VA_ARGS__)); -#define OMPT_ASSERT_SET(EventTy, ...) \ - OMPT_ASSERT_SET_EVENT("", "", EventTy, __VA_ARGS__) -#define OMPT_ASSERT_SET_GROUP(Group, EventTy, ...) \ - OMPT_ASSERT_SET_EVENT("", Group, EventTy, __VA_ARGS__) -#define OMPT_ASSERT_SET_NAMED(Name, EventTy, ...) \ - OMPT_ASSERT_SET_EVENT(Name, "", EventTy, __VA_ARGS__) - -// Handle an exact sequence of events -#define OMPT_ASSERT_SEQUENCE_EVENT(Name, Group, EventTy, ...) \ - this->SequenceAsserter.insert( \ - OmptAssertEvent::EventTy(Name, Group, __VA_ARGS__)); -#define OMPT_ASSERT_SEQUENCE(EventTy, ...) \ - OMPT_ASSERT_SEQUENCE_EVENT("", "", EventTy, __VA_ARGS__) -#define OMPT_ASSERT_GROUPED_SEQUENCE(Group, EventTy, ...) \ - OMPT_ASSERT_SEQUENCE_EVENT("", Group, EventTy, __VA_ARGS__) -#define OMPT_ASSERT_NAMED_SEQUENCE(Name, EventTy, ...) \ - OMPT_ASSERT_SEQUENCE_EVENT(Name, "", EventTy, __VA_ARGS__) - -// Enable / disable asserters entirley -#define OMPT_ASSERTER_DISABLE(AsserterName) this->AsserterName.setActive(false); -#define OMPT_ASSERTER_ENABLE(AsserterName) this->AsserterName.setActive(true); -#define OMPT_ASSERT_SET_DISABLE() OMPT_ASSERTER_DISABLE(SetAsserter) -#define OMPT_ASSERT_SET_ENABLE() OMPT_ASSERTER_ENABLE(SetAsserter) -#define OMPT_REPORT_EVENT_DISABLE() OMPT_ASSERTER_DISABLE(EventReporter) -#define OMPT_REPORT_EVENT_ENABLE() OMPT_ASSERTER_ENABLE(EventReporter) -#define OMPT_ASSERT_SEQUENCE_DISABLE() OMPT_ASSERTER_DISABLE(SequenceAsserter) -#define OMPT_ASSERT_SEQUENCE_ENABLE() OMPT_ASSERTER_ENABLE(SequenceAsserter) - -// Enable / disable certain event types for asserters -#define OMPT_ASSERTER_PERMIT_EVENT(Asserter, EventTy) \ - this->Asserter.permitEvent(EventTy); -#define OMPT_ASSERTER_SUPPRESS_EVENT(Asserter, EventTy) \ - this->Asserter.suppressEvent(EventTy); -#define OMPT_PERMIT_EVENT(EventTy) \ - OMPT_ASSERTER_PERMIT_EVENT(SetAsserter, EventTy); \ - OMPT_ASSERTER_PERMIT_EVENT(EventReporter, EventTy); \ - OMPT_ASSERTER_PERMIT_EVENT(SequenceAsserter, EventTy); -#define OMPT_SUPPRESS_EVENT(EventTy) \ - OMPT_ASSERTER_SUPPRESS_EVENT(SetAsserter, EventTy); \ - OMPT_ASSERTER_SUPPRESS_EVENT(EventReporter, EventTy); \ - OMPT_ASSERTER_SUPPRESS_EVENT(SequenceAsserter, EventTy); - -/// MACRO TO DEFINE A TESTSUITE + TESTCASE (like GoogleTest does) -#define OMPTTESTCASE(SuiteName, CaseName) \ - struct SuiteName##_##CaseName : public TestCase { \ - SuiteName##_##CaseName() : TestCase(XQUOTE(CaseName)) {} \ - virtual void execImpl() override; \ - }; \ - static Registerer R_##SuiteName##CaseName(new SuiteName##_##CaseName(), \ - #SuiteName); \ - void SuiteName##_##CaseName::execImpl() - #endif // include guard diff --git a/openmp/libomptarget/test/ompTest/src/OmptAsserter.cpp b/openmp/libomptarget/test/ompTest/src/OmptAsserter.cpp index f17833a47804fa..e16cff223067af 100644 --- a/openmp/libomptarget/test/ompTest/src/OmptAsserter.cpp +++ b/openmp/libomptarget/test/ompTest/src/OmptAsserter.cpp @@ -178,7 +178,8 @@ void OmptSequencedAsserter::notifyImpl(OmptAssertEvent &&AE) { AssertState OmptSequencedAsserter::getState() { // This is called after the testcase executed. - // Once, reached, no more events should be in the queue + // Once reached the number of successful notifications should be equal to the + // number of expected events. if (NextEvent < Events.size()) State = AssertState::fail; @@ -193,14 +194,29 @@ void OmptEventAsserter::notifyImpl(OmptAssertEvent &&AE) { if (Events.empty() || !isActive() || isSuppressedEventType(AE.getEventType())) return; + if (NumEvents == 0) + NumEvents = Events.size(); + + ++NumNotifications; + for (size_t I = 0; I < Events.size(); ++I) { if (Events[I] == AE) { Events.erase(Events.begin() + I); + ++NumAssertSuccesses; return; } } } +AssertState OmptEventAsserter::getState() { + // This is called after the testcase executed. + // Once reached no more events should be in the queue + if (!Events.empty()) + State = AssertState::fail; + + return State; +} + void OmptEventReporter::notify(OmptAssertEvent &&AE) { if (!isActive() || isSuppressedEventType(AE.getEventType())) return;