[OpenMP][OMPT] Add "expected fail" testcases to libomptest

Added macro `OMPTTESTCASE_XFAIL` to allow for expected fails Moved macros into separate header file Added `printSummary` function which prints an execution summary Fixed smaller issues Change-Id: Ib19097642afa9e68522290aba4c3dd336a8ab290
ROCm · Oct 25, 2023 · 35c7ec5 · 35c7ec5
1 parent 560e0b7
commit 35c7ec5
Show file tree

Hide file tree

Showing 8 changed files with 150 additions and 76 deletions.
diff --git a/openmp/libomptarget/test/ompTest/CMakeLists.txt b/openmp/libomptarget/test/ompTest/CMakeLists.txt
@@ -6,6 +6,7 @@
 add_llvm_library(omptest
 SHARED
 
+./include/AssertMacros.h
 ./include/InternalEvent.h
 ./include/InternalEventCommon.h
 ./include/OmptAsserter.h

diff --git a/openmp/libomptarget/test/ompTest/README.md b/openmp/libomptarget/test/ompTest/README.md
@@ -9,13 +9,15 @@ TBD
 Testing macros
 ==============
 
-## OMPT_SEQ_ASSERT
+## OMPT_ASSERT_SEQUENCE
 TBD
 
-## OMPT_ASSERT_GROUPED_LIST
+## OMPT_ASSERT_NAMED_SEQUENCE
+This macro will act like `OMPT_ASSERT_SEQUENCE` but internally the provided name
+will be stored within the created event and may be printed in messages.
 
-## OMPT_ASSERT_GROUPED_LIST
-This macro will act like `OMPT_SEQ_ASSERT` with the addition of grouping.
+## OMPT_ASSERT_GROUPED_SEQUENCE
+This macro will act like `OMPT_ASSERT_SEQUENCE` with the addition of grouping.
 
 ### Grouping
 

diff --git a/openmp/libomptarget/test/ompTest/include/AssertMacros.h b/openmp/libomptarget/test/ompTest/include/AssertMacros.h
@@ -0,0 +1,78 @@
+#ifndef OPENMP_LIBOMPTARGET_TEST_OMPTEST_ASSERTMACROS_H
+#define OPENMP_LIBOMPTARGET_TEST_OMPTEST_ASSERTMACROS_H
+
+#define XQUOTE(str) QUOTE(str)
+#define QUOTE(str) #str
+
+/// ASSERT MACROS TO BE USED BY THE USER
+
+// Not implemented yet
+#define OMPT_ASSERT_EVENT(Event, ...)
+#define OMPT_ASSERT_SEQUENCE_NOT(Event, ...)
+
+// Handle a minimum unordered set of events
+#define OMPT_ASSERT_SET_EVENT(Name, Group, EventTy, ...)                       \
+  this->SetAsserter.insert(OmptAssertEvent::EventTy(Name, Group, __VA_ARGS__));
+#define OMPT_ASSERT_SET(EventTy, ...)                                          \
+  OMPT_ASSERT_SET_EVENT("", "", EventTy, __VA_ARGS__)
+#define OMPT_ASSERT_SET_GROUP(Group, EventTy, ...)                             \
+  OMPT_ASSERT_SET_EVENT("", Group, EventTy, __VA_ARGS__)
+#define OMPT_ASSERT_SET_NAMED(Name, EventTy, ...)                              \
+  OMPT_ASSERT_SET_EVENT(Name, "", EventTy, __VA_ARGS__)
+
+// Handle an exact sequence of events
+#define OMPT_ASSERT_SEQUENCE_EVENT(Name, Group, EventTy, ...)                  \
+  this->SequenceAsserter.insert(                                               \
+      OmptAssertEvent::EventTy(Name, Group, __VA_ARGS__));
+#define OMPT_ASSERT_SEQUENCE(EventTy, ...)                                     \
+  OMPT_ASSERT_SEQUENCE_EVENT("", "", EventTy, __VA_ARGS__)
+#define OMPT_ASSERT_GROUPED_SEQUENCE(Group, EventTy, ...)                      \
+  OMPT_ASSERT_SEQUENCE_EVENT("", Group, EventTy, __VA_ARGS__)
+#define OMPT_ASSERT_NAMED_SEQUENCE(Name, EventTy, ...)                         \
+  OMPT_ASSERT_SEQUENCE_EVENT(Name, "", EventTy, __VA_ARGS__)
+
+// Enable / disable asserters entirely
+#define OMPT_ASSERTER_DISABLE(AsserterName) this->AsserterName.setActive(false);
+#define OMPT_ASSERTER_ENABLE(AsserterName) this->AsserterName.setActive(true);
+#define OMPT_ASSERT_SET_DISABLE() OMPT_ASSERTER_DISABLE(SetAsserter)
+#define OMPT_ASSERT_SET_ENABLE() OMPT_ASSERTER_ENABLE(SetAsserter)
+#define OMPT_REPORT_EVENT_DISABLE() OMPT_ASSERTER_DISABLE(EventReporter)
+#define OMPT_REPORT_EVENT_ENABLE() OMPT_ASSERTER_ENABLE(EventReporter)
+#define OMPT_ASSERT_SEQUENCE_DISABLE() OMPT_ASSERTER_DISABLE(SequenceAsserter)
+#define OMPT_ASSERT_SEQUENCE_ENABLE() OMPT_ASSERTER_ENABLE(SequenceAsserter)
+
+// Enable / disable certain event types for asserters
+#define OMPT_ASSERTER_PERMIT_EVENT(Asserter, EventTy)                          \
+  this->Asserter.permitEvent(EventTy);
+#define OMPT_ASSERTER_SUPPRESS_EVENT(Asserter, EventTy)                        \
+  this->Asserter.suppressEvent(EventTy);
+#define OMPT_PERMIT_EVENT(EventTy)                                             \
+  OMPT_ASSERTER_PERMIT_EVENT(SetAsserter, EventTy);                            \
+  OMPT_ASSERTER_PERMIT_EVENT(EventReporter, EventTy);                          \
+  OMPT_ASSERTER_PERMIT_EVENT(SequenceAsserter, EventTy);
+#define OMPT_SUPPRESS_EVENT(EventTy)                                           \
+  OMPT_ASSERTER_SUPPRESS_EVENT(SetAsserter, EventTy);                          \
+  OMPT_ASSERTER_SUPPRESS_EVENT(EventReporter, EventTy);                        \
+  OMPT_ASSERTER_SUPPRESS_EVENT(SequenceAsserter, EventTy);
+
+/// MACROS TO DEFINE A TESTSUITE + TESTCASE (like GoogleTest does)
+#define OMPTTESTCASE(SuiteName, CaseName)                                      \
+  struct SuiteName##_##CaseName : public TestCase {                            \
+    SuiteName##_##CaseName() : TestCase(XQUOTE(CaseName)) {}                   \
+    virtual void execImpl() override;                                          \
+  };                                                                           \
+  static Registerer R_##SuiteName##CaseName(new SuiteName##_##CaseName(),      \
+                                            #SuiteName);                       \
+  void SuiteName##_##CaseName::execImpl()
+
+#define OMPTTESTCASE_XFAIL(SuiteName, CaseName)                                \
+  struct SuiteName##_##CaseName : public TestCase {                            \
+    SuiteName##_##CaseName()                                                   \
+        : TestCase(XQUOTE(CaseName), omptest::AssertState::fail) {}            \
+    virtual void execImpl() override;                                          \
+  };                                                                           \
+  static Registerer R_##SuiteName##CaseName(new SuiteName##_##CaseName(),      \
+                                            #SuiteName);                       \
+  void SuiteName##_##CaseName::execImpl()
+
+#endif // include guard
diff --git a/openmp/libomptarget/test/ompTest/include/OmptAliases.h b/openmp/libomptarget/test/ompTest/include/OmptAliases.h
@@ -22,7 +22,7 @@ constexpr ompt_scope_endpoint_t BEGIN = ompt_scope_begin;
 constexpr ompt_scope_endpoint_t END = ompt_scope_end;
 constexpr ompt_scope_endpoint_t BEGINEND = ompt_scope_beginend;
 
-/// Aliases for enum: ompt_scope_endpoint_t
+/// Aliases for enum: ompt_target_t
 constexpr ompt_target_t TARGET = ompt_target;
 constexpr ompt_target_t ENTER_DATA = ompt_target_enter_data;
 constexpr ompt_target_t EXIT_DATA = ompt_target_exit_data;

diff --git a/openmp/libomptarget/test/ompTest/include/OmptAssertEvent.h b/openmp/libomptarget/test/ompTest/include/OmptAssertEvent.h
@@ -166,8 +166,6 @@ struct OmptAssertEvent {
 
   internal::EventTy getEventType() const;
 
-  // ToDo: Maybe look for another way to get access to or copy the underlying
-  // event. Note: This can easily result in slicing!
   internal::InternalEvent *getEvent() const;
 
   /// Make events comparable

diff --git a/openmp/libomptarget/test/ompTest/include/OmptAsserter.h b/openmp/libomptarget/test/ompTest/include/OmptAsserter.h
@@ -83,7 +83,7 @@ class OmptAsserter : public OmptListener {
   std::map<std::string, omptest::AssertEventGroup> EventGroups{};
 };
 
-/// Class that can assert in a sequenced fashion, i.e., events hace to occur in
+/// Class that can assert in a sequenced fashion, i.e., events have to occur in
 /// the order they were registered
 struct OmptSequencedAsserter : public OmptAsserter {
   OmptSequencedAsserter() : NextEvent(0), Events() {}
@@ -110,6 +110,12 @@ struct OmptEventAsserter : public OmptAsserter {
   /// Implements the asserter's logic
   virtual void notifyImpl(omptest::OmptAssertEvent &&AE) override;
 
+  omptest::AssertState getState() override;
+
+  int NumAssertSuccesses{0};
+  int NumNotifications{0};
+  size_t NumEvents{0};
+
   /// For now use vector (but do set semantics)
   std::vector<omptest::OmptAssertEvent> Events; // TODO std::unordered_set?
 };

diff --git a/openmp/libomptarget/test/ompTest/include/OmptTester.h b/openmp/libomptarget/test/ompTest/include/OmptTester.h
@@ -1,12 +1,14 @@
 #ifndef OPENMP_LIBOMPTARGET_TEST_OMPTEST_OMPTTESTER_H
 #define OPENMP_LIBOMPTARGET_TEST_OMPTEST_OMPTTESTER_H
 
+#include "AssertMacros.h"
 #include "OmptAliases.h"
 #include "OmptAssertEvent.h"
 #include "OmptAsserter.h"
 #include "OmptCallbackHandler.h"
 
 #include <cassert>
+#include <iomanip>
 #include <iostream>
 #include <memory>
 #include <string>
@@ -16,10 +18,6 @@
 
 #include <omp-tools.h>
 
-#define XQUOTE(str) QUOTE(str)
-#define QUOTE(str) #str
-#define BAR_IT(Id) Id##_
-
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -41,9 +39,12 @@ struct Error {
 /// A pretty crude test case abstraction
 struct TestCase {
   TestCase(const std::string &name) : Name(name) {}
+  TestCase(const std::string &name, const omptest::AssertState &expected)
+      : Name(name), ExpectedState(expected) {}
   virtual ~TestCase() {}
   std::string Name;
-  omptest::AssertState AS;
+  omptest::AssertState ExpectedState{omptest::AssertState::pass};
+  omptest::AssertState ResultState{omptest::AssertState::pass};
   Error exec() {
     OmptCallbackHandler::get().subscribe(&SequenceAsserter);
     OmptCallbackHandler::get().subscribe(&SetAsserter);
@@ -56,8 +57,20 @@ struct TestCase {
     // We remove subscribers to not be notified of events after our test case
     // finished.
     OmptCallbackHandler::get().clearSubscribers();
-    if (SequenceAsserter.getState() == omptest::AssertState::fail)
+
+    bool AnyFail = SequenceAsserter.getState() == omptest::AssertState::fail ||
+                   SetAsserter.getState() == omptest::AssertState::fail;
+    bool AllPass = SequenceAsserter.getState() == omptest::AssertState::pass &&
+                   SetAsserter.getState() == omptest::AssertState::pass;
+
+    if (ExpectedState == omptest::AssertState::pass && AnyFail)
+      E.Fail = true;
+    else if (ExpectedState == omptest::AssertState::fail && AllPass)
       E.Fail = true;
+
+    if (AnyFail)
+      ResultState = omptest::AssertState::fail;
+
     return E;
   };
 
@@ -148,74 +161,34 @@ struct Runner {
       }
       TS.teardown();
     }
+
+    printSummary();
+
     return 0;
   }
 
   void reportError(const Error &Err) {}
   void abortOrKeepGoing() {}
 
+  // Print an execution summary of all testsuites and their corresponding
+  // testcases.
+  void printSummary() {
+    std::cout << "\n====== SUMMARY\n";
+    for (auto &TS : TestSuites) {
+      std::cout << "  - " << TS.Name;
+      for (auto &TC : TS) {
+        std::string Result =
+            (TC->ResultState == TC->ExpectedState)
+                ? (TC->ResultState == omptest::AssertState::pass) ? "PASS"
+                                                                  : "XFAIL"
+                : "FAIL";
+        std::cout << "\n      " << std::setw(5) << Result << " : " << TC->Name;
+      }
+      std::cout << std::endl;
+    }
+  }
+
   std::vector<TestSuite> TestSuites;
 };
 
-/// ASSERT MACROS TO BE USED BY THE USER
-
-// Not implemented yet
-#define OMPT_ASSERT_EVENT(Event, ...)
-#define OMPT_ASSERT_SEQUENCE_NOT(Event, ...)
-
-// Handle a minimum unordered set of events
-#define OMPT_ASSERT_SET_EVENT(Name, Group, EventTy, ...)                       \
-  this->SetAsserter.insert(OmptAssertEvent::EventTy(Name, Group, __VA_ARGS__));
-#define OMPT_ASSERT_SET(EventTy, ...)                                          \
-  OMPT_ASSERT_SET_EVENT("", "", EventTy, __VA_ARGS__)
-#define OMPT_ASSERT_SET_GROUP(Group, EventTy, ...)                             \
-  OMPT_ASSERT_SET_EVENT("", Group, EventTy, __VA_ARGS__)
-#define OMPT_ASSERT_SET_NAMED(Name, EventTy, ...)                              \
-  OMPT_ASSERT_SET_EVENT(Name, "", EventTy, __VA_ARGS__)
-
-// Handle an exact sequence of events
-#define OMPT_ASSERT_SEQUENCE_EVENT(Name, Group, EventTy, ...)                  \
-  this->SequenceAsserter.insert(                                               \
-      OmptAssertEvent::EventTy(Name, Group, __VA_ARGS__));
-#define OMPT_ASSERT_SEQUENCE(EventTy, ...)                                     \
-  OMPT_ASSERT_SEQUENCE_EVENT("", "", EventTy, __VA_ARGS__)
-#define OMPT_ASSERT_GROUPED_SEQUENCE(Group, EventTy, ...)                      \
-  OMPT_ASSERT_SEQUENCE_EVENT("", Group, EventTy, __VA_ARGS__)
-#define OMPT_ASSERT_NAMED_SEQUENCE(Name, EventTy, ...)                         \
-  OMPT_ASSERT_SEQUENCE_EVENT(Name, "", EventTy, __VA_ARGS__)
-
-// Enable / disable asserters entirley
-#define OMPT_ASSERTER_DISABLE(AsserterName) this->AsserterName.setActive(false);
-#define OMPT_ASSERTER_ENABLE(AsserterName) this->AsserterName.setActive(true);
-#define OMPT_ASSERT_SET_DISABLE() OMPT_ASSERTER_DISABLE(SetAsserter)
-#define OMPT_ASSERT_SET_ENABLE() OMPT_ASSERTER_ENABLE(SetAsserter)
-#define OMPT_REPORT_EVENT_DISABLE() OMPT_ASSERTER_DISABLE(EventReporter)
-#define OMPT_REPORT_EVENT_ENABLE() OMPT_ASSERTER_ENABLE(EventReporter)
-#define OMPT_ASSERT_SEQUENCE_DISABLE() OMPT_ASSERTER_DISABLE(SequenceAsserter)
-#define OMPT_ASSERT_SEQUENCE_ENABLE() OMPT_ASSERTER_ENABLE(SequenceAsserter)
-
-// Enable / disable certain event types for asserters
-#define OMPT_ASSERTER_PERMIT_EVENT(Asserter, EventTy)                          \
-  this->Asserter.permitEvent(EventTy);
-#define OMPT_ASSERTER_SUPPRESS_EVENT(Asserter, EventTy)                        \
-  this->Asserter.suppressEvent(EventTy);
-#define OMPT_PERMIT_EVENT(EventTy)                                             \
-  OMPT_ASSERTER_PERMIT_EVENT(SetAsserter, EventTy);                            \
-  OMPT_ASSERTER_PERMIT_EVENT(EventReporter, EventTy);                          \
-  OMPT_ASSERTER_PERMIT_EVENT(SequenceAsserter, EventTy);
-#define OMPT_SUPPRESS_EVENT(EventTy)                                           \
-  OMPT_ASSERTER_SUPPRESS_EVENT(SetAsserter, EventTy);                          \
-  OMPT_ASSERTER_SUPPRESS_EVENT(EventReporter, EventTy);                        \
-  OMPT_ASSERTER_SUPPRESS_EVENT(SequenceAsserter, EventTy);
-
-/// MACRO TO DEFINE A TESTSUITE + TESTCASE (like GoogleTest does)
-#define OMPTTESTCASE(SuiteName, CaseName)                                      \
-  struct SuiteName##_##CaseName : public TestCase {                            \
-    SuiteName##_##CaseName() : TestCase(XQUOTE(CaseName)) {}                   \
-    virtual void execImpl() override;                                          \
-  };                                                                           \
-  static Registerer R_##SuiteName##CaseName(new SuiteName##_##CaseName(),      \
-                                            #SuiteName);                       \
-  void SuiteName##_##CaseName::execImpl()
-
 #endif // include guard
diff --git a/openmp/libomptarget/test/ompTest/src/OmptAsserter.cpp b/openmp/libomptarget/test/ompTest/src/OmptAsserter.cpp
@@ -178,7 +178,8 @@ void OmptSequencedAsserter::notifyImpl(OmptAssertEvent &&AE) {
 
 AssertState OmptSequencedAsserter::getState() {
   // This is called after the testcase executed.
-  // Once, reached, no more events should be in the queue
+  // Once reached the number of successful notifications should be equal to the
+  // number of expected events.
   if (NextEvent < Events.size())
     State = AssertState::fail;
 
@@ -193,14 +194,29 @@ void OmptEventAsserter::notifyImpl(OmptAssertEvent &&AE) {
   if (Events.empty() || !isActive() || isSuppressedEventType(AE.getEventType()))
     return;
 
+  if (NumEvents == 0)
+    NumEvents = Events.size();
+
+  ++NumNotifications;
+
   for (size_t I = 0; I < Events.size(); ++I) {
     if (Events[I] == AE) {
       Events.erase(Events.begin() + I);
+      ++NumAssertSuccesses;
       return;
     }
   }
 }
 
+AssertState OmptEventAsserter::getState() {
+  // This is called after the testcase executed.
+  // Once reached no more events should be in the queue
+  if (!Events.empty())
+    State = AssertState::fail;
+
+  return State;
+}
+
 void OmptEventReporter::notify(OmptAssertEvent &&AE) {
   if (!isActive() || isSuppressedEventType(AE.getEventType()))
     return;