pr-540 - [cgroups2] Introduces a Controller abstraction for cgroups v2.

src/CMakeLists.txt

@@ -355,7 +355,9 @@ endif ()
 if (ENABLE_CGROUPS_v2)
   list(APPEND LINUX_SRC
     linux/cgroups2.cpp
-    linux/ebpf.cpp)
+    linux/ebpf.cpp
+    slave/containerizer/mesos/isolators/cgroups2/controller.cpp)
+
 endif ()
 
 if (ENABLE_LAUNCHER_SEALING)

src/linux/cgroups2.cpp

@@ -58,6 +58,44 @@ Try<Nothing> write(
     const T& value);
 
 
+template <>
+Try<string> read(const string& cgroup, const string& control)
+{
+  return os::read(path::join(cgroups2::path(cgroup), control));
+}
+
+
+template <>
+Try<uint64_t> read(const string& cgroup, const string& control)
+{
+  Try<string> content = read<string>(cgroup, control);
+  if (content.isError()) {
+    return Error(content.error());
+  }
+
+  return numify<uint64_t>(strings::trim(*content));
+}
+
+
+template <>
+Try<Nothing> write(
+    const string& cgroup,
+    const string& control,
+    const string& value)
+{
+  return os::write(path::join(cgroups2::path(cgroup), control), value);
+}
+
+
+template <>
+Try<Nothing> write(
+    const string& cgroup,
+    const string& control,
+    const uint64_t& value)
+{
+  return write(cgroup, control, stringify(value));
+}
+
 namespace control {
 
 // Interface files found in all cgroups.
@@ -106,6 +144,13 @@ struct State
     _disabled.insert(controller);
   }
 
+  void disable(const set<string>& controllers)
+  {
+    foreach (const string& controller, controllers) {
+      disable(controller);
+    }
+  }
+
   set<string> enabled()  const { return _enabled; }
   set<string> disabled() const { return _disabled; }
 
@@ -148,48 +193,31 @@ std::ostream& operator<<(std::ostream& stream, const State& state)
   return stream;
 }
 
-} // namespace subtree_control {
 
-} // namespace control {
-
-
-template <>
-Try<string> read(const string& cgroup, const string& control)
+Try<State> read(const string& cgroup)
 {
-  return os::read(path::join(cgroups2::path(cgroup), control));
-}
-
+  Try<string> contents =
+    cgroups2::read<string>(cgroup, cgroups2::control::SUBTREE_CONTROLLERS);
 
-template <>
-Try<uint64_t> read(const string& cgroup, const string& control)
-{
-  Try<string> content = read<string>(cgroup, control);
-  if (content.isError()) {
-    return Error(content.error());
+  if (contents.isError()) {
+    return Error(
+        "Failed to read 'cgroup.subtree_control' for cgroup '" + cgroup + "': "
+        + contents.error());
   }
 
-  return numify<uint64_t>(strings::trim(*content));
+  return State::parse(*contents);
 }
 
 
-template <>
-Try<Nothing> write(
-    const string& cgroup,
-    const string& control,
-    const string& value)
+Try<Nothing> write(const string& cgroup, const State& state)
 {
-  return os::write(path::join(cgroups2::path(cgroup), control), value);
+  return cgroups2::write(
+      cgroup, control::SUBTREE_CONTROLLERS, stringify(state));
 }
 
+} // namespace subtree_control {
 
-template <>
-Try<Nothing> write(
-    const string& cgroup,
-    const string& control,
-    const uint64_t& value)
-{
-  return write(cgroup, control, stringify(value));
-}
+} // namespace control {
 
 
 bool enabled()
@@ -418,20 +446,29 @@ Try<set<string>> available(const string& cgroup)
 
 Try<Nothing> enable(const string& cgroup, const vector<string>& controllers)
 {
-  Try<string> contents =
-    cgroups2::read<string>(cgroup, cgroups2::control::SUBTREE_CONTROLLERS);
+  using State = control::subtree_control::State;
+  Try<State> control = cgroups2::control::subtree_control::read(cgroup);
 
-  if (contents.isError()) {
-    return Error(contents.error());
+  if (control.isError()) {
+    return Error(control.error());
   }
 
+  control->enable(controllers);
+  return cgroups2::control::subtree_control::write(cgroup, *control);
+}
+
+
+Try<Nothing> disable(const string& cgroup, const set<string>& controllers)
+{
   using State = control::subtree_control::State;
-  State control = State::parse(*contents);
-  control.enable(controllers);
-  return cgroups2::write(
-      cgroup,
-      control::SUBTREE_CONTROLLERS,
-      stringify(control));
+  Try<State> control = cgroups2::control::subtree_control::read(cgroup);
+
+  if (control.isError()) {
+    return Error(control.error());
+  }
+
+  control->disable(controllers);
+  return cgroups2::control::subtree_control::write(cgroup, *control);
 }
 
 
@@ -465,6 +502,48 @@ const std::string UCLAMP_MIN = "cpu.uclamp.min";
 const std::string WEIGHT = "cpu.weight";
 const std::string WEIGHT_NICE = "cpu.weight.nice";
 
+namespace stat {
+
+Try<Stats> parse(const string& content)
+{
+  const vector<string> lines = strings::split(content, "\n");
+  cpu::Stats stats;
+
+  foreach (const string& line, lines) {
+    if (line.empty()) {
+      continue;
+    }
+
+    vector<string> tokens = strings::split(line, " ");
+    if (tokens.size() != 2) {
+      return Error("Invalid line format in 'cpu.stat' expected "
+                   "<key> <value> received: '" + line + "'");
+    }
+
+    const string& field = tokens[0];
+    const string& value = tokens[1];
+
+    Try<uint64_t> number = numify<uint64_t>(value);
+    if (number.isError()) {
+      return Error("Failed to parse '" + field + "': " + number.error());
+    }
+    Duration duration = Microseconds(static_cast<int64_t>(*number));
+
+    if      (field == "usage_usec")     { stats.usage = duration; }
+    else if (field == "user_usec")      { stats.user_time = duration; }
+    else if (field == "system_usec")    { stats.system_time = duration; }
+    else if (field == "nr_periods")     { stats.periods = *number; }
+    else if (field == "nr_throttled")   { stats.throttled = *number; }
+    else if (field == "throttled_usec") { stats.throttle_time = duration; }
+    else if (field == "nr_burst")       { stats.bursts = *number; }
+    else if (field == "burst_usec")     { stats.bursts_time = duration; }
+  }
+
+  return stats;
+}
+
+} // namespace stat {
+
 } // namespace control {
 
 Try<Nothing> weight(const string& cgroup, uint64_t weight)
@@ -486,6 +565,20 @@ Try<uint64_t> weight(const string& cgroup)
   return cgroups2::read<uint64_t>(cgroup, cpu::control::WEIGHT);
 }
 
+
+Try<cpu::Stats> stats(const string& cgroup)
+{
+  Try<string> content = cgroups2::read<string>(
+      cgroup, cgroups2::cpu::control::STATS);
+
+  if (content.isError()) {
+    return Error("Failed to read 'cpu.stat' for the cgroup '" + cgroup + "': "
+                 + content.error());
+  }
+
+  return cpu::control::stat::parse(*content);
+}
+
 } // namespace cpu {
 
 namespace devices {

src/linux/cgroups2.hpp

@@ -21,7 +21,9 @@
 #include <string>
 #include <vector>
 
+#include <stout/duration.hpp>
 #include <stout/nothing.hpp>
+#include <stout/option.hpp>
 #include <stout/try.hpp>
 
 #include "linux/cgroups.hpp"
@@ -100,6 +102,12 @@ Try<Nothing> enable(
     const std::vector<std::string>& controllers);
 
 
+// Disables controllers in the cgroup. No-op if the controller is not enabled.
+Try<Nothing> disable(
+    const std::string& cgroup,
+    const std::set<std::string>& controllers);
+
+
 // Get all the controllers that are enabled for a cgroup.
 Try<std::set<std::string>> enabled(const std::string& cgroup);
 
@@ -116,6 +124,53 @@ Try<Nothing> weight(const std::string& cgroup, uint64_t weight);
 // Cannot be used for the root cgroup.
 Try<uint64_t> weight(const std::string& cgroup);
 
+
+// CPU usage statistics for a cgroup.
+// Represents a snapshot of the "cpu.stat" control.
+struct Stats
+{
+  // Number of scheduling periods that have elapsed.
+  // "nr_periods" key in "cpu.stat"
+  // Only available if the "cpu" controller is enabled.
+  Option<uint64_t> periods;
+
+  // Number of tasks that have been throttled.
+  // "nr_throttled" key in "cpu.stat"
+  // Only available if the "cpu" controller is enabled.
+  Option<uint64_t> throttled;
+
+  // Total time that tasks have been throttled.
+  // "throttled_usec" key in "cpu.stat"
+  // Only available if the "cpu" controller is enabled.
+  Option<Duration> throttle_time;
+
+  // Number of times tasks have exceeded their burst limit.
+  // "nr_burst" key in "cpu.stat"
+  // Only available if the "cpu" controller is enabled.
+  Option<uint64_t> bursts;
+
+  // Total time tasks have spent while exceeding their burst limit.
+  // "burst_usec" key in "cpu.stat"
+  // Only available if the "cpu" controller is enabled.
+  Option<Duration> bursts_time;
+
+  // Combined time spent in user and kernel space.
+  // "usage_usec" key in "cpu.stat"
+  Duration usage;
+
+  // Time spent in user space.
+  // "user_usec key" in "cpu.stat"
+  Duration user_time;
+
+  // Time spent in kernel space.
+  // "system_usec" key in "cpu.stat"
+  Duration system_time;
+};
+
+
+// Get the CPU usage statistics for a cgroup.
+Try<Stats> stats(const std::string& cgroup);
+
 } // namespace cpu {
 
 namespace devices {

src/slave/containerizer/mesos/isolators/cgroups2/cgroups2.cpp

@@ -31,19 +31,19 @@ namespace internal {
 namespace slave {
 
 Cgroups2IsolatorProcess::Cgroups2IsolatorProcess(
-  const hashmap<string, Owned<Subsystem>>& _subsystems)
-  : ProcessBase(process::ID::generate("cgroups2-isolator")),
-    subsystems(_subsystems) {}
+    const hashmap<string, Owned<Controller>>& _controllers)
+    : ProcessBase(process::ID::generate("cgroups2-isolator")),
+    controllers(_controllers) {}
 
 
 Cgroups2IsolatorProcess::~Cgroups2IsolatorProcess() {}
 
 
 Try<Isolator*> Cgroups2IsolatorProcess::create(const Flags& flags)
 {
-  hashmap<string, Owned<Subsystem>> subsystems;
+  hashmap<string, Owned<Controller>> controllers;
 
-  Owned<MesosIsolatorProcess> process(new Cgroups2IsolatorProcess(subsystems));
+  Owned<MesosIsolatorProcess> process(new Cgroups2IsolatorProcess(controllers));
   return new MesosIsolator(process);
 }
 

src/slave/containerizer/mesos/isolators/cgroups2/cgroups2.hpp

@@ -22,11 +22,10 @@
 #include <process/owned.hpp>
 
 #include <stout/hashmap.hpp>
-#include <stout/hashset.hpp>
 #include <stout/try.hpp>
 
 #include "slave/containerizer/mesos/isolator.hpp"
-#include "slave/containerizer/mesos/isolators/cgroups/subsystem.hpp"
+#include "slave/containerizer/mesos/isolators/cgroups2/controller.hpp"
 #include "slave/flags.hpp"
 
 namespace mesos {
@@ -46,10 +45,10 @@ class Cgroups2IsolatorProcess : public MesosIsolatorProcess
 
 private:
   Cgroups2IsolatorProcess(
-      const hashmap<std::string, process::Owned<Subsystem>>& _subsystems);
+      const hashmap<std::string, process::Owned<Controller>>& controllers);
 
-  // Maps each subsystems to the `Subsystem` isolator that manages it.
-  const hashmap<std::string, process::Owned<Subsystem>> subsystems;
+  // Maps each controller to the `Controller` isolator that manages it.
+  const hashmap<std::string, process::Owned<Controller>> controllers;
 };
 
 } // namespace slave {