Skip to content

Commit

Permalink
Sync cgroup v2 in libraries with coreclr (dotnet#34665)
Browse files Browse the repository at this point in the history 
This commit brings in two changes from coreclr to libraries:

1. dotnet#980

   "Fix named cgroup handling in docker"

   This fixes getting cgroup information for named cgroups inside containers.

2. dotnet#34334

   "Add cgroup v2 support to coreclr"

   This is essentially the same change pushed to corefx (now libraries)
   to add cgroupv2 support, but this newer coreclr change has one major
   difference: it determines whether the system is using cgroup v1 or
   cgroup v2 once, and then explicitly uses that (only). This avoids
   issues on systems where both cgroup v1 and v2 are enabled, (but only
   one is being used by default).
  • Loading branch information
@omajid
omajid authored Apr 14, 2020
1 parent d71cf79 commit 3d0ef96
Show file tree
Hide file tree
Showing 4 changed files with 175 additions and 83 deletions.
189 changes: 132 additions & 57 deletions src/libraries/Common/src/Interop/Linux/cgroups/Interop.cgroups.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,23 @@ internal static partial class cgroups
{
// For cgroup v1, see https://www.kernel.org/doc/Documentation/cgroup-v1/
// For cgroup v2, see https://www.kernel.org/doc/Documentation/cgroup-v2.txt
// For disambiguation, see https://systemd.io/CGROUP_DELEGATION/#three-different-tree-setups-

/// <summary>The version of cgroup that's being used </summary>
/// <summary>The supported versions of cgroup.</summary>
internal enum CGroupVersion { None, CGroup1, CGroup2 };

/// <summary>Path to cgroup filesystem that tells us which version of cgroup is in use.</summary>
private const string SysFsCgroupFileSystemPath = "/sys/fs/cgroup";
/// <summary>Path to mountinfo file in procfs for the current process.</summary>
private const string ProcMountInfoFilePath = "/proc/self/mountinfo";
/// <summary>Path to cgroup directory in procfs for the current process.</summary>
private const string ProcCGroupFilePath = "/proc/self/cgroup";

/// <summary>The version of cgroup that's being used. Mutated by tests only.</summary>
internal static readonly CGroupVersion s_cgroupVersion = FindCGroupVersion();

/// <summary>Path to the found cgroup memory limit path, or null if it couldn't be found.</summary>
internal static readonly string? s_cgroupMemoryLimitPath = FindCGroupMemoryLimitPath();
internal static readonly string? s_cgroupMemoryLimitPath = FindCGroupMemoryLimitPath(s_cgroupVersion);

/// <summary>Tries to read the memory limit from the cgroup memory location.</summary>
/// <param name="limit">The read limit, or 0 if it couldn't be read.</param>
Expand Down Expand Up @@ -102,19 +108,39 @@ internal static bool TryReadMemoryValueFromFile(string path, out ulong result)
return false;
}

/// <summary>Find the cgroup version in use on the system.</summary>
/// <returns>The cgroup version.</returns>
private static CGroupVersion FindCGroupVersion()
{
try
{
return new DriveInfo(SysFsCgroupFileSystemPath).DriveFormat switch
{
"cgroup2fs" => CGroupVersion.CGroup2,
"tmpfs" => CGroupVersion.CGroup1,
_ => CGroupVersion.None,
};
}
catch (Exception ex) when (ex is DriveNotFoundException || ex is ArgumentException)
{
return CGroupVersion.None;
}
}

/// <summary>Find the cgroup memory limit path.</summary>
/// <param name="cgroupVersion">The cgroup version currently in use on the system.</param>
/// <returns>The limit path if found; otherwise, null.</returns>
private static string? FindCGroupMemoryLimitPath()
private static string? FindCGroupMemoryLimitPath(CGroupVersion cgroupVersion)
{
string? cgroupMemoryPath = FindCGroupPath("memory", out CGroupVersion version);
string? cgroupMemoryPath = FindCGroupPath(cgroupVersion, "memory");
if (cgroupMemoryPath != null)
{
if (version == CGroupVersion.CGroup1)
if (cgroupVersion == CGroupVersion.CGroup1)
{
return cgroupMemoryPath + "/memory.limit_in_bytes";
}

if (version == CGroupVersion.CGroup2)
if (cgroupVersion == CGroupVersion.CGroup2)
{
// 'memory.high' is a soft limit; the process may get throttled
// 'memory.max' is where OOM killer kicks in
Expand All @@ -126,34 +152,71 @@ internal static bool TryReadMemoryValueFromFile(string path, out ulong result)
}

/// <summary>Find the cgroup path for the specified subsystem.</summary>
/// <param name="cgroupVersion">The cgroup version currently in use on the system.</param>
/// <param name="subsystem">The subsystem, e.g. "memory".</param>
/// <returns>The cgroup path if found; otherwise, null.</returns>
private static string? FindCGroupPath(string subsystem, out CGroupVersion version)
private static string? FindCGroupPath(CGroupVersion cgroupVersion, string subsystem)
{
if (TryFindHierarchyMount(subsystem, out version, out string? hierarchyRoot, out string? hierarchyMount) &&
TryFindCGroupPathForSubsystem(subsystem, out string? cgroupPathRelativeToMount))
if (cgroupVersion == CGroupVersion.None)
{
return null;
}

if (TryFindHierarchyMount(cgroupVersion, subsystem, out string? hierarchyRoot, out string? hierarchyMount) &&
TryFindCGroupPathForSubsystem(cgroupVersion, subsystem, out string? cgroupPathRelativeToMount))
{
// For a host cgroup, we need to append the relative path.
// In a docker container, the root and relative path are the same and we don't need to append.
return (hierarchyRoot != cgroupPathRelativeToMount) ?
hierarchyMount + cgroupPathRelativeToMount :
hierarchyMount;
return FindCGroupPath(hierarchyRoot, hierarchyMount, cgroupPathRelativeToMount);
}

return null;
}

internal static string FindCGroupPath(string hierarchyRoot, string hierarchyMount, string cgroupPathRelativeToMount)
{
// For a host cgroup, we need to append the relative path.
// The root and cgroup path can share a common prefix of the path that should not be appended.
// Example 1 (docker):
// hierarchyMount: /sys/fs/cgroup/cpu
// hierarchyRoot: /docker/87ee2de57e51bc75175a4d2e81b71d162811b179d549d6601ed70b58cad83578
// cgroupPathRelativeToMount: /docker/87ee2de57e51bc75175a4d2e81b71d162811b179d549d6601ed70b58cad83578/my_named_cgroup
// append to the cgroupPath: /my_named_cgroup
// final cgroupPath: /sys/fs/cgroup/cpu/my_named_cgroup
//
// Example 2 (out of docker)
// hierarchyMount: /sys/fs/cgroup/cpu
// hierarchyRoot: /
// cgroupPathRelativeToMount: /my_named_cgroup
// append to the cgroupPath: /my_named_cgroup
// final cgroupPath: /sys/fs/cgroup/cpu/my_named_cgroup

int commonPathPrefixLength = hierarchyRoot.Length;
if ((commonPathPrefixLength == 1) || !cgroupPathRelativeToMount.StartsWith(hierarchyRoot, StringComparison.Ordinal))
{
commonPathPrefixLength = 0;
}

return string.Concat(hierarchyMount, cgroupPathRelativeToMount.AsSpan(commonPathPrefixLength));
}

/// <summary>Find the cgroup mount information for the specified subsystem.</summary>
/// <param name="cgroupVersion">The cgroup version currently in use on the system.</param>
/// <param name="subsystem">The subsystem, e.g. "memory".</param>
/// <param name="root">The path of the directory in the filesystem which forms the root of this mount; null if not found.</param>
/// <param name="path">The path of the mount point relative to the process's root directory; null if not found.</param>
/// <returns>true if the mount was found; otherwise, null.</returns>
private static bool TryFindHierarchyMount(string subsystem, out CGroupVersion version, [NotNullWhen(true)] out string? root, [NotNullWhen(true)] out string? path)
private static bool TryFindHierarchyMount(CGroupVersion cgroupVersion, string subsystem, [NotNullWhen(true)] out string? root, [NotNullWhen(true)] out string? path)
{
return TryFindHierarchyMount(ProcMountInfoFilePath, subsystem, out version, out root, out path);
return TryFindHierarchyMount(cgroupVersion, ProcMountInfoFilePath, subsystem, out root, out path);
}

internal static bool TryFindHierarchyMount(string mountInfoFilePath, string subsystem, out CGroupVersion version, [NotNullWhen(true)] out string? root, [NotNullWhen(true)] out string? path)
/// <summary>Find the cgroup mount information for the specified subsystem.</summary>
/// <param name="cgroupVersion">The cgroup version currently in use on the system.</param>
/// <param name="mountInfoFilePath">The path to the /mountinfo file. Useful for tests.</param>
/// <param name="subsystem">The subsystem, e.g. "memory".</param>
/// <param name="root">The path of the directory in the filesystem which forms the root of this mount; null if not found.</param>
/// <param name="path">The path of the mount point relative to the process's root directory; null if not found.</param>
/// <returns>true if the mount was found; otherwise, null.</returns>
internal static bool TryFindHierarchyMount(CGroupVersion cgroupVersion, string mountInfoFilePath, string subsystem, [NotNullWhen(true)] out string? root, [NotNullWhen(true)] out string? path)
{
if (File.Exists(mountInfoFilePath))
{
Expand Down Expand Up @@ -188,31 +251,30 @@ internal static bool TryFindHierarchyMount(string mountInfoFilePath, string subs
continue;
}

bool validCGroup1Entry = ((postSeparatorlineParts[0] == "cgroup") &&
(Array.IndexOf(postSeparatorlineParts[2].Split(','), subsystem) >= 0));
bool validCGroup2Entry = postSeparatorlineParts[0] == "cgroup2";

if (!validCGroup1Entry && !validCGroup2Entry)
if (cgroupVersion == CGroupVersion.CGroup1)
{
// Not the relevant entry.
continue;
bool validCGroup1Entry = ((postSeparatorlineParts[0] == "cgroup") &&
(Array.IndexOf(postSeparatorlineParts[2].Split(','), subsystem) >= 0));
if (!validCGroup1Entry)
{
continue;
}
}
else if (cgroupVersion == CGroupVersion.CGroup2)
{
bool validCGroup2Entry = postSeparatorlineParts[0] == "cgroup2";
if (!validCGroup2Entry)
{
continue;
}

// Found the relevant entry. Extract the cgroup version, mount root and path.
switch (postSeparatorlineParts[0])
}
else
{
case "cgroup":
version = CGroupVersion.CGroup1;
break;
case "cgroup2":
version = CGroupVersion.CGroup2;
break;
default:
version = CGroupVersion.None;
Debug.Fail($"invalid value for CGroupVersion \"{postSeparatorlineParts[0]}\"");
break;
Debug.Fail($"Unexpected cgroup version \"{cgroupVersion}\"");
}


string[] lineParts = line.Substring(0, endOfOptionalFields).Split(' ');
root = lineParts[3];
path = lineParts[4];
Expand All @@ -227,22 +289,27 @@ internal static bool TryFindHierarchyMount(string mountInfoFilePath, string subs
}
}

version = CGroupVersion.None;
root = null;
path = null;
return false;
}

/// <summary>Find the cgroup relative path for the specified subsystem.</summary>
/// <param name="cgroupVersion">The cgroup version currently in use on the system.</param>
/// <param name="subsystem">The subsystem, e.g. "memory".</param>
/// <param name="path">The found path, or null if it couldn't be found.</param>
/// <returns></returns>
private static bool TryFindCGroupPathForSubsystem(string subsystem, [NotNullWhen(true)] out string? path)
/// <returns>true if a cgroup path for the subsystem is found.</returns>
private static bool TryFindCGroupPathForSubsystem(CGroupVersion cgroupVersion, string subsystem, [NotNullWhen(true)] out string? path)
{
return TryFindCGroupPathForSubsystem(ProcCGroupFilePath, subsystem, out path);
return TryFindCGroupPathForSubsystem(cgroupVersion, ProcCGroupFilePath, subsystem, out path);
}

internal static bool TryFindCGroupPathForSubsystem(string procCGroupFilePath, string subsystem, [NotNullWhen(true)] out string? path)
/// <summary>Find the cgroup relative path for the specified subsystem.</summary>
/// <param name="cgroupVersion">The cgroup version currently in use on the system.</param>
/// <param name="subsystem">The subsystem, e.g. "memory".</param>
/// <param name="path">The found path, or null if it couldn't be found.</param>
/// <returns>true if a cgroup path for the subsystem is found.</returns>
internal static bool TryFindCGroupPathForSubsystem(CGroupVersion cgroupVersion, string procCGroupFilePath, string subsystem, [NotNullWhen(true)] out string? path)
{
if (File.Exists(procCGroupFilePath))
{
Expand All @@ -261,28 +328,36 @@ internal static bool TryFindCGroupPathForSubsystem(string procCGroupFilePath, st
continue;
}

// cgroup v2: Find the first entry that matches the cgroup v2 hierarchy:
// 0::$PATH

if ((lineParts[0] == "0") && (string.Empty == lineParts[1]))
if (cgroupVersion == CGroupVersion.CGroup1)
{
// cgroup v1: Find the first entry that has the subsystem listed in its controller
// list. See man page for cgroups for /proc/[pid]/cgroups format, e.g:
// hierarchy-ID:controller-list:cgroup-path
// 5:cpuacct,cpu,cpuset:/daemons
if (Array.IndexOf(lineParts[1].Split(','), subsystem) < 0)
{
// Not the relevant entry.
continue;
}

path = lineParts[2];
return true;
}

// cgroup v1: Find the first entry that has the subsystem listed in its controller
// list. See man page for cgroups for /proc/[pid]/cgroups format, e.g:
// hierarchy-ID:controller-list:cgroup-path
// 5:cpuacct,cpu,cpuset:/daemons

if (Array.IndexOf(lineParts[1].Split(','), subsystem) < 0)
else if (cgroupVersion == CGroupVersion.CGroup2)
{
// Not the relevant entry.
continue;
// cgroup v2: Find the first entry that matches the cgroup v2 hierarchy:
// 0::$PATH

if ((lineParts[0] == "0") && (lineParts[1] == string.Empty))
{
path = lineParts[2];
return true;
}
}
else
{
Debug.Fail($"Unexpected cgroup version: \"{cgroupVersion}\"");
}

path = lineParts[2];
return true;
}
}
}
Expand Down
Loading

0 comments on commit 3d0ef96

Please sign in to comment.