Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add support for storing method handle histograms in profiles #67919

Merged
merged 8 commits into from
May 3, 2022
Merged

Add support for storing method handle histograms in profiles #67919

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
26ad0a8
Add support for storing method handle histograms in profiles
jakobbotsch Apr 12, 2022
6959656
Fix R2R reader for PGO schema entries without data
jakobbotsch Apr 12, 2022
5b31bee
Fix writing out methods
jakobbotsch Apr 14, 2022
7596146
Remove an unnecessary check
jakobbotsch Apr 15, 2022
a1c3a7e
Merge remote-tracking branch 'upstream/main' into method-handle-histo…
jakobbotsch Apr 15, 2022
5114f2c
Fix build and some logic inversions
jakobbotsch Apr 15, 2022
decb84d
Bump R2R minor version and JIT-EE GUID
jakobbotsch Apr 18, 2022
baf3662
Merge remote-tracking branch 'upstream/main' into method-handle-histo…
jakobbotsch Apr 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions src/coreclr/inc/corjit.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -368,6 +368,7 @@ class ICorJitInfo : public ICorDynamicInfo
FourByte = 1,
EightByte = 2,
TypeHandle = 3,
MethodHandle = 4,

// Mask of all schema data types
MarshalMask = 0xF,
Expand All @@ -385,9 +386,10 @@ class ICorJitInfo : public ICorDynamicInfo
Done = None, // All instrumentation schemas must end with a record which is "Done"
BasicBlockIntCount = (DescriptorMin * 1) | FourByte, // basic block counter using unsigned 4 byte int
BasicBlockLongCount = (DescriptorMin * 1) | EightByte, // basic block counter using unsigned 8 byte int
TypeHandleHistogramIntCount = (DescriptorMin * 2) | FourByte | AlignPointer, // 4 byte counter that is part of a type histogram. Aligned to match ClassProfile32's alignment.
TypeHandleHistogramLongCount = (DescriptorMin * 2) | EightByte, // 8 byte counter that is part of a type histogram
TypeHandleHistogramTypeHandle = (DescriptorMin * 3) | TypeHandle, // TypeHandle that is part of a type histogram
HandleHistogramIntCount = (DescriptorMin * 2) | FourByte | AlignPointer, // 4 byte counter that is part of a type histogram. Aligned to match ClassProfile32's alignment.
HandleHistogramLongCount = (DescriptorMin * 2) | EightByte, // 8 byte counter that is part of a type histogram
HandleHistogramTypes = (DescriptorMin * 3) | TypeHandle, // Histogram of type handles
HandleHistogramMethods = (DescriptorMin * 3) | MethodHandle, // Histogram of method handles
Version = (DescriptorMin * 4) | None, // Version is encoded in the Other field of the schema
NumRuns = (DescriptorMin * 5) | None, // Number of runs is encoded in the Other field of the schema
EdgeIntCount = (DescriptorMin * 6) | FourByte, // edge counter using unsigned 4 byte int
Expand Down Expand Up @@ -416,12 +418,12 @@ class ICorJitInfo : public ICorDynamicInfo
};

#define DEFAULT_UNKNOWN_TYPEHANDLE 1
#define UNKNOWN_TYPEHANDLE_MIN 1
#define UNKNOWN_TYPEHANDLE_MAX 33
#define UNKNOWN_HANDLE_MIN 1
#define UNKNOWN_HANDLE_MAX 33

static inline bool IsUnknownTypeHandle(intptr_t typeHandle)
static inline bool IsUnknownHandle(intptr_t handle)
{
return ((typeHandle >= UNKNOWN_TYPEHANDLE_MIN) && (typeHandle <= UNKNOWN_TYPEHANDLE_MAX));
return ((handle >= UNKNOWN_HANDLE_MIN) && (handle <= UNKNOWN_HANDLE_MAX));
}

// get profile information to be used for optimizing a current method. The format
Expand Down
4 changes: 2 additions & 2 deletions src/coreclr/inc/eventtracebase.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -939,7 +939,7 @@ namespace ETW
static VOID MethodRestored(MethodDesc * pMethodDesc);
static VOID MethodTableRestored(MethodTable * pMethodTable);
static VOID DynamicMethodDestroyed(MethodDesc *pMethodDesc);
static VOID LogMethodInstrumentationData(MethodDesc* method, uint32_t cbData, BYTE *data, TypeHandle* pTypeHandles, uint32_t typeHandles);
static VOID LogMethodInstrumentationData(MethodDesc* method, uint32_t cbData, BYTE *data, TypeHandle* pTypeHandles, uint32_t numTypeHandles, MethodDesc** pMethods, uint32_t numMethods);
#else // FEATURE_EVENT_TRACE
public:
static VOID GetR2RGetEntryPointStart(MethodDesc *pMethodDesc) {};
Expand All @@ -951,7 +951,7 @@ namespace ETW
static VOID MethodRestored(MethodDesc * pMethodDesc) {};
static VOID MethodTableRestored(MethodTable * pMethodTable) {};
static VOID DynamicMethodDestroyed(MethodDesc *pMethodDesc) {};
static VOID LogMethodInstrumentationData(MethodDesc* method, uint32_t cbData, BYTE *data, TypeHandle* pTypeHandles, uint32_t typeHandles) {};
static VOID LogMethodInstrumentationData(MethodDesc* method, uint32_t cbData, BYTE *data, TypeHandle* pTypeHandles, uint32_t numTypeHandles, MethodDesc** pMethods, uint32_t numMethods) {};
#endif // FEATURE_EVENT_TRACE
};

Expand Down
10 changes: 5 additions & 5 deletions src/coreclr/inc/jiteeversionguid.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,11 @@ typedef const GUID *LPCGUID;
#define GUID_DEFINED
#endif // !GUID_DEFINED

constexpr GUID JITEEVersionIdentifier = { /* b0719856-6fe6-407c-bf40-7a57e22b2382 */
0xb0719856,
0x6fe6,
0x407c,
{0xbf, 0x40, 0x7a, 0x57, 0xe2, 0x2b, 0x23, 0x82}
constexpr GUID JITEEVersionIdentifier = { /* 7503fe09-4852-40f6-829a-ff91402c9604 */
0x7503fe09,
0x4852,
0x40f6,
{0x82, 0x9a, 0xff, 0x91, 0x40, 0x2c, 0x96, 0x04}
};

//////////////////////////////////////////////////////////////////////////////////////////////////////////
Expand Down
41 changes: 29 additions & 12 deletions src/coreclr/inc/pgo_formatprocessing.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,7 @@

#ifdef FEATURE_PGO

inline bool AddTypeHandleToUnknownTypeHandleMask(INT_PTR typeHandle, uint32_t *unknownTypeHandleMask)
{
uint32_t bitMask = (uint32_t)(1 << (typeHandle - UNKNOWN_TYPEHANDLE_MIN));
bool result = (bitMask & *unknownTypeHandleMask) == 0;
*unknownTypeHandleMask |= bitMask;
return result;
}

inline INT_PTR HashToPgoUnknownTypeHandle(uint32_t hash)
inline INT_PTR HashToPgoUnknownHandle(uint32_t hash)
{
// Map from a 32bit hash to the 32 different unknown type handle values
return (hash & 0x1F) + 1;
Expand Down Expand Up @@ -53,6 +45,7 @@ inline uint32_t InstrumentationKindToSize(ICorJitInfo::PgoInstrumentationKind ki
case ICorJitInfo::PgoInstrumentationKind::EightByte:
return 8;
case ICorJitInfo::PgoInstrumentationKind::TypeHandle:
case ICorJitInfo::PgoInstrumentationKind::MethodHandle:
return TARGET_POINTER_SIZE;
default:
_ASSERTE(FALSE);
Expand Down Expand Up @@ -242,6 +235,7 @@ bool ReadInstrumentationData(const uint8_t *pByte, size_t cbDataMax, SchemaAndDa
bool done = false;
int64_t lastDataValue = 0;
int64_t lastTypeDataValue = 0;
int64_t lastMethodDataValue = 0;
int32_t dataCountToRead = 0;

ReadCompressedInts(pByte, cbDataMax, [&](int64_t curValue)
Expand All @@ -267,8 +261,16 @@ bool ReadInstrumentationData(const uint8_t *pByte, size_t cbDataMax, SchemaAndDa
return false;
}
break;
case ICorJitInfo::PgoInstrumentationKind::MethodHandle:
lastMethodDataValue += curValue;

if (!handler(schemaHandler.GetSchema(), lastMethodDataValue, schemaHandler.GetSchema().Count - dataCountToRead))
{
return false;
}
break;
default:
assert(false);
assert(!"Unexpected PGO instrumentation data type");
break;
}
dataCountToRead--;
Expand Down Expand Up @@ -516,6 +518,7 @@ class SchemaAndDataWriter
ICorJitInfo::PgoInstrumentationSchema prevSchema = {};
int64_t lastIntDataWritten = 0;
int64_t lastTypeDataWritten = 0;
int64_t lastMethodDataWritten = 0;

public:
SchemaAndDataWriter(const ByteWriter& byteWriter, uint8_t* pInstrumentationData) :
Expand All @@ -532,8 +535,8 @@ class SchemaAndDataWriter
return true;
}

template<class TypeHandleProcessor>
bool AppendDataFromLastSchema(TypeHandleProcessor& thProcessor)
template<class TypeHandleProcessor, class MethodHandleProcessor>
bool AppendDataFromLastSchema(TypeHandleProcessor& thProcessor, MethodHandleProcessor& mhProcessor)
{
uint8_t *pData = (pInstrumentationData + prevSchema.Offset);
for (int32_t iDataElem = 0; iDataElem < prevSchema.Count; iDataElem++)
Expand Down Expand Up @@ -577,6 +580,20 @@ class SchemaAndDataWriter
pData += sizeof(intptr_t);
break;
}
case ICorJitInfo::PgoInstrumentationKind::MethodHandle:
{
logicalDataToWrite = *(volatile intptr_t*)pData;

// As there could be tearing otherwise, inform the caller of exactly what value was written.
mhProcessor(logicalDataToWrite);

bool returnValue = WriteCompressedIntToBytes(logicalDataToWrite - lastMethodDataWritten, byteWriter);
lastMethodDataWritten = logicalDataToWrite;
if (!returnValue)
return false;
pData += sizeof(intptr_t);
break;
}
default:
_ASSERTE(!"Unexpected type");
return false;
Expand Down
2 changes: 1 addition & 1 deletion src/coreclr/inc/readytorun.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

// Keep these in sync with src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs
#define READYTORUN_MAJOR_VERSION 0x0006
#define READYTORUN_MINOR_VERSION 0x0000
#define READYTORUN_MINOR_VERSION 0x0001

#define MINIMUM_READYTORUN_MAJOR_VERSION 0x006

Expand Down
1 change: 1 addition & 0 deletions src/coreclr/jit/compiler.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5443,6 +5443,7 @@ class Compiler
UINT32 fgPgoBlockCounts;
UINT32 fgPgoEdgeCounts;
UINT32 fgPgoClassProfiles;
UINT32 fgPgoMethodProfiles;
unsigned fgPgoInlineePgo;
unsigned fgPgoInlineeNoPgo;
unsigned fgPgoInlineeNoPgoSingleBlock;
Expand Down
1 change: 1 addition & 0 deletions src/coreclr/jit/fgbasic.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,7 @@ void Compiler::fgInit()
fgPgoBlockCounts = 0;
fgPgoEdgeCounts = 0;
fgPgoClassProfiles = 0;
fgPgoMethodProfiles = 0;
fgPgoInlineePgo = 0;
fgPgoInlineeNoPgo = 0;
fgPgoInlineeNoPgoSingleBlock = 0;
Expand Down
39 changes: 30 additions & 9 deletions src/coreclr/jit/fgprofile.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1495,15 +1495,15 @@ class BuildClassProbeSchemaGen
}

schemaElem.InstrumentationKind = JitConfig.JitCollect64BitCounts()
? ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramLongCount
: ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramIntCount;
? ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount
: ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount;
schemaElem.ILOffset = (int32_t)call->gtClassProfileCandidateInfo->ilOffset;
schemaElem.Offset = 0;

m_schema.push_back(schemaElem);

// Re-using ILOffset and Other fields from schema item for TypeHandleHistogramCount
schemaElem.InstrumentationKind = ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramTypeHandle;
schemaElem.InstrumentationKind = ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes;
schemaElem.Count = ICorJitInfo::ClassProfile32::SIZE;
m_schema.push_back(schemaElem);

Expand Down Expand Up @@ -1550,9 +1550,9 @@ class ClassProbeInserter
//
assert(m_schema[*m_currentSchemaIndex].ILOffset == (int32_t)call->gtClassProfileCandidateInfo->ilOffset);
bool is32 = m_schema[*m_currentSchemaIndex].InstrumentationKind ==
ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramIntCount;
ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount;
bool is64 = m_schema[*m_currentSchemaIndex].InstrumentationKind ==
ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramLongCount;
ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount;
assert(is32 || is64);

// Figure out where the table is located.
Expand Down Expand Up @@ -2047,12 +2047,32 @@ PhaseStatus Compiler::fgIncorporateProfileData()
fgPgoEdgeCounts++;
break;

case ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramIntCount:
case ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramLongCount:
case ICorJitInfo::PgoInstrumentationKind::GetLikelyClass:
fgPgoClassProfiles++;
break;

case ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount:
case ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount:
if (iSchema + 1 < fgPgoSchemaCount)
{
if (fgPgoSchema[iSchema + 1].InstrumentationKind ==
ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes)
{
fgPgoClassProfiles++;
iSchema++;
break;
}
if (fgPgoSchema[iSchema + 1].InstrumentationKind ==
ICorJitInfo::PgoInstrumentationKind::HandleHistogramMethods)
{
fgPgoMethodProfiles++;
iSchema++;
break;
}
}

__fallthrough;

default:
JITDUMP("Unknown PGO record type 0x%x in schema entry %u (offset 0x%x count 0x%x other 0x%x)\n",
fgPgoSchema[iSchema].InstrumentationKind, iSchema, fgPgoSchema[iSchema].ILOffset,
Expand All @@ -2067,8 +2087,9 @@ PhaseStatus Compiler::fgIncorporateProfileData()
fgNumProfileRuns = 1;
}

JITDUMP("Profile summary: %d runs, %d block probes, %d edge probes, %d class profiles, %d other records\n",
fgNumProfileRuns, fgPgoBlockCounts, fgPgoEdgeCounts, fgPgoClassProfiles, otherRecords);
JITDUMP("Profile summary: %d runs, %d block probes, %d edge probes, %d class profiles, %d method profiles, %d "
"other records\n",
fgNumProfileRuns, fgPgoBlockCounts, fgPgoEdgeCounts, fgPgoClassProfiles, fgPgoMethodProfiles, otherRecords);

const bool haveBlockCounts = fgPgoBlockCounts > 0;
const bool haveEdgeCounts = fgPgoEdgeCounts > 0;
Expand Down
30 changes: 15 additions & 15 deletions src/coreclr/jit/likelyclass.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord*
(schema[i].Count == 1))
{
INT_PTR result = *(INT_PTR*)(pInstrumentationData + schema[i].Offset);
if (ICorJitInfo::IsUnknownTypeHandle(result))
if (ICorJitInfo::IsUnknownHandle(result))
{
return 0;
}
Expand All @@ -168,11 +168,11 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord*
}

const bool isHistogramCount =
(schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramIntCount) ||
(schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramLongCount);
(schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount) ||
(schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount);

if (isHistogramCount && (schema[i].Count == 1) && ((i + 1) < countSchemaItems) &&
(schema[i + 1].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramTypeHandle))
(schema[i + 1].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes))
{
// Form a histogram
//
Expand All @@ -191,7 +191,7 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord*
{
LikelyClassHistogramEntry const hist0 = h.HistogramEntryAt(0);
// Fast path for monomorphic cases
if (ICorJitInfo::IsUnknownTypeHandle(hist0.m_mt))
if (ICorJitInfo::IsUnknownHandle(hist0.m_mt))
{
return 0;
}
Expand All @@ -205,12 +205,12 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord*
LikelyClassHistogramEntry const hist0 = h.HistogramEntryAt(0);
LikelyClassHistogramEntry const hist1 = h.HistogramEntryAt(1);
// Fast path for two classes
if ((hist0.m_count >= hist1.m_count) && !ICorJitInfo::IsUnknownTypeHandle(hist0.m_mt))
if ((hist0.m_count >= hist1.m_count) && !ICorJitInfo::IsUnknownHandle(hist0.m_mt))
{
pLikelyClasses[0].likelihood = (100 * hist0.m_count) / h.m_totalCount;
pLikelyClasses[0].clsHandle = (CORINFO_CLASS_HANDLE)hist0.m_mt;

if ((maxLikelyClasses > 1) && !ICorJitInfo::IsUnknownTypeHandle(hist1.m_mt))
if ((maxLikelyClasses > 1) && !ICorJitInfo::IsUnknownHandle(hist1.m_mt))
{
pLikelyClasses[1].likelihood = (100 * hist1.m_count) / h.m_totalCount;
pLikelyClasses[1].clsHandle = (CORINFO_CLASS_HANDLE)hist1.m_mt;
Expand All @@ -219,12 +219,12 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord*
return 1;
}

if (!ICorJitInfo::IsUnknownTypeHandle(hist1.m_mt))
if (!ICorJitInfo::IsUnknownHandle(hist1.m_mt))
{
pLikelyClasses[0].likelihood = (100 * hist1.m_count) / h.m_totalCount;
pLikelyClasses[0].clsHandle = (CORINFO_CLASS_HANDLE)hist1.m_mt;

if ((maxLikelyClasses > 1) && !ICorJitInfo::IsUnknownTypeHandle(hist0.m_mt))
if ((maxLikelyClasses > 1) && !ICorJitInfo::IsUnknownHandle(hist0.m_mt))
{
pLikelyClasses[1].likelihood = (100 * hist0.m_count) / h.m_totalCount;
pLikelyClasses[1].clsHandle = (CORINFO_CLASS_HANDLE)hist0.m_mt;
Expand All @@ -244,7 +244,7 @@ extern "C" DLLEXPORT UINT32 WINAPI getLikelyClasses(LikelyClassRecord*
for (unsigned m = 0; m < h.countHistogramElements; m++)
{
LikelyClassHistogramEntry const hist = h.HistogramEntryAt(m);
if (!ICorJitInfo::IsUnknownTypeHandle(hist.m_mt))
if (!ICorJitInfo::IsUnknownHandle(hist.m_mt))
{
sortedEntries[knownHandles++] = hist;
}
Expand Down Expand Up @@ -311,7 +311,7 @@ CORINFO_CLASS_HANDLE Compiler::getRandomClass(ICorJitInfo::PgoInstrumentationSch
(schema[i].Count == 1))
{
INT_PTR result = *(INT_PTR*)(pInstrumentationData + schema[i].Offset);
if (ICorJitInfo::IsUnknownTypeHandle(result))
if (ICorJitInfo::IsUnknownHandle(result))
{
return NO_CLASS_HANDLE;
}
Expand All @@ -322,11 +322,11 @@ CORINFO_CLASS_HANDLE Compiler::getRandomClass(ICorJitInfo::PgoInstrumentationSch
}

bool isHistogramCount =
(schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramIntCount) ||
(schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramLongCount);
(schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramIntCount) ||
(schema[i].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramLongCount);

if (isHistogramCount && (schema[i].Count == 1) && ((i + 1) < countSchemaItems) &&
(schema[i + 1].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::TypeHandleHistogramTypeHandle))
(schema[i + 1].InstrumentationKind == ICorJitInfo::PgoInstrumentationKind::HandleHistogramTypes))
{
// Form a histogram
//
Expand All @@ -342,7 +342,7 @@ CORINFO_CLASS_HANDLE Compiler::getRandomClass(ICorJitInfo::PgoInstrumentationSch
unsigned randomEntryIndex = random->Next(0, h.countHistogramElements);
LikelyClassHistogramEntry randomEntry = h.HistogramEntryAt(randomEntryIndex);

if (ICorJitInfo::IsUnknownTypeHandle(randomEntry.m_mt))
if (ICorJitInfo::IsUnknownHandle(randomEntry.m_mt))
{
return NO_CLASS_HANDLE;
}
Expand Down
2 changes: 1 addition & 1 deletion src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ internal struct ReadyToRunHeaderConstants
public const uint Signature = 0x00525452; // 'RTR'

public const ushort CurrentMajorVersion = 6;
public const ushort CurrentMinorVersion = 0;
public const ushort CurrentMinorVersion = 1;
}

#pragma warning disable 0169
Expand Down
Loading