From 3713607636f887883eb31285ec52eaa7bd26a744 Mon Sep 17 00:00:00 2001 From: Ikey Doherty Date: Mon, 22 May 2023 12:02:28 +0100 Subject: [PATCH] ldc-profdata: Add llvm-profdata from LLVM release/16.x Modified to have an explicit `main()` per the other profdata imports in this tree. Signed-off-by: Ikey Doherty --- tools/ldc-profdata/llvm-profdata-16.0.cpp | 3013 +++++++++++++++++++++ 1 file changed, 3013 insertions(+) create mode 100644 tools/ldc-profdata/llvm-profdata-16.0.cpp diff --git a/tools/ldc-profdata/llvm-profdata-16.0.cpp b/tools/ldc-profdata/llvm-profdata-16.0.cpp new file mode 100644 index 0000000000..57dfd18076 --- /dev/null +++ b/tools/ldc-profdata/llvm-profdata-16.0.cpp @@ -0,0 +1,3013 @@ +//===- llvm-profdata.cpp - LLVM profile data tool -------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// llvm-profdata merges .profdata files. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/Object/Binary.h" +#include "llvm/ProfileData/InstrProfCorrelator.h" +#include "llvm/ProfileData/InstrProfReader.h" +#include "llvm/ProfileData/InstrProfWriter.h" +#include "llvm/ProfileData/MemProf.h" +#include "llvm/ProfileData/ProfileCommon.h" +#include "llvm/ProfileData/RawMemProfReader.h" +#include "llvm/ProfileData/SampleProfReader.h" +#include "llvm/ProfileData/SampleProfWriter.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Discriminator.h" +#include "llvm/Support/Errc.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/FormattedStream.h" +#include "llvm/Support/InitLLVM.h" +#include "llvm/Support/MD5.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/ThreadPool.h" +#include "llvm/Support/Threading.h" +#include "llvm/Support/WithColor.h" +#include "llvm/Support/raw_ostream.h" +#include +#include +#include +#include + +using namespace llvm; + +// We use this string to indicate that there are +// multiple static functions map to the same name. +const std::string DuplicateNameStr = "----"; + +enum ProfileFormat { + PF_None = 0, + PF_Text, + PF_Compact_Binary, + PF_Ext_Binary, + PF_GCC, + PF_Binary +}; + +enum class ShowFormat { Text, Json, Yaml }; + +static void warn(Twine Message, std::string Whence = "", + std::string Hint = "") { + WithColor::warning(); + if (!Whence.empty()) + errs() << Whence << ": "; + errs() << Message << "\n"; + if (!Hint.empty()) + WithColor::note() << Hint << "\n"; +} + +static void warn(Error E, StringRef Whence = "") { + if (E.isA()) { + handleAllErrors(std::move(E), [&](const InstrProfError &IPE) { + warn(IPE.message(), std::string(Whence), std::string("")); + }); + } +} + +static void exitWithError(Twine Message, std::string Whence = "", + std::string Hint = "") { + WithColor::error(); + if (!Whence.empty()) + errs() << Whence << ": "; + errs() << Message << "\n"; + if (!Hint.empty()) + WithColor::note() << Hint << "\n"; + ::exit(1); +} + +static void exitWithError(Error E, StringRef Whence = "") { + if (E.isA()) { + handleAllErrors(std::move(E), [&](const InstrProfError &IPE) { + instrprof_error instrError = IPE.get(); + StringRef Hint = ""; + if (instrError == instrprof_error::unrecognized_format) { + // Hint in case user missed specifying the profile type. + Hint = "Perhaps you forgot to use the --sample or --memory option?"; + } + exitWithError(IPE.message(), std::string(Whence), std::string(Hint)); + }); + return; + } + + exitWithError(toString(std::move(E)), std::string(Whence)); +} + +static void exitWithErrorCode(std::error_code EC, StringRef Whence = "") { + exitWithError(EC.message(), std::string(Whence)); +} + +namespace { +enum ProfileKinds { instr, sample, memory }; +enum FailureMode { failIfAnyAreInvalid, failIfAllAreInvalid }; +} + +static void warnOrExitGivenError(FailureMode FailMode, std::error_code EC, + StringRef Whence = "") { + if (FailMode == failIfAnyAreInvalid) + exitWithErrorCode(EC, Whence); + else + warn(EC.message(), std::string(Whence)); +} + +static void handleMergeWriterError(Error E, StringRef WhenceFile = "", + StringRef WhenceFunction = "", + bool ShowHint = true) { + if (!WhenceFile.empty()) + errs() << WhenceFile << ": "; + if (!WhenceFunction.empty()) + errs() << WhenceFunction << ": "; + + auto IPE = instrprof_error::success; + E = handleErrors(std::move(E), + [&IPE](std::unique_ptr E) -> Error { + IPE = E->get(); + return Error(std::move(E)); + }); + errs() << toString(std::move(E)) << "\n"; + + if (ShowHint) { + StringRef Hint = ""; + if (IPE != instrprof_error::success) { + switch (IPE) { + case instrprof_error::hash_mismatch: + case instrprof_error::count_mismatch: + case instrprof_error::value_site_count_mismatch: + Hint = "Make sure that all profile data to be merged is generated " + "from the same binary."; + break; + default: + break; + } + } + + if (!Hint.empty()) + errs() << Hint << "\n"; + } +} + +namespace { +/// A remapper from original symbol names to new symbol names based on a file +/// containing a list of mappings from old name to new name. +class SymbolRemapper { + std::unique_ptr File; + DenseMap RemappingTable; + +public: + /// Build a SymbolRemapper from a file containing a list of old/new symbols. + static std::unique_ptr create(StringRef InputFile) { + auto BufOrError = MemoryBuffer::getFileOrSTDIN(InputFile); + if (!BufOrError) + exitWithErrorCode(BufOrError.getError(), InputFile); + + auto Remapper = std::make_unique(); + Remapper->File = std::move(BufOrError.get()); + + for (line_iterator LineIt(*Remapper->File, /*SkipBlanks=*/true, '#'); + !LineIt.is_at_eof(); ++LineIt) { + std::pair Parts = LineIt->split(' '); + if (Parts.first.empty() || Parts.second.empty() || + Parts.second.count(' ')) { + exitWithError("unexpected line in remapping file", + (InputFile + ":" + Twine(LineIt.line_number())).str(), + "expected 'old_symbol new_symbol'"); + } + Remapper->RemappingTable.insert(Parts); + } + return Remapper; + } + + /// Attempt to map the given old symbol into a new symbol. + /// + /// \return The new symbol, or \p Name if no such symbol was found. + StringRef operator()(StringRef Name) { + StringRef New = RemappingTable.lookup(Name); + return New.empty() ? Name : New; + } +}; +} + +struct WeightedFile { + std::string Filename; + uint64_t Weight; +}; +typedef SmallVector WeightedFileVector; + +/// Keep track of merged data and reported errors. +struct WriterContext { + std::mutex Lock; + InstrProfWriter Writer; + std::vector> Errors; + std::mutex &ErrLock; + SmallSet &WriterErrorCodes; + + WriterContext(bool IsSparse, std::mutex &ErrLock, + SmallSet &WriterErrorCodes) + : Writer(IsSparse), ErrLock(ErrLock), WriterErrorCodes(WriterErrorCodes) { + } +}; + +/// Computer the overlap b/w profile BaseFilename and TestFileName, +/// and store the program level result to Overlap. +static void overlapInput(const std::string &BaseFilename, + const std::string &TestFilename, WriterContext *WC, + OverlapStats &Overlap, + const OverlapFuncFilters &FuncFilter, + raw_fd_ostream &OS, bool IsCS) { + auto ReaderOrErr = InstrProfReader::create(TestFilename); + if (Error E = ReaderOrErr.takeError()) { + // Skip the empty profiles by returning sliently. + instrprof_error IPE = InstrProfError::take(std::move(E)); + if (IPE != instrprof_error::empty_raw_profile) + WC->Errors.emplace_back(make_error(IPE), TestFilename); + return; + } + + auto Reader = std::move(ReaderOrErr.get()); + for (auto &I : *Reader) { + OverlapStats FuncOverlap(OverlapStats::FunctionLevel); + FuncOverlap.setFuncInfo(I.Name, I.Hash); + + WC->Writer.overlapRecord(std::move(I), Overlap, FuncOverlap, FuncFilter); + FuncOverlap.dump(OS); + } +} + +/// Load an input into a writer context. +static void loadInput(const WeightedFile &Input, SymbolRemapper *Remapper, + const InstrProfCorrelator *Correlator, + const StringRef ProfiledBinary, WriterContext *WC) { + std::unique_lock CtxGuard{WC->Lock}; + + // Copy the filename, because llvm::ThreadPool copied the input "const + // WeightedFile &" by value, making a reference to the filename within it + // invalid outside of this packaged task. + std::string Filename = Input.Filename; + + using ::llvm::memprof::RawMemProfReader; + if (RawMemProfReader::hasFormat(Input.Filename)) { + auto ReaderOrErr = RawMemProfReader::create(Input.Filename, ProfiledBinary); + if (!ReaderOrErr) { + exitWithError(ReaderOrErr.takeError(), Input.Filename); + } + std::unique_ptr Reader = std::move(ReaderOrErr.get()); + // Check if the profile types can be merged, e.g. clang frontend profiles + // should not be merged with memprof profiles. + if (Error E = WC->Writer.mergeProfileKind(Reader->getProfileKind())) { + consumeError(std::move(E)); + WC->Errors.emplace_back( + make_error( + "Cannot merge MemProf profile with Clang generated profile.", + std::error_code()), + Filename); + return; + } + + auto MemProfError = [&](Error E) { + instrprof_error IPE = InstrProfError::take(std::move(E)); + WC->Errors.emplace_back(make_error(IPE), Filename); + }; + + // Add the frame mappings into the writer context. + const auto &IdToFrame = Reader->getFrameMapping(); + for (const auto &I : IdToFrame) { + bool Succeeded = WC->Writer.addMemProfFrame( + /*Id=*/I.first, /*Frame=*/I.getSecond(), MemProfError); + // If we weren't able to add the frame mappings then it doesn't make sense + // to try to add the records from this profile. + if (!Succeeded) + return; + } + const auto &FunctionProfileData = Reader->getProfileData(); + // Add the memprof records into the writer context. + for (const auto &I : FunctionProfileData) { + WC->Writer.addMemProfRecord(/*Id=*/I.first, /*Record=*/I.second); + } + return; + } + + auto ReaderOrErr = InstrProfReader::create(Input.Filename, Correlator); + if (Error E = ReaderOrErr.takeError()) { + // Skip the empty profiles by returning sliently. + instrprof_error IPE = InstrProfError::take(std::move(E)); + if (IPE != instrprof_error::empty_raw_profile) + WC->Errors.emplace_back(make_error(IPE), Filename); + return; + } + + auto Reader = std::move(ReaderOrErr.get()); + if (Error E = WC->Writer.mergeProfileKind(Reader->getProfileKind())) { + consumeError(std::move(E)); + WC->Errors.emplace_back( + make_error( + "Merge IR generated profile with Clang generated profile.", + std::error_code()), + Filename); + return; + } + + for (auto &I : *Reader) { + if (Remapper) + I.Name = (*Remapper)(I.Name); + const StringRef FuncName = I.Name; + bool Reported = false; + WC->Writer.addRecord(std::move(I), Input.Weight, [&](Error E) { + if (Reported) { + consumeError(std::move(E)); + return; + } + Reported = true; + // Only show hint the first time an error occurs. + instrprof_error IPE = InstrProfError::take(std::move(E)); + std::unique_lock ErrGuard{WC->ErrLock}; + bool firstTime = WC->WriterErrorCodes.insert(IPE).second; + handleMergeWriterError(make_error(IPE), Input.Filename, + FuncName, firstTime); + }); + } + + if (Reader->hasError()) { + if (Error E = Reader->getError()) + WC->Errors.emplace_back(std::move(E), Filename); + } + + std::vector BinaryIds; + if (Error E = Reader->readBinaryIds(BinaryIds)) + WC->Errors.emplace_back(std::move(E), Filename); + WC->Writer.addBinaryIds(BinaryIds); +} + +/// Merge the \p Src writer context into \p Dst. +static void mergeWriterContexts(WriterContext *Dst, WriterContext *Src) { + for (auto &ErrorPair : Src->Errors) + Dst->Errors.push_back(std::move(ErrorPair)); + Src->Errors.clear(); + + if (Error E = Dst->Writer.mergeProfileKind(Src->Writer.getProfileKind())) + exitWithError(std::move(E)); + + Dst->Writer.mergeRecordsFromWriter(std::move(Src->Writer), [&](Error E) { + instrprof_error IPE = InstrProfError::take(std::move(E)); + std::unique_lock ErrGuard{Dst->ErrLock}; + bool firstTime = Dst->WriterErrorCodes.insert(IPE).second; + if (firstTime) + warn(toString(make_error(IPE))); + }); +} + +static void writeInstrProfile(StringRef OutputFilename, + ProfileFormat OutputFormat, + InstrProfWriter &Writer) { + std::error_code EC; + raw_fd_ostream Output(OutputFilename.data(), EC, + OutputFormat == PF_Text ? sys::fs::OF_TextWithCRLF + : sys::fs::OF_None); + if (EC) + exitWithErrorCode(EC, OutputFilename); + + if (OutputFormat == PF_Text) { + if (Error E = Writer.writeText(Output)) + warn(std::move(E)); + } else { + if (Output.is_displayed()) + exitWithError("cannot write a non-text format profile to the terminal"); + if (Error E = Writer.write(Output)) + warn(std::move(E)); + } +} + +static void mergeInstrProfile(const WeightedFileVector &Inputs, + StringRef DebugInfoFilename, + SymbolRemapper *Remapper, + StringRef OutputFilename, + ProfileFormat OutputFormat, bool OutputSparse, + unsigned NumThreads, FailureMode FailMode, + const StringRef ProfiledBinary) { + if (OutputFormat != PF_Binary && OutputFormat != PF_Compact_Binary && + OutputFormat != PF_Ext_Binary && OutputFormat != PF_Text) + exitWithError("unknown format is specified"); + + std::unique_ptr Correlator; + if (!DebugInfoFilename.empty()) { + if (auto Err = + InstrProfCorrelator::get(DebugInfoFilename).moveInto(Correlator)) + exitWithError(std::move(Err), DebugInfoFilename); + if (auto Err = Correlator->correlateProfileData()) + exitWithError(std::move(Err), DebugInfoFilename); + } + + std::mutex ErrorLock; + SmallSet WriterErrorCodes; + + // If NumThreads is not specified, auto-detect a good default. + if (NumThreads == 0) + NumThreads = std::min(hardware_concurrency().compute_thread_count(), + unsigned((Inputs.size() + 1) / 2)); + + // Initialize the writer contexts. + SmallVector, 4> Contexts; + for (unsigned I = 0; I < NumThreads; ++I) + Contexts.emplace_back(std::make_unique( + OutputSparse, ErrorLock, WriterErrorCodes)); + + if (NumThreads == 1) { + for (const auto &Input : Inputs) + loadInput(Input, Remapper, Correlator.get(), ProfiledBinary, + Contexts[0].get()); + } else { + ThreadPool Pool(hardware_concurrency(NumThreads)); + + // Load the inputs in parallel (N/NumThreads serial steps). + unsigned Ctx = 0; + for (const auto &Input : Inputs) { + Pool.async(loadInput, Input, Remapper, Correlator.get(), ProfiledBinary, + Contexts[Ctx].get()); + Ctx = (Ctx + 1) % NumThreads; + } + Pool.wait(); + + // Merge the writer contexts together (~ lg(NumThreads) serial steps). + unsigned Mid = Contexts.size() / 2; + unsigned End = Contexts.size(); + assert(Mid > 0 && "Expected more than one context"); + do { + for (unsigned I = 0; I < Mid; ++I) + Pool.async(mergeWriterContexts, Contexts[I].get(), + Contexts[I + Mid].get()); + Pool.wait(); + if (End & 1) { + Pool.async(mergeWriterContexts, Contexts[0].get(), + Contexts[End - 1].get()); + Pool.wait(); + } + End = Mid; + Mid /= 2; + } while (Mid > 0); + } + + // Handle deferred errors encountered during merging. If the number of errors + // is equal to the number of inputs the merge failed. + unsigned NumErrors = 0; + for (std::unique_ptr &WC : Contexts) { + for (auto &ErrorPair : WC->Errors) { + ++NumErrors; + warn(toString(std::move(ErrorPair.first)), ErrorPair.second); + } + } + if (NumErrors == Inputs.size() || + (NumErrors > 0 && FailMode == failIfAnyAreInvalid)) + exitWithError("no profile can be merged"); + + writeInstrProfile(OutputFilename, OutputFormat, Contexts[0]->Writer); +} + +/// The profile entry for a function in instrumentation profile. +struct InstrProfileEntry { + uint64_t MaxCount = 0; + uint64_t NumEdgeCounters = 0; + float ZeroCounterRatio = 0.0; + InstrProfRecord *ProfRecord; + InstrProfileEntry(InstrProfRecord *Record); + InstrProfileEntry() = default; +}; + +InstrProfileEntry::InstrProfileEntry(InstrProfRecord *Record) { + ProfRecord = Record; + uint64_t CntNum = Record->Counts.size(); + uint64_t ZeroCntNum = 0; + for (size_t I = 0; I < CntNum; ++I) { + MaxCount = std::max(MaxCount, Record->Counts[I]); + ZeroCntNum += !Record->Counts[I]; + } + ZeroCounterRatio = (float)ZeroCntNum / CntNum; + NumEdgeCounters = CntNum; +} + +/// Either set all the counters in the instr profile entry \p IFE to +/// -1 / -2 /in order to drop the profile or scale up the +/// counters in \p IFP to be above hot / cold threshold. We use +/// the ratio of zero counters in the profile of a function to +/// decide the profile is helpful or harmful for performance, +/// and to choose whether to scale up or drop it. +static void updateInstrProfileEntry(InstrProfileEntry &IFE, bool SetToHot, + uint64_t HotInstrThreshold, + uint64_t ColdInstrThreshold, + float ZeroCounterThreshold) { + InstrProfRecord *ProfRecord = IFE.ProfRecord; + if (!IFE.MaxCount || IFE.ZeroCounterRatio > ZeroCounterThreshold) { + // If all or most of the counters of the function are zero, the + // profile is unaccountable and should be dropped. Reset all the + // counters to be -1 / -2 and PGO profile-use will drop the profile. + // All counters being -1 also implies that the function is hot so + // PGO profile-use will also set the entry count metadata to be + // above hot threshold. + // All counters being -2 implies that the function is warm so + // PGO profile-use will also set the entry count metadata to be + // above cold threshold. + auto Kind = + (SetToHot ? InstrProfRecord::PseudoHot : InstrProfRecord::PseudoWarm); + ProfRecord->setPseudoCount(Kind); + return; + } + + // Scale up the MaxCount to be multiple times above hot / cold threshold. + const unsigned MultiplyFactor = 3; + uint64_t Threshold = (SetToHot ? HotInstrThreshold : ColdInstrThreshold); + uint64_t Numerator = Threshold * MultiplyFactor; + + // Make sure Threshold for warm counters is below the HotInstrThreshold. + if (!SetToHot && Threshold >= HotInstrThreshold) { + Threshold = (HotInstrThreshold + ColdInstrThreshold) / 2; + } + + uint64_t Denominator = IFE.MaxCount; + if (Numerator <= Denominator) + return; + ProfRecord->scale(Numerator, Denominator, [&](instrprof_error E) { + warn(toString(make_error(E))); + }); +} + +const uint64_t ColdPercentileIdx = 15; +const uint64_t HotPercentileIdx = 11; + +using sampleprof::FSDiscriminatorPass; + +// Internal options to set FSDiscriminatorPass. Used in merge and show +// commands. +static cl::opt FSDiscriminatorPassOption( + "fs-discriminator-pass", cl::init(PassLast), cl::Hidden, + cl::desc("Zero out the discriminator bits for the FS discrimiantor " + "pass beyond this value. The enum values are defined in " + "Support/Discriminator.h"), + cl::values(clEnumVal(Base, "Use base discriminators only"), + clEnumVal(Pass1, "Use base and pass 1 discriminators"), + clEnumVal(Pass2, "Use base and pass 1-2 discriminators"), + clEnumVal(Pass3, "Use base and pass 1-3 discriminators"), + clEnumVal(PassLast, "Use all discriminator bits (default)"))); + +static unsigned getDiscriminatorMask() { + return getN1Bits(getFSPassBitEnd(FSDiscriminatorPassOption.getValue())); +} + +/// Adjust the instr profile in \p WC based on the sample profile in +/// \p Reader. +static void +adjustInstrProfile(std::unique_ptr &WC, + std::unique_ptr &Reader, + unsigned SupplMinSizeThreshold, float ZeroCounterThreshold, + unsigned InstrProfColdThreshold) { + // Function to its entry in instr profile. + StringMap InstrProfileMap; + StringMap StaticFuncMap; + InstrProfSummaryBuilder IPBuilder(ProfileSummaryBuilder::DefaultCutoffs); + + auto checkSampleProfileHasFUnique = [&Reader]() { + for (const auto &PD : Reader->getProfiles()) { + auto &FContext = PD.first; + if (FContext.toString().find(FunctionSamples::UniqSuffix) != + std::string::npos) { + return true; + } + } + return false; + }; + + bool SampleProfileHasFUnique = checkSampleProfileHasFUnique(); + + auto buildStaticFuncMap = [&StaticFuncMap, + SampleProfileHasFUnique](const StringRef Name) { + std::string Prefixes[] = {".cpp:", "cc:", ".c:", ".hpp:", ".h:"}; + size_t PrefixPos = StringRef::npos; + for (auto &Prefix : Prefixes) { + PrefixPos = Name.find_insensitive(Prefix); + if (PrefixPos == StringRef::npos) + continue; + PrefixPos += Prefix.size(); + break; + } + + if (PrefixPos == StringRef::npos) { + return; + } + + StringRef NewName = Name.drop_front(PrefixPos); + StringRef FName = Name.substr(0, PrefixPos - 1); + if (NewName.size() == 0) { + return; + } + + // This name should have a static linkage. + size_t PostfixPos = NewName.find(FunctionSamples::UniqSuffix); + bool ProfileHasFUnique = (PostfixPos != StringRef::npos); + + // If sample profile and instrumented profile do not agree on symbol + // uniqification. + if (SampleProfileHasFUnique != ProfileHasFUnique) { + // If instrumented profile uses -funique-internal-linakge-symbols, + // we need to trim the name. + if (ProfileHasFUnique) { + NewName = NewName.substr(0, PostfixPos); + } else { + // If sample profile uses -funique-internal-linakge-symbols, + // we build the map. + std::string NStr = + NewName.str() + getUniqueInternalLinkagePostfix(FName); + NewName = StringRef(NStr); + StaticFuncMap[NewName] = Name; + return; + } + } + + if (StaticFuncMap.find(NewName) == StaticFuncMap.end()) { + StaticFuncMap[NewName] = Name; + } else { + StaticFuncMap[NewName] = DuplicateNameStr; + } + }; + + // We need to flatten the SampleFDO profile as the InstrFDO + // profile does not have inlined callsite profiles. + // One caveat is the pre-inlined function -- their samples + // should be collapsed into the caller function. + // Here we do a DFS traversal to get the flatten profile + // info: the sum of entrycount and the max of maxcount. + // Here is the algorithm: + // recursive (FS, root_name) { + // name = FS->getName(); + // get samples for FS; + // if (InstrProf.find(name) { + // root_name = name; + // } else { + // if (name is in static_func map) { + // root_name = static_name; + // } + // } + // update the Map entry for root_name; + // for (subfs: FS) { + // recursive(subfs, root_name); + // } + // } + // + // Here is an example. + // + // SampleProfile: + // foo:12345:1000 + // 1: 1000 + // 2.1: 1000 + // 15: 5000 + // 4: bar:1000 + // 1: 1000 + // 2: goo:3000 + // 1: 3000 + // 8: bar:40000 + // 1: 10000 + // 2: goo:30000 + // 1: 30000 + // + // InstrProfile has two entries: + // foo + // bar.cc:bar + // + // After BuildMaxSampleMap, we should have the following in FlattenSampleMap: + // {"foo", {1000, 5000}} + // {"bar.cc:bar", {11000, 30000}} + // + // foo's has an entry count of 1000, and max body count of 5000. + // bar.cc:bar has an entry count of 11000 (sum two callsites of 1000 and + // 10000), and max count of 30000 (from the callsite in line 8). + // + // Note that goo's count will remain in bar.cc:bar() as it does not have an + // entry in InstrProfile. + DenseMap> FlattenSampleMap; + auto BuildMaxSampleMap = [&FlattenSampleMap, &StaticFuncMap, + &InstrProfileMap](const FunctionSamples &FS, + const StringRef &RootName) { + auto BuildMaxSampleMapImpl = [&](const FunctionSamples &FS, + const StringRef &RootName, + auto &BuildImpl) -> void { + const StringRef &Name = FS.getName(); + const StringRef *NewRootName = &RootName; + uint64_t EntrySample = FS.getHeadSamplesEstimate(); + uint64_t MaxBodySample = FS.getMaxCountInside(/* SkipCallSite*/ true); + + auto It = InstrProfileMap.find(Name); + if (It != InstrProfileMap.end()) { + NewRootName = &Name; + } else { + auto NewName = StaticFuncMap.find(Name); + if (NewName != StaticFuncMap.end()) { + It = InstrProfileMap.find(NewName->second.str()); + if (NewName->second != DuplicateNameStr) { + NewRootName = &NewName->second; + } + } else { + // Here the EntrySample is of an inlined function, so we should not + // update the EntrySample in the map. + EntrySample = 0; + } + } + EntrySample += FlattenSampleMap[*NewRootName].first; + MaxBodySample = + std::max(FlattenSampleMap[*NewRootName].second, MaxBodySample); + FlattenSampleMap[*NewRootName] = + std::make_pair(EntrySample, MaxBodySample); + + for (const auto &C : FS.getCallsiteSamples()) + for (const auto &F : C.second) + BuildImpl(F.second, *NewRootName, BuildImpl); + }; + BuildMaxSampleMapImpl(FS, RootName, BuildMaxSampleMapImpl); + }; + + for (auto &PD : WC->Writer.getProfileData()) { + // Populate IPBuilder. + for (const auto &PDV : PD.getValue()) { + InstrProfRecord Record = PDV.second; + IPBuilder.addRecord(Record); + } + + // If a function has multiple entries in instr profile, skip it. + if (PD.getValue().size() != 1) + continue; + + // Initialize InstrProfileMap. + InstrProfRecord *R = &PD.getValue().begin()->second; + StringRef FullName = PD.getKey(); + InstrProfileMap[FullName] = InstrProfileEntry(R); + buildStaticFuncMap(FullName); + } + + for (auto &PD : Reader->getProfiles()) { + sampleprof::FunctionSamples &FS = PD.second; + BuildMaxSampleMap(FS, FS.getName()); + } + + ProfileSummary InstrPS = *IPBuilder.getSummary(); + ProfileSummary SamplePS = Reader->getSummary(); + + // Compute cold thresholds for instr profile and sample profile. + uint64_t HotSampleThreshold = + ProfileSummaryBuilder::getEntryForPercentile( + SamplePS.getDetailedSummary(), + ProfileSummaryBuilder::DefaultCutoffs[HotPercentileIdx]) + .MinCount; + uint64_t ColdSampleThreshold = + ProfileSummaryBuilder::getEntryForPercentile( + SamplePS.getDetailedSummary(), + ProfileSummaryBuilder::DefaultCutoffs[ColdPercentileIdx]) + .MinCount; + uint64_t HotInstrThreshold = + ProfileSummaryBuilder::getEntryForPercentile( + InstrPS.getDetailedSummary(), + ProfileSummaryBuilder::DefaultCutoffs[HotPercentileIdx]) + .MinCount; + uint64_t ColdInstrThreshold = + InstrProfColdThreshold + ? InstrProfColdThreshold + : ProfileSummaryBuilder::getEntryForPercentile( + InstrPS.getDetailedSummary(), + ProfileSummaryBuilder::DefaultCutoffs[ColdPercentileIdx]) + .MinCount; + + // Find hot/warm functions in sample profile which is cold in instr profile + // and adjust the profiles of those functions in the instr profile. + for (const auto &E : FlattenSampleMap) { + uint64_t SampleMaxCount = std::max(E.second.first, E.second.second); + if (SampleMaxCount < ColdSampleThreshold) + continue; + const StringRef &Name = E.first; + auto It = InstrProfileMap.find(Name); + if (It == InstrProfileMap.end()) { + auto NewName = StaticFuncMap.find(Name); + if (NewName != StaticFuncMap.end()) { + It = InstrProfileMap.find(NewName->second.str()); + if (NewName->second == DuplicateNameStr) { + WithColor::warning() + << "Static function " << Name + << " has multiple promoted names, cannot adjust profile.\n"; + } + } + } + if (It == InstrProfileMap.end() || + It->second.MaxCount > ColdInstrThreshold || + It->second.NumEdgeCounters < SupplMinSizeThreshold) + continue; + bool SetToHot = SampleMaxCount >= HotSampleThreshold; + updateInstrProfileEntry(It->second, SetToHot, HotInstrThreshold, + ColdInstrThreshold, ZeroCounterThreshold); + } +} + +/// The main function to supplement instr profile with sample profile. +/// \Inputs contains the instr profile. \p SampleFilename specifies the +/// sample profile. \p OutputFilename specifies the output profile name. +/// \p OutputFormat specifies the output profile format. \p OutputSparse +/// specifies whether to generate sparse profile. \p SupplMinSizeThreshold +/// specifies the minimal size for the functions whose profile will be +/// adjusted. \p ZeroCounterThreshold is the threshold to check whether +/// a function contains too many zero counters and whether its profile +/// should be dropped. \p InstrProfColdThreshold is the user specified +/// cold threshold which will override the cold threshold got from the +/// instr profile summary. +static void supplementInstrProfile( + const WeightedFileVector &Inputs, StringRef SampleFilename, + StringRef OutputFilename, ProfileFormat OutputFormat, bool OutputSparse, + unsigned SupplMinSizeThreshold, float ZeroCounterThreshold, + unsigned InstrProfColdThreshold) { + if (OutputFilename.compare("-") == 0) + exitWithError("cannot write indexed profdata format to stdout"); + if (Inputs.size() != 1) + exitWithError("expect one input to be an instr profile"); + if (Inputs[0].Weight != 1) + exitWithError("expect instr profile doesn't have weight"); + + StringRef InstrFilename = Inputs[0].Filename; + + // Read sample profile. + LLVMContext Context; + auto ReaderOrErr = sampleprof::SampleProfileReader::create( + SampleFilename.str(), Context, FSDiscriminatorPassOption); + if (std::error_code EC = ReaderOrErr.getError()) + exitWithErrorCode(EC, SampleFilename); + auto Reader = std::move(ReaderOrErr.get()); + if (std::error_code EC = Reader->read()) + exitWithErrorCode(EC, SampleFilename); + + // Read instr profile. + std::mutex ErrorLock; + SmallSet WriterErrorCodes; + auto WC = std::make_unique(OutputSparse, ErrorLock, + WriterErrorCodes); + loadInput(Inputs[0], nullptr, nullptr, /*ProfiledBinary=*/"", WC.get()); + if (WC->Errors.size() > 0) + exitWithError(std::move(WC->Errors[0].first), InstrFilename); + + adjustInstrProfile(WC, Reader, SupplMinSizeThreshold, ZeroCounterThreshold, + InstrProfColdThreshold); + writeInstrProfile(OutputFilename, OutputFormat, WC->Writer); +} + +/// Make a copy of the given function samples with all symbol names remapped +/// by the provided symbol remapper. +static sampleprof::FunctionSamples +remapSamples(const sampleprof::FunctionSamples &Samples, + SymbolRemapper &Remapper, sampleprof_error &Error) { + sampleprof::FunctionSamples Result; + Result.setName(Remapper(Samples.getName())); + Result.addTotalSamples(Samples.getTotalSamples()); + Result.addHeadSamples(Samples.getHeadSamples()); + for (const auto &BodySample : Samples.getBodySamples()) { + uint32_t MaskedDiscriminator = + BodySample.first.Discriminator & getDiscriminatorMask(); + Result.addBodySamples(BodySample.first.LineOffset, MaskedDiscriminator, + BodySample.second.getSamples()); + for (const auto &Target : BodySample.second.getCallTargets()) { + Result.addCalledTargetSamples(BodySample.first.LineOffset, + MaskedDiscriminator, + Remapper(Target.first()), Target.second); + } + } + for (const auto &CallsiteSamples : Samples.getCallsiteSamples()) { + sampleprof::FunctionSamplesMap &Target = + Result.functionSamplesAt(CallsiteSamples.first); + for (const auto &Callsite : CallsiteSamples.second) { + sampleprof::FunctionSamples Remapped = + remapSamples(Callsite.second, Remapper, Error); + MergeResult(Error, + Target[std::string(Remapped.getName())].merge(Remapped)); + } + } + return Result; +} + +static sampleprof::SampleProfileFormat FormatMap[] = { + sampleprof::SPF_None, + sampleprof::SPF_Text, + sampleprof::SPF_Compact_Binary, + sampleprof::SPF_Ext_Binary, + sampleprof::SPF_GCC, + sampleprof::SPF_Binary}; + +static std::unique_ptr +getInputFileBuf(const StringRef &InputFile) { + if (InputFile == "") + return {}; + + auto BufOrError = MemoryBuffer::getFileOrSTDIN(InputFile); + if (!BufOrError) + exitWithErrorCode(BufOrError.getError(), InputFile); + + return std::move(*BufOrError); +} + +static void populateProfileSymbolList(MemoryBuffer *Buffer, + sampleprof::ProfileSymbolList &PSL) { + if (!Buffer) + return; + + SmallVector SymbolVec; + StringRef Data = Buffer->getBuffer(); + Data.split(SymbolVec, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false); + + for (StringRef SymbolStr : SymbolVec) + PSL.add(SymbolStr.trim()); +} + +static void handleExtBinaryWriter(sampleprof::SampleProfileWriter &Writer, + ProfileFormat OutputFormat, + MemoryBuffer *Buffer, + sampleprof::ProfileSymbolList &WriterList, + bool CompressAllSections, bool UseMD5, + bool GenPartialProfile) { + populateProfileSymbolList(Buffer, WriterList); + if (WriterList.size() > 0 && OutputFormat != PF_Ext_Binary) + warn("Profile Symbol list is not empty but the output format is not " + "ExtBinary format. The list will be lost in the output. "); + + Writer.setProfileSymbolList(&WriterList); + + if (CompressAllSections) { + if (OutputFormat != PF_Ext_Binary) + warn("-compress-all-section is ignored. Specify -extbinary to enable it"); + else + Writer.setToCompressAllSections(); + } + if (UseMD5) { + if (OutputFormat != PF_Ext_Binary) + warn("-use-md5 is ignored. Specify -extbinary to enable it"); + else + Writer.setUseMD5(); + } + if (GenPartialProfile) { + if (OutputFormat != PF_Ext_Binary) + warn("-gen-partial-profile is ignored. Specify -extbinary to enable it"); + else + Writer.setPartialProfile(); + } +} + +static void +mergeSampleProfile(const WeightedFileVector &Inputs, SymbolRemapper *Remapper, + StringRef OutputFilename, ProfileFormat OutputFormat, + StringRef ProfileSymbolListFile, bool CompressAllSections, + bool UseMD5, bool GenPartialProfile, bool GenCSNestedProfile, + bool SampleMergeColdContext, bool SampleTrimColdContext, + bool SampleColdContextFrameDepth, FailureMode FailMode, + bool DropProfileSymbolList) { + using namespace sampleprof; + SampleProfileMap ProfileMap; + SmallVector, 5> Readers; + LLVMContext Context; + sampleprof::ProfileSymbolList WriterList; + std::optional ProfileIsProbeBased; + std::optional ProfileIsCS; + for (const auto &Input : Inputs) { + auto ReaderOrErr = SampleProfileReader::create(Input.Filename, Context, + FSDiscriminatorPassOption); + if (std::error_code EC = ReaderOrErr.getError()) { + warnOrExitGivenError(FailMode, EC, Input.Filename); + continue; + } + + // We need to keep the readers around until after all the files are + // read so that we do not lose the function names stored in each + // reader's memory. The function names are needed to write out the + // merged profile map. + Readers.push_back(std::move(ReaderOrErr.get())); + const auto Reader = Readers.back().get(); + if (std::error_code EC = Reader->read()) { + warnOrExitGivenError(FailMode, EC, Input.Filename); + Readers.pop_back(); + continue; + } + + SampleProfileMap &Profiles = Reader->getProfiles(); + if (ProfileIsProbeBased && + ProfileIsProbeBased != FunctionSamples::ProfileIsProbeBased) + exitWithError( + "cannot merge probe-based profile with non-probe-based profile"); + ProfileIsProbeBased = FunctionSamples::ProfileIsProbeBased; + if (ProfileIsCS && ProfileIsCS != FunctionSamples::ProfileIsCS) + exitWithError("cannot merge CS profile with non-CS profile"); + ProfileIsCS = FunctionSamples::ProfileIsCS; + for (SampleProfileMap::iterator I = Profiles.begin(), E = Profiles.end(); + I != E; ++I) { + sampleprof_error Result = sampleprof_error::success; + FunctionSamples Remapped = + Remapper ? remapSamples(I->second, *Remapper, Result) + : FunctionSamples(); + FunctionSamples &Samples = Remapper ? Remapped : I->second; + SampleContext FContext = Samples.getContext(); + MergeResult(Result, ProfileMap[FContext].merge(Samples, Input.Weight)); + if (Result != sampleprof_error::success) { + std::error_code EC = make_error_code(Result); + handleMergeWriterError(errorCodeToError(EC), Input.Filename, + FContext.toString()); + } + } + + if (!DropProfileSymbolList) { + std::unique_ptr ReaderList = + Reader->getProfileSymbolList(); + if (ReaderList) + WriterList.merge(*ReaderList); + } + } + + if (ProfileIsCS && (SampleMergeColdContext || SampleTrimColdContext)) { + // Use threshold calculated from profile summary unless specified. + SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs); + auto Summary = Builder.computeSummaryForProfiles(ProfileMap); + uint64_t SampleProfColdThreshold = + ProfileSummaryBuilder::getColdCountThreshold( + (Summary->getDetailedSummary())); + + // Trim and merge cold context profile using cold threshold above; + SampleContextTrimmer(ProfileMap) + .trimAndMergeColdContextProfiles( + SampleProfColdThreshold, SampleTrimColdContext, + SampleMergeColdContext, SampleColdContextFrameDepth, false); + } + + if (ProfileIsCS && GenCSNestedProfile) { + CSProfileConverter CSConverter(ProfileMap); + CSConverter.convertProfiles(); + ProfileIsCS = FunctionSamples::ProfileIsCS = false; + } + + auto WriterOrErr = + SampleProfileWriter::create(OutputFilename, FormatMap[OutputFormat]); + if (std::error_code EC = WriterOrErr.getError()) + exitWithErrorCode(EC, OutputFilename); + + auto Writer = std::move(WriterOrErr.get()); + // WriterList will have StringRef refering to string in Buffer. + // Make sure Buffer lives as long as WriterList. + auto Buffer = getInputFileBuf(ProfileSymbolListFile); + handleExtBinaryWriter(*Writer, OutputFormat, Buffer.get(), WriterList, + CompressAllSections, UseMD5, GenPartialProfile); + if (std::error_code EC = Writer->write(ProfileMap)) + exitWithErrorCode(std::move(EC)); +} + +static WeightedFile parseWeightedFile(const StringRef &WeightedFilename) { + StringRef WeightStr, FileName; + std::tie(WeightStr, FileName) = WeightedFilename.split(','); + + uint64_t Weight; + if (WeightStr.getAsInteger(10, Weight) || Weight < 1) + exitWithError("input weight must be a positive integer"); + + return {std::string(FileName), Weight}; +} + +static void addWeightedInput(WeightedFileVector &WNI, const WeightedFile &WF) { + StringRef Filename = WF.Filename; + uint64_t Weight = WF.Weight; + + // If it's STDIN just pass it on. + if (Filename == "-") { + WNI.push_back({std::string(Filename), Weight}); + return; + } + + llvm::sys::fs::file_status Status; + llvm::sys::fs::status(Filename, Status); + if (!llvm::sys::fs::exists(Status)) + exitWithErrorCode(make_error_code(errc::no_such_file_or_directory), + Filename); + // If it's a source file, collect it. + if (llvm::sys::fs::is_regular_file(Status)) { + WNI.push_back({std::string(Filename), Weight}); + return; + } + + if (llvm::sys::fs::is_directory(Status)) { + std::error_code EC; + for (llvm::sys::fs::recursive_directory_iterator F(Filename, EC), E; + F != E && !EC; F.increment(EC)) { + if (llvm::sys::fs::is_regular_file(F->path())) { + addWeightedInput(WNI, {F->path(), Weight}); + } + } + if (EC) + exitWithErrorCode(EC, Filename); + } +} + +static void parseInputFilenamesFile(MemoryBuffer *Buffer, + WeightedFileVector &WFV) { + if (!Buffer) + return; + + SmallVector Entries; + StringRef Data = Buffer->getBuffer(); + Data.split(Entries, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false); + for (const StringRef &FileWeightEntry : Entries) { + StringRef SanitizedEntry = FileWeightEntry.trim(" \t\v\f\r"); + // Skip comments. + if (SanitizedEntry.startswith("#")) + continue; + // If there's no comma, it's an unweighted profile. + else if (!SanitizedEntry.contains(',')) + addWeightedInput(WFV, {std::string(SanitizedEntry), 1}); + else + addWeightedInput(WFV, parseWeightedFile(SanitizedEntry)); + } +} + +static int merge_main(int argc, const char *argv[]) { + cl::list InputFilenames(cl::Positional, + cl::desc("")); + cl::list WeightedInputFilenames("weighted-input", + cl::desc(",")); + cl::opt InputFilenamesFile( + "input-files", cl::init(""), + cl::desc("Path to file containing newline-separated " + "[,] entries")); + cl::alias InputFilenamesFileA("f", cl::desc("Alias for --input-files"), + cl::aliasopt(InputFilenamesFile)); + cl::opt DumpInputFileList( + "dump-input-file-list", cl::init(false), cl::Hidden, + cl::desc("Dump the list of input files and their weights, then exit")); + cl::opt RemappingFile("remapping-file", cl::value_desc("file"), + cl::desc("Symbol remapping file")); + cl::alias RemappingFileA("r", cl::desc("Alias for --remapping-file"), + cl::aliasopt(RemappingFile)); + cl::opt OutputFilename("output", cl::value_desc("output"), + cl::init("-"), cl::desc("Output file")); + cl::alias OutputFilenameA("o", cl::desc("Alias for --output"), + cl::aliasopt(OutputFilename)); + cl::opt ProfileKind( + cl::desc("Profile kind:"), cl::init(instr), + cl::values(clEnumVal(instr, "Instrumentation profile (default)"), + clEnumVal(sample, "Sample profile"))); + cl::opt OutputFormat( + cl::desc("Format of output profile"), cl::init(PF_Binary), + cl::values( + clEnumValN(PF_Binary, "binary", "Binary encoding (default)"), + clEnumValN(PF_Compact_Binary, "compbinary", + "Compact binary encoding"), + clEnumValN(PF_Ext_Binary, "extbinary", "Extensible binary encoding"), + clEnumValN(PF_Text, "text", "Text encoding"), + clEnumValN(PF_GCC, "gcc", + "GCC encoding (only meaningful for -sample)"))); + cl::opt FailureMode( + "failure-mode", cl::init(failIfAnyAreInvalid), cl::desc("Failure mode:"), + cl::values(clEnumValN(failIfAnyAreInvalid, "any", + "Fail if any profile is invalid."), + clEnumValN(failIfAllAreInvalid, "all", + "Fail only if all profiles are invalid."))); + cl::opt OutputSparse("sparse", cl::init(false), + cl::desc("Generate a sparse profile (only meaningful for -instr)")); + cl::opt NumThreads( + "num-threads", cl::init(0), + cl::desc("Number of merge threads to use (default: autodetect)")); + cl::alias NumThreadsA("j", cl::desc("Alias for --num-threads"), + cl::aliasopt(NumThreads)); + cl::opt ProfileSymbolListFile( + "prof-sym-list", cl::init(""), + cl::desc("Path to file containing the list of function symbols " + "used to populate profile symbol list")); + cl::opt CompressAllSections( + "compress-all-sections", cl::init(false), cl::Hidden, + cl::desc("Compress all sections when writing the profile (only " + "meaningful for -extbinary)")); + cl::opt UseMD5( + "use-md5", cl::init(false), cl::Hidden, + cl::desc("Choose to use MD5 to represent string in name table (only " + "meaningful for -extbinary)")); + cl::opt SampleMergeColdContext( + "sample-merge-cold-context", cl::init(false), cl::Hidden, + cl::desc( + "Merge context sample profiles whose count is below cold threshold")); + cl::opt SampleTrimColdContext( + "sample-trim-cold-context", cl::init(false), cl::Hidden, + cl::desc( + "Trim context sample profiles whose count is below cold threshold")); + cl::opt SampleColdContextFrameDepth( + "sample-frame-depth-for-cold-context", cl::init(1), + cl::desc("Keep the last K frames while merging cold profile. 1 means the " + "context-less base profile")); + cl::opt GenPartialProfile( + "gen-partial-profile", cl::init(false), cl::Hidden, + cl::desc("Generate a partial profile (only meaningful for -extbinary)")); + cl::opt SupplInstrWithSample( + "supplement-instr-with-sample", cl::init(""), cl::Hidden, + cl::desc("Supplement an instr profile with sample profile, to correct " + "the profile unrepresentativeness issue. The sample " + "profile is the input of the flag. Output will be in instr " + "format (The flag only works with -instr)")); + cl::opt ZeroCounterThreshold( + "zero-counter-threshold", cl::init(0.7), cl::Hidden, + cl::desc("For the function which is cold in instr profile but hot in " + "sample profile, if the ratio of the number of zero counters " + "divided by the total number of counters is above the " + "threshold, the profile of the function will be regarded as " + "being harmful for performance and will be dropped.")); + cl::opt SupplMinSizeThreshold( + "suppl-min-size-threshold", cl::init(10), cl::Hidden, + cl::desc("If the size of a function is smaller than the threshold, " + "assume it can be inlined by PGO early inliner and it won't " + "be adjusted based on sample profile.")); + cl::opt InstrProfColdThreshold( + "instr-prof-cold-threshold", cl::init(0), cl::Hidden, + cl::desc("User specified cold threshold for instr profile which will " + "override the cold threshold got from profile summary. ")); + cl::opt GenCSNestedProfile( + "gen-cs-nested-profile", cl::Hidden, cl::init(false), + cl::desc("Generate nested function profiles for CSSPGO")); + cl::opt DebugInfoFilename( + "debug-info", cl::init(""), + cl::desc("Use the provided debug info to correlate the raw profile.")); + cl::opt ProfiledBinary( + "profiled-binary", cl::init(""), + cl::desc("Path to binary from which the profile was collected.")); + cl::opt DropProfileSymbolList( + "drop-profile-symbol-list", cl::init(false), cl::Hidden, + cl::desc("Drop the profile symbol list when merging AutoFDO profiles " + "(only meaningful for -sample)")); + + cl::ParseCommandLineOptions(argc, argv, "LLVM profile data merger\n"); + + WeightedFileVector WeightedInputs; + for (StringRef Filename : InputFilenames) + addWeightedInput(WeightedInputs, {std::string(Filename), 1}); + for (StringRef WeightedFilename : WeightedInputFilenames) + addWeightedInput(WeightedInputs, parseWeightedFile(WeightedFilename)); + + // Make sure that the file buffer stays alive for the duration of the + // weighted input vector's lifetime. + auto Buffer = getInputFileBuf(InputFilenamesFile); + parseInputFilenamesFile(Buffer.get(), WeightedInputs); + + if (WeightedInputs.empty()) + exitWithError("no input files specified. See " + + sys::path::filename(argv[0]) + " -help"); + + if (DumpInputFileList) { + for (auto &WF : WeightedInputs) + outs() << WF.Weight << "," << WF.Filename << "\n"; + return 0; + } + + std::unique_ptr Remapper; + if (!RemappingFile.empty()) + Remapper = SymbolRemapper::create(RemappingFile); + + if (!SupplInstrWithSample.empty()) { + if (ProfileKind != instr) + exitWithError( + "-supplement-instr-with-sample can only work with -instr. "); + + supplementInstrProfile(WeightedInputs, SupplInstrWithSample, OutputFilename, + OutputFormat, OutputSparse, SupplMinSizeThreshold, + ZeroCounterThreshold, InstrProfColdThreshold); + return 0; + } + + if (ProfileKind == instr) + mergeInstrProfile(WeightedInputs, DebugInfoFilename, Remapper.get(), + OutputFilename, OutputFormat, OutputSparse, NumThreads, + FailureMode, ProfiledBinary); + else + mergeSampleProfile( + WeightedInputs, Remapper.get(), OutputFilename, OutputFormat, + ProfileSymbolListFile, CompressAllSections, UseMD5, GenPartialProfile, + GenCSNestedProfile, SampleMergeColdContext, SampleTrimColdContext, + SampleColdContextFrameDepth, FailureMode, DropProfileSymbolList); + return 0; +} + +/// Computer the overlap b/w profile BaseFilename and profile TestFilename. +static void overlapInstrProfile(const std::string &BaseFilename, + const std::string &TestFilename, + const OverlapFuncFilters &FuncFilter, + raw_fd_ostream &OS, bool IsCS) { + std::mutex ErrorLock; + SmallSet WriterErrorCodes; + WriterContext Context(false, ErrorLock, WriterErrorCodes); + WeightedFile WeightedInput{BaseFilename, 1}; + OverlapStats Overlap; + Error E = Overlap.accumulateCounts(BaseFilename, TestFilename, IsCS); + if (E) + exitWithError(std::move(E), "error in getting profile count sums"); + if (Overlap.Base.CountSum < 1.0f) { + OS << "Sum of edge counts for profile " << BaseFilename << " is 0.\n"; + exit(0); + } + if (Overlap.Test.CountSum < 1.0f) { + OS << "Sum of edge counts for profile " << TestFilename << " is 0.\n"; + exit(0); + } + loadInput(WeightedInput, nullptr, nullptr, /*ProfiledBinary=*/"", &Context); + overlapInput(BaseFilename, TestFilename, &Context, Overlap, FuncFilter, OS, + IsCS); + Overlap.dump(OS); +} + +namespace { +struct SampleOverlapStats { + SampleContext BaseName; + SampleContext TestName; + // Number of overlap units + uint64_t OverlapCount; + // Total samples of overlap units + uint64_t OverlapSample; + // Number of and total samples of units that only present in base or test + // profile + uint64_t BaseUniqueCount; + uint64_t BaseUniqueSample; + uint64_t TestUniqueCount; + uint64_t TestUniqueSample; + // Number of units and total samples in base or test profile + uint64_t BaseCount; + uint64_t BaseSample; + uint64_t TestCount; + uint64_t TestSample; + // Number of and total samples of units that present in at least one profile + uint64_t UnionCount; + uint64_t UnionSample; + // Weighted similarity + double Similarity; + // For SampleOverlapStats instances representing functions, weights of the + // function in base and test profiles + double BaseWeight; + double TestWeight; + + SampleOverlapStats() + : OverlapCount(0), OverlapSample(0), BaseUniqueCount(0), + BaseUniqueSample(0), TestUniqueCount(0), TestUniqueSample(0), + BaseCount(0), BaseSample(0), TestCount(0), TestSample(0), UnionCount(0), + UnionSample(0), Similarity(0.0), BaseWeight(0.0), TestWeight(0.0) {} +}; +} // end anonymous namespace + +namespace { +struct FuncSampleStats { + uint64_t SampleSum; + uint64_t MaxSample; + uint64_t HotBlockCount; + FuncSampleStats() : SampleSum(0), MaxSample(0), HotBlockCount(0) {} + FuncSampleStats(uint64_t SampleSum, uint64_t MaxSample, + uint64_t HotBlockCount) + : SampleSum(SampleSum), MaxSample(MaxSample), + HotBlockCount(HotBlockCount) {} +}; +} // end anonymous namespace + +namespace { +enum MatchStatus { MS_Match, MS_FirstUnique, MS_SecondUnique, MS_None }; + +// Class for updating merging steps for two sorted maps. The class should be +// instantiated with a map iterator type. +template class MatchStep { +public: + MatchStep() = delete; + + MatchStep(T FirstIter, T FirstEnd, T SecondIter, T SecondEnd) + : FirstIter(FirstIter), FirstEnd(FirstEnd), SecondIter(SecondIter), + SecondEnd(SecondEnd), Status(MS_None) {} + + bool areBothFinished() const { + return (FirstIter == FirstEnd && SecondIter == SecondEnd); + } + + bool isFirstFinished() const { return FirstIter == FirstEnd; } + + bool isSecondFinished() const { return SecondIter == SecondEnd; } + + /// Advance one step based on the previous match status unless the previous + /// status is MS_None. Then update Status based on the comparison between two + /// container iterators at the current step. If the previous status is + /// MS_None, it means two iterators are at the beginning and no comparison has + /// been made, so we simply update Status without advancing the iterators. + void updateOneStep(); + + T getFirstIter() const { return FirstIter; } + + T getSecondIter() const { return SecondIter; } + + MatchStatus getMatchStatus() const { return Status; } + +private: + // Current iterator and end iterator of the first container. + T FirstIter; + T FirstEnd; + // Current iterator and end iterator of the second container. + T SecondIter; + T SecondEnd; + // Match status of the current step. + MatchStatus Status; +}; +} // end anonymous namespace + +template void MatchStep::updateOneStep() { + switch (Status) { + case MS_Match: + ++FirstIter; + ++SecondIter; + break; + case MS_FirstUnique: + ++FirstIter; + break; + case MS_SecondUnique: + ++SecondIter; + break; + case MS_None: + break; + } + + // Update Status according to iterators at the current step. + if (areBothFinished()) + return; + if (FirstIter != FirstEnd && + (SecondIter == SecondEnd || FirstIter->first < SecondIter->first)) + Status = MS_FirstUnique; + else if (SecondIter != SecondEnd && + (FirstIter == FirstEnd || SecondIter->first < FirstIter->first)) + Status = MS_SecondUnique; + else + Status = MS_Match; +} + +// Return the sum of line/block samples, the max line/block sample, and the +// number of line/block samples above the given threshold in a function +// including its inlinees. +static void getFuncSampleStats(const sampleprof::FunctionSamples &Func, + FuncSampleStats &FuncStats, + uint64_t HotThreshold) { + for (const auto &L : Func.getBodySamples()) { + uint64_t Sample = L.second.getSamples(); + FuncStats.SampleSum += Sample; + FuncStats.MaxSample = std::max(FuncStats.MaxSample, Sample); + if (Sample >= HotThreshold) + ++FuncStats.HotBlockCount; + } + + for (const auto &C : Func.getCallsiteSamples()) { + for (const auto &F : C.second) + getFuncSampleStats(F.second, FuncStats, HotThreshold); + } +} + +/// Predicate that determines if a function is hot with a given threshold. We +/// keep it separate from its callsites for possible extension in the future. +static bool isFunctionHot(const FuncSampleStats &FuncStats, + uint64_t HotThreshold) { + // We intentionally compare the maximum sample count in a function with the + // HotThreshold to get an approximate determination on hot functions. + return (FuncStats.MaxSample >= HotThreshold); +} + +namespace { +class SampleOverlapAggregator { +public: + SampleOverlapAggregator(const std::string &BaseFilename, + const std::string &TestFilename, + double LowSimilarityThreshold, double Epsilon, + const OverlapFuncFilters &FuncFilter) + : BaseFilename(BaseFilename), TestFilename(TestFilename), + LowSimilarityThreshold(LowSimilarityThreshold), Epsilon(Epsilon), + FuncFilter(FuncFilter) {} + + /// Detect 0-sample input profile and report to output stream. This interface + /// should be called after loadProfiles(). + bool detectZeroSampleProfile(raw_fd_ostream &OS) const; + + /// Write out function-level similarity statistics for functions specified by + /// options --function, --value-cutoff, and --similarity-cutoff. + void dumpFuncSimilarity(raw_fd_ostream &OS) const; + + /// Write out program-level similarity and overlap statistics. + void dumpProgramSummary(raw_fd_ostream &OS) const; + + /// Write out hot-function and hot-block statistics for base_profile, + /// test_profile, and their overlap. For both cases, the overlap HO is + /// calculated as follows: + /// Given the number of functions (or blocks) that are hot in both profiles + /// HCommon and the number of functions (or blocks) that are hot in at + /// least one profile HUnion, HO = HCommon / HUnion. + void dumpHotFuncAndBlockOverlap(raw_fd_ostream &OS) const; + + /// This function tries matching functions in base and test profiles. For each + /// pair of matched functions, it aggregates the function-level + /// similarity into a profile-level similarity. It also dump function-level + /// similarity information of functions specified by --function, + /// --value-cutoff, and --similarity-cutoff options. The program-level + /// similarity PS is computed as follows: + /// Given function-level similarity FS(A) for all function A, the + /// weight of function A in base profile WB(A), and the weight of function + /// A in test profile WT(A), compute PS(base_profile, test_profile) = + /// sum_A(FS(A) * avg(WB(A), WT(A))) ranging in [0.0f to 1.0f] with 0.0 + /// meaning no-overlap. + void computeSampleProfileOverlap(raw_fd_ostream &OS); + + /// Initialize ProfOverlap with the sum of samples in base and test + /// profiles. This function also computes and keeps the sum of samples and + /// max sample counts of each function in BaseStats and TestStats for later + /// use to avoid re-computations. + void initializeSampleProfileOverlap(); + + /// Load profiles specified by BaseFilename and TestFilename. + std::error_code loadProfiles(); + + using FuncSampleStatsMap = + std::unordered_map; + +private: + SampleOverlapStats ProfOverlap; + SampleOverlapStats HotFuncOverlap; + SampleOverlapStats HotBlockOverlap; + std::string BaseFilename; + std::string TestFilename; + std::unique_ptr BaseReader; + std::unique_ptr TestReader; + // BaseStats and TestStats hold FuncSampleStats for each function, with + // function name as the key. + FuncSampleStatsMap BaseStats; + FuncSampleStatsMap TestStats; + // Low similarity threshold in floating point number + double LowSimilarityThreshold; + // Block samples above BaseHotThreshold or TestHotThreshold are considered hot + // for tracking hot blocks. + uint64_t BaseHotThreshold; + uint64_t TestHotThreshold; + // A small threshold used to round the results of floating point accumulations + // to resolve imprecision. + const double Epsilon; + std::multimap> + FuncSimilarityDump; + // FuncFilter carries specifications in options --value-cutoff and + // --function. + OverlapFuncFilters FuncFilter; + // Column offsets for printing the function-level details table. + static const unsigned int TestWeightCol = 15; + static const unsigned int SimilarityCol = 30; + static const unsigned int OverlapCol = 43; + static const unsigned int BaseUniqueCol = 53; + static const unsigned int TestUniqueCol = 67; + static const unsigned int BaseSampleCol = 81; + static const unsigned int TestSampleCol = 96; + static const unsigned int FuncNameCol = 111; + + /// Return a similarity of two line/block sample counters in the same + /// function in base and test profiles. The line/block-similarity BS(i) is + /// computed as follows: + /// For an offsets i, given the sample count at i in base profile BB(i), + /// the sample count at i in test profile BT(i), the sum of sample counts + /// in this function in base profile SB, and the sum of sample counts in + /// this function in test profile ST, compute BS(i) = 1.0 - fabs(BB(i)/SB - + /// BT(i)/ST), ranging in [0.0f to 1.0f] with 0.0 meaning no-overlap. + double computeBlockSimilarity(uint64_t BaseSample, uint64_t TestSample, + const SampleOverlapStats &FuncOverlap) const; + + void updateHotBlockOverlap(uint64_t BaseSample, uint64_t TestSample, + uint64_t HotBlockCount); + + void getHotFunctions(const FuncSampleStatsMap &ProfStats, + FuncSampleStatsMap &HotFunc, + uint64_t HotThreshold) const; + + void computeHotFuncOverlap(); + + /// This function updates statistics in FuncOverlap, HotBlockOverlap, and + /// Difference for two sample units in a matched function according to the + /// given match status. + void updateOverlapStatsForFunction(uint64_t BaseSample, uint64_t TestSample, + uint64_t HotBlockCount, + SampleOverlapStats &FuncOverlap, + double &Difference, MatchStatus Status); + + /// This function updates statistics in FuncOverlap, HotBlockOverlap, and + /// Difference for unmatched callees that only present in one profile in a + /// matched caller function. + void updateForUnmatchedCallee(const sampleprof::FunctionSamples &Func, + SampleOverlapStats &FuncOverlap, + double &Difference, MatchStatus Status); + + /// This function updates sample overlap statistics of an overlap function in + /// base and test profile. It also calculates a function-internal similarity + /// FIS as follows: + /// For offsets i that have samples in at least one profile in this + /// function A, given BS(i) returned by computeBlockSimilarity(), compute + /// FIS(A) = (2.0 - sum_i(1.0 - BS(i))) / 2, ranging in [0.0f to 1.0f] with + /// 0.0 meaning no overlap. + double computeSampleFunctionInternalOverlap( + const sampleprof::FunctionSamples &BaseFunc, + const sampleprof::FunctionSamples &TestFunc, + SampleOverlapStats &FuncOverlap); + + /// Function-level similarity (FS) is a weighted value over function internal + /// similarity (FIS). This function computes a function's FS from its FIS by + /// applying the weight. + double weightForFuncSimilarity(double FuncSimilarity, uint64_t BaseFuncSample, + uint64_t TestFuncSample) const; + + /// The function-level similarity FS(A) for a function A is computed as + /// follows: + /// Compute a function-internal similarity FIS(A) by + /// computeSampleFunctionInternalOverlap(). Then, with the weight of + /// function A in base profile WB(A), and the weight of function A in test + /// profile WT(A), compute FS(A) = FIS(A) * (1.0 - fabs(WB(A) - WT(A))) + /// ranging in [0.0f to 1.0f] with 0.0 meaning no overlap. + double + computeSampleFunctionOverlap(const sampleprof::FunctionSamples *BaseFunc, + const sampleprof::FunctionSamples *TestFunc, + SampleOverlapStats *FuncOverlap, + uint64_t BaseFuncSample, + uint64_t TestFuncSample); + + /// Profile-level similarity (PS) is a weighted aggregate over function-level + /// similarities (FS). This method weights the FS value by the function + /// weights in the base and test profiles for the aggregation. + double weightByImportance(double FuncSimilarity, uint64_t BaseFuncSample, + uint64_t TestFuncSample) const; +}; +} // end anonymous namespace + +bool SampleOverlapAggregator::detectZeroSampleProfile( + raw_fd_ostream &OS) const { + bool HaveZeroSample = false; + if (ProfOverlap.BaseSample == 0) { + OS << "Sum of sample counts for profile " << BaseFilename << " is 0.\n"; + HaveZeroSample = true; + } + if (ProfOverlap.TestSample == 0) { + OS << "Sum of sample counts for profile " << TestFilename << " is 0.\n"; + HaveZeroSample = true; + } + return HaveZeroSample; +} + +double SampleOverlapAggregator::computeBlockSimilarity( + uint64_t BaseSample, uint64_t TestSample, + const SampleOverlapStats &FuncOverlap) const { + double BaseFrac = 0.0; + double TestFrac = 0.0; + if (FuncOverlap.BaseSample > 0) + BaseFrac = static_cast(BaseSample) / FuncOverlap.BaseSample; + if (FuncOverlap.TestSample > 0) + TestFrac = static_cast(TestSample) / FuncOverlap.TestSample; + return 1.0 - std::fabs(BaseFrac - TestFrac); +} + +void SampleOverlapAggregator::updateHotBlockOverlap(uint64_t BaseSample, + uint64_t TestSample, + uint64_t HotBlockCount) { + bool IsBaseHot = (BaseSample >= BaseHotThreshold); + bool IsTestHot = (TestSample >= TestHotThreshold); + if (!IsBaseHot && !IsTestHot) + return; + + HotBlockOverlap.UnionCount += HotBlockCount; + if (IsBaseHot) + HotBlockOverlap.BaseCount += HotBlockCount; + if (IsTestHot) + HotBlockOverlap.TestCount += HotBlockCount; + if (IsBaseHot && IsTestHot) + HotBlockOverlap.OverlapCount += HotBlockCount; +} + +void SampleOverlapAggregator::getHotFunctions( + const FuncSampleStatsMap &ProfStats, FuncSampleStatsMap &HotFunc, + uint64_t HotThreshold) const { + for (const auto &F : ProfStats) { + if (isFunctionHot(F.second, HotThreshold)) + HotFunc.emplace(F.first, F.second); + } +} + +void SampleOverlapAggregator::computeHotFuncOverlap() { + FuncSampleStatsMap BaseHotFunc; + getHotFunctions(BaseStats, BaseHotFunc, BaseHotThreshold); + HotFuncOverlap.BaseCount = BaseHotFunc.size(); + + FuncSampleStatsMap TestHotFunc; + getHotFunctions(TestStats, TestHotFunc, TestHotThreshold); + HotFuncOverlap.TestCount = TestHotFunc.size(); + HotFuncOverlap.UnionCount = HotFuncOverlap.TestCount; + + for (const auto &F : BaseHotFunc) { + if (TestHotFunc.count(F.first)) + ++HotFuncOverlap.OverlapCount; + else + ++HotFuncOverlap.UnionCount; + } +} + +void SampleOverlapAggregator::updateOverlapStatsForFunction( + uint64_t BaseSample, uint64_t TestSample, uint64_t HotBlockCount, + SampleOverlapStats &FuncOverlap, double &Difference, MatchStatus Status) { + assert(Status != MS_None && + "Match status should be updated before updating overlap statistics"); + if (Status == MS_FirstUnique) { + TestSample = 0; + FuncOverlap.BaseUniqueSample += BaseSample; + } else if (Status == MS_SecondUnique) { + BaseSample = 0; + FuncOverlap.TestUniqueSample += TestSample; + } else { + ++FuncOverlap.OverlapCount; + } + + FuncOverlap.UnionSample += std::max(BaseSample, TestSample); + FuncOverlap.OverlapSample += std::min(BaseSample, TestSample); + Difference += + 1.0 - computeBlockSimilarity(BaseSample, TestSample, FuncOverlap); + updateHotBlockOverlap(BaseSample, TestSample, HotBlockCount); +} + +void SampleOverlapAggregator::updateForUnmatchedCallee( + const sampleprof::FunctionSamples &Func, SampleOverlapStats &FuncOverlap, + double &Difference, MatchStatus Status) { + assert((Status == MS_FirstUnique || Status == MS_SecondUnique) && + "Status must be either of the two unmatched cases"); + FuncSampleStats FuncStats; + if (Status == MS_FirstUnique) { + getFuncSampleStats(Func, FuncStats, BaseHotThreshold); + updateOverlapStatsForFunction(FuncStats.SampleSum, 0, + FuncStats.HotBlockCount, FuncOverlap, + Difference, Status); + } else { + getFuncSampleStats(Func, FuncStats, TestHotThreshold); + updateOverlapStatsForFunction(0, FuncStats.SampleSum, + FuncStats.HotBlockCount, FuncOverlap, + Difference, Status); + } +} + +double SampleOverlapAggregator::computeSampleFunctionInternalOverlap( + const sampleprof::FunctionSamples &BaseFunc, + const sampleprof::FunctionSamples &TestFunc, + SampleOverlapStats &FuncOverlap) { + + using namespace sampleprof; + + double Difference = 0; + + // Accumulate Difference for regular line/block samples in the function. + // We match them through sort-merge join algorithm because + // FunctionSamples::getBodySamples() returns a map of sample counters ordered + // by their offsets. + MatchStep BlockIterStep( + BaseFunc.getBodySamples().cbegin(), BaseFunc.getBodySamples().cend(), + TestFunc.getBodySamples().cbegin(), TestFunc.getBodySamples().cend()); + BlockIterStep.updateOneStep(); + while (!BlockIterStep.areBothFinished()) { + uint64_t BaseSample = + BlockIterStep.isFirstFinished() + ? 0 + : BlockIterStep.getFirstIter()->second.getSamples(); + uint64_t TestSample = + BlockIterStep.isSecondFinished() + ? 0 + : BlockIterStep.getSecondIter()->second.getSamples(); + updateOverlapStatsForFunction(BaseSample, TestSample, 1, FuncOverlap, + Difference, BlockIterStep.getMatchStatus()); + + BlockIterStep.updateOneStep(); + } + + // Accumulate Difference for callsite lines in the function. We match + // them through sort-merge algorithm because + // FunctionSamples::getCallsiteSamples() returns a map of callsite records + // ordered by their offsets. + MatchStep CallsiteIterStep( + BaseFunc.getCallsiteSamples().cbegin(), + BaseFunc.getCallsiteSamples().cend(), + TestFunc.getCallsiteSamples().cbegin(), + TestFunc.getCallsiteSamples().cend()); + CallsiteIterStep.updateOneStep(); + while (!CallsiteIterStep.areBothFinished()) { + MatchStatus CallsiteStepStatus = CallsiteIterStep.getMatchStatus(); + assert(CallsiteStepStatus != MS_None && + "Match status should be updated before entering loop body"); + + if (CallsiteStepStatus != MS_Match) { + auto Callsite = (CallsiteStepStatus == MS_FirstUnique) + ? CallsiteIterStep.getFirstIter() + : CallsiteIterStep.getSecondIter(); + for (const auto &F : Callsite->second) + updateForUnmatchedCallee(F.second, FuncOverlap, Difference, + CallsiteStepStatus); + } else { + // There may be multiple inlinees at the same offset, so we need to try + // matching all of them. This match is implemented through sort-merge + // algorithm because callsite records at the same offset are ordered by + // function names. + MatchStep CalleeIterStep( + CallsiteIterStep.getFirstIter()->second.cbegin(), + CallsiteIterStep.getFirstIter()->second.cend(), + CallsiteIterStep.getSecondIter()->second.cbegin(), + CallsiteIterStep.getSecondIter()->second.cend()); + CalleeIterStep.updateOneStep(); + while (!CalleeIterStep.areBothFinished()) { + MatchStatus CalleeStepStatus = CalleeIterStep.getMatchStatus(); + if (CalleeStepStatus != MS_Match) { + auto Callee = (CalleeStepStatus == MS_FirstUnique) + ? CalleeIterStep.getFirstIter() + : CalleeIterStep.getSecondIter(); + updateForUnmatchedCallee(Callee->second, FuncOverlap, Difference, + CalleeStepStatus); + } else { + // An inlined function can contain other inlinees inside, so compute + // the Difference recursively. + Difference += 2.0 - 2 * computeSampleFunctionInternalOverlap( + CalleeIterStep.getFirstIter()->second, + CalleeIterStep.getSecondIter()->second, + FuncOverlap); + } + CalleeIterStep.updateOneStep(); + } + } + CallsiteIterStep.updateOneStep(); + } + + // Difference reflects the total differences of line/block samples in this + // function and ranges in [0.0f to 2.0f]. Take (2.0 - Difference) / 2 to + // reflect the similarity between function profiles in [0.0f to 1.0f]. + return (2.0 - Difference) / 2; +} + +double SampleOverlapAggregator::weightForFuncSimilarity( + double FuncInternalSimilarity, uint64_t BaseFuncSample, + uint64_t TestFuncSample) const { + // Compute the weight as the distance between the function weights in two + // profiles. + double BaseFrac = 0.0; + double TestFrac = 0.0; + assert(ProfOverlap.BaseSample > 0 && + "Total samples in base profile should be greater than 0"); + BaseFrac = static_cast(BaseFuncSample) / ProfOverlap.BaseSample; + assert(ProfOverlap.TestSample > 0 && + "Total samples in test profile should be greater than 0"); + TestFrac = static_cast(TestFuncSample) / ProfOverlap.TestSample; + double WeightDistance = std::fabs(BaseFrac - TestFrac); + + // Take WeightDistance into the similarity. + return FuncInternalSimilarity * (1 - WeightDistance); +} + +double +SampleOverlapAggregator::weightByImportance(double FuncSimilarity, + uint64_t BaseFuncSample, + uint64_t TestFuncSample) const { + + double BaseFrac = 0.0; + double TestFrac = 0.0; + assert(ProfOverlap.BaseSample > 0 && + "Total samples in base profile should be greater than 0"); + BaseFrac = static_cast(BaseFuncSample) / ProfOverlap.BaseSample / 2.0; + assert(ProfOverlap.TestSample > 0 && + "Total samples in test profile should be greater than 0"); + TestFrac = static_cast(TestFuncSample) / ProfOverlap.TestSample / 2.0; + return FuncSimilarity * (BaseFrac + TestFrac); +} + +double SampleOverlapAggregator::computeSampleFunctionOverlap( + const sampleprof::FunctionSamples *BaseFunc, + const sampleprof::FunctionSamples *TestFunc, + SampleOverlapStats *FuncOverlap, uint64_t BaseFuncSample, + uint64_t TestFuncSample) { + // Default function internal similarity before weighted, meaning two functions + // has no overlap. + const double DefaultFuncInternalSimilarity = 0; + double FuncSimilarity; + double FuncInternalSimilarity; + + // If BaseFunc or TestFunc is nullptr, it means the functions do not overlap. + // In this case, we use DefaultFuncInternalSimilarity as the function internal + // similarity. + if (!BaseFunc || !TestFunc) { + FuncInternalSimilarity = DefaultFuncInternalSimilarity; + } else { + assert(FuncOverlap != nullptr && + "FuncOverlap should be provided in this case"); + FuncInternalSimilarity = computeSampleFunctionInternalOverlap( + *BaseFunc, *TestFunc, *FuncOverlap); + // Now, FuncInternalSimilarity may be a little less than 0 due to + // imprecision of floating point accumulations. Make it zero if the + // difference is below Epsilon. + FuncInternalSimilarity = (std::fabs(FuncInternalSimilarity - 0) < Epsilon) + ? 0 + : FuncInternalSimilarity; + } + FuncSimilarity = weightForFuncSimilarity(FuncInternalSimilarity, + BaseFuncSample, TestFuncSample); + return FuncSimilarity; +} + +void SampleOverlapAggregator::computeSampleProfileOverlap(raw_fd_ostream &OS) { + using namespace sampleprof; + + std::unordered_map + BaseFuncProf; + const auto &BaseProfiles = BaseReader->getProfiles(); + for (const auto &BaseFunc : BaseProfiles) { + BaseFuncProf.emplace(BaseFunc.second.getContext(), &(BaseFunc.second)); + } + ProfOverlap.UnionCount = BaseFuncProf.size(); + + const auto &TestProfiles = TestReader->getProfiles(); + for (const auto &TestFunc : TestProfiles) { + SampleOverlapStats FuncOverlap; + FuncOverlap.TestName = TestFunc.second.getContext(); + assert(TestStats.count(FuncOverlap.TestName) && + "TestStats should have records for all functions in test profile " + "except inlinees"); + FuncOverlap.TestSample = TestStats[FuncOverlap.TestName].SampleSum; + + bool Matched = false; + const auto Match = BaseFuncProf.find(FuncOverlap.TestName); + if (Match == BaseFuncProf.end()) { + const FuncSampleStats &FuncStats = TestStats[FuncOverlap.TestName]; + ++ProfOverlap.TestUniqueCount; + ProfOverlap.TestUniqueSample += FuncStats.SampleSum; + FuncOverlap.TestUniqueSample = FuncStats.SampleSum; + + updateHotBlockOverlap(0, FuncStats.SampleSum, FuncStats.HotBlockCount); + + double FuncSimilarity = computeSampleFunctionOverlap( + nullptr, nullptr, nullptr, 0, FuncStats.SampleSum); + ProfOverlap.Similarity += + weightByImportance(FuncSimilarity, 0, FuncStats.SampleSum); + + ++ProfOverlap.UnionCount; + ProfOverlap.UnionSample += FuncStats.SampleSum; + } else { + ++ProfOverlap.OverlapCount; + + // Two functions match with each other. Compute function-level overlap and + // aggregate them into profile-level overlap. + FuncOverlap.BaseName = Match->second->getContext(); + assert(BaseStats.count(FuncOverlap.BaseName) && + "BaseStats should have records for all functions in base profile " + "except inlinees"); + FuncOverlap.BaseSample = BaseStats[FuncOverlap.BaseName].SampleSum; + + FuncOverlap.Similarity = computeSampleFunctionOverlap( + Match->second, &TestFunc.second, &FuncOverlap, FuncOverlap.BaseSample, + FuncOverlap.TestSample); + ProfOverlap.Similarity += + weightByImportance(FuncOverlap.Similarity, FuncOverlap.BaseSample, + FuncOverlap.TestSample); + ProfOverlap.OverlapSample += FuncOverlap.OverlapSample; + ProfOverlap.UnionSample += FuncOverlap.UnionSample; + + // Accumulate the percentage of base unique and test unique samples into + // ProfOverlap. + ProfOverlap.BaseUniqueSample += FuncOverlap.BaseUniqueSample; + ProfOverlap.TestUniqueSample += FuncOverlap.TestUniqueSample; + + // Remove matched base functions for later reporting functions not found + // in test profile. + BaseFuncProf.erase(Match); + Matched = true; + } + + // Print function-level similarity information if specified by options. + assert(TestStats.count(FuncOverlap.TestName) && + "TestStats should have records for all functions in test profile " + "except inlinees"); + if (TestStats[FuncOverlap.TestName].MaxSample >= FuncFilter.ValueCutoff || + (Matched && FuncOverlap.Similarity < LowSimilarityThreshold) || + (Matched && !FuncFilter.NameFilter.empty() && + FuncOverlap.BaseName.toString().find(FuncFilter.NameFilter) != + std::string::npos)) { + assert(ProfOverlap.BaseSample > 0 && + "Total samples in base profile should be greater than 0"); + FuncOverlap.BaseWeight = + static_cast(FuncOverlap.BaseSample) / ProfOverlap.BaseSample; + assert(ProfOverlap.TestSample > 0 && + "Total samples in test profile should be greater than 0"); + FuncOverlap.TestWeight = + static_cast(FuncOverlap.TestSample) / ProfOverlap.TestSample; + FuncSimilarityDump.emplace(FuncOverlap.BaseWeight, FuncOverlap); + } + } + + // Traverse through functions in base profile but not in test profile. + for (const auto &F : BaseFuncProf) { + assert(BaseStats.count(F.second->getContext()) && + "BaseStats should have records for all functions in base profile " + "except inlinees"); + const FuncSampleStats &FuncStats = BaseStats[F.second->getContext()]; + ++ProfOverlap.BaseUniqueCount; + ProfOverlap.BaseUniqueSample += FuncStats.SampleSum; + + updateHotBlockOverlap(FuncStats.SampleSum, 0, FuncStats.HotBlockCount); + + double FuncSimilarity = computeSampleFunctionOverlap( + nullptr, nullptr, nullptr, FuncStats.SampleSum, 0); + ProfOverlap.Similarity += + weightByImportance(FuncSimilarity, FuncStats.SampleSum, 0); + + ProfOverlap.UnionSample += FuncStats.SampleSum; + } + + // Now, ProfSimilarity may be a little greater than 1 due to imprecision + // of floating point accumulations. Make it 1.0 if the difference is below + // Epsilon. + ProfOverlap.Similarity = (std::fabs(ProfOverlap.Similarity - 1) < Epsilon) + ? 1 + : ProfOverlap.Similarity; + + computeHotFuncOverlap(); +} + +void SampleOverlapAggregator::initializeSampleProfileOverlap() { + const auto &BaseProf = BaseReader->getProfiles(); + for (const auto &I : BaseProf) { + ++ProfOverlap.BaseCount; + FuncSampleStats FuncStats; + getFuncSampleStats(I.second, FuncStats, BaseHotThreshold); + ProfOverlap.BaseSample += FuncStats.SampleSum; + BaseStats.emplace(I.second.getContext(), FuncStats); + } + + const auto &TestProf = TestReader->getProfiles(); + for (const auto &I : TestProf) { + ++ProfOverlap.TestCount; + FuncSampleStats FuncStats; + getFuncSampleStats(I.second, FuncStats, TestHotThreshold); + ProfOverlap.TestSample += FuncStats.SampleSum; + TestStats.emplace(I.second.getContext(), FuncStats); + } + + ProfOverlap.BaseName = StringRef(BaseFilename); + ProfOverlap.TestName = StringRef(TestFilename); +} + +void SampleOverlapAggregator::dumpFuncSimilarity(raw_fd_ostream &OS) const { + using namespace sampleprof; + + if (FuncSimilarityDump.empty()) + return; + + formatted_raw_ostream FOS(OS); + FOS << "Function-level details:\n"; + FOS << "Base weight"; + FOS.PadToColumn(TestWeightCol); + FOS << "Test weight"; + FOS.PadToColumn(SimilarityCol); + FOS << "Similarity"; + FOS.PadToColumn(OverlapCol); + FOS << "Overlap"; + FOS.PadToColumn(BaseUniqueCol); + FOS << "Base unique"; + FOS.PadToColumn(TestUniqueCol); + FOS << "Test unique"; + FOS.PadToColumn(BaseSampleCol); + FOS << "Base samples"; + FOS.PadToColumn(TestSampleCol); + FOS << "Test samples"; + FOS.PadToColumn(FuncNameCol); + FOS << "Function name\n"; + for (const auto &F : FuncSimilarityDump) { + double OverlapPercent = + F.second.UnionSample > 0 + ? static_cast(F.second.OverlapSample) / F.second.UnionSample + : 0; + double BaseUniquePercent = + F.second.BaseSample > 0 + ? static_cast(F.second.BaseUniqueSample) / + F.second.BaseSample + : 0; + double TestUniquePercent = + F.second.TestSample > 0 + ? static_cast(F.second.TestUniqueSample) / + F.second.TestSample + : 0; + + FOS << format("%.2f%%", F.second.BaseWeight * 100); + FOS.PadToColumn(TestWeightCol); + FOS << format("%.2f%%", F.second.TestWeight * 100); + FOS.PadToColumn(SimilarityCol); + FOS << format("%.2f%%", F.second.Similarity * 100); + FOS.PadToColumn(OverlapCol); + FOS << format("%.2f%%", OverlapPercent * 100); + FOS.PadToColumn(BaseUniqueCol); + FOS << format("%.2f%%", BaseUniquePercent * 100); + FOS.PadToColumn(TestUniqueCol); + FOS << format("%.2f%%", TestUniquePercent * 100); + FOS.PadToColumn(BaseSampleCol); + FOS << F.second.BaseSample; + FOS.PadToColumn(TestSampleCol); + FOS << F.second.TestSample; + FOS.PadToColumn(FuncNameCol); + FOS << F.second.TestName.toString() << "\n"; + } +} + +void SampleOverlapAggregator::dumpProgramSummary(raw_fd_ostream &OS) const { + OS << "Profile overlap infomation for base_profile: " + << ProfOverlap.BaseName.toString() + << " and test_profile: " << ProfOverlap.TestName.toString() + << "\nProgram level:\n"; + + OS << " Whole program profile similarity: " + << format("%.3f%%", ProfOverlap.Similarity * 100) << "\n"; + + assert(ProfOverlap.UnionSample > 0 && + "Total samples in two profile should be greater than 0"); + double OverlapPercent = + static_cast(ProfOverlap.OverlapSample) / ProfOverlap.UnionSample; + assert(ProfOverlap.BaseSample > 0 && + "Total samples in base profile should be greater than 0"); + double BaseUniquePercent = static_cast(ProfOverlap.BaseUniqueSample) / + ProfOverlap.BaseSample; + assert(ProfOverlap.TestSample > 0 && + "Total samples in test profile should be greater than 0"); + double TestUniquePercent = static_cast(ProfOverlap.TestUniqueSample) / + ProfOverlap.TestSample; + + OS << " Whole program sample overlap: " + << format("%.3f%%", OverlapPercent * 100) << "\n"; + OS << " percentage of samples unique in base profile: " + << format("%.3f%%", BaseUniquePercent * 100) << "\n"; + OS << " percentage of samples unique in test profile: " + << format("%.3f%%", TestUniquePercent * 100) << "\n"; + OS << " total samples in base profile: " << ProfOverlap.BaseSample << "\n" + << " total samples in test profile: " << ProfOverlap.TestSample << "\n"; + + assert(ProfOverlap.UnionCount > 0 && + "There should be at least one function in two input profiles"); + double FuncOverlapPercent = + static_cast(ProfOverlap.OverlapCount) / ProfOverlap.UnionCount; + OS << " Function overlap: " << format("%.3f%%", FuncOverlapPercent * 100) + << "\n"; + OS << " overlap functions: " << ProfOverlap.OverlapCount << "\n"; + OS << " functions unique in base profile: " << ProfOverlap.BaseUniqueCount + << "\n"; + OS << " functions unique in test profile: " << ProfOverlap.TestUniqueCount + << "\n"; +} + +void SampleOverlapAggregator::dumpHotFuncAndBlockOverlap( + raw_fd_ostream &OS) const { + assert(HotFuncOverlap.UnionCount > 0 && + "There should be at least one hot function in two input profiles"); + OS << " Hot-function overlap: " + << format("%.3f%%", static_cast(HotFuncOverlap.OverlapCount) / + HotFuncOverlap.UnionCount * 100) + << "\n"; + OS << " overlap hot functions: " << HotFuncOverlap.OverlapCount << "\n"; + OS << " hot functions unique in base profile: " + << HotFuncOverlap.BaseCount - HotFuncOverlap.OverlapCount << "\n"; + OS << " hot functions unique in test profile: " + << HotFuncOverlap.TestCount - HotFuncOverlap.OverlapCount << "\n"; + + assert(HotBlockOverlap.UnionCount > 0 && + "There should be at least one hot block in two input profiles"); + OS << " Hot-block overlap: " + << format("%.3f%%", static_cast(HotBlockOverlap.OverlapCount) / + HotBlockOverlap.UnionCount * 100) + << "\n"; + OS << " overlap hot blocks: " << HotBlockOverlap.OverlapCount << "\n"; + OS << " hot blocks unique in base profile: " + << HotBlockOverlap.BaseCount - HotBlockOverlap.OverlapCount << "\n"; + OS << " hot blocks unique in test profile: " + << HotBlockOverlap.TestCount - HotBlockOverlap.OverlapCount << "\n"; +} + +std::error_code SampleOverlapAggregator::loadProfiles() { + using namespace sampleprof; + + LLVMContext Context; + auto BaseReaderOrErr = SampleProfileReader::create(BaseFilename, Context, + FSDiscriminatorPassOption); + if (std::error_code EC = BaseReaderOrErr.getError()) + exitWithErrorCode(EC, BaseFilename); + + auto TestReaderOrErr = SampleProfileReader::create(TestFilename, Context, + FSDiscriminatorPassOption); + if (std::error_code EC = TestReaderOrErr.getError()) + exitWithErrorCode(EC, TestFilename); + + BaseReader = std::move(BaseReaderOrErr.get()); + TestReader = std::move(TestReaderOrErr.get()); + + if (std::error_code EC = BaseReader->read()) + exitWithErrorCode(EC, BaseFilename); + if (std::error_code EC = TestReader->read()) + exitWithErrorCode(EC, TestFilename); + if (BaseReader->profileIsProbeBased() != TestReader->profileIsProbeBased()) + exitWithError( + "cannot compare probe-based profile with non-probe-based profile"); + if (BaseReader->profileIsCS() != TestReader->profileIsCS()) + exitWithError("cannot compare CS profile with non-CS profile"); + + // Load BaseHotThreshold and TestHotThreshold as 99-percentile threshold in + // profile summary. + ProfileSummary &BasePS = BaseReader->getSummary(); + ProfileSummary &TestPS = TestReader->getSummary(); + BaseHotThreshold = + ProfileSummaryBuilder::getHotCountThreshold(BasePS.getDetailedSummary()); + TestHotThreshold = + ProfileSummaryBuilder::getHotCountThreshold(TestPS.getDetailedSummary()); + + return std::error_code(); +} + +void overlapSampleProfile(const std::string &BaseFilename, + const std::string &TestFilename, + const OverlapFuncFilters &FuncFilter, + uint64_t SimilarityCutoff, raw_fd_ostream &OS) { + using namespace sampleprof; + + // We use 0.000005 to initialize OverlapAggr.Epsilon because the final metrics + // report 2--3 places after decimal point in percentage numbers. + SampleOverlapAggregator OverlapAggr( + BaseFilename, TestFilename, + static_cast(SimilarityCutoff) / 1000000, 0.000005, FuncFilter); + if (std::error_code EC = OverlapAggr.loadProfiles()) + exitWithErrorCode(EC); + + OverlapAggr.initializeSampleProfileOverlap(); + if (OverlapAggr.detectZeroSampleProfile(OS)) + return; + + OverlapAggr.computeSampleProfileOverlap(OS); + + OverlapAggr.dumpProgramSummary(OS); + OverlapAggr.dumpHotFuncAndBlockOverlap(OS); + OverlapAggr.dumpFuncSimilarity(OS); +} + +static int overlap_main(int argc, const char *argv[]) { + cl::opt BaseFilename(cl::Positional, cl::Required, + cl::desc("")); + cl::opt TestFilename(cl::Positional, cl::Required, + cl::desc("")); + cl::opt Output("output", cl::value_desc("output"), cl::init("-"), + cl::desc("Output file")); + cl::alias OutputA("o", cl::desc("Alias for --output"), cl::aliasopt(Output)); + cl::opt IsCS( + "cs", cl::init(false), + cl::desc("For context sensitive PGO counts. Does not work with CSSPGO.")); + cl::opt ValueCutoff( + "value-cutoff", cl::init(-1), + cl::desc( + "Function level overlap information for every function (with calling " + "context for csspgo) in test " + "profile with max count value greater then the parameter value")); + cl::opt FuncNameFilter( + "function", + cl::desc("Function level overlap information for matching functions. For " + "CSSPGO this takes a a function name with calling context")); + cl::opt SimilarityCutoff( + "similarity-cutoff", cl::init(0), + cl::desc("For sample profiles, list function names (with calling context " + "for csspgo) for overlapped functions " + "with similarities below the cutoff (percentage times 10000).")); + cl::opt ProfileKind( + cl::desc("Profile kind:"), cl::init(instr), + cl::values(clEnumVal(instr, "Instrumentation profile (default)"), + clEnumVal(sample, "Sample profile"))); + cl::ParseCommandLineOptions(argc, argv, "LLVM profile data overlap tool\n"); + + std::error_code EC; + raw_fd_ostream OS(Output.data(), EC, sys::fs::OF_TextWithCRLF); + if (EC) + exitWithErrorCode(EC, Output); + + if (ProfileKind == instr) + overlapInstrProfile(BaseFilename, TestFilename, + OverlapFuncFilters{ValueCutoff, FuncNameFilter}, OS, + IsCS); + else + overlapSampleProfile(BaseFilename, TestFilename, + OverlapFuncFilters{ValueCutoff, FuncNameFilter}, + SimilarityCutoff, OS); + + return 0; +} + +namespace { +struct ValueSitesStats { + ValueSitesStats() + : TotalNumValueSites(0), TotalNumValueSitesWithValueProfile(0), + TotalNumValues(0) {} + uint64_t TotalNumValueSites; + uint64_t TotalNumValueSitesWithValueProfile; + uint64_t TotalNumValues; + std::vector ValueSitesHistogram; +}; +} // namespace + +static void traverseAllValueSites(const InstrProfRecord &Func, uint32_t VK, + ValueSitesStats &Stats, raw_fd_ostream &OS, + InstrProfSymtab *Symtab) { + uint32_t NS = Func.getNumValueSites(VK); + Stats.TotalNumValueSites += NS; + for (size_t I = 0; I < NS; ++I) { + uint32_t NV = Func.getNumValueDataForSite(VK, I); + std::unique_ptr VD = Func.getValueForSite(VK, I); + Stats.TotalNumValues += NV; + if (NV) { + Stats.TotalNumValueSitesWithValueProfile++; + if (NV > Stats.ValueSitesHistogram.size()) + Stats.ValueSitesHistogram.resize(NV, 0); + Stats.ValueSitesHistogram[NV - 1]++; + } + + uint64_t SiteSum = 0; + for (uint32_t V = 0; V < NV; V++) + SiteSum += VD[V].Count; + if (SiteSum == 0) + SiteSum = 1; + + for (uint32_t V = 0; V < NV; V++) { + OS << "\t[ " << format("%2u", I) << ", "; + if (Symtab == nullptr) + OS << format("%4" PRIu64, VD[V].Value); + else + OS << Symtab->getFuncName(VD[V].Value); + OS << ", " << format("%10" PRId64, VD[V].Count) << " ] (" + << format("%.2f%%", (VD[V].Count * 100.0 / SiteSum)) << ")\n"; + } + } +} + +static void showValueSitesStats(raw_fd_ostream &OS, uint32_t VK, + ValueSitesStats &Stats) { + OS << " Total number of sites: " << Stats.TotalNumValueSites << "\n"; + OS << " Total number of sites with values: " + << Stats.TotalNumValueSitesWithValueProfile << "\n"; + OS << " Total number of profiled values: " << Stats.TotalNumValues << "\n"; + + OS << " Value sites histogram:\n\tNumTargets, SiteCount\n"; + for (unsigned I = 0; I < Stats.ValueSitesHistogram.size(); I++) { + if (Stats.ValueSitesHistogram[I] > 0) + OS << "\t" << I + 1 << ", " << Stats.ValueSitesHistogram[I] << "\n"; + } +} + +static int showInstrProfile(const std::string &Filename, bool ShowCounts, + uint32_t TopN, bool ShowIndirectCallTargets, + bool ShowMemOPSizes, bool ShowDetailedSummary, + std::vector DetailedSummaryCutoffs, + bool ShowAllFunctions, bool ShowCS, + uint64_t ValueCutoff, bool OnlyListBelow, + const std::string &ShowFunction, bool TextFormat, + bool ShowBinaryIds, bool ShowCovered, + bool ShowProfileVersion, ShowFormat SFormat, + raw_fd_ostream &OS) { + if (SFormat == ShowFormat::Json) + exitWithError("JSON output is not supported for instr profiles"); + if (SFormat == ShowFormat::Yaml) + exitWithError("YAML output is not supported for instr profiles"); + auto ReaderOrErr = InstrProfReader::create(Filename); + std::vector Cutoffs = std::move(DetailedSummaryCutoffs); + if (ShowDetailedSummary && Cutoffs.empty()) { + Cutoffs = ProfileSummaryBuilder::DefaultCutoffs; + } + InstrProfSummaryBuilder Builder(std::move(Cutoffs)); + if (Error E = ReaderOrErr.takeError()) + exitWithError(std::move(E), Filename); + + auto Reader = std::move(ReaderOrErr.get()); + bool IsIRInstr = Reader->isIRLevelProfile(); + size_t ShownFunctions = 0; + size_t BelowCutoffFunctions = 0; + int NumVPKind = IPVK_Last - IPVK_First + 1; + std::vector VPStats(NumVPKind); + + auto MinCmp = [](const std::pair &v1, + const std::pair &v2) { + return v1.second > v2.second; + }; + + std::priority_queue, + std::vector>, + decltype(MinCmp)> + HottestFuncs(MinCmp); + + if (!TextFormat && OnlyListBelow) { + OS << "The list of functions with the maximum counter less than " + << ValueCutoff << ":\n"; + } + + // Add marker so that IR-level instrumentation round-trips properly. + if (TextFormat && IsIRInstr) + OS << ":ir\n"; + + for (const auto &Func : *Reader) { + if (Reader->isIRLevelProfile()) { + bool FuncIsCS = NamedInstrProfRecord::hasCSFlagInHash(Func.Hash); + if (FuncIsCS != ShowCS) + continue; + } + bool Show = ShowAllFunctions || + (!ShowFunction.empty() && Func.Name.contains(ShowFunction)); + + bool doTextFormatDump = (Show && TextFormat); + + if (doTextFormatDump) { + InstrProfSymtab &Symtab = Reader->getSymtab(); + InstrProfWriter::writeRecordInText(Func.Name, Func.Hash, Func, Symtab, + OS); + continue; + } + + assert(Func.Counts.size() > 0 && "function missing entry counter"); + Builder.addRecord(Func); + + if (ShowCovered) { + if (llvm::any_of(Func.Counts, [](uint64_t C) { return C; })) + OS << Func.Name << "\n"; + continue; + } + + uint64_t FuncMax = 0; + uint64_t FuncSum = 0; + + auto PseudoKind = Func.getCountPseudoKind(); + if (PseudoKind != InstrProfRecord::NotPseudo) { + if (Show) { + if (!ShownFunctions) + OS << "Counters:\n"; + ++ShownFunctions; + OS << " " << Func.Name << ":\n" + << " Hash: " << format("0x%016" PRIx64, Func.Hash) << "\n" + << " Counters: " << Func.Counts.size(); + if (PseudoKind == InstrProfRecord::PseudoHot) + OS << " \n"; + else if (PseudoKind == InstrProfRecord::PseudoWarm) + OS << " \n"; + else + llvm_unreachable("Unknown PseudoKind"); + } + continue; + } + + for (size_t I = 0, E = Func.Counts.size(); I < E; ++I) { + FuncMax = std::max(FuncMax, Func.Counts[I]); + FuncSum += Func.Counts[I]; + } + + if (FuncMax < ValueCutoff) { + ++BelowCutoffFunctions; + if (OnlyListBelow) { + OS << " " << Func.Name << ": (Max = " << FuncMax + << " Sum = " << FuncSum << ")\n"; + } + continue; + } else if (OnlyListBelow) + continue; + + if (TopN) { + if (HottestFuncs.size() == TopN) { + if (HottestFuncs.top().second < FuncMax) { + HottestFuncs.pop(); + HottestFuncs.emplace(std::make_pair(std::string(Func.Name), FuncMax)); + } + } else + HottestFuncs.emplace(std::make_pair(std::string(Func.Name), FuncMax)); + } + + if (Show) { + if (!ShownFunctions) + OS << "Counters:\n"; + + ++ShownFunctions; + + OS << " " << Func.Name << ":\n" + << " Hash: " << format("0x%016" PRIx64, Func.Hash) << "\n" + << " Counters: " << Func.Counts.size() << "\n"; + if (!IsIRInstr) + OS << " Function count: " << Func.Counts[0] << "\n"; + + if (ShowIndirectCallTargets) + OS << " Indirect Call Site Count: " + << Func.getNumValueSites(IPVK_IndirectCallTarget) << "\n"; + + uint32_t NumMemOPCalls = Func.getNumValueSites(IPVK_MemOPSize); + if (ShowMemOPSizes && NumMemOPCalls > 0) + OS << " Number of Memory Intrinsics Calls: " << NumMemOPCalls + << "\n"; + + if (ShowCounts) { + OS << " Block counts: ["; + size_t Start = (IsIRInstr ? 0 : 1); + for (size_t I = Start, E = Func.Counts.size(); I < E; ++I) { + OS << (I == Start ? "" : ", ") << Func.Counts[I]; + } + OS << "]\n"; + } + + if (ShowIndirectCallTargets) { + OS << " Indirect Target Results:\n"; + traverseAllValueSites(Func, IPVK_IndirectCallTarget, + VPStats[IPVK_IndirectCallTarget], OS, + &(Reader->getSymtab())); + } + + if (ShowMemOPSizes && NumMemOPCalls > 0) { + OS << " Memory Intrinsic Size Results:\n"; + traverseAllValueSites(Func, IPVK_MemOPSize, VPStats[IPVK_MemOPSize], OS, + nullptr); + } + } + } + if (Reader->hasError()) + exitWithError(Reader->getError(), Filename); + + if (TextFormat || ShowCovered) + return 0; + std::unique_ptr PS(Builder.getSummary()); + bool IsIR = Reader->isIRLevelProfile(); + OS << "Instrumentation level: " << (IsIR ? "IR" : "Front-end"); + if (IsIR) + OS << " entry_first = " << Reader->instrEntryBBEnabled(); + OS << "\n"; + if (ShowAllFunctions || !ShowFunction.empty()) + OS << "Functions shown: " << ShownFunctions << "\n"; + OS << "Total functions: " << PS->getNumFunctions() << "\n"; + if (ValueCutoff > 0) { + OS << "Number of functions with maximum count (< " << ValueCutoff + << "): " << BelowCutoffFunctions << "\n"; + OS << "Number of functions with maximum count (>= " << ValueCutoff + << "): " << PS->getNumFunctions() - BelowCutoffFunctions << "\n"; + } + OS << "Maximum function count: " << PS->getMaxFunctionCount() << "\n"; + OS << "Maximum internal block count: " << PS->getMaxInternalCount() << "\n"; + + if (TopN) { + std::vector> SortedHottestFuncs; + while (!HottestFuncs.empty()) { + SortedHottestFuncs.emplace_back(HottestFuncs.top()); + HottestFuncs.pop(); + } + OS << "Top " << TopN + << " functions with the largest internal block counts: \n"; + for (auto &hotfunc : llvm::reverse(SortedHottestFuncs)) + OS << " " << hotfunc.first << ", max count = " << hotfunc.second << "\n"; + } + + if (ShownFunctions && ShowIndirectCallTargets) { + OS << "Statistics for indirect call sites profile:\n"; + showValueSitesStats(OS, IPVK_IndirectCallTarget, + VPStats[IPVK_IndirectCallTarget]); + } + + if (ShownFunctions && ShowMemOPSizes) { + OS << "Statistics for memory intrinsic calls sizes profile:\n"; + showValueSitesStats(OS, IPVK_MemOPSize, VPStats[IPVK_MemOPSize]); + } + + if (ShowDetailedSummary) { + OS << "Total number of blocks: " << PS->getNumCounts() << "\n"; + OS << "Total count: " << PS->getTotalCount() << "\n"; + PS->printDetailedSummary(OS); + } + + if (ShowBinaryIds) + if (Error E = Reader->printBinaryIds(OS)) + exitWithError(std::move(E), Filename); + + if (ShowProfileVersion) + OS << "Profile version: " << Reader->getVersion() << "\n"; + return 0; +} + +static void showSectionInfo(sampleprof::SampleProfileReader *Reader, + raw_fd_ostream &OS) { + if (!Reader->dumpSectionInfo(OS)) { + WithColor::warning() << "-show-sec-info-only is only supported for " + << "sample profile in extbinary format and is " + << "ignored for other formats.\n"; + return; + } +} + +namespace { +struct HotFuncInfo { + std::string FuncName; + uint64_t TotalCount; + double TotalCountPercent; + uint64_t MaxCount; + uint64_t EntryCount; + + HotFuncInfo() + : TotalCount(0), TotalCountPercent(0.0f), MaxCount(0), EntryCount(0) {} + + HotFuncInfo(StringRef FN, uint64_t TS, double TSP, uint64_t MS, uint64_t ES) + : FuncName(FN.begin(), FN.end()), TotalCount(TS), TotalCountPercent(TSP), + MaxCount(MS), EntryCount(ES) {} +}; +} // namespace + +// Print out detailed information about hot functions in PrintValues vector. +// Users specify titles and offset of every columns through ColumnTitle and +// ColumnOffset. The size of ColumnTitle and ColumnOffset need to be the same +// and at least 4. Besides, users can optionally give a HotFuncMetric string to +// print out or let it be an empty string. +static void dumpHotFunctionList(const std::vector &ColumnTitle, + const std::vector &ColumnOffset, + const std::vector &PrintValues, + uint64_t HotFuncCount, uint64_t TotalFuncCount, + uint64_t HotProfCount, uint64_t TotalProfCount, + const std::string &HotFuncMetric, + uint32_t TopNFunctions, raw_fd_ostream &OS) { + assert(ColumnOffset.size() == ColumnTitle.size() && + "ColumnOffset and ColumnTitle should have the same size"); + assert(ColumnTitle.size() >= 4 && + "ColumnTitle should have at least 4 elements"); + assert(TotalFuncCount > 0 && + "There should be at least one function in the profile"); + double TotalProfPercent = 0; + if (TotalProfCount > 0) + TotalProfPercent = static_cast(HotProfCount) / TotalProfCount * 100; + + formatted_raw_ostream FOS(OS); + FOS << HotFuncCount << " out of " << TotalFuncCount + << " functions with profile (" + << format("%.2f%%", + (static_cast(HotFuncCount) / TotalFuncCount * 100)) + << ") are considered hot functions"; + if (!HotFuncMetric.empty()) + FOS << " (" << HotFuncMetric << ")"; + FOS << ".\n"; + FOS << HotProfCount << " out of " << TotalProfCount << " profile counts (" + << format("%.2f%%", TotalProfPercent) << ") are from hot functions.\n"; + + for (size_t I = 0; I < ColumnTitle.size(); ++I) { + FOS.PadToColumn(ColumnOffset[I]); + FOS << ColumnTitle[I]; + } + FOS << "\n"; + + uint32_t Count = 0; + for (const auto &R : PrintValues) { + if (TopNFunctions && (Count++ == TopNFunctions)) + break; + FOS.PadToColumn(ColumnOffset[0]); + FOS << R.TotalCount << " (" << format("%.2f%%", R.TotalCountPercent) << ")"; + FOS.PadToColumn(ColumnOffset[1]); + FOS << R.MaxCount; + FOS.PadToColumn(ColumnOffset[2]); + FOS << R.EntryCount; + FOS.PadToColumn(ColumnOffset[3]); + FOS << R.FuncName << "\n"; + } +} + +static int showHotFunctionList(const sampleprof::SampleProfileMap &Profiles, + ProfileSummary &PS, uint32_t TopN, + raw_fd_ostream &OS) { + using namespace sampleprof; + + const uint32_t HotFuncCutoff = 990000; + auto &SummaryVector = PS.getDetailedSummary(); + uint64_t MinCountThreshold = 0; + for (const ProfileSummaryEntry &SummaryEntry : SummaryVector) { + if (SummaryEntry.Cutoff == HotFuncCutoff) { + MinCountThreshold = SummaryEntry.MinCount; + break; + } + } + + // Traverse all functions in the profile and keep only hot functions. + // The following loop also calculates the sum of total samples of all + // functions. + std::multimap, + std::greater> + HotFunc; + uint64_t ProfileTotalSample = 0; + uint64_t HotFuncSample = 0; + uint64_t HotFuncCount = 0; + + for (const auto &I : Profiles) { + FuncSampleStats FuncStats; + const FunctionSamples &FuncProf = I.second; + ProfileTotalSample += FuncProf.getTotalSamples(); + getFuncSampleStats(FuncProf, FuncStats, MinCountThreshold); + + if (isFunctionHot(FuncStats, MinCountThreshold)) { + HotFunc.emplace(FuncProf.getTotalSamples(), + std::make_pair(&(I.second), FuncStats.MaxSample)); + HotFuncSample += FuncProf.getTotalSamples(); + ++HotFuncCount; + } + } + + std::vector ColumnTitle{"Total sample (%)", "Max sample", + "Entry sample", "Function name"}; + std::vector ColumnOffset{0, 24, 42, 58}; + std::string Metric = + std::string("max sample >= ") + std::to_string(MinCountThreshold); + std::vector PrintValues; + for (const auto &FuncPair : HotFunc) { + const FunctionSamples &Func = *FuncPair.second.first; + double TotalSamplePercent = + (ProfileTotalSample > 0) + ? (Func.getTotalSamples() * 100.0) / ProfileTotalSample + : 0; + PrintValues.emplace_back( + HotFuncInfo(Func.getContext().toString(), Func.getTotalSamples(), + TotalSamplePercent, FuncPair.second.second, + Func.getHeadSamplesEstimate())); + } + dumpHotFunctionList(ColumnTitle, ColumnOffset, PrintValues, HotFuncCount, + Profiles.size(), HotFuncSample, ProfileTotalSample, + Metric, TopN, OS); + + return 0; +} + +static int showSampleProfile(const std::string &Filename, bool ShowCounts, + uint32_t TopN, bool ShowAllFunctions, + bool ShowDetailedSummary, + const std::string &ShowFunction, + bool ShowProfileSymbolList, + bool ShowSectionInfoOnly, bool ShowHotFuncList, + ShowFormat SFormat, raw_fd_ostream &OS) { + if (SFormat == ShowFormat::Yaml) + exitWithError("YAML output is not supported for sample profiles"); + using namespace sampleprof; + LLVMContext Context; + auto ReaderOrErr = + SampleProfileReader::create(Filename, Context, FSDiscriminatorPassOption); + if (std::error_code EC = ReaderOrErr.getError()) + exitWithErrorCode(EC, Filename); + + auto Reader = std::move(ReaderOrErr.get()); + if (ShowSectionInfoOnly) { + showSectionInfo(Reader.get(), OS); + return 0; + } + + if (std::error_code EC = Reader->read()) + exitWithErrorCode(EC, Filename); + + if (ShowAllFunctions || ShowFunction.empty()) { + if (SFormat == ShowFormat::Json) + Reader->dumpJson(OS); + else + Reader->dump(OS); + } else { + if (SFormat == ShowFormat::Json) + exitWithError( + "the JSON format is supported only when all functions are to " + "be printed"); + + // TODO: parse context string to support filtering by contexts. + Reader->dumpFunctionProfile(StringRef(ShowFunction), OS); + } + + if (ShowProfileSymbolList) { + std::unique_ptr ReaderList = + Reader->getProfileSymbolList(); + ReaderList->dump(OS); + } + + if (ShowDetailedSummary) { + auto &PS = Reader->getSummary(); + PS.printSummary(OS); + PS.printDetailedSummary(OS); + } + + if (ShowHotFuncList || TopN) + showHotFunctionList(Reader->getProfiles(), Reader->getSummary(), TopN, OS); + + return 0; +} + +static int showMemProfProfile(const std::string &Filename, + const std::string &ProfiledBinary, + ShowFormat SFormat, raw_fd_ostream &OS) { + if (SFormat == ShowFormat::Json) + exitWithError("JSON output is not supported for MemProf"); + auto ReaderOr = llvm::memprof::RawMemProfReader::create( + Filename, ProfiledBinary, /*KeepNames=*/true); + if (Error E = ReaderOr.takeError()) + // Since the error can be related to the profile or the binary we do not + // pass whence. Instead additional context is provided where necessary in + // the error message. + exitWithError(std::move(E), /*Whence*/ ""); + + std::unique_ptr Reader( + ReaderOr.get().release()); + + Reader->printYAML(OS); + return 0; +} + +static int showDebugInfoCorrelation(const std::string &Filename, + bool ShowDetailedSummary, + bool ShowProfileSymbolList, + ShowFormat SFormat, raw_fd_ostream &OS) { + if (SFormat == ShowFormat::Json) + exitWithError("JSON output is not supported for debug info correlation"); + std::unique_ptr Correlator; + if (auto Err = InstrProfCorrelator::get(Filename).moveInto(Correlator)) + exitWithError(std::move(Err), Filename); + if (SFormat == ShowFormat::Yaml) { + if (auto Err = Correlator->dumpYaml(OS)) + exitWithError(std::move(Err), Filename); + return 0; + } + + if (auto Err = Correlator->correlateProfileData()) + exitWithError(std::move(Err), Filename); + + InstrProfSymtab Symtab; + if (auto Err = Symtab.create( + StringRef(Correlator->getNamesPointer(), Correlator->getNamesSize()))) + exitWithError(std::move(Err), Filename); + + if (ShowProfileSymbolList) + Symtab.dumpNames(OS); + // TODO: Read "Profile Data Type" from debug info to compute and show how many + // counters the section holds. + if (ShowDetailedSummary) + OS << "Counters section size: 0x" + << Twine::utohexstr(Correlator->getCountersSectionSize()) << " bytes\n"; + OS << "Found " << Correlator->getDataSize() << " functions\n"; + + return 0; +} + +static int show_main(int argc, const char *argv[]) { + cl::opt Filename(cl::Positional, cl::desc("")); + + cl::opt ShowCounts("counts", cl::init(false), + cl::desc("Show counter values for shown functions")); + cl::opt SFormat( + "show-format", cl::init(ShowFormat::Text), + cl::desc("Emit output in the selected format if supported"), + cl::values(clEnumValN(ShowFormat::Text, "text", + "emit normal text output (default)"), + clEnumValN(ShowFormat::Json, "json", "emit JSON"), + clEnumValN(ShowFormat::Yaml, "yaml", "emit YAML"))); + // TODO: Consider replacing this with `--show-format=text-encoding`. + cl::opt TextFormat( + "text", cl::init(false), + cl::desc("Show instr profile data in text dump format")); + cl::opt JsonFormat( + "json", cl::desc("Show sample profile data in the JSON format " + "(deprecated, please use --show-format=json)")); + cl::opt ShowIndirectCallTargets( + "ic-targets", cl::init(false), + cl::desc("Show indirect call site target values for shown functions")); + cl::opt ShowMemOPSizes( + "memop-sizes", cl::init(false), + cl::desc("Show the profiled sizes of the memory intrinsic calls " + "for shown functions")); + cl::opt ShowDetailedSummary("detailed-summary", cl::init(false), + cl::desc("Show detailed profile summary")); + cl::list DetailedSummaryCutoffs( + cl::CommaSeparated, "detailed-summary-cutoffs", + cl::desc( + "Cutoff percentages (times 10000) for generating detailed summary"), + cl::value_desc("800000,901000,999999")); + cl::opt ShowHotFuncList( + "hot-func-list", cl::init(false), + cl::desc("Show profile summary of a list of hot functions")); + cl::opt ShowAllFunctions("all-functions", cl::init(false), + cl::desc("Details for every function")); + cl::opt ShowCS("showcs", cl::init(false), + cl::desc("Show context sensitive counts")); + cl::opt ShowFunction("function", + cl::desc("Details for matching functions")); + + cl::opt OutputFilename("output", cl::value_desc("output"), + cl::init("-"), cl::desc("Output file")); + cl::alias OutputFilenameA("o", cl::desc("Alias for --output"), + cl::aliasopt(OutputFilename)); + cl::opt ProfileKind( + cl::desc("Profile kind:"), cl::init(instr), + cl::values(clEnumVal(instr, "Instrumentation profile (default)"), + clEnumVal(sample, "Sample profile"), + clEnumVal(memory, "MemProf memory access profile"))); + cl::opt TopNFunctions( + "topn", cl::init(0), + cl::desc("Show the list of functions with the largest internal counts")); + cl::opt ValueCutoff( + "value-cutoff", cl::init(0), + cl::desc("Set the count value cutoff. Functions with the maximum count " + "less than this value will not be printed out. (Default is 0)")); + cl::opt OnlyListBelow( + "list-below-cutoff", cl::init(false), + cl::desc("Only output names of functions whose max count values are " + "below the cutoff value")); + cl::opt ShowProfileSymbolList( + "show-prof-sym-list", cl::init(false), + cl::desc("Show profile symbol list if it exists in the profile. ")); + cl::opt ShowSectionInfoOnly( + "show-sec-info-only", cl::init(false), + cl::desc("Show the information of each section in the sample profile. " + "The flag is only usable when the sample profile is in " + "extbinary format")); + cl::opt ShowBinaryIds("binary-ids", cl::init(false), + cl::desc("Show binary ids in the profile. ")); + cl::opt DebugInfoFilename( + "debug-info", cl::init(""), + cl::desc("Read and extract profile metadata from debug info and show " + "the functions it found.")); + cl::opt ShowCovered( + "covered", cl::init(false), + cl::desc("Show only the functions that have been executed.")); + cl::opt ProfiledBinary( + "profiled-binary", cl::init(""), + cl::desc("Path to binary from which the profile was collected.")); + cl::opt ShowProfileVersion("profile-version", cl::init(false), + cl::desc("Show profile version. ")); + cl::ParseCommandLineOptions(argc, argv, "LLVM profile data summary\n"); + + if (Filename.empty() && DebugInfoFilename.empty()) + exitWithError( + "the positional argument '' is required unless '--" + + DebugInfoFilename.ArgStr + "' is provided"); + + if (Filename == OutputFilename) { + errs() << sys::path::filename(argv[0]) + << ": Input file name cannot be the same as the output file name!\n"; + return 1; + } + if (JsonFormat) + SFormat = ShowFormat::Json; + + std::error_code EC; + raw_fd_ostream OS(OutputFilename.data(), EC, sys::fs::OF_TextWithCRLF); + if (EC) + exitWithErrorCode(EC, OutputFilename); + + if (ShowAllFunctions && !ShowFunction.empty()) + WithColor::warning() << "-function argument ignored: showing all functions\n"; + + if (!DebugInfoFilename.empty()) + return showDebugInfoCorrelation(DebugInfoFilename, ShowDetailedSummary, + ShowProfileSymbolList, SFormat, OS); + + if (ProfileKind == instr) + return showInstrProfile( + Filename, ShowCounts, TopNFunctions, ShowIndirectCallTargets, + ShowMemOPSizes, ShowDetailedSummary, DetailedSummaryCutoffs, + ShowAllFunctions, ShowCS, ValueCutoff, OnlyListBelow, ShowFunction, + TextFormat, ShowBinaryIds, ShowCovered, ShowProfileVersion, SFormat, + OS); + if (ProfileKind == sample) + return showSampleProfile(Filename, ShowCounts, TopNFunctions, + ShowAllFunctions, ShowDetailedSummary, + ShowFunction, ShowProfileSymbolList, + ShowSectionInfoOnly, ShowHotFuncList, SFormat, OS); + return showMemProfProfile(Filename, ProfiledBinary, SFormat, OS); +} + +int main(int argc, const char *argv[]) { + InitLLVM X(argc, argv); + + StringRef ProgName(sys::path::filename(argv[0])); + if (argc > 1) { + int (*func)(int, const char *[]) = nullptr; + + if (strcmp(argv[1], "merge") == 0) + func = merge_main; + else if (strcmp(argv[1], "show") == 0) + func = show_main; + else if (strcmp(argv[1], "overlap") == 0) + func = overlap_main; + + if (func) { + std::string Invocation(ProgName.str() + " " + argv[1]); + argv[1] = Invocation.c_str(); + return func(argc - 1, argv + 1); + } + + if (strcmp(argv[1], "-h") == 0 || strcmp(argv[1], "-help") == 0 || + strcmp(argv[1], "--help") == 0) { + + errs() << "OVERVIEW: LLVM profile data tools\n\n" + << "USAGE: " << ProgName << " [args...]\n" + << "USAGE: " << ProgName << " -help\n\n" + << "See each individual command --help for more details.\n" + << "Available commands: merge, show, overlap\n"; + return 0; + } + } + + if (argc < 2) + errs() << ProgName << ": No command specified!\n"; + else + errs() << ProgName << ": Unknown command!\n"; + + errs() << "USAGE: " << ProgName << " [args...]\n"; + return 1; +}