Cuda Performance Metrics/Profiling

tc/aten/aten_compiler-inl.h

@@ -111,6 +111,26 @@ Duration ATenCompilationUnit<ExecutorType>::run(
       handle, inputDLTensorsPair.first, outputDLTensorsPair.first, profile);
 }
 
+template <typename ExecutorType>
+typename ExecutorType::ProfilingInfoType
+ATenCompilationUnit<ExecutorType>::profile(
+    const std::string& name,
+    const std::vector<at::Tensor>& inputs,
+    std::vector<at::Tensor>& outputs,
+    size_t handle) {
+  at::Backend backend = inputs[0].type().backend();
+  auto inputDLTensorsPair = toConstDlpackTensors(inputs);
+  ScopeGuard g1([&]() { deleteDlmTensors(inputDLTensorsPair.second); });
+  auto outTensorInfo =
+      executionEngine_->inferOutputTensorInfo(name, inputDLTensorsPair.first);
+  prepareOutputs(
+      executionEngine_->treeForFunction(name), outTensorInfo, backend, outputs);
+  auto outputDLTensorsPair = toDlpackTensors(outputs);
+  ScopeGuard g2([&]() { deleteDlmTensors(outputDLTensorsPair.second); });
+  return executionEngine_->profile(
+      handle, inputDLTensorsPair.first, outputDLTensorsPair.first);
+}
+
 template <typename ExecutorType>
 void ATenCompilationUnit<ExecutorType>::uncheckedRun(
     const std::vector<at::Tensor>& inputs,

tc/aten/aten_compiler.h

@@ -62,6 +62,12 @@ class ATenCompilationUnit {
       size_t handle,
       bool profile = false);
 
+  typename ExecutorType::ProfilingInfoType profile(
+      const std::string& name,
+      const std::vector<at::Tensor>& inputs,
+      std::vector<at::Tensor>& outputs,
+      size_t handle);
+
   /// This is the "low-latency" mode in which we just propagate ATen tensors
   /// Sizes are not checked and it is the user's responsibility to ensure that
   /// they match. If the user doesn't then segfault will likely occur.

tc/autotuner/genetic_autotuner.cc

@@ -57,8 +57,10 @@ void GeneticAutotuner::storeCaches(const std::string& filename) {
   } else {
     std::cout << "Dumping cache to " << filename << ".cuda/options"
               << std::endl;
-    tc::OptionsCache::getCache()->keepOnlyBestCandidates(
-        tc::FLAGS_tuner_save_best_candidates_count);
+    if (not FLAGS_tuner_gen_profiled_run) {
+      tc::OptionsCache::getCache()->keepOnlyBestCandidates(
+          tc::FLAGS_tuner_save_best_candidates_count);
+    }
     tc::OptionsCache::dumpCacheToProtobuf(tc::makeOptionsFilename(filename));
 
     tc::OptionsCache::getCache()->keepOnlyBestCandidates(1);

tc/autotuner/genetic_tuning_harness.cc

@@ -408,7 +408,11 @@ void GeneticTunerHarness::doGpuWork(
       } else {
         runtimes.reserve(kReducedBenchmarkIterations);
         for (size_t i = 0; i < kReducedBenchmarkIterations; ++i) {
-          runtimes.push_back(engine.run(handle, inputs, outputs, true));
+          if (FLAGS_tuner_gen_profiled_run) {
+            runtimes.push_back(engine.profile(handle, inputs, outputs).runtime);
+          } else {
+            runtimes.push_back(engine.run(handle, inputs, outputs, true));
+          }
         }
         engine.clear(handle);
       }

tc/core/CMakeLists.txt

@@ -157,15 +157,17 @@ if (WITH_CUDA)
     cuda/cuda_compilation_cache.cc
     cuda/cuda_rtc.cc
     cuda/cuda_tc_executor.cc
+    cuda/cuda_profile.cc
   )
-  target_include_directories(tc_cuda PUBLIC ${LLVM_INCLUDE_DIRS})
+  target_include_directories(tc_cuda PUBLIC ${LLVM_INCLUDE_DIRS} ${CUDA_TOOLKIT_ROOT_DIR}/extras/CUPTI/include)
   target_link_libraries(
     tc_cuda
 
     ${CUDA_CUDA_LIBRARIES}
     ${CUDA_curand_LIBRARY}
     ${CUDA_LIBRARIES}
     ${CUDA_NVRTC_LIBRARIES}
+    ${CUDA_cupti_LIBRARY}
     ${ISL_LIBRARIES}
 
     tc_lang

tc/core/cpu/cpu_tc_executor.h

@@ -33,10 +33,12 @@ struct CpuRTCFunction {
   void clear() {}
 };
 
+struct CpuProfilingInfo {};
+
 class CpuTcExecutor : public ::tc::TcExecutor {
  public:
   using MappingOptionsType = CpuMappingOptions;
-
+  using ProfilingInfoType = CpuProfilingInfo;
   CpuTcExecutor(
       std::string id,
       const std::vector<const DLTensor*>& inputsInfo,

tc/core/cuda/cuda.h

@@ -57,6 +57,19 @@
     }                                                           \
   } while (0)
 
+#define TC_CUPTI_CHECK(condition)                                       \
+  do {                                                                  \
+    CUptiResult result = condition;                                     \
+    if (result != CUPTI_SUCCESS) {                                      \
+      const char* msg;                                                  \
+      cuptiGetResultString(result, &msg);                               \
+      std::stringstream ss;                                             \
+      ss << "Error at: " << __FILE__ << ":" << __LINE__ << ": " << msg; \
+      LOG(WARNING) << ss.str();                                         \
+      throw std::runtime_error(ss.str().c_str());                       \
+    }                                                                   \
+  } while (0)
+
 #define TC_CUDA_RUNTIMEAPI_ENFORCE(condition)                   \
   do {                                                          \
     cudaError_t result = condition;                             \

tc/core/cuda/cuda_compilation_cache.cc

@@ -302,6 +302,29 @@ OptionsCachedEntry::OptionsCachedEntry(
   values.emplace_back(options, runtime);
 }
 
+OptionsCachedEntry::OptionsCachedEntry(
+    const std::string& id,
+    const std::vector<const DLTensor*>& inputs,
+    const std::vector<const DLTensor*>& outputs,
+    const std::string& deviceStr,
+    const CudaMappingOptions& options,
+    const CudaProfilingInfo& pInfo)
+    : key(id, inputs, outputs, deviceStr, git_version) {
+  values.emplace_back(options, pInfo);
+}
+
+OptionsCachedEntry::OptionsCachedEntry(
+    const std::string& id,
+    const std::vector<const DLTensor*>& inputs,
+    const std::vector<const DLTensor*>& outputs,
+    const std::string& deviceStr,
+    const CudaMappingOptions& options,
+    Duration runtime,
+    const CudaProfilingInfo& pInfo)
+    : key(id, inputs, outputs, deviceStr, git_version) {
+  values.emplace_back(options, runtime, pInfo);
+}
+
 OptionsCachedEntry::Key::Key(
     const std::string& id,
     const std::vector<const DLTensor*>& inputs_,
@@ -333,8 +356,37 @@ OptionsCachedEntry::Values::Values(
 
 OptionsCachedEntry::Values::Values(
     const CudaMappingOptions& options,
-    std::vector<Duration>&& runtimes)
-    : mappingOptions(options), recordedRuntimes(std::move(runtimes)) {}
+    const CudaProfilingInfo& pInfo)
+    : mappingOptions(options), profiles{pInfo} {}
+
+OptionsCachedEntry::Values::Values(
+    const CudaMappingOptions& options,
+    Duration runtime,
+    const CudaProfilingInfo& pInfo)
+    : mappingOptions(options), recordedRuntimes{runtime}, profiles{pInfo} {}
+
+OptionsCachedEntry::Values::Values(
+    const CudaMappingOptions& options,
+    std::vector<Duration>&& runtimes,
+    std::vector<CudaProfilingInfo>&& pInfos)
+    : mappingOptions(options),
+      recordedRuntimes(std::move(runtimes)),
+      profiles(std::move(pInfos)) {}
+
+namespace {
+tc::CudaProfilingInfo fromProto(const tc::CudaProfilingProto& buf) {
+  tc::CudaProfilingInfo pInfo;
+  pInfo.runtime = std::chrono::microseconds(buf.runtime());
+  pInfo.ipc = buf.ipc();
+  pInfo.globalLoadEfficiency = buf.globalloadefficiency();
+  pInfo.globalStoreEfficiency = buf.globalstoreefficiency();
+  pInfo.sharedMemoryEfficiency = buf.sharedmemoryefficiency();
+  pInfo.localMemoryOverhead = buf.localmemoryoverhead();
+  pInfo.achievedOccupancy = buf.achievedoccupancy();
+  pInfo.warpExecutionEfficiency = buf.warpexecutionefficiency();
+  return pInfo;
+}
+} // namespace
 
 OptionsCachedEntry::OptionsCachedEntry(const OptionsCacheEntryProto& buf)
     : key(buf.id(),
@@ -348,7 +400,7 @@ OptionsCachedEntry::OptionsCachedEntry(const OptionsCacheEntryProto& buf)
   }
 
   for (const auto& value : buf.values()) {
-    if (value.recorded_runtimes_size() == 0) {
+    if (value.recorded_runtimes_size() == 0 and value.profiles_size() == 0) {
       throw std::invalid_argument(
           "OptionsCachedEntry invalid protobuf: each entry value should have at least one recorded runtime.");
     }
@@ -359,11 +411,39 @@ OptionsCachedEntry::OptionsCachedEntry(const OptionsCacheEntryProto& buf)
         value.recorded_runtimes().end(),
         std::back_inserter(runtimes),
         [](int64_t us) { return std::chrono::microseconds(us); });
+    std::vector<CudaProfilingInfo> profiles;
+    profiles.reserve(value.profiles_size());
+    std::transform(
+        value.profiles().begin(),
+        value.profiles().end(),
+        std::back_inserter(profiles),
+        [](const CudaProfilingProto& buf) { return fromProto(buf); });
+
     values.emplace_back(
-        CudaMappingOptions(value.kernel_options()), std::move(runtimes));
+        CudaMappingOptions(value.kernel_options()),
+        std::move(runtimes),
+        std::move(profiles));
   }
 }
 
+namespace {
+tc::CudaProfilingProto toProto(const tc::CudaProfilingInfo& pInfo) {
+  tc::CudaProfilingProto buf;
+  buf.set_runtime(
+      std::chrono::duration_cast<std::chrono::microseconds>(pInfo.runtime)
+          .count());
+  buf.set_ipc(pInfo.ipc);
+  buf.set_globalloadefficiency(pInfo.globalLoadEfficiency);
+  buf.set_globalstoreefficiency(pInfo.globalStoreEfficiency);
+  buf.set_sharedmemoryefficiency(pInfo.sharedMemoryEfficiency);
+  buf.set_localmemoryoverhead(pInfo.localMemoryOverhead);
+  buf.set_achievedoccupancy(pInfo.achievedOccupancy);
+  buf.set_warpexecutionefficiency(pInfo.warpExecutionEfficiency);
+
+  return buf;
+}
+} // namespace
+
 OptionsCacheEntryProto OptionsCachedEntry::toProtobuf() const {
   OptionsCacheEntryProto buf;
   buf.set_id(key.id);
@@ -392,6 +472,9 @@ OptionsCacheEntryProto OptionsCachedEntry::toProtobuf() const {
           buf.add_recorded_runtimes(
               std::chrono::duration_cast<std::chrono::microseconds>(r).count());
         }
+        for (const auto& p : v.profiles) {
+          *buf.add_profiles() = toProto(p);
+        }
         return buf;
       });
   return buf;
@@ -460,6 +543,69 @@ void OptionsCache::recordRuntime(
   v->recordedRuntimes.push_back(runtime);
 }
 
+void OptionsCache::recordProfilingInfo(
+    const std::string& id,
+    const CudaMappingOptions& options,
+    const std::vector<const DLTensor*>& inputs,
+    const std::vector<const DLTensor*>& outputs,
+    const CudaProfilingInfo pInfo) {
+  std::lock_guard<std::mutex> lock(mtx_);
+  ++numberCacheAttemps;
+  auto gpuStr = CudaGPUInfo::GPUInfo().GetCudaDeviceStr();
+
+  auto kernel = searchKernel(entries_, id, inputs, outputs);
+  if (not kernel) {
+    entries_.emplace_back(
+        id, inputs, outputs, gpuStr, options, pInfo.runtime, pInfo);
+    return;
+  }
+  auto v = std::find_if(
+      kernel->values.begin(),
+      kernel->values.end(),
+      [&options](const CachedEntry::Values& v) {
+        return v.mappingOptions == options;
+      });
+  if (v == kernel->values.end()) {
+    kernel->values.emplace_back(options, pInfo);
+    return;
+  }
+
+  v->recordedRuntimes.push_back(pInfo.runtime);
+  v->profiles.push_back(pInfo);
+}
+
+std::vector<OptionsCacheRetrievalResult>
+OptionsCache::retrieveOptionsAndProfilingInfo(
+    const std::string& id,
+    const std::vector<const DLTensor*>& inputs,
+    const std::vector<const DLTensor*>& outputs) const {
+  std::lock_guard<std::mutex> lock(mtx_);
+  ++numberAttemptedRetrievals;
+  auto ret = searchKernel(entries_, id, inputs, outputs);
+  if (not ret) {
+    return {};
+  }
+  ++numberSuccessfulRetrievals;
+  std::vector<OptionsCacheRetrievalResult> res;
+  res.reserve(ret->values.size());
+  std::transform(
+      ret->values.begin(),
+      ret->values.end(),
+      std::back_inserter(res),
+      [](const CachedEntry::Values& v) -> OptionsCacheRetrievalResult {
+        return {v.mappingOptions, v.recordedRuntimes, v.profiles};
+      });
+  res.erase(
+      std::remove_if(
+          res.begin(),
+          res.end(),
+          [](const OptionsCacheRetrievalResult& rr) {
+            return rr.profilingInfo.empty();
+          }),
+      res.end());
+  return res;
+}
+
 std::vector<OptionsCacheRetrievalResult>
 OptionsCache::retrieveOptionsAndRuntimes(
     const std::string& id,

tc/core/cuda/cuda_compilation_cache.h

@@ -53,6 +53,22 @@ struct OptionsCachedEntry {
       const std::string& deviceStr,
       const CudaMappingOptions& options,
       Duration runtime);
+  OptionsCachedEntry(
+      const std::string& id,
+      const std::vector<const DLTensor*>& inputs,
+      const std::vector<const DLTensor*>& outputs,
+      const std::string& deviceStr,
+      const CudaMappingOptions& options,
+      const CudaProfilingInfo& pInfo);
+  OptionsCachedEntry(
+      const std::string& id,
+      const std::vector<const DLTensor*>& inputs,
+      const std::vector<const DLTensor*>& outputs,
+      const std::string& deviceStr,
+      const CudaMappingOptions& options,
+      Duration runtime,
+      const CudaProfilingInfo& pInfo);
+
   OptionsCachedEntry(const OptionsCacheEntryProto& buf);
   OptionsCacheEntryProto toProtobuf() const;
 
@@ -78,9 +94,19 @@ struct OptionsCachedEntry {
 
   struct Values {
     Values(const CudaMappingOptions& options, Duration runtime);
+    Values(const CudaMappingOptions& options, const CudaProfilingInfo& pInfo);
+    Values(
+        const CudaMappingOptions& options,
+        Duration runtime,
+        const CudaProfilingInfo& pInfo);
     Values(const CudaMappingOptions& options, std::vector<Duration>&& runtimes);
+    Values(
+        const CudaMappingOptions& options,
+        std::vector<Duration>&& runtimes,
+        std::vector<CudaProfilingInfo>&& pInfos);
     CudaMappingOptions mappingOptions;
     std::vector<Duration> recordedRuntimes;
+    std::vector<CudaProfilingInfo> profiles;
   };
   Key key;
   std::vector<Values> values;
@@ -89,6 +115,7 @@ struct OptionsCachedEntry {
 struct OptionsCacheRetrievalResult {
   CudaMappingOptions options;
   std::vector<Duration> recordedRuntimes;
+  std::vector<CudaProfilingInfo> profilingInfo;
 };
 
 class OptionsCache : public Cache<OptionsCache, OptionsCachedEntry> {
@@ -114,6 +141,18 @@ class OptionsCache : public Cache<OptionsCache, OptionsCachedEntry> {
       const std::vector<const DLTensor*>& outputs,
       Duration runtime);
 
+  void recordProfilingInfo(
+      const std::string& id,
+      const CudaMappingOptions& options,
+      const std::vector<const DLTensor*>& inputs,
+      const std::vector<const DLTensor*>& outputs,
+      CudaProfilingInfo pIfno);
+
+  std::vector<OptionsCacheRetrievalResult> retrieveOptionsAndProfilingInfo(
+      const std::string& id,
+      const std::vector<const DLTensor*>& inputs,
+      const std::vector<const DLTensor*>& outputs) const;
+
   std::vector<OptionsCacheRetrievalResult> retrieveOptionsAndRuntimes(
       const std::string& id,
       const std::vector<const DLTensor*>& inputs,