critical-core.cpp

/*
 * critPathAnalysis.cpp -- Critical path analysis runtime library, build for
 * hybrid OpenMp and MPI applications
 */

//===----------------------------------------------------------------------===//
//
// Based on the ompt-tsan.cpp of the LLVM Project
// version as of 06/24/2021
// parent 82e4e50 commit 08d8f1a958bd8be681e3e1f346be80818a83a556
// See https://llvm.org/LICENSE.txt for details.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "criticalPath.h"
#include "errorhandler.h"
#include "parse_flags.h"
#include <time.h>
#ifdef USE_MPI
#include <mpi.h>
#endif

#ifdef EE_CPT
#include <dlfcn.h>
#include <cstdlib>
#include <thread>
#include <atomic>
#include <chrono>
#include <fstream>
#include <condition_variable>

#include <iostream>
uint32_t endProgrammEnergy;

using MainFunc = int(*)(int, char**, char**);

static MainFunc real_main = nullptr;
static bool finalized = false;

std::atomic<bool> is_monitor_thread_running{true};
std::thread monitorThread;
double totalEnergy = 0;
std::mutex mtx;
std::condition_variable cv;

void monitor_thread() {
  uint32_t lastEnergy = likwidPowerStart(0, PKG);
  auto interval = static_cast<uint64_t>(likwidRAPLWrapEnergy(PKG) / likwidRAPLMaxPower(PKG) * 10e5 * 0.8);
  std::unique_lock<std::mutex> lock(mtx);
  while (is_monitor_thread_running) {
    cv.wait_for(lock, std::chrono::seconds(interval), [](){ return !is_monitor_thread_running.load(); });
    uint32_t currentEnergy = likwidPowerStart(0, PKG);
    totalEnergy += likwidPowerPrint(lastEnergy, currentEnergy, PKG);
    lastEnergy = currentEnergy;
  }
}

void stop_monitor_thread() {
  {
      std::lock_guard<std::mutex> lock(mtx);
      is_monitor_thread_running = false;
  }
  cv.notify_all();
  if (monitorThread.joinable()) {
    monitorThread.join();
  }
}
#endif

extern "C" void __cxa_pure_virtual() {
  assert(false && "Pure virtual function must not be called");
  abort();
}

#ifdef USE_ERRHANDLER
#include <execinfo.h>
#include <signal.h>
#include <sys/types.h>
#include <unistd.h>

#define CALLSTACK_SIZE 20
static void print_stack(void) {
  int nptrs;
  void *buf[CALLSTACK_SIZE + 1];
  nptrs = backtrace(buf, CALLSTACK_SIZE);
  backtrace_symbols_fd(buf, nptrs, STDOUT_FILENO);
}

static void set_signalhandlers(sighandler_t handler) {
  signal(SIGSEGV, handler);
  signal(SIGINT, handler);
  signal(SIGHUP, handler);
  signal(SIGABRT, handler);
  signal(SIGTERM, handler);
  signal(SIGUSR2, handler);
  signal(SIGQUIT, handler);
  signal(SIGALRM, handler);
}

void disable_signalhandlers() { set_signalhandlers(SIG_DFL); }

void mySignalHandler(int signum) {
  disable_signalhandlers();
  printf("pid %i caught signal nr %i\n", getpid(), signum);
  if (signum == SIGINT || signum == SIGKILL) {
    print_stack();
    _exit(signum + 128);
  }
  if (signum == SIGTERM || signum == SIGUSR2) {
    print_stack();
    fflush(stdout);
    sleep(1);
    _exit(signum + 128);
  }
  print_stack();

  printf("Waiting up to %i seconds to attach with a debugger.\n", 20);
  sleep(20);
  _exit(1);
}

void init_signalhandlers() {
  if (!analysis_flags->stacktrace)
    return;
  if (analysis_flags->verbose)
    fprintf(analysis_flags->output, "Setting signal handlers\n");
  set_signalhandlers(mySignalHandler);
}

#endif

double localTimeOffset{0};
double getTime() {
  struct timespec curr;
  clock_gettime(CLOCK_REALTIME, &curr);
  return curr.tv_sec + curr.tv_nsec * 1e-9 - localTimeOffset;
}

int myProcId = 0;
bool useMpi = false;

ompt_finalize_tool_t critical_ompt_finalize_tool{nullptr};

uint64_t my_next_id() {
  static uint64_t ID = 0;
  uint64_t ret = __sync_fetch_and_add(&ID, 1);
  return ret;
}

double totalProgrammTime = 0;
double startProgrammTime = getTime(), endProgrammTime;
double crit_path_useful_time = 0;

Vector<THREAD_CLOCK *> *thread_clocks = nullptr;
Vector<omptCounts *> *thread_counts = nullptr;
thread_local THREAD_CLOCK *thread_local_clock = nullptr;

void resetMpiClock(THREAD_CLOCK *thread_clock) {
  if (!analysis_flags->running) {
    thread_clock->stopped_mpi_clock = true;
    thread_clock->outsidempi_proc.Reset();
    thread_clock->outsidempi_critical.Reset();
    thread_clock->outsidempi_thread.Reset();
  } else if (!thread_clock->openmp_thread) {
    thread_clock->stopped_mpi_clock = true;
    thread_clock->stopped_clock = true;
    thread_clock->stopped_omp_clock = false;
    OmpClockReset(thread_clock);
  } else {
    thread_clock->assertMpiStopped(false);
    thread_clock->stopped_mpi_clock = true;
    thread_clock->outsidempi_proc.Reset();
    thread_clock->outsidempi_critical.Reset();
    thread_clock->outsidempi_thread.Reset();
    if (thread_clock->stopped_clock == false) {
      thread_clock->stopped_clock = true;
      thread_clock->useful_computation_proc.Reset();
      thread_clock->useful_computation_critical.Reset();
      thread_clock->useful_computation_thread.Reset();
    }
    if (thread_local_clock->stopped_omp_clock == true)
      thread_local_clock->Start(CLOCK_OMP_ONLY, __func__);
  }
}

void startMeasurement(double time) {
  auto initialStart = startProgrammTime;
  startProgrammTime = time;
  if (analysis_flags->verbose)
    fprintf(analysis_flags->output, "Initialize after %lf s\n",
            startProgrammTime - initialStart);
}

void stopMeasurement(double time) {
  endProgrammTime = time;
#ifdef EE_CPT
  endProgrammEnergy = likwidPowerStop(0, PKG);
  stop_monitor_thread();
#endif
}

template <>
double atomic_add<double>(std::atomic<double> &operand, double value_to_add) {
  double old = operand.load(std::memory_order_consume);
  double desired = old + value_to_add;
  while (!operand.compare_exchange_weak(old, desired, std::memory_order_release,
                                        std::memory_order_consume))
    desired = old + value_to_add;

  return desired;
}

#define NUM_SHARED_METRICS 7

void finishMeasurement() {
  static int finished = 0;
  if (finished)
    return;
  finished = 1;
  int number_of_procs = 1;
  int total_threads = 0;
  int num_threads = 0;
  if (thread_clocks)
    total_threads = num_threads = thread_clocks->Size();

  double avgComputation[NUM_SHARED_METRICS] = {0};
  double maxComputation[NUM_SHARED_METRICS] = {0};
  double uc_avg[NUM_SHARED_METRICS] = {0};
  double uc_max[NUM_SHARED_METRICS] = {0};

#ifdef EE_CPT
  double avgComputationEnergy[NUM_SHARED_METRICS] = {0};
  double maxComputationEnergy[NUM_SHARED_METRICS] = {0};
  double uc_avg_e[NUM_SHARED_METRICS] = {0};
  double uc_max_e[NUM_SHARED_METRICS] = {0};
#endif

  if (analysis_flags->running) {
    endProgrammTime = getTime();
    analysis_flags->running = false;
  }
  if (thread_local_clock->stopped_omp_clock == false)
    thread_local_clock->Stop(endProgrammTime, CLOCK_OMP_ONLY, __func__);

  double totalRuntimeReal = endProgrammTime - startProgrammTime;
#ifdef EE_CPT
  thread_local_clock->StopEnergy(endProgrammEnergy);
  if (!finalized) {
    finalized = true;
    stop_monitor_thread();
  }

//  Old version: Overflow for longer than 20 minutes at CLAIX23
//  double totalRuntimeRealEnergy = likwidPowerPrint(startProgrammEnergy, endProgrammEnergy, PKG);

  double totalRuntimeRealEnergy = totalEnergy;
//  std::cout << "Total Time : " << totalRuntimeReal << "s, Total Energy: " << totalRuntimeRealEnergy << "J\n";
#endif
  // get max and avg Computation accross all threads
  MPI_COUNTS proc_counts, total_counts;
  if (num_threads > 0) {
    for (int i = 0; i < num_threads; i++) {
      auto *tclock = (*thread_clocks)[i];
      proc_counts.add(*tclock);
      tclock->assertUCStopped(true);
      tclock->assertOmpStopped(true);
      double curr_uc = tclock->useful_computation_thread.getTime();
      double curr_oot = tclock->outsideomp_thread.getTime();
#ifdef EE_CPT
      double curr_uc_e = tclock->useful_computation_thread.getEnergy();
      double curr_oot_e = tclock->outsideomp_thread.getEnergy();
#ifdef EE_CPT_DEBUG
      std::cout << "MPI rank " << myProcId << " (Thread " << i << "):\n"
                << "UC Critical Time (Rank: " << myProcId << " Thread: " << i << "): " << tclock->useful_computation_critical.getTime() << "s\n"
                << "UC Proc Time (Rank: " << myProcId << " Thread: " << i << "): " << tclock->useful_computation_proc.getTime() << "s\n"
                << "UC Thread Time (Rank: " << myProcId << " Thread: " << i << "): " << tclock->useful_computation_thread.getTime() << "s\n"
                << "Outside OMP Critical Time (Rank: " << myProcId << " Thread: " << i << "): " << tclock->outsideomp_critical.getTime() << "s\n"
                << "Outside OMP Proc Time (Rank: " << myProcId << " Thread: " << i << "): " << tclock->outsideomp_proc.getTime() << "s\n"
                << "Outside OMP Thread Time (Rank: " << myProcId << " Thread: " << i << "): " << tclock->outsideomp_thread.getTime() << "s\n\n"
                << "UC Critical Energy (Rank: " << myProcId << " Thread: " << i << "): " << tclock->useful_computation_critical.getEnergy() << "J\n"
                << "UC Proc Energy (Rank: " << myProcId << " Thread: " << i << "): " << tclock->useful_computation_proc.getEnergy() << "J\n"
                << "UC Thread Energy (Rank: " << myProcId << " Thread: " << i << "): " << tclock->useful_computation_thread.getEnergy() << "J\n"
                << "Outside OMP Critical Energy (Rank: " << myProcId << " Thread: " << i << "): " << tclock->outsideomp_critical.getEnergy() << "J\n"
                << "Outside OMP Proc Energy (Rank: " << myProcId << " Thread: " << i << "): " << tclock->outsideomp_proc.getEnergy() << "J\n"
                << "Outside OMP Thread Energy (Rank: " << myProcId << " Thread: " << i << "): " << tclock->outsideomp_thread.getEnergy() << "J\n"
                << std::endl;
#endif
      std::ofstream reportFile;
      std::string filename = "report_rank" + std::to_string(myProcId) + ".csv";
      if (i == 0) {
        reportFile.open(filename, std::ios::out);
        reportFile << "rank,thr,uc_ct,uc_pt,uc_tt,"
                      "ooc_ct,ooc_pt,ooc_tt,total_runtime,"
                      "uc_ce,uc_pe,uc_te,ooc_ce,ooc_pe,ooc_te,"
                      "total_energy\n";
      } else {
        reportFile.open(filename, std::ios::app);
      }

      reportFile
            << myProcId << "," << i << ","
            << tclock->useful_computation_critical.getTime() << ","
            << tclock->useful_computation_proc.getTime() << ","
            << tclock->useful_computation_thread.getTime() << ","
            << tclock->outsideomp_critical.getTime() << ","
            << tclock->outsideomp_proc.getTime() << ","
            << tclock->outsideomp_thread.getTime() << ","
            << totalRuntimeReal << ","
            << tclock->useful_computation_critical.getEnergy() << ","
            << tclock->useful_computation_proc.getEnergy() << ","
            << tclock->useful_computation_thread.getEnergy() << ","
            << tclock->outsideomp_critical.getEnergy() << ","
            << tclock->outsideomp_proc.getEnergy() << ","
            << tclock->outsideomp_thread.getEnergy() << ","
            << totalRuntimeRealEnergy
            << std::endl;

      reportFile.close();
#endif
      if (curr_uc > uc_max[0]) {
        uc_max[0] = curr_uc;
#ifdef EE_CPT
        uc_max_e[0] = curr_uc_e;
#endif
      }
      if (curr_oot > uc_max[2]) {
        uc_max[2] = curr_oot;
#ifdef EE_CPT
        uc_max_e[2] = curr_oot_e;
#endif
      }
      uc_avg[0] += curr_uc;
      uc_avg[2] += curr_oot;
#ifdef EE_CPT
      uc_avg_e[0] += curr_uc_e;
      uc_avg_e[2] += curr_oot_e;
#endif
    }
    uc_avg[0] = uc_avg[0];

    uc_avg[2] = uc_avg[2];
    if (thread_counts)
      for (int i = 1; i < thread_counts->Size(); i++)
        (*thread_counts)[0]->add(*(*thread_counts)[i]);
  } else {
    num_threads = 1;
    uc_max[0] = uc_avg[0] =
        thread_local_clock->useful_computation_thread.getTime();
    uc_max[2] = uc_avg[2] = thread_local_clock->outsideomp_thread.getTime();
#ifdef EE_CPT
    uc_max_e[0] = uc_avg_e[0] =
        thread_local_clock->useful_computation_thread.getEnergy();
    uc_max_e[2] = uc_avg_e[2] = thread_local_clock->outsideomp_thread.getEnergy();
#endif
  }
  uc_max[1] = uc_avg[1] = thread_local_clock->outsidempi_proc.getTime();
  uc_max[3] = uc_avg[3] = thread_local_clock->useful_computation_proc.getTime();
  uc_max[4] = uc_avg[4] = thread_local_clock->outsideomp_proc.getTime();
  uc_max[5] = uc_avg[5] = uc_avg[3] - uc_avg[4];
  uc_avg[5] = uc_avg[5] * num_threads;
  uc_max[6] = uc_max[0];
  uc_avg[6] = uc_max[0] * num_threads;

#ifdef EE_CPT
  uc_max_e[1] = uc_avg_e[1] = thread_local_clock->outsidempi_proc.getEnergy();
  uc_max_e[3] = uc_avg_e[3] = thread_local_clock->useful_computation_proc.getEnergy();
  uc_max_e[4] = uc_avg_e[4] = thread_local_clock->outsideomp_proc.getEnergy();
  uc_max_e[5] = uc_avg_e[5] = uc_avg_e[3] - uc_avg_e[4];
  uc_avg_e[5] = uc_avg_e[5] * num_threads;
  uc_max_e[6] = uc_max_e[0];
  uc_avg_e[6] = uc_max_e[0] * num_threads;
#endif

#ifdef USE_MPI
  if (useMpi) {
    // aggregate all max and avg computations on the master thread
    PMPI_Reduce(&uc_max, &maxComputation, NUM_SHARED_METRICS, MPI_DOUBLE,
                MPI_MAX, 0, MPI_COMM_WORLD);
    PMPI_Reduce(&uc_avg, &avgComputation, NUM_SHARED_METRICS, MPI_DOUBLE,
                MPI_SUM, 0, MPI_COMM_WORLD);
#ifdef EE_CPT
    PMPI_Reduce(&uc_max_e, &maxComputationEnergy, NUM_SHARED_METRICS, MPI_DOUBLE,
                MPI_MAX, 0, MPI_COMM_WORLD);
    PMPI_Reduce(&uc_avg_e, &avgComputationEnergy, NUM_SHARED_METRICS, MPI_DOUBLE,
                MPI_SUM, 0, MPI_COMM_WORLD);
#endif
    PMPI_Reduce(&num_threads, &total_threads, 1, MPI_INT, MPI_SUM, 0,
                MPI_COMM_WORLD);
    PMPI_Reduce(&proc_counts, &total_counts, sizeof(proc_counts) / sizeof(int),
                MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
    double localRuntimeReal = totalRuntimeReal;
    PMPI_Reduce(&localRuntimeReal, &totalRuntimeReal, 1, MPI_DOUBLE, MPI_MAX, 0,
                MPI_COMM_WORLD);
#ifdef EE_CPT
    double localRuntimeRealEnergy = totalRuntimeRealEnergy;
    PMPI_Reduce(&localRuntimeRealEnergy, &totalRuntimeRealEnergy, 1, MPI_DOUBLE, MPI_MAX, 0,
                MPI_COMM_WORLD);
#endif
    PMPI_Comm_size(MPI_COMM_WORLD, &number_of_procs);
    avgComputation[1] = avgComputation[1] / number_of_procs;
    avgComputation[3] = avgComputation[3] / number_of_procs;
    avgComputation[4] = avgComputation[4] / number_of_procs;
    avgComputation[0] = avgComputation[0] / total_threads;
    avgComputation[2] = avgComputation[2] / total_threads;
    avgComputation[5] = avgComputation[5] / total_threads;
    avgComputation[6] = avgComputation[6] / total_threads;
#ifdef EE_CPT
    avgComputationEnergy[1] = avgComputationEnergy[1] / number_of_procs;
    avgComputationEnergy[3] = avgComputationEnergy[3] / number_of_procs;
    avgComputationEnergy[4] = avgComputationEnergy[4] / number_of_procs;
    avgComputationEnergy[0] = avgComputationEnergy[0] / total_threads;
    avgComputationEnergy[2] = avgComputationEnergy[2] / total_threads;
    avgComputationEnergy[5] = avgComputationEnergy[5] / total_threads;
    avgComputationEnergy[6] = avgComputationEnergy[6] / total_threads;
#endif
  } else
#endif
  {
    for (int i = 0; i < NUM_SHARED_METRICS; i++) {
      maxComputation[i] = uc_max[i];
      avgComputation[i] = uc_avg[i];
#ifdef EE_CPT
      maxComputationEnergy[i] = uc_max_e[i];
      avgComputationEnergy[i] = uc_avg_e[i];
#endif
    }
    avgComputation[0]/=num_threads;
    avgComputation[2]/=num_threads;
    avgComputation[5]/=num_threads;
    avgComputation[6]/=num_threads;
#ifdef EE_CPT
    avgComputationEnergy[0]/=num_threads;
    avgComputationEnergy[2]/=num_threads;
    avgComputationEnergy[5]/=num_threads;
    avgComputationEnergy[6]/=num_threads;
#endif
  }
  if (myProcId == 0) { // display results on master thread
                       // calculate pop metrics
    double totalRuntimeIdeal =
        thread_local_clock->useful_computation_critical.getTime();
    double totalOutsideMPIIdeal =
        thread_local_clock->outsidempi_critical.getTime();
    double totalOutsideOMPIdeal =
        thread_local_clock->outsideomp_critical.getTime();
#ifdef EE_CPT
    double totalRuntimeIdealEnergy =
        thread_local_clock->useful_computation_critical.getEnergy();
    double totalOutsideMPIIdealEnergy =
        thread_local_clock->outsidempi_critical.getEnergy();
    double totalOutsideOMPIdealEnergy =
        thread_local_clock->outsideomp_critical.getEnergy();
#endif
    avgComputation[5] = avgComputation[5] + totalRuntimeReal;
    maxComputation[5] = maxComputation[5] + totalRuntimeReal;
#ifdef EE_CPT
    avgComputationEnergy[5] = avgComputationEnergy[5] + totalRuntimeRealEnergy;
    maxComputationEnergy[5] = maxComputationEnergy[5] + totalRuntimeRealEnergy;
#endif
    double CommE = maxComputation[0] / totalRuntimeReal;
    double TE = totalRuntimeIdeal / totalRuntimeReal;
    double SerE = maxComputation[0] / totalRuntimeIdeal;
    double LB = avgComputation[0] / maxComputation[0];
    double PE = LB * CommE;
#ifdef EE_CPT
    double CommE_e = maxComputationEnergy[0] / totalRuntimeRealEnergy;
    double TE_e = totalRuntimeIdealEnergy / totalRuntimeRealEnergy;
    double SerE_e = maxComputationEnergy[0] / totalRuntimeIdealEnergy;
    double LB_e = avgComputationEnergy[0] / maxComputationEnergy[0];
    double PE_e = LB_e * CommE_e;
#endif

    double mpiLB = avgComputation[6] / maxComputation[0];
    double ompLB = LB / mpiLB;
#ifdef EE_CPT
    double mpiLB_e = avgComputationEnergy[6] / maxComputationEnergy[0];
    double ompLB_e = LB_e / mpiLB_e;
#endif
    double ompTE = totalOutsideOMPIdeal / totalRuntimeReal;
    double mpiTE = TE / ompTE;
#ifdef EE_CPT
    double ompTE_e = totalOutsideOMPIdealEnergy / totalRuntimeRealEnergy;
    double mpiTE_e = TE_e / ompTE_e;
#endif
    double mpiSerE = maxComputation[3] / totalRuntimeIdeal;
    double ompSerE = SerE / mpiSerE;
#ifdef EE_CPT
    double mpiSerE_e = maxComputationEnergy[3] / totalRuntimeIdealEnergy;
    double ompSerE_e = SerE_e / mpiSerE_e;
#endif
    double mpiCommE = mpiSerE * mpiTE;
    double ompCommE = ompSerE * ompTE;
#ifdef EE_CPT
    double mpiCommE_e = mpiSerE_e * mpiTE_e;
    double ompCommE_e = ompSerE_e * ompTE_e;
#endif
    double ompPE = ompLB * ompCommE;
    double mpiPE = mpiLB * mpiCommE;
#ifdef EE_CPT
    double ompPE_e = ompLB_e * ompCommE_e;
    double mpiPE_e = mpiLB_e * mpiCommE_e;
#endif
    FILE *of = stdout;
    bool openedFile{false};

    if (analysis_flags->data_path) {
      if (strcmp("stdout", analysis_flags->data_path) == 0) {
        of = stdout;
      } else if (strcmp("stderr", analysis_flags->data_path) == 0) {
        of = stderr;
      } else {
        auto len = strlen(analysis_flags->data_path);
        char *tempath = (char *)malloc(len + 15);
        sprintf(tempath, "%s-%ix%i.txt", analysis_flags->data_path,
                number_of_procs, num_threads);
        of = fopen(tempath, "w");
        if (analysis_flags->verbose) {
          fprintf(analysis_flags->output, "Opened file %s for output\n",
                  tempath);
        }
        free(tempath);
        openedFile = true;
      }
    }

    if (analysis_flags->verbose) {
      fprintf(
          of,
          "\n[pop] "
          "sere:%6.3lf:te:%6.3lf:comme:%6.3lf:lb:%6.3lf:pe:%6.3lf:crittime:%2."
          "5lf:totaltime:%6.3lf:avgcomputation:%6.3lf:maxcomputation:%6.3lf\n",
          SerE, TE, CommE, LB, PE, totalRuntimeIdeal, totalRuntimeReal,
          avgComputation[0], maxComputation[0]);
      fprintf(of, "\n\n--------MPI stats:--------\n");
      if (total_counts.send)
        fprintf(of, "MPI_*send: %i\n", total_counts.send);
      if (total_counts.isend)
        fprintf(of, "MPI_I*send: %i\n", total_counts.isend);
      if (total_counts.probe)
        fprintf(of, "MPI_*mprobe: %i\n", total_counts.probe);
      if (total_counts.recv)
        fprintf(of, "MPI_Recv: %i\n", total_counts.recv);
      if (total_counts.irecv)
        fprintf(of, "MPI_Irecv: %i\n", total_counts.irecv);
      if (total_counts.coll)
        fprintf(of, "MPI_*coll: %i\n", total_counts.coll);
      if (total_counts.coll)
        fprintf(of, "MPI_I*coll: %i\n", total_counts.coll);
      if (total_counts.test)
        fprintf(of, "MPI_Test*: %i\n", total_counts.test);
      if (total_counts.wait)
        fprintf(of, "MPI_Wait*: %i\n", total_counts.wait);

      if(thread_counts){
        fprintf(of, "\n\n--------OMPT stats:--------\n");
        auto *tCounts = (*thread_counts)[0];
        if(tCounts->taskCreate)
          fprintf(of, "taskCreate: %i\n", tCounts->taskCreate);
        if(tCounts->taskSchedule)
          fprintf(of, "taskSchedule: %i\n", tCounts->taskSchedule);
        if(tCounts->implTaskBegin)
          fprintf(of, "implTaskBegin: %i\n", tCounts->implTaskBegin);
        if(tCounts->implTaskEnd)
          fprintf(of, "implTaskEnd: %i\n", tCounts->implTaskEnd);
        if(tCounts->syncRegionBegin)
          fprintf(of, "syncRegionBegin: %i\n", tCounts->syncRegionBegin);
        if(tCounts->syncRegionEnd)
          fprintf(of, "syncRegionEnd: %i\n", tCounts->syncRegionEnd);
        if(tCounts->mutexAcquire)
          fprintf(of, "mutexAcquire: %i\n", tCounts->mutexAcquire);
      }

      fprintf(of, "\n\n--------CritPath Analysis Tool results:--------\n");
      fprintf(of, "=> Number of processes:          %i\n", number_of_procs);
      fprintf(of, "=> Number of threads:            %i\n", total_threads);
      fprintf(of, "=> Average Computation (in s):   %6.3lf\n",
              avgComputation[0]);
      fprintf(of, "=> Maximum Computation (in s):   %6.3lf\n",
              maxComputation[0]);
      fprintf(of, "=> Max crit. computation (in s): %6.3lf\n",
              totalRuntimeIdeal);
      fprintf(of, "=> Average crit. proc-local computation (in s):    %6.3lf\n",
              avgComputation[3]);
      fprintf(of, "=> Maximum crit. proc-local computation (in s):    %6.3lf\n",
              maxComputation[3]);
      fprintf(of, "=> Average crit. proc-local Outside OpenMP (in s): %6.3lf\n",
              avgComputation[4]);
      fprintf(of, "=> Maximum crit. proc-local Outside OpenMP (in s): %6.3lf\n",
              maxComputation[4]);
      fprintf(of, "=> Average proc-local rumtime (in s):   %6.3lf\n",
              avgComputation[5]);
      fprintf(of, "=> Maximum proc-local runtime (in s):   %6.3lf\n",
              maxComputation[5]);
      fprintf(of, "=> Average PL-max Computation (in s):   %6.3lf\n",
              avgComputation[6]);
      fprintf(of, "=> Maximum PL-max Computation (in s):   %6.3lf\n",
              maxComputation[6]);
      fprintf(of, "=> Average Outside OpenMP (in s):   %6.3lf\n",
              avgComputation[2]);
      fprintf(of, "=> Maximum Outside OpenMP (in s):   %6.3lf\n",
              maxComputation[2]);
      fprintf(of, "=> Max crit. Outside OpenMP (in s): %6.3lf\n",
              totalOutsideOMPIdeal);
      fprintf(of, "=> Total runtime (in s):         %6.3lf\n",
              totalRuntimeReal);
    }

    fprintf(of, "\n----------------POP metrics----------------\n");
    fprintf(of, "Time Parallel Efficiency:                %6.3lf\n",
            PE * analysis_flags->metric_factor);
    fprintf(of, "Time   Load Balance:                     %6.3lf\n",
            LB * analysis_flags->metric_factor);
    fprintf(of, "Time   Communication Efficiency:         %6.3lf\n",
            CommE * analysis_flags->metric_factor);
    fprintf(of, "Time     Serialisation Efficiency:       %6.3lf\n",
            SerE * analysis_flags->metric_factor);
    fprintf(of, "Time     Transfer Efficiency:            %6.3lf\n",
            TE * analysis_flags->metric_factor);
    fprintf(of, "Time   MPI Parallel Efficiency:          %6.3lf\n",
            mpiPE * analysis_flags->metric_factor);
    fprintf(of, "Time     MPI Load Balance:               %6.3lf\n",
            mpiLB * analysis_flags->metric_factor);
    fprintf(of, "Time     MPI Communication Efficiency:   %6.3lf\n",
            mpiCommE * analysis_flags->metric_factor);
    fprintf(of, "Time       MPI Serialisation Efficiency: %6.3lf\n",
            mpiSerE * analysis_flags->metric_factor);
    fprintf(of, "Time       MPI Transfer Efficiency:      %6.3lf\n",
            mpiTE * analysis_flags->metric_factor);
    fprintf(of, "Time   OMP Parallel Efficiency:          %6.3lf\n",
            ompPE * analysis_flags->metric_factor);
    fprintf(of, "Time     OMP Load Balance:               %6.3lf\n",
            ompLB * analysis_flags->metric_factor);
    fprintf(of, "Time     OMP Communication Efficiency:   %6.3lf\n",
            ompCommE * analysis_flags->metric_factor);
    fprintf(of, "Time       OMP Serialisation Efficiency: %6.3lf\n",
            ompSerE * analysis_flags->metric_factor);
    fprintf(of, "Time       OMP Transfer Efficiency:      %6.3lf\n",
            ompTE * analysis_flags->metric_factor);
    fprintf(of, "-------------------------------------------\n");

#ifdef EE_CPT
    fprintf(of, "\n-----------Energy metrics (PKG)------------\n");
    fprintf(of, "Energy Parallel Efficiency:                %6.3lf\n",
            PE_e * analysis_flags->metric_factor);
    fprintf(of, "Energy   Load Balance:                     %6.3lf\n",
            LB_e * analysis_flags->metric_factor);
    fprintf(of, "Energy   Communication Efficiency:         %6.3lf\n",
            CommE_e * analysis_flags->metric_factor);
    fprintf(of, "Energy     Serialisation Efficiency:       %6.3lf\n",
            SerE_e * analysis_flags->metric_factor);
    fprintf(of, "Energy     Transfer Efficiency:            %6.3lf\n",
            TE_e * analysis_flags->metric_factor);
    fprintf(of, "Energy   MPI Parallel Efficiency:          %6.3lf\n",
            mpiPE_e * analysis_flags->metric_factor);
    fprintf(of, "Energy     MPI Load Balance:               %6.3lf\n",
            mpiLB_e * analysis_flags->metric_factor);
    fprintf(of, "Energy     MPI Communication Efficiency:   %6.3lf\n",
            mpiCommE_e * analysis_flags->metric_factor);
    fprintf(of, "Energy       MPI Serialisation Efficiency: %6.3lf\n",
            mpiSerE_e * analysis_flags->metric_factor);
    fprintf(of, "Energy       MPI Transfer Efficiency:      %6.3lf\n",
            mpiTE_e * analysis_flags->metric_factor);
    fprintf(of, "Energy   OMP Parallel Efficiency:          %6.3lf\n",
            ompPE_e * analysis_flags->metric_factor);
    fprintf(of, "Energy     OMP Load Balance:               %6.3lf\n",
            ompLB_e * analysis_flags->metric_factor);
    fprintf(of, "Energy     OMP Communication Efficiency:   %6.3lf\n",
            ompCommE_e * analysis_flags->metric_factor);
    fprintf(of, "Energy       OMP Serialisation Efficiency: %6.3lf\n",
            ompSerE_e * analysis_flags->metric_factor);
    fprintf(of, "Energy       OMP Transfer Efficiency:      %6.3lf\n",
            ompTE_e * analysis_flags->metric_factor);
    fprintf(of, "-------------------------------------------\n");
#endif
    if (openedFile) {
      fclose(of);
    }
  }
}

void OmpClockReset(THREAD_CLOCK *cv) {
  if (!cv || cv->openmp_thread)
    return;
  OmpClockReset(static_cast<SYNC_CLOCK *>(cv));
}

void OmpClockReset(SYNC_CLOCK *cv) {
  if (!analysis_flags->running)
    return;
  if (cv == nullptr)
    fprintf(stderr, "%s: unexpected NULL arg\n", __PRETTY_FUNCTION__);
  else {
    cv->useful_computation_thread.Reset(-1e50);
    cv->useful_computation_proc.Reset(-1e50);
    cv->useful_computation_critical.Reset(-1e50);
    cv->outsidempi_proc.Reset(-1e50);
    cv->outsidempi_thread.Reset(-1e50);
    cv->outsidempi_critical.Reset(-1e50);
    cv->outsideomp_thread.Reset(-1e50);
    cv->outsideomp_critical.Reset(-1e50);
    cv->outsideomp_proc.Reset(-1e50);
  }
}

void startTool(bool toolControl, ClockContext cc) {
  if (analysis_flags->stopped && !toolControl)
    return;
  if (!analysis_flags->running) {
    thread_local_clock->assertUCStopped(true);
    thread_local_clock->assertMpiStopped(true);
    thread_local_clock->assertOmpStopped(true);

    assert(thread_local_clock->useful_computation_thread.getTime() == 0);
    assert(thread_local_clock->useful_computation_proc.getTime() == 0);
    assert(thread_local_clock->useful_computation_critical.getTime() == 0);
    assert(thread_local_clock->outsidempi_proc.getTime() == 0);
    assert(thread_local_clock->outsidempi_thread.getTime() == 0);
    assert(thread_local_clock->outsidempi_critical.getTime() == 0);
    assert(thread_local_clock->outsideomp_thread.getTime() == 0);
    assert(thread_local_clock->outsideomp_critical.getTime() == 0);
    assert(thread_local_clock->outsideomp_proc.getTime() == 0);

#if 0 && defined(USE_MPI)
    if (useMpi) {
      if (analysis_flags->barrier) {
        PMPI_Barrier(MPI_COMM_WORLD);
      }
    }
#endif

#ifdef EE_CPT
    likwidPowerInit();
#ifdef EE_CPT_DEBUG
    likwidPowerInfo();
#endif
    monitorThread = std::thread(monitor_thread);
#endif

    if (analysis_flags->verbose)
      fprintf(analysis_flags->output, "starting tool\n");
    double time = getTime();
    analysis_flags->running = true;
    startMeasurement(time);
    thread_local_clock->Start(time, cc, __func__);
  }
}

void stopTool() {
#if 0 && defined(USE_MPI)
  if (useMpi) {
    if (analysis_flags->barrier) {
      PMPI_Barrier(MPI_COMM_WORLD);
    }
  }
#endif
  if (analysis_flags->running) {
    if (analysis_flags->verbose)
      fprintf(analysis_flags->output, "ending tool\n");
    double time = getTime();
    thread_local_clock->Stop(time, CLOCK_ALL, __func__);
    analysis_flags->running = false;
    stopMeasurement(time);
  }
  if (analysis_flags->dump_on_stop) {
    finishMeasurement();
  }
}

#ifdef EE_CPT
static int my_main(int argc, char** argv, char** envp) {

  int ret = real_main(argc, argv, envp);

  if (!finalized) {
    finalized = true;
    endProgrammEnergy = likwidPowerStop(0, PKG);
    stop_monitor_thread();
  }

  return ret;
}

extern "C"
int __libc_start_main(
    MainFunc main_func,
    int argc, char** ubp_av,
    void (*init)(void),
    void (*fini)(void),
    void (*rtld_fini)(void),
    void* stack_end)
{
  static auto orig_libc_start_main = reinterpret_cast<
      int(*)(MainFunc, int, char**, void(*)(void), void(*)(void), void(*)(void), void*)
      >(dlsym(RTLD_NEXT, "__libc_start_main"));

  if (!orig_libc_start_main) {
    std::cerr << "Error: Failed to locate original __libc_start_main!\n";
  }

  real_main = main_func;

  return orig_libc_start_main(my_main, argc, ubp_av, init, fini, rtld_fini, stack_end);
}
#endif