cpucounters.h

Go to the documentation of this file.

// SPDX-License-Identifier: BSD-3-Clause
// Copyright (c) 2009-2020, Intel Corporation
// written by Roman Dementiev
//            Thomas Willhalm

#ifndef CPUCOUNTERS_HEADER
#define CPUCOUNTERS_HEADER

#include "version.h"

#ifndef PCM_API
#define PCM_API
#endif

#undef PCM_HA_REQUESTS_READS_ONLY
#undef PCM_DEBUG_TOPOLOGY // debug of topology enumeration routine
#undef PCM_UNCORE_PMON_BOX_CHECK_STATUS // debug only

#include "types.h"
#include "msr.h"
#include "pci.h"
#include "bw.h"
#include "width_extender.h"
#include "exceptions/unsupported_processor_exception.hpp"

#include <vector>
#include <array>
#include <limits>
#include <string>
#include <memory>
#include <map>
#include <unordered_map>
#include <string.h>
#include <assert.h>

#ifdef PCM_USE_PERF
#include <linux/perf_event.h>
#include <errno.h>
#define PCM_PERF_COUNT_HW_REF_CPU_CYCLES (9)
#endif

#ifndef _MSC_VER
#define NOMINMAX
#include <semaphore.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/syscall.h>
#include <unistd.h>
#endif

#ifdef _MSC_VER
#if _MSC_VER>= 1600
#include <intrin.h>
#endif
#endif

#include "resctrl.h"

namespace pcm {

#ifdef _MSC_VER
void PCM_API restrictDriverAccess(LPCTSTR path);
#endif

class SystemCounterState;
class SocketCounterState;
class CoreCounterState;
class BasicCounterState;
class ServerUncoreCounterState;
class PCM;
class CoreTaskQueue;
class SystemRoot;

/*
        CPU performance monitoring routines

        A set of performance monitoring routines for recent Intel CPUs
*/

struct PCM_API TopologyEntry // describes a core
{
    int32 os_id;
    int32 thread_id;
    int32 core_id;
    int32 tile_id; // tile is a constalation of 1 or more cores sharing salem L2 cache. Unique for entire system
    int32 socket;
    int32 native_cpu_model = -1;
    enum CoreType
    {
        Atom = 0x20,
        Core = 0x40,
        Invalid = -1
    };
    CoreType core_type = Invalid;

    TopologyEntry() : os_id(-1), thread_id (-1), core_id(-1), tile_id(-1), socket(-1) { }
    const char* getCoreTypeStr()
    {
        switch (core_type)
        {
            case Atom:
                return "Atom";
            case Core:
                return "Core";
            case Invalid:
                return "invalid";
        }
        return "unknown";
    }
};

class HWRegister
{
public:
    virtual void operator = (uint64 val) = 0; // write operation
    virtual operator uint64 () = 0; //read operation
    virtual ~HWRegister() {}
};

class PCICFGRegister64 : public HWRegister
{
    std::shared_ptr<PciHandleType> handle;
    size_t offset;
public:
    PCICFGRegister64(const std::shared_ptr<PciHandleType> & handle_, size_t offset_) :
        handle(handle_),
        offset(offset_)
    {
    }
    void operator = (uint64 val) override
    {
        cvt_ds cvt;
        cvt.ui64 = val;
        handle->write32(offset, cvt.ui32.low);
        handle->write32(offset + sizeof(uint32), cvt.ui32.high);
    }
    operator uint64 ()  override
    {
        uint64 result = 0;
        handle->read64(offset, &result);
        return result;
    }
};

class PCICFGRegister32 : public HWRegister
{
    std::shared_ptr<PciHandleType> handle;
    size_t offset;
public:
    PCICFGRegister32(const std::shared_ptr<PciHandleType> & handle_, size_t offset_) :
        handle(handle_),
        offset(offset_)
    {
    }
    void operator = (uint64 val) override
    {
        handle->write32(offset, (uint32)val);
    }
    operator uint64 () override
    {
        uint32 result = 0;
        handle->read32(offset, &result);
        return result;
    }
};

class MMIORegister64 : public HWRegister
{
    std::shared_ptr<MMIORange> handle;
    size_t offset;
public:
    MMIORegister64(const std::shared_ptr<MMIORange> & handle_, size_t offset_) :
        handle(handle_),
        offset(offset_)
    {
    }
    void operator = (uint64 val) override
    {
        handle->write64(offset, val);
    }
    operator uint64 () override
    {
        return handle->read64(offset);
    }
};

class MMIORegister32 : public HWRegister
{
    std::shared_ptr<MMIORange> handle;
    size_t offset;
public:
    MMIORegister32(const std::shared_ptr<MMIORange> & handle_, size_t offset_) :
        handle(handle_),
        offset(offset_)
    {
    }
    void operator = (uint64 val) override
    {
        handle->write32(offset, (uint32)val);
    }
    operator uint64 () override
    {
        return (uint64)handle->read32(offset);
    }
};

class MSRRegister : public HWRegister
{
    std::shared_ptr<SafeMsrHandle> handle;
    size_t offset;
public:
    MSRRegister(const std::shared_ptr<SafeMsrHandle> & handle_, size_t offset_) :
        handle(handle_),
        offset(offset_)
    {
    }
    void operator = (uint64 val) override
    {
        handle->write(offset, val);
    }
    operator uint64 () override
    {
        uint64 value = 0;
        handle->read(offset, &value);
        return value;
    }
};

class CounterWidthExtenderRegister : public HWRegister
{
    std::shared_ptr<CounterWidthExtender> handle;
public:
    CounterWidthExtenderRegister(const std::shared_ptr<CounterWidthExtender> & handle_) :
        handle(handle_)
    {
    }
    void operator = (uint64 val) override
    {
        if (val == 0)
        {
            handle->reset();
        }
        else
        {
            std::cerr << "ERROR: writing non-zero values to CounterWidthExtenderRegister is not supported\n";
            throw std::exception();
        }
    }
    operator uint64 () override
    {
        return handle->read();;
    }
};

class UncorePMU
{
    typedef std::shared_ptr<HWRegister> HWRegisterPtr;
    HWRegisterPtr unitControl;
public:
    HWRegisterPtr counterControl[4];
    HWRegisterPtr counterValue[4];
    HWRegisterPtr fixedCounterControl;
    HWRegisterPtr fixedCounterValue;
    HWRegisterPtr filter[2];

    UncorePMU(const HWRegisterPtr & unitControl_,
        const HWRegisterPtr & counterControl0,
        const HWRegisterPtr & counterControl1,
        const HWRegisterPtr & counterControl2,
        const HWRegisterPtr & counterControl3,
        const HWRegisterPtr & counterValue0,
        const HWRegisterPtr & counterValue1,
        const HWRegisterPtr & counterValue2,
        const HWRegisterPtr & counterValue3,
        const HWRegisterPtr & fixedCounterControl_ = HWRegisterPtr(),
        const HWRegisterPtr & fixedCounterValue_ = HWRegisterPtr(),
        const HWRegisterPtr & filter0 = HWRegisterPtr(),
        const HWRegisterPtr & filter1 = HWRegisterPtr()
    ) :
        unitControl(unitControl_),
        counterControl{ counterControl0, counterControl1, counterControl2, counterControl3 },
        counterValue{ counterValue0, counterValue1, counterValue2, counterValue3 },
        fixedCounterControl(fixedCounterControl_),
        fixedCounterValue(fixedCounterValue_),
        filter{ filter0 , filter1 }
    {
    }
    UncorePMU() {}
    virtual ~UncorePMU() {}
    bool valid() const
    {
        return unitControl.get() != nullptr;
    }
    void writeUnitControl(const uint32 value)
    {
        *unitControl = value;
    }
    void cleanup();
    void freeze(const uint32 extra);
    bool initFreeze(const uint32 extra, const char* xPICheckMsg = nullptr);
    void unfreeze(const uint32 extra);
    void resetUnfreeze(const uint32 extra);
};

enum ServerUncoreMemoryMetrics
{
    PartialWrites,
    Pmem,
    PmemMemoryMode,
    PmemMixedMode
};

class ServerPCICFGUncore
{
    friend class PCM;
    int32 iMCbus,UPIbus,M2Mbus;
    uint32 groupnr;
    int32 cpu_model;
    typedef std::vector<UncorePMU> UncorePMUVector;
    UncorePMUVector imcPMUs;
    UncorePMUVector edcPMUs;
    UncorePMUVector xpiPMUs;
    UncorePMUVector m3upiPMUs;
    UncorePMUVector m2mPMUs;
    UncorePMUVector haPMUs;
    std::vector<UncorePMUVector*> allPMUs{ &imcPMUs, &edcPMUs, &xpiPMUs, &m3upiPMUs , &m2mPMUs, &haPMUs };
    std::vector<uint64> qpi_speed;
    std::vector<uint32> num_imc_channels; // number of memory channels in each memory controller
    std::vector<std::pair<uint32, uint32> > XPIRegisterLocation; // (device, function)
    std::vector<std::pair<uint32, uint32> > M3UPIRegisterLocation; // (device, function)
    std::vector<std::vector< std::pair<uint32, uint32> > > MCRegisterLocation; // MCRegisterLocation[controller]: (device, function)
    std::vector<std::pair<uint32, uint32> > EDCRegisterLocation; // EDCRegisterLocation: (device, function)
    std::vector<std::pair<uint32, uint32> > M2MRegisterLocation; // M2MRegisterLocation: (device, function)
    std::vector<std::pair<uint32, uint32> > HARegisterLocation;  // HARegisterLocation: (device, function)

    static std::vector<std::pair<uint32, uint32> > socket2iMCbus;
    static std::vector<std::pair<uint32, uint32> > socket2UPIbus;
    static std::vector<std::pair<uint32, uint32> > socket2M2Mbus;

    ServerPCICFGUncore();                                         // forbidden
    ServerPCICFGUncore(ServerPCICFGUncore &);                     // forbidden
    ServerPCICFGUncore & operator = (const ServerPCICFGUncore &); // forbidden
    PciHandleType * createIntelPerfMonDevice(uint32 groupnr, int32 bus, uint32 dev, uint32 func, bool checkVendor = false);
    void programIMC(const uint32 * MCCntConfig);
    void programEDC(const uint32 * EDCCntConfig);
    void programM2M(const uint64 * M2MCntConfig);
    void programM2M();
    void programHA(const uint32 * config);
    void programHA();
    void programXPI(const uint32 * XPICntConfig);
    void programM3UPI(const uint32* M3UPICntConfig);
    typedef std::pair<size_t, std::vector<uint64 *> > MemTestParam;
    void initMemTest(MemTestParam & param);
    void doMemTest(const MemTestParam & param);
    void cleanupMemTest(const MemTestParam & param);
    void cleanupQPIHandles();
    void cleanupPMUs();
    void writeAllUnitControl(const uint32 value);
    void initDirect(uint32 socket_, const PCM * pcm);
    void initPerf(uint32 socket_, const PCM * pcm);
    void initBuses(uint32 socket_, const PCM * pcm);
    void initRegisterLocations(const PCM * pcm);
    uint64 getPMUCounter(std::vector<UncorePMU> & pmu, const uint32 id, const uint32 counter);

public:
    enum EventPosition {
        READ=0,
        WRITE=1,
        READ_RANK_A=0,
        WRITE_RANK_A=1,
        READ_RANK_B=2,
        WRITE_RANK_B=3,
        PARTIAL=2,
        PMM_READ=2,
        PMM_WRITE=3,
        PMM_MM_MISS_CLEAN=2,
        PMM_MM_MISS_DIRTY=3,
        NM_HIT=0,  // NM :  Near Memory (DRAM cache) in Memory Mode
        M2M_CLOCKTICKS=1
    };
    ServerPCICFGUncore(uint32 socket_, const PCM * pcm);
    void program();
    uint64 getImcReads();
    uint64 getImcReadsForController(uint32 controller);
    uint64 getImcReadsForChannels(uint32 beginChannel, uint32 endChannel);
    uint64 getImcWrites();
    uint64 getHALocalRequests();
    uint64 getHARequests();

    uint64 getPMMReads();
    uint64 getPMMWrites();

    uint64 getEdcReads();
    uint64 getEdcWrites();

    uint64 getIncomingDataFlits(uint32 port);

    uint64 getOutgoingFlits(uint32 port);

    ~ServerPCICFGUncore();

    void program_power_metrics(int mc_profile);

    void programServerUncoreMemoryMetrics(const ServerUncoreMemoryMetrics & metrics, const int rankA = -1, const int rankB = -1);

    uint64 getQPIClocks(uint32 port);

    uint64 getQPIL0pTxCycles(uint32 port);
    uint64 getUPIL0TxCycles(uint32 port);
    uint64 getQPIL1Cycles(uint32 port);
    uint64 getDRAMClocks(uint32 channel);
    uint64 getMCDRAMClocks(uint32 channel);
    uint64 getMCCounter(uint32 channel, uint32 counter);
    uint64 getEDCCounter(uint32 channel, uint32 counter);
    uint64 getQPILLCounter(uint32 port, uint32 counter);
    uint64 getM3UPICounter(uint32 port, uint32 counter);
    uint64 getM2MCounter(uint32 box, uint32 counter);

    void freezeCounters();
    void unfreezeCounters();

    uint64 computeQPISpeed(const uint32 ref_core, const int cpumodel);

    void enableJKTWorkaround(bool enable);

    size_t getNumQPIPorts() const { return xpiPMUs.size(); }

    uint64 getQPILinkSpeed(const uint32 linkNr) const
    {
        return qpi_speed.empty() ? 0 : qpi_speed[linkNr];
    }

    void reportQPISpeed() const;

    uint32 getNumMC() const { return (uint32)num_imc_channels.size(); }

    size_t getNumMCChannels() const { return (size_t)imcPMUs.size(); }

    size_t getNumMCChannels(const uint32 controller) const;

    size_t getNumEDCChannels() const { return edcPMUs.size(); }
};

class SimpleCounterState
{
    template <class T>
    friend uint64 getNumberOfEvents(const T & before, const T & after);
    friend class PCM;
    uint64 data;

public:
    SimpleCounterState() : data(0)
    { }
    virtual ~SimpleCounterState() { }
};

typedef SimpleCounterState PCIeCounterState;
typedef SimpleCounterState IIOCounterState;
typedef std::vector<uint64> eventGroup_t;

class PerfVirtualControlRegister;

class PCM_API PCM
{
    friend class BasicCounterState;
    friend class UncoreCounterState;
    friend class Socket;
    friend class ServerUncore;
    friend class PerfVirtualControlRegister;
    friend class Aggregator;
    friend class ServerPCICFGUncore;
    PCM();     // forbidden to call directly because it is a singleton
    PCM(const PCM &) = delete;
    PCM & operator = (const PCM &) = delete;

    int32 cpu_family;
    int32 cpu_model;
    bool hybrid = false;
    int32 cpu_stepping;
    int64 cpu_microcode_level;
    uint32 max_cpuid;
    int32 threads_per_core;
    int32 num_cores;
    int32 num_sockets;
    int32 num_phys_cores_per_socket;
    int32 num_online_cores;
    int32 num_online_sockets;
    uint32 core_gen_counter_num_max;
    uint32 core_gen_counter_num_used;
    uint32 core_gen_counter_width;
    uint32 core_fixed_counter_num_max;
    uint32 core_fixed_counter_num_used;
    uint32 core_fixed_counter_width;
    uint64 core_global_ctrl_value{0ULL};
    uint32 uncore_gen_counter_num_max;
    uint32 uncore_gen_counter_num_used;
    uint32 uncore_gen_counter_width;
    uint32 uncore_fixed_counter_num_max;
    uint32 uncore_fixed_counter_num_used;
    uint32 uncore_fixed_counter_width;
    uint32 perfmon_version;
    int32 perfmon_config_anythread;
    uint64 nominal_frequency;
    uint64 max_qpi_speed; // in GBytes/second
    uint32 L3ScalingFactor;
    int32 pkgThermalSpecPower, pkgMinimumPower, pkgMaximumPower;

    std::vector<TopologyEntry> topology;
    SystemRoot* systemTopology;
    std::string errorMessage;

    static PCM * instance;
    bool programmed_core_pmu{false};
    std::vector<std::shared_ptr<SafeMsrHandle> > MSR;
    std::vector<std::shared_ptr<ServerPCICFGUncore> > server_pcicfg_uncore;
    std::vector<UncorePMU> pcuPMUs;
    std::vector<std::map<int32, UncorePMU> > iioPMUs;
    std::vector<std::map<int32, UncorePMU> > irpPMUs;
    std::vector<UncorePMU> uboxPMUs;
    double joulesPerEnergyUnit;
    std::vector<std::shared_ptr<CounterWidthExtender> > energy_status;
    std::vector<std::shared_ptr<CounterWidthExtender> > dram_energy_status;
    std::vector<std::vector<UncorePMU> > cboPMUs;

    std::vector<std::shared_ptr<CounterWidthExtender> > memory_bw_local;
    std::vector<std::shared_ptr<CounterWidthExtender> > memory_bw_total;
#ifdef __linux__
    Resctrl resctrl;
#endif
    bool useResctrl;

    std::shared_ptr<FreeRunningBWCounters> clientBW;
    std::shared_ptr<CounterWidthExtender> clientImcReads;
    std::shared_ptr<CounterWidthExtender> clientImcWrites;
    std::shared_ptr<CounterWidthExtender> clientGtRequests;
    std::shared_ptr<CounterWidthExtender> clientIaRequests;
    std::shared_ptr<CounterWidthExtender> clientIoRequests;

    std::vector<std::shared_ptr<ServerBW> > serverBW;

    bool disable_JKT_workaround;
    bool blocked;              // track if time-driven counter update is running or not: PCM is blocked

    uint64 * coreCStateMsr;    // MSR addresses of core C-state free-running counters
    uint64 * pkgCStateMsr;     // MSR addresses of package C-state free-running counters

    std::vector<std::shared_ptr<CoreTaskQueue> > coreTaskQueues;

    bool L2CacheHitRatioAvailable;
    bool L3CacheHitRatioAvailable;
    bool L3CacheMissesAvailable;
    bool L2CacheMissesAvailable;
    bool L2CacheHitsAvailable;
    bool L3CacheHitsNoSnoopAvailable;
    bool L3CacheHitsSnoopAvailable;
    bool L3CacheHitsAvailable;

    bool forceRTMAbortMode;

    std::vector<uint64> FrontendBoundSlots, BadSpeculationSlots, BackendBoundSlots, RetiringSlots, AllSlotsRaw;
    bool isFixedCounterSupported(unsigned c);
    bool vm = false;
    bool linux_arch_perfmon = false;

public:
    enum { MAX_C_STATE = 10 }; // max C-state on Intel architecture

    bool isCoreCStateResidencySupported(int state)
    {
        if (state == 0 || state == 1)
            return true;

        return (coreCStateMsr != NULL && state <= ((int)MAX_C_STATE) && coreCStateMsr[state] != 0);
    }

    bool isPackageCStateResidencySupported(int state)
    {
        if (state == 0)
        {
            return true;
        }
        return (pkgCStateMsr != NULL && state <= ((int)MAX_C_STATE) && pkgCStateMsr[state] != 0);
    }

    static void setOutput(const std::string filename, const bool cerrToo = false);

    void restoreOutput();

    // Arguments:
    //  -- 1 - program is running
    //  -- 0 -pgram is sleeping
    void setRunState(int new_state) { run_state = new_state; }

    // Results:
    //  -- 1 - program is running
    //  -- 0 -pgram is sleeping
    int getRunState(void) { return run_state; }

    bool isBlocked(void) { return blocked; }
    void setBlocked(const bool new_blocked) { blocked = new_blocked; }

    enum ProgramMode {
        DEFAULT_EVENTS = 0,         
        CUSTOM_CORE_EVENTS = 1,     
        EXT_CUSTOM_CORE_EVENTS = 2, 
        INVALID_MODE                
    };

    enum ErrorCode {
        Success = 0,
        MSRAccessDenied = 1,
        PMUBusy = 2,
        UnknownError
    };

    enum PerfmonField {
        INVALID, /* Use to parse invalid field */
        OPCODE,
        EVENT_SELECT,
        UMASK,
        RESET,
        EDGE_DET,
        IGNORED,
        OVERFLOW_ENABLE,
        ENABLE,
        INVERT,
        THRESH,
        CH_MASK,
        FC_MASK,
        /* Below are not part of perfmon definition */
        H_EVENT_NAME,
        V_EVENT_NAME,
        MULTIPLIER,
        DIVIDER,
        COUNTER_INDEX
    };

    enum PCIeWidthMode {
        X1,
        X4,
        X8,
        X16,
        XFF
    };

    enum { // offsets/enumeration of IIO stacks
        IIO_CBDMA = 0, // shared with DMI
        IIO_PCIe0 = 1,
        IIO_PCIe1 = 2,
        IIO_PCIe2 = 3,
        IIO_MCP0 = 4,
        IIO_MCP1 = 5,
        IIO_STACK_COUNT = 6
    };

    // Offsets/enumeration of IIO stacks Skylake server.
    enum SkylakeIIOStacks {
        SKX_IIO_CBDMA_DMI   = 0,
        SKX_IIO_PCIe0       = 1,
        SKX_IIO_PCIe1       = 2,
        SKX_IIO_PCIe2       = 3,
        SKX_IIO_MCP0        = 4,
        SKX_IIO_MCP1        = 5,
        SKX_IIO_STACK_COUNT = 6
    };

     // Offsets/enumeration of IIO stacks for IceLake server.
    enum IcelakeIIOStacks {
        ICX_IIO_PCIe0       = 0,
        ICX_IIO_PCIe1       = 1,
        ICX_IIO_MCP0        = 2,
        ICX_IIO_PCIe2       = 3,
        ICX_IIO_PCIe3       = 4,
        ICX_IIO_CBDMA_DMI   = 5,
        ICX_IIO_STACK_COUNT = 6
    };

    // Offsets/enumeration of IIO stacks for IceLake server.
    enum SnowridgeIIOStacks {
        SNR_IIO_QAT         = 0,
        SNR_IIO_CBDMA_DMI   = 1,
        SNR_IIO_NIS         = 2,
        SNR_IIO_HQM         = 3,
        SNR_IIO_PCIe0       = 4,
        SNR_IIO_STACK_COUNT = 5
    };

    struct SimplePCIeDevInfo
    {
        enum PCIeWidthMode width;
        std::string pciDevName;
        std::string busNumber;

        SimplePCIeDevInfo() : width(XFF) { }
    };

    struct CustomCoreEventDescription
    {
        int32 event_number = 0, umask_value = 0;
    };

    struct ExtendedCustomCoreEventDescription
    {
        FixedEventControlRegister * fixedCfg; // if NULL, then default configuration performed for fixed counters
        uint32 nGPCounters;                   // number of general purpose counters
        EventSelectRegister * gpCounterCfg;   // general purpose counters, if NULL, then default configuration performed for GP counters
        EventSelectRegister * gpCounterHybridAtomCfg; // general purpose counters for Atom cores in hybrid processors
        uint64 OffcoreResponseMsrValue[2];
        uint64 LoadLatencyMsrValue, FrontendMsrValue;
        bool defaultUncoreProgramming{true};
        static uint64 invalidMsrValue() { return ~0ULL; }
        ExtendedCustomCoreEventDescription() : fixedCfg(NULL), nGPCounters(0), gpCounterCfg(nullptr), gpCounterHybridAtomCfg(nullptr), LoadLatencyMsrValue(invalidMsrValue()), FrontendMsrValue(invalidMsrValue())
        {
            OffcoreResponseMsrValue[0] = 0;
            OffcoreResponseMsrValue[1] = 0;
        }
    };

    struct CustomIIOEventDescription
    {
        /* We program the same counters to every IIO Stacks */
        std::string eventNames[4];
        IIOPMUCNTCTLRegister eventOpcodes[4];
        int multiplier[4]; //Some IIO event requires transformation to get meaningful output (i.e. DWord to bytes)
        int divider[4]; //We usually like to have some kind of divider (i.e. /10e6 )
    };

    enum MSREventPosition
    {
        index = 0,
        type = 1
    };
    enum MSRType
    {
        Static = 0,
        Freerun = 1
    };

private:
    ProgramMode mode;
    CustomCoreEventDescription coreEventDesc[PERF_MAX_CUSTOM_COUNTERS];
    CustomCoreEventDescription hybridAtomEventDesc[PERF_MAX_CUSTOM_COUNTERS];

    std::vector<int32> socketRefCore;

    bool canUsePerf;
#ifdef PCM_USE_PERF
    typedef std::vector<std::vector<int> > PerfEventHandleContainer;
    PerfEventHandleContainer perfEventHandle;
    std::vector<PerfEventHandleContainer> perfEventTaskHandle;
    void readPerfData(uint32 core, std::vector<uint64> & data);
    void closePerfHandles(const bool silent = false);

    enum {
        PERF_INST_RETIRED_POS = 0,
        PERF_CPU_CLK_UNHALTED_THREAD_POS = 1,
        PERF_CPU_CLK_UNHALTED_REF_POS = 2,
        PERF_GEN_EVENT_0_POS = 3,
        PERF_GEN_EVENT_1_POS = 4,
        PERF_GEN_EVENT_2_POS = 5,
        PERF_GEN_EVENT_3_POS = 6,
        PERF_TOPDOWN_SLOTS_POS = PERF_GEN_EVENT_0_POS + PERF_MAX_CUSTOM_COUNTERS,
        PERF_TOPDOWN_FRONTEND_POS = PERF_TOPDOWN_SLOTS_POS + 1,
        PERF_TOPDOWN_BADSPEC_POS = PERF_TOPDOWN_SLOTS_POS + 2,
        PERF_TOPDOWN_BACKEND_POS = PERF_TOPDOWN_SLOTS_POS + 3,
        PERF_TOPDOWN_RETIRING_POS = PERF_TOPDOWN_SLOTS_POS + 4
    };

    std::array<int, (PERF_TOPDOWN_RETIRING_POS + 1)> perfTopDownPos;

    enum {
        PERF_GROUP_LEADER_COUNTER = PERF_INST_RETIRED_POS,
        PERF_TOPDOWN_GROUP_LEADER_COUNTER = PERF_TOPDOWN_SLOTS_POS
    };
#endif
    static std::ofstream * outfile;       // output file stream
    static std::streambuf * backup_ofile; // backup of original output = cout
    static std::streambuf * backup_ofile_cerr; // backup of original output = cerr
    int run_state;                 // either running (1) or sleeping (0)

    bool needToRestoreNMIWatchdog;

    std::vector<std::vector<EventSelectRegister> > lastProgrammedCustomCounters;
    uint32 checkCustomCoreProgramming(std::shared_ptr<SafeMsrHandle> msr);
    ErrorCode programCoreCounters(int core, const PCM::ProgramMode mode, const ExtendedCustomCoreEventDescription * pExtDesc,
        std::vector<EventSelectRegister> & programmedCustomCounters, const std::vector<int> & tids);

    bool PMUinUse();
    void cleanupPMU(const bool silent = false);
    void cleanupRDT(const bool silent = false);

    void computeQPISpeedBeckton(int core_nr);
    void destroyMSR();
    void computeNominalFrequency();
    static bool isCPUModelSupported(const int model_);
    std::string getSupportedUarchCodenames() const;
    std::string getUnsupportedMessage() const;
    bool detectModel();
    bool checkModel();

    void initCStateSupportTables();
    bool discoverSystemTopology();
    void printSystemTopology() const;
    bool initMSR();
    bool detectNominalFrequency();
    void showSpecControlMSRs();
    void initEnergyMonitoring();
    void initUncoreObjects();
    void initRDT();
    void initQOSevent(const uint64 event, const int32 core);
    void programBecktonUncore(int core);
    void programNehalemEPUncore(int core);
    void enableJKTWorkaround(bool enable);
    template <class CounterStateType>
    void readAndAggregateMemoryBWCounters(const uint32 core, CounterStateType & counterState);
    template <class CounterStateType>
    void readAndAggregateUncoreMCCounters(const uint32 socket, CounterStateType & counterState);
    template <class CounterStateType>
    void readAndAggregateEnergyCounters(const uint32 socket, CounterStateType & counterState);
    template <class CounterStateType>
    void readPackageThermalHeadroom(const uint32 socket, CounterStateType & counterState);
    template <class CounterStateType>
    void readAndAggregatePackageCStateResidencies(std::shared_ptr<SafeMsrHandle> msr, CounterStateType & result);
public:
    struct RawPMUConfig;
private:
    template <class CounterStateType>
    void readMSRs(std::shared_ptr<SafeMsrHandle> msr, const RawPMUConfig & msrConfig, CounterStateType & result);
    void readQPICounters(SystemCounterState & counterState);
    void reportQPISpeed() const;
    void readCoreCounterConfig(const bool complainAboutMSR = false);
    void readCPUMicrocodeLevel();

    uint64 CX_MSR_PMON_CTRY(uint32 Cbo, uint32 Ctr) const;
    uint64 CX_MSR_PMON_BOX_FILTER(uint32 Cbo) const;
    uint64 CX_MSR_PMON_BOX_FILTER1(uint32 Cbo) const;
    uint64 CX_MSR_PMON_CTLY(uint32 Cbo, uint32 Ctl) const;
    uint64 CX_MSR_PMON_BOX_CTL(uint32 Cbo) const;
    void programCboOpcodeFilter(const uint32 opc0, UncorePMU & pmu, const uint32 nc_, const uint32 opc1, const uint32 loc, const uint32 rem);
    void initLLCReadMissLatencyEvents(uint64 * events, uint32 & opCode);
    void initCHARequestEvents(uint64 * events);
    void programCbo();
    uint64 getCBOCounterState(const uint32 socket, const uint32 ctr_);
    template <class Iterator>
    static void program(UncorePMU& pmu, const Iterator& eventsBegin, const Iterator& eventsEnd, const uint32 extra)
    {
        if (!eventsBegin) return;
        Iterator curEvent = eventsBegin;
        for (int c = 0; curEvent != eventsEnd; ++c, ++curEvent)
        {
            auto ctrl = pmu.counterControl[c];
            if (ctrl.get() != nullptr)
            {
                *ctrl = MC_CH_PCI_PMON_CTL_EN;
                *ctrl = MC_CH_PCI_PMON_CTL_EN | *curEvent;
            }
        }
        if (extra)
        {
            pmu.resetUnfreeze(extra);
        }
    }
    void programPCU(uint32 * events, const uint64 filter);
    void programUBOX(const uint64* events);

    void cleanupUncorePMUs(const bool silent = false);

    bool isCLX() const // Cascade Lake-SP
    {
        return (PCM::SKX == cpu_model) && (cpu_stepping > 4 && cpu_stepping < 8);
    }

    static bool isCPX(int cpu_model_, int cpu_stepping_) // Cooper Lake
    {
        return (PCM::SKX == cpu_model_) && (cpu_stepping_ >= 10);
    }

    bool isCPX() const
    {
        return isCPX(cpu_model, cpu_stepping);
    }

    void initUncorePMUsDirect();
    void initUncorePMUsPerf();
    bool isRDTDisabled() const;

public:
    bool isHWTMAL1Supported() const;

    enum EventPosition
    {
        TOR_OCCUPANCY = 0,
        TOR_INSERTS = 1,
        REQUESTS_ALL = 2,
        REQUESTS_LOCAL = 3
    };
    bool isSecureBoot() const;

    bool useLinuxPerfForUncore() const;

    SystemRoot const & getSystemTopology() const {
        return *systemTopology;
    }

    void printDetailedSystemTopology();

    bool QOSMetricAvailable() const;
    bool L3QOSMetricAvailable() const;
    bool L3CacheOccupancyMetricAvailable() const;
    bool CoreLocalMemoryBWMetricAvailable() const;
    bool CoreRemoteMemoryBWMetricAvailable() const;
    unsigned getMaxRMID() const;

    uint32 getMaxNumOfCBoxes() const;

    uint32 getMaxNumOfIIOStacks() const;

    static PCM * getInstance();        // the only way to get access

    bool good();                       // true if access to CPU counters works

    const std::string & getErrorMessage() const
    {
        return errorMessage;
    }

    ErrorCode program(const ProgramMode mode_ = DEFAULT_EVENTS, const void * parameter_ = NULL, const bool silent = false, const int pid = -1); // program counters and start counting

    void checkError(const ErrorCode code);

    ErrorCode programServerUncoreLatencyMetrics(bool enable_pmm);

    ErrorCode programServerUncorePowerMetrics(int mc_profile, int pcu_profile, int * freq_bands = NULL);

    /*  \brief Program memory counters (disables programming performance counters)
        \param rankA count DIMM rank1 statistics (disables memory channel monitoring)
        \param rankB count DIMM rank2 statistics (disables memory channel monitoring)
        \brief metrics metric set (see the ServerUncoreMemoryMetrics enum)

        Call this method before you start using the memory counter routines on microarchitecture codename SandyBridge-EP and later Xeon uarch

        \warning Using this routines with other tools that *program* Performance Monitoring
        Units (PMUs) on CPUs is not recommended because PMU can not be shared. Tools that are known to
        program PMUs: Intel(r) VTune(tm), Intel(r) Performance Tuning Utility (PTU). This code may make
        VTune or PTU measurements invalid. VTune or PTU measurement may make measurement with this code invalid. Please enable either usage of these routines or VTune/PTU/etc.
    */
    ErrorCode programServerUncoreMemoryMetrics(const ServerUncoreMemoryMetrics & metrics, int rankA = -1, int rankB = -1);

    // vector of IDs. E.g. for core {raw event} or {raw event, offcore response1 msr value, } or {raw event, offcore response1 msr value, offcore response2}
    // or for cha/cbo {raw event, filter value}, etc
    // + user-supplied name
    typedef std::pair<std::array<uint64, 5>, std::string> RawEventConfig;
    struct RawPMUConfig
    {
        std::vector<RawEventConfig> programmable;
        std::vector<RawEventConfig> fixed;
    };
    enum {
        OCR0Pos = 1,
        OCR1Pos = 2,
        LoadLatencyPos = 3,
        FrontendPos = 4
    };
    typedef std::map<std::string, RawPMUConfig> RawPMUConfigs;
    ErrorCode program(const RawPMUConfigs& curPMUConfigs, const bool silent = false, const int pid = -1);

    std::pair<unsigned, unsigned> getOCREventNr(const int event, const unsigned coreID) const
    {
       assert (coreID < topology.size());
       if (hybrid)
       {
            switch (cpu_model)
            {
            case ADL:
                if (topology[coreID].core_type == TopologyEntry::Atom)
                {
                    return std::make_pair(OFFCORE_RESPONSE_0_EVTNR, event + 1);
                }
                break;
            }
       }
       bool useGLCOCREvent = false;
       switch (cpu_model)
       {
       case ADL: // ADL big core (GLC)
           useGLCOCREvent = true;
           break;
       }
       switch (event)
       {
            case 0:
                return std::make_pair(useGLCOCREvent ? GLC_OFFCORE_RESPONSE_0_EVTNR : OFFCORE_RESPONSE_0_EVTNR, OFFCORE_RESPONSE_0_UMASK);
            case 1:
                return std::make_pair(useGLCOCREvent ? GLC_OFFCORE_RESPONSE_1_EVTNR : OFFCORE_RESPONSE_1_EVTNR, OFFCORE_RESPONSE_1_UMASK);
       }
       assert (false && "wrong event nr in getOCREventNr");
       return std::make_pair(0U, 0U);
    }

    void freezeServerUncoreCounters();

    void unfreezeServerUncoreCounters();

    ServerUncoreCounterState getServerUncoreCounterState(uint32 socket);

    void cleanup(const bool silent = false);

    void resetPMU();

    void getAllCounterStates(SystemCounterState & systemState, std::vector<SocketCounterState> & socketStates, std::vector<CoreCounterState> & coreStates, const bool readAndAggregateSocketUncoreCounters = true);

    void getUncoreCounterStates(SystemCounterState & systemState, std::vector<SocketCounterState> & socketStates);

    bool isCoreOnline(int32 os_core_id) const;

    bool isSocketOnline(int32 socket_id) const;

    SystemCounterState getSystemCounterState();

    SocketCounterState getSocketCounterState(uint32 socket);

    CoreCounterState getCoreCounterState(uint32 core);

    uint32 getNumCores() const;

    uint32 getNumOnlineCores() const;

    uint32 getNumSockets() const;

    uint32 getNumOnlineSockets() const;

    uint32 getThreadsPerCore() const;

    bool getSMT() const; // returns true iff SMT ("Hyperthreading") is on

    uint64 getNominalFrequency() const; // in Hz

    uint32 getL3ScalingFactor() const;

    bool isSomeCoreOfflined();

    int32 getMaxCustomCoreEvents();

    static int getCPUModelFromCPUID();

    enum SupportedCPUModels
    {
        NEHALEM_EP = 26,
        NEHALEM = 30,
        ATOM = 28,
        ATOM_2 = 53,
        CENTERTON = 54,
        BAYTRAIL = 55,
        AVOTON = 77,
        CHERRYTRAIL = 76,
        APOLLO_LAKE = 92,
        DENVERTON = 95,
        SNOWRIDGE = 134,
        CLARKDALE = 37,
        WESTMERE_EP = 44,
        NEHALEM_EX = 46,
        WESTMERE_EX = 47,
        SANDY_BRIDGE = 42,
        JAKETOWN = 45,
        IVY_BRIDGE = 58,
        HASWELL = 60,
        HASWELL_ULT = 69,
        HASWELL_2 = 70,
        IVYTOWN = 62,
        HASWELLX = 63,
        BROADWELL = 61,
        BROADWELL_XEON_E3 = 71,
        BDX_DE = 86,
        SKL_UY = 78,
        KBL = 158,
        KBL_1 = 142,
        CML = 166,
        CML_1 = 165,
        ICL = 126,
        ICL_1 = 125,
        RKL = 167,
        TGL = 140,
        TGL_1 = 141,
        ADL = 151,
        ADL_1 = 154,
        BDX = 79,
        KNL = 87,
        SKL = 94,
        SKX = 85,
        ICX_D = 108,
        ICX = 106,
        END_OF_MODEL_LIST = 0x0ffff
    };

#define PCM_SKL_PATH_CASES \
        case PCM::SKL_UY:  \
        case PCM::KBL:     \
        case PCM::KBL_1:   \
        case PCM::CML:     \
        case PCM::ICL:     \
        case PCM::RKL:     \
        case PCM::TGL:     \
        case PCM::SKL:

private:
    bool useSKLPath() const
    {
        switch (cpu_model)
        {
            PCM_SKL_PATH_CASES
                return true;
        }
        return false;
    }
    RawPMUConfig threadMSRConfig{}, packageMSRConfig{};
public:

    uint32 getCPUModel() const { return (uint32)cpu_model; }

    uint32 getCPUStepping() const { return (uint32)cpu_stepping; }

    int32 getThreadId(uint32 os_id) const { return (int32)topology[os_id].thread_id; }

    int32 getCoreId(uint32 os_id) const { return (int32)topology[os_id].core_id; }

    int32 getTileId(uint32 os_id) const { return (int32)topology[os_id].tile_id; }

    int32 getSocketId(uint32 core_id) const { return (int32)topology[core_id].socket; }

    uint64 getQPILinksPerSocket() const
    {
        switch (cpu_model)
        {
        case NEHALEM_EP:
        case WESTMERE_EP:
        case CLARKDALE:
            if (num_sockets == 2)
                return 2;
            else
                return 1;
        case NEHALEM_EX:
        case WESTMERE_EX:
            return 4;
        case JAKETOWN:
        case IVYTOWN:
        case HASWELLX:
        case BDX_DE:
        case BDX:
        case SKX:
        case ICX:
            return (server_pcicfg_uncore.size() && server_pcicfg_uncore[0].get()) ? (server_pcicfg_uncore[0]->getNumQPIPorts()) : 0;
        }
        return 0;
    }

    uint32 getMCPerSocket() const
    {
        switch (cpu_model)
        {
        case NEHALEM_EP:
        case WESTMERE_EP:
        case CLARKDALE:
            return 1;
        case NEHALEM_EX:
        case WESTMERE_EX:
            return 2;
        case JAKETOWN:
        case IVYTOWN:
        case HASWELLX:
        case BDX_DE:
        case SKX:
        case ICX:
        case BDX:
        case KNL:
            return (server_pcicfg_uncore.size() && server_pcicfg_uncore[0].get()) ? (server_pcicfg_uncore[0]->getNumMC()) : 0;
        }
        return 0;
    }

    size_t getMCChannelsPerSocket() const
    {
        switch (cpu_model)
        {
        case NEHALEM_EP:
        case WESTMERE_EP:
        case CLARKDALE:
            return 3;
        case NEHALEM_EX:
        case WESTMERE_EX:
            return 4;
        case JAKETOWN:
        case IVYTOWN:
        case HASWELLX:
        case BDX_DE:
        case SKX:
        case ICX:
        case BDX:
        case KNL:
        case SNOWRIDGE:
            return (server_pcicfg_uncore.size() && server_pcicfg_uncore[0].get()) ? (server_pcicfg_uncore[0]->getNumMCChannels()) : 0;
        }
        return 0;
    }

    size_t getMCChannels(uint32 socket, uint32 controller) const
    {
        switch (cpu_model)
        {
        case NEHALEM_EP:
        case WESTMERE_EP:
        case CLARKDALE:
            return 3;
        case NEHALEM_EX:
        case WESTMERE_EX:
            return 4;
        case JAKETOWN:
        case IVYTOWN:
        case HASWELLX:
        case BDX_DE:
        case SKX:
        case ICX:
        case BDX:
        case KNL:
        case SNOWRIDGE:
            return (socket < server_pcicfg_uncore.size() && server_pcicfg_uncore[socket].get()) ? (server_pcicfg_uncore[socket]->getNumMCChannels(controller)) : 0;
        }
        return 0;
    }


    size_t getEDCChannelsPerSocket() const
    {
        switch (cpu_model)
        {
        case KNL:
            return (server_pcicfg_uncore.size() && server_pcicfg_uncore[0].get()) ? (server_pcicfg_uncore[0]->getNumEDCChannels()) : 0;
        }
        return 0;
    }


    uint32 getMaxIPC() const
    {
        if (ICL == cpu_model || TGL == cpu_model || RKL == cpu_model) return 5;
        switch (cpu_model)
        {
        case ADL:
            return 6;
        case SNOWRIDGE:
            return 4;
        case DENVERTON:
            return 3;
        case NEHALEM_EP:
        case WESTMERE_EP:
        case NEHALEM_EX:
        case WESTMERE_EX:
        case CLARKDALE:
        case SANDY_BRIDGE:
        case JAKETOWN:
        case IVYTOWN:
        case IVY_BRIDGE:
        case HASWELL:
        case HASWELLX:
        case BROADWELL:
        case BDX_DE:
        case BDX:
        PCM_SKL_PATH_CASES
        case SKX:
            return 4;
        case KNL:
            return 2;
        case ICX:
            return 5;
        }
        if (isAtom())
        {
            return 2;
        }
        std::cerr << "MaxIPC is not defined for your cpu model " << cpu_model << '\n';
        assert (0);
        return 0;
    }

    uint64 getPCUFrequency() const
    {
        switch (cpu_model)
        {
        case JAKETOWN:
        case IVYTOWN:
            return 800000000ULL;  // 800 MHz
        case HASWELLX:
        case BDX_DE:
        case BDX:
        case KNL:
            return 1000000000ULL; // 1 GHz
        case SKX:
        case ICX:
        case SNOWRIDGE:
            return 1100000000ULL; // 1.1 GHz
        }
        return 0;
    }

    bool isServerCPU() const
    {
        switch (cpu_model)
        {
        case NEHALEM_EP:
        case NEHALEM_EX:
        case WESTMERE_EP:
        case WESTMERE_EX:
        case JAKETOWN:
        case IVYTOWN:
        case HASWELLX:
        case BDX:
        case BDX_DE:
        case SKX:
        case ICX:
        case SNOWRIDGE:
        case KNL:
            return true;
        default:
            return false;
        };
    }

    bool isClientCPU() const
    {
        return !isServerCPU();
    }
    uint64 getTickCount(uint64 multiplier = 1000 /* ms */, uint32 core = 0);

    uint64 getInvariantTSC_Fast(uint32 core = 0);

    uint64 getUncoreClocks(const uint32 socket_);

    uint64 getQPILinkSpeed(uint32 socketNr, uint32 linkNr) const
    {
        return hasPCICFGUncore() ? server_pcicfg_uncore[socketNr]->getQPILinkSpeed(linkNr) : max_qpi_speed;
    }

    double getJoulesPerEnergyUnit() const { return joulesPerEnergyUnit; }

    int32 getPackageThermalSpecPower() const { return pkgThermalSpecPower; }

    int32 getPackageMinimumPower() const { return pkgMinimumPower; }

    int32 getPackageMaximumPower() const { return pkgMaximumPower; }

    #ifndef NO_WINRING // In cases where loading the WinRing0 driver is not desirable as a fallback to MSR.sys, add -DNO_WINRING to compile command to remove ability to load driver
    static bool initWinRing0Lib();
    #endif // NO_WINRING

    inline void disableJKTWorkaround() { disable_JKT_workaround = true; }

    enum PCIeEventCode
    {
        // PCIe read events (PCI devices reading from memory - application writes to disk/network/PCIe device)
        PCIeRdCur = 0x19E, // PCIe read current (full cache line)
        PCIeNSRd = 0x1E4,  // PCIe non-snoop read (full cache line)
        // PCIe write events (PCI devices writing to memory - application reads from disk/network/PCIe device)
        PCIeWiLF = 0x194,  // PCIe Write (non-allocating) (full cache line)
        PCIeItoM = 0x19C,  // PCIe Write (allocating) (full cache line)
        PCIeNSWr = 0x1E5,  // PCIe Non-snoop write (partial cache line)
        PCIeNSWrF = 0x1E6, // PCIe Non-snoop write (full cache line)
        // events shared by CPU and IO
        RFO = 0x180,       // Demand Data RFO; share the same code for CPU, use tid to filter PCIe only traffic
        CRd = 0x181,       // Demand Code Read
        DRd = 0x182,       // Demand Data Read
        PRd = 0x187,       // Partial Reads (UC) (MMIO Read)
        WiL = 0x18F,       // Write Invalidate Line - partial (MMIO write), PL: Not documented in HSX/IVT
        ItoM = 0x1C8,      // Request Invalidate Line; share the same code for CPU, use tid to filter PCIe only traffic

        SKX_RFO = 0x200,
        SKX_CRd = 0x201,
        SKX_DRd = 0x202,
        SKX_PRd = 0x207,
        SKX_WiL = 0x20F,
        SKX_RdCur = 0x21E,
        SKX_ItoM = 0x248,
    };

    enum ChaPipelineQueue
    {
        None,
        IRQ,
        PRQ,
    };

    enum CBoEventTid
    {
        RFOtid = 0x3E,
        ItoMtid = 0x3E,
    };

    void programPCIeEventGroup(eventGroup_t &eventGroup);
    uint64 getPCIeCounterData(const uint32 socket_, const uint32 ctr_);

    void programCbo(const uint64 * events, const uint32 opCode = 0, const uint32 nc_ = 0, const uint32 llc_lookup_tid_filter = 0, const uint32 loc = 1, const uint32 rem = 1);

    void programCboRaw(const uint64* events, const uint64 filter0, const uint64 filter1);

    PCIeCounterState getPCIeCounterState(const uint32 socket_, const uint32 ctr_ = 0);

    void programIIOCounters(uint64 rawEvents[4], int IIOStack = -1);

    void programIRPCounters(uint64 rawEvents[4], int IIOStack = -1);

    IIOCounterState getIIOCounterState(int socket, int IIOStack, int counter);

    void getIIOCounterStates(int socket, int IIOStack, IIOCounterState * result);

    uint64 extractCoreGenCounterValue(uint64 val);
    uint64 extractCoreFixedCounterValue(uint64 val);
    uint64 extractUncoreGenCounterValue(uint64 val);
    uint64 extractUncoreFixedCounterValue(uint64 val);
    uint64 extractQOSMonitoring(uint64 val);

    const char * getUArchCodename(const int32 cpu_model_ = -1) const;

    static std::string getCPUBrandString();
    std::string getCPUFamilyModelString();


    void enableForceRTMAbortMode(const bool silent = false);

    bool isForceRTMAbortModeEnabled() const;

    void disableForceRTMAbortMode(const bool silent = false);

    bool isForceRTMAbortModeAvailable() const;

    int64 getCPUMicrocodeLevel() const { return cpu_microcode_level; }

    static bool isAtom(const int32 cpu_model_)
    {
        return cpu_model_ == ATOM
            || cpu_model_ == ATOM_2
            || cpu_model_ == CENTERTON
            || cpu_model_ == BAYTRAIL
            || cpu_model_ == AVOTON
            || cpu_model_ == CHERRYTRAIL
            || cpu_model_ == APOLLO_LAKE
            || cpu_model_ == DENVERTON
            // || cpu_model_ == SNOWRIDGE do not use Atom code for SNOWRIDGE
            ;
    }

    bool isAtom() const
    {
        return isAtom(cpu_model);
    }

    bool packageEnergyMetricsAvailable() const
    {
        return (
                    cpu_model == PCM::JAKETOWN
                 || cpu_model == PCM::IVYTOWN
                 || cpu_model == PCM::SANDY_BRIDGE
                 || cpu_model == PCM::IVY_BRIDGE
                 || cpu_model == PCM::HASWELL
                 || cpu_model == PCM::AVOTON
                 || cpu_model == PCM::CHERRYTRAIL
                 || cpu_model == PCM::BAYTRAIL
                 || cpu_model == PCM::APOLLO_LAKE
                 || cpu_model == PCM::DENVERTON
                 || cpu_model == PCM::SNOWRIDGE
                 || cpu_model == PCM::HASWELLX
                 || cpu_model == PCM::BROADWELL
                 || cpu_model == PCM::BDX_DE
                 || cpu_model == PCM::BDX
                 || cpu_model == PCM::KNL
                 || useSKLPath()
                 || cpu_model == PCM::SKX
                 || cpu_model == PCM::ICX
                 || cpu_model == PCM::ADL
               );
    }

    bool dramEnergyMetricsAvailable() const
    {
        return (
             cpu_model == PCM::JAKETOWN
          || cpu_model == PCM::IVYTOWN
          || cpu_model == PCM::HASWELLX
          || cpu_model == PCM::BDX_DE
          || cpu_model == PCM::BDX
          || cpu_model == PCM::KNL
          || cpu_model == PCM::SKX
          || cpu_model == PCM::ICX
          );
    }

    bool packageThermalMetricsAvailable() const
    {
        return packageEnergyMetricsAvailable();
    }

    bool outgoingQPITrafficMetricsAvailable() const
    {
        return getQPILinksPerSocket() > 0 &&
            (
                cpu_model == PCM::NEHALEM_EX
            ||  cpu_model == PCM::WESTMERE_EX
            ||  cpu_model == PCM::JAKETOWN
            ||  cpu_model == PCM::IVYTOWN
            ||  cpu_model == PCM::HASWELLX
            ||  cpu_model == PCM::BDX
            ||  cpu_model == PCM::SKX
            ||  cpu_model == PCM::ICX
            );
    }

    bool incomingQPITrafficMetricsAvailable() const
    {
        return getQPILinksPerSocket() > 0 &&
            (
                cpu_model == PCM::NEHALEM_EX
            ||  cpu_model == PCM::WESTMERE_EX
            ||  cpu_model == PCM::JAKETOWN
            ||  cpu_model == PCM::IVYTOWN
            || (cpu_model == PCM::SKX && cpu_stepping > 1)
            ||  cpu_model == PCM::ICX
               );
    }

    bool localMemoryRequestRatioMetricAvailable() const
    {
        return cpu_model == PCM::HASWELLX
            || cpu_model == PCM::BDX
            || cpu_model == PCM::SKX
            || cpu_model == PCM::ICX
            ;
    }

    bool qpiUtilizationMetricsAvailable() const
    {
        return outgoingQPITrafficMetricsAvailable();
    }

    bool memoryTrafficMetricsAvailable() const
    {
        return (!(isAtom() || cpu_model == PCM::CLARKDALE))
               ;
    }

    bool MCDRAMmemoryTrafficMetricsAvailable() const
    {
        return (cpu_model == PCM::KNL);
    }

    bool memoryIOTrafficMetricAvailable() const
    {
        if (cpu_model == TGL) return false;
        return (
            cpu_model == PCM::SANDY_BRIDGE
            || cpu_model == PCM::IVY_BRIDGE
            || cpu_model == PCM::HASWELL
            || cpu_model == PCM::BROADWELL
            || useSKLPath()
            );
    }

    bool IIOEventsAvailable() const
    {
        return (
               cpu_model == PCM::SKX
            || cpu_model == PCM::ICX
        || cpu_model  == PCM::SNOWRIDGE
        );
    }

    bool uncoreFrequencyMetricAvailable() const
    {
        return MSR.empty() == false && uboxPMUs.size() == getNumSockets() && getNumCores() == getNumOnlineCores();
    }

    bool LatencyMetricsAvailable() const
    {
        return (
            cpu_model == PCM::HASWELLX
            || cpu_model == PCM::BDX
            || cpu_model == PCM::SKX
            || cpu_model == PCM::ICX
            || useSKLPath()
            );
    }

    bool DDRLatencyMetricsAvailable() const
    {
        return (
            cpu_model == PCM::SKX
            || cpu_model == PCM::ICX
            );
    }

    bool PMMTrafficMetricsAvailable() const
    {
        return (
            isCLX()
                    ||  isCPX()
                     || cpu_model == PCM::ICX
                     || cpu_model == PCM::SNOWRIDGE
        );
    }

    bool LLCReadMissLatencyMetricsAvailable() const
    {
        return (
               HASWELLX == cpu_model
            || BDX_DE == cpu_model
            || BDX == cpu_model
            || isCLX()
            || isCPX()
#ifdef PCM_ENABLE_LLCRDLAT_SKX_MP
            || SKX == cpu_model
#else
            || ((SKX == cpu_model) && (num_sockets == 1))
#endif
            || ICX == cpu_model
            || SNOWRIDGE == cpu_model
               );
    }

    bool hasBecktonUncore() const
    {
        return (
            cpu_model == PCM::NEHALEM_EX
            || cpu_model == PCM::WESTMERE_EX
            );
    }
    bool hasPCICFGUncore() const // has PCICFG uncore PMON
    {
        return (
            cpu_model == PCM::JAKETOWN
            || cpu_model == PCM::SNOWRIDGE
            || cpu_model == PCM::IVYTOWN
            || cpu_model == PCM::HASWELLX
            || cpu_model == PCM::BDX_DE
            || cpu_model == PCM::SKX
            || cpu_model == PCM::ICX
            || cpu_model == PCM::BDX
            || cpu_model == PCM::KNL
            );
    }

    bool isSkxCompatible() const
    {
        return (
            cpu_model == PCM::SKX
               );
    }

    static bool hasUPI(const int32 cpu_model_) // Intel(r) Ultra Path Interconnect
    {
        return (
            cpu_model_ == PCM::SKX
         || cpu_model_ == PCM::ICX
               );
    }

    bool hasUPI() const
    {
        return hasUPI(cpu_model);
    }

    const char * xPI() const
    {
        if (hasUPI())
            return "UPI";

        return "QPI";
    }

    bool hasCHA() const
    {
        return (
            cpu_model == PCM::SKX
         || cpu_model == PCM::ICX
               );
    }

    bool supportsHLE() const;
    bool supportsRTM() const;
    bool supportsRDTSCP() const;

    bool useSkylakeEvents() const
    {
        return    useSKLPath()
               || PCM::SKX == cpu_model
               || PCM::ICX == cpu_model
               ;
    }

    bool hasClientMCCounters() const
    {
        return  cpu_model == SANDY_BRIDGE
            || cpu_model == IVY_BRIDGE
            || cpu_model == HASWELL
            || cpu_model == BROADWELL
            || useSKLPath()
            ;
    }

    static double getBytesPerFlit(int32 cpu_model_)
    {
        if (hasUPI(cpu_model_))
        {
            // 172 bits per UPI flit
            return 172./8.;
        }
        // 8 bytes per QPI flit
        return 8.;
    }

    double getBytesPerFlit() const
    {
        return getBytesPerFlit(cpu_model);
    }

    static double getDataBytesPerFlit(int32 cpu_model_)
    {
        if (hasUPI(cpu_model_))
        {
            // 9 UPI flits to transfer 64 bytes
            return 64./9.;
        }
        // 8 bytes per QPI flit
        return 8.;
    }

    double getDataBytesPerFlit() const
    {
        return getDataBytesPerFlit(cpu_model);
    }

    static double getFlitsPerLinkCycle(int32 cpu_model_)
    {
        if (hasUPI(cpu_model_))
        {
            // 5 UPI flits sent every 6 link cycles
            return 5./6.;
        }
        return 2.;
    }

    static double getBytesPerLinkCycle(int32 cpu_model_)
    {
        return getBytesPerFlit(cpu_model_) * getFlitsPerLinkCycle(cpu_model_);
    }

    double getBytesPerLinkCycle() const
    {
        return getBytesPerLinkCycle(cpu_model);
    }

    static double getLinkTransfersPerLinkCycle()
    {
        return 8.;
    }

    double getBytesPerLinkTransfer() const
    {
        return getBytesPerLinkCycle() / getLinkTransfersPerLinkCycle();
    }

    void setupCustomCoreEventsForNuma(PCM::ExtendedCustomCoreEventDescription& conf) const;

    #define PCM_GENERATE_METRIC_AVAILABLE_FUNCTION(m) bool is##m() const { return m; }

    PCM_GENERATE_METRIC_AVAILABLE_FUNCTION(L2CacheHitRatioAvailable)
    PCM_GENERATE_METRIC_AVAILABLE_FUNCTION(L3CacheHitRatioAvailable)
    PCM_GENERATE_METRIC_AVAILABLE_FUNCTION(L3CacheMissesAvailable)
    PCM_GENERATE_METRIC_AVAILABLE_FUNCTION(L2CacheMissesAvailable)
    PCM_GENERATE_METRIC_AVAILABLE_FUNCTION(L2CacheHitsAvailable)
    PCM_GENERATE_METRIC_AVAILABLE_FUNCTION(L3CacheHitsNoSnoopAvailable)
    PCM_GENERATE_METRIC_AVAILABLE_FUNCTION(L3CacheHitsSnoopAvailable)
    PCM_GENERATE_METRIC_AVAILABLE_FUNCTION(L3CacheHitsAvailable)

    #undef PCM_GEN_METRIC_AVAILABLE_FUNCTION

    bool isActiveRelativeFrequencyAvailable() const
    {
        return !isAtom();
    }

    ~PCM();
};

class BasicCounterState
{
    friend class PCM;
    friend class JSONPrinter;
    template <class CounterStateType>
    friend double getExecUsage(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend double getIPC(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend double getAverageFrequency(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend double getAverageFrequencyFromClocks(const int64 clocks, const CounterStateType& before, const CounterStateType& after);
    template <class CounterStateType>
    friend double getActiveAverageFrequency(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend double getRelativeFrequency(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend double getActiveRelativeFrequency(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend double getL2CacheHitRatio(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend double getL3CacheHitRatio(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getL3CacheMisses(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getL2CacheMisses(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getL2CacheHits(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getL3CacheHitsNoSnoop(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getL3CacheHitsSnoop(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getL3CacheHits(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getL3CacheOccupancy(const CounterStateType & now);
    template <class CounterStateType>
    friend uint64 getLocalMemoryBW(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getRemoteMemoryBW(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getCycles(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getInstructionsRetired(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getCycles(const CounterStateType & now);
    template <class CounterStateType>
    friend uint64 getInstructionsRetired(const CounterStateType & now);
    template <class CounterStateType>
    friend uint64 getNumberOfCustomEvents(int32 eventCounterNr, const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getInvariantTSC(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getRefCycles(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend double getCoreCStateResidency(int state, const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getCoreCStateResidency(int state, const CounterStateType& now);
    template <class CounterStateType>
    friend uint64 getSMICount(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getAllSlotsRaw(const CounterStateType& before, const CounterStateType& after);
    template <class CounterStateType>
    friend uint64 getAllSlots(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend double getBackendBound(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend double getFrontendBound(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend double getBadSpeculation(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend double getRetiring(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getMSREvent(const uint64 & index, const PCM::MSRType & type, const CounterStateType& before, const CounterStateType& after);
protected:
    checked_uint64 InstRetiredAny{};
    checked_uint64 CpuClkUnhaltedThread{};
    checked_uint64 CpuClkUnhaltedRef{};
    checked_uint64 Event[PERF_MAX_CUSTOM_COUNTERS];
    enum
    {
               L3MissPos = 0,
          ArchLLCMissPos = 0,
        L3UnsharedHitPos = 1,
           ArchLLCRefPos = 1,
             SKLL3HitPos = 1,
               L2HitMPos = 2,
            SKLL2MissPos = 2,
                L2HitPos = 3
    };
    uint64 InvariantTSC; // invariant time stamp counter
    uint64 CStateResidency[PCM::MAX_C_STATE + 1];
    int32 ThermalHeadroom;
    uint64 L3Occupancy;
    uint64 MemoryBWLocal;
    uint64 MemoryBWTotal;
    uint64 SMICount;
    uint64 FrontendBoundSlots, BadSpeculationSlots, BackendBoundSlots, RetiringSlots, AllSlotsRaw;
    std::unordered_map<uint64, uint64> MSRValues;

public:
    BasicCounterState() :
        InvariantTSC(0),
        ThermalHeadroom(PCM_INVALID_THERMAL_HEADROOM),
        L3Occupancy(0),
        MemoryBWLocal(0),
        MemoryBWTotal(0),
        SMICount(0),
    FrontendBoundSlots(0),
    BadSpeculationSlots(0),
    BackendBoundSlots(0),
    RetiringSlots(0),
    AllSlotsRaw(0)
    {
        std::fill(CStateResidency, CStateResidency + PCM::MAX_C_STATE + 1, 0);
    }
    virtual ~BasicCounterState() { }

    BasicCounterState( const BasicCounterState& ) = default;
    BasicCounterState( BasicCounterState&& ) = default;
    BasicCounterState & operator = ( BasicCounterState&& ) = default;

    BasicCounterState & operator += (const BasicCounterState & o)
    {
        InstRetiredAny += o.InstRetiredAny;
        CpuClkUnhaltedThread += o.CpuClkUnhaltedThread;
        CpuClkUnhaltedRef += o.CpuClkUnhaltedRef;
        for (int i = 0; i < PERF_MAX_CUSTOM_COUNTERS; ++i)
        {
            Event[i] += o.Event[i];
        }
        InvariantTSC += o.InvariantTSC;
        for (int i = 0; i <= (int)PCM::MAX_C_STATE; ++i)
            CStateResidency[i] += o.CStateResidency[i];
        // ThermalHeadroom is not accumulative
        L3Occupancy += o.L3Occupancy;
        MemoryBWLocal += o.MemoryBWLocal;
        MemoryBWTotal += o.MemoryBWTotal;
        SMICount += o.SMICount;
        // std::cout << "before PCM debug aggregate "<< FrontendBoundSlots << " " << BadSpeculationSlots << " " << BackendBoundSlots << " " <<RetiringSlots << std::endl;
        BasicCounterState old = *this;
        FrontendBoundSlots += o.FrontendBoundSlots;
        BadSpeculationSlots += o.BadSpeculationSlots;
        BackendBoundSlots += o.BackendBoundSlots;
        RetiringSlots += o.RetiringSlots;
        AllSlotsRaw += o.AllSlotsRaw;
        //std::cout << "after PCM debug aggregate "<< FrontendBoundSlots << " " << BadSpeculationSlots << " " << BackendBoundSlots << " " <<RetiringSlots << std::endl;
        assert(FrontendBoundSlots >= old.FrontendBoundSlots);
        assert(BadSpeculationSlots >= old.BadSpeculationSlots);
        assert(BackendBoundSlots >= old.BackendBoundSlots);
        assert(RetiringSlots >= old.RetiringSlots);
        return *this;
    }

    void readAndAggregate(std::shared_ptr<SafeMsrHandle>);
    void readAndAggregateTSC(std::shared_ptr<SafeMsrHandle>);

    int32 getThermalHeadroom() const { return ThermalHeadroom; }
};

inline uint64 RDTSC()
{
        uint64 result = 0;
#ifdef _MSC_VER
        // Windows
        #if _MSC_VER>= 1600
        result = static_cast<uint64>(__rdtsc());
        #endif
#else
        // Linux
        uint32 high = 0, low = 0;
        asm volatile("rdtsc" : "=a" (low), "=d" (high));
        result = low + (uint64(high)<<32ULL);
#endif
        return result;

}

inline uint64 RDTSCP()
{
    uint64 result = 0;
#ifdef _MSC_VER
    // Windows
    #if _MSC_VER>= 1600
    unsigned int Aux;
    result = __rdtscp(&Aux);
    #endif
#else
    // Linux and OS X
    uint32 high = 0, low = 0;
    asm volatile (
       "rdtscp\n\t"
       "mov %%edx, %0\n\t"
       "mov %%eax, %1\n\t":
       "=r" (high), "=r" (low) :: "%rax", "%rcx", "%rdx");
    result = low + (uint64(high)<<32ULL);
#endif
    return result;
}

template <class CounterStateType>
int32 getThermalHeadroom(const CounterStateType & /* before */, const CounterStateType & after)
{
    return after.getThermalHeadroom();
}

template <class CounterStateType>
double getNormalizedQPIL0pTxCycles(uint32 port, const CounterStateType & before, const CounterStateType & after)
{
    return double(getQPIL0pTxCycles(port, before, after)) / double(getQPIClocks(port, before, after));
}

template <class CounterStateType>
double getNormalizedQPIL1Cycles(uint32 port, const CounterStateType & before, const CounterStateType & after)
{
    return double(getQPIL1Cycles(port, before, after)) / double(getQPIClocks(port, before, after));
}

template <class CounterStateType>
uint64 getDRAMClocks(uint32 channel, const CounterStateType & before, const CounterStateType & after)
{
    const auto clk = after.DRAMClocks[channel] - before.DRAMClocks[channel];
    const auto cpu_model = PCM::getInstance()->getCPUModel();
    if (cpu_model == PCM::ICX || cpu_model == PCM::SNOWRIDGE)
    {
        return 2 * clk;
    }
    return clk;
}

template <class CounterStateType>
uint64 getMCDRAMClocks(uint32 channel, const CounterStateType & before, const CounterStateType & after)
{
    return after.MCDRAMClocks[channel] - before.MCDRAMClocks[channel];
}


template <class CounterStateType>
uint64 getMCCounter(uint32 channel, uint32 counter, const CounterStateType & before, const CounterStateType & after)
{
    return after.MCCounter[channel][counter] - before.MCCounter[channel][counter];
}

template <class CounterStateType>
uint64 getM3UPICounter(uint32 port, uint32 counter, const CounterStateType& before, const CounterStateType& after)
{
    return after.M3UPICounter[port][counter] - before.M3UPICounter[port][counter];
}

template <class CounterStateType>
uint64 getCBOCounter(uint32 cbo, uint32 counter, const CounterStateType& before, const CounterStateType& after)
{
    return after.CBOCounter[cbo][counter] - before.CBOCounter[cbo][counter];
}

template <class CounterStateType>
uint64 getUBOXCounter(uint32 counter, const CounterStateType& before, const CounterStateType& after)
{
    return after.UBOXCounter[counter] - before.UBOXCounter[counter];
}

template <class CounterStateType>
uint64 getIIOCounter(uint32 stack, uint32 counter, const CounterStateType& before, const CounterStateType& after)
{
    return after.IIOCounter[stack][counter] - before.IIOCounter[stack][counter];
}

template <class CounterStateType>
uint64 getIRPCounter(uint32 stack, uint32 counter, const CounterStateType& before, const CounterStateType& after)
{
    return after.IRPCounter[stack][counter] - before.IRPCounter[stack][counter];
}

template <class CounterStateType>
uint64 getXPICounter(uint32 port, uint32 counter, const CounterStateType& before, const CounterStateType& after)
{
    return after.xPICounter[port][counter] - before.xPICounter[port][counter];
}

template <class CounterStateType>
uint64 getM2MCounter(uint32 controller, uint32 counter, const CounterStateType & before, const CounterStateType & after)
{
    return after.M2MCounter[controller][counter] - before.M2MCounter[controller][counter];
}


template <class CounterStateType>
uint64 getEDCCounter(uint32 channel, uint32 counter, const CounterStateType & before, const CounterStateType & after)
{
    if (PCM::getInstance()->MCDRAMmemoryTrafficMetricsAvailable())
        return after.EDCCounter[channel][counter] - before.EDCCounter[channel][counter];
    return 0ULL;
}

template <class CounterStateType>
uint64 getPCUCounter(uint32 counter, const CounterStateType & before, const CounterStateType & after)
{
    return after.PCUCounter[counter] - before.PCUCounter[counter];
}

template <class CounterStateType>
uint64 getPCUClocks(const CounterStateType & before, const CounterStateType & after)
{
    return getPCUCounter(0, before, after);
}

template <class CounterStateType>
uint64 getConsumedEnergy(const CounterStateType & before, const CounterStateType & after)
{
    return after.PackageEnergyStatus - before.PackageEnergyStatus;
}

template <class CounterStateType>
uint64 getDRAMConsumedEnergy(const CounterStateType & before, const CounterStateType & after)
{
    return after.DRAMEnergyStatus - before.DRAMEnergyStatus;
}


template <class CounterStateType>
int64 getFreeRunningCounter(const typename CounterStateType::FreeRunningCounterID & counter, const CounterStateType & before, const CounterStateType & after)
{
    const auto beforeIt = before.freeRunningCounter.find(counter);
    const auto afterIt = after.freeRunningCounter.find(counter);
    if (beforeIt != before.freeRunningCounter.end() &&
        afterIt != after.freeRunningCounter.end())
    {
        return afterIt->second - beforeIt->second;
    }
    return -1;
}


template <class CounterStateType>
uint64 getUncoreClocks(const CounterStateType& before, const CounterStateType& after)
{
    return after.UncClocks - before.UncClocks;
}

template <class CounterStateType>
double getConsumedJoules(const CounterStateType & before, const CounterStateType & after)
{
    PCM * m = PCM::getInstance();
    if (!m) return -1.;

    return double(getConsumedEnergy(before, after)) * m->getJoulesPerEnergyUnit();
}

template <class CounterStateType>
double getDRAMConsumedJoules(const CounterStateType & before, const CounterStateType & after)
{
    PCM * m = PCM::getInstance();
    if (!m) return -1.;
    double dram_joules_per_energy_unit = 0.;
    const auto cpu_model = m->getCPUModel();

    if (PCM::HASWELLX == cpu_model
        || PCM::BDX_DE == cpu_model
        || PCM::BDX == cpu_model
        || PCM::SKX == cpu_model
        || PCM::ICX == cpu_model
        || PCM::KNL == cpu_model
        ) {
/* as described in sections 5.3.2 (DRAM_POWER_INFO) and 5.3.3 (DRAM_ENERGY_STATUS) of
 * Volume 2 (Registers) of
 * Intel Xeon E5-1600 v3 and Intel Xeon E5-2600 v3 (Haswell-EP) Datasheet (Ref 330784-001, Sept.2014)
 * ENERGY_UNIT for DRAM domain is fixed to 15.3 uJ for server HSX, BDW and KNL processors.
 */
        dram_joules_per_energy_unit = 0.0000153;
    } else {
/* for all other processors (including Haswell client/mobile SKUs) the ENERGY_UNIT for DRAM domain
 * should be read from PACKAGE_POWER_SKU register (usually value around ~61uJ)
 */
        dram_joules_per_energy_unit = m->getJoulesPerEnergyUnit();
    }
    return double(getDRAMConsumedEnergy(before, after)) * dram_joules_per_energy_unit;
}

class UncoreCounterState
{
    friend class PCM;
    friend class JSONPrinter;
    template <class CounterStateType>
    friend uint64 getBytesReadFromMC(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getBytesWrittenToMC(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getBytesReadFromPMM(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getBytesWrittenToPMM(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getBytesReadFromEDC(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getBytesWrittenToEDC(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getGTRequestBytesFromMC(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getIARequestBytesFromMC(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getIORequestBytesFromMC(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getConsumedEnergy(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getDRAMConsumedEnergy(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getUncoreClocks(const CounterStateType& before, const CounterStateType& after);
    template <class CounterStateType>
    friend double getPackageCStateResidency(int state, const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getPackageCStateResidency(int state, const CounterStateType& now);
    template <class CounterStateType>
    friend double getLLCReadMissLatency(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend double getLocalMemoryRequestRatio(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend double getAverageUncoreFrequency(const CounterStateType& before, const CounterStateType& after);
    template <class CounterStateType>
    friend double getAverageFrequencyFromClocks(const int64 clocks, const CounterStateType& before, const CounterStateType& after);

protected:
    uint64 UncMCFullWrites;
    uint64 UncMCNormalReads;
    uint64 UncHARequests;
    uint64 UncHALocalRequests;
    uint64 UncPMMWrites;
    uint64 UncPMMReads;
    uint64 UncEDCFullWrites;
    uint64 UncEDCNormalReads;
    uint64 UncMCGTRequests;
    uint64 UncMCIARequests;
    uint64 UncMCIORequests;
    uint64 PackageEnergyStatus;
    uint64 DRAMEnergyStatus;
    uint64 TOROccupancyIAMiss;
    uint64 TORInsertsIAMiss;
    uint64 UncClocks;
    uint64 CStateResidency[PCM::MAX_C_STATE + 1];
    void readAndAggregate(std::shared_ptr<SafeMsrHandle>);

public:
    UncoreCounterState() :
        UncMCFullWrites(0),
        UncMCNormalReads(0),
        UncHARequests(0),
        UncHALocalRequests(0),
        UncPMMWrites(0),
        UncPMMReads(0),
        UncEDCFullWrites(0),
        UncEDCNormalReads(0),
        UncMCGTRequests(0),
        UncMCIARequests(0),
        UncMCIORequests(0),
        PackageEnergyStatus(0),
        DRAMEnergyStatus(0),
        TOROccupancyIAMiss(0),
        TORInsertsIAMiss(0),
        UncClocks(0)
    {
        std::fill(CStateResidency, CStateResidency + PCM::MAX_C_STATE + 1, 0);
    }
    virtual ~UncoreCounterState() { }

    UncoreCounterState( const UncoreCounterState& ) = default;
    UncoreCounterState( UncoreCounterState&& ) = default;
    UncoreCounterState & operator = ( UncoreCounterState&& ) = default;

    UncoreCounterState & operator += (const UncoreCounterState & o)
    {
        UncMCFullWrites += o.UncMCFullWrites;
        UncMCNormalReads += o.UncMCNormalReads;
        UncHARequests += o.UncHARequests;
        UncHALocalRequests += o.UncHALocalRequests;
        UncPMMReads += o.UncPMMReads;
        UncPMMWrites += o.UncPMMWrites;
        UncEDCFullWrites += o.UncEDCFullWrites;
        UncEDCNormalReads += o.UncEDCNormalReads;
        UncMCGTRequests += o.UncMCGTRequests;
        UncMCIARequests += o.UncMCIARequests;
        UncMCIORequests += o.UncMCIORequests;
        PackageEnergyStatus += o.PackageEnergyStatus;
        DRAMEnergyStatus += o.DRAMEnergyStatus;
        TOROccupancyIAMiss += o.TOROccupancyIAMiss;
        TORInsertsIAMiss += o.TORInsertsIAMiss;
        UncClocks += o.UncClocks;
        for (int i = 0; i <= (int)PCM::MAX_C_STATE; ++i)
            CStateResidency[i] += o.CStateResidency[i];
        return *this;
    }
};


class ServerUncoreCounterState : public UncoreCounterState
{
public:
    enum {
        maxControllers = 4,
        maxChannels = 12,
        maxXPILinks = 6,
        maxCBOs = 128,
        maxIIOStacks = 16,
        maxCounters = 4
    };
    enum EventPosition
    {
        xPI_TxL0P_POWER_CYCLES = 0,
        xPI_L1_POWER_CYCLES = 2,
        xPI_CLOCKTICKS = 3
    };
    enum FreeRunningCounterID
    {
        ImcReads,
        ImcWrites,
        PMMReads,
        PMMWrites
    };
private:
    std::array<std::array<uint64, maxCounters>, maxXPILinks> xPICounter;
    std::array<std::array<uint64, maxCounters>, maxXPILinks> M3UPICounter;
    std::array<std::array<uint64, maxCounters>, maxCBOs> CBOCounter;
    std::array<std::array<uint64, maxCounters>, maxIIOStacks> IIOCounter;
    std::array<std::array<uint64, maxCounters>, maxIIOStacks> IRPCounter;
    std::array<uint64, maxCounters> UBOXCounter;
    std::array<uint64, maxChannels> DRAMClocks;
    std::array<uint64, maxChannels> MCDRAMClocks;
    std::array<std::array<uint64, maxCounters>, maxChannels> MCCounter; // channel X counter
    std::array<std::array<uint64, maxCounters>, maxControllers> M2MCounter; // M2M/iMC boxes x counter
    std::array<std::array<uint64, maxCounters>, maxChannels> EDCCounter; // EDC controller X counter
    std::array<uint64, maxCounters> PCUCounter;
    std::unordered_map<int, uint64> freeRunningCounter;
    int32 PackageThermalHeadroom;
    uint64 InvariantTSC;    // invariant time stamp counter
    friend class PCM;
    template <class CounterStateType>
    friend uint64 getDRAMClocks(uint32 channel, const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getMCDRAMClocks(uint32 channel, const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getMCCounter(uint32 channel, uint32 counter, const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getM3UPICounter(uint32 port, uint32 counter, const CounterStateType& before, const CounterStateType& after);
    template <class CounterStateType>
    friend uint64 getCBOCounter(uint32 cbo, uint32 counter, const CounterStateType& before, const CounterStateType& after);
    template <class CounterStateType>
    friend uint64 getUBOXCounter(uint32 counter, const CounterStateType& before, const CounterStateType& after);
    template <class CounterStateType>
    friend uint64 getIIOCounter(uint32 stack, uint32 counter, const CounterStateType& before, const CounterStateType& after);
    template <class CounterStateType>
    friend uint64 getIRPCounter(uint32 stack, uint32 counter, const CounterStateType& before, const CounterStateType& after);
    template <class CounterStateType>
    friend uint64 getXPICounter(uint32 port, uint32 counter, const CounterStateType& before, const CounterStateType& after);
    template <class CounterStateType>
    friend uint64 getM2MCounter(uint32 controller, uint32 counter, const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getEDCCounter(uint32 channel, uint32 counter, const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getPCUCounter(uint32 counter, const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getConsumedEnergy(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getDRAMConsumedEnergy(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend uint64 getInvariantTSC(const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend int64 getFreeRunningCounter(const typename CounterStateType::FreeRunningCounterID &, const CounterStateType & before, const CounterStateType & after);
    template <class CounterStateType>
    friend double getAverageFrequencyFromClocks(const int64 clocks, const CounterStateType& before, const CounterStateType& after);

public:
    int32 getPackageThermalHeadroom() const { return PackageThermalHeadroom; }
    ServerUncoreCounterState() :
        xPICounter{{}},
        M3UPICounter{{}},
        CBOCounter{{}},
        IIOCounter{{}},
        IRPCounter{{}},
        UBOXCounter{{}},
        DRAMClocks{{}},
        MCDRAMClocks{{}},
        MCCounter{{}},
        M2MCounter{{}},
        EDCCounter{{}},
        PCUCounter{{}},
        PackageThermalHeadroom(0),
        InvariantTSC(0)
    {
    }
};

template <class CounterStateType>
uint64 getQPIClocks(uint32 port, const CounterStateType& before, const CounterStateType& after)
{
    return getXPICounter(port, ServerUncoreCounterState::EventPosition::xPI_CLOCKTICKS, before, after);
}

template <class CounterStateType>
uint64 getQPIL0pTxCycles(uint32 port, const CounterStateType& before, const CounterStateType& after)
{
    return getXPICounter(port, ServerUncoreCounterState::EventPosition::xPI_TxL0P_POWER_CYCLES, before, after);
}

template <class CounterStateType>
uint64 getQPIL1Cycles(uint32 port, const CounterStateType& before, const CounterStateType& after)
{
    return getXPICounter(port, ServerUncoreCounterState::EventPosition::xPI_L1_POWER_CYCLES, before, after);
}

class CoreCounterState : public BasicCounterState
{
    friend class PCM;

public:
    CoreCounterState() = default;
    CoreCounterState( const CoreCounterState& ) = default;
    CoreCounterState( CoreCounterState&& ) = default;
    CoreCounterState & operator= ( CoreCounterState&& ) = default;
    virtual ~ CoreCounterState() {}
};

class SocketCounterState : public BasicCounterState, public UncoreCounterState
{
    friend class PCM;

protected:
    void readAndAggregate(std::shared_ptr<SafeMsrHandle> handle)
    {
        BasicCounterState::readAndAggregate(handle);
        UncoreCounterState::readAndAggregate(handle);
    }

public:
    SocketCounterState& operator += ( const BasicCounterState& ccs )
    {
        BasicCounterState::operator += ( ccs );

        return *this;
    }

    SocketCounterState& operator += ( const UncoreCounterState& ucs )
    {
        UncoreCounterState::operator += ( ucs );

        return *this;
    }

    SocketCounterState() = default;
    SocketCounterState( const SocketCounterState& ) = default;
    SocketCounterState( SocketCounterState&& ) = default;
    SocketCounterState & operator = ( SocketCounterState&& ) = default;

    SocketCounterState & operator = ( UncoreCounterState&& ucs ) {
        UncoreCounterState::operator = ( std::move(ucs) );
        return *this;
    }

    virtual ~ SocketCounterState() {}
};

class SystemCounterState : public SocketCounterState
{
    friend class PCM;

    std::vector<std::vector<uint64> > incomingQPIPackets; // each 64 byte
    std::vector<std::vector<uint64> > outgoingQPIFlits; // idle or data/non-data flits depending on the architecture
    std::vector<std::vector<uint64> > TxL0Cycles;
    uint64 uncoreTSC;

protected:
    void readAndAggregate(std::shared_ptr<SafeMsrHandle> handle)
    {
        BasicCounterState::readAndAggregate(handle);
        UncoreCounterState::readAndAggregate(handle);
    }

public:
    friend uint64 getIncomingQPILinkBytes(uint32 socketNr, uint32 linkNr, const SystemCounterState & before, const SystemCounterState & after);
    friend uint64 getIncomingQPILinkBytes(uint32 socketNr, uint32 linkNr, const SystemCounterState & now);
    friend double getOutgoingQPILinkUtilization(uint32 socketNr, uint32 linkNr, const SystemCounterState & before, const SystemCounterState & after);
    friend uint64 getOutgoingQPILinkBytes(uint32 socketNr, uint32 linkNr, const SystemCounterState & before, const SystemCounterState & after);
    friend uint64 getOutgoingQPILinkBytes(uint32 socketNr, uint32 linkNr, const SystemCounterState & now);

    SystemCounterState() :
        uncoreTSC(0)
    {
        PCM * m = PCM::getInstance();
        incomingQPIPackets.resize(m->getNumSockets(),
                                  std::vector<uint64>((uint32)m->getQPILinksPerSocket(), 0));
        outgoingQPIFlits.resize(m->getNumSockets(),
                                    std::vector<uint64>((uint32)m->getQPILinksPerSocket(), 0));
        TxL0Cycles.resize(m->getNumSockets(),
                                    std::vector<uint64>((uint32)m->getQPILinksPerSocket(), 0));
    }

    SystemCounterState( const SystemCounterState& ) = default;
    SystemCounterState( SystemCounterState&& ) = default;
    SystemCounterState & operator = ( SystemCounterState&& ) = default;

    SystemCounterState & operator += ( const SocketCounterState& scs )
    {
        BasicCounterState::operator += ( scs );
        UncoreCounterState::operator += ( scs );

        return *this;
    }

    SystemCounterState & operator += ( const UncoreCounterState& ucs )
    {
        UncoreCounterState::operator += ( ucs );

        return *this;
    }
    virtual ~ SystemCounterState() {}
};

PCM_API SystemCounterState getSystemCounterState();

PCM_API SocketCounterState getSocketCounterState(uint32 socket);

PCM_API CoreCounterState getCoreCounterState(uint32 core);


template <class CounterStateType>
double getIPC(const CounterStateType & before, const CounterStateType & after) // instructions per cycle
{
    int64 clocks = after.CpuClkUnhaltedThread - before.CpuClkUnhaltedThread;
    if (clocks != 0)
        return double(after.InstRetiredAny - before.InstRetiredAny) / double(clocks);
    return -1;
}


template <class CounterStateType>
uint64 getInstructionsRetired(const CounterStateType & before, const CounterStateType & after) // instructions
{
    return after.InstRetiredAny - before.InstRetiredAny;
}

template <class CounterStateType>
double getExecUsage(const CounterStateType & before, const CounterStateType & after) // usage
{
    int64 timer_clocks = after.InvariantTSC - before.InvariantTSC;
    if (timer_clocks != 0)
        return double(after.InstRetiredAny - before.InstRetiredAny) / double(timer_clocks);
    return -1;
}

template <class CounterStateType>
uint64 getInstructionsRetired(const CounterStateType & now) // instructions
{
    return now.InstRetiredAny.getRawData_NoOverflowProtection();
}

template <class CounterStateType>
uint64 getCycles(const CounterStateType & before, const CounterStateType & after) // clocks
{
    return after.CpuClkUnhaltedThread - before.CpuClkUnhaltedThread;
}

template <class CounterStateType>
uint64 getRefCycles(const CounterStateType & before, const CounterStateType & after) // clocks
{
    return after.CpuClkUnhaltedRef - before.CpuClkUnhaltedRef;
}

template <class CounterStateType>
uint64 getCycles(const CounterStateType & now) // clocks
{
    return now.CpuClkUnhaltedThread.getRawData_NoOverflowProtection();
}

template <class CounterStateType>
inline double getCoreIPC(const CounterStateType & before, const CounterStateType & after) // instructions per cycle
{
    double ipc = getIPC(before, after);
    PCM * m = PCM::getInstance();
    if (ipc >= 0. && m && (m->getNumCores() == m->getNumOnlineCores()))
        return ipc * double(m->getThreadsPerCore());
    return -1;
}

template <class CounterStateType>
inline double getTotalExecUsage(const CounterStateType & before, const CounterStateType & after) // usage
{
    double usage = getExecUsage(before, after);
    PCM * m = PCM::getInstance();
    if (usage >= 0. && m && (m->getNumCores() == m->getNumOnlineCores()))
        return usage * double(m->getThreadsPerCore());
    return -1;
}

template <class StateType>
double getAverageFrequencyFromClocks(const int64 clocks, const StateType& before, const StateType& after) // in Hz
{
    const int64 timer_clocks = after.InvariantTSC - before.InvariantTSC;
    PCM* m = PCM::getInstance();
    if (timer_clocks != 0 && m)
        return double(m->getNominalFrequency()) * double(clocks) / double(timer_clocks);
    return -1;
}

template <class CounterStateType>
double getAverageFrequency(const CounterStateType & before, const CounterStateType & after) // in Hz
{
    return getAverageFrequencyFromClocks(after.CpuClkUnhaltedThread - before.CpuClkUnhaltedThread, before, after);
}

template <class UncoreStateType>
double getAverageUncoreFrequency(const UncoreStateType& before, const UncoreStateType & after) // in Hz
{
    auto m = PCM::getInstance();
    assert(m);
    return double(m->getNumOnlineCores()) * getAverageFrequencyFromClocks(after.UncClocks - before.UncClocks, before, after) / double(m->getNumOnlineSockets());
}

template <class CounterStateType>
double getActiveAverageFrequency(const CounterStateType & before, const CounterStateType & after) // in Hz
{
    int64 clocks = after.CpuClkUnhaltedThread - before.CpuClkUnhaltedThread;
    int64 ref_clocks = after.CpuClkUnhaltedRef - before.CpuClkUnhaltedRef;
    PCM * m = PCM::getInstance();
    if (ref_clocks != 0 && m)
        return double(m->getNominalFrequency()) * double(clocks) / double(ref_clocks);
    return -1;
}

template <class CounterStateType>
double getRelativeFrequency(const CounterStateType & before, const CounterStateType & after) // fraction of nominal frequency
{
    int64 clocks = after.CpuClkUnhaltedThread - before.CpuClkUnhaltedThread;
    int64 timer_clocks = after.InvariantTSC - before.InvariantTSC;
    if (timer_clocks != 0)
        return double(clocks) / double(timer_clocks);
    return -1;
}

template <class CounterStateType>
double getActiveRelativeFrequency(const CounterStateType & before, const CounterStateType & after) // fraction of nominal frequency
{
    if (!PCM::getInstance()->isActiveRelativeFrequencyAvailable()) return -1.;
    int64 clocks = after.CpuClkUnhaltedThread - before.CpuClkUnhaltedThread;
    int64 ref_clocks = after.CpuClkUnhaltedRef - before.CpuClkUnhaltedRef;
    if (ref_clocks != 0)
        return double(clocks) / double(ref_clocks);
    return -1;
}

template <class CounterStateType>
double getL2CacheHitRatio(const CounterStateType& before, const CounterStateType& after) // 0.0 - 1.0
{
    if (!PCM::getInstance()->isL2CacheHitRatioAvailable()) return 0;
    const auto hits = getL2CacheHits(before, after);
    const auto misses = getL2CacheMisses(before, after);
    return double(hits) / double(hits + misses);
}

template <class CounterStateType>
double getL3CacheHitRatio(const CounterStateType& before, const CounterStateType& after) // 0.0 - 1.0
{
    if (!PCM::getInstance()->isL3CacheHitRatioAvailable()) return 0;
    const auto hits = getL3CacheHits(before, after);
    const auto misses = getL3CacheMisses(before, after);
    return double(hits) / double(hits + misses);
}

template <class CounterStateType>
uint64 getL3CacheMisses(const CounterStateType & before, const CounterStateType & after)
{
    if (!PCM::getInstance()->isL3CacheMissesAvailable()) return 0;
    return after.Event[BasicCounterState::L3MissPos] - before.Event[BasicCounterState::L3MissPos];
}

template <class CounterStateType>
uint64 getL2CacheMisses(const CounterStateType & before, const CounterStateType & after)
{
    auto pcm = PCM::getInstance();
    if (pcm->isL2CacheMissesAvailable() == false) return 0ULL;
    const auto cpu_model = pcm->getCPUModel();
    if (pcm->useSkylakeEvents() || cpu_model == PCM::SNOWRIDGE || cpu_model == PCM::ADL) {
        return after.Event[BasicCounterState::SKLL2MissPos] - before.Event[BasicCounterState::SKLL2MissPos];
    }
    if (pcm->isAtom() || cpu_model == PCM::KNL)
    {
        return after.Event[BasicCounterState::ArchLLCMissPos] - before.Event[BasicCounterState::ArchLLCMissPos];
    }
    uint64 L3Miss = after.Event[BasicCounterState::L3MissPos] - before.Event[BasicCounterState::L3MissPos];
    uint64 L3UnsharedHit = after.Event[BasicCounterState::L3UnsharedHitPos] - before.Event[BasicCounterState::L3UnsharedHitPos];
    uint64 L2HitM = after.Event[BasicCounterState::L2HitMPos] - before.Event[BasicCounterState::L2HitMPos];
    return L2HitM + L3UnsharedHit + L3Miss;
}

template <class CounterStateType>
uint64 getL2CacheHits(const CounterStateType & before, const CounterStateType & after)
{
    auto pcm = PCM::getInstance();
    if (pcm->isL2CacheHitsAvailable() == false) return 0ULL;
    if (pcm->isAtom() || pcm->getCPUModel() == PCM::KNL)
    {
        uint64 L2Miss = after.Event[BasicCounterState::ArchLLCMissPos] - before.Event[BasicCounterState::ArchLLCMissPos];
        uint64 L2Ref = after.Event[BasicCounterState::ArchLLCRefPos] - before.Event[BasicCounterState::ArchLLCRefPos];
        return L2Ref - L2Miss;
    }
    return after.Event[BasicCounterState::L2HitPos] - before.Event[BasicCounterState::L2HitPos];
}

template <class CounterStateType>
uint64 getL3CacheOccupancy(const CounterStateType & now)
{
    if (PCM::getInstance()->L3CacheOccupancyMetricAvailable() == false) return 0ULL;
    return now.L3Occupancy;
}
template <class CounterStateType>
uint64 getLocalMemoryBW(const CounterStateType & before, const CounterStateType & after)
{
    if (PCM::getInstance()->CoreLocalMemoryBWMetricAvailable() == false) return 0ULL;
    return after.MemoryBWLocal - before.MemoryBWLocal;
}

template <class CounterStateType>
uint64 getRemoteMemoryBW(const CounterStateType & before, const CounterStateType & after)
{
    if (PCM::getInstance()->CoreRemoteMemoryBWMetricAvailable() == false) return 0ULL;
    const uint64 total = after.MemoryBWTotal - before.MemoryBWTotal;
    const uint64 local = getLocalMemoryBW(before, after);
    if (total > local)
        return total - local;

    return 0;
}

template <class CounterStateType>
uint64 getL3CacheHitsNoSnoop(const CounterStateType & before, const CounterStateType & after)
{
    if (!PCM::getInstance()->isL3CacheHitsNoSnoopAvailable()) return 0;
    return after.Event[BasicCounterState::L3UnsharedHitPos] - before.Event[BasicCounterState::L3UnsharedHitPos];
}

template <class CounterStateType>
uint64 getL3CacheHitsSnoop(const CounterStateType & before, const CounterStateType & after)
{
    auto pcm = PCM::getInstance();
    if (!pcm->isL3CacheHitsSnoopAvailable()) return 0;
    const auto cpu_model = pcm->getCPUModel();
    if (cpu_model == PCM::SNOWRIDGE || cpu_model == PCM::ADL)
    {
        const int64 misses = getL3CacheMisses(before, after);
        const int64 refs = after.Event[BasicCounterState::ArchLLCRefPos] - before.Event[BasicCounterState::ArchLLCRefPos];
        const int64 hits = refs - misses;
        return (hits > 0)? hits : 0;
    }
    if (pcm->useSkylakeEvents()) {
        return after.Event[BasicCounterState::SKLL3HitPos] - before.Event[BasicCounterState::SKLL3HitPos];
    }
    return after.Event[BasicCounterState::L2HitMPos] - before.Event[BasicCounterState::L2HitMPos];
}


template <class CounterStateType>
uint64 getL3CacheHits(const CounterStateType & before, const CounterStateType & after)
{
    if (!PCM::getInstance()->isL3CacheHitsAvailable()) return 0;
    return getL3CacheHitsSnoop(before, after) + getL3CacheHitsNoSnoop(before, after);
}

template <class CounterStateType>
uint64 getInvariantTSC(const CounterStateType & before, const CounterStateType & after)
{
    return after.InvariantTSC - before.InvariantTSC;
}

template <class CounterStateType>
inline double getCoreCStateResidency(int state, const CounterStateType & before, const CounterStateType & after)
{
    const double tsc = double(getInvariantTSC(before, after));

    if (state == 0) return double(getRefCycles(before, after)) / tsc;

    if (state == 1)
    {
        PCM * m = PCM::getInstance();
        double result = 1.0 - double(getRefCycles(before, after)) / tsc; // 1.0 - cC0
        for (int i = 2; i <= PCM::MAX_C_STATE; ++i)
            if (m->isCoreCStateResidencySupported(state))
                result -= (after.BasicCounterState::CStateResidency[i] - before.BasicCounterState::CStateResidency[i]) / tsc;

        if (result < 0.) result = 0.;       // fix counter dissynchronization
        else if (result > 1.) result = 1.;  // fix counter dissynchronization

        return result;
    }
    return (after.BasicCounterState::CStateResidency[state] - before.BasicCounterState::CStateResidency[state]) / tsc;
}

template <class CounterStateType>
inline uint64 getCoreCStateResidency(int state, const CounterStateType& now)
{
    if (state == 0) return now.CpuClkUnhaltedRef.getRawData_NoOverflowProtection();

    return now.BasicCounterState::CStateResidency[state];
}

template <class CounterStateType>
inline double getPackageCStateResidency(int state, const CounterStateType & before, const CounterStateType & after)
{
    const double tsc = double(getInvariantTSC(before, after));
    if (state == 0)
    {
        PCM * m = PCM::getInstance();
        double result = 1.0;
        for (int i = 1; i <= PCM::MAX_C_STATE; ++i)
            if (m->isPackageCStateResidencySupported(state))
                result -= (after.UncoreCounterState::CStateResidency[i] - before.UncoreCounterState::CStateResidency[i]) / tsc;

        if (result < 0.) result = 0.;       // fix counter dissynchronization
        else if (result > 1.) result = 1.;  // fix counter dissynchronization

        return result;
    }
    return double(after.UncoreCounterState::CStateResidency[state] - before.UncoreCounterState::CStateResidency[state]) / tsc;
}

template <class CounterStateType>
inline uint64 getPackageCStateResidency(int state, const CounterStateType& now)
{
    return now.UncoreCounterState::CStateResidency[state];
}

template <class CounterStateType>
uint64 getBytesReadFromMC(const CounterStateType & before, const CounterStateType & after)
{
    if (PCM::getInstance()->memoryTrafficMetricsAvailable())
        return (after.UncMCNormalReads - before.UncMCNormalReads) * 64;
    return 0ULL;
}

template <class CounterStateType>
uint64 getBytesWrittenToMC(const CounterStateType & before, const CounterStateType & after)
{
    if (PCM::getInstance()->memoryTrafficMetricsAvailable())
        return (after.UncMCFullWrites - before.UncMCFullWrites) * 64;
    return 0ULL;
}

template <class CounterStateType>
uint64 getBytesReadFromPMM(const CounterStateType & before, const CounterStateType & after)
{
    if (PCM::getInstance()->PMMTrafficMetricsAvailable())
        return (after.UncPMMReads - before.UncPMMReads) * 64;
    return 0ULL;
}

template <class CounterStateType>
uint64 getBytesWrittenToPMM(const CounterStateType & before, const CounterStateType & after)
{
    if (PCM::getInstance()->PMMTrafficMetricsAvailable())
        return (after.UncPMMWrites - before.UncPMMWrites) * 64;
    return 0ULL;
}

template <class CounterStateType>
uint64 getBytesReadFromEDC(const CounterStateType & before, const CounterStateType & after)
{
    if (PCM::getInstance()->MCDRAMmemoryTrafficMetricsAvailable())
        return (after.UncEDCNormalReads - before.UncEDCNormalReads) * 64;
    return 0ULL;
}

template <class CounterStateType>
uint64 getBytesWrittenToEDC(const CounterStateType & before, const CounterStateType & after)
{
    if (PCM::getInstance()->MCDRAMmemoryTrafficMetricsAvailable())
        return (after.UncEDCFullWrites - before.UncEDCFullWrites) * 64;
    return 0ULL;
}

template <class CounterStateType>
uint64 getGTRequestBytesFromMC(const CounterStateType & before, const CounterStateType & after)
{
    if (PCM::getInstance()->memoryIOTrafficMetricAvailable())
        return (after.UncMCGTRequests - before.UncMCGTRequests) * 64;
    return 0ULL;
}

template <class CounterStateType>
uint64 getIARequestBytesFromMC(const CounterStateType & before, const CounterStateType & after)
{
    if (PCM::getInstance()->memoryIOTrafficMetricAvailable())
        return (after.UncMCIARequests - before.UncMCIARequests) * 64;
    return 0ULL;
}

template <class CounterStateType>
uint64 getIORequestBytesFromMC(const CounterStateType & before, const CounterStateType & after)
{
    if (PCM::getInstance()->memoryIOTrafficMetricAvailable())
        return (after.UncMCIORequests - before.UncMCIORequests) * 64;
    return 0ULL;
}

template <class CounterStateType>
uint64 getSMICount(const CounterStateType & before, const CounterStateType & after)
{
    return after.SMICount - before.SMICount;
}

template <class CounterStateType>
uint64 getNumberOfCustomEvents(int32 eventCounterNr, const CounterStateType & before, const CounterStateType & after)
{
    return after.Event[eventCounterNr] - before.Event[eventCounterNr];
}

inline uint64 getIncomingQPILinkBytes(uint32 socketNr, uint32 linkNr, const SystemCounterState & before, const SystemCounterState & after)
{
    if (!PCM::getInstance()->incomingQPITrafficMetricsAvailable()) return 0ULL;
    uint64 b = before.incomingQPIPackets[socketNr][linkNr];
    uint64 a = after.incomingQPIPackets[socketNr][linkNr];
    // prevent overflows due to counter dissynchronisation
    return (a > b) ? (64 * (a - b)) : 0;
}

inline double getIncomingQPILinkUtilization(uint32 socketNr, uint32 linkNr, const SystemCounterState & before, const SystemCounterState & after)
{
    PCM * m = PCM::getInstance();
    if (!(m->qpiUtilizationMetricsAvailable())) return 0.;

    const double bytes = (double)getIncomingQPILinkBytes(socketNr, linkNr, before, after);
    const uint64 max_speed = m->getQPILinkSpeed(socketNr, linkNr);
    const double max_bytes = (double)(double(max_speed) * double(getInvariantTSC(before, after) / double(m->getNumOnlineCores())) / double(m->getNominalFrequency()));
    return bytes / max_bytes;
}

inline double getOutgoingQPILinkUtilization(uint32 socketNr, uint32 linkNr, const SystemCounterState & before, const SystemCounterState & after)
{
    PCM * m = PCM::getInstance();

    if (m->outgoingQPITrafficMetricsAvailable() == false) return 0.;

    if (m->hasBecktonUncore())
    {
        const uint64 b = before.outgoingQPIFlits[socketNr][linkNr]; // idle flits
        const uint64 a = after.outgoingQPIFlits[socketNr][linkNr];  // idle flits
        // prevent overflows due to counter dissynchronisation
        const double idle_flits = (double)((a > b) ? (a - b) : 0);
        const uint64 bTSC = before.uncoreTSC;
        const uint64 aTSC = after.uncoreTSC;
        const double tsc = (double)((aTSC > bTSC) ? (aTSC - bTSC) : 0);
        if (idle_flits >= tsc) return 0.; // prevent overflows due to potential counter dissynchronization

        return (1. - (idle_flits / tsc));
    } else if (m->hasPCICFGUncore())
    {
        const uint64 b = before.outgoingQPIFlits[socketNr][linkNr]; // data + non-data flits or idle (null) flits
        const uint64 a = after.outgoingQPIFlits[socketNr][linkNr]; // data + non-data flits or idle (null) flits
        // prevent overflows due to counter dissynchronisation
        double flits = (double)((a > b) ? (a - b) : 0);
        const double max_flits = ((double(getInvariantTSC(before, after)) * double(m->getQPILinkSpeed(socketNr, linkNr)) / m->getBytesPerFlit()) / double(m->getNominalFrequency())) / double(m->getNumOnlineCores());
        if(m->hasUPI())
        {
            flits = flits/3.;
        }
        if (flits > max_flits) return 1.; // prevent overflows due to potential counter dissynchronization
        return (flits / max_flits);
    }

    return 0;
}

inline uint64 getOutgoingQPILinkBytes(uint32 socketNr, uint32 linkNr, const SystemCounterState & before, const SystemCounterState & after)
{
    PCM * m = PCM::getInstance();
    if (!(m->outgoingQPITrafficMetricsAvailable())) return 0ULL;

    const double util = getOutgoingQPILinkUtilization(socketNr, linkNr, before, after);
    const double max_bytes = (double(m->getQPILinkSpeed(socketNr, linkNr)) * double(getInvariantTSC(before, after) / double(m->getNumOnlineCores())) / double(m->getNominalFrequency()));

    return (uint64)(max_bytes * util);
}


inline uint64 getAllIncomingQPILinkBytes(const SystemCounterState & before, const SystemCounterState & after)
{
    PCM * m = PCM::getInstance();
    const uint32 ns = m->getNumSockets();
    const uint32 qpiLinks = (uint32)m->getQPILinksPerSocket();
    uint64 sum = 0;

    for (uint32 s = 0; s < ns; ++s)
        for (uint32 q = 0; q < qpiLinks; ++q)
            sum += getIncomingQPILinkBytes(s, q, before, after);

    return sum;
}

inline uint64 getAllOutgoingQPILinkBytes(const SystemCounterState & before, const SystemCounterState & after)
{
    PCM * m = PCM::getInstance();
    const uint32 ns = m->getNumSockets();
    const uint32 qpiLinks = (uint32)m->getQPILinksPerSocket();
    uint64 sum = 0;

    for (uint32 s = 0; s < ns; ++s)
        for (uint32 q = 0; q < qpiLinks; ++q)
            sum += getOutgoingQPILinkBytes(s, q, before, after);

    return sum;
}


inline uint64 getIncomingQPILinkBytes(uint32 socketNr, uint32 linkNr, const SystemCounterState & now)
{
    if (PCM::getInstance()->incomingQPITrafficMetricsAvailable())
        return 64 * now.incomingQPIPackets[socketNr][linkNr];
    return 0ULL;
}

inline uint64 getSocketIncomingQPILinkBytes(uint32 socketNr, const SystemCounterState & now)
{
    PCM * m = PCM::getInstance();
    const uint32 qpiLinks = (uint32)m->getQPILinksPerSocket();
    uint64 sum = 0;

    for (uint32 q = 0; q < qpiLinks; ++q)
        sum += getIncomingQPILinkBytes(socketNr, q, now);

    return sum;
}

inline uint64 getAllIncomingQPILinkBytes(const SystemCounterState & now)
{
    PCM * m = PCM::getInstance();
    const uint32 ns = m->getNumSockets();
    uint64 sum = 0;

    for (uint32 s = 0; s < ns; ++s)
        sum += getSocketIncomingQPILinkBytes(s, now);
    return sum;
}


inline double getQPItoMCTrafficRatio(const SystemCounterState & before, const SystemCounterState & after)
{
    const uint64 totalQPI = getAllIncomingQPILinkBytes(before, after);
    uint64 memTraffic = getBytesReadFromMC(before, after) + getBytesWrittenToMC(before, after);
    if (PCM::getInstance()->PMMTrafficMetricsAvailable())
    {
        memTraffic += getBytesReadFromPMM(before, after) + getBytesWrittenToPMM(before, after);
    }
    return double(totalQPI) / double(memTraffic);
}

template <class CounterStateType>
inline double getLocalMemoryRequestRatio(const CounterStateType & before, const CounterStateType & after)
{
    if (PCM::getInstance()->localMemoryRequestRatioMetricAvailable() == false) return -1.;
    const auto all = after.UncHARequests - before.UncHARequests;
    const auto local = after.UncHALocalRequests - before.UncHALocalRequests;
    // std::cout << "PCM DEBUG "<< 64*all/1e6 << " " << 64*local/1e6 << "\n";
    return double(local)/double(all);
}

template <class CounterType>
inline uint64 getNumberOfEvents(const CounterType & before, const CounterType & after)
{
    return after.data - before.data;
}

template <class CounterStateType>
inline double getLLCReadMissLatency(const CounterStateType & before, const CounterStateType & after)
{
    auto * m = PCM::getInstance();
    if (m->LLCReadMissLatencyMetricsAvailable() == false) return -1.;
    const double occupancy = double(after.TOROccupancyIAMiss) - double(before.TOROccupancyIAMiss);
    const double inserts = double(after.TORInsertsIAMiss) - double(before.TORInsertsIAMiss);
    const double unc_clocks = double(after.UncClocks) - double(before.UncClocks);
    const double seconds = double(getInvariantTSC(before, after)) / double(m->getNumOnlineCores()/m->getNumSockets()) / double(m->getNominalFrequency());
    return 1e9*seconds*(occupancy/inserts)/unc_clocks;
}

template <class CounterStateType>
inline uint64 getAllSlots(const CounterStateType & before, const CounterStateType & after)
{
    const int64 a = after.BackendBoundSlots - before.BackendBoundSlots;
    const int64 b = after.FrontendBoundSlots - before.FrontendBoundSlots;
    const int64 c = after.BadSpeculationSlots - before.BadSpeculationSlots;
    const int64 d = after.RetiringSlots - before.RetiringSlots;
    // std::cout << "before DEBUG: " << before.FrontendBoundSlots << " " << before.BadSpeculationSlots << " "<< before.BackendBoundSlots << " " << before.RetiringSlots << std::endl;
    // std::cout << "after DEBUG: " <<  after.FrontendBoundSlots << " " << after.BadSpeculationSlots << " " << after.BackendBoundSlots << " " << after.RetiringSlots << std::endl;
    assert(a >= 0);
    assert(b >= 0);
    assert(c >= 0);
    assert(d >= 0);
    return a + b + c + d;
}

template <class CounterStateType>
inline uint64 getAllSlotsRaw(const CounterStateType& before, const CounterStateType& after)
{
    return after.AllSlotsRaw - before.AllSlotsRaw;
}

template <class CounterStateType>
inline double getBackendBound(const CounterStateType & before, const CounterStateType & after)
{
//    std::cout << "DEBUG: "<< after.BackendBoundSlots - before.BackendBoundSlots << " " << getAllSlots(before, after) << std::endl;
    if (PCM::getInstance()->isHWTMAL1Supported())
        return double(after.BackendBoundSlots - before.BackendBoundSlots)/double(getAllSlots(before, after));
    return 0.;
}

template <class CounterStateType>
inline double getFrontendBound(const CounterStateType & before, const CounterStateType & after)
{
//    std::cout << "DEBUG: "<< after.FrontendBoundSlots - before.FrontendBoundSlots << " " << getAllSlots(before, after) << std::endl;
    if (PCM::getInstance()->isHWTMAL1Supported())
        return double(after.FrontendBoundSlots - before.FrontendBoundSlots)/double(getAllSlots(before, after));
    return 0.;
}

template <class CounterStateType>
inline double getBadSpeculation(const CounterStateType & before, const CounterStateType & after)
{
//    std::cout << "DEBUG: "<< after.BadSpeculationSlots - before.BadSpeculationSlots << " " << getAllSlots(before, after) << std::endl;
    if (PCM::getInstance()->isHWTMAL1Supported())
        return double(after.BadSpeculationSlots - before.BadSpeculationSlots)/double(getAllSlots(before, after));
    return 0.;
}

template <class CounterStateType>
inline double getRetiring(const CounterStateType & before, const CounterStateType & after)
{
//    std::cout << "DEBUG: "<< after.RetiringSlots - before.RetiringSlots << " " << getAllSlots(before, after) << std::endl;
    if (PCM::getInstance()->isHWTMAL1Supported())
        return double(after.RetiringSlots - before.RetiringSlots)/double(getAllSlots(before, after));
    return 0.;
}

template <class CounterStateType>
uint64 getMSREvent(const uint64& index, const PCM::MSRType& type, const CounterStateType& before, const CounterStateType& after)
{
    switch (type)
    {
    case PCM::MSRType::Freerun:
        {
            const auto beforeIt = before.MSRValues.find(index);
            const auto afterIt = after.MSRValues.find(index);
            if (beforeIt != before.MSRValues.end() && afterIt != after.MSRValues.end())
            {
                return afterIt->second - beforeIt->second;
            }
            break;
        }
    case PCM::MSRType::Static:
        {
            const auto result = after.MSRValues.find(index);
            if (result != after.MSRValues.end())
            {
                return result->second;
            }
            break;
        }
    }
    return 0ULL;
}

} // namespace pcm

#endif