I am running perf c2c for a multi-threaded example (a threadpool that calculates a matrix multiplication in a work-stealing manner) and obtain curious results (cacheline contention for const-variables and few contentions for write-to variables at addresses I don’t understand.) The threads share two non-const variables:

• An atomic counter that threads decrement to pick their next work item
• An output buffer (c) to which the threads write the result of their computation at an offset related to the counter.

The code is (and gets compiled with g++ main.cpp -std=c++20 -g -O1):

#include <atomic>
#include <barrier>
#include <cstddef>
#include <functional>
#include <iostream>
#include <numeric>
#include <thread>
#include <vector>

class ThreadPool {
   public:
    friend void ThreadMain(std::stop_token, ThreadPool*, int core_id);
    using function = void(const void* context, float* c, size_t idx);
    explicit ThreadPool(size_t num_threads);
    ~ThreadPool();
    void QueueTask(function*, const void* context, float* __restrict__ c,
                   int range);

   private:
    std::barrier<std::function<void()>> sync_point;
    std::atomic_flag start{false};
    std::atomic<int> idx{0};
    function* task{nullptr};
    const void* ctx{nullptr};
    float* __restrict__ c_;
    std::vector<std::jthread> threads;
};

int pin_thread(int core_id, int tid) {
    cpu_set_t cpuset;
    CPU_ZERO(&cpuset);
    CPU_SET(core_id, &cpuset);
    int rc = sched_setaffinity(tid, sizeof(cpu_set_t), &cpuset);
    return rc;
}

void ThreadMain(std::stop_token stoken, ThreadPool* pool, int core_id) {
    pin_thread(core_id, gettid());
    while (!stoken.stop_requested()) {
        pool->start.wait(false);
        if (stoken.stop_requested()) {
            return;
        }
        int i{0};
        while ((i = pool->idx.fetch_sub(1) - 1) > -1) {
            (*(pool->task))(pool->ctx, pool->c_, i);
        }
        pool->sync_point.arrive_and_wait();
    }
}

ThreadPool::ThreadPool(size_t num_threads)
    : sync_point(num_threads, [&]() noexcept { this->start.clear(); }),
      threads(num_threads - 1) {
    pin_thread(0, gettid());
    for (uint32_t ii = 1; ii < num_threads; ++ii) {
        threads[ii - 1] = std::jthread(ThreadMain, this, ii);
    }
}

void ThreadPool::QueueTask(ThreadPool::function* function, const void* context,
                           float* __restrict__ c, int r) {
    task = function;
    ctx = context;
    c_ = c;
    idx.store(r);
    start.test_and_set();
    start.notify_all();
    int i{0};
    while ((i = idx.fetch_sub(1) - 1) > -1) {
        (*(function))(context, c_, i);
    }
    sync_point.arrive_and_wait();
}

ThreadPool::~ThreadPool() {
    for (auto& t : threads) {
        t.request_stop();
    }
    start.test_and_set();
    start.notify_all();
}

void __attribute__((noinline))
fma_f32_single(const float* __restrict__ aptr, const float* __restrict__ bptr,
               size_t M, size_t N, size_t K, float* __restrict__ cptr) {
    float c0{0};
    for (size_t i = 0; i < K; ++i) {
        c0 += (*(aptr + i)) * (*(bptr + N * i));
    }
    *cptr = c0;
}

struct pthreadpool_context {
    const float* __restrict__ a;
    const float* __restrict__ b;
    size_t M;
    size_t N;
    size_t K;
    const std::vector<std::pair<size_t, size_t>> indices;
};

void __attribute__((noinline))
work(const void* ctx, float* __restrict__ c, size_t i) {
    const auto* context = (const pthreadpool_context*)ctx;
    const auto [row, col] = context->indices[i];
    const float* aptr = context->a + row * context->K;
    const float* bptr = context->b + col;
    size_t N = context->N;
    size_t M = context->M;
    size_t K = context->K;
    float* cptr = c + row * N + col;
    fma_f32_single(aptr, bptr, M, N, K, cptr);
}

void __attribute__((noinline))
dispatch_threadpool(const float* __restrict__ a, const float* __restrict__ b,
                    float* __restrict__ c, size_t M, size_t N, size_t K,
                    ThreadPool& threadpool) {
    std::vector<std::pair<size_t, size_t>> indices;
    for (size_t i = 0; i < M; ++i) {
        for (size_t j = 0; j < N; ++j) {
            indices.emplace_back(i, j);
        }
    }
    const pthreadpool_context context = {a, b, M, N, K, indices};
    threadpool.QueueTask(work, (const void*)&context, c, indices.size());
}

int main() {
    using In_T = float;
    using Out_T = float;
    constexpr size_t M = 1024;
    constexpr size_t N = 1024;
    constexpr size_t K = 1024;
    std::vector<In_T> A(M * K);
    std::vector<In_T> B(K * N);
    std::vector<Out_T> C(M * N, 0);
    std::cout << "The address range for the output array C is: " << C.data()
              << " til " << C.data() + C.size() << std::endl;
    std::iota(A.begin(), A.end(), 0);
    std::iota(B.begin(), B.end(), 0);
    ThreadPool threadpool{4};
    dispatch_threadpool(A.data(), B.data(), C.data(), M, N, K, threadpool);
}

Running perf c2c at a high enough frequency shows me the following results:

=================================================
                 c2c details                     
=================================================
  Events                            : cpu_atom/mem-loads,ldlat=30/P
                                    : cpu_core/mem-loads-aux/
                                    : cpu_core/mem-loads,ldlat=30/
                                    : cpu_atom/mem-stores/P
                                    : cpu_core/mem-stores/P
                                    : dummy:u
  Cachelines sort on                : Total HITMs
  Cacheline data grouping           : offset,iaddr

=================================================
      Shared Cache Line Distribution Pareto      
=================================================
#
#        ----- HITM -----  ------- Store Refs ------  --------- Data address ---------                      ---------- cycles ----------    Total       cpu                                  Shared                         
#   Num  RmtHitm  LclHitm   L1 Hit  L1 Miss      N/A              Offset  Node  PA cnt        Code address  rmt hitm  lcl hitm      load  records       cnt                          Symbol  Object        Source:Line  Node
# .....  .......  .......  .......  .......  .......  ..................  ....  ......  ..................  ........  ........  ........  .......  ........  ..............................  .....  .................  ....
#
  ----------------------------------------------------------------------
      0        0       38        2        0        0      0x7ffe28f87740
  ----------------------------------------------------------------------
           0.00%   10.53%    0.00%    0.00%    0.00%                0x24     0       1            0x4029a7         0       373       296      141         3  [.] ThreadMain(std::stop_token  a.out  atomic_base.h:645   0
           0.00%    0.00%  100.00%    0.00%    0.00%                0x24     0       1            0x402a59         0         0       310       51         1  [.] ThreadPool::QueueTask(void  a.out  atomic_base.h:645   0
           0.00%   18.42%    0.00%    0.00%    0.00%                0x28     0       1            0x4029a2         0       195       165       95         3  [.] ThreadMain(std::stop_token  a.out  main.cpp:46         0
           0.00%   18.42%    0.00%    0.00%    0.00%                0x30     0       1            0x40299e         0       168       147       69         3  [.] ThreadMain(std::stop_token  a.out  main.cpp:46         0
           0.00%   39.47%    0.00%    0.00%    0.00%                0x38     0       1            0x40299a         0       204       172       91         3  [.] ThreadMain(std::stop_token  a.out  main.cpp:46         0
           0.00%   13.16%    0.00%    0.00%    0.00%                0x38     0       1            0x402a4c         0       270       200       21         1  [.] ThreadPool::QueueTask(void  a.out  main.cpp:71         0

  ----------------------------------------------------------------------
      1        0        3       33        0        0      0x7ffe28f87680
  ----------------------------------------------------------------------
           0.00%   33.33%    0.00%    0.00%    0.00%                 0x0     0       1            0x402391         0       167         5        7         3  [.] work(void const*, float*,   a.out  main.cpp:113     0
           0.00%   33.33%    0.00%    0.00%    0.00%                0x10     0       1            0x40237f         0       179        61        9         4  [.] work(void const*, float*,   a.out  main.cpp:107     0
           0.00%   33.33%    0.00%    0.00%    0.00%                0x18     0       1            0x402374         0       216        54       10         4  [.] work(void const*, float*,   a.out  main.cpp:106     0
           0.00%    0.00%  100.00%    0.00%    0.00%                0x38     0       1            0x402c8d         0         0         0       33         1  [.] dispatch_threadpool(float   a.out  vector.tcc:119   0

  ----------------------------------------------------------------------
      2        0        1        0        0        0  0xffff93e6df534bc0
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x28     0       1  0xffffffffa31585f1         0       208         0        1         1  [k] idle_cpu  [kernel.kallsyms]  idle_cpu+81   0

  ----------------------------------------------------------------------
      3        0        1        0        0        0      0x7a9862a070c0
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x28     0       1            0x40237b         0       118         0        1         1  [.] work(void const*, float*,   a.out  main.cpp:106   0

  ----------------------------------------------------------------------
      4        0        1        0        0        0      0x7a9862d91ac0
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x20     0       1            0x402378         0       157         0        1         1  [.] work(void const*, float*,   a.out  main.cpp:106   0

  ----------------------------------------------------------------------
      5        0        1        0        0        0      0x7a9862ddf240
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x38     0       1            0x40237b         0       118         0        1         1  [.] work(void const*, float*,   a.out  main.cpp:106   0

  ----------------------------------------------------------------------
      6        0        1        0        0        0      0x7a9862e45600
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x28     0       1            0x40237b         0       161         0        1         1  [.] work(void const*, float*,   a.out  main.cpp:106   0

  ----------------------------------------------------------------------
      7        0        1        0        0        0      0x7a9862ed3500
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x10     0       1            0x402378         0       161         0        1         1  [.] work(void const*, float*,   a.out  main.cpp:106   0

  ----------------------------------------------------------------------
      8        0        1        0        0        0      0x7a9862f592c0
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x20     0       1            0x402378         0       189         0        1         1  [.] work(void const*, float*,   a.out  main.cpp:106   0

  ----------------------------------------------------------------------
      9        0        1        0        0        0      0x7a98630fff00
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x38     0       1            0x40237b         0       157         0        1         1  [.] work(void const*, float*,   a.out  main.cpp:106   0

  ----------------------------------------------------------------------
     10        0        1        0        0        0      0x7a98631b9c00
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x30     0       1            0x402378         0       189         0        1         1  [.] work(void const*, float*,   a.out  main.cpp:106   0

  ----------------------------------------------------------------------
     11        0        1        0        0        0      0x7a9863214e40
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x30     0       1            0x402378         0       204         0        1         1  [.] work(void const*, float*,   a.out  main.cpp:106   0

  ----------------------------------------------------------------------
     12        0        1        0        0        0      0x7a986327ee00
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x18     0       1            0x40237b         0       168         0        1         1  [.] work(void const*, float*,   a.out  main.cpp:106   0

  ----------------------------------------------------------------------
     13        0        1        0        0        0      0x7a98633afd00
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x38     0       1            0x40237b         0       159         0        1         1  [.] work(void const*, float*,   a.out  main.cpp:106   0

  ----------------------------------------------------------------------
     14        0        1        0        0        0      0x7a98633c5780
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x30     0       1            0x402378         0       172         0        1         1  [.] work(void const*, float*,   a.out  main.cpp:106   0

  ----------------------------------------------------------------------
     15        0        1        0        0        0      0x7a986346a740
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x38     0       1            0x40237b         0       168         0        1         1  [.] work(void const*, float*,   a.out  main.cpp:106   0

  ----------------------------------------------------------------------
     16        0        1        0        0        0      0x7a986357f8c0
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x30     0       1            0x402378         0       168         0        1         1  [.] work(void const*, float*,   a.out  main.cpp:106   0

  ----------------------------------------------------------------------
     17        0        1        0        0        0      0x7a98635ad980
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x38     0       1            0x40237b         0       118         0        1         1  [.] work(void const*, float*,   a.out  main.cpp:106   0

  ----------------------------------------------------------------------
     18        0        1        0        0        0      0x7a986369be00
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x28     0       1            0x40237b         0       173         0        1         1  [.] work(void const*, float*,   a.out  main.cpp:106   0

  ----------------------------------------------------------------------
     19        0        1        0        0        0      0x7a9863711600
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x28     0       1            0x40237b         0       156         0        1         1  [.] work(void const*, float*,   a.out  main.cpp:106   0

  ----------------------------------------------------------------------
     20        0        1        0        0        0      0x7a9863778ec0
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x10     0       1            0x402378         0       168         0        1         1  [.] work(void const*, float*,   a.out  main.cpp:106   0

  ----------------------------------------------------------------------
     21        0        1        0        0        0      0x7a986378a040
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x28     0       1            0x40237b         0       119         0        1         1  [.] work(void const*, float*,   a.out  main.cpp:106   0

  ----------------------------------------------------------------------
     22        0        1        0        0        0      0x7a986378b2c0
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x30     0       1            0x402378         0       175         0        1         1  [.] work(void const*, float*,   a.out  main.cpp:106   0

  ----------------------------------------------------------------------
     23        0        1        0        0        0      0x7a986378c6c0
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x20     0       1            0x402378         0       159         0        1         1  [.] work(void const*, float*,   a.out  main.cpp:106   0

  ----------------------------------------------------------------------
     24        0        1        0        0        0      0x7a986379c8c0
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x30     0       1            0x402378         0       177         0        1         1  [.] work(void const*, float*,   a.out  main.cpp:106   0

  ----------------------------------------------------------------------
     25        0        1        0        0        0      0x7a98637b7980
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x18     0       1            0x40237b         0       161         0        1         1  [.] work(void const*, float*,   a.out  main.cpp:106   0

  ----------------------------------------------------------------------
     26        0        1        0        0        0      0x7a98637db180
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                 0x8     0       1            0x40237b         0       119         0        1         1  [.] work(void const*, float*,   a.out  main.cpp:106   0

  ----------------------------------------------------------------------
     27        0        1        0        0        0      0x7a98637ef0c0
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x28     0       1            0x40237b         0       132         0        1         1  [.] work(void const*, float*,   a.out  main.cpp:106   0

  ----------------------------------------------------------------------
     28        0        1        0        0        0      0x7a9863869740
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x38     0       1            0x40237b         0       179         0        1         1  [.] work(void const*, float*,   a.out  main.cpp:106   0

  ----------------------------------------------------------------------
     29        0        1        0        0        0      0x7a98660e3dc0
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x3c     0       1            0x402346         0       336       225        2         2  [.] fma_f32_single(float const  a.out  main.cpp:89   0

  ----------------------------------------------------------------------
     30        0        1        0        0        0      0x7a9866109f00
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x20     0       1            0x402346         0       242         0        1         1  [.] fma_f32_single(float const  a.out  main.cpp:89   0

  ----------------------------------------------------------------------
     31        0        1        0        0        0      0x7a986615ef40
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x30     0       1            0x402346         0       345         0        1         1  [.] fma_f32_single(float const  a.out  main.cpp:89   0

  ----------------------------------------------------------------------
     32        0        1        0        0        0      0x7a9866208000
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x2c     0       1            0x402346         0       168         0        1         1  [.] fma_f32_single(float const  a.out  main.cpp:89   0

  ----------------------------------------------------------------------
     33        0        1        0        0        0      0x7a98662564c0
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x30     0       1            0x402346         0       304       153        2         2  [.] fma_f32_single(float const  a.out  main.cpp:89   0

  ----------------------------------------------------------------------
     34        0        1        0        0        0      0x7a98662d0f80
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x30     0       1            0x402346         0       201         0        1         1  [.] fma_f32_single(float const  a.out  main.cpp:89   0

  ----------------------------------------------------------------------
     35        0        1        0        0        0      0x7a98663d7b40
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x3c     0       1            0x402346         0       274       360        2         1  [.] fma_f32_single(float const  a.out  main.cpp:89   0

  ----------------------------------------------------------------------
     36        0        1        0        0        0      0x7a98663f4400
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                0x34     0       1            0x402346         0       526       185        2         1  [.] fma_f32_single(float const  a.out  main.cpp:89   0

  ----------------------------------------------------------------------
     37        0        1        0        0        0      0x7a9866622a80
  ----------------------------------------------------------------------
           0.00%  100.00%    0.00%    0.00%    0.00%                 0x4     0       1            0x402342         0       159         0        1         1  [.] fma_f32_single(float const  a.out  main.cpp:89   0

I find these results very curious for the following reasons:

Cacheline Contention for Read-Only Variable
Many cacheline contentions are at position main.cpp:106. This is const auto [row, col] = context->indices[i]. The variable context is const. I expect each thread to hold a local copy of the indices array. Why does this variable show up in the cacheline-contention statistics? Is this maybe an artifact because snooping from other caches is faster than reading from memory?

Few Contentions at write-to Array
Why are there so few contentions at the output buffer C? Furthermore, what address does this relate to? Consider item 29, which has an address of 0x7a98660e3dc0. The beginning of the code prints the address range of C as 0x7a9865a00010–0x7a9865e00010. That item doesn’t lie inside that range. Moreover, the code address is at main.cpp:89 (more precisely 0x402346), not at the exact write to the shared object (which would be 0x40235).

Low CPU count
Finally, many contentions show a CPU count of 1. Why’s that contented?

How can I make sense of the outputs? The test system is an Alder-Lake laptop, but I obtained similar results with a Graviton 3 instance.