Ceph » RADOS

asy to visualize the number of ops
in each subqueue of mclock.

Add request statistics for each
op_scheduler_class

ceph daemon osd.0 perf dump osd
{
    "osd": {
        "mclock_immdiate_op": 640,
        "mclock_client_op": 1229,
        "mclock_recovery_op": 887,
        "mclock_best_effort_op": 925
    }
}

https://github.com/ceph/ceph/pull/55295

The reasons are as follows:

1. op request type

enum class op_scheduler_class : uint8_t {
  background_recovery = 0,
  background_best_effort,
  immediate,
  client,
};

2. Current tools for viewing mclock queue
- Only the number of clients and requests of shard-queue can be displayed.
- Problem: There is no way to distinguish the number of handles for each request type

# ceph tell osd.10 dump_op_pq_state | jq
{
  "OSD:ShardedOpWQ:0": {
    "queue_sizes": {
      "immediate": 0,
      "scheduler": 0
    },
    "mClockClients": {
      "client_count": 2,
      "clients": "{ PriorityQueue::  { client:{ class_id: 2 client_id: 2 profile_id: 0 }, record:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5866 p:246674.9139 l:149134.2172 } req_count:0 top_req:none } }  { client:{ class_id: 4 client_id: 4 profile_id: 0 }, record:{ ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.0831 p:225897.1451 l:149134.0831 } req_count:0 top_req:none } } { reservation_top:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5866 p:246674.9139 l:149134.2172 } req_count:0 top_req:none } } { ready_top:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5866 p:246674.9139 l:149134.2172 } req_count:0 top_req:none } } { limit_top:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5866 p:246674.9139 l:149134.2172 } req_count:0 top_req:none } } }" 
    },
    "mClockQueues": {
      "queues": "RESER:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5866 p:246674.9139 l:149134.2172 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.0831 p:225897.1451 l:149134.0831 } req_count:0 top_req:none }LIMIT:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5866 p:246674.9139 l:149134.2172 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.0831 p:225897.1451 l:149134.0831 } req_count:0 top_req:none }READY:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5866 p:246674.9139 l:149134.2172 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.0831 p:225897.1451 l:149134.0831 } req_count:0 top_req:none }" 
    }
  },
  "OSD:ShardedOpWQ:1": {
    "queue_sizes": {
      "immediate": 0,
      "scheduler": 0
    },
    "mClockClients": {
      "client_count": 2,
      "clients": "{ PriorityQueue::  { client:{ class_id: 2 client_id: 2 profile_id: 0 }, record:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.8296 p:248969.0061 l:149134.2010 } req_count:0 top_req:none } }  { client:{ class_id: 4 client_id: 4 profile_id: 0 }, record:{ ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.2180 p:227152.5730 l:149134.2180 } req_count:0 top_req:none } } { reservation_top:{ ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.2180 p:227152.5730 l:149134.2180 } req_count:0 top_req:none } } { ready_top:{ ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.2180 p:227152.5730 l:149134.2180 } req_count:0 top_req:none } } { limit_top:{ ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.2180 p:227152.5730 l:149134.2180 } req_count:0 top_req:none } } }" 
    },
    "mClockQueues": {
      "queues": "RESER:{ ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.2180 p:227152.5730 l:149134.2180 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.8296 p:248969.0061 l:149134.2010 } req_count:0 top_req:none }LIMIT:{ ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.2180 p:227152.5730 l:149134.2180 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.8296 p:248969.0061 l:149134.2010 } req_count:0 top_req:none }READY:{ ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.2180 p:227152.5730 l:149134.2180 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.8296 p:248969.0061 l:149134.2010 } req_count:0 top_req:none }" 
    }
  },
  "OSD:ShardedOpWQ:2": {
    "queue_sizes": {
      "immediate": 0,
      "scheduler": 0
    },
    "mClockClients": {
      "client_count": 2,
      "clients": "{ PriorityQueue::  { client:{ class_id: 2 client_id: 2 profile_id: 0 }, record:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5392 p:247277.9751 l:149134.2089 } req_count:0 top_req:none } }  { client:{ class_id: 4 client_id: 4 profile_id: 0 }, record:{ ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.1760 p:225219.9794 l:149134.1760 } req_count:0 top_req:none } } { reservation_top:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5392 p:247277.9751 l:149134.2089 } req_count:0 top_req:none } } { ready_top:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5392 p:247277.9751 l:149134.2089 } req_count:0 top_req:none } } { limit_top:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5392 p:247277.9751 l:149134.2089 } req_count:0 top_req:none } } }" 
    },
    "mClockQueues": {
      "queues": "RESER:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5392 p:247277.9751 l:149134.2089 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.1760 p:225219.9794 l:149134.1760 } req_count:0 top_req:none }LIMIT:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5392 p:247277.9751 l:149134.2089 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.1760 p:225219.9794 l:149134.1760 } req_count:0 top_req:none }READY:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.5392 p:247277.9751 l:149134.2089 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.1760 p:225219.9794 l:149134.1760 } req_count:0 top_req:none }" 
    }
  },
  "OSD:ShardedOpWQ:3": {
    "queue_sizes": {
      "immediate": 0,
      "scheduler": 0
    },
    "mClockClients": {
      "client_count": 2,
      "clients": "{ PriorityQueue::  { client:{ class_id: 2 client_id: 2 profile_id: 0 }, record:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.9434 p:127110.0675 l:149134.2163 } req_count:0 top_req:none } }  { client:{ class_id: 4 client_id: 4 profile_id: 0 }, record:{ ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.1993 p:213865.4879 l:149134.1993 } req_count:0 top_req:none } } { reservation_top:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.9434 p:127110.0675 l:149134.2163 } req_count:0 top_req:none } } { ready_top:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.9434 p:127110.0675 l:149134.2163 } req_count:0 top_req:none } } { limit_top:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.9434 p:127110.0675 l:149134.2163 } req_count:0 top_req:none } } }" 
    },
    "mClockQueues": {
      "queues": "RESER:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.9434 p:127110.0675 l:149134.2163 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.1993 p:213865.4879 l:149134.1993 } req_count:0 top_req:none }LIMIT:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.9434 p:127110.0675 l:149134.2163 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.1993 p:213865.4879 l:149134.1993 } req_count:0 top_req:none }READY:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.9434 p:127110.0675 l:149134.2163 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.1993 p:213865.4879 l:149134.1993 } req_count:0 top_req:none }" 
    }
  },
  "OSD:ShardedOpWQ:4": {
    "queue_sizes": {
      "immediate": 0,
      "scheduler": 0
    },
    "mClockClients": {
      "client_count": 2,
      "clients": "{ PriorityQueue::  { client:{ class_id: 2 client_id: 2 profile_id: 0 }, record:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.6596 p:125844.9819 l:149134.2203 } req_count:0 top_req:none } }  { client:{ class_id: 4 client_id: 4 profile_id: 0 }, record:{ ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.1605 p:211845.4401 l:149134.1695 } req_count:0 top_req:none } } { reservation_top:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.6596 p:125844.9819 l:149134.2203 } req_count:0 top_req:none } } { ready_top:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.6596 p:125844.9819 l:149134.2203 } req_count:0 top_req:none } } { limit_top:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.6596 p:125844.9819 l:149134.2203 } req_count:0 top_req:none } } }" 
    },
    "mClockQueues": {
      "queues": "RESER:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.6596 p:125844.9819 l:149134.2203 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.1605 p:211845.4401 l:149134.1695 } req_count:0 top_req:none }LIMIT:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.6596 p:125844.9819 l:149134.2203 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.1605 p:211845.4401 l:149134.1695 } req_count:0 top_req:none }READY:{ ClientRec:: client:{ class_id: 2 client_id: 2 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149133.6596 p:125844.9819 l:149134.2203 } req_count:0 top_req:none }, { ClientRec:: client:{ class_id: 4 client_id: 4 profile_id: 0 } prev_tag:{ RequestTag:: ready:false r:149134.1605 p:211845.4401 l:149134.1695 } req_count:0 top_req:none }" 
    }
  }
}

The design of this plan:
1. By adding mclock perfcounter to osd,
2. Before requesting to join the team,
3. Count requests for background_recovery / background_best_effort / immediate / client respectively.

benefit:
1. You can quickly export the processing counts of various requests through ceph tell osd.x perf dump osd.
2. It is convenient for the monitoring component to trace the history of the number of various requests processed by mclock.
3. When analyzing problems, it helps to analyze the source of disk pressure.