Skip to content

libbeat/cmd/instance: cgroup metrics no longer being reported #27598

Closed
#27620
Closed
libbeat/cmd/instance: cgroup metrics no longer being reported#27598
#27620
Assignees
fearful-symmetry
Labels
Team:IntegrationsLabel for the Integrations teambugv7.15.0
@axw

Description

@axw
axw
opened on Aug 26, 2021

libbeat is meant to record cgroup metrics for stack monitoring, in order to calculate CPU utilisation in containerised environments. These metrics are not being captured in 7.15.0-SNAPSHOT. Compare the two captured beats_stats monitoring docs from 7.14.0 and 7.15.0-SNAPSHOT:

7.14.0 
{
  "took" : 6,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 10,
      "relation" : "eq"
    },
    "max_score" : 1.0,
    "hits" : [
      {
        "_index" : ".monitoring-beats-7-2021.08.24",
        "_type" : "_doc",
        "_id" : "SY4md3sB3FXlYVJA2IsP",
        "_score" : 1.0,
        "_source" : {
          "cluster_uuid" : "tQQkEZaXQaeuqYfa3JDbDw",
          "timestamp" : "2021-08-24T07:53:58.943Z",
          "interval_ms" : 10000,
          "type" : "beats_stats",
          "source_node" : {
            "uuid" : "gdRkH_MyRAG4ju7s7H64rQ",
            "host" : "10.40.152.5",
            "transport_address" : "10.40.152.5:19823",
            "ip" : "10.40.152.5",
            "name" : "instance-0000000001",
            "timestamp" : "2021-08-24T07:53:58.944Z"
          },
          "beats_stats" : {
            "timestamp" : "2021-08-24T07:53:58.932Z",
            "beat" : {
              "version" : "7.14.0",
              "name" : "instance-0000000000",
              "host" : "ebd47fe5f705",
              "uuid" : "fa3dba82-27c8-4ebf-92d2-21a558e468af",
              "type" : "apm-server"
            },
            "metrics" : {
              "system" : {
                "load" : {
                  "1" : 1.62,
                  "5" : 1.32,
                  "15" : 1.28,
                  "norm" : {
                    "1" : 0.0506,
                    "5" : 0.0413,
                    "15" : 0.04
                  }
                },
                "cpu" : {
                  "cores" : 32
                }
              },
              "beat" : {
                "memstats" : {
                  "memory_total" : 68182944,
                  "memory_alloc" : 15684176,
                  "memory_sys" : 79119368,
                  "gc_next" : 19701082,
                  "rss" : 122650624
                },
                "cpu" : {
                  "total" : {
                    "value" : 860,
                    "ticks" : 860,
                    "time" : {
                      "ms" : 868
                    }
                  },
                  "user" : {
                    "ticks" : 600,
                    "time" : {
                      "ms" : 603
                    }
                  },
                  "system" : {
                    "ticks" : 260,
                    "time" : {
                      "ms" : 265
                    }
                  }
                },
                "runtime" : {
                  "goroutines" : 80
                },
                "cgroup" : {
                  "cpu" : {
                    "id" : "ebd47fe5f70595e26ac9ee7a770996d3e1d672326d98e31090fb5041c75be035",
                    "cfs" : {
                      "period" : {
                        "us" : 100000
                      },
                      "quota" : {
                        "us" : 19672
                      }
                    },
                    "stats" : {
                      "periods" : 426,
                      "throttled" : {
                        "periods" : 142,
                        "ns" : 38214584358
                      }
                    }
                  },
                  "cpuacct" : {
                    "total" : {
                      "ns" : 11540118524
                    },
                    "id" : "ebd47fe5f70595e26ac9ee7a770996d3e1d672326d98e31090fb5041c75be035"
                  },
                  "memory" : {
                    "mem" : {
                      "limit" : {
                        "bytes" : 536870912
                      },
                      "usage" : {
                        "bytes" : 446816256
                      }
                    },
                    "id" : "ebd47fe5f70595e26ac9ee7a770996d3e1d672326d98e31090fb5041c75be035"
                  }
                },
                "handles" : {
                  "open" : 21,
                  "limit" : {
                    "hard" : 1048576,
                    "soft" : 1048576
                  }
                },
                "info" : {
                  "version" : "7.14.0",
                  "uptime" : {
                    "ms" : 130889
                  },
                  "ephemeral_id" : "ef2c00a2-7c9f-448b-aaa4-48fcd9e7f3ca"
                }
              },
              "libbeat" : {
                "config" : {
                  "scans" : 0,
                  "reloads" : 0,
                  "module" : {
                    "running" : 0,
                    "starts" : 0,
                    "stops" : 0
                  }
                },
                "output" : {
                  "read" : {
                    "bytes" : 73094,
                    "errors" : 0
                  },
                  "type" : "elasticsearch",
                  "events" : {
                    "dropped" : 0,
                    "duplicates" : 0,
                    "active" : 0,
                    "toomany" : 0,
                    "batches" : 1,
                    "total" : 1,
                    "acked" : 1,
                    "failed" : 0
                  },
                  "write" : {
                    "bytes" : 137716,
                    "errors" : 0
                  }
                },
                "pipeline" : {
                  "clients" : 1,
                  "events" : {
                    "total" : 1,
                    "filtered" : 0,
                    "published" : 1,
                    "failed" : 0,
                    "dropped" : 0,
                    "retry" : 5,
                    "active" : 0
                  },
                  "queue" : {
                    "acked" : 1,
                    "max_events" : 2000
                  }
                }
              },
              "apm-server" : {
                "server" : {
                  "request" : {
                    "count" : 0
                  },
                  "response" : {
                    "count" : 0,
                    "errors" : {
                      "forbidden" : 0,
                      "validate" : 0,
                      "count" : 0,
                      "internal" : 0,
                      "decode" : 0,
                      "notfound" : 0,
                      "ratelimit" : 0,
                      "invalidquery" : 0,
                      "closed" : 0,
                      "method" : 0,
                      "queue" : 0,
                      "timeout" : 0,
                      "toolarge" : 0,
                      "unauthorized" : 0,
                      "unavailable" : 0
                    },
                    "valid" : {
                      "count" : 0,
                      "ok" : 0,
                      "accepted" : 0,
                      "notmodified" : 0
                    }
                  },
                  "unset" : 0
                },
                "profile" : {
                  "response" : {
                    "count" : 0,
                    "errors" : {
                      "internal" : 0,
                      "validate" : 0,
                      "unavailable" : 0,
                      "method" : 0,
                      "unauthorized" : 0,
                      "queue" : 0,
                      "timeout" : 0,
                      "decode" : 0,
                      "forbidden" : 0,
                      "count" : 0,
                      "ratelimit" : 0,
                      "toolarge" : 0,
                      "invalidquery" : 0,
                      "notfound" : 0,
                      "closed" : 0
                    },
                    "valid" : {
                      "ok" : 0,
                      "accepted" : 0,
                      "count" : 0,
                      "notmodified" : 0
                    }
                  },
                  "unset" : 0,
                  "request" : {
                    "count" : 0
                  }
                },
                "acm" : {
                  "response" : {
                    "count" : 0,
                    "errors" : {
                      "timeout" : 0,
                      "count" : 0,
                      "internal" : 0,
                      "ratelimit" : 0,
                      "method" : 0,
                      "queue" : 0,
                      "unavailable" : 0,
                      "notfound" : 0,
                      "forbidden" : 0,
                      "invalidquery" : 0,
                      "closed" : 0,
                      "validate" : 0,
                      "toolarge" : 0,
                      "decode" : 0,
                      "unauthorized" : 0
                    },
                    "valid" : {
                      "count" : 0,
                      "notmodified" : 0,
                      "ok" : 0,
                      "accepted" : 0
                    }
                  },
                  "unset" : 0,
                  "request" : {
                    "count" : 0
                  }
                },
                "sourcemap" : {
                  "response" : {
                    "valid" : {
                      "accepted" : 0,
                      "count" : 0,
                      "notmodified" : 0,
                      "ok" : 0
                    },
                    "count" : 0,
                    "errors" : {
                      "unauthorized" : 0,
                      "closed" : 0,
                      "unavailable" : 0,
                      "notfound" : 0,
                      "decode" : 0,
                      "invalidquery" : 0,
                      "timeout" : 0,
                      "toolarge" : 0,
                      "ratelimit" : 0,
                      "forbidden" : 0,
                      "method" : 0,
                      "validate" : 0,
                      "queue" : 0,
                      "internal" : 0,
                      "count" : 0
                    }
                  },
                  "unset" : 0,
                  "decoding" : {
                    "errors" : 0,
                    "count" : 0
                  },
                  "validation" : {
                    "count" : 0,
                    "errors" : 0
                  },
                  "request" : {
                    "count" : 0
                  }
                },
                "aggregation" : {
                  "txmetrics" : {
                    "active_groups" : 0,
                    "overflowed" : 0
                  }
                },
                "processor" : {
                  "error" : {
                    "transformations" : 0,
                    "stacktraces" : 0,
                    "frames" : 0
                  },
                  "metric" : {
                    "transformations" : 0
                  },
                  "sourcemap" : {
                    "counter" : 0
                  },
                  "span" : {
                    "transformations" : 0,
                    "stacktraces" : 0,
                    "frames" : 0
                  },
                  "transaction" : {
                    "transformations" : 0
                  },
                  "stream" : {
                    "errors" : {
                      "toolarge" : 0,
                      "invalid" : 0
                    },
                    "accepted" : 0
                  }
                },
                "jaeger" : {
                  "grpc" : {
                    "collect" : {
                      "request" : {
                        "count" : 0
                      },
                      "response" : {
                        "valid" : {
                          "count" : 0
                        },
                        "count" : 0,
                        "errors" : {
                          "count" : 0,
                          "ratelimit" : 0,
                          "timeout" : 0,
                          "unauthorized" : 0
                        }
                      }
                    },
                    "sampling" : {
                      "request" : {
                        "count" : 0
                      },
                      "response" : {
                        "count" : 0,
                        "errors" : {
                          "count" : 0
                        },
                        "valid" : {
                          "count" : 0
                        }
                      },
                      "event" : {
                        "received" : {
                          "count" : 0
                        }
                      }
                    }
                  },
                  "http" : {
                    "request" : {
                      "count" : 0
                    },
                    "response" : {
                      "count" : 0,
                      "errors" : {
                        "count" : 0
                      },
                      "valid" : {
                        "count" : 0
                      }
                    },
                    "event" : {
                      "received" : {
                        "count" : 0
                      }
                    }
                  }
                },
                "decoder" : {
                  "gzip" : {
                    "content-length" : 0,
                    "count" : 0
                  },
                  "uncompressed" : {
                    "content-length" : 0,
                    "count" : 0
                  },
                  "reader" : {
                    "count" : 0
                  },
                  "missing-content-length" : {
                    "count" : 0
                  },
                  "deflate" : {
                    "content-length" : 0,
                    "count" : 0
                  }
                },
                "root" : {
                  "request" : {
                    "count" : 14
                  },
                  "response" : {
                    "count" : 14,
                    "errors" : {
                      "toolarge" : 0,
                      "forbidden" : 0,
                      "timeout" : 0,
                      "invalidquery" : 0,
                      "unauthorized" : 0,
                      "internal" : 0,
                      "queue" : 0,
                      "notfound" : 0,
                      "unavailable" : 0,
                      "closed" : 0,
                      "method" : 0,
                      "decode" : 0,
                      "ratelimit" : 0,
                      "validate" : 0,
                      "count" : 0
                    },
                    "valid" : {
                      "accepted" : 0,
                      "count" : 14,
                      "notmodified" : 0,
                      "ok" : 14
                    }
                  },
                  "unset" : 0
                },
                "otlp" : {
                  "grpc" : {
                    "metrics" : {
                      "request" : {
                        "count" : 0
                      },
                      "response" : {
                        "valid" : {
                          "count" : 0
                        },
                        "count" : 0,
                        "errors" : {
                          "count" : 0,
                          "ratelimit" : 0,
                          "timeout" : 0,
                          "unauthorized" : 0
                        }
                      },
                      "consumer" : {
                        "unsupported_dropped" : 0
                      }
                    },
                    "traces" : {
                      "request" : {
                        "count" : 0
                      },
                      "response" : {
                        "valid" : {
                          "count" : 0
                        },
                        "count" : 0,
                        "errors" : {
                          "count" : 0,
                          "ratelimit" : 0,
                          "timeout" : 0,
                          "unauthorized" : 0
                        }
                      }
                    }
                  }
                },
                "sampling" : {
                  "transactions_dropped" : 0
                }
              }
            }
          }
        }
      }
    ]
  }
}
7.15.0-SNAPSHOT 
{
  "took" : 5,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 180,
      "relation" : "eq"
    },
    "max_score" : 1.0,
    "hits" : [
      {
        "_index" : ".monitoring-beats-7-2021.08.24",
        "_type" : "_doc",
        "_id" : "K0cJd3sBGs3E_2OrMDi6",
        "_score" : 1.0,
        "_source" : {
          "cluster_uuid" : "3S3M5ta2SKe8-HLdWdO8Cw",
          "timestamp" : "2021-08-24T07:21:35.636Z",
          "interval_ms" : 10000,
          "type" : "beats_stats",
          "source_node" : {
            "uuid" : "abBsu4wCTbGYOsPIz-dhbg",
            "host" : "10.41.0.53",
            "transport_address" : "10.41.0.53:19952",
            "ip" : "10.41.0.53",
            "name" : "instance-0000000000",
            "timestamp" : "2021-08-24T07:21:35.636Z"
          },
          "beats_stats" : {
            "timestamp" : "2021-08-24T07:21:35.404Z",
            "beat" : {
              "type" : "apm-server",
              "version" : "7.15.0",
              "name" : "instance-0000000000",
              "host" : "c628e081d1d6",
              "uuid" : "44ba57b4-d4cc-4cd3-94d0-188181013764"
            },
            "metrics" : {
              "libbeat" : {
                "config" : {
                  "scans" : 0,
                  "reloads" : 0,
                  "module" : {
                    "starts" : 0,
                    "stops" : 0,
                    "running" : 0
                  }
                },
                "output" : {
                  "write" : {
                    "bytes" : 188503,
                    "errors" : 0
                  },
                  "read" : {
                    "errors" : 0,
                    "bytes" : 104600
                  },
                  "type" : "elasticsearch",
                  "events" : {
                    "dropped" : 0,
                    "duplicates" : 0,
                    "active" : 0,
                    "toomany" : 0,
                    "batches" : 1,
                    "total" : 1,
                    "acked" : 1,
                    "failed" : 0
                  }
                },
                "pipeline" : {
                  "queue" : {
                    "acked" : 1,
                    "max_events" : 4000
                  },
                  "clients" : 1,
                  "events" : {
                    "published" : 1,
                    "failed" : 0,
                    "dropped" : 0,
                    "retry" : 7,
                    "active" : 0,
                    "total" : 1,
                    "filtered" : 0
                  }
                }
              },
              "apm-server" : {
                "sourcemap" : {
                  "validation" : {
                    "count" : 0,
                    "errors" : 0
                  },
                  "request" : {
                    "count" : 0
                  },
                  "response" : {
                    "count" : 0,
                    "errors" : {
                      "count" : 0,
                      "validate" : 0,
                      "ratelimit" : 0,
                      "closed" : 0,
                      "unavailable" : 0,
                      "queue" : 0,
                      "internal" : 0,
                      "timeout" : 0,
                      "invalidquery" : 0,
                      "toolarge" : 0,
                      "notfound" : 0,
                      "forbidden" : 0,
                      "decode" : 0,
                      "method" : 0,
                      "unauthorized" : 0
                    },
                    "valid" : {
                      "accepted" : 0,
                      "notmodified" : 0,
                      "count" : 0,
                      "ok" : 0
                    }
                  },
                  "unset" : 0,
                  "decoding" : {
                    "count" : 0,
                    "errors" : 0
                  }
                },
                "sampling" : {
                  "transactions_dropped" : 0
                },
                "server" : {
                  "request" : {
                    "count" : 0
                  },
                  "response" : {
                    "errors" : {
                      "invalidquery" : 0,
                      "decode" : 0,
                      "queue" : 0,
                      "timeout" : 0,
                      "unauthorized" : 0,
                      "toolarge" : 0,
                      "method" : 0,
                      "internal" : 0,
                      "notfound" : 0,
                      "validate" : 0,
                      "ratelimit" : 0,
                      "count" : 0,
                      "closed" : 0,
                      "unavailable" : 0,
                      "forbidden" : 0
                    },
                    "valid" : {
                      "count" : 0,
                      "ok" : 0,
                      "accepted" : 0,
                      "notmodified" : 0
                    },
                    "count" : 0
                  },
                  "unset" : 0
                },
                "otlp" : {
                  "grpc" : {
                    "metrics" : {
                      "consumer" : {
                        "unsupported_dropped" : 0
                      },
                      "request" : {
                        "count" : 0
                      },
                      "response" : {
                        "valid" : {
                          "count" : 0
                        },
                        "count" : 0,
                        "errors" : {
                          "count" : 0,
                          "ratelimit" : 0,
                          "timeout" : 0,
                          "unauthorized" : 0
                        }
                      }
                    },
                    "traces" : {
                      "request" : {
                        "count" : 0
                      },
                      "response" : {
                        "errors" : {
                          "unauthorized" : 0,
                          "count" : 0,
                          "ratelimit" : 0,
                          "timeout" : 0
                        },
                        "valid" : {
                          "count" : 0
                        },
                        "count" : 0
                      }
                    }
                  }
                },
                "processor" : {
                  "transaction" : {
                    "transformations" : 0
                  },
                  "sourcemap" : {
                    "counter" : 0
                  },
                  "stream" : {
                    "accepted" : 0,
                    "errors" : {
                      "toolarge" : 0,
                      "invalid" : 0
                    }
                  },
                  "error" : {
                    "transformations" : 0,
                    "stacktraces" : 0,
                    "frames" : 0
                  },
                  "metric" : {
                    "transformations" : 0
                  },
                  "span" : {
                    "transformations" : 0,
                    "stacktraces" : 0,
                    "frames" : 0
                  }
                },
                "jaeger" : {
                  "grpc" : {
                    "sampling" : {
                      "request" : {
                        "count" : 0
                      },
                      "response" : {
                        "count" : 0,
                        "errors" : {
                          "count" : 0
                        },
                        "valid" : {
                          "count" : 0
                        }
                      },
                      "event" : {
                        "received" : {
                          "count" : 0
                        }
                      }
                    },
                    "collect" : {
                      "request" : {
                        "count" : 0
                      },
                      "response" : {
                        "count" : 0,
                        "errors" : {
                          "ratelimit" : 0,
                          "timeout" : 0,
                          "unauthorized" : 0,
                          "count" : 0
                        },
                        "valid" : {
                          "count" : 0
                        }
                      }
                    }
                  },
                  "http" : {
                    "response" : {
                      "count" : 0,
                      "errors" : {
                        "count" : 0
                      },
                      "valid" : {
                        "count" : 0
                      }
                    },
                    "event" : {
                      "received" : {
                        "count" : 0
                      }
                    },
                    "request" : {
                      "count" : 0
                    }
                  }
                },
                "root" : {
                  "request" : {
                    "count" : 39
                  },
                  "response" : {
                    "errors" : {
                      "decode" : 0,
                      "notfound" : 0,
                      "unauthorized" : 0,
                      "ratelimit" : 0,
                      "closed" : 0,
                      "toolarge" : 0,
                      "internal" : 0,
                      "forbidden" : 0,
                      "queue" : 0,
                      "timeout" : 0,
                      "unavailable" : 0,
                      "count" : 0,
                      "method" : 0,
                      "validate" : 0,
                      "invalidquery" : 0
                    },
                    "valid" : {
                      "notmodified" : 0,
                      "ok" : 39,
                      "count" : 39,
                      "accepted" : 0
                    },
                    "count" : 39
                  },
                  "unset" : 0
                },
                "aggregation" : {
                  "txmetrics" : {
                    "active_groups" : 0,
                    "overflowed" : 0
                  }
                },
                "acm" : {
                  "unset" : 0,
                  "request" : {
                    "count" : 0
                  },
                  "response" : {
                    "valid" : {
                      "count" : 0,
                      "ok" : 0,
                      "accepted" : 0,
                      "notmodified" : 0
                    },
                    "count" : 0,
                    "errors" : {
                      "internal" : 0,
                      "unauthorized" : 0,
                      "ratelimit" : 0,
                      "queue" : 0,
                      "toolarge" : 0,
                      "unavailable" : 0,
                      "invalidquery" : 0,
                      "decode" : 0,
                      "method" : 0,
                      "timeout" : 0,
                      "closed" : 0,
                      "forbidden" : 0,
                      "notfound" : 0,
                      "validate" : 0,
                      "count" : 0
                    }
                  }
                },
                "decoder" : {
                  "gzip" : {
                    "content-length" : 0,
                    "count" : 0
                  },
                  "uncompressed" : {
                    "content-length" : 0,
                    "count" : 0
                  },
                  "reader" : {
                    "count" : 0
                  },
                  "missing-content-length" : {
                    "count" : 0
                  },
                  "deflate" : {
                    "count" : 0,
                    "content-length" : 0
                  }
                },
                "profile" : {
                  "response" : {
                    "valid" : {
                      "accepted" : 0,
                      "notmodified" : 0,
                      "count" : 0,
                      "ok" : 0
                    },
                    "count" : 0,
                    "errors" : {
                      "queue" : 0,
                      "closed" : 0,
                      "decode" : 0,
                      "unauthorized" : 0,
                      "internal" : 0,
                      "timeout" : 0,
                      "forbidden" : 0,
                      "invalidquery" : 0,
                      "notfound" : 0,
                      "validate" : 0,
                      "unavailable" : 0,
                      "method" : 0,
                      "count" : 0,
                      "ratelimit" : 0,
                      "toolarge" : 0
                    }
                  },
                  "unset" : 0,
                  "request" : {
                    "count" : 0
                  }
                }
              },
              "system" : {
                "cpu" : {
                  "cores" : 32
                },
                "load" : {
                  "5" : 1.15,
                  "15" : 0.9,
                  "norm" : {
                    "5" : 0.0359,
                    "15" : 0.0281,
                    "1" : 0.0338
                  },
                  "1" : 1.08
                }
              },
              "beat" : {
                "cpu" : {
                  "user" : {
                    "ticks" : 790,
                    "time" : {
                      "ms" : 792
                    }
                  },
                  "system" : {
                    "ticks" : 330,
                    "time" : {
                      "ms" : 335
                    }
                  },
                  "total" : {
                    "value" : 1120,
                    "ticks" : 1120,
                    "time" : {
                      "ms" : 1127
                    }
                  }
                },
                "runtime" : {
                  "goroutines" : 81
                },
                "handles" : {
                  "open" : 21,
                  "limit" : {
                    "hard" : 1048576,
                    "soft" : 1048576
                  }
                },
                "info" : {
                  "version" : "7.15.0",
                  "uptime" : {
                    "ms" : 240615
                  },
                  "ephemeral_id" : "1fbaca05-9195-43d5-be2b-58b384b2e5da"
                },
                "memstats" : {
                  "memory_total" : 88490512,
                  "memory_alloc" : 17635768,
                  "memory_sys" : 79971336,
                  "gc_next" : 19815499,
                  "rss" : 124682240
                }
              }
            }
          }
        }
      }
    ]
  }
}

Metadata

Metadata

Assignees

Labels

Team:IntegrationsLabel for the Integrations teambugv7.15.0

Type

No type

Fields

No fields configured for issues without a type.

Projects

No projects

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions