[APM] Collect telemetry about data/API performance by dgieselaar · Pull Request #51612 · elastic/kibana

dgieselaar · 2019-11-25T15:31:40Z

An example of collected telemetry:

{
  "counts": {
    "error": {
      "1d": 162227,
      "all": 6809905
    },
    "metric": {
      "1d": 975978,
      "all": 48222846
    },
    "span": {
      "1d": 10592884,
      "all": 483896330
    },
    "transaction": {
      "1d": 1950203,
      "all": 92890244
    },
    "onboarding": {
      "1d": 0,
      "all": 7
    },
    "sourcemap": {
      "1d": 0,
      "all": 0
    },
    "agent_configuration": {
      "all": 3
    },
    "max_error_groups_per_service": {
      "1d": 1838
    },
    "max_transaction_groups_per_service": {
      "1d": 25
    },
    "traces": {
      "1d": 1483642
    },
    "services": {
      "1d": 8
    }
  },
  "retainment": {
    "error": {
      "ms": 4445241078
    },
    "metric": {
      "ms": 4445248747
    },
    "span": {
      "ms": 4445259937
    },
    "transaction": {
      "ms": 4445275561
    },
    "onboarding": {
      "ms": 4445288673
    }
  },
  "tasks": {
    "processor_events": {
      "took": {
        "ms": 32468
      }
    },
    "agent_configuration": {
      "took": {
        "ms": 19
      }
    },
    "services": {
      "took": {
        "ms": 3339
      }
    },
    "versions": {
      "took": {
        "ms": 102
      }
    },
    "groupings": {
      "took": {
        "ms": 3462
      }
    },
    "integrations": {
      "took": {
        "ms": 51
      }
    },
    "agents": {
      "took": {
        "ms": 12967
      }
    },
    "indices_stats": {
      "took": {
        "ms": 71
      }
    }
  },
  "has_any_services": true,
  "services_per_agent": {
    "java": 1,
    "js-base": 1,
    "rum-js": 0,
    "dotnet": 1,
    "go": 2,
    "nodejs": 1,
    "python": 1,
    "ruby": 1
  },
  "versions": {
    "apm_server": {
      "major": 8,
      "minor": 0,
      "patch": 0
    }
  },
  "integrations": {
    "ml": {
      "all_jobs_count": 1
    }
  },
  "agents": {
    "java": {
      "agent": {
        "version": [
          "1.13.1-SNAPSHOT"
        ]
      },
      "service": {
        "framework": {
          "name": [],
          "version": [],
          "composite": []
        },
        "language": {
          "name": [
            "Java"
          ],
          "version": [
            "10.0.2"
          ],
          "composite": [
            "Java/10.0.2"
          ]
        },
        "runtime": {
          "name": [
            "Java"
          ],
          "version": [
            "10.0.2"
          ],
          "composite": [
            "Java/10.0.2"
          ]
        }
      }
    },
    "js-base": {
      "agent": {
        "version": [
          "4.9.0"
        ]
      },
      "service": {
        "framework": {
          "name": [],
          "version": [],
          "composite": []
        },
        "language": {
          "name": [
            "javascript"
          ],
          "version": [],
          "composite": []
        },
        "runtime": {
          "name": [],
          "version": [],
          "composite": []
        }
      }
    },
    "rum-js": {
      "agent": {
        "version": []
      },
      "service": {
        "framework": {
          "name": [],
          "version": [],
          "composite": []
        },
        "language": {
          "name": [],
          "version": [],
          "composite": []
        },
        "runtime": {
          "name": [],
          "version": [],
          "composite": []
        }
      }
    },
    "dotnet": {
      "agent": {
        "version": [
          "1.3.1"
        ]
      },
      "service": {
        "framework": {
          "name": [
            "ASP.NET Core"
          ],
          "version": [
            "2.2.0.0"
          ],
          "composite": [
            "ASP.NET Core/2.2.0.0"
          ]
        },
        "language": {
          "name": [
            "C#"
          ],
          "version": [],
          "composite": []
        },
        "runtime": {
          "name": [
            ".NET Core"
          ],
          "version": [
            "2.2.8"
          ],
          "composite": [
            ".NET Core/2.2.8"
          ]
        }
      }
    },
    "go": {
      "agent": {
        "version": [
          "1.7.0",
          "1.7.1"
        ]
      },
      "service": {
        "framework": {
          "name": [
            "gin"
          ],
          "version": [
            "v1.4.0"
          ],
          "composite": [
            "gin/v1.4.0"
          ]
        },
        "language": {
          "name": [
            "go"
          ],
          "version": [
            "go1.13.8",
            "go1.14"
          ],
          "composite": [
            "go/go1.13.8",
            "go/go1.14"
          ]
        },
        "runtime": {
          "name": [
            "gc"
          ],
          "version": [
            "go1.13.8",
            "go1.14"
          ],
          "composite": [
            "gc/go1.13.8",
            "gc/go1.14"
          ]
        }
      }
    },
    "nodejs": {
      "agent": {
        "version": [
          "3.5.0"
        ]
      },
      "service": {
        "framework": {
          "name": [
            "express"
          ],
          "version": [
            "4.17.1"
          ],
          "composite": [
            "express/4.17.1"
          ]
        },
        "language": {
          "name": [
            "javascript"
          ],
          "version": [],
          "composite": []
        },
        "runtime": {
          "name": [
            "node"
          ],
          "version": [
            "12.16.1"
          ],
          "composite": [
            "node/12.16.1"
          ]
        }
      }
    },
    "python": {
      "agent": {
        "version": [
          "5.4.3"
        ]
      },
      "service": {
        "framework": {
          "name": [
            "django"
          ],
          "version": [
            "2.1.13"
          ],
          "composite": [
            "django/2.1.13"
          ]
        },
        "language": {
          "name": [
            "python"
          ],
          "version": [
            "3.6.10"
          ],
          "composite": [
            "python/3.6.10"
          ]
        },
        "runtime": {
          "name": [
            "CPython"
          ],
          "version": [
            "3.6.10"
          ],
          "composite": [
            "CPython/3.6.10"
          ]
        }
      }
    },
    "ruby": {
      "agent": {
        "version": [
          "3.5.0"
        ]
      },
      "service": {
        "framework": {
          "name": [
            "Ruby on Rails"
          ],
          "version": [
            "5.2.4.1"
          ],
          "composite": [
            "Ruby on Rails/5.2.4.1"
          ]
        },
        "language": {
          "name": [
            "ruby"
          ],
          "version": [
            "2.6.5"
          ],
          "composite": [
            "ruby/2.6.5"
          ]
        },
        "runtime": {
          "name": [
            "ruby"
          ],
          "version": [
            "2.6.5"
          ],
          "composite": [
            "ruby/2.6.5"
          ]
        }
      }
    }
  },
  "indices": {
    "shards": {
      "total": 24
    },
    "all": {
      "total": {
        "docs": {
          "count": 1135679324
        },
        "store": {
          "size_in_bytes": 570622947437
        }
      }
    }
  }
}

elasticmachine · 2019-11-25T16:59:19Z

💔 Build Failed

continuous-integration/kibana-ci/pull-request
Commit: 0d39b5ac589526014f9f7e9e55dcb1c03a7ea1f2

dgieselaar · 2019-11-27T09:02:05Z

Spoke to the telemetry folks, we should wrap this in a task that stores data to saved objects (similar to what we had), and we should investigate more aggressive pre-processing on our side (e.g., min/max/avg/percentiles)

elasticmachine · 2019-12-03T13:09:47Z

💔 Build Failed

continuous-integration/kibana-ci/pull-request
Commit: d83b37ec830f08ab9d0b23b89df7ac365ac26080

elasticmachine · 2019-12-05T13:22:10Z

💔 Build Failed

continuous-integration/kibana-ci/pull-request
Commit: a2d96cc728d1df322f2d1fc35f8450e6f4d5f810

elasticmachine · 2020-02-07T10:50:01Z

Pinging @elastic/apm-ui (Team:apm)

x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/download-telemetry-mapping.ts

TinaHeiligers

I'm concerned over the current implementation, specifically about:

writing directly to the xpack-phone-home index
using a dynamic mapping for the apm data that's being overwritten in the xpack-phone-home index which is, on purpose, not mapped dynamically.
At the moment, the xpack-home-phone template is updated when new mappings are added
Adding yet another collector

I'd like some other members of the Pulse team to take a look before specifically approving/requesting changes and, while I realize the target is for 7.7, I hope you understand the concerns.

TinaHeiligers · 2020-03-16T23:39:22Z

x-pack/legacy/plugins/apm/mappings.json

Dynamic mappings aren't set within the current telemetry service. The properties have to be defined explicitly.

As far as I understand these mappings are for the SavedObjects. It should not cause any effect on the telemetry service. But it's true I think it's considered an antipattern the dynamic: true mappings in the SavedObjects as well (maybe the @elastic/kibana-platform team can confirm).
@dgieselaar mentioned it was dynamic only during the development phase. Just please don't forget to change this to the actual mapping when done :)

The reason for having dynamic:true was given in a Slack message:

...the dynamic mapping is temporary. Once the data that we are collecting is finalized I was planning on adding a static mapping

The mapping for saved objects is no longer dynamic, thanks!

x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/index.ts

x-pack/legacy/plugins/apm/server/lib/apm_telemetry/types.ts

afharo

Overall, it looks great! I only added some FYIs and nits.

The only main thing I'd personally don't do here is the upload-telemetry-data script. Unless there's a reason to have it I can't think of at the moment.

afharo · 2020-03-17T09:45:09Z

x-pack/legacy/plugins/apm/common/agent_name.ts

Suggested change

export function isAgentName(agentName: string): boolean {

export function isAgentName(agentName: string): agentName is AgentName {

afharo · 2020-03-17T09:45:48Z

x-pack/legacy/plugins/apm/common/agent_name.ts

Suggested change

): agentName is AgentName {

): agentName is 'js-base' | 'rum-js' {

afharo · 2020-03-17T09:47:11Z

x-pack/legacy/plugins/apm/common/agent_name.ts

afharo · 2020-03-17T09:55:22Z

x-pack/legacy/plugins/apm/mappings.json

As far as I understand these mappings are for the SavedObjects. It should not cause any effect on the telemetry service. But it's true I think it's considered an antipattern the dynamic: true mappings in the SavedObjects as well (maybe the @elastic/kibana-platform team can confirm).
@dgieselaar mentioned it was dynamic only during the development phase. Just please don't forget to change this to the actual mapping when done :)

x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/index.ts

afharo · 2020-03-17T10:48:59Z

x-pack/legacy/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts

see other reply

afharo · 2020-03-17T10:57:02Z

x-pack/legacy/plugins/apm/server/lib/apm_telemetry/index.ts

I love this taskManager approach to get all the stats ready to consume from the savedObject at any point when the usage collector requires it.

Especially taking into account all the requests we do to retrieve this telemetry. That's a good amount of information! Well done! 👏

Just FYI, we usually report telemetry every 24h so, maybe running the collection logic every minute is a bit too much? ;)

changed this to 24h - another artifact of "dev mode" 😅

btw, let me know if it should be anything else than 24h

Changed it to 720m (12 hours, the task manager only supports seconds and minutes)

afharo · 2020-03-17T11:00:14Z

x-pack/legacy/plugins/apm/server/lib/apm_telemetry/index.ts

There is a risk we might delete existing telemetry because of
https://github.com/elastic/kibana/blob/caee49b307bf9a6c8c1c146b1a7c35fd044fd894/x-pack/legacy/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/index.ts#L64-L68

But the chances should be low so I wouldn't worry too much about it (just a heads up) ;)

afharo · 2020-03-17T11:09:09Z

x-pack/legacy/plugins/apm/server/lib/apm_telemetry/types.ts

It's amazing to have this! This will help us much to create the mapping in the telemetry cluster! Thanks for being so thoughtful and strict with the types! ❤️

afharo · 2020-03-17T11:16:51Z

x-pack/plugins/apm/kibana.json

the recommendation is to import this plugin as optional:
https://github.com/elastic/kibana/blob/master/src/plugins/usage_collection/README.md#new-platform

done, thanks!

TinaHeiligers · 2020-03-17T15:16:25Z

Overall, it looks great! I only added some FYIs and nits.

The only main thing I'd personally don't do here is the upload-telemetry-data script. Unless there's a reason to have it I can't think of at the moment.

From a Slack message:

...script that collects data and writes it to a specified index, but that’s just for testing purposes

afharo

LGTM!

afharo · 2020-03-19T15:13:09Z

x-pack/legacy/plugins/apm/mappings.json

thank you! this is so awesome will be so useful when we add the mapping in the telemetry cluster. Please refer to this file when requesting it :)

afharo · 2020-03-19T15:13:44Z

x-pack/legacy/plugins/apm/mappings.json

I wonder if the removal of the "null_value" might cause any issues?

Feel free to dismiss this if this change is intended :)

good catch - I don't think we need a fallback anymore but I'll put it there to prevent any backwards compat issues.

sorenlouv

lgtm 👍

sorenlouv · 2020-03-19T22:09:00Z

x-pack/legacy/plugins/apm/scripts/upload-telemetry-data.js

Maybe add a description to this script about what it does

aye, I could do it in this file, in dev_docs or in a README.md file in this folder, any preferences?

sorenlouv · 2020-03-19T22:16:50Z

x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/index.ts

Why is githubToken not required? If the repo is private I'd expect this to require a token.

Yeah, I'll just fail the process in that case. Can't remember why I made it optional.

sorenlouv · 2020-03-19T22:17:59Z

x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/index.ts

Perhaps it would be cleaner with safeLoad done separately?
Also: shouldn't elasticsearch.hosts be in the explicit type?

I'll split it up a bit. elasticsearch.hosts is already in the type because it has a default value, but it doesn't hurt to have it in both, I'll add it there as well (it looks odd).

sorenlouv · 2020-03-19T22:24:21Z

x-pack/plugins/apm/common/agent_name.ts

sorenlouv · 2020-03-19T22:26:51Z

x-pack/plugins/apm/common/elasticsearch_fieldnames.ts

I think SERVICE_AGENT_NAME and SERVICE_AGENT_VERSION should be called AGENT_NAME and AGENT_VERSION since they are no longer on the "service" context (they used to be afair)

ah yeah was wondering why that was the case. I'll update

sorenlouv · 2020-03-19T22:33:33Z

x-pack/plugins/apm/server/plugin.ts

Why no longer in the context of 'apm'?

it's already scoped, so scoping it again leads to [apm] [apm] output

Closes elastic#50757.

afharo · 2020-03-23T11:10:28Z

x-pack/legacy/plugins/apm/index.ts

+        serviceMapEnabled: Joi.boolean().default(true),
+
+        // telemetry
+        telemetryCollectionEnabled: Joi.boolean().default(true)


do we want a specific flag to disable APM-only telemetry (while the rest of the telemetry is enabled)? Wouldn't that complicate the analysis about the usage? aka: how can I tell APM is used but not reporting telemetry because it's disabled?
We could do that by checking the application_usage.apm stats... but I'd like @alexfrancoeur thoughts on this.

Would be great to get some additional feedback 👍🏻the reason why I've added it because we want an escape hatch in case the queries are too expensive, without requiring our users to disable telemetry altogether in those cases.

@afharo If you don't mind, I'll merge this in given the time constraints, but let me know if there's anything we need to change.

kibanamachine · 2020-03-23T11:55:02Z

💚 Build Succeeded

continuous-integration/kibana-ci/pull-request
Commit: bf7a247

History

💚 Build #35299 succeeded 72b7ae9
💔 Build #34932 failed cd8d7531bbbb8f514384352429d79d0f5ca6d067
💔 Build #34894 failed 642543faf31ed70b1b06c409286eb42778731cc9
💔 Build #34837 failed cabc3a036bf201891701d0f3a10f61cc9ff39658
💔 Build #34670 failed aa917d7cc574c3dd5909f7347dfe1b98bdf577d0

To update your PR or re-run it, just comment with:
@elasticmachine merge upstream

spalger

yarn.lock config changes LGTM

This reverts commit 13baa51.

spalger · 2020-03-24T05:28:12Z

This was causing type check failures in master once it was merged so it was reverted in 6de7f2a

* master: (34 commits) [APM] add service map config options to legacy plugin (elastic#61002) [App Arch] migrate legacy CSS to new platform (core_plugins/kibana_react) (elastic#59882) Migrated styles for "share" plugin to new platform (elastic#59981) [ML] Module setup with dynamic model memory estimation (elastic#60656) Drilldowns (elastic#59632) Upgrade mocha dev-dependency from 6.2.2 to 7.1.1 (elastic#60779) [SIEM] Overview: Recent cases widget (elastic#60993) [ML] Functional tests - stabilize df analytics clone tests (elastic#60497) [SIEM] Updates process and TLS tables to use ECS 1.5 fields (elastic#60854) Migrate doc view part of discover (elastic#58094) Revert "[APM] Collect telemetry about data/API performance (elastic#51612)" fix(NA): log rotation watchers usage (elastic#60956) [SIEM] [CASES] Build lego blocks case details view (elastic#60864) Create Painless Lab app (elastic#57538) [SIEM] Move Timeline Template field to first step of rule creation (elastic#60840) [Reporting/New Platform Migration] Use a new config service on server-side (elastic#55882) [Alerting] allow email action to not require auth (elastic#60839) [Maps] Default ES document layer scaling type to clusters and show scaling UI in the create wizard (elastic#60668) [APM] Collect telemetry about data/API performance (elastic#51612) Implement Kibana Login Selector (elastic#53010) ...

…lastic#51612)"" This reverts commit 6de7f2a.

* Revert "Revert "[APM] Collect telemetry about data/API performance (#51612)"" This reverts commit 6de7f2a. * Update transaction mock data to reflect the type

…ic#61030) * Revert "Revert "[APM] Collect telemetry about data/API performance (elastic#51612)"" This reverts commit 6de7f2a. * Update transaction mock data to reflect the type

* Revert "Revert "[APM] Collect telemetry about data/API performance (#51612)"" This reverts commit 6de7f2a. * Update transaction mock data to reflect the type

dgieselaar force-pushed the apm-data-telemetry branch from 07a0d9b to 0d39b5a Compare November 25, 2019 15:36

dgieselaar force-pushed the apm-data-telemetry branch from 0d39b5a to d83b37e Compare December 3, 2019 11:56

dgieselaar mentioned this pull request Dec 5, 2019

[APM] Service Map data background task and API endpoint #50844

Closed

dgieselaar force-pushed the apm-data-telemetry branch from d83b37e to a2d96cc Compare December 5, 2019 12:05

dgieselaar force-pushed the apm-data-telemetry branch from a2d96cc to caee49b Compare January 3, 2020 12:31

streamich added Feature:Telemetry Team:APM - DEPRECATED Use Team:obs-ux-infra_services. labels Feb 7, 2020

kobelb mentioned this pull request Mar 10, 2020

[APM] Collect telemetry about data/queries #50757

Closed

TinaHeiligers requested review from a team and Bamieh and removed request for Bamieh March 16, 2020 20:45

TinaHeiligers reviewed Mar 16, 2020

View reviewed changes

x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/download-telemetry-mapping.ts Outdated Show resolved Hide resolved

TinaHeiligers reviewed Mar 17, 2020

View reviewed changes

afharo reviewed Mar 17, 2020

View reviewed changes

dgieselaar force-pushed the apm-data-telemetry branch 4 times, most recently from 5b0c188 to 2197df8 Compare March 18, 2020 19:43

dgieselaar marked this pull request as ready for review March 18, 2020 20:44

dgieselaar requested a review from a team March 18, 2020 20:44

dgieselaar added release_note:enhancement v7.7.0 labels Mar 18, 2020

dgieselaar force-pushed the apm-data-telemetry branch from 65f17d2 to 55355a6 Compare March 19, 2020 07:33

afharo approved these changes Mar 19, 2020

View reviewed changes

sorenlouv approved these changes Mar 19, 2020

View reviewed changes

dgieselaar force-pushed the apm-data-telemetry branch 3 times, most recently from 642543f to cd8d753 Compare March 20, 2020 15:00

dgieselaar added 2 commits March 23, 2020 09:13

[APM] Collect telemetry about data/API performance

cb9118a

Closes elastic#50757.

Ignore apm scripts package.json

72b7ae9

dgieselaar force-pushed the apm-data-telemetry branch from cd8d753 to 72b7ae9 Compare March 23, 2020 08:20

dgieselaar requested a review from a team as a code owner March 23, 2020 08:20

Config flag for enabling/disabling telemetry collection

bf7a247

afharo reviewed Mar 23, 2020

View reviewed changes

spalger approved these changes Mar 23, 2020

View reviewed changes

dgieselaar merged commit 13baa51 into elastic:master Mar 23, 2020

dgieselaar deleted the apm-data-telemetry branch March 23, 2020 22:08

spalger added a commit that referenced this pull request Mar 24, 2020

Revert "[APM] Collect telemetry about data/API performance (#51612)"

6de7f2a

This reverts commit 13baa51.

spalger added the reverted label Mar 24, 2020

spalger added the backport:skip This PR does not require backporting label Mar 24, 2020

dgieselaar mentioned this pull request Mar 24, 2020

[APM] Re-revert "Collect telemetry about data/API performance" #61030

Merged

dgieselaar added a commit to dgieselaar/kibana that referenced this pull request Mar 24, 2020

Revert "Revert "[APM] Collect telemetry about data/API performance (e…

146a149

…lastic#51612)"" This reverts commit 6de7f2a.

cauemarcondes added the apm-test-plan-7.7.0 label Mar 30, 2020

ogupte self-assigned this Apr 5, 2020

ogupte mentioned this pull request Apr 5, 2020

[APM] Service telemetry missing rum-js agent #62559

Closed

dgieselaar mentioned this pull request Apr 15, 2020

[APM] Telemetry: Report whether a given cluster has enabled the ML integration for APM #41772

Closed

	export function isAgentName(agentName: string): boolean {
	export function isAgentName(agentName: string): agentName is AgentName {

	): agentName is AgentName {
	): agentName is 'js-base' \| 'rum-js' {

Conversation

dgieselaar commented Nov 25, 2019 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

elasticmachine commented Nov 25, 2019

💔 Build Failed

Uh oh!

dgieselaar commented Nov 27, 2019 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

elasticmachine commented Dec 3, 2019

💔 Build Failed

Uh oh!

elasticmachine commented Dec 5, 2019

💔 Build Failed

Uh oh!

elasticmachine commented Feb 7, 2020

Uh oh!

Uh oh!

TinaHeiligers left a comment

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Uh oh!

Uh oh!

afharo left a comment

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

TinaHeiligers commented Mar 17, 2020

Uh oh!

afharo left a comment

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Choose a reason for hiding this comment

Uh oh!

sorenlouv left a comment

dgieselaar commented Nov 25, 2019 •

edited

Loading

dgieselaar commented Nov 27, 2019 •

edited

Loading