Merge branch 'main' into reverting-memory-fix-for-cypress

achyutjhunjhunwala · web-flow · commit d33db45a5728 · 2023-10-02T15:21:28.000+02:00
diff --git a/.github/workflows/create-deploy-tag.yml b/.github/workflows/create-deploy-tag.yml
@@ -102,7 +102,8 @@ jobs:
               "<https://github.com/elastic/kibana/actions/runs/${{ github.run_id }}|GitHub Workflow run>",
               "<https://buildkite.com/elastic/kibana-serverless-release/builds?branch=${{ env.TAG_NAME }}|Kibana Serverless Release pipeline>",
               "<https://argo-workflows.us-central1.gcp.qa.cld.elstc.co/workflows?label=hash%3D${{ env.COMMIT }}|Argo Workflow> (use Elastic Cloud Staging VPN)",
-              "<https://overview.qa.cld.elstc.co/app/dashboards#/view/serverless-tooling-gpctl-deployment-status?_g=(refreshInterval:(pause:!t,value:0),time:(from:now-1d,to:now))&service-name=kibana&_a=(controlGroupInput:(chainingSystem:HIERARCHICAL,controlStyle:oneLine,ignoreParentSettings:(ignoreFilters:!f,ignoreQuery:!f,ignoreTimerange:!f,ignoreValidations:!f),panels:('18201b8e-3aae-4459-947d-21e007b6a3a5':(explicitInput:(dataViewId:'serverless.logs-*',enhancements:(),fieldName:commit-hash,id:'18201b8e-3aae-4459-947d-21e007b6a3a5',selectedOptions:!('${{ env.COMMIT }}'),title:commit-hash),grow:!t,order:1,type:optionsListControl,width:medium),'41060e65-ce4c-414e-b8cf-492ccb19245f':(explicitInput:(dataViewId:'serverless.logs-*',enhancements:(),fieldName:service-name,id:'41060e65-ce4c-414e-b8cf-492ccb19245f',selectedOptions:!(kibana),title:service-name),grow:!t,order:0,type:optionsListControl,width:medium),ed96828e-efe9-43ad-be3f-0e04218f79af:(explicitInput:(dataViewId:'serverless.logs-*',enhancements:(),fieldName:to-env,id:ed96828e-efe9-43ad-be3f-0e04218f79af,selectedOptions:!(qa),title:to-env),grow:!t,order:2,type:optionsListControl,width:medium))))|GPCTL Deployment Status dashboard>",
+              "<https://overview.qa.cld.elstc.co/app/dashboards#/view/serverless-tooling-gpctl-deployment-status?_g=(refreshInterval:(pause:!t,value:0),time:(from:now-1d,to:now))&service-name=kibana&_a=(controlGroupInput:(chainingSystem:HIERARCHICAL,controlStyle:oneLine,ignoreParentSettings:(ignoreFilters:!f,ignoreQuery:!f,ignoreTimerange:!f,ignoreValidations:!f),panels:('18201b8e-3aae-4459-947d-21e007b6a3a5':(explicitInput:(dataViewId:'serverless.logs-*',enhancements:(),fieldName:commit-hash,id:'18201b8e-3aae-4459-947d-21e007b6a3a5',selectedOptions:!('${{ env.COMMIT }}'),title:commit-hash),grow:!t,order:1,type:optionsListControl,width:medium),'41060e65-ce4c-414e-b8cf-492ccb19245f':(explicitInput:(dataViewId:'serverless.logs-*',enhancements:(),fieldName:service-name,id:'41060e65-ce4c-414e-b8cf-492ccb19245f',selectedOptions:!(kibana),title:service-name),grow:!t,order:0,type:optionsListControl,width:medium),ed96828e-efe9-43ad-be3f-0e04218f79af:(explicitInput:(dataViewId:'serverless.logs-*',enhancements:(),fieldName:to-env,id:ed96828e-efe9-43ad-be3f-0e04218f79af,selectedOptions:!(qa),title:to-env),grow:!t,order:2,type:optionsListControl,width:medium))))|GPCTL Deployment Status dashboard for nonprod >",
+              "<https://overview.elastic-cloud.com/app/dashboards#/view/serverless-tooling-gpctl-deployment-status?_g=(refreshInterval:(pause:!t,value:0),time:(from:now-1d,to:now))&service-name=kibana&_a=(controlGroupInput:(chainingSystem:HIERARCHICAL,controlStyle:oneLine,ignoreParentSettings:(ignoreFilters:!f,ignoreQuery:!f,ignoreTimerange:!f,ignoreValidations:!f),panels:('18201b8e-3aae-4459-947d-21e007b6a3a5':(explicitInput:(dataViewId:'serverless.logs-*',enhancements:(),fieldName:commit-hash,id:'18201b8e-3aae-4459-947d-21e007b6a3a5',selectedOptions:!('${{ env.COMMIT }}'),title:commit-hash),grow:!t,order:1,type:optionsListControl,width:medium),'41060e65-ce4c-414e-b8cf-492ccb19245f':(explicitInput:(dataViewId:'serverless.logs-*',enhancements:(),fieldName:service-name,id:'41060e65-ce4c-414e-b8cf-492ccb19245f',selectedOptions:!(kibana),title:service-name),grow:!t,order:0,type:optionsListControl,width:medium),ed96828e-efe9-43ad-be3f-0e04218f79af:(explicitInput:(dataViewId:'serverless.logs-*',enhancements:(),fieldName:to-env,id:ed96828e-efe9-43ad-be3f-0e04218f79af,selectedOptions:!(production),title:to-env),grow:!t,order:2,type:optionsListControl,width:medium))))|GPCTL Deployment Status dashboard>",
               "<https://buildkite.com/elastic/kibana-tests/builds?branch=main|Quality Gate pipeline>"
             ]
       - name: Post Slack failure message
diff --git a/x-pack/plugins/apm/public/components/app/transaction_details/waterfall_with_summary/transaction_tabs.tsx b/x-pack/plugins/apm/public/components/app/transaction_details/waterfall_with_summary/transaction_tabs.tsx
@@ -154,7 +154,7 @@ function TimelineTabContent({
 }
 
 function MetadataTabContent({ transaction }: { transaction: Transaction }) {
-  return <TransactionMetadata transactionId={transaction.transaction.id} />;
+  return <TransactionMetadata transaction={transaction} />;
 }
 
 function LogsTabContent({
diff --git a/x-pack/plugins/apm/public/components/app/transaction_details/waterfall_with_summary/waterfall_container/waterfall/span_flyout/index.tsx b/x-pack/plugins/apm/public/components/app/transaction_details/waterfall_with_summary/waterfall_container/waterfall/span_flyout/index.tsx
@@ -228,7 +228,7 @@ function SpanFlyoutBody({
       content: (
         <Fragment>
           <EuiSpacer size="m" />
-          <SpanMetadata spanId={span.span.id} />
+          <SpanMetadata span={span} />
         </Fragment>
       ),
     },
diff --git a/x-pack/plugins/apm/public/components/app/transaction_details/waterfall_with_summary/waterfall_container/waterfall/transaction_flyout/index.tsx b/x-pack/plugins/apm/public/components/app/transaction_details/waterfall_with_summary/waterfall_container/waterfall/transaction_flyout/index.tsx
@@ -141,7 +141,7 @@ function TransactionFlyoutBody({
       content: (
         <>
           <EuiSpacer size="m" />
-          <TransactionMetadata transactionId={transaction.transaction.id} />
+          <TransactionMetadata transaction={transaction} />
         </>
       ),
     },
diff --git a/x-pack/plugins/apm/public/components/shared/metadata_table/error_metadata/index.tsx b/x-pack/plugins/apm/public/components/shared/metadata_table/error_metadata/index.tsx
@@ -27,11 +27,15 @@ export function ErrorMetadata({ error }: Props) {
               processorEvent: ProcessorEvent.error,
               id: error.error.id,
             },
+            query: {
+              start: error['@timestamp'],
+              end: error['@timestamp'],
+            },
           },
         }
       );
     },
-    [error.error.id]
+    [error]
   );
 
   const sections = useMemo(
diff --git a/x-pack/plugins/apm/public/components/shared/metadata_table/span_metadata/index.tsx b/x-pack/plugins/apm/public/components/shared/metadata_table/span_metadata/index.tsx
@@ -7,15 +7,16 @@
 
 import React, { useMemo } from 'react';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
+import { Span } from '../../../../../typings/es_schemas/ui/span';
 import { getSectionsFromFields } from '../helper';
 import { MetadataTable } from '..';
 import { FETCH_STATUS, useFetcher } from '../../../../hooks/use_fetcher';
 
 interface Props {
-  spanId: string;
+  span: Span;
 }
 
-export function SpanMetadata({ spanId }: Props) {
+export function SpanMetadata({ span }: Props) {
   const { data: spanEvent, status } = useFetcher(
     (callApmApi) => {
       return callApmApi(
@@ -24,13 +25,17 @@ export function SpanMetadata({ spanId }: Props) {
           params: {
             path: {
               processorEvent: ProcessorEvent.span,
-              id: spanId,
+              id: span.span.id,
+            },
+            query: {
+              start: span['@timestamp'],
+              end: span['@timestamp'],
             },
           },
         }
       );
     },
-    [spanId]
+    [span]
   );
 
   const sections = useMemo(
diff --git a/x-pack/plugins/apm/public/components/shared/metadata_table/transaction_metadata/index.tsx b/x-pack/plugins/apm/public/components/shared/metadata_table/transaction_metadata/index.tsx
@@ -7,15 +7,16 @@
 
 import React, { useMemo } from 'react';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
+import { Transaction } from '../../../../../typings/es_schemas/ui/transaction';
 import { getSectionsFromFields } from '../helper';
 import { MetadataTable } from '..';
 import { FETCH_STATUS, useFetcher } from '../../../../hooks/use_fetcher';
 
 interface Props {
-  transactionId: string;
+  transaction: Transaction;
 }
 
-export function TransactionMetadata({ transactionId }: Props) {
+export function TransactionMetadata({ transaction }: Props) {
   const { data: transactionEvent, status } = useFetcher(
     (callApmApi) => {
       return callApmApi(
@@ -24,13 +25,17 @@ export function TransactionMetadata({ transactionId }: Props) {
           params: {
             path: {
               processorEvent: ProcessorEvent.transaction,
-              id: transactionId,
+              id: transaction.transaction.id,
+            },
+            query: {
+              start: transaction['@timestamp'],
+              end: transaction['@timestamp'],
             },
           },
         }
       );
     },
-    [transactionId]
+    [transaction]
   );
 
   const sections = useMemo(
diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts
@@ -11,6 +11,7 @@ import {
   Logger,
   Plugin,
   PluginInitializerContext,
+  SavedObjectsClient,
 } from '@kbn/core/server';
 import { isEmpty, mapValues } from 'lodash';
 import { Dataset } from '@kbn/rule-registry-plugin/server';
@@ -48,6 +49,7 @@ import { scheduleSourceMapMigration } from './routes/source_maps/schedule_source
 import { createApmSourceMapIndexTemplate } from './routes/source_maps/create_apm_source_map_index_template';
 import { addApiKeysToEveryPackagePolicyIfMissing } from './routes/fleet/api_keys/add_api_keys_to_policies_if_missing';
 import { apmTutorialCustomIntegration } from '../common/tutorial/tutorials';
+import { APM_STATIC_DATA_VIEW_ID } from '../common/data_view_constants';
 
 export class APMPlugin
   implements
@@ -119,6 +121,26 @@ export class APMPlugin
       ],
     });
 
+    // ensure that the APM data view is globally available
+    getCoreStart()
+      .then(async (coreStart) => {
+        const soClient = new SavedObjectsClient(
+          coreStart.savedObjects.createInternalRepository()
+        );
+
+        await soClient.updateObjectsSpaces(
+          [{ id: APM_STATIC_DATA_VIEW_ID, type: 'index-pattern' }],
+          ['*'],
+          []
+        );
+      })
+      .catch((e) => {
+        this.logger?.error(
+          'Failed to make APM data view available globally',
+          e
+        );
+      });
+
     const resourcePlugins = mapValues(plugins, (value, key) => {
       return {
         setup: value,
diff --git a/x-pack/plugins/apm/server/routes/event_metadata/get_event_metadata.ts b/x-pack/plugins/apm/server/routes/event_metadata/get_event_metadata.ts
@@ -5,8 +5,8 @@
  * 2.0.
  */
 
-import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
+import { rangeQuery } from '@kbn/observability-plugin/server';
 import {
   ERROR_ID,
   SPAN_ID,
@@ -18,43 +18,26 @@ export async function getEventMetadata({
   apmEventClient,
   processorEvent,
   id,
+  start,
+  end,
 }: {
   apmEventClient: APMEventClient;
   processorEvent: ProcessorEvent;
   id: string;
+  start: number;
+  end: number;
 }) {
-  const filter: QueryDslQueryContainer[] = [];
-
-  switch (processorEvent) {
-    case ProcessorEvent.error:
-      filter.push({
-        term: { [ERROR_ID]: id },
-      });
-      break;
-
-    case ProcessorEvent.transaction:
-      filter.push({
-        term: {
-          [TRANSACTION_ID]: id,
-        },
-      });
-      break;
-
-    case ProcessorEvent.span:
-      filter.push({
-        term: { [SPAN_ID]: id },
-      });
-      break;
-  }
-
+  const fieldName = getFieldName(processorEvent);
   const response = await apmEventClient.search('get_event_metadata', {
     apm: {
       events: [processorEvent],
     },
     body: {
       track_total_hits: false,
       query: {
-        bool: { filter },
+        bool: {
+          filter: [...rangeQuery(start, end), { term: { [fieldName]: id } }],
+        },
       },
       size: 1,
       _source: false,
@@ -65,3 +48,19 @@ export async function getEventMetadata({
 
   return response.hits.hits[0].fields;
 }
+
+function getFieldName(processorEvent: ProcessorEvent) {
+  switch (processorEvent) {
+    case ProcessorEvent.error:
+      return ERROR_ID;
+
+    case ProcessorEvent.transaction:
+      return TRANSACTION_ID;
+
+    case ProcessorEvent.span:
+      return SPAN_ID;
+
+    default:
+      throw new Error('Unknown processor event');
+  }
+}
diff --git a/x-pack/plugins/apm/server/routes/event_metadata/route.ts b/x-pack/plugins/apm/server/routes/event_metadata/route.ts
@@ -10,6 +10,7 @@ import { createApmServerRoute } from '../apm_routes/create_apm_server_route';
 import { getEventMetadata } from './get_event_metadata';
 import { processorEventRt } from '../../../common/processor_event';
 import { getApmEventClient } from '../../lib/helpers/get_apm_event_client';
+import { rangeRt } from '../default_api_types';
 
 const eventMetadataRoute = createApmServerRoute({
   endpoint: 'GET /internal/apm/event_metadata/{processorEvent}/{id}',
@@ -19,20 +20,22 @@ const eventMetadataRoute = createApmServerRoute({
       processorEvent: processorEventRt,
       id: t.string,
     }),
+    query: rangeRt,
   }),
   handler: async (
     resources
   ): Promise<{ metadata: Partial<Record<string, unknown[]>> }> => {
     const apmEventClient = await getApmEventClient(resources);
-
-    const {
-      path: { processorEvent, id },
-    } = resources.params;
+    const { params } = resources;
+    const { start, end } = params.query;
+    const { processorEvent, id } = params.path;
 
     const metadata = await getEventMetadata({
       apmEventClient,
       processorEvent,
       id,
+      start,
+      end,
     });
 
     return {
diff --git a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts
@@ -81,7 +81,7 @@ export const FLEET_COMPONENT_TEMPLATES = [
   },
 ];
 
-export const FLEET_FINAL_PIPELINE_VERSION = 3;
+export const FLEET_FINAL_PIPELINE_VERSION = 4;
 
 // If the content is updated you probably need to update the FLEET_FINAL_PIPELINE_VERSION too to allow upgrade of the pipeline
 export const FLEET_FINAL_PIPELINE_CONTENT = `---
@@ -92,21 +92,28 @@ _meta:
 description: >
   Final pipeline for processing all incoming Fleet Agent documents.
 processors:
-  - date:
+  - script:
       description: Add time when event was ingested (and remove sub-seconds to improve storage efficiency)
       tag: truncate-subseconds-event-ingested
-      field: _ingest.timestamp
-      target_field: event.ingested
-      formats:
-        - ISO8601
-      output_format: date_time_no_millis
       ignore_failure: true
+      source: |-
+        if (ctx?.event == null) {
+          ctx.event = [:];
+        }
+
+        ctx.event.ingested = metadata().now.withNano(0).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);
   - remove:
       description: Remove any pre-existing untrusted values.
       field:
         - event.agent_id_status
         - _security
       ignore_missing: true
+  - remove:
+      description: Remove event.original unless the preserve_original_event tag is set
+      field: event.original
+      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+      ignore_failure: true
+      ignore_missing: true
   - set_security_user:
       field: _security
       properties:
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/create_esql_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/create_esql_alert_type.ts
@@ -7,6 +7,7 @@
 
 import { validateNonExact } from '@kbn/securitysolution-io-ts-utils';
 import { ESQL_RULE_TYPE_ID } from '@kbn/securitysolution-rules';
+import { DEFAULT_APP_CATEGORIES } from '@kbn/core-application-common';
 
 import { SERVER_APP_ID } from '../../../../../common/constants';
 import type { EsqlRuleParams } from '../../rule_schema';
@@ -47,6 +48,7 @@ export const createEsqlAlertType = (
     },
     minimumLicenseRequired: 'basic',
     isExportable: false,
+    category: DEFAULT_APP_CATEGORIES.security.id,
     producer: SERVER_APP_ID,
     executor: (params) => esqlExecutor({ ...params, version }),
   };
diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/components/settings/private_locations/location_form.tsx b/x-pack/plugins/synthetics/public/apps/synthetics/components/settings/private_locations/location_form.tsx
diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/components/settings/private_locations/policy_hosts.tsx b/x-pack/plugins/synthetics/public/apps/synthetics/components/settings/private_locations/policy_hosts.tsx
diff --git a/x-pack/test/apm_api_integration/tests/event_metadata/event_metadata.spec.ts b/x-pack/test/apm_api_integration/tests/event_metadata/event_metadata.spec.ts
diff --git a/x-pack/test/fleet_api_integration/apis/agent_policy/agent_policy.ts b/x-pack/test/fleet_api_integration/apis/agent_policy/agent_policy.ts
diff --git a/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts b/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts
diff --git a/x-pack/test/functional/services/cases/list.ts b/x-pack/test/functional/services/cases/list.ts

Original file line number	Diff line number	Diff line change
`@@ -154,7 +154,7 @@ function TimelineTabContent({`
`154`	`154`	`}`
`155`	`155`
`156`	`156`	`function MetadataTabContent({ transaction }: { transaction: Transaction }) {`
`157`		`- return <TransactionMetadata transactionId={transaction.transaction.id} />;`
	`157`	`+ return <TransactionMetadata transaction={transaction} />;`
`158`	`158`	`}`
`159`	`159`
`160`	`160`	`function LogsTabContent({`