Merge branch '7.9' into backport/7.9/pr-74409

elasticmachine · web-flow · commit 39ec06511bbf · 2020-09-08T07:44:55.000-06:00
diff --git a/package.json b/package.json
@@ -92,6 +92,7 @@
     "**/istanbul-instrumenter-loader/schema-utils": "1.0.0",
     "**/image-diff/gm/debug": "^2.6.9",
     "**/load-grunt-config/lodash": "^4.17.20",
+    "**/node-jose/node-forge": "^0.10.0",
     "**/react-dom": "^16.12.0",
     "**/react": "^16.12.0",
     "**/react-test-renderer": "^16.12.0",
@@ -228,7 +229,7 @@
     "mustache": "2.3.2",
     "ngreact": "0.5.1",
     "node-fetch": "1.7.3",
-    "node-forge": "^0.9.1",
+    "node-forge": "^0.10.0",
     "opn": "^5.5.0",
     "oppsy": "^2.0.0",
     "pegjs": "0.10.0",
@@ -365,7 +366,7 @@
     "@types/moment-timezone": "^0.5.12",
     "@types/mustache": "^0.8.31",
     "@types/node": ">=10.17.17 <10.20.0",
-    "@types/node-forge": "^0.9.0",
+    "@types/node-forge": "^0.9.5",
     "@types/normalize-path": "^3.0.0",
     "@types/opn": "^5.1.0",
     "@types/pegjs": "^0.10.1",
diff --git a/src/legacy/server/logging/log_format.js b/src/legacy/server/logging/log_format.js
@@ -91,7 +91,7 @@ export default class TransformObjStream extends Stream.Transform {
         method: event.method || '',
         headers: event.headers,
         remoteAddress: source.remoteAddress,
-        userAgent: source.remoteAddress,
+        userAgent: source.userAgent,
         referer: source.referer,
       };
 
diff --git a/src/legacy/server/logging/log_format_json.test.js b/src/legacy/server/logging/log_format_json.test.js
@@ -65,12 +65,14 @@ describe('KbnLoggerJsonFormat', () => {
         },
       };
       const result = await createPromiseFromStreams([createListStream([event]), format]);
-      const { type, method, statusCode, message } = JSON.parse(result);
+      const { type, method, statusCode, message, req } = JSON.parse(result);
 
       expect(type).toBe('response');
       expect(method).toBe('GET');
       expect(statusCode).toBe(200);
       expect(message).toBe('GET /path/to/resource 200 12000ms - 13.0B');
+      expect(req.remoteAddress).toBe('127.0.0.1');
+      expect(req.userAgent).toBe('Test Thing');
     });
 
     it('ops', async () => {
diff --git a/x-pack/plugins/index_management/server/lib/fetch_indices.ts b/x-pack/plugins/index_management/server/lib/fetch_indices.ts
@@ -44,7 +44,9 @@ async function fetchIndicesCall(
   // This call retrieves alias and settings (incl. hidden status) information about indices
   const indices: GetIndicesResponse = await callAsCurrentUser('transport.request', {
     method: 'GET',
-    path: `/${indexNamesString}`,
+    // transport.request doesn't do any URI encoding, unlike other JS client APIs. This enables
+    // working with Logstash indices with names like %{[@metadata][beat]}-%{[@metadata][version]}.
+    path: `/${encodeURIComponent(indexNamesString)}`,
     query: {
       expand_wildcards: 'hidden,all',
     },
diff --git a/x-pack/plugins/logstash/public/application/components/pipeline_editor/pipeline_editor.js b/x-pack/plugins/logstash/public/application/components/pipeline_editor/pipeline_editor.js
@@ -125,11 +125,11 @@ class PipelineEditorUi extends React.Component {
 
   onPipelineSave = () => {
     const { pipelineService, toastNotifications, intl } = this.props;
-    const { id } = this.state.pipeline;
+    const { id, ...pipelineToStore } = this.state.pipeline;
     return pipelineService
       .savePipeline({
         id,
-        upstreamJSON: this.state.pipeline,
+        upstreamJSON: pipelineToStore,
       })
       .then(() => {
         toastNotifications.addSuccess(
diff --git a/x-pack/plugins/logstash/server/models/pipeline/pipeline.ts b/x-pack/plugins/logstash/server/models/pipeline/pipeline.ts
@@ -11,14 +11,14 @@ import { i18n } from '@kbn/i18n';
 
 interface PipelineOptions {
   id: string;
-  description: string;
+  description?: string;
   pipeline: string;
   username?: string;
   settings?: Record<string, any>;
 }
 
 interface DownstreamPipeline {
-  description: string;
+  description?: string;
   pipeline: string;
   settings?: Record<string, any>;
 }
@@ -27,7 +27,7 @@ interface DownstreamPipeline {
  */
 export class Pipeline {
   public readonly id: string;
-  public readonly description: string;
+  public readonly description?: string;
   public readonly username?: string;
   public readonly pipeline: string;
   private readonly settings: Record<string, any>;
diff --git a/x-pack/plugins/logstash/server/routes/pipeline/save.ts b/x-pack/plugins/logstash/server/routes/pipeline/save.ts
@@ -22,8 +22,7 @@ export function registerPipelineSaveRoute(router: IRouter, security?: SecurityPl
           id: schema.string(),
         }),
         body: schema.object({
-          id: schema.string(),
-          description: schema.string(),
+          description: schema.maybe(schema.string()),
           pipeline: schema.string(),
           settings: schema.maybe(schema.object({}, { unknowns: 'allow' })),
         }),
diff --git a/x-pack/plugins/ml/public/application/explorer/swimlane_container.tsx b/x-pack/plugins/ml/public/application/explorer/swimlane_container.tsx
@@ -22,7 +22,6 @@ import {
 
 import { MlTooltipComponent } from '../../application/components/chart_tooltip';
 import { SwimLanePagination } from './swimlane_pagination';
-import { SWIMLANE_TYPE } from './explorer_constants';
 import { ViewBySwimLaneData } from './explorer_utils';
 
 /**
@@ -94,7 +93,6 @@ export const SwimlaneContainer: FC<
     (showSwimlane || isLoading) &&
     swimlaneLimit !== undefined &&
     onPaginationChange &&
-    props.swimlaneType === SWIMLANE_TYPE.VIEW_BY &&
     fromPage &&
     perPage;
 
diff --git a/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx b/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx
@@ -189,10 +189,8 @@ const ExplorerUrlStateManager: FC<ExplorerUrlStateManagerProps> = ({ jobsWithTim
       loadExplorerData({
         ...loadExplorerDataConfig,
         swimlaneLimit:
-          explorerState?.viewBySwimlaneData &&
-          isViewBySwimLaneData(explorerState?.viewBySwimlaneData)
-            ? explorerState?.viewBySwimlaneData.cardinality
-            : undefined,
+          isViewBySwimLaneData(explorerState?.viewBySwimlaneData) &&
+          explorerState?.viewBySwimlaneData.cardinality,
       });
     }
   }, [JSON.stringify(loadExplorerDataConfig)]);
diff --git a/x-pack/plugins/ml/public/application/services/results_service/results_service.js b/x-pack/plugins/ml/public/application/services/results_service/results_service.js
@@ -87,6 +87,11 @@ export function resultsServiceProvider(mlApiServices) {
                 },
               },
               aggs: {
+                jobsCardinality: {
+                  cardinality: {
+                    field: 'job_id',
+                  },
+                },
                 jobId: {
                   terms: {
                     field: 'job_id',
@@ -147,6 +152,7 @@ export function resultsServiceProvider(mlApiServices) {
               });
               obj.results[jobId] = resultsForTime;
             });
+            obj.cardinality = resp.aggregations?.jobsCardinality?.value ?? 0;
 
             resolve(obj);
           })
diff --git a/x-pack/plugins/security_solution/public/common/components/ml_popover/jobs_table/__snapshots__/jobs_table.test.tsx.snap b/x-pack/plugins/security_solution/public/common/components/ml_popover/jobs_table/__snapshots__/jobs_table.test.tsx.snap
diff --git a/x-pack/plugins/security_solution/public/common/components/ml_popover/jobs_table/jobs_table.tsx b/x-pack/plugins/security_solution/public/common/components/ml_popover/jobs_table/jobs_table.tsx
@@ -125,7 +125,7 @@ export const JobsTableComponent = ({ isLoading, jobs, onJobStateChange }: JobTab
       columns={getJobsTableColumns(isLoading, onJobStateChange, basePath)}
       items={getPaginatedItems(jobs, pageIndex, pageSize)}
       loading={isLoading}
-      noItemsMessage={<NoItemsMessage />}
+      noItemsMessage={<NoItemsMessage basePath={basePath} />}
       pagination={pagination}
       responsive={false}
       onChange={({ page }: { page: { index: number } }) => {
@@ -141,13 +141,13 @@ export const JobsTable = React.memo(JobsTableComponent);
 
 JobsTable.displayName = 'JobsTable';
 
-export const NoItemsMessage = React.memo(() => (
+export const NoItemsMessage = React.memo(({ basePath }: { basePath: string }) => (
   <EuiEmptyPrompt
     title={<h3>{i18n.NO_ITEMS_TEXT}</h3>}
     titleSize="xs"
     actions={
       <EuiButton
-        href="ml#/jobs/new_job/step/index_or_search"
+        href={`${basePath}/app/ml#/jobs/new_job/step/index_or_search`}
         iconType="popout"
         iconSide="right"
         size="s"
diff --git a/x-pack/plugins/security_solution/public/common/components/ml_popover/jobs_table/translations.ts b/x-pack/plugins/security_solution/public/common/components/ml_popover/jobs_table/translations.ts
@@ -30,7 +30,7 @@ export const COLUMN_RUN_JOB = i18n.translate(
 export const NO_ITEMS_TEXT = i18n.translate(
   'xpack.securitySolution.components.mlPopup.jobsTable.noItemsDescription',
   {
-    defaultMessage: 'No SIEM Machine Learning jobs found',
+    defaultMessage: 'No Security Machine Learning jobs found',
   }
 );
 
diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/policy/index.ts b/x-pack/plugins/security_solution/server/endpoint/routes/policy/index.ts
@@ -11,6 +11,8 @@ import { getHostPolicyResponseHandler } from './handlers';
 
 export const BASE_POLICY_RESPONSE_ROUTE = `/api/endpoint/policy_response`;
 
+export const INITIAL_POLICY_ID = '00000000-0000-0000-0000-000000000000';
+
 export function registerPolicyRoutes(router: IRouter, endpointAppContext: EndpointAppContext) {
   router.get(
     {
diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/policy/service.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/policy/service.test.ts
@@ -5,6 +5,7 @@
  */
 
 import { GetPolicyResponseSchema } from '../../../../common/endpoint/schema/policy';
+import { getESQueryPolicyResponseByHostID } from './service';
 
 describe('test policy handlers schema', () => {
   it('validate that get policy response query schema', async () => {
@@ -17,3 +18,21 @@ describe('test policy handlers schema', () => {
     expect(() => GetPolicyResponseSchema.query.validate({})).toThrowError();
   });
 });
+
+describe('test policy query', () => {
+  it('queries for the correct host', async () => {
+    const hostID = 'f757d3c0-e874-11ea-9ad9-015510b487f4';
+    const query = getESQueryPolicyResponseByHostID(hostID, 'anyindex');
+    expect(query.body.query.bool.filter.term).toEqual({ 'host.id': hostID });
+  });
+
+  it('filters out initial policy by ID', async () => {
+    const query = getESQueryPolicyResponseByHostID(
+      'f757d3c0-e874-11ea-9ad9-015510b487f4',
+      'anyindex'
+    );
+    expect(query.body.query.bool.must_not.term).toEqual({
+      'Endpoint.policy.applied.id': '00000000-0000-0000-0000-000000000000',
+    });
+  });
+});
diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/policy/service.ts b/x-pack/plugins/security_solution/server/endpoint/routes/policy/service.ts
@@ -7,13 +7,23 @@
 import { SearchResponse } from 'elasticsearch';
 import { ILegacyScopedClusterClient } from 'kibana/server';
 import { GetHostPolicyResponse, HostPolicyResponse } from '../../../../common/endpoint/types';
+import { INITIAL_POLICY_ID } from './index';
 
 export function getESQueryPolicyResponseByHostID(hostID: string, index: string) {
   return {
     body: {
       query: {
-        match: {
-          'host.id': hostID,
+        bool: {
+          filter: {
+            term: {
+              'host.id': hostID,
+            },
+          },
+          must_not: {
+            term: {
+              'Endpoint.policy.applied.id': INITIAL_POLICY_ID,
+            },
+          },
         },
       },
       sort: [
diff --git a/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/fields/indices_and_data_streams_field/indices_and_data_streams_field.tsx b/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/fields/indices_and_data_streams_field/indices_and_data_streams_field.tsx
@@ -67,7 +67,7 @@ export const IndicesAndDataStreamsField: FunctionComponent<Props> = ({
 
   // We assume all indices if the config has no indices entry or if we receive an empty array
   const [isAllIndices, setIsAllIndices] = useState<boolean>(
-    !config.indices || (Array.isArray(config.indices) && config.indices.length === 0)
+    config.indices == null || (Array.isArray(config.indices) && config.indices.length === 0)
   );
 
   const onUpdate = (data: IndicesConfig) => {
@@ -135,12 +135,17 @@ export const IndicesAndDataStreamsField: FunctionComponent<Props> = ({
           );
           onUpdate({ indices: undefined });
         } else {
-          onUpdate({
-            indices:
-              selectIndicesMode === 'custom'
-                ? indexPatterns.join(',')
-                : [...(indicesAndDataStreamsSelection || [])],
-          });
+          _onUpdate(
+            {
+              indices:
+                selectIndicesMode === 'custom'
+                  ? indexPatterns.join(',')
+                  : [...(indicesAndDataStreamsSelection || [])],
+            },
+            {
+              validateIndicesCount: true,
+            }
+          );
         }
       }}
     />
diff --git a/x-pack/test/api_integration/apis/logstash/pipeline/delete.ts b/x-pack/test/api_integration/apis/logstash/pipeline/delete.ts
@@ -18,7 +18,6 @@ export default function ({ getService }: FtrProviderContext) {
         .put('/api/logstash/pipeline/fast_generator')
         .set('kbn-xsrf', 'xxx')
         .send({
-          id: 'fast_generator',
           description: 'foobar baz',
           pipeline: 'input { generator {} }\n\n output { stdout {} }',
         })
diff --git a/x-pack/test/api_integration/apis/logstash/pipeline/save.ts b/x-pack/test/api_integration/apis/logstash/pipeline/save.ts
@@ -26,7 +26,6 @@ export default function ({ getService }: FtrProviderContext) {
         .put('/api/logstash/pipeline/fast_generator')
         .set('kbn-xsrf', 'xxx')
         .send({
-          id: 'fast_generator',
           description: 'foobar baz',
           pipeline: 'input { generator {} }\n\n output { stdout {} }',
         })
diff --git a/yarn.lock b/yarn.lock
@@ -5553,10 +5553,10 @@
   dependencies:
     "@types/node" "*"
 
-"@types/node-forge@^0.9.0":
-  version "0.9.0"
-  resolved "https://registry.yarnpkg.com/@types/node-forge/-/node-forge-0.9.0.tgz#e9f678ec09283f9f35cb8de6c01f86be9278ac08"
-  integrity sha512-J00+BIHJOfagO1Qs67Jp5CZO3VkFxY8YKMt44oBhXr+3ZYNnl8wv/vtcJyPjuH0QZ+q7+5nnc6o/YH91ZJy2pQ==
+"@types/node-forge@^0.9.5":
+  version "0.9.5"
+  resolved "https://registry.yarnpkg.com/@types/node-forge/-/node-forge-0.9.5.tgz#648231d79da197216290429020698d4e767365a0"
+  integrity sha512-rrN3xfA/oZIzwOnO3d2wRQz7UdeVkmMMPjWUCfpPTPuKFVb3D6G10LuiVHYYmvrivBBLMx4m0P/FICoDbNZUMA==
   dependencies:
     "@types/node" "*"
 
@@ -8727,11 +8727,12 @@ bit-twiddle@^1.0.0:
   integrity sha1-DGwfq+KyPRcXPZpht7cJPrnhdp4=
 
 bl@^1.0.0:
-  version "1.2.1"
-  resolved "https://registry.yarnpkg.com/bl/-/bl-1.2.1.tgz#cac328f7bee45730d404b692203fcb590e172d5e"
-  integrity sha1-ysMo977kVzDUBLaSID/LWQ4XLV4=
+  version "1.2.3"
+  resolved "https://registry.yarnpkg.com/bl/-/bl-1.2.3.tgz#1e8dd80142eac80d7158c9dccc047fb620e035e7"
+  integrity sha512-pvcNpa0UU69UT341rO6AYy4FVAIkUHuZXRIWbq+zHnsVcRzDDjIAhGuuYoi0d//cwIwtt4pkpKycWEfjdV+vww==
   dependencies:
-    readable-stream "^2.0.5"
+    readable-stream "^2.3.5"
+    safe-buffer "^5.1.1"
 
 bl@^4.0.1:
   version "4.0.3"
@@ -22634,15 +22635,10 @@ node-forge@0.9.0:
   resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.9.0.tgz#d624050edbb44874adca12bb9a52ec63cb782579"
   integrity sha512-7ASaDa3pD+lJ3WvXFsxekJQelBKRpne+GOVbLbtHYdd7pFspyeuJHnWfLplGf3SwKGbfs/aYl5V/JCIaHVUKKQ==
 
-node-forge@^0.7.6:
-  version "0.7.6"
-  resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.7.6.tgz#fdf3b418aee1f94f0ef642cd63486c77ca9724ac"
-  integrity sha512-sol30LUpz1jQFBjOKwbjxijiE3b6pjd74YwfD0fJOKPjF+fONKb2Yg8rYgS6+bK6VDl+/wfr4IYpC7jDzLUIfw==
-
-node-forge@^0.9.1:
-  version "0.9.1"
-  resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.9.1.tgz#775368e6846558ab6676858a4d8c6e8d16c677b5"
-  integrity sha512-G6RlQt5Sb4GMBzXvhfkeFmbqR6MzhtnT7VTHuLadjkii3rdYHNdw0m8zA4BTxVIh68FicCQ2NSUANpsqkr9jvQ==
+node-forge@^0.10.0, node-forge@^0.7.6:
+  version "0.10.0"
+  resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.10.0.tgz#32dea2afb3e9926f02ee5ce8794902691a676bf3"
+  integrity sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==
 
 node-gyp@^3.8.0:
   version "3.8.0"
