Skip to content

Commit 45ef7ba

Browse files
kibanamachinedasansol92
kibanamachine
and
dasansol92
authored
[Security solution][Endpoint] Removes zip compression when creating artifacts (#101379) (#102033)
* Removes zlib compression when creating artifacts. Also fixes related unit tests and removes old code * Replaces artifact in new manifest using the ones from fleet client with zlip compression * Fixes create_policy_artifact_manifest pushArtifacts missing new manifest. Also fixes unit tests Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: David Sánchez <davidsansol92@gmail.com>
1 parent df39255 commit 45ef7ba

17 files changed

Lines changed: 158 additions & 193 deletions

File tree

‎x-pack/plugins/security_solution/common/endpoint/schema/common.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ export type CompressionAlgorithm = t.TypeOf<typeof compressionAlgorithm>;
1515

1616
export const compressionAlgorithmDispatch = t.keyof({
1717
zlib: null,
18+
none: null,
1819
});
1920
export type CompressionAlgorithmDispatch = t.TypeOf<typeof compressionAlgorithmDispatch>;
2021

‎x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts‎

Lines changed: 0 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
*/
77

88
import { createHash } from 'crypto';
9-
import { deflate } from 'zlib';
109
import type {
1110
Entry,
1211
EntryNested,
@@ -21,9 +20,7 @@ import {
2120
} from '@kbn/securitysolution-list-constants';
2221
import { ExceptionListClient } from '../../../../../lists/server';
2322
import {
24-
internalArtifactCompleteSchema,
2523
InternalArtifactCompleteSchema,
26-
InternalArtifactSchema,
2724
TranslatedEntry,
2825
translatedEntry as translatedEntryType,
2926
translatedEntryMatchAnyMatcher,
@@ -60,28 +57,6 @@ export async function buildArtifact(
6057
};
6158
}
6259

63-
export async function maybeCompressArtifact(
64-
uncompressedArtifact: InternalArtifactSchema
65-
): Promise<InternalArtifactSchema> {
66-
const compressedArtifact = { ...uncompressedArtifact };
67-
if (internalArtifactCompleteSchema.is(uncompressedArtifact)) {
68-
const compressedArtifactBody = await compressExceptionList(
69-
Buffer.from(uncompressedArtifact.body, 'base64')
70-
);
71-
compressedArtifact.body = compressedArtifactBody.toString('base64');
72-
compressedArtifact.encodedSize = compressedArtifactBody.byteLength;
73-
compressedArtifact.compressionAlgorithm = 'zlib';
74-
compressedArtifact.encodedSha256 = createHash('sha256')
75-
.update(compressedArtifactBody)
76-
.digest('hex');
77-
}
78-
return compressedArtifact;
79-
}
80-
81-
export function isCompressed(artifact: InternalArtifactSchema) {
82-
return artifact.compressionAlgorithm === 'zlib';
83-
}
84-
8560
export async function getFilteredEndpointExceptionList(
8661
eClient: ExceptionListClient,
8762
schemaVersion: string,
@@ -297,15 +272,3 @@ function translateEntry(
297272
}
298273
}
299274
}
300-
301-
export async function compressExceptionList(buffer: Buffer): Promise<Buffer> {
302-
return new Promise((resolve, reject) => {
303-
deflate(buffer, function (err, buf) {
304-
if (err) {
305-
reject(err);
306-
} else {
307-
resolve(buf);
308-
}
309-
});
310-
});
311-
}

‎x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@ describe('manifest', () => {
3737
let ARTIFACT_COPY_TRUSTED_APPS_WINDOWS: InternalArtifactCompleteSchema;
3838

3939
beforeAll(async () => {
40-
ARTIFACTS = await getMockArtifacts({ compress: true });
41-
ARTIFACTS_COPY = await getMockArtifacts({ compress: true });
40+
ARTIFACTS = await getMockArtifacts();
41+
ARTIFACTS_COPY = await getMockArtifacts();
4242
ARTIFACT_EXCEPTIONS_MACOS = ARTIFACTS[0];
4343
ARTIFACT_EXCEPTIONS_WINDOWS = ARTIFACTS[1];
4444
ARTIFACT_EXCEPTIONS_LINUX = ARTIFACTS[2];

‎x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import {
1212
InternalArtifactSchema,
1313
InternalManifestSchema,
1414
InternalManifestEntrySchema,
15+
InternalArtifactCompleteSchema,
1516
} from '../../schemas/artifacts';
1617
import {
1718
ManifestSchemaVersion,
@@ -139,6 +140,27 @@ export class Manifest {
139140
return this.allEntries.get(getArtifactId(artifact))?.specificTargetPolicies;
140141
}
141142

143+
/**
144+
* Replaces an artifact from all the collections.
145+
*
146+
* @param artifact An InternalArtifactCompleteSchema representing the artifact.
147+
*/
148+
public replaceArtifact(artifact: InternalArtifactCompleteSchema) {
149+
const existingEntry = this.allEntries.get(getArtifactId(artifact));
150+
if (existingEntry) {
151+
existingEntry.entry = new ManifestEntry(artifact);
152+
153+
this.allEntries.set(getArtifactId(artifact), existingEntry);
154+
this.defaultEntries.set(getArtifactId(artifact), existingEntry.entry);
155+
156+
existingEntry.specificTargetPolicies.forEach((policyId) => {
157+
const entries = this.policySpecificEntries.get(policyId) || new Map();
158+
entries.set(existingEntry.entry.getDocId(), existingEntry.entry);
159+
this.policySpecificEntries.set(policyId, entries);
160+
});
161+
}
162+
}
163+
142164
public diff(manifest: Manifest): ManifestDiff {
143165
const diff: ManifestDiff = {
144166
additions: [],

‎x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ describe('manifest_entry', () => {
1515
let manifestEntry: ManifestEntry;
1616

1717
beforeAll(async () => {
18-
artifact = await getInternalArtifactMock('windows', 'v1', { compress: true });
18+
artifact = await getInternalArtifactMock('windows', 'v1');
1919
manifestEntry = new ManifestEntry(artifact);
2020
});
2121

@@ -35,15 +35,15 @@ describe('manifest_entry', () => {
3535

3636
test('Correct sha256 is returned', () => {
3737
expect(manifestEntry.getEncodedSha256()).toEqual(
38-
'975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e'
38+
'96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3'
3939
);
4040
expect(manifestEntry.getDecodedSha256()).toEqual(
4141
'96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3'
4242
);
4343
});
4444

4545
test('Correct size is returned', () => {
46-
expect(manifestEntry.getEncodedSize()).toEqual(147);
46+
expect(manifestEntry.getEncodedSize()).toEqual(432);
4747
expect(manifestEntry.getDecodedSize()).toEqual(432);
4848
});
4949

@@ -59,12 +59,12 @@ describe('manifest_entry', () => {
5959

6060
test('Correct record is returned', () => {
6161
expect(manifestEntry.getRecord()).toEqual({
62-
compression_algorithm: 'zlib',
62+
compression_algorithm: 'none',
6363
encryption_algorithm: 'none',
6464
decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3',
65-
encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e',
65+
encoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3',
6666
decoded_size: 432,
67-
encoded_size: 147,
67+
encoded_size: 432,
6868
relative_url:
6969
'/api/fleet/artifacts/endpoint-exceptionlist-windows-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3',
7070
});

‎x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrate_artifacts_to_fleet.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ describe('When migrating artifacts to fleet', () => {
4949
type: '',
5050
id: 'abc123',
5151
references: [],
52-
attributes: await getInternalArtifactMock('windows', 'v1', { compress: true }),
52+
attributes: await getInternalArtifactMock('windows', 'v1'),
5353
},
5454
])
5555
);

‎x-pack/plugins/security_solution/server/endpoint/lib/artifacts/mocks.ts‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,49 +17,49 @@ import {
1717
import { ArtifactConstants } from './common';
1818
import { Manifest } from './manifest';
1919

20-
export const getMockArtifacts = async (opts?: { compress: boolean }) => {
20+
export const getMockArtifacts = async () => {
2121
return Promise.all([
2222
// Exceptions items
2323
...ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.map<Promise<InternalArtifactCompleteSchema>>(
2424
async (os) => {
25-
return getInternalArtifactMock(os, 'v1', opts);
25+
return getInternalArtifactMock(os, 'v1');
2626
}
2727
),
2828
// Trusted Apps items
2929
...ArtifactConstants.SUPPORTED_TRUSTED_APPS_OPERATING_SYSTEMS.map<
3030
Promise<InternalArtifactCompleteSchema>
3131
>(async (os) => {
32-
return getInternalArtifactMock(os, 'v1', opts, ArtifactConstants.GLOBAL_TRUSTED_APPS_NAME);
32+
return getInternalArtifactMock(os, 'v1', ArtifactConstants.GLOBAL_TRUSTED_APPS_NAME);
3333
}),
3434
]);
3535
};
3636

37-
export const getMockArtifactsWithDiff = async (opts?: { compress: boolean }) => {
37+
export const getMockArtifactsWithDiff = async () => {
3838
return Promise.all(
3939
ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.map<Promise<InternalArtifactCompleteSchema>>(
4040
async (os) => {
4141
if (os === 'macos') {
4242
return getInternalArtifactMockWithDiffs(os, 'v1');
4343
}
44-
return getInternalArtifactMock(os, 'v1', opts);
44+
return getInternalArtifactMock(os, 'v1');
4545
}
4646
)
4747
);
4848
};
4949

50-
export const getEmptyMockArtifacts = async (opts?: { compress: boolean }) => {
50+
export const getEmptyMockArtifacts = async () => {
5151
return Promise.all(
5252
ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.map<Promise<InternalArtifactCompleteSchema>>(
5353
async (os) => {
54-
return getEmptyInternalArtifactMock(os, 'v1', opts);
54+
return getEmptyInternalArtifactMock(os, 'v1');
5555
}
5656
)
5757
);
5858
};
5959

60-
export const getMockManifest = async (opts?: { compress: boolean }) => {
60+
export const getMockManifest = async () => {
6161
const manifest = new Manifest();
62-
const artifacts = await getMockArtifacts(opts);
62+
const artifacts = await getMockArtifacts();
6363
artifacts.forEach((artifact) => manifest.addEntry(artifact));
6464
return manifest;
6565
};

‎x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts‎

Lines changed: 19 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ describe('task', () => {
106106
let ARTIFACT_TRUSTED_APPS_MACOS: InternalArtifactCompleteSchema;
107107

108108
beforeAll(async () => {
109-
const artifacts = await getMockArtifacts({ compress: true });
109+
const artifacts = await getMockArtifacts();
110110
ARTIFACT_EXCEPTIONS_MACOS = artifacts[0];
111111
ARTIFACT_EXCEPTIONS_WINDOWS = artifacts[1];
112112
ARTIFACT_TRUSTED_APPS_MACOS = artifacts[2];
@@ -167,7 +167,7 @@ describe('task', () => {
167167

168168
expect(manifestManager.getLastComputedManifest).toHaveBeenCalled();
169169
expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(lastManifest);
170-
expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([]);
170+
expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([], newManifest);
171171
expect(manifestManager.commit).not.toHaveBeenCalled();
172172
expect(manifestManager.tryDispatch).toHaveBeenCalledWith(newManifest);
173173
expect(manifestManager.deleteArtifacts).toHaveBeenCalledWith([]);
@@ -192,10 +192,10 @@ describe('task', () => {
192192

193193
expect(manifestManager.getLastComputedManifest).toHaveBeenCalled();
194194
expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(lastManifest);
195-
expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([
196-
ARTIFACT_EXCEPTIONS_MACOS,
197-
ARTIFACT_TRUSTED_APPS_MACOS,
198-
]);
195+
expect(manifestManager.pushArtifacts).toHaveBeenCalledWith(
196+
[ARTIFACT_EXCEPTIONS_MACOS, ARTIFACT_TRUSTED_APPS_MACOS],
197+
newManifest
198+
);
199199
expect(manifestManager.commit).not.toHaveBeenCalled();
200200
expect(manifestManager.tryDispatch).not.toHaveBeenCalled();
201201
expect(manifestManager.deleteArtifacts).not.toHaveBeenCalled();
@@ -221,10 +221,10 @@ describe('task', () => {
221221

222222
expect(manifestManager.getLastComputedManifest).toHaveBeenCalled();
223223
expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(lastManifest);
224-
expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([
225-
ARTIFACT_EXCEPTIONS_MACOS,
226-
ARTIFACT_TRUSTED_APPS_MACOS,
227-
]);
224+
expect(manifestManager.pushArtifacts).toHaveBeenCalledWith(
225+
[ARTIFACT_EXCEPTIONS_MACOS, ARTIFACT_TRUSTED_APPS_MACOS],
226+
newManifest
227+
);
228228
expect(manifestManager.commit).toHaveBeenCalledWith(newManifest);
229229
expect(manifestManager.tryDispatch).not.toHaveBeenCalled();
230230
expect(manifestManager.deleteArtifacts).not.toHaveBeenCalled();
@@ -251,10 +251,10 @@ describe('task', () => {
251251

252252
expect(manifestManager.getLastComputedManifest).toHaveBeenCalled();
253253
expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(lastManifest);
254-
expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([
255-
ARTIFACT_EXCEPTIONS_MACOS,
256-
ARTIFACT_TRUSTED_APPS_MACOS,
257-
]);
254+
expect(manifestManager.pushArtifacts).toHaveBeenCalledWith(
255+
[ARTIFACT_EXCEPTIONS_MACOS, ARTIFACT_TRUSTED_APPS_MACOS],
256+
newManifest
257+
);
258258
expect(manifestManager.commit).toHaveBeenCalledWith(newManifest);
259259
expect(manifestManager.tryDispatch).toHaveBeenCalledWith(newManifest);
260260
expect(manifestManager.deleteArtifacts).not.toHaveBeenCalled();
@@ -284,7 +284,10 @@ describe('task', () => {
284284

285285
expect(manifestManager.getLastComputedManifest).toHaveBeenCalled();
286286
expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(lastManifest);
287-
expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([ARTIFACT_TRUSTED_APPS_MACOS]);
287+
expect(manifestManager.pushArtifacts).toHaveBeenCalledWith(
288+
[ARTIFACT_TRUSTED_APPS_MACOS],
289+
newManifest
290+
);
288291
expect(manifestManager.commit).toHaveBeenCalledWith(newManifest);
289292
expect(manifestManager.tryDispatch).toHaveBeenCalledWith(newManifest);
290293
expect(manifestManager.deleteArtifacts).toHaveBeenCalledWith([ARTIFACT_ID_1]);
@@ -314,7 +317,7 @@ describe('task', () => {
314317

315318
expect(manifestManager.getLastComputedManifest).toHaveBeenCalled();
316319
expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(lastManifest);
317-
expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([]);
320+
expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([], newManifest);
318321
expect(manifestManager.commit).toHaveBeenCalledWith(newManifest);
319322
expect(manifestManager.tryDispatch).toHaveBeenCalledWith(newManifest);
320323
expect(manifestManager.deleteArtifacts).toHaveBeenCalledWith([]);

‎x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,8 @@ export class ManifestTask {
130130
const diff = newManifest.diff(oldManifest);
131131

132132
const persistErrors = await manifestManager.pushArtifacts(
133-
diff.additions as InternalArtifactCompleteSchema[]
133+
diff.additions as InternalArtifactCompleteSchema[],
134+
newManifest
134135
);
135136
if (persistErrors.length) {
136137
reportErrors(this.logger, persistErrors);

‎x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts‎

Lines changed: 6 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -5,33 +5,13 @@
55
* 2.0.
66
*/
77

8-
import {
9-
buildArtifact,
10-
maybeCompressArtifact,
11-
isCompressed,
12-
ArtifactConstants,
13-
} from '../../lib/artifacts';
8+
import { buildArtifact, ArtifactConstants } from '../../lib/artifacts';
149
import { getTranslatedExceptionListMock } from './lists.mock';
15-
import {
16-
InternalManifestSchema,
17-
internalArtifactCompleteSchema,
18-
InternalArtifactCompleteSchema,
19-
} from './saved_objects';
20-
21-
const compressArtifact = async (artifact: InternalArtifactCompleteSchema) => {
22-
const compressedArtifact = await maybeCompressArtifact(artifact);
23-
if (!isCompressed(compressedArtifact)) {
24-
throw new Error(`Unable to compress artifact: ${artifact.identifier}`);
25-
} else if (!internalArtifactCompleteSchema.is(compressedArtifact)) {
26-
throw new Error(`Incomplete artifact detected: ${artifact.identifier}`);
27-
}
28-
return compressedArtifact;
29-
};
10+
import { InternalManifestSchema, InternalArtifactCompleteSchema } from './saved_objects';
3011

3112
export const getInternalArtifactMock = async (
3213
os: string,
3314
schemaVersion: string,
34-
opts?: { compress: boolean },
3515
artifactName: string = ArtifactConstants.GLOBAL_ALLOWLIST_NAME
3616
): Promise<InternalArtifactCompleteSchema> => {
3717
const artifact = await buildArtifact(
@@ -40,23 +20,21 @@ export const getInternalArtifactMock = async (
4020
os,
4121
artifactName
4222
);
43-
return opts?.compress ? compressArtifact(artifact) : artifact;
23+
return artifact;
4424
};
4525

4626
export const getEmptyInternalArtifactMock = async (
4727
os: string,
4828
schemaVersion: string,
49-
opts?: { compress: boolean },
5029
artifactName: string = ArtifactConstants.GLOBAL_ALLOWLIST_NAME
5130
): Promise<InternalArtifactCompleteSchema> => {
5231
const artifact = await buildArtifact({ entries: [] }, schemaVersion, os, artifactName);
53-
return opts?.compress ? compressArtifact(artifact) : artifact;
32+
return artifact;
5433
};
5534

5635
export const getInternalArtifactMockWithDiffs = async (
5736
os: string,
58-
schemaVersion: string,
59-
opts?: { compress: boolean }
37+
schemaVersion: string
6038
): Promise<InternalArtifactCompleteSchema> => {
6139
const mock = getTranslatedExceptionListMock();
6240
mock.entries.pop();
@@ -66,7 +44,7 @@ export const getInternalArtifactMockWithDiffs = async (
6644
os,
6745
ArtifactConstants.GLOBAL_ALLOWLIST_NAME
6846
);
69-
return opts?.compress ? compressArtifact(artifact) : artifact;
47+
return artifact;
7048
};
7149

7250
export const getInternalManifestMock = (): InternalManifestSchema => ({

0 commit comments

Comments
 (0)