Skip to content

Commit 7e72b46

Browse files
kdelemmekdelemme
committed
feat(slo): lock resource installation (#219747)
(cherry picked from commit f0537b8) # Conflicts: # x-pack/solutions/observability/plugins/slo/tsconfig.json
1 parent 0002d09 commit 7e72b46

5 files changed

Lines changed: 29 additions & 89 deletions

File tree

‎x-pack/solutions/observability/plugins/slo/common/constants.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ export const SUPPRESSED_PRIORITY_ACTION = {
5353
}),
5454
};
5555

56+
export const LOCK_ID_RESOURCE_INSTALLER = 'slo:resource_installer';
57+
5658
export const SLO_MODEL_VERSION = 2;
5759
export const SLO_RESOURCES_VERSION = 3.4;
5860
export const SLO_RESOURCES_VERSION_MAJOR = 3;
@@ -91,8 +93,6 @@ export const getWildcardPipelineId = (sloId: string, sloRevision: number) =>
9193
export const SYNTHETICS_INDEX_PATTERN = 'synthetics-*';
9294
export const SYNTHETICS_DEFAULT_GROUPINGS = ['monitor.name', 'observer.geo.name', 'monitor.id'];
9395

94-
// in hours
9596
export const DEFAULT_STALE_SLO_THRESHOLD_HOURS = 48;
96-
9797
export const DEFAULT_SLO_PAGE_SIZE = 25;
9898
export const DEFAULT_SLO_GROUPS_PAGE_SIZE = 25;

‎x-pack/solutions/observability/plugins/slo/server/plugin.ts‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import { i18n } from '@kbn/i18n';
2121
import { AlertsLocatorDefinition, sloFeatureId } from '@kbn/observability-plugin/common';
2222
import { SLO_BURN_RATE_RULE_TYPE_ID } from '@kbn/rule-data-utils';
2323
import { mapValues } from 'lodash';
24+
import { LockAcquisitionError, LockManagerService } from '@kbn/lock-manager';
2425
import { getSloClientWithRequest } from './client';
2526
import { registerSloUsageCollector } from './lib/collectors/register';
2627
import { registerBurnRateRule } from './lib/rules/register_burn_rate_rule';
@@ -47,6 +48,7 @@ import type {
4748
SLOServerSetup,
4849
SLOServerStart,
4950
} from './types';
51+
import { LOCK_ID_RESOURCE_INSTALLER } from '../common/constants';
5052

5153
const sloRuleTypes = [SLO_BURN_RATE_RULE_TYPE_ID];
5254

@@ -70,6 +72,7 @@ export class SLOPlugin
7072
core: CoreSetup<SLOPluginStartDependencies, SLOServerStart>,
7173
plugins: SLOPluginSetupDependencies
7274
): SLOServerSetup {
75+
const lockManager = new LockManagerService(core, this.logger);
7376
const alertsLocator = plugins.share.url.locators.create(new AlertsLocatorDefinition());
7477

7578
const savedObjectTypes = [SO_SLO_TYPE, SO_SLO_SETTINGS_TYPE];
@@ -214,10 +217,14 @@ export class SLOPlugin
214217
.then(async ([coreStart, pluginStart]) => {
215218
const esInternalClient = coreStart.elasticsearch.client.asInternalUser;
216219
const sloResourceInstaller = new DefaultResourceInstaller(esInternalClient, this.logger);
217-
await sloResourceInstaller.ensureCommonResourcesInstalled();
220+
await lockManager.withLock(LOCK_ID_RESOURCE_INSTALLER, () =>
221+
sloResourceInstaller.ensureCommonResourcesInstalled()
222+
);
218223
})
219-
.catch(() => {
220-
// noop - error already logged from the installer
224+
.catch((err) => {
225+
if (err instanceof LockAcquisitionError) {
226+
this.logger.debug('Cannot install SLO resources, another process is already doing it');
227+
}
221228
});
222229

223230
this.sloOrphanCleanupTask = new SloOrphanSummaryCleanupTask(

‎x-pack/solutions/observability/plugins/slo/server/services/resource_installer.test.ts‎

Lines changed: 0 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -122,42 +122,4 @@ describe('resourceInstaller', () => {
122122
expect(mockClusterClient.cluster.putComponentTemplate).not.toHaveBeenCalled();
123123
expect(mockClusterClient.indices.putIndexTemplate).not.toHaveBeenCalled();
124124
});
125-
126-
it('runs the installation only once at a time', async () => {
127-
const mockClusterClient = elasticsearchServiceMock.createElasticsearchClient();
128-
mockClusterClient.cluster.getComponentTemplate.mockImplementation(
129-
() =>
130-
new Promise((resolve) =>
131-
setTimeout(
132-
() =>
133-
resolve({
134-
component_templates: [
135-
{
136-
name: SLI_INDEX_TEMPLATE_NAME,
137-
component_template: {
138-
_meta: {
139-
version: SLO_RESOURCES_VERSION - 1,
140-
},
141-
template: {
142-
settings: {},
143-
},
144-
},
145-
},
146-
],
147-
}),
148-
1000
149-
)
150-
)
151-
);
152-
153-
const installer = new DefaultResourceInstaller(mockClusterClient, loggerMock.create());
154-
155-
await Promise.all([
156-
installer.ensureCommonResourcesInstalled(),
157-
installer.ensureCommonResourcesInstalled(),
158-
]);
159-
160-
// Ensure that the installation was only run once, e.g. 4 calls to the put component template API, and not 2x 4 calls
161-
expect(mockClusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4);
162-
});
163125
});

‎x-pack/solutions/observability/plugins/slo/server/services/resource_installer.ts‎

Lines changed: 15 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -28,20 +28,10 @@ export interface ResourceInstaller {
2828
}
2929

3030
export class DefaultResourceInstaller implements ResourceInstaller {
31-
private isInstalling: boolean = false;
32-
3331
constructor(private esClient: ElasticsearchClient, private logger: Logger) {}
3432

35-
public async ensureCommonResourcesInstalled() {
36-
if (this.isInstalling) {
37-
return;
38-
}
39-
this.isInstalling = true;
40-
41-
let installTimeout;
33+
public async ensureCommonResourcesInstalled(): Promise<void> {
4234
try {
43-
installTimeout = setTimeout(() => (this.isInstalling = false), 60000);
44-
4535
this.logger.debug('Installing SLO shared resources');
4636
await Promise.all([
4737
this.createOrUpdateComponentTemplate(SLI_MAPPINGS_TEMPLATE),
@@ -58,18 +48,11 @@ export class DefaultResourceInstaller implements ResourceInstaller {
5848
await this.createIndex(SUMMARY_TEMP_INDEX_NAME);
5949
} catch (err) {
6050
this.logger.error(`Error while installing SLO shared resources: ${err}`);
61-
} finally {
62-
this.isInstalling = false;
63-
clearTimeout(installTimeout);
6451
}
6552
}
6653

6754
private async createOrUpdateComponentTemplate(template: ClusterPutComponentTemplateRequest) {
68-
const currentVersion = await fetchComponentTemplateVersion(
69-
template.name,
70-
this.logger,
71-
this.esClient
72-
);
55+
const currentVersion = await this.fetchComponentTemplateVersion(template.name);
7356
if (template._meta?.version && currentVersion === template._meta.version) {
7457
this.logger.debug(`SLO component template found with version [${template._meta.version}]`);
7558
} else {
@@ -79,11 +62,7 @@ export class DefaultResourceInstaller implements ResourceInstaller {
7962
}
8063

8164
private async createOrUpdateIndexTemplate(template: IndicesPutIndexTemplateRequest) {
82-
const currentVersion = await fetchIndexTemplateVersion(
83-
template.name,
84-
this.logger,
85-
this.esClient
86-
);
65+
const currentVersion = await this.fetchIndexTemplateVersion(template.name);
8766

8867
if (template._meta?.version && currentVersion === template._meta.version) {
8968
this.logger.debug(`SLO index template found with version [${template._meta.version}]`);
@@ -106,30 +85,20 @@ export class DefaultResourceInstaller implements ResourceInstaller {
10685
private async execute<T>(esCall: () => Promise<T>): Promise<T> {
10786
return await retryTransientEsErrors(esCall, { logger: this.logger });
10887
}
109-
}
11088

111-
async function fetchComponentTemplateVersion(
112-
name: string,
113-
logger: Logger,
114-
esClient: ElasticsearchClient
115-
) {
116-
const getTemplateRes = await retryTransientEsErrors(
117-
() => esClient.cluster.getComponentTemplate({ name }, { ignore: [404] }),
118-
{ logger }
119-
);
89+
private async fetchComponentTemplateVersion(name: string) {
90+
const getTemplateRes = await this.execute(() =>
91+
this.esClient.cluster.getComponentTemplate({ name }, { ignore: [404] })
92+
);
12093

121-
return getTemplateRes?.component_templates?.[0]?.component_template?._meta?.version ?? null;
122-
}
94+
return getTemplateRes?.component_templates?.[0]?.component_template?._meta?.version ?? null;
95+
}
12396

124-
async function fetchIndexTemplateVersion(
125-
name: string,
126-
logger: Logger,
127-
esClient: ElasticsearchClient
128-
) {
129-
const getTemplateRes = await retryTransientEsErrors(
130-
() => esClient.indices.getIndexTemplate({ name }, { ignore: [404] }),
131-
{ logger }
132-
);
97+
private async fetchIndexTemplateVersion(name: string) {
98+
const getTemplateRes = await this.execute(() =>
99+
this.esClient.indices.getIndexTemplate({ name }, { ignore: [404] })
100+
);
133101

134-
return getTemplateRes?.index_templates?.[0]?.index_template?._meta?.version ?? null;
102+
return getTemplateRes?.index_templates?.[0]?.index_template?._meta?.version ?? null;
103+
}
135104
}

‎x-pack/solutions/observability/plugins/slo/tsconfig.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,5 +106,7 @@
106106
"@kbn/response-ops-rule-form",
107107
"@kbn/fields-metadata-plugin",
108108
"@kbn/server-route-repository-utils",
109+
"@kbn/apm-sources-access-plugin",
110+
"@kbn/lock-manager",
109111
]
110112
}

0 commit comments

Comments
 (0)