Skip to content

Commit f684ea4

Browse files
sorenlouvviduni94
sorenlouv
and
viduni94
authored
[LockManager] Ensure index template are created (#218901)
Closes: #218944 The index template for the Lock Manager was not created, causing index mappings and settings to be incorrect. Root cause: the function responsible for creating the index template (`ensureTemplatesAndIndexCreated`) was never invoked - only during automated testing 🤦 ## Manual testing steps The mappings for the lock manager index (`.kibana_locks-000001`) should match this: ```ts { mappings: { dynamic: false, properties: { token: { type: 'keyword' }, metadata: { enabled: false }, createdAt: { type: 'date' }, expiresAt: { type: 'date' }, }, }, } ``` ### Test 1 In this test we make sure that the LockManager library can recover and fix the mappings if the existing index has invalid mappings #### Delete existing assets if they already exist ``` DELETE .kibana_locks-000001 DELETE _index_template/.kibana_locks-index-template DELETE _component_template/.kibana_locks-component ``` #### Create lock index. This is invalid because it does not have the correct mappings ``` PUT .kibana_locks-000001 ``` (Restart Kibana) #### Check mappings are correct ``` GET .kibana_locks-000001/_mapping ``` ### Test 2 In this test we make sure that out of the box, the LockManager library creates an index with the correct mappings ``` DELETE .kibana_locks-000001 DELETE _index_template/.kibana_locks-index-template DELETE _component_template/.kibana_locks-component ``` (Restart Kibana) #### Check mappings are correct ``` GET .kibana_locks-000001/_mapping ``` Related: #216916 #216397 --------- Co-authored-by: Viduni Wickramarachchi <viduni.ushanka@gmail.com>
1 parent 8cdf6ff commit f684ea4

3 files changed

Lines changed: 218 additions & 51 deletions

File tree

‎x-pack/platform/plugins/shared/observability_ai_assistant/server/service/distributed_lock_manager/lock_manager_client.ts‎

Lines changed: 9 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,7 @@ import prettyMilliseconds from 'pretty-ms';
1313
import { once } from 'lodash';
1414
import { duration } from 'moment';
1515
import { ElasticsearchClient } from '@kbn/core/server';
16-
17-
export const LOCKS_INDEX_ALIAS = '.kibana_locks';
18-
export const LOCKS_CONCRETE_INDEX_NAME = `${LOCKS_INDEX_ALIAS}-000001`;
16+
import { LOCKS_CONCRETE_INDEX_NAME, setuplockManagerIndex } from './setup_lock_manager_index';
1917

2018
export type LockId = string;
2119
export interface LockDocument {
@@ -38,7 +36,12 @@ export interface AcquireOptions {
3836
ttl?: number;
3937
}
4038

41-
const createLocksWriteIndexOnce = once(createLocksWriteIndex);
39+
// The index assets should only be set up once
40+
// For testing purposes, we need to be able to set it up every time
41+
let runSetupIndexAssetOnce = once(setuplockManagerIndex);
42+
export function runSetupIndexAssetEveryTime() {
43+
runSetupIndexAssetOnce = setuplockManagerIndex;
44+
}
4245

4346
export class LockManager {
4447
private token = uuid();
@@ -58,7 +61,8 @@ export class LockManager {
5861
ttl = duration(30, 'seconds').asMilliseconds(),
5962
}: AcquireOptions = {}): Promise<boolean> {
6063
let response: Awaited<ReturnType<ElasticsearchClient['update']>>;
61-
await createLocksWriteIndexOnce(this.esClient);
64+
65+
await runSetupIndexAssetOnce(this.esClient, this.logger);
6266
this.token = uuid();
6367

6468
try {
@@ -303,45 +307,6 @@ export async function withLock<T>(
303307
}
304308
}
305309

306-
export async function ensureTemplatesAndIndexCreated(esClient: ElasticsearchClient): Promise<void> {
307-
const COMPONENT_TEMPLATE_NAME = `${LOCKS_INDEX_ALIAS}-component`;
308-
const INDEX_TEMPLATE_NAME = `${LOCKS_INDEX_ALIAS}-index-template`;
309-
const INDEX_PATTERN = `${LOCKS_INDEX_ALIAS}*`;
310-
311-
await esClient.cluster.putComponentTemplate({
312-
name: COMPONENT_TEMPLATE_NAME,
313-
template: {
314-
mappings: {
315-
dynamic: false,
316-
properties: {
317-
token: { type: 'keyword' },
318-
metadata: { enabled: false },
319-
createdAt: { type: 'date' },
320-
expiresAt: { type: 'date' },
321-
},
322-
},
323-
},
324-
});
325-
326-
await esClient.indices.putIndexTemplate({
327-
name: INDEX_TEMPLATE_NAME,
328-
index_patterns: [INDEX_PATTERN],
329-
composed_of: [COMPONENT_TEMPLATE_NAME],
330-
priority: 500,
331-
template: {
332-
settings: {
333-
number_of_shards: 1,
334-
auto_expand_replicas: '0-1',
335-
hidden: true,
336-
},
337-
},
338-
});
339-
}
340-
341-
export async function createLocksWriteIndex(esClient: ElasticsearchClient): Promise<void> {
342-
await esClient.indices.create({ index: LOCKS_CONCRETE_INDEX_NAME }, { ignore: [400] });
343-
}
344-
345310
function isVersionConflictException(e: Error): boolean {
346311
return (
347312
e instanceof errors.ResponseError && e.body?.error?.type === 'version_conflict_engine_exception'

‎x-pack/platform/plugins/shared/observability_ai_assistant/server/service/distributed_lock_manager/setup_lock_manager_index.ts‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import { errors } from '@elastic/elasticsearch';
9+
import { ElasticsearchClient, Logger } from '@kbn/core/server';
10+
import { IndicesGetMappingResponse } from '@elastic/elasticsearch/lib/api/types';
11+
12+
const LOCKS_INDEX_ALIAS = '.kibana_locks';
13+
export const LOCKS_CONCRETE_INDEX_NAME = `${LOCKS_INDEX_ALIAS}-000001`;
14+
export const LOCKS_COMPONENT_TEMPLATE_NAME = `${LOCKS_INDEX_ALIAS}-component`;
15+
export const LOCKS_INDEX_TEMPLATE_NAME = `${LOCKS_INDEX_ALIAS}-index-template`;
16+
17+
export async function removeLockIndexWithIncorrectMappings(
18+
esClient: ElasticsearchClient,
19+
logger: Logger
20+
) {
21+
let res: IndicesGetMappingResponse;
22+
try {
23+
res = await esClient.indices.getMapping({ index: LOCKS_CONCRETE_INDEX_NAME });
24+
} catch (error) {
25+
const isNotFoundError = error instanceof errors.ResponseError && error.statusCode === 404;
26+
if (!isNotFoundError) {
27+
logger.error(
28+
`Failed to get mapping for lock index "${LOCKS_CONCRETE_INDEX_NAME}": ${error.message}`
29+
);
30+
}
31+
32+
return;
33+
}
34+
35+
const { mappings } = res[LOCKS_CONCRETE_INDEX_NAME];
36+
const hasIncorrectMappings =
37+
mappings.properties?.token?.type !== 'keyword' ||
38+
mappings.properties?.expiresAt?.type !== 'date';
39+
40+
if (hasIncorrectMappings) {
41+
logger.warn(`Lock index "${LOCKS_CONCRETE_INDEX_NAME}" has incorrect mappings.`);
42+
try {
43+
await esClient.indices.delete({ index: LOCKS_CONCRETE_INDEX_NAME });
44+
logger.info(`Lock index "${LOCKS_CONCRETE_INDEX_NAME}" removed successfully.`);
45+
} catch (error) {
46+
logger.error(`Failed to remove lock index "${LOCKS_CONCRETE_INDEX_NAME}": ${error.message}`);
47+
}
48+
}
49+
}
50+
51+
export async function ensureTemplatesAndIndexCreated(
52+
esClient: ElasticsearchClient,
53+
logger: Logger
54+
): Promise<void> {
55+
const INDEX_PATTERN = `${LOCKS_INDEX_ALIAS}*`;
56+
57+
await esClient.cluster.putComponentTemplate({
58+
name: LOCKS_COMPONENT_TEMPLATE_NAME,
59+
template: {
60+
mappings: {
61+
dynamic: false,
62+
properties: {
63+
token: { type: 'keyword' },
64+
metadata: { enabled: false },
65+
createdAt: { type: 'date' },
66+
expiresAt: { type: 'date' },
67+
},
68+
},
69+
},
70+
});
71+
logger.info(
72+
`Component template ${LOCKS_COMPONENT_TEMPLATE_NAME} created or updated successfully.`
73+
);
74+
75+
await esClient.indices.putIndexTemplate({
76+
name: LOCKS_INDEX_TEMPLATE_NAME,
77+
index_patterns: [INDEX_PATTERN],
78+
composed_of: [LOCKS_COMPONENT_TEMPLATE_NAME],
79+
priority: 500,
80+
template: {
81+
settings: {
82+
number_of_shards: 1,
83+
auto_expand_replicas: '0-1',
84+
hidden: true,
85+
},
86+
},
87+
});
88+
logger.info(`Index template ${LOCKS_INDEX_TEMPLATE_NAME} created or updated successfully.`);
89+
90+
await esClient.indices.create({ index: LOCKS_CONCRETE_INDEX_NAME }, { ignore: [400] });
91+
logger.info(`Index ${LOCKS_CONCRETE_INDEX_NAME} created or updated successfully.`);
92+
}
93+
94+
export async function setuplockManagerIndex(esClient: ElasticsearchClient, logger: Logger) {
95+
await removeLockIndexWithIncorrectMappings(esClient, logger); // TODO: should be removed in the future (after 9.1). See https://github.com/elastic/kibana/issues/218944
96+
await ensureTemplatesAndIndexCreated(esClient, logger);
97+
}

‎x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/distributed_lock_manager/distributed_lock_manager.spec.ts‎

Lines changed: 112 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,18 +10,21 @@ import { v4 as uuid } from 'uuid';
1010
import prettyMilliseconds from 'pretty-ms';
1111
import {
1212
LockId,
13-
ensureTemplatesAndIndexCreated,
1413
LockManager,
1514
LockDocument,
16-
LOCKS_CONCRETE_INDEX_NAME,
17-
createLocksWriteIndex,
1815
withLock,
16+
runSetupIndexAssetEveryTime,
1917
} from '@kbn/observability-ai-assistant-plugin/server/service/distributed_lock_manager/lock_manager_client';
2018
import nock from 'nock';
2119
import { Client } from '@elastic/elasticsearch';
2220
import { times } from 'lodash';
2321
import { ToolingLog } from '@kbn/tooling-log';
2422
import pRetry from 'p-retry';
23+
import {
24+
LOCKS_COMPONENT_TEMPLATE_NAME,
25+
LOCKS_CONCRETE_INDEX_NAME,
26+
LOCKS_INDEX_TEMPLATE_NAME,
27+
} from '@kbn/observability-ai-assistant-plugin/server/service/distributed_lock_manager/setup_lock_manager_index';
2528
import type { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context';
2629
import { getLoggerMock } from '../utils/logger';
2730
import { dateAsTimestamp, durationAsMs, sleep } from '../utils/time';
@@ -32,11 +35,21 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon
3235
const logger = getLoggerMock(log);
3336

3437
describe('LockManager', function () {
38+
before(async () => {
39+
// delete existing index mappings to ensure we start from a clean state
40+
await deleteLockIndexAssets(es, log);
41+
42+
// ensure that the index and templates are created
43+
runSetupIndexAssetEveryTime();
44+
});
45+
46+
after(async () => {
47+
await deleteLockIndexAssets(es, log);
48+
});
49+
3550
describe('Manual locking API', function () {
3651
this.tags(['failsOnMKI']);
3752
before(async () => {
38-
await ensureTemplatesAndIndexCreated(es);
39-
await createLocksWriteIndex(es);
4053
await clearAllLocks(es, log);
4154
});
4255

@@ -439,8 +452,6 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon
439452
describe('withLock API', function () {
440453
this.tags(['failsOnMKI']);
441454
before(async () => {
442-
await ensureTemplatesAndIndexCreated(es);
443-
await createLocksWriteIndex(es);
444455
await clearAllLocks(es, log);
445456
});
446457

@@ -642,9 +653,91 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon
642653
});
643654
});
644655
});
656+
657+
describe('index assets', () => {
658+
describe('when lock index is created with incorrect mappings', () => {
659+
before(async () => {
660+
await deleteLockIndexAssets(es, log);
661+
await es.index({
662+
refresh: true,
663+
index: LOCKS_CONCRETE_INDEX_NAME,
664+
id: 'my_lock_with_incorrect_mappings',
665+
document: {
666+
token: 'my token',
667+
expiresAt: new Date(Date.now() + 100000),
668+
createdAt: new Date(),
669+
metadata: { foo: 'bar' },
670+
},
671+
});
672+
});
673+
674+
it('should delete the index and re-create it', async () => {
675+
const mappingsBefore = await getMappings(es);
676+
log.debug(`Mappings before: ${JSON.stringify(mappingsBefore)}`);
677+
expect(mappingsBefore.properties?.token.type).to.eql('text');
678+
679+
// Simulate a scenario where the index mappings are incorrect and a lock is added
680+
// it should delete the index and re-create it with the correct mappings
681+
await withLock({ esClient: es, lockId: uuid(), logger }, async () => {});
682+
683+
const mappingsAfter = await getMappings(es);
684+
log.debug(`Mappings after: ${JSON.stringify(mappingsAfter)}`);
685+
expect(mappingsAfter.properties?.token.type).to.be('keyword');
686+
});
687+
});
688+
689+
describe('when lock index is created with correct mappings', () => {
690+
before(async () => {
691+
await withLock({ esClient: es, lockId: uuid(), logger }, async () => {});
692+
693+
// wait for the index to be created
694+
await es.indices.refresh({ index: LOCKS_CONCRETE_INDEX_NAME });
695+
});
696+
697+
it('should have the correct mappings for the lock index', async () => {
698+
const mappings = await getMappings(es);
699+
700+
const expectedMapping = {
701+
dynamic: 'false',
702+
properties: {
703+
token: { type: 'keyword' },
704+
expiresAt: { type: 'date' },
705+
createdAt: { type: 'date' },
706+
metadata: { enabled: false, type: 'object' },
707+
},
708+
};
709+
710+
expect(mappings).to.eql(expectedMapping);
711+
});
712+
713+
it('has the right number_of_replicas', async () => {
714+
const settings = await getSettings(es);
715+
expect(settings?.index?.auto_expand_replicas).to.eql('0-1');
716+
});
717+
718+
it('does not delete the index when adding a new lock', async () => {
719+
const settingsBefore = await getSettings(es);
720+
721+
await withLock({ esClient: es, lockId: uuid(), logger }, async () => {});
722+
723+
const settingsAfter = await getSettings(es);
724+
expect(settingsAfter?.uuid).to.be(settingsBefore?.uuid);
725+
});
726+
});
727+
});
645728
});
646729
}
647730

731+
async function deleteLockIndexAssets(es: Client, log: ToolingLog) {
732+
log.debug(`Deleting index assets`);
733+
await es.indices.delete({ index: LOCKS_CONCRETE_INDEX_NAME }, { ignore: [404] });
734+
await es.indices.deleteIndexTemplate({ name: LOCKS_INDEX_TEMPLATE_NAME }, { ignore: [404] });
735+
await es.cluster.deleteComponentTemplate(
736+
{ name: LOCKS_COMPONENT_TEMPLATE_NAME },
737+
{ ignore: [404] }
738+
);
739+
}
740+
648741
function clearAllLocks(es: Client, log: ToolingLog) {
649742
try {
650743
return es.deleteByQuery(
@@ -696,3 +789,15 @@ async function getLockById(esClient: Client, lockId: LockId): Promise<LockDocume
696789

697790
return res._source;
698791
}
792+
793+
async function getMappings(es: Client) {
794+
const res = await es.indices.getMapping({ index: LOCKS_CONCRETE_INDEX_NAME });
795+
const { mappings } = res[LOCKS_CONCRETE_INDEX_NAME];
796+
return mappings;
797+
}
798+
799+
async function getSettings(es: Client) {
800+
const res = await es.indices.getSettings({ index: LOCKS_CONCRETE_INDEX_NAME });
801+
const { settings } = res[LOCKS_CONCRETE_INDEX_NAME];
802+
return settings;
803+
}

0 commit comments

Comments
 (0)