Skip to content

Commit 5768b3e

Browse files
mattkimekibanamachine
mattkime
and
kibanamachine
committed
[data views] data views + rollup index referenced by alias (#212592)
## Summary Upgrading to 9.x involves reindexing indices created in 7.x, which does include rollup indices. Reindexing means relying on aliases to preserve existing index names. As it turns out, our existing code did not work with rollups that referenced aliases, rather than indices. This is because the index name is used as an object key even if it was retrieved via alias. Note - I need to verify this on 9.0 from scratch. I used upgraded data and need to verify the steps to make this work when testing. To test 1. Add sample data 2. Create a rollup job that references the sample data. 3. Create a data view that references the rollup index. It may take a few minutes for the rollup index to be populated. 4. Create an alias from the dev console, like such - ``` POST _aliases { "actions": [ { "add": { "index": "rollup", "alias": "my-alias" } } ] } ``` 5. Create a rollup data view based in the alias you just created. Part of #211850 --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> (cherry picked from commit 5b6dbf2)
1 parent 5b945f7 commit 5768b3e

9 files changed

Lines changed: 109 additions & 14 deletions

File tree

‎src/plugins/data_view_editor/public/components/data_view_editor_flyout_content.tsx‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ const IndexPatternEditorFlyoutContentComponent = ({
142142
params: {
143143
rollup_index: rollupIndex,
144144
},
145-
aggs: rollupIndicesCapabilities[rollupIndex].aggs,
145+
aggs: rollupCaps?.aggs,
146146
};
147147
}
148148

@@ -176,6 +176,7 @@ const IndexPatternEditorFlyoutContentComponent = ({
176176
const isLoadingSources = useObservable(dataViewEditorService.isLoadingSources$, true);
177177
const existingDataViewNames = useObservable(dataViewEditorService.dataViewNames$);
178178
const rollupIndex = useObservable(dataViewEditorService.rollupIndex$);
179+
const rollupCaps = useObservable(dataViewEditorService.rollupCaps$);
179180
const rollupIndicesCapabilities = useObservable(dataViewEditorService.rollupIndicesCaps$, {});
180181

181182
useDebounce(
@@ -194,7 +195,7 @@ const IndexPatternEditorFlyoutContentComponent = ({
194195
dataViewEditorService.setType(type);
195196
}, [dataViewEditorService, type]);
196197

197-
const getRollupIndices = (rollupCaps: RollupIndicesCapsResponse) => Object.keys(rollupCaps);
198+
const getRollupIndices = (rollupCapsRes: RollupIndicesCapsResponse) => Object.keys(rollupCapsRes);
198199

199200
const onTypeChange = useCallback(
200201
(newType: INDEX_PATTERN_TYPE) => {

‎src/plugins/data_view_editor/public/components/form_fields/title_field.tsx‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -80,8 +80,15 @@ const createMatchesIndicesValidator = ({
8080
}
8181

8282
// A rollup index pattern needs to match one and only one rollup index.
83-
const rollupIndexMatches = matchedIndices.exactMatchedIndices.filter((matchedIndex) =>
84-
rollupIndices.includes(matchedIndex.name)
83+
const rollupIndexMatches = matchedIndices.exactMatchedIndices.filter(
84+
(matchedIndex) =>
85+
rollupIndices.includes(matchedIndex.name) ||
86+
// matched item is alias
87+
(matchedIndex.item.indices?.length === 1 &&
88+
rollupIndices.includes(matchedIndex.item.indices[0])) ||
89+
// matched item is an index referenced by an alias
90+
(matchedIndex.item.aliases?.length === 1 &&
91+
rollupIndices.includes(matchedIndex.item.aliases[0]))
8592
);
8693

8794
if (!rollupIndexMatches.length) {

‎src/plugins/data_view_editor/public/data_view_editor_service.ts‎

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,12 @@ import {
2727
DataViewField,
2828
} from '@kbn/data-views-plugin/public';
2929

30-
import { RollupIndicesCapsResponse, MatchedIndicesSet, TimestampOption } from './types';
30+
import {
31+
RollupIndicesCapsResponse,
32+
RollupIndiciesCapability,
33+
MatchedIndicesSet,
34+
TimestampOption,
35+
} from './types';
3136
import { getMatchedIndices, ensureMinimumTime, extractTimeFields, removeSpaces } from './lib';
3237
import { GetFieldsOptions } from './shared_imports';
3338

@@ -70,6 +75,7 @@ interface DataViewEditorState {
7075
loadingTimestampFields: boolean;
7176
timestampFieldOptions: TimestampOption[];
7277
rollupIndexName?: string | null;
78+
rollupCaps?: RollupIndiciesCapability;
7379
}
7480

7581
const defaultDataViewEditorState: DataViewEditorState = {
@@ -119,6 +125,7 @@ export class DataViewEditorService {
119125
this.loadingTimestampFields$ = stateSelector((state) => state.loadingTimestampFields);
120126
this.timestampFieldOptions$ = stateSelector((state) => state.timestampFieldOptions);
121127
this.rollupIndex$ = stateSelector((state) => state.rollupIndexName);
128+
this.rollupCaps$ = stateSelector((state) => state.rollupCaps);
122129

123130
// when list of matched indices is updated always update timestamp fields
124131
this.loadTimestampFieldsSub = this.matchedIndices$.subscribe(() => this.loadTimestampFields());
@@ -162,6 +169,8 @@ export class DataViewEditorService {
162169

163170
// current matched rollup index
164171
rollupIndex$: Observable<string | undefined | null>;
172+
// current matched rollup capabilities
173+
rollupCaps$: Observable<RollupIndiciesCapability | undefined>;
165174
// alernates between value and undefined so validation can treat new value as thought its a promise
166175
private rollupIndexForProvider$ = new Subject<string | undefined | null>();
167176

@@ -244,11 +253,27 @@ export class DataViewEditorService {
244253
// verify we're looking at the current result
245254
if (currentLoadingMatchedIndicesIdx === this.currentLoadingMatchedIndices) {
246255
if (type === INDEX_PATTERN_TYPE.ROLLUP) {
247-
const rollupIndices = exactMatched.filter((index) => isRollupIndex(index.name));
256+
const rollupIndices = exactMatched.filter(
257+
(index) =>
258+
isRollupIndex(index.name) ||
259+
// if its an alias
260+
(index.item.indices?.length === 1 && isRollupIndex(index.item.indices[0])) ||
261+
// if its an index referenced by an alias
262+
(index.item.aliases?.length === 1 && isRollupIndex(index.item.aliases[0]))
263+
);
264+
248265
newRollupIndexName = rollupIndices.length === 1 ? rollupIndices[0].name : null;
249-
this.updateState({ rollupIndexName: newRollupIndexName });
266+
const newRollupCaps = await this.rollupCapsResponse.then((response) => {
267+
return (
268+
response[newRollupIndexName || ''] ||
269+
// if its an alias
270+
response[rollupIndices[0]?.item.indices?.[0] || '']
271+
);
272+
});
273+
274+
this.updateState({ rollupIndexName: newRollupIndexName, rollupCaps: newRollupCaps });
250275
} else {
251-
this.updateState({ rollupIndexName: null });
276+
this.updateState({ rollupIndexName: null, rollupCaps: undefined });
252277
}
253278

254279
this.updateState({ matchedIndices });

‎src/plugins/data_views/public/services/get_indices.test.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ export const successfulResolveResponse = {
2121
aliases: [
2222
{
2323
name: 'f-alias',
24-
indices: ['freeze-index', 'my-index'],
24+
indices: ['my-index'],
2525
},
2626
],
2727
data_streams: [
@@ -69,6 +69,16 @@ describe('getIndices', () => {
6969
expect(result[2].name).toBe('remoteCluster1:bar-01');
7070
});
7171

72+
it('should work with rollup indices based on aliases', async () => {
73+
const isRollupIdx = (indexName: string) => indexName === 'my-index';
74+
const result = await getIndices({
75+
http,
76+
pattern: 'kibana',
77+
isRollupIndex: isRollupIdx,
78+
});
79+
expect(result[0].tags[1].key).toBe('rollup');
80+
});
81+
7282
it('should ignore ccs query-all', async () => {
7383
expect((await getIndices({ http, pattern: '*:', isRollupIndex })).length).toBe(0);
7484
});

‎src/plugins/data_views/public/services/get_indices.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,9 @@ export const responseToItemArray = (
119119
const isFrozen = (index.attributes || []).includes(ResolveIndexResponseItemIndexAttrs.FROZEN);
120120

121121
tags.push(...getTags(index.name));
122+
index.aliases?.forEach((alias) => {
123+
tags.push(...getTags(alias));
124+
});
122125
if (isFrozen) {
123126
tags.push({ name: frozenLabel, key: 'frozen', color: 'danger' });
124127
}
@@ -130,11 +133,15 @@ export const responseToItemArray = (
130133
});
131134
});
132135
(response.aliases || []).forEach((alias) => {
133-
source.push({
136+
const item = {
134137
name: alias.name,
135138
tags: [{ key: 'alias', name: aliasLabel, color: 'default' }],
136139
item: alias,
137-
});
140+
};
141+
// we only need to check the first index to see if its a rollup since there can only be one alias match
142+
item.tags.push(...getTags(alias.indices[0]));
143+
item.tags.push(...getTags(alias.name));
144+
source.push(item);
138145
});
139146
(response.data_streams || []).forEach((dataStream) => {
140147
source.push({

‎src/plugins/data_views/server/fetcher/index_patterns_fetcher.test.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,19 @@ describe('Index Pattern Fetcher - server', () => {
7171
expect(esClient.rollup.getRollupIndexCaps).toHaveBeenCalledTimes(1);
7272
});
7373

74+
it("works with index aliases - when rollup response doesn't have index as key", async () => {
75+
esClient.rollup.getRollupIndexCaps.mockResponse(
76+
rollupResponse as unknown as estypes.RollupGetRollupIndexCapsResponse
77+
);
78+
indexPatterns = new IndexPatternsFetcher(esClient, optionalParams);
79+
await indexPatterns.getFieldsForWildcard({
80+
pattern: patternList,
81+
type: DataViewType.ROLLUP,
82+
rollupIndex: 'foo',
83+
});
84+
expect(esClient.rollup.getRollupIndexCaps).toHaveBeenCalledTimes(1);
85+
});
86+
7487
it("doesn't call rollup api when given rollup data view and rollups are disabled", async () => {
7588
esClient.rollup.getRollupIndexCaps.mockResponse(
7689
rollupResponse as unknown as estypes.RollupGetRollupIndexCapsResponse

‎src/plugins/data_views/server/fetcher/index_patterns_fetcher.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -116,11 +116,15 @@ export class IndexPatternsFetcher {
116116

117117
if (this.rollupsEnabled && type === DataViewType.ROLLUP && rollupIndex) {
118118
const rollupFields: FieldDescriptor[] = [];
119-
const capabilityCheck = getCapabilitiesForRollupIndices(
119+
const capabilities = getCapabilitiesForRollupIndices(
120120
await this.elasticsearchClient.rollup.getRollupIndexCaps({
121121
index: rollupIndex,
122122
})
123-
)[rollupIndex];
123+
);
124+
125+
const capabilityCheck =
126+
// use the rollup index name BUT if its an alias, we'll take the first one
127+
capabilities[rollupIndex] || capabilities[Object.keys(capabilities)[0]];
124128

125129
if (capabilityCheck.error) {
126130
throw new Error(capabilityCheck.error);

‎x-pack/plugins/rollup/server/rollup_data_enricher.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
import { IScopedClusterClient } from '@kbn/core/server';
99
import { Index } from '@kbn/index-management-plugin/server';
10+
import { isArray } from 'lodash';
1011

1112
export const rollupDataEnricher = async (indicesList: Index[], client: IScopedClusterClient) => {
1213
if (!indicesList || !indicesList.length) {
@@ -19,7 +20,10 @@ export const rollupDataEnricher = async (indicesList: Index[], client: IScopedCl
1920
});
2021

2122
return indicesList.map((index) => {
22-
const isRollupIndex = !!rollupJobData[index.name];
23+
let isRollupIndex = !!rollupJobData[index.name];
24+
if (!isRollupIndex && isArray(index.aliases)) {
25+
isRollupIndex = index.aliases.some((alias) => !!rollupJobData[alias]);
26+
}
2327
return {
2428
...index,
2529
isRollupIndex,

‎x-pack/test/functional/apps/rollup_job/hybrid_index_pattern.js‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,30 @@ export default function ({ getService, getPageObjects }) {
113113
expect(fields).to.eql(['@timestamp', '_id', '_ignored', '_index', '_score', '_source']);
114114
});
115115

116+
it('create hybrid index pattern - with alias to rollup index', async () => {
117+
const rollupAlias = 'rollup-alias';
118+
await es.indices.putAlias({
119+
index: rollupTargetIndexName,
120+
name: rollupAlias,
121+
});
122+
await PageObjects.common.navigateToApp('settings');
123+
await PageObjects.settings.createIndexPattern(rollupAlias, '@timestamp', false);
124+
125+
await PageObjects.settings.clickKibanaIndexPatterns();
126+
const indexPatternNames = await PageObjects.settings.getAllIndexPatternNames();
127+
//The assertion is going to check that the string has the right name and that the text Rollup
128+
//is included (since there is a Rollup tag).
129+
const filteredIndexPatternNames = indexPatternNames.filter(
130+
(i) => i.includes(rollupIndexPatternName) && i.includes('Rollup')
131+
);
132+
expect(filteredIndexPatternNames.length).to.be(1);
133+
134+
// ensure all fields are available
135+
await PageObjects.settings.clickIndexPatternByName(rollupAlias);
136+
const fields = await PageObjects.settings.getFieldNames();
137+
expect(fields).to.eql(['@timestamp', '_id', '_ignored', '_index', '_score', '_source']);
138+
});
139+
116140
after(async () => {
117141
// Delete the rollup job.
118142
await es.rollup.deleteJob({ id: rollupJobName });

0 commit comments

Comments
 (0)