Skip to content

Commit 95f8d9a

Browse files
petardzpetardz
authored
Rollover for alerts/findings history indices (opensearch-project#82)
Signed-off-by: Petar Dzepina <petar.dzepina@gmail.com>
1 parent bfb2b23 commit 95f8d9a

11 files changed

Lines changed: 1277 additions & 30 deletions

File tree

‎src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
package org.opensearch.securityanalytics;
66

77
import java.util.Collection;
8+
import java.util.Collections;
89
import java.util.List;
910
import java.util.function.Supplier;
1011
import org.opensearch.action.ActionRequest;
@@ -13,6 +14,7 @@
1314
import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
1415
import org.opensearch.cluster.node.DiscoveryNodes;
1516
import org.opensearch.cluster.service.ClusterService;
17+
import org.opensearch.common.component.LifecycleComponent;
1618
import org.opensearch.common.io.stream.NamedWriteableRegistry;
1719
import org.opensearch.common.settings.ClusterSettings;
1820
import org.opensearch.common.settings.IndexScopedSettings;
@@ -39,6 +41,7 @@
3941
import org.opensearch.securityanalytics.action.IndexDetectorAction;
4042
import org.opensearch.securityanalytics.action.SearchDetectorAction;
4143
import org.opensearch.securityanalytics.action.UpdateIndexMappingsAction;
44+
import org.opensearch.securityanalytics.indexmanagment.DetectorIndexManagementService;
4245
import org.opensearch.securityanalytics.action.ValidateRulesAction;
4346
import org.opensearch.securityanalytics.mapper.MapperService;
4447
import org.opensearch.securityanalytics.resthandler.RestAcknowledgeAlertsAction;
@@ -102,6 +105,8 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin {
102105

103106
private RuleIndices ruleIndices;
104107

108+
private DetectorIndexManagementService detectorIndexManagementService;
109+
105110
@Override
106111
public Collection<Object> createComponents(Client client,
107112
ClusterService clusterService,
@@ -121,6 +126,11 @@ public Collection<Object> createComponents(Client client,
121126
return List.of(detectorIndices, ruleTopicIndices, ruleIndices, mapperService);
122127
}
123128

129+
@Override
130+
public Collection<Class<? extends LifecycleComponent>> getGuiceServiceClasses() {
131+
return Collections.singletonList(DetectorIndexManagementService.class);
132+
}
133+
124134
@Override
125135
public List<RestHandler> getRestHandlers(Settings settings,
126136
RestController restController,
@@ -160,7 +170,20 @@ public List<NamedXContentRegistry.Entry> getNamedXContent() {
160170
@Override
161171
public List<Setting<?>> getSettings() {
162172
return List.of(
163-
SecurityAnalyticsSettings.INDEX_TIMEOUT
173+
SecurityAnalyticsSettings.INDEX_TIMEOUT,
174+
SecurityAnalyticsSettings.ALERT_HISTORY_ENABLED,
175+
SecurityAnalyticsSettings.ALERT_HISTORY_ROLLOVER_PERIOD,
176+
SecurityAnalyticsSettings.ALERT_HISTORY_INDEX_MAX_AGE,
177+
SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS,
178+
SecurityAnalyticsSettings.ALERT_HISTORY_RETENTION_PERIOD,
179+
SecurityAnalyticsSettings.REQUEST_TIMEOUT,
180+
SecurityAnalyticsSettings.MAX_ACTION_THROTTLE_VALUE,
181+
SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES,
182+
SecurityAnalyticsSettings.FINDING_HISTORY_ENABLED,
183+
SecurityAnalyticsSettings.FINDING_HISTORY_MAX_DOCS,
184+
SecurityAnalyticsSettings.FINDING_HISTORY_INDEX_MAX_AGE,
185+
SecurityAnalyticsSettings.FINDING_HISTORY_ROLLOVER_PERIOD,
186+
SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD
164187
);
165188
}
166189

‎src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -243,7 +243,7 @@ public void getAlerts(List<String> alertIds,
243243
"ALL",
244244
"ALL",
245245
null,
246-
DetectorMonitorConfig.getAlertsIndex(detector.getDetectorType()),
246+
DetectorMonitorConfig.getAllAlertsIndicesPattern(detector.getDetectorType()),
247247
null,
248248
alertIds);
249249
AlertingPluginInterface.INSTANCE.getAlerts(

‎src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java‎

Lines changed: 42 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
*/
55
package org.opensearch.securityanalytics.config.monitors;
66

7+
import java.util.ArrayList;
8+
import java.util.List;
79
import org.opensearch.securityanalytics.model.Detector;
810

911
import java.util.Arrays;
@@ -13,71 +15,80 @@
1315

1416

1517
public class DetectorMonitorConfig {
18+
1619
public static final String OPENSEARCH_DEFAULT_RULE_INDEX = ".opensearch-sap-detectors-queries-default";
1720
public static final String OPENSEARCH_DEFAULT_ALERT_INDEX = ".opensearch-sap-alerts-default";
1821
public static final String OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX = ".opensearch-sap-alerts-history-default";
1922
public static final String OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX_PATTERN = "<.opensearch-sap-alerts-history-default-{now/d}-1>";
2023
public static final String OPENSEARCH_DEFAULT_FINDINGS_INDEX = ".opensearch-sap-findings-default";
2124
public static final String OPENSEARCH_DEFAULT_FINDINGS_INDEX_PATTERN = "<.opensearch-sap-findings-default-{now/d}-1>";
2225

23-
private static Map<String, MonitorConfig> ruleIndexByDetectorTypeMap;
26+
private static Map<String, MonitorConfig> detectorTypeToIndicesMapping;
2427

2528
static {
26-
ruleIndexByDetectorTypeMap = new HashMap<>();
29+
detectorTypeToIndicesMapping = new HashMap<>();
2730
Arrays.stream(Detector.DetectorType.values()).forEach(
2831
detectorType -> {
2932
String ruleIndex = String.format(
30-
Locale.getDefault(), ".opensearch-sap-detectors-queries-%s", detectorType.getDetectorType());
33+
Locale.getDefault(), ".opensearch-sap-%s-detectors-queries", detectorType.getDetectorType());
3134
String alertsIndex = String.format(
32-
Locale.getDefault(), ".opensearch-sap-alerts-%s", detectorType.getDetectorType());
35+
Locale.getDefault(), ".opensearch-sap-%s-alerts", detectorType.getDetectorType());
3336
String alertsHistoryIndex = String.format(
34-
Locale.getDefault(), ".opensearch-sap-alerts-history-%s", detectorType.getDetectorType());
37+
Locale.getDefault(), ".opensearch-sap-%s-alerts-history", detectorType.getDetectorType());
3538
String alertsHistoryIndexPattern = String.format(
36-
Locale.getDefault(), "<.opensearch-sap-alerts-history-%s-{now/d}-1>", detectorType.getDetectorType());
39+
Locale.getDefault(), "<.opensearch-sap-%s-alerts-history-{now/d}-1>", detectorType.getDetectorType());
40+
String allAlertsIndicesPattern = String.format(
41+
Locale.getDefault(), ".opensearch-sap-%s-alerts*", detectorType.getDetectorType());
3742
String findingsIndex = String.format(
38-
Locale.getDefault(), ".opensearch-sap-findings-%s", detectorType.getDetectorType());
43+
Locale.getDefault(), ".opensearch-sap-%s-findings", detectorType.getDetectorType());
3944
String findingsIndexPattern = String.format(
40-
Locale.getDefault(), "<.opensearch-sap-findings-%s-{now/d}-1>", detectorType.getDetectorType());
45+
Locale.getDefault(), "<.opensearch-sap-%s-findings-{now/d}-1>", detectorType.getDetectorType());
4146

42-
MonitorConfig monitor = new MonitorConfig(alertsIndex, alertsHistoryIndex, alertsHistoryIndexPattern, findingsIndex, findingsIndexPattern, ruleIndex);
43-
ruleIndexByDetectorTypeMap.put(detectorType.getDetectorType(), monitor);
47+
MonitorConfig monitor = new MonitorConfig(alertsIndex, alertsHistoryIndex, alertsHistoryIndexPattern, allAlertsIndicesPattern, findingsIndex, findingsIndexPattern, ruleIndex);
48+
detectorTypeToIndicesMapping.put(detectorType.getDetectorType(), monitor);
4449
});
4550
}
4651

4752
public static String getRuleIndex(String detectorType) {
48-
return ruleIndexByDetectorTypeMap.containsKey(detectorType) ?
49-
ruleIndexByDetectorTypeMap.get(detectorType).getRuleIndex() :
53+
return detectorTypeToIndicesMapping.containsKey(detectorType) ?
54+
detectorTypeToIndicesMapping.get(detectorType).getRuleIndex() :
5055
OPENSEARCH_DEFAULT_RULE_INDEX;
5156
}
5257

5358
public static String getAlertsIndex(String detectorType) {
54-
return ruleIndexByDetectorTypeMap.containsKey(detectorType) ?
55-
ruleIndexByDetectorTypeMap.get(detectorType).getAlertsIndex() :
59+
return detectorTypeToIndicesMapping.containsKey(detectorType) ?
60+
detectorTypeToIndicesMapping.get(detectorType).getAlertsIndex() :
5661
OPENSEARCH_DEFAULT_ALERT_INDEX;
5762
}
5863

5964
public static String getAlertsHistoryIndex(String detectorType) {
60-
return ruleIndexByDetectorTypeMap.containsKey(detectorType) ?
61-
ruleIndexByDetectorTypeMap.get(detectorType).getAlertsHistoryIndex() :
65+
return detectorTypeToIndicesMapping.containsKey(detectorType) ?
66+
detectorTypeToIndicesMapping.get(detectorType).getAlertsHistoryIndex() :
6267
OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX;
6368
}
6469

6570
public static String getAlertsHistoryIndexPattern(String detectorType) {
66-
return ruleIndexByDetectorTypeMap.containsKey(detectorType) ?
67-
ruleIndexByDetectorTypeMap.get(detectorType).getAlertsHistoryIndexPattern() :
71+
return detectorTypeToIndicesMapping.containsKey(detectorType) ?
72+
detectorTypeToIndicesMapping.get(detectorType).getAlertsHistoryIndexPattern() :
6873
OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX_PATTERN;
6974
}
7075

76+
public static String getAllAlertsIndicesPattern(String detectorType) {
77+
return detectorTypeToIndicesMapping.containsKey(detectorType) ?
78+
detectorTypeToIndicesMapping.get(detectorType).getAllAlertsIndicesPattern() :
79+
"*";
80+
}
81+
7182
public static String getFindingsIndex(String detectorType) {
72-
return ruleIndexByDetectorTypeMap.containsKey(detectorType) ?
73-
ruleIndexByDetectorTypeMap.get(detectorType).getFindingsIndex() :
83+
return detectorTypeToIndicesMapping.containsKey(detectorType) ?
84+
detectorTypeToIndicesMapping.get(detectorType).getFindingsIndex() :
7485
OPENSEARCH_DEFAULT_FINDINGS_INDEX;
7586
}
7687

7788
public static String getFindingsIndexPattern(String detectorType) {
78-
return ruleIndexByDetectorTypeMap.containsKey(detectorType) ?
79-
ruleIndexByDetectorTypeMap.get(detectorType).getFindingsIndexPattern() :
80-
OPENSEARCH_DEFAULT_FINDINGS_INDEX;
89+
return detectorTypeToIndicesMapping.containsKey(detectorType) ?
90+
detectorTypeToIndicesMapping.get(detectorType).getFindingsIndexPattern() :
91+
OPENSEARCH_DEFAULT_FINDINGS_INDEX_PATTERN;
8192
}
8293

8394
public static Map<String, Map<String, String>> getRuleIndexMappingsByType(String detectorType) {
@@ -88,10 +99,11 @@ public static Map<String, Map<String, String>> getRuleIndexMappingsByType(String
8899
return fieldMappingProperties;
89100
}
90101

91-
private static class MonitorConfig {
102+
public static class MonitorConfig {
92103
private final String alertsIndex;
93104
private final String alertsHistoryIndex;
94105
private final String alertsHistoryIndexPattern;
106+
private final String allAlertsIndicesPattern;
95107
private final String findingIndex;
96108
private final String findingsIndexPattern;
97109
private final String ruleIndex;
@@ -100,13 +112,15 @@ private MonitorConfig(
100112
String alertsIndex,
101113
String alertsHistoryIndex,
102114
String alertsHistoryIndexPattern,
115+
String allAlertsIndicesPattern,
103116
String findingsIndex,
104117
String findingsIndexPattern,
105118
String ruleIndex
106119
) {
107120
this.alertsIndex = alertsIndex;
108121
this.alertsHistoryIndex = alertsHistoryIndex;
109122
this.alertsHistoryIndexPattern = alertsHistoryIndexPattern;
123+
this.allAlertsIndicesPattern = allAlertsIndicesPattern;
110124
this.findingIndex = findingsIndex;
111125
this.findingsIndexPattern = findingsIndexPattern;
112126
this.ruleIndex = ruleIndex;
@@ -124,6 +138,10 @@ public String getAlertsHistoryIndexPattern() {
124138
return alertsHistoryIndexPattern;
125139
}
126140

141+
public String getAllAlertsIndicesPattern() {
142+
return allAlertsIndicesPattern;
143+
}
144+
127145
public String getFindingsIndex() {
128146
return findingIndex;
129147
}

0 commit comments

Comments
 (0)