Skip to content

Commit 1d08187

Browse files
vamsimanoharvamsimanohar
committed
Added new interface methods in DataSourceService and DataSourceMetadataStorage
Signed-off-by: Vamsi Manohar <reddyvam@amazon.com>
1 parent e49d8f5 commit 1d08187

12 files changed

Lines changed: 170 additions & 44 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,3 +46,4 @@ gen
4646
/.prom.pid.lock
4747

4848
.java-version
49+
.worktrees

‎core/src/main/java/org/opensearch/sql/analysis/Analyzer.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,6 @@
6464
import org.opensearch.sql.data.model.ExprMissingValue;
6565
import org.opensearch.sql.data.type.ExprCoreType;
6666
import org.opensearch.sql.datasource.DataSourceService;
67-
import org.opensearch.sql.datasource.model.DataSource;
6867
import org.opensearch.sql.datasource.model.DataSourceMetadata;
6968
import org.opensearch.sql.exception.SemanticCheckException;
7069
import org.opensearch.sql.expression.DSL;
@@ -135,7 +134,7 @@ public LogicalPlan analyze(UnresolvedPlan unresolved, AnalysisContext context) {
135134
@Override
136135
public LogicalPlan visitRelation(Relation node, AnalysisContext context) {
137136
QualifiedName qualifiedName = node.getTableQualifiedName();
138-
Set<String> allowedDataSourceNames = dataSourceService.getMaskedDataSourceMetadataSet()
137+
Set<String> allowedDataSourceNames = dataSourceService.getDataSourceMetadataSet()
139138
.stream()
140139
.map(DataSourceMetadata::getName)
141140
.collect(Collectors.toSet());
@@ -183,7 +182,7 @@ public LogicalPlan visitRelationSubquery(RelationSubquery node, AnalysisContext
183182
@Override
184183
public LogicalPlan visitTableFunction(TableFunction node, AnalysisContext context) {
185184
QualifiedName qualifiedName = node.getFunctionName();
186-
Set<String> allowedDataSourceNames = dataSourceService.getMaskedDataSourceMetadataSet()
185+
Set<String> allowedDataSourceNames = dataSourceService.getDataSourceMetadataSet()
187186
.stream()
188187
.map(DataSourceMetadata::getName)
189188
.collect(Collectors.toSet());

‎core/src/main/java/org/opensearch/sql/datasource/DataSourceMetadataStorage.java‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,17 +9,56 @@
99

1010
import java.util.List;
1111
import java.util.Optional;
12+
import javax.xml.crypto.Data;
13+
import org.opensearch.sql.datasource.model.DataSource;
1214
import org.opensearch.sql.datasource.model.DataSourceMetadata;
1315

1416
/**
15-
* Interface for DataSourceMetadata Storage.
17+
* Interface for DataSourceMetadata Storage
18+
* which will be only used by DataSourceService for Storage.
1619
*/
1720
public interface DataSourceMetadataStorage {
1821

22+
/**
23+
* Returns all dataSource Metadata objects. The returned objects won't contain
24+
* any of the credential info.
25+
*
26+
* @return list of {@link DataSourceMetadata}.
27+
*/
1928
List<DataSourceMetadata> getDataSourceMetadata();
2029

30+
31+
/**
32+
* Gets {@link DataSourceMetadata} corresponding to the
33+
* datasourceName from underlying storage.
34+
*
35+
* @param datasourceName name of the {@link DataSource}.
36+
*/
2137
Optional<DataSourceMetadata> getDataSourceMetadata(String datasourceName);
2238

39+
40+
/**
41+
* Stores {@link DataSourceMetadata} in underlying storage.
42+
*
43+
* @param dataSourceMetadata {@link DataSourceMetadata}.
44+
*/
2345
void createDataSourceMetadata(DataSourceMetadata dataSourceMetadata);
2446

47+
48+
/**
49+
* Updates {@link DataSourceMetadata} in underlying storage.
50+
*
51+
* @param dataSourceMetadata {@link DataSourceMetadata}.
52+
*/
53+
void updateDataSourceMetadata(DataSourceMetadata dataSourceMetadata);
54+
55+
56+
/**
57+
* Deletes {@link DataSourceMetadata} corresponding to the
58+
* datasourceName from underlying storage.
59+
*
60+
* @param datasourceName name of the {@link DataSource}.
61+
*/
62+
void deleteDataSourceMetadata(String datasourceName);
63+
2564
}

‎core/src/main/java/org/opensearch/sql/datasource/DataSourceService.java‎

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,26 +15,49 @@
1515
public interface DataSourceService {
1616

1717
/**
18-
* Returns all DataSource objects.
18+
* Returns {@link DataSource} corresponding to the DataSource name.
1919
*
20-
* @return set of {@link DataSource}.
20+
* @param dataSourceName Name of the {@link DataSource}.
21+
* @return {@link DataSource}.
2122
*/
22-
Set<DataSourceMetadata> getMaskedDataSourceMetadataSet();
23+
DataSource getDataSource(String dataSourceName);
24+
2325

2426
/**
25-
* Returns {@link DataSource} with corresponding to the DataSource name.
27+
* Returns all dataSource Metadata objects. The returned objects won't contain
28+
* any of the credential info.
2629
*
27-
* @param dataSourceName Name of the {@link DataSource}.
28-
* @return {@link DataSource}.
30+
* @return set of {@link DataSourceMetadata}.
2931
*/
30-
DataSource getDataSource(String dataSourceName);
32+
Set<DataSourceMetadata> getDataSourceMetadataSet();
3133

3234
/**
3335
* Register {@link DataSource} defined by {@link DataSourceMetadata}.
3436
*
3537
* @param metadatas list of {@link DataSourceMetadata}.
3638
*/
37-
void addDataSource(DataSourceMetadata... metadatas);
39+
void createDataSource(DataSourceMetadata... metadatas);
40+
41+
/**
42+
* Updates {@link DataSource} corresponding to dataSourceMetadata.
43+
*
44+
* @param dataSourceMetadata {@link DataSourceMetadata}.
45+
*/
46+
void updateDataSource(DataSourceMetadata dataSourceMetadata);
47+
48+
49+
/**
50+
* Deletes {@link DataSource} corresponding to the DataSource name.
51+
*
52+
* @param dataSourceName name of the {@link DataSource}.
53+
*/
54+
void deleteDataSource(String dataSourceName);
55+
56+
/**
57+
* This method is to bootstrap
58+
* datasources during the startup of the plugin.
59+
*/
60+
void bootstrapDataSources();
3861

3962
/**
4063
* remove all the registered {@link DataSource}.

‎core/src/main/java/org/opensearch/sql/datasource/DataSourceServiceImpl.java‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ public DataSourceServiceImpl(Set<DataSourceFactory> dataSourceFactories) {
4747
}
4848

4949
@Override
50-
public Set<DataSourceMetadata> getMaskedDataSourceMetadataSet() {
50+
public Set<DataSourceMetadata> getDataSourceMetadataSet() {
5151
return dataSourceMap.values().stream()
5252
.map(dataSource
5353
-> new DataSourceMetadata(dataSource.getName(),
@@ -65,7 +65,7 @@ public DataSource getDataSource(String dataSourceName) {
6565
}
6666

6767
@Override
68-
public void addDataSource(DataSourceMetadata... metadatas) {
68+
public void createDataSource(DataSourceMetadata... metadatas) {
6969
for (DataSourceMetadata metadata : metadatas) {
7070
validateDataSourceMetaData(metadata);
7171
dataSourceMap.put(
@@ -74,6 +74,21 @@ public void addDataSource(DataSourceMetadata... metadatas) {
7474
}
7575
}
7676

77+
@Override
78+
public void updateDataSource(DataSourceMetadata dataSourceMetadata) {
79+
throw new UnsupportedOperationException("will be supported in future");
80+
}
81+
82+
@Override
83+
public void deleteDataSource(String dataSourceName) {
84+
throw new UnsupportedOperationException("will be supported in future");
85+
}
86+
87+
@Override
88+
public void bootstrapDataSources() {
89+
throw new UnsupportedOperationException("will be supported in future");
90+
}
91+
7792
@Override
7893
public void clear() {
7994
dataSourceMap.clear();

‎core/src/main/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableScan.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,14 +48,14 @@ public String explain() {
4848
public void open() {
4949
List<ExprValue> exprValues = new ArrayList<>();
5050
Set<DataSourceMetadata> dataSourceMetadataSet
51-
= dataSourceService.getMaskedDataSourceMetadataSet();
52-
for (DataSourceMetadata dataSource : dataSourceMetadataSet) {
51+
= dataSourceService.getDataSourceMetadataSet();
52+
for (DataSourceMetadata dataSourceMetadata : dataSourceMetadataSet) {
5353
exprValues.add(
5454
new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of(
5555
"DATASOURCE_NAME",
56-
ExprValueUtils.stringValue(dataSource.getName()),
56+
ExprValueUtils.stringValue(dataSourceMetadata.getName()),
5757
"CONNECTOR_TYPE",
58-
ExprValueUtils.stringValue(dataSource.getConnector().name())))));
58+
ExprValueUtils.stringValue(dataSourceMetadata.getConnector().name())))));
5959
}
6060
iterator = exprValues.iterator();
6161
}

‎core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ private class DefaultDataSourceService implements DataSourceService {
165165

166166

167167
@Override
168-
public Set<DataSourceMetadata> getMaskedDataSourceMetadataSet() {
168+
public Set<DataSourceMetadata> getDataSourceMetadataSet() {
169169
return ImmutableSet.of(new DataSourceMetadata(dataSource.getName(),
170170
dataSource.getConnectorType(), ImmutableMap.of()));
171171
}
@@ -176,10 +176,25 @@ public DataSource getDataSource(String dataSourceName) {
176176
}
177177

178178
@Override
179-
public void addDataSource(DataSourceMetadata... metadatas) {
179+
public void createDataSource(DataSourceMetadata... metadatas) {
180180
throw new UnsupportedOperationException();
181181
}
182182

183+
@Override
184+
public void updateDataSource(DataSourceMetadata dataSourceMetadata) {
185+
186+
}
187+
188+
@Override
189+
public void deleteDataSource(String dataSourceName) {
190+
191+
}
192+
193+
@Override
194+
public void bootstrapDataSources() {
195+
196+
}
197+
183198
@Override
184199
public void clear() {
185200
throw new UnsupportedOperationException();

‎core/src/test/java/org/opensearch/sql/datasource/DataSourceServiceImplTest.java‎

Lines changed: 40 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
package org.opensearch.sql.datasource;
77

88
import static org.junit.jupiter.api.Assertions.assertEquals;
9+
import static org.junit.jupiter.api.Assertions.assertNull;
910
import static org.junit.jupiter.api.Assertions.assertThrows;
1011
import static org.mockito.ArgumentMatchers.any;
1112
import static org.mockito.Mockito.lenient;
@@ -65,7 +66,7 @@ public void clear() {
6566

6667
@Test
6768
void getDataSourceSuccess() {
68-
dataSourceService.addDataSource(DataSourceMetadata.defaultOpenSearchDataSourceMetadata());
69+
dataSourceService.createDataSource(DataSourceMetadata.defaultOpenSearchDataSourceMetadata());
6970

7071
assertEquals(
7172
new DataSource(DEFAULT_DATASOURCE_NAME, DataSourceType.OPENSEARCH, storageEngine),
@@ -81,19 +82,21 @@ void getNotExistDataSourceShouldFail() {
8182

8283
@Test
8384
void getAddDataSourcesShouldSuccess() {
84-
assertEquals(0, dataSourceService.getMaskedDataSourceMetadataSet().size());
85+
assertEquals(0, dataSourceService.getDataSourceMetadataSet().size());
8586

86-
dataSourceService.addDataSource(metadata(NAME, DataSourceType.OPENSEARCH, ImmutableMap.of()));
87-
assertEquals(1, dataSourceService.getMaskedDataSourceMetadataSet().size());
87+
dataSourceService.createDataSource(metadata(NAME,
88+
DataSourceType.OPENSEARCH, ImmutableMap.of()));
89+
assertEquals(1, dataSourceService.getDataSourceMetadataSet().size());
8890
}
8991

9092
@Test
9193
void noDataSourceExistAfterClear() {
92-
dataSourceService.addDataSource(metadata(NAME, DataSourceType.OPENSEARCH, ImmutableMap.of()));
93-
assertEquals(1, dataSourceService.getMaskedDataSourceMetadataSet().size());
94+
dataSourceService.createDataSource(metadata(NAME,
95+
DataSourceType.OPENSEARCH, ImmutableMap.of()));
96+
assertEquals(1, dataSourceService.getDataSourceMetadataSet().size());
9497

9598
dataSourceService.clear();
96-
assertEquals(0, dataSourceService.getMaskedDataSourceMetadataSet().size());
99+
assertEquals(0, dataSourceService.getDataSourceMetadataSet().size());
97100
}
98101

99102
@Test
@@ -102,7 +105,7 @@ void metaDataMissingNameShouldFail() {
102105
assertThrows(
103106
IllegalArgumentException.class,
104107
() ->
105-
dataSourceService.addDataSource(
108+
dataSourceService.createDataSource(
106109
metadata(null, DataSourceType.OPENSEARCH, ImmutableMap.of())));
107110
assertEquals(
108111
"Missing Name Field from a DataSource. Name is a required parameter.",
@@ -115,7 +118,7 @@ void metaDataHasIllegalDataSourceNameShouldFail() {
115118
assertThrows(
116119
IllegalArgumentException.class,
117120
() ->
118-
dataSourceService.addDataSource(
121+
dataSourceService.createDataSource(
119122
metadata("prometheus.test", DataSourceType.OPENSEARCH, ImmutableMap.of())));
120123
assertEquals(
121124
"DataSource Name: prometheus.test contains illegal characters. "
@@ -128,26 +131,50 @@ void metaDataMissingPropertiesShouldFail() {
128131
IllegalArgumentException exception =
129132
assertThrows(
130133
IllegalArgumentException.class,
131-
() -> dataSourceService.addDataSource(metadata(NAME, DataSourceType.OPENSEARCH, null)));
134+
() -> dataSourceService.createDataSource(metadata(NAME,
135+
DataSourceType.OPENSEARCH, null)));
132136
assertEquals(
133137
"Missing properties field in catalog configuration. Properties are required parameters.",
134138
exception.getMessage());
135139
}
136140

137141
@Test
138142
void metaDataHasDuplicateNameShouldFail() {
139-
dataSourceService.addDataSource(metadata(NAME, DataSourceType.OPENSEARCH, ImmutableMap.of()));
140-
assertEquals(1, dataSourceService.getMaskedDataSourceMetadataSet().size());
143+
dataSourceService.createDataSource(metadata(NAME,
144+
DataSourceType.OPENSEARCH, ImmutableMap.of()));
145+
assertEquals(1, dataSourceService.getDataSourceMetadataSet().size());
141146

142147
IllegalArgumentException exception =
143148
assertThrows(
144149
IllegalArgumentException.class,
145-
() -> dataSourceService.addDataSource(metadata(NAME, DataSourceType.OPENSEARCH, null)));
150+
() -> dataSourceService.createDataSource(metadata(NAME,
151+
DataSourceType.OPENSEARCH, null)));
146152
assertEquals(
147153
String.format("Datasource name should be unique, Duplicate datasource found %s.", NAME),
148154
exception.getMessage());
149155
}
150156

157+
@Test
158+
void testUpdateDatasource() {
159+
assertThrows(
160+
UnsupportedOperationException.class,
161+
() -> dataSourceService.updateDataSource(new DataSourceMetadata()));
162+
}
163+
164+
@Test
165+
void testDeleteDatasource() {
166+
assertThrows(
167+
UnsupportedOperationException.class,
168+
() -> dataSourceService.deleteDataSource(NAME));
169+
}
170+
171+
@Test
172+
void testLoadDatasource() {
173+
assertThrows(
174+
UnsupportedOperationException.class,
175+
() -> dataSourceService.bootstrapDataSources());
176+
}
177+
151178
DataSourceMetadata metadata(String name, DataSourceType type, Map<String, String> properties) {
152179
DataSourceMetadata dataSourceMetadata = new DataSourceMetadata();
153180
dataSourceMetadata.setName(name);

‎core/src/test/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableScanTest.java‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.mockito.Mock;
2424
import org.mockito.junit.jupiter.MockitoExtension;
2525
import org.opensearch.sql.data.model.ExprTupleValue;
26+
import org.opensearch.sql.data.model.ExprValue;
2627
import org.opensearch.sql.data.model.ExprValueUtils;
2728
import org.opensearch.sql.datasource.DataSourceService;
2829
import org.opensearch.sql.datasource.model.DataSource;
@@ -59,17 +60,23 @@ void testIterator() {
5960
Set<DataSourceMetadata> dataSourceMetadata = dataSourceSet.stream()
6061
.map(dataSource -> new DataSourceMetadata(dataSource.getName(),
6162
dataSource.getConnectorType(), ImmutableMap.of())).collect(Collectors.toSet());
62-
when(dataSourceService.getMaskedDataSourceMetadataSet()).thenReturn(dataSourceMetadata);
63+
when(dataSourceService.getDataSourceMetadataSet()).thenReturn(dataSourceMetadata);
6364

6465
assertFalse(dataSourceTableScan.hasNext());
6566
dataSourceTableScan.open();
6667
assertTrue(dataSourceTableScan.hasNext());
68+
Set<ExprValue> exprTupleValues = new HashSet<>();
69+
while (dataSourceTableScan.hasNext()) {
70+
exprTupleValues.add(dataSourceTableScan.next());
71+
}
72+
73+
Set<ExprValue> expectedExprTupleValues = new HashSet<>();
6774
for (DataSource dataSource : dataSourceSet) {
68-
assertEquals(new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of(
69-
"DATASOURCE_NAME", ExprValueUtils.stringValue(dataSource.getName()),
70-
"CONNECTOR_TYPE", ExprValueUtils.stringValue(dataSource.getConnectorType().name())))),
71-
dataSourceTableScan.next());
75+
expectedExprTupleValues.add(new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of(
76+
"DATASOURCE_NAME", ExprValueUtils.stringValue(dataSource.getName()),
77+
"CONNECTOR_TYPE", ExprValueUtils.stringValue(dataSource.getConnectorType().name())))));
7278
}
79+
assertEquals(expectedExprTupleValues, exprTupleValues);
7380
}
7481

7582
}

0 commit comments

Comments
 (0)