Skip to content

Commit 19afa29

Browse files
rfecherrfecher
committed
stats calculation tool using CLI framework (#387)
1 parent 68e9359 commit 19afa29

13 files changed

Lines changed: 429 additions & 254 deletions

File tree

‎core/ingest/src/main/java/mil/nga/giat/geowave/core/ingest/ClearNamespaceDriver.java‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,11 @@
33
import java.util.List;
44

55
import mil.nga.giat.geowave.core.store.IndexWriter;
6+
import mil.nga.giat.geowave.datastore.accumulo.AccumuloCommandLineOptions;
67

78
import org.apache.accumulo.core.client.AccumuloException;
89
import org.apache.accumulo.core.client.AccumuloSecurityException;
10+
import org.apache.accumulo.core.client.TableNotFoundException;
911
import org.apache.commons.cli.CommandLine;
1012
import org.apache.commons.cli.Options;
1113
import org.apache.commons.cli.ParseException;
@@ -20,6 +22,7 @@ public class ClearNamespaceDriver extends
2022
{
2123
private final static Logger LOGGER = Logger.getLogger(ClearNamespaceDriver.class);
2224
protected AccumuloCommandLineOptions accumulo;
25+
protected IngestCommandLineOptions ingest;
2326
protected IndexWriter indexWriter;
2427

2528
public ClearNamespaceDriver(
@@ -33,12 +36,14 @@ public void parseOptionsInternal(
3336
final CommandLine commandLine )
3437
throws ParseException {
3538
accumulo = AccumuloCommandLineOptions.parseOptions(commandLine);
39+
ingest = IngestCommandLineOptions.parseOptions(commandLine);
3640
}
3741

3842
@Override
3943
public void applyOptionsInternal(
4044
final Options allOptions ) {
4145
AccumuloCommandLineOptions.applyOptions(allOptions);
46+
IngestCommandLineOptions.applyOptions(allOptions);
4247
}
4348

4449
@Override
@@ -47,9 +52,9 @@ protected void runInternal(
4752
final List<IngestFormatPluginProviderSpi<?, ?>> pluginProviders ) {
4853
// just check if the flag to clear namespaces is set, and even if it is
4954
// not, clear it, but only if a namespace is provided
50-
if (!accumulo.isClearNamespace()) {
55+
if (!ingest.isClearNamespace()) {
5156
try {
52-
accumulo.clearNamespace();
57+
clearNamespace();
5358
}
5459
catch (AccumuloException | AccumuloSecurityException e) {
5560
LOGGER.fatal(
@@ -58,4 +63,23 @@ protected void runInternal(
5863
}
5964
}
6065
}
66+
67+
protected void clearNamespace()
68+
throws AccumuloException,
69+
AccumuloSecurityException {
70+
// don't delete all tables in the case that no namespace is given
71+
if ((accumulo.getNamespace() != null) && !accumulo.getNamespace().isEmpty()) {
72+
LOGGER.info("deleting all tables prefixed by '" + accumulo.getNamespace() + "'");
73+
try {
74+
accumulo.getAccumuloOperations().deleteAll();
75+
}
76+
catch (TableNotFoundException | AccumuloSecurityException | AccumuloException e) {
77+
LOGGER.error("Unable to clear accumulo namespace");
78+
}
79+
80+
}
81+
else {
82+
LOGGER.error("cannot clear a namespace if no namespace is provided");
83+
}
84+
}
6185
}

core/ingest/src/main/java/mil/nga/giat/geowave/core/ingest/AccumuloCommandLineOptions.java renamed to core/ingest/src/main/java/mil/nga/giat/geowave/core/ingest/IngestCommandLineOptions.java

Lines changed: 52 additions & 206 deletions
Original file line numberDiff line numberDiff line change
@@ -5,106 +5,32 @@
55
import java.util.Map;
66
import java.util.ServiceLoader;
77

8-
import mil.nga.giat.geowave.core.ingest.IndexCompatibilityVisitor;
9-
import mil.nga.giat.geowave.core.ingest.IngestDimensionalityTypeProviderSpi;
108
import mil.nga.giat.geowave.core.store.index.Index;
11-
import mil.nga.giat.geowave.datastore.accumulo.AccumuloOperations;
12-
import mil.nga.giat.geowave.datastore.accumulo.BasicAccumuloOperations;
139

14-
import org.apache.accumulo.core.client.AccumuloException;
15-
import org.apache.accumulo.core.client.AccumuloSecurityException;
16-
import org.apache.accumulo.core.client.TableNotFoundException;
1710
import org.apache.commons.cli.CommandLine;
1811
import org.apache.commons.cli.Option;
1912
import org.apache.commons.cli.Options;
2013
import org.apache.commons.cli.ParseException;
2114
import org.slf4j.Logger;
2215
import org.slf4j.LoggerFactory;
2316

24-
/**
25-
* This class encapsulates all of the options and parsed values specific to
26-
* setting up GeoWave to appropriately connect to Accumulo. This class also can
27-
* perform the function of clearing data for a namespace if that option is
28-
* activated.
29-
*
30-
*/
31-
public class AccumuloCommandLineOptions
17+
public class IngestCommandLineOptions
3218
{
33-
private final static Logger LOGGER = LoggerFactory.getLogger(AccumuloCommandLineOptions.class);
19+
private final static Logger LOGGER = LoggerFactory.getLogger(IngestCommandLineOptions.class);
20+
3421
private static Map<String, IndexCompatibilityVisitor> registeredDimensionalityTypes = null;
3522
private static String defaultDimensionalityType;
36-
private final String zookeepers;
37-
private final String instanceId;
38-
private final String user;
39-
private final String password;
40-
private final String namespace;
4123
private final String visibility;
4224
private final boolean clearNamespace;
4325
private final String dimensionalityType;
44-
private AccumuloOperations operations;
4526

46-
public AccumuloCommandLineOptions(
47-
final String zookeepers,
48-
final String instanceId,
49-
final String user,
50-
final String password,
51-
final String namespace,
27+
public IngestCommandLineOptions(
5228
final String visibility,
5329
final boolean clearNamespace,
54-
final String dimensionalityType )
55-
throws AccumuloException,
56-
AccumuloSecurityException {
57-
this.zookeepers = zookeepers;
58-
this.instanceId = instanceId;
59-
this.user = user;
60-
this.password = password;
61-
this.namespace = namespace;
30+
final String dimensionalityType ) {
6231
this.visibility = visibility;
6332
this.clearNamespace = clearNamespace;
6433
this.dimensionalityType = dimensionalityType;
65-
66-
if (clearNamespace) {
67-
clearNamespace();
68-
}
69-
}
70-
71-
protected void clearNamespace()
72-
throws AccumuloException,
73-
AccumuloSecurityException {
74-
// don't delete all tables in the case that no namespace is given
75-
if ((namespace != null) && !namespace.isEmpty()) {
76-
LOGGER.info("deleting all tables prefixed by '" + namespace + "'");
77-
try {
78-
getAccumuloOperations().deleteAll();
79-
}
80-
catch (TableNotFoundException | AccumuloSecurityException | AccumuloException e) {
81-
LOGGER.error("Unable to clear accumulo namespace");
82-
}
83-
84-
}
85-
else {
86-
LOGGER.error("cannot clear a namespace if no namespace is provided");
87-
}
88-
}
89-
90-
public String getZookeepers() {
91-
return zookeepers;
92-
}
93-
94-
public String getInstanceId() {
95-
return instanceId;
96-
}
97-
98-
public String getUser() {
99-
return user;
100-
}
101-
102-
public String getPassword() {
103-
return password;
104-
}
105-
106-
public String getNamespace() {
107-
return namespace;
10834
}
10935

11036
public String getVisibility() {
@@ -119,20 +45,6 @@ public boolean isClearNamespace() {
11945
return clearNamespace;
12046
}
12147

122-
public synchronized AccumuloOperations getAccumuloOperations()
123-
throws AccumuloException,
124-
AccumuloSecurityException {
125-
if (operations == null) {
126-
operations = new BasicAccumuloOperations(
127-
zookeepers,
128-
instanceId,
129-
user,
130-
password,
131-
namespace);
132-
}
133-
return operations;
134-
}
135-
13648
public Index getIndex(
13749
final Index[] supportedIndices ) {
13850
final IndexCompatibilityVisitor compatibilityVisitor = getSelectedIndexCompatibility(getDimensionalityType());
@@ -149,119 +61,6 @@ public boolean isSupported(
14961
return (getIndex(supportedIndices) != null);
15062
}
15163

152-
public static AccumuloCommandLineOptions parseOptions(
153-
final CommandLine commandLine )
154-
throws ParseException {
155-
boolean success = true;
156-
final String zookeepers = commandLine.getOptionValue("z");
157-
final String instanceId = commandLine.getOptionValue("i");
158-
final String user = commandLine.getOptionValue("u");
159-
final String password = commandLine.getOptionValue("p");
160-
boolean clearNamespace = false;
161-
if (commandLine.hasOption("c")) {
162-
clearNamespace = true;
163-
}
164-
String visibility = null;
165-
if (commandLine.hasOption("v")) {
166-
visibility = commandLine.getOptionValue("v");
167-
}
168-
final String namespace = commandLine.getOptionValue(
169-
"n",
170-
"");
171-
final String dimensionalityType = commandLine.getOptionValue(
172-
"dim",
173-
getDefaultDimensionalityType());
174-
if (zookeepers == null) {
175-
success = false;
176-
LOGGER.error("Zookeeper URL not set");
177-
}
178-
if (instanceId == null) {
179-
success = false;
180-
LOGGER.error("Accumulo instance ID not set");
181-
}
182-
if (user == null) {
183-
success = false;
184-
LOGGER.error("Accumulo user ID not set");
185-
}
186-
if (password == null) {
187-
success = false;
188-
LOGGER.error("Accumulo password not set");
189-
}
190-
if (!success) {
191-
throw new ParseException(
192-
"Required option is missing");
193-
}
194-
try {
195-
return new AccumuloCommandLineOptions(
196-
zookeepers,
197-
instanceId,
198-
user,
199-
password,
200-
namespace,
201-
visibility,
202-
clearNamespace,
203-
dimensionalityType);
204-
}
205-
catch (AccumuloException | AccumuloSecurityException e) {
206-
LOGGER.error(
207-
"Unable to connect to Accumulo with the specified options",
208-
e);
209-
}
210-
return null;
211-
}
212-
213-
public static void applyOptions(
214-
final Options allOptions ) {
215-
final Option zookeeperUrl = new Option(
216-
"z",
217-
"zookeepers",
218-
true,
219-
"A comma-separated list of zookeeper servers that an Accumulo instance is using");
220-
allOptions.addOption(zookeeperUrl);
221-
final Option instanceId = new Option(
222-
"i",
223-
"instance-id",
224-
true,
225-
"The Accumulo instance ID");
226-
allOptions.addOption(instanceId);
227-
final Option user = new Option(
228-
"u",
229-
"user",
230-
true,
231-
"A valid Accumulo user ID");
232-
allOptions.addOption(user);
233-
final Option password = new Option(
234-
"p",
235-
"password",
236-
true,
237-
"The password for the user");
238-
allOptions.addOption(password);
239-
final Option visibility = new Option(
240-
"v",
241-
"visibility",
242-
true,
243-
"The visibility of the data ingested (optional; default is 'public')");
244-
allOptions.addOption(visibility);
245-
246-
final Option namespace = new Option(
247-
"n",
248-
"namespace",
249-
true,
250-
"The table namespace (optional; default is no namespace)");
251-
allOptions.addOption(namespace);
252-
final Option dimensionalityType = new Option(
253-
"dim",
254-
"dimensionality",
255-
true,
256-
"The preferred dimensionality type to index the data for this ingest operation. " + getDimensionalityTypeOptionDescription());
257-
allOptions.addOption(dimensionalityType);
258-
allOptions.addOption(new Option(
259-
"c",
260-
"clear",
261-
false,
262-
"Clear ALL data stored with the same prefix as this namespace (optional; default is to append data to the namespace if it exists)"));
263-
}
264-
26564
private static synchronized String getDimensionalityTypeOptionDescription() {
26665
if (registeredDimensionalityTypes == null) {
26766
initDimensionalityTypeRegistry();
@@ -339,4 +138,51 @@ private static synchronized void initDimensionalityTypeRegistry() {
339138
}
340139
}
341140
}
141+
142+
public static IngestCommandLineOptions parseOptions(
143+
final CommandLine commandLine )
144+
throws ParseException {
145+
final boolean success = true;
146+
boolean clearNamespace = false;
147+
if (commandLine.hasOption("c")) {
148+
clearNamespace = true;
149+
}
150+
String visibility = null;
151+
if (commandLine.hasOption("v")) {
152+
visibility = commandLine.getOptionValue("v");
153+
}
154+
final String dimensionalityType = commandLine.getOptionValue(
155+
"dim",
156+
getDefaultDimensionalityType());
157+
if (!success) {
158+
throw new ParseException(
159+
"Required option is missing");
160+
}
161+
return new IngestCommandLineOptions(
162+
visibility,
163+
clearNamespace,
164+
dimensionalityType);
165+
}
166+
167+
public static void applyOptions(
168+
final Options allOptions ) {
169+
final Option visibility = new Option(
170+
"v",
171+
"visibility",
172+
true,
173+
"The visibility of the data ingested (optional; default is 'public')");
174+
allOptions.addOption(visibility);
175+
176+
final Option dimensionalityType = new Option(
177+
"dim",
178+
"dimensionality",
179+
true,
180+
"The preferred dimensionality type to index the data for this ingest operation. " + getDimensionalityTypeOptionDescription());
181+
allOptions.addOption(dimensionalityType);
182+
allOptions.addOption(new Option(
183+
"c",
184+
"clear",
185+
false,
186+
"Clear ALL data stored with the same prefix as this namespace (optional; default is to append data to the namespace if it exists)"));
187+
}
342188
}

0 commit comments

Comments
 (0)