Skip to content

Commit e2df87b

Browse files
vamsimanoharvamsimanohar
authored
refactor span (#4334)
Signed-off-by: Vamsi Manohar <reddyvam@amazon.com>
1 parent 326a678 commit e2df87b

17 files changed

Lines changed: 197 additions & 188 deletions

File tree

‎core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -478,6 +478,37 @@ public static Span span(UnresolvedExpression field, UnresolvedExpression value,
478478
return new Span(field, value, unit);
479479
}
480480

481+
/**
482+
* Creates a Span expression from a field and a span length literal. Parses string literals to
483+
* extract numeric value and time unit (e.g., "1h" -> value=1, unit=h).
484+
*
485+
* @param field The field expression to apply the span to
486+
* @param spanLengthLiteral The literal value containing either a string with embedded unit (e.g.,
487+
* "1h", "30m") or a plain number
488+
* @return A Span expression with parsed value and unit
489+
*/
490+
public static Span spanFromSpanLengthLiteral(
491+
UnresolvedExpression field, Literal spanLengthLiteral) {
492+
if (spanLengthLiteral.getType() == DataType.STRING) {
493+
String spanText = spanLengthLiteral.getValue().toString();
494+
String valueStr = spanText.replaceAll("[^0-9]", "");
495+
String unitStr = spanText.replaceAll("[0-9]", "");
496+
497+
if (valueStr.isEmpty()) {
498+
// No numeric value found, use the literal as-is
499+
return new Span(field, spanLengthLiteral, SpanUnit.NONE);
500+
} else {
501+
// Parse numeric value and unit
502+
Integer value = Integer.parseInt(valueStr);
503+
SpanUnit unit = unitStr.isEmpty() ? SpanUnit.NONE : SpanUnit.of(unitStr);
504+
return span(field, intLiteral(value), unit);
505+
}
506+
} else {
507+
// Non-string literal (e.g., integer)
508+
return span(field, spanLengthLiteral, SpanUnit.NONE);
509+
}
510+
}
511+
481512
public static Sort sort(UnresolvedPlan input, Field... sorts) {
482513
return new Sort(Arrays.asList(sorts)).attach(input);
483514
}

‎core/src/main/java/org/opensearch/sql/ast/expression/SpanUnit.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,20 +17,37 @@ public enum SpanUnit {
1717
NONE(""),
1818
MILLISECOND("ms"),
1919
MS("ms"),
20+
SECONDS("s"),
2021
SECOND("s"),
22+
SECS("s"),
23+
SEC("s"),
2124
S("s"),
25+
MINUTES("m"),
2226
MINUTE("m"),
27+
MINS("m"),
28+
MIN("m"),
2329
m("m"),
30+
HOURS("h"),
2431
HOUR("h"),
32+
HRS("h"),
33+
HR("h"),
2534
H("h"),
35+
DAYS("d"),
2636
DAY("d"),
2737
D("d"),
38+
WEEKS("w"),
2839
WEEK("w"),
2940
W("w"),
3041
MONTH("M"),
42+
MONTHS("M"),
43+
MON("M"),
3144
M("M"),
45+
QUARTERS("q"),
3246
QUARTER("q"),
47+
QTRS("q"),
48+
QTR("q"),
3349
Q("q"),
50+
YEARS("y"),
3451
YEAR("y"),
3552
Y("y");
3653

‎integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java‎

Lines changed: 0 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -506,33 +506,6 @@ public void testBinSpanWithStartEndNeverShrinkRange() throws IOException {
506506
rows("39-40", 39));
507507
}
508508

509-
@Test
510-
public void testBinFloatingPointSpanBasicFunctionality() throws IOException {
511-
JSONObject result =
512-
executeQuery(
513-
String.format(
514-
"source=%s | bin age span=2.5 | fields age | head 3", TEST_INDEX_ACCOUNT));
515-
verifySchema(result, schema("age", null, "string"));
516-
517-
// Test that floating point spans work with proper range formatting
518-
verifyDataRows(result, rows("27.5-30.0"), rows("30.0-32.5"), rows("35.0-37.5"));
519-
}
520-
521-
@Test
522-
public void testBinFloatingPointSpanWithStats() throws IOException {
523-
JSONObject result =
524-
executeQuery(
525-
String.format(
526-
"source=%s | bin balance span=15000.5 | fields balance | sort balance |"
527-
+ " head 2",
528-
TEST_INDEX_ACCOUNT));
529-
530-
verifySchema(result, schema("balance", null, "string"));
531-
532-
// Test floating point spans without aggregation - verify proper decimal formatting
533-
verifyDataRows(result, rows("0.0-15000.5"), rows("0.0-15000.5"));
534-
}
535-
536509
@Test
537510
@Ignore
538511
public void testBinWithNumericSpanStatsCount() throws IOException {

‎integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java‎

Lines changed: 15 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -458,7 +458,7 @@ public void testAvgByTimeSpanAndFields() throws IOException {
458458
JSONObject actual =
459459
executeQuery(
460460
String.format(
461-
"source=%s | stats avg(balance) by span(birthdate, 1 month) as age_balance",
461+
"source=%s | stats avg(balance) by span(birthdate, 1month) as age_balance",
462462
TEST_INDEX_BANK));
463463
verifySchema(actual, schema("age_balance", "timestamp"), schema("avg(balance)", "double"));
464464
verifyDataRows(
@@ -475,7 +475,7 @@ public void testCountByCustomTimeSpanWithDifferentUnits() throws IOException {
475475
JSONObject actual =
476476
executeQuery(
477477
String.format(
478-
"source=%s | head 5 | stats count(datetime0) by span(datetime0, 15 minute) as"
478+
"source=%s | head 5 | stats count(datetime0) by span(datetime0, 15minute) as"
479479
+ " datetime_span",
480480
TEST_INDEX_CALCS));
481481
verifySchema(
@@ -491,7 +491,7 @@ public void testCountByCustomTimeSpanWithDifferentUnits() throws IOException {
491491
actual =
492492
executeQuery(
493493
String.format(
494-
"source=%s | head 5 | stats count(datetime0) by span(datetime0, 5 second) as"
494+
"source=%s | head 5 | stats count(datetime0) by span(datetime0, 5second) as"
495495
+ " datetime_span",
496496
TEST_INDEX_CALCS));
497497
verifySchema(
@@ -507,7 +507,7 @@ public void testCountByCustomTimeSpanWithDifferentUnits() throws IOException {
507507
actual =
508508
executeQuery(
509509
String.format(
510-
"source=%s | head 5 | stats count(datetime0) by span(datetime0, 3 month) as"
510+
"source=%s | head 5 | stats count(datetime0) by span(datetime0, 3month) as"
511511
+ " datetime_span",
512512
TEST_INDEX_CALCS));
513513
verifySchema(
@@ -521,7 +521,7 @@ public void testCountByNullableTimeSpan() throws IOException {
521521
executeQuery(
522522
String.format(
523523
"source=%s | head 5 | stats count(datetime0), count(datetime1) by span(time1,"
524-
+ " 15 minute) as time_span",
524+
+ " 15minute) as time_span",
525525
TEST_INDEX_CALCS));
526526
verifySchema(
527527
actual,
@@ -541,24 +541,23 @@ public void testCountByDateTypeSpanWithDifferentUnits() throws IOException {
541541
JSONObject actual =
542542
executeQuery(
543543
String.format(
544-
"source=%s | stats count(strict_date) by span(strict_date, 1 day) as"
545-
+ " date_span",
544+
"source=%s | stats count(strict_date) by span(strict_date, 1day) as" + " date_span",
546545
TEST_INDEX_DATE_FORMATS));
547546
verifySchema(actual, schema("date_span", "date"), schema("count(strict_date)", "bigint"));
548547
verifyDataRows(actual, rows(2, "1984-04-12"));
549548

550549
actual =
551550
executeQuery(
552551
String.format(
553-
"source=%s | stats count(basic_date) by span(basic_date, 1 year) as" + " date_span",
552+
"source=%s | stats count(basic_date) by span(basic_date, 1year) as" + " date_span",
554553
TEST_INDEX_DATE_FORMATS));
555554
verifySchema(actual, schema("date_span", "date"), schema("count(basic_date)", "bigint"));
556555
verifyDataRows(actual, rows(2, "1984-01-01"));
557556

558557
actual =
559558
executeQuery(
560559
String.format(
561-
"source=%s | stats count(year_month_day) by span(year_month_day, 1 month)"
560+
"source=%s | stats count(year_month_day) by span(year_month_day, 1month)"
562561
+ " as date_span",
563562
TEST_INDEX_DATE_FORMATS));
564563
verifySchema(actual, schema("date_span", "date"), schema("count(year_month_day)", "bigint"));
@@ -571,7 +570,7 @@ public void testCountByTimeTypeSpanWithDifferentUnits() throws IOException {
571570
executeQuery(
572571
String.format(
573572
"source=%s | stats count(hour_minute_second) by span(hour_minute_second, 1"
574-
+ " minute) as time_span",
573+
+ "minute) as time_span",
575574
TEST_INDEX_DATE_FORMATS));
576575
verifySchema(
577576
actual, schema("time_span", "time"), schema("count(hour_minute_second)", "bigint"));
@@ -580,7 +579,7 @@ public void testCountByTimeTypeSpanWithDifferentUnits() throws IOException {
580579
actual =
581580
executeQuery(
582581
String.format(
583-
"source=%s | stats count(custom_time) by span(custom_time, 1 second) as"
582+
"source=%s | stats count(custom_time) by span(custom_time, 1second) as"
584583
+ " time_span",
585584
TEST_INDEX_DATE_FORMATS));
586585
verifySchema(actual, schema("time_span", "time"), schema("count(custom_time)", "bigint"));
@@ -589,7 +588,7 @@ public void testCountByTimeTypeSpanWithDifferentUnits() throws IOException {
589588
actual =
590589
executeQuery(
591590
String.format(
592-
"source=%s | stats count(hour) by span(hour, 6 hour) as time_span",
591+
"source=%s | stats count(hour) by span(hour, 6hour) as time_span",
593592
TEST_INDEX_DATE_FORMATS));
594593
verifySchema(actual, schema("time_span", "time"), schema("count(hour)", "bigint"));
595594
verifyDataRows(actual, rows(2, "06:00:00"));
@@ -689,7 +688,7 @@ public void testCountBySpanForCustomFormats() throws IOException {
689688
executeQuery(
690689
String.format(
691690
"source=%s | stats count(custom_date_or_date) by span(custom_date_or_date, 1"
692-
+ " month) as date_span",
691+
+ "month) as date_span",
693692
TEST_INDEX_DATE_FORMATS));
694693
verifySchema(
695694
actual, schema("date_span", "date"), schema("count(custom_date_or_date)", "bigint"));
@@ -699,7 +698,7 @@ public void testCountBySpanForCustomFormats() throws IOException {
699698
executeQuery(
700699
String.format(
701700
"source=%s | stats count(custom_date_or_custom_time) by"
702-
+ " span(custom_date_or_custom_time, 1 hour) as timestamp_span",
701+
+ " span(custom_date_or_custom_time, 1hour) as timestamp_span",
703702
TEST_INDEX_DATE_FORMATS));
704703
verifySchema(
705704
actual,
@@ -711,7 +710,7 @@ public void testCountBySpanForCustomFormats() throws IOException {
711710
executeQuery(
712711
String.format(
713712
"source=%s | stats count(custom_no_delimiter_ts) by span(custom_no_delimiter_ts, 1"
714-
+ " hour) as timestamp_span",
713+
+ "hour) as timestamp_span",
715714
TEST_INDEX_DATE_FORMATS));
716715
verifySchema(
717716
actual,
@@ -723,7 +722,7 @@ public void testCountBySpanForCustomFormats() throws IOException {
723722
executeQuery(
724723
String.format(
725724
"source=%s | stats count(incomplete_custom_time) by span(incomplete_custom_time, 12"
726-
+ " hour) as time_span",
725+
+ "hour) as time_span",
727726
TEST_INDEX_DATE_FORMATS));
728727
verifySchema(
729728
actual, schema("time_span", "time"), schema("count(incomplete_custom_time)", "bigint"));

‎integ-test/src/test/java/org/opensearch/sql/ppl/SearchCommandIT.java‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -917,4 +917,77 @@ public void testSearchWithDateINOperator() throws IOException {
917917
rows("2024-01-15 10:30:00.123456789", "INFO"),
918918
rows("2024-01-15 10:30:01.23456789", "ERROR"));
919919
}
920+
921+
@Test
922+
public void testSearchWithTraceId() throws IOException {
923+
// Test 1: Search for specific traceId
924+
JSONObject specificTraceId =
925+
executeQuery(
926+
String.format(
927+
"search source=%s b3cb01a03c846973fd496b973f49be85 | fields" + " traceId, body",
928+
TEST_INDEX_OTEL_LOGS));
929+
verifyDataRows(
930+
specificTraceId,
931+
rows(
932+
"b3cb01a03c846973fd496b973f49be85",
933+
"User e1ce63e6-8501-11f0-930d-c2fcbdc05f14 adding 4 of product HQTGWGPNH4 to cart"));
934+
}
935+
936+
@Test
937+
public void testSearchWithSpanLength() throws IOException {
938+
// Test searching for SPANLENGTH keyword in free text search
939+
// This tests that SPANLENGTH tokens like "3month" are searchable
940+
JSONObject result =
941+
executeQuery(
942+
String.format(
943+
"search source=%s 3month | fields body, `attributes.span.duration`",
944+
TEST_INDEX_OTEL_LOGS));
945+
verifyDataRows(result, rows("Processing data for 3month period", "3month"));
946+
}
947+
948+
@Test
949+
public void testSearchWithSpanLengthInField() throws IOException {
950+
// Test searching for SPANLENGTH value in a specific field
951+
JSONObject result =
952+
executeQuery(
953+
String.format(
954+
"search source=%s `attributes.span.duration`=\\\"3month\\\" | fields body,"
955+
+ " `attributes.span.duration`",
956+
TEST_INDEX_OTEL_LOGS));
957+
verifyDataRows(result, rows("Processing data for 3month period", "3month"));
958+
}
959+
960+
@Test
961+
public void testSearchWithNumericIdVsSpanLength() throws IOException {
962+
// Test that NUMERIC_ID tokens like "1s4f7" (which start with what could be a SPANLENGTH like
963+
// "1s")
964+
// are properly searchable as complete tokens
965+
// This verifies that NUMERIC_ID takes precedence over SPANLENGTH when applicable
966+
967+
// Test 1: Search for the NUMERIC_ID token in free text
968+
JSONObject numericIdResult =
969+
executeQuery(
970+
String.format(
971+
"search source=%s 1s4f7 | fields body, `attributes.transaction.id`",
972+
TEST_INDEX_OTEL_LOGS));
973+
verifyDataRows(numericIdResult, rows("Transaction ID 1s4f7 processed successfully", "1s4f7"));
974+
975+
// Test 2: Search for NUMERIC_ID in specific field
976+
JSONObject fieldSearchResult =
977+
executeQuery(
978+
String.format(
979+
"search source=%s `attributes.transaction.id`=1s4f7 | fields body,"
980+
+ " `attributes.transaction.id`",
981+
TEST_INDEX_OTEL_LOGS));
982+
verifyDataRows(fieldSearchResult, rows("Transaction ID 1s4f7 processed successfully", "1s4f7"));
983+
984+
// Test 3: Verify that searching for just "1s" (which would be a SPANLENGTH)
985+
// does NOT match the "1s4f7" token
986+
JSONObject spanLengthSearchResult =
987+
executeQuery(
988+
String.format(
989+
"search source=%s `attributes.transaction.id`=1s | fields body",
990+
TEST_INDEX_OTEL_LOGS));
991+
verifyDataRows(spanLengthSearchResult); // Should return no results
992+
}
920993
}

‎integ-test/src/test/resources/expectedOutput/calcite/explain_search_basic_text.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"calcite": {
3-
"logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$153], severityNumber=[$154], time=[$155], body=[$156])\n LogicalFilter(condition=[query_string(MAP('query', 'ERROR':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]])\n",
3+
"logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$162], severityNumber=[$163], time=[$164], body=[$165])\n LogicalFilter(condition=[query_string(MAP('query', 'ERROR':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]])\n",
44
"physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[PROJECT->[spanId, traceId, @timestamp, instrumentationScope, severityText, resource, flags, attributes, droppedAttributesCount, severityNumber, time, body], FILTER->query_string(MAP('query', 'ERROR':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"ERROR\",\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"spanId\",\"traceId\",\"@timestamp\",\"instrumentationScope\",\"severityText\",\"resource\",\"flags\",\"attributes\",\"droppedAttributesCount\",\"severityNumber\",\"time\",\"body\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n"
55
}
66
}

‎integ-test/src/test/resources/expectedOutput/calcite/explain_search_numeric_comparison.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"calcite": {
3-
"logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$153], severityNumber=[$154], time=[$155], body=[$156])\n LogicalFilter(condition=[query_string(MAP('query', 'severityNumber:>15':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]])\n",
3+
"logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(spanId=[$0], traceId=[$1], @timestamp=[$2], instrumentationScope=[$3], severityText=[$7], resource=[$8], flags=[$23], attributes=[$24], droppedAttributesCount=[$162], severityNumber=[$163], time=[$164], body=[$165])\n LogicalFilter(condition=[query_string(MAP('query', 'severityNumber:>15':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]])\n",
44
"physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[PROJECT->[spanId, traceId, @timestamp, instrumentationScope, severityText, resource, flags, attributes, droppedAttributesCount, severityNumber, time, body], FILTER->query_string(MAP('query', 'severityNumber:>15':VARCHAR)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"severityNumber:>15\",\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\":true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"spanId\",\"traceId\",\"@timestamp\",\"instrumentationScope\",\"severityText\",\"resource\",\"flags\",\"attributes\",\"droppedAttributesCount\",\"severityNumber\",\"time\",\"body\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n"
55
}
66
}

0 commit comments

Comments
 (0)