parser grammar OpenSearchPPLParser;

options { tokenVocab = OpenSearchPPLLexer; }

: pplStatement? EOF

;

: explainStatement

| queryStatement

;

: PIPE? commands (PIPE commands?)*

;

: (PIPE)? pplCommands (PIPE commands?)*

;

: EXPLAIN (explainMode)? queryStatement

;

: SIMPLE

| STANDARD

| COST

| EXTENDED

;

: searchCommand (PIPE commands?)*

;

: describeCommand

| showDataSourcesCommand

| searchCommand

| multisearchCommand

| graphLookupCommand

| unionCommand

;

: whereCommand

| fieldsCommand

| tableCommand

| joinCommand

| renameCommand

| statsCommand

| eventstatsCommand

| streamstatsCommand

| dedupCommand

| sortCommand

| evalCommand

| headCommand

| binCommand

| rareTopCommand

| grokCommand

| parseCommand

| spathCommand

| patternsCommand

| lookupCommand

| kmeansCommand

| adCommand

| mlCommand

| fillnullCommand

| convertCommand

| trendlineCommand

| appendcolCommand

| addtotalsCommand

| addcoltotalsCommand

| appendCommand

| expandCommand

| mvexpandCommand

| flattenCommand

| reverseCommand

| regexCommand

| chartCommand

| timechartCommand

| transposeCommand

| rexCommand

| appendPipeCommand

| replaceCommand

| mvcombineCommand

| fieldformatCommand

| nomvCommand

| graphLookupCommand

| unionCommand

;

: SEARCH

| DESCRIBE

| SHOW

| WHERE

| FIELDS

| TABLE

| JOIN

| RENAME

| STATS

| EVENTSTATS

| STREAMSTATS

| DEDUP

| SORT

| EVAL

| FIELDFORMAT

| HEAD

| BIN

| TOP

| RARE

| GROK

| PARSE

| PATTERNS

| LOOKUP

| KMEANS

| AD

| ML

| FILLNULL

| CONVERT

| EXPAND

| MVEXPAND

| FLATTEN

| TRENDLINE

| TIMECHART

| EXPLAIN

| REVERSE

| REGEX

| ADDTOTALS

| ADDCOLTOTALS

| APPEND

| MULTISEARCH

| UNION

| REX

| APPENDPIPE

| REPLACE

| MVCOMBINE

| NOMV

| TRANSPOSE

| GRAPHLOOKUP

;

: (SEARCH)? (searchExpression)* fromClause (searchExpression)* # searchFrom

;

: timeModifier # timeModifierExpression

| LT_PRTHS searchExpression RT_PRTHS # groupedExpression

| NOT searchExpression # notExpression

| searchExpression OR searchExpression # orExpression

| searchExpression AND searchExpression # andExpression

| searchTerm # termExpression

;

: searchFieldComparison # searchComparisonTerm

| searchFieldInList # searchInListTerm

| searchLiteral # searchLiteralTerm

;

: numericLiteral

| booleanLiteral

| ID

| NUMERIC_ID

| stringLiteral

| searchableKeyWord

;

: fieldExpression searchComparisonOperator searchLiteral # searchFieldCompare

;

: fieldExpression IN LT_PRTHS searchLiteralList RT_PRTHS # searchFieldInValues

;

: searchLiteral (COMMA searchLiteral)* # searchLiterals

;

: EQUAL # equals

| NOT_EQUAL # notEquals

| LESS # lessThan

| NOT_GREATER # lessOrEqual

| GREATER # greaterThan

| NOT_LESS # greaterOrEqual

;

: DESCRIBE tableSourceClause

;

: SHOW DATASOURCES

;

: WHERE logicalExpression

;

: FIELDS fieldsCommandBody

;

: TABLE fieldsCommandBody

;

: (PLUS | MINUS)? wcFieldList

;

: selectFieldExpression (COMMA? selectFieldExpression)*

;

: RENAME renameClause (COMMA? renameClause)*

;

: REPLACE replacePair (COMMA replacePair)* IN fieldList

;

: pattern=stringLiteral WITH replacement=stringLiteral

;

: STATS statsArgs statsAggTerm (COMMA statsAggTerm)* (statsByClause)? (dedupSplitArg)?

;

: APPENDPIPE LT_SQR_PRTHS subPipeline RT_SQR_PRTHS

;

: (partitionsArg | allnumArg | delimArg | bucketNullableArg)*

;

: PARTITIONS EQUAL partitions = integerLiteral

;

: ALLNUM EQUAL allnum = booleanLiteral

;

: DELIM EQUAL delim = stringLiteral

;

: BUCKET_NULLABLE EQUAL bucket_nullable = booleanLiteral

;

: DEDUP_SPLITVALUES EQUAL dedupsplit = booleanLiteral

;

: EVENTSTATS (bucketNullableArg)? eventstatsAggTerm (COMMA eventstatsAggTerm)* (statsByClause)?

;

: STREAMSTATS streamstatsArgs streamstatsAggTerm (COMMA streamstatsAggTerm)* (statsByClause)?

;

: (currentArg | windowArg | globalArg | resetBeforeArg | resetAfterArg | bucketNullableArg)*

;

: CURRENT EQUAL current = booleanLiteral

;

: WINDOW EQUAL window = integerLiteral

;

: GLOBAL EQUAL global = booleanLiteral

;

: RESET_BEFORE EQUAL logicalExpression

;

: RESET_AFTER EQUAL logicalExpression

;

: DEDUP (number = integerLiteral)? fieldList (KEEPEMPTY EQUAL keepempty = booleanLiteral)? (CONSECUTIVE EQUAL consecutive = booleanLiteral)?

;

: SORT (count = integerLiteral)? sortbyClause

;

: REVERSE

;

: CHART chartOptions* statsAggTerm (OVER rowSplit)? (BY columnSplit)? chartOptions*

| CHART chartOptions* statsAggTerm BY rowSplit (COMMA)? columnSplit chartOptions*

;

: LIMIT EQUAL integerLiteral

| LIMIT EQUAL (TOP_K | BOTTOM_K)

| USEOTHER EQUAL booleanLiteral

| OTHERSTR EQUAL stringLiteral

| USENULL EQUAL booleanLiteral

| NULLSTR EQUAL stringLiteral

;

: fieldExpression binOption*

;

: fieldExpression binOption*

;

: TIMECHART timechartParameter* statsAggTerm (BY fieldExpression)? timechartParameter*

;

: TRANSPOSE transposeParameter*

;

: (number = integerLiteral)

| (COLUMN_NAME EQUAL stringLiteral)

;

: LIMIT EQUAL integerLiteral

| SPAN EQUAL spanLiteral

| USEOTHER EQUAL (booleanLiteral | ident)

| TIMEFIELD EQUAL (ident | stringLiteral)

;

: SPANLENGTH

| DECIMAL_SPANLENGTH

| DOUBLE_LITERAL // 1.5d can also represent decimal span length

| INTEGER_LITERAL

| DECIMAL_LITERAL

;

: EVAL evalClause (COMMA evalClause)*

;

: FIELDFORMAT fieldFormatEvalClause (COMMA fieldFormatEvalClause)*

;

: HEAD (number = integerLiteral)? (FROM from = integerLiteral)?

;

: BIN fieldExpression binOption* (AS alias = qualifiedName)?

;

: SPAN EQUAL span = binSpanValue

| BINS EQUAL bins = integerLiteral

| MINSPAN EQUAL minspan = spanLiteral

| ALIGNTIME EQUAL aligntime = aligntimeValue

| START EQUAL start = numericLiteral

| END EQUAL end = numericLiteral

;

: EARLIEST

| LATEST

| literalValue

;

: spanLiteral # numericSpanValue

| logSpanValue # logBasedSpanValue

;

: LOG_WITH_BASE # logWithBaseSpan

;

: (TOP | RARE) (number = integerLiteral)? rareTopOption* fieldList (byClause)?

;

: COUNTFIELD EQUAL countField = stringLiteral

| SHOWCOUNT EQUAL showCount = booleanLiteral

| USENULL EQUAL useNull = booleanLiteral

;

: GROK (source_field = expression) (pattern = stringLiteral)

;

: PARSE (source_field = expression) (pattern = stringLiteral)

;

: SPATH spathParameter*

;

: (INPUT EQUAL input = expression)

| (OUTPUT EQUAL output = expression)

| ((PATH EQUAL)? path = indexablePath)

;

: pathElement (DOT pathElement)*

| stringLiteral

;

: ident pathArrayAccess?

;

: LT_CURLY (INTEGER_LITERAL)? RT_CURLY

;

: REGEX regexExpr

;

: field=qualifiedName operator=(EQUAL | NOT_EQUAL) pattern=stringLiteral

;

: REX rexExpr

;

: FIELD EQUAL field=qualifiedName (rexOption)* pattern=stringLiteral (rexOption)*

;

: MAX_MATCH EQUAL maxMatch=integerLiteral

| MODE EQUAL (EXTRACT | SED)

| OFFSET_FIELD EQUAL offsetField=qualifiedName

;

: PUNCT

| REGEX

;

: PATTERNS (source_field = expression) (statsByClause)? (patternsCommandOption)* (patternsParameter)*

;

: (METHOD EQUAL method = patternMethod)

| (MODE EQUAL pattern_mode = patternMode)

| (MAX_SAMPLE_COUNT EQUAL max_sample_count = integerLiteral)

| (BUFFER_LIMIT EQUAL buffer_limit = integerLiteral)

| (SHOW_NUMBERED_TOKEN EQUAL show_numbered_token = booleanLiteral)

;

: (PATTERN EQUAL pattern = stringLiteral)

| (NEW_FIELD EQUAL new_field = stringLiteral)

| (VARIABLE_COUNT_THRESHOLD EQUAL variable_count_threshold = integerLiteral)

| (FREQUENCY_THRESHOLD_PERCENTAGE EQUAL frequency_threshold_percentage = decimalLiteral)

;

: SIMPLE_PATTERN

| BRAIN

;

: LABEL

| AGGREGATION

;

: LOOKUP tableSource lookupMappingList ((APPEND | REPLACE | OUTPUT) outputCandidateList)?

;

: lookupPair (COMMA lookupPair)*

;

: lookupPair (COMMA lookupPair)*

;

// The lookup pair will generate a K-V pair.

// The format is Key -> Alias(outputFieldName, inputField), Value -> outputField. For example:

// 1. When lookupPair is "name AS cName", the key will be Alias(cName, Field(name)), the value will be Field(cName)

// 2. When lookupPair is "dept", the key is Alias(dept, Field(dept)), value is Field(dept)

: inputField = fieldExpression (AS outputField = fieldExpression)?

;

: FILLNULL fillNullWith # fillNullWithClause

| FILLNULL fillNullUsing # fillNullUsingClause

| FILLNULL VALUE EQUAL replacement = valueExpression fieldList # fillNullValueWithFields

| FILLNULL VALUE EQUAL replacement = valueExpression # fillNullValueAllFields

;

: WITH replacement = valueExpression (IN fieldList)?

;

: USING replacementPair (COMMA replacementPair)*

;

: fieldExpression EQUAL replacement = valueExpression

;

: CONVERT (TIMEFORMAT EQUAL timeFormat=stringLiteral)? convertFunction (COMMA? convertFunction)*

;

: functionName = ident LT_PRTHS fieldExpression RT_PRTHS (AS alias = fieldExpression)?

;

: TRENDLINE (SORT sortField)? trendlineClause (trendlineClause)*

;

: trendlineType LT_PRTHS numberOfDataPoints = integerLiteral COMMA field = fieldExpression RT_PRTHS (AS alias = qualifiedName)?

;

: SMA

| WMA

;

: EXPAND fieldExpression (AS alias = qualifiedName)?

;

: MVCOMBINE fieldExpression (DELIM EQUAL stringLiteral)?

;

: NOMV fieldExpression

;

: MVEXPAND fieldExpression (LIMIT EQUAL INTEGER_LITERAL)?

;

: FLATTEN fieldExpression (AS aliases = identifierSeq)?

;

: APPENDCOL (OVERRIDE EQUAL override = booleanLiteral)? LT_SQR_PRTHS commands (PIPE commands)* RT_SQR_PRTHS

;

: APPEND LT_SQR_PRTHS searchCommand? (PIPE commands)* RT_SQR_PRTHS

;

: MULTISEARCH (LT_SQR_PRTHS subSearch RT_SQR_PRTHS)+

;

: UNION subsearchOptions? unionDataset (COMMA? unionDataset)*

;

: (MAXOUT EQUAL maxout=integerLiteral)?

;

: LT_SQR_PRTHS subSearch RT_SQR_PRTHS

| tableSource

;

: KMEANS (kmeansParameter)*

;

: (CENTROIDS EQUAL centroids = integerLiteral)

| (ITERATIONS EQUAL iterations = integerLiteral)

| (DISTANCE_TYPE EQUAL distance_type = stringLiteral)

;

: AD (adParameter)*

;

: (NUMBER_OF_TREES EQUAL number_of_trees = integerLiteral)

| (SHINGLE_SIZE EQUAL shingle_size = integerLiteral)

| (SAMPLE_SIZE EQUAL sample_size = integerLiteral)

| (OUTPUT_AFTER EQUAL output_after = integerLiteral)

| (TIME_DECAY EQUAL time_decay = decimalLiteral)

| (ANOMALY_RATE EQUAL anomaly_rate = decimalLiteral)

| (CATEGORY_FIELD EQUAL category_field = stringLiteral)

| (TIME_FIELD EQUAL time_field = stringLiteral)

| (DATE_FORMAT EQUAL date_format = stringLiteral)

| (TIME_ZONE EQUAL time_zone = stringLiteral)

| (TRAINING_DATA_SIZE EQUAL training_data_size = integerLiteral)

| (ANOMALY_SCORE_THRESHOLD EQUAL anomaly_score_threshold = decimalLiteral)

;

: ML (mlArg)*

;

: (argName = ident EQUAL argValue = literalValue)

;

: ADDTOTALS (fieldList)? addtotalsOption*

| ADDTOTALS addtotalsOption* (fieldList)?

;

: (LABEL EQUAL stringLiteral)

| (LABELFIELD EQUAL stringLiteral)

| (FIELDNAME EQUAL stringLiteral)

| (ROW EQUAL booleanLiteral)

| (COL EQUAL booleanLiteral)

;

: ADDCOLTOTALS (fieldList)? addcoltotalsOption*

| ADDCOLTOTALS addcoltotalsOption* (fieldList)?

;

: (LABEL EQUAL stringLiteral)

| (LABELFIELD EQUAL stringLiteral)

;

: GRAPHLOOKUP lookupTable = tableSourceClause startClause edgeClause graphLookupArgs* AS outputField = fieldExpression

;

: START EQUAL valueList

| START EQUAL startField = fieldExpression

| START EQUAL startValue = literalValue

;

: edgeClauseToken = EDGE_CLAUSE

;

: (MAX_DEPTH EQUAL integerLiteral)

| (DEPTH_FIELD EQUAL fieldExpression)

| (SUPPORT_ARRAY EQUAL booleanLiteral)

| (BATCH_MODE EQUAL booleanLiteral)

| (USE_PIT EQUAL booleanLiteral)

| (FILTER EQUAL LT_PRTHS logicalExpression RT_PRTHS)

;

: SOURCE EQUAL tableOrSubqueryClause

| INDEX EQUAL tableOrSubqueryClause

| SOURCE EQUAL tableFunction

| INDEX EQUAL tableFunction

| SOURCE EQUAL dynamicSourceClause

| INDEX EQUAL dynamicSourceClause

;

: LT_SQR_PRTHS subSearch RT_SQR_PRTHS (AS alias = qualifiedName)?

| tableSourceClause

;

: tableSource (COMMA tableSource)* (AS alias = qualifiedName)?

;

: LT_SQR_PRTHS (sourceReference | sourceFilterArg) (COMMA (sourceReference | sourceFilterArg))* RT_SQR_PRTHS

;

: (CLUSTER)? wcQualifiedName

;

: ident EQUAL literalValue

| ident IN LT_PRTHS valueList RT_PRTHS

;

: JOIN (joinOption)* (fieldList)? right = tableOrSubqueryClause

| sqlLikeJoinType? JOIN (joinOption)* sideAlias joinHintList? joinCriteria right = tableOrSubqueryClause

;

: INNER

| CROSS

| (LEFT OUTER? | OUTER)

| RIGHT OUTER?

| FULL OUTER?

| LEFT? SEMI

| LEFT? ANTI

;

: INNER

| CROSS

| OUTER

| LEFT

| RIGHT

| FULL

| SEMI

| ANTI

;

: (LEFT EQUAL leftAlias = qualifiedName)? COMMA? (RIGHT EQUAL rightAlias = qualifiedName)?

;

: (ON | WHERE) logicalExpression

;

: hintPair (COMMA? hintPair)*

;

: leftHintKey = LEFT_HINT DOT ID EQUAL leftHintValue = ident #leftHint

| rightHintKey = RIGHT_HINT DOT ID EQUAL rightHintValue = ident #rightHint

;

: OVERWRITE EQUAL booleanLiteral # overwriteOption

| TYPE EQUAL joinType # typeOption

| MAX EQUAL integerLiteral # maxOption

;

: orignalField = renameFieldExpression AS renamedField = renameFieldExpression

;

: BY fieldList

;

: BY fieldList

| BY bySpanClause

| BY bySpanClause COMMA fieldList

| BY fieldList COMMA bySpanClause

;

: spanClause (AS alias = qualifiedName)?

;

: SPAN LT_PRTHS (fieldExpression COMMA)? value = spanLiteral RT_PRTHS

;

: sortField (COMMA sortField)*

;

: fieldExpression EQUAL logicalExpression

;

: fieldExpression EQUAL ffLogicalExpression

;

: windowFunction (AS alias = wcFieldExpression)?

;

: windowFunction (AS alias = wcFieldExpression)?

;

: windowFunctionName LT_PRTHS functionArgs RT_PRTHS

;

: statsFunctionName

| scalarWindowFunctionName

;

: ROW_NUMBER

| RANK

| DENSE_RANK

| PERCENT_RANK

| CUME_DIST

| FIRST

| LAST

| NTH

| NTILE

| DISTINCT_COUNT

| DC

;

: statsFunction (AS alias = wcFieldExpression)?

;

: (COUNT | C) LT_PRTHS evalExpression RT_PRTHS # countEvalFunctionCall

| (COUNT | C) (LT_PRTHS RT_PRTHS)? # countAllFunctionCall

| PERCENTILE_SHORTCUT LT_PRTHS valueExpression RT_PRTHS # percentileShortcutFunctionCall

| (DISTINCT_COUNT | DC | DISTINCT_COUNT_APPROX) LT_PRTHS valueExpression RT_PRTHS # distinctCountFunctionCall

| takeAggFunction # takeAggFunctionCall

| valuesAggFunction # valuesAggFunctionCall

| percentileApproxFunction # percentileApproxFunctionCall

| perFunction # perFunctionCall

| statsFunctionName LT_PRTHS functionArgs RT_PRTHS # statsFunctionCall

;

: AVG

| COUNT

| SUM

| MIN

| MAX

| VAR_SAMP

| VAR_POP

| STDDEV_SAMP

| STDDEV_POP

| PERCENTILE

| PERCENTILE_APPROX

| MEDIAN

| LIST

| FIRST

| EARLIEST

| LATEST

| LAST

;

: TAKE LT_PRTHS fieldExpression (COMMA size = integerLiteral)? RT_PRTHS

;

: VALUES LT_PRTHS valueExpression RT_PRTHS

;

: (PERCENTILE | PERCENTILE_APPROX) LT_PRTHS aggField = valueExpression

COMMA percent = numericLiteral (COMMA compression = numericLiteral)? RT_PRTHS

;

: funcName=(PER_SECOND | PER_MINUTE | PER_HOUR | PER_DAY) LT_PRTHS functionArg RT_PRTHS

;

: integerLiteral

| decimalLiteral

| doubleLiteral

| floatLiteral

;

: stringLiteral DOT logicalExpression # stringDotlogicalExpression

| stringLiteral DOT logicalExpression DOT stringLiteral # stringDotlogicalExpressionDotString

| logicalExpression DOT stringLiteral # logicalExpressionDotString

| logicalExpression # ffStandardLogicalExpression

;

: NOT logicalExpression # logicalNot

| left = logicalExpression AND right = logicalExpression # logicalAnd

| left = logicalExpression XOR right = logicalExpression # logicalXor

| left = logicalExpression OR right = logicalExpression # logicalOr

| expression # logicalExpr

;

: valueExpression # valueExpr

| relevanceExpression # relevanceExpr

| left = expression comparisonOperator right = expression # compareExpr

| expression NOT? IN LT_PRTHS valueList RT_PRTHS # inExpr

| expression NOT? BETWEEN expression AND expression # between

| expression IS nullNotnull # isNullPredicate

;

: NOT? NULL

;

: left = valueExpression binaryOperator = (STAR | DIVIDE | MODULE) right = valueExpression # binaryArithmetic

| left = valueExpression binaryOperator = (PLUS | MINUS) right = valueExpression # binaryArithmetic

| literalValue # literalValueExpr

| functionCall # functionCallExpr

| lambda # lambdaExpr

| LT_SQR_PRTHS subSearch RT_SQR_PRTHS # scalarSubqueryExpr

| valueExpression NOT? IN LT_SQR_PRTHS subSearch RT_SQR_PRTHS # inSubqueryExpr

| LT_PRTHS valueExpression (COMMA valueExpression)* RT_PRTHS NOT? IN LT_SQR_PRTHS subSearch RT_SQR_PRTHS # inSubqueryExpr

| EXISTS LT_SQR_PRTHS subSearch RT_SQR_PRTHS # existsSubqueryExpr

| fieldExpression # fieldExpr

| LT_PRTHS logicalExpression RT_PRTHS # nestedValueExpr

;

: EVAL LT_PRTHS logicalExpression RT_PRTHS

;

: mvmapFunctionCall

| evalFunctionCall

| dataTypeFunctionCall

| positionFunctionCall

| caseFunctionCall

| timestampFunctionCall

| extractFunctionCall

| getFormatFunctionCall

;

: MVMAP LT_PRTHS functionArg COMMA functionArg RT_PRTHS

;

: positionFunctionName LT_PRTHS functionArg IN functionArg RT_PRTHS

;

: CASE LT_PRTHS logicalExpression COMMA valueExpression (COMMA logicalExpression COMMA valueExpression)* (ELSE valueExpression)? RT_PRTHS

;

: singleFieldRelevanceFunction

| multiFieldRelevanceFunction

;

: singleFieldRelevanceFunctionName LT_PRTHS field = relevanceField COMMA query = relevanceQuery (COMMA relevanceArg)* RT_PRTHS

;

: multiFieldRelevanceFunctionName LT_PRTHS (LT_SQR_PRTHS field = relevanceFieldAndWeight (COMMA field = relevanceFieldAndWeight)* RT_SQR_PRTHS COMMA)? query = relevanceQuery (COMMA relevanceArg)* RT_PRTHS

;

: (EARLIEST | LATEST) EQUAL timeModifierValue

;

: NOW

| NOW LT_PRTHS RT_PRTHS