apache · zhouyuan · Apr 17, 2023 · May 5, 2023
diff --git a/.github/workflows/velox_be.yml b/.github/workflows/velox_be.yml
diff --git a/...src/test/scala/org/apache/spark/sql/GlutenApproxCountDistinctForIntervalsQuerySuite.scala b/...src/test/scala/org/apache/spark/sql/GlutenApproxCountDistinctForIntervalsQuerySuite.scala
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+class GlutenApproxCountDistinctForIntervalsQuerySuite
+  extends ApproxCountDistinctForIntervalsQuerySuite with GlutenSQLTestsTrait {
+}
diff --git a/...t/spark33/src/test/scala/org/apache/spark/sql/GlutenApproximatePercentileQuerySuite.scala b/...t/spark33/src/test/scala/org/apache/spark/sql/GlutenApproximatePercentileQuerySuite.scala
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+class GlutenApproximatePercentileQuerySuite extends ApproximatePercentileQuerySuite
+  with GlutenSQLTestsTrait {
+
+  override def testFile(fileName: String): String = {
+    Thread.currentThread().getContextClassLoader.getResource(fileName)
+      .toString
+  }
+}
diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCTEHintSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCTEHintSuite.scala
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+class GlutenCTEHintSuite extends CTEHintSuite with GlutenSQLTestsTrait {
+}
diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCTEInlineSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCTEInlineSuite.scala
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+class GlutenCTEInlineSuiteAEOff extends CTEInlineSuiteAEOff
+  with GlutenSQLTestsTrait
+
+class GlutenCTEInlineSuiteAEOn extends CTEInlineSuiteAEOn
+  with GlutenSQLTestsTrait
diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import org.apache.spark.SparkConf
+
+class GlutenCachedTableSuite extends CachedTableSuite with GlutenSQLTestsTrait {
+  override def sparkConf: SparkConf = {
+    super.sparkConf.set("spark.sql.shuffle.partitions", "5")
+  }
+}
diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCharVarcharTestSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCharVarcharTestSuite.scala
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+class GlutenFileSourceCharVarcharTestSuite extends FileSourceCharVarcharTestSuite
+  with GlutenSQLTestsTrait {
+}
+
+class GlutenDSV2CharVarcharTestSuite extends DSV2CharVarcharTestSuite
+  with GlutenSQLTestsTrait {
+}
diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenColumnExpressionSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenColumnExpressionSuite.scala
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+class GlutenColumnExpressionSuite extends ColumnExpressionSuite with GlutenSQLTestsTrait {
+
+}
diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenComplexTypesSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenComplexTypesSuite.scala
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+class GlutenComplexTypesSuite extends ComplexTypesSuite with GlutenSQLTestsTrait {
+  import testImplicits._
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    spark.range(10).selectExpr(
+      "(id % 2 = 0) as bool",
+      "cast(id as BYTE) as i8",
+      "cast(id as SHORT) as i16",
+      "cast(id as FLOAT) as fp32",
+      "cast(id as DOUBLE) as fp64",
+      "cast(id as DECIMAL(4, 2)) as dec",
+      "cast(cast(id as BYTE) as BINARY) as vbin",
+      "binary(id) as vbin1",
+      "map_from_arrays(array(id),array(id+2)) as map",
+      "array(id, id+1, id+2) as list",
+      "struct(cast(id as LONG) as a, cast(id+1 as STRING) as b) as struct"
+    ).write.saveAsTable("tab_types")
+  }
+
+  override def afterAll(): Unit = {
+   try {
+      spark.sql("DROP TABLE IF EXISTS tab_types")
+    } finally {
+      super.afterAll()
+    }
+  }
+
+  test(GlutenTestConstants.GLUTEN_TEST +
+    "types bool/byte/short/float/double/decimal/binary/map/array/struct") {
+    val df = spark.table("tab_types").selectExpr(
+      "bool",
+      "i8",
+      "i16",
+      "fp32",
+      "fp64",
+      "dec",
+      "vbin",
+      "length(vbin)",
+      "vbin1",
+      "length(vbin1)",
+      "struct",
+      "struct.a",
+      "list",
+      "map"
+    ).sort("i8").limit(1)
+
+    checkAnswer(df, Seq(
+      Row(
+        true,
+        0.toByte,
+        0.toShort,
+        0.toFloat,
+        0.toDouble,
+        BigDecimal(0),
+        Array.fill[Byte](1)(0.toByte),
+        1.toInt,
+        Array.fill[Byte](8)(0.toByte),
+        8.toInt,
+        Row(0.toLong, "1"),
+        0.toLong,
+        Array(0, 1, 2),
+        Map(0 -> 2)
+      )))
+
+    checkNamedStruct(df.queryExecution.optimizedPlan, expectedCount = 0)
+  }
+}
diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenConfigBehaviorSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenConfigBehaviorSuite.scala
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+class GlutenConfigBehaviorSuite extends ConfigBehaviorSuite with GlutenSQLTestsTrait {
+}
diff --git a/...en-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCountMinSketchAggQuerySuite.scala b/...en-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCountMinSketchAggQuerySuite.scala
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.util.sketch.CountMinSketch
+
+/**
+ * End-to-end test suite for count_min_sketch.
+ */
+class GlutenCountMinSketchAggQuerySuite extends CountMinSketchAggQuerySuite
+  with GlutenSQLTestsTrait {
+}
diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCsvFunctionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCsvFunctionsSuite.scala
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+class GlutenCsvFunctionsSuite extends CsvFunctionsSuite with GlutenSQLTestsTrait {
+}