apache · LuciferYang · Jun 12, 2023 · Jun 12, 2023 · Jun 12, 2023 · Jun 12, 2023
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
@@ -175,12 +175,27 @@ jobs:
             hadoop: ${{ inputs.hadoop }}
             hive: hive2.3
             included-tags: org.apache.spark.tags.ExtendedSQLTest
+            comment: "- extended tests"
+          - modules: sql
+            java: ${{ inputs.java }}
+            hadoop: ${{ inputs.hadoop }}
+            hive: hive2.3
+            # Using a tag that will not appear in sql module for placeholder, branch-3.3 and branch-3.4 will not run any UTs.
+            included-tags: >-
+              ${{
+                ((inputs.branch == 'branch-3.3' || inputs.branch == 'branch-3.4') && 'org.apache.spark.tags.SlowHiveTest')
+                || 'org.apache.spark.tags.SlowSQLTest'
+              }}
             comment: "- slow tests"
           - modules: sql
             java: ${{ inputs.java }}
             hadoop: ${{ inputs.hadoop }}
             hive: hive2.3
-            excluded-tags: org.apache.spark.tags.ExtendedSQLTest
+            excluded-tags: >-
+              ${{
+                ((inputs.branch == 'branch-3.3' || inputs.branch == 'branch-3.4') && 'org.apache.spark.tags.ExtendedSQLTest')
+                || 'org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowSQLTest'
+              }}
             comment: "- other tests"
     env:
       MODULES_TO_TEST: ${{ matrix.modules }}

diff --git a/common/tags/src/test/java/org/apache/spark/tags/SlowSQLTest.java b/common/tags/src/test/java/org/apache/spark/tags/SlowSQLTest.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.tags;
+
+import org.scalatest.TagAnnotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@TagAnnotation
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.METHOD, ElementType.TYPE})
+public @interface SlowSQLTest { }
diff --git a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/functions.scala b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/functions.scala
@@ -1251,6 +1251,30 @@ object functions {
    */
   def bool_or(e: Column): Column = Column.fn("bool_or", e)
 
+  /**
+   * Aggregate function: returns the bitwise AND of all non-null input values, or null if none.
+   *
+   * @group agg_funcs
+   * @since 3.5.0
+   */
+  def bit_and(e: Column): Column = Column.fn("bit_and", e)
+
+  /**
+   * Aggregate function: returns the bitwise OR of all non-null input values, or null if none.
+   *
+   * @group agg_funcs
+   * @since 3.5.0
+   */
+  def bit_or(e: Column): Column = Column.fn("bit_or", e)
+
+  /**
+   * Aggregate function: returns the bitwise XOR of all non-null input values, or null if none.
+   *
+   * @group agg_funcs
+   * @since 3.5.0
+   */
+  def bit_xor(e: Column): Column = Column.fn("bit_xor", e)
+
   //////////////////////////////////////////////////////////////////////////////////////////////
   // Window functions
   //////////////////////////////////////////////////////////////////////////////////////////////
@@ -1851,6 +1875,33 @@ object functions {
    */
   def bitwise_not(e: Column): Column = Column.fn("~", e)
 
+  /**
+   * Returns the number of bits that are set in the argument expr as an unsigned 64-bit integer,
+   * or NULL if the argument is NULL.
+   *
+   * @group bitwise_funcs
+   * @since 3.5.0
+   */
+  def bit_count(e: Column): Column = Column.fn("bit_count", e)
+
+  /**
+   * Returns the value of the bit (0 or 1) at the specified position. The positions are numbered
+   * from right to left, starting at zero. The position argument cannot be negative.
+   *
+   * @group bitwise_funcs
+   * @since 3.5.0
+   */
+  def bit_get(e: Column, pos: Column): Column = Column.fn("bit_get", e, pos)
+
+  /**
+   * Returns the value of the bit (0 or 1) at the specified position. The positions are numbered
+   * from right to left, starting at zero. The position argument cannot be negative.
+   *
+   * @group bitwise_funcs
+   * @since 3.5.0
+   */
+  def getbit(e: Column, pos: Column): Column = Column.fn("getbit", e, pos)
+
   /**
    * Parses the expression string into the column that it represents, similar to
    * [[Dataset#selectExpr]].

diff --git a/...ctor/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala b/...ctor/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
@@ -1009,6 +1009,18 @@ class PlanGenerationTestSuite
     fn.histogram_numeric(fn.col("a"), lit(10))
   }
 
+  functionTest("bit_and") {
+    fn.bit_and(fn.col("a"))
+  }
+
+  functionTest("bit_or") {
+    fn.bit_or(fn.col("a"))
+  }
+
+  functionTest("bit_xor") {
+    fn.bit_xor(fn.col("a"))
+  }
+
   functionTest("mode") {
     fn.mode(fn.col("a"))
   }
@@ -1209,6 +1221,18 @@ class PlanGenerationTestSuite
     fn.bitwise_not(fn.col("a"))
   }
 
+  functionTest("bit_count") {
+    fn.bit_count(fn.col("a"))
+  }
+
+  functionTest("bit_get") {
+    fn.bit_get(fn.col("a"), lit(0))
+  }
+
+  functionTest("getbit") {
+    fn.getbit(fn.col("a"), lit(0))
+  }
+
   functionTest("expr") {
     fn.expr("a + 1")
   }

diff --git a/...or/connect/common/src/test/resources/query-tests/explain-results/function_bit_and.explain b/...or/connect/common/src/test/resources/query-tests/explain-results/function_bit_and.explain
@@ -0,0 +1,2 @@
+Aggregate [bit_and(a#0) AS bit_and(a)#0]
++- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git a/.../connect/common/src/test/resources/query-tests/explain-results/function_bit_count.explain b/.../connect/common/src/test/resources/query-tests/explain-results/function_bit_count.explain
@@ -0,0 +1,2 @@
+Project [bit_count(a#0) AS bit_count(a)#0]
++- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git a/...or/connect/common/src/test/resources/query-tests/explain-results/function_bit_get.explain b/...or/connect/common/src/test/resources/query-tests/explain-results/function_bit_get.explain
@@ -0,0 +1,2 @@
+Project [bit_get(a#0, 0) AS bit_get(a, 0)#0]
++- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git a/...tor/connect/common/src/test/resources/query-tests/explain-results/function_bit_or.explain b/...tor/connect/common/src/test/resources/query-tests/explain-results/function_bit_or.explain
@@ -0,0 +1,2 @@
+Aggregate [bit_or(a#0) AS bit_or(a)#0]
++- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git a/...or/connect/common/src/test/resources/query-tests/explain-results/function_bit_xor.explain b/...or/connect/common/src/test/resources/query-tests/explain-results/function_bit_xor.explain
@@ -0,0 +1,2 @@
+Aggregate [bit_xor(a#0) AS bit_xor(a)#0]
++- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git a/...tor/connect/common/src/test/resources/query-tests/explain-results/function_getbit.explain b/...tor/connect/common/src/test/resources/query-tests/explain-results/function_getbit.explain
@@ -0,0 +1,2 @@
+Project [getbit(a#0, 0) AS getbit(a, 0)#0]
++- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_bit_and.json b/connector/connect/common/src/test/resources/query-tests/queries/function_bit_and.json
@@ -0,0 +1,25 @@
+{
+  "common": {
+    "planId": "1"
+  },
+  "project": {
+    "input": {
+      "common": {
+        "planId": "0"
+      },
+      "localRelation": {
+        "schema": "struct\u003cid:bigint,a:int,b:double,d:struct\u003cid:bigint,a:int,b:double\u003e,e:array\u003cint\u003e,f:map\u003cstring,struct\u003cid:bigint,a:int,b:double\u003e\u003e,g:string\u003e"
+      }
+    },
+    "expressions": [{
+      "unresolvedFunction": {
+        "functionName": "bit_and",
+        "arguments": [{
+          "unresolvedAttribute": {
+            "unparsedIdentifier": "a"
+          }
+        }]
+      }
+    }]
+  }
+}
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_bit_and.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_bit_and.proto.bin
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_bit_count.json b/connector/connect/common/src/test/resources/query-tests/queries/function_bit_count.json
@@ -0,0 +1,25 @@
+{
+  "common": {
+    "planId": "1"
+  },
+  "project": {
+    "input": {
+      "common": {
+        "planId": "0"
+      },
+      "localRelation": {
+        "schema": "struct\u003cid:bigint,a:int,b:double,d:struct\u003cid:bigint,a:int,b:double\u003e,e:array\u003cint\u003e,f:map\u003cstring,struct\u003cid:bigint,a:int,b:double\u003e\u003e,g:string\u003e"
+      }
+    },
+    "expressions": [{
+      "unresolvedFunction": {
+        "functionName": "bit_count",
+        "arguments": [{
+          "unresolvedAttribute": {
+            "unparsedIdentifier": "a"
+          }
+        }]
+      }
+    }]
+  }
+}
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_bit_count.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_bit_count.proto.bin
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_bit_get.json b/connector/connect/common/src/test/resources/query-tests/queries/function_bit_get.json
@@ -0,0 +1,29 @@
+{
+  "common": {
+    "planId": "1"
+  },
+  "project": {
+    "input": {
+      "common": {
+        "planId": "0"
+      },
+      "localRelation": {
+        "schema": "struct\u003cid:bigint,a:int,b:double,d:struct\u003cid:bigint,a:int,b:double\u003e,e:array\u003cint\u003e,f:map\u003cstring,struct\u003cid:bigint,a:int,b:double\u003e\u003e,g:string\u003e"
+      }
+    },
+    "expressions": [{
+      "unresolvedFunction": {
+        "functionName": "bit_get",
+        "arguments": [{
+          "unresolvedAttribute": {
+            "unparsedIdentifier": "a"
+          }
+        }, {
+          "literal": {
+            "integer": 0
+          }
+        }]
+      }
+    }]
+  }
+}
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_bit_get.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_bit_get.proto.bin
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_bit_or.json b/connector/connect/common/src/test/resources/query-tests/queries/function_bit_or.json
@@ -0,0 +1,25 @@
+{
+  "common": {
+    "planId": "1"
+  },
+  "project": {
+    "input": {
+      "common": {
+        "planId": "0"
+      },
+      "localRelation": {
+        "schema": "struct\u003cid:bigint,a:int,b:double,d:struct\u003cid:bigint,a:int,b:double\u003e,e:array\u003cint\u003e,f:map\u003cstring,struct\u003cid:bigint,a:int,b:double\u003e\u003e,g:string\u003e"
+      }
+    },
+    "expressions": [{
+      "unresolvedFunction": {
+        "functionName": "bit_or",
+        "arguments": [{
+          "unresolvedAttribute": {
+            "unparsedIdentifier": "a"
+          }
+        }]
+      }
+    }]
+  }
+}
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_bit_or.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_bit_or.proto.bin
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_bit_xor.json b/connector/connect/common/src/test/resources/query-tests/queries/function_bit_xor.json
@@ -0,0 +1,25 @@
+{
+  "common": {
+    "planId": "1"
+  },
+  "project": {
+    "input": {
+      "common": {
+        "planId": "0"
+      },
+      "localRelation": {
+        "schema": "struct\u003cid:bigint,a:int,b:double,d:struct\u003cid:bigint,a:int,b:double\u003e,e:array\u003cint\u003e,f:map\u003cstring,struct\u003cid:bigint,a:int,b:double\u003e\u003e,g:string\u003e"
+      }
+    },
+    "expressions": [{
+      "unresolvedFunction": {
+        "functionName": "bit_xor",
+        "arguments": [{
+          "unresolvedAttribute": {
+            "unparsedIdentifier": "a"
+          }
+        }]
+      }
+    }]
+  }
+}
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_bit_xor.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_bit_xor.proto.bin
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_getbit.json b/connector/connect/common/src/test/resources/query-tests/queries/function_getbit.json
@@ -0,0 +1,29 @@
+{
+  "common": {
+    "planId": "1"
+  },
+  "project": {
+    "input": {
+      "common": {
+        "planId": "0"
+      },
+      "localRelation": {
+        "schema": "struct\u003cid:bigint,a:int,b:double,d:struct\u003cid:bigint,a:int,b:double\u003e,e:array\u003cint\u003e,f:map\u003cstring,struct\u003cid:bigint,a:int,b:double\u003e\u003e,g:string\u003e"
+      }
+    },
+    "expressions": [{
+      "unresolvedFunction": {
+        "functionName": "getbit",
+        "arguments": [{
+          "unresolvedAttribute": {
+            "unparsedIdentifier": "a"
+          }
+        }, {
+          "literal": {
+            "integer": 0
+          }
+        }]
+      }
+    }]
+  }
+}
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_getbit.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_getbit.proto.bin
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Aggregate [bit_and(a#0) AS bit_and(a)#0]
		+- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Project [bit_count(a#0) AS bit_count(a)#0]
		+- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Project [bit_get(a#0, 0) AS bit_get(a, 0)#0]
		+- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Aggregate [bit_or(a#0) AS bit_or(a)#0]
		+- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Aggregate [bit_xor(a#0) AS bit_xor(a)#0]
		+- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Project [getbit(a#0, 0) AS getbit(a, 0)#0]
		+- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]