Skip to content

Commit 5b955d1

Browse files
pdabre12pdabre12
committed
Add arrow federation connector
1 parent 48eaf8a commit 5b955d1

47 files changed

Lines changed: 4039 additions & 17 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/arrow-flight-tests.yml‎

Lines changed: 75 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ jobs:
8787
# Run Maven tests for the target module, excluding native tests
8888
- name: Maven Tests
8989
if: needs.changes.outputs.codechange == 'true'
90-
run: ./mvnw test ${MAVEN_TEST} -pl ${{ matrix.modules }} -Dtest="*,!TestArrowFlightNativeQueries*"
90+
run: ./mvnw test ${MAVEN_TEST} -pl ${{ matrix.modules }} -Dtest="*,!TestArrowFlightNativeQueries*,!TestArrowFederationNativeQueries*"
9191

9292
prestocpp-linux-build-for-test:
9393
runs-on: ubuntu-22.04
@@ -320,3 +320,77 @@ jobs:
320320
-DDATA_DIR=${RUNNER_TEMP} \
321321
-Duser.timezone=America/Bahia_Banderas \
322322
-T1C
323+
324+
arrowfederation-native-e2e-tests:
325+
needs: prestocpp-linux-build-for-test
326+
runs-on: ubuntu-22.04
327+
container:
328+
image: prestodb/presto-native-dependency:0.297-202602271419-160459b8
329+
env:
330+
INSTALL_PREFIX: "${{ github.workspace }}/adapter-deps/install"
331+
strategy:
332+
fail-fast: false
333+
matrix:
334+
modules:
335+
- :presto-flight-shim # Only run tests for the `presto-flight-shim` module
336+
337+
timeout-minutes: 80
338+
concurrency:
339+
group: ${{ github.workflow }}-test-${{ matrix.modules }}-${{ github.event.pull_request.number }}
340+
cancel-in-progress: true
341+
342+
steps:
343+
- uses: actions/checkout@v4
344+
with:
345+
persist-credentials: false
346+
347+
- name: Fix git permissions
348+
# Usually actions/checkout does this but as we run in a container
349+
# it doesn't work
350+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
351+
352+
- name: Download artifacts
353+
uses: actions/download-artifact@v4
354+
with:
355+
name: arrow-flight-presto-native-build
356+
path: presto-native-execution/_build/release/presto_cpp/main
357+
358+
- name: Download Arrow Flight install artifacts
359+
uses: actions/download-artifact@v4
360+
with:
361+
name: arrow-flight-install
362+
path: ${{ env.INSTALL_PREFIX }}/lib64
363+
364+
# Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38
365+
- name: Restore execute permissions and library path
366+
run: |
367+
chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server
368+
# Ensure transitive dependency libboost-iostreams is found.
369+
ldconfig /usr/local/lib
370+
- name: Install OpenJDK8
371+
uses: actions/setup-java@v4
372+
with:
373+
distribution: temurin
374+
java-version: 17.0.15
375+
cache: maven
376+
- name: Download nodejs to maven cache
377+
run: .github/bin/download_nodejs
378+
379+
- name: Maven install
380+
env:
381+
# Use different Maven options to install.
382+
MAVEN_OPTS: -Xmx2G -XX:+ExitOnOutOfMemoryError
383+
run: |
384+
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
385+
./mvnw install ${MAVEN_FAST_INSTALL} -am -pl ${{ matrix.modules }}
386+
- name: Run arrowflight native e2e tests
387+
run: |
388+
export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server"
389+
mvn test \
390+
${MAVEN_TEST} \
391+
-pl ${{ matrix.modules }} \
392+
-Dtest="TestArrowFederationNativeQueries*" \
393+
-DPRESTO_SERVER=${PRESTO_SERVER_PATH} \
394+
-DDATA_DIR=${RUNNER_TEMP} \
395+
-Duser.timezone=America/Bahia_Banderas \
396+
-T1C

‎.github/workflows/test-other-modules.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,4 +72,4 @@ jobs:
7272
./mvnw install ${MAVEN_FAST_INSTALL} -pl '!:presto-docs,!:presto-server,!presto-test-coverage'
7373
- name: Maven Tests
7474
if: needs.changes.outputs.codechange == 'true'
75-
run: ./mvnw test -T 1 ${MAVEN_TEST} -P skip-native-sidecar-tests -pl '!presto-tests, !presto-native-tests, !presto-accumulo, !presto-cassandra, !presto-hive, !presto-kudu, !presto-docs, !presto-server, !presto-main, !presto-main-base, !presto-mongodb, !presto-spark-package, !presto-spark-launcher, !presto-spark-testing, !presto-spark-base, !presto-redis, !presto-elasticsearch, !presto-orc, !presto-thrift-connector, !presto-native-execution, !presto-test-coverage, !presto-iceberg, !presto-singlestore, !presto-base-arrow-flight, !presto-plan-checker-router-plugin, !presto-mysql, !presto-postgresql'
75+
run: ./mvnw test -T 1 ${MAVEN_TEST} -P skip-native-sidecar-tests -pl '!presto-tests, !presto-native-tests, !presto-accumulo, !presto-cassandra, !presto-hive, !presto-kudu, !presto-docs, !presto-server, !presto-main, !presto-main-base, !presto-mongodb, !presto-spark-package, !presto-spark-launcher, !presto-spark-testing, !presto-spark-base, !presto-redis, !presto-elasticsearch, !presto-orc, !presto-thrift-connector, !presto-native-execution, !presto-test-coverage, !presto-iceberg, !presto-singlestore, !presto-base-arrow-flight, !presto-plan-checker-router-plugin, !presto-mysql, !presto-postgresql, !presto-flight-shim'

‎presto-flight-shim/etc/catalog/mongodb.properties‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
connector.name=mongodb
2+
mongodb.seeds=localhost:27017
3+
mongodb.socket-keep-alive=true

‎presto-flight-shim/etc/catalog/mysql.properties‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
connector.name=mysql
2+
connection-url=jdbc:mysql://localhost:3306
3+
connection-user=root
4+
connection-password=mysql

‎presto-flight-shim/etc/catalog/postgresql.properties‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
connector.name=postgresql
2+
connection-url=jdbc:postgresql://localhost:5432/tpch
3+
connection-user=postgres
4+
connection-password=postgres

‎presto-flight-shim/pom.xml‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
<air.main.basedir>${project.parent.basedir}</air.main.basedir>
1717
<air.test.jvm.additional-arguments>-Xss10M</air.test.jvm.additional-arguments>
1818
<air.check.skip-modernizer>true</air.check.skip-modernizer>
19+
<mongo-java.version>3.12.14</mongo-java.version>
20+
<mongo-server.version>1.47.0</mongo-server.version>
1921
</properties>
2022

2123
<dependencyManagement>
@@ -199,6 +201,21 @@
199201
<artifactId>jmxutils</artifactId>
200202
</dependency>
201203

204+
<dependency>
205+
<groupId>io.airlift.resolver</groupId>
206+
<artifactId>resolver</artifactId>
207+
</dependency>
208+
209+
<dependency>
210+
<groupId>org.jetbrains</groupId>
211+
<artifactId>annotations</artifactId>
212+
</dependency>
213+
214+
<dependency>
215+
<groupId>com.facebook.airlift</groupId>
216+
<artifactId>units</artifactId>
217+
</dependency>
218+
202219
<!-- for testing -->
203220

204221
<dependency>
@@ -220,6 +237,12 @@
220237
<scope>test</scope>
221238
</dependency>
222239

240+
<dependency>
241+
<groupId>com.facebook.presto</groupId>
242+
<artifactId>presto-main</artifactId>
243+
<scope>test</scope>
244+
</dependency>
245+
223246
<dependency>
224247
<groupId>com.facebook.presto</groupId>
225248
<artifactId>presto-base-arrow-flight</artifactId>
@@ -256,6 +279,14 @@
256279
<scope>test</scope>
257280
</dependency>
258281

282+
<dependency>
283+
<groupId>com.facebook.presto</groupId>
284+
<artifactId>presto-base-arrow-flight</artifactId>
285+
<version>${project.version}</version>
286+
<scope>test</scope>
287+
<type>test-jar</type>
288+
</dependency>
289+
259290
<dependency>
260291
<groupId>com.h2database</groupId>
261292
<artifactId>h2</artifactId>
@@ -366,6 +397,89 @@
366397
<type>test-jar</type>
367398
<scope>test</scope>
368399
</dependency>
400+
401+
<dependency>
402+
<groupId>com.facebook.presto</groupId>
403+
<artifactId>presto-mongodb</artifactId>
404+
<version>${project.version}</version>
405+
<scope>test</scope>
406+
</dependency>
407+
408+
<dependency>
409+
<groupId>com.facebook.presto</groupId>
410+
<artifactId>presto-mongodb</artifactId>
411+
<version>${project.version}</version>
412+
<scope>test</scope>
413+
<type>test-jar</type>
414+
</dependency>
415+
416+
<dependency>
417+
<groupId>de.bwaldvogel</groupId>
418+
<artifactId>mongo-java-server</artifactId>
419+
<version>${mongo-server.version}</version>
420+
<exclusions>
421+
<exclusion>
422+
<groupId>org.mongodb</groupId>
423+
<artifactId>mongo-java-driver</artifactId>
424+
</exclusion>
425+
</exclusions>
426+
<scope>test</scope>
427+
</dependency>
428+
429+
<dependency>
430+
<groupId>de.bwaldvogel</groupId>
431+
<artifactId>mongo-java-server-core</artifactId>
432+
<version>${mongo-server.version}</version>
433+
<exclusions>
434+
<exclusion>
435+
<groupId>io.netty</groupId>
436+
<artifactId>*</artifactId>
437+
</exclusion>
438+
</exclusions>
439+
<scope>test</scope>
440+
</dependency>
441+
442+
<dependency>
443+
<groupId>de.bwaldvogel</groupId>
444+
<artifactId>mongo-java-server-memory-backend</artifactId>
445+
<version>${mongo-server.version}</version>
446+
<scope>test</scope>
447+
</dependency>
448+
<dependency>
449+
<groupId>com.facebook.presto</groupId>
450+
<artifactId>presto-cassandra</artifactId>
451+
<version>${project.version}</version>
452+
<scope>test</scope>
453+
</dependency>
454+
455+
<dependency>
456+
<groupId>com.facebook.presto</groupId>
457+
<artifactId>presto-cassandra</artifactId>
458+
<version>${project.version}</version>
459+
<scope>test</scope>
460+
<type>test-jar</type>
461+
</dependency>
462+
463+
<dependency>
464+
<groupId>com.facebook.presto</groupId>
465+
<artifactId>presto-redis</artifactId>
466+
<version>${project.version}</version>
467+
<scope>test</scope>
468+
</dependency>
469+
470+
<dependency>
471+
<groupId>com.facebook.presto</groupId>
472+
<artifactId>presto-redis</artifactId>
473+
<version>${project.version}</version>
474+
<scope>test</scope>
475+
<type>test-jar</type>
476+
</dependency>
477+
478+
<dependency>
479+
<groupId>com.orange.redis-embedded</groupId>
480+
<artifactId>embedded-redis</artifactId>
481+
<scope>test</scope>
482+
</dependency>
369483
</dependencies>
370484

371485
<build>
@@ -384,6 +498,8 @@
384498
<ignoredNonTestScopedDependency>javax.inject:javax.inject</ignoredNonTestScopedDependency>
385499
<ignoredNonTestScopedDependency>com.facebook.presto:presto-base-jdbc</ignoredNonTestScopedDependency>
386500
<ignoredNonTestScopedDependency>com.google.errorprone:error_prone_annotations</ignoredNonTestScopedDependency>
501+
<ignoredNonTestScopedDependency>org.jetbrains:annotations</ignoredNonTestScopedDependency>
502+
<ignoredNonTestScopedDependency>com.facebook.airlift:units</ignoredNonTestScopedDependency>
387503
</ignoredNonTestScopedDependencies>
388504
</configuration>
389505
</plugin>

‎presto-flight-shim/src/main/java/com/facebook/presto/flightshim/FlightShimProducer.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import com.facebook.presto.spi.connector.ConnectorTransactionHandle;
3939
import com.facebook.presto.spi.security.Identity;
4040
import com.facebook.presto.split.PageSourceManager;
41+
import com.facebook.presto.split.RecordPageSourceProvider;
4142
import com.facebook.presto.type.TypeDeserializer;
4243
import com.google.common.collect.ImmutableMap;
4344
import org.apache.arrow.flight.BackpressureStrategy;

0 commit comments

Comments
 (0)