Skip to content

Commit 0a7bade

Browse files
finnegancarrollfinnegancarroll
committed
Add gRPC server as transport-grpc plugin (#16534)
Introduce auxiliary transport to NetworkPlugin and add gRPC plugin. Auxiliary transports are optional lifecycle components provided by network plugins which run in parallel to the http server/native transport. They are distinct from the existing NetworkPlugin interfaces of 'getTransports' and 'getHttpTransports' as auxiliary transports are optional. Each AuxTransport implements it's own 'aux.transport.type' and 'aux.transport.<type>.ports' setting. Since Security.java initializes previous to Node.java during bootstrap socket binding permissions are granted based on 'aux.transport.<type>.ports' for each enabled 'aux.transport.type', falling back to a default if no ports are specified. Signed-off-by: Finn Carroll <carrofin@amazon.com> (cherry picked from commit 7a0e8fb)
1 parent cbcc150 commit 0a7bade

42 files changed

Lines changed: 2149 additions & 188 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎CHANGELOG.md‎

Lines changed: 60 additions & 67 deletions
Large diffs are not rendered by default.

‎gradle/libs.versions.toml‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
[versions]
2+
opensearch = "2.19.0"
3+
lucene = "9.12.1"
4+
5+
bundled_jdk_vendor = "adoptium"
6+
bundled_jdk = "21.0.5+11"
7+
8+
# optional dependencies
9+
spatial4j = "0.7"
10+
jts = "1.15.0"
11+
jackson = "2.18.2"
12+
jackson_databind = "2.18.2"
13+
snakeyaml = "2.1"
14+
icu4j = "75.1"
15+
supercsv = "2.4.0"
16+
log4j = "2.21.0"
17+
slf4j = "1.7.36"
18+
asm = "9.7"
19+
jettison = "1.5.4"
20+
woodstox = "6.4.0"
21+
kotlin = "1.7.10"
22+
antlr4 = "4.13.1"
23+
guava = "33.2.1-jre"
24+
protobuf = "3.25.5"
25+
jakarta_annotation = "1.3.5"
26+
google_http_client = "1.44.1"
27+
google_auth = "1.29.0"
28+
tdigest = "3.2"
29+
hdrhistogram = "2.2.2"
30+
grpc = "1.68.0"
31+
32+
# when updating the JNA version, also update the version in buildSrc/build.gradle
33+
jna = "5.13.0"
34+
35+
netty = "4.1.115.Final"
36+
joda = "2.12.7"
37+
roaringbitmap = "1.3.0"
38+
39+
# project reactor
40+
reactor_netty = "1.1.23"
41+
reactor = "3.5.20"
42+
43+
# client dependencies
44+
httpclient = "4.5.14"
45+
httpcore = "4.4.16"
46+
httpasyncclient = "4.1.5"
47+
commonslogging = "1.2"
48+
commonscodec = "1.16.1"
49+
commonslang = "3.14.0"
50+
commonscompress = "1.26.1"
51+
commonsio = "2.16.0"
52+
# plugin dependencies
53+
aws = "2.20.86"
54+
reactivestreams = "1.0.4"
55+
56+
# when updating this version, you need to ensure compatibility with:
57+
# - plugins/ingest-attachment (transitive dependency, check the upstream POM)
58+
# - distribution/tools/plugin-cli
59+
bouncycastle="1.78"
60+
# test dependencies
61+
randomizedrunner = "2.7.1"
62+
junit = "4.13.2"
63+
hamcrest = "2.1"
64+
mockito = "5.14.2"
65+
objenesis = "3.3"
66+
bytebuddy = "1.15.10"
67+
68+
# benchmark dependencies
69+
jmh = "1.35"
70+
71+
# compression
72+
zstd = "1.5.5-5"
73+
74+
jzlib = "1.1.3"
75+
76+
resteasy = "6.2.4.Final"
77+
78+
# opentelemetry dependencies
79+
opentelemetry = "1.41.0"
80+
opentelemetrysemconv = "1.27.0-alpha"
81+
82+
# arrow dependencies
83+
arrow = "17.0.0"
84+
flatbuffers = "2.0.0"
85+
86+
[libraries]
87+
hdrhistogram = { group = "org.hdrhistogram", name = "HdrHistogram", version.ref = "hdrhistogram" }
88+
jakartaannotation = { group = "jakarta.annotation", name = "jakarta.annotation-api", version.ref = "jakarta_annotation" }
89+
jodatime = { group = "joda-time", name = "joda-time", version.ref = "joda" }
90+
jna = { group = "net.java.dev.jna", name = "jna", version.ref = "jna" }
91+
jtscore = { group = "org.locationtech.jts", name = "jts-core", version.ref = "jts" }
92+
jzlib = { group = "com.jcraft", name = "jzlib", version.ref = "jzlib" }
93+
log4japi = { group = "org.apache.logging.log4j", name = "log4j-api", version.ref = "log4j" }
94+
log4jjul = { group = "org.apache.logging.log4j", name = "log4j-jul", version.ref = "log4j" }
95+
log4jcore = { group = "org.apache.logging.log4j", name = "log4j-core", version.ref = "log4j" }
96+
lucene-core = { group = "org.apache.lucene", name = "lucene-core", version.ref = "lucene" }
97+
lucene-analysis-common = { group = "org.apache.lucene", name = "lucene-analysis-common", version.ref = "lucene" }
98+
lucene-backward-codecs = { group = "org.apache.lucene", name = "lucene-backward-codecs", version.ref = "lucene" }
99+
lucene-grouping = { group = "org.apache.lucene", name = "lucene-grouping", version.ref = "lucene" }
100+
lucene-highlighter = { group = "org.apache.lucene", name = "lucene-highlighter", version.ref = "lucene" }
101+
lucene-join = { group = "org.apache.lucene", name = "lucene-join", version.ref = "lucene" }
102+
lucene-memory = { group = "org.apache.lucene", name = "lucene-memory", version.ref = "lucene" }
103+
lucene-misc = { group = "org.apache.lucene", name = "lucene-misc", version.ref = "lucene" }
104+
lucene-queries = { group = "org.apache.lucene", name = "lucene-queries", version.ref = "lucene" }
105+
lucene-queryparser = { group = "org.apache.lucene", name = "lucene-queryparser", version.ref = "lucene" }
106+
lucene-sandbox = { group = "org.apache.lucene", name = "lucene-sandbox", version.ref = "lucene" }
107+
lucene-spatial-extras = { group = "org.apache.lucene", name = "lucene-spatial-extras", version.ref = "lucene" }
108+
lucene-spatial3d = { group = "org.apache.lucene", name = "lucene-spatial3d", version.ref = "lucene" }
109+
lucene-suggest = { group = "org.apache.lucene", name = "lucene-suggest", version.ref = "lucene" }
110+
protobuf = { group = "com.google.protobuf", name = "protobuf-java", version.ref = "protobuf" }
111+
reactivestreams = { group = "io.projectreactor", name = "reactor-core", version.ref = "reactor" }
112+
reactorcore = { group = "org.reactivestreams", name = "reactive-streams", version.ref = "reactivestreams" }
113+
roaringbitmap = { group = "org.roaringbitmap", name = "RoaringBitmap", version.ref = "roaringbitmap" }
114+
spatial4j = { group = "org.locationtech.spatial4j", name = "spatial4j", version.ref = "spatial4j" }
115+
tdigest = { group = "com.tdunning", name = "t-digest", version.ref = "tdigest" }
116+
117+
[bundles]
118+
lucene = [
119+
"lucene-core",
120+
"lucene-analysis-common",
121+
"lucene-backward-codecs",
122+
"lucene-grouping",
123+
"lucene-highlighter",
124+
"lucene-join",
125+
"lucene-memory",
126+
"lucene-misc",
127+
"lucene-queries",
128+
"lucene-queryparser",
129+
"lucene-sandbox",
130+
"lucene-spatial-extras",
131+
"lucene-spatial3d",
132+
"lucene-suggest"
133+
]

‎plugins/discovery-gce/licenses/grpc-api-1.68.2.jar.sha1‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
a257a5dd25dda1c97a99b56d5b9c1e56c12ae554

‎plugins/repository-gcs/licenses/grpc-api-1.68.2.jar.sha1‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
a257a5dd25dda1c97a99b56d5b9c1e56c12ae554

‎plugins/transport-grpc/build.gradle‎

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
import org.gradle.api.attributes.java.TargetJvmEnvironment
2+
3+
/*
4+
* SPDX-License-Identifier: Apache-2.0
5+
*
6+
* The OpenSearch Contributors require contributions made to
7+
* this file be licensed under the Apache-2.0 license or a
8+
* compatible open source license.
9+
*/
10+
11+
opensearchplugin {
12+
description 'gRPC based transport implementation'
13+
classname 'org.opensearch.transport.grpc.GrpcPlugin'
14+
}
15+
16+
dependencies {
17+
compileOnly "com.google.code.findbugs:jsr305:3.0.2"
18+
runtimeOnly "com.google.guava:guava:${versions.guava}"
19+
implementation "com.google.errorprone:error_prone_annotations:2.24.1"
20+
implementation "com.google.guava:failureaccess:1.0.1"
21+
implementation "io.grpc:grpc-api:${versions.grpc}"
22+
implementation "io.grpc:grpc-core:${versions.grpc}"
23+
implementation "io.grpc:grpc-netty-shaded:${versions.grpc}"
24+
implementation "io.grpc:grpc-protobuf-lite:${versions.grpc}"
25+
implementation "io.grpc:grpc-protobuf:${versions.grpc}"
26+
implementation "io.grpc:grpc-services:${versions.grpc}"
27+
implementation "io.grpc:grpc-stub:${versions.grpc}"
28+
implementation "io.grpc:grpc-util:${versions.grpc}"
29+
implementation "io.perfmark:perfmark-api:0.26.0"
30+
}
31+
32+
tasks.named("dependencyLicenses").configure {
33+
mapping from: /grpc-.*/, to: 'grpc'
34+
}
35+
36+
thirdPartyAudit {
37+
ignoreMissingClasses(
38+
'com.aayushatharva.brotli4j.Brotli4jLoader',
39+
'com.aayushatharva.brotli4j.decoder.DecoderJNI$Status',
40+
'com.aayushatharva.brotli4j.decoder.DecoderJNI$Wrapper',
41+
'com.aayushatharva.brotli4j.encoder.BrotliEncoderChannel',
42+
'com.aayushatharva.brotli4j.encoder.Encoder$Mode',
43+
'com.aayushatharva.brotli4j.encoder.Encoder$Parameters',
44+
// classes are missing
45+
46+
// from io.netty.logging.CommonsLoggerFactory (netty)
47+
'org.apache.commons.logging.Log',
48+
'org.apache.commons.logging.LogFactory',
49+
50+
// from Log4j (deliberate, Netty will fallback to Log4j 2)
51+
'org.apache.log4j.Level',
52+
'org.apache.log4j.Logger',
53+
54+
// from io.netty.handler.ssl.util.BouncyCastleSelfSignedCertGenerator (netty)
55+
'org.bouncycastle.cert.X509v3CertificateBuilder',
56+
'org.bouncycastle.cert.jcajce.JcaX509CertificateConverter',
57+
'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder',
58+
'org.bouncycastle.openssl.PEMEncryptedKeyPair',
59+
'org.bouncycastle.openssl.PEMParser',
60+
'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter',
61+
'org.bouncycastle.openssl.jcajce.JceOpenSSLPKCS8DecryptorProviderBuilder',
62+
'org.bouncycastle.openssl.jcajce.JcePEMDecryptorProviderBuilder',
63+
'org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo',
64+
65+
// from io.netty.handler.ssl.JettyNpnSslEngine (netty)
66+
'org.eclipse.jetty.npn.NextProtoNego$ClientProvider',
67+
'org.eclipse.jetty.npn.NextProtoNego$ServerProvider',
68+
'org.eclipse.jetty.npn.NextProtoNego',
69+
70+
// from io.netty.handler.codec.marshalling.ChannelBufferByteInput (netty)
71+
'org.jboss.marshalling.ByteInput',
72+
73+
// from io.netty.handler.codec.marshalling.ChannelBufferByteOutput (netty)
74+
'org.jboss.marshalling.ByteOutput',
75+
76+
// from io.netty.handler.codec.marshalling.CompatibleMarshallingEncoder (netty)
77+
'org.jboss.marshalling.Marshaller',
78+
79+
// from io.netty.handler.codec.marshalling.ContextBoundUnmarshallerProvider (netty)
80+
'org.jboss.marshalling.MarshallerFactory',
81+
'org.jboss.marshalling.MarshallingConfiguration',
82+
'org.jboss.marshalling.Unmarshaller',
83+
84+
// from io.netty.util.internal.logging.InternalLoggerFactory (netty) - it's optional
85+
'org.slf4j.helpers.FormattingTuple',
86+
'org.slf4j.helpers.MessageFormatter',
87+
'org.slf4j.Logger',
88+
'org.slf4j.LoggerFactory',
89+
'org.slf4j.spi.LocationAwareLogger',
90+
91+
'com.google.gson.stream.JsonReader',
92+
'com.google.gson.stream.JsonToken',
93+
'com.google.protobuf.util.Durations',
94+
'com.google.protobuf.util.Timestamps',
95+
'com.google.protobuf.nano.CodedOutputByteBufferNano',
96+
'com.google.protobuf.nano.MessageNano',
97+
'com.google.rpc.Status',
98+
'com.google.rpc.Status$Builder',
99+
'com.ning.compress.BufferRecycler',
100+
'com.ning.compress.lzf.ChunkDecoder',
101+
'com.ning.compress.lzf.ChunkEncoder',
102+
'com.ning.compress.lzf.LZFChunk',
103+
'com.ning.compress.lzf.LZFEncoder',
104+
'com.ning.compress.lzf.util.ChunkDecoderFactory',
105+
'com.ning.compress.lzf.util.ChunkEncoderFactory',
106+
'lzma.sdk.lzma.Encoder',
107+
'net.jpountz.lz4.LZ4Compressor',
108+
'net.jpountz.lz4.LZ4Factory',
109+
'net.jpountz.lz4.LZ4FastDecompressor',
110+
'net.jpountz.xxhash.XXHash32',
111+
'net.jpountz.xxhash.XXHashFactory',
112+
'org.eclipse.jetty.alpn.ALPN$ClientProvider',
113+
'org.eclipse.jetty.alpn.ALPN$ServerProvider',
114+
'org.eclipse.jetty.alpn.ALPN',
115+
116+
'org.conscrypt.AllocatedBuffer',
117+
'org.conscrypt.BufferAllocator',
118+
'org.conscrypt.Conscrypt',
119+
'org.conscrypt.HandshakeListener',
120+
121+
'reactor.blockhound.BlockHound$Builder',
122+
'reactor.blockhound.integration.BlockHoundIntegration'
123+
)
124+
125+
ignoreViolations(
126+
// uses internal java api: sun.misc.Unsafe
127+
'com.google.common.cache.Striped64',
128+
'com.google.common.cache.Striped64$1',
129+
'com.google.common.cache.Striped64$Cell',
130+
'com.google.common.hash.Striped64',
131+
'com.google.common.hash.Striped64$1',
132+
'com.google.common.hash.Striped64$Cell',
133+
'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
134+
'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
135+
'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
136+
'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper',
137+
'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
138+
'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
139+
'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
140+
141+
'io.grpc.netty.shaded.io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator',
142+
'io.grpc.netty.shaded.io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$1',
143+
'io.grpc.netty.shaded.io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$2',
144+
'io.grpc.netty.shaded.io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$3',
145+
'io.grpc.netty.shaded.io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$4',
146+
'io.grpc.netty.shaded.io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$5',
147+
'io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0',
148+
'io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0$1',
149+
'io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0$2',
150+
'io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0$3',
151+
'io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0$4',
152+
'io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0$6',
153+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueConsumerNodeRef',
154+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueProducerNodeRef',
155+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueColdProducerFields',
156+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueConsumerFields',
157+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueProducerFields',
158+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.LinkedQueueNode',
159+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueConsumerIndexField',
160+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueProducerIndexField',
161+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField',
162+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField',
163+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField',
164+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess',
165+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.util.UnsafeLongArrayAccess',
166+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess'
167+
)
168+
}

‎plugins/transport-grpc/licenses/error_prone_annotations-2.24.1.jar.sha1‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
32b299e45105aa9b0df8279c74dc1edfcf313ff0

0 commit comments

Comments
 (0)