Skip to content

Commit 0131353

Browse files
raulchenpcmoritz
raulchen
authored and
pcmoritz
committed
[gRPC] Migrate gcs data structures to protobuf (#5024)
1 parent bd8aceb commit 0131353

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+1465
-1642
lines changed

‎BUILD.bazel‎

Lines changed: 47 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,55 @@
11
# Bazel build
22
# C/C++ documentation: https://docs.bazel.build/versions/master/be/c-cpp.html
33

4-
load("@com_github_grpc_grpc//bazel:grpc_build_system.bzl", "grpc_proto_library")
4+
load("@com_github_grpc_grpc//bazel:cc_grpc_library.bzl", "cc_grpc_library")
5+
load("@build_stack_rules_proto//python:python_proto_compile.bzl", "python_proto_compile")
56
load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library")
67
load("@//bazel:ray.bzl", "flatbuffer_py_library")
78
load("@//bazel:cython_library.bzl", "pyx_library")
89

910
COPTS = ["-DRAY_USE_GLOG"]
1011

11-
# Node manager gRPC lib.
12-
grpc_proto_library(
13-
name = "node_manager_grpc_lib",
12+
# === Begin of protobuf definitions ===
13+
14+
proto_library(
15+
name = "gcs_proto",
16+
srcs = ["src/ray/protobuf/gcs.proto"],
17+
visibility = ["//java:__subpackages__"],
18+
)
19+
20+
cc_proto_library(
21+
name = "gcs_cc_proto",
22+
deps = [":gcs_proto"],
23+
)
24+
25+
python_proto_compile(
26+
name = "gcs_py_proto",
27+
deps = [":gcs_proto"],
28+
)
29+
30+
proto_library(
31+
name = "node_manager_proto",
1432
srcs = ["src/ray/protobuf/node_manager.proto"],
1533
)
1634

35+
cc_proto_library(
36+
name = "node_manager_cc_proto",
37+
deps = ["node_manager_proto"],
38+
)
39+
40+
# === End of protobuf definitions ===
41+
42+
# Node manager gRPC lib.
43+
cc_grpc_library(
44+
name = "node_manager_cc_grpc",
45+
srcs = [":node_manager_proto"],
46+
grpc_only = True,
47+
deps = [":node_manager_cc_proto"],
48+
)
49+
1750
# Node manager server and client.
1851
cc_library(
19-
name = "node_manager_rpc_lib",
52+
name = "node_manager_rpc",
2053
srcs = glob([
2154
"src/ray/rpc/*.cc",
2255
]),
@@ -25,7 +58,7 @@ cc_library(
2558
]),
2659
copts = COPTS,
2760
deps = [
28-
":node_manager_grpc_lib",
61+
":node_manager_cc_grpc",
2962
":ray_common",
3063
"@boost//:asio",
3164
"@com_github_grpc_grpc//:grpc++",
@@ -114,7 +147,7 @@ cc_library(
114147
":gcs",
115148
":gcs_fbs",
116149
":node_manager_fbs",
117-
":node_manager_rpc_lib",
150+
":node_manager_rpc",
118151
":object_manager",
119152
":ray_common",
120153
":ray_util",
@@ -422,9 +455,11 @@ cc_library(
422455
"src/ray/gcs/format",
423456
],
424457
deps = [
458+
":gcs_cc_proto",
425459
":gcs_fbs",
426460
":hiredis",
427461
":node_manager_fbs",
462+
":node_manager_rpc",
428463
":ray_common",
429464
":ray_util",
430465
":stats_lib",
@@ -555,46 +590,6 @@ filegroup(
555590
visibility = ["//java:__subpackages__"],
556591
)
557592

558-
flatbuffer_py_library(
559-
name = "python_gcs_fbs",
560-
srcs = [
561-
":gcs_fbs_file",
562-
],
563-
outs = [
564-
"ActorCheckpointIdData.py",
565-
"ActorState.py",
566-
"ActorTableData.py",
567-
"Arg.py",
568-
"ClassTableData.py",
569-
"ClientTableData.py",
570-
"ConfigTableData.py",
571-
"CustomSerializerData.py",
572-
"DriverTableData.py",
573-
"EntryType.py",
574-
"ErrorTableData.py",
575-
"ErrorType.py",
576-
"FunctionTableData.py",
577-
"GcsEntry.py",
578-
"HeartbeatBatchTableData.py",
579-
"HeartbeatTableData.py",
580-
"Language.py",
581-
"ObjectTableData.py",
582-
"ProfileEvent.py",
583-
"ProfileTableData.py",
584-
"RayResource.py",
585-
"ResourcePair.py",
586-
"SchedulingState.py",
587-
"TablePrefix.py",
588-
"TablePubsub.py",
589-
"TaskInfo.py",
590-
"TaskLeaseData.py",
591-
"TaskReconstructionData.py",
592-
"TaskTableData.py",
593-
"TaskTableTestAndUpdate.py",
594-
],
595-
out_prefix = "python/ray/core/generated/",
596-
)
597-
598593
flatbuffer_py_library(
599594
name = "python_node_manager_fbs",
600595
srcs = [
@@ -679,6 +674,7 @@ cc_binary(
679674
linkstatic = 1,
680675
visibility = ["//java:__subpackages__"],
681676
deps = [
677+
":gcs_cc_proto",
682678
":ray_common",
683679
],
684680
)
@@ -688,7 +684,7 @@ genrule(
688684
srcs = [
689685
"python/ray/_raylet.so",
690686
"//:python_sources",
691-
"//:python_gcs_fbs",
687+
"//:gcs_py_proto",
692688
"//:python_node_manager_fbs",
693689
"//:redis-server",
694690
"//:redis-cli",
@@ -710,11 +706,13 @@ genrule(
710706
cp -f $(location //:raylet_monitor) $$WORK_DIR/python/ray/core/src/ray/raylet/ &&
711707
cp -f $(location @plasma//:plasma_store_server) $$WORK_DIR/python/ray/core/src/plasma/ &&
712708
cp -f $(location //:raylet) $$WORK_DIR/python/ray/core/src/ray/raylet/ &&
713-
for f in $(locations //:python_gcs_fbs); do cp -f $$f $$WORK_DIR/python/ray/core/generated/; done &&
714709
mkdir -p $$WORK_DIR/python/ray/core/generated/ray/protocol/ &&
715710
for f in $(locations //:python_node_manager_fbs); do
716711
cp -f $$f $$WORK_DIR/python/ray/core/generated/ray/protocol/;
717712
done &&
713+
for f in $(locations //:gcs_py_proto); do
714+
cp -f $$f $$WORK_DIR/python/ray/core/generated/;
715+
done &&
718716
echo $$WORK_DIR > $@
719717
""",
720718
local = 1,

‎bazel/ray_deps_build_all.bzl‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ load("@com_github_jupp0r_prometheus_cpp//:repositories.bzl", "prometheus_cpp_rep
44
load("@com_github_ray_project_ray//bazel:python_configure.bzl", "python_configure")
55
load("@com_github_checkstyle_java//:repo.bzl", "checkstyle_deps")
66
load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")
7+
load("@build_stack_rules_proto//java:deps.bzl", "java_proto_compile")
8+
load("@build_stack_rules_proto//python:deps.bzl", "python_proto_compile")
79

810

911
def ray_deps_build_all():
@@ -13,4 +15,6 @@ def ray_deps_build_all():
1315
prometheus_cpp_repositories()
1416
python_configure(name = "local_config_python")
1517
grpc_deps()
18+
java_proto_compile()
19+
python_proto_compile()
1620

‎bazel/ray_deps_setup.bzl‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,14 @@ def ray_deps_setup():
105105
http_archive(
106106
name = "com_github_grpc_grpc",
107107
urls = [
108-
"https://github.com/grpc/grpc/archive/7741e806a213cba63c96234f16d712a8aa101a49.tar.gz",
108+
"https://github.com/grpc/grpc/archive/76a381869413834692b8ed305fbe923c0f9c4472.tar.gz",
109109
],
110-
strip_prefix = "grpc-7741e806a213cba63c96234f16d712a8aa101a49",
110+
strip_prefix = "grpc-76a381869413834692b8ed305fbe923c0f9c4472",
111+
)
112+
113+
http_archive(
114+
name = "build_stack_rules_proto",
115+
urls = ["https://github.com/stackb/rules_proto/archive/b93b544f851fdcd3fc5c3d47aee3b7ca158a8841.tar.gz"],
116+
sha256 = "c62f0b442e82a6152fcd5b1c0b7c4028233a9e314078952b6b04253421d56d61",
117+
strip_prefix = "rules_proto-b93b544f851fdcd3fc5c3d47aee3b7ca158a8841",
111118
)

‎doc/source/conf.py‎

Lines changed: 1 addition & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -23,20 +23,7 @@
2323
"gym.spaces",
2424
"ray._raylet",
2525
"ray.core.generated",
26-
"ray.core.generated.ActorCheckpointIdData",
27-
"ray.core.generated.ClientTableData",
28-
"ray.core.generated.DriverTableData",
29-
"ray.core.generated.EntryType",
30-
"ray.core.generated.ErrorTableData",
31-
"ray.core.generated.ErrorType",
32-
"ray.core.generated.GcsEntry",
33-
"ray.core.generated.HeartbeatBatchTableData",
34-
"ray.core.generated.HeartbeatTableData",
35-
"ray.core.generated.Language",
36-
"ray.core.generated.ObjectTableData",
37-
"ray.core.generated.ProfileTableData",
38-
"ray.core.generated.TablePrefix",
39-
"ray.core.generated.TablePubsub",
26+
"ray.core.generated.gcs_pb2",
4027
"ray.core.generated.ray.protocol.Task",
4128
"scipy",
4229
"scipy.signal",

‎java/BUILD.bazel‎

Lines changed: 20 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
load("//bazel:ray.bzl", "flatbuffer_java_library", "define_java_module")
2+
load("@build_stack_rules_proto//java:java_proto_compile.bzl", "java_proto_compile")
23

34
exports_files([
45
"testng.xml",
@@ -50,6 +51,7 @@ define_java_module(
5051
name = "runtime",
5152
additional_srcs = [
5253
":generate_java_gcs_fbs",
54+
":gcs_java_proto",
5355
],
5456
additional_resources = [
5557
":java_native_deps",
@@ -68,6 +70,7 @@ define_java_module(
6870
"@plasma//:org_apache_arrow_arrow_plasma",
6971
"@maven//:com_github_davidmoten_flatbuffers_java",
7072
"@maven//:com_google_guava_guava",
73+
"@maven//:com_google_protobuf_protobuf_java",
7174
"@maven//:com_typesafe_config",
7275
"@maven//:commons_io_commons_io",
7376
"@maven//:de_ruedigermoeller_fst",
@@ -148,38 +151,16 @@ java_binary(
148151
],
149152
)
150153

154+
java_proto_compile(
155+
name = "gcs_java_proto",
156+
deps = ["@//:gcs_proto"],
157+
)
158+
151159
flatbuffers_generated_files = [
152-
"ActorCheckpointData.java",
153-
"ActorCheckpointIdData.java",
154-
"ActorState.java",
155-
"ActorTableData.java",
156160
"Arg.java",
157-
"ClassTableData.java",
158-
"ClientTableData.java",
159-
"ConfigTableData.java",
160-
"CustomSerializerData.java",
161-
"DriverTableData.java",
162-
"EntryType.java",
163-
"ErrorTableData.java",
164-
"ErrorType.java",
165-
"FunctionTableData.java",
166-
"GcsEntry.java",
167-
"HeartbeatBatchTableData.java",
168-
"HeartbeatTableData.java",
169161
"Language.java",
170-
"ObjectTableData.java",
171-
"ProfileEvent.java",
172-
"ProfileTableData.java",
173-
"RayResource.java",
174-
"ResourcePair.java",
175-
"SchedulingState.java",
176-
"TablePrefix.java",
177-
"TablePubsub.java",
178162
"TaskInfo.java",
179-
"TaskLeaseData.java",
180-
"TaskReconstructionData.java",
181-
"TaskTableData.java",
182-
"TaskTableTestAndUpdate.java",
163+
"ResourcePair.java",
183164
]
184165

185166
flatbuffer_java_library(
@@ -198,7 +179,7 @@ genrule(
198179
cmd = """
199180
for f in $(locations //java:java_gcs_fbs); do
200181
chmod +w $$f
201-
cp -f $$f $(@D)/runtime/src/main/java/org/ray/runtime/generated
182+
mv -f $$f $(@D)/runtime/src/main/java/org/ray/runtime/generated
202183
done
203184
python $$(pwd)/java/modify_generated_java_flatbuffers_files.py $(@D)/..
204185
""",
@@ -221,8 +202,10 @@ filegroup(
221202
genrule(
222203
name = "gen_maven_deps",
223204
srcs = [
224-
":java_native_deps",
205+
":gcs_java_proto",
225206
":generate_java_gcs_fbs",
207+
":java_native_deps",
208+
":copy_pom_file",
226209
"@plasma//:org_apache_arrow_arrow_plasma",
227210
],
228211
outs = ["gen_maven_deps.out"],
@@ -237,10 +220,15 @@ genrule(
237220
chmod +w $$f
238221
cp $$f $$NATIVE_DEPS_DIR
239222
done
240-
# Copy flatbuffers-generated files
223+
# Copy protobuf-generated files.
241224
GENERATED_DIR=$$WORK_DIR/java/runtime/src/main/java/org/ray/runtime/generated
242225
rm -rf $$GENERATED_DIR
243226
mkdir -p $$GENERATED_DIR
227+
for f in $(locations //java:gcs_java_proto); do
228+
unzip $$f
229+
mv org/ray/runtime/generated/* $$GENERATED_DIR
230+
done
231+
# Copy flatbuffers-generated files
244232
for f in $(locations //java:generate_java_gcs_fbs); do
245233
cp $$f $$GENERATED_DIR
246234
done
@@ -250,6 +238,7 @@ genrule(
250238
echo $$(date) > $@
251239
""",
252240
local = 1,
241+
tags = ["no-cache"],
253242
)
254243

255244
genrule(

‎java/dependencies.bzl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ def gen_java_deps():
66
"com.beust:jcommander:1.72",
77
"com.github.davidmoten:flatbuffers-java:1.9.0.1",
88
"com.google.guava:guava:27.0.1-jre",
9+
"com.google.protobuf:protobuf-java:3.8.0",
910
"com.puppycrawl.tools:checkstyle:8.15",
1011
"com.sun.xml.bind:jaxb-core:2.3.0",
1112
"com.sun.xml.bind:jaxb-impl:2.3.0",

‎java/modify_generated_java_flatbuffers_files.py‎

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44

55
import os
66
import sys
7-
87
"""
98
This script is used for modifying the generated java flatbuffer
109
files for the reason: The package declaration in Java is different
@@ -21,35 +20,30 @@
2120
PACKAGE_DECLARATION = "package org.ray.runtime.generated;"
2221

2322

24-
def add_new_line(file, line_num, text):
23+
def add_package(file):
2524
with open(file, "r") as file_handler:
2625
lines = file_handler.readlines()
27-
if (line_num <= 0) or (line_num > len(lines) + 1):
28-
return False
2926

30-
lines.insert(line_num - 1, text + os.linesep)
27+
if "FlatBuffers" not in lines[0]:
28+
return
29+
30+
lines.insert(1, PACKAGE_DECLARATION + os.linesep)
3131
with open(file, "w") as file_handler:
3232
for line in lines:
3333
file_handler.write(line)
3434

35-
return True
36-
3735

3836
def add_package_declarations(generated_root_path):
3937
file_names = os.listdir(generated_root_path)
4038
for file_name in file_names:
4139
if not file_name.endswith(".java"):
4240
continue
4341
full_name = os.path.join(generated_root_path, file_name)
44-
success = add_new_line(full_name, 2, PACKAGE_DECLARATION)
45-
if not success:
46-
raise RuntimeError("Failed to add package declarations, "
47-
"file name is %s" % full_name)
42+
add_package(full_name)
4843

4944

5045
if __name__ == "__main__":
5146
ray_home = sys.argv[1]
5247
root_path = os.path.join(
53-
ray_home,
54-
"java/runtime/src/main/java/org/ray/runtime/generated")
48+
ray_home, "java/runtime/src/main/java/org/ray/runtime/generated")
5549
add_package_declarations(root_path)

0 commit comments

Comments
 (0)