Skip to content

Commit 7840504

Browse files
raymondxyangwschin
raymondxyang
authored and
wschin
committed
Update gen_doc script to validate proto3 files (#2122)
* Update gen_doc script to validate proto3 files * Update CMakeLists.txt
1 parent bd35e62 commit 7840504

4 files changed

Lines changed: 48 additions & 3 deletions

File tree

‎CMakeLists.txt‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,18 @@ if(NOT DEFINED ONNX_ML)
3232
endif()
3333
option(ONNX_ML "Enable traditional ML API." ${DEFAULT_ONNX_ML})
3434
endif()
35+
3536
option(ONNXIFI_DUMMY_BACKEND "Use dummy backend in onnxifi test driver." OFF)
3637

38+
if(NOT DEFINED ONNX_VERIFY_PROTO3)
39+
if(DEFINED ENV{ONNX_VERIFY_PROTO3})
40+
set(PROTO3_ENABLED $ENV{ONNX_VERIFY_PROTO3})
41+
else()
42+
set(PROTO3_ENABLED OFF)
43+
endif()
44+
option(ONNX_VERIFY_PROTO3 "Generate code by proto3" ${PROTO3_ENABLED})
45+
endif()
46+
3747
# Set C++11 as standard for the whole project
3848
if(NOT MSVC)
3949
set(CMAKE_CXX_STANDARD 11)
@@ -193,13 +203,21 @@ function(RELATIVE_PROTOBUF_GENERATE_CPP NAME SRCS HDRS ROOT_DIR DEPEND)
193203
if(ONNX_USE_LITE_PROTO)
194204
list(APPEND GEN_PROTO_ARGS -l)
195205
endif()
206+
if(ONNX_VERIFY_PROTO3)
207+
if(NOT ONNX_PROTOC_EXECUTABLE)
208+
message(FATAL_ERROR "Protobuf compiler not found")
209+
endif()
210+
list(APPEND GEN_PROTO_ARGS --protoc_path)
211+
list(APPEND GEN_PROTO_ARGS "${ONNX_PROTOC_EXECUTABLE}")
212+
endif()
213+
196214
add_custom_command(OUTPUT "${GENERATED_PROTO}"
197215
COMMAND "${_python_exe}" "${GEN_PROTO_PY}"
198216
ARGS ${GEN_PROTO_ARGS}
199217
DEPENDS ${INFILE}
200218
COMMENT "Running gen_proto.py on ${INFILE}"
201219
VERBATIM)
202-
220+
message("Generated: ${GENERATED_PROTO}")
203221
set(PROTOC_ARGS
204222
${GENERATED_PROTO}
205223
-I

‎appveyor.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,10 @@ environment:
1818

1919
matrix:
2020
- CONDA_PREFIX: C:\Miniconda35-x64
21+
ONNX_VERIFY_PROTO3: 1
2122

2223
- CONDA_PREFIX: C:\Miniconda36-x64
24+
ONNX_VERIFY_PROTO3: 1
2325

2426
- CONDA_PREFIX: C:\Miniconda36
2527

‎onnx/gen_proto.py‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
import io
99
import os
1010
import re
11+
import glob
12+
import subprocess
1113
from textwrap import dedent
1214

1315
autogen_header = """\
@@ -106,6 +108,13 @@ def convert_to_proto3(lines): # type: (Iterable[Text]) -> Iterable[Text]
106108
yield line
107109

108110

111+
def gen_proto3_code(protoc_path, proto3_path, include_path, cpp_out, python_out): # type: (Text, Text, Text, Text, Text) -> None
112+
print("Generate pb3 code using {}".format(protoc_path))
113+
build_args = [protoc_path, proto3_path, '-I', include_path]
114+
build_args.extend(['--cpp_out', cpp_out, '--python_out', python_out])
115+
subprocess.check_call(build_args)
116+
117+
109118
def translate(source, proto, onnx_ml, package_name): # type: (Text, int, bool, Text) -> Text
110119
lines = source.splitlines() # type: Iterable[Text]
111120
lines = process_ifs(lines, onnx_ml=onnx_ml)
@@ -121,7 +130,7 @@ def qualify(f, pardir=os.path.realpath(os.path.dirname(__file__))): # type: (Te
121130
return os.path.join(pardir, f)
122131

123132

124-
def convert(stem, package_name, output, do_onnx_ml=False, lite=False): # type: (Text, Text, Text, bool, bool) -> None
133+
def convert(stem, package_name, output, do_onnx_ml=False, lite=False, protoc_path=''): # type: (Text, Text, Text, bool, bool, Text) -> None
125134
proto_in = qualify("{}.in.proto".format(stem))
126135
need_rename = (package_name != DEFAULT_PACKAGE_NAME)
127136
if do_onnx_ml:
@@ -146,6 +155,15 @@ def convert(stem, package_name, output, do_onnx_ml=False, lite=False): # type:
146155
fout.write(translate(source, proto=3, onnx_ml=do_onnx_ml, package_name=package_name))
147156
if lite:
148157
fout.write(LITE_OPTION)
158+
if protoc_path:
159+
porto3_dir = os.path.dirname(proto3)
160+
base_dir = os.path.dirname(porto3_dir)
161+
gen_proto3_code(protoc_path, proto3, base_dir, base_dir, base_dir)
162+
pb3_files = glob.glob(os.path.join(porto3_dir, '*.proto3.*'))
163+
for pb3_file in pb3_files:
164+
print("Removing {}".format(pb3_file))
165+
os.remove(pb3_file)
166+
149167
if need_rename:
150168
if do_onnx_ml:
151169
proto_header = qualify("{}-ml.pb.h".format(stem), pardir=output)
@@ -193,6 +211,9 @@ def main(): # type: () -> None
193211
parser.add_argument('-o', '--output',
194212
default=os.path.realpath(os.path.dirname(__file__)),
195213
help='output directory (default: %(default)s)')
214+
parser.add_argument('--protoc_path',
215+
default='',
216+
help='path to protoc for proto3 file validation')
196217
parser.add_argument('stems', nargs='*', default=['onnx', 'onnx-operators'],
197218
help='list of .in.proto file stems '
198219
'(default: %(default)s)')
@@ -206,7 +227,8 @@ def main(): # type: () -> None
206227
package_name=args.package,
207228
output=args.output,
208229
do_onnx_ml=args.ml,
209-
lite=args.lite)
230+
lite=args.lite,
231+
protoc_path=args.protoc_path)
210232

211233

212234
if __name__ == '__main__':

‎setup.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
################################################################################
4343

4444
ONNX_ML = not bool(os.getenv('ONNX_ML') == '0')
45+
ONNX_VERIFY_PROTO3 = bool(os.getenv('ONNX_VERIFY_PROTO3') == '1')
4546
ONNX_NAMESPACE = os.getenv('ONNX_NAMESPACE', 'onnx')
4647
ONNX_BUILD_TESTS = bool(os.getenv('ONNX_BUILD_TESTS') == '1')
4748

@@ -180,6 +181,8 @@ def run(self):
180181
cmake_args.append('-DCMAKE_GENERATOR_PLATFORM=x64')
181182
if ONNX_ML:
182183
cmake_args.append('-DONNX_ML=1')
184+
if ONNX_VERIFY_PROTO3:
185+
cmake_args.append('-DONNX_VERIFY_PROTO3=1')
183186
if ONNX_BUILD_TESTS:
184187
cmake_args.append('-DONNX_BUILD_TESTS=ON')
185188
if 'CMAKE_ARGS' in os.environ:

0 commit comments

Comments
 (0)