Skip to content

Commit 8480654

Browse files
arawindarawind
committed
Merge branch 'float32' of github.com:arawind/DataflowTemplates into float32
2 parents c6364b7 + afd1dc2 commit 8480654

76 files changed

Lines changed: 3320 additions & 861 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/CODEOWNERS‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,12 @@ v1/src/test/java/com/google/cloud/teleport/templates/common/SpannerConverterTest
1313
# Datastream To spanner template
1414
v2/datastream-to-spanner/ @GoogleCloudPlatform/spanner-migrations-team
1515
v2/spanner-common/ @GoogleCloudPlatform/spanner-migrations-team
16-
v2/spanner-migrations-sdk/ @GoogleCloudPlatform/spanner-migrations-team
1716

1817
# Spanner To other database reverse replication templates
1918
v2/spanner-change-streams-to-sharded-file-sink/ @GoogleCloudPlatform/spanner-migrations-team
2019
v2/gcs-to-sourcedb/ @GoogleCloudPlatform/spanner-migrations-team
20+
v2/spanner-migrations-sdk/ @GoogleCloudPlatform/spanner-migrations-team
21+
v2/spanner-custom-shard/ @GoogleCloudPlatform/spanner-migrations-team
22+
23+
# Spanner Bulk migration template
24+
v2/sourcedb-to-spanner/ @GoogleCloudPlatform/spanner-migrations-team

‎.github/codecov.yml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
comment:
2+
layout: "header, diff, components, files" # show component info in the PR comment
3+
4+
coverage:
5+
status:
6+
patch:
7+
default:
8+
target: 80%
9+
informational: true
10+
11+
component_management:
12+
individual_components:
13+
- component_id: spanner-templates
14+
name: spanner-templates
15+
paths:
16+
- "v1/src/main/java/com/google/cloud/teleport/spanner/**"
17+
- "v1/src/main/java/com/google/cloud/teleport/templates/SpannerToText.java"
18+
- "v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java"
19+
- "v2/datastream-to-spanner/**"
20+
- "v2/spanner-common/**"
21+
- "v2/spanner-change-streams-to-sharded-file-sink/**"
22+
- "v2/gcs-to-sourcedb/**"
23+
- "v2/spanner-migrations-sdk/**"
24+
- "v2/spanner-custom-shard/**"
25+
- "v2/sourcedb-to-spanner/**"
26+
statuses:
27+
- type: project
28+
informational: true
29+
- type: patch
30+
target: 80%
31+
- component_id: spanner-import-export
32+
name: spanner-import-export
33+
paths:
34+
- "v1/src/main/java/com/google/cloud/teleport/spanner/**"
35+
- "v1/src/main/java/com/google/cloud/teleport/templates/SpannerToText.java"
36+
- "v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java"
37+
- component_id: spanner-live-forward-migration
38+
name: spanner-live-forward-migration
39+
paths:
40+
- "v2/datastream-to-spanner/**"
41+
- "v2/spanner-common/**"
42+
- component_id: spanner-live-reverse-replication
43+
name: spanner-live-reverse-replication
44+
paths:
45+
- "v2/spanner-change-streams-to-sharded-file-sink/**"
46+
- "v2/gcs-to-sourcedb/**"
47+
- "v2/spanner-migrations-sdk/**"
48+
- "v2/spanner-custom-shard/**"
49+
- "v2/spanner-common/**"
50+
- component_id: spanner-bulk-migration
51+
name: spanner-bulk-migration
52+
paths:
53+
- "v2/sourcedb-to-spanner/**"
54+
- "v2/spanner-common/**"

‎.github/workflows/java-pr-report.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@
1717
name: Java PR Report
1818
on:
1919
workflow_run:
20-
workflows: [ Java PR ]
20+
workflows:
21+
- Java PR
22+
- Release
2123
types: [ completed ]
2224

2325
permissions:

‎.github/workflows/java-pr.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,12 @@ jobs:
9494
name: surefire-test-results
9595
path: '**/surefire-reports/TEST-*.xml'
9696
retention-days: 1
97+
- name: Upload coverage reports to Codecov
98+
uses: codecov/codecov-action@v4.0.1
99+
with:
100+
token: ${{ secrets.CODECOV_TOKEN }}
101+
slug: GoogleCloudPlatform/DataflowTemplates
102+
files: 'target/site/jacoco-aggregate/jacoco.xml'
97103
- name: Cleanup Java Environment
98104
uses: ./.github/actions/cleanup-java-env
99105
java_integration_smoke_tests_templates:

‎it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import com.google.cloud.teleport.metadata.DirectRunnerTest;
2626
import com.google.cloud.teleport.metadata.MultiTemplateIntegrationTest;
2727
import com.google.cloud.teleport.metadata.Template;
28+
import com.google.cloud.teleport.metadata.Template.TemplateType;
2829
import com.google.cloud.teleport.metadata.TemplateCreationParameter;
2930
import com.google.cloud.teleport.metadata.TemplateCreationParameters;
3031
import com.google.cloud.teleport.metadata.TemplateIntegrationTest;
@@ -377,6 +378,11 @@ private String[] buildMavenStageCommand(
377378
moduleBuild = ".";
378379
}
379380

381+
// Skip shading for now due to flakiness / slowness in the process, except for XLANG
382+
// templates, which MUST use shading because they're built using a custom Dockerfile
383+
// that will copy only the shaded jar to the docker image.
384+
boolean skipShade = templateMetadata.type() != TemplateType.XLANG;
385+
380386
return new String[] {
381387
"mvn",
382388
"compile",
@@ -390,7 +396,7 @@ private String[] buildMavenStageCommand(
390396
"-PtemplatesStage,pluginOutputDir,splunkDeps",
391397
"-DpluginRunId=" + RandomStringUtils.randomAlphanumeric(16),
392398
// Skip shading for now due to flakiness / slowness in the process.
393-
"-DskipShade",
399+
"-DskipShade=" + skipShade,
394400
"-DskipTests",
395401
"-Dmaven.test.skip",
396402
"-Dcheckstyle.skip",

‎metadata/src/main/java/com/google/cloud/teleport/metadata/Template.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@
4343

4444
String yamlTemplateName() default "";
4545

46+
String xlangContainerName() default "";
47+
4648
/** The category of the template. */
4749
TemplateCategory category();
4850

@@ -104,7 +106,8 @@
104106
enum TemplateType {
105107
JAVA,
106108
PYTHON,
107-
YAML
109+
YAML,
110+
XLANG
108111
}
109112

110113
/** Marker if the template is still in preview / pre-GA. */

‎plugins/core-plugin/src/main/java/com/google/cloud/teleport/plugin/PythonDockerfileGenerator.java‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import java.nio.charset.StandardCharsets;
2727
import java.nio.file.Files;
2828
import java.nio.file.Path;
29+
import java.nio.file.Paths;
2930
import java.util.HashMap;
3031
import java.util.Map;
3132
import java.util.logging.Logger;
@@ -66,4 +67,77 @@ public static void generateDockerfile(
6667
throw e;
6768
}
6869
}
70+
71+
public static void generateXlangDockerfile(
72+
String basePythonContainerImage,
73+
String containerName,
74+
File targetDirectory,
75+
File artifactFile,
76+
String commandSpec)
77+
throws IOException, TemplateException {
78+
Configuration freemarkerConfig = new Configuration(Configuration.VERSION_2_3_32);
79+
freemarkerConfig.setDefaultEncoding("UTF-8");
80+
freemarkerConfig.setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER);
81+
freemarkerConfig.setLogTemplateExceptions(true);
82+
freemarkerConfig.setClassForTemplateLoading(PythonDockerfileGenerator.class, "/");
83+
84+
String classesDirectory = targetDirectory.getPath() + "/classes";
85+
Map<String, Object> parameters = new HashMap<>();
86+
parameters.put("baseContainerImage", basePythonContainerImage);
87+
parameters.put("commandSpec", commandSpec);
88+
parameters.put("containerName", containerName);
89+
90+
Template template = freemarkerConfig.getTemplate("Dockerfile-xlang-template");
91+
92+
ByteArrayOutputStream baos = new ByteArrayOutputStream();
93+
OutputStreamWriter writer = new OutputStreamWriter(baos);
94+
95+
try {
96+
template.process(parameters, writer);
97+
writer.flush();
98+
99+
Files.createDirectories(Path.of(classesDirectory + "/" + containerName));
100+
101+
Files.write(
102+
Path.of(classesDirectory + "/" + containerName + "/Dockerfile"),
103+
baos.toString(StandardCharsets.UTF_8).getBytes());
104+
105+
} catch (Exception e) {
106+
LOG.warning("Unable to generate Dockerfile for " + containerName);
107+
throw e;
108+
}
109+
110+
try {
111+
Files.createDirectories(Path.of(classesDirectory + "/" + containerName + "/classpath"));
112+
Files.createDirectories(Path.of(classesDirectory + "/" + containerName + "/libs"));
113+
114+
String artifactPath = artifactFile.getPath();
115+
String targetArtifactPath =
116+
artifactPath.substring(artifactPath.lastIndexOf("/"), artifactPath.length());
117+
118+
Files.copy(
119+
Path.of(targetDirectory.getPath() + targetArtifactPath),
120+
Path.of(classesDirectory + "/" + containerName + "/classpath" + targetArtifactPath));
121+
String sourceLibsDirectory = targetDirectory.getPath() + "/extra_libs";
122+
String destLibsDirectory = classesDirectory + "/" + containerName + "/libs/";
123+
Files.walk(Paths.get(sourceLibsDirectory))
124+
.forEach(
125+
source -> {
126+
LOG.warning("current source: " + source.toString());
127+
LOG.warning("current source libs directory: " + sourceLibsDirectory);
128+
Path dest =
129+
Paths.get(
130+
destLibsDirectory,
131+
source.toString().substring(sourceLibsDirectory.length()));
132+
try {
133+
Files.copy(source, dest);
134+
} catch (IOException e) {
135+
LOG.warning("Unable to copy contents of " + sourceLibsDirectory);
136+
}
137+
});
138+
} catch (Exception e) {
139+
LOG.warning("unable to copy jar files");
140+
throw e;
141+
}
142+
}
69143
}

‎plugins/core-plugin/src/main/java/com/google/cloud/teleport/plugin/TemplateSpecsGenerator.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -147,15 +147,16 @@ public File saveCommandSpec(TemplateDefinitions definition, File targetDirectory
147147

148148
String containerName = templateAnnotation.flexContainerName();
149149

150-
if (definition.getTemplateAnnotation().type() == TemplateType.JAVA) {
150+
if (definition.getTemplateAnnotation().type() == TemplateType.JAVA
151+
|| definition.getTemplateAnnotation().type() == TemplateType.XLANG) {
151152
writer.write(
152153
"{\n"
153154
+ " \"mainClass\": \""
154155
+ definition.getTemplateClass().getName()
155156
+ "\",\n"
156157
+ " \"classPath\": \"/template/"
157158
+ containerName
158-
+ "/libs/conscrypt-openjdk-uber-*.jar:/template/"
159+
+ "/libs/conscrypt-openjdk-uber.jar:/template/"
159160
+ containerName
160161
+ "/libs/*:/template/"
161162
+ containerName

‎plugins/core-plugin/src/main/java/com/google/cloud/teleport/plugin/model/TemplateDefinitions.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,14 @@ public ImageSpec buildSpecModel(boolean validateFlag) {
9999
ImageSpec imageSpec = new ImageSpec();
100100

101101
SdkInfo sdkInfo = new SdkInfo();
102-
sdkInfo.setLanguage(templateAnnotation.type().toString());
102+
103+
// Xlang templates require the java language.
104+
if (templateAnnotation.type() == TemplateType.XLANG) {
105+
sdkInfo.setLanguage("JAVA");
106+
} else {
107+
sdkInfo.setLanguage(templateAnnotation.type().toString());
108+
}
109+
103110
imageSpec.setSdkInfo(sdkInfo);
104111

105112
ImageSpecMetadata metadata = new ImageSpecMetadata();

‎plugins/core-plugin/src/main/resources/Dockerfile-xlang-template‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#========================================================================#
2+
# Create build environment from base Python 3.11 template launcher image #
3+
#========================================================================#
4+
FROM gcr.io/dataflow-templates-base/java11-template-launcher-base-distroless as java-base
5+
6+
7+
#============================================================#
8+
# Create Distroless xlang image compatible with YamlTemplate #
9+
#============================================================#
10+
FROM gcr.io/dataflow-templates-base/python311-template-launcher-base:latest
11+
12+
13+
ARG REQUIREMENTS_FILE=requirements.txt
14+
15+
COPY --from=busybox:1.35.0-uclibc /bin/sh /bin/sh
16+
# Copy template files to /template
17+
ARG WORKDIR=/template
18+
RUN mkdir -p $WORKDIR
19+
COPY $REQUIREMENTS_FILE /template
20+
COPY ${containerName}/ /template/${containerName}/
21+
COPY ${containerName}-generated-metadata.json /template
22+
23+
WORKDIR $WORKDIR
24+
25+
# Set environment variables
26+
ENV DATAFLOW_JAVA_COMMAND_SPEC=${commandSpec}
27+
ENV FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE=requirements.txt
28+
29+
ENV SITE_PACKAGES=/root/.apache_beam/cache/venvs/py-3.11-beam-2.54.0-da39a3ee5e6b4b0d3255bfef95601890afd80709
30+
RUN python -m venv $SITE_PACKAGES
31+
32+
# pip install dependencies and cache wheels
33+
RUN source $SITE_PACKAGES/bin/activate \
34+
&& pip install --no-cache-dir --upgrade pip \
35+
&& pip install --no-cache-dir --upgrade setuptools \
36+
&& pip install --no-cache-dir pyparsing==2.4.2 \
37+
&& pip install --no-cache-dir -U -r $REQUIREMENTS_FILE \
38+
&& pip download --no-cache-dir --dest /tmp/dataflow-requirements-cache -r $REQUIREMENTS_FILE
39+
40+
COPY --from=java-base /usr/local/jdk-11.0.20+8 /usr/local/jdk-11.0.20+8
41+
RUN ln -s /usr/local/jdk-11.0.20+8/bin/java /usr/local/bin/java
42+
COPY --from=java-base /opt/google/dataflow/java_template_launcher /opt/google/dataflow/java_template_launcher
43+
COPY --from=java-base /usr/licenses/LICENSE-java11 /usr/licenses/LICENSE-java11
44+
45+
WORKDIR /template
46+
47+
ENV PIP_NO_DEPS=True
48+
ENTRYPOINT ["/opt/google/dataflow/java_template_launcher"]

0 commit comments

Comments
 (0)