Skip to content

Commit b84b33a

Browse files
jieguangzhoujieguangzhou
committed
Changing the parameter hierarchy and add meaningful key
1 parent 9c91e39 commit b84b33a

18 files changed

Lines changed: 269 additions & 333 deletions

‎dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/Condition.yaml‎

Lines changed: 22 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -15,46 +15,29 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717

18-
# Define the process
19-
process:
18+
# Define the workflow
19+
workflow:
2020
name: "Condition"
2121

22-
# Define the tasks under the process
22+
# Define the tasks under the workflow
2323
tasks:
24-
- {
25-
"task_type": "Shell",
26-
"params": { "name": "pre_task_1", "command": "echo pre_task_1" },
27-
}
28-
- {
29-
"task_type": "Shell",
30-
"params": { "name": "pre_task_2", "command": "echo pre_task_2" },
31-
}
32-
- {
33-
"task_type": "Shell",
34-
"params": { "name": "pre_task_3", "command": "echo pre_task_3" },
35-
}
36-
- {
37-
"task_type": "Shell",
38-
"params": { "name": "success_branch", "command": "echo success_branch" },
39-
}
40-
- {
41-
"task_type": "Shell",
42-
"params": { "name": "fail_branch", "command": "echo fail_branch" },
43-
}
24+
- { "task_type": "Shell", "name": "pre_task_1", "command": "echo pre_task_1" }
25+
- { "task_type": "Shell", "name": "pre_task_2", "command": "echo pre_task_2" }
26+
- { "task_type": "Shell", "name": "pre_task_3", "command": "echo pre_task_3" }
27+
- { "task_type": "Shell", "name": "success_branch", "command": "echo success_branch" }
28+
- { "task_type": "Shell", "name": "fail_branch", "command": "echo fail_branch" }
29+
4430
- task_type: Condition
45-
params:
46-
name: condition
47-
success_task: success_branch
48-
failed_task: fail_branch
49-
OP: AND
50-
groups:
51-
- OP: AND
52-
groups:
53-
- [pre_task_1, true]
54-
- [pre_task_2, true]
55-
- [pre_task_3, false]
56-
- OP: AND
57-
groups:
58-
- [pre_task_1, false]
59-
- [pre_task_2, true]
60-
- [pre_task_3, true]
31+
name: condition
32+
success_task: success_branch
33+
failed_task: fail_branch
34+
op: AND
35+
groups:
36+
- op: AND
37+
groups:
38+
- task: pre_task_1
39+
flag: true
40+
- task: pre_task_2
41+
flag: true
42+
- task: pre_task_3
43+
flag: false

‎dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/DataX.yaml‎

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -15,21 +15,19 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717

18-
# Define the process
19-
process:
18+
# Define the workflow
19+
workflow:
2020
name: "DataX"
2121

22-
# Define the tasks under the process
22+
# Define the tasks under the workflow
2323
tasks:
2424
- task_type: DataX
25-
params:
26-
name: task
27-
datasource_name: db
28-
datatarget_name: db
29-
sql: show tables;
30-
target_table: table_test
25+
name: task
26+
datasource_name: db
27+
datatarget_name: db
28+
sql: show tables;
29+
target_table: table_test
3130

3231
- task_type: CustomDataX
33-
params:
34-
name: task_custon_config
35-
json: $FILE{"example_datax.json"}
32+
name: task_custon_config
33+
json: $FILE{"example_datax.json"}

‎dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/Dependent.yaml‎

Lines changed: 53 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -15,62 +15,62 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717

18-
process:
18+
workflow:
1919
name: "Dependent"
2020

21-
# Define the tasks under the process
21+
# Define the tasks under the workflow
2222
tasks:
2323
- task_type: Dependent
24-
params:
25-
name: dependent
26-
denpendence:
27-
OP: AND
28-
groups:
29-
- OP: Or
30-
groups:
31-
- [project-pydolphin, task_dependent_external, task_1]
32-
- [project-pydolphin, task_dependent_external, task_2]
33-
- OP: And
34-
groups:
35-
- [
36-
project-pydolphin,
37-
task_dependent_external,
38-
task_1,
39-
LAST_WEDNESDAY,
40-
]
41-
- [project-pydolphin, task_dependent_external, task_2, last24Hours]
24+
name: dependent
25+
denpendence:
26+
op: and
27+
groups:
28+
- op: or
29+
groups:
30+
- project_name: pydolphin
31+
process_definition_name: task_dependent_external
32+
dependent_task_name: task_1
33+
34+
- project_name: pydolphin
35+
process_definition_name: task_dependent_external
36+
dependent_task_name: task_2
37+
38+
- op: and
39+
groups:
40+
- project_name: pydolphin
41+
process_definition_name: task_dependent_external
42+
dependent_task_name: task_1
43+
dependent_date: LAST_WEDNESDAY
44+
45+
- project_name: pydolphin
46+
process_definition_name: task_dependent_external
47+
dependent_task_name: task_2
48+
dependent_date: last24Hours
4249

4350
- task_type: Dependent
44-
params:
45-
name: dependent_var
46-
denpendence:
47-
OP: AND
48-
groups:
49-
- OP: Or
50-
# we can use ${CONFIG.WORKFLOW_PROJECT} to set the value to configuration.WORKFLOW_PROJECT
51-
# we can use $PROCESS{"Dependent_External.yaml"} to create or update a process from dependent_external.yaml and set the value to that process name
52-
groups:
53-
- [
54-
"${CONFIG.WORKFLOW_PROJECT}",
55-
'$PROCESS{"Dependent_External.yaml"}',
56-
task_1,
57-
]
58-
- [
59-
"${CONFIG.WORKFLOW_PROJECT}",
60-
'$PROCESS{"Dependent_External.yaml"}',
61-
task_2,
62-
]
63-
- OP: And
64-
groups:
65-
- [
66-
"${CONFIG.WORKFLOW_PROJECT}",
67-
'$PROCESS{"Dependent_External.yaml"}',
68-
task_1,
69-
LAST_WEDNESDAY,
70-
]
71-
- [
72-
"${CONFIG.WORKFLOW_PROJECT}",
73-
'$PROCESS{"Dependent_External.yaml"}',
74-
task_2,
75-
last24Hours,
76-
]
51+
name: dependent_var
52+
denpendence:
53+
op: and
54+
groups:
55+
- op: or
56+
# we can use ${CONFIG.WORKFLOW_PROJECT} to set the value to configuration.WORKFLOW_PROJECT
57+
# we can use $WORKFLOW{"Dependent_External.yaml"} to create or update a workflow from dependent_external.yaml and set the value to that workflow name
58+
groups:
59+
- project_name: ${CONFIG.WORKFLOW_PROJECT}
60+
process_definition_name: $WORKFLOW{"Dependent_External.yaml"}
61+
dependent_task_name: task_1
62+
63+
- project_name: ${CONFIG.WORKFLOW_PROJECT}
64+
process_definition_name: $WORKFLOW{"Dependent_External.yaml"}
65+
dependent_task_name: task_2
66+
- op: and
67+
groups:
68+
- project_name: ${CONFIG.WORKFLOW_PROJECT}
69+
process_definition_name: $WORKFLOW{"Dependent_External.yaml"}
70+
dependent_task_name: task_1
71+
dependent_date: LAST_WEDNESDAY
72+
73+
- project_name: ${CONFIG.WORKFLOW_PROJECT}
74+
process_definition_name: $WORKFLOW{"Dependent_External.yaml"}
75+
dependent_task_name: task_2
76+
dependent_date: last24Hours

‎dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/Dependent_External.yaml‎

Lines changed: 6 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -15,21 +15,12 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717

18-
# Define the process
19-
process:
18+
# Define the workflow
19+
workflow:
2020
name: "task_dependent_external"
2121

22-
# Define the tasks under the process
22+
# Define the tasks under the workflow
2323
tasks:
24-
- {
25-
"task_type": "Shell",
26-
"params": { "name": "task_1", "command": "echo task 1" },
27-
}
28-
- {
29-
"task_type": "Shell",
30-
"params": { "name": "task_2", "command": "echo task 2" },
31-
}
32-
- {
33-
"task_type": "Shell",
34-
"params": { "name": "task_3", "command": "echo task 3" },
35-
}
24+
- { "task_type": "Shell", "name": "task_1", "command": "echo task 1" }
25+
- { "task_type": "Shell", "name": "task_2", "command": "echo task 2" }
26+
- { "task_type": "Shell", "name": "task_3", "command": "echo task 3" }

‎dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/Flink.yaml‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,15 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717

18-
# Define the process
19-
process:
18+
# Define the workflow
19+
workflow:
2020
name: "Flink"
2121

22-
# Define the tasks under the process
22+
# Define the tasks under the workflow
2323
tasks:
2424
- task_type: Flink
25-
params:
26-
name: task
27-
main_class: org.apache.flink.streaming.examples.wordcount.WordCount
28-
main_package: test_java.jar
29-
program_type: JAVA
30-
deploy_mode: local
25+
name: task
26+
main_class: org.apache.flink.streaming.examples.wordcount.WordCount
27+
main_package: test_java.jar
28+
program_type: JAVA
29+
deploy_mode: local

‎dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/Http.yaml‎

Lines changed: 16 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -15,24 +15,23 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717

18-
# Define the process
19-
process:
18+
# Define the workflow
19+
workflow:
2020
name: "Http"
2121

22-
# Define the tasks under the process
22+
# Define the tasks under the workflow
2323
tasks:
2424
- task_type: Http
25-
params:
26-
name: task
27-
url: "https://httpbin.org/get"
28-
http_method: "GET"
29-
http_params:
30-
- { "prop": "a", "httpParametersType": "PARAMETER", "value": "1" }
31-
- { "prop": "b", "httpParametersType": "PARAMETER", "value": "2" }
32-
- {
33-
"prop": "Content-Type",
34-
"httpParametersType": "header",
35-
"value": "test",
36-
}
37-
http_check_condition: "STATUS_CODE_CUSTOM"
38-
condition: "404"
25+
name: task
26+
url: "https://httpbin.org/get"
27+
http_method: "GET"
28+
http_params:
29+
- { "prop": "a", "httpParametersType": "PARAMETER", "value": "1" }
30+
- { "prop": "b", "httpParametersType": "PARAMETER", "value": "2" }
31+
- {
32+
"prop": "Content-Type",
33+
"httpParametersType": "header",
34+
"value": "test",
35+
}
36+
http_check_condition: "STATUS_CODE_CUSTOM"
37+
condition: "404"

‎dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/MapReduce.yaml‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,15 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717

18-
# Define the process
19-
process:
18+
# Define the workflow
19+
workflow:
2020
name: "MapReduce"
2121

22-
# Define the tasks under the process
22+
# Define the tasks under the workflow
2323
tasks:
2424
- task_type: MR
25-
params:
26-
name: task
27-
main_class: wordcount
28-
main_package: test_java.jar
29-
program_type: SCALA
30-
main_args: /dolphinscheduler/tenant_exists/resources/file.txt /output/ds
25+
name: task
26+
main_class: wordcount
27+
main_package: test_java.jar
28+
program_type: SCALA
29+
main_args: /dolphinscheduler/tenant_exists/resources/file.txt /output/ds

‎dolphinscheduler-python/pydolphinscheduler/examples/yaml_define/MoreConfiguration.yaml‎

Lines changed: 17 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -15,27 +15,26 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717

18-
# Define the process
19-
process:
18+
# Define the workflow
19+
workflow:
2020
name: "MoreConfiguration"
2121
param:
2222
n: 1
2323

24-
# Define the tasks under the process
24+
# Define the tasks under the workflow
2525
tasks:
2626
- task_type: Shell
27-
params:
28-
name: shell_0
29-
description: "yaml define task"
30-
flag: "YES"
31-
command: |
32-
echo "$ENV{HOME}"
33-
echo "${n}"
34-
task_priority: "HIGH"
35-
delay_time: 20
36-
fail_retry_times: 30
37-
fail_retry_interval: 5
38-
timeout_flag: "CLOSE"
39-
timeout: 60
40-
local_params:
41-
- { "prop": "n", "direct": "IN", "type": "VARCHAR", "value": "${n}" }
27+
name: shell_0
28+
description: "yaml define task"
29+
flag: "YES"
30+
command: |
31+
echo "$ENV{HOME}"
32+
echo "${n}"
33+
task_priority: "HIGH"
34+
delay_time: 20
35+
fail_retry_times: 30
36+
fail_retry_interval: 5
37+
timeout_flag: "CLOSE"
38+
timeout: 60
39+
local_params:
40+
- { "prop": "n", "direct": "IN", "type": "VARCHAR", "value": "${n}" }

0 commit comments

Comments
 (0)