[fix][plugin]: fix spark sql use custom parma. (#10821)

huangchenguang123 · huaangcg · web-flow · commit 427c58546c93 · 2022-07-13T11:44:13.000+08:00
* [fix][plugin]: fix spark sql use custom parma.

* [fix][plugin] roll back part of the code

Co-authored-by: huaangcg &lt;huangcg@getui.com&gt;
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java
@@ -30,6 +30,7 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ArrayNode;
@@ -90,9 +91,8 @@ public Map<String, Property> getLocalParametersMap() {
     public Map<String, Property> getInputLocalParametersMap() {
         Map<String, Property> localParametersMaps = new LinkedHashMap<>();
         if (localParams != null) {
-
             for (Property property : localParams) {
-                if (property.getDirect().equals(Direct.IN)) {
+                if (Objects.equals(Direct.IN, property.getDirect())) {
                     localParametersMaps.put(property.getProp(), property);
                 }
             }
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java
@@ -229,7 +229,7 @@ private String generateScriptFile() {
         Path path = file.toPath();
 
         if (!Files.exists(path)) {
-            String script = sparkParameters.getRawScript().replaceAll("\\r\\n", "\n");
+            String script = replaceParam(sparkParameters.getRawScript());
             sparkParameters.setRawScript(script);
 
             logger.info("raw script : {}", sparkParameters.getRawScript());
@@ -255,6 +255,14 @@ private String generateScriptFile() {
         return scriptFileName;
     }
 
+    private String replaceParam(String script) {
+        script = script.replaceAll("\\r\\n", "\n");
+        // replace placeholder, and combining local and global parameters
+        Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap();
+        script = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap));
+        return script;
+    }
+
     @Override
     protected void setMainJarName() {
         // main jar
diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-spark.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-spark.ts
@@ -130,7 +130,7 @@ export function useSpark(model: { [field: string]: any }): IJsonItem[] {
       }
     },
     useResources(),
-    ...useCustomParams({ model, field: 'localParams', isSimple: true })
+    ...useCustomParams({ model, field: 'localParams', isSimple: false })
   ]
 }
 

Original file line number	Diff line number	Diff line change
`@@ -130,7 +130,7 @@ export function useSpark(model: { [field: string]: any }): IJsonItem[] {`
`130`	`130`	`}`
`131`	`131`	`},`
`132`	`132`	`useResources(),`
`133`		`- ...useCustomParams({ model, field: 'localParams', isSimple: true })`
	`133`	`+ ...useCustomParams({ model, field: 'localParams', isSimple: false })`
`134`	`134`	`]`
`135`	`135`	`}`
`136`	`136`