From d23fb7459713962184e6fcdc8736004da3cf197e Mon Sep 17 00:00:00 2001 From: Nikhil Bharadwaj Ramashasthri Date: Wed, 22 Jul 2026 05:40:34 -0700 Subject: [PATCH 1/6] [Fix-18389][DataX] Read job definition from attached resource file when custom json is empty --- .../plugin/task/datax/DataxParameters.java | 5 +++- .../plugin/task/datax/DataxTask.java | 24 ++++++++++++++++++- .../task/datax/DataxParametersTest.java | 22 +++++++++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java index 07c20c320392..8706ca3d916c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; import org.apache.dolphinscheduler.spi.enums.Flag; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import java.util.List; @@ -116,7 +117,9 @@ public boolean checkParameters() { && StringUtils.isNotEmpty(sql) && StringUtils.isNotEmpty(targetTable); } else { - return StringUtils.isNotEmpty(json); + // Custom config is valid with either inline json or an attached resource file + // carrying the job definition (issue #18389) + return StringUtils.isNotEmpty(json) || CollectionUtils.isNotEmpty(resourceList); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java index e82666d53006..32405b7e537e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java @@ -32,6 +32,7 @@ import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; +import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext; import org.apache.dolphinscheduler.plugin.task.api.shell.IShellInterceptorBuilder; import org.apache.dolphinscheduler.plugin.task.api.shell.ShellInterceptorBuilderFactory; import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; @@ -168,6 +169,18 @@ public void cancel() throws TaskException { } } + /** + * Reads the DataX job definition from the first attached resource file. The worker has + * already downloaded resources into the execution directory by the time the task runs. + */ + private String readJsonFromResourceFile() throws Exception { + String resourceFileName = dataXParameters.getResourceList().get(0).getResourceName(); + ResourceContext resourceContext = taskRequest.getResourceContext(); + return FileUtils.readFileToString( + new File(resourceContext.getResourceItem(resourceFileName).getResourceAbsolutePathInLocal()), + StandardCharsets.UTF_8); + } + /** * build datax configuration file * @@ -185,7 +198,16 @@ private String buildDataxJsonFile(Map paramsMap) throws Except } if (dataXParameters.getCustomConfig() == Flag.YES.ordinal()) { - json = dataXParameters.getJson().replaceAll("\\r\\n", System.lineSeparator()); + // An attached resource file is a valid way to supply the job definition. Without + // this branch the worker downloads the resource but the plugin runs with the empty + // inline json and the job fails (issue #18389). + if (StringUtils.isEmpty(dataXParameters.getJson()) + && CollectionUtils.isNotEmpty(dataXParameters.getResourceList())) { + json = readJsonFromResourceFile(); + } else { + json = dataXParameters.getJson(); + } + json = json.replaceAll("\\r\\n", System.lineSeparator()); } else { ObjectNode job = JSONUtils.createObjectNode(); job.putArray("content").addAll(buildDataxJobContentJson()); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParametersTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParametersTest.java index 736d0aab941e..f352545667d1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParametersTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParametersTest.java @@ -32,6 +32,28 @@ public class DataxParametersTest { */ public static final String JVM_PARAM = " --jvm=\"-Xms%sG -Xmx%sG\" "; + @Test + public void testCheckParametersWithCustomConfig() { + DataxParameters withInlineJson = new DataxParameters(); + withInlineJson.setCustomConfig(1); + withInlineJson.setJson("{\"job\":{}}"); + Assertions.assertTrue(withInlineJson.checkParameters()); + + // an attached resource file is a valid alternative to inline json (issue #18389) + DataxParameters withResourceFile = new DataxParameters(); + withResourceFile.setCustomConfig(1); + ResourceInfo resource = new ResourceInfo(); + resource.setResourceName("/datax/job.json"); + List resources = new ArrayList<>(); + resources.add(resource); + withResourceFile.setResourceList(resources); + Assertions.assertTrue(withResourceFile.checkParameters()); + + DataxParameters withNeither = new DataxParameters(); + withNeither.setCustomConfig(1); + Assertions.assertFalse(withNeither.checkParameters()); + } + @Test public void testLoadJvmEnv() { From 0d5af8f16cd89f1922858d9f2b98574e2f89d0c3 Mon Sep 17 00:00:00 2001 From: Nikhil Bharadwaj Ramashasthri Date: Thu, 23 Jul 2026 11:52:14 -0700 Subject: [PATCH 2/6] [Fix-18389][DataX] Address review: treat {} placeholder as absent, align UI validation, add task-level regression test --- .../plugin/task/datax/DataxParameters.java | 6 +- .../plugin/task/datax/DataxTask.java | 14 ++++- .../plugin/task/datax/DataxTaskTest.java | 57 +++++++++++++++++++ .../task/components/node/fields/use-datax.ts | 9 ++- 4 files changed, 80 insertions(+), 6 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java index 8706ca3d916c..681f920b712f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java @@ -118,8 +118,10 @@ public boolean checkParameters() { && StringUtils.isNotEmpty(targetTable); } else { // Custom config is valid with either inline json or an attached resource file - // carrying the job definition (issue #18389) - return StringUtils.isNotEmpty(json) || CollectionUtils.isNotEmpty(resourceList); + // carrying the job definition (issue #18389). "{}" is the UI placeholder and + // does not count as an inline definition. + boolean hasInlineJson = StringUtils.isNotBlank(json) && !"{}".equals(json.trim()); + return hasInlineJson || CollectionUtils.isNotEmpty(resourceList); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java index 32405b7e537e..a70ed60fa3dc 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java @@ -169,6 +169,15 @@ public void cancel() throws TaskException { } } + /** + * Returns true when no usable inline job definition was provided. The UI historically + * stored the placeholder {@code {}} in the json field, so a blank value and the empty + * object placeholder are both treated as absent. + */ + static boolean isInlineJsonAbsent(String json) { + return StringUtils.isBlank(json) || "{}".equals(json.trim()); + } + /** * Reads the DataX job definition from the first attached resource file. The worker has * already downloaded resources into the execution directory by the time the task runs. @@ -200,8 +209,9 @@ private String buildDataxJsonFile(Map paramsMap) throws Except if (dataXParameters.getCustomConfig() == Flag.YES.ordinal()) { // An attached resource file is a valid way to supply the job definition. Without // this branch the worker downloads the resource but the plugin runs with the empty - // inline json and the job fails (issue #18389). - if (StringUtils.isEmpty(dataXParameters.getJson()) + // inline json and the job fails (issue #18389). Existing tasks created through the + // UI carry "{}" as a placeholder, treat it the same as no inline json. + if (isInlineJsonAbsent(dataXParameters.getJson()) && CollectionUtils.isNotEmpty(dataXParameters.getResourceList())) { json = readJsonFromResourceFile(); } else { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java index 17687e15c117..1159e06c3caa 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java @@ -37,8 +37,10 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.TaskRunStatus; import org.apache.dolphinscheduler.plugin.task.api.model.ApplicationInfo; import org.apache.dolphinscheduler.plugin.task.api.model.Property; +import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; +import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext; import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; import org.apache.dolphinscheduler.spi.enums.DbType; @@ -271,6 +273,61 @@ private Map createPrepareParamsMap() { return paramsMap; } + @Test + public void testCustomConfigReadsJobDefinitionFromResourceFile() throws Exception { + // a real resource file carrying the job definition, with the UI placeholder "{}" inline + String resourceJson = "{\"job\":{\"content\":[{\"reader\":{\"name\":\"mysqlreader\"}}]}}"; + File resourceFile = File.createTempFile("datax-job", ".json"); + resourceFile.deleteOnExit(); + java.nio.file.Files.write(resourceFile.toPath(), + resourceJson.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + + DataxParameters parameters = new DataxParameters(); + parameters.setCustomConfig(1); + parameters.setJson("{}"); + parameters.setXms(1); + parameters.setXmx(1); + ResourceInfo resourceInfo = new ResourceInfo(); + resourceInfo.setResourceName("/datax/job.json"); + parameters.setResourceList(java.util.Collections.singletonList(resourceInfo)); + + TaskExecutionContext taskExecutionContext = buildTestTaskExecutionContext(); + // own app id so the generated file cannot collide with other tests' job files + taskExecutionContext.setTaskAppId("app-id-resource"); + taskExecutionContext.setPrepareParamsMap(null); + taskExecutionContext.setTaskParams(JSONUtils.toJsonString(parameters)); + ResourceContext resourceContext = new ResourceContext(); + resourceContext.addResourceItem(ResourceContext.ResourceItem.builder() + .resourceAbsolutePathInStorage("/datax/job.json") + .resourceAbsolutePathInLocal(resourceFile.getAbsolutePath()) + .build()); + taskExecutionContext.setResourceContext(resourceContext); + + DataxTask dataxTask = new DataxTask(taskExecutionContext); + dataxTask.init(); + + ShellCommandExecutor shellCommandExecutor = mock(ShellCommandExecutor.class); + Field shellCommandExecutorFiled = DataxTask.class.getDeclaredField("shellCommandExecutor"); + shellCommandExecutorFiled.setAccessible(true); + shellCommandExecutorFiled.set(dataxTask, shellCommandExecutor); + + TaskResponse taskResponse = new TaskResponse(); + taskResponse.setStatus(TaskRunStatus.SUCCESS); + taskResponse.setExitStatusCode(0); + taskResponse.setProcessId(1); + when(shellCommandExecutor.run(any(), eq(taskCallBack))).thenReturn(taskResponse); + + dataxTask.handle(taskCallBack); + Assertions.assertEquals(0, dataxTask.getExitStatusCode()); + + // the generated job file must carry the resource content, not the "{}" placeholder + File jsonFile = new File("/tmp/execution/app-id-resource_job.json"); + String generated = FileUtils.readFile2Str(Files.newInputStream(jsonFile.toPath())); + Assertions.assertTrue(generated.contains("mysqlreader"), + "generated job file should contain the resource file definition, was: " + generated); + Assertions.assertTrue(jsonFile.delete()); + } + private TaskExecutionContext buildTestTaskExecutionContext() { TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); taskExecutionContext.setTaskAppId("app-id"); diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts index 8b5c3ae934cf..7cebaf99b0f3 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts @@ -197,14 +197,19 @@ export function useDataX(model: { [field: string]: any }): IJsonItem[] { span: jsonEditorSpan, validate: { trigger: ['input', 'trigger'], - required: true, + required: false, validator() { + const hasResource = + model.resourceList && (model.resourceList as string[]).length > 0 if ( model.json === '' || model.json === undefined || model.json === null ) { - return new Error(t('project.node.sql_empty_tips')) + // an attached resource file may carry the job definition instead + return hasResource + ? undefined + : new Error(t('project.node.sql_empty_tips')) } if (!utils.isJson(model.json)) { return new Error(t('project.node.json_format_tips')) From 0b5687a501cd17f0cbab6c693f308b9d767f8988 Mon Sep 17 00:00:00 2001 From: Nikhil Bharadwaj Ramashasthri Date: Mon, 27 Jul 2026 19:50:09 -0700 Subject: [PATCH 3/6] [Fix-18389][DataX] Detect empty job json semantically and centralize the check in DataxParameters --- .../plugin/task/datax/DataxParameters.java | 28 ++++++++-- .../plugin/task/datax/DataxTask.java | 13 +---- .../task/datax/DataxParametersTest.java | 51 +++++++++++++++---- .../plugin/task/datax/DataxTaskTest.java | 5 +- .../task/components/node/fields/use-datax.ts | 12 +++++ 5 files changed, 83 insertions(+), 26 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java index 681f920b712f..524faac326cb 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java @@ -33,6 +33,8 @@ import lombok.Data; +import com.fasterxml.jackson.databind.node.ObjectNode; + /** * DataX parameter */ @@ -118,10 +120,28 @@ public boolean checkParameters() { && StringUtils.isNotEmpty(targetTable); } else { // Custom config is valid with either inline json or an attached resource file - // carrying the job definition (issue #18389). "{}" is the UI placeholder and - // does not count as an inline definition. - boolean hasInlineJson = StringUtils.isNotBlank(json) && !"{}".equals(json.trim()); - return hasInlineJson || CollectionUtils.isNotEmpty(resourceList); + // carrying the job definition (issue #18389). + return !isInlineJsonAbsent() || CollectionUtils.isNotEmpty(resourceList); + } + } + + /** + * Returns true when the json field carries no usable inline job definition. The UI + * historically stored an empty object placeholder in the json field, so a blank value + * and any semantically empty JSON object (for example {@code {}}, {@code { }} or a + * formatted multi-line empty object) are all treated as absent (issue #18389). + */ + public boolean isInlineJsonAbsent() { + if (StringUtils.isBlank(json)) { + return true; + } + try { + ObjectNode node = JSONUtils.parseObject(json); + return node == null || node.isEmpty(); + } catch (Exception e) { + // not parseable as a JSON object, so there is inline content: downstream + // validation reports the malformed definition + return false; } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java index a70ed60fa3dc..87372b579720 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java @@ -169,15 +169,6 @@ public void cancel() throws TaskException { } } - /** - * Returns true when no usable inline job definition was provided. The UI historically - * stored the placeholder {@code {}} in the json field, so a blank value and the empty - * object placeholder are both treated as absent. - */ - static boolean isInlineJsonAbsent(String json) { - return StringUtils.isBlank(json) || "{}".equals(json.trim()); - } - /** * Reads the DataX job definition from the first attached resource file. The worker has * already downloaded resources into the execution directory by the time the task runs. @@ -210,8 +201,8 @@ private String buildDataxJsonFile(Map paramsMap) throws Except // An attached resource file is a valid way to supply the job definition. Without // this branch the worker downloads the resource but the plugin runs with the empty // inline json and the job fails (issue #18389). Existing tasks created through the - // UI carry "{}" as a placeholder, treat it the same as no inline json. - if (isInlineJsonAbsent(dataXParameters.getJson()) + // UI carry an empty object placeholder, treat it the same as no inline json. + if (dataXParameters.isInlineJsonAbsent() && CollectionUtils.isNotEmpty(dataXParameters.getResourceList())) { json = readJsonFromResourceFile(); } else { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParametersTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParametersTest.java index f352545667d1..65e71f64cb12 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParametersTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParametersTest.java @@ -39,21 +39,54 @@ public void testCheckParametersWithCustomConfig() { withInlineJson.setJson("{\"job\":{}}"); Assertions.assertTrue(withInlineJson.checkParameters()); - // an attached resource file is a valid alternative to inline json (issue #18389) - DataxParameters withResourceFile = new DataxParameters(); - withResourceFile.setCustomConfig(1); - ResourceInfo resource = new ResourceInfo(); - resource.setResourceName("/datax/job.json"); - List resources = new ArrayList<>(); - resources.add(resource); - withResourceFile.setResourceList(resources); - Assertions.assertTrue(withResourceFile.checkParameters()); + // a blank json field or any semantically empty JSON object is no inline + // definition: invalid without a resource file, valid with one because the + // resource then carries the job definition (issue #18389) + String[] absentJsonVariants = {null, "", " ", "{}", "{ }", "{\n\n}", " { } "}; + for (String variant : absentJsonVariants) { + DataxParameters withoutResource = new DataxParameters(); + withoutResource.setCustomConfig(1); + withoutResource.setJson(variant); + Assertions.assertTrue(withoutResource.isInlineJsonAbsent(), + "expected inline json to be absent for: [" + variant + "]"); + Assertions.assertFalse(withoutResource.checkParameters(), + "expected invalid without resource for json: [" + variant + "]"); + + DataxParameters withResource = new DataxParameters(); + withResource.setCustomConfig(1); + withResource.setJson(variant); + withResource.setResourceList(buildResourceList()); + Assertions.assertTrue(withResource.checkParameters(), + "expected valid with resource for json: [" + variant + "]"); + } + + // a non-empty inline definition stays inline even when a resource is attached + DataxParameters inlineWithResource = new DataxParameters(); + inlineWithResource.setCustomConfig(1); + inlineWithResource.setJson("{\"job\":{}}"); + inlineWithResource.setResourceList(buildResourceList()); + Assertions.assertFalse(inlineWithResource.isInlineJsonAbsent()); + Assertions.assertTrue(inlineWithResource.checkParameters()); + + // malformed json is not treated as absent, downstream validation reports it + DataxParameters malformed = new DataxParameters(); + malformed.setCustomConfig(1); + malformed.setJson("{invalid"); + Assertions.assertFalse(malformed.isInlineJsonAbsent()); DataxParameters withNeither = new DataxParameters(); withNeither.setCustomConfig(1); Assertions.assertFalse(withNeither.checkParameters()); } + private List buildResourceList() { + ResourceInfo resource = new ResourceInfo(); + resource.setResourceName("/datax/job.json"); + List resources = new ArrayList<>(); + resources.add(resource); + return resources; + } + @Test public void testLoadJvmEnv() { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java index 1159e06c3caa..5b2799b5413e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java @@ -275,7 +275,8 @@ private Map createPrepareParamsMap() { @Test public void testCustomConfigReadsJobDefinitionFromResourceFile() throws Exception { - // a real resource file carrying the job definition, with the UI placeholder "{}" inline + // a real resource file carrying the job definition, with a formatted empty object + // placeholder inline (the semantic-absence rule, not a literal "{}" compare) String resourceJson = "{\"job\":{\"content\":[{\"reader\":{\"name\":\"mysqlreader\"}}]}}"; File resourceFile = File.createTempFile("datax-job", ".json"); resourceFile.deleteOnExit(); @@ -284,7 +285,7 @@ public void testCustomConfigReadsJobDefinitionFromResourceFile() throws Exceptio DataxParameters parameters = new DataxParameters(); parameters.setCustomConfig(1); - parameters.setJson("{}"); + parameters.setJson("{\n \n}"); parameters.setXms(1); parameters.setXmx(1); ResourceInfo resourceInfo = new ResourceInfo(); diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts index 7cebaf99b0f3..76e656e7d23c 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts @@ -214,6 +214,18 @@ export function useDataX(model: { [field: string]: any }): IJsonItem[] { if (!utils.isJson(model.json)) { return new Error(t('project.node.json_format_tips')) } + // A semantically empty object ({}, { }, formatted) is the historical UI + // placeholder and does not count as an inline definition. Same rule as + // DataxParameters.isInlineJsonAbsent on the backend. + const parsed = JSON.parse(model.json) + const isEmptyObject = + parsed !== null && + typeof parsed === 'object' && + !Array.isArray(parsed) && + Object.keys(parsed).length === 0 + if (isEmptyObject && !hasResource) { + return new Error(t('project.node.sql_empty_tips')) + } } } }, From 6c6c9190eb0a9d4fe9ad0b2afb3c15d086f075a7 Mon Sep 17 00:00:00 2001 From: Nikhil Bharadwaj Ramashasthri Date: Mon, 10 Aug 2026 19:54:37 -0700 Subject: [PATCH 4/6] [Fix-18389][DataX] Identify the job definition as the single .json resource, not resourceList.get(0) resourceList is multi-select and also carries auxiliary files such as Kerberos keytabs and xml configs. Taking resourceList.get(0) as the DataX job definition could read a keytab as the job when it is listed first, and a task carrying only auxiliary resources passed validation with no job definition at all. The job definition is now identified as the single resource whose name ends with .json. Validation rejects the no-json and ambiguous multi-json cases, and the worker reads that designated resource, failing loudly if it is absent. Added coverage for a keytab before the job file, multiple json resources, and an auxiliary-only list. --- .../plugin/task/datax/DataxParameters.java | 28 +++++++- .../plugin/task/datax/DataxTask.java | 19 ++++-- .../task/datax/DataxParametersTest.java | 47 +++++++++++++ .../plugin/task/datax/DataxTaskTest.java | 66 +++++++++++++++++++ 4 files changed, 151 insertions(+), 9 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java index 524faac326cb..b7e51d44d154 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Objects; +import java.util.stream.Collectors; import lombok.Data; @@ -119,12 +120,33 @@ public boolean checkParameters() { && StringUtils.isNotEmpty(sql) && StringUtils.isNotEmpty(targetTable); } else { - // Custom config is valid with either inline json or an attached resource file - // carrying the job definition (issue #18389). - return !isInlineJsonAbsent() || CollectionUtils.isNotEmpty(resourceList); + // Custom config is valid with either inline json or an attached resource file that + // unambiguously carries the job definition, identified as the single .json resource + // (issue #18389). resourceList is multi-select and also holds auxiliary files, so a + // non-empty list on its own is not enough. + return !isInlineJsonAbsent() || getJobDefinitionResource() != null; } } + /** + * When the inline json is absent the job definition must come from an attached resource file. + * resourceList is multi-select and also carries auxiliary files such as Kerberos keytabs and + * xml configs, so the job definition is identified as the single resource whose name ends with + * {@code .json} rather than the first entry in the list (issue #18389). Returns that resource, + * or {@code null} when there is not exactly one json resource, which the caller treats as a + * missing or ambiguous job definition. + */ + public ResourceInfo getJobDefinitionResource() { + if (CollectionUtils.isEmpty(resourceList)) { + return null; + } + List jsonResources = resourceList.stream() + .filter(Objects::nonNull) + .filter(resource -> StringUtils.endsWithIgnoreCase(resource.getResourceName(), ".json")) + .collect(Collectors.toList()); + return jsonResources.size() == 1 ? jsonResources.get(0) : null; + } + /** * Returns true when the json field carries no usable inline job definition. The UI * historically stored an empty object placeholder in the json field, so a blank value diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java index 87372b579720..021fa783c0e2 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java @@ -30,6 +30,7 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.log.SensitiveDataConverter; import org.apache.dolphinscheduler.plugin.task.api.model.Property; +import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext; @@ -170,11 +171,11 @@ public void cancel() throws TaskException { } /** - * Reads the DataX job definition from the first attached resource file. The worker has + * Reads the DataX job definition from the designated json resource file. The worker has * already downloaded resources into the execution directory by the time the task runs. */ - private String readJsonFromResourceFile() throws Exception { - String resourceFileName = dataXParameters.getResourceList().get(0).getResourceName(); + private String readJsonFromResourceFile(ResourceInfo jobResource) throws Exception { + String resourceFileName = jobResource.getResourceName(); ResourceContext resourceContext = taskRequest.getResourceContext(); return FileUtils.readFileToString( new File(resourceContext.getResourceItem(resourceFileName).getResourceAbsolutePathInLocal()), @@ -202,9 +203,15 @@ private String buildDataxJsonFile(Map paramsMap) throws Except // this branch the worker downloads the resource but the plugin runs with the empty // inline json and the job fails (issue #18389). Existing tasks created through the // UI carry an empty object placeholder, treat it the same as no inline json. - if (dataXParameters.isInlineJsonAbsent() - && CollectionUtils.isNotEmpty(dataXParameters.getResourceList())) { - json = readJsonFromResourceFile(); + if (dataXParameters.isInlineJsonAbsent()) { + // the job definition is the single attached .json resource, never the first + // entry in resourceList, which may be an auxiliary keytab or xml (issue #18389) + ResourceInfo jobResource = dataXParameters.getJobDefinitionResource(); + if (jobResource == null) { + throw new TaskException( + "DataX job definition is missing, provide inline json or attach exactly one .json resource file"); + } + json = readJsonFromResourceFile(jobResource); } else { json = dataXParameters.getJson(); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParametersTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParametersTest.java index 65e71f64cb12..fc161abb2326 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParametersTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParametersTest.java @@ -87,6 +87,53 @@ private List buildResourceList() { return resources; } + @Test + public void testJobDefinitionResourceIsTheSingleJsonResource() { + // resourceList is multi-select and also carries auxiliary files, so the job definition + // is identified as the single .json resource, not resourceList.get(0) (issue #18389) + + // only an auxiliary keytab and no json: no job definition, invalid + DataxParameters onlyAuxiliary = new DataxParameters(); + onlyAuxiliary.setCustomConfig(1); + onlyAuxiliary.setResourceList(resources("/datax/hdfs.keytab")); + Assertions.assertNull(onlyAuxiliary.getJobDefinitionResource()); + Assertions.assertFalse(onlyAuxiliary.checkParameters(), + "a task carrying only auxiliary resources has no job definition"); + + // a keytab listed before the job file: the .json is chosen, not the first entry + DataxParameters auxiliaryBeforeJob = new DataxParameters(); + auxiliaryBeforeJob.setCustomConfig(1); + auxiliaryBeforeJob.setResourceList(resources("/datax/hdfs.keytab", "/datax/job.json")); + Assertions.assertEquals("/datax/job.json", + auxiliaryBeforeJob.getJobDefinitionResource().getResourceName()); + Assertions.assertTrue(auxiliaryBeforeJob.checkParameters()); + + // two json resources are ambiguous: no single job definition, invalid + DataxParameters twoJson = new DataxParameters(); + twoJson.setCustomConfig(1); + twoJson.setResourceList(resources("/datax/a.json", "/datax/b.json")); + Assertions.assertNull(twoJson.getJobDefinitionResource()); + Assertions.assertFalse(twoJson.checkParameters()); + + // exactly one json resource is the job definition + DataxParameters singleJson = new DataxParameters(); + singleJson.setCustomConfig(1); + singleJson.setResourceList(resources("/datax/job.json")); + Assertions.assertEquals("/datax/job.json", + singleJson.getJobDefinitionResource().getResourceName()); + Assertions.assertTrue(singleJson.checkParameters()); + } + + private List resources(String... names) { + List list = new ArrayList<>(); + for (String name : names) { + ResourceInfo resource = new ResourceInfo(); + resource.setResourceName(name); + list.add(resource); + } + return list; + } + @Test public void testLoadJvmEnv() { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java index 5b2799b5413e..5d75150edf95 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java @@ -329,6 +329,72 @@ public void testCustomConfigReadsJobDefinitionFromResourceFile() throws Exceptio Assertions.assertTrue(jsonFile.delete()); } + @Test + public void testCustomConfigReadsJobFromJsonResourceNotFirstAuxiliaryResource() throws Exception { + // resourceList carries a keytab BEFORE the job file. The worker must read the .json job + // definition, not resourceList.get(0) which is the keytab (issue #18389, review by SbloodyS) + String jobJson = "{\"job\":{\"content\":[{\"reader\":{\"name\":\"mysqlreader\"}}]}}"; + File jobFile = File.createTempFile("datax-job", ".json"); + jobFile.deleteOnExit(); + java.nio.file.Files.write(jobFile.toPath(), + jobJson.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + File keytabFile = File.createTempFile("hdfs", ".keytab"); + keytabFile.deleteOnExit(); + java.nio.file.Files.write(keytabFile.toPath(), + "keytab-binary-not-json".getBytes(java.nio.charset.StandardCharsets.UTF_8)); + + DataxParameters parameters = new DataxParameters(); + parameters.setCustomConfig(1); + parameters.setJson("{}"); + parameters.setXms(1); + parameters.setXmx(1); + ResourceInfo keytab = new ResourceInfo(); + keytab.setResourceName("/datax/hdfs.keytab"); + ResourceInfo job = new ResourceInfo(); + job.setResourceName("/datax/job.json"); + parameters.setResourceList(java.util.Arrays.asList(keytab, job)); + + TaskExecutionContext taskExecutionContext = buildTestTaskExecutionContext(); + taskExecutionContext.setTaskAppId("app-id-multi-resource"); + taskExecutionContext.setPrepareParamsMap(null); + taskExecutionContext.setTaskParams(JSONUtils.toJsonString(parameters)); + ResourceContext resourceContext = new ResourceContext(); + resourceContext.addResourceItem(ResourceContext.ResourceItem.builder() + .resourceAbsolutePathInStorage("/datax/hdfs.keytab") + .resourceAbsolutePathInLocal(keytabFile.getAbsolutePath()) + .build()); + resourceContext.addResourceItem(ResourceContext.ResourceItem.builder() + .resourceAbsolutePathInStorage("/datax/job.json") + .resourceAbsolutePathInLocal(jobFile.getAbsolutePath()) + .build()); + taskExecutionContext.setResourceContext(resourceContext); + + DataxTask dataxTask = new DataxTask(taskExecutionContext); + dataxTask.init(); + + ShellCommandExecutor shellCommandExecutor = mock(ShellCommandExecutor.class); + Field shellCommandExecutorFiled = DataxTask.class.getDeclaredField("shellCommandExecutor"); + shellCommandExecutorFiled.setAccessible(true); + shellCommandExecutorFiled.set(dataxTask, shellCommandExecutor); + + TaskResponse taskResponse = new TaskResponse(); + taskResponse.setStatus(TaskRunStatus.SUCCESS); + taskResponse.setExitStatusCode(0); + taskResponse.setProcessId(1); + when(shellCommandExecutor.run(any(), eq(taskCallBack))).thenReturn(taskResponse); + + dataxTask.handle(taskCallBack); + Assertions.assertEquals(0, dataxTask.getExitStatusCode()); + + File jsonFile = new File("/tmp/execution/app-id-multi-resource_job.json"); + String generated = FileUtils.readFile2Str(Files.newInputStream(jsonFile.toPath())); + Assertions.assertTrue(generated.contains("mysqlreader"), + "generated job file should carry the .json resource content, was: " + generated); + Assertions.assertFalse(generated.contains("keytab-binary-not-json"), + "generated job file must not read the auxiliary keytab as the job definition"); + Assertions.assertTrue(jsonFile.delete()); + } + private TaskExecutionContext buildTestTaskExecutionContext() { TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); taskExecutionContext.setTaskAppId("app-id"); From ce13dca0f9e9d01844243335858f0ca1daad3037 Mon Sep 17 00:00:00 2001 From: Nikhil Bharadwaj Ramashasthri Date: Tue, 11 Aug 2026 00:44:20 -0700 Subject: [PATCH 5/6] [Fix-18389][UI] Require exactly one .json resource in the datax validator when inline json is absent The backend now identifies the datax job definition as the single .json resource in resourceList and rejects the no-json and ambiguous multi-json cases. The UI validator still accepted any non-empty resourceList, so an auxiliary-only or multi-json config passed UI validation and was only rejected later at worker init with a generic error. Apply the same rule in the UI json validator: when the inline json is absent, require exactly one resource whose name ends with .json. Add an explicit message key in the en and zh locales. --- .../src/locales/en_US/project.ts | 2 ++ .../src/locales/zh_CN/project.ts | 2 ++ .../task/components/node/fields/use-datax.ts | 22 +++++++++++++------ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/dolphinscheduler-ui/src/locales/en_US/project.ts b/dolphinscheduler-ui/src/locales/en_US/project.ts index 534c11915b84..cc88f08190a0 100644 --- a/dolphinscheduler-ui/src/locales/en_US/project.ts +++ b/dolphinscheduler-ui/src/locales/en_US/project.ts @@ -645,6 +645,8 @@ export default { or: 'or', datax_custom_template: 'Custom Template', datax_json_template: 'JSON', + datax_custom_json_resource_tips: + 'When the custom JSON is empty, attach exactly one .json resource file that carries the DataX job definition.', datax_target_datasource_type: 'Target Datasource Types', datax_target_database: 'Target Database', datax_target_table: 'Target Table', diff --git a/dolphinscheduler-ui/src/locales/zh_CN/project.ts b/dolphinscheduler-ui/src/locales/zh_CN/project.ts index 429fd4990bf7..24552393a40c 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/project.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/project.ts @@ -626,6 +626,8 @@ export default { or: '或', datax_custom_template: '自定义模板', datax_json_template: 'JSON', + datax_custom_json_resource_tips: + '当自定义 JSON 为空时, 需要且仅需要附加一个携带 DataX 任务定义的 .json 资源文件。', datax_target_datasource_type: '目标源类型', datax_target_database: '目标源实例', datax_target_table: '目标表', diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts index 76e656e7d23c..db556971bc8b 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts @@ -199,17 +199,25 @@ export function useDataX(model: { [field: string]: any }): IJsonItem[] { trigger: ['input', 'trigger'], required: false, validator() { - const hasResource = - model.resourceList && (model.resourceList as string[]).length > 0 + // When the inline json is absent the job definition must come from exactly one + // attached .json resource. resourceList is multi-select and also carries auxiliary + // files such as keytabs and xml, so the same rule as the backend + // DataxParameters.getJobDefinitionResource applies here (issue #18389). + const resourceList = (model.resourceList as string[]) || [] + const hasSingleJsonResource = + resourceList.filter( + (fullName) => + typeof fullName === 'string' && + fullName.toLowerCase().endsWith('.json') + ).length === 1 if ( model.json === '' || model.json === undefined || model.json === null ) { - // an attached resource file may carry the job definition instead - return hasResource + return hasSingleJsonResource ? undefined - : new Error(t('project.node.sql_empty_tips')) + : new Error(t('project.node.datax_custom_json_resource_tips')) } if (!utils.isJson(model.json)) { return new Error(t('project.node.json_format_tips')) @@ -223,8 +231,8 @@ export function useDataX(model: { [field: string]: any }): IJsonItem[] { typeof parsed === 'object' && !Array.isArray(parsed) && Object.keys(parsed).length === 0 - if (isEmptyObject && !hasResource) { - return new Error(t('project.node.sql_empty_tips')) + if (isEmptyObject && !hasSingleJsonResource) { + return new Error(t('project.node.datax_custom_json_resource_tips')) } } } From 2e2de6bf1fe92daa63287e0c56ae29deb7692497 Mon Sep 17 00:00:00 2001 From: Nikhil Bharadwaj Ramashasthri Date: Sun, 16 Aug 2026 23:25:56 -0700 Subject: [PATCH 6/6] [Fix-18389][UI] Treat whitespace-only inline json as absent in the datax validator --- .../task/components/node/fields/use-datax.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts index db556971bc8b..45698908bd70 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts @@ -210,11 +210,15 @@ export function useDataX(model: { [field: string]: any }): IJsonItem[] { typeof fullName === 'string' && fullName.toLowerCase().endsWith('.json') ).length === 1 - if ( - model.json === '' || + // Treat a whitespace-only value as absent too, matching the backend + // DataxParameters.isInlineJsonAbsent which uses StringUtils.isBlank. Otherwise a + // blank inline json would reach utils.isJson below and be rejected even when exactly + // one valid .json resource is attached, a configuration the worker would have accepted. + const inlineJsonAbsent = model.json === undefined || - model.json === null - ) { + model.json === null || + (model.json as string).trim() === '' + if (inlineJsonAbsent) { return hasSingleJsonResource ? undefined : new Error(t('project.node.datax_custom_json_resource_tips'))