From e5f5ab909be9dc6087c6c5c3c73f2d5a5d02e8d5 Mon Sep 17 00:00:00 2001 From: taojinlong Date: Tue, 26 Apr 2022 18:57:42 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E5=90=8C=E6=97=B6=E5=88=9B=E5=BB=BA?= =?UTF-8?q?=E5=A4=9A=E4=B8=AA=E5=AE=9A=E6=97=B6=E5=90=8C=E6=AD=A5=E7=9A=84?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=9B=86=EF=BC=8C=E4=BC=9A=E5=87=BA=E7=8E=B0?= =?UTF-8?q?=E9=83=A8=E5=88=86=E6=95=B0=E6=8D=AE=E9=9B=86=E6=9B=B4=E6=96=B0?= =?UTF-8?q?=E4=BF=A1=E6=81=AF=E6=98=BE=E7=A4=BA=E6=88=90=E5=8A=9F=E4=BD=86?= =?UTF-8?q?=E6=98=AF=E9=A2=84=E8=A7=88=E6=95=B0=E6=8D=AE=E4=B8=BA=E7=A9=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/dataset/ExtractDataService.java | 38 +++++++++++++++---- frontend/src/views/system/datasource/form.vue | 5 ++- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index 524b467ff2..dc24ed3a8f 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -120,6 +120,19 @@ public class ExtractDataService { " exit 1\n" + "fi\n"; + private static final String shellScriptForDeleteFile = "result=`curl --location-trusted -u %s:%s -H \"label:%s\" -H \"column_separator:%s\" -H \"columns:%s\" -H \"merge_type: %s\" -T %s -XPUT http://%s:%s/api/%s/%s/_stream_load`\n" + + "rm -rf %s \n" + + "if [ $? -eq 0 ] ; then\n" + + " failstatus=$(echo $result | grep '\"Status\": \"Fail\"')\n" + + " if [ \"x${failstatus}\" != \"x\" ];then" + + " echo $result\n" + + " exit 1\n" + + " fi\n" + + "else\n" + + " echo $result\n" + + " exit 1\n" + + "fi\n"; + public synchronized boolean existSyncTask(DatasetTable datasetTable, DatasetTableTask datasetTableTask, Long startTime) { datasetTable.setSyncStatus(JobStatus.Underway.name()); DatasetTableExample example = new DatasetTableExample(); @@ -445,14 +458,20 @@ public class ExtractDataService { String dataFile = null; String script = null; + String streamLoadScript = ""; + if(kettleFilesKeep){ + streamLoadScript = shellScript; + }else { + streamLoadScript = shellScriptForDeleteFile; + } switch (extractType) { case "all_scope": dataFile = root_path + TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())) + "." + extention; - script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId()))); + script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId()))); break; default: dataFile = root_path + TableUtils.addName(TableUtils.tableName(datasetTable.getId())) + "." + extention; - script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId())); + script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId())); break; } @@ -744,7 +763,6 @@ public class ExtractDataService { DataEaseException.throwException(transStatus.getLoggingString()); return; } - executing = true; String lastCarteObjectId = Job.sendToSlaveServer(jobMeta, jobExecutionConfiguration, repository, null); SlaveServerJobStatus jobStatus = null; @@ -759,7 +777,7 @@ public class ExtractDataService { if (jobStatus.getStatusDescription().equals("Finished")) { return; } else { - DataEaseException.throwException((jobStatus.getLoggingString())); + DataEaseException.throwException(jobStatus.getLoggingString()); } } @@ -773,20 +791,26 @@ public class ExtractDataService { Datasource dorisDatasource = engineService.getDeEngine(); DorisConfiguration dorisConfiguration = new Gson().fromJson(dorisDatasource.getConfiguration(), DorisConfiguration.class); String columns = columnFields + ",dataease_uuid"; + String streamLoadScript = ""; + if(kettleFilesKeep){ + streamLoadScript = shellScript; + }else { + streamLoadScript = shellScriptForDeleteFile; + } switch (extractType) { case "all_scope": outFile = TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())); jobName = "job_" + TableUtils.tableName(datasetTable.getId()); - script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())), root_path + outFile + "." + extention); + script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), datasetTable.getId() + System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())), root_path + outFile + "." + extention); break; case "incremental_add": outFile = TableUtils.addName(datasetTable.getId()); jobName = "job_add_" + TableUtils.tableName(datasetTable.getId()); - script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention); + script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), datasetTable.getId() + System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention); break; case "incremental_delete": outFile = TableUtils.deleteName(TableUtils.tableName(datasetTable.getId())); - script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "DELETE", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention); + script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), datasetTable.getId() + System.currentTimeMillis(), separator, columns, "DELETE", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention); jobName = "job_delete_" + TableUtils.tableName(datasetTable.getId()); break; default: diff --git a/frontend/src/views/system/datasource/form.vue b/frontend/src/views/system/datasource/form.vue index c557158fd2..70afbfe964 100644 --- a/frontend/src/views/system/datasource/form.vue +++ b/frontend/src/views/system/datasource/form.vue @@ -277,6 +277,7 @@ export default { this.form = JSON.parse(JSON.stringify(row)) this.originConfiguration = this.form.configuration if (row.type === 'api') { + } else { this.form.configuration = JSON.parse(this.form.configuration) } @@ -494,7 +495,9 @@ export default { changeType() { for (let i = 0; i < this.dsTypes.length; i++) { if (this.dsTypes[i].type === this.form.type) { - this.form.configuration.extraParams = this.dsTypes[i].extraParams + if(row.type !== 'api'){ + this.form.configuration.extraParams = this.dsTypes[i].extraParams + } this.datasourceType = this.dsTypes[i] } }