center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

fix: 同时创建多个定时同步的数据集，会出现部分数据集更新信息显示成功但是预览数据为空

Browse Source
This commit is contained in:
taojinlong 2022-04-26 18:57:42 +08:00
parent 657265cc2f
commit e5f5ab909b
2 changed files with 35 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -120,6 +120,19 @@ public class ExtractDataService {
" exit 1\n" +
"fi\n";
private static final String shellScriptForDeleteFile = "result=`curl --location-trusted -u %s:%s -H \"label:%s\" -H \"column_separator:%s\" -H \"columns:%s\" -H \"merge_type: %s\" -T %s -XPUT http://%s:%s/api/%s/%s/_stream_load`\n" +
"rm -rf %s \n" +
"if [ $? -eq 0 ] ; then\n" +
" failstatus=$(echo $result | grep '\"Status\": \"Fail\"')\n" +
" if [ \"x${failstatus}\" != \"x\" ];then" +
" echo $result\n" +
" exit 1\n" +
" fi\n" +
"else\n" +
" echo $result\n" +
" exit 1\n" +
"fi\n";
public synchronized boolean existSyncTask(DatasetTable datasetTable, DatasetTableTask datasetTableTask, Long startTime) {
datasetTable.setSyncStatus(JobStatus.Underway.name());
DatasetTableExample example = new DatasetTableExample();
@ -445,14 +458,20 @@ public class ExtractDataService {
String dataFile = null;
String script = null;
String streamLoadScript = "";
if(kettleFilesKeep){
streamLoadScript = shellScript;
}else {
streamLoadScript = shellScriptForDeleteFile;
}
switch (extractType) {
case "all_scope":
dataFile = root_path + TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())) + "." + extention;
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())));
script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())));
break;
default:
dataFile = root_path + TableUtils.addName(TableUtils.tableName(datasetTable.getId())) + "." + extention;
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()));
script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()));
break;
}
@ -744,7 +763,6 @@ public class ExtractDataService {
DataEaseException.throwException(transStatus.getLoggingString());
return;
}
executing = true;
String lastCarteObjectId = Job.sendToSlaveServer(jobMeta, jobExecutionConfiguration, repository, null);
SlaveServerJobStatus jobStatus = null;
@ -759,7 +777,7 @@ public class ExtractDataService {
if (jobStatus.getStatusDescription().equals("Finished")) {
return;
} else {
DataEaseException.throwException((jobStatus.getLoggingString()));
DataEaseException.throwException(jobStatus.getLoggingString());
}
}
@ -773,20 +791,26 @@ public class ExtractDataService {
Datasource dorisDatasource = engineService.getDeEngine();
DorisConfiguration dorisConfiguration = new Gson().fromJson(dorisDatasource.getConfiguration(), DorisConfiguration.class);
String columns = columnFields + ",dataease_uuid";
String streamLoadScript = "";
if(kettleFilesKeep){
streamLoadScript = shellScript;
}else {
streamLoadScript = shellScriptForDeleteFile;
}
switch (extractType) {
case "all_scope":
outFile = TableUtils.tmpName(TableUtils.tableName(datasetTable.getId()));
jobName = "job_" + TableUtils.tableName(datasetTable.getId());
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())), root_path + outFile + "." + extention);
script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), datasetTable.getId() + System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tmpName(TableUtils.tableName(datasetTable.getId())), root_path + outFile + "." + extention);
break;
case "incremental_add":
outFile = TableUtils.addName(datasetTable.getId());
jobName = "job_add_" + TableUtils.tableName(datasetTable.getId());
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention);
script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), datasetTable.getId() + System.currentTimeMillis(), separator, columns, "APPEND", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention);
break;
case "incremental_delete":
outFile = TableUtils.deleteName(TableUtils.tableName(datasetTable.getId()));
script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "DELETE", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention);
script = String.format(streamLoadScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), datasetTable.getId() + System.currentTimeMillis(), separator, columns, "DELETE", root_path + outFile + "." + extention, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), TableUtils.tableName(datasetTable.getId()), root_path + outFile + "." + extention);
jobName = "job_delete_" + TableUtils.tableName(datasetTable.getId());
break;
default:

5
frontend/src/views/system/datasource/form.vue
View File

@ -277,6 +277,7 @@ export default {
this.form = JSON.parse(JSON.stringify(row))
this.originConfiguration = this.form.configuration
if (row.type === 'api') {
} else {
this.form.configuration = JSON.parse(this.form.configuration)
}
@ -494,7 +495,9 @@ export default {
changeType() {
for (let i = 0; i < this.dsTypes.length; i++) {
if (this.dsTypes[i].type === this.form.type) {
this.form.configuration.extraParams = this.dsTypes[i].extraParams
if(row.type !== 'api'){
this.form.configuration.extraParams = this.dsTypes[i].extraParams
}
this.datasourceType = this.dsTypes[i]
}
}