center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

refactor: 整理代码

Browse Source
This commit is contained in:
taojinlong 2022-03-16 15:44:59 +08:00
parent 1f5fd16564
commit 98dc2436a2
3 changed files with 35 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@ -2188,9 +2188,9 @@ public class DataSetTableService {
+ qrtzSchedulerState.getCheckinInterval() + 1000 > utilMapper.currentTimestamp())
.map(QrtzSchedulerStateKey::getInstanceName).collect(Collectors.toList());
List<DatasetTable> jobStoppeddDatasetTables = new ArrayList<>();
DatasetTableExample example = new DatasetTableExample();
example.createCriteria().andSyncStatusEqualTo(JobStatus.Underway.name());
datasetTableMapper.selectByExample(example).forEach(datasetTable -> {
if (StringUtils.isEmpty(datasetTable.getQrtzInstance()) || !activeQrtzInstances.contains(
datasetTable.getQrtzInstance().substring(0, datasetTable.getQrtzInstance().length() - 13))) {
@ -2202,6 +2202,7 @@ public class DataSetTableService {
return;
}
//DatasetTable
DatasetTable record = new DatasetTable();
record.setSyncStatus(JobStatus.Error.name());
example.clear();
@ -2209,6 +2210,14 @@ public class DataSetTableService {
.andIdIn(jobStoppeddDatasetTables.stream().map(DatasetTable::getId).collect(Collectors.toList()));
datasetTableMapper.updateByExampleSelective(record, example);
//Task
DatasetTableTaskExample datasetTableTaskExample = new DatasetTableTaskExample();
DatasetTableTaskExample.Criteria criteria = datasetTableTaskExample.createCriteria();
criteria.andTableIdIn(jobStoppeddDatasetTables.stream().map(DatasetTable::getId).collect(Collectors.toList())).andStatusEqualTo(JobStatus.Underway.name());
List<DatasetTableTask> datasetTableTasks = dataSetTableTaskService.list(datasetTableTaskExample);
dataSetTableTaskService.updateTaskStatus(datasetTableTasks, JobStatus.Error);
//TaskLog
DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
datasetTableTaskLog.setStatus(JobStatus.Error.name());
datasetTableTaskLog.setInfo("Job stopped due to system error.");
@ -2216,12 +2225,8 @@ public class DataSetTableService {
DatasetTableTaskLogExample datasetTableTaskLogExample = new DatasetTableTaskLogExample();
datasetTableTaskLogExample.createCriteria().andStatusEqualTo(JobStatus.Underway.name())
.andTableIdIn(jobStoppeddDatasetTables.stream().map(DatasetTable::getId).collect(Collectors.toList()));
List<String> taskIds = datasetTableTaskLogMapper.selectByExample(datasetTableTaskLogExample).stream()
.map(DatasetTableTaskLog::getTaskId).collect(Collectors.toList());
datasetTableTaskLogMapper.updateByExampleSelective(datasetTableTaskLog, datasetTableTaskLogExample);
dataSetTableTaskService.updateTaskStatus(taskIds, JobStatus.Error);
for (DatasetTable jobStoppeddDatasetTable : jobStoppeddDatasetTables) {
extractDataService.deleteFile("all_scope", jobStoppeddDatasetTable.getId());
extractDataService.deleteFile("incremental_add", jobStoppeddDatasetTable.getId());

16
backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java
View File

@ -161,13 +161,13 @@ public class DataSetTableTaskService {
return datasetTableTaskMapper.selectByPrimaryKey(id);
}
public void updateTaskStatus(List<String> taskIds, JobStatus lastExecStatus) {
if (CollectionUtils.isEmpty(taskIds)) {
return;
}
DatasetTableTaskExample example = new DatasetTableTaskExample();
example.createCriteria().andIdIn(taskIds);
List<DatasetTableTask> datasetTableTasks = datasetTableTaskMapper.selectByExample(example);
public List<DatasetTableTask> list(DatasetTableTaskExample example) {
return datasetTableTaskMapper.selectByExample(example);
}
public void updateTaskStatus(List<DatasetTableTask> datasetTableTasks, JobStatus lastExecStatus) {
for (DatasetTableTask tableTask : datasetTableTasks) {
updateTaskStatus(tableTask, lastExecStatus);
}
@ -202,7 +202,7 @@ public class DataSetTableTaskService {
if (datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.name())) {
datasetTableTask.setStatus(TaskStatus.Stopped.name());
} else {
datasetTableTask = datasetTableTaskMapper.selectByPrimaryKey(datasetTableTask.getId());
// datasetTableTask = datasetTableTaskMapper.selectByPrimaryKey(datasetTableTask.getId());
datasetTableTask.setLastExecStatus(lastExecStatus.name());
if (StringUtils.isNotEmpty(datasetTableTask.getEnd()) && datasetTableTask.getEnd().equalsIgnoreCase("1")) {
BaseGridRequest request = new BaseGridRequest();

52
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -140,7 +140,7 @@ public class ExtractDataService {
datasetTableTaskLog.setTableId(datasetTable.getId());
datasetTableTaskLog.setStatus(JobStatus.Underway.name());
List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog);
return !CollectionUtils.isNotEmpty(datasetTableTaskLogs) || !datasetTableTaskLogs.get(0).getTriggerType().equalsIgnoreCase(TriggerType.Custom.name());
return CollectionUtils.isEmpty(datasetTableTaskLogs) || !datasetTableTaskLogs.get(0).getTriggerType().equalsIgnoreCase(TriggerType.Custom.name());
} else {
datasetTableTask.setLastExecTime(startTime);
datasetTableTask.setLastExecStatus(JobStatus.Underway.name());
@ -159,7 +159,7 @@ public class ExtractDataService {
return;
}
UpdateType updateType = UpdateType.valueOf(type);
DatasetTableTaskLog datasetTableTaskLog;
if (datasetTableFields == null) {
datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTable.getId()).build());
}
@ -174,10 +174,10 @@ public class ExtractDataService {
return o1.getColumnIndex().compareTo(o2.getColumnIndex());
});
DatasetTableTaskLog datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops);
switch (updateType) {
case all_scope: // 全量更新
try {
datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops);
createEngineTable(TableUtils.tableName(datasetTableId), datasetTableFields);
createEngineTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId)), datasetTableFields);
Long execTime = System.currentTimeMillis();
@ -222,7 +222,7 @@ public class ExtractDataService {
toDelete.forEach(datasetTableField -> dataSetTableFieldsService.delete(datasetTableField.getId()));
}
} catch (Exception e) {
saveErrorLog(datasetTableId, null, e);
saveErrorLog(datasetTableTaskLog, e);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null);
dropDorisTable(TableUtils.tmpName(TableUtils.tableName(datasetTableId)));
} finally {
@ -233,7 +233,6 @@ public class ExtractDataService {
case add_scope: // 增量更新
try {
datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops);
Long execTime = System.currentTimeMillis();
if (!engineService.isSimpleMode()) {
generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, null);
@ -245,7 +244,7 @@ public class ExtractDataService {
saveSuccessLog(datasetTableTaskLog);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime);
} catch (Exception e) {
saveErrorLog(datasetTableId, null, e);
saveErrorLog(datasetTableTaskLog, e);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null);
} finally {
deleteFile("incremental_add", datasetTableId);
@ -312,7 +311,7 @@ public class ExtractDataService {
msg = true;
lastExecStatus = JobStatus.Completed;
} catch (Exception e) {
saveErrorLog(datasetTableId, taskId, e);
saveErrorLog(datasetTableTaskLog, e);
msg = false;
lastExecStatus = JobStatus.Error;
execTime = null;
@ -372,7 +371,7 @@ public class ExtractDataService {
msg = true;
lastExecStatus = JobStatus.Completed;
} catch (Exception e) {
saveErrorLog(datasetTableId, taskId, e);
saveErrorLog(datasetTableTaskLog, e);
msg = false;
lastExecStatus = JobStatus.Error;
execTime = null;
@ -592,26 +591,14 @@ public class ExtractDataService {
dataSetTableTaskLogService.save(datasetTableTaskLog);
}
private void saveErrorLog(String datasetTableId, String taskId, Exception e) {
LogUtil.error("Extract data error: " + datasetTableId, e);
DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
datasetTableTaskLog.setTableId(datasetTableId);
datasetTableTaskLog.setStatus(JobStatus.Underway.name());
if (StringUtils.isNotEmpty(taskId)) {
datasetTableTaskLog.setTaskId(taskId);
}
List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog);
if (CollectionUtils.isNotEmpty(datasetTableTaskLogs)) {
datasetTableTaskLog = datasetTableTaskLogs.get(0);
datasetTableTaskLog.setStatus(JobStatus.Error.name());
datasetTableTaskLog.setInfo(e.getMessage());
datasetTableTaskLog.setEndTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);
}
private void saveErrorLog(DatasetTableTaskLog datasetTableTaskLog, Exception e) {
LogUtil.error("Extract data error: " + datasetTableTaskLog.getTaskId(), e);
datasetTableTaskLog.setStatus(JobStatus.Error.name());
datasetTableTaskLog.setInfo(e.getMessage());
datasetTableTaskLog.setEndTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);
}
private void createEngineTable(String tableName, List<DatasetTableField> datasetTableFields) throws Exception {
Datasource engine = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
@ -671,14 +658,9 @@ public class ExtractDataService {
datasetTableTaskLog.setTaskId(taskId);
datasetTableTaskLog.setStatus(JobStatus.Underway.name());
datasetTableTaskLog.setTriggerType(TriggerType.Cron.name());
List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog);
if (CollectionUtils.isEmpty(datasetTableTaskLogs)) {
datasetTableTaskLog.setStartTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);
return datasetTableTaskLog;
} else {
return datasetTableTaskLogs.get(0);
}
datasetTableTaskLog.setStartTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);
return datasetTableTaskLog;
}
private DatasetTableTaskLog getDatasetTableTaskLog(String datasetTableId, String taskId, Long startTime) {
@ -687,7 +669,7 @@ public class ExtractDataService {
datasetTableTaskLog.setTaskId(taskId);
datasetTableTaskLog.setStatus(JobStatus.Underway.name());
datasetTableTaskLog.setTriggerType(TriggerType.Custom.name());
for (int i = 0; i < 10; i++) {
for (int i = 0; i < 5; i++) {
List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.select(datasetTableTaskLog);
if (CollectionUtils.isNotEmpty(datasetTableTaskLogs)) {
return datasetTableTaskLogs.get(0);