center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

fix: 更新任务状态

Browse Source
This commit is contained in:
taojinlong 2021-07-23 18:35:37 +08:00
parent 82c2b1259e
commit d324ff8c6e
12 changed files with 92 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
backend/src/main/java/io/dataease/controller/dataset/DataSetTableTaskLogController.java
View File

@ -33,10 +33,10 @@ public class DataSetTableTaskLogController {
dataSetTableTaskLogService.delete(id);
}
@PostMapping("list/{goPage}/{pageSize}")
public Pager<List<DataSetTaskLogDTO>> list(@RequestBody BaseGridRequest request, @PathVariable int goPage, @PathVariable int pageSize) {
@PostMapping("list/{type}/{goPage}/{pageSize}")
public Pager<List<DataSetTaskLogDTO>> list(@RequestBody BaseGridRequest request, @PathVariable String type, @PathVariable int goPage, @PathVariable int pageSize) {
Page<Object> page = PageHelper.startPage(goPage, pageSize, true);
return PageUtils.setPageInfo(page, dataSetTableTaskLogService.list(request));
return PageUtils.setPageInfo(page, dataSetTableTaskLogService.list(request, type));
}
}

1
backend/src/main/java/io/dataease/dto/dataset/DataTableInfoDTO.java
View File

@ -14,6 +14,7 @@ import java.util.List;
public class DataTableInfoDTO {
private String table;
private String sql;
private List<String> sheets;
private String data;// file path
private List<DataTableInfoCustomUnion> list;
}

60
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@ -101,7 +101,7 @@ public class DataSetTableService {
private void extractData(DataSetTableRequest datasetTable) throws Exception {
if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) {
commonThreadPool.addTask(() -> {
extractDataService.extractExcelData(datasetTable.getId(), "all_scope");
extractDataService.extractExcelData(datasetTable.getId(), "all_scope", "初始导入");
});
return;
}
@ -131,7 +131,6 @@ public class DataSetTableService {
datasetTable.setId(UUID.randomUUID().toString());
datasetTable.setCreateBy(AuthUtils.getUser().getUsername());
datasetTable.setCreateTime(System.currentTimeMillis());
DataTableInfoDTO dataTableInfoDTO = new DataTableInfoDTO();
int insert = datasetTableMapper.insert(datasetTable);
// 添加表成功后，获取当前表字段和类型，抽象到dataease数据库
if (insert == 1) {
@ -151,11 +150,11 @@ public class DataSetTableService {
if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) {
if (datasetTable.getEditType() == 0) {
commonThreadPool.addTask(() -> {
extractDataService.extractExcelData(datasetTable.getId(), "all_scope");
extractDataService.extractExcelData(datasetTable.getId(), "all_scope", "替换");
});
} else if (datasetTable.getEditType() == 1) {
commonThreadPool.addTask(() -> {
extractDataService.extractExcelData(datasetTable.getId(), "add_scope");
extractDataService.extractExcelData(datasetTable.getId(), "add_scope", "追加");
});
}
}
@ -416,38 +415,27 @@ public class DataSetTableService {
}
}
} else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) {
if (StringUtils.isEmpty(datasetTable.getSyncStatus()) || datasetTable.getSyncStatus().equalsIgnoreCase(JobStatus.Underway.name())) {
map.put("status", "warnning");
map.put("msg", Translator.get("i18n_processing_data"));
dataSetPreviewPage.setTotal(0);
} else if (datasetTable.getSyncStatus().equalsIgnoreCase(JobStatus.Error.name())) {
List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.getByTableId(datasetTable.getId());
map.put("status", "error");
if (CollectionUtils.isNotEmpty(datasetTableTaskLogs)) {
map.put("msg", "Failed to extract data: " + datasetTableTaskLogs.get(0).getInfo());
} else {
map.put("msg", "Failed to extract data.");
}
dataSetPreviewPage.setTotal(0);
} else {
Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize));
try {
data.addAll(jdbcProvider.getData(datasourceRequest));
} catch (Exception e) {
e.printStackTrace();
}
try {
datasourceRequest.setQuery(qp.createQueryTableWithLimit(table, fields, Integer.valueOf(dataSetTableRequest.getRow())));
dataSetPreviewPage.setTotal(Integer.valueOf(jdbcProvider.getData(datasourceRequest).get(0)[0]));
} catch (Exception e) {
e.printStackTrace();
}
if (!checkDorisTableIsExists(dataSetTableRequest.getId())) {
throw new RuntimeException(Translator.get("i18n_data_not_sync"));
}
Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize));
try {
data.addAll(jdbcProvider.getData(datasourceRequest));
} catch (Exception e) {
e.printStackTrace();
}
try {
datasourceRequest.setQuery(qp.createQueryTableWithLimit(table, fields, Integer.valueOf(dataSetTableRequest.getRow())));
dataSetPreviewPage.setTotal(Integer.valueOf(jdbcProvider.getData(datasourceRequest).get(0)[0]));
} catch (Exception e) {
e.printStackTrace();
}
} else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "custom")) {
if (datasetTable.getMode() == 0) {

29
backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java
View File

@ -47,18 +47,27 @@ public class DataSetTableTaskLogService {
datasetTableTaskLogMapper.deleteByPrimaryKey(id);
}
public List<DataSetTaskLogDTO> list(BaseGridRequest request) {
ConditionEntity entity = new ConditionEntity();
entity.setField("task_id");
entity.setOperator("not null");
List<ConditionEntity> conditionEntities = request.getConditions();
if(CollectionUtils.isEmpty(conditionEntities)){
conditionEntities = new ArrayList<>();
public List<DataSetTaskLogDTO> list(BaseGridRequest request, String type) {
if(!type.equalsIgnoreCase("excel")){
ConditionEntity entity = new ConditionEntity();
entity.setField("task_id");
entity.setOperator("not null");
List<ConditionEntity> conditionEntities = request.getConditions();
if(CollectionUtils.isEmpty(conditionEntities)){
conditionEntities = new ArrayList<>();
}
conditionEntities.add(entity);
request.setConditions(conditionEntities);
}
conditionEntities.add(entity);
request.setConditions(conditionEntities);
GridExample gridExample = request.convertExample();
return extDataSetTaskMapper.list(gridExample);
List<DataSetTaskLogDTO> dataSetTaskLogDTOS = extDataSetTaskMapper.list(gridExample);
dataSetTaskLogDTOS.forEach(dataSetTaskLogDTO -> {
if(StringUtils.isEmpty(dataSetTaskLogDTO.getName())){
dataSetTaskLogDTO.setName(dataSetTaskLogDTO.getTaskId());
}
});
return dataSetTaskLogDTOS;
}
public void deleteByTaskId(String taskId){

16
backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java
View File

@ -13,6 +13,7 @@ import io.dataease.commons.constants.TaskStatus;
import io.dataease.commons.constants.TriggerType;
import io.dataease.controller.request.dataset.DataSetTaskRequest;
import io.dataease.controller.sys.base.BaseGridRequest;
import io.dataease.controller.sys.base.ConditionEntity;
import io.dataease.controller.sys.response.SysUserGridResponse;
import io.dataease.controller.sys.response.SysUserRole;
import io.dataease.dto.dataset.DataSetTaskDTO;
@ -28,6 +29,7 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import javax.annotation.Resource;
import java.util.Arrays;
import java.util.List;
import java.util.UUID;
import java.util.stream.Collectors;
@ -177,8 +179,20 @@ public class DataSetTableTaskService {
datasetTableTask.setStatus(TaskStatus.Stopped.name());
}else {
if(StringUtils.isNotEmpty(datasetTableTask.getEnd()) && datasetTableTask.getEnd().equalsIgnoreCase("1")){
if(utilMapper.currentTimestamp() > datasetTableTask.getEndTime()){
BaseGridRequest request = new BaseGridRequest();
ConditionEntity conditionEntity = new ConditionEntity();
conditionEntity.setField("dataset_table_task.id");
conditionEntity.setOperator("eq");
conditionEntity.setValue(datasetTableTask.getId());
request.setConditions(Arrays.asList(conditionEntity));
List<DataSetTaskDTO> dataSetTaskDTOS = taskList(request);
if(CollectionUtils.isEmpty(dataSetTaskDTOS)){
return;
}
if(dataSetTaskDTOS.get(0).getNextExecTime() == null || dataSetTaskDTOS.get(0).getNextExecTime() <= 0){
datasetTableTask.setStatus(TaskStatus.Stopped.name());
}else {
datasetTableTask.setStatus(TaskStatus.Underway.name());
}
}else {
datasetTableTask.setStatus(TaskStatus.Underway.name());

66
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -158,7 +158,7 @@ public class ExtractDataService {
}
}
public void extractExcelData(String datasetTableId, String type) {
public void extractExcelData(String datasetTableId, String type, String ops) {
Datasource datasource = new Datasource();
datasource.setType("excel");
DatasetTable datasetTable = getDatasetTable(datasetTableId);
@ -182,15 +182,11 @@ public class ExtractDataService {
switch (updateType) {
case all_scope: // 全量更新
try {
datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, null);
datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops);
createDorisTable(DorisTableUtils.dorisName(datasetTableId), dorisTablColumnSql);
createDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)), dorisTablColumnSql);
generateTransFile("all_scope", datasetTable, datasource, datasetTableFields, null);
if (datasetTable.getType().equalsIgnoreCase("sql")) {
generateJobFile("all_scope", datasetTable, fetchSqlField(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getSql(), datasource));
} else {
generateJobFile("all_scope", datasetTable, String.join(",", datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList())));
}
generateJobFile("all_scope", datasetTable, String.join(",", datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList())));
Long execTime = System.currentTimeMillis();
extractData(datasetTable, "all_scope");
replaceTable(DorisTableUtils.dorisName(datasetTableId));
@ -210,17 +206,17 @@ public class ExtractDataService {
case add_scope: // 增量更新
try {
datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, null);
datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops);
generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, null);
generateJobFile("incremental_add", datasetTable, String.join(",", datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList())));
Long execTime = System.currentTimeMillis();
extractData(datasetTable, "incremental_add");
saveSucessLog(datasetTableTaskLog);
sendWebMsg(datasetTable, null, true);
// sendWebMsg(datasetTable, null, true);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime);
} catch (Exception e) {
saveErrorLog(datasetTableId, null, e);
sendWebMsg(datasetTable, null, false);
// sendWebMsg(datasetTable, null, false);
updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null);
deleteFile("incremental_add", datasetTableId);
deleteFile("incremental_delete", datasetTableId);
@ -256,7 +252,7 @@ public class ExtractDataService {
return;
}
DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
DatasetTableTaskLog datasetTableTaskLog = getDatasetTableTaskLog(datasetTableId, taskId);
UpdateType updateType = UpdateType.valueOf(type);
if (context != null) {
datasetTable.setQrtzInstance(context.getFireInstanceId());
@ -283,11 +279,8 @@ public class ExtractDataService {
switch (updateType) {
case all_scope: // 全量更新
try {
if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.CRON.toString())) {
datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId);
}
if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) {
datasetTableTaskLog = getDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId);
if (datasetTableTask == null ) {
datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, taskId);
}
createDorisTable(DorisTableUtils.dorisName(datasetTableId), dorisTablColumnSql);
createDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)), dorisTablColumnSql);
@ -336,11 +329,8 @@ public class ExtractDataService {
return;
}
if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.CRON.toString())) {
datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId);
}
if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) {
datasetTableTaskLog = getDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId);
if (datasetTableTask == null ) {
datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, taskId);
}
Long execTime = System.currentTimeMillis();
if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) {// 增量添加
@ -533,7 +523,8 @@ public class ExtractDataService {
return null;
}
private DatasetTableTaskLog writeDatasetTableTaskLog(DatasetTableTaskLog datasetTableTaskLog, String datasetTableId, String taskId) {
private DatasetTableTaskLog writeDatasetTableTaskLog(String datasetTableId, String taskId) {
DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
datasetTableTaskLog.setTableId(datasetTableId);
datasetTableTaskLog.setTaskId(taskId);
datasetTableTaskLog.setStatus(JobStatus.Underway.name());
@ -548,7 +539,8 @@ public class ExtractDataService {
}
}
private DatasetTableTaskLog getDatasetTableTaskLog(DatasetTableTaskLog datasetTableTaskLog, String datasetTableId, String taskId) {
private DatasetTableTaskLog getDatasetTableTaskLog(String datasetTableId, String taskId) {
DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
datasetTableTaskLog.setTableId(datasetTableId);
datasetTableTaskLog.setTaskId(taskId);
datasetTableTaskLog.setStatus(JobStatus.Underway.name());
@ -765,8 +757,7 @@ public class ExtractDataService {
udjcStep = udjc(datasetTableFields, DatasourceTypes.oracle);
break;
case excel:
String filePath = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getData();
inputStep = excelInputStep(filePath, datasetTableFields);
inputStep = excelInputStep(datasetTable.getInfo(), datasetTableFields);
udjcStep = udjc(datasetTableFields, DatasourceTypes.excel);
default:
break;
@ -837,33 +828,20 @@ public class ExtractDataService {
return fromStep;
}
private StepMeta excelInputStep(String filePath, List<DatasetTableField> datasetTableFields) {
String suffix = filePath.substring(filePath.lastIndexOf(".") + 1);
private StepMeta excelInputStep(String Info, List<DatasetTableField> datasetTableFields){
DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(Info, DataTableInfoDTO.class);
String suffix = dataTableInfoDTO.getData().substring(dataTableInfoDTO.getData().lastIndexOf(".") + 1);
ExcelInputMeta excelInputMeta = new ExcelInputMeta();
if (StringUtils.equalsIgnoreCase(suffix, "xlsx")) {
excelInputMeta.setSpreadSheetType(SpreadSheetType.SAX_POI);
try {
InputStream inputStream = new FileInputStream(filePath);
XSSFWorkbook xssfWorkbook = new XSSFWorkbook(inputStream);
XSSFSheet sheet0 = xssfWorkbook.getSheetAt(0);
excelInputMeta.setSheetName(new String[]{sheet0.getSheetName()});
} catch (Exception e) {
e.printStackTrace();
}
excelInputMeta.setSheetName(new String[]{dataTableInfoDTO.getSheets().get(0)});
}
if (StringUtils.equalsIgnoreCase(suffix, "xls")) {
excelInputMeta.setSpreadSheetType(SpreadSheetType.JXL);
try {
InputStream inputStream = new FileInputStream(filePath);
HSSFWorkbook workbook = new HSSFWorkbook(inputStream);
HSSFSheet sheet0 = workbook.getSheetAt(0);
excelInputMeta.setSheetName(new String[]{sheet0.getSheetName()});
} catch (Exception e) {
e.printStackTrace();
}
excelInputMeta.setSheetName(new String[]{dataTableInfoDTO.getSheets().get(0)});
}
excelInputMeta.setPassword("Encrypted");
excelInputMeta.setFileName(new String[]{filePath});
excelInputMeta.setFileName(new String[]{dataTableInfoDTO.getData()});
excelInputMeta.setStartsWithHeader(true);
excelInputMeta.setIgnoreEmptyRows(true);
ExcelInputField[] fields = new ExcelInputField[datasetTableFields.size()];

2
backend/src/main/resources/db/migration/V13__system_task.sql
View File

@ -23,4 +23,4 @@ ALTER TABLE `dataset_table_task` ADD COLUMN `extra_data` LONGTEXT NULL AFTER `la
update dataset_table_task_log set trigger_type='Cron';
update dataset_table_task_log set dataset_table_task_log.task_id='初始导入' where dataset_table_task_log.task_id is null;

2
frontend/src/views/dataset/add/AddExcel.vue
View File

@ -258,7 +258,7 @@ export default {
type: 'excel',
mode: parseInt(this.mode),
// info: '{"data":"' + this.path + '"}',
info: JSON.stringify({ data: this.path }),
info: JSON.stringify({ data: this.path, sheets: [this.sheets[0]]}),
fields: this.fields
}
} else {

6
frontend/src/views/dataset/data/UpdateInfo.vue
View File

@ -1,7 +1,7 @@
<template>
<el-col>
<el-row>
<el-button v-if="hasDataPermission('manage',param.privileges)" icon="el-icon-setting" size="mini" @click="showConfig">
<el-button v-if="hasDataPermission('manage',param.privileges) || table.type !== 'excel'" icon="el-icon-setting" size="mini" @click="showConfig">
{{ $t('dataset.update_setting') }}
</el-button>
<el-button icon="el-icon-refresh" size="mini" @click="refreshLog">
@ -615,8 +615,8 @@ export default {
}
},
listTaskLog(loading = true) {
const params = {"conditions":[{"field":"dataset_table_task.table_id","operator":"eq","value": this.table.id}],"orders":[]}
post('/dataset/taskLog/list/' + this.page.currentPage + '/' + this.page.pageSize, params, loading).then(response => {
const params = {"conditions":[{"field":"dataset_table_task_log.table_id","operator":"eq","value": this.table.id}],"orders":[]}
post('/dataset/taskLog/list/' + this.table.type + '/' + this.page.currentPage + '/' + this.page.pageSize, params, loading).then(response => {
this.taskLogData = response.data.listObject
this.page.total = response.data.itemCount
})

2
frontend/src/views/dataset/data/ViewTable.vue
View File

@ -51,7 +51,7 @@
<el-tab-pane v-if="table.type !== 'custom'" :label="$t('dataset.join_view')" name="joinView">
<union-view :param="param" :table="table" />
</el-tab-pane>
<el-tab-pane v-if="table.mode === 1 && (table.type === 'db' || table.type === 'sql')" :label="$t('dataset.update_info')" name="updateInfo">
<el-tab-pane v-if="table.mode === 1 && (table.type === 'excel' || table.type === 'db' || table.type === 'sql')" :label="$t('dataset.update_info')" name="updateInfo">
<update-info :param="param" :table="table" />
</el-tab-pane>
</el-tabs>

2
frontend/src/views/system/task/DatasetTaskList.vue
View File

@ -47,7 +47,7 @@
<el-table-column prop="nextExecTime" :label="$t('dataset.task.next_exec_time')">
<template slot-scope="scope" >
<span v-if="scope.row.nextExecTime && scope.row.rate !== 'SIMPLE'">
<span v-if="scope.row.nextExecTime && scope.row.nextExecTime !== -1 && scope.row.rate !== 'SIMPLE'">
{{ scope.row.nextExecTime | timestampFormatDate }}
</span>
<span v-if="!scope.row.nextExecTime || scope.row.rate === 'SIMPLE'"></span>

2
frontend/src/views/system/task/TaskRecord.vue
View File

@ -198,7 +198,7 @@ export default {
const temp = formatCondition(condition)
const param = temp || {}
param['orders'] = formatOrders(this.orderConditions)
post('/dataset/taskLog/list/' + this.paginationConfig.currentPage + '/' + this.paginationConfig.pageSize, param, showLoading).then(response => {
post('/dataset/taskLog/list/notexcel/' + this.paginationConfig.currentPage + '/' + this.paginationConfig.pageSize, param, showLoading).then(response => {
this.data = response.data.listObject
this.paginationConfig.total = response.data.itemCount
})