fix: 更新任务状态

backend/src/main/java/io/dataease/controller/dataset/DataSetTableTaskLogController.java

@@ -33,10 +33,10 @@ public class DataSetTableTaskLogController {
         dataSetTableTaskLogService.delete(id);
     }
 
-    @PostMapping("list/{goPage}/{pageSize}")
+    @PostMapping("list/{type}/{goPage}/{pageSize}")
-    public Pager<List<DataSetTaskLogDTO>> list(@RequestBody BaseGridRequest request, @PathVariable int goPage, @PathVariable int pageSize) {
+    public Pager<List<DataSetTaskLogDTO>> list(@RequestBody BaseGridRequest request, @PathVariable String type, @PathVariable int goPage, @PathVariable int pageSize) {
         Page<Object> page = PageHelper.startPage(goPage, pageSize, true);
-        return PageUtils.setPageInfo(page, dataSetTableTaskLogService.list(request));
+        return PageUtils.setPageInfo(page, dataSetTableTaskLogService.list(request, type));
     }
 
 }

backend/src/main/java/io/dataease/dto/dataset/DataTableInfoDTO.java

@@ -14,6 +14,7 @@ import java.util.List;
 public class DataTableInfoDTO {
     private String table;
     private String sql;
+    private List<String> sheets;
     private String data;// file path
     private List<DataTableInfoCustomUnion> list;
 }

backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java

@@ -101,7 +101,7 @@ public class DataSetTableService {
     private void extractData(DataSetTableRequest datasetTable) throws Exception {
         if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) {
             commonThreadPool.addTask(() -> {
-                extractDataService.extractExcelData(datasetTable.getId(), "all_scope");
+                extractDataService.extractExcelData(datasetTable.getId(), "all_scope", "初始导入");
             });
             return;
         }
@@ -131,7 +131,6 @@ public class DataSetTableService {
             datasetTable.setId(UUID.randomUUID().toString());
             datasetTable.setCreateBy(AuthUtils.getUser().getUsername());
             datasetTable.setCreateTime(System.currentTimeMillis());
-            DataTableInfoDTO dataTableInfoDTO = new DataTableInfoDTO();
             int insert = datasetTableMapper.insert(datasetTable);
             // 添加表成功后，获取当前表字段和类型，抽象到dataease数据库
             if (insert == 1) {
@@ -151,11 +150,11 @@ public class DataSetTableService {
                     if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) {
                         if (datasetTable.getEditType() == 0) {
                             commonThreadPool.addTask(() -> {
-                                extractDataService.extractExcelData(datasetTable.getId(), "all_scope");
+                                extractDataService.extractExcelData(datasetTable.getId(), "all_scope", "替换");
                             });
                         } else if (datasetTable.getEditType() == 1) {
                             commonThreadPool.addTask(() -> {
-                                extractDataService.extractExcelData(datasetTable.getId(), "add_scope");
+                                extractDataService.extractExcelData(datasetTable.getId(), "add_scope", "追加");
                             });
                         }
                     }
@@ -416,38 +415,27 @@ public class DataSetTableService {
                 }
             }
         } else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) {
-            if (StringUtils.isEmpty(datasetTable.getSyncStatus()) || datasetTable.getSyncStatus().equalsIgnoreCase(JobStatus.Underway.name())) {
+            if (!checkDorisTableIsExists(dataSetTableRequest.getId())) {
-                map.put("status", "warnning");
+                throw new RuntimeException(Translator.get("i18n_data_not_sync"));
-                map.put("msg", Translator.get("i18n_processing_data"));
+            }
-                dataSetPreviewPage.setTotal(0);
+
-            } else if (datasetTable.getSyncStatus().equalsIgnoreCase(JobStatus.Error.name())) {
+            Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
-                List<DatasetTableTaskLog> datasetTableTaskLogs = dataSetTableTaskLogService.getByTableId(datasetTable.getId());
+            JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
-                map.put("status", "error");
+            DatasourceRequest datasourceRequest = new DatasourceRequest();
-                if (CollectionUtils.isNotEmpty(datasetTableTaskLogs)) {
+            datasourceRequest.setDatasource(ds);
-                    map.put("msg", "Failed to extract data: " + datasetTableTaskLogs.get(0).getInfo());
+            String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
-                } else {
+            QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
-                    map.put("msg", "Failed to extract data.");
+            datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize));
-                }
+            try {
-                dataSetPreviewPage.setTotal(0);
+                data.addAll(jdbcProvider.getData(datasourceRequest));
-            } else {
+            } catch (Exception e) {
-                Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
+                e.printStackTrace();
-                JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
+            }
-                DatasourceRequest datasourceRequest = new DatasourceRequest();
+            try {
-                datasourceRequest.setDatasource(ds);
+                datasourceRequest.setQuery(qp.createQueryTableWithLimit(table, fields, Integer.valueOf(dataSetTableRequest.getRow())));
-                String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
+                dataSetPreviewPage.setTotal(Integer.valueOf(jdbcProvider.getData(datasourceRequest).get(0)[0]));
-                QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
+            } catch (Exception e) {
-                datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize));
+                e.printStackTrace();
-                try {
-                    data.addAll(jdbcProvider.getData(datasourceRequest));
-                } catch (Exception e) {
-                    e.printStackTrace();
-                }
-                try {
-                    datasourceRequest.setQuery(qp.createQueryTableWithLimit(table, fields, Integer.valueOf(dataSetTableRequest.getRow())));
-                    dataSetPreviewPage.setTotal(Integer.valueOf(jdbcProvider.getData(datasourceRequest).get(0)[0]));
-                } catch (Exception e) {
-                    e.printStackTrace();
-                }
             }
         } else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "custom")) {
             if (datasetTable.getMode() == 0) {

backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java

@@ -47,18 +47,27 @@ public class DataSetTableTaskLogService {
         datasetTableTaskLogMapper.deleteByPrimaryKey(id);
     }
 
-    public List<DataSetTaskLogDTO> list(BaseGridRequest request) {
+    public List<DataSetTaskLogDTO> list(BaseGridRequest request, String type) {
-        ConditionEntity entity = new ConditionEntity();
+        if(!type.equalsIgnoreCase("excel")){
-        entity.setField("task_id");
+            ConditionEntity entity = new ConditionEntity();
-        entity.setOperator("not null");
+            entity.setField("task_id");
-        List<ConditionEntity> conditionEntities = request.getConditions();
+            entity.setOperator("not null");
-        if(CollectionUtils.isEmpty(conditionEntities)){
+            List<ConditionEntity> conditionEntities = request.getConditions();
-            conditionEntities = new ArrayList<>();
+            if(CollectionUtils.isEmpty(conditionEntities)){
+                conditionEntities = new ArrayList<>();
+            }
+            conditionEntities.add(entity);
+            request.setConditions(conditionEntities);
         }
-        conditionEntities.add(entity);
+
-        request.setConditions(conditionEntities);
         GridExample gridExample = request.convertExample();
-        return extDataSetTaskMapper.list(gridExample);
+        List<DataSetTaskLogDTO> dataSetTaskLogDTOS = extDataSetTaskMapper.list(gridExample);
+        dataSetTaskLogDTOS.forEach(dataSetTaskLogDTO -> {
+            if(StringUtils.isEmpty(dataSetTaskLogDTO.getName())){
+                dataSetTaskLogDTO.setName(dataSetTaskLogDTO.getTaskId());
+            }
+        });
+        return dataSetTaskLogDTOS;
     }
 
     public void deleteByTaskId(String taskId){

backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java

@@ -13,6 +13,7 @@ import io.dataease.commons.constants.TaskStatus;
 import io.dataease.commons.constants.TriggerType;
 import io.dataease.controller.request.dataset.DataSetTaskRequest;
 import io.dataease.controller.sys.base.BaseGridRequest;
+import io.dataease.controller.sys.base.ConditionEntity;
 import io.dataease.controller.sys.response.SysUserGridResponse;
 import io.dataease.controller.sys.response.SysUserRole;
 import io.dataease.dto.dataset.DataSetTaskDTO;
@@ -28,6 +29,7 @@ import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
 import javax.annotation.Resource;
+import java.util.Arrays;
 import java.util.List;
 import java.util.UUID;
 import java.util.stream.Collectors;
@@ -177,8 +179,20 @@ public class DataSetTableTaskService {
             datasetTableTask.setStatus(TaskStatus.Stopped.name());
         }else {
             if(StringUtils.isNotEmpty(datasetTableTask.getEnd()) && datasetTableTask.getEnd().equalsIgnoreCase("1")){
-                if(utilMapper.currentTimestamp() > datasetTableTask.getEndTime()){
+                BaseGridRequest request = new BaseGridRequest();
+                ConditionEntity conditionEntity = new ConditionEntity();
+                conditionEntity.setField("dataset_table_task.id");
+                conditionEntity.setOperator("eq");
+                conditionEntity.setValue(datasetTableTask.getId());
+                request.setConditions(Arrays.asList(conditionEntity));
+                List<DataSetTaskDTO> dataSetTaskDTOS = taskList(request);
+                if(CollectionUtils.isEmpty(dataSetTaskDTOS)){
+                    return;
+                }
+                if(dataSetTaskDTOS.get(0).getNextExecTime() == null || dataSetTaskDTOS.get(0).getNextExecTime() <= 0){
                     datasetTableTask.setStatus(TaskStatus.Stopped.name());
+                }else {
+                    datasetTableTask.setStatus(TaskStatus.Underway.name());
                 }
             }else {
                 datasetTableTask.setStatus(TaskStatus.Underway.name());

backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java

@@ -158,7 +158,7 @@ public class ExtractDataService {
         }
     }
 
-    public void extractExcelData(String datasetTableId, String type) {
+    public void extractExcelData(String datasetTableId, String type, String ops) {
         Datasource datasource = new Datasource();
         datasource.setType("excel");
         DatasetTable datasetTable = getDatasetTable(datasetTableId);
@@ -182,15 +182,11 @@ public class ExtractDataService {
         switch (updateType) {
             case all_scope:  // 全量更新
                 try {
-                    datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, null);
+                    datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops);
                     createDorisTable(DorisTableUtils.dorisName(datasetTableId), dorisTablColumnSql);
                     createDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)), dorisTablColumnSql);
                     generateTransFile("all_scope", datasetTable, datasource, datasetTableFields, null);
-                    if (datasetTable.getType().equalsIgnoreCase("sql")) {
+                    generateJobFile("all_scope", datasetTable, String.join(",", datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList())));
-                        generateJobFile("all_scope", datasetTable, fetchSqlField(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getSql(), datasource));
-                    } else {
-                        generateJobFile("all_scope", datasetTable, String.join(",", datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList())));
-                    }
                     Long execTime = System.currentTimeMillis();
                     extractData(datasetTable, "all_scope");
                     replaceTable(DorisTableUtils.dorisName(datasetTableId));
@@ -210,17 +206,17 @@ public class ExtractDataService {
 
             case add_scope: // 增量更新
                 try {
-                    datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, null);
+                    datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops);
                     generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, null);
                     generateJobFile("incremental_add", datasetTable, String.join(",", datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList())));
                     Long execTime = System.currentTimeMillis();
                     extractData(datasetTable, "incremental_add");
                     saveSucessLog(datasetTableTaskLog);
-                    sendWebMsg(datasetTable, null, true);
+//                    sendWebMsg(datasetTable, null, true);
                     updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime);
                 } catch (Exception e) {
                     saveErrorLog(datasetTableId, null, e);
-                    sendWebMsg(datasetTable, null, false);
+//                    sendWebMsg(datasetTable, null, false);
                     updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null);
                     deleteFile("incremental_add", datasetTableId);
                     deleteFile("incremental_delete", datasetTableId);
@@ -256,7 +252,7 @@ public class ExtractDataService {
             return;
         }
 
-        DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
+        DatasetTableTaskLog datasetTableTaskLog = getDatasetTableTaskLog(datasetTableId, taskId);
         UpdateType updateType = UpdateType.valueOf(type);
         if (context != null) {
             datasetTable.setQrtzInstance(context.getFireInstanceId());
@@ -283,11 +279,8 @@ public class ExtractDataService {
         switch (updateType) {
             case all_scope:  // 全量更新
                 try {
-                    if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.CRON.toString())) {
+                    if (datasetTableTask == null ) {
-                        datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId);
+                        datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, taskId);
-                    }
-                    if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) {
-                        datasetTableTaskLog = getDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId);
                     }
                     createDorisTable(DorisTableUtils.dorisName(datasetTableId), dorisTablColumnSql);
                     createDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)), dorisTablColumnSql);
@@ -336,11 +329,8 @@ public class ExtractDataService {
                         return;
                     }
 
-                    if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.CRON.toString())) {
+                    if (datasetTableTask == null ) {
-                        datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId);
+                        datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, taskId);
-                    }
-                    if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) {
-                        datasetTableTaskLog = getDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId);
                     }
                     Long execTime = System.currentTimeMillis();
                     if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) {// 增量添加
@@ -533,7 +523,8 @@ public class ExtractDataService {
         return null;
     }
 
-    private DatasetTableTaskLog writeDatasetTableTaskLog(DatasetTableTaskLog datasetTableTaskLog, String datasetTableId, String taskId) {
+    private DatasetTableTaskLog writeDatasetTableTaskLog(String datasetTableId, String taskId) {
+        DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
         datasetTableTaskLog.setTableId(datasetTableId);
         datasetTableTaskLog.setTaskId(taskId);
         datasetTableTaskLog.setStatus(JobStatus.Underway.name());
@@ -548,7 +539,8 @@ public class ExtractDataService {
         }
     }
 
-    private DatasetTableTaskLog getDatasetTableTaskLog(DatasetTableTaskLog datasetTableTaskLog, String datasetTableId, String taskId) {
+    private DatasetTableTaskLog getDatasetTableTaskLog(String datasetTableId, String taskId) {
+        DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
         datasetTableTaskLog.setTableId(datasetTableId);
         datasetTableTaskLog.setTaskId(taskId);
         datasetTableTaskLog.setStatus(JobStatus.Underway.name());
@@ -765,8 +757,7 @@ public class ExtractDataService {
                 udjcStep = udjc(datasetTableFields, DatasourceTypes.oracle);
                 break;
             case excel:
-                String filePath = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getData();
+                inputStep = excelInputStep(datasetTable.getInfo(), datasetTableFields);
-                inputStep = excelInputStep(filePath, datasetTableFields);
                 udjcStep = udjc(datasetTableFields, DatasourceTypes.excel);
             default:
                 break;
@@ -837,33 +828,20 @@ public class ExtractDataService {
         return fromStep;
     }
 
-    private StepMeta excelInputStep(String filePath, List<DatasetTableField> datasetTableFields) {
+    private StepMeta excelInputStep(String Info, List<DatasetTableField> datasetTableFields){
-        String suffix = filePath.substring(filePath.lastIndexOf(".") + 1);
+        DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(Info, DataTableInfoDTO.class);
+        String suffix = dataTableInfoDTO.getData().substring(dataTableInfoDTO.getData().lastIndexOf(".") + 1);
         ExcelInputMeta excelInputMeta = new ExcelInputMeta();
         if (StringUtils.equalsIgnoreCase(suffix, "xlsx")) {
             excelInputMeta.setSpreadSheetType(SpreadSheetType.SAX_POI);
-            try {
+            excelInputMeta.setSheetName(new String[]{dataTableInfoDTO.getSheets().get(0)});
-                InputStream inputStream = new FileInputStream(filePath);
-                XSSFWorkbook xssfWorkbook = new XSSFWorkbook(inputStream);
-                XSSFSheet sheet0 = xssfWorkbook.getSheetAt(0);
-                excelInputMeta.setSheetName(new String[]{sheet0.getSheetName()});
-            } catch (Exception e) {
-                e.printStackTrace();
-            }
         }
         if (StringUtils.equalsIgnoreCase(suffix, "xls")) {
             excelInputMeta.setSpreadSheetType(SpreadSheetType.JXL);
-            try {
+            excelInputMeta.setSheetName(new String[]{dataTableInfoDTO.getSheets().get(0)});
-                InputStream inputStream = new FileInputStream(filePath);
-                HSSFWorkbook workbook = new HSSFWorkbook(inputStream);
-                HSSFSheet sheet0 = workbook.getSheetAt(0);
-                excelInputMeta.setSheetName(new String[]{sheet0.getSheetName()});
-            } catch (Exception e) {
-                e.printStackTrace();
-            }
         }
         excelInputMeta.setPassword("Encrypted");
-        excelInputMeta.setFileName(new String[]{filePath});
+        excelInputMeta.setFileName(new String[]{dataTableInfoDTO.getData()});
         excelInputMeta.setStartsWithHeader(true);
         excelInputMeta.setIgnoreEmptyRows(true);
         ExcelInputField[] fields = new ExcelInputField[datasetTableFields.size()];

backend/src/main/resources/db/migration/V13__system_task.sql

@@ -23,4 +23,4 @@ ALTER TABLE `dataset_table_task` ADD COLUMN `extra_data` LONGTEXT NULL AFTER `la
 
 update dataset_table_task_log set trigger_type='Cron';
 
-
+update dataset_table_task_log set dataset_table_task_log.task_id='初始导入' where dataset_table_task_log.task_id is null;

frontend/src/views/dataset/add/AddExcel.vue

@@ -258,7 +258,7 @@ export default {
           type: 'excel',
           mode: parseInt(this.mode),
           // info: '{"data":"' + this.path + '"}',
-          info: JSON.stringify({ data: this.path }),
+          info: JSON.stringify({ data: this.path, sheets: [this.sheets[0]]}),
           fields: this.fields
         }
       } else {

frontend/src/views/dataset/data/UpdateInfo.vue

@@ -1,7 +1,7 @@
 <template>
   <el-col>
     <el-row>
-      <el-button v-if="hasDataPermission('manage',param.privileges)" icon="el-icon-setting" size="mini" @click="showConfig">
+      <el-button v-if="hasDataPermission('manage',param.privileges) || table.type !== 'excel'" icon="el-icon-setting" size="mini" @click="showConfig">
         {{ $t('dataset.update_setting') }}
       </el-button>
       <el-button icon="el-icon-refresh" size="mini" @click="refreshLog">
@@ -615,8 +615,8 @@ export default {
       }
     },
     listTaskLog(loading = true) {
-      const params = {"conditions":[{"field":"dataset_table_task.table_id","operator":"eq","value": this.table.id}],"orders":[]}
+      const params = {"conditions":[{"field":"dataset_table_task_log.table_id","operator":"eq","value": this.table.id}],"orders":[]}
-      post('/dataset/taskLog/list/' + this.page.currentPage + '/' + this.page.pageSize, params, loading).then(response => {
+      post('/dataset/taskLog/list/' +  this.table.type + '/' + this.page.currentPage + '/' + this.page.pageSize, params, loading).then(response => {
         this.taskLogData = response.data.listObject
         this.page.total = response.data.itemCount
       })

frontend/src/views/dataset/data/ViewTable.vue

@@ -51,7 +51,7 @@
       <el-tab-pane v-if="table.type !== 'custom'" :label="$t('dataset.join_view')" name="joinView">
         <union-view :param="param" :table="table" />
       </el-tab-pane>
-      <el-tab-pane v-if="table.mode === 1 && (table.type === 'db' || table.type === 'sql')" :label="$t('dataset.update_info')" name="updateInfo">
+      <el-tab-pane v-if="table.mode === 1 && (table.type === 'excel' || table.type === 'db' || table.type === 'sql')" :label="$t('dataset.update_info')" name="updateInfo">
         <update-info :param="param" :table="table" />
       </el-tab-pane>
     </el-tabs>

frontend/src/views/system/task/DatasetTaskList.vue

@@ -47,7 +47,7 @@
 
         <el-table-column prop="nextExecTime" :label="$t('dataset.task.next_exec_time')">
           <template slot-scope="scope" >
-            <span v-if="scope.row.nextExecTime && scope.row.rate !== 'SIMPLE'">
+            <span v-if="scope.row.nextExecTime && scope.row.nextExecTime !== -1 && scope.row.rate !== 'SIMPLE'">
               {{ scope.row.nextExecTime | timestampFormatDate }}
             </span>
             <span v-if="!scope.row.nextExecTime || scope.row.rate === 'SIMPLE'"></span>

frontend/src/views/system/task/TaskRecord.vue

@@ -198,7 +198,7 @@ export default {
       const temp = formatCondition(condition)
       const param = temp || {}
       param['orders'] = formatOrders(this.orderConditions)
-      post('/dataset/taskLog/list/' + this.paginationConfig.currentPage + '/' + this.paginationConfig.pageSize, param, showLoading).then(response => {
+      post('/dataset/taskLog/list/notexcel/' + this.paginationConfig.currentPage + '/' + this.paginationConfig.pageSize, param, showLoading).then(response => {
         this.data = response.data.listObject
         this.paginationConfig.total = response.data.itemCount
       })