diff --git a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableTaskLogController.java b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableTaskLogController.java index 4b0123fe3c..6b7a84c6be 100644 --- a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableTaskLogController.java +++ b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableTaskLogController.java @@ -33,10 +33,10 @@ public class DataSetTableTaskLogController { dataSetTableTaskLogService.delete(id); } - @PostMapping("list/{goPage}/{pageSize}") - public Pager> list(@RequestBody BaseGridRequest request, @PathVariable int goPage, @PathVariable int pageSize) { + @PostMapping("list/{type}/{goPage}/{pageSize}") + public Pager> list(@RequestBody BaseGridRequest request, @PathVariable String type, @PathVariable int goPage, @PathVariable int pageSize) { Page page = PageHelper.startPage(goPage, pageSize, true); - return PageUtils.setPageInfo(page, dataSetTableTaskLogService.list(request)); + return PageUtils.setPageInfo(page, dataSetTableTaskLogService.list(request, type)); } } diff --git a/backend/src/main/java/io/dataease/dto/dataset/DataTableInfoDTO.java b/backend/src/main/java/io/dataease/dto/dataset/DataTableInfoDTO.java index ffa73619cd..a75f12dbba 100644 --- a/backend/src/main/java/io/dataease/dto/dataset/DataTableInfoDTO.java +++ b/backend/src/main/java/io/dataease/dto/dataset/DataTableInfoDTO.java @@ -14,6 +14,7 @@ import java.util.List; public class DataTableInfoDTO { private String table; private String sql; + private List sheets; private String data;// file path private List list; } diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index 00c54ea4e5..f0507b0148 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -101,7 +101,7 @@ public class DataSetTableService { private void extractData(DataSetTableRequest datasetTable) throws Exception { if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) { commonThreadPool.addTask(() -> { - extractDataService.extractExcelData(datasetTable.getId(), "all_scope"); + extractDataService.extractExcelData(datasetTable.getId(), "all_scope", "初始导入"); }); return; } @@ -131,7 +131,6 @@ public class DataSetTableService { datasetTable.setId(UUID.randomUUID().toString()); datasetTable.setCreateBy(AuthUtils.getUser().getUsername()); datasetTable.setCreateTime(System.currentTimeMillis()); - DataTableInfoDTO dataTableInfoDTO = new DataTableInfoDTO(); int insert = datasetTableMapper.insert(datasetTable); // 添加表成功后，获取当前表字段和类型，抽象到dataease数据库 if (insert == 1) { @@ -151,11 +150,11 @@ public class DataSetTableService { if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) { if (datasetTable.getEditType() == 0) { commonThreadPool.addTask(() -> { - extractDataService.extractExcelData(datasetTable.getId(), "all_scope"); + extractDataService.extractExcelData(datasetTable.getId(), "all_scope", "替换"); }); } else if (datasetTable.getEditType() == 1) { commonThreadPool.addTask(() -> { - extractDataService.extractExcelData(datasetTable.getId(), "add_scope"); + extractDataService.extractExcelData(datasetTable.getId(), "add_scope", "追加"); }); } } @@ -416,38 +415,27 @@ public class DataSetTableService { } } } else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) { - if (StringUtils.isEmpty(datasetTable.getSyncStatus()) || datasetTable.getSyncStatus().equalsIgnoreCase(JobStatus.Underway.name())) { - map.put("status", "warnning"); - map.put("msg", Translator.get("i18n_processing_data")); - dataSetPreviewPage.setTotal(0); - } else if (datasetTable.getSyncStatus().equalsIgnoreCase(JobStatus.Error.name())) { - List datasetTableTaskLogs = dataSetTableTaskLogService.getByTableId(datasetTable.getId()); - map.put("status", "error"); - if (CollectionUtils.isNotEmpty(datasetTableTaskLogs)) { - map.put("msg", "Failed to extract data: " + datasetTableTaskLogs.get(0).getInfo()); - } else { - map.put("msg", "Failed to extract data."); - } - dataSetPreviewPage.setTotal(0); - } else { - Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); - JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); - DatasourceRequest datasourceRequest = new DatasourceRequest(); - datasourceRequest.setDatasource(ds); - String table = DorisTableUtils.dorisName(dataSetTableRequest.getId()); - QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); - datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize)); - try { - data.addAll(jdbcProvider.getData(datasourceRequest)); - } catch (Exception e) { - e.printStackTrace(); - } - try { - datasourceRequest.setQuery(qp.createQueryTableWithLimit(table, fields, Integer.valueOf(dataSetTableRequest.getRow()))); - dataSetPreviewPage.setTotal(Integer.valueOf(jdbcProvider.getData(datasourceRequest).get(0)[0])); - } catch (Exception e) { - e.printStackTrace(); - } + if (!checkDorisTableIsExists(dataSetTableRequest.getId())) { + throw new RuntimeException(Translator.get("i18n_data_not_sync")); + } + + Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(ds); + String table = DorisTableUtils.dorisName(dataSetTableRequest.getId()); + QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); + datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize)); + try { + data.addAll(jdbcProvider.getData(datasourceRequest)); + } catch (Exception e) { + e.printStackTrace(); + } + try { + datasourceRequest.setQuery(qp.createQueryTableWithLimit(table, fields, Integer.valueOf(dataSetTableRequest.getRow()))); + dataSetPreviewPage.setTotal(Integer.valueOf(jdbcProvider.getData(datasourceRequest).get(0)[0])); + } catch (Exception e) { + e.printStackTrace(); } } else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "custom")) { if (datasetTable.getMode() == 0) { diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java index c44a62b9fe..80117e76c8 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java @@ -47,18 +47,27 @@ public class DataSetTableTaskLogService { datasetTableTaskLogMapper.deleteByPrimaryKey(id); } - public List list(BaseGridRequest request) { - ConditionEntity entity = new ConditionEntity(); - entity.setField("task_id"); - entity.setOperator("not null"); - List conditionEntities = request.getConditions(); - if(CollectionUtils.isEmpty(conditionEntities)){ - conditionEntities = new ArrayList<>(); + public List list(BaseGridRequest request, String type) { + if(!type.equalsIgnoreCase("excel")){ + ConditionEntity entity = new ConditionEntity(); + entity.setField("task_id"); + entity.setOperator("not null"); + List conditionEntities = request.getConditions(); + if(CollectionUtils.isEmpty(conditionEntities)){ + conditionEntities = new ArrayList<>(); + } + conditionEntities.add(entity); + request.setConditions(conditionEntities); } - conditionEntities.add(entity); - request.setConditions(conditionEntities); + GridExample gridExample = request.convertExample(); - return extDataSetTaskMapper.list(gridExample); + List dataSetTaskLogDTOS = extDataSetTaskMapper.list(gridExample); + dataSetTaskLogDTOS.forEach(dataSetTaskLogDTO -> { + if(StringUtils.isEmpty(dataSetTaskLogDTO.getName())){ + dataSetTaskLogDTO.setName(dataSetTaskLogDTO.getTaskId()); + } + }); + return dataSetTaskLogDTOS; } public void deleteByTaskId(String taskId){ diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java index bd491abcc4..0278a7cd93 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java @@ -13,6 +13,7 @@ import io.dataease.commons.constants.TaskStatus; import io.dataease.commons.constants.TriggerType; import io.dataease.controller.request.dataset.DataSetTaskRequest; import io.dataease.controller.sys.base.BaseGridRequest; +import io.dataease.controller.sys.base.ConditionEntity; import io.dataease.controller.sys.response.SysUserGridResponse; import io.dataease.controller.sys.response.SysUserRole; import io.dataease.dto.dataset.DataSetTaskDTO; @@ -28,6 +29,7 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import javax.annotation.Resource; +import java.util.Arrays; import java.util.List; import java.util.UUID; import java.util.stream.Collectors; @@ -177,8 +179,20 @@ public class DataSetTableTaskService { datasetTableTask.setStatus(TaskStatus.Stopped.name()); }else { if(StringUtils.isNotEmpty(datasetTableTask.getEnd()) && datasetTableTask.getEnd().equalsIgnoreCase("1")){ - if(utilMapper.currentTimestamp() > datasetTableTask.getEndTime()){ + BaseGridRequest request = new BaseGridRequest(); + ConditionEntity conditionEntity = new ConditionEntity(); + conditionEntity.setField("dataset_table_task.id"); + conditionEntity.setOperator("eq"); + conditionEntity.setValue(datasetTableTask.getId()); + request.setConditions(Arrays.asList(conditionEntity)); + List dataSetTaskDTOS = taskList(request); + if(CollectionUtils.isEmpty(dataSetTaskDTOS)){ + return; + } + if(dataSetTaskDTOS.get(0).getNextExecTime() == null || dataSetTaskDTOS.get(0).getNextExecTime() <= 0){ datasetTableTask.setStatus(TaskStatus.Stopped.name()); + }else { + datasetTableTask.setStatus(TaskStatus.Underway.name()); } }else { datasetTableTask.setStatus(TaskStatus.Underway.name()); diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index 9624aa842a..c40599503b 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -158,7 +158,7 @@ public class ExtractDataService { } } - public void extractExcelData(String datasetTableId, String type) { + public void extractExcelData(String datasetTableId, String type, String ops) { Datasource datasource = new Datasource(); datasource.setType("excel"); DatasetTable datasetTable = getDatasetTable(datasetTableId); @@ -182,15 +182,11 @@ public class ExtractDataService { switch (updateType) { case all_scope: // 全量更新 try { - datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, null); + datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops); createDorisTable(DorisTableUtils.dorisName(datasetTableId), dorisTablColumnSql); createDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)), dorisTablColumnSql); generateTransFile("all_scope", datasetTable, datasource, datasetTableFields, null); - if (datasetTable.getType().equalsIgnoreCase("sql")) { - generateJobFile("all_scope", datasetTable, fetchSqlField(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getSql(), datasource)); - } else { - generateJobFile("all_scope", datasetTable, String.join(",", datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList()))); - } + generateJobFile("all_scope", datasetTable, String.join(",", datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList()))); Long execTime = System.currentTimeMillis(); extractData(datasetTable, "all_scope"); replaceTable(DorisTableUtils.dorisName(datasetTableId)); @@ -210,17 +206,17 @@ public class ExtractDataService { case add_scope: // 增量更新 try { - datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, null); + datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops); generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, null); generateJobFile("incremental_add", datasetTable, String.join(",", datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList()))); Long execTime = System.currentTimeMillis(); extractData(datasetTable, "incremental_add"); saveSucessLog(datasetTableTaskLog); - sendWebMsg(datasetTable, null, true); +// sendWebMsg(datasetTable, null, true); updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime); } catch (Exception e) { saveErrorLog(datasetTableId, null, e); - sendWebMsg(datasetTable, null, false); +// sendWebMsg(datasetTable, null, false); updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null); deleteFile("incremental_add", datasetTableId); deleteFile("incremental_delete", datasetTableId); @@ -256,7 +252,7 @@ public class ExtractDataService { return; } - DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); + DatasetTableTaskLog datasetTableTaskLog = getDatasetTableTaskLog(datasetTableId, taskId); UpdateType updateType = UpdateType.valueOf(type); if (context != null) { datasetTable.setQrtzInstance(context.getFireInstanceId()); @@ -283,11 +279,8 @@ public class ExtractDataService { switch (updateType) { case all_scope: // 全量更新 try { - if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.CRON.toString())) { - datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId); - } - if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { - datasetTableTaskLog = getDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId); + if (datasetTableTask == null ) { + datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, taskId); } createDorisTable(DorisTableUtils.dorisName(datasetTableId), dorisTablColumnSql); createDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)), dorisTablColumnSql); @@ -336,11 +329,8 @@ public class ExtractDataService { return; } - if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.CRON.toString())) { - datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId); - } - if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { - datasetTableTaskLog = getDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId); + if (datasetTableTask == null ) { + datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, taskId); } Long execTime = System.currentTimeMillis(); if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) {// 增量添加 @@ -533,7 +523,8 @@ public class ExtractDataService { return null; } - private DatasetTableTaskLog writeDatasetTableTaskLog(DatasetTableTaskLog datasetTableTaskLog, String datasetTableId, String taskId) { + private DatasetTableTaskLog writeDatasetTableTaskLog(String datasetTableId, String taskId) { + DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); datasetTableTaskLog.setTableId(datasetTableId); datasetTableTaskLog.setTaskId(taskId); datasetTableTaskLog.setStatus(JobStatus.Underway.name()); @@ -548,7 +539,8 @@ public class ExtractDataService { } } - private DatasetTableTaskLog getDatasetTableTaskLog(DatasetTableTaskLog datasetTableTaskLog, String datasetTableId, String taskId) { + private DatasetTableTaskLog getDatasetTableTaskLog(String datasetTableId, String taskId) { + DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); datasetTableTaskLog.setTableId(datasetTableId); datasetTableTaskLog.setTaskId(taskId); datasetTableTaskLog.setStatus(JobStatus.Underway.name()); @@ -765,8 +757,7 @@ public class ExtractDataService { udjcStep = udjc(datasetTableFields, DatasourceTypes.oracle); break; case excel: - String filePath = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getData(); - inputStep = excelInputStep(filePath, datasetTableFields); + inputStep = excelInputStep(datasetTable.getInfo(), datasetTableFields); udjcStep = udjc(datasetTableFields, DatasourceTypes.excel); default: break; @@ -837,33 +828,20 @@ public class ExtractDataService { return fromStep; } - private StepMeta excelInputStep(String filePath, List datasetTableFields) { - String suffix = filePath.substring(filePath.lastIndexOf(".") + 1); + private StepMeta excelInputStep(String Info, List datasetTableFields){ + DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(Info, DataTableInfoDTO.class); + String suffix = dataTableInfoDTO.getData().substring(dataTableInfoDTO.getData().lastIndexOf(".") + 1); ExcelInputMeta excelInputMeta = new ExcelInputMeta(); if (StringUtils.equalsIgnoreCase(suffix, "xlsx")) { excelInputMeta.setSpreadSheetType(SpreadSheetType.SAX_POI); - try { - InputStream inputStream = new FileInputStream(filePath); - XSSFWorkbook xssfWorkbook = new XSSFWorkbook(inputStream); - XSSFSheet sheet0 = xssfWorkbook.getSheetAt(0); - excelInputMeta.setSheetName(new String[]{sheet0.getSheetName()}); - } catch (Exception e) { - e.printStackTrace(); - } + excelInputMeta.setSheetName(new String[]{dataTableInfoDTO.getSheets().get(0)}); } if (StringUtils.equalsIgnoreCase(suffix, "xls")) { excelInputMeta.setSpreadSheetType(SpreadSheetType.JXL); - try { - InputStream inputStream = new FileInputStream(filePath); - HSSFWorkbook workbook = new HSSFWorkbook(inputStream); - HSSFSheet sheet0 = workbook.getSheetAt(0); - excelInputMeta.setSheetName(new String[]{sheet0.getSheetName()}); - } catch (Exception e) { - e.printStackTrace(); - } + excelInputMeta.setSheetName(new String[]{dataTableInfoDTO.getSheets().get(0)}); } excelInputMeta.setPassword("Encrypted"); - excelInputMeta.setFileName(new String[]{filePath}); + excelInputMeta.setFileName(new String[]{dataTableInfoDTO.getData()}); excelInputMeta.setStartsWithHeader(true); excelInputMeta.setIgnoreEmptyRows(true); ExcelInputField[] fields = new ExcelInputField[datasetTableFields.size()]; diff --git a/backend/src/main/resources/db/migration/V13__system_task.sql b/backend/src/main/resources/db/migration/V13__system_task.sql index 4caafbc099..b3a7d7433a 100644 --- a/backend/src/main/resources/db/migration/V13__system_task.sql +++ b/backend/src/main/resources/db/migration/V13__system_task.sql @@ -23,4 +23,4 @@ ALTER TABLE `dataset_table_task` ADD COLUMN `extra_data` LONGTEXT NULL AFTER `la update dataset_table_task_log set trigger_type='Cron'; - +update dataset_table_task_log set dataset_table_task_log.task_id='初始导入' where dataset_table_task_log.task_id is null; diff --git a/frontend/src/views/dataset/add/AddExcel.vue b/frontend/src/views/dataset/add/AddExcel.vue index 40de024f65..55f99bdff2 100644 --- a/frontend/src/views/dataset/add/AddExcel.vue +++ b/frontend/src/views/dataset/add/AddExcel.vue @@ -258,7 +258,7 @@ export default { type: 'excel', mode: parseInt(this.mode), // info: '{"data":"' + this.path + '"}', - info: JSON.stringify({ data: this.path }), + info: JSON.stringify({ data: this.path, sheets: [this.sheets[0]]}), fields: this.fields } } else { diff --git a/frontend/src/views/dataset/data/UpdateInfo.vue b/frontend/src/views/dataset/data/UpdateInfo.vue index 3bd10fd436..0d4ee78f38 100644 --- a/frontend/src/views/dataset/data/UpdateInfo.vue +++ b/frontend/src/views/dataset/data/UpdateInfo.vue @@ -1,7 +1,7 @@