diff --git a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java index 877c19e66f..bb09b35534 100644 --- a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java +++ b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java @@ -81,7 +81,7 @@ public class DatasetDataManage { private static Logger logger = LoggerFactory.getLogger(DatasetDataManage.class); - private static final List notFullDs = List.of("mysql", "mariadb", "Excel", "API"); + public static final List notFullDs = List.of("mysql", "mariadb", "Excel", "API"); public List getTableFields(DatasetTableDTO datasetTableDTO) throws Exception { List list = null; diff --git a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetGroupManage.java b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetGroupManage.java index 683cb561a6..d7cec6ff26 100644 --- a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetGroupManage.java +++ b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetGroupManage.java @@ -559,7 +559,7 @@ public class DatasetGroupManage { } } - private void geFullName(Long pid, List fullName) { + public void geFullName(Long pid, List fullName) { CoreDatasetGroup parent = coreDatasetGroupMapper.selectById(pid);// 查找父级folder fullName.add(parent.getName()); if (parent.getPid() != null && parent.getPid() != 0) { diff --git a/core/core-backend/src/main/java/io/dataease/dataset/server/DatasetTreeServer.java b/core/core-backend/src/main/java/io/dataease/dataset/server/DatasetTreeServer.java index 3e0f9e8aa9..3eb2f52478 100644 --- a/core/core-backend/src/main/java/io/dataease/dataset/server/DatasetTreeServer.java +++ b/core/core-backend/src/main/java/io/dataease/dataset/server/DatasetTreeServer.java @@ -1,12 +1,15 @@ package io.dataease.dataset.server; import io.dataease.api.dataset.DatasetTreeApi; +import io.dataease.api.dataset.dto.DataSetExportRequest; import io.dataease.api.dataset.dto.DatasetNodeDTO; import io.dataease.api.dataset.union.DatasetGroupInfoDTO; import io.dataease.api.dataset.vo.DataSetBarVO; import io.dataease.constant.LogOT; import io.dataease.constant.LogST; import io.dataease.dataset.manage.DatasetGroupManage; +import io.dataease.exportCenter.manage.ExportCenterManage; +import io.dataease.exportCenter.server.ExportCenterServer; import io.dataease.extensions.datasource.dto.DatasetTableDTO; import io.dataease.extensions.view.dto.SqlVariableDetails; import io.dataease.log.DeLog; @@ -23,6 +26,8 @@ import java.util.List; public class DatasetTreeServer implements DatasetTreeApi { @Resource private DatasetGroupManage datasetGroupManage; + @Resource + private ExportCenterManage exportCenterManage; @DeLog(id = "#p0.id", ot = LogOT.MODIFY, st = LogST.DATASET) @@ -95,4 +100,9 @@ public class DatasetTreeServer implements DatasetTreeApi { return datasetGroupManage.getDetailWithPerm(ids); } + @Override + public void exportDataset(DataSetExportRequest request) throws Exception { + exportCenterManage.addTask(request.getId(), "dataset", request); + } + } diff --git a/core/core-backend/src/main/java/io/dataease/datasource/provider/CalciteProvider.java b/core/core-backend/src/main/java/io/dataease/datasource/provider/CalciteProvider.java index e038611fbb..45a7a28ea3 100644 --- a/core/core-backend/src/main/java/io/dataease/datasource/provider/CalciteProvider.java +++ b/core/core-backend/src/main/java/io/dataease/datasource/provider/CalciteProvider.java @@ -947,7 +947,7 @@ public class CalciteProvider extends Provider { if (database.contains(".")) { sql = "select * from " + datasourceRequest.getTable() + " limit 0 offset 0 "; } else { - sql = String.format("SELECT TABLE_NAME,TABLE_COMMENT FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '%s' ;", database); + sql = String.format("SELECT COLUMN_NAME,DATA_TYPE,COLUMN_COMMENT FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s'", database, datasourceRequest.getTable()); } break; case mysql: diff --git a/core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterManage.java b/core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterManage.java index 7b8638641e..d406b741e8 100644 --- a/core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterManage.java +++ b/core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterManage.java @@ -2,17 +2,42 @@ package io.dataease.exportCenter.manage; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.fasterxml.jackson.core.type.TypeReference; import io.dataease.api.chart.dto.ViewDetailField; import io.dataease.api.chart.request.ChartExcelRequest; import io.dataease.api.chart.request.ChartExcelRequestInner; -import io.dataease.api.exportCenter.vo.ExportTaskDTO; +import io.dataease.api.dataset.dto.DataSetExportRequest; +import io.dataease.api.dataset.union.DatasetGroupInfoDTO; +import io.dataease.api.dataset.union.UnionDTO; +import io.dataease.model.ExportTaskDTO; +import io.dataease.api.permissions.dataset.dto.DataSetRowPermissionsTreeDTO; import io.dataease.auth.bo.TokenUserBO; import io.dataease.chart.dao.auto.mapper.CoreChartViewMapper; import io.dataease.chart.server.ChartDataServer; +import io.dataease.dataset.dao.auto.entity.CoreDatasetGroup; +import io.dataease.dataset.dao.auto.mapper.CoreDatasetGroupMapper; +import io.dataease.dataset.manage.*; +import io.dataease.datasource.utils.DatasourceUtils; +import io.dataease.engine.constant.DeTypeConstants; +import io.dataease.engine.sql.SQLProvider; +import io.dataease.engine.trans.Field2SQLObj; +import io.dataease.engine.trans.Order2SQLObj; +import io.dataease.engine.trans.Table2SQLObj; +import io.dataease.engine.trans.WhereTree2Str; +import io.dataease.engine.utils.Utils; import io.dataease.exception.DEException; import io.dataease.exportCenter.dao.auto.entity.CoreExportTask; import io.dataease.exportCenter.dao.auto.mapper.CoreExportTaskMapper; +import io.dataease.extensions.datasource.dto.DatasetTableFieldDTO; +import io.dataease.extensions.datasource.dto.DatasourceRequest; +import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO; +import io.dataease.extensions.datasource.factory.ProviderFactory; +import io.dataease.extensions.datasource.model.SQLMeta; +import io.dataease.extensions.datasource.provider.Provider; +import io.dataease.extensions.view.dto.ColumnPermissionItem; +import io.dataease.i18n.Translator; import io.dataease.license.config.XpackInteract; +import io.dataease.system.manage.CoreUserManage; import io.dataease.system.manage.SysParameterManage; import io.dataease.utils.*; import io.dataease.visualization.server.DataVisualizationServer; @@ -22,6 +47,7 @@ import jakarta.annotation.PostConstruct; import jakarta.annotation.Resource; import jakarta.servlet.http.HttpServletResponse; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.poi.ss.usermodel.*; import org.apache.poi.xssf.streaming.SXSSFWorkbook; @@ -33,10 +59,12 @@ import org.springframework.transaction.annotation.Transactional; import java.io.*; import java.net.InetAddress; +import java.text.DecimalFormat; import java.util.*; import java.util.concurrent.Future; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; @Component @Transactional(rollbackFor = Exception.class) @@ -44,6 +72,8 @@ public class ExportCenterManage { @Resource private CoreExportTaskMapper exportTaskMapper; @Resource + private DatasetGroupManage datasetGroupManage; + @Resource DataVisualizationServer dataVisualizationServer; @Resource private CoreChartViewMapper coreChartViewMapper; @@ -68,6 +98,18 @@ public class ExportCenterManage { private Map Running_Task = new HashMap<>(); @Resource private ChartDataServer chartDataServer; + @Resource + private CoreDatasetGroupMapper coreDatasetGroupMapper; + @Resource + private CoreUserManage coreUserManage; + @Resource + private DatasetSQLManage datasetSQLManage; + @Resource + private PermissionManage permissionManage; + @Resource + private DatasetTableFieldManage datasetTableFieldManage; + @Resource + private DatasetDataManage datasetDataManage; @PostConstruct public void init() { @@ -158,7 +200,7 @@ public class ExportCenterManage { public void retry(String id) { CoreExportTask exportTask = exportTaskMapper.selectById(id); - if(!exportTask.getExportStatus().equalsIgnoreCase("FAILED")){ + if (!exportTask.getExportStatus().equalsIgnoreCase("FAILED")) { DEException.throwException("正在导出中!"); } exportTask.setExportStatus("PENDING"); @@ -198,7 +240,8 @@ public class ExportCenterManage { } @XpackInteract(value = "exportCenter", before = false) - public void setOrg(ExportTaskDTO exportTaskDTO) {} + public void setOrg(ExportTaskDTO exportTaskDTO) { + } private ExportCenterManage proxy() { return CommonBeanFactory.getBean(ExportCenterManage.class); @@ -208,6 +251,13 @@ public class ExportCenterManage { if (exportTaskDTO.getExportFromType().equalsIgnoreCase("chart")) { exportTaskDTO.setExportFromName(dataVisualizationServer.getAbsPath(exportTaskDTO.getExportFrom())); } + if (exportTaskDTO.getExportFromType().equalsIgnoreCase("dataset")) { + List fullName = new ArrayList<>(); + datasetGroupManage.geFullName(Long.valueOf(exportTaskDTO.getExportFrom()), fullName); + Collections.reverse(fullName); + List finalFullName = fullName; + exportTaskDTO.setExportFromName(String.join("/", finalFullName)); + } } private void setExportFromName(ExportTaskDTO exportTaskDTO) { @@ -243,6 +293,216 @@ public class ExportCenterManage { startViewTask(exportTask, request); } + public void addTask(Long exportFrom, String exportFromType, DataSetExportRequest request) { + CoreExportTask exportTask = new CoreExportTask(); + exportTask.setId(UUID.randomUUID().toString()); + exportTask.setUserId(AuthUtils.getUser().getUserId()); + exportTask.setExportFrom(String.valueOf(exportFrom)); + exportTask.setExportFromType(exportFromType); + exportTask.setExportStatus("PENDING"); + exportTask.setFileName(request.getFilename() + ".xlsx"); + exportTask.setExportProgress("0"); + exportTask.setExportTime(System.currentTimeMillis()); + exportTask.setParams(JsonUtil.toJSONString(request).toString()); + exportTask.setExportMachineName(hostName()); + exportTaskMapper.insert(exportTask); + startDatasetTask(exportTask, request); + } + + private void startDatasetTask(CoreExportTask exportTask, DataSetExportRequest request) { + String dataPath = exportData_path + exportTask.getId(); + File directory = new File(dataPath); + boolean isCreated = directory.mkdir(); + + TokenUserBO tokenUserBO = AuthUtils.getUser(); + Future future = scheduledThreadPoolExecutor.submit(() -> { + AuthUtils.setUser(tokenUserBO); + try { + exportTask.setExportStatus("IN_PROGRESS"); + exportTaskMapper.updateById(exportTask); + CoreDatasetGroup coreDatasetGroup = coreDatasetGroupMapper.selectById(exportTask.getExportFrom()); + if (coreDatasetGroup == null) { + throw new Exception("Not found dataset group: " + exportTask.getExportFrom()); + } + DatasetGroupInfoDTO dto = new DatasetGroupInfoDTO(); + BeanUtils.copyBean(dto, coreDatasetGroup); + dto.setUnionSql(null); + List unionDTOList = JsonUtil.parseList(coreDatasetGroup.getInfo(), new TypeReference<>() { + }); + dto.setUnion(unionDTOList); + List dsFields = datasetTableFieldManage.selectByDatasetGroupId(Long.valueOf(exportTask.getExportFrom())); + List allFields = dsFields.stream().map(ele -> { + DatasetTableFieldDTO datasetTableFieldDTO = new DatasetTableFieldDTO(); + BeanUtils.copyBean(datasetTableFieldDTO, ele); + datasetTableFieldDTO.setFieldShortName(ele.getDataeaseName()); + return datasetTableFieldDTO; + }).collect(Collectors.toList()); + dto.setAllFields(allFields); + Map sqlMap = datasetSQLManage.getUnionSQLForEdit(dto, null); + String sql = (String) sqlMap.get("sql"); + if (ObjectUtils.isEmpty(allFields)) { + DEException.throwException(Translator.get("i18n_no_fields")); + } + Map desensitizationList = new HashMap<>(); + allFields = permissionManage.filterColumnPermissions(allFields, desensitizationList, dto.getId(), null); + if (ObjectUtils.isEmpty(allFields)) { + DEException.throwException(Translator.get("i18n_no_column_permission")); + } + datasetDataManage.buildFieldName(sqlMap, allFields); + Map dsMap = (Map) sqlMap.get("dsMap"); + DatasourceUtils.checkDsStatus(dsMap); + List dsList = new ArrayList<>(); + for (Map.Entry next : dsMap.entrySet()) { + dsList.add(next.getValue().getType()); + } + boolean needOrder = Utils.isNeedOrder(dsList); + boolean crossDs = Utils.isCrossDs(dsMap); + if (!crossDs) { + if (datasetDataManage.notFullDs.contains(dsMap.entrySet().iterator().next().getValue().getType()) && (boolean) sqlMap.get("isFullJoin")) { + DEException.throwException(Translator.get("i18n_not_full")); + } + sql = Utils.replaceSchemaAlias(sql, dsMap); + } + + List rowPermissionsTree = new ArrayList<>(); + TokenUserBO user = AuthUtils.getUser(); + if (user != null) { + rowPermissionsTree = permissionManage.getRowPermissionsTree(dto.getId(), user.getUserId()); + } + + Provider provider; + if (crossDs) { + provider = ProviderFactory.getDefaultProvider(); + } else { + provider = ProviderFactory.getProvider(dsList.getFirst()); + } + SQLMeta sqlMeta = new SQLMeta(); + Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs); + Field2SQLObj.field2sqlObj(sqlMeta, allFields, allFields, crossDs, dsMap, Utils.getParams(allFields), null); + WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, allFields, crossDs, dsMap, Utils.getParams(allFields), null); + Order2SQLObj.getOrders(sqlMeta, dto.getSortFields(), allFields, crossDs, dsMap, Utils.getParams(allFields), null); + String replaceSql = provider.rebuildSQL(SQLProvider.createQuerySQL(sqlMeta, false, false, false), sqlMeta, crossDs, dsMap); + Long totalCount = datasetDataManage.getDatasetTotal(dto, replaceSql, null); + totalCount = totalCount > limit ? limit : totalCount; + Long totalPage = (totalCount / extractPageSize) + (totalCount % extractPageSize > 0 ? 1 : 0); + + + Workbook wb = new SXSSFWorkbook(); + FileOutputStream fileOutputStream = new FileOutputStream(dataPath + "/" + request.getFilename() + ".xlsx"); + Sheet detailsSheet = wb.createSheet("数据"); + + for (Integer p = 0; p < totalPage; p++) { + String querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, p * extractPageSize, p * extractPageSize + extractPageSize); + querySQL = provider.rebuildSQL(querySQL, sqlMeta, crossDs, dsMap); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setQuery(querySQL); + datasourceRequest.setDsList(dsMap); + Map previewData = datasetDataManage.buildPreviewData(provider.fetchResultField(datasourceRequest), allFields, desensitizationList); + List> data = (List>) previewData.get("data"); + if (p == 1L) { + CellStyle cellStyle = wb.createCellStyle(); + Font font = wb.createFont(); + font.setFontHeightInPoints((short) 12); + font.setBold(true); + cellStyle.setFont(font); + cellStyle.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex()); + cellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND); + List header = new ArrayList<>(); + for (DatasetTableFieldDTO field : allFields) { + header.add(field.getName()); + } + List> details = new ArrayList<>(); + details.add(header); + for (Map obj : data) { + List row = new ArrayList<>(); + for (DatasetTableFieldDTO field : allFields) { + String string = (String) obj.get(field.getDataeaseName()); + row.add(string); + } + details.add(row); + } + if (CollectionUtils.isNotEmpty(details)) { + for (int i = 0; i < details.size(); i++) { + Row row = detailsSheet.createRow(i); + List rowData = details.get(i); + if (rowData != null) { + for (int j = 0; j < rowData.size(); j++) { + Cell cell = row.createCell(j); + if (i == 0) { + cell.setCellValue(rowData.get(j)); + cell.setCellStyle(cellStyle); + detailsSheet.setColumnWidth(j, 255 * 20); + } else { + if ((allFields.get(j).getDeType().equals(DeTypeConstants.DE_INT) || allFields.get(j).getDeType() == DeTypeConstants.DE_FLOAT) && StringUtils.isNotEmpty(rowData.get(j))) { + try { + cell.setCellValue(Double.valueOf(rowData.get(j))); + } catch (Exception e) { + LogUtil.warn("export excel data transform error"); + } + } else { + cell.setCellValue(rowData.get(j)); + } + } + } + } + } + } + } else { + List> details = new ArrayList<>(); + for (Map obj : data) { + List row = new ArrayList<>(); + for (DatasetTableFieldDTO field : allFields) { + String string = (String) obj.get(field.getDataeaseName()); + row.add(string); + } + details.add(row); + } + int lastNum = detailsSheet.getLastRowNum(); + for (int i = 0; i < details.size(); i++) { + Row row = detailsSheet.createRow(i + lastNum + 1); + List rowData = details.get(i); + if (rowData != null) { + for (int j = 0; j < rowData.size(); j++) { + Cell cell = row.createCell(j); + if ((allFields.get(j).getDeType().equals(DeTypeConstants.DE_INT) || allFields.get(j).getDeType() == DeTypeConstants.DE_FLOAT) && StringUtils.isNotEmpty(rowData.get(j))) { + try { + cell.setCellValue(Double.valueOf(rowData.get(j))); + } catch (Exception e) { + LogUtil.warn("export excel data transform error"); + } + } else { + cell.setCellValue(rowData.get(j)); + } + } + } + } + } + exportTask.setExportStatus("IN_PROGRESS"); + double exportRogress = (double) ((double) p / (double) totalPage); + DecimalFormat df = new DecimalFormat("#.##"); + String formattedResult = df.format(exportRogress * 100); + exportTask.setExportProgress(formattedResult); + exportTaskMapper.updateById(exportTask); + } + wb.write(fileOutputStream); + fileOutputStream.flush(); + fileOutputStream.close(); + wb.close(); + exportTask.setExportProgress("100"); + exportTask.setExportStatus("SUCCESS"); + setFileSize(dataPath + "/" + request.getFilename() + ".xlsx", exportTask); + + } catch (Exception e) { + LogUtil.error("Failed to export data", e); + exportTask.setMsg(e.getMessage()); + exportTask.setExportStatus("FAILED"); + } finally { + exportTaskMapper.updateById(exportTask); + } + }); + Running_Task.put(exportTask.getId(), future); + } + private void startViewTask(CoreExportTask exportTask, ChartExcelRequest request) { String dataPath = exportData_path + exportTask.getId(); File directory = new File(dataPath); diff --git a/core/core-backend/src/main/java/io/dataease/exportCenter/server/ExportCenterServer.java b/core/core-backend/src/main/java/io/dataease/exportCenter/server/ExportCenterServer.java index a416178e38..d3f180a7c6 100644 --- a/core/core-backend/src/main/java/io/dataease/exportCenter/server/ExportCenterServer.java +++ b/core/core-backend/src/main/java/io/dataease/exportCenter/server/ExportCenterServer.java @@ -1,7 +1,7 @@ package io.dataease.exportCenter.server; import io.dataease.api.exportCenter.ExportCenterApi; -import io.dataease.api.exportCenter.vo.ExportTaskDTO; +import io.dataease.model.ExportTaskDTO; import io.dataease.exportCenter.manage.ExportCenterManage; import jakarta.annotation.Resource; import jakarta.servlet.http.HttpServletResponse; diff --git a/core/core-backend/src/main/resources/application-standalone.yml b/core/core-backend/src/main/resources/application-standalone.yml index b521aa59d0..a61ebf66be 100644 --- a/core/core-backend/src/main/resources/application-standalone.yml +++ b/core/core-backend/src/main/resources/application-standalone.yml @@ -1,8 +1,8 @@ spring: datasource: - url: jdbc:mysql://localhost:3306/dataease?autoReconnect=false&useUnicode=true&characterEncoding=UTF-8&characterSetResults=UTF-8&zeroDateTimeBehavior=convertToNull&useSSL=false&allowPublicKeyRetrieval=true + url: jdbc:mysql://39.101.192.183:13306/dataease?autoReconnect=false&useUnicode=true&characterEncoding=UTF-8&characterSetResults=UTF-8&zeroDateTimeBehavior=convertToNull&useSSL=false&allowPublicKeyRetrieval=true username: root - password: 123456 + password: Password123@mysql messages: basename: i18n/lic,i18n/core,i18n/permissions,i18n/xpack flyway: diff --git a/core/core-frontend/src/api/dataset.ts b/core/core-frontend/src/api/dataset.ts index d765b63460..f013b8ca14 100644 --- a/core/core-frontend/src/api/dataset.ts +++ b/core/core-frontend/src/api/dataset.ts @@ -120,6 +120,16 @@ export const delDatasetTree = async (id): Promise => { }) } +export const exportDatasetData = async (data): Promise => { + return request.post({ + url: '/datasetTree/exportDataset', + method: 'post', + data: data, + loading: true, + responseType: 'blob' + }) +} + export const perDelete = async (id): Promise => { return request.post({ url: `/datasetTree/perDelete/${id}`, data: {} }).then(res => { return res?.data diff --git a/core/core-frontend/src/views/visualized/data/dataset/index.vue b/core/core-frontend/src/views/visualized/data/dataset/index.vue index e6ecd9c881..e95b6aae84 100644 --- a/core/core-frontend/src/views/visualized/data/dataset/index.vue +++ b/core/core-frontend/src/views/visualized/data/dataset/index.vue @@ -1,6 +1,18 @@