diff --git a/core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterManage.java b/core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterManage.java index 61e95591d2..93acc84134 100644 --- a/core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterManage.java +++ b/core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterManage.java @@ -96,8 +96,8 @@ public class ExportCenterManage implements BaseExportApi { @Value("${dataease.export.max.size:10}") private int max; - @Value("${dataease.export.dataset.limit:100000}") - private int limit; + @Value("${dataease.export.dataset.limit:100000000}") + private Long limit; private final static String DATA_URL_TITLE = "data:image/jpeg;base64,"; private static final String exportData_path = "/opt/dataease2.0/data/exportData/"; @Value("${dataease.export.page.size:50000}") @@ -162,7 +162,9 @@ public class ExportCenterManage implements BaseExportApi { } private Long getExportLimit() { - return Math.min(f2CLicLimitedManage.checkDatasetLimit(), limit); + System.out.println("checkDatasetLimit: " + f2CLicLimitedManage.checkDatasetLimit()); +// return Math.min(f2CLicLimitedManage.checkDatasetLimit(), limit); + return limit; } public void download(String id, HttpServletResponse response) throws Exception { @@ -496,97 +498,105 @@ public class ExportCenterManage implements BaseExportApi { Field2SQLObj.field2sqlObj(sqlMeta, allFields, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage); WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage); Order2SQLObj.getOrders(sqlMeta, dto.getSortFields(), allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage); - String replaceSql = provider.rebuildSQL(SQLProvider.createQuerySQL(sqlMeta, false, false, false), sqlMeta, crossDs, dsMap); Long totalCount = datasetDataManage.getDatasetTotal(dto, replaceSql, null); Long curLimit = getExportLimit(); totalCount = totalCount > curLimit ? curLimit : totalCount; - Long totalPage = (totalCount / extractPageSize) + (totalCount % extractPageSize > 0 ? 1 : 0); - + Long sheetLimit = 1000000L; + Long sheetCount = (totalCount / sheetLimit) + (totalCount % sheetLimit > 0 ? 1 : 0); Workbook wb = new SXSSFWorkbook(); FileOutputStream fileOutputStream = new FileOutputStream(dataPath + "/" + request.getFilename() + ".xlsx"); - Sheet detailsSheet = wb.createSheet("数据"); - List> details = new ArrayList<>(); - - for (Integer p = 0; p < totalPage; p++) { - String querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, p * extractPageSize + extractPageSize, extractPageSize); - if (totalPage == 1) { - querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, 0, totalCount.intValue()); + for (Long s = 1L; s < sheetCount + 1; s++) { + Long sheetSize; + if (s.equals(sheetCount)) { + sheetSize = totalCount - sheetCount * (s - 1) * sheetLimit; + } else { + sheetSize = sheetLimit; } - querySQL = provider.rebuildSQL(querySQL, sqlMeta, crossDs, dsMap); - DatasourceRequest datasourceRequest = new DatasourceRequest(); - datasourceRequest.setQuery(querySQL); - datasourceRequest.setDsList(dsMap); - Map previewData = datasetDataManage.buildPreviewData(provider.fetchResultField(datasourceRequest), allFields, desensitizationList); - List> data = (List>) previewData.get("data"); - if (p == 0L) { - CellStyle cellStyle = wb.createCellStyle(); - Font font = wb.createFont(); - font.setFontHeightInPoints((short) 12); - font.setBold(true); - cellStyle.setFont(font); - cellStyle.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex()); - cellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND); - List header = new ArrayList<>(); - for (DatasetTableFieldDTO field : allFields) { - header.add(field.getName()); + Long pageSize = (sheetSize / extractPageSize) + (sheetSize % extractPageSize > 0 ? 1 : 0); + Sheet detailsSheet = null; + List> details = new ArrayList<>(); + for (Long p = 0L; p < pageSize; p++) { + String querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, p.intValue() * extractPageSize, extractPageSize); + if (pageSize == 1) { + querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, 0, sheetSize.intValue()); } - - details.add(header); - for (Map obj : data) { - List row = new ArrayList<>(); + querySQL = provider.rebuildSQL(querySQL, sqlMeta, crossDs, dsMap); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setQuery(querySQL); + datasourceRequest.setDsList(dsMap); + Map previewData = datasetDataManage.buildPreviewData(provider.fetchResultField(datasourceRequest), allFields, desensitizationList); + List> data = (List>) previewData.get("data"); + if (p.equals(0L)) { + detailsSheet = wb.createSheet("数据-" + s); + CellStyle cellStyle = wb.createCellStyle(); + Font font = wb.createFont(); + font.setFontHeightInPoints((short) 12); + font.setBold(true); + cellStyle.setFont(font); + cellStyle.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex()); + cellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND); + List header = new ArrayList<>(); for (DatasetTableFieldDTO field : allFields) { - String string = (String) obj.get(field.getDataeaseName()); - row.add(string); + header.add(field.getName()); } - details.add(row); - } - if (CollectionUtils.isNotEmpty(details)) { - for (int i = 0; i < details.size(); i++) { - Row row = detailsSheet.createRow(i); - List rowData = details.get(i); - if (rowData != null) { - for (int j = 0; j < rowData.size(); j++) { - Cell cell = row.createCell(j); - if (i == 0) { - cell.setCellValue(rowData.get(j)); - cell.setCellStyle(cellStyle); - detailsSheet.setColumnWidth(j, 255 * 20); - } else { - cell.setCellValue(rowData.get(j)); + details.add(header); + for (Map obj : data) { + List row = new ArrayList<>(); + for (DatasetTableFieldDTO field : allFields) { + String string = (String) obj.get(field.getDataeaseName()); + row.add(string); + } + details.add(row); + } + if (CollectionUtils.isNotEmpty(details)) { + for (int i = 0; i < details.size(); i++) { + Row row = detailsSheet.createRow(i); + List rowData = details.get(i); + if (rowData != null) { + for (int j = 0; j < rowData.size(); j++) { + Cell cell = row.createCell(j); + if (i == 0) { + cell.setCellValue(rowData.get(j)); + cell.setCellStyle(cellStyle); + detailsSheet.setColumnWidth(j, 255 * 20); + } else { + cell.setCellValue(rowData.get(j)); + } } } } } - } - } else { - details.clear(); - for (Map obj : data) { - List row = new ArrayList<>(); - for (DatasetTableFieldDTO field : allFields) { - String string = (String) obj.get(field.getDataeaseName()); - row.add(string); + } else { + details.clear(); + for (Map obj : data) { + List row = new ArrayList<>(); + for (DatasetTableFieldDTO field : allFields) { + String string = (String) obj.get(field.getDataeaseName()); + row.add(string); + } + details.add(row); } - details.add(row); - } - int lastNum = detailsSheet.getLastRowNum(); - for (int i = 0; i < details.size(); i++) { - Row row = detailsSheet.createRow(i + lastNum + 1); - List rowData = details.get(i); - if (rowData != null) { - for (int j = 0; j < rowData.size(); j++) { - Cell cell = row.createCell(j); - cell.setCellValue(rowData.get(j)); + int lastNum = detailsSheet.getLastRowNum(); + for (int i = 0; i < details.size(); i++) { + Row row = detailsSheet.createRow(i + lastNum + 1); + List rowData = details.get(i); + if (rowData != null) { + for (int j = 0; j < rowData.size(); j++) { + Cell cell = row.createCell(j); + cell.setCellValue(rowData.get(j)); + } } } } + exportTask.setExportStatus("IN_PROGRESS"); + double exportRogress2 = (double) ((double) s / (double) sheetCount); + double exportRogress = (double) ((double) p / (double) pageSize) * ((double) 1 / sheetCount); + DecimalFormat df = new DecimalFormat("#.##"); + String formattedResult = df.format((exportRogress + exportRogress2) * 100); + exportTask.setExportProgress(formattedResult); + exportTaskMapper.updateById(exportTask); } - exportTask.setExportStatus("IN_PROGRESS"); - double exportRogress = (double) ((double) p / (double) totalPage); - DecimalFormat df = new DecimalFormat("#.##"); - String formattedResult = df.format(exportRogress * 100); - exportTask.setExportProgress(formattedResult); - exportTaskMapper.updateById(exportTask); } wb.write(fileOutputStream); fileOutputStream.flush();