center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

fix: 数据导出超过100w时，分多个sheet

Browse Source
This commit is contained in:
taojinlong 2024-10-12 18:00:58 +08:00
parent 3760e58166
commit 9424f3fb0f
1 changed files with 85 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

160
core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterManage.java
View File

@ -96,8 +96,8 @@ public class ExportCenterManage implements BaseExportApi {
@Value("${dataease.export.max.size:10}") @Value("${dataease.export.max.size:10}")
private int max; private int max;
@Value("${dataease.export.dataset.limit:100000}") @Value("${dataease.export.dataset.limit:100000000}")
private int limit; private Long limit;
private final static String DATA_URL_TITLE = "data:image/jpeg;base64,"; private final static String DATA_URL_TITLE = "data:image/jpeg;base64,";
private static final String exportData_path = "/opt/dataease2.0/data/exportData/"; private static final String exportData_path = "/opt/dataease2.0/data/exportData/";
@Value("${dataease.export.page.size:50000}") @Value("${dataease.export.page.size:50000}")
@ -162,7 +162,9 @@ public class ExportCenterManage implements BaseExportApi {
} }
private Long getExportLimit() { private Long getExportLimit() {
return Math.min(f2CLicLimitedManage.checkDatasetLimit(), limit); System.out.println("checkDatasetLimit: " + f2CLicLimitedManage.checkDatasetLimit());
// return Math.min(f2CLicLimitedManage.checkDatasetLimit(), limit);
return limit;
} }
public void download(String id, HttpServletResponse response) throws Exception { public void download(String id, HttpServletResponse response) throws Exception {
@ -496,97 +498,105 @@ public class ExportCenterManage implements BaseExportApi {
Field2SQLObj.field2sqlObj(sqlMeta, allFields, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage); Field2SQLObj.field2sqlObj(sqlMeta, allFields, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage); WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
Order2SQLObj.getOrders(sqlMeta, dto.getSortFields(), allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage); Order2SQLObj.getOrders(sqlMeta, dto.getSortFields(), allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
String replaceSql = provider.rebuildSQL(SQLProvider.createQuerySQL(sqlMeta, false, false, false), sqlMeta, crossDs, dsMap); String replaceSql = provider.rebuildSQL(SQLProvider.createQuerySQL(sqlMeta, false, false, false), sqlMeta, crossDs, dsMap);
Long totalCount = datasetDataManage.getDatasetTotal(dto, replaceSql, null); Long totalCount = datasetDataManage.getDatasetTotal(dto, replaceSql, null);
Long curLimit = getExportLimit(); Long curLimit = getExportLimit();
totalCount = totalCount > curLimit ? curLimit : totalCount; totalCount = totalCount > curLimit ? curLimit : totalCount;
Long totalPage = (totalCount / extractPageSize) + (totalCount % extractPageSize > 0 ? 1 : 0); Long sheetLimit = 1000000L;
Long sheetCount = (totalCount / sheetLimit) + (totalCount % sheetLimit > 0 ? 1 : 0);
Workbook wb = new SXSSFWorkbook(); Workbook wb = new SXSSFWorkbook();
FileOutputStream fileOutputStream = new FileOutputStream(dataPath + "/" + request.getFilename() + ".xlsx"); FileOutputStream fileOutputStream = new FileOutputStream(dataPath + "/" + request.getFilename() + ".xlsx");
Sheet detailsSheet = wb.createSheet("数据"); for (Long s = 1L; s < sheetCount + 1; s++) {
List<List<String>> details = new ArrayList<>(); Long sheetSize;
if (s.equals(sheetCount)) {
for (Integer p = 0; p < totalPage; p++) { sheetSize = totalCount - sheetCount * (s - 1) * sheetLimit;
String querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, p * extractPageSize + extractPageSize, extractPageSize); } else {
if (totalPage == 1) { sheetSize = sheetLimit;
querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, 0, totalCount.intValue());
} }
querySQL = provider.rebuildSQL(querySQL, sqlMeta, crossDs, dsMap); Long pageSize = (sheetSize / extractPageSize) + (sheetSize % extractPageSize > 0 ? 1 : 0);
DatasourceRequest datasourceRequest = new DatasourceRequest(); Sheet detailsSheet = null;
datasourceRequest.setQuery(querySQL); List<List<String>> details = new ArrayList<>();
datasourceRequest.setDsList(dsMap); for (Long p = 0L; p < pageSize; p++) {
Map<String, Object> previewData = datasetDataManage.buildPreviewData(provider.fetchResultField(datasourceRequest), allFields, desensitizationList); String querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, p.intValue() * extractPageSize, extractPageSize);
List<Map<String, Object>> data = (List<Map<String, Object>>) previewData.get("data"); if (pageSize == 1) {
if (p == 0L) { querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, 0, sheetSize.intValue());
CellStyle cellStyle = wb.createCellStyle();
Font font = wb.createFont();
font.setFontHeightInPoints((short) 12);
font.setBold(true);
cellStyle.setFont(font);
cellStyle.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex());
cellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND);
List<String> header = new ArrayList<>();
for (DatasetTableFieldDTO field : allFields) {
header.add(field.getName());
} }
querySQL = provider.rebuildSQL(querySQL, sqlMeta, crossDs, dsMap);
details.add(header); DatasourceRequest datasourceRequest = new DatasourceRequest();
for (Map<String, Object> obj : data) { datasourceRequest.setQuery(querySQL);
List<String> row = new ArrayList<>(); datasourceRequest.setDsList(dsMap);
Map<String, Object> previewData = datasetDataManage.buildPreviewData(provider.fetchResultField(datasourceRequest), allFields, desensitizationList);
List<Map<String, Object>> data = (List<Map<String, Object>>) previewData.get("data");
if (p.equals(0L)) {
detailsSheet = wb.createSheet("数据-" + s);
CellStyle cellStyle = wb.createCellStyle();
Font font = wb.createFont();
font.setFontHeightInPoints((short) 12);
font.setBold(true);
cellStyle.setFont(font);
cellStyle.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex());
cellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND);
List<String> header = new ArrayList<>();
for (DatasetTableFieldDTO field : allFields) { for (DatasetTableFieldDTO field : allFields) {
String string = (String) obj.get(field.getDataeaseName()); header.add(field.getName());
row.add(string);
} }
details.add(row); details.add(header);
} for (Map<String, Object> obj : data) {
if (CollectionUtils.isNotEmpty(details)) { List<String> row = new ArrayList<>();
for (int i = 0; i < details.size(); i++) { for (DatasetTableFieldDTO field : allFields) {
Row row = detailsSheet.createRow(i); String string = (String) obj.get(field.getDataeaseName());
List<String> rowData = details.get(i); row.add(string);
if (rowData != null) { }
for (int j = 0; j < rowData.size(); j++) { details.add(row);
Cell cell = row.createCell(j); }
if (i == 0) { if (CollectionUtils.isNotEmpty(details)) {
cell.setCellValue(rowData.get(j)); for (int i = 0; i < details.size(); i++) {
cell.setCellStyle(cellStyle); Row row = detailsSheet.createRow(i);
detailsSheet.setColumnWidth(j, 255 * 20); List<String> rowData = details.get(i);
} else { if (rowData != null) {
cell.setCellValue(rowData.get(j)); for (int j = 0; j < rowData.size(); j++) {
Cell cell = row.createCell(j);
if (i == 0) {
cell.setCellValue(rowData.get(j));
cell.setCellStyle(cellStyle);
detailsSheet.setColumnWidth(j, 255 * 20);
} else {
cell.setCellValue(rowData.get(j));
}
} }
} }
} }
} }
} } else {
} else { details.clear();
details.clear(); for (Map<String, Object> obj : data) {
for (Map<String, Object> obj : data) { List<String> row = new ArrayList<>();
List<String> row = new ArrayList<>(); for (DatasetTableFieldDTO field : allFields) {
for (DatasetTableFieldDTO field : allFields) { String string = (String) obj.get(field.getDataeaseName());
String string = (String) obj.get(field.getDataeaseName()); row.add(string);
row.add(string); }
details.add(row);
} }
details.add(row); int lastNum = detailsSheet.getLastRowNum();
} for (int i = 0; i < details.size(); i++) {
int lastNum = detailsSheet.getLastRowNum(); Row row = detailsSheet.createRow(i + lastNum + 1);
for (int i = 0; i < details.size(); i++) { List<String> rowData = details.get(i);
Row row = detailsSheet.createRow(i + lastNum + 1); if (rowData != null) {
List<String> rowData = details.get(i); for (int j = 0; j < rowData.size(); j++) {
if (rowData != null) { Cell cell = row.createCell(j);
for (int j = 0; j < rowData.size(); j++) { cell.setCellValue(rowData.get(j));
Cell cell = row.createCell(j); }
cell.setCellValue(rowData.get(j));
} }
} }
} }
exportTask.setExportStatus("IN_PROGRESS");
double exportRogress2 = (double) ((double) s / (double) sheetCount);
double exportRogress = (double) ((double) p / (double) pageSize) * ((double) 1 / sheetCount);
DecimalFormat df = new DecimalFormat("#.##");
String formattedResult = df.format((exportRogress + exportRogress2) * 100);
exportTask.setExportProgress(formattedResult);
exportTaskMapper.updateById(exportTask);
} }
exportTask.setExportStatus("IN_PROGRESS");
double exportRogress = (double) ((double) p / (double) totalPage);
DecimalFormat df = new DecimalFormat("#.##");
String formattedResult = df.format(exportRogress * 100);
exportTask.setExportProgress(formattedResult);
exportTaskMapper.updateById(exportTask);
} }
wb.write(fileOutputStream); wb.write(fileOutputStream);
fileOutputStream.flush(); fileOutputStream.flush();