center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

fix: 数据导出超过100w时，分多个sheet

Browse Source
This commit is contained in:
taojinlong 2024-10-12 18:00:58 +08:00
parent 3760e58166
commit 9424f3fb0f
1 changed files with 85 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterManage.java
View File

@ -96,8 +96,8 @@ public class ExportCenterManage implements BaseExportApi {
@Value("${dataease.export.max.size:10}")
private int max;
@Value("${dataease.export.dataset.limit:100000}")
private int limit;
@Value("${dataease.export.dataset.limit:100000000}")
private Long limit;
private final static String DATA_URL_TITLE = "data:image/jpeg;base64,";
private static final String exportData_path = "/opt/dataease2.0/data/exportData/";
@Value("${dataease.export.page.size:50000}")
@ -162,7 +162,9 @@ public class ExportCenterManage implements BaseExportApi {
}
private Long getExportLimit() {
return Math.min(f2CLicLimitedManage.checkDatasetLimit(), limit);
System.out.println("checkDatasetLimit: " + f2CLicLimitedManage.checkDatasetLimit());
// return Math.min(f2CLicLimitedManage.checkDatasetLimit(), limit);
return limit;
}
public void download(String id, HttpServletResponse response) throws Exception {
@ -496,22 +498,28 @@ public class ExportCenterManage implements BaseExportApi {
Field2SQLObj.field2sqlObj(sqlMeta, allFields, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
Order2SQLObj.getOrders(sqlMeta, dto.getSortFields(), allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
String replaceSql = provider.rebuildSQL(SQLProvider.createQuerySQL(sqlMeta, false, false, false), sqlMeta, crossDs, dsMap);
Long totalCount = datasetDataManage.getDatasetTotal(dto, replaceSql, null);
Long curLimit = getExportLimit();
totalCount = totalCount > curLimit ? curLimit : totalCount;
Long totalPage = (totalCount / extractPageSize) + (totalCount % extractPageSize > 0 ? 1 : 0);
Long sheetLimit = 1000000L;
Long sheetCount = (totalCount / sheetLimit) + (totalCount % sheetLimit > 0 ? 1 : 0);
Workbook wb = new SXSSFWorkbook();
FileOutputStream fileOutputStream = new FileOutputStream(dataPath + "/" + request.getFilename() + ".xlsx");
Sheet detailsSheet = wb.createSheet("数据");
for (Long s = 1L; s < sheetCount + 1; s++) {
Long sheetSize;
if (s.equals(sheetCount)) {
sheetSize = totalCount - sheetCount * (s - 1) * sheetLimit;
} else {
sheetSize = sheetLimit;
}
Long pageSize = (sheetSize / extractPageSize) + (sheetSize % extractPageSize > 0 ? 1 : 0);
Sheet detailsSheet = null;
List<List<String>> details = new ArrayList<>();
for (Integer p = 0; p < totalPage; p++) {
String querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, p * extractPageSize + extractPageSize, extractPageSize);
if (totalPage == 1) {
querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, 0, totalCount.intValue());
for (Long p = 0L; p < pageSize; p++) {
String querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, p.intValue() * extractPageSize, extractPageSize);
if (pageSize == 1) {
querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, 0, sheetSize.intValue());
}
querySQL = provider.rebuildSQL(querySQL, sqlMeta, crossDs, dsMap);
DatasourceRequest datasourceRequest = new DatasourceRequest();
@ -519,7 +527,8 @@ public class ExportCenterManage implements BaseExportApi {
datasourceRequest.setDsList(dsMap);
Map<String, Object> previewData = datasetDataManage.buildPreviewData(provider.fetchResultField(datasourceRequest), allFields, desensitizationList);
List<Map<String, Object>> data = (List<Map<String, Object>>) previewData.get("data");
if (p == 0L) {
if (p.equals(0L)) {
detailsSheet = wb.createSheet("数据-" + s);
CellStyle cellStyle = wb.createCellStyle();
Font font = wb.createFont();
font.setFontHeightInPoints((short) 12);
@ -531,7 +540,6 @@ public class ExportCenterManage implements BaseExportApi {
for (DatasetTableFieldDTO field : allFields) {
header.add(field.getName());
}
details.add(header);
for (Map<String, Object> obj : data) {
List<String> row = new ArrayList<>();
@ -582,12 +590,14 @@ public class ExportCenterManage implements BaseExportApi {
}
}
exportTask.setExportStatus("IN_PROGRESS");
double exportRogress = (double) ((double) p / (double) totalPage);
double exportRogress2 = (double) ((double) s / (double) sheetCount);
double exportRogress = (double) ((double) p / (double) pageSize) * ((double) 1 / sheetCount);
DecimalFormat df = new DecimalFormat("#.##");
String formattedResult = df.format(exportRogress * 100);
String formattedResult = df.format((exportRogress + exportRogress2) * 100);
exportTask.setExportProgress(formattedResult);
exportTaskMapper.updateById(exportTask);
}
}
wb.write(fileOutputStream);
fileOutputStream.flush();
fileOutputStream.close();