forked from github/dataease
fix: 数据导出超过100w时，分多个sheet
This commit is contained in:
taojinlong
parent
3760e58166
commit
9424f3fb0f
@ -96,8 +96,8 @@ public class ExportCenterManage implements BaseExportApi {
|
||||
@Value("${dataease.export.max.size:10}")
|
||||
private int max;
|
||||
|
||||
@Value("${dataease.export.dataset.limit:100000}")
|
||||
private int limit;
|
||||
@Value("${dataease.export.dataset.limit:100000000}")
|
||||
private Long limit;
|
||||
private final static String DATA_URL_TITLE = "data:image/jpeg;base64,";
|
||||
private static final String exportData_path = "/opt/dataease2.0/data/exportData/";
|
||||
@Value("${dataease.export.page.size:50000}")
|
||||
@ -162,7 +162,9 @@ public class ExportCenterManage implements BaseExportApi {
|
||||
}
|
||||
|
||||
private Long getExportLimit() {
|
||||
return Math.min(f2CLicLimitedManage.checkDatasetLimit(), limit);
|
||||
System.out.println("checkDatasetLimit: " + f2CLicLimitedManage.checkDatasetLimit());
|
||||
// return Math.min(f2CLicLimitedManage.checkDatasetLimit(), limit);
|
||||
return limit;
|
||||
}
|
||||
|
||||
public void download(String id, HttpServletResponse response) throws Exception {
|
||||
@ -496,97 +498,105 @@ public class ExportCenterManage implements BaseExportApi {
|
||||
Field2SQLObj.field2sqlObj(sqlMeta, allFields, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
|
||||
WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
|
||||
Order2SQLObj.getOrders(sqlMeta, dto.getSortFields(), allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
|
||||
|
||||
String replaceSql = provider.rebuildSQL(SQLProvider.createQuerySQL(sqlMeta, false, false, false), sqlMeta, crossDs, dsMap);
|
||||
Long totalCount = datasetDataManage.getDatasetTotal(dto, replaceSql, null);
|
||||
Long curLimit = getExportLimit();
|
||||
totalCount = totalCount > curLimit ? curLimit : totalCount;
|
||||
Long totalPage = (totalCount / extractPageSize) + (totalCount % extractPageSize > 0 ? 1 : 0);
|
||||
|
||||
Long sheetLimit = 1000000L;
|
||||
Long sheetCount = (totalCount / sheetLimit) + (totalCount % sheetLimit > 0 ? 1 : 0);
|
||||
Workbook wb = new SXSSFWorkbook();
|
||||
FileOutputStream fileOutputStream = new FileOutputStream(dataPath + "/" + request.getFilename() + ".xlsx");
|
||||
Sheet detailsSheet = wb.createSheet("数据");
|
||||
List<List<String>> details = new ArrayList<>();
|
||||
|
||||
for (Integer p = 0; p < totalPage; p++) {
|
||||
String querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, p * extractPageSize + extractPageSize, extractPageSize);
|
||||
if (totalPage == 1) {
|
||||
querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, 0, totalCount.intValue());
|
||||
for (Long s = 1L; s < sheetCount + 1; s++) {
|
||||
Long sheetSize;
|
||||
if (s.equals(sheetCount)) {
|
||||
sheetSize = totalCount - sheetCount * (s - 1) * sheetLimit;
|
||||
} else {
|
||||
sheetSize = sheetLimit;
|
||||
}
|
||||
querySQL = provider.rebuildSQL(querySQL, sqlMeta, crossDs, dsMap);
|
||||
DatasourceRequest datasourceRequest = new DatasourceRequest();
|
||||
datasourceRequest.setQuery(querySQL);
|
||||
datasourceRequest.setDsList(dsMap);
|
||||
Map<String, Object> previewData = datasetDataManage.buildPreviewData(provider.fetchResultField(datasourceRequest), allFields, desensitizationList);
|
||||
List<Map<String, Object>> data = (List<Map<String, Object>>) previewData.get("data");
|
||||
if (p == 0L) {
|
||||
CellStyle cellStyle = wb.createCellStyle();
|
||||
Font font = wb.createFont();
|
||||
font.setFontHeightInPoints((short) 12);
|
||||
font.setBold(true);
|
||||
cellStyle.setFont(font);
|
||||
cellStyle.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex());
|
||||
cellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND);
|
||||
List<String> header = new ArrayList<>();
|
||||
for (DatasetTableFieldDTO field : allFields) {
|
||||
header.add(field.getName());
|
||||
Long pageSize = (sheetSize / extractPageSize) + (sheetSize % extractPageSize > 0 ? 1 : 0);
|
||||
Sheet detailsSheet = null;
|
||||
List<List<String>> details = new ArrayList<>();
|
||||
for (Long p = 0L; p < pageSize; p++) {
|
||||
String querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, p.intValue() * extractPageSize, extractPageSize);
|
||||
if (pageSize == 1) {
|
||||
querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, 0, sheetSize.intValue());
|
||||
}
|
||||
|
||||
details.add(header);
|
||||
for (Map<String, Object> obj : data) {
|
||||
List<String> row = new ArrayList<>();
|
||||
querySQL = provider.rebuildSQL(querySQL, sqlMeta, crossDs, dsMap);
|
||||
DatasourceRequest datasourceRequest = new DatasourceRequest();
|
||||
datasourceRequest.setQuery(querySQL);
|
||||
datasourceRequest.setDsList(dsMap);
|
||||
Map<String, Object> previewData = datasetDataManage.buildPreviewData(provider.fetchResultField(datasourceRequest), allFields, desensitizationList);
|
||||
List<Map<String, Object>> data = (List<Map<String, Object>>) previewData.get("data");
|
||||
if (p.equals(0L)) {
|
||||
detailsSheet = wb.createSheet("数据-" + s);
|
||||
CellStyle cellStyle = wb.createCellStyle();
|
||||
Font font = wb.createFont();
|
||||
font.setFontHeightInPoints((short) 12);
|
||||
font.setBold(true);
|
||||
cellStyle.setFont(font);
|
||||
cellStyle.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex());
|
||||
cellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND);
|
||||
List<String> header = new ArrayList<>();
|
||||
for (DatasetTableFieldDTO field : allFields) {
|
||||
String string = (String) obj.get(field.getDataeaseName());
|
||||
row.add(string);
|
||||
header.add(field.getName());
|
||||
}
|
||||
details.add(row);
|
||||
}
|
||||
if (CollectionUtils.isNotEmpty(details)) {
|
||||
for (int i = 0; i < details.size(); i++) {
|
||||
Row row = detailsSheet.createRow(i);
|
||||
List<String> rowData = details.get(i);
|
||||
if (rowData != null) {
|
||||
for (int j = 0; j < rowData.size(); j++) {
|
||||
Cell cell = row.createCell(j);
|
||||
if (i == 0) {
|
||||
cell.setCellValue(rowData.get(j));
|
||||
cell.setCellStyle(cellStyle);
|
||||
detailsSheet.setColumnWidth(j, 255 * 20);
|
||||
} else {
|
||||
cell.setCellValue(rowData.get(j));
|
||||
details.add(header);
|
||||
for (Map<String, Object> obj : data) {
|
||||
List<String> row = new ArrayList<>();
|
||||
for (DatasetTableFieldDTO field : allFields) {
|
||||
String string = (String) obj.get(field.getDataeaseName());
|
||||
row.add(string);
|
||||
}
|
||||
details.add(row);
|
||||
}
|
||||
if (CollectionUtils.isNotEmpty(details)) {
|
||||
for (int i = 0; i < details.size(); i++) {
|
||||
Row row = detailsSheet.createRow(i);
|
||||
List<String> rowData = details.get(i);
|
||||
if (rowData != null) {
|
||||
for (int j = 0; j < rowData.size(); j++) {
|
||||
Cell cell = row.createCell(j);
|
||||
if (i == 0) {
|
||||
cell.setCellValue(rowData.get(j));
|
||||
cell.setCellStyle(cellStyle);
|
||||
detailsSheet.setColumnWidth(j, 255 * 20);
|
||||
} else {
|
||||
cell.setCellValue(rowData.get(j));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
details.clear();
|
||||
for (Map<String, Object> obj : data) {
|
||||
List<String> row = new ArrayList<>();
|
||||
for (DatasetTableFieldDTO field : allFields) {
|
||||
String string = (String) obj.get(field.getDataeaseName());
|
||||
row.add(string);
|
||||
} else {
|
||||
details.clear();
|
||||
for (Map<String, Object> obj : data) {
|
||||
List<String> row = new ArrayList<>();
|
||||
for (DatasetTableFieldDTO field : allFields) {
|
||||
String string = (String) obj.get(field.getDataeaseName());
|
||||
row.add(string);
|
||||
}
|
||||
details.add(row);
|
||||
}
|
||||
details.add(row);
|
||||
}
|
||||
int lastNum = detailsSheet.getLastRowNum();
|
||||
for (int i = 0; i < details.size(); i++) {
|
||||
Row row = detailsSheet.createRow(i + lastNum + 1);
|
||||
List<String> rowData = details.get(i);
|
||||
if (rowData != null) {
|
||||
for (int j = 0; j < rowData.size(); j++) {
|
||||
Cell cell = row.createCell(j);
|
||||
cell.setCellValue(rowData.get(j));
|
||||
int lastNum = detailsSheet.getLastRowNum();
|
||||
for (int i = 0; i < details.size(); i++) {
|
||||
Row row = detailsSheet.createRow(i + lastNum + 1);
|
||||
List<String> rowData = details.get(i);
|
||||
if (rowData != null) {
|
||||
for (int j = 0; j < rowData.size(); j++) {
|
||||
Cell cell = row.createCell(j);
|
||||
cell.setCellValue(rowData.get(j));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
exportTask.setExportStatus("IN_PROGRESS");
|
||||
double exportRogress2 = (double) ((double) s / (double) sheetCount);
|
||||
double exportRogress = (double) ((double) p / (double) pageSize) * ((double) 1 / sheetCount);
|
||||
DecimalFormat df = new DecimalFormat("#.##");
|
||||
String formattedResult = df.format((exportRogress + exportRogress2) * 100);
|
||||
exportTask.setExportProgress(formattedResult);
|
||||
exportTaskMapper.updateById(exportTask);
|
||||
}
|
||||
exportTask.setExportStatus("IN_PROGRESS");
|
||||
double exportRogress = (double) ((double) p / (double) totalPage);
|
||||
DecimalFormat df = new DecimalFormat("#.##");
|
||||
String formattedResult = df.format(exportRogress * 100);
|
||||
exportTask.setExportProgress(formattedResult);
|
||||
exportTaskMapper.updateById(exportTask);
|
||||
}
|
||||
wb.write(fileOutputStream);
|
||||
fileOutputStream.flush();
|
||||
|
||||
Loading…
Reference in New Issue
Block a user