From 7285eb11bf7483f19211e4dd711f434f8ac714b0 Mon Sep 17 00:00:00 2001 From: taojinlong Date: Wed, 21 Jul 2021 15:07:51 +0800 Subject: [PATCH 1/9] =?UTF-8?q?feat:=20SAX=20=E8=A7=A3=E6=9E=90=20excel?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../commons/utils/ExcelReaderUtil.java | 75 +++ .../commons/utils/ExcelXlsReader.java | 401 +++++++++++++++ .../commons/utils/ExcelXlsxReader.java | 470 ++++++++++++++++++ .../datasource/provider/JdbcProvider.java | 3 + .../dataease/dto/dataset/ExcelSheetData.java | 13 + .../service/dataset/DataSetTableService.java | 49 +- 6 files changed, 1010 insertions(+), 1 deletion(-) create mode 100644 backend/src/main/java/io/dataease/commons/utils/ExcelReaderUtil.java create mode 100644 backend/src/main/java/io/dataease/commons/utils/ExcelXlsReader.java create mode 100644 backend/src/main/java/io/dataease/commons/utils/ExcelXlsxReader.java create mode 100644 backend/src/main/java/io/dataease/dto/dataset/ExcelSheetData.java diff --git a/backend/src/main/java/io/dataease/commons/utils/ExcelReaderUtil.java b/backend/src/main/java/io/dataease/commons/utils/ExcelReaderUtil.java new file mode 100644 index 0000000000..4ae57a6f5f --- /dev/null +++ b/backend/src/main/java/io/dataease/commons/utils/ExcelReaderUtil.java @@ -0,0 +1,75 @@ +package io.dataease.commons.utils; +import com.google.gson.Gson; +import io.dataease.datasource.dto.TableFiled; + +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class ExcelReaderUtil { + //excel2003扩展名 + public static final String EXCEL03_EXTENSION = ".xls"; + //excel2007扩展名 + public static final String EXCEL07_EXTENSION = ".xlsx"; + + public static void sendRows(String filePath, String sheetName, int sheetIndex, int curRow, List cellList) { + StringBuffer oneLineSb = new StringBuffer(); + oneLineSb.append(filePath); + oneLineSb.append("--"); + oneLineSb.append("sheet" + sheetIndex); + oneLineSb.append("::" + sheetName);//加上sheet名 + oneLineSb.append("--"); + oneLineSb.append("row" + curRow); + oneLineSb.append("::"); + + // map.put(cellList.get(9),cellList.get(0)) ; + + for (String cell : cellList) { + oneLineSb.append(cell.trim()); + oneLineSb.append("|"); + } + String oneLine = oneLineSb.toString(); + if (oneLine.endsWith("|")) { + oneLine = oneLine.substring(0, oneLine.lastIndexOf("|")); + }// 去除最后一个分隔符 + + System.out.println(oneLine); + } + + /** + * 读取excel文件路径 + * @param fileName 文件路径 + * @throws Exception + */ + public static void readExcel(String fileName, InputStream inputStream) throws Exception { + if (fileName.endsWith(EXCEL03_EXTENSION)) { //处理excel2003文件 + ExcelXlsReader excelXls=new ExcelXlsReader(); + excelXls.process(inputStream); + System.out.println(excelXls.totalSheets.size()); + System.out.println(excelXls.totalSheets.get(0).getSheetName()); + for (TableFiled field : excelXls.totalSheets.get(0).getFields()) { + System.out.println(new Gson().toJson(field)); + } + System.out.println(excelXls.totalSheets.get(0).getData().get(0)); + + } else if (fileName.endsWith(EXCEL07_EXTENSION)) {//处理excel2007文件 + ExcelXlsxReader excelXlsxReader = new ExcelXlsxReader(); + excelXlsxReader.process(inputStream); + System.out.println(excelXlsxReader.totalSheets.size()); + System.out.println(excelXlsxReader.totalSheets.get(0).getSheetName()); + for (TableFiled field : excelXlsxReader.totalSheets.get(0).getFields()) { + System.out.println(new Gson().toJson(field)); + } + System.out.println(excelXlsxReader.totalSheets.get(0).getData().get(0)); + + } else { + throw new Exception("文件格式错误，fileName的扩展名只能是xls或xlsx。"); + } + } + + public static void main(String[] args) throws Exception { + ExcelReaderUtil.readExcel("111.xls", new FileInputStream("/Users/taojinlong/Desktop/111.xls")); + } +} diff --git a/backend/src/main/java/io/dataease/commons/utils/ExcelXlsReader.java b/backend/src/main/java/io/dataease/commons/utils/ExcelXlsReader.java new file mode 100644 index 0000000000..7a12665e48 --- /dev/null +++ b/backend/src/main/java/io/dataease/commons/utils/ExcelXlsReader.java @@ -0,0 +1,401 @@ +package io.dataease.commons.utils; + +import com.google.gson.Gson; +import io.dataease.datasource.dto.TableFiled; +import io.dataease.dto.dataset.ExcelSheetData; +import io.dataease.i18n.Translator; +import org.apache.poi.hssf.eventusermodel.*; +import org.apache.poi.hssf.eventusermodel.dummyrecord.LastCellOfRowDummyRecord; +import org.apache.poi.hssf.eventusermodel.dummyrecord.MissingCellDummyRecord; +import org.apache.poi.hssf.model.HSSFFormulaParser; +import org.apache.poi.hssf.record.*; +import org.apache.poi.hssf.usermodel.HSSFDataFormatter; +import org.apache.poi.hssf.usermodel.HSSFWorkbook; +import org.apache.poi.poifs.filesystem.POIFSFileSystem; + +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * @author y + * @create 2018-01-19 14:18 + * @desc 用于解决.xls2003版本大数据量问题 + **/ +public class ExcelXlsReader implements HSSFListener { + + public ExcelReaderUtil excelReaderUtil = new ExcelReaderUtil(); + + private int minColums = -1; + + private POIFSFileSystem fs; + + /** + * 总行数 + */ + private int totalRows = 0; + + /** + * 上一行row的序号 + */ + private int lastRowNumber; + + /** + * 上一单元格的序号 + */ + private int lastColumnNumber; + + /** + * 是否输出formula，还是它对应的值 + */ + private boolean outputFormulaValues = true; + + /** + * 用于转换formulas + */ + private EventWorkbookBuilder.SheetRecordCollectingListener workbookBuildingListener; + + //excel2003工作簿 + private HSSFWorkbook stubWorkbook; + + private SSTRecord sstRecord; + + private FormatTrackingHSSFListener formatListener; + + private final HSSFDataFormatter formatter = new HSSFDataFormatter(); + + /** + * 文件的绝对路径 + */ + private String filePath = ""; + + //表索引 + private int sheetIndex = 0; + + private BoundSheetRecord[] orderedBSRs; + + @SuppressWarnings("unchecked") + private ArrayList boundSheetRecords = new ArrayList(); + + private int nextRow; + + private int nextColumn; + + private boolean outputNextStringRecord; + + //当前行 + private int curRow = 0; + + //存储一行记录所有单元格的容器 + private List cellList = new ArrayList(); + + /** + * 判断整行是否为空行的标记 + */ + private boolean flag = false; + + @SuppressWarnings("unused") + private String sheetName; + + public List fields = new ArrayList<>(); + public List> data = new ArrayList<>(); + public List totalSheets = new ArrayList<>(); + /** + * 是否为日期 + */ + private boolean isDateFormat = false; + + + public List getFields() { + return fields; + } + + public void setFields(List fields) { + this.fields = fields; + } + + public List> getData() { + return data; + } + + public void setData(List> data) { + this.data = data; + } + + + + /** + * 遍历excel下所有的sheet + * + * @param inputStream + * @throws Exception + */ + public int process(InputStream inputStream) throws Exception { + this.fs = new POIFSFileSystem(inputStream); + MissingRecordAwareHSSFListener listener = new MissingRecordAwareHSSFListener(this); + formatListener = new FormatTrackingHSSFListener(listener); + HSSFEventFactory factory = new HSSFEventFactory(); + HSSFRequest request = new HSSFRequest(); + if (outputFormulaValues) { + request.addListenerForAllRecords(formatListener); + } else { + workbookBuildingListener = new EventWorkbookBuilder.SheetRecordCollectingListener(formatListener); + request.addListenerForAllRecords(workbookBuildingListener); + } + factory.processWorkbookEvents(request, fs); + + return totalRows; //返回该excel文件的总行数，不包括首列和空行 + } + + /** + * HSSFListener 监听方法，处理Record + * 处理每个单元格 + * + * @param record + */ + @SuppressWarnings("unchecked") + public void processRecord(Record record) { + int thisRow = -1; + int thisColumn = -1; + String thisStr = null; + String value = null; + switch (record.getSid()) { + case BoundSheetRecord.sid: + boundSheetRecords.add(record); + break; + case BOFRecord.sid: //开始处理每个sheet + BOFRecord br = (BOFRecord) record; + if (br.getType() == BOFRecord.TYPE_WORKSHEET) { + //如果有需要，则建立子工作簿 + if (workbookBuildingListener != null && stubWorkbook == null) { + stubWorkbook = workbookBuildingListener.getStubHSSFWorkbook(); + } + + if (orderedBSRs == null) { + orderedBSRs = BoundSheetRecord.orderByBofPosition(boundSheetRecords); + } + sheetName = orderedBSRs[sheetIndex].getSheetname(); + sheetIndex++; + } + break; + case MergeCellsRecord.sid: + throw new RuntimeException(Translator.get("i18n_excel_have_merge_region")); + case SSTRecord.sid: + sstRecord = (SSTRecord) record; + break; + case BlankRecord.sid: //单元格为空白 + BlankRecord brec = (BlankRecord) record; + thisRow = brec.getRow(); + thisColumn = brec.getColumn(); + thisStr = ""; + cellList.add(thisColumn, thisStr); + break; + case BoolErrRecord.sid: //单元格为布尔类型 + BoolErrRecord berec = (BoolErrRecord) record; + thisRow = berec.getRow(); + thisColumn = berec.getColumn(); + thisStr = berec.getBooleanValue() + ""; + cellList.add(thisColumn, thisStr); + + checkRowIsNull(thisStr); //如果里面某个单元格含有值，则标识该行不为空行 + break; + case FormulaRecord.sid://单元格为公式类型 + FormulaRecord frec = (FormulaRecord) record; + thisRow = frec.getRow(); + thisColumn = frec.getColumn(); + thisStr = String.valueOf(frec.getValue()); +// if (outputFormulaValues) { +// if (Double.isNaN(frec.getValue())) { +// outputNextStringRecord = true; +// nextRow = frec.getRow(); +// nextColumn = frec.getColumn(); +// } else { +// thisStr = '"' + HSSFFormulaParser.toFormulaString(stubWorkbook, frec.getParsedExpression()) + '"'; +// } +// } else { +// thisStr = '"' + HSSFFormulaParser.toFormulaString(stubWorkbook, frec.getParsedExpression()) + '"'; +// } + String feildType = checkType(thisStr, thisColumn); + if(feildType.equalsIgnoreCase("LONG") && thisStr.endsWith(".0")){ + thisStr = thisStr.substring(0, thisStr.length() -2); + } + cellList.add(thisColumn, thisStr); + checkRowIsNull(thisStr); //如果里面某个单元格含有值，则标识该行不为空行 + break; + case StringRecord.sid: //单元格中公式的字符串 + if (outputNextStringRecord) { + StringRecord srec = (StringRecord) record; + thisStr = srec.getString(); + thisRow = nextRow; + thisColumn = nextColumn; + outputNextStringRecord = false; + } + break; + case LabelRecord.sid: + LabelRecord lrec = (LabelRecord) record; + curRow = thisRow = lrec.getRow(); + thisColumn = lrec.getColumn(); + value = lrec.getValue().trim(); + value = value.equals("") ? "" : value; + cellList.add(thisColumn, value); + checkRowIsNull(value); //如果里面某个单元格含有值，则标识该行不为空行 + break; + case LabelSSTRecord.sid: //单元格为字符串类型 + LabelSSTRecord lsrec = (LabelSSTRecord) record; + curRow = thisRow = lsrec.getRow(); + thisColumn = lsrec.getColumn(); + if (sstRecord == null) { + cellList.add(thisColumn, ""); + } else { + value = sstRecord.getString(lsrec.getSSTIndex()).toString().trim(); + value = value.equals("") ? "" : value; + cellList.add(thisColumn, value); + checkRowIsNull(value); //如果里面某个单元格含有值，则标识该行不为空行 + } + break; + case NumberRecord.sid: //单元格为数字类型 + NumberRecord numrec = (NumberRecord) record; + curRow = thisRow = numrec.getRow(); + thisColumn = numrec.getColumn(); + //第一种方式 + //value = formatListener.formatNumberDateCell(numrec).trim();//这个被写死，采用的m/d/yy h:mm格式，不符合要求 + //第二种方式，参照formatNumberDateCell里面的实现方法编写 + Double valueDouble = ((NumberRecord) numrec).getValue(); + String formatString = formatListener.getFormatString(numrec); + if (formatString.contains("m/d/yy")) { + formatString = "yyyy-MM-dd hh:mm:ss"; + } + int formatIndex = formatListener.getFormatIndex(numrec); + + value = formatter.formatRawCellContents(valueDouble, formatIndex, formatString).trim(); + value = value.equals("") ? "" : value; + //向容器加入列值 + cellList.add(thisColumn, value); + if(formatIndex == 59){ + totalSheets.get(totalSheets.size() -1).getFields().get(thisColumn).setFieldType("DATETIME"); + }else { + checkType(value, thisColumn); + } + checkRowIsNull(value); //如果里面某个单元格含有值，则标识该行不为空行 + break; + default: + break; + } + + //遇到新行的操作 + if (thisRow != -1 && thisRow != lastRowNumber) { + lastColumnNumber = -1; + } + + //空值的操作 + if (record instanceof MissingCellDummyRecord) { + MissingCellDummyRecord mc = (MissingCellDummyRecord) record; + curRow = thisRow = mc.getRow(); + thisColumn = mc.getColumn(); + cellList.add(thisColumn, ""); + } + + //更新行和列的值 + if (thisRow > -1) + lastRowNumber = thisRow; + if (thisColumn > -1) + lastColumnNumber = thisColumn; + + //行结束时的操作 + if (record instanceof LastCellOfRowDummyRecord) { + if (minColums > 0) { + //列值重新置空 + if (lastColumnNumber == -1) { + lastColumnNumber = 0; + } + } + lastColumnNumber = -1; + + if(!totalSheets.stream().map(ExcelSheetData::getSheetName).collect(Collectors.toList()).contains(sheetName)){ + ExcelSheetData excelSheetData = new ExcelSheetData(); + excelSheetData.setSheetName(sheetName); + excelSheetData.setData(new ArrayList<>()); + excelSheetData.setFields(new ArrayList<>()); + totalSheets.add(excelSheetData); + } + + if(curRow == 0){ + for (String s : cellList) { + TableFiled tableFiled = new TableFiled(); + tableFiled.setFieldType("TEXT"); + tableFiled.setFieldSize(65533); + tableFiled.setFieldName(s); + tableFiled.setRemarks(s); + this.fields.add(tableFiled); + totalSheets.get(totalSheets.size() -1).getFields().add(tableFiled); + } + } + + + if (flag && curRow != 0) { //该行不为空行且该行不是第一行，发送（第一行为列名，不需要） + if(!totalSheets.stream().map(ExcelSheetData::getSheetName).collect(Collectors.toList()).contains(sheetName)){ + ExcelSheetData excelSheetData = new ExcelSheetData(); + excelSheetData.setData(new ArrayList<>(data)); + excelSheetData.setSheetName(sheetName); + excelSheetData.setFields(new ArrayList<>(fields)); + List tmp = new ArrayList<>(cellList); + excelSheetData.getData().add(tmp); + totalRows++; + totalSheets.add(excelSheetData); + }else { + List tmp = new ArrayList<>(cellList); + totalSheets.stream().filter(s->s.getSheetName().equalsIgnoreCase(sheetName)).collect(Collectors.toList()).get(0).getData().add(tmp); + totalRows++; + } + } + + //清空容器 + cellList.clear(); + flag = false; + } + } + + /** + * 如果里面某个单元格含有值，则标识该行不为空行 + * + * @param value + */ + public void checkRowIsNull(String value) { + if (value != null && !"".equals(value)) { + flag = true; + } + } + + + private String checkType(String str, int thisColumn){ + String type = null; + try { + double d = Double.valueOf(str); + try { + Double value = new Double(d); + double eps = 1e-10; + if (value - Math.floor(value) < eps) { + type = "LONG"; + } else { + type = "DOUBLE"; + } + } catch (Exception e) { + type = "TEXT"; + } + }catch (Exception e){ + type = "TEXT"; + } + + String oldType = totalSheets.get(totalSheets.size() -1).getFields().get(thisColumn).getFieldType(); + if(type.equalsIgnoreCase("LONG") && oldType.equalsIgnoreCase("TEXT")){ + totalSheets.get(totalSheets.size() -1).getFields().get(thisColumn).setFieldType(type); + } + if(type.equalsIgnoreCase("DOUBLE")){ + totalSheets.get(totalSheets.size() -1).getFields().get(thisColumn).setFieldType(type); + } + return type; + } + +} \ No newline at end of file diff --git a/backend/src/main/java/io/dataease/commons/utils/ExcelXlsxReader.java b/backend/src/main/java/io/dataease/commons/utils/ExcelXlsxReader.java new file mode 100644 index 0000000000..fd0aa4ca8f --- /dev/null +++ b/backend/src/main/java/io/dataease/commons/utils/ExcelXlsxReader.java @@ -0,0 +1,470 @@ +package io.dataease.commons.utils; +import com.google.gson.Gson; +import io.dataease.datasource.dto.TableFiled; +import io.dataease.dto.dataset.ExcelSheetData; +import io.dataease.i18n.Translator; +import io.dataease.service.message.MsgAop; +import org.apache.commons.lang3.StringUtils; +import org.apache.poi.openxml4j.opc.OPCPackage; +import org.apache.poi.ss.usermodel.BuiltinFormats; +import org.apache.poi.ss.usermodel.DataFormatter; +import org.apache.poi.xssf.eventusermodel.XSSFReader; +import org.apache.poi.xssf.model.SharedStringsTable; +import org.apache.poi.xssf.model.StylesTable; +import org.apache.poi.xssf.usermodel.XSSFCellStyle; +import org.apache.poi.xssf.usermodel.XSSFRichTextString; +import org.xml.sax.Attributes; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; +import org.xml.sax.XMLReader; +import org.xml.sax.helpers.DefaultHandler; +import org.xml.sax.helpers.XMLReaderFactory; + +import java.io.InputStream; +import java.util.*; + +/** + * @author y + * @create 2018-01-18 14:28 + * @desc POI读取excel有两种模式，一种是用户模式，一种是事件驱动模式 + * 采用SAX事件驱动模式解决XLSX文件，可以有效解决用户模式内存溢出的问题， + * 该模式是POI官方推荐的读取大数据的模式， + * 在用户模式下，数据量较大，Sheet较多，或者是有很多无用的空行的情况下，容易出现内存溢出 + *

+ * 用于解决.xlsx2007版本大数据量问题 + **/ +public class ExcelXlsxReader extends DefaultHandler { + + /** + * 自定义获取表格某些信息 + */ + public Map map = new TreeMap(); + /** + * 单元格中的数据可能的数据类型 + */ + enum CellDataType { + BOOL, ERROR, FORMULA, INLINESTR, SSTINDEX, NUMBER, DATE, NULL + } + + /** + * 共享字符串表 + */ + private SharedStringsTable sst; + + /** + * 上一次的索引值 + */ + private String lastIndex; + + /** + * 总行数 + */ + private int totalRows=0; + + /** + * 一行内cell集合 + */ + private List cellList = new ArrayList(); + + /** + * 判断整行是否为空行的标记 + */ + private boolean flag = false; + + /** + * 当前行 + */ + private int curRow = 1; + + /** + * 当前列 + */ + private int curCol = 0; + + /** + * T元素标识 + */ + private boolean isTElement; + + /** + * 单元格数据类型，默认为字符串类型 + */ + private CellDataType nextDataType = CellDataType.SSTINDEX; + + private final DataFormatter formatter = new DataFormatter(); + + /** + * 单元格日期格式的索引 + */ + private short formatIndex; + + /** + * 日期格式字符串 + */ + private String formatString; + + //定义前一个元素和当前元素的位置，用来计算其中空的单元格数量，如A6和A8等 + private String preRef = null, ref = null; + + //定义该文档一行最大的单元格数，用来补全一行最后可能缺失的单元格 + private String maxRef = null; + + /** + * 单元格 + */ + private StylesTable stylesTable; + + public List fields = new ArrayList<>(); + public List> data = new ArrayList<>(); + public List totalSheets = new ArrayList<>(); + /** + * 是否为日期 + */ + private boolean isDateFormat = false; + + + public List getFields() { + return fields; + } + + public void setFields(List fields) { + this.fields = fields; + } + + public List> getData() { + return data; + } + + public void setData(List> data) { + this.data = data; + } + + public int process(InputStream inputStream) throws Exception { + OPCPackage pkg = OPCPackage.open(inputStream); + XSSFReader xssfReader = new XSSFReader(pkg); + stylesTable = xssfReader.getStylesTable(); + SharedStringsTable sst = xssfReader.getSharedStringsTable(); + XMLReader parser = XMLReaderFactory.createXMLReader("org.apache.xerces.parsers.SAXParser"); + this.sst = sst; + parser.setContentHandler(this); + XSSFReader.SheetIterator sheets = (XSSFReader.SheetIterator) xssfReader.getSheetsData(); + while (sheets.hasNext()) { //遍历sheet + curRow = 1; //标记初始行为第一行 + fields.clear(); + data.clear(); + InputStream sheet = sheets.next(); //sheets.next()和sheets.getSheetName()不能换位置，否则sheetName报错 + InputSource sheetSource = new InputSource(sheet); + parser.parse(sheetSource); //解析excel的每条记录，在这个过程中startElement()、characters()、endElement()这三个函数会依次执行 + + ExcelSheetData excelSheetData = new ExcelSheetData(); + excelSheetData.setData(new ArrayList<>(data)); + excelSheetData.setSheetName(sheets.getSheetName()); + excelSheetData.setFields(new ArrayList<>(fields)); + totalSheets.add(excelSheetData); + + sheet.close(); + } + return totalRows; //返回该excel文件的总行数，不包括首列和空行 + } + + /** + * 第一个执行 + * + * @param uri + * @param localName + * @param name + * @param attributes + * @throws SAXException + */ + @Override + public void startElement(String uri, String localName, String name, Attributes attributes) throws SAXException { + if(curRow>101){ + return; + } + + if(name.equalsIgnoreCase("mergeCell")){ + throw new RuntimeException(Translator.get("i18n_excel_have_merge_region")); + } + //c => 单元格 + if ("c".equals(name)) { + //当前单元格的位置 + ref = attributes.getValue("r"); + //设定单元格类型 + this.setNextDataType(attributes); + } + + //当元素为t时 + if ("t".equals(name)) { + isTElement = true; + } else { + isTElement = false; + } + + //置空 + lastIndex = ""; + } + + /** + * 第二个执行 + * 得到单元格对应的索引值或是内容值 + * 如果单元格类型是字符串、INLINESTR、数字、日期，lastIndex则是索引值 + * 如果单元格类型是布尔值、错误、公式，lastIndex则是内容值 + * @param ch + * @param start + * @param length + * @throws SAXException + */ + @Override + public void characters(char[] ch, int start, int length) throws SAXException { + if(curRow>101){ + return; + } + lastIndex += new String(ch, start, length); + } + + /** + * 第三个执行 + * + * @param uri + * @param localName + * @param name + * @throws SAXException + */ + @Override + public void endElement(String uri, String localName, String name) throws SAXException { + if(curRow>101){ + return; + } + //t元素也包含字符串 + if (isTElement) {//这个程序没经过 + //将单元格内容加入rowlist中，在这之前先去掉字符串前后的空白符 + String value = lastIndex.trim(); + cellList.add(curCol, value); + curCol++; + isTElement = false; + //如果里面某个单元格含有值，则标识该行不为空行 + if (value != null && !"".equals(value)) { + flag = true; + } + } else if ("v".equals(name)) { + //v => 单元格的值，如果单元格是字符串，则v标签的值为该字符串在SST中的索引 + String value = this.getDataValue(lastIndex.trim(), "");//根据索引值获取对应的单元格值 + if (preRef == null) { + preRef = ref; + } + //补全单元格之间的空单元格 + if (!ref.equals(preRef)) { + int len = countNullCell(ref, preRef); + for (int i = 0; i < len; i++) { + cellList.add(curCol, ""); + curCol++; + } + } + cellList.add(curCol, value); + curCol++; + //如果里面某个单元格含有值，则标识该行不为空行 + if (value != null && !"".equals(value)) { + flag = true; + } + preRef = ref; + } else { + //如果标签名称为row，这说明已到行尾 + if ("row".equals(name)) { + //默认第一行为表头，以该行单元格数目为最大数目 + if (curRow == 1) { + maxRef = ref; + } + //补全一行尾部可能缺失的单元格 + if (maxRef != null) { + int len = countNullCell(maxRef, ref); + for (int i = 0; i <= len; i++) { + cellList.add(curCol, ""); + curCol++; + } + } + if(curRow>1){ + List tmp = new ArrayList<>(cellList); + this.getData().add(tmp); + } + totalRows++; + cellList.clear(); + curRow++; + curCol = 0; + preRef = null; + ref = null; + flag=false; + } + } + } + + /** + * 处理数据类型 + * + * @param attributes + */ + public void setNextDataType(Attributes attributes) { + nextDataType = CellDataType.NUMBER; //cellType为空，则表示该单元格类型为数字 + formatIndex = -1; + formatString = null; + String cellType = attributes.getValue("t"); //单元格类型 + if ("b".equals(cellType)) { //处理布尔值 + nextDataType = CellDataType.BOOL; + } else if ("e".equals(cellType)) { //处理错误 + nextDataType = CellDataType.ERROR; + } else if ("inlineStr".equals(cellType)) { + nextDataType = CellDataType.INLINESTR; + } else if ("s".equals(cellType)) { //处理字符串 + nextDataType = CellDataType.SSTINDEX; + } else if ("str".equals(cellType)) { + nextDataType = CellDataType.FORMULA; + } + + String cellStyleStr = attributes.getValue("s"); // + if (cellStyleStr != null) { + int styleIndex = Integer.parseInt(cellStyleStr); + XSSFCellStyle style = this.stylesTable.getStyleAt(styleIndex); + formatIndex = style.getDataFormat(); + formatString = style.getDataFormatString(); + short format = this.formatIndex; + if (format == 14 || format == 31 || format == 57 ||format == 59|| + format == 58 || (176 <= format && format <= 178) + || (182 <= format && format <= 196) || + (210 <= format && format <= 213) || (208 == format)) + { // 日期 + isDateFormat = true; + } + } + + } + + /** + * 对解析出来的数据进行类型处理 + * @param value 单元格的值， + * value代表解析：BOOL的为0或1， ERROR的为内容值，FORMULA的为内容值，INLINESTR的为索引值需转换为内容值， + * SSTINDEX的为索引值需转换为内容值， NUMBER为内容值，DATE为内容值 + * @param thisStr 一个空字符串 + * @return + */ + @SuppressWarnings("deprecation") + public String getDataValue(String value, String thisStr) { + String type = "TEXT"; + switch (nextDataType) { + // 这几个的顺序不能随便交换，交换了很可能会导致数据错误 + case BOOL: //布尔值 + char first = value.charAt(0); + thisStr = first == '0' ? "FALSE" : "TRUE"; + type = "LONG"; + break; + case ERROR: //错误 + thisStr = "\"ERROR:" + value.toString() + '"'; + break; + case FORMULA: //公式 + thisStr = '"' + value.toString() + '"'; + type = getType(thisStr); + break; + case INLINESTR: + XSSFRichTextString rtsi = new XSSFRichTextString(value.toString()); + thisStr = rtsi.toString(); + rtsi = null; + break; + case SSTINDEX: //字符串 + String sstIndex = value.toString(); + try { + int idx = Integer.parseInt(sstIndex); + XSSFRichTextString rtss = new XSSFRichTextString(sst.getEntryAt(idx));//根据idx索引值获取内容值 + thisStr = rtss.toString(); + rtss = null; + } catch (NumberFormatException ex) { + thisStr = value.toString(); + } + + break; + case NUMBER: //数字 + if (formatString != null) { + thisStr = formatter.formatRawCellContents(Double.parseDouble(value), formatIndex, formatString).trim(); + } else { + thisStr = value; + } + thisStr = thisStr.replace("_", "").trim(); + if(isDateFormat){ + type = "DATETIME";isDateFormat = false; + }else { + type = getType(thisStr); + } + break; + case DATE: //日期 + thisStr = formatter.formatRawCellContents(Double.parseDouble(value), formatIndex, formatString); + // 对日期字符串作特殊处理，去掉T + thisStr = thisStr.replace("T", " "); + type = "DATETIME"; + break; + default: + thisStr = " "; + break; + } + if(curRow==1){ + TableFiled tableFiled = new TableFiled(); + tableFiled.setFieldType(type); + tableFiled.setFieldSize(65533); + tableFiled.setFieldName(thisStr); + tableFiled.setRemarks(thisStr); + this.fields.add(tableFiled); + }else { + this.getFields().get(curCol).setFieldType(type); + } + return thisStr; + } + + private String getType(String thisStr){ + if(totalRows==0){ + return "TEXT"; + } + try{ + if(thisStr.endsWith("%")){ + thisStr = thisStr.substring(0, thisStr.length()-1); + thisStr = String.valueOf(Double.valueOf(thisStr)/100); + } + Long.valueOf(thisStr); + if(this.getFields().get(curCol).getFieldType().equalsIgnoreCase("TEXT")){ + return "LONG"; + } + }catch (Exception e){ + try { + Double.valueOf(thisStr); + return "DOUBLE"; + }catch (Exception ignore){ } + } + return "TEXT"; + } + + public int countNullCell(String ref, String preRef) { + //excel2007最大行数是1048576，最大列数是16384，最后一列列名是XFD + String xfd = ref.replaceAll("\\d+", ""); + String xfd_1 = preRef.replaceAll("\\d+", ""); + + xfd = fillChar(xfd, 3, '@', true); + xfd_1 = fillChar(xfd_1, 3, '@', true); + + char[] letter = xfd.toCharArray(); + char[] letter_1 = xfd_1.toCharArray(); + int res = (letter[0] - letter_1[0]) * 26 * 26 + (letter[1] - letter_1[1]) * 26 + (letter[2] - letter_1[2]); + return res - 1; + } + + public String fillChar(String str, int len, char let, boolean isPre) { + int len_1 = str.length(); + if (len_1 < len) { + if (isPre) { + for (int i = 0; i < (len - len_1); i++) { + str = let + str; + } + } else { + for (int i = 0; i < (len - len_1); i++) { + str = str + let; + } + } + } + return str; + } + +} \ No newline at end of file diff --git a/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java b/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java index b0da9a2813..d7c5d51301 100644 --- a/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java +++ b/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java @@ -185,6 +185,9 @@ public class JdbcProvider extends DatasourceProvider { field.setFieldType(t); field.setFieldSize(metaData.getColumnDisplaySize(j + 1)); if(t.equalsIgnoreCase("LONG")){field.setFieldSize(65533);} //oracle LONG + if(StringUtils.isNotEmpty(t) && t.toLowerCase().contains("date") && field.getFieldSize() < 50 ){ + field.setFieldSize(50); + } fieldList.add(field); } return fieldList; diff --git a/backend/src/main/java/io/dataease/dto/dataset/ExcelSheetData.java b/backend/src/main/java/io/dataease/dto/dataset/ExcelSheetData.java new file mode 100644 index 0000000000..7e81d0fefd --- /dev/null +++ b/backend/src/main/java/io/dataease/dto/dataset/ExcelSheetData.java @@ -0,0 +1,13 @@ +package io.dataease.dto.dataset; + +import io.dataease.datasource.dto.TableFiled; +import lombok.Data; + +import java.util.List; + +@Data +public class ExcelSheetData { + private String sheetName; + private List> data; + private List fields; +} diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index 897d7fbb4f..3229c3fffe 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -1012,7 +1012,7 @@ public class DataSetTableService { public Map excelSaveAndParse(MultipartFile file, String tableId) throws Exception { String filename = file.getOriginalFilename(); // parse file - Map fileMap = parseExcel(filename, file.getInputStream(), true); + Map fileMap = parseExcel2(filename, file.getInputStream(), true); if (StringUtils.isNotEmpty(tableId)) { List datasetTableFields = dataSetTableFieldsService.getFieldsByTableId(tableId); datasetTableFields.sort((o1, o2) -> { @@ -1038,6 +1038,52 @@ public class DataSetTableService { return map; } + private Map parseExcel2(String filename, InputStream inputStream, boolean isPreview) throws Exception { + String suffix = filename.substring(filename.lastIndexOf(".") + 1); + List fields = new ArrayList<>(); + List> data = new ArrayList<>(); + List> jsonArray = new ArrayList<>(); + List sheets = new ArrayList<>(); + if (StringUtils.equalsIgnoreCase(suffix, "xls")) { + ExcelXlsReader excelXlsReader = new ExcelXlsReader(); + excelXlsReader.process(inputStream); + fields = excelXlsReader.totalSheets.get(0).getFields(); + data = excelXlsReader.totalSheets.get(0).getData(); + sheets = excelXlsReader.totalSheets.stream().map(ExcelSheetData::getSheetName).collect(Collectors.toList()); + } + if (StringUtils.equalsIgnoreCase(suffix, "xlsx")) { + ExcelXlsxReader excelXlsxReader = new ExcelXlsxReader(); + excelXlsxReader.process(inputStream); + fields = excelXlsxReader.totalSheets.get(0).getFields(); + data = excelXlsxReader.totalSheets.get(0).getData(); + sheets = excelXlsxReader.totalSheets.stream().map(ExcelSheetData::getSheetName).collect(Collectors.toList()); + } + + String[] fieldArray = fields.stream().map(TableFiled::getFieldName).toArray(String[]::new); + + // 校验excel字段是否重名 + if (checkIsRepeat(fieldArray)) { + DataEaseException.throwException(Translator.get("i18n_excel_field_repeat")); + } + + if (CollectionUtils.isNotEmpty(data)) { + jsonArray = data.stream().map(ele -> { + Map map = new HashMap<>(); + for (int i = 0; i < ele.size(); i++) { + map.put(fieldArray[i], ele.get(i)); + } + return map; + }).collect(Collectors.toList()); + } + inputStream.close(); + + Map map = new HashMap<>(); + map.put("fields", fields); + map.put("data", jsonArray); + map.put("sheets", sheets); + return map; + } + private Map parseExcel(String filename, InputStream inputStream, boolean isPreview) throws Exception { String suffix = filename.substring(filename.lastIndexOf(".") + 1); List fields = new ArrayList<>(); @@ -1191,6 +1237,7 @@ public class DataSetTableService { return map; } + private String readCell(Cell cell, boolean cellType, TableFiled tableFiled) { if (cell == null) { return ""; From 66335c3880ce1edebe12003157a2c76bf72b7f4d Mon Sep 17 00:00:00 2001 From: fit2cloud-chenyw Date: Wed, 21 Jul 2021 17:53:36 +0800 Subject: [PATCH 2/9] =?UTF-8?q?fix:=20=E6=B7=BB=E5=8A=A0=E8=8F=9C=E5=8D=95?= =?UTF-8?q?=E6=BB=9A=E5=8A=A8=E6=9D=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/layout/index.vue | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/layout/index.vue b/frontend/src/layout/index.vue index d2841ae1fc..313ed1ff09 100644 --- a/frontend/src/layout/index.vue +++ b/frontend/src/layout/index.vue @@ -121,6 +121,8 @@ export default { width: 100% !important; position: initial !important; height: calc(100vh - 80px) !important; + overflow-x: hidden !important; + overflow-y: auto !important; } } From 14a58ac561f8cb1b8d72817e2cb1684768e4872f Mon Sep 17 00:00:00 2001 From: fit2cloud-chenyw Date: Thu, 22 Jul 2021 16:59:34 +0800 Subject: [PATCH 3/9] =?UTF-8?q?fix:=201.=E7=B1=BB=E5=9E=8B=E6=8B=8D?= =?UTF-8?q?=E6=88=8F=E6=8A=A5=E9=94=99=20=20=20=20=20=202.=E5=88=86?= =?UTF-8?q?=E9=A1=B5=E6=95=B0=E6=8D=AE=E9=94=99=E8=AF=AF=20=20=20=20=20=20?= =?UTF-8?q?3.=E5=85=A8=E9=83=A8=E6=B6=88=E6=81=AF=E5=8F=96=E6=B6=88?= =?UTF-8?q?=E9=BB=98=E8=AE=A4=E7=8A=B6=E6=80=81=E6=8E=92=E5=BA=8F=20=20=20?= =?UTF-8?q?=20=20=204.=E6=95=B0=E6=8D=AE=E9=9B=86=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=E6=B6=88=E6=81=AF=E8=B7=B3=E8=BD=AC=E8=B7=AF=E7=94=B1=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../base/mapper/ext/ExtSysMsgMapper.xml | 2 +- .../controller/message/MsgController.java | 13 ++++- .../service/message/SysMsgService.java | 7 ++- frontend/src/views/dataset/index.vue | 8 +-- frontend/src/views/msg/all.vue | 7 ++- frontend/src/views/msg/readed.vue | 5 +- frontend/src/views/msg/unread.vue | 5 +- frontend/src/views/system/task/TaskRecord.vue | 25 ++++++--- frontend/src/views/system/task/dataset.vue | 52 +++++++++++++++---- 9 files changed, 94 insertions(+), 30 deletions(-) diff --git a/backend/src/main/java/io/dataease/base/mapper/ext/ExtSysMsgMapper.xml b/backend/src/main/java/io/dataease/base/mapper/ext/ExtSysMsgMapper.xml index 53892fc215..90959633e4 100644 --- a/backend/src/main/java/io/dataease/base/mapper/ext/ExtSysMsgMapper.xml +++ b/backend/src/main/java/io/dataease/base/mapper/ext/ExtSysMsgMapper.xml @@ -52,7 +52,7 @@ order by ${orderByClause} - order by sm.status asc + order by sm.create_time desc diff --git a/backend/src/main/java/io/dataease/controller/message/MsgController.java b/backend/src/main/java/io/dataease/controller/message/MsgController.java index 6f84a2e4df..ec52740417 100644 --- a/backend/src/main/java/io/dataease/controller/message/MsgController.java +++ b/backend/src/main/java/io/dataease/controller/message/MsgController.java @@ -15,9 +15,15 @@ import io.dataease.controller.message.dto.SettingTreeNode; import io.dataease.service.message.SysMsgService; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; import org.springframework.web.bind.annotation.*; import javax.annotation.Resource; +import java.util.ArrayList; import java.util.List; +import java.util.stream.Collector; +import java.util.stream.Collectors; +import java.util.stream.Stream; @Api(tags = "系统：消息管理") @RequestMapping("/api/sys_msg") @@ -31,8 +37,13 @@ public class MsgController { @PostMapping("/list/{goPage}/{pageSize}") public Pager> messages(@PathVariable int goPage, @PathVariable int pageSize, @RequestBody MsgRequest msgRequest) { Long userId = AuthUtils.getUser().getUserId(); + List typeIds = null; + if (ObjectUtils.isNotEmpty(msgRequest.getType())){ + List sysMsgTypes = sysMsgService.queryMsgTypes(); + typeIds = sysMsgTypes.stream().filter(sysMsgType -> msgRequest.getType() == sysMsgType.getPid()).map(SysMsgType::getMsgTypeId).collect(Collectors.toList()); + } Page page = PageHelper.startPage(goPage, pageSize, true); - Pager> listPager = PageUtils.setPageInfo(page, sysMsgService.queryGrid(userId, msgRequest)); + Pager> listPager = PageUtils.setPageInfo(page, sysMsgService.queryGrid(userId, msgRequest, typeIds)); return listPager; } diff --git a/backend/src/main/java/io/dataease/service/message/SysMsgService.java b/backend/src/main/java/io/dataease/service/message/SysMsgService.java index 0bc5cdb8c8..a711fc54d8 100644 --- a/backend/src/main/java/io/dataease/service/message/SysMsgService.java +++ b/backend/src/main/java/io/dataease/service/message/SysMsgService.java @@ -67,7 +67,7 @@ public class SysMsgService { return sysMsgs; } - public List queryGrid(Long userId, MsgRequest msgRequest) { + public List queryGrid(Long userId, MsgRequest msgRequest, List typeIds) { String orderClause = " create_time desc"; SysMsgExample example = new SysMsgExample(); SysMsgExample.Criteria criteria = example.createCriteria(); @@ -79,13 +79,16 @@ public class SysMsgService { orderClause = String.join(", ", orders); } - if (ObjectUtils.isNotEmpty(msgRequest.getType())) { + /*if (ObjectUtils.isNotEmpty(msgRequest.getType())) { SysMsgTypeExample sysMsgTypeExample = new SysMsgTypeExample(); sysMsgTypeExample.createCriteria().andPidEqualTo(msgRequest.getType()); List sysMsgTypes = sysMsgTypeMapper.selectByExample(sysMsgTypeExample); List typeIds = sysMsgTypes.stream().map(SysMsgType::getMsgTypeId).collect(Collectors.toList()); criteria.andTypeIdIn(typeIds); + }*/ + if (CollectionUtils.isNotEmpty(typeIds)){ + criteria.andTypeIdIn(typeIds); } if (ObjectUtils.isNotEmpty(msgRequest.getStatus())) { diff --git a/frontend/src/views/dataset/index.vue b/frontend/src/views/dataset/index.vue index 49415c7a7e..3b20bc15a0 100644 --- a/frontend/src/views/dataset/index.vue +++ b/frontend/src/views/dataset/index.vue @@ -26,7 +26,7 @@ import AddExcel from './add/AddExcel' import AddCustom from './add/AddCustom' import FieldEdit from './data/FieldEdit' import { removeClass } from '@/utils' -import bus from '@/utils/bus' +// import bus from '@/utils/bus' export default { name: 'DataSet', components: { DeMainContainer, DeContainer, DeAsideContainer, Group, DataHome, ViewTable, AddDB, AddSQL, AddExcel, AddCustom }, @@ -39,9 +39,9 @@ export default { }, mounted() { removeClass(document.body, 'showRightPanel') - bus.$on('to-msg-dataset', params => { - this.toMsgShare(params) - }) + // bus.$on('to-msg-dataset', params => { + // this.toMsgShare(params) + // }) }, created() { this.$store.dispatch('app/toggleSideBarHide', true) diff --git a/frontend/src/views/msg/all.vue b/frontend/src/views/msg/all.vue index 803a24ba12..51ab3dd58b 100644 --- a/frontend/src/views/msg/all.vue +++ b/frontend/src/views/msg/all.vue @@ -36,7 +36,7 @@ - + @@ -101,7 +101,7 @@ export default { } if (this.orderConditions.length === 0) { - param.orders = [' status asc ', 'create_time desc '] + param.orders = ['create_time desc '] } else { param.orders = formatOrders(this.orderConditions) } @@ -139,6 +139,9 @@ export default { if (prop === 'createTime') { prop = 'create_time' } + if (prop === 'typeId') { + prop = 'type_id' + } addOrder({ field: prop, value: order }, this.orderConditions) this.search() } diff --git a/frontend/src/views/msg/readed.vue b/frontend/src/views/msg/readed.vue index 573f4bf313..bddc2e0f89 100644 --- a/frontend/src/views/msg/readed.vue +++ b/frontend/src/views/msg/readed.vue @@ -42,7 +42,7 @@ - + @@ -139,6 +139,9 @@ export default { if (prop === 'readTime') { prop = 'read_time' } + if (prop === 'typeId') { + prop = 'type_id' + } addOrder({ field: prop, value: order }, this.orderConditions) this.search() } diff --git a/frontend/src/views/msg/unread.vue b/frontend/src/views/msg/unread.vue index b017942771..83ecdeb2e2 100644 --- a/frontend/src/views/msg/unread.vue +++ b/frontend/src/views/msg/unread.vue @@ -45,7 +45,7 @@ - + @@ -168,6 +168,9 @@ export default { if (prop === 'createTime') { prop = 'create_time' } + if (prop === 'typeId') { + prop = 'type_id' + } addOrder({ field: prop, value: order }, this.orderConditions) this.search() } diff --git a/frontend/src/views/system/task/TaskRecord.vue b/frontend/src/views/system/task/TaskRecord.vue index 218e8e6e0b..c717c1cbcc 100644 --- a/frontend/src/views/system/task/TaskRecord.vue +++ b/frontend/src/views/system/task/TaskRecord.vue @@ -2,7 +2,7 @@ - +