From 7285eb11bf7483f19211e4dd711f434f8ac714b0 Mon Sep 17 00:00:00 2001 From: taojinlong Date: Wed, 21 Jul 2021 15:07:51 +0800 Subject: [PATCH 01/55] =?UTF-8?q?feat:=20SAX=20=E8=A7=A3=E6=9E=90=20excel?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../commons/utils/ExcelReaderUtil.java | 75 +++ .../commons/utils/ExcelXlsReader.java | 401 +++++++++++++++ .../commons/utils/ExcelXlsxReader.java | 470 ++++++++++++++++++ .../datasource/provider/JdbcProvider.java | 3 + .../dataease/dto/dataset/ExcelSheetData.java | 13 + .../service/dataset/DataSetTableService.java | 49 +- 6 files changed, 1010 insertions(+), 1 deletion(-) create mode 100644 backend/src/main/java/io/dataease/commons/utils/ExcelReaderUtil.java create mode 100644 backend/src/main/java/io/dataease/commons/utils/ExcelXlsReader.java create mode 100644 backend/src/main/java/io/dataease/commons/utils/ExcelXlsxReader.java create mode 100644 backend/src/main/java/io/dataease/dto/dataset/ExcelSheetData.java diff --git a/backend/src/main/java/io/dataease/commons/utils/ExcelReaderUtil.java b/backend/src/main/java/io/dataease/commons/utils/ExcelReaderUtil.java new file mode 100644 index 0000000000..4ae57a6f5f --- /dev/null +++ b/backend/src/main/java/io/dataease/commons/utils/ExcelReaderUtil.java @@ -0,0 +1,75 @@ +package io.dataease.commons.utils; +import com.google.gson.Gson; +import io.dataease.datasource.dto.TableFiled; + +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class ExcelReaderUtil { + //excel2003扩展名 + public static final String EXCEL03_EXTENSION = ".xls"; + //excel2007扩展名 + public static final String EXCEL07_EXTENSION = ".xlsx"; + + public static void sendRows(String filePath, String sheetName, int sheetIndex, int curRow, List cellList) { + StringBuffer oneLineSb = new StringBuffer(); + oneLineSb.append(filePath); + oneLineSb.append("--"); + oneLineSb.append("sheet" + sheetIndex); + oneLineSb.append("::" + sheetName);//加上sheet名 + oneLineSb.append("--"); + oneLineSb.append("row" + curRow); + oneLineSb.append("::"); + + // map.put(cellList.get(9),cellList.get(0)) ; + + for (String cell : cellList) { + oneLineSb.append(cell.trim()); + oneLineSb.append("|"); + } + String oneLine = oneLineSb.toString(); + if (oneLine.endsWith("|")) { + oneLine = oneLine.substring(0, oneLine.lastIndexOf("|")); + }// 去除最后一个分隔符 + + System.out.println(oneLine); + } + + /** + * 读取excel文件路径 + * @param fileName 文件路径 + * @throws Exception + */ + public static void readExcel(String fileName, InputStream inputStream) throws Exception { + if (fileName.endsWith(EXCEL03_EXTENSION)) { //处理excel2003文件 + ExcelXlsReader excelXls=new ExcelXlsReader(); + excelXls.process(inputStream); + System.out.println(excelXls.totalSheets.size()); + System.out.println(excelXls.totalSheets.get(0).getSheetName()); + for (TableFiled field : excelXls.totalSheets.get(0).getFields()) { + System.out.println(new Gson().toJson(field)); + } + System.out.println(excelXls.totalSheets.get(0).getData().get(0)); + + } else if (fileName.endsWith(EXCEL07_EXTENSION)) {//处理excel2007文件 + ExcelXlsxReader excelXlsxReader = new ExcelXlsxReader(); + excelXlsxReader.process(inputStream); + System.out.println(excelXlsxReader.totalSheets.size()); + System.out.println(excelXlsxReader.totalSheets.get(0).getSheetName()); + for (TableFiled field : excelXlsxReader.totalSheets.get(0).getFields()) { + System.out.println(new Gson().toJson(field)); + } + System.out.println(excelXlsxReader.totalSheets.get(0).getData().get(0)); + + } else { + throw new Exception("文件格式错误，fileName的扩展名只能是xls或xlsx。"); + } + } + + public static void main(String[] args) throws Exception { + ExcelReaderUtil.readExcel("111.xls", new FileInputStream("/Users/taojinlong/Desktop/111.xls")); + } +} diff --git a/backend/src/main/java/io/dataease/commons/utils/ExcelXlsReader.java b/backend/src/main/java/io/dataease/commons/utils/ExcelXlsReader.java new file mode 100644 index 0000000000..7a12665e48 --- /dev/null +++ b/backend/src/main/java/io/dataease/commons/utils/ExcelXlsReader.java @@ -0,0 +1,401 @@ +package io.dataease.commons.utils; + +import com.google.gson.Gson; +import io.dataease.datasource.dto.TableFiled; +import io.dataease.dto.dataset.ExcelSheetData; +import io.dataease.i18n.Translator; +import org.apache.poi.hssf.eventusermodel.*; +import org.apache.poi.hssf.eventusermodel.dummyrecord.LastCellOfRowDummyRecord; +import org.apache.poi.hssf.eventusermodel.dummyrecord.MissingCellDummyRecord; +import org.apache.poi.hssf.model.HSSFFormulaParser; +import org.apache.poi.hssf.record.*; +import org.apache.poi.hssf.usermodel.HSSFDataFormatter; +import org.apache.poi.hssf.usermodel.HSSFWorkbook; +import org.apache.poi.poifs.filesystem.POIFSFileSystem; + +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * @author y + * @create 2018-01-19 14:18 + * @desc 用于解决.xls2003版本大数据量问题 + **/ +public class ExcelXlsReader implements HSSFListener { + + public ExcelReaderUtil excelReaderUtil = new ExcelReaderUtil(); + + private int minColums = -1; + + private POIFSFileSystem fs; + + /** + * 总行数 + */ + private int totalRows = 0; + + /** + * 上一行row的序号 + */ + private int lastRowNumber; + + /** + * 上一单元格的序号 + */ + private int lastColumnNumber; + + /** + * 是否输出formula，还是它对应的值 + */ + private boolean outputFormulaValues = true; + + /** + * 用于转换formulas + */ + private EventWorkbookBuilder.SheetRecordCollectingListener workbookBuildingListener; + + //excel2003工作簿 + private HSSFWorkbook stubWorkbook; + + private SSTRecord sstRecord; + + private FormatTrackingHSSFListener formatListener; + + private final HSSFDataFormatter formatter = new HSSFDataFormatter(); + + /** + * 文件的绝对路径 + */ + private String filePath = ""; + + //表索引 + private int sheetIndex = 0; + + private BoundSheetRecord[] orderedBSRs; + + @SuppressWarnings("unchecked") + private ArrayList boundSheetRecords = new ArrayList(); + + private int nextRow; + + private int nextColumn; + + private boolean outputNextStringRecord; + + //当前行 + private int curRow = 0; + + //存储一行记录所有单元格的容器 + private List cellList = new ArrayList(); + + /** + * 判断整行是否为空行的标记 + */ + private boolean flag = false; + + @SuppressWarnings("unused") + private String sheetName; + + public List fields = new ArrayList<>(); + public List> data = new ArrayList<>(); + public List totalSheets = new ArrayList<>(); + /** + * 是否为日期 + */ + private boolean isDateFormat = false; + + + public List getFields() { + return fields; + } + + public void setFields(List fields) { + this.fields = fields; + } + + public List> getData() { + return data; + } + + public void setData(List> data) { + this.data = data; + } + + + + /** + * 遍历excel下所有的sheet + * + * @param inputStream + * @throws Exception + */ + public int process(InputStream inputStream) throws Exception { + this.fs = new POIFSFileSystem(inputStream); + MissingRecordAwareHSSFListener listener = new MissingRecordAwareHSSFListener(this); + formatListener = new FormatTrackingHSSFListener(listener); + HSSFEventFactory factory = new HSSFEventFactory(); + HSSFRequest request = new HSSFRequest(); + if (outputFormulaValues) { + request.addListenerForAllRecords(formatListener); + } else { + workbookBuildingListener = new EventWorkbookBuilder.SheetRecordCollectingListener(formatListener); + request.addListenerForAllRecords(workbookBuildingListener); + } + factory.processWorkbookEvents(request, fs); + + return totalRows; //返回该excel文件的总行数，不包括首列和空行 + } + + /** + * HSSFListener 监听方法，处理Record + * 处理每个单元格 + * + * @param record + */ + @SuppressWarnings("unchecked") + public void processRecord(Record record) { + int thisRow = -1; + int thisColumn = -1; + String thisStr = null; + String value = null; + switch (record.getSid()) { + case BoundSheetRecord.sid: + boundSheetRecords.add(record); + break; + case BOFRecord.sid: //开始处理每个sheet + BOFRecord br = (BOFRecord) record; + if (br.getType() == BOFRecord.TYPE_WORKSHEET) { + //如果有需要，则建立子工作簿 + if (workbookBuildingListener != null && stubWorkbook == null) { + stubWorkbook = workbookBuildingListener.getStubHSSFWorkbook(); + } + + if (orderedBSRs == null) { + orderedBSRs = BoundSheetRecord.orderByBofPosition(boundSheetRecords); + } + sheetName = orderedBSRs[sheetIndex].getSheetname(); + sheetIndex++; + } + break; + case MergeCellsRecord.sid: + throw new RuntimeException(Translator.get("i18n_excel_have_merge_region")); + case SSTRecord.sid: + sstRecord = (SSTRecord) record; + break; + case BlankRecord.sid: //单元格为空白 + BlankRecord brec = (BlankRecord) record; + thisRow = brec.getRow(); + thisColumn = brec.getColumn(); + thisStr = ""; + cellList.add(thisColumn, thisStr); + break; + case BoolErrRecord.sid: //单元格为布尔类型 + BoolErrRecord berec = (BoolErrRecord) record; + thisRow = berec.getRow(); + thisColumn = berec.getColumn(); + thisStr = berec.getBooleanValue() + ""; + cellList.add(thisColumn, thisStr); + + checkRowIsNull(thisStr); //如果里面某个单元格含有值，则标识该行不为空行 + break; + case FormulaRecord.sid://单元格为公式类型 + FormulaRecord frec = (FormulaRecord) record; + thisRow = frec.getRow(); + thisColumn = frec.getColumn(); + thisStr = String.valueOf(frec.getValue()); +// if (outputFormulaValues) { +// if (Double.isNaN(frec.getValue())) { +// outputNextStringRecord = true; +// nextRow = frec.getRow(); +// nextColumn = frec.getColumn(); +// } else { +// thisStr = '"' + HSSFFormulaParser.toFormulaString(stubWorkbook, frec.getParsedExpression()) + '"'; +// } +// } else { +// thisStr = '"' + HSSFFormulaParser.toFormulaString(stubWorkbook, frec.getParsedExpression()) + '"'; +// } + String feildType = checkType(thisStr, thisColumn); + if(feildType.equalsIgnoreCase("LONG") && thisStr.endsWith(".0")){ + thisStr = thisStr.substring(0, thisStr.length() -2); + } + cellList.add(thisColumn, thisStr); + checkRowIsNull(thisStr); //如果里面某个单元格含有值，则标识该行不为空行 + break; + case StringRecord.sid: //单元格中公式的字符串 + if (outputNextStringRecord) { + StringRecord srec = (StringRecord) record; + thisStr = srec.getString(); + thisRow = nextRow; + thisColumn = nextColumn; + outputNextStringRecord = false; + } + break; + case LabelRecord.sid: + LabelRecord lrec = (LabelRecord) record; + curRow = thisRow = lrec.getRow(); + thisColumn = lrec.getColumn(); + value = lrec.getValue().trim(); + value = value.equals("") ? "" : value; + cellList.add(thisColumn, value); + checkRowIsNull(value); //如果里面某个单元格含有值，则标识该行不为空行 + break; + case LabelSSTRecord.sid: //单元格为字符串类型 + LabelSSTRecord lsrec = (LabelSSTRecord) record; + curRow = thisRow = lsrec.getRow(); + thisColumn = lsrec.getColumn(); + if (sstRecord == null) { + cellList.add(thisColumn, ""); + } else { + value = sstRecord.getString(lsrec.getSSTIndex()).toString().trim(); + value = value.equals("") ? "" : value; + cellList.add(thisColumn, value); + checkRowIsNull(value); //如果里面某个单元格含有值，则标识该行不为空行 + } + break; + case NumberRecord.sid: //单元格为数字类型 + NumberRecord numrec = (NumberRecord) record; + curRow = thisRow = numrec.getRow(); + thisColumn = numrec.getColumn(); + //第一种方式 + //value = formatListener.formatNumberDateCell(numrec).trim();//这个被写死，采用的m/d/yy h:mm格式，不符合要求 + //第二种方式，参照formatNumberDateCell里面的实现方法编写 + Double valueDouble = ((NumberRecord) numrec).getValue(); + String formatString = formatListener.getFormatString(numrec); + if (formatString.contains("m/d/yy")) { + formatString = "yyyy-MM-dd hh:mm:ss"; + } + int formatIndex = formatListener.getFormatIndex(numrec); + + value = formatter.formatRawCellContents(valueDouble, formatIndex, formatString).trim(); + value = value.equals("") ? "" : value; + //向容器加入列值 + cellList.add(thisColumn, value); + if(formatIndex == 59){ + totalSheets.get(totalSheets.size() -1).getFields().get(thisColumn).setFieldType("DATETIME"); + }else { + checkType(value, thisColumn); + } + checkRowIsNull(value); //如果里面某个单元格含有值，则标识该行不为空行 + break; + default: + break; + } + + //遇到新行的操作 + if (thisRow != -1 && thisRow != lastRowNumber) { + lastColumnNumber = -1; + } + + //空值的操作 + if (record instanceof MissingCellDummyRecord) { + MissingCellDummyRecord mc = (MissingCellDummyRecord) record; + curRow = thisRow = mc.getRow(); + thisColumn = mc.getColumn(); + cellList.add(thisColumn, ""); + } + + //更新行和列的值 + if (thisRow > -1) + lastRowNumber = thisRow; + if (thisColumn > -1) + lastColumnNumber = thisColumn; + + //行结束时的操作 + if (record instanceof LastCellOfRowDummyRecord) { + if (minColums > 0) { + //列值重新置空 + if (lastColumnNumber == -1) { + lastColumnNumber = 0; + } + } + lastColumnNumber = -1; + + if(!totalSheets.stream().map(ExcelSheetData::getSheetName).collect(Collectors.toList()).contains(sheetName)){ + ExcelSheetData excelSheetData = new ExcelSheetData(); + excelSheetData.setSheetName(sheetName); + excelSheetData.setData(new ArrayList<>()); + excelSheetData.setFields(new ArrayList<>()); + totalSheets.add(excelSheetData); + } + + if(curRow == 0){ + for (String s : cellList) { + TableFiled tableFiled = new TableFiled(); + tableFiled.setFieldType("TEXT"); + tableFiled.setFieldSize(65533); + tableFiled.setFieldName(s); + tableFiled.setRemarks(s); + this.fields.add(tableFiled); + totalSheets.get(totalSheets.size() -1).getFields().add(tableFiled); + } + } + + + if (flag && curRow != 0) { //该行不为空行且该行不是第一行，发送（第一行为列名，不需要） + if(!totalSheets.stream().map(ExcelSheetData::getSheetName).collect(Collectors.toList()).contains(sheetName)){ + ExcelSheetData excelSheetData = new ExcelSheetData(); + excelSheetData.setData(new ArrayList<>(data)); + excelSheetData.setSheetName(sheetName); + excelSheetData.setFields(new ArrayList<>(fields)); + List tmp = new ArrayList<>(cellList); + excelSheetData.getData().add(tmp); + totalRows++; + totalSheets.add(excelSheetData); + }else { + List tmp = new ArrayList<>(cellList); + totalSheets.stream().filter(s->s.getSheetName().equalsIgnoreCase(sheetName)).collect(Collectors.toList()).get(0).getData().add(tmp); + totalRows++; + } + } + + //清空容器 + cellList.clear(); + flag = false; + } + } + + /** + * 如果里面某个单元格含有值，则标识该行不为空行 + * + * @param value + */ + public void checkRowIsNull(String value) { + if (value != null && !"".equals(value)) { + flag = true; + } + } + + + private String checkType(String str, int thisColumn){ + String type = null; + try { + double d = Double.valueOf(str); + try { + Double value = new Double(d); + double eps = 1e-10; + if (value - Math.floor(value) < eps) { + type = "LONG"; + } else { + type = "DOUBLE"; + } + } catch (Exception e) { + type = "TEXT"; + } + }catch (Exception e){ + type = "TEXT"; + } + + String oldType = totalSheets.get(totalSheets.size() -1).getFields().get(thisColumn).getFieldType(); + if(type.equalsIgnoreCase("LONG") && oldType.equalsIgnoreCase("TEXT")){ + totalSheets.get(totalSheets.size() -1).getFields().get(thisColumn).setFieldType(type); + } + if(type.equalsIgnoreCase("DOUBLE")){ + totalSheets.get(totalSheets.size() -1).getFields().get(thisColumn).setFieldType(type); + } + return type; + } + +} \ No newline at end of file diff --git a/backend/src/main/java/io/dataease/commons/utils/ExcelXlsxReader.java b/backend/src/main/java/io/dataease/commons/utils/ExcelXlsxReader.java new file mode 100644 index 0000000000..fd0aa4ca8f --- /dev/null +++ b/backend/src/main/java/io/dataease/commons/utils/ExcelXlsxReader.java @@ -0,0 +1,470 @@ +package io.dataease.commons.utils; +import com.google.gson.Gson; +import io.dataease.datasource.dto.TableFiled; +import io.dataease.dto.dataset.ExcelSheetData; +import io.dataease.i18n.Translator; +import io.dataease.service.message.MsgAop; +import org.apache.commons.lang3.StringUtils; +import org.apache.poi.openxml4j.opc.OPCPackage; +import org.apache.poi.ss.usermodel.BuiltinFormats; +import org.apache.poi.ss.usermodel.DataFormatter; +import org.apache.poi.xssf.eventusermodel.XSSFReader; +import org.apache.poi.xssf.model.SharedStringsTable; +import org.apache.poi.xssf.model.StylesTable; +import org.apache.poi.xssf.usermodel.XSSFCellStyle; +import org.apache.poi.xssf.usermodel.XSSFRichTextString; +import org.xml.sax.Attributes; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; +import org.xml.sax.XMLReader; +import org.xml.sax.helpers.DefaultHandler; +import org.xml.sax.helpers.XMLReaderFactory; + +import java.io.InputStream; +import java.util.*; + +/** + * @author y + * @create 2018-01-18 14:28 + * @desc POI读取excel有两种模式，一种是用户模式，一种是事件驱动模式 + * 采用SAX事件驱动模式解决XLSX文件，可以有效解决用户模式内存溢出的问题， + * 该模式是POI官方推荐的读取大数据的模式， + * 在用户模式下，数据量较大，Sheet较多，或者是有很多无用的空行的情况下，容易出现内存溢出 + *

+ * 用于解决.xlsx2007版本大数据量问题 + **/ +public class ExcelXlsxReader extends DefaultHandler { + + /** + * 自定义获取表格某些信息 + */ + public Map map = new TreeMap(); + /** + * 单元格中的数据可能的数据类型 + */ + enum CellDataType { + BOOL, ERROR, FORMULA, INLINESTR, SSTINDEX, NUMBER, DATE, NULL + } + + /** + * 共享字符串表 + */ + private SharedStringsTable sst; + + /** + * 上一次的索引值 + */ + private String lastIndex; + + /** + * 总行数 + */ + private int totalRows=0; + + /** + * 一行内cell集合 + */ + private List cellList = new ArrayList(); + + /** + * 判断整行是否为空行的标记 + */ + private boolean flag = false; + + /** + * 当前行 + */ + private int curRow = 1; + + /** + * 当前列 + */ + private int curCol = 0; + + /** + * T元素标识 + */ + private boolean isTElement; + + /** + * 单元格数据类型，默认为字符串类型 + */ + private CellDataType nextDataType = CellDataType.SSTINDEX; + + private final DataFormatter formatter = new DataFormatter(); + + /** + * 单元格日期格式的索引 + */ + private short formatIndex; + + /** + * 日期格式字符串 + */ + private String formatString; + + //定义前一个元素和当前元素的位置，用来计算其中空的单元格数量，如A6和A8等 + private String preRef = null, ref = null; + + //定义该文档一行最大的单元格数，用来补全一行最后可能缺失的单元格 + private String maxRef = null; + + /** + * 单元格 + */ + private StylesTable stylesTable; + + public List fields = new ArrayList<>(); + public List> data = new ArrayList<>(); + public List totalSheets = new ArrayList<>(); + /** + * 是否为日期 + */ + private boolean isDateFormat = false; + + + public List getFields() { + return fields; + } + + public void setFields(List fields) { + this.fields = fields; + } + + public List> getData() { + return data; + } + + public void setData(List> data) { + this.data = data; + } + + public int process(InputStream inputStream) throws Exception { + OPCPackage pkg = OPCPackage.open(inputStream); + XSSFReader xssfReader = new XSSFReader(pkg); + stylesTable = xssfReader.getStylesTable(); + SharedStringsTable sst = xssfReader.getSharedStringsTable(); + XMLReader parser = XMLReaderFactory.createXMLReader("org.apache.xerces.parsers.SAXParser"); + this.sst = sst; + parser.setContentHandler(this); + XSSFReader.SheetIterator sheets = (XSSFReader.SheetIterator) xssfReader.getSheetsData(); + while (sheets.hasNext()) { //遍历sheet + curRow = 1; //标记初始行为第一行 + fields.clear(); + data.clear(); + InputStream sheet = sheets.next(); //sheets.next()和sheets.getSheetName()不能换位置，否则sheetName报错 + InputSource sheetSource = new InputSource(sheet); + parser.parse(sheetSource); //解析excel的每条记录，在这个过程中startElement()、characters()、endElement()这三个函数会依次执行 + + ExcelSheetData excelSheetData = new ExcelSheetData(); + excelSheetData.setData(new ArrayList<>(data)); + excelSheetData.setSheetName(sheets.getSheetName()); + excelSheetData.setFields(new ArrayList<>(fields)); + totalSheets.add(excelSheetData); + + sheet.close(); + } + return totalRows; //返回该excel文件的总行数，不包括首列和空行 + } + + /** + * 第一个执行 + * + * @param uri + * @param localName + * @param name + * @param attributes + * @throws SAXException + */ + @Override + public void startElement(String uri, String localName, String name, Attributes attributes) throws SAXException { + if(curRow>101){ + return; + } + + if(name.equalsIgnoreCase("mergeCell")){ + throw new RuntimeException(Translator.get("i18n_excel_have_merge_region")); + } + //c => 单元格 + if ("c".equals(name)) { + //当前单元格的位置 + ref = attributes.getValue("r"); + //设定单元格类型 + this.setNextDataType(attributes); + } + + //当元素为t时 + if ("t".equals(name)) { + isTElement = true; + } else { + isTElement = false; + } + + //置空 + lastIndex = ""; + } + + /** + * 第二个执行 + * 得到单元格对应的索引值或是内容值 + * 如果单元格类型是字符串、INLINESTR、数字、日期，lastIndex则是索引值 + * 如果单元格类型是布尔值、错误、公式，lastIndex则是内容值 + * @param ch + * @param start + * @param length + * @throws SAXException + */ + @Override + public void characters(char[] ch, int start, int length) throws SAXException { + if(curRow>101){ + return; + } + lastIndex += new String(ch, start, length); + } + + /** + * 第三个执行 + * + * @param uri + * @param localName + * @param name + * @throws SAXException + */ + @Override + public void endElement(String uri, String localName, String name) throws SAXException { + if(curRow>101){ + return; + } + //t元素也包含字符串 + if (isTElement) {//这个程序没经过 + //将单元格内容加入rowlist中，在这之前先去掉字符串前后的空白符 + String value = lastIndex.trim(); + cellList.add(curCol, value); + curCol++; + isTElement = false; + //如果里面某个单元格含有值，则标识该行不为空行 + if (value != null && !"".equals(value)) { + flag = true; + } + } else if ("v".equals(name)) { + //v => 单元格的值，如果单元格是字符串，则v标签的值为该字符串在SST中的索引 + String value = this.getDataValue(lastIndex.trim(), "");//根据索引值获取对应的单元格值 + if (preRef == null) { + preRef = ref; + } + //补全单元格之间的空单元格 + if (!ref.equals(preRef)) { + int len = countNullCell(ref, preRef); + for (int i = 0; i < len; i++) { + cellList.add(curCol, ""); + curCol++; + } + } + cellList.add(curCol, value); + curCol++; + //如果里面某个单元格含有值，则标识该行不为空行 + if (value != null && !"".equals(value)) { + flag = true; + } + preRef = ref; + } else { + //如果标签名称为row，这说明已到行尾 + if ("row".equals(name)) { + //默认第一行为表头，以该行单元格数目为最大数目 + if (curRow == 1) { + maxRef = ref; + } + //补全一行尾部可能缺失的单元格 + if (maxRef != null) { + int len = countNullCell(maxRef, ref); + for (int i = 0; i <= len; i++) { + cellList.add(curCol, ""); + curCol++; + } + } + if(curRow>1){ + List tmp = new ArrayList<>(cellList); + this.getData().add(tmp); + } + totalRows++; + cellList.clear(); + curRow++; + curCol = 0; + preRef = null; + ref = null; + flag=false; + } + } + } + + /** + * 处理数据类型 + * + * @param attributes + */ + public void setNextDataType(Attributes attributes) { + nextDataType = CellDataType.NUMBER; //cellType为空，则表示该单元格类型为数字 + formatIndex = -1; + formatString = null; + String cellType = attributes.getValue("t"); //单元格类型 + if ("b".equals(cellType)) { //处理布尔值 + nextDataType = CellDataType.BOOL; + } else if ("e".equals(cellType)) { //处理错误 + nextDataType = CellDataType.ERROR; + } else if ("inlineStr".equals(cellType)) { + nextDataType = CellDataType.INLINESTR; + } else if ("s".equals(cellType)) { //处理字符串 + nextDataType = CellDataType.SSTINDEX; + } else if ("str".equals(cellType)) { + nextDataType = CellDataType.FORMULA; + } + + String cellStyleStr = attributes.getValue("s"); // + if (cellStyleStr != null) { + int styleIndex = Integer.parseInt(cellStyleStr); + XSSFCellStyle style = this.stylesTable.getStyleAt(styleIndex); + formatIndex = style.getDataFormat(); + formatString = style.getDataFormatString(); + short format = this.formatIndex; + if (format == 14 || format == 31 || format == 57 ||format == 59|| + format == 58 || (176 <= format && format <= 178) + || (182 <= format && format <= 196) || + (210 <= format && format <= 213) || (208 == format)) + { // 日期 + isDateFormat = true; + } + } + + } + + /** + * 对解析出来的数据进行类型处理 + * @param value 单元格的值， + * value代表解析：BOOL的为0或1， ERROR的为内容值，FORMULA的为内容值，INLINESTR的为索引值需转换为内容值， + * SSTINDEX的为索引值需转换为内容值， NUMBER为内容值，DATE为内容值 + * @param thisStr 一个空字符串 + * @return + */ + @SuppressWarnings("deprecation") + public String getDataValue(String value, String thisStr) { + String type = "TEXT"; + switch (nextDataType) { + // 这几个的顺序不能随便交换，交换了很可能会导致数据错误 + case BOOL: //布尔值 + char first = value.charAt(0); + thisStr = first == '0' ? "FALSE" : "TRUE"; + type = "LONG"; + break; + case ERROR: //错误 + thisStr = "\"ERROR:" + value.toString() + '"'; + break; + case FORMULA: //公式 + thisStr = '"' + value.toString() + '"'; + type = getType(thisStr); + break; + case INLINESTR: + XSSFRichTextString rtsi = new XSSFRichTextString(value.toString()); + thisStr = rtsi.toString(); + rtsi = null; + break; + case SSTINDEX: //字符串 + String sstIndex = value.toString(); + try { + int idx = Integer.parseInt(sstIndex); + XSSFRichTextString rtss = new XSSFRichTextString(sst.getEntryAt(idx));//根据idx索引值获取内容值 + thisStr = rtss.toString(); + rtss = null; + } catch (NumberFormatException ex) { + thisStr = value.toString(); + } + + break; + case NUMBER: //数字 + if (formatString != null) { + thisStr = formatter.formatRawCellContents(Double.parseDouble(value), formatIndex, formatString).trim(); + } else { + thisStr = value; + } + thisStr = thisStr.replace("_", "").trim(); + if(isDateFormat){ + type = "DATETIME";isDateFormat = false; + }else { + type = getType(thisStr); + } + break; + case DATE: //日期 + thisStr = formatter.formatRawCellContents(Double.parseDouble(value), formatIndex, formatString); + // 对日期字符串作特殊处理，去掉T + thisStr = thisStr.replace("T", " "); + type = "DATETIME"; + break; + default: + thisStr = " "; + break; + } + if(curRow==1){ + TableFiled tableFiled = new TableFiled(); + tableFiled.setFieldType(type); + tableFiled.setFieldSize(65533); + tableFiled.setFieldName(thisStr); + tableFiled.setRemarks(thisStr); + this.fields.add(tableFiled); + }else { + this.getFields().get(curCol).setFieldType(type); + } + return thisStr; + } + + private String getType(String thisStr){ + if(totalRows==0){ + return "TEXT"; + } + try{ + if(thisStr.endsWith("%")){ + thisStr = thisStr.substring(0, thisStr.length()-1); + thisStr = String.valueOf(Double.valueOf(thisStr)/100); + } + Long.valueOf(thisStr); + if(this.getFields().get(curCol).getFieldType().equalsIgnoreCase("TEXT")){ + return "LONG"; + } + }catch (Exception e){ + try { + Double.valueOf(thisStr); + return "DOUBLE"; + }catch (Exception ignore){ } + } + return "TEXT"; + } + + public int countNullCell(String ref, String preRef) { + //excel2007最大行数是1048576，最大列数是16384，最后一列列名是XFD + String xfd = ref.replaceAll("\\d+", ""); + String xfd_1 = preRef.replaceAll("\\d+", ""); + + xfd = fillChar(xfd, 3, '@', true); + xfd_1 = fillChar(xfd_1, 3, '@', true); + + char[] letter = xfd.toCharArray(); + char[] letter_1 = xfd_1.toCharArray(); + int res = (letter[0] - letter_1[0]) * 26 * 26 + (letter[1] - letter_1[1]) * 26 + (letter[2] - letter_1[2]); + return res - 1; + } + + public String fillChar(String str, int len, char let, boolean isPre) { + int len_1 = str.length(); + if (len_1 < len) { + if (isPre) { + for (int i = 0; i < (len - len_1); i++) { + str = let + str; + } + } else { + for (int i = 0; i < (len - len_1); i++) { + str = str + let; + } + } + } + return str; + } + +} \ No newline at end of file diff --git a/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java b/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java index b0da9a2813..d7c5d51301 100644 --- a/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java +++ b/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java @@ -185,6 +185,9 @@ public class JdbcProvider extends DatasourceProvider { field.setFieldType(t); field.setFieldSize(metaData.getColumnDisplaySize(j + 1)); if(t.equalsIgnoreCase("LONG")){field.setFieldSize(65533);} //oracle LONG + if(StringUtils.isNotEmpty(t) && t.toLowerCase().contains("date") && field.getFieldSize() < 50 ){ + field.setFieldSize(50); + } fieldList.add(field); } return fieldList; diff --git a/backend/src/main/java/io/dataease/dto/dataset/ExcelSheetData.java b/backend/src/main/java/io/dataease/dto/dataset/ExcelSheetData.java new file mode 100644 index 0000000000..7e81d0fefd --- /dev/null +++ b/backend/src/main/java/io/dataease/dto/dataset/ExcelSheetData.java @@ -0,0 +1,13 @@ +package io.dataease.dto.dataset; + +import io.dataease.datasource.dto.TableFiled; +import lombok.Data; + +import java.util.List; + +@Data +public class ExcelSheetData { + private String sheetName; + private List> data; + private List fields; +} diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index 897d7fbb4f..3229c3fffe 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -1012,7 +1012,7 @@ public class DataSetTableService { public Map excelSaveAndParse(MultipartFile file, String tableId) throws Exception { String filename = file.getOriginalFilename(); // parse file - Map fileMap = parseExcel(filename, file.getInputStream(), true); + Map fileMap = parseExcel2(filename, file.getInputStream(), true); if (StringUtils.isNotEmpty(tableId)) { List datasetTableFields = dataSetTableFieldsService.getFieldsByTableId(tableId); datasetTableFields.sort((o1, o2) -> { @@ -1038,6 +1038,52 @@ public class DataSetTableService { return map; } + private Map parseExcel2(String filename, InputStream inputStream, boolean isPreview) throws Exception { + String suffix = filename.substring(filename.lastIndexOf(".") + 1); + List fields = new ArrayList<>(); + List> data = new ArrayList<>(); + List> jsonArray = new ArrayList<>(); + List sheets = new ArrayList<>(); + if (StringUtils.equalsIgnoreCase(suffix, "xls")) { + ExcelXlsReader excelXlsReader = new ExcelXlsReader(); + excelXlsReader.process(inputStream); + fields = excelXlsReader.totalSheets.get(0).getFields(); + data = excelXlsReader.totalSheets.get(0).getData(); + sheets = excelXlsReader.totalSheets.stream().map(ExcelSheetData::getSheetName).collect(Collectors.toList()); + } + if (StringUtils.equalsIgnoreCase(suffix, "xlsx")) { + ExcelXlsxReader excelXlsxReader = new ExcelXlsxReader(); + excelXlsxReader.process(inputStream); + fields = excelXlsxReader.totalSheets.get(0).getFields(); + data = excelXlsxReader.totalSheets.get(0).getData(); + sheets = excelXlsxReader.totalSheets.stream().map(ExcelSheetData::getSheetName).collect(Collectors.toList()); + } + + String[] fieldArray = fields.stream().map(TableFiled::getFieldName).toArray(String[]::new); + + // 校验excel字段是否重名 + if (checkIsRepeat(fieldArray)) { + DataEaseException.throwException(Translator.get("i18n_excel_field_repeat")); + } + + if (CollectionUtils.isNotEmpty(data)) { + jsonArray = data.stream().map(ele -> { + Map map = new HashMap<>(); + for (int i = 0; i < ele.size(); i++) { + map.put(fieldArray[i], ele.get(i)); + } + return map; + }).collect(Collectors.toList()); + } + inputStream.close(); + + Map map = new HashMap<>(); + map.put("fields", fields); + map.put("data", jsonArray); + map.put("sheets", sheets); + return map; + } + private Map parseExcel(String filename, InputStream inputStream, boolean isPreview) throws Exception { String suffix = filename.substring(filename.lastIndexOf(".") + 1); List fields = new ArrayList<>(); @@ -1191,6 +1237,7 @@ public class DataSetTableService { return map; } + private String readCell(Cell cell, boolean cellType, TableFiled tableFiled) { if (cell == null) { return ""; From f0a2532b515c5fa017774d243c2dd10f206650ce Mon Sep 17 00:00:00 2001 From: fit2cloud-chenyw Date: Wed, 21 Jul 2021 17:12:40 +0800 Subject: [PATCH 02/55] =?UTF-8?q?feat:=20=E8=A7=86=E5=9B=BE=E5=9B=BE?= =?UTF-8?q?=E8=A1=A8=E7=B1=BB=E5=9E=8B=E5=A2=9E=E5=8A=A0=E5=9C=B0=E5=9B=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dataease/map/dto/entity/AreaEntity.java | 2 + .../io/dataease/map/server/MapServer.java | 5 +- frontend/src/api/map/map.js | 17 ++++ frontend/src/icons/svg/chart-map.svg | 1 + frontend/src/lang/en.js | 6 +- frontend/src/lang/tw.js | 6 +- frontend/src/lang/zh.js | 6 +- frontend/src/views/chart/chart/chart.js | 34 +++++++ .../src/views/chart/chart/common/common.js | 2 +- frontend/src/views/chart/chart/map/map.js | 59 +++++++++++ .../views/chart/components/ChartComponent.vue | 29 +++++- frontend/src/views/chart/view/ChartEdit.vue | 99 +++++++++++++++---- 12 files changed, 239 insertions(+), 27 deletions(-) create mode 100644 frontend/src/api/map/map.js create mode 100644 frontend/src/icons/svg/chart-map.svg create mode 100644 frontend/src/views/chart/chart/map/map.js diff --git a/backend/src/main/java/io/dataease/map/dto/entity/AreaEntity.java b/backend/src/main/java/io/dataease/map/dto/entity/AreaEntity.java index 3b5acae163..c40d7063be 100644 --- a/backend/src/main/java/io/dataease/map/dto/entity/AreaEntity.java +++ b/backend/src/main/java/io/dataease/map/dto/entity/AreaEntity.java @@ -1,5 +1,6 @@ package io.dataease.map.dto.entity; +import com.fasterxml.jackson.annotation.JsonInclude; import lombok.Builder; import lombok.Data; @@ -10,6 +11,7 @@ import java.util.Optional; @Data @Builder +@JsonInclude(JsonInclude.Include.NON_NULL) public class AreaEntity implements Serializable { private static final long serialVersionUID = -1326667005437020282L; diff --git a/backend/src/main/java/io/dataease/map/server/MapServer.java b/backend/src/main/java/io/dataease/map/server/MapServer.java index 10dab5d7c8..ca88e64f55 100644 --- a/backend/src/main/java/io/dataease/map/server/MapServer.java +++ b/backend/src/main/java/io/dataease/map/server/MapServer.java @@ -5,6 +5,7 @@ import io.dataease.map.api.MapApi; import io.dataease.map.dto.entity.AreaEntity; import io.dataease.map.service.MapService; import io.dataease.map.utils.MapUtils; +import org.apache.commons.lang3.StringUtils; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RestController; import javax.annotation.Resource; @@ -40,7 +41,9 @@ public class MapServer implements MapApi { @Override public List areaEntitys(@PathVariable String pcode) { List areaEntities = mapService.areaEntities(); - + if (StringUtils.equals(pcode, "0")) { + return areaEntities; + } return mapService.entitysByPid(areaEntities, pcode); } diff --git a/frontend/src/api/map/map.js b/frontend/src/api/map/map.js new file mode 100644 index 0000000000..73463c11b9 --- /dev/null +++ b/frontend/src/api/map/map.js @@ -0,0 +1,17 @@ +import request from '@/utils/request' + +export const areaMapping = () => { + return request({ + url: '/api/map/areaEntitys/0', + method: 'get', + loading: true + }) +} + +export function geoJson(areaCode) { + return request({ + url: '/api/map/resourceFull/' + areaCode, + method: 'get', + loading: true + }) +} diff --git a/frontend/src/icons/svg/chart-map.svg b/frontend/src/icons/svg/chart-map.svg new file mode 100644 index 0000000000..677cd07630 --- /dev/null +++ b/frontend/src/icons/svg/chart-map.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/src/lang/en.js b/frontend/src/lang/en.js index 48f7ba3b24..75483877d3 100644 --- a/frontend/src/lang/en.js +++ b/frontend/src/lang/en.js @@ -750,6 +750,7 @@ export default { chart_funnel: 'Funnel', chart_radar: 'Radar', chart_gauge: 'Gauge', + chart_map: 'Map', dateStyle: 'Date Style', datePattern: 'Date Format', y: 'Year', @@ -817,7 +818,10 @@ export default { drag_block_funnel_width: 'Funnel Width', drag_block_funnel_split: 'Funnel Split', drag_block_radar_length: 'Branch Length', - drag_block_radar_label: 'Branch Label' + drag_block_radar_label: 'Branch Label', + map_range: 'Map range', + select_map_range: 'Please select map range', + area: 'Area' }, dataset: { sheet_warn: 'There are multiple sheet pages, and the first one is extracted by default', diff --git a/frontend/src/lang/tw.js b/frontend/src/lang/tw.js index 10415d2bf6..2f333dde3a 100644 --- a/frontend/src/lang/tw.js +++ b/frontend/src/lang/tw.js @@ -750,6 +750,7 @@ export default { chart_funnel: '漏鬥圖', chart_radar: '雷達圖', chart_gauge: '儀表盤', + chart_map: '地圖', dateStyle: '日期顯示', datePattern: '日期格式', y: '年', @@ -817,7 +818,10 @@ export default { drag_block_funnel_width: '漏鬥層寬', drag_block_funnel_split: '漏鬥分層', drag_block_radar_length: '分支長度', - drag_block_radar_label: '分支標簽' + drag_block_radar_label: '分支標簽', + map_range: '地圖範圍', + select_map_range: '請選擇地圖範圍', + area: '地區' }, dataset: { sheet_warn: '有多個sheet頁面，默認抽取第一個', diff --git a/frontend/src/lang/zh.js b/frontend/src/lang/zh.js index 3c8da35366..b54dd28e7b 100644 --- a/frontend/src/lang/zh.js +++ b/frontend/src/lang/zh.js @@ -750,6 +750,7 @@ export default { chart_funnel: '漏斗图', chart_radar: '雷达图', chart_gauge: '仪表盘', + chart_map: '地图', dateStyle: '日期显示', datePattern: '日期格式', y: '年', @@ -817,7 +818,10 @@ export default { drag_block_funnel_width: '漏斗层宽', drag_block_funnel_split: '漏斗分层', drag_block_radar_length: '分支长度', - drag_block_radar_label: '分支标签' + drag_block_radar_label: '分支标签', + map_range: '地图范围', + select_map_range: '请选择地图范围', + area: '地区' }, dataset: { sheet_warn: '有多个 Sheet 页，默认抽取第一个', diff --git a/frontend/src/views/chart/chart/chart.js b/frontend/src/views/chart/chart/chart.js index 88dd327a47..0ee2ce820c 100644 --- a/frontend/src/views/chart/chart/chart.js +++ b/frontend/src/views/chart/chart/chart.js @@ -528,3 +528,37 @@ export const BASE_CHART = { }, customFilter: [] } + +export const BASE_MAP = { + title: { + text: '', + textStyle: { + fontWeight: 'normal' + } + }, + + tooltip: {}, + visualMap: { + min: 50, + max: 52, + text: ['High', 'Low'], + realtime: false, + calculable: true, + inRange: { + color: ['lightskyblue', 'yellow', 'orangered'] + } + }, + // legend: {}, + series: [ + { + name: '', + type: 'map', + map: 'HK', + roam: true, + // label: { + // show: true + // }, + data: [] + } + ] +} diff --git a/frontend/src/views/chart/chart/common/common.js b/frontend/src/views/chart/chart/common/common.js index e4135b6c12..a1ce01bbe2 100644 --- a/frontend/src/views/chart/chart/common/common.js +++ b/frontend/src/views/chart/chart/common/common.js @@ -29,7 +29,7 @@ export function componentStyle(chart_option, chart) { customStyle.text.isBolder ? style.fontWeight = 'bold' : style.fontWeight = 'normal' chart_option.title.textStyle = style } - if (customStyle.legend) { + if (customStyle.legend && chart_option.legend) { chart_option.legend.show = customStyle.legend.show // 水平方向 if (customStyle.legend.hPosition === 'left') { diff --git a/frontend/src/views/chart/chart/map/map.js b/frontend/src/views/chart/chart/map/map.js new file mode 100644 index 0000000000..b95d9ae338 --- /dev/null +++ b/frontend/src/views/chart/chart/map/map.js @@ -0,0 +1,59 @@ +import { hexColorToRGBA } from '@/views/chart/chart/util' +import { componentStyle } from '../common/common' + +export function baseMapOption(chart_option, chart) { + // 处理shape attr + let customAttr = {} + if (chart.customAttr) { + customAttr = JSON.parse(chart.customAttr) + if (customAttr.color) { + chart_option.color = customAttr.color.colors + } + // tooltip + if (customAttr.tooltip) { + const tooltip = JSON.parse(JSON.stringify(customAttr.tooltip)) + const reg = new RegExp('\n', 'g') + tooltip.formatter = tooltip.formatter.replace(reg, '
') + chart_option.tooltip = tooltip + } + } + // 处理data + if (chart.data) { + chart_option.title.text = chart.title + if (chart.data.series.length > 0) { + chart_option.series[0].name = chart.data.series[0].name + // size + if (customAttr.size) { + chart_option.series[0].radius = [customAttr.size.pieInnerRadius + '%', customAttr.size.pieOuterRadius + '%'] + } + // label + if (customAttr.label) { + chart_option.series[0].label = customAttr.label + chart_option.series[0].labelLine = customAttr.label.labelLine + } + // visualMap + const valueArr = chart.data.series[0].data + chart_option.visualMap.min = Math.min(...valueArr) + chart_option.visualMap.max = Math.max(...valueArr) + if (customAttr.color && customAttr.color.colors) { + chart_option.visualMap.inRange.color = customAttr.color.colors + } + for (let i = 0; i < valueArr.length; i++) { + const y = { + name: chart.data.x[i], + value: valueArr[i] + } + // color + y.itemStyle = { + color: hexColorToRGBA(customAttr.color.colors[i % 9], customAttr.color.alpha), + borderRadius: 0 + } + chart_option.series[0].data.push(y) + } + } + } + // console.log(chart_option); + componentStyle(chart_option, chart) + return chart_option +} + diff --git a/frontend/src/views/chart/components/ChartComponent.vue b/frontend/src/views/chart/components/ChartComponent.vue index 963fc6f111..66f232ae31 100644 --- a/frontend/src/views/chart/components/ChartComponent.vue +++ b/frontend/src/views/chart/components/ChartComponent.vue @@ -5,15 +5,17 @@ - From 57155637148485501b93b98c3f8a4ce36d991f5a Mon Sep 17 00:00:00 2001 From: fit2cloud-chenyw Date: Wed, 21 Jul 2021 17:23:58 +0800 Subject: [PATCH 03/55] =?UTF-8?q?feat:=20=E6=9B=BF=E6=8D=A2=E7=BC=A9?= =?UTF-8?q?=E7=95=A5=E5=9B=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/icons/svg/chart-map.svg | 1 - frontend/src/icons/svg/map.svg | 1 + frontend/src/views/chart/view/ChartEdit.vue | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 frontend/src/icons/svg/chart-map.svg create mode 100644 frontend/src/icons/svg/map.svg diff --git a/frontend/src/icons/svg/chart-map.svg b/frontend/src/icons/svg/chart-map.svg deleted file mode 100644 index 677cd07630..0000000000 --- a/frontend/src/icons/svg/chart-map.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/frontend/src/icons/svg/map.svg b/frontend/src/icons/svg/map.svg new file mode 100644 index 0000000000..721251a9ba --- /dev/null +++ b/frontend/src/icons/svg/map.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/src/views/chart/view/ChartEdit.vue b/frontend/src/views/chart/view/ChartEdit.vue index 2a7302c678..194af8b662 100644 --- a/frontend/src/views/chart/view/ChartEdit.vue +++ b/frontend/src/views/chart/view/ChartEdit.vue @@ -192,7 +192,7 @@ - + From a8e6e2d0e127c13ecdcb814ce72b09492d17244d Mon Sep 17 00:00:00 2001 From: junjie Date: Wed, 21 Jul 2021 17:36:50 +0800 Subject: [PATCH 04/55] =?UTF-8?q?feat:=20=E5=A0=86=E5=8F=A0=E5=9B=BE?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E5=A0=86=E5=8F=A0=E9=A1=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../base/domain/ChartViewWithBLOBs.java | 2 + .../dataease/base/mapper/ChartViewMapper.xml | 25 +- .../io/dataease/provider/QueryProvider.java | 13 + .../provider/doris/DorisQueryProvider.java | 98 +++++++ .../provider/mysql/MysqlQueryProvider.java | 117 ++++++++ .../provider/oracle/OracleQueryProvider.java | 102 ++++++- .../sqlserver/SqlserverQueryProvider.java | 10 + .../service/chart/ChartViewService.java | 264 +++++++++++++----- .../db/migration/V16__alter_chart_view.sql | 3 + .../src/main/resources/generatorConfig.xml | 8 +- frontend/src/lang/en.js | 1 + frontend/src/lang/tw.js | 3 +- frontend/src/lang/zh.js | 3 +- .../components/drag-item/ChartDragItem.vue | 152 ++++++++++ frontend/src/views/chart/group/Group.vue | 3 + frontend/src/views/chart/view/ChartEdit.vue | 169 ++++++----- 16 files changed, 826 insertions(+), 147 deletions(-) create mode 100644 backend/src/main/resources/db/migration/V16__alter_chart_view.sql create mode 100644 frontend/src/views/chart/components/drag-item/ChartDragItem.vue diff --git a/backend/src/main/java/io/dataease/base/domain/ChartViewWithBLOBs.java b/backend/src/main/java/io/dataease/base/domain/ChartViewWithBLOBs.java index bc06da0a28..a3510da1c9 100644 --- a/backend/src/main/java/io/dataease/base/domain/ChartViewWithBLOBs.java +++ b/backend/src/main/java/io/dataease/base/domain/ChartViewWithBLOBs.java @@ -13,6 +13,8 @@ public class ChartViewWithBLOBs extends ChartView implements Serializable { private String yAxis; + private String extStack; + private String customAttr; private String customStyle; diff --git a/backend/src/main/java/io/dataease/base/mapper/ChartViewMapper.xml b/backend/src/main/java/io/dataease/base/mapper/ChartViewMapper.xml index 2d91efd41a..d89b3a0613 100644 --- a/backend/src/main/java/io/dataease/base/mapper/ChartViewMapper.xml +++ b/backend/src/main/java/io/dataease/base/mapper/ChartViewMapper.xml @@ -16,6 +16,7 @@ + @@ -84,7 +85,7 @@ style_priority - x_axis, y_axis, custom_attr, custom_style, custom_filter, snapshot + x_axis, y_axis, ext_stack, custom_attr, custom_style, custom_filter, snapshot + From 17ef8291669816aaad61d0a8a2d411d49ef940cb Mon Sep 17 00:00:00 2001 From: wangjiahao <1522128093@qq.com> Date: Fri, 23 Jul 2021 16:06:12 +0800 Subject: [PATCH 50/55] =?UTF-8?q?fix:=E5=9C=A8=E7=BC=96=E8=BE=91=E4=BB=AA?= =?UTF-8?q?=E8=A1=A8=E6=9D=BF=E6=97=B6=EF=BC=8C=E5=A4=8D=E5=88=B6=E5=A0=86?= =?UTF-8?q?=E5=8F=A0=E6=9F=B1=E7=8A=B6=E5=9B=BE=E6=97=B6=E5=87=BA=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/io/dataease/base/mapper/ext/ExtChartViewMapper.xml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/src/main/java/io/dataease/base/mapper/ext/ExtChartViewMapper.xml b/backend/src/main/java/io/dataease/base/mapper/ext/ExtChartViewMapper.xml index a7ccd9855c..f8de1cbaa0 100644 --- a/backend/src/main/java/io/dataease/base/mapper/ext/ExtChartViewMapper.xml +++ b/backend/src/main/java/io/dataease/base/mapper/ext/ExtChartViewMapper.xml @@ -45,7 +45,8 @@ `create_time`, `update_time`, `snapshot`, - `style_priority` + `style_priority`, + `ext_stack` ) SELECT #{newChartId}, GET_CHART_VIEW_COPY_NAME ( #{oldChartId} ), @@ -62,7 +63,8 @@ `create_time`, `update_time`, `snapshot`, - `style_priority` + `style_priority`, + `ext_stack` FROM chart_view WHERE From 99b2ac0c7ae8bb09571cacc55827da9318802f9d Mon Sep 17 00:00:00 2001 From: junjie Date: Fri, 23 Jul 2021 17:02:50 +0800 Subject: [PATCH 51/55] =?UTF-8?q?refactor:=20=E8=A7=86=E5=9B=BE=E6=8B=96?= =?UTF-8?q?=E6=8B=BD=E5=9D=97=E5=A2=9E=E5=8A=A0placeholder?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/lang/en.js | 2 +- frontend/src/lang/tw.js | 2 +- frontend/src/lang/zh.js | 2 +- frontend/src/views/chart/view/ChartEdit.vue | 47 +++++++++------------ 4 files changed, 23 insertions(+), 30 deletions(-) diff --git a/frontend/src/lang/en.js b/frontend/src/lang/en.js index 2911aef2da..c44b93e7c6 100644 --- a/frontend/src/lang/en.js +++ b/frontend/src/lang/en.js @@ -823,7 +823,7 @@ export default { map_range: 'Map range', select_map_range: 'Please select map range', area: 'Area', - placeholder_field: 'Drag Field' + placeholder_field: 'Drag Field To Here' }, dataset: { sheet_warn: 'There are multiple sheet pages, and the first one is extracted by default', diff --git a/frontend/src/lang/tw.js b/frontend/src/lang/tw.js index e2899fe5bf..c8ab685e18 100644 --- a/frontend/src/lang/tw.js +++ b/frontend/src/lang/tw.js @@ -823,7 +823,7 @@ export default { select_map_range: '請選擇地圖範圍', area: '地區', stack_item: '堆疊項', - placeholder_field: '拖入字段' + placeholder_field: '拖動字段至此處' }, dataset: { sheet_warn: '有多個sheet頁面，默認抽取第一個', diff --git a/frontend/src/lang/zh.js b/frontend/src/lang/zh.js index 4f096de4fc..42d6515b09 100644 --- a/frontend/src/lang/zh.js +++ b/frontend/src/lang/zh.js @@ -823,7 +823,7 @@ export default { select_map_range: '请选择地图范围', area: '地区', stack_item: '堆叠项', - placeholder_field: '拖入字段' + placeholder_field: '拖动字段至此处' }, dataset: { sheet_warn: '有多个 Sheet 页，默认抽取第一个', diff --git a/frontend/src/views/chart/view/ChartEdit.vue b/frontend/src/views/chart/view/ChartEdit.vue index 59ab020022..31a345b614 100644 --- a/frontend/src/views/chart/view/ChartEdit.vue +++ b/frontend/src/views/chart/view/ChartEdit.vue @@ -253,13 +253,13 @@ class="drag-block-style" @add="addXaxis" > - - {{ $t('chart.placeholder_field') }} - +

+ {{ $t('chart.placeholder_field') }} +
@@ -283,13 +283,13 @@ class="drag-block-style" @add="addYaxis" > - - {{ $t('chart.placeholder_field') }} - +
+ {{ $t('chart.placeholder_field') }} +
@@ -306,15 +306,15 @@ class="drag-block-style" @add="addStack" > - - {{ $t('chart.placeholder_field') }} - +
+ {{ $t('chart.placeholder_field') }} +
-
+ {{ $t('chart.result_filter') }} @@ -328,14 +328,14 @@ style="padding:2px 0 0 0;width:100%;min-height: 32px;border-radius: 4px;border: 1px solid #DCDFE6;overflow-x: auto;display: flex;align-items: center;background-color: white;" @add="addCustomFilter" > - - {{ $t('chart.placeholder_field') }} - -
+
+ {{ $t('chart.placeholder_field') }} +
+ @@ -1531,14 +1531,6 @@ export default { pointer-events:none; } - .filter-class{ - height: calc(35% - 102px); - } - - .filter-class>>>.filter-inner-class{ - height: calc(100% - 40px); - } - .chart-class{ height: 100%; padding: 10px; @@ -1604,11 +1596,12 @@ export default { align-items: center; background-color: white; } - .drag_placeholder-style{ - font-size: 12px; - color: #CCCCCC; - padding: 0 0 2px 10px; - display: inline-block; + .drag-placeholder-style{ + position: absolute; + top: calc(50% - 2px); + left: 0; width: 100%; + color: #CCCCCC; + text-align: center; } From d324ff8c6e7e16dfab9855fd03eb15f602c81a07 Mon Sep 17 00:00:00 2001 From: taojinlong Date: Fri, 23 Jul 2021 18:35:37 +0800 Subject: [PATCH 52/55] =?UTF-8?q?fix:=20=E6=9B=B4=E6=96=B0=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E7=8A=B6=E6=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DataSetTableTaskLogController.java | 6 +- .../dto/dataset/DataTableInfoDTO.java | 1 + .../service/dataset/DataSetTableService.java | 60 +++++++---------- .../dataset/DataSetTableTaskLogService.java | 29 +++++--- .../dataset/DataSetTableTaskService.java | 16 ++++- .../service/dataset/ExtractDataService.java | 66 +++++++------------ .../db/migration/V13__system_task.sql | 2 +- frontend/src/views/dataset/add/AddExcel.vue | 2 +- .../src/views/dataset/data/UpdateInfo.vue | 6 +- frontend/src/views/dataset/data/ViewTable.vue | 2 +- .../src/views/system/task/DatasetTaskList.vue | 2 +- frontend/src/views/system/task/TaskRecord.vue | 2 +- 12 files changed, 92 insertions(+), 102 deletions(-) diff --git a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableTaskLogController.java b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableTaskLogController.java index 4b0123fe3c..6b7a84c6be 100644 --- a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableTaskLogController.java +++ b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableTaskLogController.java @@ -33,10 +33,10 @@ public class DataSetTableTaskLogController { dataSetTableTaskLogService.delete(id); } - @PostMapping("list/{goPage}/{pageSize}") - public Pager> list(@RequestBody BaseGridRequest request, @PathVariable int goPage, @PathVariable int pageSize) { + @PostMapping("list/{type}/{goPage}/{pageSize}") + public Pager> list(@RequestBody BaseGridRequest request, @PathVariable String type, @PathVariable int goPage, @PathVariable int pageSize) { Page page = PageHelper.startPage(goPage, pageSize, true); - return PageUtils.setPageInfo(page, dataSetTableTaskLogService.list(request)); + return PageUtils.setPageInfo(page, dataSetTableTaskLogService.list(request, type)); } } diff --git a/backend/src/main/java/io/dataease/dto/dataset/DataTableInfoDTO.java b/backend/src/main/java/io/dataease/dto/dataset/DataTableInfoDTO.java index ffa73619cd..a75f12dbba 100644 --- a/backend/src/main/java/io/dataease/dto/dataset/DataTableInfoDTO.java +++ b/backend/src/main/java/io/dataease/dto/dataset/DataTableInfoDTO.java @@ -14,6 +14,7 @@ import java.util.List; public class DataTableInfoDTO { private String table; private String sql; + private List sheets; private String data;// file path private List list; } diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index 00c54ea4e5..f0507b0148 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -101,7 +101,7 @@ public class DataSetTableService { private void extractData(DataSetTableRequest datasetTable) throws Exception { if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) { commonThreadPool.addTask(() -> { - extractDataService.extractExcelData(datasetTable.getId(), "all_scope"); + extractDataService.extractExcelData(datasetTable.getId(), "all_scope", "初始导入"); }); return; } @@ -131,7 +131,6 @@ public class DataSetTableService { datasetTable.setId(UUID.randomUUID().toString()); datasetTable.setCreateBy(AuthUtils.getUser().getUsername()); datasetTable.setCreateTime(System.currentTimeMillis()); - DataTableInfoDTO dataTableInfoDTO = new DataTableInfoDTO(); int insert = datasetTableMapper.insert(datasetTable); // 添加表成功后，获取当前表字段和类型，抽象到dataease数据库 if (insert == 1) { @@ -151,11 +150,11 @@ public class DataSetTableService { if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) { if (datasetTable.getEditType() == 0) { commonThreadPool.addTask(() -> { - extractDataService.extractExcelData(datasetTable.getId(), "all_scope"); + extractDataService.extractExcelData(datasetTable.getId(), "all_scope", "替换"); }); } else if (datasetTable.getEditType() == 1) { commonThreadPool.addTask(() -> { - extractDataService.extractExcelData(datasetTable.getId(), "add_scope"); + extractDataService.extractExcelData(datasetTable.getId(), "add_scope", "追加"); }); } } @@ -416,38 +415,27 @@ public class DataSetTableService { } } } else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) { - if (StringUtils.isEmpty(datasetTable.getSyncStatus()) || datasetTable.getSyncStatus().equalsIgnoreCase(JobStatus.Underway.name())) { - map.put("status", "warnning"); - map.put("msg", Translator.get("i18n_processing_data")); - dataSetPreviewPage.setTotal(0); - } else if (datasetTable.getSyncStatus().equalsIgnoreCase(JobStatus.Error.name())) { - List datasetTableTaskLogs = dataSetTableTaskLogService.getByTableId(datasetTable.getId()); - map.put("status", "error"); - if (CollectionUtils.isNotEmpty(datasetTableTaskLogs)) { - map.put("msg", "Failed to extract data: " + datasetTableTaskLogs.get(0).getInfo()); - } else { - map.put("msg", "Failed to extract data."); - } - dataSetPreviewPage.setTotal(0); - } else { - Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); - JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); - DatasourceRequest datasourceRequest = new DatasourceRequest(); - datasourceRequest.setDatasource(ds); - String table = DorisTableUtils.dorisName(dataSetTableRequest.getId()); - QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); - datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize)); - try { - data.addAll(jdbcProvider.getData(datasourceRequest)); - } catch (Exception e) { - e.printStackTrace(); - } - try { - datasourceRequest.setQuery(qp.createQueryTableWithLimit(table, fields, Integer.valueOf(dataSetTableRequest.getRow()))); - dataSetPreviewPage.setTotal(Integer.valueOf(jdbcProvider.getData(datasourceRequest).get(0)[0])); - } catch (Exception e) { - e.printStackTrace(); - } + if (!checkDorisTableIsExists(dataSetTableRequest.getId())) { + throw new RuntimeException(Translator.get("i18n_data_not_sync")); + } + + Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(ds); + String table = DorisTableUtils.dorisName(dataSetTableRequest.getId()); + QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); + datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize)); + try { + data.addAll(jdbcProvider.getData(datasourceRequest)); + } catch (Exception e) { + e.printStackTrace(); + } + try { + datasourceRequest.setQuery(qp.createQueryTableWithLimit(table, fields, Integer.valueOf(dataSetTableRequest.getRow()))); + dataSetPreviewPage.setTotal(Integer.valueOf(jdbcProvider.getData(datasourceRequest).get(0)[0])); + } catch (Exception e) { + e.printStackTrace(); } } else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "custom")) { if (datasetTable.getMode() == 0) { diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java index c44a62b9fe..80117e76c8 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java @@ -47,18 +47,27 @@ public class DataSetTableTaskLogService { datasetTableTaskLogMapper.deleteByPrimaryKey(id); } - public List list(BaseGridRequest request) { - ConditionEntity entity = new ConditionEntity(); - entity.setField("task_id"); - entity.setOperator("not null"); - List conditionEntities = request.getConditions(); - if(CollectionUtils.isEmpty(conditionEntities)){ - conditionEntities = new ArrayList<>(); + public List list(BaseGridRequest request, String type) { + if(!type.equalsIgnoreCase("excel")){ + ConditionEntity entity = new ConditionEntity(); + entity.setField("task_id"); + entity.setOperator("not null"); + List conditionEntities = request.getConditions(); + if(CollectionUtils.isEmpty(conditionEntities)){ + conditionEntities = new ArrayList<>(); + } + conditionEntities.add(entity); + request.setConditions(conditionEntities); } - conditionEntities.add(entity); - request.setConditions(conditionEntities); + GridExample gridExample = request.convertExample(); - return extDataSetTaskMapper.list(gridExample); + List dataSetTaskLogDTOS = extDataSetTaskMapper.list(gridExample); + dataSetTaskLogDTOS.forEach(dataSetTaskLogDTO -> { + if(StringUtils.isEmpty(dataSetTaskLogDTO.getName())){ + dataSetTaskLogDTO.setName(dataSetTaskLogDTO.getTaskId()); + } + }); + return dataSetTaskLogDTOS; } public void deleteByTaskId(String taskId){ diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java index bd491abcc4..0278a7cd93 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java @@ -13,6 +13,7 @@ import io.dataease.commons.constants.TaskStatus; import io.dataease.commons.constants.TriggerType; import io.dataease.controller.request.dataset.DataSetTaskRequest; import io.dataease.controller.sys.base.BaseGridRequest; +import io.dataease.controller.sys.base.ConditionEntity; import io.dataease.controller.sys.response.SysUserGridResponse; import io.dataease.controller.sys.response.SysUserRole; import io.dataease.dto.dataset.DataSetTaskDTO; @@ -28,6 +29,7 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import javax.annotation.Resource; +import java.util.Arrays; import java.util.List; import java.util.UUID; import java.util.stream.Collectors; @@ -177,8 +179,20 @@ public class DataSetTableTaskService { datasetTableTask.setStatus(TaskStatus.Stopped.name()); }else { if(StringUtils.isNotEmpty(datasetTableTask.getEnd()) && datasetTableTask.getEnd().equalsIgnoreCase("1")){ - if(utilMapper.currentTimestamp() > datasetTableTask.getEndTime()){ + BaseGridRequest request = new BaseGridRequest(); + ConditionEntity conditionEntity = new ConditionEntity(); + conditionEntity.setField("dataset_table_task.id"); + conditionEntity.setOperator("eq"); + conditionEntity.setValue(datasetTableTask.getId()); + request.setConditions(Arrays.asList(conditionEntity)); + List dataSetTaskDTOS = taskList(request); + if(CollectionUtils.isEmpty(dataSetTaskDTOS)){ + return; + } + if(dataSetTaskDTOS.get(0).getNextExecTime() == null || dataSetTaskDTOS.get(0).getNextExecTime() <= 0){ datasetTableTask.setStatus(TaskStatus.Stopped.name()); + }else { + datasetTableTask.setStatus(TaskStatus.Underway.name()); } }else { datasetTableTask.setStatus(TaskStatus.Underway.name()); diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index 9624aa842a..c40599503b 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -158,7 +158,7 @@ public class ExtractDataService { } } - public void extractExcelData(String datasetTableId, String type) { + public void extractExcelData(String datasetTableId, String type, String ops) { Datasource datasource = new Datasource(); datasource.setType("excel"); DatasetTable datasetTable = getDatasetTable(datasetTableId); @@ -182,15 +182,11 @@ public class ExtractDataService { switch (updateType) { case all_scope: // 全量更新 try { - datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, null); + datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops); createDorisTable(DorisTableUtils.dorisName(datasetTableId), dorisTablColumnSql); createDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)), dorisTablColumnSql); generateTransFile("all_scope", datasetTable, datasource, datasetTableFields, null); - if (datasetTable.getType().equalsIgnoreCase("sql")) { - generateJobFile("all_scope", datasetTable, fetchSqlField(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getSql(), datasource)); - } else { - generateJobFile("all_scope", datasetTable, String.join(",", datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList()))); - } + generateJobFile("all_scope", datasetTable, String.join(",", datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList()))); Long execTime = System.currentTimeMillis(); extractData(datasetTable, "all_scope"); replaceTable(DorisTableUtils.dorisName(datasetTableId)); @@ -210,17 +206,17 @@ public class ExtractDataService { case add_scope: // 增量更新 try { - datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, null); + datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, ops); generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, null); generateJobFile("incremental_add", datasetTable, String.join(",", datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList()))); Long execTime = System.currentTimeMillis(); extractData(datasetTable, "incremental_add"); saveSucessLog(datasetTableTaskLog); - sendWebMsg(datasetTable, null, true); +// sendWebMsg(datasetTable, null, true); updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, execTime); } catch (Exception e) { saveErrorLog(datasetTableId, null, e); - sendWebMsg(datasetTable, null, false); +// sendWebMsg(datasetTable, null, false); updateTableStatus(datasetTableId, datasetTable, JobStatus.Error, null); deleteFile("incremental_add", datasetTableId); deleteFile("incremental_delete", datasetTableId); @@ -256,7 +252,7 @@ public class ExtractDataService { return; } - DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); + DatasetTableTaskLog datasetTableTaskLog = getDatasetTableTaskLog(datasetTableId, taskId); UpdateType updateType = UpdateType.valueOf(type); if (context != null) { datasetTable.setQrtzInstance(context.getFireInstanceId()); @@ -283,11 +279,8 @@ public class ExtractDataService { switch (updateType) { case all_scope: // 全量更新 try { - if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.CRON.toString())) { - datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId); - } - if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { - datasetTableTaskLog = getDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId); + if (datasetTableTask == null ) { + datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, taskId); } createDorisTable(DorisTableUtils.dorisName(datasetTableId), dorisTablColumnSql); createDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)), dorisTablColumnSql); @@ -336,11 +329,8 @@ public class ExtractDataService { return; } - if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.CRON.toString())) { - datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId); - } - if (datasetTableTask != null && datasetTableTask.getRate().equalsIgnoreCase(ScheduleType.SIMPLE.toString())) { - datasetTableTaskLog = getDatasetTableTaskLog(datasetTableTaskLog, datasetTableId, taskId); + if (datasetTableTask == null ) { + datasetTableTaskLog = writeDatasetTableTaskLog(datasetTableId, taskId); } Long execTime = System.currentTimeMillis(); if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) {// 增量添加 @@ -533,7 +523,8 @@ public class ExtractDataService { return null; } - private DatasetTableTaskLog writeDatasetTableTaskLog(DatasetTableTaskLog datasetTableTaskLog, String datasetTableId, String taskId) { + private DatasetTableTaskLog writeDatasetTableTaskLog(String datasetTableId, String taskId) { + DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); datasetTableTaskLog.setTableId(datasetTableId); datasetTableTaskLog.setTaskId(taskId); datasetTableTaskLog.setStatus(JobStatus.Underway.name()); @@ -548,7 +539,8 @@ public class ExtractDataService { } } - private DatasetTableTaskLog getDatasetTableTaskLog(DatasetTableTaskLog datasetTableTaskLog, String datasetTableId, String taskId) { + private DatasetTableTaskLog getDatasetTableTaskLog(String datasetTableId, String taskId) { + DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); datasetTableTaskLog.setTableId(datasetTableId); datasetTableTaskLog.setTaskId(taskId); datasetTableTaskLog.setStatus(JobStatus.Underway.name()); @@ -765,8 +757,7 @@ public class ExtractDataService { udjcStep = udjc(datasetTableFields, DatasourceTypes.oracle); break; case excel: - String filePath = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getData(); - inputStep = excelInputStep(filePath, datasetTableFields); + inputStep = excelInputStep(datasetTable.getInfo(), datasetTableFields); udjcStep = udjc(datasetTableFields, DatasourceTypes.excel); default: break; @@ -837,33 +828,20 @@ public class ExtractDataService { return fromStep; } - private StepMeta excelInputStep(String filePath, List datasetTableFields) { - String suffix = filePath.substring(filePath.lastIndexOf(".") + 1); + private StepMeta excelInputStep(String Info, List datasetTableFields){ + DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(Info, DataTableInfoDTO.class); + String suffix = dataTableInfoDTO.getData().substring(dataTableInfoDTO.getData().lastIndexOf(".") + 1); ExcelInputMeta excelInputMeta = new ExcelInputMeta(); if (StringUtils.equalsIgnoreCase(suffix, "xlsx")) { excelInputMeta.setSpreadSheetType(SpreadSheetType.SAX_POI); - try { - InputStream inputStream = new FileInputStream(filePath); - XSSFWorkbook xssfWorkbook = new XSSFWorkbook(inputStream); - XSSFSheet sheet0 = xssfWorkbook.getSheetAt(0); - excelInputMeta.setSheetName(new String[]{sheet0.getSheetName()}); - } catch (Exception e) { - e.printStackTrace(); - } + excelInputMeta.setSheetName(new String[]{dataTableInfoDTO.getSheets().get(0)}); } if (StringUtils.equalsIgnoreCase(suffix, "xls")) { excelInputMeta.setSpreadSheetType(SpreadSheetType.JXL); - try { - InputStream inputStream = new FileInputStream(filePath); - HSSFWorkbook workbook = new HSSFWorkbook(inputStream); - HSSFSheet sheet0 = workbook.getSheetAt(0); - excelInputMeta.setSheetName(new String[]{sheet0.getSheetName()}); - } catch (Exception e) { - e.printStackTrace(); - } + excelInputMeta.setSheetName(new String[]{dataTableInfoDTO.getSheets().get(0)}); } excelInputMeta.setPassword("Encrypted"); - excelInputMeta.setFileName(new String[]{filePath}); + excelInputMeta.setFileName(new String[]{dataTableInfoDTO.getData()}); excelInputMeta.setStartsWithHeader(true); excelInputMeta.setIgnoreEmptyRows(true); ExcelInputField[] fields = new ExcelInputField[datasetTableFields.size()]; diff --git a/backend/src/main/resources/db/migration/V13__system_task.sql b/backend/src/main/resources/db/migration/V13__system_task.sql index 4caafbc099..b3a7d7433a 100644 --- a/backend/src/main/resources/db/migration/V13__system_task.sql +++ b/backend/src/main/resources/db/migration/V13__system_task.sql @@ -23,4 +23,4 @@ ALTER TABLE `dataset_table_task` ADD COLUMN `extra_data` LONGTEXT NULL AFTER `la update dataset_table_task_log set trigger_type='Cron'; - +update dataset_table_task_log set dataset_table_task_log.task_id='初始导入' where dataset_table_task_log.task_id is null; diff --git a/frontend/src/views/dataset/add/AddExcel.vue b/frontend/src/views/dataset/add/AddExcel.vue index 40de024f65..55f99bdff2 100644 --- a/frontend/src/views/dataset/add/AddExcel.vue +++ b/frontend/src/views/dataset/add/AddExcel.vue @@ -258,7 +258,7 @@ export default { type: 'excel', mode: parseInt(this.mode), // info: '{"data":"' + this.path + '"}', - info: JSON.stringify({ data: this.path }), + info: JSON.stringify({ data: this.path, sheets: [this.sheets[0]]}), fields: this.fields } } else { diff --git a/frontend/src/views/dataset/data/UpdateInfo.vue b/frontend/src/views/dataset/data/UpdateInfo.vue index 3bd10fd436..0d4ee78f38 100644 --- a/frontend/src/views/dataset/data/UpdateInfo.vue +++ b/frontend/src/views/dataset/data/UpdateInfo.vue @@ -1,7 +1,7 @@