center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

feat: 支持excel多sheet页

Browse Source
This commit is contained in:
taojinlong 2021-08-12 19:24:18 +08:00 committed by taojinlong
commit c36fd88f3f
14 changed files with 496 additions and 248 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
backend/src/main/java/io/dataease/commons/utils/ExcelReaderUtil.java
View File

@ -1,8 +1,6 @@
package io.dataease.commons.utils;
import com.google.gson.Gson;
import io.dataease.datasource.dto.TableFiled;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.List;
@ -46,7 +44,7 @@ public class ExcelReaderUtil {
ExcelXlsReader excelXls=new ExcelXlsReader();
excelXls.process(inputStream);
System.out.println(excelXls.totalSheets.size());
System.out.println(excelXls.totalSheets.get(0).getSheetName());
System.out.println(excelXls.totalSheets.get(0).getExcelLable());
for (TableFiled field : excelXls.totalSheets.get(0).getFields()) {
System.out.println(new Gson().toJson(field));
}
@ -56,7 +54,7 @@ public class ExcelReaderUtil {
ExcelXlsxReader excelXlsxReader = new ExcelXlsxReader();
excelXlsxReader.process(inputStream);
System.out.println(excelXlsxReader.totalSheets.size());
System.out.println(excelXlsxReader.totalSheets.get(0).getSheetName());
System.out.println(excelXlsxReader.totalSheets.get(0).getExcelLable());
for (TableFiled field : excelXlsxReader.totalSheets.get(0).getFields()) {
System.out.println(new Gson().toJson(field));
}
@ -68,7 +66,7 @@ public class ExcelReaderUtil {
}
public static void main(String[] args) throws Exception {
String file ="下单渠道.xlsx";
ExcelReaderUtil.readExcel(file, new FileInputStream("/Users/taojinlong/Desktop/" + file));
// String file ="下单渠道.xlsx";
// ExcelReaderUtil.readExcel(file, new FileInputStream("/Users/taojinlong/Desktop/" + file));
}
}

10
backend/src/main/java/io/dataease/commons/utils/ExcelXlsReader.java
View File

@ -300,9 +300,9 @@ public class ExcelXlsReader implements HSSFListener {
}
lastColumnNumber = -1;
if(!totalSheets.stream().map(ExcelSheetData::getSheetName).collect(Collectors.toList()).contains(sheetName)){
if(!totalSheets.stream().map(ExcelSheetData::getExcelLable).collect(Collectors.toList()).contains(sheetName)){
ExcelSheetData excelSheetData = new ExcelSheetData();
excelSheetData.setSheetName(sheetName);
excelSheetData.setExcelLable(sheetName);
excelSheetData.setData(new ArrayList<>());
excelSheetData.setFields(new ArrayList<>());
totalSheets.add(excelSheetData);
@ -322,10 +322,10 @@ public class ExcelXlsReader implements HSSFListener {
if (flag && curRow != 0) { //该行不为空行且该行不是第一行，发送（第一行为列名，不需要）
if(!totalSheets.stream().map(ExcelSheetData::getSheetName).collect(Collectors.toList()).contains(sheetName)){
if(!totalSheets.stream().map(ExcelSheetData::getExcelLable).collect(Collectors.toList()).contains(sheetName)){
ExcelSheetData excelSheetData = new ExcelSheetData();
excelSheetData.setData(new ArrayList<>(data));
excelSheetData.setSheetName(sheetName);
excelSheetData.setExcelLable(sheetName);
excelSheetData.setFields(new ArrayList<>(fields));
List<String> tmp = new ArrayList<>(cellList);
excelSheetData.getData().add(tmp);
@ -333,7 +333,7 @@ public class ExcelXlsReader implements HSSFListener {
totalSheets.add(excelSheetData);
}else {
List<String> tmp = new ArrayList<>(cellList);
totalSheets.stream().filter(s->s.getSheetName().equalsIgnoreCase(sheetName)).collect(Collectors.toList()).get(0).getData().add(tmp);
totalSheets.stream().filter(s->s.getExcelLable().equalsIgnoreCase(sheetName)).collect(Collectors.toList()).get(0).getData().add(tmp);
totalRows++;
}
}

2
backend/src/main/java/io/dataease/commons/utils/ExcelXlsxReader.java
View File

@ -157,7 +157,7 @@ public class ExcelXlsxReader extends DefaultHandler {
ExcelSheetData excelSheetData = new ExcelSheetData();
excelSheetData.setData(new ArrayList<>(data));
excelSheetData.setSheetName(sheets.getSheetName());
excelSheetData.setExcelLable(sheets.getSheetName());
excelSheetData.setFields(new ArrayList<>(fields));
totalSheets.add(excelSheetData);

13
backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java
View File

@ -7,6 +7,7 @@ import io.dataease.base.domain.DatasetTableIncrementalConfig;
import io.dataease.controller.request.dataset.DataSetTableRequest;
import io.dataease.datasource.dto.TableFiled;
import io.dataease.dto.dataset.DataSetTableDTO;
import io.dataease.dto.dataset.ExcelFileData;
import io.dataease.service.dataset.DataSetTableService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
@ -37,8 +38,12 @@ public class DataSetTableController {
@ApiOperation("更新")
@PostMapping("update")
public DatasetTable save(@RequestBody DataSetTableRequest datasetTable) throws Exception {
return dataSetTableService.save(datasetTable);
public void save(@RequestBody DataSetTableRequest datasetTable) throws Exception {
if(datasetTable.getType().equalsIgnoreCase("excel")){
dataSetTableService.saveExcel(datasetTable);
}else {
dataSetTableService.save(datasetTable);
}
}
@ApiOperation("删除")
@ -121,8 +126,8 @@ public class DataSetTableController {
@ApiOperation("excel上传")
@PostMapping("excel/upload")
public Map<String, Object> excelUpload(@RequestParam("file") MultipartFile file, @RequestParam("tableId") String tableId) throws Exception {
return dataSetTableService.excelSaveAndParse(file, tableId);
public ExcelFileData excelUpload(@RequestParam("file") MultipartFile file, @RequestParam("tableId") String tableId, @RequestParam("editType") Integer editType ) throws Exception {
return dataSetTableService.excelSaveAndParse(file, tableId, editType);
}
@ApiOperation("检测doris")

3
backend/src/main/java/io/dataease/controller/request/dataset/DataSetTableRequest.java
View File

@ -2,6 +2,7 @@ package io.dataease.controller.request.dataset;
import io.dataease.base.domain.DatasetTable;
import io.dataease.datasource.dto.TableFiled;
import io.dataease.dto.dataset.ExcelSheetData;
import lombok.Getter;
import lombok.Setter;
@ -23,4 +24,6 @@ public class DataSetTableRequest extends DatasetTable {
private Boolean isRename;
private List<String> typeFilter;
private List<TableFiled> fields;
private List<ExcelSheetData> sheets;
private boolean mergeSheet = false;
}

2
backend/src/main/java/io/dataease/dto/dataset/DataTableInfoDTO.java
View File

@ -14,7 +14,7 @@ import java.util.List;
public class DataTableInfoDTO {
private String table;
private String sql;
private List<String> sheets;
private List<ExcelSheetData> excelSheetDataList;
private String data;// file path
private List<DataTableInfoCustomUnion> list;
}

16
backend/src/main/java/io/dataease/dto/dataset/ExcelFileData.java Normal file
View File

@ -0,0 +1,16 @@
package io.dataease.dto.dataset;
import io.dataease.datasource.dto.TableFiled;
import lombok.Data;
import java.util.List;
import java.util.Map;
@Data
public class ExcelFileData {
private String excelId;
private String excelLable;
private List<ExcelSheetData> sheets;
private String path;
private boolean isSheet = false;
}

10
backend/src/main/java/io/dataease/dto/dataset/ExcelSheetData.java
View File

@ -4,10 +4,18 @@ import io.dataease.datasource.dto.TableFiled;
import lombok.Data;
import java.util.List;
import java.util.Map;
@Data
public class ExcelSheetData {
private String sheetName;
private String excelLable;
private List<List<String>> data;
private List<TableFiled> fields;
private boolean isSheet = true;
private List<Map<String, Object>> jsonArray;
private String datasetName;
private String sheetExcelId;
private String sheetId;
private String path;
private String fieldsMd5;
}

268
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@ -100,13 +100,7 @@ public class DataSetTableService {
}
private void extractData(DataSetTableRequest datasetTable) throws Exception {
if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) {
commonThreadPool.addTask(() -> {
extractDataService.extractExcelData(datasetTable.getId(), "all_scope", "初始导入");
});
return;
}
if (StringUtils.isNotEmpty(datasetTable.getSyncType()) && datasetTable.getSyncType().equalsIgnoreCase("sync_now")) {
if (datasetTable.getMode() == 1 && StringUtils.isNotEmpty(datasetTable.getSyncType()) && datasetTable.getSyncType().equalsIgnoreCase("sync_now")) {
DataSetTaskRequest dataSetTaskRequest = new DataSetTaskRequest();
DatasetTableTask datasetTableTask = new DatasetTableTask();
datasetTableTask.setTableId(datasetTable.getId());
@ -121,6 +115,113 @@ public class DataSetTableService {
}
}
public void saveExcel(DataSetTableRequest datasetTable)throws Exception {
if (StringUtils.isEmpty(datasetTable.getId())) {
if(datasetTable.isMergeSheet()){
Map<String, List<ExcelSheetData>> map = datasetTable.getSheets().stream().collect(Collectors.groupingBy(ExcelSheetData::getFieldsMd5));
for (String s : map.keySet()) {
DataSetTableRequest sheetTable = new DataSetTableRequest();
BeanUtils.copyBean(sheetTable, datasetTable);
sheetTable.setId(UUID.randomUUID().toString());
sheetTable.setCreateBy(AuthUtils.getUser().getUsername());
sheetTable.setCreateTime(System.currentTimeMillis());
List<ExcelSheetData> excelSheetDataList = map.get(s);
sheetTable.setName(excelSheetDataList.get(0).getDatasetName());
checkName(sheetTable);
excelSheetDataList.forEach(excelSheetData -> {
String[] fieldArray = excelSheetData.getFields().stream().map(TableFiled::getFieldName).toArray(String[]::new);
if (checkIsRepeat(fieldArray)) {
DataEaseException.throwException(Translator.get("i18n_excel_field_repeat"));
}
excelSheetData.setData(null);
excelSheetData.setJsonArray(null);
});
DataTableInfoDTO info = new DataTableInfoDTO();
info.setExcelSheetDataList(excelSheetDataList);
sheetTable.setInfo(new Gson().toJson(info));
int insert = datasetTableMapper.insert(sheetTable);
if (insert == 1) {
saveExcelTableField(sheetTable.getId(), excelSheetDataList.get(0).getFields());
commonThreadPool.addTask(() -> {
extractDataService.extractExcelData(sheetTable.getId(), "all_scope", "初始导入");
});
}
}
}else {
for (ExcelSheetData sheet : datasetTable.getSheets()) {
String[] fieldArray = sheet.getFields().stream().map(TableFiled::getFieldName).toArray(String[]::new);
if (checkIsRepeat(fieldArray)) {
DataEaseException.throwException(Translator.get("i18n_excel_field_repeat"));
}
DataSetTableRequest sheetTable = new DataSetTableRequest();
BeanUtils.copyBean(sheetTable, datasetTable);
sheetTable.setId(UUID.randomUUID().toString());
sheetTable.setCreateBy(AuthUtils.getUser().getUsername());
sheetTable.setCreateTime(System.currentTimeMillis());
sheetTable.setName(sheet.getDatasetName());
checkName(sheetTable);
sheet.setData(null);
sheet.setJsonArray(null);
List<ExcelSheetData> excelSheetDataList = new ArrayList<>();
excelSheetDataList.add(sheet);
DataTableInfoDTO info = new DataTableInfoDTO();
info.setExcelSheetDataList(excelSheetDataList);
sheetTable.setInfo(new Gson().toJson(info));
int insert = datasetTableMapper.insert(sheetTable);
if (insert == 1) {
saveExcelTableField(sheetTable.getId(), sheet.getFields());
commonThreadPool.addTask(() -> {
extractDataService.extractExcelData(sheetTable.getId(), "all_scope", "初始导入");
});
}
}
}
return;
}
List<ExcelSheetData> excelSheetDataList = new ArrayList<>();
List<String> oldFields = datasetTable.getSheets().get(0).getFields().stream().map(TableFiled::getRemarks).collect(Collectors.toList());
for (ExcelSheetData sheet : datasetTable.getSheets()) {
//替换时，
if(datasetTable.getEditType() == 0){
List<String> newFields = sheet.getFields().stream().map(TableFiled::getRemarks).collect(Collectors.toList());
if (!oldFields.equals(newFields)) {
DataEaseException.throwException(Translator.get("i18n_excel_colume_change"));
}
oldFields = newFields;
}
String[] fieldArray = sheet.getFields().stream().map(TableFiled::getFieldName).toArray(String[]::new);
if (checkIsRepeat(fieldArray)) {
DataEaseException.throwException(Translator.get("i18n_excel_field_repeat"));
}
sheet.setData(null);
sheet.setJsonArray(null);
excelSheetDataList.add(sheet);
}
DataTableInfoDTO info = new DataTableInfoDTO();
info.setExcelSheetDataList(excelSheetDataList);
datasetTable.setInfo(new Gson().toJson(info));
int update = datasetTableMapper.updateByPrimaryKeySelective(datasetTable);
// 删除所有字段，重新抽象
if(datasetTable.getEditType() == 0){
dataSetTableFieldsService.deleteByTableId(datasetTable.getId());
saveExcelTableField(datasetTable.getId(), datasetTable.getSheets().get(0).getFields());
}
if (update == 1) {
if (datasetTable.getEditType() == 0) {
commonThreadPool.addTask(() -> {
extractDataService.extractExcelData(datasetTable.getId(), "all_scope", "替换");
});
} else if (datasetTable.getEditType() == 1) {
commonThreadPool.addTask(() -> {
extractDataService.extractExcelData(datasetTable.getId(), "add_scope", "追加");
});
}
}
}
public DatasetTable save(DataSetTableRequest datasetTable) throws Exception {
checkName(datasetTable);
if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "sql")) {
@ -143,22 +244,11 @@ public class DataSetTableService {
if (datasetTable.getIsRename() == null || !datasetTable.getIsRename()) {
// 更新数据和字段
if (update == 1) {
if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "sql") || StringUtils.equalsIgnoreCase(datasetTable.getType(), "custom")) {
if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "sql") || StringUtils.equalsIgnoreCase(datasetTable.getType(), "custom") ) {
// 删除所有字段，重新抽象
dataSetTableFieldsService.deleteByTableId(datasetTable.getId());
saveTableField(datasetTable);
}
if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) {
if (datasetTable.getEditType() == 0) {
commonThreadPool.addTask(() -> {
extractDataService.extractExcelData(datasetTable.getId(), "all_scope", "替换");
});
} else if (datasetTable.getEditType() == 1) {
commonThreadPool.addTask(() -> {
extractDataService.extractExcelData(datasetTable.getId(), "add_scope", "追加");
});
}
}
}
}
}
@ -810,6 +900,29 @@ public class DataSetTableService {
}
}
public void saveExcelTableField(String datasetTableId, List<TableFiled> fields) throws Exception {
if (CollectionUtils.isNotEmpty(fields)) {
for (int i = 0; i < fields.size(); i++) {
TableFiled filed = fields.get(i);
DatasetTableField datasetTableField = DatasetTableField.builder().build();
datasetTableField.setTableId(datasetTableId);
datasetTableField.setOriginName(filed.getFieldName());
datasetTableField.setName(filed.getRemarks());
datasetTableField.setDataeaseName(DorisTableUtils.columnName(filed.getFieldName()));
datasetTableField.setType(filed.getFieldType());
datasetTableField.setDeType(transFieldType(filed.getFieldType()));
datasetTableField.setDeExtractType(transFieldType(filed.getFieldType()));
datasetTableField.setSize(filed.getFieldSize());
datasetTableField.setChecked(true);
datasetTableField.setColumnIndex(i);
datasetTableField.setLastSyncTime(System.currentTimeMillis());
datasetTableField.setExtField(0);
datasetTableField.setGroupType(datasetTableField.getDeType() < 2 ? "d" : "q");
dataSetTableFieldsService.save(datasetTableField);
}
}
}
public void saveTableField(DatasetTable datasetTable) throws Exception {
Datasource ds = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId());
DataSetTableRequest dataSetTableRequest = new DataSetTableRequest();
@ -828,11 +941,6 @@ public class DataSetTableService {
datasourceRequest.setQuery(sqlAsTable);
fields = datasourceProvider.fetchResultField(datasourceRequest);
} else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) {
/*DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class);
String path = dataTableInfoDTO.getData();
File file = new File(path);
Map<String, Object> map = parseExcel(path.substring(path.lastIndexOf("/") + 1), new FileInputStream(file), false);
fields = (List<TableFiled>) map.get("fields");*/
fields = dataSetTableRequest.getFields();
} else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "custom")) {
if (datasetTable.getMode() == 1) {
@ -1038,9 +1146,6 @@ public class DataSetTableService {
}
private void checkName(DatasetTable datasetTable) {
// if (StringUtils.isEmpty(datasetTable.getId()) && StringUtils.equalsIgnoreCase("db", datasetTable.getType())) {
// return;
// }
DatasetTableExample datasetTableExample = new DatasetTableExample();
DatasetTableExample.Criteria criteria = datasetTableExample.createCriteria();
if (StringUtils.isNotEmpty(datasetTable.getId())) {
@ -1069,11 +1174,13 @@ public class DataSetTableService {
return map;
}
public Map<String, Object> excelSaveAndParse(MultipartFile file, String tableId) throws Exception {
public ExcelFileData excelSaveAndParse(MultipartFile file, String tableId, Integer editType) throws Exception {
String filename = file.getOriginalFilename();
// parse file
Map<String, Object> fileMap = parseExcel2(filename, file.getInputStream(), true);
if (StringUtils.isNotEmpty(tableId)) {
List<ExcelSheetData> excelSheetDataList = parseExcel2(filename, file.getInputStream(), true);
List<ExcelSheetData> retrunSheetDataList = new ArrayList<>();
if (StringUtils.isNotEmpty(tableId) && editType == 1 ) {
List<DatasetTableField> datasetTableFields = dataSetTableFieldsService.getFieldsByTableId(tableId);
datasetTableFields.sort((o1, o2) -> {
if (o1.getColumnIndex() == null) {
@ -1084,64 +1191,80 @@ public class DataSetTableService {
}
return o1.getColumnIndex().compareTo(o2.getColumnIndex());
});
List<TableFiled> fields = (List<TableFiled>) fileMap.get("fields");
List<String> newFields = fields.stream().map(TableFiled::getRemarks).collect(Collectors.toList());
List<String> oldFields = datasetTableFields.stream().map(DatasetTableField::getOriginName).collect(Collectors.toList());
if (!oldFields.equals(newFields)) {
for (ExcelSheetData excelSheetData : excelSheetDataList) {
List<TableFiled> fields = excelSheetData.getFields();
List<String> newFields = fields.stream().map(TableFiled::getRemarks).collect(Collectors.toList());
if (oldFields.equals(newFields)) {
retrunSheetDataList.add(excelSheetData);
}
}
if (retrunSheetDataList.size() == 0) {
DataEaseException.throwException(Translator.get("i18n_excel_colume_change"));
}
}else {
retrunSheetDataList = excelSheetDataList;
}
// save file
String filePath = saveFile(file);
Map<String, Object> map = new HashMap<>(fileMap);
map.put("path", filePath);
return map;
String excelId = UUID.randomUUID().toString();
String filePath = saveFile(file, excelId);
ExcelFileData excelFileData = new ExcelFileData();
excelFileData.setExcelLable(filename);
excelFileData.setExcelId(excelId);
excelFileData.setPath(filePath);
filename = filename.substring(0, filename.lastIndexOf('.'));
if(retrunSheetDataList.size() == 1){
retrunSheetDataList.get(0).setDatasetName(filename);
retrunSheetDataList.get(0).setSheetExcelId(excelId);
retrunSheetDataList.get(0).setSheetId(UUID.randomUUID().toString());
retrunSheetDataList.get(0).setPath(filePath);
}else {
for (ExcelSheetData excelSheetData : retrunSheetDataList) {
excelSheetData.setDatasetName(filename + "-" + excelSheetData.getExcelLable());
excelSheetData.setSheetExcelId(excelId);
excelSheetData.setSheetId(UUID.randomUUID().toString());
excelSheetData.setPath(filePath);
}
}
excelFileData.setSheets(retrunSheetDataList);
return excelFileData;
}
private Map<String, Object> parseExcel2(String filename, InputStream inputStream, boolean isPreview) throws Exception {
private List<ExcelSheetData> parseExcel2(String filename, InputStream inputStream, boolean isPreview) throws Exception {
List<ExcelSheetData> excelSheetDataList = new ArrayList<>();
String suffix = filename.substring(filename.lastIndexOf(".") + 1);
List<TableFiled> fields = new ArrayList<>();
List<List<String>> data = new ArrayList<>();
List<Map<String, Object>> jsonArray = new ArrayList<>();
List<String> sheets = new ArrayList<>();
if (StringUtils.equalsIgnoreCase(suffix, "xls")) {
ExcelXlsReader excelXlsReader = new ExcelXlsReader();
excelXlsReader.process(inputStream);
fields = excelXlsReader.totalSheets.get(0).getFields();
data = excelXlsReader.totalSheets.get(0).getData();
sheets = excelXlsReader.totalSheets.stream().map(ExcelSheetData::getSheetName).collect(Collectors.toList());
excelSheetDataList = excelXlsReader.totalSheets;
}
if (StringUtils.equalsIgnoreCase(suffix, "xlsx")) {
ExcelXlsxReader excelXlsxReader = new ExcelXlsxReader();
excelXlsxReader.process(inputStream);
fields = excelXlsxReader.totalSheets.get(0).getFields();
data = excelXlsxReader.totalSheets.get(0).getData();
sheets = excelXlsxReader.totalSheets.stream().map(ExcelSheetData::getSheetName).collect(Collectors.toList());
}
String[] fieldArray = fields.stream().map(TableFiled::getFieldName).toArray(String[]::new);
// 校验excel字段是否重名
if (checkIsRepeat(fieldArray)) {
DataEaseException.throwException(Translator.get("i18n_excel_field_repeat"));
}
if (CollectionUtils.isNotEmpty(data)) {
jsonArray = data.stream().map(ele -> {
Map<String, Object> map = new HashMap<>();
for (int i = 0; i < ele.size(); i++) {
map.put(fieldArray[i], ele.get(i));
}
return map;
}).collect(Collectors.toList());
excelSheetDataList = excelXlsxReader.totalSheets;
}
inputStream.close();
excelSheetDataList.forEach(excelSheetData -> {
List<List<String>> data = excelSheetData.getData();
String[] fieldArray = excelSheetData.getFields().stream().map(TableFiled::getFieldName).toArray(String[]::new);
List<Map<String, Object>> jsonArray = new ArrayList<>();
if (CollectionUtils.isNotEmpty(data)) {
jsonArray = data.stream().map(ele -> {
Map<String, Object> map = new HashMap<>();
for (int i = 0; i < ele.size(); i++) {
map.put(fieldArray[i], ele.get(i));
}
return map;
}).collect(Collectors.toList());
}
excelSheetData.setFieldsMd5(Md5Utils.md5(StringUtils.join(fieldArray, ",")));
excelSheetData.setJsonArray(jsonArray);
});
Map<String, Object> map = new HashMap<>();
map.put("fields", fields);
map.put("data", jsonArray);
map.put("sheets", sheets);
return map;
return excelSheetDataList;
}
private Map<String, Object> parseExcel(String filename, InputStream inputStream, boolean isPreview) throws Exception {
@ -1387,16 +1510,15 @@ public class DataSetTableService {
return "";
}
private String saveFile(MultipartFile file) throws Exception {
private String saveFile(MultipartFile file, String fileNameUUID) throws Exception {
String filename = file.getOriginalFilename();
String suffix = filename.substring(filename.lastIndexOf(".") + 1);
filename = Md5Utils.md5(filename.substring(0, filename.length() - suffix.length()));
String dirPath = path + AuthUtils.getUser().getUsername() + "/";
File p = new File(dirPath);
if (!p.exists()) {
p.mkdirs();
}
String filePath = dirPath + filename + "." + suffix;
String filePath = dirPath + fileNameUUID + "." + suffix;
File f = new File(filePath);
FileOutputStream fileOutputStream = new FileOutputStream(f);
fileOutputStream.write(file.getBytes());

28
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -17,6 +17,7 @@ import io.dataease.datasource.provider.ProviderFactory;
import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.datasource.service.DatasourceService;
import io.dataease.dto.dataset.DataTableInfoDTO;
import io.dataease.dto.dataset.ExcelSheetData;
import io.dataease.exception.DataEaseException;
import io.dataease.listener.util.CacheUtils;
import io.dataease.provider.QueryProvider;
@ -192,7 +193,9 @@ public class ExtractDataService {
dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)));
} finally {
deleteFile("all_scope", datasetTableId);
// deleteFile(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getData());
for (ExcelSheetData excelSheetData : new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getExcelSheetDataList()) {
deleteFile(excelSheetData.getPath());
}
}
break;
@ -213,6 +216,9 @@ public class ExtractDataService {
} finally {
deleteFile("incremental_add", datasetTableId);
deleteFile("incremental_delete", datasetTableId);
for (ExcelSheetData excelSheetData : new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getExcelSheetDataList()) {
deleteFile(excelSheetData.getPath());
}
}
break;
}
@ -831,18 +837,30 @@ public class ExtractDataService {
private StepMeta excelInputStep(String Info, List<DatasetTableField> datasetTableFields){
DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(Info, DataTableInfoDTO.class);
String suffix = dataTableInfoDTO.getData().substring(dataTableInfoDTO.getData().lastIndexOf(".") + 1);
List<ExcelSheetData> excelSheetDataList = dataTableInfoDTO.getExcelSheetDataList();
String suffix = excelSheetDataList.get(0).getPath().substring(excelSheetDataList.get(0).getPath().lastIndexOf(".") + 1);
ExcelInputMeta excelInputMeta = new ExcelInputMeta();
List<String> sheetNames = new ArrayList<>();
List<String> files = new ArrayList<>();
for (ExcelSheetData excelSheetData : excelSheetDataList) {
if(!sheetNames.contains(excelSheetData.getExcelLable())){
sheetNames.add(excelSheetData.getExcelLable());
}
if(!files.contains(excelSheetData.getPath())){
files.add(excelSheetData.getPath());
}
}
if (StringUtils.equalsIgnoreCase(suffix, "xlsx")) {
excelInputMeta.setSpreadSheetType(SpreadSheetType.SAX_POI);
excelInputMeta.setSheetName(new String[]{dataTableInfoDTO.getSheets().get(0)});
excelInputMeta.setSheetName(sheetNames.toArray(new String[sheetNames.size()]));
}
if (StringUtils.equalsIgnoreCase(suffix, "xls")) {
excelInputMeta.setSpreadSheetType(SpreadSheetType.JXL);
excelInputMeta.setSheetName(new String[]{dataTableInfoDTO.getSheets().get(0)});
excelInputMeta.setSheetName(sheetNames.toArray(new String[sheetNames.size()]));
}
excelInputMeta.setPassword("Encrypted");
excelInputMeta.setFileName(new String[]{dataTableInfoDTO.getData()});
excelInputMeta.setFileName( files.toArray(new String[files.size()]));
excelInputMeta.setStartsWithHeader(true);
excelInputMeta.setIgnoreEmptyRows(true);
ExcelInputField[] fields = new ExcelInputField[datasetTableFields.size()];

3
frontend/src/lang/en.js
View File

@ -1006,7 +1006,8 @@ export default {
field_manage: 'Field Manage',
edit_calc_field: 'Edit calc field',
calc_field: 'Calc Field',
show_sql: 'Show SQL'
show_sql: 'Show SQL',
ple_select_excel: 'Please select excel file to import'
},
datasource: {
datasource: 'Data Source',

3
frontend/src/lang/tw.js
View File

@ -1006,7 +1006,8 @@ export default {
field_manage: '字段管理',
edit_calc_field: '編輯計算字段',
calc_field: '計算字段',
show_sql: '顯示SQL'
show_sql: '顯示SQL',
ple_select_excel: '請選擇要導入的 Excel'
},
datasource: {
datasource: '數據源',

3
frontend/src/lang/zh.js
View File

@ -1006,7 +1006,8 @@ export default {
field_manage: '字段管理',
edit_calc_field: '编辑计算字段',
calc_field: '计算字段',
show_sql: '显示SQL'
show_sql: '显示SQL',
ple_select_excel: '请选择要导入的 Excel'
},
datasource: {
datasource: '数据源',

373
frontend/src/views/dataset/add/AddExcel.vue
View File

@ -9,115 +9,132 @@
<el-button size="mini" @click="cancel">
{{ $t('dataset.cancel') }}
</el-button>
<el-button :disabled="!name || fileList.length === 0" size="mini" type="primary" @click="save">
<el-button size="mini" type="primary" @click="save">
{{ $t('dataset.confirm') }}
</el-button>
</el-row>
</el-row>
<el-divider />
<el-row>
<el-row>
<el-col style="width: 500px;">
<el-form :inline="true" size="mini" class="row-style">
<el-form-item class="form-item">
<el-input v-show="!param.tableId" v-model="name" :placeholder="$t('commons.name')" />
</el-form-item>
<el-form-item class="form-item">
<el-upload
:action="baseUrl+'dataset/table/excel/upload'"
:multiple="false"
:show-file-list="false"
:file-list="fileList"
:data="param"
accept=".xls,.xlsx,"
:before-upload="beforeUpload"
:on-success="uploadSuccess"
:on-error="uploadFail"
name="file"
:headers="headers"
>
<el-button size="mini" type="primary" :disabled="uploading">
<span v-if="!uploading" style="font-size: 12px;">{{ $t('dataset.upload_file') }}</span>
<span v-if="uploading" style="font-size: 12px;"><i class="el-icon-loading" /> {{ $t('dataset.uploading') }}</span>
</el-button>
</el-upload>
</el-form-item>
</el-form>
</el-col>
</el-row>
</el-row>
<el-row style="margin-top: 10px;">
<el-card class="box-card dataPreview" shadow="never">
<div slot="header" class="clearfix">
<span>{{ $t('dataset.data_preview') }}</span>
<span style="font-size: 12px;color: #3d4d66;">（{{ $t('dataset.preview_100_data') }}）</span>
</div>
<div class="text item">
<ux-grid
ref="plxTable"
size="mini"
style="width: 100%;"
:height="height"
:checkbox-config="{highlight: true}"
:width-resize="true"
>
<ux-table-column
v-for="field in fields"
:key="field.fieldName"
min-width="200px"
:field="field.fieldName"
:title="field.remarks"
:resizable="true"
>
<template slot="header" slot-scope="scope">
<span v-if="!param.tableId" style="display: flex;align-items: center;">
<span style="display: inline-block;font-size: 12px;">
<div style="display: inline-block;">
<el-select v-model="field.fieldType" size="mini" style="display: inline-block;width: 120px;">
<el-option
v-for="item in fieldOptions"
:key="item.value"
:label="item.label"
:value="item.value"
>
<span style="float: left">
<svg-icon v-if="item.value === 'TEXT'" icon-class="field_text" class="field-icon-text" />
<svg-icon v-if="item.value === 'DATETIME'" icon-class="field_time" class="field-icon-time" />
<svg-icon v-if="item.value === 'LONG' || item.value === 'DOUBLE'" icon-class="field_value" class="field-icon-value" />
</span>
<span style="float: left; color: #8492a6; font-size: 12px">{{ item.label }}</span>
</el-option>
</el-select>
<el-container>
<el-aside width="200px" >
<el-row>
<el-col style="width: 200px;">
<el-form :inline="true" size="mini" class="row-style">
<el-form-item class="form-item">
<el-upload
:action="baseUrl+'dataset/table/excel/upload'"
:multiple="false"
:show-file-list="false"
:file-list="fileList"
:data="param"
accept=".xls,.xlsx,"
:before-upload="beforeUpload"
:on-success="uploadSuccess"
:on-error="uploadFail"
name="file"
:headers="headers"
>
<el-button size="mini" type="primary" :disabled="uploading">
<span v-if="!uploading" style="font-size: 12px;">{{ $t('dataset.upload_file') }}</span>
<span v-if="uploading" style="font-size: 12px;"><i class="el-icon-loading" /> {{ $t('dataset.uploading') }}</span>
</el-button>
</el-upload>
</el-form-item>
</el-form>
</el-col>
</el-row>
<el-tree ref="tree"
:data="excelData"
node-key="excelId"
:props="props"
show-checkbox
highlight-current
@node-click="handleNodeClick"
@check-change="handleCheckChange">
</el-tree>
</el-aside>
<el-container>
<el-header style="text-align: left;" height="30px">
<el-row>
<el-col style="width: 500px;">
<el-form :inline="true" size="mini" class="row-style">
<el-form-item class="form-item" :label="$t('dataset.name')" v-show="!param.tableId">
<el-input v-model="sheetObj.datasetName" :placeholder="$t('commons.name')" @change="changeDatasetName" />
</el-form-item>
<el-form-item>
<div>
<span>{{ $t('dataset.data_preview') }}</span>
<span style="font-size: 12px;color: #3d4d66;">（{{ $t('dataset.preview_100_data') }}）</span>
</div>
<!-- <span style="margin-left: 8px;">-->
<!-- <span v-if="field.fieldType === 'TEXT'">-->
<!-- <svg-icon v-if="field.fieldType === 'TEXT'" icon-class="field_text" class="field-icon-text" />-->
<!-- <span class="field-class">{{ $t('dataset.text') }}</span>-->
<!-- </span>-->
<!-- <span v-if="field.fieldType === 'DATETIME'">-->
<!-- <svg-icon v-if="field.fieldType === 'DATETIME'" icon-class="field_time" class="field-icon-time" />-->
<!-- <span class="field-class">{{ $t('dataset.time') }}</span>-->
<!-- </span>-->
<!-- <span v-if="field.fieldType === 'LONG' || field.fieldType === 'DOUBLE'">-->
<!-- <svg-icon v-if="field.fieldType === 'LONG' || field.fieldType === 'DOUBLE'" icon-class="field_value" class="field-icon-value" />-->
<!-- <span v-if="field.fieldType === 'LONG'" class="field-class">{{ $t('dataset.value') }}</span>-->
<!-- <span v-if="field.fieldType === 'DOUBLE'" class="field-class">{{ $t('dataset.value') + '(' + $t('dataset.float') + ')' }}</span>-->
<!-- </span>-->
<!-- </span>-->
</span>
<span style="font-size: 12px;margin-left: 10px;">
{{ field.remarks }}
</span>
</span>
<span v-else style="font-size: 12px;">
{{ field.remarks }}
</span>
</template>
</ux-table-column>
</ux-grid>
</div>
</el-card>
</el-form-item>
</el-form>
</el-col>
</el-row>
</el-header>
<el-main>
<div class="text item" >
<ux-grid
ref="plxTable"
size="mini"
style="width: 100%;"
:height="height"
:checkbox-config="{highlight: true}"
:width-resize="true"
>
<ux-table-column
v-for="field in sheetObj.fields"
:key="field.fieldName"
min-width="200px"
:field="field.fieldName"
:title="field.remarks"
:resizable="true"
>
<template slot="header" slot-scope="scope">
<span style="display: flex;align-items: center;">
<span style="display: inline-block;font-size: 12px;">
<div style="display: inline-block;">
<el-select v-model="field.fieldType" size="mini" style="display: inline-block;width: 120px;" @change="changeDatasetName">
<el-option
v-for="item in fieldOptions"
:key="item.value"
:label="item.label"
:value="item.value"
>
<span style="float: left">
<svg-icon v-if="item.value === 'TEXT'" icon-class="field_text" class="field-icon-text" />
<svg-icon v-if="item.value === 'DATETIME'" icon-class="field_time" class="field-icon-time" />
<svg-icon v-if="item.value === 'LONG' || item.value === 'DOUBLE'" icon-class="field_value" class="field-icon-value" />
</span>
<span style="float: left; color: #8492a6; font-size: 12px">{{ item.label }}</span>
</el-option>
</el-select>
</div>
</span>
<span style="font-size: 12px;margin-left: 10px;">
{{ field.remarks }}
</span>
</span>
<!-- <span v-else style="font-size: 12px;">-->
<!-- {{ field.remarks }}-->
<!-- </span>-->
</template>
</ux-table-column>
</ux-grid>
</div>
</el-main>
</el-container>
</el-container>
</el-row>
</el-row>
</el-col>
@ -140,12 +157,15 @@ export default {
tableId: {
type: String,
default: null
},
editType: {
type: Number,
default: 0
}
},
data() {
return {
name: '',
fields: [],
sheetObj: {datasetName: " ", fields: []},
sheets: [],
data: [],
mode: '1',
@ -160,13 +180,18 @@ export default {
{ label: this.$t('dataset.time'), value: 'DATETIME' },
{ label: this.$t('dataset.value'), value: 'LONG' },
{ label: this.$t('dataset.value') + '(' + this.$t('dataset.float') + ')', value: 'DOUBLE' }
]
],
props: {
label: 'excelLable',
children: 'sheets'
},
count: 1,
excelData: []
}
},
watch: {
},
mounted() {
// this.initDataSource()
window.onresize = () => {
this.calHeight()
}
@ -176,13 +201,33 @@ export default {
if (!this.param.tableId) {
this.param.tableId = ''
}
if (!this.param.editType) {
this.param.editType = 0
}
},
methods: {
// initDataSource() {
// listDatasource().then(response => {
// this.options = response.data
// })
// },
handleCheckChange(data, checked, indeterminate) {
},
handleNodeClick(data) {
if(data.sheet){
this.sheetObj = data
this.fields = data.fields
this.jsonArray = data.jsonArray
const datas = this.jsonArray
this.$refs.plxTable.reloadData(datas)
}
},
changeDatasetName(){
for(var i=0;i<this.excelData.length;i++){
if(this.excelData[i].excelId==this.sheetObj.sheetExcelId){
for(var j=0;j<this.excelData[i].sheets.length;j++){
if(this.excelData[i].sheets[j].excelId==this.sheetObj.sheetId){
this.excelData[i].sheets[j] = this.sheetObj
}
}
}
}
},
calHeight() {
const that = this
setTimeout(function() {
@ -204,7 +249,6 @@ export default {
this.data = []
const datas = this.data
this.$refs.plxTable.reloadData(datas)
this.name = ''
this.fileList = []
this.uploading = false
this.$message({
@ -214,52 +258,58 @@ export default {
})
},
uploadSuccess(response, file, fileList) {
this.path = response.data.path
this.fields = response.data.fields
this.sheets = response.data.sheets
if (this.sheets.length > 1) {
this.$warning(this.$t('dataset.sheet_warn'))
}
this.data = response.data.data
const datas = this.data
this.$refs.plxTable.reloadData(datas)
if (file.name.lastIndexOf('.') > 0) {
this.name = file.name.substring(0, file.name.lastIndexOf('.'))
}
this.excelData.push(response.data)
this.fileList = fileList
this.uploading = false
},
save() {
if (!this.name || this.name === '') {
this.$message({
showClose: true,
message: this.$t('dataset.pls_input_name'),
type: 'error'
})
var validate = true;
var selectedSheet = []
var sheetFileMd5 = []
var selectNode = this.$refs.tree.getCheckedNodes()
for(var i=0;i<selectNode.length;i++){
if(selectNode[i].sheet){
if (!selectNode[i].datasetName || selectNode[i].datasetName === '') {
this.$message({
showClose: true,
message: this.$t('dataset.pls_input_name'),
type: 'error'
})
return
}
if (selectNode[i].datasetName.length > 50) {
this.$message({
showClose: true,
message: this.$t('dataset.char_can_not_more_50'),
type: 'error'
})
return
}
selectedSheet.push(selectNode[i])
sheetFileMd5.push(selectNode[i].fieldsMd5)
}
}
if(selectedSheet.length == 0){
this.$message.warning(this.$t('dataset.ple_select_excel'))
return
}
if (this.name.length > 50) {
this.$message({
showClose: true,
message: this.$t('dataset.char_can_not_more_50'),
type: 'error'
})
return
if(!validate){
return;
}
let table = {}
if (!this.param.tableId) {
table = {
id: this.param.tableId,
name: this.name,
sceneId: this.param.id,
dataSourceId: null,
type: 'excel',
sheets: selectedSheet,
mode: parseInt(this.mode),
// info: '{"data":"' + this.path + '"}',
info: JSON.stringify({ data: this.path, sheets: [this.sheets[0]] }),
fields: this.fields
editType: 0
}
} else {
table = {
@ -268,22 +318,37 @@ export default {
sceneId: this.param.id,
dataSourceId: null,
type: 'excel',
sheets: selectedSheet,
mode: parseInt(this.mode),
// info: '{"data":"' + this.path + '"}',
info: JSON.stringify({ data: this.path, sheets: [this.sheets[0]] }),
editType: this.param.editType ? this.param.editType : 0
}
}
post('/dataset/table/update', table).then(response => {
// this.$store.dispatch('dataset/setSceneData', new Date().getTime())
this.$emit('saveSuccess', table)
this.cancel()
})
if(new Set(sheetFileMd5).size !== sheetFileMd5.length){
this.$confirm('The fields in the data table are consistent, whether they are merged into one table?', 'Merge data table ', {
confirmButtonText: 'Merge',
cancelButtonText: 'Do not merge',
type: 'info'
}).then(() => {
table.mergeSheet = true
post('/dataset/table/update', table).then(response => {
this.$emit('saveSuccess', table)
this.cancel()
})
}).catch(action => {
if(action == 'cancle'){
return
}
table.mergeSheet = false
post('/dataset/table/update', table).then(response => {
this.$emit('saveSuccess', table)
this.cancel()
})
});
}
},
cancel() {
this.dataReset()
// this.$router.push('/dataset/home')
if (this.param.tableId) {
this.$emit('switchComponent', { name: 'ViewTable', param: this.param.table })
} else {
@ -337,4 +402,14 @@ export default {
.dataPreview>>>.el-card__body{
padding:10px;
}
.el-header {
background-color: rgb(241, 243, 248);
color: #333;
line-height: 30px;
}
.el-main {
padding: 0px
}
</style>