center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

Merge remote-tracking branch 'origin/dev' into dev

Browse Source
This commit is contained in:
wangjiahao 2022-01-20 18:31:53 +08:00
commit fc9ba6a67e
3 changed files with 49 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
backend/src/main/java/io/dataease/auth/server/AuthServer.java
View File

@ -117,6 +117,7 @@ public class AuthServer implements AuthApi {
// 记录token操作时间
result.put("token", token);
ServletUtils.setToken(token);
authUserService.clearCache(user.getUserId());
return result;
}

42
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@ -126,6 +126,8 @@ public class DataSetTableService {
}
public void saveExcel(DataSetTableRequest datasetTable) throws Exception {
List<String> datasetIdList = new ArrayList<>();
if (StringUtils.isEmpty(datasetTable.getId())) {
if (datasetTable.isMergeSheet()) {
Map<String, List<ExcelSheetData>> map = datasetTable.getSheets().stream().collect(Collectors.groupingBy(ExcelSheetData::getFieldsMd5));
@ -149,12 +151,13 @@ public class DataSetTableService {
DataTableInfoDTO info = new DataTableInfoDTO();
info.setExcelSheetDataList(excelSheetDataList);
sheetTable.setInfo(new Gson().toJson(info));
int insert = datasetTableMapper.insert(sheetTable);
if (insert == 1) {
saveExcelTableField(sheetTable.getId(), excelSheetDataList.get(0).getFields(), true);
commonThreadPool.addTask(() -> extractDataService.extractExcelData(sheetTable.getId(), "all_scope", "初始导入", null));
}
datasetTableMapper.insert(sheetTable);
saveExcelTableField(sheetTable.getId(), excelSheetDataList.get(0).getFields(), true);
datasetIdList.add(sheetTable.getId());
}
datasetIdList.forEach(datasetId ->{
commonThreadPool.addTask(() -> extractDataService.extractExcelData(datasetId, "all_scope", "初始导入", null, datasetIdList));
});
} else {
for (ExcelSheetData sheet : datasetTable.getSheets()) {
String[] fieldArray = sheet.getFields().stream().map(TableFiled::getFieldName).toArray(String[]::new);
@ -175,12 +178,14 @@ public class DataSetTableService {
DataTableInfoDTO info = new DataTableInfoDTO();
info.setExcelSheetDataList(excelSheetDataList);
sheetTable.setInfo(new Gson().toJson(info));
int insert = datasetTableMapper.insert(sheetTable);
if (insert == 1) {
saveExcelTableField(sheetTable.getId(), sheet.getFields(), true);
commonThreadPool.addTask(() -> extractDataService.extractExcelData(sheetTable.getId(), "all_scope", "初始导入", null));
}
datasetTableMapper.insert(sheetTable);
saveExcelTableField(sheetTable.getId(), sheet.getFields(), true);
datasetIdList.add(sheetTable.getId());
}
datasetIdList.forEach(datasetId ->{
commonThreadPool.addTask(() -> extractDataService.extractExcelData(datasetId, "all_scope", "初始导入", null, datasetIdList));
});
}
return;
}
@ -210,13 +215,10 @@ public class DataSetTableService {
datasetTable.setInfo(new Gson().toJson(info));
int update = datasetTableMapper.updateByPrimaryKeySelective(datasetTable);
// 替換時，先不刪除旧字段；同步成功后再删除
if (update == 1) {
if (datasetTable.getEditType() == 0) {
commonThreadPool.addTask(() -> extractDataService.extractExcelData(datasetTable.getId(), "all_scope", "替换", saveExcelTableField(datasetTable.getId(), datasetTable.getSheets().get(0).getFields(), false)));
} else if (datasetTable.getEditType() == 1) {
commonThreadPool.addTask(() -> extractDataService.extractExcelData(datasetTable.getId(), "add_scope", "追加", null));
}
if (datasetTable.getEditType() == 0) {
commonThreadPool.addTask(() -> extractDataService.extractExcelData(datasetTable.getId(), "all_scope", "替换", saveExcelTableField(datasetTable.getId(), datasetTable.getSheets().get(0).getFields(), false), Arrays.asList(datasetTable.getId())));
} else if (datasetTable.getEditType() == 1) {
commonThreadPool.addTask(() -> extractDataService.extractExcelData(datasetTable.getId(), "add_scope", "追加", null, Arrays.asList(datasetTable.getId())));
}
}
@ -302,6 +304,12 @@ public class DataSetTableService {
return extDataSetTableMapper.search(dataSetTableRequest);
}
public List<DatasetTable> list(List<String> datasetIds) {
DatasetTableExample example = new DatasetTableExample();
example.createCriteria().andIdIn(datasetIds);
return datasetTableMapper.selectByExample(example);
}
public List<DataSetTableDTO> listAndGroup(DataSetTableRequest dataSetTableRequest) {
dataSetTableRequest.setUserId(String.valueOf(AuthUtils.getUser().getUserId()));
dataSetTableRequest.setTypeFilter(dataSetTableRequest.getTypeFilter());

27
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -150,7 +150,7 @@ public class ExtractDataService {
}
}
public void extractExcelData(String datasetTableId, String type, String ops, List<DatasetTableField> datasetTableFields) {
public void extractExcelData(String datasetTableId, String type, String ops, List<DatasetTableField> datasetTableFields, List<String> datasetTableIds) {
Datasource datasource = new Datasource();
datasource.setType("excel");
DatasetTable datasetTable = getDatasetTable(datasetTableId);
@ -223,9 +223,7 @@ public class ExtractDataService {
dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)));
} finally {
deleteFile("all_scope", datasetTableId);
for (ExcelSheetData excelSheetData : new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getExcelSheetDataList()) {
deleteFile(excelSheetData.getPath());
}
deleteExcelFile(datasetTable, datasetTableIds);
}
break;
@ -1081,6 +1079,27 @@ public class ExtractDataService {
deleteFile(root_path + transName + ".ktr");
}
private void deleteExcelFile(DatasetTable datasetTable, List<String>datasetTableIds){
List<DatasetTable> datasetTables = dataSetTableService.list(datasetTableIds);
for (ExcelSheetData excelSheetData : new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getExcelSheetDataList()) {
Boolean allIsFinished = true;
for (DatasetTable table : datasetTables) {
for(ExcelSheetData data : new Gson().fromJson(table.getInfo(), DataTableInfoDTO.class).getExcelSheetDataList()){
if(data.getPath().equalsIgnoreCase(excelSheetData.getPath())){
if(StringUtils.isEmpty(table.getSyncStatus()) || table.getSyncStatus().equalsIgnoreCase(JobStatus.Underway.name())){
allIsFinished = false;
}
}
}
}
if(allIsFinished){
deleteFile(excelSheetData.getPath());
}else {
try { Thread.sleep(5000); }catch (Exception ignore){}
deleteExcelFile(datasetTable, datasetTableIds);
}
}
}
private void deleteFile(String filePath){
if(StringUtils.isEmpty(filePath)){
return;