center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

fix(应用): 修复应用创建非直连数据集因doris表创建延迟而导致的报错问题

Browse Source
This commit is contained in:
wangjiahao 2022-11-30 13:36:23 +08:00
parent c1fca154db
commit c1509be15b
4 changed files with 58 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/main/java/io/dataease/ext/ExtDataSetTableMapper.xml
View File

@ -149,7 +149,7 @@
</select>
<select id="findByTableIds" resultMap="BaseResultMapDTO">
select dataset_table.*
select distinct dataset_table.*
from dataset_table
where dataset_table.id in
<foreach collection="tableIds" item="item" open="(" separator="," close=")">

18
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@ -2,11 +2,16 @@ package io.dataease.service.dataset;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import io.dataease.ext.ExtChartViewMapper;
import io.dataease.commons.constants.*;
import io.dataease.commons.model.AuthURD;
import io.dataease.commons.utils.*;
import io.dataease.controller.request.datasource.ApiDefinition;
import io.dataease.dto.dataset.DataTableInfoDTO;
import io.dataease.dto.dataset.ExcelSheetData;
import io.dataease.dto.datasource.*;
import io.dataease.exception.DataEaseException;
import io.dataease.ext.ExtChartViewMapper;
import io.dataease.listener.util.CacheUtils;
import io.dataease.plugins.common.base.domain.*;
import io.dataease.plugins.common.constants.DatasetType;
import io.dataease.plugins.common.constants.DatasourceTypes;
@ -17,14 +22,9 @@ import io.dataease.plugins.datasource.entity.JdbcConfiguration;
import io.dataease.plugins.datasource.provider.Provider;
import io.dataease.plugins.datasource.query.QueryProvider;
import io.dataease.provider.DDLProvider;
import io.dataease.provider.datasource.JdbcProvider;
import io.dataease.provider.ProviderFactory;
import io.dataease.dto.datasource.*;
import io.dataease.provider.datasource.JdbcProvider;
import io.dataease.service.datasource.DatasourceService;
import io.dataease.dto.dataset.DataTableInfoDTO;
import io.dataease.dto.dataset.ExcelSheetData;
import io.dataease.exception.DataEaseException;
import io.dataease.listener.util.CacheUtils;
import io.dataease.service.engine.EngineService;
import io.dataease.service.kettle.KettleService;
import io.dataease.service.message.DeMsgutil;
@ -413,7 +413,7 @@ public class ExtractDataService {
}
private List<DatasetTableField> getDatasetTableFields(String datasetTableId) {
public List<DatasetTableField> getDatasetTableFields(String datasetTableId) {
List<DatasetTableField> datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTableId).build());
datasetTableFields = datasetTableFields.stream().filter(datasetTableField -> datasetTableField.getExtField() == 0).collect(Collectors.toList());
datasetTableFields.sort((o1, o2) -> {
@ -617,7 +617,7 @@ public class ExtractDataService {
dataSetTableTaskLogService.save(datasetTableTaskLog, hasTask);
}
private void createEngineTable(String tableName, List<DatasetTableField> datasetTableFields) throws Exception {
public void createEngineTable(String tableName, List<DatasetTableField> datasetTableFields) throws Exception {
Datasource engine = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();

21
backend/src/main/java/io/dataease/service/panel/PanelAppTemplateService.java
View File

@ -20,6 +20,7 @@ import io.dataease.service.chart.ChartViewService;
import io.dataease.service.dataset.DataSetGroupService;
import io.dataease.service.dataset.DataSetTableFieldsService;
import io.dataease.service.dataset.DataSetTableService;
import io.dataease.service.dataset.ExtractDataService;
import io.dataease.service.datasource.DatasourceService;
import io.dataease.service.staticResource.StaticResourceService;
import org.apache.commons.lang3.StringUtils;
@ -68,6 +69,8 @@ public class PanelAppTemplateService {
private DataSetGroupService dataSetGroupService;
@Resource
private StaticResourceService staticResourceService;
@Resource
private ExtractDataService extractDataService;
public List<PanelAppTemplateWithBLOBs> list(PanelAppTemplateRequest request) {
return extPanelAppTemplateMapper.queryBaseInfo(request.getNodeType(), request.getPid());
@ -209,6 +212,7 @@ public class PanelAppTemplateService {
DatasetTableField newTableField = dataSetTableFieldsService.save(datasetTableField);
datasetFieldsRealMap.put(oldId, newTableField.getId());
datasetFieldsMd5FormatRealMap.put(TableUtils.fieldNameShort(oldTableId + "_" + datasetTableField.getOriginName()), TableUtils.fieldNameShort(newTableField.getTableId() + "_" + datasetTableField.getOriginName()));
datasetFieldsMd5FormatRealMap.put(TableUtils.fieldName(oldTableId + "_" + datasetTableField.getDataeaseName()), TableUtils.fieldName(newTableField.getTableId() + "_" + datasetTableField.getDataeaseName()));
}
}
//数据集计算字段替换
@ -225,6 +229,8 @@ public class PanelAppTemplateService {
DatasetTableField newTableField = dataSetTableFieldsService.save(datasetTableField);
datasetFieldsRealMap.put(oldId, newTableField.getId());
datasetFieldsMd5FormatRealMap.put(TableUtils.fieldNameShort(oldTableId + "_" + oldOriginName), TableUtils.fieldNameShort(newTableField.getTableId() + "_" + datasetTableField.getOriginName()));
datasetFieldsMd5FormatRealMap.put(TableUtils.fieldName(oldTableId + "_" + datasetTableField.getDataeaseName()), TableUtils.fieldName(newTableField.getTableId() + "_" + datasetTableField.getDataeaseName()));
}
}
@ -233,13 +239,28 @@ public class PanelAppTemplateService {
if (DatasetType.UNION.name().equalsIgnoreCase(datasetTypeRealMap.get(datasetTableField.getTableId())) || DatasetType.CUSTOM.name().equalsIgnoreCase(datasetTypeRealMap.get(datasetTableField.getTableId()))) {
DatasetTableField updateField = new DatasetTableField();
updateField.setId(datasetTableField.getId());
String newOriginName = datasetFieldsMd5FormatRealMap.get(datasetTableField.getOriginName());
String dataeaseName = datasetFieldsMd5FormatRealMap.get(datasetTableField.getDataeaseName());
if (StringUtils.isNotEmpty(newOriginName) || StringUtils.isNotEmpty(dataeaseName)) {
updateField.setOriginName(datasetFieldsMd5FormatRealMap.get(datasetTableField.getOriginName()));
updateField.setDataeaseName(datasetFieldsMd5FormatRealMap.get(datasetTableField.getDataeaseName()));
dataSetTableFieldsService.updateByPrimaryKeySelective(updateField);
}
}
}
return datasetFieldsRealMap;
}
@Transactional(rollbackFor = Exception.class)
public void createDorisTable(List<DatasetTable> datasetTablesInfo) throws Exception {
for (DatasetTable datasetTable : datasetTablesInfo) {
if (1 == datasetTable.getMode() && !(DatasetType.CUSTOM.name().equalsIgnoreCase(datasetTable.getType()) || DatasetType.UNION.name().equalsIgnoreCase(datasetTable.getType()))) {
List<DatasetTableField> fields = extractDataService.getDatasetTableFields(datasetTable.getId());
extractDataService.createEngineTable(TableUtils.tableName(datasetTable.getId()), fields);
}
}
}
@Transactional(rollbackFor = Exception.class)
public void resetCustomAndUnionDataset(List<DatasetTable> datasetTablesInfo, Map<String, String> datasetRealMap, Map<String, String> datasetFieldsRealMap) throws Exception {
for (DatasetTable datasetTable : datasetTablesInfo) {

45
backend/src/main/java/io/dataease/service/panel/PanelGroupService.java
View File

@ -30,7 +30,6 @@ import io.dataease.listener.util.CacheUtils;
import io.dataease.plugins.common.base.domain.*;
import io.dataease.plugins.common.base.mapper.*;
import io.dataease.plugins.common.constants.DeTypeConstants;
import io.dataease.service.SystemInfoService;
import io.dataease.service.chart.ChartViewService;
import io.dataease.service.dataset.DataSetGroupService;
import io.dataease.service.dataset.DataSetTableService;
@ -135,8 +134,6 @@ public class PanelGroupService {
private DatasetGroupMapper datasetGroupMapper;
@Resource
private PanelWatermarkMapper panelWatermarkMapper;
@Resource
private SystemInfoService systemInfoService;
public List<PanelGroupDTO> tree(PanelGroupRequest panelGroupRequest) {
String userId = String.valueOf(AuthUtils.getUser().getUserId());
@ -811,29 +808,6 @@ public class PanelGroupService {
List<ChartViewField> chartViewFieldsInfo = extChartViewFieldMapper.findByPanelId(panelId);
//3.获取所有数据集信息
List<DatasetTable> datasetTablesInfo = extDataSetTableMapper.findByPanelId(panelId);
List<String> attachTableIds = new ArrayList<>();
if (CollectionUtils.isNotEmpty(datasetTablesInfo)) {
for (DatasetTable datasetTable : datasetTablesInfo) {
if ("union".equals(datasetTable.getType()) && StringUtils.isNotEmpty(datasetTable.getInfo())) {
DataTableInfoDTO dt = gson.fromJson(datasetTable.getInfo(), DataTableInfoDTO.class);
DatasetUtils.getUnionTable(attachTableIds, dt.getUnion());
} else if ("custom".equals(datasetTable.getType()) && StringUtils.isNotEmpty(datasetTable.getInfo())) {
Map result = gson.fromJson(datasetTable.getInfo(), Map.class);
List<Map> list = (List<Map>) result.get("list");
if (CollectionUtils.isNotEmpty(list)) {
for (Map details : list) {
attachTableIds.add(String.valueOf(details.get("tableId")));
}
}
}
}
if (CollectionUtils.isNotEmpty(attachTableIds)) {
List<DatasetTable> attachDatasetTables = extDataSetTableMapper.findByTableIds(attachTableIds);
if (CollectionUtils.isNotEmpty(attachDatasetTables)) {
datasetTablesInfo.addAll(attachDatasetTables);
}
}
}
// dataset check
if (CollectionUtils.isEmpty(datasetTablesInfo)) {
return new PanelExport2App(Translator.get("I18N_APP_NO_DATASET_ERROR"));
@ -841,6 +815,23 @@ public class PanelGroupService {
return new PanelExport2App(Translator.get("I18N_APP_ERROR_DATASET"));
}
List<String> allTableIds = datasetTablesInfo.stream().map(DatasetTable::getId).collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(datasetTablesInfo)) {
for (DatasetTable datasetTable : datasetTablesInfo) {
if ("union".equals(datasetTable.getType()) && StringUtils.isNotEmpty(datasetTable.getInfo())) {
DataTableInfoDTO dt = gson.fromJson(datasetTable.getInfo(), DataTableInfoDTO.class);
DatasetUtils.getUnionTable(allTableIds, dt.getUnion());
} else if ("custom".equals(datasetTable.getType()) && StringUtils.isNotEmpty(datasetTable.getInfo())) {
Map result = gson.fromJson(datasetTable.getInfo(), Map.class);
List<Map> list = (List<Map>) result.get("list");
if (CollectionUtils.isNotEmpty(list)) {
for (Map details : list) {
allTableIds.add(String.valueOf(details.get("tableId")));
}
}
}
}
}
datasetTablesInfo = extDataSetTableMapper.findByTableIds(allTableIds);
//4.获取所有数据集字段信息
List<DatasetTableField> datasetTableFieldsInfo = extDataSetTableFieldMapper.findByTableIds(allTableIds);
//5.获取所有任务信息
@ -917,6 +908,8 @@ public class PanelGroupService {
Map<String, String> datasetFieldsRealMap = panelAppTemplateService.applyDatasetField(datasetTableFieldsInfo, datasetsRealMap, datasetTypeRealMap, datasetFieldsMd5FormatRealMap);
panelAppTemplateService.createDorisTable(datasetTablesInfo);
panelAppTemplateService.resetCustomAndUnionDataset(datasetTablesInfo, datasetsRealMap, datasetFieldsRealMap);
Map<String, String> chartViewsRealMap = panelAppTemplateService.applyViews(chartViewsInfo, datasetsRealMap, datasetFieldsRealMap, datasetFieldsMd5FormatRealMap, newPanelId);