fix(应用): 修复应用创建非直连数据集因doris表创建延迟而导致的报错问题

This commit is contained in:
wangjiahao 2022-11-30 13:36:23 +08:00
parent c1fca154db
commit c1509be15b
4 changed files with 58 additions and 44 deletions

View File

@ -149,7 +149,7 @@
</select> </select>
<select id="findByTableIds" resultMap="BaseResultMapDTO"> <select id="findByTableIds" resultMap="BaseResultMapDTO">
select dataset_table.* select distinct dataset_table.*
from dataset_table from dataset_table
where dataset_table.id in where dataset_table.id in
<foreach collection="tableIds" item="item" open="(" separator="," close=")"> <foreach collection="tableIds" item="item" open="(" separator="," close=")">

View File

@ -2,11 +2,16 @@ package io.dataease.service.dataset;
import com.google.gson.Gson; import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken; import com.google.gson.reflect.TypeToken;
import io.dataease.ext.ExtChartViewMapper;
import io.dataease.commons.constants.*; import io.dataease.commons.constants.*;
import io.dataease.commons.model.AuthURD; import io.dataease.commons.model.AuthURD;
import io.dataease.commons.utils.*; import io.dataease.commons.utils.*;
import io.dataease.controller.request.datasource.ApiDefinition; import io.dataease.controller.request.datasource.ApiDefinition;
import io.dataease.dto.dataset.DataTableInfoDTO;
import io.dataease.dto.dataset.ExcelSheetData;
import io.dataease.dto.datasource.*;
import io.dataease.exception.DataEaseException;
import io.dataease.ext.ExtChartViewMapper;
import io.dataease.listener.util.CacheUtils;
import io.dataease.plugins.common.base.domain.*; import io.dataease.plugins.common.base.domain.*;
import io.dataease.plugins.common.constants.DatasetType; import io.dataease.plugins.common.constants.DatasetType;
import io.dataease.plugins.common.constants.DatasourceTypes; import io.dataease.plugins.common.constants.DatasourceTypes;
@ -17,14 +22,9 @@ import io.dataease.plugins.datasource.entity.JdbcConfiguration;
import io.dataease.plugins.datasource.provider.Provider; import io.dataease.plugins.datasource.provider.Provider;
import io.dataease.plugins.datasource.query.QueryProvider; import io.dataease.plugins.datasource.query.QueryProvider;
import io.dataease.provider.DDLProvider; import io.dataease.provider.DDLProvider;
import io.dataease.provider.datasource.JdbcProvider;
import io.dataease.provider.ProviderFactory; import io.dataease.provider.ProviderFactory;
import io.dataease.dto.datasource.*; import io.dataease.provider.datasource.JdbcProvider;
import io.dataease.service.datasource.DatasourceService; import io.dataease.service.datasource.DatasourceService;
import io.dataease.dto.dataset.DataTableInfoDTO;
import io.dataease.dto.dataset.ExcelSheetData;
import io.dataease.exception.DataEaseException;
import io.dataease.listener.util.CacheUtils;
import io.dataease.service.engine.EngineService; import io.dataease.service.engine.EngineService;
import io.dataease.service.kettle.KettleService; import io.dataease.service.kettle.KettleService;
import io.dataease.service.message.DeMsgutil; import io.dataease.service.message.DeMsgutil;
@ -413,7 +413,7 @@ public class ExtractDataService {
} }
private List<DatasetTableField> getDatasetTableFields(String datasetTableId) { public List<DatasetTableField> getDatasetTableFields(String datasetTableId) {
List<DatasetTableField> datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTableId).build()); List<DatasetTableField> datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTableId).build());
datasetTableFields = datasetTableFields.stream().filter(datasetTableField -> datasetTableField.getExtField() == 0).collect(Collectors.toList()); datasetTableFields = datasetTableFields.stream().filter(datasetTableField -> datasetTableField.getExtField() == 0).collect(Collectors.toList());
datasetTableFields.sort((o1, o2) -> { datasetTableFields.sort((o1, o2) -> {
@ -617,7 +617,7 @@ public class ExtractDataService {
dataSetTableTaskLogService.save(datasetTableTaskLog, hasTask); dataSetTableTaskLogService.save(datasetTableTaskLog, hasTask);
} }
private void createEngineTable(String tableName, List<DatasetTableField> datasetTableFields) throws Exception { public void createEngineTable(String tableName, List<DatasetTableField> datasetTableFields) throws Exception {
Datasource engine = engineService.getDeEngine(); Datasource engine = engineService.getDeEngine();
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest(); DatasourceRequest datasourceRequest = new DatasourceRequest();
@ -793,7 +793,7 @@ public class ExtractDataService {
Thread.sleep(1000); Thread.sleep(1000);
} }
if (jobStatus.getStatusDescription().equals("Finished")) { if (jobStatus.getStatusDescription().equals("Finished")) {
LogUtil.info(datasetTable.getId()+ ": " + jobStatus.getLoggingString()); LogUtil.info(datasetTable.getId() + ": " + jobStatus.getLoggingString());
return; return;
} else { } else {
DataEaseException.throwException(jobStatus.getLoggingString()); DataEaseException.throwException(jobStatus.getLoggingString());
@ -1029,7 +1029,7 @@ public class ExtractDataService {
if (extractType.equalsIgnoreCase("all_scope") && datasetTable.getType().equalsIgnoreCase(DatasetType.SQL.name())) { if (extractType.equalsIgnoreCase("all_scope") && datasetTable.getType().equalsIgnoreCase(DatasetType.SQL.name())) {
DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class); DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class);
selectSQL = dataTableInfoDTO.getSql(); selectSQL = dataTableInfoDTO.getSql();
if(dataTableInfoDTO.isBase64Encryption()){ if (dataTableInfoDTO.isBase64Encryption()) {
selectSQL = new String(java.util.Base64.getDecoder().decode(selectSQL)); selectSQL = new String(java.util.Base64.getDecoder().decode(selectSQL));
} }
QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType()); QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType());
@ -1056,13 +1056,13 @@ public class ExtractDataService {
} }
private List<StepMeta> excelInputStep(String Info, List<DatasetTableField> datasetTableFields) { private List<StepMeta> excelInputStep(String Info, List<DatasetTableField> datasetTableFields) {
List<StepMeta>inputSteps = new ArrayList<>(); List<StepMeta> inputSteps = new ArrayList<>();
DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(Info, DataTableInfoDTO.class); DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(Info, DataTableInfoDTO.class);
List<ExcelSheetData> excelSheetDataList = dataTableInfoDTO.getExcelSheetDataList(); List<ExcelSheetData> excelSheetDataList = dataTableInfoDTO.getExcelSheetDataList();
List<String> sheetNames = new ArrayList<>(); List<String> sheetNames = new ArrayList<>();
int size =1; int size = 1;
for (ExcelSheetData excelSheetData : excelSheetDataList) { for (ExcelSheetData excelSheetData : excelSheetDataList) {
StepMeta fromStep = null; StepMeta fromStep = null;
String suffix = excelSheetData.getPath().substring(excelSheetDataList.get(0).getPath().lastIndexOf(".") + 1); String suffix = excelSheetData.getPath().substring(excelSheetDataList.get(0).getPath().lastIndexOf(".") + 1);
@ -1090,7 +1090,7 @@ public class ExtractDataService {
fromStep.setDraw(true); fromStep.setDraw(true);
fromStep.setLocation(100, 100 * size); fromStep.setLocation(100, 100 * size);
inputSteps.add(fromStep); inputSteps.add(fromStep);
}else { } else {
List<String> files = new ArrayList<>(); List<String> files = new ArrayList<>();
files.add(excelSheetData.getPath()); files.add(excelSheetData.getPath());
@ -1240,7 +1240,7 @@ public class ExtractDataService {
if (StringUtils.isNotEmpty(charset)) { if (StringUtils.isNotEmpty(charset)) {
String varcharFields = datasetTableFields.stream().filter(datasetTableField -> datasetTableField.getDeExtractType() == 0).map(DatasetTableField::getOriginName).collect(Collectors.joining(",")); String varcharFields = datasetTableFields.stream().filter(datasetTableField -> datasetTableField.getDeExtractType() == 0).map(DatasetTableField::getOriginName).collect(Collectors.joining(","));
tmp_code = tmp_code.replace("handleCharset", handleCharset.replace("Datasource_Charset", charset).replace("Target_Charset", targetCharset).replace("varcharFields", varcharFields)); tmp_code = tmp_code.replace("handleCharset", handleCharset.replace("Datasource_Charset", charset).replace("Target_Charset", targetCharset).replace("varcharFields", varcharFields));
}else { } else {
tmp_code = tmp_code.replace("handleCharset", ""); tmp_code = tmp_code.replace("handleCharset", "");
} }
} else { } else {

View File

@ -20,6 +20,7 @@ import io.dataease.service.chart.ChartViewService;
import io.dataease.service.dataset.DataSetGroupService; import io.dataease.service.dataset.DataSetGroupService;
import io.dataease.service.dataset.DataSetTableFieldsService; import io.dataease.service.dataset.DataSetTableFieldsService;
import io.dataease.service.dataset.DataSetTableService; import io.dataease.service.dataset.DataSetTableService;
import io.dataease.service.dataset.ExtractDataService;
import io.dataease.service.datasource.DatasourceService; import io.dataease.service.datasource.DatasourceService;
import io.dataease.service.staticResource.StaticResourceService; import io.dataease.service.staticResource.StaticResourceService;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
@ -68,6 +69,8 @@ public class PanelAppTemplateService {
private DataSetGroupService dataSetGroupService; private DataSetGroupService dataSetGroupService;
@Resource @Resource
private StaticResourceService staticResourceService; private StaticResourceService staticResourceService;
@Resource
private ExtractDataService extractDataService;
public List<PanelAppTemplateWithBLOBs> list(PanelAppTemplateRequest request) { public List<PanelAppTemplateWithBLOBs> list(PanelAppTemplateRequest request) {
return extPanelAppTemplateMapper.queryBaseInfo(request.getNodeType(), request.getPid()); return extPanelAppTemplateMapper.queryBaseInfo(request.getNodeType(), request.getPid());
@ -209,6 +212,7 @@ public class PanelAppTemplateService {
DatasetTableField newTableField = dataSetTableFieldsService.save(datasetTableField); DatasetTableField newTableField = dataSetTableFieldsService.save(datasetTableField);
datasetFieldsRealMap.put(oldId, newTableField.getId()); datasetFieldsRealMap.put(oldId, newTableField.getId());
datasetFieldsMd5FormatRealMap.put(TableUtils.fieldNameShort(oldTableId + "_" + datasetTableField.getOriginName()), TableUtils.fieldNameShort(newTableField.getTableId() + "_" + datasetTableField.getOriginName())); datasetFieldsMd5FormatRealMap.put(TableUtils.fieldNameShort(oldTableId + "_" + datasetTableField.getOriginName()), TableUtils.fieldNameShort(newTableField.getTableId() + "_" + datasetTableField.getOriginName()));
datasetFieldsMd5FormatRealMap.put(TableUtils.fieldName(oldTableId + "_" + datasetTableField.getDataeaseName()), TableUtils.fieldName(newTableField.getTableId() + "_" + datasetTableField.getDataeaseName()));
} }
} }
//数据集计算字段替换 //数据集计算字段替换
@ -225,6 +229,8 @@ public class PanelAppTemplateService {
DatasetTableField newTableField = dataSetTableFieldsService.save(datasetTableField); DatasetTableField newTableField = dataSetTableFieldsService.save(datasetTableField);
datasetFieldsRealMap.put(oldId, newTableField.getId()); datasetFieldsRealMap.put(oldId, newTableField.getId());
datasetFieldsMd5FormatRealMap.put(TableUtils.fieldNameShort(oldTableId + "_" + oldOriginName), TableUtils.fieldNameShort(newTableField.getTableId() + "_" + datasetTableField.getOriginName())); datasetFieldsMd5FormatRealMap.put(TableUtils.fieldNameShort(oldTableId + "_" + oldOriginName), TableUtils.fieldNameShort(newTableField.getTableId() + "_" + datasetTableField.getOriginName()));
datasetFieldsMd5FormatRealMap.put(TableUtils.fieldName(oldTableId + "_" + datasetTableField.getDataeaseName()), TableUtils.fieldName(newTableField.getTableId() + "_" + datasetTableField.getDataeaseName()));
} }
} }
@ -233,13 +239,28 @@ public class PanelAppTemplateService {
if (DatasetType.UNION.name().equalsIgnoreCase(datasetTypeRealMap.get(datasetTableField.getTableId())) || DatasetType.CUSTOM.name().equalsIgnoreCase(datasetTypeRealMap.get(datasetTableField.getTableId()))) { if (DatasetType.UNION.name().equalsIgnoreCase(datasetTypeRealMap.get(datasetTableField.getTableId())) || DatasetType.CUSTOM.name().equalsIgnoreCase(datasetTypeRealMap.get(datasetTableField.getTableId()))) {
DatasetTableField updateField = new DatasetTableField(); DatasetTableField updateField = new DatasetTableField();
updateField.setId(datasetTableField.getId()); updateField.setId(datasetTableField.getId());
updateField.setOriginName(datasetFieldsMd5FormatRealMap.get(datasetTableField.getOriginName())); String newOriginName = datasetFieldsMd5FormatRealMap.get(datasetTableField.getOriginName());
dataSetTableFieldsService.updateByPrimaryKeySelective(updateField); String dataeaseName = datasetFieldsMd5FormatRealMap.get(datasetTableField.getDataeaseName());
if (StringUtils.isNotEmpty(newOriginName) || StringUtils.isNotEmpty(dataeaseName)) {
updateField.setOriginName(datasetFieldsMd5FormatRealMap.get(datasetTableField.getOriginName()));
updateField.setDataeaseName(datasetFieldsMd5FormatRealMap.get(datasetTableField.getDataeaseName()));
dataSetTableFieldsService.updateByPrimaryKeySelective(updateField);
}
} }
} }
return datasetFieldsRealMap; return datasetFieldsRealMap;
} }
@Transactional(rollbackFor = Exception.class)
public void createDorisTable(List<DatasetTable> datasetTablesInfo) throws Exception {
for (DatasetTable datasetTable : datasetTablesInfo) {
if (1 == datasetTable.getMode() && !(DatasetType.CUSTOM.name().equalsIgnoreCase(datasetTable.getType()) || DatasetType.UNION.name().equalsIgnoreCase(datasetTable.getType()))) {
List<DatasetTableField> fields = extractDataService.getDatasetTableFields(datasetTable.getId());
extractDataService.createEngineTable(TableUtils.tableName(datasetTable.getId()), fields);
}
}
}
@Transactional(rollbackFor = Exception.class) @Transactional(rollbackFor = Exception.class)
public void resetCustomAndUnionDataset(List<DatasetTable> datasetTablesInfo, Map<String, String> datasetRealMap, Map<String, String> datasetFieldsRealMap) throws Exception { public void resetCustomAndUnionDataset(List<DatasetTable> datasetTablesInfo, Map<String, String> datasetRealMap, Map<String, String> datasetFieldsRealMap) throws Exception {
for (DatasetTable datasetTable : datasetTablesInfo) { for (DatasetTable datasetTable : datasetTablesInfo) {

View File

@ -30,7 +30,6 @@ import io.dataease.listener.util.CacheUtils;
import io.dataease.plugins.common.base.domain.*; import io.dataease.plugins.common.base.domain.*;
import io.dataease.plugins.common.base.mapper.*; import io.dataease.plugins.common.base.mapper.*;
import io.dataease.plugins.common.constants.DeTypeConstants; import io.dataease.plugins.common.constants.DeTypeConstants;
import io.dataease.service.SystemInfoService;
import io.dataease.service.chart.ChartViewService; import io.dataease.service.chart.ChartViewService;
import io.dataease.service.dataset.DataSetGroupService; import io.dataease.service.dataset.DataSetGroupService;
import io.dataease.service.dataset.DataSetTableService; import io.dataease.service.dataset.DataSetTableService;
@ -135,8 +134,6 @@ public class PanelGroupService {
private DatasetGroupMapper datasetGroupMapper; private DatasetGroupMapper datasetGroupMapper;
@Resource @Resource
private PanelWatermarkMapper panelWatermarkMapper; private PanelWatermarkMapper panelWatermarkMapper;
@Resource
private SystemInfoService systemInfoService;
public List<PanelGroupDTO> tree(PanelGroupRequest panelGroupRequest) { public List<PanelGroupDTO> tree(PanelGroupRequest panelGroupRequest) {
String userId = String.valueOf(AuthUtils.getUser().getUserId()); String userId = String.valueOf(AuthUtils.getUser().getUserId());
@ -811,29 +808,6 @@ public class PanelGroupService {
List<ChartViewField> chartViewFieldsInfo = extChartViewFieldMapper.findByPanelId(panelId); List<ChartViewField> chartViewFieldsInfo = extChartViewFieldMapper.findByPanelId(panelId);
//3.获取所有数据集信息 //3.获取所有数据集信息
List<DatasetTable> datasetTablesInfo = extDataSetTableMapper.findByPanelId(panelId); List<DatasetTable> datasetTablesInfo = extDataSetTableMapper.findByPanelId(panelId);
List<String> attachTableIds = new ArrayList<>();
if (CollectionUtils.isNotEmpty(datasetTablesInfo)) {
for (DatasetTable datasetTable : datasetTablesInfo) {
if ("union".equals(datasetTable.getType()) && StringUtils.isNotEmpty(datasetTable.getInfo())) {
DataTableInfoDTO dt = gson.fromJson(datasetTable.getInfo(), DataTableInfoDTO.class);
DatasetUtils.getUnionTable(attachTableIds, dt.getUnion());
} else if ("custom".equals(datasetTable.getType()) && StringUtils.isNotEmpty(datasetTable.getInfo())) {
Map result = gson.fromJson(datasetTable.getInfo(), Map.class);
List<Map> list = (List<Map>) result.get("list");
if (CollectionUtils.isNotEmpty(list)) {
for (Map details : list) {
attachTableIds.add(String.valueOf(details.get("tableId")));
}
}
}
}
if (CollectionUtils.isNotEmpty(attachTableIds)) {
List<DatasetTable> attachDatasetTables = extDataSetTableMapper.findByTableIds(attachTableIds);
if (CollectionUtils.isNotEmpty(attachDatasetTables)) {
datasetTablesInfo.addAll(attachDatasetTables);
}
}
}
// dataset check // dataset check
if (CollectionUtils.isEmpty(datasetTablesInfo)) { if (CollectionUtils.isEmpty(datasetTablesInfo)) {
return new PanelExport2App(Translator.get("I18N_APP_NO_DATASET_ERROR")); return new PanelExport2App(Translator.get("I18N_APP_NO_DATASET_ERROR"));
@ -841,6 +815,23 @@ public class PanelGroupService {
return new PanelExport2App(Translator.get("I18N_APP_ERROR_DATASET")); return new PanelExport2App(Translator.get("I18N_APP_ERROR_DATASET"));
} }
List<String> allTableIds = datasetTablesInfo.stream().map(DatasetTable::getId).collect(Collectors.toList()); List<String> allTableIds = datasetTablesInfo.stream().map(DatasetTable::getId).collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(datasetTablesInfo)) {
for (DatasetTable datasetTable : datasetTablesInfo) {
if ("union".equals(datasetTable.getType()) && StringUtils.isNotEmpty(datasetTable.getInfo())) {
DataTableInfoDTO dt = gson.fromJson(datasetTable.getInfo(), DataTableInfoDTO.class);
DatasetUtils.getUnionTable(allTableIds, dt.getUnion());
} else if ("custom".equals(datasetTable.getType()) && StringUtils.isNotEmpty(datasetTable.getInfo())) {
Map result = gson.fromJson(datasetTable.getInfo(), Map.class);
List<Map> list = (List<Map>) result.get("list");
if (CollectionUtils.isNotEmpty(list)) {
for (Map details : list) {
allTableIds.add(String.valueOf(details.get("tableId")));
}
}
}
}
}
datasetTablesInfo = extDataSetTableMapper.findByTableIds(allTableIds);
//4.获取所有数据集字段信息 //4.获取所有数据集字段信息
List<DatasetTableField> datasetTableFieldsInfo = extDataSetTableFieldMapper.findByTableIds(allTableIds); List<DatasetTableField> datasetTableFieldsInfo = extDataSetTableFieldMapper.findByTableIds(allTableIds);
//5.获取所有任务信息 //5.获取所有任务信息
@ -917,6 +908,8 @@ public class PanelGroupService {
Map<String, String> datasetFieldsRealMap = panelAppTemplateService.applyDatasetField(datasetTableFieldsInfo, datasetsRealMap, datasetTypeRealMap, datasetFieldsMd5FormatRealMap); Map<String, String> datasetFieldsRealMap = panelAppTemplateService.applyDatasetField(datasetTableFieldsInfo, datasetsRealMap, datasetTypeRealMap, datasetFieldsMd5FormatRealMap);
panelAppTemplateService.createDorisTable(datasetTablesInfo);
panelAppTemplateService.resetCustomAndUnionDataset(datasetTablesInfo, datasetsRealMap, datasetFieldsRealMap); panelAppTemplateService.resetCustomAndUnionDataset(datasetTablesInfo, datasetsRealMap, datasetFieldsRealMap);
Map<String, String> chartViewsRealMap = panelAppTemplateService.applyViews(chartViewsInfo, datasetsRealMap, datasetFieldsRealMap, datasetFieldsMd5FormatRealMap, newPanelId); Map<String, String> chartViewsRealMap = panelAppTemplateService.applyViews(chartViewsInfo, datasetsRealMap, datasetFieldsRealMap, datasetFieldsMd5FormatRealMap, newPanelId);