fix(应用): 修复含有关联数据集的仪表板应用导入后关联数据集报错问题

This commit is contained in:
wangjiahao 2022-11-28 22:23:30 +08:00
parent b4a47f6569
commit 1d08506f0c
12 changed files with 189 additions and 66 deletions

View File

@ -0,0 +1,26 @@
package io.dataease.commons.utils;
import io.dataease.dto.dataset.union.UnionDTO;
import org.apache.commons.collections4.CollectionUtils;
import java.util.List;
/**
* Author: wangjiahao
* Date: 2022/11/28
* Description:
*/
public class DatasetUtils {
public static void getUnionTable(List<String> tableIdList, List<UnionDTO> childrenDs) {
if (CollectionUtils.isNotEmpty(childrenDs)) {
for (UnionDTO unionDTO : childrenDs) {
String tableId = unionDTO.getCurrentDs().getId();
tableIdList.add(tableId);
if (CollectionUtils.isNotEmpty(unionDTO.getChildrenDs())) {
getUnionTable(tableIdList, unionDTO.getChildrenDs());
}
}
}
}
}

View File

@ -8,4 +8,6 @@ import java.util.List;
public interface ExtDataSetTableFieldMapper {
List<DatasetTableField> findByPanelId(@Param("panelId") String panelId);
List<DatasetTableField> findByTableIds(@Param("tableIds") List<String> tableIds);
}

View File

@ -7,23 +7,22 @@
</resultMap>
<select id="findByPanelId" resultMap="BaseResultMapDTO">
select
dataset_table_field.*
select dataset_table_field.*
from dataset_table_field
where table_id in (
SELECT
table_id
FROM
chart_view
WHERE
id IN (
SELECT
chart_view_id
FROM
panel_view
WHERE
panel_id = #{panelId}
)
)
where table_id in (SELECT table_id
FROM chart_view
WHERE id IN (SELECT chart_view_id
FROM panel_view
WHERE panel_id = #{panelId}))
</select>
<select id="findByTableIds" resultMap="BaseResultMapDTO">
select
dataset_table_field.*
from dataset_table_field
where dataset_table_field.table_id in
<foreach collection="tableIds" item="item" open="(" separator="," close=")">
#{item}
</foreach>
</select>
</mapper>

View File

@ -11,8 +11,13 @@ public interface ExtDataSetTableMapper {
List<DataSetTableDTO> search(DataSetTableRequest request);
DataSetTableDTO searchOne(DataSetTableRequest request);
DataSetTableDTO findOneDetails(@Param("datasetTableId") String datasetTableId);
List<DataSetTableDTO> searchDataSetTableWithPanelId(@Param("panelId") String panelId, @Param("userId") String userId);
List<DatasetTable> findByPanelId(@Param("panelId") String panelId);
List<DatasetTable> findByTableIds(@Param("tableIds") List<String> tableIds);
}

View File

@ -25,7 +25,7 @@
`info`,
create_by,
create_time,
( SELECT nick_name FROM sys_user WHERE sys_user.username = dataset_table.create_by ) AS creator_name
(SELECT nick_name FROM sys_user WHERE sys_user.username = dataset_table.create_by) AS creator_name
from dataset_table
where id = #{datasetTableId}
</select>
@ -147,4 +147,13 @@
FROM panel_view
WHERE panel_id = #{panelId}))
</select>
<select id="findByTableIds" resultMap="BaseResultMapDTO">
select dataset_table.*
from dataset_table
where dataset_table.id in
<foreach collection="tableIds" item="item" open="(" separator="," close=")">
#{item}
</foreach>
</select>
</mapper>

View File

@ -1,8 +1,8 @@
package io.dataease.ext;
import io.dataease.ext.query.GridExample;
import io.dataease.dto.dataset.DataSetTaskDTO;
import io.dataease.dto.dataset.DataSetTaskLogDTO;
import io.dataease.ext.query.GridExample;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
@ -25,4 +25,6 @@ public interface ExtDataSetTaskMapper {
List<DataSetTaskDTO> taskWithTriggers(GridExample example);
List<DataSetTaskDTO> findByPanelId(@Param("panelId") String panelId);
List<DataSetTaskDTO> findByTableIds(@Param("tableIds") List tableIds);
}

View File

@ -8,19 +8,21 @@
<result column="dataset_name" jdbcType="VARCHAR" property="datasetName"/>
</resultMap>
<resultMap id="TaskResult" type="io.dataease.dto.dataset.DataSetTaskDTO" extends="io.dataease.plugins.common.base.mapper.DatasetTableTaskMapper.BaseResultMap">
<resultMap id="TaskResult" type="io.dataease.dto.dataset.DataSetTaskDTO"
extends="io.dataease.plugins.common.base.mapper.DatasetTableTaskMapper.BaseResultMap">
<result column="table_name" jdbcType="VARCHAR" property="datasetName"/>
<result column="privileges" jdbcType="VARCHAR" property="privileges"/>
<result column="NEXT_FIRE_TIME" jdbcType="BIGINT" property="nextExecTime"/>
</resultMap>
<select id="listTaskLog" resultMap="BaseResult" parameterType="io.dataease.plugins.common.base.domain.DatasetTableTaskLog">
<select id="listTaskLog" resultMap="BaseResult"
parameterType="io.dataease.plugins.common.base.domain.DatasetTableTaskLog">
SELECT dataset_table_task_log.*, dataset_table_task.name, dataset_table.name as dataset_name
FROM dataset_table_task_log
LEFT JOIN dataset_table_task ON dataset_table_task_log.task_id = dataset_table_task.id
LEFT JOIN dataset_table ON dataset_table_task_log.table_id = dataset_table.id
<if test="_parameter != null">
<include refid="io.dataease.ext.query.GridSql.gridCondition" />
<include refid="io.dataease.ext.query.GridSql.gridCondition"/>
</if>
<if test="orderByClause != null">
order by ${orderByClause}
@ -30,13 +32,14 @@
</if>
</select>
<select id="listUserTaskLog" resultMap="BaseResult" parameterType="io.dataease.plugins.common.base.domain.DatasetTableTaskLog">
<select id="listUserTaskLog" resultMap="BaseResult"
parameterType="io.dataease.plugins.common.base.domain.DatasetTableTaskLog">
SELECT dataset_table_task_log.*, dataset_table_task.name, dataset_table.name as dataset_name
FROM dataset_table_task_log
LEFT JOIN dataset_table_task ON dataset_table_task_log.task_id = dataset_table_task.id
LEFT JOIN dataset_table ON dataset_table_task_log.table_id = dataset_table.id
<if test="_parameter != null">
<include refid="io.dataease.ext.query.GridSql.taskListGridCondition" />
<include refid="io.dataease.ext.query.GridSql.taskListGridCondition"/>
</if>
<if test="orderByClause != null">
order by ${orderByClause}
@ -47,12 +50,13 @@
</select>
<select id="taskList" resultMap="TaskResult" parameterType="io.dataease.ext.query.GridExample">
SELECT dataset_table.name as table_name, 'grant,manage,use' as `privileges`,dataset_table_task.* , qrtz_triggers.NEXT_FIRE_TIME
SELECT dataset_table.name as table_name, 'grant,manage,use' as `privileges`,dataset_table_task.* ,
qrtz_triggers.NEXT_FIRE_TIME
FROM dataset_table_task
left join dataset_table on dataset_table.id=dataset_table_task.table_id
left join qrtz_triggers on dataset_table_task.id=qrtz_triggers.TRIGGER_NAME
<if test="_parameter != null">
<include refid="io.dataease.ext.query.GridSql.gridCondition" />
<include refid="io.dataease.ext.query.GridSql.gridCondition"/>
</if>
<if test="orderByClause != null">
order by ${orderByClause}
@ -63,12 +67,13 @@
</select>
<select id="userTaskList" resultMap="TaskResult" parameterType="io.dataease.ext.query.GridExample">
SELECT dataset_table.name as table_name, get_auths(dataset_table_task.table_id,'dataset', #{extendCondition}) as `privileges`,dataset_table_task.* , qrtz_triggers.NEXT_FIRE_TIME
SELECT dataset_table.name as table_name, get_auths(dataset_table_task.table_id,'dataset', #{extendCondition}) as
`privileges`,dataset_table_task.* , qrtz_triggers.NEXT_FIRE_TIME
FROM dataset_table_task
left join dataset_table on dataset_table.id=dataset_table_task.table_id
left join qrtz_triggers on dataset_table_task.id=qrtz_triggers.TRIGGER_NAME
<if test="_parameter != null">
<include refid="io.dataease.ext.query.GridSql.taskListGridCondition" />
<include refid="io.dataease.ext.query.GridSql.taskListGridCondition"/>
</if>
<if test="orderByClause != null">
order by ${orderByClause}
@ -79,12 +84,13 @@
</select>
<select id="taskWithTriggers" resultMap="TaskResult" parameterType="io.dataease.ext.query.GridExample">
SELECT dataset_table.name as table_name, get_auths(dataset_table_task.table_id,'dataset', #{extendCondition}) as `privileges`,dataset_table_task.* , qrtz_triggers.NEXT_FIRE_TIME
SELECT dataset_table.name as table_name, get_auths(dataset_table_task.table_id,'dataset', #{extendCondition}) as
`privileges`,dataset_table_task.* , qrtz_triggers.NEXT_FIRE_TIME
FROM dataset_table_task
left join dataset_table on dataset_table.id=dataset_table_task.table_id
left join qrtz_triggers on dataset_table_task.id=qrtz_triggers.TRIGGER_NAME
<if test="_parameter != null">
<include refid="io.dataease.ext.query.GridSql.gridCondition" />
<include refid="io.dataease.ext.query.GridSql.gridCondition"/>
</if>
<if test="orderByClause != null">
order by ${orderByClause}
@ -95,26 +101,23 @@
</select>
<select id="findByPanelId" resultMap="TaskResult">
select
dataset_table_task.*
select dataset_table_task.*
from dataset_table_task
where id in (
SELECT
table_id
FROM
chart_view
WHERE
id IN (
SELECT
chart_view_id
FROM
panel_view
WHERE
panel_id = #{panelId}
)
)
where id in (SELECT table_id
FROM chart_view
WHERE id IN (SELECT chart_view_id
FROM panel_view
WHERE panel_id = #{panelId}))
</select>
<select id="findByTableIds" resultMap="TaskResult">
select dataset_table_task.*
from dataset_table_task
where dataset_table_task.table_id in
<foreach collection="tableIds" item="item" open="(" separator="," close=")">
#{item}
</foreach>
</select>
</mapper>

View File

@ -1,8 +1,8 @@
package io.dataease.ext;
import io.dataease.ext.query.GridExample;
import io.dataease.controller.request.DatasourceUnionRequest;
import io.dataease.dto.DatasourceDTO;
import io.dataease.ext.query.GridExample;
import org.apache.ibatis.annotations.Param;
import java.util.List;
@ -15,7 +15,9 @@ public interface ExtDataSourceMapper {
List<DatasourceDTO> findByPanelId(@Param("panelId") String panelId);
DatasourceDTO queryDetails(@Param("datasourceId") String datasourceId,@Param("userId") String userId);
List<DatasourceDTO> findByTableIds(@Param("tableIds") List<String> tableIds);
DatasourceDTO queryDetails(@Param("datasourceId") String datasourceId, @Param("userId") String userId);
}

View File

@ -130,6 +130,19 @@
WHERE panel_view.panel_id = #{panelId}
</select>
<select id="findByTableIds" resultMap="BaseResultMapDTO">
SELECT DISTINCT datasource.id,
datasource.`name`,
datasource.DESC,
datasource.type
FROM dataset_table
INNER JOIN datasource ON dataset_table.data_source_id = datasource.id
WHERE dataset_table.id in
<foreach collection="tableIds" item="item" open="(" separator="," close=")">
#{item}
</foreach>
</select>
<select id="queryDetails" resultMap="BaseResultMapDTO">
select datasource.*,
get_auths(id, 'link', #{userId}) as `privileges`

View File

@ -116,4 +116,8 @@ public class DataSetTableFieldsService {
public void delete(String id) {
datasetTableFieldMapper.deleteByPrimaryKey(id);
}
public void updateByPrimaryKeySelective(DatasetTableField request) {
datasetTableFieldMapper.updateByPrimaryKeySelective(request);
}
}

View File

@ -5,6 +5,7 @@ import io.dataease.commons.constants.CommonConstants;
import io.dataease.commons.constants.PanelConstants;
import io.dataease.commons.utils.AuthUtils;
import io.dataease.commons.utils.BeanUtils;
import io.dataease.commons.utils.TableUtils;
import io.dataease.controller.datasource.request.UpdataDsRequest;
import io.dataease.controller.request.dataset.DataSetTableRequest;
import io.dataease.controller.request.panel.PanelAppTemplateApplyRequest;
@ -197,28 +198,44 @@ public class PanelAppTemplateService {
}
@Transactional(rollbackFor = Exception.class)
public Map<String, String> applyDatasetField(List<DatasetTableField> datasetTableFieldsInfo, Map<String, String> datasetsRealMap) {
public Map<String, String> applyDatasetField(List<DatasetTableField> datasetTableFieldsInfo, Map<String, String> datasetsRealMap, Map<String, String> datasetTypeRealMap, Map<String, String> datasetFieldsMd5FormatRealMap) {
Map<String, String> datasetFieldsRealMap = new HashMap<>();
for (DatasetTableField datasetTableField : datasetTableFieldsInfo) {
if (datasetTableField.getExtField() != 2) {
String oldId = datasetTableField.getId();
datasetTableField.setTableId(datasetsRealMap.get(datasetTableField.getTableId()));
String oldTableId = datasetTableField.getTableId();
datasetTableField.setTableId(datasetsRealMap.get(oldTableId));
datasetTableField.setId(null);
DatasetTableField newTableField = dataSetTableFieldsService.save(datasetTableField);
datasetFieldsRealMap.put(oldId, newTableField.getId());
datasetFieldsMd5FormatRealMap.put(TableUtils.fieldNameShort(oldTableId + "_" + datasetTableField.getOriginName()), TableUtils.fieldNameShort(newTableField.getTableId() + "_" + datasetTableField.getOriginName()));
System.out.println();
}
}
//数据集计算字段替换
for (DatasetTableField datasetTableField : datasetTableFieldsInfo) {
if (datasetTableField.getExtField() == 2) {
String oldId = datasetTableField.getId();
datasetTableField.setTableId(datasetsRealMap.get(datasetTableField.getTableId()));
String oldTableId = datasetTableField.getTableId();
String oldOriginName = datasetTableField.getOriginName();
datasetTableField.setTableId(datasetsRealMap.get(oldTableId));
datasetTableField.setId(null);
datasetFieldsRealMap.forEach((k, v) -> {
datasetTableField.setOriginName(datasetTableField.getOriginName().replaceAll(k, v));
});
DatasetTableField newTableField = dataSetTableFieldsService.save(datasetTableField);
datasetFieldsRealMap.put(oldId, newTableField.getId());
datasetFieldsMd5FormatRealMap.put(TableUtils.fieldNameShort(oldTableId + "_" + oldOriginName), TableUtils.fieldNameShort(newTableField.getTableId() + "_" + datasetTableField.getOriginName()));
}
}
//custom union originName替换
for (DatasetTableField datasetTableField : datasetTableFieldsInfo) {
if (DatasetType.UNION.name().equalsIgnoreCase(datasetTypeRealMap.get(datasetTableField.getTableId())) || DatasetType.CUSTOM.name().equalsIgnoreCase(datasetTypeRealMap.get(datasetTableField.getTableId()))) {
DatasetTableField updateField = new DatasetTableField();
updateField.setId(datasetTableField.getId());
updateField.setOriginName(datasetFieldsMd5FormatRealMap.get(datasetTableField.getOriginName()));
dataSetTableFieldsService.updateByPrimaryKeySelective(updateField);
}
}
return datasetFieldsRealMap;
@ -248,7 +265,7 @@ public class PanelAppTemplateService {
}
@Transactional(rollbackFor = Exception.class)
public Map<String, String> applyViews(List<ChartViewWithBLOBs> chartViewsInfo, Map<String, String> datasetsRealMap, Map<String, String> datasetFieldsRealMap, String sceneId) throws Exception {
public Map<String, String> applyViews(List<ChartViewWithBLOBs> chartViewsInfo, Map<String, String> datasetsRealMap, Map<String, String> datasetFieldsRealMap, Map<String, String> datasetFieldsMd5FormatRealMap, String sceneId) throws Exception {
Map<String, String> chartViewsRealMap = new HashMap<>();
for (ChartViewWithBLOBs chartView : chartViewsInfo) {
String oldViewId = chartView.getId();
@ -279,6 +296,19 @@ public class PanelAppTemplateService {
chartView.setCustomFilter(chartView.getCustomFilter().replaceAll(k, v));
chartView.setDrillFields(chartView.getDrillFields().replaceAll(k, v));
});
//替换originName
datasetFieldsMd5FormatRealMap.forEach((k, v) -> {
chartView.setXAxis(chartView.getXAxis().replaceAll(k, v));
chartView.setXAxisExt(chartView.getXAxisExt().replaceAll(k, v));
chartView.setYAxis(chartView.getYAxis().replaceAll(k, v));
chartView.setYAxisExt(chartView.getYAxisExt().replaceAll(k, v));
chartView.setExtStack(chartView.getExtStack().replaceAll(k, v));
chartView.setExtBubble(chartView.getExtBubble().replaceAll(k, v));
chartView.setCustomAttr(chartView.getCustomAttr().replaceAll(k, v));
chartView.setCustomStyle(chartView.getCustomStyle().replaceAll(k, v));
chartView.setCustomFilter(chartView.getCustomFilter().replaceAll(k, v));
chartView.setDrillFields(chartView.getDrillFields().replaceAll(k, v));
});
chartView.setId(null);
chartView.setSceneId(sceneId);
ChartViewWithBLOBs newOne = chartViewService.newOne(chartView);

View File

@ -18,6 +18,7 @@ import io.dataease.dto.chart.ChartViewDTO;
import io.dataease.dto.dataset.DataSetGroupDTO;
import io.dataease.dto.dataset.DataSetTableDTO;
import io.dataease.dto.dataset.DataSetTaskDTO;
import io.dataease.dto.dataset.DataTableInfoDTO;
import io.dataease.dto.panel.PanelExport2App;
import io.dataease.dto.panel.PanelGroupDTO;
import io.dataease.dto.panel.PanelTemplateFileDTO;
@ -808,12 +809,42 @@ public class PanelGroupService {
List<ChartViewField> chartViewFieldsInfo = extChartViewFieldMapper.findByPanelId(panelId);
//3.获取所有数据集信息
List<DatasetTable> datasetTablesInfo = extDataSetTableMapper.findByPanelId(panelId);
List<String> attachTableIds = new ArrayList<>();
if (CollectionUtils.isNotEmpty(datasetTablesInfo)) {
for (DatasetTable datasetTable : datasetTablesInfo) {
if ("union".equals(datasetTable.getType()) && StringUtils.isNotEmpty(datasetTable.getInfo())) {
DataTableInfoDTO dt = gson.fromJson(datasetTable.getInfo(), DataTableInfoDTO.class);
DatasetUtils.getUnionTable(attachTableIds, dt.getUnion());
} else if ("custom".equals(datasetTable.getType()) && StringUtils.isNotEmpty(datasetTable.getInfo())) {
Map result = gson.fromJson(datasetTable.getInfo(), Map.class);
List<Map> list = (List<Map>) result.get("list");
if (CollectionUtils.isNotEmpty(list)) {
for (Map details : list) {
attachTableIds.add(String.valueOf(details.get("tableId")));
}
}
}
}
if (CollectionUtils.isNotEmpty(attachTableIds)) {
List<DatasetTable> attachDatasetTables = extDataSetTableMapper.findByTableIds(attachTableIds);
if (CollectionUtils.isNotEmpty(attachDatasetTables)) {
datasetTablesInfo.addAll(attachDatasetTables);
}
}
}
// dataset check
if (CollectionUtils.isEmpty(datasetTablesInfo)) {
return new PanelExport2App(Translator.get("I18N_APP_NO_DATASET_ERROR"));
} else if (datasetTablesInfo.stream().filter(datasetTable -> datasetTable.getType().equals("excel") || datasetTable.getType().equals("api")).collect(Collectors.toList()).size() > 0) {
return new PanelExport2App(Translator.get("I18N_APP_ERROR_DATASET"));
}
List<String> allTableIds = datasetTablesInfo.stream().map(DatasetTable::getId).collect(Collectors.toList());
//4.获取所有数据集字段信息
List<DatasetTableField> datasetTableFieldsInfo = extDataSetTableFieldMapper.findByPanelId(panelId);
List<DatasetTableField> datasetTableFieldsInfo = extDataSetTableFieldMapper.findByTableIds(allTableIds);
//5.获取所有任务信息
List<DataSetTaskDTO> dataSetTasksInfo = extDataSetTaskMapper.findByPanelId(panelId);
List<DataSetTaskDTO> dataSetTasksInfo = extDataSetTaskMapper.findByTableIds(allTableIds);
//6.获取所有数据源信息
List<DatasourceDTO> datasourceDTOS = extDataSourceMapper.findByPanelId(panelId);
List<DatasourceDTO> datasourceDTOS = extDataSourceMapper.findByTableIds(allTableIds);
List<PanelView> panelViews = panelViewService.findPanelViewsByPanelId(panelId);
@ -825,13 +856,6 @@ public class PanelGroupService {
return new PanelExport2App(Translator.get("I18N_APP_TEMPLATE_VIEW_ERROR"));
}
// dataset check
if (CollectionUtils.isEmpty(datasetTablesInfo)) {
return new PanelExport2App(Translator.get("I18N_APP_NO_DATASET_ERROR"));
} else if (datasetTablesInfo.stream().filter(datasetTable -> datasetTable.getType().equals("excel") || datasetTable.getType().equals("api")).collect(Collectors.toList()).size() > 0) {
return new PanelExport2App(Translator.get("I18N_APP_ERROR_DATASET"));
}
//datasource check
if (CollectionUtils.isEmpty(datasourceDTOS)) {
return new PanelExport2App(Translator.get("I18N_APP_NO_DATASOURCE"));
@ -885,11 +909,15 @@ public class PanelGroupService {
Map<String, String> datasetsRealMap = panelAppTemplateService.applyDataset(datasetTablesInfo, datasourceRealMap, asideDatasetGroupId);
Map<String, String> datasetFieldsRealMap = panelAppTemplateService.applyDatasetField(datasetTableFieldsInfo, datasetsRealMap);
Map<String, String> datasetTypeRealMap = datasetTablesInfo.stream().collect(Collectors.toMap(DatasetTable::getId, DatasetTable::getType));
Map<String, String> datasetFieldsMd5FormatRealMap = new HashMap<>();
Map<String, String> datasetFieldsRealMap = panelAppTemplateService.applyDatasetField(datasetTableFieldsInfo, datasetsRealMap, datasetTypeRealMap, datasetFieldsMd5FormatRealMap);
panelAppTemplateService.resetCustomAndUnionDataset(datasetTablesInfo, datasetsRealMap, datasetFieldsRealMap);
Map<String, String> chartViewsRealMap = panelAppTemplateService.applyViews(chartViewsInfo, datasetsRealMap, datasetFieldsRealMap, newPanelId);
Map<String, String> chartViewsRealMap = panelAppTemplateService.applyViews(chartViewsInfo, datasetsRealMap, datasetFieldsRealMap, datasetFieldsMd5FormatRealMap, newPanelId);
panelAppTemplateService.applyViewsField(chartViewFieldsInfo, chartViewsRealMap, datasetsRealMap, datasetFieldsRealMap);