center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

feat(数据集): 关联数据集

Browse Source
This commit is contained in:
junjie 2021-12-02 14:58:42 +08:00
parent 70cdd22f87
commit acb54573cb
18 changed files with 653 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backend/src/main/java/io/dataease/commons/utils/DorisTableUtils.java
View File

@ -22,6 +22,10 @@ public class DorisTableUtils {
return "f_" + Md5Utils.md5(dorisName);
}
public static String dorisFieldNameShort(String dorisName) {
return "f_" + Md5Utils.md5(dorisName).substring(8, 24);
}
public static String columnName(String filedName) {
return "C_" + Md5Utils.md5(filedName);
}

12
backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java
View File

@ -133,9 +133,9 @@ public class DataSetTableController {
@ApiOperation("excel上传")
@PostMapping("excel/upload")
@ApiImplicitParams({
@ApiImplicitParam(name = "file", value = "文件", required = true, dataType = "MultipartFile"),
@ApiImplicitParam(name = "tableId", value = "数据表ID", required = true, dataType = "String"),
@ApiImplicitParam(name = "editType", value = "编辑类型", required = true, dataType = "Integer")
@ApiImplicitParam(name = "file", value = "文件", required = true, dataType = "MultipartFile"),
@ApiImplicitParam(name = "tableId", value = "数据表ID", required = true, dataType = "String"),
@ApiImplicitParam(name = "editType", value = "编辑类型", required = true, dataType = "Integer")
})
public ExcelFileData excelUpload(@RequestParam("file") MultipartFile file, @RequestParam("tableId") String tableId, @RequestParam("editType") Integer editType) throws Exception {
return dataSetTableService.excelSaveAndParse(file, tableId, editType);
@ -158,4 +158,10 @@ public class DataSetTableController {
public DatasetTable syncDatasetTableField(@PathVariable String id) throws Exception {
return dataSetTableService.syncDatasetTableField(id);
}
@ApiOperation("关联数据集预览数据")
@PostMapping("unionPreview")
public Map<String, Object> unionPreview(@RequestBody DataSetTableRequest dataSetTableRequest) throws Exception {
return dataSetTableService.getUnionPreview(dataSetTableRequest);
}
}

4
backend/src/main/java/io/dataease/dto/dataset/DataTableInfoDTO.java
View File

@ -1,5 +1,6 @@
package io.dataease.dto.dataset;
import io.dataease.dto.dataset.union.UnionDTO;
import lombok.Getter;
import lombok.Setter;
@ -16,5 +17,6 @@ public class DataTableInfoDTO {
private String sql;
private List<ExcelSheetData> excelSheetDataList;
private String data;// file path
private List<DataTableInfoCustomUnion> list;
private List<DataTableInfoCustomUnion> list;// 自定义数据集
private List<UnionDTO> union;// 关联数据集
}

19
backend/src/main/java/io/dataease/dto/dataset/union/UnionDTO.java Normal file
View File

@ -0,0 +1,19 @@
package io.dataease.dto.dataset.union;
import io.dataease.base.domain.DatasetTable;
import lombok.Data;
import java.util.List;
/**
* @Author gin
* @Date 2021/12/1 3:48 下午
*/
@Data
public class UnionDTO {
private DatasetTable currentDs;
private List<String> currentDsField;
private List<UnionDTO> childrenDs;
private UnionParamDTO unionToParent;
private int allChildCount;
}

14
backend/src/main/java/io/dataease/dto/dataset/union/UnionItemDTO.java Normal file
View File

@ -0,0 +1,14 @@
package io.dataease.dto.dataset.union;
import io.dataease.base.domain.DatasetTableField;
import lombok.Data;
/**
* @Author gin
* @Date 2021/12/1 3:54 下午
*/
@Data
public class UnionItemDTO {
private DatasetTableField parentField;
private DatasetTableField currentField;
}

15
backend/src/main/java/io/dataease/dto/dataset/union/UnionParamDTO.java Normal file
View File

@ -0,0 +1,15 @@
package io.dataease.dto.dataset.union;
import lombok.Data;
import java.util.List;
/**
* @Author gin
* @Date 2021/12/1 3:53 下午
*/
@Data
public class UnionParamDTO {
private String unionType;
private List<UnionItemDTO> unionFields;
}

29
backend/src/main/java/io/dataease/service/chart/ChartViewService.java
View File

@ -13,21 +13,21 @@ import io.dataease.commons.utils.BeanUtils;
import io.dataease.commons.utils.CommonBeanFactory;
import io.dataease.commons.utils.LogUtil;
import io.dataease.controller.request.chart.*;
import io.dataease.controller.request.datasource.DatasourceRequest;
import io.dataease.controller.response.ChartDetail;
import io.dataease.controller.response.DataSetDetail;
import io.dataease.provider.datasource.DatasourceProvider;
import io.dataease.provider.ProviderFactory;
import io.dataease.controller.request.datasource.DatasourceRequest;
import io.dataease.service.datasource.DatasourceService;
import io.dataease.dto.chart.*;
import io.dataease.dto.dataset.DataSetTableUnionDTO;
import io.dataease.dto.dataset.DataTableInfoDTO;
import io.dataease.i18n.Translator;
import io.dataease.listener.util.CacheUtils;
import io.dataease.provider.ProviderFactory;
import io.dataease.provider.datasource.DatasourceProvider;
import io.dataease.provider.query.QueryProvider;
import io.dataease.service.dataset.DataSetTableFieldsService;
import io.dataease.service.dataset.DataSetTableService;
import io.dataease.service.dataset.DataSetTableUnionService;
import io.dataease.service.datasource.DatasourceService;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -162,9 +162,10 @@ public class ChartViewService {
}
public ChartViewDTO getOneWithPermission(String id) {
String userId = AuthUtils.getUser()!=null?String.valueOf(AuthUtils.getUser().getUserId()):"NONE";
return extChartViewMapper.searchOneWithPrivileges(userId,id);
String userId = AuthUtils.getUser() != null ? String.valueOf(AuthUtils.getUser().getUserId()) : "NONE";
return extChartViewMapper.searchOneWithPrivileges(userId, id);
}
public void delete(String id) {
chartViewMapper.deleteByPrimaryKey(id);
}
@ -365,6 +366,22 @@ public class ChartViewService {
DataTableInfoDTO dt = new Gson().fromJson(table.getInfo(), DataTableInfoDTO.class);
List<DataSetTableUnionDTO> list = dataSetTableUnionService.listByTableId(dt.getList().get(0).getTableId());
String sql = dataSetTableService.getCustomSQLDatasource(dt, list, ds);
if (StringUtils.equalsIgnoreCase("text", view.getType()) || StringUtils.equalsIgnoreCase("gauge", view.getType()) || StringUtils.equalsIgnoreCase("liquid", view.getType())) {
datasourceRequest.setQuery(qp.getSQLSummaryAsTmp(sql, yAxis, customFilter, extFilterList, view));
} else if (StringUtils.containsIgnoreCase(view.getType(), "stack")) {
datasourceRequest.setQuery(qp.getSQLAsTmpStack(sql, xAxis, yAxis, customFilter, extFilterList, extStack, view));
} else if (StringUtils.containsIgnoreCase(view.getType(), "scatter")) {
datasourceRequest.setQuery(qp.getSQLAsTmpScatter(sql, xAxis, yAxis, customFilter, extFilterList, extBubble, view));
} else if (StringUtils.equalsIgnoreCase("table-info", view.getType())) {
datasourceRequest.setQuery(qp.getSQLAsTmpTableInfo(sql, xAxis, customFilter, extFilterList, ds, view));
} else {
datasourceRequest.setQuery(qp.getSQLAsTmp(sql, xAxis, yAxis, customFilter, extFilterList, view));
}
} else if (StringUtils.equalsIgnoreCase(table.getType(), "union")) {
DataTableInfoDTO dt = new Gson().fromJson(table.getInfo(), DataTableInfoDTO.class);
Map<String, Object> sqlMap = dataSetTableService.getUnionSQLDatasource(dt, ds);
String sql = (String) sqlMap.get("sql");
if (StringUtils.equalsIgnoreCase("text", view.getType()) || StringUtils.equalsIgnoreCase("gauge", view.getType()) || StringUtils.equalsIgnoreCase("liquid", view.getType())) {
datasourceRequest.setQuery(qp.getSQLSummaryAsTmp(sql, yAxis, customFilter, extFilterList, view));
} else if (StringUtils.containsIgnoreCase(view.getType(), "stack")) {

417
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@ -19,6 +19,9 @@ import io.dataease.controller.request.dataset.DataSetTaskRequest;
import io.dataease.controller.request.datasource.DatasourceRequest;
import io.dataease.controller.response.DataSetDetail;
import io.dataease.dto.dataset.*;
import io.dataease.dto.dataset.union.UnionDTO;
import io.dataease.dto.dataset.union.UnionItemDTO;
import io.dataease.dto.dataset.union.UnionParamDTO;
import io.dataease.dto.datasource.TableFiled;
import io.dataease.exception.DataEaseException;
import io.dataease.i18n.Translator;
@ -237,7 +240,9 @@ public class DataSetTableService {
if (datasetTable.getIsRename() == null || !datasetTable.getIsRename()) {
// 更新数据和字段
if (update == 1) {
if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "sql") || StringUtils.equalsIgnoreCase(datasetTable.getType(), "custom")) {
if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "sql")
|| StringUtils.equalsIgnoreCase(datasetTable.getType(), "custom")
|| StringUtils.equalsIgnoreCase(datasetTable.getType(), "union")) {
saveTableField(datasetTable);
}
}
@ -660,6 +665,72 @@ public class DataSetTableService {
DEException.throwException(Translator.get("i18n_ds_error"));
}
try {
datasourceRequest.setQuery(qp.createQueryTableWithLimit(table, fields, Integer.valueOf(dataSetTableRequest.getRow()), false, ds));
dataSetPreviewPage.setTotal(jdbcProvider.getData(datasourceRequest).size());
} catch (Exception e) {
logger.error(e.getMessage());
DEException.throwException(Translator.get("i18n_ds_error"));
}
}
} else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "union")) {
if (datasetTable.getMode() == 0) {
Datasource ds = datasourceMapper.selectByPrimaryKey(dataSetTableRequest.getDataSourceId());
if (ObjectUtils.isEmpty(ds)) {
DEException.throwException(Translator.get("i18n_datasource_delete"));
}
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
DataTableInfoDTO dt = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class);
String sql = "";
try {
sql = (String) getUnionSQLDatasource(dt, ds).get("sql");
} catch (Exception e) {
logger.error(e.getMessage());
DEException.throwException(Translator.get("i18n_ds_error"));
}
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(qp.createQuerySQLWithPage(sql, fields, page, pageSize, realSize, false));
map.put("sql", datasourceRequest.getQuery());
datasourceRequest.setPage(page);
datasourceRequest.setFetchSize(Integer.parseInt(dataSetTableRequest.getRow()));
datasourceRequest.setPageSize(pageSize);
datasourceRequest.setRealSize(realSize);
datasourceRequest.setPreviewData(true);
try {
datasourceRequest.setPageable(true);
data.addAll(datasourceProvider.getData(datasourceRequest));
} catch (Exception e) {
logger.error(e.getMessage());
DEException.throwException(Translator.get("i18n_ds_error"));
}
try {
datasourceRequest.setPageable(false);
datasourceRequest.setQuery(qp.createQuerySqlWithLimit(sql, fields, Integer.valueOf(dataSetTableRequest.getRow()), false));
dataSetPreviewPage.setTotal(datasourceProvider.getData(datasourceRequest).size());
} catch (Exception e) {
logger.error(e.getMessage());
DEException.throwException(Translator.get("i18n_ds_error"));
}
} else {
Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds));
map.put("sql", datasourceRequest.getQuery());
try {
data.addAll(jdbcProvider.getData(datasourceRequest));
} catch (Exception e) {
logger.error(e.getMessage());
DEException.throwException(Translator.get("i18n_ds_error"));
}
try {
datasourceRequest.setQuery(qp.createQueryTableWithLimit(table, fields, Integer.valueOf(dataSetTableRequest.getRow()), false, ds));
dataSetPreviewPage.setTotal(jdbcProvider.getData(datasourceRequest).size());
@ -727,6 +798,70 @@ public class DataSetTableService {
return map;
}
public Map<String, Object> getUnionPreview(DataSetTableRequest dataSetTableRequest) throws Exception {
DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class);
Map<String, Object> sqlMap = new HashMap<>();
DatasourceRequest datasourceRequest = new DatasourceRequest();
Datasource ds;
if (dataSetTableRequest.getMode() == 0) {
ds = datasourceMapper.selectByPrimaryKey(dataSetTableRequest.getDataSourceId());
datasourceRequest.setDatasource(ds);
sqlMap = getUnionSQLDatasource(dataTableInfoDTO, ds);
} else {
ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
datasourceRequest.setDatasource(ds);
sqlMap = getUnionSQLDoris(dataTableInfoDTO);
}
String sql = (String) sqlMap.get("sql");
List<DatasetTableField> fieldList = (List<DatasetTableField>) sqlMap.get("field");
List<UnionParamDTO> join = (List<UnionParamDTO>) sqlMap.get("join");
// 处理结果
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
// 使用输入的sql先预执行一次,并拿到所有字段
datasourceRequest.setQuery(sql);
Map<String, Object> res = new HashMap<>();
try {
List<TableFiled> previewFields = datasourceProvider.fetchResultField(datasourceRequest);
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(qp.createSQLPreview(sql, previewFields.get(0).getFieldName()));
Map<String, List> result = datasourceProvider.fetchResultAndField(datasourceRequest);
List<String[]> data = result.get("dataList");
List<TableFiled> fields = result.get("fieldList");
String[] fieldArray = fields.stream().map(TableFiled::getFieldName).toArray(String[]::new);
List<Map<String, Object>> jsonArray = new ArrayList<>();
if (CollectionUtils.isNotEmpty(data)) {
jsonArray = data.stream().map(ele -> {
Map<String, Object> map = new HashMap<>();
for (int i = 0; i < ele.length; i++) {
map.put(fieldArray[i], ele[i]);
}
return map;
}).collect(Collectors.toList());
}
// 获取每个字段在当前de数据库中的name，作为sql查询后的remarks返回前端展示
for (DatasetTableField datasetTableField : fieldList) {
for (TableFiled tableFiled : fields) {
if (StringUtils.equalsIgnoreCase(tableFiled.getFieldName(), DorisTableUtils.dorisFieldName(datasetTableField.getTableId() + "_" + datasetTableField.getDataeaseName()))
|| StringUtils.equalsIgnoreCase(tableFiled.getFieldName(), DorisTableUtils.dorisFieldNameShort(datasetTableField.getTableId() + "_" + datasetTableField.getOriginName()))) {
tableFiled.setRemarks(datasetTableField.getName());
break;
}
}
}
res.put("fields", fields);
res.put("data", jsonArray);
return res;
} catch (Exception e) {
return res;
}
}
public Map<String, Object> getCustomPreview(DataSetTableRequest dataSetTableRequest) throws Exception {
DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class);
List<DataSetTableUnionDTO> list = dataSetTableUnionService.listByTableId(dataTableInfoDTO.getList().get(0).getTableId());
@ -915,18 +1050,249 @@ public class DataSetTableService {
private String convertUnionTypeToSQL(String unionType) {
switch (unionType) {
case "1:1":
case "inner":
return " INNER JOIN ";
case "1:N":
case "left":
return " LEFT JOIN ";
case "N:1":
case "right":
return " RIGHT JOIN ";
case "N:N":
case "full":
return " FULL JOIN ";
default:
return " INNER JOIN ";
}
}
// 关联数据集从doris里预览数据
private Map<String, Object> getUnionSQLDoris(DataTableInfoDTO dataTableInfoDTO) {
List<UnionDTO> union = dataTableInfoDTO.getUnion();
// 所有选中的字段，即select后的查询字段
Map<String, String[]> checkedInfo = new TreeMap<>();
List<UnionParamDTO> unionList = new ArrayList<>();
List<DatasetTableField> checkedFields = new ArrayList<>();
String sql = "";
for (UnionDTO unionDTO : union) {
// doris 使用数据集id做表名，拼sql将用到该名称
String tableId = unionDTO.getCurrentDs().getId();
String table = DorisTableUtils.dorisName(tableId);
DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(tableId);
if (ObjectUtils.isEmpty(datasetTable)) {
DEException.throwException(Translator.get("i18n_custom_ds_delete") + String.format(":table id [%s]", tableId));
}
List<DatasetTableField> fields = dataSetTableFieldsService.getListByIdsEach(unionDTO.getCurrentDsField());
if (CollectionUtils.isEmpty(fields)) {
DEException.throwException(Translator.get("i18n_cst_ds_tb_or_field_deleted") + String.format(":table id [%s]", tableId));
}
String[] array = fields.stream().map(f -> table + "." + f.getDataeaseName() + " AS " + DorisTableUtils.dorisFieldName(tableId + "_" + f.getDataeaseName())).toArray(String[]::new);
checkedInfo.put(table, array);
checkedFields.addAll(fields);
// 获取child的fields和union
if (CollectionUtils.isNotEmpty(unionDTO.getChildrenDs())) {
getUnionSQLDorisJoin(unionDTO.getChildrenDs(), checkedInfo, unionList, checkedFields);
}
}
// build sql
if (CollectionUtils.isNotEmpty(unionList)) {
// field
StringBuilder field = new StringBuilder();
Iterator<Map.Entry<String, String[]>> iterator = checkedInfo.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, String[]> next = iterator.next();
field.append(StringUtils.join(next.getValue(), ",")).append(",");
}
String f = field.substring(0, field.length() - 1);
// join
StringBuilder join = new StringBuilder();
for (UnionParamDTO unionParamDTO : unionList) {
String joinType = convertUnionTypeToSQL(unionParamDTO.getUnionType());
UnionItemDTO u = unionParamDTO.getUnionFields().get(0);
DatasetTableField pField = dataSetTableFieldsService.get(u.getParentField().getId());
DatasetTableField cField = dataSetTableFieldsService.get(u.getCurrentField().getId());
if (ObjectUtils.isEmpty(pField) || ObjectUtils.isEmpty(cField)) {
DEException.throwException(Translator.get("i18n_dataset_field_delete"));
}
DatasetTable parentTable = datasetTableMapper.selectByPrimaryKey(pField.getTableId());
DatasetTable currentTable = datasetTableMapper.selectByPrimaryKey(cField.getTableId());
join.append(" ").append(joinType).append(" ").append(DorisTableUtils.dorisName(currentTable.getId()))
.append(" ON ");
for (int i = 0; i < unionParamDTO.getUnionFields().size(); i++) {
UnionItemDTO unionItemDTO = unionParamDTO.getUnionFields().get(i);
// 通过field id取得field详情，并且以第一组为准，寻找dataset table
DatasetTableField parentField = dataSetTableFieldsService.get(unionItemDTO.getParentField().getId());
DatasetTableField currentField = dataSetTableFieldsService.get(unionItemDTO.getCurrentField().getId());
join.append(DorisTableUtils.dorisName(parentTable.getId())).append(".").append(parentField.getDataeaseName())
.append(" = ")
.append(DorisTableUtils.dorisName(currentTable.getId())).append(".").append(currentField.getDataeaseName());
if (i < unionParamDTO.getUnionFields().size() - 1) {
join.append(" AND ");
}
}
}
if (StringUtils.isEmpty(f)) {
DEException.throwException(Translator.get("i18n_custom_ds_delete"));
}
sql = MessageFormat.format("SELECT {0} FROM {1}", f, DorisTableUtils.dorisName(union.get(0).getCurrentDs().getId())) + join.toString();
} else {
String f = StringUtils.join(checkedInfo.get(DorisTableUtils.dorisName(union.get(0).getCurrentDs().getId())), ",");
if (StringUtils.isEmpty(f)) {
throw new RuntimeException(Translator.get("i18n_custom_ds_delete"));
}
sql = MessageFormat.format("SELECT {0} FROM {1}", f, DorisTableUtils.dorisName(union.get(0).getCurrentDs().getId()));
}
Map<String, Object> map = new HashMap<>();
map.put("sql", sql);
map.put("field", checkedFields);
map.put("join", unionList);
return map;
}
// 递归计算出所有子级的checkedFields和unionParam
private void getUnionSQLDorisJoin(List<UnionDTO> childrenDs, Map<String, String[]> checkedInfo, List<UnionParamDTO> unionList, List<DatasetTableField> checkedFields) {
for (int i = 0; i < childrenDs.size(); i++) {
UnionDTO unionDTO = childrenDs.get(i);
String tableId = unionDTO.getCurrentDs().getId();
String table = DorisTableUtils.dorisName(tableId);
DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(tableId);
if (ObjectUtils.isEmpty(datasetTable)) {
DEException.throwException(Translator.get("i18n_custom_ds_delete") + String.format(":table id [%s]", tableId));
}
List<DatasetTableField> fields = dataSetTableFieldsService.getListByIdsEach(unionDTO.getCurrentDsField());
if (CollectionUtils.isEmpty(fields)) {
DEException.throwException(Translator.get("i18n_cst_ds_tb_or_field_deleted") + String.format(":table id [%s]", tableId));
}
String[] array = fields.stream().map(f -> table + "." + f.getDataeaseName() + " AS " + DorisTableUtils.dorisFieldName(tableId + "_" + f.getDataeaseName())).toArray(String[]::new);
checkedInfo.put(table, array);
checkedFields.addAll(fields);
unionList.add(unionDTO.getUnionToParent());
if (CollectionUtils.isNotEmpty(unionDTO.getChildrenDs())) {
getUnionSQLDorisJoin(unionDTO.getChildrenDs(), checkedInfo, unionList, checkedFields);
}
}
}
// 关联数据集 直连模式
public Map<String, Object> getUnionSQLDatasource(DataTableInfoDTO dataTableInfoDTO, Datasource ds) {
DatasourceTypes datasourceTypes = DatasourceTypes.valueOf(ds.getType());
String keyword = datasourceTypes.getKeywordPrefix() + "%s" + datasourceTypes.getKeywordSuffix();
List<UnionDTO> union = dataTableInfoDTO.getUnion();
// 所有选中的字段，即select后的查询字段
Map<String, String[]> checkedInfo = new TreeMap<>();
List<UnionParamDTO> unionList = new ArrayList<>();
List<DatasetTableField> checkedFields = new ArrayList<>();
String sql = "";
String tableName = new Gson().fromJson(datasetTableMapper.selectByPrimaryKey(union.get(0).getCurrentDs().getId()).getInfo(), DataTableInfoDTO.class).getTable();
for (UnionDTO unionDTO : union) {
DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(unionDTO.getCurrentDs().getId());
String table = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable();
String tableId = unionDTO.getCurrentDs().getId();
if (ObjectUtils.isEmpty(datasetTable)) {
DEException.throwException(Translator.get("i18n_custom_ds_delete") + String.format(":table id [%s]", tableId));
}
List<DatasetTableField> fields = dataSetTableFieldsService.getListByIdsEach(unionDTO.getCurrentDsField());
if (CollectionUtils.isEmpty(fields)) {
DEException.throwException(Translator.get("i18n_cst_ds_tb_or_field_deleted") + String.format(":table id [%s]", tableId));
}
String[] array = fields.stream().map(f -> String.format(keyword, table) + "." + String.format(keyword, f.getOriginName()) + " AS " + DorisTableUtils.dorisFieldNameShort(tableId + "_" + f.getOriginName())).toArray(String[]::new);
checkedInfo.put(table, array);
checkedFields.addAll(fields);
// 获取child的fields和union
if (CollectionUtils.isNotEmpty(unionDTO.getChildrenDs())) {
getUnionSQLDatasourceJoin(unionDTO.getChildrenDs(), checkedInfo, unionList, keyword, checkedFields);
}
}
// build sql
if (CollectionUtils.isNotEmpty(unionList)) {
// field
StringBuilder field = new StringBuilder();
Iterator<Map.Entry<String, String[]>> iterator = checkedInfo.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, String[]> next = iterator.next();
field.append(StringUtils.join(next.getValue(), ",")).append(",");
}
String f = field.substring(0, field.length() - 1);
// join
StringBuilder join = new StringBuilder();
for (UnionParamDTO unionParamDTO : unionList) {
String joinType = convertUnionTypeToSQL(unionParamDTO.getUnionType());
UnionItemDTO u = unionParamDTO.getUnionFields().get(0);
DatasetTableField pField = dataSetTableFieldsService.get(u.getParentField().getId());
DatasetTableField cField = dataSetTableFieldsService.get(u.getCurrentField().getId());
if (ObjectUtils.isEmpty(pField) || ObjectUtils.isEmpty(cField)) {
DEException.throwException(Translator.get("i18n_dataset_field_delete"));
}
DatasetTable parentTable = datasetTableMapper.selectByPrimaryKey(pField.getTableId());
String parentTableName = new Gson().fromJson(parentTable.getInfo(), DataTableInfoDTO.class).getTable();
DatasetTable currentTable = datasetTableMapper.selectByPrimaryKey(cField.getTableId());
String currentTableName = new Gson().fromJson(currentTable.getInfo(), DataTableInfoDTO.class).getTable();
join.append(" ").append(joinType).append(" ").append(String.format(keyword, currentTableName))
.append(" ON ");
for (int i = 0; i < unionParamDTO.getUnionFields().size(); i++) {
UnionItemDTO unionItemDTO = unionParamDTO.getUnionFields().get(i);
// 通过field id取得field详情，并且以第一组为准，寻找dataset table
DatasetTableField parentField = dataSetTableFieldsService.get(unionItemDTO.getParentField().getId());
DatasetTableField currentField = dataSetTableFieldsService.get(unionItemDTO.getCurrentField().getId());
join.append(String.format(keyword, parentTableName)).append(".").append(parentField.getOriginName())
.append(" = ")
.append(String.format(keyword, currentTableName)).append(".").append(currentField.getOriginName());
if (i < unionParamDTO.getUnionFields().size() - 1) {
join.append(" AND ");
}
}
}
if (StringUtils.isEmpty(f)) {
DEException.throwException(Translator.get("i18n_custom_ds_delete"));
}
sql = MessageFormat.format("SELECT {0} FROM {1}", f, String.format(keyword, tableName)) + join.toString();
} else {
String f = StringUtils.join(checkedInfo.get(tableName), ",");
if (StringUtils.isEmpty(f)) {
throw new RuntimeException(Translator.get("i18n_custom_ds_delete"));
}
sql = MessageFormat.format("SELECT {0} FROM {1}", f, String.format(keyword, tableName));
}
Map<String, Object> map = new HashMap<>();
map.put("sql", sql);
map.put("field", checkedFields);
map.put("join", unionList);
return map;
}
// 递归计算出所有子级的checkedFields和unionParam
private void getUnionSQLDatasourceJoin(List<UnionDTO> childrenDs, Map<String, String[]> checkedInfo, List<UnionParamDTO> unionList, String keyword, List<DatasetTableField> checkedFields) {
for (int i = 0; i < childrenDs.size(); i++) {
UnionDTO unionDTO = childrenDs.get(i);
DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(unionDTO.getCurrentDs().getId());
String table = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable();
String tableId = unionDTO.getCurrentDs().getId();
if (ObjectUtils.isEmpty(datasetTable)) {
DEException.throwException(Translator.get("i18n_custom_ds_delete") + String.format(":table id [%s]", tableId));
}
List<DatasetTableField> fields = dataSetTableFieldsService.getListByIdsEach(unionDTO.getCurrentDsField());
if (CollectionUtils.isEmpty(fields)) {
DEException.throwException(Translator.get("i18n_cst_ds_tb_or_field_deleted") + String.format(":table id [%s]", tableId));
}
String[] array = fields.stream().map(f -> String.format(keyword, table) + "." + String.format(keyword, f.getOriginName()) + " AS " + DorisTableUtils.dorisFieldNameShort(tableId + "_" + f.getOriginName())).toArray(String[]::new);
checkedInfo.put(table, array);
checkedFields.addAll(fields);
unionList.add(unionDTO.getUnionToParent());
if (CollectionUtils.isNotEmpty(unionDTO.getChildrenDs())) {
getUnionSQLDatasourceJoin(unionDTO.getChildrenDs(), checkedInfo, unionList, keyword, checkedFields);
}
}
}
public List<DatasetTableField> saveExcelTableField(String datasetTableId, List<TableFiled> fields, boolean insert) {
List<DatasetTableField> datasetTableFields = new ArrayList<>();
if (CollectionUtils.isNotEmpty(fields)) {
@ -1024,6 +1390,55 @@ public class DataSetTableService {
}
}
}
} else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "union")) {
if (datasetTable.getMode() == 1) {
ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
// save field
DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class);
Map<String, Object> sqlMap = getUnionSQLDoris(dataTableInfoDTO);
String sql = (String) sqlMap.get("sql");
List<DatasetTableField> fieldList = (List<DatasetTableField>) sqlMap.get("field");
List<UnionParamDTO> join = (List<UnionParamDTO>) sqlMap.get("join");
// custom 创建doris视图
createDorisView(DorisTableUtils.dorisName(datasetTable.getId()), sql);
datasourceRequest.setQuery(sql);
fields = datasourceProvider.fetchResultField(datasourceRequest);
for (DatasetTableField field : fieldList) {
for (TableFiled tableFiled : fields) {
if (StringUtils.equalsIgnoreCase(DorisTableUtils.dorisFieldName(field.getTableId() + "_" + field.getDataeaseName()), tableFiled.getFieldName())) {
tableFiled.setRemarks(field.getName());
break;
}
}
}
} else {
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
DataTableInfoDTO dt = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class);
Map<String, Object> sqlMap = getUnionSQLDatasource(dt, ds);
String sql = (String) sqlMap.get("sql");
List<DatasetTableField> fieldList = (List<DatasetTableField>) sqlMap.get("field");
List<UnionParamDTO> join = (List<UnionParamDTO>) sqlMap.get("join");
datasourceRequest.setQuery(sql);
fields = datasourceProvider.fetchResultField(datasourceRequest);
for (DatasetTableField field : fieldList) {
for (TableFiled tableFiled : fields) {
if (StringUtils.equalsIgnoreCase(DorisTableUtils.dorisFieldNameShort(field.getTableId() + "_" + field.getOriginName()), tableFiled.getFieldName())) {
tableFiled.setRemarks(field.getName());
break;
}
}
}
}
}
QueryProvider qp = null;
if (!ObjectUtils.isEmpty(ds)) {

3
frontend/src/lang/en.js
View File

@ -1163,7 +1163,8 @@ export default {
field_select: 'Select Field',
add_union_field: 'Add Union Field',
union_error: 'Union relation and field can not be empty',
union_repeat: 'This dataset is already union，do not union repeat'
union_repeat: 'This dataset is already union，do not union repeat',
preview_result: 'Preview'
},
datasource: {
datasource: 'Data Source',

3
frontend/src/lang/tw.js
View File

@ -1164,7 +1164,8 @@ export default {
field_select: '字段選擇',
add_union_field: '添加關聯字段',
union_error: '關聯關系與關聯字段不能為空',
union_repeat: '當前數據集已被關聯，請勿重復關聯'
union_repeat: '當前數據集已被關聯，請勿重復關聯',
preview_result: '預覽結果'
},
datasource: {
datasource: '數據源',

3
frontend/src/lang/zh.js
View File

@ -1166,7 +1166,8 @@ export default {
field_select: '字段选择',
add_union_field: '添加关联字段',
union_error: '关联关系与关联字段不能为空',
union_repeat: '当前数据集已被关联，请勿重复关联'
union_repeat: '当前数据集已被关联，请勿重复关联',
preview_result: '预览结果'
},
datasource: {
datasource: '数据源',

47
frontend/src/views/dataset/add/AddUnion.vue
View File

@ -15,10 +15,15 @@
</el-row>
<el-divider />
<div>
<el-form :inline="true">
<el-form :inline="true" style="display: flex;align-items: center;justify-content: space-between;">
<el-form-item class="form-item">
<el-input v-model="name" size="mini" :placeholder="$t('commons.name')" clearable />
</el-form-item>
<el-form-item class="form-item">
<el-button size="mini" @click="previewData">
{{ $t('dataset.preview_result') }}
</el-button>
</el-form-item>
</el-form>
<!--添加第一个数据集按钮-->
<div v-if="dataset.length === 0">
@ -70,6 +75,11 @@
<el-button type="primary" size="mini" @click="confirmEditUnion()">{{ $t('dataset.confirm') }}</el-button>
</div>
</el-dialog>
<!--数据预览界面-->
<el-drawer v-if="showPreview" :title="$t('dataset.preview_result')" :visible.sync="showPreview" direction="btt" class="preview-style">
<union-preview :table="previewTable" :dataset="dataset" />
</el-drawer>
</el-row>
</template>
@ -79,9 +89,10 @@ import NodeItem from '@/views/dataset/add/union/NodeItem'
import DatasetGroupSelectorTree from '@/views/dataset/common/DatasetGroupSelectorTree'
import UnionEdit from '@/views/dataset/add/union/UnionEdit'
import { getTable, post } from '@/api/dataset/dataset'
import UnionPreview from '@/views/dataset/add/union/UnionPreview'
export default {
name: 'AddUnion',
components: { UnionEdit, DatasetGroupSelectorTree, NodeItem, UnionNode },
components: { UnionPreview, UnionEdit, DatasetGroupSelectorTree, NodeItem, UnionNode },
props: {
param: {
type: Object,
@ -133,7 +144,9 @@ export default {
// 弹框临时选中的数据集
tempDs: {},
editUnion: false,
unionParam: {}
unionParam: {},
showPreview: false,
previewTable: {}
}
},
watch: {
@ -169,7 +182,7 @@ export default {
dataSourceId: this.dataset[0].currentDs.dataSourceId,
type: 'union',
mode: this.dataset[0].currentDs.mode,
info: '{"list":' + JSON.stringify(this.dataset) + '}'
info: '{"union":' + JSON.stringify(this.dataset) + '}'
}
post('/dataset/table/update', table).then(response => {
this.$emit('saveSuccess', table)
@ -270,9 +283,22 @@ export default {
getTable(this.param.tableId).then(response => {
const table = JSON.parse(JSON.stringify(response.data))
this.name = table.name
this.dataset = JSON.parse(table.info).list
this.dataset = JSON.parse(table.info).union
})
}
},
previewData() {
this.previewTable = {
id: this.param.tableId,
name: this.name,
sceneId: this.param.id,
dataSourceId: this.dataset[0].currentDs.dataSourceId,
type: 'union',
mode: this.dataset[0].currentDs.mode,
info: '{"union":' + JSON.stringify(this.dataset) + '}'
}
this.showPreview = true
}
}
}
@ -294,4 +320,15 @@ export default {
.dialog-css >>> .el-dialog__body {
padding: 0 20px;
}
.preview-style >>> .el-drawer{
height: 50%!important;
}
.preview-style >>> .el-drawer .el-drawer__header{
margin-bottom: 10px!important;
padding: 10px 16px 0!important;
font-size: 14px;
}
.preview-style >>> .el-drawer .el-drawer__body{
padding: 0 16px 10px!important;
}
</style>

94
frontend/src/views/dataset/add/union/UnionPreview.vue Normal file
View File

@ -0,0 +1,94 @@
<template>
<div>
<div class="text item">
<ux-grid
ref="plxTable"
size="mini"
style="width: 100%;"
:height="height"
:checkbox-config="{highlight: true}"
:width-resize="true"
>
<ux-table-column
v-for="field in fields"
:key="field.fieldName"
min-width="200px"
:field="field.fieldName"
:title="field.remarks"
:resizable="true"
/>
</ux-grid>
</div>
<span class="table-count">
{{ $t('dataset.preview_show') }}
<span class="span-number">1000</span>
{{ $t('dataset.preview_item') }}
</span>
</div>
</template>
<script>
import { post } from '@/api/dataset/dataset'
export default {
name: 'UnionPreview',
props: {
table: {
type: Object,
required: true
},
dataset: {
type: Array,
required: true
}
},
data() {
return {
height: 'auto',
fields: [],
data: []
}
},
watch: {
'table': function() {
this.initPreview()
}
},
mounted() {
this.initHeight()
this.initPreview()
},
methods: {
initHeight() {
this.height = (document.getElementsByClassName('el-drawer__body')[0].clientHeight - 40) + 'px'
},
initPreview() {
if (this.dataset && this.dataset.length > 0) {
post('/dataset/table/unionPreview', this.table).then(response => {
this.fields = response.data.fields
this.data = response.data.data
const datas = this.data
this.$refs.plxTable.reloadData(datas)
})
} else {
this.fields = []
this.data = []
const datas = this.data
this.$refs.plxTable.reloadData(datas)
}
}
}
}
</script>
<style scoped>
.span-number{
color: #0a7be0;
}
.table-count{
color: #606266;
}
span{
font-size: 12px;
}
</style>

1
frontend/src/views/dataset/common/DatasetGroupSelectorTree.vue
View File

@ -61,6 +61,7 @@
<svg-icon v-if="data.modelInnerType === 'sql'" icon-class="ds-sql" class="ds-icon-sql" />
<svg-icon v-if="data.modelInnerType === 'excel'" icon-class="ds-excel" class="ds-icon-excel" />
<svg-icon v-if="data.modelInnerType === 'custom'" icon-class="ds-custom" class="ds-icon-custom" />
<svg-icon v-if="data.modelInnerType === 'union'" icon-class="ds-union" class="ds-icon-union" />
</span>
<span v-if="data.modelInnerType === 'db' || data.modelInnerType === 'sql'">
<span v-if="data.mode === 0" style="margin-left: 6px"><i class="el-icon-s-operation" /></span>

6
frontend/src/views/dataset/data/FieldEdit.vue
View File

@ -15,7 +15,7 @@
<el-form :inline="true">
<el-form-item class="form-item">
<el-button v-if="hasDataPermission('manage',param.privileges)" size="mini" icon="el-icon-circle-plus-outline" @click="addCalcField">{{ $t('dataset.add_calc_field') }}</el-button>
<el-button v-if="hasDataPermission('manage',param.privileges) && table.type !== 'excel' && table.type !== 'custom'" size="mini" :loading="isSyncField" icon="el-icon-refresh-left" @click="syncField">{{ $t('dataset.sync_field') }}</el-button>
<el-button v-if="hasDataPermission('manage',param.privileges) && table.type !== 'excel' && table.type !== 'custom' && table.type !== 'union'" size="mini" :loading="isSyncField" icon="el-icon-refresh-left" @click="syncField">{{ $t('dataset.sync_field') }}</el-button>
</el-form-item>
<el-form-item class="form-item" style="float: right;margin-right: 0;">
<el-input
@ -43,7 +43,7 @@
<el-input v-model="scope.row.name" size="mini" :disabled="!hasDataPermission('manage',param.privileges)" @blur="saveEdit(scope.row)" @keyup.enter.native="saveEdit(scope.row)" />
</template>
</el-table-column>
<el-table-column v-if="!(param.mode === 0 && param.type === 'custom')" property="originName" :label="$t('dataset.field_origin_name')" width="100">
<el-table-column v-if="!(table.mode === 0 && (table.type === 'custom' || table.type === 'union'))" property="originName" :label="$t('dataset.field_origin_name')" width="100">
<template slot-scope="scope">
<span v-if="scope.row.extField === 0" :title="scope.row.originName" class="field-class" style="width: 100%;white-space: nowrap;overflow: hidden;text-overflow: ellipsis;">
<span style="font-size: 12px;">{{ scope.row.originName }}</span>
@ -149,7 +149,7 @@
<el-input v-model="scope.row.name" size="mini" :disabled="!hasDataPermission('manage',param.privileges)" @blur="saveEdit(scope.row)" @keyup.enter.native="saveEdit(scope.row)" />
</template>
</el-table-column>
<el-table-column v-if="!(param.mode === 0 && param.type === 'custom')" property="originName" :label="$t('dataset.field_origin_name')" width="100">
<el-table-column v-if="!(table.mode === 0 && (table.type === 'custom' || table.type === 'union'))" property="originName" :label="$t('dataset.field_origin_name')" width="100">
<template slot-scope="scope">
<span v-if="scope.row.extField === 0" :title="scope.row.originName" class="field-class" style="width: 100%;white-space: nowrap;overflow: hidden;text-overflow: ellipsis;">
<span style="font-size: 12px;">{{ scope.row.originName }}</span>

2
frontend/src/views/dataset/data/TabDataPreview.vue
View File

@ -37,7 +37,7 @@
</ux-table-column>
</ux-grid>
<el-row style="margin-top: 4px;">
<span v-if="table.type === 'excel' || table.type === 'custom'" class="table-count">
<span v-if="table.type === 'excel' || table.type === 'custom' || table.type === 'union'" class="table-count">
<span v-if="page.total <= currentPage.show">
{{ $t('dataset.preview_total') }}
<span class="span-number">{{ page.total }}</span>

3
frontend/src/views/dataset/data/ViewTable.vue
View File

@ -56,9 +56,6 @@
<el-tab-pane :label="$t('dataset.field_manage')" name="fieldEdit">
<field-edit :param="param" :table="table" />
</el-tab-pane>
<el-tab-pane v-if="table.type !== 'custom' && !(table.type === 'sql' && table.mode === 0)" :label="$t('dataset.join_view')" name="joinView">
<union-view :param="param" :table="table" />
</el-tab-pane>
<el-tab-pane v-if="table.mode === 1 && (table.type === 'excel' || table.type === 'db' || table.type === 'sql')" :label="$t('dataset.update_info')" name="updateInfo">
<update-info v-if="tabActive=='updateInfo'" :param="param" :table="table" />
</el-tab-pane>

6
frontend/src/views/dataset/group/Group.vue
View File

@ -89,9 +89,9 @@
<svg-icon icon-class="ds-excel" class="ds-icon-excel" />
{{ $t('dataset.excel_data') }}
</el-dropdown-item>
<el-dropdown-item :command="beforeClickAddData('custom',data)">
<svg-icon icon-class="ds-custom" class="ds-icon-custom" />
{{ $t('dataset.custom_data') }}
<el-dropdown-item :command="beforeClickAddData('union',data)">
<svg-icon icon-class="ds-union" class="ds-icon-union" />
{{ $t('dataset.union_data') }}
</el-dropdown-item>
</el-dropdown-menu>
</el-dropdown>