From acb54573cb3d79e90f9db279c717541fa1a1ec3e Mon Sep 17 00:00:00 2001 From: junjie Date: Thu, 2 Dec 2021 14:58:42 +0800 Subject: [PATCH] =?UTF-8?q?feat(=E6=95=B0=E6=8D=AE=E9=9B=86):=20=E5=85=B3?= =?UTF-8?q?=E8=81=94=E6=95=B0=E6=8D=AE=E9=9B=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../commons/utils/DorisTableUtils.java | 4 + .../dataset/DataSetTableController.java | 12 +- .../dto/dataset/DataTableInfoDTO.java | 4 +- .../dataease/dto/dataset/union/UnionDTO.java | 19 + .../dto/dataset/union/UnionItemDTO.java | 14 + .../dto/dataset/union/UnionParamDTO.java | 15 + .../service/chart/ChartViewService.java | 29 +- .../service/dataset/DataSetTableService.java | 417 +++++++++++++++++- frontend/src/lang/en.js | 3 +- frontend/src/lang/tw.js | 3 +- frontend/src/lang/zh.js | 3 +- frontend/src/views/dataset/add/AddUnion.vue | 47 +- .../views/dataset/add/union/UnionPreview.vue | 94 ++++ .../common/DatasetGroupSelectorTree.vue | 1 + frontend/src/views/dataset/data/FieldEdit.vue | 6 +- .../src/views/dataset/data/TabDataPreview.vue | 2 +- frontend/src/views/dataset/data/ViewTable.vue | 3 - frontend/src/views/dataset/group/Group.vue | 6 +- 18 files changed, 653 insertions(+), 29 deletions(-) create mode 100644 backend/src/main/java/io/dataease/dto/dataset/union/UnionDTO.java create mode 100644 backend/src/main/java/io/dataease/dto/dataset/union/UnionItemDTO.java create mode 100644 backend/src/main/java/io/dataease/dto/dataset/union/UnionParamDTO.java create mode 100644 frontend/src/views/dataset/add/union/UnionPreview.vue diff --git a/backend/src/main/java/io/dataease/commons/utils/DorisTableUtils.java b/backend/src/main/java/io/dataease/commons/utils/DorisTableUtils.java index 970078deed..154b5aadb4 100644 --- a/backend/src/main/java/io/dataease/commons/utils/DorisTableUtils.java +++ b/backend/src/main/java/io/dataease/commons/utils/DorisTableUtils.java @@ -22,6 +22,10 @@ public class DorisTableUtils { return "f_" + Md5Utils.md5(dorisName); } + public static String dorisFieldNameShort(String dorisName) { + return "f_" + Md5Utils.md5(dorisName).substring(8, 24); + } + public static String columnName(String filedName) { return "C_" + Md5Utils.md5(filedName); } diff --git a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java index 13bf1f1f11..c75cef5f5e 100644 --- a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java +++ b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java @@ -133,9 +133,9 @@ public class DataSetTableController { @ApiOperation("excel上传") @PostMapping("excel/upload") @ApiImplicitParams({ - @ApiImplicitParam(name = "file", value = "文件", required = true, dataType = "MultipartFile"), - @ApiImplicitParam(name = "tableId", value = "数据表ID", required = true, dataType = "String"), - @ApiImplicitParam(name = "editType", value = "编辑类型", required = true, dataType = "Integer") + @ApiImplicitParam(name = "file", value = "文件", required = true, dataType = "MultipartFile"), + @ApiImplicitParam(name = "tableId", value = "数据表ID", required = true, dataType = "String"), + @ApiImplicitParam(name = "editType", value = "编辑类型", required = true, dataType = "Integer") }) public ExcelFileData excelUpload(@RequestParam("file") MultipartFile file, @RequestParam("tableId") String tableId, @RequestParam("editType") Integer editType) throws Exception { return dataSetTableService.excelSaveAndParse(file, tableId, editType); @@ -158,4 +158,10 @@ public class DataSetTableController { public DatasetTable syncDatasetTableField(@PathVariable String id) throws Exception { return dataSetTableService.syncDatasetTableField(id); } + + @ApiOperation("关联数据集预览数据") + @PostMapping("unionPreview") + public Map unionPreview(@RequestBody DataSetTableRequest dataSetTableRequest) throws Exception { + return dataSetTableService.getUnionPreview(dataSetTableRequest); + } } diff --git a/backend/src/main/java/io/dataease/dto/dataset/DataTableInfoDTO.java b/backend/src/main/java/io/dataease/dto/dataset/DataTableInfoDTO.java index 1eed9391d1..00f2eea13f 100644 --- a/backend/src/main/java/io/dataease/dto/dataset/DataTableInfoDTO.java +++ b/backend/src/main/java/io/dataease/dto/dataset/DataTableInfoDTO.java @@ -1,5 +1,6 @@ package io.dataease.dto.dataset; +import io.dataease.dto.dataset.union.UnionDTO; import lombok.Getter; import lombok.Setter; @@ -16,5 +17,6 @@ public class DataTableInfoDTO { private String sql; private List excelSheetDataList; private String data;// file path - private List list; + private List list;// 自定义数据集 + private List union;// 关联数据集 } diff --git a/backend/src/main/java/io/dataease/dto/dataset/union/UnionDTO.java b/backend/src/main/java/io/dataease/dto/dataset/union/UnionDTO.java new file mode 100644 index 0000000000..aa289b412a --- /dev/null +++ b/backend/src/main/java/io/dataease/dto/dataset/union/UnionDTO.java @@ -0,0 +1,19 @@ +package io.dataease.dto.dataset.union; + +import io.dataease.base.domain.DatasetTable; +import lombok.Data; + +import java.util.List; + +/** + * @Author gin + * @Date 2021/12/1 3:48 下午 + */ +@Data +public class UnionDTO { + private DatasetTable currentDs; + private List currentDsField; + private List childrenDs; + private UnionParamDTO unionToParent; + private int allChildCount; +} diff --git a/backend/src/main/java/io/dataease/dto/dataset/union/UnionItemDTO.java b/backend/src/main/java/io/dataease/dto/dataset/union/UnionItemDTO.java new file mode 100644 index 0000000000..cea88e9f54 --- /dev/null +++ b/backend/src/main/java/io/dataease/dto/dataset/union/UnionItemDTO.java @@ -0,0 +1,14 @@ +package io.dataease.dto.dataset.union; + +import io.dataease.base.domain.DatasetTableField; +import lombok.Data; + +/** + * @Author gin + * @Date 2021/12/1 3:54 下午 + */ +@Data +public class UnionItemDTO { + private DatasetTableField parentField; + private DatasetTableField currentField; +} diff --git a/backend/src/main/java/io/dataease/dto/dataset/union/UnionParamDTO.java b/backend/src/main/java/io/dataease/dto/dataset/union/UnionParamDTO.java new file mode 100644 index 0000000000..a5bbc2bdae --- /dev/null +++ b/backend/src/main/java/io/dataease/dto/dataset/union/UnionParamDTO.java @@ -0,0 +1,15 @@ +package io.dataease.dto.dataset.union; + +import lombok.Data; + +import java.util.List; + +/** + * @Author gin + * @Date 2021/12/1 3:53 下午 + */ +@Data +public class UnionParamDTO { + private String unionType; + private List unionFields; +} diff --git a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java index 3f2ae4d72d..d90a240ebb 100644 --- a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java +++ b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java @@ -13,21 +13,21 @@ import io.dataease.commons.utils.BeanUtils; import io.dataease.commons.utils.CommonBeanFactory; import io.dataease.commons.utils.LogUtil; import io.dataease.controller.request.chart.*; +import io.dataease.controller.request.datasource.DatasourceRequest; import io.dataease.controller.response.ChartDetail; import io.dataease.controller.response.DataSetDetail; -import io.dataease.provider.datasource.DatasourceProvider; -import io.dataease.provider.ProviderFactory; -import io.dataease.controller.request.datasource.DatasourceRequest; -import io.dataease.service.datasource.DatasourceService; import io.dataease.dto.chart.*; import io.dataease.dto.dataset.DataSetTableUnionDTO; import io.dataease.dto.dataset.DataTableInfoDTO; import io.dataease.i18n.Translator; import io.dataease.listener.util.CacheUtils; +import io.dataease.provider.ProviderFactory; +import io.dataease.provider.datasource.DatasourceProvider; import io.dataease.provider.query.QueryProvider; import io.dataease.service.dataset.DataSetTableFieldsService; import io.dataease.service.dataset.DataSetTableService; import io.dataease.service.dataset.DataSetTableUnionService; +import io.dataease.service.datasource.DatasourceService; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -162,9 +162,10 @@ public class ChartViewService { } public ChartViewDTO getOneWithPermission(String id) { - String userId = AuthUtils.getUser()!=null?String.valueOf(AuthUtils.getUser().getUserId()):"NONE"; - return extChartViewMapper.searchOneWithPrivileges(userId,id); + String userId = AuthUtils.getUser() != null ? String.valueOf(AuthUtils.getUser().getUserId()) : "NONE"; + return extChartViewMapper.searchOneWithPrivileges(userId, id); } + public void delete(String id) { chartViewMapper.deleteByPrimaryKey(id); } @@ -365,6 +366,22 @@ public class ChartViewService { DataTableInfoDTO dt = new Gson().fromJson(table.getInfo(), DataTableInfoDTO.class); List list = dataSetTableUnionService.listByTableId(dt.getList().get(0).getTableId()); String sql = dataSetTableService.getCustomSQLDatasource(dt, list, ds); + if (StringUtils.equalsIgnoreCase("text", view.getType()) || StringUtils.equalsIgnoreCase("gauge", view.getType()) || StringUtils.equalsIgnoreCase("liquid", view.getType())) { + datasourceRequest.setQuery(qp.getSQLSummaryAsTmp(sql, yAxis, customFilter, extFilterList, view)); + } else if (StringUtils.containsIgnoreCase(view.getType(), "stack")) { + datasourceRequest.setQuery(qp.getSQLAsTmpStack(sql, xAxis, yAxis, customFilter, extFilterList, extStack, view)); + } else if (StringUtils.containsIgnoreCase(view.getType(), "scatter")) { + datasourceRequest.setQuery(qp.getSQLAsTmpScatter(sql, xAxis, yAxis, customFilter, extFilterList, extBubble, view)); + } else if (StringUtils.equalsIgnoreCase("table-info", view.getType())) { + datasourceRequest.setQuery(qp.getSQLAsTmpTableInfo(sql, xAxis, customFilter, extFilterList, ds, view)); + } else { + datasourceRequest.setQuery(qp.getSQLAsTmp(sql, xAxis, yAxis, customFilter, extFilterList, view)); + } + } else if (StringUtils.equalsIgnoreCase(table.getType(), "union")) { + DataTableInfoDTO dt = new Gson().fromJson(table.getInfo(), DataTableInfoDTO.class); + Map sqlMap = dataSetTableService.getUnionSQLDatasource(dt, ds); + String sql = (String) sqlMap.get("sql"); + if (StringUtils.equalsIgnoreCase("text", view.getType()) || StringUtils.equalsIgnoreCase("gauge", view.getType()) || StringUtils.equalsIgnoreCase("liquid", view.getType())) { datasourceRequest.setQuery(qp.getSQLSummaryAsTmp(sql, yAxis, customFilter, extFilterList, view)); } else if (StringUtils.containsIgnoreCase(view.getType(), "stack")) { diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index 58edb770b0..dfa03a002c 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -19,6 +19,9 @@ import io.dataease.controller.request.dataset.DataSetTaskRequest; import io.dataease.controller.request.datasource.DatasourceRequest; import io.dataease.controller.response.DataSetDetail; import io.dataease.dto.dataset.*; +import io.dataease.dto.dataset.union.UnionDTO; +import io.dataease.dto.dataset.union.UnionItemDTO; +import io.dataease.dto.dataset.union.UnionParamDTO; import io.dataease.dto.datasource.TableFiled; import io.dataease.exception.DataEaseException; import io.dataease.i18n.Translator; @@ -237,7 +240,9 @@ public class DataSetTableService { if (datasetTable.getIsRename() == null || !datasetTable.getIsRename()) { // 更新数据和字段 if (update == 1) { - if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "sql") || StringUtils.equalsIgnoreCase(datasetTable.getType(), "custom")) { + if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "sql") + || StringUtils.equalsIgnoreCase(datasetTable.getType(), "custom") + || StringUtils.equalsIgnoreCase(datasetTable.getType(), "union")) { saveTableField(datasetTable); } } @@ -660,6 +665,72 @@ public class DataSetTableService { DEException.throwException(Translator.get("i18n_ds_error")); } + try { + datasourceRequest.setQuery(qp.createQueryTableWithLimit(table, fields, Integer.valueOf(dataSetTableRequest.getRow()), false, ds)); + dataSetPreviewPage.setTotal(jdbcProvider.getData(datasourceRequest).size()); + } catch (Exception e) { + logger.error(e.getMessage()); + DEException.throwException(Translator.get("i18n_ds_error")); + } + } + } else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "union")) { + if (datasetTable.getMode() == 0) { + Datasource ds = datasourceMapper.selectByPrimaryKey(dataSetTableRequest.getDataSourceId()); + if (ObjectUtils.isEmpty(ds)) { + DEException.throwException(Translator.get("i18n_datasource_delete")); + } + DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(ds); + + DataTableInfoDTO dt = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class); + + String sql = ""; + try { + sql = (String) getUnionSQLDatasource(dt, ds).get("sql"); + } catch (Exception e) { + logger.error(e.getMessage()); + DEException.throwException(Translator.get("i18n_ds_error")); + } + QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); + datasourceRequest.setQuery(qp.createQuerySQLWithPage(sql, fields, page, pageSize, realSize, false)); + map.put("sql", datasourceRequest.getQuery()); + datasourceRequest.setPage(page); + datasourceRequest.setFetchSize(Integer.parseInt(dataSetTableRequest.getRow())); + datasourceRequest.setPageSize(pageSize); + datasourceRequest.setRealSize(realSize); + datasourceRequest.setPreviewData(true); + try { + datasourceRequest.setPageable(true); + data.addAll(datasourceProvider.getData(datasourceRequest)); + } catch (Exception e) { + logger.error(e.getMessage()); + DEException.throwException(Translator.get("i18n_ds_error")); + } + try { + datasourceRequest.setPageable(false); + datasourceRequest.setQuery(qp.createQuerySqlWithLimit(sql, fields, Integer.valueOf(dataSetTableRequest.getRow()), false)); + dataSetPreviewPage.setTotal(datasourceProvider.getData(datasourceRequest).size()); + } catch (Exception e) { + logger.error(e.getMessage()); + DEException.throwException(Translator.get("i18n_ds_error")); + } + } else { + Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + JdbcProvider jdbcProvider = CommonBeanFactory.getBean(JdbcProvider.class); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(ds); + String table = DorisTableUtils.dorisName(dataSetTableRequest.getId()); + QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); + datasourceRequest.setQuery(qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds)); + map.put("sql", datasourceRequest.getQuery()); + try { + data.addAll(jdbcProvider.getData(datasourceRequest)); + } catch (Exception e) { + logger.error(e.getMessage()); + DEException.throwException(Translator.get("i18n_ds_error")); + } + try { datasourceRequest.setQuery(qp.createQueryTableWithLimit(table, fields, Integer.valueOf(dataSetTableRequest.getRow()), false, ds)); dataSetPreviewPage.setTotal(jdbcProvider.getData(datasourceRequest).size()); @@ -727,6 +798,70 @@ public class DataSetTableService { return map; } + public Map getUnionPreview(DataSetTableRequest dataSetTableRequest) throws Exception { + DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class); + Map sqlMap = new HashMap<>(); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + Datasource ds; + if (dataSetTableRequest.getMode() == 0) { + ds = datasourceMapper.selectByPrimaryKey(dataSetTableRequest.getDataSourceId()); + datasourceRequest.setDatasource(ds); + sqlMap = getUnionSQLDatasource(dataTableInfoDTO, ds); + } else { + ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + datasourceRequest.setDatasource(ds); + sqlMap = getUnionSQLDoris(dataTableInfoDTO); + } + String sql = (String) sqlMap.get("sql"); + List fieldList = (List) sqlMap.get("field"); + List join = (List) sqlMap.get("join"); + + // 处理结果 + DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); + // 使用输入的sql先预执行一次,并拿到所有字段 + datasourceRequest.setQuery(sql); + + Map res = new HashMap<>(); + try { + List previewFields = datasourceProvider.fetchResultField(datasourceRequest); + + QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); + datasourceRequest.setQuery(qp.createSQLPreview(sql, previewFields.get(0).getFieldName())); + Map result = datasourceProvider.fetchResultAndField(datasourceRequest); + List data = result.get("dataList"); + List fields = result.get("fieldList"); + String[] fieldArray = fields.stream().map(TableFiled::getFieldName).toArray(String[]::new); + + List> jsonArray = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(data)) { + jsonArray = data.stream().map(ele -> { + Map map = new HashMap<>(); + for (int i = 0; i < ele.length; i++) { + map.put(fieldArray[i], ele[i]); + } + return map; + }).collect(Collectors.toList()); + } + + // 获取每个字段在当前de数据库中的name，作为sql查询后的remarks返回前端展示 + for (DatasetTableField datasetTableField : fieldList) { + for (TableFiled tableFiled : fields) { + if (StringUtils.equalsIgnoreCase(tableFiled.getFieldName(), DorisTableUtils.dorisFieldName(datasetTableField.getTableId() + "_" + datasetTableField.getDataeaseName())) + || StringUtils.equalsIgnoreCase(tableFiled.getFieldName(), DorisTableUtils.dorisFieldNameShort(datasetTableField.getTableId() + "_" + datasetTableField.getOriginName()))) { + tableFiled.setRemarks(datasetTableField.getName()); + break; + } + } + } + + res.put("fields", fields); + res.put("data", jsonArray); + return res; + } catch (Exception e) { + return res; + } + } + public Map getCustomPreview(DataSetTableRequest dataSetTableRequest) throws Exception { DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class); List list = dataSetTableUnionService.listByTableId(dataTableInfoDTO.getList().get(0).getTableId()); @@ -915,18 +1050,249 @@ public class DataSetTableService { private String convertUnionTypeToSQL(String unionType) { switch (unionType) { case "1:1": + case "inner": return " INNER JOIN "; case "1:N": + case "left": return " LEFT JOIN "; case "N:1": + case "right": return " RIGHT JOIN "; case "N:N": + case "full": return " FULL JOIN "; default: return " INNER JOIN "; } } + // 关联数据集从doris里预览数据 + private Map getUnionSQLDoris(DataTableInfoDTO dataTableInfoDTO) { + List union = dataTableInfoDTO.getUnion(); + // 所有选中的字段，即select后的查询字段 + Map checkedInfo = new TreeMap<>(); + List unionList = new ArrayList<>(); + List checkedFields = new ArrayList<>(); + String sql = ""; + for (UnionDTO unionDTO : union) { + // doris 使用数据集id做表名，拼sql将用到该名称 + String tableId = unionDTO.getCurrentDs().getId(); + String table = DorisTableUtils.dorisName(tableId); + DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(tableId); + if (ObjectUtils.isEmpty(datasetTable)) { + DEException.throwException(Translator.get("i18n_custom_ds_delete") + String.format(":table id [%s]", tableId)); + } + List fields = dataSetTableFieldsService.getListByIdsEach(unionDTO.getCurrentDsField()); + if (CollectionUtils.isEmpty(fields)) { + DEException.throwException(Translator.get("i18n_cst_ds_tb_or_field_deleted") + String.format(":table id [%s]", tableId)); + } + String[] array = fields.stream().map(f -> table + "." + f.getDataeaseName() + " AS " + DorisTableUtils.dorisFieldName(tableId + "_" + f.getDataeaseName())).toArray(String[]::new); + checkedInfo.put(table, array); + checkedFields.addAll(fields); + // 获取child的fields和union + if (CollectionUtils.isNotEmpty(unionDTO.getChildrenDs())) { + getUnionSQLDorisJoin(unionDTO.getChildrenDs(), checkedInfo, unionList, checkedFields); + } + } + // build sql + if (CollectionUtils.isNotEmpty(unionList)) { + // field + StringBuilder field = new StringBuilder(); + Iterator> iterator = checkedInfo.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry next = iterator.next(); + field.append(StringUtils.join(next.getValue(), ",")).append(","); + } + String f = field.substring(0, field.length() - 1); + // join + StringBuilder join = new StringBuilder(); + for (UnionParamDTO unionParamDTO : unionList) { + String joinType = convertUnionTypeToSQL(unionParamDTO.getUnionType()); + UnionItemDTO u = unionParamDTO.getUnionFields().get(0); + DatasetTableField pField = dataSetTableFieldsService.get(u.getParentField().getId()); + DatasetTableField cField = dataSetTableFieldsService.get(u.getCurrentField().getId()); + if (ObjectUtils.isEmpty(pField) || ObjectUtils.isEmpty(cField)) { + DEException.throwException(Translator.get("i18n_dataset_field_delete")); + } + DatasetTable parentTable = datasetTableMapper.selectByPrimaryKey(pField.getTableId()); + DatasetTable currentTable = datasetTableMapper.selectByPrimaryKey(cField.getTableId()); + + join.append(" ").append(joinType).append(" ").append(DorisTableUtils.dorisName(currentTable.getId())) + .append(" ON "); + for (int i = 0; i < unionParamDTO.getUnionFields().size(); i++) { + UnionItemDTO unionItemDTO = unionParamDTO.getUnionFields().get(i); + // 通过field id取得field详情，并且以第一组为准，寻找dataset table + DatasetTableField parentField = dataSetTableFieldsService.get(unionItemDTO.getParentField().getId()); + DatasetTableField currentField = dataSetTableFieldsService.get(unionItemDTO.getCurrentField().getId()); + + join.append(DorisTableUtils.dorisName(parentTable.getId())).append(".").append(parentField.getDataeaseName()) + .append(" = ") + .append(DorisTableUtils.dorisName(currentTable.getId())).append(".").append(currentField.getDataeaseName()); + if (i < unionParamDTO.getUnionFields().size() - 1) { + join.append(" AND "); + } + } + } + if (StringUtils.isEmpty(f)) { + DEException.throwException(Translator.get("i18n_custom_ds_delete")); + } + sql = MessageFormat.format("SELECT {0} FROM {1}", f, DorisTableUtils.dorisName(union.get(0).getCurrentDs().getId())) + join.toString(); + } else { + String f = StringUtils.join(checkedInfo.get(DorisTableUtils.dorisName(union.get(0).getCurrentDs().getId())), ","); + if (StringUtils.isEmpty(f)) { + throw new RuntimeException(Translator.get("i18n_custom_ds_delete")); + } + sql = MessageFormat.format("SELECT {0} FROM {1}", f, DorisTableUtils.dorisName(union.get(0).getCurrentDs().getId())); + } + Map map = new HashMap<>(); + map.put("sql", sql); + map.put("field", checkedFields); + map.put("join", unionList); + return map; + } + + // 递归计算出所有子级的checkedFields和unionParam + private void getUnionSQLDorisJoin(List childrenDs, Map checkedInfo, List unionList, List checkedFields) { + for (int i = 0; i < childrenDs.size(); i++) { + UnionDTO unionDTO = childrenDs.get(i); + String tableId = unionDTO.getCurrentDs().getId(); + String table = DorisTableUtils.dorisName(tableId); + DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(tableId); + if (ObjectUtils.isEmpty(datasetTable)) { + DEException.throwException(Translator.get("i18n_custom_ds_delete") + String.format(":table id [%s]", tableId)); + } + List fields = dataSetTableFieldsService.getListByIdsEach(unionDTO.getCurrentDsField()); + if (CollectionUtils.isEmpty(fields)) { + DEException.throwException(Translator.get("i18n_cst_ds_tb_or_field_deleted") + String.format(":table id [%s]", tableId)); + } + String[] array = fields.stream().map(f -> table + "." + f.getDataeaseName() + " AS " + DorisTableUtils.dorisFieldName(tableId + "_" + f.getDataeaseName())).toArray(String[]::new); + checkedInfo.put(table, array); + checkedFields.addAll(fields); + + unionList.add(unionDTO.getUnionToParent()); + if (CollectionUtils.isNotEmpty(unionDTO.getChildrenDs())) { + getUnionSQLDorisJoin(unionDTO.getChildrenDs(), checkedInfo, unionList, checkedFields); + } + } + } + + // 关联数据集 直连模式 + public Map getUnionSQLDatasource(DataTableInfoDTO dataTableInfoDTO, Datasource ds) { + DatasourceTypes datasourceTypes = DatasourceTypes.valueOf(ds.getType()); + String keyword = datasourceTypes.getKeywordPrefix() + "%s" + datasourceTypes.getKeywordSuffix(); + + List union = dataTableInfoDTO.getUnion(); + // 所有选中的字段，即select后的查询字段 + Map checkedInfo = new TreeMap<>(); + List unionList = new ArrayList<>(); + List checkedFields = new ArrayList<>(); + String sql = ""; + String tableName = new Gson().fromJson(datasetTableMapper.selectByPrimaryKey(union.get(0).getCurrentDs().getId()).getInfo(), DataTableInfoDTO.class).getTable(); + for (UnionDTO unionDTO : union) { + DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(unionDTO.getCurrentDs().getId()); + String table = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable(); + String tableId = unionDTO.getCurrentDs().getId(); + if (ObjectUtils.isEmpty(datasetTable)) { + DEException.throwException(Translator.get("i18n_custom_ds_delete") + String.format(":table id [%s]", tableId)); + } + List fields = dataSetTableFieldsService.getListByIdsEach(unionDTO.getCurrentDsField()); + if (CollectionUtils.isEmpty(fields)) { + DEException.throwException(Translator.get("i18n_cst_ds_tb_or_field_deleted") + String.format(":table id [%s]", tableId)); + } + String[] array = fields.stream().map(f -> String.format(keyword, table) + "." + String.format(keyword, f.getOriginName()) + " AS " + DorisTableUtils.dorisFieldNameShort(tableId + "_" + f.getOriginName())).toArray(String[]::new); + checkedInfo.put(table, array); + checkedFields.addAll(fields); + // 获取child的fields和union + if (CollectionUtils.isNotEmpty(unionDTO.getChildrenDs())) { + getUnionSQLDatasourceJoin(unionDTO.getChildrenDs(), checkedInfo, unionList, keyword, checkedFields); + } + } + // build sql + if (CollectionUtils.isNotEmpty(unionList)) { + // field + StringBuilder field = new StringBuilder(); + Iterator> iterator = checkedInfo.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry next = iterator.next(); + field.append(StringUtils.join(next.getValue(), ",")).append(","); + } + String f = field.substring(0, field.length() - 1); + // join + StringBuilder join = new StringBuilder(); + for (UnionParamDTO unionParamDTO : unionList) { + String joinType = convertUnionTypeToSQL(unionParamDTO.getUnionType()); + UnionItemDTO u = unionParamDTO.getUnionFields().get(0); + DatasetTableField pField = dataSetTableFieldsService.get(u.getParentField().getId()); + DatasetTableField cField = dataSetTableFieldsService.get(u.getCurrentField().getId()); + if (ObjectUtils.isEmpty(pField) || ObjectUtils.isEmpty(cField)) { + DEException.throwException(Translator.get("i18n_dataset_field_delete")); + } + DatasetTable parentTable = datasetTableMapper.selectByPrimaryKey(pField.getTableId()); + String parentTableName = new Gson().fromJson(parentTable.getInfo(), DataTableInfoDTO.class).getTable(); + DatasetTable currentTable = datasetTableMapper.selectByPrimaryKey(cField.getTableId()); + String currentTableName = new Gson().fromJson(currentTable.getInfo(), DataTableInfoDTO.class).getTable(); + + join.append(" ").append(joinType).append(" ").append(String.format(keyword, currentTableName)) + .append(" ON "); + for (int i = 0; i < unionParamDTO.getUnionFields().size(); i++) { + UnionItemDTO unionItemDTO = unionParamDTO.getUnionFields().get(i); + // 通过field id取得field详情，并且以第一组为准，寻找dataset table + DatasetTableField parentField = dataSetTableFieldsService.get(unionItemDTO.getParentField().getId()); + DatasetTableField currentField = dataSetTableFieldsService.get(unionItemDTO.getCurrentField().getId()); + + join.append(String.format(keyword, parentTableName)).append(".").append(parentField.getOriginName()) + .append(" = ") + .append(String.format(keyword, currentTableName)).append(".").append(currentField.getOriginName()); + if (i < unionParamDTO.getUnionFields().size() - 1) { + join.append(" AND "); + } + } + } + if (StringUtils.isEmpty(f)) { + DEException.throwException(Translator.get("i18n_custom_ds_delete")); + } + sql = MessageFormat.format("SELECT {0} FROM {1}", f, String.format(keyword, tableName)) + join.toString(); + } else { + String f = StringUtils.join(checkedInfo.get(tableName), ","); + if (StringUtils.isEmpty(f)) { + throw new RuntimeException(Translator.get("i18n_custom_ds_delete")); + } + sql = MessageFormat.format("SELECT {0} FROM {1}", f, String.format(keyword, tableName)); + } + Map map = new HashMap<>(); + map.put("sql", sql); + map.put("field", checkedFields); + map.put("join", unionList); + return map; + } + + // 递归计算出所有子级的checkedFields和unionParam + private void getUnionSQLDatasourceJoin(List childrenDs, Map checkedInfo, List unionList, String keyword, List checkedFields) { + for (int i = 0; i < childrenDs.size(); i++) { + UnionDTO unionDTO = childrenDs.get(i); + + DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(unionDTO.getCurrentDs().getId()); + String table = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable(); + String tableId = unionDTO.getCurrentDs().getId(); + + if (ObjectUtils.isEmpty(datasetTable)) { + DEException.throwException(Translator.get("i18n_custom_ds_delete") + String.format(":table id [%s]", tableId)); + } + List fields = dataSetTableFieldsService.getListByIdsEach(unionDTO.getCurrentDsField()); + if (CollectionUtils.isEmpty(fields)) { + DEException.throwException(Translator.get("i18n_cst_ds_tb_or_field_deleted") + String.format(":table id [%s]", tableId)); + } + String[] array = fields.stream().map(f -> String.format(keyword, table) + "." + String.format(keyword, f.getOriginName()) + " AS " + DorisTableUtils.dorisFieldNameShort(tableId + "_" + f.getOriginName())).toArray(String[]::new); + checkedInfo.put(table, array); + checkedFields.addAll(fields); + + unionList.add(unionDTO.getUnionToParent()); + if (CollectionUtils.isNotEmpty(unionDTO.getChildrenDs())) { + getUnionSQLDatasourceJoin(unionDTO.getChildrenDs(), checkedInfo, unionList, keyword, checkedFields); + } + } + } + public List saveExcelTableField(String datasetTableId, List fields, boolean insert) { List datasetTableFields = new ArrayList<>(); if (CollectionUtils.isNotEmpty(fields)) { @@ -1024,6 +1390,55 @@ public class DataSetTableService { } } } + } else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "union")) { + if (datasetTable.getMode() == 1) { + ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(ds); + // save field + DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class); + Map sqlMap = getUnionSQLDoris(dataTableInfoDTO); + String sql = (String) sqlMap.get("sql"); + List fieldList = (List) sqlMap.get("field"); + List join = (List) sqlMap.get("join"); + + // custom 创建doris视图 + createDorisView(DorisTableUtils.dorisName(datasetTable.getId()), sql); + + datasourceRequest.setQuery(sql); + fields = datasourceProvider.fetchResultField(datasourceRequest); + for (DatasetTableField field : fieldList) { + for (TableFiled tableFiled : fields) { + if (StringUtils.equalsIgnoreCase(DorisTableUtils.dorisFieldName(field.getTableId() + "_" + field.getDataeaseName()), tableFiled.getFieldName())) { + tableFiled.setRemarks(field.getName()); + break; + } + } + } + } else { + DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(ds); + DataTableInfoDTO dt = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class); + + Map sqlMap = getUnionSQLDatasource(dt, ds); + String sql = (String) sqlMap.get("sql"); + List fieldList = (List) sqlMap.get("field"); + List join = (List) sqlMap.get("join"); + + datasourceRequest.setQuery(sql); + fields = datasourceProvider.fetchResultField(datasourceRequest); + + for (DatasetTableField field : fieldList) { + for (TableFiled tableFiled : fields) { + if (StringUtils.equalsIgnoreCase(DorisTableUtils.dorisFieldNameShort(field.getTableId() + "_" + field.getOriginName()), tableFiled.getFieldName())) { + tableFiled.setRemarks(field.getName()); + break; + } + } + } + } } QueryProvider qp = null; if (!ObjectUtils.isEmpty(ds)) { diff --git a/frontend/src/lang/en.js b/frontend/src/lang/en.js index 8ca1dcfea5..41ca099a94 100644 --- a/frontend/src/lang/en.js +++ b/frontend/src/lang/en.js @@ -1163,7 +1163,8 @@ export default { field_select: 'Select Field', add_union_field: 'Add Union Field', union_error: 'Union relation and field can not be empty', - union_repeat: 'This dataset is already union，do not union repeat' + union_repeat: 'This dataset is already union，do not union repeat', + preview_result: 'Preview' }, datasource: { datasource: 'Data Source', diff --git a/frontend/src/lang/tw.js b/frontend/src/lang/tw.js index a555c6ca9f..53ee617b71 100644 --- a/frontend/src/lang/tw.js +++ b/frontend/src/lang/tw.js @@ -1164,7 +1164,8 @@ export default { field_select: '字段選擇', add_union_field: '添加關聯字段', union_error: '關聯關系與關聯字段不能為空', - union_repeat: '當前數據集已被關聯，請勿重復關聯' + union_repeat: '當前數據集已被關聯，請勿重復關聯', + preview_result: '預覽結果' }, datasource: { datasource: '數據源', diff --git a/frontend/src/lang/zh.js b/frontend/src/lang/zh.js index 9e2fea741c..29199428e0 100644 --- a/frontend/src/lang/zh.js +++ b/frontend/src/lang/zh.js @@ -1166,7 +1166,8 @@ export default { field_select: '字段选择', add_union_field: '添加关联字段', union_error: '关联关系与关联字段不能为空', - union_repeat: '当前数据集已被关联，请勿重复关联' + union_repeat: '当前数据集已被关联，请勿重复关联', + preview_result: '预览结果' }, datasource: { datasource: '数据源', diff --git a/frontend/src/views/dataset/add/AddUnion.vue b/frontend/src/views/dataset/add/AddUnion.vue index 6fcf5b7013..c87cb68323 100644 --- a/frontend/src/views/dataset/add/AddUnion.vue +++ b/frontend/src/views/dataset/add/AddUnion.vue @@ -15,10 +15,15 @@
- + + + + {{ $t('dataset.preview_result') }} + +
@@ -70,6 +75,11 @@ {{ $t('dataset.confirm') }}
+ + + + + @@ -79,9 +89,10 @@ import NodeItem from '@/views/dataset/add/union/NodeItem' import DatasetGroupSelectorTree from '@/views/dataset/common/DatasetGroupSelectorTree' import UnionEdit from '@/views/dataset/add/union/UnionEdit' import { getTable, post } from '@/api/dataset/dataset' +import UnionPreview from '@/views/dataset/add/union/UnionPreview' export default { name: 'AddUnion', - components: { UnionEdit, DatasetGroupSelectorTree, NodeItem, UnionNode }, + components: { UnionPreview, UnionEdit, DatasetGroupSelectorTree, NodeItem, UnionNode }, props: { param: { type: Object, @@ -133,7 +144,9 @@ export default { // 弹框临时选中的数据集 tempDs: {}, editUnion: false, - unionParam: {} + unionParam: {}, + showPreview: false, + previewTable: {} } }, watch: { @@ -169,7 +182,7 @@ export default { dataSourceId: this.dataset[0].currentDs.dataSourceId, type: 'union', mode: this.dataset[0].currentDs.mode, - info: '{"list":' + JSON.stringify(this.dataset) + '}' + info: '{"union":' + JSON.stringify(this.dataset) + '}' } post('/dataset/table/update', table).then(response => { this.$emit('saveSuccess', table) @@ -270,9 +283,22 @@ export default { getTable(this.param.tableId).then(response => { const table = JSON.parse(JSON.stringify(response.data)) this.name = table.name - this.dataset = JSON.parse(table.info).list + this.dataset = JSON.parse(table.info).union }) } + }, + + previewData() { + this.previewTable = { + id: this.param.tableId, + name: this.name, + sceneId: this.param.id, + dataSourceId: this.dataset[0].currentDs.dataSourceId, + type: 'union', + mode: this.dataset[0].currentDs.mode, + info: '{"union":' + JSON.stringify(this.dataset) + '}' + } + this.showPreview = true } } } @@ -294,4 +320,15 @@ export default { .dialog-css >>> .el-dialog__body { padding: 0 20px; } +.preview-style >>> .el-drawer{ + height: 50%!important; +} +.preview-style >>> .el-drawer .el-drawer__header{ + margin-bottom: 10px!important; + padding: 10px 16px 0!important; + font-size: 14px; +} +.preview-style >>> .el-drawer .el-drawer__body{ + padding: 0 16px 10px!important; +} diff --git a/frontend/src/views/dataset/add/union/UnionPreview.vue b/frontend/src/views/dataset/add/union/UnionPreview.vue new file mode 100644 index 0000000000..6d86576f21 --- /dev/null +++ b/frontend/src/views/dataset/add/union/UnionPreview.vue @@ -0,0 +1,94 @@ + + + + + diff --git a/frontend/src/views/dataset/common/DatasetGroupSelectorTree.vue b/frontend/src/views/dataset/common/DatasetGroupSelectorTree.vue index a566005fbc..18eddb08be 100644 --- a/frontend/src/views/dataset/common/DatasetGroupSelectorTree.vue +++ b/frontend/src/views/dataset/common/DatasetGroupSelectorTree.vue @@ -61,6 +61,7 @@ + diff --git a/frontend/src/views/dataset/data/FieldEdit.vue b/frontend/src/views/dataset/data/FieldEdit.vue index 9ce0272696..5338dbe9a6 100644 --- a/frontend/src/views/dataset/data/FieldEdit.vue +++ b/frontend/src/views/dataset/data/FieldEdit.vue @@ -15,7 +15,7 @@ {{ $t('dataset.add_calc_field') }} - {{ $t('dataset.sync_field') }} + {{ $t('dataset.sync_field') }} - + - +