github/dataease
github/dataease
3
0
Fork 1
mirror of https://github.com/dataease/dataease.git synced 2025-02-25 12:03:05 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/v1.7' into v1.7

Browse Source
This commit is contained in:
wangjiahao 2022-01-26 14:15:53 +08:00
commit 0b4581fd98
4 changed files with 274 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

379
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@ -57,7 +57,7 @@ import java.text.SimpleDateFormat;
import java.util.*; import java.util.*;
import java.util.stream.Collectors; import java.util.stream.Collectors;
;import static io.dataease.commons.constants.ColumnPermissionConstants.Desensitization_desc; import static io.dataease.commons.constants.ColumnPermissionConstants.Desensitization_desc;
/** /**
* @Author gin * @Author gin
@ -110,7 +110,8 @@ public class DataSetTableService {
} }
private void extractData(DataSetTableRequest datasetTable) throws Exception { private void extractData(DataSetTableRequest datasetTable) throws Exception {
if (datasetTable.getMode() == 1 && StringUtils.isNotEmpty(datasetTable.getSyncType()) && datasetTable.getSyncType().equalsIgnoreCase("sync_now")) { if (datasetTable.getMode() == 1 && StringUtils.isNotEmpty(datasetTable.getSyncType())
&& datasetTable.getSyncType().equalsIgnoreCase("sync_now")) {
DataSetTaskRequest dataSetTaskRequest = new DataSetTaskRequest(); DataSetTaskRequest dataSetTaskRequest = new DataSetTaskRequest();
DatasetTableTask datasetTableTask = new DatasetTableTask(); DatasetTableTask datasetTableTask = new DatasetTableTask();
datasetTableTask.setTableId(datasetTable.getId()); datasetTableTask.setTableId(datasetTable.getId());
@ -130,7 +131,8 @@ public class DataSetTableService {
if (StringUtils.isEmpty(datasetTable.getId())) { if (StringUtils.isEmpty(datasetTable.getId())) {
if (datasetTable.isMergeSheet()) { if (datasetTable.isMergeSheet()) {
Map<String, List<ExcelSheetData>> map = datasetTable.getSheets().stream().collect(Collectors.groupingBy(ExcelSheetData::getFieldsMd5)); Map<String, List<ExcelSheetData>> map = datasetTable.getSheets().stream()
.collect(Collectors.groupingBy(ExcelSheetData::getFieldsMd5));
for (String s : map.keySet()) { for (String s : map.keySet()) {
DataSetTableRequest sheetTable = new DataSetTableRequest(); DataSetTableRequest sheetTable = new DataSetTableRequest();
BeanUtils.copyBean(sheetTable, datasetTable); BeanUtils.copyBean(sheetTable, datasetTable);
@ -141,7 +143,8 @@ public class DataSetTableService {
sheetTable.setName(excelSheetDataList.get(0).getDatasetName()); sheetTable.setName(excelSheetDataList.get(0).getDatasetName());
checkName(sheetTable); checkName(sheetTable);
excelSheetDataList.forEach(excelSheetData -> { excelSheetDataList.forEach(excelSheetData -> {
String[] fieldArray = excelSheetData.getFields().stream().map(TableFiled::getFieldName).toArray(String[]::new); String[] fieldArray = excelSheetData.getFields().stream().map(TableFiled::getFieldName)
.toArray(String[]::new);
if (checkIsRepeat(fieldArray)) { if (checkIsRepeat(fieldArray)) {
DataEaseException.throwException(Translator.get("i18n_excel_field_repeat")); DataEaseException.throwException(Translator.get("i18n_excel_field_repeat"));
} }
@ -156,11 +159,13 @@ public class DataSetTableService {
datasetIdList.add(sheetTable.getId()); datasetIdList.add(sheetTable.getId());
} }
datasetIdList.forEach(datasetId -> { datasetIdList.forEach(datasetId -> {
commonThreadPool.addTask(() -> extractDataService.extractExcelData(datasetId, "all_scope", "初始导入", null, datasetIdList)); commonThreadPool.addTask(() -> extractDataService.extractExcelData(datasetId, "all_scope", "初始导入",
null, datasetIdList));
}); });
} else { } else {
for (ExcelSheetData sheet : datasetTable.getSheets()) { for (ExcelSheetData sheet : datasetTable.getSheets()) {
String[] fieldArray = sheet.getFields().stream().map(TableFiled::getFieldName).toArray(String[]::new); String[] fieldArray = sheet.getFields().stream().map(TableFiled::getFieldName)
.toArray(String[]::new);
if (checkIsRepeat(fieldArray)) { if (checkIsRepeat(fieldArray)) {
DataEaseException.throwException(Translator.get("i18n_excel_field_repeat")); DataEaseException.throwException(Translator.get("i18n_excel_field_repeat"));
} }
@ -183,7 +188,8 @@ public class DataSetTableService {
datasetIdList.add(sheetTable.getId()); datasetIdList.add(sheetTable.getId());
} }
datasetIdList.forEach(datasetId -> { datasetIdList.forEach(datasetId -> {
commonThreadPool.addTask(() -> extractDataService.extractExcelData(datasetId, "all_scope", "初始导入", null, datasetIdList)); commonThreadPool.addTask(() -> extractDataService.extractExcelData(datasetId, "all_scope", "初始导入",
null, datasetIdList));
}); });
} }
@ -191,11 +197,13 @@ public class DataSetTableService {
} }
List<ExcelSheetData> excelSheetDataList = new ArrayList<>(); List<ExcelSheetData> excelSheetDataList = new ArrayList<>();
List<String> oldFields = datasetTable.getSheets().get(0).getFields().stream().map(TableFiled::getRemarks).collect(Collectors.toList()); List<String> oldFields = datasetTable.getSheets().get(0).getFields().stream().map(TableFiled::getRemarks)
.collect(Collectors.toList());
for (ExcelSheetData sheet : datasetTable.getSheets()) { for (ExcelSheetData sheet : datasetTable.getSheets()) {
//替换时， // 替换时，
if (datasetTable.getEditType() == 0) { if (datasetTable.getEditType() == 0) {
List<String> newFields = sheet.getFields().stream().map(TableFiled::getRemarks).collect(Collectors.toList()); List<String> newFields = sheet.getFields().stream().map(TableFiled::getRemarks)
.collect(Collectors.toList());
if (!oldFields.equals(newFields)) { if (!oldFields.equals(newFields)) {
DataEaseException.throwException(Translator.get("i18n_excel_column_inconsistent")); DataEaseException.throwException(Translator.get("i18n_excel_column_inconsistent"));
} }
@ -216,9 +224,12 @@ public class DataSetTableService {
int update = datasetTableMapper.updateByPrimaryKeySelective(datasetTable); int update = datasetTableMapper.updateByPrimaryKeySelective(datasetTable);
// 替換時，先不刪除旧字段；同步成功后再删除 // 替換時，先不刪除旧字段；同步成功后再删除
if (datasetTable.getEditType() == 0) { if (datasetTable.getEditType() == 0) {
commonThreadPool.addTask(() -> extractDataService.extractExcelData(datasetTable.getId(), "all_scope", "替换", saveExcelTableField(datasetTable.getId(), datasetTable.getSheets().get(0).getFields(), false), Arrays.asList(datasetTable.getId()))); commonThreadPool.addTask(() -> extractDataService.extractExcelData(datasetTable.getId(), "all_scope", "替换",
saveExcelTableField(datasetTable.getId(), datasetTable.getSheets().get(0).getFields(), false),
Arrays.asList(datasetTable.getId())));
} else if (datasetTable.getEditType() == 1) { } else if (datasetTable.getEditType() == 1) {
commonThreadPool.addTask(() -> extractDataService.extractExcelData(datasetTable.getId(), "add_scope", "追加", null, Arrays.asList(datasetTable.getId()))); commonThreadPool.addTask(() -> extractDataService.extractExcelData(datasetTable.getId(), "add_scope", "追加",
null, Arrays.asList(datasetTable.getId())));
} }
} }
@ -285,7 +296,8 @@ public class DataSetTableService {
DatasourceRequest datasourceRequest = new DatasourceRequest(); DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dorisDatasource); datasourceRequest.setDatasource(dorisDatasource);
DDLProvider ddlProvider = ProviderFactory.getDDLProvider(dorisDatasource.getType()); DDLProvider ddlProvider = ProviderFactory.getDDLProvider(dorisDatasource.getType());
if (StringUtils.equalsIgnoreCase("custom", table.getType()) || StringUtils.equalsIgnoreCase("union", table.getType())) { if (StringUtils.equalsIgnoreCase("custom", table.getType())
|| StringUtils.equalsIgnoreCase("union", table.getType())) {
datasourceRequest.setQuery(ddlProvider.dropView(dorisTableName)); datasourceRequest.setQuery(ddlProvider.dropView(dorisTableName));
jdbcProvider.exec(datasourceRequest); jdbcProvider.exec(datasourceRequest);
datasourceRequest.setQuery(ddlProvider.dropView(DorisTableUtils.dorisTmpName(dorisTableName))); datasourceRequest.setQuery(ddlProvider.dropView(DorisTableUtils.dorisTmpName(dorisTableName)));
@ -402,7 +414,8 @@ public class DataSetTableService {
return datasourceProvider.getTableFileds(datasourceRequest); return datasourceProvider.getTableFileds(datasourceRequest);
} }
public Map<String, List<DatasetTableField>> getFieldsFromDE(DataSetTableRequest dataSetTableRequest) throws Exception { public Map<String, List<DatasetTableField>> getFieldsFromDE(DataSetTableRequest dataSetTableRequest)
throws Exception {
DatasetTableField datasetTableField = DatasetTableField.builder().build(); DatasetTableField datasetTableField = DatasetTableField.builder().build();
datasetTableField.setTableId(dataSetTableRequest.getId()); datasetTableField.setTableId(dataSetTableRequest.getId());
datasetTableField.setChecked(Boolean.TRUE); datasetTableField.setChecked(Boolean.TRUE);
@ -441,10 +454,11 @@ public class DataSetTableService {
return map; return map;
} }
public Map<String, Object> getPreviewData(DataSetTableRequest dataSetTableRequest, Integer page, Integer pageSize,
public Map<String, Object> getPreviewData(DataSetTableRequest dataSetTableRequest, Integer page, Integer pageSize, List<DatasetTableField> extFields) throws Exception { List<DatasetTableField> extFields) throws Exception {
Map<String, Object> map = new HashMap<>(); Map<String, Object> map = new HashMap<>();
DatasetTableField datasetTableField = DatasetTableField.builder().tableId(dataSetTableRequest.getId()).checked(Boolean.TRUE).build(); DatasetTableField datasetTableField = DatasetTableField.builder().tableId(dataSetTableRequest.getId())
.checked(Boolean.TRUE).build();
List<DatasetTableField> fields = dataSetTableFieldsService.list(datasetTableField); List<DatasetTableField> fields = dataSetTableFieldsService.list(datasetTableField);
if (CollectionUtils.isNotEmpty(extFields)) { if (CollectionUtils.isNotEmpty(extFields)) {
fields.addAll(extFields); fields.addAll(extFields);
@ -456,7 +470,7 @@ public class DataSetTableService {
return map; return map;
} }
DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(dataSetTableRequest.getId()); DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(dataSetTableRequest.getId());
//列权限 // 列权限
List<String> desensitizationList = new ArrayList<>(); List<String> desensitizationList = new ArrayList<>();
fields = permissionService.filterColumnPermissons(fields, desensitizationList, datasetTable.getId(), null); fields = permissionService.filterColumnPermissons(fields, desensitizationList, datasetTable.getId(), null);
if (CollectionUtils.isEmpty(fields)) { if (CollectionUtils.isEmpty(fields)) {
@ -465,19 +479,20 @@ public class DataSetTableService {
map.put("page", new DataSetPreviewPage()); map.put("page", new DataSetPreviewPage());
return map; return map;
} }
//行权限 // 行权限
List<ChartFieldCustomFilterDTO> customFilter = permissionService.getCustomFilters(fields, datasetTable, null); List<ChartFieldCustomFilterDTO> customFilter = permissionService.getCustomFilters(fields, datasetTable, null);
String[] fieldArray = fields.stream().map(DatasetTableField::getDataeaseName).toArray(String[]::new); String[] fieldArray = fields.stream().map(DatasetTableField::getDataeaseName).toArray(String[]::new);
DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class); DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class);
List<String[]> data = new ArrayList<>(); List<String[]> data = new ArrayList<>();
DataSetPreviewPage dataSetPreviewPage = new DataSetPreviewPage(); DataSetPreviewPage dataSetPreviewPage = new DataSetPreviewPage();
dataSetPreviewPage.setShow(Integer.valueOf(dataSetTableRequest.getRow())); dataSetPreviewPage.setShow(Integer.valueOf(dataSetTableRequest.getRow()));
dataSetPreviewPage.setPage(page); dataSetPreviewPage.setPage(page);
dataSetPreviewPage.setPageSize(pageSize); dataSetPreviewPage.setPageSize(pageSize);
int realSize = Integer.parseInt(dataSetTableRequest.getRow()) < pageSize ? Integer.parseInt(dataSetTableRequest.getRow()) : pageSize; int realSize = Integer.parseInt(dataSetTableRequest.getRow()) < pageSize
? Integer.parseInt(dataSetTableRequest.getRow())
: pageSize;
if (page == Integer.parseInt(dataSetTableRequest.getRow()) / pageSize + 1) { if (page == Integer.parseInt(dataSetTableRequest.getRow()) / pageSize + 1) {
realSize = Integer.parseInt(dataSetTableRequest.getRow()) % pageSize; realSize = Integer.parseInt(dataSetTableRequest.getRow()) % pageSize;
} }
@ -496,8 +511,8 @@ public class DataSetTableService {
String table = dataTableInfoDTO.getTable(); String table = dataTableInfoDTO.getTable();
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(
datasourceRequest.setQuery(qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter)); qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter));
map.put("sql", datasourceRequest.getQuery()); map.put("sql", datasourceRequest.getQuery());
datasourceRequest.setPage(page); datasourceRequest.setPage(page);
@ -514,7 +529,8 @@ public class DataSetTableService {
} }
try { try {
datasourceRequest.setQuery(qp.createQueryTableWithLimit(table, fields, Integer.valueOf(dataSetTableRequest.getRow()), false, ds, customFilter)); datasourceRequest.setQuery(qp.createQueryTableWithLimit(table, fields,
Integer.valueOf(dataSetTableRequest.getRow()), false, ds, customFilter));
datasourceRequest.setPageable(false); datasourceRequest.setPageable(false);
dataSetPreviewPage.setTotal(datasourceProvider.getData(datasourceRequest).size()); dataSetPreviewPage.setTotal(datasourceProvider.getData(datasourceRequest).size());
} catch (Exception e) { } catch (Exception e) {
@ -532,7 +548,8 @@ public class DataSetTableService {
datasourceRequest.setDatasource(ds); datasourceRequest.setDatasource(ds);
String table = DorisTableUtils.dorisName(dataSetTableRequest.getId()); String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter)); datasourceRequest.setQuery(
qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter));
map.put("sql", datasourceRequest.getQuery()); map.put("sql", datasourceRequest.getQuery());
try { try {
data.addAll(jdbcProvider.getData(datasourceRequest)); data.addAll(jdbcProvider.getData(datasourceRequest));
@ -541,7 +558,8 @@ public class DataSetTableService {
DEException.throwException(Translator.get("i18n_ds_error") + "->" + e.getMessage()); DEException.throwException(Translator.get("i18n_ds_error") + "->" + e.getMessage());
} }
try { try {
datasourceRequest.setQuery(qp.createQueryTableWithLimit(table, fields, Integer.valueOf(dataSetTableRequest.getRow()), false, ds, customFilter)); datasourceRequest.setQuery(qp.createQueryTableWithLimit(table, fields,
Integer.valueOf(dataSetTableRequest.getRow()), false, ds, customFilter));
dataSetPreviewPage.setTotal(jdbcProvider.getData(datasourceRequest).size()); dataSetPreviewPage.setTotal(jdbcProvider.getData(datasourceRequest).size());
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage()); logger.error(e.getMessage());
@ -564,7 +582,8 @@ public class DataSetTableService {
String sql = dataTableInfoDTO.getSql(); String sql = dataTableInfoDTO.getSql();
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(qp.createQuerySQLWithPage(sql, fields, page, pageSize, realSize, false, customFilter)); datasourceRequest.setQuery(
qp.createQuerySQLWithPage(sql, fields, page, pageSize, realSize, false, customFilter));
map.put("sql", datasourceRequest.getQuery()); map.put("sql", datasourceRequest.getQuery());
datasourceRequest.setPage(page); datasourceRequest.setPage(page);
datasourceRequest.setFetchSize(Integer.parseInt(dataSetTableRequest.getRow())); datasourceRequest.setFetchSize(Integer.parseInt(dataSetTableRequest.getRow()));
@ -580,7 +599,8 @@ public class DataSetTableService {
} }
try { try {
datasourceRequest.setPageable(false); datasourceRequest.setPageable(false);
datasourceRequest.setQuery(qp.createQuerySqlWithLimit(sql, fields, Integer.valueOf(dataSetTableRequest.getRow()), false, customFilter)); datasourceRequest.setQuery(qp.createQuerySqlWithLimit(sql, fields,
Integer.valueOf(dataSetTableRequest.getRow()), false, customFilter));
dataSetPreviewPage.setTotal(datasourceProvider.getData(datasourceRequest).size()); dataSetPreviewPage.setTotal(datasourceProvider.getData(datasourceRequest).size());
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage()); logger.error(e.getMessage());
@ -597,7 +617,8 @@ public class DataSetTableService {
datasourceRequest.setDatasource(ds); datasourceRequest.setDatasource(ds);
String table = DorisTableUtils.dorisName(dataSetTableRequest.getId()); String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter)); datasourceRequest.setQuery(
qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter));
map.put("sql", datasourceRequest.getQuery()); map.put("sql", datasourceRequest.getQuery());
try { try {
data.addAll(jdbcProvider.getData(datasourceRequest)); data.addAll(jdbcProvider.getData(datasourceRequest));
@ -606,7 +627,8 @@ public class DataSetTableService {
DEException.throwException(Translator.get("i18n_ds_error") + "->" + e.getMessage()); DEException.throwException(Translator.get("i18n_ds_error") + "->" + e.getMessage());
} }
try { try {
datasourceRequest.setQuery(qp.createQueryTableWithLimit(table, fields, Integer.valueOf(dataSetTableRequest.getRow()), false, ds, customFilter)); datasourceRequest.setQuery(qp.createQueryTableWithLimit(table, fields,
Integer.valueOf(dataSetTableRequest.getRow()), false, ds, customFilter));
dataSetPreviewPage.setTotal(jdbcProvider.getData(datasourceRequest).size()); dataSetPreviewPage.setTotal(jdbcProvider.getData(datasourceRequest).size());
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage()); logger.error(e.getMessage());
@ -624,7 +646,8 @@ public class DataSetTableService {
datasourceRequest.setDatasource(ds); datasourceRequest.setDatasource(ds);
String table = DorisTableUtils.dorisName(dataSetTableRequest.getId()); String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter)); datasourceRequest.setQuery(
qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter));
map.put("sql", datasourceRequest.getQuery()); map.put("sql", datasourceRequest.getQuery());
try { try {
data.addAll(jdbcProvider.getData(datasourceRequest)); data.addAll(jdbcProvider.getData(datasourceRequest));
@ -633,7 +656,8 @@ public class DataSetTableService {
DEException.throwException(Translator.get("i18n_ds_error") + "->" + e.getMessage()); DEException.throwException(Translator.get("i18n_ds_error") + "->" + e.getMessage());
} }
try { try {
datasourceRequest.setQuery(qp.createQueryTableWithLimit(table, fields, Integer.valueOf(dataSetTableRequest.getRow()), false, ds, customFilter)); datasourceRequest.setQuery(qp.createQueryTableWithLimit(table, fields,
Integer.valueOf(dataSetTableRequest.getRow()), false, ds, customFilter));
dataSetPreviewPage.setTotal(jdbcProvider.getData(datasourceRequest).size()); dataSetPreviewPage.setTotal(jdbcProvider.getData(datasourceRequest).size());
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage()); logger.error(e.getMessage());
@ -650,7 +674,8 @@ public class DataSetTableService {
datasourceRequest.setDatasource(ds); datasourceRequest.setDatasource(ds);
DataTableInfoDTO dt = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class); DataTableInfoDTO dt = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class);
List<DataSetTableUnionDTO> list = dataSetTableUnionService.listByTableId(dt.getList().get(0).getTableId()); List<DataSetTableUnionDTO> list = dataSetTableUnionService
.listByTableId(dt.getList().get(0).getTableId());
String sql = ""; String sql = "";
try { try {
@ -660,7 +685,8 @@ public class DataSetTableService {
DEException.throwException(Translator.get("i18n_ds_error") + "->" + e.getMessage()); DEException.throwException(Translator.get("i18n_ds_error") + "->" + e.getMessage());
} }
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(qp.createQuerySQLWithPage(sql, fields, page, pageSize, realSize, false, customFilter)); datasourceRequest.setQuery(
qp.createQuerySQLWithPage(sql, fields, page, pageSize, realSize, false, customFilter));
map.put("sql", datasourceRequest.getQuery()); map.put("sql", datasourceRequest.getQuery());
datasourceRequest.setPage(page); datasourceRequest.setPage(page);
datasourceRequest.setFetchSize(Integer.parseInt(dataSetTableRequest.getRow())); datasourceRequest.setFetchSize(Integer.parseInt(dataSetTableRequest.getRow()));
@ -676,7 +702,8 @@ public class DataSetTableService {
} }
try { try {
datasourceRequest.setPageable(false); datasourceRequest.setPageable(false);
datasourceRequest.setQuery(qp.createQuerySqlWithLimit(sql, fields, Integer.valueOf(dataSetTableRequest.getRow()), false, customFilter)); datasourceRequest.setQuery(qp.createQuerySqlWithLimit(sql, fields,
Integer.valueOf(dataSetTableRequest.getRow()), false, customFilter));
dataSetPreviewPage.setTotal(datasourceProvider.getData(datasourceRequest).size()); dataSetPreviewPage.setTotal(datasourceProvider.getData(datasourceRequest).size());
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage()); logger.error(e.getMessage());
@ -689,7 +716,8 @@ public class DataSetTableService {
datasourceRequest.setDatasource(ds); datasourceRequest.setDatasource(ds);
String table = DorisTableUtils.dorisName(dataSetTableRequest.getId()); String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter)); datasourceRequest.setQuery(
qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter));
map.put("sql", datasourceRequest.getQuery()); map.put("sql", datasourceRequest.getQuery());
try { try {
data.addAll(jdbcProvider.getData(datasourceRequest)); data.addAll(jdbcProvider.getData(datasourceRequest));
@ -699,7 +727,8 @@ public class DataSetTableService {
} }
try { try {
datasourceRequest.setQuery(qp.createQueryTableWithLimit(table, fields, Integer.valueOf(dataSetTableRequest.getRow()), false, ds, customFilter)); datasourceRequest.setQuery(qp.createQueryTableWithLimit(table, fields,
Integer.valueOf(dataSetTableRequest.getRow()), false, ds, customFilter));
dataSetPreviewPage.setTotal(jdbcProvider.getData(datasourceRequest).size()); dataSetPreviewPage.setTotal(jdbcProvider.getData(datasourceRequest).size());
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage()); logger.error(e.getMessage());
@ -726,7 +755,8 @@ public class DataSetTableService {
DEException.throwException(Translator.get("i18n_ds_error") + "->" + e.getMessage()); DEException.throwException(Translator.get("i18n_ds_error") + "->" + e.getMessage());
} }
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(qp.createQuerySQLWithPage(sql, fields, page, pageSize, realSize, false, customFilter)); datasourceRequest.setQuery(
qp.createQuerySQLWithPage(sql, fields, page, pageSize, realSize, false, customFilter));
map.put("sql", datasourceRequest.getQuery()); map.put("sql", datasourceRequest.getQuery());
datasourceRequest.setPage(page); datasourceRequest.setPage(page);
datasourceRequest.setFetchSize(Integer.parseInt(dataSetTableRequest.getRow())); datasourceRequest.setFetchSize(Integer.parseInt(dataSetTableRequest.getRow()));
@ -742,7 +772,8 @@ public class DataSetTableService {
} }
try { try {
datasourceRequest.setPageable(false); datasourceRequest.setPageable(false);
datasourceRequest.setQuery(qp.createQuerySqlWithLimit(sql, fields, Integer.valueOf(dataSetTableRequest.getRow()), false, customFilter)); datasourceRequest.setQuery(qp.createQuerySqlWithLimit(sql, fields,
Integer.valueOf(dataSetTableRequest.getRow()), false, customFilter));
dataSetPreviewPage.setTotal(datasourceProvider.getData(datasourceRequest).size()); dataSetPreviewPage.setTotal(datasourceProvider.getData(datasourceRequest).size());
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage()); logger.error(e.getMessage());
@ -755,7 +786,8 @@ public class DataSetTableService {
datasourceRequest.setDatasource(ds); datasourceRequest.setDatasource(ds);
String table = DorisTableUtils.dorisName(dataSetTableRequest.getId()); String table = DorisTableUtils.dorisName(dataSetTableRequest.getId());
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
datasourceRequest.setQuery(qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter)); datasourceRequest.setQuery(
qp.createQueryTableWithPage(table, fields, page, pageSize, realSize, false, ds, customFilter));
map.put("sql", datasourceRequest.getQuery()); map.put("sql", datasourceRequest.getQuery());
try { try {
data.addAll(jdbcProvider.getData(datasourceRequest)); data.addAll(jdbcProvider.getData(datasourceRequest));
@ -765,7 +797,8 @@ public class DataSetTableService {
} }
try { try {
datasourceRequest.setQuery(qp.createQueryTableWithLimit(table, fields, Integer.valueOf(dataSetTableRequest.getRow()), false, ds, customFilter)); datasourceRequest.setQuery(qp.createQueryTableWithLimit(table, fields,
Integer.valueOf(dataSetTableRequest.getRow()), false, ds, customFilter));
dataSetPreviewPage.setTotal(jdbcProvider.getData(datasourceRequest).size()); dataSetPreviewPage.setTotal(jdbcProvider.getData(datasourceRequest).size());
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage()); logger.error(e.getMessage());
@ -880,8 +913,12 @@ public class DataSetTableService {
// 获取每个字段在当前de数据库中的name，作为sql查询后的remarks返回前端展示 // 获取每个字段在当前de数据库中的name，作为sql查询后的remarks返回前端展示
for (DatasetTableField datasetTableField : fieldList) { for (DatasetTableField datasetTableField : fieldList) {
for (TableFiled tableFiled : fields) { for (TableFiled tableFiled : fields) {
if (StringUtils.equalsIgnoreCase(tableFiled.getFieldName(), DorisTableUtils.dorisFieldName(datasetTableField.getTableId() + "_" + datasetTableField.getDataeaseName())) if (StringUtils.equalsIgnoreCase(tableFiled.getFieldName(),
|| StringUtils.equalsIgnoreCase(tableFiled.getFieldName(), DorisTableUtils.dorisFieldNameShort(datasetTableField.getTableId() + "_" + datasetTableField.getOriginName()))) { DorisTableUtils.dorisFieldName(
datasetTableField.getTableId() + "_" + datasetTableField.getDataeaseName()))
|| StringUtils.equalsIgnoreCase(tableFiled.getFieldName(),
DorisTableUtils.dorisFieldNameShort(datasetTableField.getTableId() + "_"
+ datasetTableField.getOriginName()))) {
tableFiled.setRemarks(datasetTableField.getName()); tableFiled.setRemarks(datasetTableField.getName());
break; break;
} }
@ -898,7 +935,8 @@ public class DataSetTableService {
public Map<String, Object> getCustomPreview(DataSetTableRequest dataSetTableRequest) throws Exception { public Map<String, Object> getCustomPreview(DataSetTableRequest dataSetTableRequest) throws Exception {
DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class); DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class);
List<DataSetTableUnionDTO> list = dataSetTableUnionService.listByTableId(dataTableInfoDTO.getList().get(0).getTableId()); List<DataSetTableUnionDTO> list = dataSetTableUnionService
.listByTableId(dataTableInfoDTO.getList().get(0).getTableId());
String sql; String sql;
DatasourceRequest datasourceRequest = new DatasourceRequest(); DatasourceRequest datasourceRequest = new DatasourceRequest();
@ -935,11 +973,15 @@ public class DataSetTableService {
// 获取每个字段在当前de数据库中的name，作为sql查询后的remarks返回前端展示 // 获取每个字段在当前de数据库中的name，作为sql查询后的remarks返回前端展示
List<DatasetTableField> checkedFieldList = new ArrayList<>(); List<DatasetTableField> checkedFieldList = new ArrayList<>();
dataTableInfoDTO.getList().forEach(ele -> checkedFieldList.addAll(dataSetTableFieldsService.getListByIds(ele.getCheckedFields()))); dataTableInfoDTO.getList().forEach(
ele -> checkedFieldList.addAll(dataSetTableFieldsService.getListByIds(ele.getCheckedFields())));
for (DatasetTableField datasetTableField : checkedFieldList) { for (DatasetTableField datasetTableField : checkedFieldList) {
for (TableFiled tableFiled : fields) { for (TableFiled tableFiled : fields) {
if (StringUtils.equalsIgnoreCase(tableFiled.getFieldName(), DorisTableUtils.dorisFieldName(datasetTableField.getTableId() + "_" + datasetTableField.getDataeaseName())) if (StringUtils.equalsIgnoreCase(tableFiled.getFieldName(),
|| StringUtils.equalsIgnoreCase(tableFiled.getFieldName(), DorisTableUtils.dorisFieldName(datasetTableField.getTableId() + "_" + datasetTableField.getOriginName()))) { DorisTableUtils.dorisFieldName(
datasetTableField.getTableId() + "_" + datasetTableField.getDataeaseName()))
|| StringUtils.equalsIgnoreCase(tableFiled.getFieldName(), DorisTableUtils.dorisFieldName(
datasetTableField.getTableId() + "_" + datasetTableField.getOriginName()))) {
tableFiled.setRemarks(datasetTableField.getName()); tableFiled.setRemarks(datasetTableField.getName());
break; break;
} }
@ -967,7 +1009,10 @@ public class DataSetTableService {
if (CollectionUtils.isEmpty(fields)) { if (CollectionUtils.isEmpty(fields)) {
throw new RuntimeException(Translator.get("i18n_cst_ds_tb_or_field_deleted")); throw new RuntimeException(Translator.get("i18n_cst_ds_tb_or_field_deleted"));
} }
String[] array = fields.stream().map(f -> table + "." + f.getDataeaseName() + " AS " + DorisTableUtils.dorisFieldName(ele.getTableId() + "_" + f.getDataeaseName())).toArray(String[]::new); String[] array = fields.stream()
.map(f -> table + "." + f.getDataeaseName() + " AS "
+ DorisTableUtils.dorisFieldName(ele.getTableId() + "_" + f.getDataeaseName()))
.toArray(String[]::new);
customInfo.put(table, array); customInfo.put(table, array);
}); });
DataTableInfoCustomUnion first = dataTableInfoDTO.getList().get(0); DataTableInfoCustomUnion first = dataTableInfoDTO.getList().get(0);
@ -994,25 +1039,32 @@ public class DataSetTableService {
join.append(convertUnionTypeToSQL(dto.getSourceUnionRelation())) join.append(convertUnionTypeToSQL(dto.getSourceUnionRelation()))
.append(DorisTableUtils.dorisName(dto.getTargetTableId())) .append(DorisTableUtils.dorisName(dto.getTargetTableId()))
.append(" ON ") .append(" ON ")
.append(DorisTableUtils.dorisName(dto.getSourceTableId())).append(".").append(sourceField.getDataeaseName()) .append(DorisTableUtils.dorisName(dto.getSourceTableId())).append(".")
.append(sourceField.getDataeaseName())
.append(" = ") .append(" = ")
.append(DorisTableUtils.dorisName(dto.getTargetTableId())).append(".").append(targetField.getDataeaseName()); .append(DorisTableUtils.dorisName(dto.getTargetTableId())).append(".")
.append(targetField.getDataeaseName());
} }
} }
} }
if (StringUtils.isEmpty(f)) { if (StringUtils.isEmpty(f)) {
throw new RuntimeException(Translator.get("i18n_custom_ds_delete")); throw new RuntimeException(Translator.get("i18n_custom_ds_delete"));
} }
return MessageFormat.format("SELECT {0} FROM {1}", f, DorisTableUtils.dorisName(first.getTableId())) + join.toString(); return MessageFormat.format("SELECT {0} FROM {1}", f, DorisTableUtils.dorisName(first.getTableId()))
+ join.toString();
} else { } else {
if (StringUtils.isEmpty(StringUtils.join(customInfo.get(DorisTableUtils.dorisName(first.getTableId())), ","))) { if (StringUtils
.isEmpty(StringUtils.join(customInfo.get(DorisTableUtils.dorisName(first.getTableId())), ","))) {
throw new RuntimeException(Translator.get("i18n_custom_ds_delete")); throw new RuntimeException(Translator.get("i18n_custom_ds_delete"));
} }
return MessageFormat.format("SELECT {0} FROM {1}", StringUtils.join(customInfo.get(DorisTableUtils.dorisName(first.getTableId())), ","), DorisTableUtils.dorisName(first.getTableId())); return MessageFormat.format("SELECT {0} FROM {1}",
StringUtils.join(customInfo.get(DorisTableUtils.dorisName(first.getTableId())), ","),
DorisTableUtils.dorisName(first.getTableId()));
} }
} }
public String getCustomSQLDatasource(DataTableInfoDTO dataTableInfoDTO, List<DataSetTableUnionDTO> list, Datasource ds) { public String getCustomSQLDatasource(DataTableInfoDTO dataTableInfoDTO, List<DataSetTableUnionDTO> list,
Datasource ds) {
DatasourceTypes datasourceTypes = DatasourceTypes.valueOf(ds.getType()); DatasourceTypes datasourceTypes = DatasourceTypes.valueOf(ds.getType());
String keyword = datasourceTypes.getKeywordPrefix() + "%s" + datasourceTypes.getKeywordSuffix(); String keyword = datasourceTypes.getKeywordPrefix() + "%s" + datasourceTypes.getKeywordSuffix();
Map<String, String[]> customInfo = new TreeMap<>(); Map<String, String[]> customInfo = new TreeMap<>();
@ -1026,7 +1078,10 @@ public class DataSetTableService {
if (CollectionUtils.isEmpty(fields)) { if (CollectionUtils.isEmpty(fields)) {
throw new RuntimeException(Translator.get("i18n_cst_ds_tb_or_field_deleted")); throw new RuntimeException(Translator.get("i18n_cst_ds_tb_or_field_deleted"));
} }
String[] array = fields.stream().map(f -> String.format(keyword, table) + "." + String.format(keyword, f.getOriginName()) + " AS " + DorisTableUtils.dorisFieldName(ele.getTableId() + "_" + f.getOriginName())).toArray(String[]::new); String[] array = fields.stream()
.map(f -> String.format(keyword, table) + "." + String.format(keyword, f.getOriginName()) + " AS "
+ DorisTableUtils.dorisFieldName(ele.getTableId() + "_" + f.getOriginName()))
.toArray(String[]::new);
customInfo.put(table, array); customInfo.put(table, array);
} }
DataTableInfoCustomUnion first = dataTableInfoDTO.getList().get(0); DataTableInfoCustomUnion first = dataTableInfoDTO.getList().get(0);
@ -1052,15 +1107,19 @@ public class DataSetTableService {
DEException.throwException(Translator.get("i18n_dataset_field_delete")); DEException.throwException(Translator.get("i18n_dataset_field_delete"));
} }
DatasetTable sourceTable = datasetTableMapper.selectByPrimaryKey(dto.getSourceTableId()); DatasetTable sourceTable = datasetTableMapper.selectByPrimaryKey(dto.getSourceTableId());
String sourceTableName = new Gson().fromJson(sourceTable.getInfo(), DataTableInfoDTO.class).getTable(); String sourceTableName = new Gson().fromJson(sourceTable.getInfo(), DataTableInfoDTO.class)
.getTable();
DatasetTable targetTable = datasetTableMapper.selectByPrimaryKey(dto.getTargetTableId()); DatasetTable targetTable = datasetTableMapper.selectByPrimaryKey(dto.getTargetTableId());
String targetTableName = new Gson().fromJson(targetTable.getInfo(), DataTableInfoDTO.class).getTable(); String targetTableName = new Gson().fromJson(targetTable.getInfo(), DataTableInfoDTO.class)
.getTable();
join.append(convertUnionTypeToSQL(dto.getSourceUnionRelation())) join.append(convertUnionTypeToSQL(dto.getSourceUnionRelation()))
.append(String.format(keyword, targetTableName)) .append(String.format(keyword, targetTableName))
.append(" ON ") .append(" ON ")
.append(String.format(keyword, sourceTableName)).append(".").append(String.format(keyword, sourceField.getOriginName())) .append(String.format(keyword, sourceTableName)).append(".")
.append(String.format(keyword, sourceField.getOriginName()))
.append(" = ") .append(" = ")
.append(String.format(keyword, targetTableName)).append(".").append(String.format(keyword, targetField.getOriginName())); .append(String.format(keyword, targetTableName)).append(".")
.append(String.format(keyword, targetField.getOriginName()));
} }
} }
} }
@ -1072,7 +1131,8 @@ public class DataSetTableService {
if (StringUtils.isEmpty(StringUtils.join(customInfo.get(tableName), ","))) { if (StringUtils.isEmpty(StringUtils.join(customInfo.get(tableName), ","))) {
throw new RuntimeException(Translator.get("i18n_custom_ds_delete")); throw new RuntimeException(Translator.get("i18n_custom_ds_delete"));
} }
return MessageFormat.format("SELECT {0} FROM {1}", StringUtils.join(customInfo.get(tableName), ","), String.format(keyword, tableName)); return MessageFormat.format("SELECT {0} FROM {1}", StringUtils.join(customInfo.get(tableName), ","),
String.format(keyword, tableName));
} }
} }
@ -1109,11 +1169,15 @@ public class DataSetTableService {
String table = DorisTableUtils.dorisName(tableId); String table = DorisTableUtils.dorisName(tableId);
DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(tableId); DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(tableId);
if (ObjectUtils.isEmpty(datasetTable)) { if (ObjectUtils.isEmpty(datasetTable)) {
DEException.throwException(Translator.get("i18n_custom_ds_delete") + String.format(":table id [%s]", tableId)); DEException.throwException(
Translator.get("i18n_custom_ds_delete") + String.format(":table id [%s]", tableId));
} }
List<DatasetTableField> fields = dataSetTableFieldsService.getListByIdsEach(unionDTO.getCurrentDsField()); List<DatasetTableField> fields = dataSetTableFieldsService.getListByIdsEach(unionDTO.getCurrentDsField());
String[] array = fields.stream().map(f -> table + "." + f.getDataeaseName() + " AS " + DorisTableUtils.dorisFieldName(tableId + "_" + f.getDataeaseName())).toArray(String[]::new); String[] array = fields.stream()
.map(f -> table + "." + f.getDataeaseName() + " AS "
+ DorisTableUtils.dorisFieldName(tableId + "_" + f.getDataeaseName()))
.toArray(String[]::new);
checkedInfo.put(table, array); checkedInfo.put(table, array);
checkedFields.addAll(fields); checkedFields.addAll(fields);
// 获取child的fields和union // 获取child的fields和union
@ -1149,12 +1213,16 @@ public class DataSetTableService {
for (int i = 0; i < unionParamDTO.getUnionFields().size(); i++) { for (int i = 0; i < unionParamDTO.getUnionFields().size(); i++) {
UnionItemDTO unionItemDTO = unionParamDTO.getUnionFields().get(i); UnionItemDTO unionItemDTO = unionParamDTO.getUnionFields().get(i);
// 通过field id取得field详情，并且以第一组为准，寻找dataset table // 通过field id取得field详情，并且以第一组为准，寻找dataset table
DatasetTableField parentField = dataSetTableFieldsService.get(unionItemDTO.getParentField().getId()); DatasetTableField parentField = dataSetTableFieldsService
DatasetTableField currentField = dataSetTableFieldsService.get(unionItemDTO.getCurrentField().getId()); .get(unionItemDTO.getParentField().getId());
DatasetTableField currentField = dataSetTableFieldsService
.get(unionItemDTO.getCurrentField().getId());
join.append(DorisTableUtils.dorisName(parentTable.getId())).append(".").append(parentField.getDataeaseName()) join.append(DorisTableUtils.dorisName(parentTable.getId())).append(".")
.append(parentField.getDataeaseName())
.append(" = ") .append(" = ")
.append(DorisTableUtils.dorisName(currentTable.getId())).append(".").append(currentField.getDataeaseName()); .append(DorisTableUtils.dorisName(currentTable.getId())).append(".")
.append(currentField.getDataeaseName());
if (i < unionParamDTO.getUnionFields().size() - 1) { if (i < unionParamDTO.getUnionFields().size() - 1) {
join.append(" AND "); join.append(" AND ");
} }
@ -1163,13 +1231,16 @@ public class DataSetTableService {
if (StringUtils.isEmpty(f)) { if (StringUtils.isEmpty(f)) {
DEException.throwException(Translator.get("i18n_union_ds_no_checked")); DEException.throwException(Translator.get("i18n_union_ds_no_checked"));
} }
sql = MessageFormat.format("SELECT {0} FROM {1}", f, DorisTableUtils.dorisName(union.get(0).getCurrentDs().getId())) + join.toString(); sql = MessageFormat.format("SELECT {0} FROM {1}", f,
DorisTableUtils.dorisName(union.get(0).getCurrentDs().getId())) + join.toString();
} else { } else {
String f = StringUtils.join(checkedInfo.get(DorisTableUtils.dorisName(union.get(0).getCurrentDs().getId())), ","); String f = StringUtils.join(checkedInfo.get(DorisTableUtils.dorisName(union.get(0).getCurrentDs().getId())),
",");
if (StringUtils.isEmpty(f)) { if (StringUtils.isEmpty(f)) {
throw new RuntimeException(Translator.get("i18n_union_ds_no_checked")); throw new RuntimeException(Translator.get("i18n_union_ds_no_checked"));
} }
sql = MessageFormat.format("SELECT {0} FROM {1}", f, DorisTableUtils.dorisName(union.get(0).getCurrentDs().getId())); sql = MessageFormat.format("SELECT {0} FROM {1}", f,
DorisTableUtils.dorisName(union.get(0).getCurrentDs().getId()));
} }
Map<String, Object> map = new HashMap<>(); Map<String, Object> map = new HashMap<>();
map.put("sql", sql); map.put("sql", sql);
@ -1179,18 +1250,23 @@ public class DataSetTableService {
} }
// 递归计算出所有子级的checkedFields和unionParam // 递归计算出所有子级的checkedFields和unionParam
private void getUnionSQLDorisJoin(List<UnionDTO> childrenDs, Map<String, String[]> checkedInfo, List<UnionParamDTO> unionList, List<DatasetTableField> checkedFields) { private void getUnionSQLDorisJoin(List<UnionDTO> childrenDs, Map<String, String[]> checkedInfo,
List<UnionParamDTO> unionList, List<DatasetTableField> checkedFields) {
for (int i = 0; i < childrenDs.size(); i++) { for (int i = 0; i < childrenDs.size(); i++) {
UnionDTO unionDTO = childrenDs.get(i); UnionDTO unionDTO = childrenDs.get(i);
String tableId = unionDTO.getCurrentDs().getId(); String tableId = unionDTO.getCurrentDs().getId();
String table = DorisTableUtils.dorisName(tableId); String table = DorisTableUtils.dorisName(tableId);
DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(tableId); DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(tableId);
if (ObjectUtils.isEmpty(datasetTable)) { if (ObjectUtils.isEmpty(datasetTable)) {
DEException.throwException(Translator.get("i18n_custom_ds_delete") + String.format(":table id [%s]", tableId)); DEException.throwException(
Translator.get("i18n_custom_ds_delete") + String.format(":table id [%s]", tableId));
} }
List<DatasetTableField> fields = dataSetTableFieldsService.getListByIdsEach(unionDTO.getCurrentDsField()); List<DatasetTableField> fields = dataSetTableFieldsService.getListByIdsEach(unionDTO.getCurrentDsField());
String[] array = fields.stream().map(f -> table + "." + f.getDataeaseName() + " AS " + DorisTableUtils.dorisFieldName(tableId + "_" + f.getDataeaseName())).toArray(String[]::new); String[] array = fields.stream()
.map(f -> table + "." + f.getDataeaseName() + " AS "
+ DorisTableUtils.dorisFieldName(tableId + "_" + f.getDataeaseName()))
.toArray(String[]::new);
checkedInfo.put(table, array); checkedInfo.put(table, array);
checkedFields.addAll(fields); checkedFields.addAll(fields);
@ -1212,17 +1288,24 @@ public class DataSetTableService {
List<UnionParamDTO> unionList = new ArrayList<>(); List<UnionParamDTO> unionList = new ArrayList<>();
List<DatasetTableField> checkedFields = new ArrayList<>(); List<DatasetTableField> checkedFields = new ArrayList<>();
String sql = ""; String sql = "";
String tableName = new Gson().fromJson(datasetTableMapper.selectByPrimaryKey(union.get(0).getCurrentDs().getId()).getInfo(), DataTableInfoDTO.class).getTable(); String tableName = new Gson()
.fromJson(datasetTableMapper.selectByPrimaryKey(union.get(0).getCurrentDs().getId()).getInfo(),
DataTableInfoDTO.class)
.getTable();
for (UnionDTO unionDTO : union) { for (UnionDTO unionDTO : union) {
DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(unionDTO.getCurrentDs().getId()); DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(unionDTO.getCurrentDs().getId());
String table = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable(); String table = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable();
String tableId = unionDTO.getCurrentDs().getId(); String tableId = unionDTO.getCurrentDs().getId();
if (ObjectUtils.isEmpty(datasetTable)) { if (ObjectUtils.isEmpty(datasetTable)) {
DEException.throwException(Translator.get("i18n_custom_ds_delete") + String.format(":table id [%s]", tableId)); DEException.throwException(
Translator.get("i18n_custom_ds_delete") + String.format(":table id [%s]", tableId));
} }
List<DatasetTableField> fields = dataSetTableFieldsService.getListByIdsEach(unionDTO.getCurrentDsField()); List<DatasetTableField> fields = dataSetTableFieldsService.getListByIdsEach(unionDTO.getCurrentDsField());
String[] array = fields.stream().map(f -> String.format(keyword, table) + "." + String.format(keyword, f.getOriginName()) + " AS " + DorisTableUtils.dorisFieldNameShort(tableId + "_" + f.getOriginName())).toArray(String[]::new); String[] array = fields.stream()
.map(f -> String.format(keyword, table) + "." + String.format(keyword, f.getOriginName()) + " AS "
+ DorisTableUtils.dorisFieldNameShort(tableId + "_" + f.getOriginName()))
.toArray(String[]::new);
checkedInfo.put(table, array); checkedInfo.put(table, array);
checkedFields.addAll(fields); checkedFields.addAll(fields);
// 获取child的fields和union // 获取child的fields和union
@ -1253,19 +1336,24 @@ public class DataSetTableService {
DatasetTable parentTable = datasetTableMapper.selectByPrimaryKey(pField.getTableId()); DatasetTable parentTable = datasetTableMapper.selectByPrimaryKey(pField.getTableId());
String parentTableName = new Gson().fromJson(parentTable.getInfo(), DataTableInfoDTO.class).getTable(); String parentTableName = new Gson().fromJson(parentTable.getInfo(), DataTableInfoDTO.class).getTable();
DatasetTable currentTable = datasetTableMapper.selectByPrimaryKey(cField.getTableId()); DatasetTable currentTable = datasetTableMapper.selectByPrimaryKey(cField.getTableId());
String currentTableName = new Gson().fromJson(currentTable.getInfo(), DataTableInfoDTO.class).getTable(); String currentTableName = new Gson().fromJson(currentTable.getInfo(), DataTableInfoDTO.class)
.getTable();
join.append(" ").append(joinType).append(" ").append(String.format(keyword, currentTableName)) join.append(" ").append(joinType).append(" ").append(String.format(keyword, currentTableName))
.append(" ON "); .append(" ON ");
for (int i = 0; i < unionParamDTO.getUnionFields().size(); i++) { for (int i = 0; i < unionParamDTO.getUnionFields().size(); i++) {
UnionItemDTO unionItemDTO = unionParamDTO.getUnionFields().get(i); UnionItemDTO unionItemDTO = unionParamDTO.getUnionFields().get(i);
// 通过field id取得field详情，并且以第一组为准，寻找dataset table // 通过field id取得field详情，并且以第一组为准，寻找dataset table
DatasetTableField parentField = dataSetTableFieldsService.get(unionItemDTO.getParentField().getId()); DatasetTableField parentField = dataSetTableFieldsService
DatasetTableField currentField = dataSetTableFieldsService.get(unionItemDTO.getCurrentField().getId()); .get(unionItemDTO.getParentField().getId());
DatasetTableField currentField = dataSetTableFieldsService
.get(unionItemDTO.getCurrentField().getId());
join.append(String.format(keyword, parentTableName)).append(".").append(String.format(keyword, parentField.getOriginName())) join.append(String.format(keyword, parentTableName)).append(".")
.append(String.format(keyword, parentField.getOriginName()))
.append(" = ") .append(" = ")
.append(String.format(keyword, currentTableName)).append(".").append(String.format(keyword, currentField.getOriginName())); .append(String.format(keyword, currentTableName)).append(".")
.append(String.format(keyword, currentField.getOriginName()));
if (i < unionParamDTO.getUnionFields().size() - 1) { if (i < unionParamDTO.getUnionFields().size() - 1) {
join.append(" AND "); join.append(" AND ");
} }
@ -1300,20 +1388,25 @@ public class DataSetTableService {
} }
// 递归计算出所有子级的checkedFields和unionParam // 递归计算出所有子级的checkedFields和unionParam
private void getUnionSQLDatasourceJoin(List<UnionDTO> childrenDs, Map<String, String[]> checkedInfo, List<UnionParamDTO> unionList, String keyword, List<DatasetTableField> checkedFields) { private void getUnionSQLDatasourceJoin(List<UnionDTO> childrenDs, Map<String, String[]> checkedInfo,
List<UnionParamDTO> unionList, String keyword, List<DatasetTableField> checkedFields) {
for (int i = 0; i < childrenDs.size(); i++) { for (int i = 0; i < childrenDs.size(); i++) {
UnionDTO unionDTO = childrenDs.get(i); UnionDTO unionDTO = childrenDs.get(i);
DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(unionDTO.getCurrentDs().getId()); DatasetTable datasetTable = datasetTableMapper.selectByPrimaryKey(unionDTO.getCurrentDs().getId());
String tableId = unionDTO.getCurrentDs().getId(); String tableId = unionDTO.getCurrentDs().getId();
if (ObjectUtils.isEmpty(datasetTable)) { if (ObjectUtils.isEmpty(datasetTable)) {
DEException.throwException(Translator.get("i18n_custom_ds_delete") + String.format(":table id [%s]", tableId)); DEException.throwException(
Translator.get("i18n_custom_ds_delete") + String.format(":table id [%s]", tableId));
} }
String table = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable(); String table = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable();
List<DatasetTableField> fields = dataSetTableFieldsService.getListByIdsEach(unionDTO.getCurrentDsField()); List<DatasetTableField> fields = dataSetTableFieldsService.getListByIdsEach(unionDTO.getCurrentDsField());
String[] array = fields.stream().map(f -> String.format(keyword, table) + "." + String.format(keyword, f.getOriginName()) + " AS " + DorisTableUtils.dorisFieldNameShort(tableId + "_" + f.getOriginName())).toArray(String[]::new); String[] array = fields.stream()
.map(f -> String.format(keyword, table) + "." + String.format(keyword, f.getOriginName()) + " AS "
+ DorisTableUtils.dorisFieldNameShort(tableId + "_" + f.getOriginName()))
.toArray(String[]::new);
checkedInfo.put(table, array); checkedInfo.put(table, array);
checkedFields.addAll(fields); checkedFields.addAll(fields);
@ -1366,7 +1459,8 @@ public class DataSetTableService {
DatasourceRequest datasourceRequest = new DatasourceRequest(); DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds); datasourceRequest.setDatasource(ds);
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
String sqlAsTable = qp.createSQLPreview(new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class).getSql(), null); String sqlAsTable = qp.createSQLPreview(
new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class).getSql(), null);
datasourceRequest.setQuery(sqlAsTable); datasourceRequest.setQuery(sqlAsTable);
fields = datasourceProvider.fetchResultField(datasourceRequest); fields = datasourceProvider.fetchResultField(datasourceRequest);
} else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) { } else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "excel")) {
@ -1374,12 +1468,15 @@ public class DataSetTableService {
} else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "custom")) { } else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "custom")) {
if (datasetTable.getMode() == 1) { if (datasetTable.getMode() == 1) {
// save field // save field
DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class); DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(dataSetTableRequest.getInfo(),
DataTableInfoDTO.class);
List<DataTableInfoCustomUnion> list = dataTableInfoDTO.getList(); List<DataTableInfoCustomUnion> list = dataTableInfoDTO.getList();
List<DatasetTableField> fieldList = new ArrayList<>(); List<DatasetTableField> fieldList = new ArrayList<>();
list.forEach(ele -> { list.forEach(ele -> {
List<DatasetTableField> listByIds = dataSetTableFieldsService.getListByIdsEach(ele.getCheckedFields()); List<DatasetTableField> listByIds = dataSetTableFieldsService
listByIds.forEach(f -> f.setDataeaseName(DorisTableUtils.dorisFieldName(ele.getTableId() + "_" + f.getDataeaseName()))); .getListByIdsEach(ele.getCheckedFields());
listByIds.forEach(f -> f.setDataeaseName(
DorisTableUtils.dorisFieldName(ele.getTableId() + "_" + f.getDataeaseName())));
fieldList.addAll(listByIds); fieldList.addAll(listByIds);
}); });
for (int i = 0; i < fieldList.size(); i++) { for (int i = 0; i < fieldList.size(); i++) {
@ -1392,7 +1489,8 @@ public class DataSetTableService {
dataSetTableFieldsService.batchEdit(fieldList); dataSetTableFieldsService.batchEdit(fieldList);
// custom 创建doris视图 // custom 创建doris视图
if (datasetTable.getMode() == 1) { if (datasetTable.getMode() == 1) {
createDorisView(DorisTableUtils.dorisName(datasetTable.getId()), getCustomSQLDoris(dataTableInfoDTO, dataSetTableUnionService.listByTableId(dataTableInfoDTO.getList().get(0).getTableId()))); createDorisView(DorisTableUtils.dorisName(datasetTable.getId()), getCustomSQLDoris(dataTableInfoDTO,
dataSetTableUnionService.listByTableId(dataTableInfoDTO.getList().get(0).getTableId())));
} }
return; return;
} else { } else {
@ -1400,21 +1498,26 @@ public class DataSetTableService {
DatasourceRequest datasourceRequest = new DatasourceRequest(); DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds); datasourceRequest.setDatasource(ds);
DataTableInfoDTO dt = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class); DataTableInfoDTO dt = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class);
List<DataSetTableUnionDTO> list = dataSetTableUnionService.listByTableId(dt.getList().get(0).getTableId()); List<DataSetTableUnionDTO> list = dataSetTableUnionService
.listByTableId(dt.getList().get(0).getTableId());
String sqlAsTable = getCustomSQLDatasource(dt, list, ds); String sqlAsTable = getCustomSQLDatasource(dt, list, ds);
datasourceRequest.setQuery(sqlAsTable); datasourceRequest.setQuery(sqlAsTable);
fields = datasourceProvider.fetchResultField(datasourceRequest); fields = datasourceProvider.fetchResultField(datasourceRequest);
DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class); DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(dataSetTableRequest.getInfo(),
DataTableInfoDTO.class);
List<DataTableInfoCustomUnion> listField = dataTableInfoDTO.getList(); List<DataTableInfoCustomUnion> listField = dataTableInfoDTO.getList();
List<DatasetTableField> fieldList = new ArrayList<>(); List<DatasetTableField> fieldList = new ArrayList<>();
listField.forEach(ele -> { listField.forEach(ele -> {
List<DatasetTableField> listByIds = dataSetTableFieldsService.getListByIdsEach(ele.getCheckedFields()); List<DatasetTableField> listByIds = dataSetTableFieldsService
.getListByIdsEach(ele.getCheckedFields());
fieldList.addAll(listByIds); fieldList.addAll(listByIds);
}); });
for (DatasetTableField field : fieldList) { for (DatasetTableField field : fieldList) {
for (TableFiled tableFiled : fields) { for (TableFiled tableFiled : fields) {
if (StringUtils.equalsIgnoreCase(DorisTableUtils.dorisFieldName(field.getTableId() + "_" + field.getOriginName()), tableFiled.getFieldName())) { if (StringUtils.equalsIgnoreCase(
DorisTableUtils.dorisFieldName(field.getTableId() + "_" + field.getOriginName()),
tableFiled.getFieldName())) {
tableFiled.setRemarks(field.getName()); tableFiled.setRemarks(field.getName());
break; break;
} }
@ -1428,7 +1531,8 @@ public class DataSetTableService {
DatasourceRequest datasourceRequest = new DatasourceRequest(); DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds); datasourceRequest.setDatasource(ds);
// save field // save field
DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class); DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(dataSetTableRequest.getInfo(),
DataTableInfoDTO.class);
Map<String, Object> sqlMap = getUnionSQLDoris(dataTableInfoDTO); Map<String, Object> sqlMap = getUnionSQLDoris(dataTableInfoDTO);
String sql = (String) sqlMap.get("sql"); String sql = (String) sqlMap.get("sql");
List<DatasetTableField> fieldList = (List<DatasetTableField>) sqlMap.get("field"); List<DatasetTableField> fieldList = (List<DatasetTableField>) sqlMap.get("field");
@ -1441,7 +1545,9 @@ public class DataSetTableService {
fields = datasourceProvider.fetchResultField(datasourceRequest); fields = datasourceProvider.fetchResultField(datasourceRequest);
for (DatasetTableField field : fieldList) { for (DatasetTableField field : fieldList) {
for (TableFiled tableFiled : fields) { for (TableFiled tableFiled : fields) {
if (StringUtils.equalsIgnoreCase(DorisTableUtils.dorisFieldName(field.getTableId() + "_" + field.getDataeaseName()), tableFiled.getFieldName())) { if (StringUtils.equalsIgnoreCase(
DorisTableUtils.dorisFieldName(field.getTableId() + "_" + field.getDataeaseName()),
tableFiled.getFieldName())) {
tableFiled.setRemarks(field.getName()); tableFiled.setRemarks(field.getName());
break; break;
} }
@ -1463,7 +1569,9 @@ public class DataSetTableService {
for (DatasetTableField field : fieldList) { for (DatasetTableField field : fieldList) {
for (TableFiled tableFiled : fields) { for (TableFiled tableFiled : fields) {
if (StringUtils.equalsIgnoreCase(DorisTableUtils.dorisFieldNameShort(field.getTableId() + "_" + field.getOriginName()), tableFiled.getFieldName())) { if (StringUtils.equalsIgnoreCase(
DorisTableUtils.dorisFieldNameShort(field.getTableId() + "_" + field.getOriginName()),
tableFiled.getFieldName())) {
tableFiled.setRemarks(field.getName()); tableFiled.setRemarks(field.getName());
break; break;
} }
@ -1484,8 +1592,10 @@ public class DataSetTableService {
// 物理字段名设定为唯一，查询当前数据集下是否已存在该字段，存在则update，不存在则insert // 物理字段名设定为唯一，查询当前数据集下是否已存在该字段，存在则update，不存在则insert
DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample();
// 字段名一致，认为字段没有改变 // 字段名一致，认为字段没有改变
datasetTableFieldExample.createCriteria().andTableIdEqualTo(datasetTable.getId()).andOriginNameEqualTo(filed.getFieldName()); datasetTableFieldExample.createCriteria().andTableIdEqualTo(datasetTable.getId())
List<DatasetTableField> datasetTableFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); .andOriginNameEqualTo(filed.getFieldName());
List<DatasetTableField> datasetTableFields = datasetTableFieldMapper
.selectByExample(datasetTableFieldExample);
if (CollectionUtils.isNotEmpty(datasetTableFields)) { if (CollectionUtils.isNotEmpty(datasetTableFields)) {
datasetTableField.setId(datasetTableFields.get(0).getId()); datasetTableField.setId(datasetTableFields.get(0).getId());
datasetTableField.setOriginName(filed.getFieldName()); datasetTableField.setOriginName(filed.getFieldName());
@ -1526,7 +1636,8 @@ public class DataSetTableService {
} }
// delete 数据库中多余的字段 // delete 数据库中多余的字段
DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample();
datasetTableFieldExample.createCriteria().andTableIdEqualTo(datasetTable.getId()).andExtFieldEqualTo(0).andOriginNameNotIn(originNameList); datasetTableFieldExample.createCriteria().andTableIdEqualTo(datasetTable.getId()).andExtFieldEqualTo(0)
.andOriginNameNotIn(originNameList);
datasetTableFieldMapper.deleteByExample(datasetTableFieldExample); datasetTableFieldMapper.deleteByExample(datasetTableFieldExample);
} }
} }
@ -1560,7 +1671,8 @@ public class DataSetTableService {
} }
} }
public DatasetTableIncrementalConfig incrementalConfig(DatasetTableIncrementalConfig datasetTableIncrementalConfig) { public DatasetTableIncrementalConfig incrementalConfig(
DatasetTableIncrementalConfig datasetTableIncrementalConfig) {
if (StringUtils.isEmpty(datasetTableIncrementalConfig.getTableId())) { if (StringUtils.isEmpty(datasetTableIncrementalConfig.getTableId())) {
return new DatasetTableIncrementalConfig(); return new DatasetTableIncrementalConfig();
} }
@ -1581,7 +1693,6 @@ public class DataSetTableService {
return incrementalConfig(datasetTableIncrementalConfig); return incrementalConfig(datasetTableIncrementalConfig);
} }
public void saveIncrementalConfig(DatasetTableIncrementalConfig datasetTableIncrementalConfig) throws Exception { public void saveIncrementalConfig(DatasetTableIncrementalConfig datasetTableIncrementalConfig) throws Exception {
if (datasetTableIncrementalConfig == null || StringUtils.isEmpty(datasetTableIncrementalConfig.getTableId())) { if (datasetTableIncrementalConfig == null || StringUtils.isEmpty(datasetTableIncrementalConfig.getTableId())) {
return; return;
@ -1608,21 +1719,25 @@ public class DataSetTableService {
return o1.getColumnIndex().compareTo(o2.getColumnIndex()); return o1.getColumnIndex().compareTo(o2.getColumnIndex());
}); });
List<String> originNameFileds = datasetTableFields.stream().map(DatasetTableField::getOriginName).collect(Collectors.toList()); List<String> originNameFileds = datasetTableFields.stream().map(DatasetTableField::getOriginName)
.collect(Collectors.toList());
Datasource ds = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId()); Datasource ds = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId());
QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType());
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest(); DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds); datasourceRequest.setDatasource(ds);
if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) {// 增量添加 if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd())
String sql = datasetTableIncrementalConfig.getIncrementalAdd().replace(lastUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) {// 增量添加
String sql = datasetTableIncrementalConfig.getIncrementalAdd()
.replace(lastUpdateTime, Long.valueOf(System.currentTimeMillis()).toString())
.replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()); .replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString());
datasourceRequest.setQuery(qp.wrapSql(sql)); datasourceRequest.setQuery(qp.wrapSql(sql));
List<String> sqlFileds = new ArrayList<>(); List<String> sqlFileds = new ArrayList<>();
try { try {
datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName).forEach(filed -> { datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName)
sqlFileds.add(filed); .forEach(filed -> {
}); sqlFileds.add(filed);
});
} catch (Exception e) { } catch (Exception e) {
DataEaseException.throwException(Translator.get("i18n_check_sql_error") + e.getMessage()); DataEaseException.throwException(Translator.get("i18n_check_sql_error") + e.getMessage());
} }
@ -1631,13 +1746,16 @@ public class DataSetTableService {
DataEaseException.throwException(Translator.get("i18n_sql_add_not_matching") + sqlFileds.toString()); DataEaseException.throwException(Translator.get("i18n_sql_add_not_matching") + sqlFileds.toString());
} }
} }
if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete().replace(" ", ""))) {// 增量删除 if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete())
String sql = datasetTableIncrementalConfig.getIncrementalDelete().replace(lastUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete().replace(" ", ""))) {// 增量删除
String sql = datasetTableIncrementalConfig.getIncrementalDelete()
.replace(lastUpdateTime, Long.valueOf(System.currentTimeMillis()).toString())
.replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()); .replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString());
datasourceRequest.setQuery(qp.wrapSql(sql)); datasourceRequest.setQuery(qp.wrapSql(sql));
List<String> sqlFileds = new ArrayList<>(); List<String> sqlFileds = new ArrayList<>();
try { try {
datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName).forEach(filed -> sqlFileds.add(filed)); datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName)
.forEach(filed -> sqlFileds.add(filed));
} catch (Exception e) { } catch (Exception e) {
DataEaseException.throwException(Translator.get("i18n_check_sql_error") + e.getMessage()); DataEaseException.throwException(Translator.get("i18n_check_sql_error") + e.getMessage());
} }
@ -1686,7 +1804,8 @@ public class DataSetTableService {
if (StringUtils.isNotEmpty(tableId) && editType == 1) { if (StringUtils.isNotEmpty(tableId) && editType == 1) {
List<DatasetTableField> datasetTableFields = dataSetTableFieldsService.getFieldsByTableId(tableId); List<DatasetTableField> datasetTableFields = dataSetTableFieldsService.getFieldsByTableId(tableId);
datasetTableFields.stream().filter(datasetTableField -> datasetTableField.getExtField() == 0).collect(Collectors.toList()); datasetTableFields.stream().filter(datasetTableField -> datasetTableField.getExtField() == 0)
.collect(Collectors.toList());
datasetTableFields.sort((o1, o2) -> { datasetTableFields.sort((o1, o2) -> {
if (o1.getColumnIndex() == null) { if (o1.getColumnIndex() == null) {
return -1; return -1;
@ -1696,7 +1815,8 @@ public class DataSetTableService {
} }
return o1.getColumnIndex().compareTo(o2.getColumnIndex()); return o1.getColumnIndex().compareTo(o2.getColumnIndex());
}); });
List<String> oldFields = datasetTableFields.stream().map(DatasetTableField::getOriginName).collect(Collectors.toList()); List<String> oldFields = datasetTableFields.stream().map(DatasetTableField::getOriginName)
.collect(Collectors.toList());
for (ExcelSheetData excelSheetData : excelSheetDataList) { for (ExcelSheetData excelSheetData : excelSheetDataList) {
List<TableFiled> fields = excelSheetData.getFields(); List<TableFiled> fields = excelSheetData.getFields();
List<String> newFields = fields.stream().map(TableFiled::getRemarks).collect(Collectors.toList()); List<String> newFields = fields.stream().map(TableFiled::getRemarks).collect(Collectors.toList());
@ -1711,7 +1831,9 @@ public class DataSetTableService {
} else { } else {
retrunSheetDataList = excelSheetDataList; retrunSheetDataList = excelSheetDataList;
} }
retrunSheetDataList = retrunSheetDataList.stream().filter(excelSheetData -> CollectionUtils.isNotEmpty(excelSheetData.getFields())).collect(Collectors.toList()); retrunSheetDataList = retrunSheetDataList.stream()
.filter(excelSheetData -> CollectionUtils.isNotEmpty(excelSheetData.getFields()))
.collect(Collectors.toList());
// save file // save file
String excelId = UUID.randomUUID().toString(); String excelId = UUID.randomUUID().toString();
String filePath = saveFile(file, excelId); String filePath = saveFile(file, excelId);
@ -1738,7 +1860,8 @@ public class DataSetTableService {
return excelFileData; return excelFileData;
} }
private List<ExcelSheetData> parseExcel2(String filename, InputStream inputStream, boolean isPreview) throws Exception { private List<ExcelSheetData> parseExcel2(String filename, InputStream inputStream, boolean isPreview)
throws Exception {
List<ExcelSheetData> excelSheetDataList = new ArrayList<>(); List<ExcelSheetData> excelSheetDataList = new ArrayList<>();
String suffix = filename.substring(filename.lastIndexOf(".") + 1); String suffix = filename.substring(filename.lastIndexOf(".") + 1);
if (StringUtils.equalsIgnoreCase(suffix, "xls")) { if (StringUtils.equalsIgnoreCase(suffix, "xls")) {
@ -1754,7 +1877,8 @@ public class DataSetTableService {
inputStream.close(); inputStream.close();
excelSheetDataList.forEach(excelSheetData -> { excelSheetDataList.forEach(excelSheetData -> {
List<List<String>> data = excelSheetData.getData(); List<List<String>> data = excelSheetData.getData();
String[] fieldArray = excelSheetData.getFields().stream().map(TableFiled::getFieldName).toArray(String[]::new); String[] fieldArray = excelSheetData.getFields().stream().map(TableFiled::getFieldName)
.toArray(String[]::new);
List<Map<String, Object>> jsonArray = new ArrayList<>(); List<Map<String, Object>> jsonArray = new ArrayList<>();
if (CollectionUtils.isNotEmpty(data)) { if (CollectionUtils.isNotEmpty(data)) {
jsonArray = data.stream().map(ele -> { jsonArray = data.stream().map(ele -> {
@ -1772,7 +1896,8 @@ public class DataSetTableService {
return excelSheetDataList; return excelSheetDataList;
} }
private Map<String, Object> parseExcel(String filename, InputStream inputStream, boolean isPreview) throws Exception { private Map<String, Object> parseExcel(String filename, InputStream inputStream, boolean isPreview)
throws Exception {
String suffix = filename.substring(filename.lastIndexOf(".") + 1); String suffix = filename.substring(filename.lastIndexOf(".") + 1);
List<TableFiled> fields = new ArrayList<>(); List<TableFiled> fields = new ArrayList<>();
List<String[]> data = new ArrayList<>(); List<String[]> data = new ArrayList<>();
@ -1925,7 +2050,6 @@ public class DataSetTableService {
return map; return map;
} }
private String readCell(Cell cell, boolean cellType, TableFiled tableFiled) { private String readCell(Cell cell, boolean cellType, TableFiled tableFiled) {
if (cell == null) { if (cell == null) {
return ""; return "";
@ -1939,7 +2063,8 @@ public class DataSetTableService {
double eps = 1e-10; double eps = 1e-10;
if (value - Math.floor(value) < eps) { if (value - Math.floor(value) < eps) {
if (cellType) { if (cellType) {
if (StringUtils.isEmpty(tableFiled.getFieldType()) || tableFiled.getFieldType().equalsIgnoreCase("TEXT")) { if (StringUtils.isEmpty(tableFiled.getFieldType())
|| tableFiled.getFieldType().equalsIgnoreCase("TEXT")) {
tableFiled.setFieldType("LONG"); tableFiled.setFieldType("LONG");
} }
} }
@ -1990,7 +2115,8 @@ public class DataSetTableService {
double eps = 1e-10; double eps = 1e-10;
if (value - Math.floor(value) < eps) { if (value - Math.floor(value) < eps) {
if (cellType) { if (cellType) {
if (StringUtils.isEmpty(tableFiled.getFieldType()) || tableFiled.getFieldType().equalsIgnoreCase("TEXT")) { if (StringUtils.isEmpty(tableFiled.getFieldType())
|| tableFiled.getFieldType().equalsIgnoreCase("TEXT")) {
tableFiled.setFieldType("LONG"); tableFiled.setFieldType("LONG");
} }
} }
@ -2048,13 +2174,17 @@ public class DataSetTableService {
public void updateDatasetTableStatus() { public void updateDatasetTableStatus() {
List<QrtzSchedulerState> qrtzSchedulerStates = qrtzSchedulerStateMapper.selectByExample(null); List<QrtzSchedulerState> qrtzSchedulerStates = qrtzSchedulerStateMapper.selectByExample(null);
List<String> activeQrtzInstances = qrtzSchedulerStates.stream().filter(qrtzSchedulerState -> qrtzSchedulerState.getLastCheckinTime() + qrtzSchedulerState.getCheckinInterval() + 1000 > utilMapper.currentTimestamp()).map(QrtzSchedulerStateKey::getInstanceName).collect(Collectors.toList()); List<String> activeQrtzInstances = qrtzSchedulerStates.stream()
.filter(qrtzSchedulerState -> qrtzSchedulerState.getLastCheckinTime()
+ qrtzSchedulerState.getCheckinInterval() + 1000 > utilMapper.currentTimestamp())
.map(QrtzSchedulerStateKey::getInstanceName).collect(Collectors.toList());
List<DatasetTable> jobStoppeddDatasetTables = new ArrayList<>(); List<DatasetTable> jobStoppeddDatasetTables = new ArrayList<>();
DatasetTableExample example = new DatasetTableExample(); DatasetTableExample example = new DatasetTableExample();
example.createCriteria().andSyncStatusEqualTo(JobStatus.Underway.name()); example.createCriteria().andSyncStatusEqualTo(JobStatus.Underway.name());
datasetTableMapper.selectByExample(example).forEach(datasetTable -> { datasetTableMapper.selectByExample(example).forEach(datasetTable -> {
if (StringUtils.isEmpty(datasetTable.getQrtzInstance()) || !activeQrtzInstances.contains(datasetTable.getQrtzInstance().substring(0, datasetTable.getQrtzInstance().length() - 13))) { if (StringUtils.isEmpty(datasetTable.getQrtzInstance()) || !activeQrtzInstances.contains(
datasetTable.getQrtzInstance().substring(0, datasetTable.getQrtzInstance().length() - 13))) {
jobStoppeddDatasetTables.add(datasetTable); jobStoppeddDatasetTables.add(datasetTable);
} }
}); });
@ -2066,7 +2196,8 @@ public class DataSetTableService {
DatasetTable record = new DatasetTable(); DatasetTable record = new DatasetTable();
record.setSyncStatus(JobStatus.Error.name()); record.setSyncStatus(JobStatus.Error.name());
example.clear(); example.clear();
example.createCriteria().andSyncStatusEqualTo(JobStatus.Underway.name()).andIdIn(jobStoppeddDatasetTables.stream().map(DatasetTable::getId).collect(Collectors.toList())); example.createCriteria().andSyncStatusEqualTo(JobStatus.Underway.name())
.andIdIn(jobStoppeddDatasetTables.stream().map(DatasetTable::getId).collect(Collectors.toList()));
datasetTableMapper.updateByExampleSelective(record, example); datasetTableMapper.updateByExampleSelective(record, example);
DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog(); DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
@ -2074,8 +2205,10 @@ public class DataSetTableService {
datasetTableTaskLog.setInfo("Job stopped due to system error."); datasetTableTaskLog.setInfo("Job stopped due to system error.");
DatasetTableTaskLogExample datasetTableTaskLogExample = new DatasetTableTaskLogExample(); DatasetTableTaskLogExample datasetTableTaskLogExample = new DatasetTableTaskLogExample();
datasetTableTaskLogExample.createCriteria().andStatusEqualTo(JobStatus.Underway.name()).andTableIdIn(jobStoppeddDatasetTables.stream().map(DatasetTable::getId).collect(Collectors.toList())); datasetTableTaskLogExample.createCriteria().andStatusEqualTo(JobStatus.Underway.name())
List<String> taskIds = datasetTableTaskLogMapper.selectByExample(datasetTableTaskLogExample).stream().map(DatasetTableTaskLog::getTaskId).collect(Collectors.toList()); .andTableIdIn(jobStoppeddDatasetTables.stream().map(DatasetTable::getId).collect(Collectors.toList()));
List<String> taskIds = datasetTableTaskLogMapper.selectByExample(datasetTableTaskLogExample).stream()
.map(DatasetTableTaskLog::getTaskId).collect(Collectors.toList());
datasetTableTaskLogMapper.updateByExampleSelective(datasetTableTaskLog, datasetTableTaskLogExample); datasetTableTaskLogMapper.updateByExampleSelective(datasetTableTaskLog, datasetTableTaskLogExample);
dataSetTableTaskService.updateTaskStatus(taskIds, JobStatus.Error); dataSetTableTaskService.updateTaskStatus(taskIds, JobStatus.Error);

14
frontend/src/components/widget/DeWidget/DeSelectGrid.vue
View File

@ -127,14 +127,20 @@ export default {
}, },
'element.options.attrs.multiple': function(value, old) { 'element.options.attrs.multiple': function(value, old) {
if (typeof old === 'undefined' || value === old) return if (typeof old === 'undefined' || value === old) return
// if (!this.inDraw) { if (!this.inDraw) {
this.value = value ? [] : null this.value = value ? [] : null
this.element.options.value = '' this.element.options.value = ''
// } } else {
this.value = this.fillValueDerfault()
}
this.show = false this.show = false
this.$nextTick(() => { this.$nextTick(() => {
this.show = true this.show = true
if (value) {
this.checkAll = this.value.length === this.datas.length
this.isIndeterminate = this.value.length > 0 && this.value.length < this.datas.length
}
}) })
} }
}, },

5
frontend/src/views/system/plugin/dynamic.vue
View File

@ -1,5 +1,5 @@
<template> <template>
<layout-content v-if="!noLayout" v-loading="$store.getters.loadingMap[$store.getters.currentPath]" :header="header" :back-name="backName"> <layout-content v-if="!noLayout" v-loading="jsname && !innerLoadingNames.includes(jsname) && $store.getters.loadingMap[$store.getters.currentPath]" :header="header" :back-name="backName">
<async-component v-if="showAsync" :url="url" @execute-axios="executeAxios" @on-add-languanges="addLanguages" @on-plugin-layout="setLayoutInfo" @plugin-call-back="pluginCallBack" /> <async-component v-if="showAsync" :url="url" @execute-axios="executeAxios" @on-add-languanges="addLanguages" @on-plugin-layout="setLayoutInfo" @plugin-call-back="pluginCallBack" />
<div v-else> <div v-else>
<h1>未知组件无法展示</h1> <h1>未知组件无法展示</h1>
@ -46,7 +46,8 @@ export default {
header: null, header: null,
backName: null, backName: null,
baseUrl: '/api/pluginCommon/async/', baseUrl: '/api/pluginCommon/async/',
url: null url: null,
innerLoadingNames: ['SystemDept', 'SystemRole']
} }
}, },
created() { created() {

9
frontend/src/views/system/user/index.vue
View File

@ -1,7 +1,8 @@
<template> <template>
<layout-content v-loading="$store.getters.loadingMap[$store.getters.currentPath]"> <layout-content>
<complex-table <complex-table
v-if="canLoadDom" v-if="canLoadDom"
v-loading="$store.getters.loadingMap[$store.getters.currentPath]"
:data="data" :data="data"
:columns="columns" :columns="columns"
local-key="userGrid" local-key="userGrid"
@ -107,9 +108,9 @@
:load-options="loadDepts" :load-options="loadDepts"
style="width: 430px" style="width: 430px"
:placeholder="$t('user.choose_org')" :placeholder="$t('user.choose_org')"
:noChildrenText="$t('commons.treeselect.no_children_text')" :no-children-text="$t('commons.treeselect.no_children_text')"
:noOptionsText="$t('commons.treeselect.no_options_text')" :no-options-text="$t('commons.treeselect.no_options_text')"
:noResultsText="$t('commons.treeselect.no_results_text')" :no-results-text="$t('commons.treeselect.no_results_text')"
/> />
</el-form-item> </el-form-item>
<el-form-item style="margin-bottom: 0;" :label="$t('commons.role')" prop="roleIds"> <el-form-item style="margin-bottom: 0;" :label="$t('commons.role')" prop="roleIds">