github/dataease
github/dataease
3
0
Fork 1
mirror of https://github.com/dataease/dataease.git synced 2025-02-24 11:32:57 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

perf(X-Pack): 数据源应用数量控制

Browse Source
This commit is contained in:
fit2cloud-chenyw 2024-12-01 14:44:36 +08:00
parent 310fd0151f
commit bb01077806
5 changed files with 36 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java
View File

@ -16,6 +16,7 @@ import io.dataease.dataset.utils.FieldUtils;
import io.dataease.dataset.utils.TableUtils;
import io.dataease.datasource.dao.auto.entity.CoreDatasource;
import io.dataease.datasource.dao.auto.mapper.CoreDatasourceMapper;
import io.dataease.datasource.manage.DataSourceManage;
import io.dataease.datasource.manage.EngineManage;
import io.dataease.datasource.utils.DatasourceUtils;
import io.dataease.engine.constant.ExtFieldConstant;
@ -80,6 +81,9 @@ public class DatasetDataManage {
@Resource
private CorePermissionManage corePermissionManage;
@Resource
private DataSourceManage dataSourceManage;
private static Logger logger = LoggerFactory.getLogger(DatasetDataManage.class);
public static final List<String> notFullDs = List.of("mysql", "mariadb", "Excel", "API");
@ -90,7 +94,7 @@ public class DatasetDataManage {
String type = datasetTableDTO.getType();
DatasetTableInfoDTO tableInfoDTO = JsonUtil.parseObject(datasetTableDTO.getInfo(), DatasetTableInfoDTO.class);
if (StringUtils.equalsIgnoreCase(type, DatasetTableType.DB) || StringUtils.equalsIgnoreCase(type, DatasetTableType.SQL)) {
CoreDatasource coreDatasource = coreDatasourceMapper.selectById(datasetTableDTO.getDatasourceId());
CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(datasetTableDTO.getDatasourceId());
DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO();
if (StringUtils.equalsIgnoreCase("excel", coreDatasource.getType()) || StringUtils.equalsIgnoreCase("api", coreDatasource.getType())) {
coreDatasource = engineManage.getDeEngine();
@ -133,7 +137,7 @@ public class DatasetDataManage {
tableFields = provider.fetchTableField(datasourceRequest);
} else if (StringUtils.equalsIgnoreCase(type, DatasetTableType.Es)) {
CoreDatasource coreDatasource = coreDatasourceMapper.selectById(datasetTableDTO.getDatasourceId());
CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(datasetTableDTO.getDatasourceId());
Provider provider = ProviderFactory.getProvider(type);
DatasourceRequest datasourceRequest = new DatasourceRequest();
DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO();
@ -381,7 +385,7 @@ public class DatasetDataManage {
}
public Map<String, Object> previewSql(PreviewSqlDTO dto) throws DEException {
CoreDatasource coreDatasource = coreDatasourceMapper.selectById(dto.getDatasourceId());
CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(dto.getDatasourceId());
DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO();
if (coreDatasource.getType().equalsIgnoreCase("API") || coreDatasource.getType().equalsIgnoreCase("Excel")) {
BeanUtils.copyBean(datasourceSchemaDTO, engineManage.getDeEngine());

10
core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLManage.java
View File

@ -10,6 +10,7 @@ import io.dataease.dataset.utils.SqlUtils;
import io.dataease.dataset.utils.TableUtils;
import io.dataease.datasource.dao.auto.entity.CoreDatasource;
import io.dataease.datasource.dao.auto.mapper.CoreDatasourceMapper;
import io.dataease.datasource.manage.DataSourceManage;
import io.dataease.datasource.manage.EngineManage;
import io.dataease.engine.constant.ExtFieldConstant;
import io.dataease.engine.constant.SQLConstants;
@ -62,6 +63,9 @@ public class DatasetSQLManage {
@Autowired(required = false)
private PluginManageApi pluginManage;
@Resource
private DataSourceManage dataSourceManage;
private static Logger logger = LoggerFactory.getLogger(DatasetSQLManage.class);
private List<SqlVariableDetails> filterParameters(ChartExtRequest chartExtRequest, Long datasetTableId) {
@ -379,7 +383,7 @@ public class DatasetSQLManage {
DatasourceSchemaDTO datasourceSchemaDTO = dsMap.get(datasourceId);
String type;
if (datasourceSchemaDTO == null) {
CoreDatasource coreDatasource = coreDatasourceMapper.selectById(datasourceId);
CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(datasourceId);
if (coreDatasource == null) {
DEException.throwException(Translator.get("i18n_dataset_ds_error") + ",ID:" + datasourceId);
}
@ -479,7 +483,7 @@ public class DatasetSQLManage {
String schemaAlias;
if (StringUtils.equalsIgnoreCase(ds.getType(), DatasetTableType.DB) || StringUtils.equalsIgnoreCase(ds.getType(), DatasetTableType.SQL)) {
CoreDatasource coreDatasource = coreDatasourceMapper.selectById(ds.getDatasourceId());
CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(ds.getDatasourceId());
if (coreDatasource == null) {
DEException.throwException(Translator.get("i18n_dataset_ds_error") + ",ID:" + ds.getDatasourceId());
}
@ -501,7 +505,7 @@ public class DatasetSQLManage {
dsMap.put(coreDatasource.getId(), datasourceSchemaDTO);
}
} else if (StringUtils.equalsIgnoreCase(ds.getType(), DatasetTableType.Es)) {
CoreDatasource coreDatasource = coreDatasourceMapper.selectById(ds.getDatasourceId());
CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(ds.getDatasourceId());
schemaAlias = String.format(SQLConstants.SCHEMA, coreDatasource.getId());
if (!dsMap.containsKey(coreDatasource.getId())) {
DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO();

11
core/core-backend/src/main/java/io/dataease/datasource/manage/DataSourceManage.java
View File

@ -140,7 +140,7 @@ public class DataSourceManage {
public void move(DatasourceDTO dataSourceDTO) {
Long id = dataSourceDTO.getId();
CoreDatasource sourceData = null;
if (ObjectUtils.isEmpty(id) || ObjectUtils.isEmpty(sourceData = coreDatasourceMapper.selectById(id))) {
if (ObjectUtils.isEmpty(id) || ObjectUtils.isEmpty(sourceData = getCoreDatasource(id))) {
DEException.throwException("resource not exist");
}
checkName(dataSourceDTO);
@ -157,14 +157,19 @@ public class DataSourceManage {
}
public void encryptDsConfig(){
public void encryptDsConfig() {
coreDatasourceMapper.selectList(null).forEach(dataSource -> {
coreDatasourceMapper.updateById(dataSource);
});
}
@XpackInteract(value = "datasourceResourceTree", before = false)
public CoreDatasource getCoreDatasource(Long id) {
return coreDatasourceMapper.selectById(id);
}
public DatasourceDTO getDs(Long id) {
CoreDatasource coreDatasource = coreDatasourceMapper.selectById(id);
CoreDatasource coreDatasource = getCoreDatasource(id);
DatasourceDTO dto = new DatasourceDTO();
BeanUtils.copyBean(dto, coreDatasource);
return dto;

28
core/core-backend/src/main/java/io/dataease/datasource/server/DatasourceServer.java
View File

@ -107,8 +107,6 @@ public class DatasourceServer implements DatasourceApi {
private PluginManageApi pluginManage;
@Autowired(required = false)
private RelationApi relationManage;
@Autowired
private CoreDatasourceMapper coreDatasourceMapper;
public enum UpdateType {
all_scope, add_scope
@ -208,7 +206,7 @@ public class DatasourceServer implements DatasourceApi {
if (StringUtils.isEmpty(dataSourceDTO.getName())) {
DEException.throwException("名称不能为空！");
}
CoreDatasource datasource = datasourceMapper.selectById(dataSourceDTO.getId());
CoreDatasource datasource = dataSourceManage.getCoreDatasource(dataSourceDTO.getId());
datasource.setName(dataSourceDTO.getName());
dataSourceDTO.setPid(datasource.getPid());
dataSourceManage.checkName(dataSourceDTO);
@ -490,7 +488,7 @@ public class DatasourceServer implements DatasourceApi {
@Override
public DatasourceDTO getSimpleDs(Long datasourceId) throws DEException {
CoreDatasource datasource = datasourceMapper.selectById(datasourceId);
CoreDatasource datasource = dataSourceManage.getCoreDatasource(datasourceId);
if (datasource == null) {
DEException.throwException("不存在的数据源！");
}
@ -517,7 +515,7 @@ public class DatasourceServer implements DatasourceApi {
@Override
public String getName(Long datasourceId) throws DEException {
CoreDatasource datasource = datasourceMapper.selectById(datasourceId);
CoreDatasource datasource = dataSourceManage.getCoreDatasource(datasourceId);
if (datasource == null) {
DEException.throwException("不存在的数据源！");
}
@ -578,7 +576,7 @@ public class DatasourceServer implements DatasourceApi {
}
public void recursionDel(Long datasourceId) throws DEException {
CoreDatasource coreDatasource = datasourceMapper.selectById(datasourceId);
CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(datasourceId);
if (ObjectUtils.isEmpty(coreDatasource)) {
return;
}
@ -635,7 +633,7 @@ public class DatasourceServer implements DatasourceApi {
@Override
public DatasourceDTO validate(Long datasourceId) throws DEException {
CoreDatasource coreDatasource = new CoreDatasource();
BeanUtils.copyBean(coreDatasource, datasourceMapper.selectById(datasourceId));
BeanUtils.copyBean(coreDatasource, dataSourceManage.getCoreDatasource(datasourceId));
return validate(coreDatasource);
}
@ -671,7 +669,7 @@ public class DatasourceServer implements DatasourceApi {
@Override
public List<DatasetTableDTO> getTables(DatasetTableDTO datasetTableDTO) throws DEException {
CoreDatasource coreDatasource = datasourceMapper.selectById(datasetTableDTO.getDatasourceId());
CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(datasetTableDTO.getDatasourceId());
if (coreDatasource == null) {
DEException.throwException("无效数据源！");
}
@ -706,7 +704,7 @@ public class DatasourceServer implements DatasourceApi {
if (!getTables(datasetTableDTO).stream().map(DatasetTableDTO::getTableName).collect(Collectors.toList()).contains(tableName)) {
DEException.throwException("无效的表名！");
}
CoreDatasource coreDatasource = datasourceMapper.selectById(datasourceId);
CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(Long.parseLong(datasourceId));
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(transDTO(coreDatasource));
if (coreDatasource.getType().equals("API") || coreDatasource.getType().equals("Excel")) {
@ -746,7 +744,7 @@ public class DatasourceServer implements DatasourceApi {
public void syncApiDs(Map<String, String> req) throws Exception {
Long datasourceId = Long.valueOf(req.get("datasourceId"));
CoreDatasourceTask coreDatasourceTask = datasourceTaskServer.selectByDSId(datasourceId);
CoreDatasource coreDatasource = datasourceMapper.selectById(datasourceId);
CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(datasourceId);
DatasourceServer.UpdateType updateType = DatasourceServer.UpdateType.valueOf(coreDatasourceTask.getUpdateType());
datasourceSyncManage.extractedData(null, coreDatasource, updateType, MANUAL.toString());
}
@ -774,7 +772,7 @@ public class DatasourceServer implements DatasourceApi {
private static final Integer append = 1;
public ExcelFileData excelUpload(@RequestParam("file") MultipartFile file, @RequestParam("id") long datasourceId, @RequestParam("editType") Integer editType) throws DEException {
CoreDatasource coreDatasource = datasourceMapper.selectById(datasourceId);
CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(datasourceId);
ExcelUtils excelUtils = new ExcelUtils();
ExcelFileData excelFileData = excelUtils.excelSaveAndParse(file);
@ -983,7 +981,7 @@ public class DatasourceServer implements DatasourceApi {
wrapper.orderByDesc("start_time");
Page<CoreDatasourceTaskLogDTO> page = new Page<>(goPage, pageSize);
IPage<CoreDatasourceTaskLogDTO> pager = taskLogExtMapper.pager(page, wrapper);
CoreDatasource coreDatasource = datasourceMapper.selectById(dsId);
CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(dsId);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(transDTO(coreDatasource));
List<DatasetTableDTO> datasetTableDTOS = ApiUtils.getTables(datasourceRequest);
@ -1083,7 +1081,7 @@ public class DatasourceServer implements DatasourceApi {
}
private void getParents(Long pid, List<Long> ids) {
CoreDatasource parent = datasourceMapper.selectById(pid);// 查找父级folder
CoreDatasource parent = dataSourceManage.getCoreDatasource(pid);// 查找父级folder
ids.add(parent.getId());
if (parent.getPid() != null && parent.getPid() != 0) {
getParents(parent.getPid(), ids);
@ -1138,7 +1136,7 @@ public class DatasourceServer implements DatasourceApi {
}
private DatasourceDTO getDatasourceDTOById(Long datasourceId, boolean hidePw) throws DEException {
CoreDatasource datasource = datasourceMapper.selectById(datasourceId);
CoreDatasource datasource = dataSourceManage.getCoreDatasource(datasourceId);
if (datasource == null) {
DEException.throwException("不存在的数据源！");
}
@ -1248,7 +1246,7 @@ public class DatasourceServer implements DatasourceApi {
@Override
public DsSimpleVO simple(Long id) {
if (ObjectUtils.isEmpty(id)) DEException.throwException("id is null");
CoreDatasource coreDatasource = coreDatasourceMapper.selectById(id);
CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(id);
if (ObjectUtils.isEmpty(coreDatasource)) return null;
DsSimpleVO vo = new DsSimpleVO();
vo.setName(coreDatasource.getName());

2
de-xpack

@ -1 +1 @@
Subproject commit fd5754d9d6d6615f9ae49addc9bbf1ad9839e29f
Subproject commit ef5980c7a78b82eb17c9a1376ad7258806014636