refactor: 数据源接口抽象(部分)

This commit is contained in:
junjun 2024-06-26 15:35:43 +08:00
parent ae0d9387b3
commit b8bc449b11
61 changed files with 266 additions and 195 deletions

View File

@ -1,21 +1,21 @@
package io.dataease.chart.charts.impl;
import io.dataease.chart.charts.AbstractChartHandler;
import io.dataease.chart.charts.ChartHandlerManager;
import io.dataease.chart.constant.ChartConstants;
import io.dataease.chart.manage.ChartViewManege;
import io.dataease.chart.utils.ChartDataBuild;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.dataset.manage.DatasetTableFieldManage;
import io.dataease.dataset.utils.SqlUtils;
import io.dataease.datasource.provider.CalciteProvider;
import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.sql.SQLProvider;
import io.dataease.engine.trans.Dimension2SQLObj;
import io.dataease.engine.trans.Quota2SQLObj;
import io.dataease.engine.utils.Utils;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.dto.*;
import io.dataease.chart.charts.ChartHandlerManager;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.util.ChartDataUtil;
import io.dataease.extensions.view.util.FieldUtil;
@ -33,7 +33,6 @@ import java.math.RoundingMode;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Component
public class DefaultChartHandler extends AbstractChartHandler {
@ -49,7 +48,7 @@ public class DefaultChartHandler extends AbstractChartHandler {
private String type = "*";
@PostConstruct
public void init(){
public void init() {
chartHandlerManager.registerChartHandler(this.getRender(), this.getType(), this);
}
@ -392,7 +391,7 @@ public class DefaultChartHandler extends AbstractChartHandler {
return false;
}
protected boolean checkYoyFilter(List<ChartExtFilterDTO> filter, List<ChartViewFieldDTO> yoyAxis){
protected boolean checkYoyFilter(List<ChartExtFilterDTO> filter, List<ChartViewFieldDTO> yoyAxis) {
boolean flag = false;
for (ChartExtFilterDTO filterDTO : filter) {
for (ChartViewFieldDTO chartViewFieldDTO : yoyAxis) {
@ -419,7 +418,9 @@ public class DefaultChartHandler extends AbstractChartHandler {
}
}
return flag;
};
}
;
protected void groupStackDrill(List<ChartViewFieldDTO> xAxis,
List<ChartExtFilterDTO> filterList,

View File

@ -2,13 +2,13 @@ package io.dataease.chart.charts.impl;
import com.fasterxml.jackson.core.type.TypeReference;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.dataset.utils.SqlUtils;
import io.dataease.datasource.provider.CalciteProvider;
import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.engine.sql.SQLProvider;
import io.dataease.engine.trans.ExtWhere2Str;
import io.dataease.engine.utils.Utils;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.dto.*;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.util.FieldUtil;
@ -54,10 +54,11 @@ public class YoyChartHandler extends DefaultChartHandler {
/**
* 构建同环比类型的数据
* @param view 视图对象
*
* @param view 视图对象
* @param formatResult 处理后的轴
* @param filterResult 处理后的过滤器
* @param data 原始数据
* @param data 原始数据
* @return 视图构建结果
*/
public Map<String, Object> buildNormalResult(ChartViewDTO view, AxisFormatResult formatResult, CustomFilterResult filterResult, List<String[]> data) {

View File

@ -1,28 +1,18 @@
package io.dataease.chart.charts.impl.bar;
import com.fasterxml.jackson.core.type.TypeReference;
import io.dataease.chart.charts.impl.YoyChartHandler;
import io.dataease.chart.constant.ChartConstants;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.dataset.utils.SqlUtils;
import io.dataease.datasource.provider.CalciteProvider;
import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.engine.sql.SQLProvider;
import io.dataease.engine.trans.ExtWhere2Str;
import io.dataease.engine.utils.SQLUtils;
import io.dataease.engine.utils.Utils;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.dto.*;
import io.dataease.chart.charts.impl.DefaultChartHandler;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.util.FieldUtil;
import io.dataease.utils.JsonUtil;
import lombok.Getter;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import java.util.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@Component
public class BarHandler extends YoyChartHandler {

View File

@ -1,17 +1,15 @@
package io.dataease.chart.charts.impl.line;
import io.dataease.chart.charts.impl.GroupChartHandler;
import io.dataease.chart.charts.impl.YoyChartHandler;
import io.dataease.chart.utils.ChartDataBuild;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.datasource.provider.CalciteProvider;
import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.engine.utils.Utils;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.dto.*;
import io.dataease.extensions.view.model.SQLMeta;
import lombok.Getter;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import java.util.ArrayList;

View File

@ -2,16 +2,15 @@ package io.dataease.chart.charts.impl.line;
import io.dataease.chart.charts.impl.YoyChartHandler;
import io.dataease.chart.utils.ChartDataBuild;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.datasource.provider.CalciteProvider;
import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.engine.utils.Utils;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.dto.*;
import io.dataease.extensions.view.model.SQLMeta;
import lombok.Getter;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import java.util.ArrayList;

View File

@ -2,14 +2,14 @@ package io.dataease.chart.charts.impl.numberic;
import io.dataease.chart.charts.impl.DefaultChartHandler;
import io.dataease.chart.utils.ChartDataBuild;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.dataset.utils.SqlUtils;
import io.dataease.datasource.provider.CalciteProvider;
import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.engine.sql.SQLProvider;
import io.dataease.engine.trans.Quota2SQLObj;
import io.dataease.engine.utils.Utils;
import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.dto.*;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.util.FieldUtil;
@ -17,7 +17,6 @@ import io.dataease.i18n.Translator;
import io.dataease.utils.BeanUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;

View File

@ -2,14 +2,13 @@ package io.dataease.chart.charts.impl.table;
import io.dataease.api.chart.dto.PageInfo;
import io.dataease.chart.charts.impl.DefaultChartHandler;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.dataset.utils.SqlUtils;
import io.dataease.datasource.provider.CalciteProvider;
import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.engine.sql.SQLProvider;
import io.dataease.engine.trans.Dimension2SQLObj;
import io.dataease.engine.trans.Quota2SQLObj;
import io.dataease.engine.utils.Utils;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.dto.*;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.util.ChartDataUtil;

View File

@ -11,20 +11,20 @@ import io.dataease.chart.charts.ChartHandlerManager;
import io.dataease.chart.constant.ChartConstants;
import io.dataease.chart.utils.ChartDataBuild;
import io.dataease.constant.AuthEnum;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.dataset.manage.DatasetGroupManage;
import io.dataease.dataset.manage.DatasetSQLManage;
import io.dataease.dataset.manage.DatasetTableFieldManage;
import io.dataease.dataset.manage.PermissionManage;
import io.dataease.dataset.utils.SqlUtils;
import io.dataease.datasource.provider.CalciteProvider;
import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.sql.SQLProvider;
import io.dataease.engine.trans.*;
import io.dataease.engine.utils.SQLUtils;
import io.dataease.engine.utils.Utils;
import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.dto.*;
import io.dataease.extensions.view.factory.PluginsChartFactory;
import io.dataease.extensions.view.filter.FilterTreeObj;

View File

@ -1,9 +1,9 @@
package io.dataease.commons.utils;
import com.fasterxml.jackson.core.type.TypeReference;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.dto.SqlVariableDetails;
import io.dataease.api.ds.vo.DatasourceConfiguration;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.exception.DEException;
import io.dataease.i18n.Translator;
import io.dataease.utils.JsonUtil;

View File

@ -1,12 +0,0 @@
package io.dataease.dataset.dto;
import io.dataease.datasource.dao.auto.entity.CoreDatasource;
import lombok.Data;
/**
* @Author Junjun
*/
@Data
public class DatasourceSchemaDTO extends CoreDatasource {
private String schemaAlias;
}

View File

@ -5,20 +5,17 @@ import io.dataease.api.chart.dto.DeSortField;
import io.dataease.api.dataset.dto.*;
import io.dataease.api.dataset.union.DatasetGroupInfoDTO;
import io.dataease.api.dataset.union.DatasetTableInfoDTO;
import io.dataease.api.ds.vo.TableField;
import io.dataease.api.permissions.dataset.dto.DataSetRowPermissionsTreeDTO;
import io.dataease.auth.bo.TokenUserBO;
import io.dataease.chart.manage.ChartViewManege;
import io.dataease.chart.utils.ChartDataBuild;
import io.dataease.commons.utils.SqlparserUtils;
import io.dataease.dataset.constant.DatasetTableType;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.dataset.utils.*;
import io.dataease.datasource.dao.auto.entity.CoreDatasource;
import io.dataease.datasource.dao.auto.mapper.CoreDatasourceMapper;
import io.dataease.datasource.manage.EngineManage;
import io.dataease.datasource.provider.CalciteProvider;
import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.datasource.utils.DatasourceUtils;
import io.dataease.engine.constant.ExtFieldConstant;
import io.dataease.engine.constant.SQLConstants;
@ -28,6 +25,10 @@ import io.dataease.engine.trans.*;
import io.dataease.engine.utils.SQLUtils;
import io.dataease.engine.utils.Utils;
import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.dto.DatasetTableDTO;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.datasource.dto.TableField;
import io.dataease.extensions.view.dto.ChartExtFilterDTO;
import io.dataease.extensions.view.dto.ChartExtRequest;
import io.dataease.extensions.view.dto.DatasetTableFieldDTO;

View File

@ -2,11 +2,11 @@ package io.dataease.dataset.manage;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import io.dataease.api.dataset.dto.DatasetTableDTO;
import io.dataease.extensions.datasource.dto.DatasetTableDTO;
import io.dataease.extensions.datasource.dto.DatasourceDTO;
import io.dataease.api.dataset.union.DatasetGroupInfoDTO;
import io.dataease.api.dataset.union.UnionDTO;
import io.dataease.api.dataset.vo.DataSetBarVO;
import io.dataease.api.ds.vo.DatasourceDTO;
import io.dataease.commons.constants.OptConstants;
import io.dataease.dataset.dao.auto.entity.CoreDatasetGroup;
import io.dataease.dataset.dao.auto.entity.CoreDatasetTable;

View File

@ -1,8 +1,9 @@
package io.dataease.dataset.manage;
import io.dataease.extensions.datasource.dto.DatasetTableDTO;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.dto.ChartExtFilterDTO;
import io.dataease.extensions.view.dto.ChartExtRequest;
import io.dataease.api.dataset.dto.DatasetTableDTO;
import io.dataease.extensions.view.dto.SqlVariableDetails;
import io.dataease.api.dataset.union.*;
import io.dataease.extensions.view.model.SQLObj;
@ -11,7 +12,6 @@ import io.dataease.api.permissions.auth.dto.BusiPerCheckDTO;
import io.dataease.commons.utils.SqlparserUtils;
import io.dataease.constant.AuthEnum;
import io.dataease.dataset.constant.DatasetTableType;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.dataset.utils.DatasetTableTypeConstants;
import io.dataease.dataset.utils.SqlUtils;
import io.dataease.dataset.utils.TableUtils;

View File

@ -1,10 +1,10 @@
package io.dataease.dataset.manage;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import io.dataease.api.dataset.dto.DatasetTableDTO;
import io.dataease.dataset.dao.auto.entity.CoreDatasetTable;
import io.dataease.dataset.dao.auto.mapper.CoreDatasetTableMapper;
import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.dto.DatasetTableDTO;
import io.dataease.i18n.Translator;
import io.dataease.utils.BeanUtils;
import io.dataease.utils.IDUtils;

View File

@ -2,11 +2,11 @@ package io.dataease.dataset.server;
import io.dataease.api.dataset.DatasetDataApi;
import io.dataease.api.dataset.dto.BaseTreeNodeDTO;
import io.dataease.api.dataset.dto.DatasetTableDTO;
import io.dataease.api.dataset.dto.EnumValueRequest;
import io.dataease.api.dataset.dto.PreviewSqlDTO;
import io.dataease.api.dataset.union.DatasetGroupInfoDTO;
import io.dataease.dataset.manage.DatasetDataManage;
import io.dataease.extensions.datasource.dto.DatasetTableDTO;
import io.dataease.extensions.view.dto.DatasetTableFieldDTO;
import io.dataease.utils.LogUtil;
import jakarta.annotation.Resource;

View File

@ -2,7 +2,7 @@ package io.dataease.dataset.server;
import io.dataease.api.dataset.DatasetTreeApi;
import io.dataease.api.dataset.dto.DatasetNodeDTO;
import io.dataease.api.dataset.dto.DatasetTableDTO;
import io.dataease.extensions.datasource.dto.DatasetTableDTO;
import io.dataease.extensions.view.dto.SqlVariableDetails;
import io.dataease.api.dataset.union.DatasetGroupInfoDTO;
import io.dataease.api.dataset.vo.DataSetBarVO;

View File

@ -1,9 +1,9 @@
package io.dataease.dataset.utils;
import com.google.common.collect.ImmutableList;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.api.ds.vo.DatasourceConfiguration;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.exception.DEException;
import org.apache.calcite.config.Lex;

View File

@ -1,8 +1,8 @@
package io.dataease.dataset.utils;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.model.SQLObj;
import io.dataease.api.ds.vo.DatasourceConfiguration;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.utils.Md5Utils;
import org.apache.calcite.avatica.util.Quoting;
import org.apache.commons.lang3.StringUtils;

View File

@ -2,7 +2,6 @@ package io.dataease.datasource.manage;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper;
import io.dataease.api.ds.vo.DatasourceDTO;
import io.dataease.commons.constants.OptConstants;
import io.dataease.constant.DataSourceType;
import io.dataease.datasource.dao.auto.entity.CoreDatasource;
@ -11,12 +10,14 @@ import io.dataease.datasource.dao.ext.mapper.DataSourceExtMapper;
import io.dataease.datasource.dao.ext.po.DataSourceNodePO;
import io.dataease.datasource.dto.DatasourceNodeBO;
import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.dto.DatasourceDTO;
import io.dataease.i18n.Translator;
import io.dataease.license.config.XpackInteract;
import io.dataease.model.BusiNodeRequest;
import io.dataease.model.BusiNodeVO;
import io.dataease.operation.manage.CoreOptRecentManage;
import io.dataease.utils.AuthUtils;
import io.dataease.utils.BeanUtils;
import io.dataease.utils.TreeUtils;
import jakarta.annotation.Resource;
import org.apache.commons.collections4.CollectionUtils;
@ -134,4 +135,11 @@ public class DataSourceManage {
coreDatasourceMapper.updateById(sourceData);
coreOptRecentManage.saveOpt(sourceData.getId(), OptConstants.OPT_RESOURCE_TYPE.DATASOURCE, OptConstants.OPT_TYPE.UPDATE);
}
public DatasourceDTO getDs(Long id) {
CoreDatasource coreDatasource = coreDatasourceMapper.selectById(id);
DatasourceDTO dto = new DatasourceDTO();
BeanUtils.copyBean(dto,coreDatasource);
return dto;
}
}

View File

@ -1,8 +1,6 @@
package io.dataease.datasource.manage;
import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper;
import io.dataease.api.dataset.dto.DatasetTableDTO;
import io.dataease.api.ds.vo.TableField;
import io.dataease.commons.constants.TaskStatus;
import io.dataease.dataset.utils.TableUtils;
import io.dataease.datasource.dao.auto.entity.CoreDatasource;
@ -14,13 +12,17 @@ import io.dataease.datasource.provider.ApiUtils;
import io.dataease.datasource.provider.EngineProvider;
import io.dataease.datasource.provider.ExcelUtils;
import io.dataease.datasource.provider.ProviderUtil;
import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.datasource.request.EngineRequest;
import io.dataease.datasource.server.DatasourceServer;
import io.dataease.datasource.server.DatasourceTaskServer;
import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.dto.DatasetTableDTO;
import io.dataease.extensions.datasource.dto.DatasourceDTO;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.extensions.datasource.dto.TableField;
import io.dataease.job.schedule.ExtractDataJob;
import io.dataease.job.schedule.ScheduleManager;
import io.dataease.utils.BeanUtils;
import io.dataease.utils.LogUtil;
import jakarta.annotation.Resource;
import org.apache.commons.lang3.StringUtils;
@ -55,7 +57,7 @@ public class DatasourceSyncManage {
}
DatasourceServer.UpdateType updateType = DatasourceServer.UpdateType.valueOf(type);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(coreDatasource);
datasourceRequest.setDatasource(transDTO(coreDatasource));
List<DatasetTableDTO> tables = ExcelUtils.getTables(datasourceRequest);
for (DatasetTableDTO tableDTO : tables) {
CoreDatasourceTaskLog datasetTableTaskLog = datasourceTaskServer.initTaskLog(coreDatasource.getId(), null, tableDTO.getTableName(), CRON.toString());
@ -78,7 +80,8 @@ public class DatasourceSyncManage {
if (updateType.equals(DatasourceServer.UpdateType.all_scope)) {
dropEngineTable(TableUtils.tmpName(datasourceRequest.getTable()));
}
}catch (Exception ignore){}
} catch (Exception ignore) {
}
datasetTableTaskLog.setTaskStatus(TaskStatus.Error.toString());
datasetTableTaskLog.setInfo(datasetTableTaskLog.getInfo() + "/n Failed to sync datatable: " + datasourceRequest.getTable() + ", " + e.getMessage());
@ -127,9 +130,9 @@ public class DatasourceSyncManage {
}
}
public void extractedData(Long taskId, CoreDatasource coreDatasource, DatasourceServer.UpdateType updateType, String scheduleType ) {
public void extractedData(Long taskId, CoreDatasource coreDatasource, DatasourceServer.UpdateType updateType, String scheduleType) {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(coreDatasource);
datasourceRequest.setDatasource(transDTO(coreDatasource));
List<DatasetTableDTO> tables = ApiUtils.getTables(datasourceRequest);
for (DatasetTableDTO api : tables) {
CoreDatasourceTaskLog datasetTableTaskLog = datasourceTaskServer.initTaskLog(coreDatasource.getId(), taskId, api.getTableName(), scheduleType);
@ -164,7 +167,7 @@ public class DatasourceSyncManage {
}
}
private void updateDsTaskStatus(Long datasourceId){
private void updateDsTaskStatus(Long datasourceId) {
UpdateWrapper<CoreDatasource> updateWrapper = new UpdateWrapper<>();
updateWrapper.eq("id", datasourceId);
CoreDatasource record = new CoreDatasource();
@ -179,13 +182,13 @@ public class DatasourceSyncManage {
LogUtil.error("Can not find datasource: " + datasourceId);
return;
}
CoreDatasourceTaskLog datasetTableTaskLog = datasourceTaskServer.initTaskLog(datasourceId, null, tableName, MANUAL.toString());
CoreDatasourceTaskLog datasetTableTaskLog = datasourceTaskServer.initTaskLog(datasourceId, null, tableName, MANUAL.toString());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(coreDatasource);
datasourceRequest.setDatasource(transDTO(coreDatasource));
List<DatasetTableDTO> tables = ApiUtils.getTables(datasourceRequest);
for (DatasetTableDTO api : tables) {
if(api.getTableName().equalsIgnoreCase(tableName)){
if (api.getTableName().equalsIgnoreCase(tableName)) {
datasourceRequest.setTable(api.getTableName());
List<TableField> tableFields = ApiUtils.getTableFields(datasourceRequest);
try {
@ -206,11 +209,12 @@ public class DatasourceSyncManage {
if (updateType.equals(DatasourceServer.UpdateType.all_scope)) {
dropEngineTable(TableUtils.tmpName(datasourceRequest.getTable()));
}
}catch (Exception ignore){}
} catch (Exception ignore) {
}
datasetTableTaskLog.setInfo(datasetTableTaskLog.getInfo() + "/n Failed to sync datatable: " + datasourceRequest.getTable() + ", " + e.getMessage());
datasetTableTaskLog.setTaskStatus(TaskStatus.Error.name());
datasetTableTaskLog.setEndTime(System.currentTimeMillis());
}finally {
} finally {
datasourceTaskServer.saveLog(datasetTableTaskLog);
}
}
@ -301,7 +305,7 @@ public class DatasourceSyncManage {
engineProvider.exec(engineRequest);
}
public void dropEngineTable(String tableName) throws Exception{
public void dropEngineTable(String tableName) throws Exception {
CoreDeEngine engine = engineManage.info();
EngineRequest engineRequest = new EngineRequest();
engineRequest.setEngine(engine);
@ -348,4 +352,10 @@ public class DatasourceSyncManage {
public void fireNow(CoreDatasourceTask datasourceTask) throws Exception {
scheduleManager.fireNow(datasourceTask.getId().toString(), datasourceTask.getDsId().toString());
}
private DatasourceDTO transDTO(CoreDatasource record) {
DatasourceDTO datasourceDTO = new DatasourceDTO();
BeanUtils.copyBean(datasourceDTO, record);
return datasourceDTO;
}
}

View File

@ -7,10 +7,11 @@ import io.dataease.datasource.dao.auto.mapper.CoreDatasourceMapper;
import io.dataease.datasource.dao.auto.mapper.CoreDeEngineMapper;
import io.dataease.datasource.provider.EngineProvider;
import io.dataease.datasource.provider.ProviderUtil;
import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.datasource.type.H2;
import io.dataease.datasource.type.Mysql;
import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.dto.DatasourceDTO;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.result.ResultMessage;
import io.dataease.utils.BeanUtils;
import io.dataease.utils.JsonUtil;
@ -76,7 +77,7 @@ public class EngineManage {
try {
EngineProvider provider = ProviderUtil.getEngineProvider(engine.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
CoreDatasource datasource = new CoreDatasource();
DatasourceDTO datasource = new DatasourceDTO();
BeanUtils.copyBean(datasource, engine);
datasourceRequest.setDatasource(datasource);
provider.checkStatus(datasourceRequest);
@ -155,23 +156,23 @@ public class EngineManage {
}
}
public void initLocalDataSource(){
public void initLocalDataSource() {
QueryWrapper<CoreDatasource> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("id",985188400292302848L);
queryWrapper.ne("create_time",1715053684176L);
if(!datasourceMapper.exists(queryWrapper) && !ModelUtils.isDesktop()){
queryWrapper.eq("id", 985188400292302848L);
queryWrapper.ne("create_time", 1715053684176L);
if (!datasourceMapper.exists(queryWrapper) && !ModelUtils.isDesktop()) {
Pattern WITH_SQL_FRAGMENT = Pattern.compile("jdbc:mysql://(.*):(\\d+)/(.*)\\?(.*)");
Matcher matcher = WITH_SQL_FRAGMENT.matcher(env.getProperty("spring.datasource.url"));
if (!matcher.find()) {
return;
}
Map configuration = new HashMap<>();
configuration.put("dataBase",matcher.group(3));
configuration.put("username",env.getProperty("spring.datasource.username"));
configuration.put("password",env.getProperty("spring.datasource.password"));
configuration.put("host",matcher.group(1));
configuration.put("port",Integer.valueOf(matcher.group(2)));
configuration.put("extraParams","");
configuration.put("dataBase", matcher.group(3));
configuration.put("username", env.getProperty("spring.datasource.username"));
configuration.put("password", env.getProperty("spring.datasource.password"));
configuration.put("host", matcher.group(1));
configuration.put("port", Integer.valueOf(matcher.group(2)));
configuration.put("extraParams", "");
CoreDatasource initDatasource = new CoreDatasource();
initDatasource.setId(985188400292302848L);

View File

@ -6,12 +6,12 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.jayway.jsonpath.JsonPath;
import io.dataease.api.dataset.dto.DatasetTableDTO;
import io.dataease.api.ds.vo.ApiDefinition;
import io.dataease.api.ds.vo.ApiDefinitionRequest;
import io.dataease.api.ds.vo.TableField;
import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.dto.DatasetTableDTO;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.extensions.datasource.dto.TableField;
import io.dataease.utils.CommonBeanFactory;
import io.dataease.utils.HttpClientConfig;
import io.dataease.utils.HttpClientUtil;

View File

@ -1,22 +1,19 @@
package io.dataease.datasource.provider;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import io.dataease.api.dataset.dto.DatasetTableDTO;
import io.dataease.api.ds.vo.DatasourceConfiguration;
import io.dataease.api.ds.vo.DatasourceConfiguration.DatasourceType;
import io.dataease.api.ds.vo.DatasourceDTO;
import io.dataease.api.ds.vo.TableField;
import io.dataease.commons.utils.CommonThreadPool;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.dataset.utils.FieldUtils;
import io.dataease.datasource.dao.auto.entity.CoreDatasource;
import io.dataease.datasource.dao.auto.entity.CoreDriver;
import io.dataease.datasource.dao.auto.mapper.CoreDatasourceMapper;
import io.dataease.datasource.manage.EngineManage;
import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.datasource.type.*;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.dto.*;
import io.dataease.extensions.datasource.provider.Provider;
import io.dataease.i18n.Translator;
import io.dataease.utils.BeanUtils;
import io.dataease.utils.CommonBeanFactory;
@ -45,7 +42,7 @@ import java.util.stream.Collectors;
@Component("calciteProvider")
public class CalciteProvider {
public class CalciteProvider extends Provider {
@Resource
protected CoreDatasourceMapper coreDatasourceMapper;
@ -84,6 +81,7 @@ public class CalciteProvider {
}
}
@Override
public List<String> getSchema(DatasourceRequest datasourceRequest) {
List<String> schemas = new ArrayList<>();
String queryStr = getSchemaSql(datasourceRequest.getDatasource());
@ -97,6 +95,7 @@ public class CalciteProvider {
return schemas;
}
@Override
public List<DatasetTableDTO> getTables(DatasourceRequest datasourceRequest) {
List<DatasetTableDTO> tables = new ArrayList<>();
List<String> tablesSqls = getTablesSql(datasourceRequest);
@ -132,6 +131,7 @@ public class CalciteProvider {
return drivers;
}
@Override
public String checkStatus(DatasourceRequest datasourceRequest) throws Exception {
DatasourceConfiguration.DatasourceType datasourceType = DatasourceConfiguration.DatasourceType.valueOf(datasourceRequest.getDatasource().getType());
switch (datasourceType) {
@ -153,6 +153,7 @@ public class CalciteProvider {
return "Success";
}
@Override
public Map<String, Object> fetchResultField(DatasourceRequest datasourceRequest) throws DEException {
// 不跨数据源
if (datasourceRequest.getDsList().size() == 1) {
@ -547,7 +548,7 @@ public class CalciteProvider {
return tableField;
}
@Override
public List<TableField> fetchTableField(DatasourceRequest datasourceRequest) throws DEException {
List<TableField> datasetTableFields = new ArrayList<>();
DatasourceSchemaDTO datasourceSchemaDTO = datasourceRequest.getDsList().entrySet().iterator().next().getValue();
@ -967,7 +968,7 @@ public class CalciteProvider {
}
private String getSchemaSql(CoreDatasource datasource) throws DEException {
private String getSchemaSql(DatasourceDTO datasource) throws DEException {
DatasourceConfiguration.DatasourceType datasourceType = DatasourceConfiguration.DatasourceType.valueOf(datasource.getType());
switch (datasourceType) {
case oracle:
@ -986,8 +987,8 @@ public class CalciteProvider {
}
}
public Connection getConnection(CoreDatasource coreDatasource) throws DEException {
@Override
public Connection getConnection(DatasourceDTO coreDatasource) throws DEException {
DatasourceConfiguration configuration = null;
DatasourceConfiguration.DatasourceType datasourceType = DatasourceConfiguration.DatasourceType.valueOf(coreDatasource.getType());
switch (datasourceType) {

View File

@ -1,11 +1,8 @@
package io.dataease.datasource.provider;
import io.dataease.datasource.dao.auto.entity.CoreDeEngine;
import io.dataease.api.ds.vo.TableField;
import io.dataease.datasource.request.EngineRequest;
import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.dto.TableField;
import java.util.List;

View File

@ -10,13 +10,13 @@ import com.alibaba.excel.read.metadata.ReadSheet;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.dataease.api.dataset.dto.DatasetTableDTO;
import io.dataease.api.ds.vo.ExcelFileData;
import io.dataease.api.ds.vo.ExcelSheetData;
import io.dataease.api.ds.vo.TableField;
import io.dataease.datasource.dao.auto.entity.CoreDatasource;
import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.dto.DatasetTableDTO;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.extensions.datasource.dto.TableField;
import io.dataease.utils.AuthUtils;
import io.dataease.utils.JsonUtil;
import lombok.Data;
@ -26,7 +26,6 @@ import org.springframework.web.multipart.MultipartFile;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

View File

@ -2,12 +2,12 @@ package io.dataease.datasource.provider;
import io.dataease.api.ds.vo.DatasourceConfiguration;
import io.dataease.api.ds.vo.TableField;
import io.dataease.dataset.utils.TableUtils;
import io.dataease.datasource.dao.auto.entity.CoreDatasource;
import io.dataease.datasource.dao.auto.entity.CoreDeEngine;
import io.dataease.datasource.request.EngineRequest;
import io.dataease.datasource.type.H2;
import io.dataease.extensions.datasource.dto.DatasourceDTO;
import io.dataease.extensions.datasource.dto.TableField;
import io.dataease.utils.BeanUtils;
import io.dataease.utils.JsonUtil;
import org.apache.commons.lang3.StringUtils;
@ -26,7 +26,7 @@ public class H2EngineProvider extends EngineProvider {
public void exec(EngineRequest engineRequest) throws Exception {
DatasourceConfiguration configuration = JsonUtil.parseObject(engineRequest.getEngine().getConfiguration(), H2.class);
int queryTimeout = configuration.getQueryTimeout();
CoreDatasource datasource = new CoreDatasource();
DatasourceDTO datasource = new DatasourceDTO();
BeanUtils.copyBean(datasource, engineRequest.getEngine());
try (Connection connection = getConnection(datasource); Statement stat = getStatement(connection, queryTimeout)) {
PreparedStatement preparedStatement = connection.prepareStatement(engineRequest.getQuery());
@ -81,7 +81,7 @@ public class H2EngineProvider extends EngineProvider {
@Override
public String replaceTable(String name) {
return "ALTER TABLE `FROM_TABLE` rename to `FROM_TABLE_tmp`; ALTER TABLE `TO_TABLE` rename to `FROM_TABLE`; DROP TABLE IF EXISTS `FROM_TABLE_tmp`;".replace("FROM_TABLE", name).replace("TO_TABLE", TableUtils.tmpName(name));
return "ALTER TABLE `FROM_TABLE` rename to `FROM_TABLE_tmp`; ALTER TABLE `TO_TABLE` rename to `FROM_TABLE`; DROP TABLE IF EXISTS `FROM_TABLE_tmp`;".replace("FROM_TABLE", name).replace("TO_TABLE", TableUtils.tmpName(name));
}

View File

@ -2,12 +2,12 @@ package io.dataease.datasource.provider;
import io.dataease.api.ds.vo.DatasourceConfiguration;
import io.dataease.api.ds.vo.TableField;
import io.dataease.dataset.utils.TableUtils;
import io.dataease.datasource.dao.auto.entity.CoreDatasource;
import io.dataease.datasource.dao.auto.entity.CoreDeEngine;
import io.dataease.datasource.request.EngineRequest;
import io.dataease.datasource.type.Mysql;
import io.dataease.extensions.datasource.dto.DatasourceDTO;
import io.dataease.extensions.datasource.dto.TableField;
import io.dataease.utils.BeanUtils;
import io.dataease.utils.JsonUtil;
import org.apache.commons.lang3.StringUtils;
@ -30,7 +30,7 @@ public class MysqlEngineProvider extends EngineProvider {
public void exec(EngineRequest engineRequest) throws Exception {
DatasourceConfiguration configuration = JsonUtil.parseObject(engineRequest.getEngine().getConfiguration(), Mysql.class);
int queryTimeout = configuration.getQueryTimeout();
CoreDatasource datasource = new CoreDatasource();
DatasourceDTO datasource = new DatasourceDTO();
BeanUtils.copyBean(datasource, engineRequest.getEngine());
try (Connection connection = getConnection(datasource); Statement stat = getStatement(connection, queryTimeout)) {
PreparedStatement preparedStatement = connection.prepareStatement(engineRequest.getQuery());
@ -91,7 +91,6 @@ public class MysqlEngineProvider extends EngineProvider {
return replaceTableSql + ";" + dropTableSql;
}
@Override
public String createTableSql(String tableName, List<TableField> tableFields, CoreDeEngine engine) {
String dorisTableColumnSql = createTableSql(tableFields);

View File

@ -1,12 +1,9 @@
package io.dataease.datasource.request;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.datasource.dao.auto.entity.CoreDatasource;
import io.dataease.datasource.dao.auto.entity.CoreDeEngine;
import lombok.Data;
import org.springframework.util.StringUtils;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

View File

@ -3,13 +3,13 @@ package io.dataease.datasource.server;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import io.dataease.api.ds.DatasourceDriverApi;
import io.dataease.api.ds.vo.DatasourceDTO;
import io.dataease.api.ds.vo.DriveDTO;
import io.dataease.api.ds.vo.DriveJarDTO;
import io.dataease.datasource.dao.auto.entity.CoreDriver;
import io.dataease.datasource.dao.auto.entity.CoreDriverJar;
import io.dataease.datasource.dao.auto.mapper.CoreDriverJarMapper;
import io.dataease.datasource.dao.auto.mapper.CoreDriverMapper;
import io.dataease.extensions.datasource.dto.DatasourceDTO;
import io.dataease.utils.BeanUtils;
import io.dataease.utils.FileUtils;
import io.dataease.utils.Md5Utils;

View File

@ -7,7 +7,6 @@ import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.dataease.api.dataset.dto.DatasetTableDTO;
import io.dataease.api.dataset.dto.PreviewSqlDTO;
import io.dataease.api.ds.DatasourceApi;
import io.dataease.api.ds.vo.*;
@ -16,7 +15,6 @@ import io.dataease.commons.utils.CommonThreadPool;
import io.dataease.constant.DataSourceType;
import io.dataease.constant.LogOT;
import io.dataease.constant.LogST;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.dataset.manage.DatasetDataManage;
import io.dataease.dataset.utils.TableUtils;
import io.dataease.datasource.dao.auto.entity.*;
@ -31,9 +29,9 @@ import io.dataease.datasource.manage.EngineManage;
import io.dataease.datasource.provider.ApiUtils;
import io.dataease.datasource.provider.CalciteProvider;
import io.dataease.datasource.provider.ExcelUtils;
import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.dto.*;
import io.dataease.i18n.Translator;
import io.dataease.job.schedule.CheckDsStatusJob;
import io.dataease.job.schedule.ScheduleManager;
@ -249,23 +247,24 @@ public class DatasourceServer implements DatasourceApi {
}
preCheckDs(dataSourceDTO);
dataSourceDTO.setId(IDUtils.snowID());
dataSourceDTO.setCreateTime(System.currentTimeMillis());
dataSourceDTO.setUpdateTime(System.currentTimeMillis());
try {
checkDatasourceStatus(dataSourceDTO);
} catch (Exception ignore) {
dataSourceDTO.setStatus("Error");
}
dataSourceDTO.setTaskStatus(TaskStatus.WaitingForExecution.name());
dataSourceDTO.setCreateBy(AuthUtils.getUser().getUserId().toString());
dataSourceDTO.setUpdateBy(AuthUtils.getUser().getUserId());
CoreDatasource coreDatasource = new CoreDatasource();
BeanUtils.copyBean(coreDatasource, dataSourceDTO);
coreDatasource.setCreateTime(System.currentTimeMillis());
coreDatasource.setUpdateTime(System.currentTimeMillis());
try {
checkDatasourceStatus(coreDatasource);
} catch (Exception ignore) {
coreDatasource.setStatus("Error");
}
coreDatasource.setTaskStatus(TaskStatus.WaitingForExecution.name());
coreDatasource.setCreateBy(AuthUtils.getUser().getUserId().toString());
coreDatasource.setUpdateBy(AuthUtils.getUser().getUserId());
dataSourceManage.innerSave(coreDatasource);
if (dataSourceDTO.getType().equals(DatasourceConfiguration.DatasourceType.Excel.name())) {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(coreDatasource);
datasourceRequest.setDatasource(dataSourceDTO);
List<DatasetTableDTO> tables = ExcelUtils.getTables(datasourceRequest);
for (DatasetTableDTO table : tables) {
datasourceRequest.setTable(table.getTableName());
@ -298,7 +297,7 @@ public class DatasourceServer implements DatasourceApi {
datasourceTaskServer.insert(coreDatasourceTask);
datasourceSyncManage.addSchedule(coreDatasourceTask);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(coreDatasource);
datasourceRequest.setDatasource(dataSourceDTO);
List<DatasetTableDTO> tables = ApiUtils.getTables(datasourceRequest);
checkName(tables.stream().map(DatasetTableDTO::getName).collect(Collectors.toList()));
for (DatasetTableDTO api : tables) {
@ -325,25 +324,26 @@ public class DatasourceServer implements DatasourceApi {
if (ObjectUtils.isEmpty(pk = dataSourceDTO.getId())) {
return save(dataSourceDTO);
}
CoreDatasource sourceData = datasourceMapper.selectById(pk);
DatasourceDTO sourceData = dataSourceManage.getDs(pk);
dataSourceDTO.setConfiguration(new String(Base64.getDecoder().decode(dataSourceDTO.getConfiguration())));
dataSourceDTO.setPid(sourceData.getPid());
preCheckDs(dataSourceDTO);
dataSourceDTO.setUpdateTime(System.currentTimeMillis());
dataSourceDTO.setUpdateBy(AuthUtils.getUser().getUserId());
try {
checkDatasourceStatus(dataSourceDTO);
} catch (Exception e) {
dataSourceDTO.setStatus("Error");
}
CoreDatasource requestDatasource = new CoreDatasource();
BeanUtils.copyBean(requestDatasource, dataSourceDTO);
requestDatasource.setUpdateTime(System.currentTimeMillis());
requestDatasource.setUpdateBy(AuthUtils.getUser().getUserId());
try {
checkDatasourceStatus(requestDatasource);
} catch (Exception e) {
requestDatasource.setStatus("Error");
}
DatasourceRequest sourceTableRequest = new DatasourceRequest();
sourceTableRequest.setDatasource(sourceData);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(requestDatasource);
datasourceRequest.setDatasource(dataSourceDTO);
List<String> toCreateTables = new ArrayList<>();
List<String> toDeleteTables = new ArrayList<>();
if (dataSourceDTO.getType().equals(DatasourceConfiguration.DatasourceType.API.name())) {
@ -474,7 +474,7 @@ public class DatasourceServer implements DatasourceApi {
dataSourceDTO.setConfiguration(new String(Base64.getDecoder().decode(dataSourceDTO.getConfiguration())));
CoreDatasource coreDatasource = new CoreDatasource();
BeanUtils.copyBean(coreDatasource, dataSourceDTO);
checkDatasourceStatus(coreDatasource);
checkDatasourceStatus(dataSourceDTO);
DatasourceDTO result = new DatasourceDTO();
result.setStatus(coreDatasource.getStatus());
return result;
@ -486,7 +486,7 @@ public class DatasourceServer implements DatasourceApi {
CoreDatasource coreDatasource = new CoreDatasource();
BeanUtils.copyBean(coreDatasource, dataSourceDTO);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(coreDatasource);
datasourceRequest.setDatasource(dataSourceDTO);
return calciteProvider.getSchema(datasourceRequest);
}
@ -592,9 +592,11 @@ public class DatasourceServer implements DatasourceApi {
if (ObjectUtils.isEmpty(coreDatasource)) {
return;
}
DatasourceDTO datasourceDTO = new DatasourceDTO();
BeanUtils.copyBean(datasourceDTO, coreDatasource);
if (coreDatasource.getType().equals(DatasourceConfiguration.DatasourceType.Excel.name())) {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(coreDatasource);
datasourceRequest.setDatasource(datasourceDTO);
List<DatasetTableDTO> tables = ExcelUtils.getTables(datasourceRequest);
for (DatasetTableDTO table : tables) {
datasourceRequest.setTable(table.getTableName());
@ -607,7 +609,7 @@ public class DatasourceServer implements DatasourceApi {
}
if (coreDatasource.getType().equals(DatasourceConfiguration.DatasourceType.API.name())) {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(coreDatasource);
datasourceRequest.setDatasource(datasourceDTO);
List<DatasetTableDTO> tables = ApiUtils.getTables(datasourceRequest);
for (DatasetTableDTO api : tables) {
datasourceRequest.setTable(api.getTableName());
@ -677,7 +679,7 @@ public class DatasourceServer implements DatasourceApi {
DatasourceDTO datasourceDTO = new DatasourceDTO();
BeanUtils.copyBean(datasourceDTO, coreDatasource);
try {
checkDatasourceStatus(coreDatasource);
checkDatasourceStatus(datasourceDTO);
calciteProvider.updateDsPoolAfterCheckStatus(datasourceDTO);
} catch (Exception e) {
coreDatasource.setStatus("Error");
@ -736,8 +738,10 @@ public class DatasourceServer implements DatasourceApi {
@Override
public List<DatasetTableDTO> getTables(DatasetTableDTO datasetTableDTO) throws DEException {
CoreDatasource coreDatasource = datasourceMapper.selectById(datasetTableDTO.getDatasourceId());
DatasourceDTO datasourceDTO = new DatasourceDTO();
BeanUtils.copyBean(datasourceDTO, coreDatasource);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(coreDatasource);
datasourceRequest.setDatasource(datasourceDTO);
if (coreDatasource.getType().equals("API")) {
List<DatasetTableDTO> datasetTableDTOS = ApiUtils.getTables(datasourceRequest);
datasetTableDTOS.forEach(datasetTableDTO1 -> {
@ -761,9 +765,9 @@ public class DatasourceServer implements DatasourceApi {
String datasourceId = req.get("datasourceId");
CoreDatasource coreDatasource = datasourceMapper.selectById(datasourceId);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(coreDatasource);
datasourceRequest.setDatasource(transDTO(coreDatasource));
if (coreDatasource.getType().equals("API") || coreDatasource.getType().equals("Excel")) {
datasourceRequest.setDatasource(engineManage.getDeEngine());
datasourceRequest.setDatasource(transDTO(engineManage.getDeEngine()));
DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO();
BeanUtils.copyBean(datasourceSchemaDTO, engineManage.getDeEngine());
datasourceSchemaDTO.setSchemaAlias(String.format(SQLConstants.SCHEMA, datasourceSchemaDTO.getId()));
@ -809,7 +813,7 @@ public class DatasourceServer implements DatasourceApi {
CoreDatasource coreDatasource = datasourceMapper.selectById(datasourceId);
if (coreDatasource != null) {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(coreDatasource);
datasourceRequest.setDatasource(transDTO(coreDatasource));
List<DatasetTableDTO> datasetTableDTOS = ExcelUtils.getTables(datasourceRequest);
List<ExcelSheetData> excelSheetDataList = new ArrayList<>();
for (ExcelSheetData sheet : excelFileData.getSheets()) {
@ -877,7 +881,7 @@ public class DatasourceServer implements DatasourceApi {
}
}
public void checkDatasourceStatus(CoreDatasource coreDatasource) {
public void checkDatasourceStatus(DatasourceDTO coreDatasource) {
if (coreDatasource.getType().equals(DatasourceConfiguration.DatasourceType.Excel.name()) || coreDatasource.getType().equals(DatasourceConfiguration.DatasourceType.folder.name())) {
return;
}
@ -944,7 +948,7 @@ public class DatasourceServer implements DatasourceApi {
IPage<CoreDatasourceTaskLogDTO> pager = taskLogExtMapper.pager(page, wrapper);
CoreDatasource coreDatasource = datasourceMapper.selectById(dsId);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(coreDatasource);
datasourceRequest.setDatasource(transDTO(coreDatasource));
List<DatasetTableDTO> datasetTableDTOS = ApiUtils.getTables(datasourceRequest);
for (int i = 0; i < pager.getRecords().size(); i++) {
for (int i1 = 0; i1 < datasetTableDTOS.size(); i1++) {
@ -1029,4 +1033,10 @@ public class DatasourceServer implements DatasourceApi {
coreDsFinishPage.setId(AuthUtils.getUser().getUserId());
coreDsFinishPageMapper.insert(coreDsFinishPage);
}
private DatasourceDTO transDTO(CoreDatasource record) {
DatasourceDTO datasourceDTO = new DatasourceDTO();
BeanUtils.copyBean(datasourceDTO, record);
return datasourceDTO;
}
}

View File

@ -1,11 +1,11 @@
package io.dataease.datasource.server;
import io.dataease.api.ds.vo.DatasourceDTO;
import io.dataease.api.ds.EngineApi;
import io.dataease.datasource.dao.auto.entity.CoreDeEngine;
import io.dataease.datasource.dao.auto.mapper.CoreDeEngineMapper;
import io.dataease.datasource.manage.EngineManage;
import io.dataease.datasource.provider.CalciteProvider;
import io.dataease.extensions.datasource.dto.DatasourceDTO;
import io.dataease.utils.BeanUtils;
import io.dataease.utils.IDUtils;
import jakarta.annotation.Resource;

View File

@ -1,7 +1,7 @@
package io.dataease.datasource.utils;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.i18n.Translator;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;

View File

@ -1,10 +1,10 @@
package io.dataease.engine.trans;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.filter.FilterTreeItem;
import io.dataease.extensions.view.filter.FilterTreeObj;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.model.SQLObj;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.dto.DatasetTableFieldDTO;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;

View File

@ -1,9 +1,9 @@
package io.dataease.engine.trans;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.dto.ChartViewFieldDTO;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.model.SQLObj;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.dto.DatasetTableFieldDTO;
import io.dataease.engine.constant.DeTypeConstants;
import io.dataease.engine.constant.ExtFieldConstant;

View File

@ -1,9 +1,9 @@
package io.dataease.engine.trans;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.dto.ChartExtFilterDTO;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.model.SQLObj;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.dto.DatasetTableFieldDTO;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;

View File

@ -1,8 +1,8 @@
package io.dataease.engine.trans;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.model.SQLObj;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.dto.DatasetTableFieldDTO;
import io.dataease.engine.constant.DeTypeConstants;
import io.dataease.engine.constant.ExtFieldConstant;

View File

@ -1,9 +1,9 @@
package io.dataease.engine.trans;
import io.dataease.api.chart.dto.DeSortField;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.model.SQLObj;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.dto.DatasetTableFieldDTO;
import io.dataease.engine.constant.DeTypeConstants;
import io.dataease.engine.constant.ExtFieldConstant;

View File

@ -1,9 +1,9 @@
package io.dataease.engine.trans;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.dto.ChartViewFieldDTO;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.model.SQLObj;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.dto.DatasetTableFieldDTO;
import io.dataease.engine.constant.DeTypeConstants;
import io.dataease.engine.constant.ExtFieldConstant;

View File

@ -1,11 +1,11 @@
package io.dataease.engine.trans;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.model.SQLMeta;
import io.dataease.extensions.view.model.SQLObj;
import io.dataease.api.permissions.dataset.dto.DataSetRowPermissionsTreeDTO;
import io.dataease.extensions.view.dto.DatasetRowPermissionsTreeItem;
import io.dataease.extensions.view.dto.DatasetRowPermissionsTreeObj;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.dto.DatasetTableFieldDTO;
import io.dataease.engine.constant.ExtFieldConstant;
import io.dataease.engine.constant.SQLConstants;

View File

@ -1,11 +1,11 @@
package io.dataease.engine.utils;
import io.dataease.api.ds.vo.DatasourceConfiguration;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.engine.constant.ExtFieldConstant;
import io.dataease.engine.constant.SQLConstants;
import io.dataease.engine.constant.SqlPlaceholderConstants;
import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.view.dto.DatasetTableFieldDTO;
import io.dataease.extensions.view.model.SQLObj;
import io.dataease.i18n.Translator;

View File

@ -2,10 +2,10 @@ package io.dataease.api.dataset;
import com.github.xiaoymin.knife4j.annotations.ApiSupport;
import io.dataease.api.dataset.dto.BaseTreeNodeDTO;
import io.dataease.api.dataset.dto.DatasetTableDTO;
import io.dataease.api.dataset.dto.EnumValueRequest;
import io.dataease.api.dataset.dto.PreviewSqlDTO;
import io.dataease.api.dataset.union.DatasetGroupInfoDTO;
import io.dataease.extensions.datasource.dto.DatasetTableDTO;
import io.dataease.extensions.view.dto.DatasetTableFieldDTO;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;

View File

@ -2,11 +2,11 @@ package io.dataease.api.dataset;
import com.github.xiaoymin.knife4j.annotations.ApiSupport;
import io.dataease.api.dataset.dto.DatasetNodeDTO;
import io.dataease.api.dataset.dto.DatasetTableDTO;
import io.dataease.api.dataset.union.DatasetGroupInfoDTO;
import io.dataease.api.dataset.vo.DataSetBarVO;
import io.dataease.auth.DeApiPath;
import io.dataease.auth.DePermit;
import io.dataease.extensions.datasource.dto.DatasetTableDTO;
import io.dataease.extensions.view.dto.SqlVariableDetails;
import io.dataease.model.BusiNodeRequest;
import io.dataease.model.BusiNodeVO;

View File

@ -1,6 +1,6 @@
package io.dataease.api.dataset.union;
import io.dataease.api.dataset.dto.DatasetTableDTO;
import io.dataease.extensions.datasource.dto.DatasetTableDTO;
import io.dataease.extensions.view.dto.DatasetTableFieldDTO;
import lombok.Data;

View File

@ -1,6 +1,6 @@
package io.dataease.api.dataset.union;
import io.dataease.api.dataset.dto.DatasetTableDTO;
import io.dataease.extensions.datasource.dto.DatasetTableDTO;
import io.dataease.extensions.view.model.SQLObj;
import lombok.Data;

View File

@ -2,7 +2,7 @@ package io.dataease.api.dataset.vo;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
import io.dataease.api.ds.vo.DatasourceDTO;
import io.dataease.extensions.datasource.dto.DatasourceDTO;
import lombok.Data;
import java.io.Serial;

View File

@ -2,11 +2,13 @@ package io.dataease.api.ds;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.github.xiaoymin.knife4j.annotations.ApiSupport;
import io.dataease.api.dataset.dto.DatasetTableDTO;
import io.dataease.api.ds.vo.*;
import io.dataease.auth.DeApiPath;
import io.dataease.auth.DePermit;
import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.dto.DatasetTableDTO;
import io.dataease.extensions.datasource.dto.DatasourceDTO;
import io.dataease.extensions.datasource.dto.TableField;
import io.dataease.model.BusiNodeRequest;
import io.dataease.model.BusiNodeVO;
import io.swagger.v3.oas.annotations.tags.Tag;

View File

@ -2,9 +2,9 @@ package io.dataease.api.ds;
import com.github.xiaoymin.knife4j.annotations.ApiSupport;
import io.dataease.api.ds.vo.DatasourceDTO;
import io.dataease.api.ds.vo.DriveDTO;
import io.dataease.api.ds.vo.DriveJarDTO;
import io.dataease.extensions.datasource.dto.DatasourceDTO;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;

View File

@ -1,8 +1,8 @@
package io.dataease.api.ds;
import com.github.xiaoymin.knife4j.annotations.ApiSupport;
import io.dataease.api.ds.vo.DatasourceDTO;
import io.dataease.auth.DeApiPath;
import io.dataease.extensions.datasource.dto.DatasourceDTO;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.springframework.web.bind.annotation.*;

View File

@ -1,6 +1,7 @@
package io.dataease.api.ds.vo;
import io.dataease.extensions.datasource.dto.TableField;
import lombok.Data;
import java.util.ArrayList;

View File

@ -1,5 +1,6 @@
package io.dataease.api.ds.vo;
import io.dataease.extensions.datasource.dto.TableField;
import lombok.Data;
import java.util.List;

View File

@ -130,6 +130,12 @@
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.dataease</groupId>
<artifactId>extensions-datasource</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</project>

View File

@ -0,0 +1,27 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>io.dataease</groupId>
<artifactId>extensions</artifactId>
<version>${dataease.version}</version>
</parent>
<artifactId>extensions-datasource</artifactId>
<properties>
<maven.compiler.source>21</maven.compiler.source>
<maven.compiler.target>21</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>io.dataease</groupId>
<artifactId>extensions-view</artifactId>
<version>${dataease.version}</version>
</dependency>
</dependencies>
</project>

View File

@ -1,4 +1,4 @@
package io.dataease.api.dataset.dto;
package io.dataease.extensions.datasource.dto;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;

View File

@ -1,4 +1,4 @@
package io.dataease.api.ds.vo;
package io.dataease.extensions.datasource.dto;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
@ -58,6 +58,7 @@ public class DatasourceDTO implements Serializable {
*/
private Long updateTime;
private Long updateBy;
/**
* 创建人ID
*/
@ -80,4 +81,12 @@ public class DatasourceDTO implements Serializable {
* 上次成功更新时间
*/
private Long lastSyncTime;
private String qrtzInstance;
/**
* 任务状态
*/
private String taskStatus;
}

View File

@ -1,21 +1,20 @@
package io.dataease.datasource.request;
package io.dataease.extensions.datasource.dto;
import io.dataease.dataset.dto.DatasourceSchemaDTO;
import io.dataease.datasource.dao.auto.entity.CoreDatasource;
import lombok.Data;
import org.springframework.util.StringUtils;
import java.io.Serializable;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@Data
public class DatasourceRequest {
public class DatasourceRequest implements Serializable {
private final String REG_WITH_SQL_FRAGMENT = "((?i)WITH[\\s\\S]+(?i)AS?\\s*\\([\\s\\S]+\\))\\s*(?i)SELECT";
private Pattern WITH_SQL_FRAGMENT = Pattern.compile("((?i)WITH[\\s\\S]+(?i)AS?\\s*\\([\\s\\S]+\\))\\s*(?i)SELECT");
protected String query;
protected String table;
protected CoreDatasource datasource;
protected DatasourceDTO datasource;
private Integer pageSize;
private Integer page;
private Integer realSize;

View File

@ -0,0 +1,11 @@
package io.dataease.extensions.datasource.dto;
import lombok.Data;
/**
* @Author Junjun
*/
@Data
public class DatasourceSchemaDTO extends DatasourceDTO {
private String schemaAlias;
}

View File

@ -1,4 +1,4 @@
package io.dataease.api.ds.vo;
package io.dataease.extensions.datasource.dto;
import lombok.Data;

View File

@ -1,4 +1,4 @@
package io.dataease.api.ds.vo;
package io.dataease.extensions.datasource.dto;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;

View File

@ -0,0 +1,16 @@
package io.dataease.extensions.datasource.factory;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
/**
* @Author Junjun
*/
public class ProviderFactory implements ApplicationContextAware {
@Override
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
}
}

View File

@ -1,10 +1,10 @@
package io.dataease.datasource.provider;
package io.dataease.extensions.datasource.provider;
import io.dataease.api.dataset.dto.DatasetTableDTO;
import io.dataease.api.ds.vo.TableField;
import io.dataease.datasource.dao.auto.entity.CoreDatasource;
import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.exception.DEException;
import io.dataease.extensions.datasource.dto.DatasetTableDTO;
import io.dataease.extensions.datasource.dto.DatasourceDTO;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.extensions.datasource.dto.TableField;
import java.sql.Connection;
import java.util.List;
@ -18,7 +18,7 @@ public abstract class Provider {
public abstract List<DatasetTableDTO> getTables(DatasourceRequest datasourceRequest);
public abstract Connection getConnection(CoreDatasource coreDatasource) throws DEException;
public abstract Connection getConnection(DatasourceDTO coreDatasource) throws DEException;
public abstract String checkStatus(DatasourceRequest datasourceRequest) throws Exception;

View File

@ -12,6 +12,7 @@
<packaging>pom</packaging>
<modules>
<module>extensions-view</module>
<module>extensions-datasource</module>
</modules>
<artifactId>extensions</artifactId>
@ -35,4 +36,4 @@
</dependency>
</dependencies>
</project>
</project>