Merge branch 'dev' into pr@dev@feat_new-template-import

This commit is contained in:
王嘉豪 2022-03-31 10:31:50 +08:00 committed by GitHub
commit 0be2a080af
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 123 additions and 93 deletions

View File

@ -32,8 +32,6 @@ public class DataSetGroupController {
@Resource
private DataSetGroupService dataSetGroupService;
@Resource
private ExtractDataService extractDataService;
@Resource
private KettleService kettleService;
@DePermissions(value = {

View File

@ -62,13 +62,27 @@ public class ApiProvider extends DatasourceProvider{
ApiDefinition apiDefinition = checkApiDefinition(datasourceRequest);
String response = execHttpRequest(apiDefinition);
fieldList = getTableFileds(datasourceRequest);
fieldList = getTableFileds(apiDefinition, response);
result.put("fieldList", fieldList);
dataList = fetchResult(response, apiDefinition);
result.put("dataList", dataList);
return result;
}
private List<TableField> getTableFileds(ApiDefinition apiDefinition, String response) throws Exception {
List<TableField> tableFields = new ArrayList<>();
for (DatasetTableField field : checkApiDefinition(apiDefinition, response).getFields()) {
TableField tableField = new TableField();
tableField.setFieldName(field.getOriginName());
tableField.setRemarks(field.getName());
tableField.setFieldSize(field.getSize());
tableField.setFieldType(field.getDeExtractType().toString());
tableFields.add(tableField);
}
return tableFields;
}
@Override
public void handleDatasource(DatasourceRequest datasourceRequest, String type) throws Exception {
@ -83,9 +97,10 @@ public class ApiProvider extends DatasourceProvider{
public List<TableField> getTableFileds(DatasourceRequest datasourceRequest) throws Exception {
List<ApiDefinition> lists = JSONObject.parseArray(datasourceRequest.getDatasource().getConfiguration(), ApiDefinition.class);
List<TableField> tableFields = new ArrayList<>();
for (ApiDefinition list : lists) {
if(datasourceRequest.getTable().equalsIgnoreCase(list.getName())){
for (DatasetTableField field : list.getFields()) {
for (ApiDefinition apiDefinition : lists) {
if(datasourceRequest.getTable().equalsIgnoreCase(apiDefinition.getName())){
String response = ApiProvider.execHttpRequest(apiDefinition);
for (DatasetTableField field : checkApiDefinition(apiDefinition, response).getFields()) {
TableField tableField = new TableField();
tableField.setFieldName(field.getOriginName());
tableField.setRemarks(field.getName());
@ -170,6 +185,50 @@ public class ApiProvider extends DatasourceProvider{
return response;
}
static public ApiDefinition checkApiDefinition(ApiDefinition apiDefinition, String response)throws Exception{
if(StringUtils.isEmpty(response)){
throw new Exception("该请求返回数据为空");
}
List<LinkedHashMap> datas = new ArrayList<>();
try {
datas = JsonPath.read(response,apiDefinition.getDataPath());
}catch (Exception e){
throw new Exception("jsonPath 路径错误:" + e.getMessage());
}
List<JSONObject> dataList = new ArrayList<>();
List<DatasetTableField> fields = new ArrayList<>();
Set<String> fieldKeys = new HashSet<>();
//第一遍获取 field
for (LinkedHashMap data : datas) {
Set<String> keys = data.keySet();
for (String key : keys) {
if(!fieldKeys.contains(key)){
fieldKeys.add(key);
DatasetTableField tableField = new DatasetTableField();
tableField.setOriginName(key);
tableField.setName(key);
tableField.setSize(65535);
tableField.setDeExtractType(0);
tableField.setDeType(0);
tableField.setExtField(0);
fields.add(tableField);
}
}
}
//第二遍获取 data
for (LinkedHashMap data : datas) {
JSONObject jsonObject = new JSONObject();
for (String key : fieldKeys) {
jsonObject.put(key, Optional.ofNullable(data.get(key)).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " "));
}
dataList.add(jsonObject);
}
apiDefinition.setDatas(dataList);
apiDefinition.setFields(fields);
return apiDefinition;
}
private List<String[]> fetchResult(String result, ApiDefinition apiDefinition){
List<String[]> dataList = new LinkedList<>();
List<LinkedHashMap> datas = JsonPath.read(result, apiDefinition.getDataPath());

View File

@ -812,8 +812,8 @@ public class Db2QueryProvider extends QueryProvider {
whereValue = "''";
} else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "not_empty")) {
whereValue = "''";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in")) {
whereValue = "('" + StringUtils.join(value, "','") + "')";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in") || StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "not in")) {
whereValue = "('" + String.join("','", value.split(",")) + "')";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "like")) {
whereValue = "'%" + value + "%'";
} else {

View File

@ -858,8 +858,8 @@ public class EsQueryProvider extends QueryProvider {
whereValue = "''";
} else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "not_empty")) {
whereValue = "''";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in")) {
whereValue = "('" + StringUtils.join(value, "','") + "')";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in") || StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "not in")) {
whereValue = "('" + String.join("','", value.split(",")) + "')";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "like")) {
whereValue = "'%" + value + "%'";
} else {

View File

@ -794,8 +794,8 @@ public class HiveQueryProvider extends QueryProvider {
whereValue = "''";
} else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "not_empty")) {
whereValue = "''";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in")) {
whereValue = "('" + StringUtils.join(value, "','") + "')";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in") || StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "not in")) {
whereValue = "('" + String.join("','", value.split(",")) + "')";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "like")) {
whereValue = "'%" + value + "%'";
} else {

View File

@ -791,8 +791,8 @@ public class ImpalaQueryProvider extends QueryProvider {
whereValue = "''";
} else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "not_empty")) {
whereValue = "''";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in")) {
whereValue = "('" + StringUtils.join(value, "','") + "')";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in") || StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "not in")) {
whereValue = "('" + String.join("','", value.split(",")) + "')";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "like")) {
whereValue = "'%" + value + "%'";
} else {

View File

@ -776,8 +776,8 @@ public class MongoQueryProvider extends QueryProvider {
whereValue = "''";
} else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "not_empty")) {
whereValue = "''";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in")) {
whereValue = "('" + StringUtils.join(value, "','") + "')";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in") || StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "not in")) {
whereValue = "('" + String.join("','", value.split(",")) + "')";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "like")) {
whereValue = "'%" + value + "%'";
} else {

View File

@ -799,8 +799,8 @@ public class MysqlQueryProvider extends QueryProvider {
whereValue = "''";
} else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "not_empty")) {
whereValue = "''";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in")) {
whereValue = "('" + StringUtils.join(value, "','") + "')";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in") || StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "not in")) {
whereValue = "('" + String.join("','", value.split(",")) + "')";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "like")) {
whereValue = "'%" + value + "%'";
} else {

View File

@ -850,8 +850,8 @@ public class OracleQueryProvider extends QueryProvider {
whereValue = "''";
} else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "not_empty")) {
whereValue = "''";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in")) {
whereValue = "('" + StringUtils.join(value, "','") + "')";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in") || StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "not in")) {
whereValue = "('" + String.join("','", value.split(",")) + "')";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "like")) {
whereValue = "'%" + value + "%'";
} else {

View File

@ -825,8 +825,8 @@ public class PgQueryProvider extends QueryProvider {
whereValue = "''";
} else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "not_empty")) {
whereValue = "''";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in")) {
whereValue = "('" + StringUtils.join(value, "','") + "')";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in") || StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "not in")) {
whereValue = "('" + String.join("','", value.split(",")) + "')";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "like")) {
whereValue = "'%" + value + "%'";
} else {

View File

@ -758,8 +758,8 @@ public class RedshiftQueryProvider extends QueryProvider {
whereValue = PgConstants.WHERE_VALUE_NULL;
} else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "not_null")) {
whereTerm = String.format(whereTerm, originName);
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in")) {
whereValue = "('" + StringUtils.join(value, "','") + "')";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in") || StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "not in")) {
whereValue = "('" + String.join("','", value.split(",")) + "')";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "like")) {
whereValue = "'%" + value + "%'";
} else {

View File

@ -859,8 +859,8 @@ public class SqlserverQueryProvider extends QueryProvider {
whereValue = "''";
} else if (StringUtils.equalsIgnoreCase(filterItemDTO.getTerm(), "not_empty")) {
whereValue = "''";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in")) {
whereValue = "('" + StringUtils.join(value, "','") + "')";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "in") || StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "not in")) {
whereValue = "('" + String.join("','", value.split(",")) + "')";
} else if (StringUtils.containsIgnoreCase(filterItemDTO.getTerm(), "like")) {
whereValue = "'%" + value + "%'";
} else {

View File

@ -45,11 +45,11 @@ public class DataSetGroupService {
@DeCleaner(DePermissionType.DATASET)
public DataSetGroupDTO save(DatasetGroup datasetGroup) throws Exception {
if (StringUtils.isEmpty(datasetGroup.getType())) {
throw new Exception("type can not be empty");
}
checkName(datasetGroup);
if (StringUtils.isEmpty(datasetGroup.getId())) {
if (StringUtils.isEmpty(datasetGroup.getType())) {
throw new Exception("type can not be empty");
}
datasetGroup.setId(UUID.randomUUID().toString());
datasetGroup.setCreateBy(AuthUtils.getUser().getUsername());
datasetGroup.setCreateTime(System.currentTimeMillis());

View File

@ -30,7 +30,7 @@ public class PermissionService {
public List<ChartFieldCustomFilterDTO> getCustomFilters(List<DatasetTableField> fields, DatasetTable datasetTable, Long user) {
List<ChartFieldCustomFilterDTO> customFilter = new ArrayList<>();
Map<String, Object> values = new HashMap<>();
Map<String, String> values = new HashMap<>();
for (DatasetRowPermissions datasetRowPermissions : rowPermissions(datasetTable.getId(), user, values)) {
ChartFieldCustomFilterDTO dto = new ChartFieldCustomFilterDTO();
if (StringUtils.isEmpty(datasetRowPermissions.getDatasetFieldId())) {
@ -51,6 +51,7 @@ public class PermissionService {
lists.forEach(chartCustomFilterDTO -> {
chartCustomFilterDTO.setFieldId(field.getId());
if(datasetRowPermissions.getAuthTargetType().equalsIgnoreCase("sysParams")){
System.out.println(values.get(chartCustomFilterDTO.getValue()).toString());
chartCustomFilterDTO.setValue(values.get(chartCustomFilterDTO.getValue()).toString());
}
});
@ -96,7 +97,7 @@ public class PermissionService {
}
private List<DatasetRowPermissions> rowPermissions(String datasetId, Long userId, Map<String, Object> values) {
private List<DatasetRowPermissions> rowPermissions(String datasetId, Long userId, Map<String, String> values) {
List<DatasetRowPermissions> datasetRowPermissions = new ArrayList<>();
Map<String, RowPermissionService> beansOfType = SpringContextUtil.getApplicationContext().getBeansOfType((RowPermissionService.class));
if (beansOfType.keySet().size() == 0) {
@ -139,7 +140,7 @@ public class PermissionService {
values.put("${sysParams.userEmail}", userEntity.getEmail());
values.put("${sysParams.userSource}", userEntity.getFrom() == 0 ? "LOCAL" : "OIDC");
values.put("${sysParams.dept}", userEntity.getDeptName());
values.put("${sysParams.roles}", StringUtils.joinWith(",", currentRoleDtos.stream().map(CurrentRoleDto::getName).collect(Collectors.toList())));
values.put("${sysParams.roles}", String.join(",", currentRoleDtos.stream().map(CurrentRoleDto::getName).collect(Collectors.toList())));
return datasetRowPermissions;
}

View File

@ -57,7 +57,7 @@ public class DatasourceService {
private DataSetGroupService dataSetGroupService;
@Resource
private CommonThreadPool commonThreadPool;
private static List<String> dsTypes = Arrays.asList("excel", "mysql", "hive", "impala", "mariadb", "ds_doris", "pg", "sqlServer", "oracle", "mongo", "ck", "db2", "es", "redshift", "api");
private static List<String> dsTypes = Arrays.asList("TiDB", "StarRocks", "excel", "mysql", "hive", "impala", "mariadb", "ds_doris", "pg", "sqlServer", "oracle", "mongo", "ck", "db2", "es", "redshift", "api");
@DeCleaner(DePermissionType.DATASOURCE)
public Datasource addDatasource(Datasource datasource) throws Exception{
@ -366,47 +366,7 @@ public class DatasourceService {
public ApiDefinition checkApiDatasource(ApiDefinition apiDefinition) throws Exception {
String response = ApiProvider.execHttpRequest(apiDefinition);
if(StringUtils.isEmpty(response)){
throw new Exception("该请求返回数据为空");
}
List<LinkedHashMap> datas = new ArrayList<>();
try {
datas = JsonPath.read(response,apiDefinition.getDataPath());
}catch (Exception e){
throw new Exception("jsonPath 路径错误:" + e.getMessage());
}
List<JSONObject> dataList = new ArrayList<>();
List<DatasetTableField> fields = new ArrayList<>();
Set<String> fieldKeys = new HashSet<>();
//第一遍获取 field
for (LinkedHashMap data : datas) {
Set<String> keys = data.keySet();
for (String key : keys) {
if(!fieldKeys.contains(key)){
fieldKeys.add(key);
DatasetTableField tableField = new DatasetTableField();
tableField.setOriginName(key);
tableField.setName(key);
tableField.setSize(65535);
tableField.setDeExtractType(0);
tableField.setDeType(0);
tableField.setExtField(0);
fields.add(tableField);
}
}
}
//第二遍获取 data
for (LinkedHashMap data : datas) {
JSONObject jsonObject = new JSONObject();
for (String key : fieldKeys) {
jsonObject.put(key, Optional.ofNullable(data.get(key)).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " "));
}
dataList.add(jsonObject);
}
apiDefinition.setDatas(dataList);
apiDefinition.setFields(fields);
return apiDefinition;
return ApiProvider.checkApiDefinition(apiDefinition, response);
}
private void checkAndUpdateDatasourceStatus(Datasource datasource){

View File

@ -250,7 +250,7 @@ END
;;
delimiter ;
INSERT INTO `my_plugin`(`plugin_id`, `name`, `store`, `free`, `cost`, `category`, `descript`, `version`, `install_type`, `creator`, `load_mybatis`, `release_time`, `install_time`, `module_name`, `icon`) VALUES (3, 'tabs插件', 'default', 0, 20000, 'panel', 'tabs插件', '1.0-SNAPSHOT', NULL, 'fit2cloud-chenyw', 0, NULL, NULL, 'dataease-extensions-tabs-backend', NULL);
INSERT INTO `my_plugin`(`plugin_id`, `name`, `store`, `free`, `cost`, `category`, `descript`, `version`, `install_type`, `creator`, `load_mybatis`, `release_time`, `install_time`, `module_name`, `icon`) VALUES (3, '选项卡插件', 'default', 0, 20000, 'panel', '选项卡插件', '1.0-SNAPSHOT', NULL, 'fit2cloud-chenyw', 0, NULL, NULL, 'dataease-extensions-tabs-backend', NULL);
ALTER TABLE `panel_link_jump_info`
ADD COLUMN `attach_params` tinyint(1) NULL COMMENT '是否附加点击参数' AFTER `checked`;
@ -334,3 +334,5 @@ CREATE TABLE `panel_outer_params_target_view_info` (
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE utf8mb4_general_ci;
SET FOREIGN_KEY_CHECKS = 1;
update `my_plugin` set `name` = 'X-Pack默认插件' where `plugin_id` = 1;
update `my_plugin` set `module_name` = 'view-bubblemap-backend' where `plugin_id` = 2;

View File

@ -896,6 +896,8 @@ export default {
filter_value_can_null: 'Filter value can not empty',
filter_like: 'Contain',
filter_not_like: 'Not Contain',
filter_in: 'IN',
filter_not_in: 'NOT IN',
chart_details: 'Chart Details',
export_details: 'Export Details',
color_light: 'Light',

View File

@ -897,6 +897,8 @@ export default {
filter_value_can_null: '過濾值不能爲空',
filter_like: '包含',
filter_not_like: '不包含',
filter_in: '屬於',
filter_not_in: '不屬於',
color_light: '明亮',
color_classical: '經典',
color_fresh: '清新',

View File

@ -899,6 +899,8 @@ export default {
filter_value_can_null: '过滤值不能为空',
filter_like: '包含',
filter_not_like: '不包含',
filter_in: '属于',
filter_not_in: '不属于',
color_light: '明亮',
color_classical: '经典',
color_fresh: '清新',

View File

@ -300,8 +300,8 @@
:param="param"
:index="index"
:item="item"
:dimension-data="dimensionData"
:quota-data="quotaData"
:dimension-data="dimension"
:quota-data="quota"
@onDimensionItemChange="dimensionItemChange"
@onDimensionItemRemove="dimensionItemRemove"
@editItemFilter="showDimensionEditFilter"
@ -364,8 +364,8 @@
:param="param"
:index="index"
:item="item"
:dimension-data="dimensionData"
:quota-data="quotaData"
:dimension-data="dimension"
:quota-data="quota"
@onDimensionItemChange="dimensionItemChange"
@onDimensionItemRemove="dimensionItemRemove"
@editItemFilter="showDimensionEditFilter"
@ -438,8 +438,8 @@
:index="index"
:item="item"
:chart="chart"
:dimension-data="dimensionData"
:quota-data="quotaData"
:dimension-data="dimensionD"
:quota-data="quota"
@onQuotaItemChange="quotaItemChange"
@onQuotaItemRemove="quotaItemRemove"
@editItemFilter="showQuotaEditFilter"
@ -476,8 +476,8 @@
:index="index"
:item="item"
:chart="chart"
:dimension-data="dimensionData"
:quota-data="quotaData"
:dimension-data="dimension"
:quota-data="quota"
@onQuotaItemChange="quotaItemChange"
@onQuotaItemRemove="quotaItemRemove"
@editItemFilter="showQuotaEditFilter"
@ -514,8 +514,8 @@
:param="param"
:index="index"
:item="item"
:dimension-data="dimensionData"
:quota-data="quotaData"
:dimension-data="dimension"
:quota-data="quota"
@onItemChange="stackItemChange"
@onItemRemove="stackItemRemove"
/>
@ -559,8 +559,8 @@
:param="param"
:index="index"
:item="item"
:dimension-data="dimensionData"
:quota-data="quotaData"
:dimension-data="dimension"
:quota-data="quota"
@onItemChange="bubbleItemChange"
@onItemRemove="bubbleItemRemove"
/>
@ -592,8 +592,8 @@
:param="param"
:index="index"
:item="item"
:dimension-data="dimensionData"
:quota-data="quotaData"
:dimension-data="dimension"
:quota-data="quota"
@onFilterItemRemove="filterItemRemove"
@editItemFilter="showEditFilter"
/>
@ -629,8 +629,8 @@
:param="param"
:index="index"
:item="item"
:dimension-data="dimensionData"
:quota-data="quotaData"
:dimension-data="dimension"
:quota-data="quota"
@onDimensionItemChange="drillItemChange"
@onDimensionItemRemove="drillItemRemove"
/>
@ -1701,9 +1701,9 @@ export default {
// this.closeChangeChart()
// })
// },
calcData() {
calcData(getData, trigger, needRefreshGroup = false, switchType = false) {
this.changeEditStatus(true)
const view = this.buildParam(true, 'chart', false, false)
const view = this.buildParam(true, 'chart', false, switchType)
if (!view) return
save2Cache(this.panelInfo.id, view).then(() => {
bus.$emit('view-in-cache', { type: 'propChange', viewId: this.param.id })

View File

@ -226,6 +226,10 @@ export default {
return 'API'
} else if (type === 'impala') {
return 'Apache Impala'
}if (type === 'TiDB') {
return 'TiDB'
}if (type === 'StarRocks') {
return 'StarRocks'
}
},