diff --git a/backend/src/main/java/io/dataease/controller/request/datasource/ApiDefinition.java b/backend/src/main/java/io/dataease/controller/request/datasource/ApiDefinition.java index 7885b80684..b409549ae8 100644 --- a/backend/src/main/java/io/dataease/controller/request/datasource/ApiDefinition.java +++ b/backend/src/main/java/io/dataease/controller/request/datasource/ApiDefinition.java @@ -1,7 +1,7 @@ package io.dataease.controller.request.datasource; -import com.google.gson.JsonObject; -import io.dataease.plugins.common.base.domain.DatasetTableField; +import com.alibaba.fastjson.JSONObject; +import io.dataease.dto.dataset.DatasetTableFieldDTO; import lombok.Data; import java.util.ArrayList; @@ -14,9 +14,13 @@ public class ApiDefinition { private String desc; private String url; private String method = "GET"; - private List fields; + private List fields; private ApiDefinitionRequest request; private String dataPath; private String status; private List> datas = new ArrayList<>(); + private List jsonFields = new ArrayList<>(); + private int previewNum = 10; + private int maxPreviewNum = 10; + } diff --git a/backend/src/main/java/io/dataease/dto/dataset/DatasetTableFieldDTO.java b/backend/src/main/java/io/dataease/dto/dataset/DatasetTableFieldDTO.java new file mode 100644 index 0000000000..83a65cb94d --- /dev/null +++ b/backend/src/main/java/io/dataease/dto/dataset/DatasetTableFieldDTO.java @@ -0,0 +1,9 @@ +package io.dataease.dto.dataset; + +import io.dataease.plugins.common.base.domain.DatasetTableField; +import lombok.Data; + +@Data +public class DatasetTableFieldDTO extends DatasetTableField { + private String jsonPath; +} diff --git a/backend/src/main/java/io/dataease/provider/datasource/ApiProvider.java b/backend/src/main/java/io/dataease/provider/datasource/ApiProvider.java index d588d617d8..ceddb3075a 100644 --- a/backend/src/main/java/io/dataease/provider/datasource/ApiProvider.java +++ b/backend/src/main/java/io/dataease/provider/datasource/ApiProvider.java @@ -1,13 +1,17 @@ package io.dataease.provider.datasource; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; import com.google.gson.*; import com.google.gson.reflect.TypeToken; +import io.dataease.commons.utils.Md5Utils; +import io.dataease.dto.dataset.DatasetTableFieldDTO; import io.dataease.plugins.common.dto.datasource.TableDesc; import io.dataease.plugins.common.dto.datasource.TableField; import io.dataease.plugins.common.request.datasource.DatasourceRequest; import io.dataease.plugins.datasource.provider.Provider; import com.jayway.jsonpath.JsonPath; -import io.dataease.plugins.common.base.domain.DatasetTableField; import io.dataease.commons.utils.HttpClientConfig; import io.dataease.commons.utils.HttpClientUtil; import io.dataease.controller.request.datasource.ApiDefinition; @@ -20,6 +24,7 @@ import org.springframework.stereotype.Service; import java.util.*; +import java.util.stream.Collectors; @Service("apiProvider") public class ApiProvider extends Provider { @@ -68,7 +73,7 @@ public class ApiProvider extends Provider { private List getTableFileds(ApiDefinition apiDefinition, String response) throws Exception { List tableFields = new ArrayList<>(); - for (DatasetTableField field : checkApiDefinition(apiDefinition, response).getFields()) { + for (DatasetTableFieldDTO field : checkApiDefinition(apiDefinition, response).getFields()) { TableField tableField = new TableField(); tableField.setFieldName(field.getOriginName()); tableField.setRemarks(field.getName()); @@ -86,7 +91,7 @@ public class ApiProvider extends Provider { for (ApiDefinition apiDefinition : lists) { if (datasourceRequest.getTable().equalsIgnoreCase(apiDefinition.getName())) { String response = ApiProvider.execHttpRequest(apiDefinition); - for (DatasetTableField field : checkApiDefinition(apiDefinition, response).getFields()) { + for (DatasetTableFieldDTO field : checkApiDefinition(apiDefinition, response).getFields()) { TableField tableField = new TableField(); tableField.setFieldName(field.getOriginName()); tableField.setRemarks(field.getName()); @@ -171,77 +176,171 @@ public class ApiProvider extends Provider { return response; } + static public ApiDefinition checkApiDefinition(ApiDefinition apiDefinition, String response) throws Exception { if (StringUtils.isEmpty(response)) { throw new Exception("该请求返回数据为空"); } - List datas = new ArrayList<>(); - try { - Object object = JsonPath.read(response, apiDefinition.getDataPath()); + List jsonFields = new ArrayList<>(); + String rootPath; + if (response.startsWith("[")) { + rootPath = "$[*]"; + JSONArray jsonArray = JSONObject.parseArray(response); + for (Object o : jsonArray) { + handleStr(apiDefinition, o.toString(), jsonFields, rootPath); + } + } else { + rootPath = "$"; + handleStr(apiDefinition, response, jsonFields, rootPath); + } + apiDefinition.setJsonFields(jsonFields); + return apiDefinition; + } + + + static private void handleStr(ApiDefinition apiDefinition, String jsonStr, List objects, String rootPath) { + if (jsonStr.startsWith("[")) { + JSONArray jsonArray = JSONObject.parseArray(jsonStr); + for (Object o : jsonArray) { + handleStr(apiDefinition, o.toString(), objects, rootPath); + } + } else { + JSONObject jsonObject = JSONObject.parseObject(jsonStr); + for (String s : jsonObject.keySet()) { + String value = jsonObject.getString(s); + if (StringUtils.isNotEmpty(value) && value.startsWith("[")) { + rootPath = rootPath + "." + s; + JSONArray jsonArray = JSONObject.parseArray(jsonObject.getString(s)); + List children = new ArrayList<>(); + for (Object o : jsonArray) { + handleStr(apiDefinition, o.toString(), children, rootPath + "[*]"); + } + JSONObject o = new JSONObject(); + o.put("children", children); + o.put("childrenDataType", "LIST"); + o.put("jsonPath", rootPath); + setProperty(apiDefinition, o, s); + if (!hasItem(objects, o, null)) { + objects.add(o); + } + } else if (StringUtils.isNotEmpty(value) && value.startsWith("{")) { + List children = new ArrayList<>(); + rootPath = rootPath + "." + s; + handleStr(apiDefinition, jsonObject.getString(s), children, rootPath); + JSONObject o = new JSONObject(); + o.put("children", children); + o.put("childrenDataType", "OBJECT"); + o.put("jsonPath", rootPath); + setProperty(apiDefinition, o, s); + if (!hasItem(objects, o, null)) { + objects.add(o); + } + } else { + JSONObject o = new JSONObject(); + o.put("children", null); + o.put("jsonPath", rootPath + "." + s); + setProperty(apiDefinition, o, s); + if (!hasItem(objects, o, StringUtils.isNotEmpty(jsonObject.getString(s))? jsonObject.getString(s) : "")) { + JSONArray array = new JSONArray(); + array.add(StringUtils.isNotEmpty(jsonObject.getString(s))? jsonObject.getString(s) : ""); + o.put("value", array); + objects.add(o); + } + } + + } + } + } + + static private void setProperty(ApiDefinition apiDefinition, JSONObject o, String s) { + o.put("originName", s); + o.put("name", s); + o.put("type", "STRING"); + o.put("checked", false); + o.put("size", 65535); + o.put("deExtractType", 0); + o.put("deType", 0); + o.put("extField", 0); + o.put("checked", false); + for (DatasetTableFieldDTO fieldDTO : apiDefinition.getFields()) { + if (fieldDTO.getJsonPath().equals(o.getString("jsonPath"))) { + o.put("checked", true); + o.put("deExtractType", fieldDTO.getDeExtractType()); + o.put("name", fieldDTO.getName()); + } + } + + } + + static private boolean hasItem(List objects, JSONObject o, String value) { + boolean has = false; + for (JSONObject object : objects) { + + JSONObject jsonObject = JSONObject.parseObject(object.toJSONString()); + jsonObject.remove("value"); + jsonObject.remove("id"); + if (Md5Utils.md5(jsonObject.toString()).equals(Md5Utils.md5(o.toString()))) { + has = true; + if(value != null){ + JSONArray array = object.getJSONArray("value"); + array.add(value); + object.put("value", array); + } + break; + } + } + + return has; + } + + private List fetchResult(String result, ApiDefinition apiDefinition) { + List dataList = new LinkedList<>(); + if (StringUtils.isNotEmpty(apiDefinition.getDataPath()) && CollectionUtils.isNotEmpty(apiDefinition.getJsonFields())) { + List datas = new ArrayList<>(); + Object object = JsonPath.read(result, apiDefinition.getDataPath()); if (object instanceof List) { datas = (List) object; } else { datas.add((LinkedHashMap) object); } - } catch (Exception e) { - throw new Exception("jsonPath 路径错误：" + e.getMessage()); - } - - List> dataList = new ArrayList<>(); - List fields = new ArrayList<>(); - Set fieldKeys = new HashSet<>(); - //第一遍获取 field - for (LinkedHashMap data : datas) { - Set keys = data.keySet(); - for (String key : keys) { - if (!fieldKeys.contains(key)) { - fieldKeys.add(key); - DatasetTableField tableField = new DatasetTableField(); - tableField.setOriginName(key); - tableField.setName(key); - tableField.setSize(65535); - tableField.setDeExtractType(0); - tableField.setDeType(0); - tableField.setExtField(0); - fields.add(tableField); + for (LinkedHashMap data : datas) { + String[] row = new String[apiDefinition.getFields().size()]; + int i = 0; + for (DatasetTableFieldDTO field : apiDefinition.getFields()) { + row[i] = Optional.ofNullable(data.get(field.getOriginName())).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " "); + i++; + } + dataList.add(row); + } + } else { + List jsonPaths = apiDefinition.getFields().stream().map(DatasetTableFieldDTO::getJsonPath).collect(Collectors.toList()); + Long maxLenth = 0l; + List> columnDataList = new ArrayList<>(); + for (int i = 0; i < jsonPaths.size(); i++) { + List datas = new ArrayList<>(); + Object object = JsonPath.read(result, jsonPaths.get(i)); + if (object instanceof List) { + datas = (List) object; + } else { + datas.add((String) object); + } + maxLenth = maxLenth > datas.size() ? maxLenth : datas.size(); + columnDataList.add(datas); + } + for (int i = 0; i < maxLenth; i++) { + String[] row = new String[apiDefinition.getFields().size()]; + dataList.add(row); + } + for (int i = 0; i < columnDataList.size(); i++) { + for (int j = 0; j < columnDataList.get(i).size(); j++) { + dataList.get(j)[i] = String.valueOf(columnDataList.get(i).get(j)); } } } - //第二遍获取 data - for (LinkedHashMap data : datas) { - Map mapData = new HashMap<>(); - for (String key : fieldKeys) { - mapData.put(key, Optional.ofNullable(data.get(key)).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " ")); - } - dataList.add(mapData); - } - apiDefinition.setDatas(dataList); - apiDefinition.setFields(fields); - return apiDefinition; - } - - private List fetchResult(String result, ApiDefinition apiDefinition) { - List dataList = new LinkedList<>(); - List datas = new ArrayList<>(); - - Object object = JsonPath.read(result, apiDefinition.getDataPath()); - if (object instanceof List) { - datas = (List) object; - } else { - datas.add((LinkedHashMap) object); - } - for (LinkedHashMap data : datas) { - String[] row = new String[apiDefinition.getFields().size()]; - int i = 0; - for (DatasetTableField field : apiDefinition.getFields()) { - row[i] = Optional.ofNullable(data.get(field.getOriginName())).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " "); - i++; - } - dataList.add(row); - } return dataList; } + private ApiDefinition checkApiDefinition(DatasourceRequest datasourceRequest) throws Exception { List apiDefinitionList = new ArrayList<>(); List apiDefinitionListTemp = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), new TypeToken>() {}.getType()); diff --git a/frontend/src/lang/en.js b/frontend/src/lang/en.js index e8da510e74..66399ddf4a 100644 --- a/frontend/src/lang/en.js +++ b/frontend/src/lang/en.js @@ -1124,6 +1124,7 @@ export default { max_more_than_mix: 'Max must more than Min' }, dataset: { + field_rename: 'Rename Field', params_work: 'Effective only when editing SQL', sql_variable_limit_1: '1、SQL variables can only be used in where conditions', sql_variable_limit_2: '2、Example：select * from table_name where column_name1=${parm_name1} and column_name2 in ${parm_name2}', @@ -1455,6 +1456,7 @@ export default { add_api_table: 'Add API table', edit_api_table: 'Edit API table', base_info: 'Basic information', + column_info: 'Data structure', request: 'Request', path_all_info: 'Please fill in the full address', req_param: 'Request parameters', @@ -1477,7 +1479,9 @@ export default { api_table_not_empty: 'API data table cannot be empty', has_repeat_name: 'Duplicate API data table name', valid: 'Valid', - invalid: 'Invalid' + invalid: 'Invalid', + api_step_1: 'Connection API', + api_step_2: 'Extract data' }, pblink: { key_pwd: 'Please enter the password to open the link', diff --git a/frontend/src/lang/tw.js b/frontend/src/lang/tw.js index f0398bcb86..8ebeed339f 100644 --- a/frontend/src/lang/tw.js +++ b/frontend/src/lang/tw.js @@ -1124,6 +1124,7 @@ export default { max_more_than_mix: '最大值必須大於最小值' }, dataset: { + field_rename: '字段重命名', params_work: '僅在編輯 sql 時生效', sql_variable_limit_1: '1、SQL變數只能在WHERE條件中使用', sql_variable_limit_2: '2、示例：select * from table_name where column_name1=${parm_name1} and column_name2 in ${parm_name2}', @@ -1456,6 +1457,7 @@ export default { add_api_table: '添加 API 數據表', edit_api_table: '編輯 API 數據表', base_info: '基礎信息', + column_info: '資料結構', request: '請求', path_all_info: '請輸入完整地址', req_param: '請求參數', @@ -1478,7 +1480,9 @@ export default { api_table_not_empty: 'API 數據表不能為空', has_repeat_name: 'API 數據表名稱重複', valid: '有效', - invalid: '無效' + invalid: '無效', + api_step_1: '連接API', + api_step_2: '選取數據' }, pblink: { key_pwd: '請輸入密碼打開鏈接', diff --git a/frontend/src/lang/zh.js b/frontend/src/lang/zh.js index e3ff9dcd52..2ddfca37ab 100644 --- a/frontend/src/lang/zh.js +++ b/frontend/src/lang/zh.js @@ -1126,6 +1126,7 @@ export default { max_more_than_mix: '最大值必须大于最小值' }, dataset: { + field_rename: '字段重命名', params_work: '仅在编辑sql时生效', select_year: '选择年', sql_variable_limit_1: '1、SQL 变量只能在 WHERE 条件中使用', @@ -1464,6 +1465,7 @@ export default { add_api_table: '添加API数据表', edit_api_table: '编辑API数据表', base_info: '基础信息', + column_info: '数据结构', request: '请求', path_all_info: '请填入完整地址', req_param: '请求参数', @@ -1486,7 +1488,9 @@ export default { api_table_not_empty: 'API 数据表不能为空', has_repeat_name: 'API 数据表名称重复', valid: '有效', - invalid: '无效' + invalid: '无效', + api_step_1: '连接API', + api_step_2: '提取数据' }, pblink: { key_pwd: '请输入密码打开链接', diff --git a/frontend/src/views/system/datasource/DsConfiguration.vue b/frontend/src/views/system/datasource/DsConfiguration.vue index 38844dc326..c92aaac26c 100644 --- a/frontend/src/views/system/datasource/DsConfiguration.vue +++ b/frontend/src/views/system/datasource/DsConfiguration.vue @@ -3,13 +3,13 @@ @@ -46,12 +46,13 @@ - - + + - +

{{ $t('datasource.base_info') }}

@@ -76,34 +77,74 @@ - - - - - - - - - - - - + +

{{ $t('datasource.column_info') }}

+ + + + + + + + + + + + + + + +

{{ $t('dataset.data_preview') }}

+ + + + + + +
+ {{ $t('commons.cancel') }} {{ $t('fu.steps.next') }} @@ -143,14 +184,14 @@ @@ -164,7 +205,7 @@ :label="$t('datasource.keytab_Key_path')">

- {{$t('datasource.kerbers_info')}} + {{ $t('datasource.kerbers_info') }}

@@ -172,13 +213,15 @@ - + - + @@ -203,7 +246,7 @@ + v-if="form.type=='oracle' || form.type=='sqlServer' || form.type=='pg' || form.type=='redshift' || form.type=='db2'"> {{ $t('datasource.get_schema') }} @@ -211,8 +254,8 @@ + v-if="form.type=='oracle' || form.type=='sqlServer' || form.type=='pg' || form.type=='redshift' || form.type=='db2'" + :label="$t('datasource.schema')"> @@ -248,7 +291,8 @@ - + @@ -413,7 +457,9 @@ export default { }, authManager: {} }, - fields: [] + fields: [], + jsonFields: [], + maxPreviewNum: '' }, reqOptions: [{id: 'GET', label: 'GET'}, {id: 'POST', label: 'POST'}], loading: false, @@ -435,7 +481,12 @@ export default { }, { id: 'kerberos', label: 'Kerberos' - }] + }], + fieldOptions: [ + {label: this.$t('dataset.text'), value: 0}, + {label: this.$t('dataset.value'), value: 2}, + {label: this.$t('dataset.value') + '(' + this.$t('dataset.float') + ')', value: 3}, + ], } }, created() { @@ -488,8 +539,10 @@ export default { this.apiItem.status = 'Success' this.$success(i18n.t('commons.success')) this.active++ - this.apiItem.fields = res.data.fields - this.$refs.plxTable.reloadData(res.data.datas) + this.apiItem.jsonFields = res.data.jsonFields + this.apiItem.maxPreviewNum = res.data.maxPreviewNum + this.handleFiledChange(); + this.previewData() }).catch(res => { this.loading = false this.disabledNext = false @@ -534,30 +587,50 @@ export default { deleteItem(item) { this.form.apiConfiguration.splice(this.form.apiConfiguration.indexOf(item), 1) }, - validateApi(item) { - if (undefined) { - - } else { - this.$refs.apiItem.validate(valid => { - if (valid) { - const data = JSON.parse(JSON.stringify(this.apiItem)) - data.request = JSON.stringify(data.request) - this.loading = true - checkApiDatasource(data).then(res => { - this.loading = false - this.$success(i18n.t('commons.success')) - this.apiItem.fields = res.data.fields - this.$refs.plxTable.reloadData(res.data.datas) - }).catch(res => { - this.loading = false - }) - } else { - return false - } - }) + handleCheckAllChange(row) { + this.handleCheckChange(row) + this.handleFiledChange() + this.previewData() + }, + handleFiledChange(data) { + this.apiItem.fields = [] + let jsonField = data === undefined ? this.apiItem.jsonFields : data; + jsonField.forEach((item) => { + if (item.checked && item.children === null) { + this.apiItem.fields.push(item) + } + if (item.children !== null) { + this.handleFiledChange(item.children) + } + }) + }, + previewData() { + let datas = []; + for (let i = 0; i < this.apiItem.maxPreviewNum; i++) { + datas.push({}) } - } + for (let i = 0; i < this.apiItem.fields.length; i++) { + for (let j = 0; j < this.apiItem.fields[i].value.length; j++) { + this.$set(datas[j], this.apiItem.fields[i].name , this.apiItem.fields[i].value[j]); + } + } + this.$refs.plxTable.reloadData(datas) + }, + handleCheckChange(node) { + if (node.children !== null) { + node.children.forEach((item) => { + item.checked = node.checked; + this.handleCheckChange(item) + }); + } + + }, + fieldNameChange(row) { + this.previewData() + }, + fieldTypeChange(row) { + } } } @@ -586,4 +659,12 @@ export default { float: right; margin-right: 45px; } + +.tip { + padding: 3px 5px; + font-size: 16px; + border-radius: 0; + border-left: 4px solid #409EFF; + margin: 5px 5px 10px 5px; +}