center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

fix: API 数据集校验错误

Browse Source
This commit is contained in:
taojinlong 2022-05-28 21:57:46 +08:00
parent fb446e3943
commit 7879eed3ff
6 changed files with 48 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/main/java/io/dataease/controller/datasource/DatasourceController.java
View File

@ -61,7 +61,7 @@ public class DatasourceController {
@ApiIgnore @ApiIgnore
@PostMapping("/validate") @PostMapping("/validate")
public ResultHolder validate(@RequestBody DatasourceDTO datasource) throws Exception { public ResultHolder validate(@RequestBody Datasource datasource) throws Exception {
return datasourceService.validate(datasource); return datasourceService.validate(datasource);
} }

5
backend/src/main/java/io/dataease/controller/request/datasource/ApiDefinition.java
View File

@ -6,6 +6,7 @@ import lombok.Data;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map;
@Data @Data
public class ApiDefinition { public class ApiDefinition {
@ -14,8 +15,8 @@ public class ApiDefinition {
private String url; private String url;
private String method = "GET"; private String method = "GET";
private List<DatasetTableField> fields; private List<DatasetTableField> fields;
private String request; private ApiDefinitionRequest request;
private String dataPath; private String dataPath;
private String status; private String status;
private List<JsonObject> datas = new ArrayList<>(); private List<Map<String,String>> datas = new ArrayList<>();
} }

6
backend/src/main/java/io/dataease/controller/request/datasource/ApiDefinitionRequest.java
View File

@ -3,12 +3,14 @@ package io.dataease.controller.request.datasource;
import com.google.gson.JsonObject; import com.google.gson.JsonObject;
import lombok.Data; import lombok.Data;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map;
@Data @Data
public class ApiDefinitionRequest { public class ApiDefinitionRequest {
private List<JsonObject> headers = new ArrayList<>(); private List<Map<String, String>> headers = new ArrayList<>();
private JsonObject body = new JsonObject(); private Map<String, Object> body = new HashMap<>();
private AuthManager authManager = new AuthManager(); private AuthManager authManager = new AuthManager();

6
backend/src/main/java/io/dataease/dto/DatasourceDTO.java
View File

@ -1,11 +1,13 @@
package io.dataease.dto; package io.dataease.dto;
import com.google.gson.JsonArray; import io.dataease.controller.request.datasource.ApiDefinition;
import io.dataease.plugins.common.base.domain.Datasource; import io.dataease.plugins.common.base.domain.Datasource;
import io.dataease.plugins.common.constants.DatasourceCalculationMode; import io.dataease.plugins.common.constants.DatasourceCalculationMode;
import io.swagger.annotations.ApiModelProperty; import io.swagger.annotations.ApiModelProperty;
import lombok.Data; import lombok.Data;
import java.util.List;
/** /**
* Author: wangjiahao * Author: wangjiahao
* Date: 2021-05-18 * Date: 2021-05-18
@ -16,7 +18,7 @@ public class DatasourceDTO extends Datasource {
@ApiModelProperty("权限") @ApiModelProperty("权限")
private String privileges; private String privileges;
private JsonArray apiConfiguration; private List<ApiDefinition> apiConfiguration;
private String typeDesc; private String typeDesc;
private DatasourceCalculationMode calculationMode; private DatasourceCalculationMode calculationMode;
} }

33
backend/src/main/java/io/dataease/provider/datasource/ApiProvider.java
View File

@ -1,8 +1,6 @@
package io.dataease.provider.datasource; package io.dataease.provider.datasource;
import com.google.gson.Gson; import com.google.gson.*;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.google.gson.reflect.TypeToken; import com.google.gson.reflect.TypeToken;
import io.dataease.plugins.common.dto.datasource.TableDesc; import io.dataease.plugins.common.dto.datasource.TableDesc;
import io.dataease.plugins.common.dto.datasource.TableField; import io.dataease.plugins.common.dto.datasource.TableField;
@ -22,7 +20,6 @@ import org.springframework.stereotype.Service;
import java.util.*; import java.util.*;
import java.util.stream.Collectors;
@Service("apiProvider") @Service("apiProvider")
public class ApiProvider extends Provider { public class ApiProvider extends Provider {
@ -122,11 +119,10 @@ public class ApiProvider extends Provider {
static public String execHttpRequest(ApiDefinition apiDefinition) throws Exception { static public String execHttpRequest(ApiDefinition apiDefinition) throws Exception {
String response = ""; String response = "";
HttpClientConfig httpClientConfig = new HttpClientConfig(); HttpClientConfig httpClientConfig = new HttpClientConfig();
ApiDefinitionRequest apiDefinitionRequest = new Gson().fromJson(apiDefinition.getRequest(), ApiDefinitionRequest.class); ApiDefinitionRequest apiDefinitionRequest = apiDefinition.getRequest();
//headers for (Map header : apiDefinitionRequest.getHeaders()) {
for (JsonObject header : apiDefinitionRequest.getHeaders()) { if (header.get("name") != null && StringUtils.isNotEmpty(header.get("name").toString()) && header.get("value") != null && StringUtils.isNotEmpty(header.get("value").toString())) {
if (header.get("name") != null && StringUtils.isNotEmpty(header.get("name").getAsString()) && header.get("value") != null && StringUtils.isNotEmpty(header.get("value").getAsString())) { httpClientConfig.addHeader(header.get("name").toString(), header.get("value").toString());
httpClientConfig.addHeader(header.get("name").getAsString(), header.get("value").getAsString());
} }
} }
@ -147,18 +143,18 @@ public class ApiProvider extends Provider {
if (apiDefinitionRequest.getBody().get("type") == null) { if (apiDefinitionRequest.getBody().get("type") == null) {
throw new Exception("请求类型不能为空"); throw new Exception("请求类型不能为空");
} }
String type = apiDefinitionRequest.getBody().get("type").getAsString(); String type = apiDefinitionRequest.getBody().get("type").toString();
if (StringUtils.equalsAny(type, "JSON", "XML", "Raw")) { if (StringUtils.equalsAny(type, "JSON", "XML", "Raw")) {
String raw = null; String raw = null;
if (apiDefinitionRequest.getBody().get("raw") != null) { if (apiDefinitionRequest.getBody().get("raw") != null) {
raw = apiDefinitionRequest.getBody().get("raw").getAsString(); raw = apiDefinitionRequest.getBody().get("raw").toString();
response = HttpClientUtil.post(apiDefinition.getUrl(), raw, httpClientConfig); response = HttpClientUtil.post(apiDefinition.getUrl(), raw, httpClientConfig);
} }
} }
if (StringUtils.equalsAny(type, "Form_Data", "WWW_FORM")) { if (StringUtils.equalsAny(type, "Form_Data", "WWW_FORM")) {
if (apiDefinitionRequest.getBody().get("kvs") != null) { if (apiDefinitionRequest.getBody().get("kvs") != null) {
Map<String, String> body = new HashMap<>(); Map<String, String> body = new HashMap<>();
JsonArray kvsArr = apiDefinitionRequest.getBody().getAsJsonArray("kvs"); JsonArray kvsArr = JsonParser.parseString(apiDefinitionRequest.getBody().get("kvs").toString()).getAsJsonArray();
for (int i = 0; i < kvsArr.size(); i++) { for (int i = 0; i < kvsArr.size(); i++) {
JsonObject kv = kvsArr.get(i).getAsJsonObject(); JsonObject kv = kvsArr.get(i).getAsJsonObject();
if (kv.get("name") != null) { if (kv.get("name") != null) {
@ -191,7 +187,7 @@ public class ApiProvider extends Provider {
throw new Exception("jsonPath 路径错误：" + e.getMessage()); throw new Exception("jsonPath 路径错误：" + e.getMessage());
} }
List<JsonObject> dataList = new ArrayList<>(); List<Map<String,String>> dataList = new ArrayList<>();
List<DatasetTableField> fields = new ArrayList<>(); List<DatasetTableField> fields = new ArrayList<>();
Set<String> fieldKeys = new HashSet<>(); Set<String> fieldKeys = new HashSet<>();
//第一遍获取 field //第一遍获取 field
@ -213,11 +209,11 @@ public class ApiProvider extends Provider {
} }
//第二遍获取 data //第二遍获取 data
for (LinkedHashMap data : datas) { for (LinkedHashMap data : datas) {
JsonObject jsonObject = new JsonObject(); Map<String,String> mapData = new HashMap<>();
for (String key : fieldKeys) { for (String key : fieldKeys) {
jsonObject.addProperty(key, Optional.ofNullable(data.get(key)).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " ")); mapData.put(key, Optional.ofNullable(data.get(key)).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " "));
} }
dataList.add(jsonObject); dataList.add(mapData);
} }
apiDefinition.setDatas(dataList); apiDefinition.setDatas(dataList);
apiDefinition.setFields(fields); apiDefinition.setFields(fields);
@ -248,9 +244,8 @@ public class ApiProvider extends Provider {
private ApiDefinition checkApiDefinition(DatasourceRequest datasourceRequest) throws Exception { private ApiDefinition checkApiDefinition(DatasourceRequest datasourceRequest) throws Exception {
List<ApiDefinition> apiDefinitionList = new ArrayList<>(); List<ApiDefinition> apiDefinitionList = new ArrayList<>();
List<ApiDefinition> apiDefinitionListTemp = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), new TypeToken<List<ApiDefinition>>() { List<ApiDefinition> apiDefinitionListTemp = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), new TypeToken<List<ApiDefinition>>() {}.getType());
}.getType()); if (CollectionUtils.isNotEmpty(apiDefinitionListTemp)) {
if (CollectionUtils.isEmpty(apiDefinitionListTemp)) {
for (ApiDefinition apiDefinition : apiDefinitionListTemp) { for (ApiDefinition apiDefinition : apiDefinitionListTemp) {
if (apiDefinition.getName().equalsIgnoreCase(datasourceRequest.getTable())) { if (apiDefinition.getName().equalsIgnoreCase(datasourceRequest.getTable())) {
apiDefinitionList.add(apiDefinition); apiDefinitionList.add(apiDefinition);

43
backend/src/main/java/io/dataease/service/datasource/DatasourceService.java
View File

@ -7,6 +7,7 @@ import com.google.gson.JsonParser;
import com.google.gson.reflect.TypeToken; import com.google.gson.reflect.TypeToken;
import io.dataease.auth.annotation.DeCleaner; import io.dataease.auth.annotation.DeCleaner;
import io.dataease.commons.constants.RedisConstants; import io.dataease.commons.constants.RedisConstants;
import io.dataease.commons.utils.BeanUtils;
import io.dataease.ext.ExtDataSourceMapper; import io.dataease.ext.ExtDataSourceMapper;
import io.dataease.ext.query.GridExample; import io.dataease.ext.query.GridExample;
import io.dataease.commons.constants.DePermissionType; import io.dataease.commons.constants.DePermissionType;
@ -144,16 +145,15 @@ public class DatasourceService {
datasourceDTO.setConfiguration(datasourceDTO.getConfiguration()); datasourceDTO.setConfiguration(datasourceDTO.getConfiguration());
} }
if (datasourceDTO.getType().equalsIgnoreCase(DatasourceTypes.api.toString())) { if (datasourceDTO.getType().equalsIgnoreCase(DatasourceTypes.api.toString())) {
JsonArray apiDefinitionList = JsonParser.parseString(datasourceDTO.getConfiguration()).getAsJsonArray(); List<ApiDefinition> apiDefinitionList = new Gson().fromJson(datasourceDTO.getConfiguration(), new TypeToken<ArrayList<ApiDefinition>>() {}.getType());
JsonArray apiDefinitionListWithStatus = new JsonArray(); List<ApiDefinition> apiDefinitionListWithStatus = new ArrayList<>();
int success = 0; int success = 0;
if (StringUtils.isNotEmpty(datasourceDTO.getStatus())) { if (StringUtils.isNotEmpty(datasourceDTO.getStatus())) {
JsonObject apiItemStatuses = JsonParser.parseString(datasourceDTO.getStatus()).getAsJsonObject(); JsonObject apiItemStatuses = JsonParser.parseString(datasourceDTO.getStatus()).getAsJsonObject();
for (Object apiDefinition : apiDefinitionList) { for (ApiDefinition apiDefinition : apiDefinitionList) {
String status = apiItemStatuses.get(JsonParser.parseString(apiDefinition.toString()).getAsJsonObject().get("name").getAsString()).getAsString(); String status = apiItemStatuses.get(apiDefinition.getName()).getAsString();
JsonObject object = JsonParser.parseString(apiDefinition.toString()).getAsJsonObject(); apiDefinition.setStatus(status);
object.addProperty("status", status); apiDefinitionListWithStatus.add(apiDefinition);
apiDefinitionListWithStatus.add(object);
if (StringUtils.isNotEmpty(status) && status.equalsIgnoreCase("Success")) { if (StringUtils.isNotEmpty(status) && status.equalsIgnoreCase("Success")) {
success++; success++;
} }
@ -232,40 +232,41 @@ public class DatasourceService {
} }
} }
public ResultHolder validate(DatasourceDTO datasource) throws Exception { public ResultHolder validate(Datasource datasource) throws Exception {
DatasourceDTO datasourceDTO = new DatasourceDTO();
BeanUtils.copyBean(datasourceDTO, datasource);
try { try {
Provider datasourceProvider = ProviderFactory.getProvider(datasource.getType()); Provider datasourceProvider = ProviderFactory.getProvider(datasource.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest(); DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(datasource); datasourceRequest.setDatasource(datasource);
String datasourceStatus = datasourceProvider.checkStatus(datasourceRequest); String datasourceStatus = datasourceProvider.checkStatus(datasourceRequest);
if (datasource.getType().equalsIgnoreCase("api")) { if (datasource.getType().equalsIgnoreCase("api")) {
Gson gson = new Gson();
int success = 0; int success = 0;
JsonArray apiDefinitionList = JsonParser.parseString(datasource.getConfiguration()).getAsJsonArray(); List<ApiDefinition> apiDefinitionList = new Gson().fromJson(datasource.getConfiguration(), new TypeToken<List<ApiDefinition>>() {}.getType());
JsonArray apiDefinitionListWithStatus = new JsonArray(); List<ApiDefinition> apiDefinitionListWithStatus = new ArrayList<>();
if (StringUtils.isNotEmpty(datasourceStatus)) { if (StringUtils.isNotEmpty(datasourceStatus)) {
JsonObject apiItemStatuses = JsonParser.parseString(datasourceStatus).getAsJsonObject(); JsonObject apiItemStatuses = JsonParser.parseString(datasourceStatus).getAsJsonObject();
for (Object apiDefinition : apiDefinitionList) { for (ApiDefinition apiDefinition : apiDefinitionList) {
String status = apiItemStatuses.get(JsonParser.parseString(apiDefinition.toString()).getAsJsonObject().get("name").getAsString()).getAsString(); String status = apiItemStatuses.get(apiDefinition.getName()).getAsString();
JsonObject object = JsonParser.parseString(apiDefinition.toString()).getAsJsonObject(); apiDefinition.setStatus(status);
object.addProperty("status", status); apiDefinitionListWithStatus.add(apiDefinition);
apiDefinitionListWithStatus.add(object);
if (StringUtils.isNotEmpty(status) && status.equalsIgnoreCase("Success")) { if (StringUtils.isNotEmpty(status) && status.equalsIgnoreCase("Success")) {
success++; success++;
} }
} }
} }
datasource.setApiConfiguration(apiDefinitionListWithStatus); datasourceDTO.setApiConfiguration(apiDefinitionListWithStatus);
if (success == apiDefinitionList.size()) { if (success == apiDefinitionList.size()) {
return ResultHolder.success(datasource); return ResultHolder.success(datasourceDTO);
} }
if (success > 0 && success < apiDefinitionList.size()) { if (success > 0 && success < apiDefinitionList.size()) {
return ResultHolder.error("Datasource has invalid tables", datasource); return ResultHolder.error("Datasource has invalid tables", datasourceDTO);
} }
return ResultHolder.error("Datasource is invalid.", datasource); return ResultHolder.error("Datasource is invalid.", datasourceDTO);
} }
return ResultHolder.success(datasource); return ResultHolder.success(datasourceDTO);
} catch (Exception e) { } catch (Exception e) {
return ResultHolder.error("Datasource is invalid: " + e.getMessage()); return ResultHolder.error("Datasource is invalid: " + e.getMessage());
} }