center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

fix(数据源): API 数据源超时设置

Browse Source
This commit is contained in:
taojinlong 2022-07-26 17:26:50 +08:00
parent d0906e6f6c
commit 798b1ba778
9 changed files with 64 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
backend/src/main/java/io/dataease/commons/utils/HttpClientConfig.java
View File

@ -19,13 +19,13 @@ public class HttpClientConfig {
// 设置从connect Manager获取Connection 超时时间，单位毫秒。这个属性是新加的属性，因为目前版本是可以共享连接池的
private int connectionRequestTimeout = 5000;
// 请求获取数据的超时时间，单位毫秒。 如果访问一个接口，多少时间内无法返回数据，就直接放弃此次调用
private int cocketTimeout = 60000;
private int socketTimeout = 60000;
public RequestConfig buildRequestConfig() {
Builder builder = RequestConfig.custom();
builder.setConnectTimeout(connectTimeout);
builder.setConnectionRequestTimeout(connectionRequestTimeout);
builder.setSocketTimeout(cocketTimeout);
builder.setSocketTimeout(socketTimeout);
return builder.build();
}
@ -61,12 +61,12 @@ public class HttpClientConfig {
this.connectionRequestTimeout = connectionRequestTimeout;
}
public int getCocketTimeout() {
return cocketTimeout;
public int getSocketTimeout() {
return socketTimeout;
}
public void setCocketTimeout(int cocketTimeout) {
this.cocketTimeout = cocketTimeout;
public void setSocketTimeout(int cocketTimeout) {
this.socketTimeout = cocketTimeout;
}
}

1
backend/src/main/java/io/dataease/controller/request/datasource/ApiDefinition.java
View File

@ -22,5 +22,6 @@ public class ApiDefinition {
private List<JSONObject> jsonFields = new ArrayList<>();
private int previewNum = 10;
private int maxPreviewNum = 10;
private int serialNumber;
}

18
backend/src/main/java/io/dataease/provider/datasource/ApiProvider.java
View File

@ -5,6 +5,7 @@ import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.gson.*;
import com.google.gson.reflect.TypeToken;
import io.dataease.controller.sys.response.BasicInfo;
import io.dataease.dto.dataset.DatasetTableFieldDTO;
import io.dataease.plugins.common.dto.datasource.TableDesc;
import io.dataease.plugins.common.dto.datasource.TableField;
@ -16,23 +17,28 @@ import io.dataease.commons.utils.HttpClientUtil;
import io.dataease.controller.request.datasource.ApiDefinition;
import io.dataease.controller.request.datasource.ApiDefinitionRequest;
import io.dataease.service.system.SystemParameterService;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.*;
import java.util.stream.Collectors;
@Service("apiProvider")
public class ApiProvider extends Provider {
@Resource
private SystemParameterService systemParameterService;
@Override
public List<String[]> getData(DatasourceRequest datasourceRequest) throws Exception {
BasicInfo basicInfo = systemParameterService.basicInfo();
ApiDefinition apiDefinition = checkApiDefinition(datasourceRequest);
String response = execHttpRequest(apiDefinition);
String response = execHttpRequest(apiDefinition, StringUtils.isNotBlank(basicInfo.getFrontTimeOut()) ? Integer.parseInt(basicInfo.getFrontTimeOut()) : 10);
return fetchResult(response, apiDefinition);
}
@ -56,11 +62,12 @@ public class ApiProvider extends Provider {
}
public Map<String, List> fetchResultAndField(DatasourceRequest datasourceRequest) throws Exception {
BasicInfo basicInfo = systemParameterService.basicInfo();
Map<String, List> result = new HashMap<>();
List<String[]> dataList = new ArrayList<>();
List<TableField> fieldList = new ArrayList<>();
ApiDefinition apiDefinition = checkApiDefinition(datasourceRequest);
String response = execHttpRequest(apiDefinition);
String response = execHttpRequest(apiDefinition, StringUtils.isNotBlank(basicInfo.getFrontTimeOut()) ? Integer.parseInt(basicInfo.getFrontTimeOut()) : 10);
fieldList = getTableFileds(apiDefinition, response);
result.put("fieldList", fieldList);
@ -84,12 +91,13 @@ public class ApiProvider extends Provider {
}
public List<TableField> getTableFileds(DatasourceRequest datasourceRequest) throws Exception {
BasicInfo basicInfo = systemParameterService.basicInfo();
List<ApiDefinition> lists = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), new TypeToken<List<ApiDefinition>>() {
}.getType());
List<TableField> tableFields = new ArrayList<>();
for (ApiDefinition apiDefinition : lists) {
if (datasourceRequest.getTable().equalsIgnoreCase(apiDefinition.getName())) {
String response = ApiProvider.execHttpRequest(apiDefinition);
String response = ApiProvider.execHttpRequest(apiDefinition, StringUtils.isNotBlank(basicInfo.getFrontTimeOut()) ? Integer.parseInt(basicInfo.getFrontTimeOut()) : 10);
for (DatasetTableFieldDTO field : checkApiDefinition(apiDefinition, response).getFields()) {
TableField tableField = new TableField();
tableField.setFieldName(field.getOriginName());
@ -120,9 +128,11 @@ public class ApiProvider extends Provider {
return gson.toJson(apiItemStatuses);
}
static public String execHttpRequest(ApiDefinition apiDefinition) throws Exception {
static public String execHttpRequest(ApiDefinition apiDefinition, int socketTimeout) throws Exception {
String response = "";
HttpClientConfig httpClientConfig = new HttpClientConfig();
httpClientConfig.setSocketTimeout(socketTimeout * 1000);
ApiDefinitionRequest apiDefinitionRequest = apiDefinition.getRequest();
for (Map header : apiDefinitionRequest.getHeaders()) {
if (header.get("name") != null && StringUtils.isNotEmpty(header.get("name").toString()) && header.get("value") != null && StringUtils.isNotEmpty(header.get("value").toString())) {

32
backend/src/main/java/io/dataease/service/datasource/DatasourceService.java
View File

@ -10,6 +10,7 @@ import com.google.gson.reflect.TypeToken;
import io.dataease.auth.annotation.DeCleaner;
import io.dataease.commons.constants.RedisConstants;
import io.dataease.commons.utils.BeanUtils;
import io.dataease.controller.sys.response.BasicInfo;
import io.dataease.ext.ExtDataSourceMapper;
import io.dataease.ext.query.GridExample;
import io.dataease.commons.constants.DePermissionType;
@ -46,6 +47,7 @@ import io.dataease.provider.datasource.ApiProvider;
import io.dataease.service.dataset.DataSetGroupService;
import io.dataease.service.message.DeMsgutil;
import io.dataease.service.sys.SysAuthService;
import io.dataease.service.system.SystemParameterService;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -76,6 +78,8 @@ public class DatasourceService {
private SysAuthService sysAuthService;
@Resource
private Environment env;
@Resource
private SystemParameterService systemParameterService;
public Collection<DataSourceType> types() {
Collection<DataSourceType> types = new ArrayList<>();
@ -124,7 +128,7 @@ public class DatasourceService {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(datasource);
datasourceProvider.handleDatasource(datasourceRequest, type);
LogUtil.info("Succsss to {} datasource connection pool: {}", type, datasource.getName());
LogUtil.info("Success to {} datasource connection pool: {}", type, datasource.getName());
} catch (Exception e) {
LogUtil.error("Failed to handle datasource connection pool: " + datasource.getName(), e);
}
@ -147,26 +151,29 @@ public class DatasourceService {
datasourceDTO.setCalculationMode(DatasourceCalculationMode.DIRECT);
}
JSONObject jsonObject = JSONObject.parseObject(datasourceDTO.getConfiguration());
if(jsonObject.getString("queryTimeout") == null){
if (jsonObject.getString("queryTimeout") == null) {
jsonObject.put("queryTimeout", 30);
datasourceDTO.setConfiguration(jsonObject.toString());
}
}
if(datasourceDTO.getType().equalsIgnoreCase(DatasourceTypes.mysql.toString())){
if (datasourceDTO.getType().equalsIgnoreCase(DatasourceTypes.mysql.toString())) {
MysqlConfiguration mysqlConfiguration = new Gson().fromJson(datasourceDTO.getConfiguration(), MysqlConfiguration.class);
datasourceDTO.setConfiguration(new Gson().toJson(mysqlConfiguration));
}
if (datasourceDTO.getType().equalsIgnoreCase(DatasourceTypes.api.toString())) {
List<ApiDefinition> apiDefinitionList = new Gson().fromJson(datasourceDTO.getConfiguration(), new TypeToken<ArrayList<ApiDefinition>>() {}.getType());
List<ApiDefinition> apiDefinitionList = new Gson().fromJson(datasourceDTO.getConfiguration(), new TypeToken<ArrayList<ApiDefinition>>() {
}.getType());
List<ApiDefinition> apiDefinitionListWithStatus = new ArrayList<>();
int success = 0;
if (StringUtils.isNotEmpty(datasourceDTO.getStatus())) {
JsonObject apiItemStatuses = JsonParser.parseString(datasourceDTO.getStatus()).getAsJsonObject();
for (ApiDefinition apiDefinition : apiDefinitionList) {
String status = apiItemStatuses.get(apiDefinition.getName()).getAsString();
apiDefinition.setStatus(status);
apiDefinitionListWithStatus.add(apiDefinition);
for (int i = 0; i < apiDefinitionList.size(); i++) {
String status = apiItemStatuses.get(apiDefinitionList.get(i).getName()).getAsString();
apiDefinitionList.get(i).setStatus(status);
apiDefinitionList.get(i).setSerialNumber(i);
apiDefinitionListWithStatus.add(apiDefinitionList.get(i));
if (StringUtils.isNotEmpty(status) && status.equalsIgnoreCase("Success")) {
success++;
}
@ -255,7 +262,8 @@ public class DatasourceService {
String datasourceStatus = datasourceProvider.checkStatus(datasourceRequest);
if (datasource.getType().equalsIgnoreCase("api")) {
int success = 0;
List<ApiDefinition> apiDefinitionList = new Gson().fromJson(datasource.getConfiguration(), new TypeToken<List<ApiDefinition>>() {}.getType());
List<ApiDefinition> apiDefinitionList = new Gson().fromJson(datasource.getConfiguration(), new TypeToken<List<ApiDefinition>>() {
}.getType());
List<ApiDefinition> apiDefinitionListWithStatus = new ArrayList<>();
if (StringUtils.isNotEmpty(datasourceStatus)) {
@ -298,7 +306,8 @@ public class DatasourceService {
datasource.setStatus(datasourceStatus);
if (datasource.getType().equalsIgnoreCase("api")) {
List<ApiDefinition> apiDefinitionList = new Gson().fromJson(datasource.getConfiguration(),new TypeToken<List<ApiDefinition>>(){}.getType());
List<ApiDefinition> apiDefinitionList = new Gson().fromJson(datasource.getConfiguration(), new TypeToken<List<ApiDefinition>>() {
}.getType());
JsonObject apiItemStatuses = JsonParser.parseString(datasourceStatus).getAsJsonObject();
int success = 0;
for (ApiDefinition apiDefinition : apiDefinitionList) {
@ -419,7 +428,8 @@ public class DatasourceService {
}
public ApiDefinition checkApiDatasource(ApiDefinition apiDefinition) throws Exception {
String response = ApiProvider.execHttpRequest(apiDefinition);
BasicInfo basicInfo = systemParameterService.basicInfo();
String response = ApiProvider.execHttpRequest(apiDefinition, StringUtils.isNotBlank(basicInfo.getFrontTimeOut()) ? Integer.parseInt(basicInfo.getFrontTimeOut()) : 10);
return ApiProvider.checkApiDefinition(apiDefinition, response);
}

2
backend/src/main/java/io/dataease/service/wizard/ReptileService.java
View File

@ -25,7 +25,7 @@ public class ReptileService {
List result = new ArrayList();
try {
HttpClientConfig config = new HttpClientConfig();
config.setCocketTimeout(5000);
config.setSocketTimeout(5000);
//爬取最新数据
Document doc = Jsoup.parse(HttpClientUtil.get(blogUrl, config));
Elements elementsContent = doc.getElementsByAttributeValue("rel", "bookmark");

1
frontend/src/lang/en.js
View File

@ -1541,6 +1541,7 @@ export default {
api_table_not_empty: 'API data table cannot be empty',
has_repeat_name: 'Duplicate API data table name',
has_repeat_field_name: 'The field name is duplicate, please modify it before selecting',
api_field_not_empty: 'Field cannot be empty',
valid: 'Valid',
invalid: 'Invalid',
api_step_1: 'Connection API',

1
frontend/src/lang/tw.js
View File

@ -1541,6 +1541,7 @@ export default {
api_table_not_empty: 'API 數據表不能為空',
has_repeat_name: 'API 數據表名稱重複',
has_repeat_field_name: '欄位名重複，請修改後再選擇',
api_field_not_empty: '欄位不能為空',
valid: '有效',
invalid: '無效',
api_step_1: '連接API',

1
frontend/src/lang/zh.js
View File

@ -1549,6 +1549,7 @@ export default {
api_table_not_empty: 'API 数据表不能为空',
has_repeat_name: 'API 数据表名称重复',
has_repeat_field_name: '字段名重复，请修改后再选择',
api_field_not_empty: '字段不能为空',
valid: '有效',
invalid: '无效',
api_step_1: '连接API',

41
frontend/src/views/system/datasource/DsConfiguration.vue
View File

@ -925,23 +925,11 @@ export default {
next() {
if (this.active === 1) {
let hasRepeatName = false;
if (this.add_api_item) {
this.form.apiConfiguration.forEach((item) => {
if (item.name === this.apiItem.name) {
hasRepeatName = true;
}
});
} else {
const index = this.form.apiConfiguration.indexOf(this.apiItem);
for (let i = 0; i < this.form.apiConfiguration.length; i++) {
if (
i !== index &&
this.form.apiConfiguration[i].name === this.apiItem.name
) {
hasRepeatName = true;
}
this.form.apiConfiguration.forEach((item) => {
if (item.name === this.apiItem.name && item.serialNumber !== this.apiItem.serialNumber) {
hasRepeatName = true;
}
}
});
if (hasRepeatName) {
this.$message.error(i18n.t("datasource.has_repeat_name"));
return;
@ -986,17 +974,27 @@ export default {
this.edit_api_item = false;
},
saveItem() {
if(this.apiItem.fields.length === 0){
this.$message.warning(i18n.t('datasource.api_field_not_empty'))
return
}
this.active = 0;
this.edit_api_item = false;
if (this.add_api_item) {
this.form.apiConfiguration.push(this.apiItem);
if (!this.add_api_item) {
for (var i = 0; i < this.form.apiConfiguration.length; i++) {
if (this.form.apiConfiguration[i].serialNumber === this.apiItem.serialNumber) {
this.form.apiConfiguration.splice(this.form.apiConfiguration.indexOf(this.form.apiConfiguration[i]), 1);
}
}
}
this.form.apiConfiguration.push(this.apiItem);
},
addApiItem(item) {
if (item) {
this.add_api_item = false;
this.api_table_title = this.$t("datasource.edit_api_table");
this.apiItem = item;
this.apiItem = JSON.parse(JSON.stringify(item));
} else {
this.add_api_item = true;
this.apiItem = JSON.parse(JSON.stringify(this.defaultApiItem));
@ -1006,10 +1004,7 @@ export default {
this.edit_api_item = true;
},
deleteItem(item) {
this.form.apiConfiguration.splice(
this.form.apiConfiguration.indexOf(item),
1
);
this.form.apiConfiguration.splice(this.form.apiConfiguration.indexOf(item), 1);
},
handleCheckAllChange(row) {
this.handleCheckChange(row);