center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

fix: api 数据集补全字段

Browse Source
This commit is contained in:
taojinlong 2022-02-25 17:32:17 +08:00
parent bc0962f27b
commit 03a2e92254
4 changed files with 37 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
backend/src/main/java/io/dataease/auth/api/demo.java
View File

@ -3,13 +3,10 @@ package io.dataease.auth.api;
import cn.hutool.json.JSONArray;
import cn.hutool.json.JSONObject;
import com.github.xiaoymin.knife4j.annotations.ApiSupport;
import io.dataease.base.domain.DatasetTableFunction;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.springframework.web.bind.annotation.*;
import java.util.List;
@RestController
@ -19,10 +16,10 @@ public class demo {
@GetMapping("demo")
public Object listByTableId() {
JSONArray jsonArray = new JSONArray();
for(int i=0;i<100;i++){
for(int i=0;i<10;i++){
JSONObject jsonObject = new JSONObject();
for(int j=0;j<10;j++){
jsonObject.set("column" + j, "value"+j);
for(int j=0;j<1;j++){
jsonObject.set("column" +i + j, "value"+i+j);
}
jsonArray.put(jsonObject);
}

21
backend/src/main/java/io/dataease/commons/utils/HttpClientUtil.java
View File

@ -1,8 +1,8 @@
package io.dataease.commons.utils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.EntityBuilder;
import org.apache.http.client.entity.UrlEncodedFormEntity;
@ -90,8 +90,7 @@ public class HttpClientUtil {
httpGet.addHeader(HTTP.CONTENT_ENCODING, config.getCharset());
HttpResponse response = httpClient.execute(httpGet);
HttpEntity entity = response.getEntity();
return getResponseStr(response, entity, config);
return getResponseStr(response, config);
} catch (Exception e) {
logger.error("HttpClient查询失败", e);
throw new RuntimeException("HttpClient查询失败: " + e.getMessage());
@ -136,8 +135,7 @@ public class HttpClientUtil {
httpPost.setEntity(requestEntity);
HttpResponse response = httpClient.execute(httpPost);
HttpEntity entity = response.getEntity();
return getResponseStr(response, entity, config);
return getResponseStr(response, config);
} catch (Exception e) {
logger.error("HttpClient查询失败", e);
throw new RuntimeException("HttpClient查询失败: " + e.getMessage());
@ -198,8 +196,7 @@ public class HttpClientUtil {
}
HttpResponse response = httpClient.execute(httpPost);
HttpEntity entity = response.getEntity();
return getResponseStr(response, entity, config);
return getResponseStr(response, config);
} catch (Exception e) {
logger.error("HttpClient查询失败", e);
throw new RuntimeException("HttpClient查询失败: " + e.getMessage());
@ -212,10 +209,14 @@ public class HttpClientUtil {
}
}
private static String getResponseStr(HttpResponse response, HttpEntity entity, HttpClientConfig config) throws Exception{
private static String getResponseStr(HttpResponse response, HttpClientConfig config) throws Exception{
if(response.getStatusLine().getStatusCode() >= 400){
throw new Exception(EntityUtils.toString(entity, config.getCharset()));
String msg = EntityUtils.toString(response.getEntity(), config.getCharset());
if(StringUtils.isEmpty(msg)){
msg = "StatusCode: " + response.getStatusLine().getStatusCode();
}
throw new Exception(msg);
}
return EntityUtils.toString(entity, config.getCharset());
return EntityUtils.toString(response.getEntity(), config.getCharset());
}
}

16
backend/src/main/java/io/dataease/provider/datasource/ApiProvider.java
View File

@ -28,7 +28,7 @@ public class ApiProvider extends DatasourceProvider{
public List<String[]> getData(DatasourceRequest datasourceRequest) throws Exception {
ApiDefinition apiDefinition = checkApiDefinition(datasourceRequest);
String response = execHttpRequest(apiDefinition);
return fetchResult(response, apiDefinition.getDataPath());
return fetchResult(response, apiDefinition);
}
@Override
@ -64,7 +64,7 @@ public class ApiProvider extends DatasourceProvider{
fieldList = getTableFileds(datasourceRequest);
result.put("fieldList", fieldList);
dataList = fetchResult(response, apiDefinition.getDataPath());
dataList = fetchResult(response, apiDefinition);
result.put("dataList", dataList);
return result;
}
@ -170,16 +170,14 @@ public class ApiProvider extends DatasourceProvider{
return response;
}
private List<String[]> fetchResult(String result, String path){
private List<String[]> fetchResult(String result, ApiDefinition apiDefinition){
List<String[]> dataList = new LinkedList<>();
List<LinkedHashMap> datas = JsonPath.read(result, path);
List<LinkedHashMap> datas = JsonPath.read(result, apiDefinition.getDataPath());
for (LinkedHashMap data : datas) {
String[] row = new String[data.entrySet().size()];
Iterator it = data.entrySet().iterator();
String[] row = new String[apiDefinition.getFields().size()];
int i = 0;
while (it.hasNext()){
Map.Entry entry = (Map.Entry)it.next();
row[i] = Optional.ofNullable(entry.getValue()).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " ");
for (DatasetTableField field : apiDefinition.getFields()) {
row[i] = Optional.ofNullable(data.get(field.getOriginName())).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " ");
i++;
}
dataList.add(row);

26
backend/src/main/java/io/dataease/service/datasource/DatasourceService.java
View File

@ -362,18 +362,16 @@ public class DatasourceService {
List<JSONObject> dataList = new ArrayList<>();
List<DatasetTableField> fields = new ArrayList<>();
Boolean getFileds = true;
Set<String> fieldKeys = new HashSet<>();
//第一遍获取 field
for (LinkedHashMap data : datas) {
JSONObject jsonObject = new JSONObject();
Iterator it = data.entrySet().iterator();
while (it.hasNext()){
Map.Entry entry = (Map.Entry)it.next();
jsonObject.put((String) entry.getKey(), Optional.ofNullable(entry.getValue()).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " "));
if(getFileds) {
Set<String> keys = data.keySet();
for (String key : keys) {
if(!fieldKeys.contains(key)){
fieldKeys.add(key);
DatasetTableField tableField = new DatasetTableField();
tableField.setOriginName((String) entry.getKey());
tableField.setName((String) entry.getKey());
tableField.setOriginName(key);
tableField.setName(key);
tableField.setSize(65535);
tableField.setDeExtractType(0);
tableField.setDeType(0);
@ -381,7 +379,13 @@ public class DatasourceService {
fields.add(tableField);
}
}
getFileds = false;
}
//第二遍获取 data
for (LinkedHashMap data : datas) {
JSONObject jsonObject = new JSONObject();
for (String key : fieldKeys) {
jsonObject.put(key, Optional.ofNullable(data.get(key)).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " "));
}
dataList.add(jsonObject);
}
apiDefinition.setDatas(dataList);