center/dataease-dm
center/dataease-dm
13
0
Fork 0
forked from github/dataease
Code Pull Requests Activity

fix: api 数据集补全字段

Browse Source
This commit is contained in:
taojinlong 2022-02-25 17:32:17 +08:00
parent bc0962f27b
commit 03a2e92254
4 changed files with 37 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
backend/src/main/java/io/dataease/auth/api/demo.java
View File

@ -3,13 +3,10 @@ package io.dataease.auth.api;
import cn.hutool.json.JSONArray; import cn.hutool.json.JSONArray;
import cn.hutool.json.JSONObject; import cn.hutool.json.JSONObject;
import com.github.xiaoymin.knife4j.annotations.ApiSupport;
import io.dataease.base.domain.DatasetTableFunction;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiOperation;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.*;
import java.util.List;
@RestController @RestController
@ -19,10 +16,10 @@ public class demo {
@GetMapping("demo") @GetMapping("demo")
public Object listByTableId() { public Object listByTableId() {
JSONArray jsonArray = new JSONArray(); JSONArray jsonArray = new JSONArray();
for(int i=0;i<100;i++){ for(int i=0;i<10;i++){
JSONObject jsonObject = new JSONObject(); JSONObject jsonObject = new JSONObject();
for(int j=0;j<10;j++){ for(int j=0;j<1;j++){
jsonObject.set("column" + j, "value"+j); jsonObject.set("column" +i + j, "value"+i+j);
} }
jsonArray.put(jsonObject); jsonArray.put(jsonObject);
} }

21
backend/src/main/java/io/dataease/commons/utils/HttpClientUtil.java
View File

@ -1,8 +1,8 @@
package io.dataease.commons.utils; package io.dataease.commons.utils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpEntity; import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse; import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.NameValuePair; import org.apache.http.NameValuePair;
import org.apache.http.client.entity.EntityBuilder; import org.apache.http.client.entity.EntityBuilder;
import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.entity.UrlEncodedFormEntity;
@ -90,8 +90,7 @@ public class HttpClientUtil {
httpGet.addHeader(HTTP.CONTENT_ENCODING, config.getCharset()); httpGet.addHeader(HTTP.CONTENT_ENCODING, config.getCharset());
HttpResponse response = httpClient.execute(httpGet); HttpResponse response = httpClient.execute(httpGet);
HttpEntity entity = response.getEntity(); return getResponseStr(response, config);
return getResponseStr(response, entity, config);
} catch (Exception e) { } catch (Exception e) {
logger.error("HttpClient查询失败", e); logger.error("HttpClient查询失败", e);
throw new RuntimeException("HttpClient查询失败: " + e.getMessage()); throw new RuntimeException("HttpClient查询失败: " + e.getMessage());
@ -136,8 +135,7 @@ public class HttpClientUtil {
httpPost.setEntity(requestEntity); httpPost.setEntity(requestEntity);
HttpResponse response = httpClient.execute(httpPost); HttpResponse response = httpClient.execute(httpPost);
HttpEntity entity = response.getEntity(); return getResponseStr(response, config);
return getResponseStr(response, entity, config);
} catch (Exception e) { } catch (Exception e) {
logger.error("HttpClient查询失败", e); logger.error("HttpClient查询失败", e);
throw new RuntimeException("HttpClient查询失败: " + e.getMessage()); throw new RuntimeException("HttpClient查询失败: " + e.getMessage());
@ -198,8 +196,7 @@ public class HttpClientUtil {
} }
HttpResponse response = httpClient.execute(httpPost); HttpResponse response = httpClient.execute(httpPost);
HttpEntity entity = response.getEntity(); return getResponseStr(response, config);
return getResponseStr(response, entity, config);
} catch (Exception e) { } catch (Exception e) {
logger.error("HttpClient查询失败", e); logger.error("HttpClient查询失败", e);
throw new RuntimeException("HttpClient查询失败: " + e.getMessage()); throw new RuntimeException("HttpClient查询失败: " + e.getMessage());
@ -212,10 +209,14 @@ public class HttpClientUtil {
} }
} }
private static String getResponseStr(HttpResponse response, HttpEntity entity, HttpClientConfig config) throws Exception{ private static String getResponseStr(HttpResponse response, HttpClientConfig config) throws Exception{
if(response.getStatusLine().getStatusCode() >= 400){ if(response.getStatusLine().getStatusCode() >= 400){
throw new Exception(EntityUtils.toString(entity, config.getCharset())); String msg = EntityUtils.toString(response.getEntity(), config.getCharset());
if(StringUtils.isEmpty(msg)){
msg = "StatusCode: " + response.getStatusLine().getStatusCode();
}
throw new Exception(msg);
} }
return EntityUtils.toString(entity, config.getCharset()); return EntityUtils.toString(response.getEntity(), config.getCharset());
} }
} }

16
backend/src/main/java/io/dataease/provider/datasource/ApiProvider.java
View File

@ -28,7 +28,7 @@ public class ApiProvider extends DatasourceProvider{
public List<String[]> getData(DatasourceRequest datasourceRequest) throws Exception { public List<String[]> getData(DatasourceRequest datasourceRequest) throws Exception {
ApiDefinition apiDefinition = checkApiDefinition(datasourceRequest); ApiDefinition apiDefinition = checkApiDefinition(datasourceRequest);
String response = execHttpRequest(apiDefinition); String response = execHttpRequest(apiDefinition);
return fetchResult(response, apiDefinition.getDataPath()); return fetchResult(response, apiDefinition);
} }
@Override @Override
@ -64,7 +64,7 @@ public class ApiProvider extends DatasourceProvider{
fieldList = getTableFileds(datasourceRequest); fieldList = getTableFileds(datasourceRequest);
result.put("fieldList", fieldList); result.put("fieldList", fieldList);
dataList = fetchResult(response, apiDefinition.getDataPath()); dataList = fetchResult(response, apiDefinition);
result.put("dataList", dataList); result.put("dataList", dataList);
return result; return result;
} }
@ -170,16 +170,14 @@ public class ApiProvider extends DatasourceProvider{
return response; return response;
} }
private List<String[]> fetchResult(String result, String path){ private List<String[]> fetchResult(String result, ApiDefinition apiDefinition){
List<String[]> dataList = new LinkedList<>(); List<String[]> dataList = new LinkedList<>();
List<LinkedHashMap> datas = JsonPath.read(result, path); List<LinkedHashMap> datas = JsonPath.read(result, apiDefinition.getDataPath());
for (LinkedHashMap data : datas) { for (LinkedHashMap data : datas) {
String[] row = new String[data.entrySet().size()]; String[] row = new String[apiDefinition.getFields().size()];
Iterator it = data.entrySet().iterator();
int i = 0; int i = 0;
while (it.hasNext()){ for (DatasetTableField field : apiDefinition.getFields()) {
Map.Entry entry = (Map.Entry)it.next(); row[i] = Optional.ofNullable(data.get(field.getOriginName())).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " ");
row[i] = Optional.ofNullable(entry.getValue()).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " ");
i++; i++;
} }
dataList.add(row); dataList.add(row);

26
backend/src/main/java/io/dataease/service/datasource/DatasourceService.java
View File

@ -362,18 +362,16 @@ public class DatasourceService {
List<JSONObject> dataList = new ArrayList<>(); List<JSONObject> dataList = new ArrayList<>();
List<DatasetTableField> fields = new ArrayList<>(); List<DatasetTableField> fields = new ArrayList<>();
Boolean getFileds = true; Set<String> fieldKeys = new HashSet<>();
//第一遍获取 field
for (LinkedHashMap data : datas) { for (LinkedHashMap data : datas) {
JSONObject jsonObject = new JSONObject(); Set<String> keys = data.keySet();
Iterator it = data.entrySet().iterator(); for (String key : keys) {
while (it.hasNext()){ if(!fieldKeys.contains(key)){
Map.Entry entry = (Map.Entry)it.next(); fieldKeys.add(key);
jsonObject.put((String) entry.getKey(), Optional.ofNullable(entry.getValue()).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " "));
if(getFileds) {
DatasetTableField tableField = new DatasetTableField(); DatasetTableField tableField = new DatasetTableField();
tableField.setOriginName((String) entry.getKey()); tableField.setOriginName(key);
tableField.setName((String) entry.getKey()); tableField.setName(key);
tableField.setSize(65535); tableField.setSize(65535);
tableField.setDeExtractType(0); tableField.setDeExtractType(0);
tableField.setDeType(0); tableField.setDeType(0);
@ -381,7 +379,13 @@ public class DatasourceService {
fields.add(tableField); fields.add(tableField);
} }
} }
getFileds = false; }
//第二遍获取 data
for (LinkedHashMap data : datas) {
JSONObject jsonObject = new JSONObject();
for (String key : fieldKeys) {
jsonObject.put(key, Optional.ofNullable(data.get(key)).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " "));
}
dataList.add(jsonObject); dataList.add(jsonObject);
} }
apiDefinition.setDatas(dataList); apiDefinition.setDatas(dataList);