diff --git a/backend/pom.xml b/backend/pom.xml index 0ffce51cf3..a5f0f7fcd5 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -268,6 +268,12 @@ jsoup 1.14.3 + + + com.jayway.jsonpath + json-path + 2.4.0 + diff --git a/backend/src/main/java/io/dataease/auth/api/demo.java b/backend/src/main/java/io/dataease/auth/api/demo.java new file mode 100644 index 0000000000..1fcab1b099 --- /dev/null +++ b/backend/src/main/java/io/dataease/auth/api/demo.java @@ -0,0 +1,31 @@ +package io.dataease.auth.api; + + +import cn.hutool.json.JSONArray; +import cn.hutool.json.JSONObject; +import com.github.xiaoymin.knife4j.annotations.ApiSupport; +import io.dataease.base.domain.DatasetTableFunction; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.springframework.web.bind.annotation.*; + +import java.util.List; + + +@RestController +@RequestMapping("api/data") +public class demo { + @ApiOperation("查询") + @GetMapping("demo") + public Object listByTableId() { + JSONArray jsonArray = new JSONArray(); + for(int i=0;i<100;i++){ + JSONObject jsonObject = new JSONObject(); + for(int j=0;j<10;j++){ + jsonObject.set("column" + j, "value"+j); + } + jsonArray.put(jsonObject); + } + return jsonArray; + } +} diff --git a/backend/src/main/java/io/dataease/commons/constants/DatasourceTypes.java b/backend/src/main/java/io/dataease/commons/constants/DatasourceTypes.java index df4a77332e..1250a90a08 100644 --- a/backend/src/main/java/io/dataease/commons/constants/DatasourceTypes.java +++ b/backend/src/main/java/io/dataease/commons/constants/DatasourceTypes.java @@ -14,7 +14,8 @@ public enum DatasourceTypes { ck("ch", "ch", "ru.yandex.clickhouse.ClickHouseDriver", "`", "`", "'", "'"), db2("db2", "db2", "com.ibm.db2.jcc.DB2Driver", "\"", "\"", "\"", "\""), es("es", "es", "", "\"", "\"", "\"", "\""), - redshift("redshift", "redshift", "org.postgresql.Driver", "\"", "\"", "\"", "\""); + redshift("redshift", "redshift", "org.postgresql.Driver", "\"", "\"", "\"", "\""), + api("api", "api", "", "\"", "\"", "\"", "\""); private String feature; diff --git a/backend/src/main/java/io/dataease/commons/utils/ExcelXlsReader.java b/backend/src/main/java/io/dataease/commons/utils/ExcelXlsReader.java index d2665d98f3..f32e47bbe8 100644 --- a/backend/src/main/java/io/dataease/commons/utils/ExcelXlsReader.java +++ b/backend/src/main/java/io/dataease/commons/utils/ExcelXlsReader.java @@ -1,6 +1,6 @@ package io.dataease.commons.utils; -import io.dataease.dto.datasource.TableFiled; +import io.dataease.dto.datasource.TableField; import io.dataease.dto.dataset.ExcelSheetData; import io.dataease.i18n.Translator; import org.apache.poi.hssf.eventusermodel.*; @@ -94,7 +94,7 @@ public class ExcelXlsReader implements HSSFListener { @SuppressWarnings("unused") private String sheetName; - public List fields = new ArrayList<>(); + public List fields = new ArrayList<>(); public List> data = new ArrayList<>(); public List totalSheets = new ArrayList<>(); /** @@ -103,11 +103,11 @@ public class ExcelXlsReader implements HSSFListener { private boolean isDateFormat = false; - public List getFields() { + public List getFields() { return fields; } - public void setFields(List fields) { + public void setFields(List fields) { this.fields = fields; } @@ -308,13 +308,13 @@ public class ExcelXlsReader implements HSSFListener { if(curRow == 0){ for (String s : cellList) { - TableFiled tableFiled = new TableFiled(); - tableFiled.setFieldType("TEXT"); - tableFiled.setFieldSize(65533); - tableFiled.setFieldName(s); - tableFiled.setRemarks(s); - this.fields.add(tableFiled); - totalSheets.get(totalSheets.size() -1).getFields().add(tableFiled); + TableField tableField = new TableField(); + tableField.setFieldType("TEXT"); + tableField.setFieldSize(65533); + tableField.setFieldName(s); + tableField.setRemarks(s); + this.fields.add(tableField); + totalSheets.get(totalSheets.size() -1).getFields().add(tableField); } } diff --git a/backend/src/main/java/io/dataease/commons/utils/ExcelXlsxReader.java b/backend/src/main/java/io/dataease/commons/utils/ExcelXlsxReader.java index d8ad9349a2..fc96df9a6b 100644 --- a/backend/src/main/java/io/dataease/commons/utils/ExcelXlsxReader.java +++ b/backend/src/main/java/io/dataease/commons/utils/ExcelXlsxReader.java @@ -1,5 +1,5 @@ package io.dataease.commons.utils; -import io.dataease.dto.datasource.TableFiled; +import io.dataease.dto.datasource.TableField; import io.dataease.dto.dataset.ExcelSheetData; import io.dataease.i18n.Translator; import org.apache.commons.collections4.CollectionUtils; @@ -111,7 +111,7 @@ public class ExcelXlsxReader extends DefaultHandler { */ private StylesTable stylesTable; - public List fields = new ArrayList<>(); + public List fields = new ArrayList<>(); public List> data = new ArrayList<>(); public List totalSheets = new ArrayList<>(); /** @@ -120,11 +120,11 @@ public class ExcelXlsxReader extends DefaultHandler { private boolean isDateFormat = false; - public List getFields() { + public List getFields() { return fields; } - public void setFields(List fields) { + public void setFields(List fields) { this.fields = fields; } @@ -238,12 +238,12 @@ public class ExcelXlsxReader extends DefaultHandler { //将单元格内容加入rowlist中，在这之前先去掉字符串前后的空白符 String value = lastIndex.trim(); if(curRow==1){ - TableFiled tableFiled = new TableFiled(); - tableFiled.setFieldType("TEXT"); - tableFiled.setFieldSize(65533); - tableFiled.setFieldName(value); - tableFiled.setRemarks(value); - this.fields.add(tableFiled); + TableField tableField = new TableField(); + tableField.setFieldType("TEXT"); + tableField.setFieldSize(65533); + tableField.setFieldName(value); + tableField.setRemarks(value); + this.fields.add(tableField); } cellList.add(curCol, value); curCol++; @@ -451,15 +451,15 @@ public class ExcelXlsxReader extends DefaultHandler { } private void addField(String columeName, Integer index){ - TableFiled tableFiled = new TableFiled(); - tableFiled.setFieldType("TEXT"); - tableFiled.setFieldSize(65533); - tableFiled.setFieldName(columeName); - tableFiled.setRemarks(columeName); + TableField tableField = new TableField(); + tableField.setFieldType("TEXT"); + tableField.setFieldSize(65533); + tableField.setFieldName(columeName); + tableField.setRemarks(columeName); if(index != null){ - this.fields.add(index, tableFiled); + this.fields.add(index, tableField); }else { - this.fields.add(tableFiled); + this.fields.add(tableField); } } private String getType(String thisStr){ diff --git a/backend/src/main/java/io/dataease/commons/utils/HttpClientUtil.java b/backend/src/main/java/io/dataease/commons/utils/HttpClientUtil.java index 8454df061b..d00974d658 100755 --- a/backend/src/main/java/io/dataease/commons/utils/HttpClientUtil.java +++ b/backend/src/main/java/io/dataease/commons/utils/HttpClientUtil.java @@ -2,6 +2,7 @@ package io.dataease.commons.utils; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; import org.apache.http.NameValuePair; import org.apache.http.client.entity.EntityBuilder; import org.apache.http.client.entity.UrlEncodedFormEntity; @@ -90,10 +91,10 @@ public class HttpClientUtil { HttpResponse response = httpClient.execute(httpGet); HttpEntity entity = response.getEntity(); - return EntityUtils.toString(entity, config.getCharset()); + return getResponseStr(response, entity, config); } catch (Exception e) { logger.error("HttpClient查询失败", e); - throw new RuntimeException("HttpClient查询失败", e); + throw new RuntimeException("HttpClient查询失败: " + e.getMessage()); } finally { try { httpClient.close(); @@ -136,10 +137,10 @@ public class HttpClientUtil { HttpResponse response = httpClient.execute(httpPost); HttpEntity entity = response.getEntity(); - return EntityUtils.toString(entity, config.getCharset()); + return getResponseStr(response, entity, config); } catch (Exception e) { logger.error("HttpClient查询失败", e); - throw new RuntimeException("HttpClient查询失败", e); + throw new RuntimeException("HttpClient查询失败: " + e.getMessage()); } finally { try { httpClient.close(); @@ -198,10 +199,10 @@ public class HttpClientUtil { HttpResponse response = httpClient.execute(httpPost); HttpEntity entity = response.getEntity(); - return EntityUtils.toString(entity, config.getCharset()); + return getResponseStr(response, entity, config); } catch (Exception e) { logger.error("HttpClient查询失败", e); - throw new RuntimeException("HttpClient查询失败", e); + throw new RuntimeException("HttpClient查询失败: " + e.getMessage()); } finally { try { httpClient.close(); @@ -211,5 +212,10 @@ public class HttpClientUtil { } } - + private static String getResponseStr(HttpResponse response, HttpEntity entity, HttpClientConfig config) throws Exception{ + if(response.getStatusLine().getStatusCode() >= 400){ + throw new Exception(EntityUtils.toString(entity, config.getCharset())); + } + return EntityUtils.toString(entity, config.getCharset()); + } } diff --git a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java index 54841a4805..1fd35a4c09 100644 --- a/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java +++ b/backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java @@ -6,7 +6,7 @@ import io.dataease.base.domain.DatasetTableField; import io.dataease.base.domain.DatasetTableIncrementalConfig; import io.dataease.controller.request.dataset.DataSetTableRequest; import io.dataease.controller.response.DataSetDetail; -import io.dataease.dto.datasource.TableFiled; +import io.dataease.dto.datasource.TableField; import io.dataease.dto.dataset.DataSetTableDTO; import io.dataease.dto.dataset.ExcelFileData; import io.dataease.service.dataset.DataSetTableService; @@ -84,7 +84,7 @@ public class DataSetTableController { @ApiOperation("查询原始字段") @PostMapping("getFields") - public List getFields(@RequestBody DatasetTable datasetTable) throws Exception { + public List getFields(@RequestBody DatasetTable datasetTable) throws Exception { return dataSetTableService.getFields(datasetTable); } diff --git a/backend/src/main/java/io/dataease/controller/datasource/DatasourceController.java b/backend/src/main/java/io/dataease/controller/datasource/DatasourceController.java index df0015db1d..4d3601e7f3 100644 --- a/backend/src/main/java/io/dataease/controller/datasource/DatasourceController.java +++ b/backend/src/main/java/io/dataease/controller/datasource/DatasourceController.java @@ -9,6 +9,7 @@ import io.dataease.commons.utils.PageUtils; import io.dataease.commons.utils.Pager; import io.dataease.controller.ResultHolder; import io.dataease.controller.request.DatasourceUnionRequest; +import io.dataease.controller.request.datasource.ApiDefinition; import io.dataease.controller.sys.base.BaseGridRequest; import io.dataease.dto.datasource.DBTableDTO; import io.dataease.service.datasource.DatasourceService; @@ -95,5 +96,11 @@ public class DatasourceController { return datasourceService.getSchema(datasource); } + @ApiOperation("校验API数据源") + @PostMapping("/checkApiDatasource") + public ApiDefinition checkApiDatasource(@RequestBody ApiDefinition apiDefinition) throws Exception { + return datasourceService.checkApiDatasource(apiDefinition); + } + } diff --git a/backend/src/main/java/io/dataease/controller/request/dataset/DataSetTableRequest.java b/backend/src/main/java/io/dataease/controller/request/dataset/DataSetTableRequest.java index 85fae8e129..451d4f39e3 100644 --- a/backend/src/main/java/io/dataease/controller/request/dataset/DataSetTableRequest.java +++ b/backend/src/main/java/io/dataease/controller/request/dataset/DataSetTableRequest.java @@ -1,7 +1,7 @@ package io.dataease.controller.request.dataset; import io.dataease.base.domain.DatasetTable; -import io.dataease.dto.datasource.TableFiled; +import io.dataease.dto.datasource.TableField; import io.dataease.dto.dataset.ExcelSheetData; import io.swagger.annotations.ApiModelProperty; import lombok.Getter; @@ -33,7 +33,7 @@ public class DataSetTableRequest extends DatasetTable { @ApiModelProperty("类型过滤条件集合") private List typeFilter; @ApiModelProperty("字段集合") - private List fields; + private List fields; @ApiModelProperty("excel sheet集合") private List sheets; @ApiModelProperty("是否合并sheet") diff --git a/backend/src/main/java/io/dataease/controller/request/datasource/ApiDefinition.java b/backend/src/main/java/io/dataease/controller/request/datasource/ApiDefinition.java new file mode 100644 index 0000000000..36e4d8e41a --- /dev/null +++ b/backend/src/main/java/io/dataease/controller/request/datasource/ApiDefinition.java @@ -0,0 +1,20 @@ +package io.dataease.controller.request.datasource; + +import com.alibaba.fastjson.JSONObject; +import io.dataease.base.domain.DatasetTableField; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +public class ApiDefinition { + private String name; + private String desc; + private String url; + private String method = "GET"; + private List fields; + private String request; + private String dataPath; + private List datas = new ArrayList<>(); +} diff --git a/backend/src/main/java/io/dataease/controller/request/datasource/ApiDefinitionRequest.java b/backend/src/main/java/io/dataease/controller/request/datasource/ApiDefinitionRequest.java new file mode 100644 index 0000000000..d975d25a41 --- /dev/null +++ b/backend/src/main/java/io/dataease/controller/request/datasource/ApiDefinitionRequest.java @@ -0,0 +1,21 @@ +package io.dataease.controller.request.datasource; + +import com.alibaba.fastjson.JSONObject; +import lombok.Data; +import java.util.ArrayList; +import java.util.List; + +@Data +public class ApiDefinitionRequest { + private List headers = new ArrayList<>(); + private JSONObject body = new JSONObject(); + private AuthManager authManager = new AuthManager(); + + + @Data + public class AuthManager{ + private String password; + private String username; + private String verification = ""; + } +} diff --git a/backend/src/main/java/io/dataease/dto/DatasourceDTO.java b/backend/src/main/java/io/dataease/dto/DatasourceDTO.java index 63c3e8760b..4ef5406536 100644 --- a/backend/src/main/java/io/dataease/dto/DatasourceDTO.java +++ b/backend/src/main/java/io/dataease/dto/DatasourceDTO.java @@ -1,5 +1,6 @@ package io.dataease.dto; +import com.alibaba.fastjson.JSONArray; import io.dataease.base.domain.Datasource; import io.swagger.annotations.ApiModelProperty; import lombok.Data; @@ -14,5 +15,5 @@ public class DatasourceDTO extends Datasource { @ApiModelProperty("权限") private String privileges; - + private JSONArray apiConfiguration; } diff --git a/backend/src/main/java/io/dataease/dto/dataset/ExcelSheetData.java b/backend/src/main/java/io/dataease/dto/dataset/ExcelSheetData.java index 27356e2d93..afd54ab8a5 100644 --- a/backend/src/main/java/io/dataease/dto/dataset/ExcelSheetData.java +++ b/backend/src/main/java/io/dataease/dto/dataset/ExcelSheetData.java @@ -1,6 +1,6 @@ package io.dataease.dto.dataset; -import io.dataease.dto.datasource.TableFiled; +import io.dataease.dto.datasource.TableField; import io.swagger.annotations.ApiModelProperty; import lombok.Data; @@ -14,7 +14,7 @@ public class ExcelSheetData { @ApiModelProperty("数据集合") private List> data; @ApiModelProperty("字段集合") - private List fields; + private List fields; @ApiModelProperty("是否sheet") private boolean isSheet = true; @ApiModelProperty("json数组") diff --git a/backend/src/main/java/io/dataease/dto/datasource/TableFiled.java b/backend/src/main/java/io/dataease/dto/datasource/TableField.java similarity index 94% rename from backend/src/main/java/io/dataease/dto/datasource/TableFiled.java rename to backend/src/main/java/io/dataease/dto/datasource/TableField.java index 2c2024be1e..d718128e55 100644 --- a/backend/src/main/java/io/dataease/dto/datasource/TableFiled.java +++ b/backend/src/main/java/io/dataease/dto/datasource/TableField.java @@ -6,7 +6,7 @@ import lombok.Setter; @Setter @Getter -public class TableFiled { +public class TableField { @ApiModelProperty("字段名称") private String fieldName; @ApiModelProperty("重新标记") diff --git a/backend/src/main/java/io/dataease/provider/ProviderFactory.java b/backend/src/main/java/io/dataease/provider/ProviderFactory.java index 6d68d973a2..b901fbb02b 100644 --- a/backend/src/main/java/io/dataease/provider/ProviderFactory.java +++ b/backend/src/main/java/io/dataease/provider/ProviderFactory.java @@ -4,6 +4,7 @@ import io.dataease.commons.constants.DatasourceTypes; import io.dataease.provider.datasource.DatasourceProvider; import io.dataease.provider.query.DDLProvider; import io.dataease.provider.query.QueryProvider; +import io.dataease.provider.query.api.ApiProvider; import org.springframework.beans.BeansException; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; @@ -25,6 +26,8 @@ public class ProviderFactory implements ApplicationContextAware { switch (datasourceType) { case es: return context.getBean("es", DatasourceProvider.class); + case api: + return context.getBean("api", DatasourceProvider.class); default: return context.getBean("jdbc", DatasourceProvider.class); } @@ -57,6 +60,8 @@ public class ProviderFactory implements ApplicationContextAware { return context.getBean("hiveQuery", QueryProvider.class); case db2: return context.getBean("db2Query", QueryProvider.class); + case api: + return context.getBean("apiQuery", ApiProvider.class); default: return context.getBean("mysqlQuery", QueryProvider.class); } diff --git a/backend/src/main/java/io/dataease/provider/datasource/ApiProvider.java b/backend/src/main/java/io/dataease/provider/datasource/ApiProvider.java new file mode 100644 index 0000000000..2edeb52c86 --- /dev/null +++ b/backend/src/main/java/io/dataease/provider/datasource/ApiProvider.java @@ -0,0 +1,214 @@ +package io.dataease.provider.datasource; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.google.gson.Gson; +import com.jayway.jsonpath.JsonPath; +import io.dataease.base.domain.DatasetTableField; +import io.dataease.commons.utils.HttpClientConfig; +import io.dataease.commons.utils.HttpClientUtil; +import io.dataease.controller.request.datasource.ApiDefinition; +import io.dataease.controller.request.datasource.ApiDefinitionRequest; +import io.dataease.controller.request.datasource.DatasourceRequest; +import io.dataease.dto.datasource.TableDesc; +import io.dataease.dto.datasource.TableField; +import io.dataease.exception.DataEaseException; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpHeaders; +import org.springframework.stereotype.Service; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.*; +import java.util.stream.Collectors; + +@Service("api") +public class ApiProvider extends DatasourceProvider{ + @Override + public List getData(DatasourceRequest datasourceRequest) throws Exception { + ApiDefinition apiDefinition = checkApiDefinition(datasourceRequest); + String response = execHttpRequest(apiDefinition); + return fetchResult(response, apiDefinition.getDataPath()); + } + + @Override + public List getTables(DatasourceRequest datasourceRequest) throws Exception { + List tableDescs = new ArrayList<>(); + List lists = JSONObject.parseArray(datasourceRequest.getDatasource().getConfiguration(), ApiDefinition.class); + for (ApiDefinition apiDefinition : lists) { + TableDesc tableDesc = new TableDesc(); + tableDesc.setName(apiDefinition.getName()); + tableDesc.setRemark(apiDefinition.getDesc()); + tableDescs.add(tableDesc); + } + return tableDescs; + } + + @Override + public List fetchResult(DatasourceRequest datasourceRequest) throws Exception { + return null; + } + + @Override + public List fetchResultField(DatasourceRequest datasourceRequest) throws Exception { + return null; + } + + @Override + public Map fetchResultAndField(DatasourceRequest datasourceRequest) throws Exception { + Map result = new HashMap<>(); + List dataList = new ArrayList<>(); + List fieldList = new ArrayList<>();; + ApiDefinition apiDefinition = checkApiDefinition(datasourceRequest); + String response = execHttpRequest(apiDefinition); + + fieldList = getTableFileds(datasourceRequest); + result.put("fieldList", fieldList); + dataList = fetchResult(response, apiDefinition.getDataPath()); + result.put("dataList", dataList); + return result; + } + + @Override + public void handleDatasource(DatasourceRequest datasourceRequest, String type) throws Exception { + + } + + @Override + public List getSchema(DatasourceRequest datasourceRequest) throws Exception { + return null; + } + + @Override + public List getTableFileds(DatasourceRequest datasourceRequest) throws Exception { + List lists = JSONObject.parseArray(datasourceRequest.getDatasource().getConfiguration(), ApiDefinition.class); + List tableFields = new ArrayList<>(); + for (ApiDefinition list : lists) { + if(datasourceRequest.getTable().equalsIgnoreCase(list.getName())){ + for (DatasetTableField field : list.getFields()) { + TableField tableField = new TableField(); + tableField.setFieldName(field.getOriginName()); + tableField.setRemarks(field.getName()); + tableField.setFieldSize(field.getSize()); + tableField.setFieldType(field.getDeExtractType().toString()); + tableFields.add(tableField); + } + } + } + return tableFields; + } + + @Override + public String checkStatus(DatasourceRequest datasourceRequest) throws Exception { + List apiDefinitionList = JSONObject.parseArray(datasourceRequest.getDatasource().getConfiguration(), ApiDefinition.class).stream().filter(item -> item.getName().equalsIgnoreCase(datasourceRequest.getTable())).collect(Collectors.toList()); + int success = 0; + for (ApiDefinition apiDefinition : apiDefinitionList) { + datasourceRequest.setTable(apiDefinition.getName()); + try { + getData(datasourceRequest); + success++; + }catch (Exception ignore){} + } + if(success == apiDefinitionList.size()){ + return "Success"; + } + if(success > 0 && success < apiDefinitionList.size() ){ + return "Warning"; + } + return "Error"; + } + + static public String execHttpRequest(ApiDefinition apiDefinition) throws Exception{ + String response = ""; + HttpClientConfig httpClientConfig = new HttpClientConfig(); + ApiDefinitionRequest apiDefinitionRequest = JSONObject.parseObject(apiDefinition.getRequest(), ApiDefinitionRequest.class); + System.out.println(new Gson().toJson(apiDefinitionRequest.getAuthManager())); + //headers + for (JSONObject header : apiDefinitionRequest.getHeaders()) { + if(StringUtils.isNotEmpty(header.getString("name")) && StringUtils.isNotEmpty(header.getString("value"))){ + httpClientConfig.addHeader(header.getString("name"), header.getString("value")); + } + } + + if(apiDefinitionRequest.getAuthManager() != null + && StringUtils.isNotBlank(apiDefinitionRequest.getAuthManager().getUsername()) + && StringUtils.isNotBlank(apiDefinitionRequest.getAuthManager().getPassword()) + && apiDefinitionRequest.getAuthManager().getVerification().equals("Basic Auth")){ + String authValue = "Basic " + Base64.getUrlEncoder().encodeToString((apiDefinitionRequest.getAuthManager().getUsername() + + ":" + apiDefinitionRequest.getAuthManager().getPassword()).getBytes()); + httpClientConfig.addHeader("Authorization", authValue); + } + + switch (apiDefinition.getMethod()){ + case "GET": + response = HttpClientUtil.get(apiDefinition.getUrl(), httpClientConfig); + break; + case "POST": + if (!apiDefinitionRequest.getBody().containsKey("type")) { + throw new Exception("请求类型不能为空"); + } + String type = apiDefinitionRequest.getBody().getString("type"); + if (StringUtils.equalsAny(type, "JSON", "XML", "Raw")) { + String raw = null; + if (apiDefinitionRequest.getBody().containsKey("raw")) { + raw = apiDefinitionRequest.getBody().getString("raw"); + response = HttpClientUtil.post(apiDefinition.getUrl(), raw, httpClientConfig); + } + } + if (StringUtils.equalsAny(type, "Form_Data", "WWW_FORM")) { + if (apiDefinitionRequest.getBody().containsKey("kvs")) { + Map body = new HashMap<>(); + JSONArray kvsArr = apiDefinitionRequest.getBody().getJSONArray("kvs"); + for (int i = 0; i < kvsArr.size(); i++) { + JSONObject kv = kvsArr.getJSONObject(i); + if (kv.containsKey("name")) { + body.put(kv.getString("name"), kv.getString("value")); + } + } + response = HttpClientUtil.post(apiDefinition.getUrl(), body, httpClientConfig); + } + } + break; + default: + break; + } + System.out.println("response: " + response); + return response; + } + + private List fetchResult(String result, String path){ + List dataList = new LinkedList<>(); + List datas = JsonPath.read(result, path); + for (LinkedHashMap data : datas) { + String[] row = new String[data.entrySet().size()]; + Iterator it = data.entrySet().iterator(); + int i = 0; + while (it.hasNext()){ + Map.Entry entry = (Map.Entry)it.next(); + row[i] = entry.getValue().toString(); + i++; + } + dataList.add(row); + } + return dataList; + } + + private ApiDefinition checkApiDefinition(DatasourceRequest datasourceRequest)throws Exception{ + List apiDefinitionList = JSONObject.parseArray(datasourceRequest.getDatasource().getConfiguration(), ApiDefinition.class).stream().filter(item -> item.getName().equalsIgnoreCase(datasourceRequest.getTable())).collect(Collectors.toList()); + if(CollectionUtils.isEmpty(apiDefinitionList)){ + throw new Exception("未找到API数据表"); + } + if(apiDefinitionList.size() > 1 ){ + throw new Exception("存在重名的API数据表"); + } + for (ApiDefinition apiDefinition : apiDefinitionList) { + if (apiDefinition.getName().equalsIgnoreCase(datasourceRequest.getTable())){ + return apiDefinition; + } + } + throw new Exception("未找到API数据表"); + } + +} diff --git a/backend/src/main/java/io/dataease/provider/datasource/DatasourceProvider.java b/backend/src/main/java/io/dataease/provider/datasource/DatasourceProvider.java index d99d7a80b0..67ed045cb2 100644 --- a/backend/src/main/java/io/dataease/provider/datasource/DatasourceProvider.java +++ b/backend/src/main/java/io/dataease/provider/datasource/DatasourceProvider.java @@ -1,7 +1,7 @@ package io.dataease.provider.datasource; import io.dataease.dto.datasource.TableDesc; -import io.dataease.dto.datasource.TableFiled; +import io.dataease.dto.datasource.TableField; import io.dataease.controller.request.datasource.DatasourceRequest; import java.util.List; @@ -15,13 +15,14 @@ public abstract class DatasourceProvider { abstract public List getTables(DatasourceRequest datasourceRequest) throws Exception; - public void checkStatus(DatasourceRequest datasourceRequest) throws Exception { + public String checkStatus(DatasourceRequest datasourceRequest) throws Exception { getData(datasourceRequest); + return "Success"; } abstract public List fetchResult(DatasourceRequest datasourceRequest) throws Exception; - abstract public List fetchResultField(DatasourceRequest datasourceRequest) throws Exception; + abstract public List fetchResultField(DatasourceRequest datasourceRequest) throws Exception; abstract public Map fetchResultAndField(DatasourceRequest datasourceRequest) throws Exception; @@ -29,5 +30,5 @@ public abstract class DatasourceProvider { abstract public List getSchema(DatasourceRequest datasourceRequest) throws Exception; - public abstract List getTableFileds(DatasourceRequest datasourceRequest) throws Exception; + public abstract List getTableFileds(DatasourceRequest datasourceRequest) throws Exception; } diff --git a/backend/src/main/java/io/dataease/provider/datasource/EsProvider.java b/backend/src/main/java/io/dataease/provider/datasource/EsProvider.java index 3b1e9c7c61..411b1b13f8 100644 --- a/backend/src/main/java/io/dataease/provider/datasource/EsProvider.java +++ b/backend/src/main/java/io/dataease/provider/datasource/EsProvider.java @@ -10,7 +10,7 @@ import io.dataease.controller.request.datasource.es.RequstWithCursor; import io.dataease.controller.request.datasource.DatasourceRequest; import io.dataease.dto.datasource.EsConfiguration; import io.dataease.dto.datasource.TableDesc; -import io.dataease.dto.datasource.TableFiled; +import io.dataease.dto.datasource.TableField; import io.dataease.exception.DataEaseException; import io.dataease.i18n.Translator; import io.dataease.provider.ProviderFactory; @@ -98,7 +98,7 @@ public class EsProvider extends DatasourceProvider { } @Override - public List getTableFileds(DatasourceRequest datasourceRequest) throws Exception { + public List getTableFileds(DatasourceRequest datasourceRequest) throws Exception { QueryProvider qp = ProviderFactory.getQueryProvider(datasourceRequest.getDatasource().getType()); datasourceRequest.setQuery(qp.convertTableToSql(datasourceRequest.getTable(), datasourceRequest.getDatasource())); return fetchResultField(datasourceRequest); @@ -119,26 +119,26 @@ public class EsProvider extends DatasourceProvider { } @Override - public List fetchResultField(DatasourceRequest datasourceRequest) throws Exception { - List tableFileds = new ArrayList<>(); + public List fetchResultField(DatasourceRequest datasourceRequest) throws Exception { + List tableFields = new ArrayList<>(); try { String response = exexQuery(datasourceRequest, datasourceRequest.getQuery(), "?format=json"); - tableFileds = fetchResultField4Sql(response); + tableFields = fetchResultField4Sql(response); } catch (Exception e) { DataEaseException.throwException(e); } - return tableFileds; + return tableFields; } - private List fetchResultField(String response) throws Exception { - List fieldList = new ArrayList<>(); + private List fetchResultField(String response) throws Exception { + List fieldList = new ArrayList<>(); EsReponse esReponse = new Gson().fromJson(response, EsReponse.class); if (esReponse.getError() != null) { throw new Exception(esReponse.getError().getReason()); } for (String[] row : esReponse.getRows()) { - TableFiled field = new TableFiled(); + TableField field = new TableField(); field.setFieldName(row[0]); field.setRemarks(row[0]); field.setFieldType(row[2]); @@ -148,15 +148,15 @@ public class EsProvider extends DatasourceProvider { return fieldList; } - private List fetchResultField4Sql(String response) throws Exception { - List fieldList = new ArrayList<>(); + private List fetchResultField4Sql(String response) throws Exception { + List fieldList = new ArrayList<>(); EsReponse esReponse = new Gson().fromJson(response, EsReponse.class); if (esReponse.getError() != null) { throw new Exception(esReponse.getError().getReason()); } for (EsReponse.Column column : esReponse.getColumns()) { - TableFiled field = new TableFiled(); + TableField field = new TableField(); field.setFieldName(column.getName()); field.setRemarks(column.getName()); field.setFieldType(column.getType()); @@ -225,7 +225,7 @@ public class EsProvider extends DatasourceProvider { @Override - public void checkStatus(DatasourceRequest datasourceRequest) throws Exception { + public String checkStatus(DatasourceRequest datasourceRequest) throws Exception { EsConfiguration esConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), EsConfiguration.class); String response = exexGetQuery(datasourceRequest); @@ -246,6 +246,7 @@ public class EsProvider extends DatasourceProvider { } datasourceRequest.getDatasource().setConfiguration(new Gson().toJson(esConfiguration)); getTables(datasourceRequest); + return "Success"; } private String exexQuery(DatasourceRequest datasourceRequest, String sql, String uri) { diff --git a/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java b/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java index 64975c4fd8..af864f0fa4 100644 --- a/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java +++ b/backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java @@ -138,12 +138,12 @@ public class JdbcProvider extends DatasourceProvider { } @Override - public List getTableFileds(DatasourceRequest datasourceRequest) throws Exception { + public List getTableFileds(DatasourceRequest datasourceRequest) throws Exception { if(datasourceRequest.getDatasource().getType().equalsIgnoreCase("mongo")){ datasourceRequest.setQuery("select * from " + datasourceRequest.getTable()); return fetchResultField(datasourceRequest); } - List list = new LinkedList<>(); + List list = new LinkedList<>(); try (Connection connection = getConnectionFromPool(datasourceRequest)) { if (datasourceRequest.getDatasource().getType().equalsIgnoreCase("oracle")) { Method setRemarksReporting = extendedJdbcClassLoader.loadClass("oracle.jdbc.driver.OracleConnection").getMethod("setRemarksReporting",boolean.class); @@ -161,13 +161,13 @@ public class JdbcProvider extends DatasourceProvider { } if (database != null) { if (tableName.equals(datasourceRequest.getTable()) && database.equalsIgnoreCase(getDatabase(datasourceRequest))) { - TableFiled tableFiled = getTableFiled(resultSet, datasourceRequest); - list.add(tableFiled); + TableField tableField = getTableFiled(resultSet, datasourceRequest); + list.add(tableField); } } else { if (tableName.equals(datasourceRequest.getTable())) { - TableFiled tableFiled = getTableFiled(resultSet, datasourceRequest); - list.add(tableFiled); + TableField tableField = getTableFiled(resultSet, datasourceRequest); + list.add(tableField); } } } @@ -186,40 +186,40 @@ public class JdbcProvider extends DatasourceProvider { return list; } - private TableFiled getTableFiled(ResultSet resultSet, DatasourceRequest datasourceRequest) throws SQLException { - TableFiled tableFiled = new TableFiled(); + private TableField getTableFiled(ResultSet resultSet, DatasourceRequest datasourceRequest) throws SQLException { + TableField tableField = new TableField(); String colName = resultSet.getString("COLUMN_NAME"); - tableFiled.setFieldName(colName); + tableField.setFieldName(colName); String remarks = resultSet.getString("REMARKS"); if (remarks == null || remarks.equals("")) { remarks = colName; } - tableFiled.setRemarks(remarks); + tableField.setRemarks(remarks); String dbType = resultSet.getString("TYPE_NAME").toUpperCase(); - tableFiled.setFieldType(dbType); + tableField.setFieldType(dbType); if (dbType.equalsIgnoreCase("LONG")) { - tableFiled.setFieldSize(65533); + tableField.setFieldSize(65533); } - if (StringUtils.isNotEmpty(dbType) && dbType.toLowerCase().contains("date") && tableFiled.getFieldSize() < 50) { - tableFiled.setFieldSize(50); + if (StringUtils.isNotEmpty(dbType) && dbType.toLowerCase().contains("date") && tableField.getFieldSize() < 50) { + tableField.setFieldSize(50); } if (datasourceRequest.getDatasource().getType().equalsIgnoreCase(DatasourceTypes.ck.name())) { QueryProvider qp = ProviderFactory.getQueryProvider(datasourceRequest.getDatasource().getType()); - tableFiled.setFieldSize(qp.transFieldSize(dbType)); + tableField.setFieldSize(qp.transFieldSize(dbType)); } else { - if (datasourceRequest.getDatasource().getType().equalsIgnoreCase(DatasourceTypes.hive.name()) && tableFiled.getFieldType().equalsIgnoreCase("BOOLEAN")) { - tableFiled.setFieldSize(1); + if (datasourceRequest.getDatasource().getType().equalsIgnoreCase(DatasourceTypes.hive.name()) && tableField.getFieldType().equalsIgnoreCase("BOOLEAN")) { + tableField.setFieldSize(1); } else { String size = resultSet.getString("COLUMN_SIZE"); if (size == null) { - tableFiled.setFieldSize(1); + tableField.setFieldSize(1); } else { - tableFiled.setFieldSize(Integer.valueOf(size)); + tableField.setFieldSize(Integer.valueOf(size)); } } } - return tableFiled; + return tableField; } private String getDatabase(DatasourceRequest datasourceRequest) { @@ -244,7 +244,7 @@ public class JdbcProvider extends DatasourceProvider { } @Override - public List fetchResultField(DatasourceRequest datasourceRequest) throws Exception { + public List fetchResultField(DatasourceRequest datasourceRequest) throws Exception { try (Connection connection = getConnectionFromPool(datasourceRequest); Statement stat = connection.createStatement(); ResultSet rs = stat.executeQuery(rebuildSqlWithFragment(datasourceRequest.getQuery()))) { return fetchResultField(rs, datasourceRequest); } catch (SQLException e) { @@ -259,7 +259,7 @@ public class JdbcProvider extends DatasourceProvider { public Map fetchResultAndField(DatasourceRequest datasourceRequest) throws Exception { Map result = new HashMap<>(); List dataList; - List fieldList; + List fieldList; try (Connection connection = getConnectionFromPool(datasourceRequest); Statement stat = connection.createStatement(); ResultSet rs = stat.executeQuery(rebuildSqlWithFragment(datasourceRequest.getQuery()))) { fieldList = fetchResultField(rs, datasourceRequest); result.put("fieldList", fieldList); @@ -274,8 +274,8 @@ public class JdbcProvider extends DatasourceProvider { return new HashMap<>(); } - private List fetchResultField(ResultSet rs, DatasourceRequest datasourceRequest) throws Exception { - List fieldList = new ArrayList<>(); + private List fetchResultField(ResultSet rs, DatasourceRequest datasourceRequest) throws Exception { + List fieldList = new ArrayList<>(); ResultSetMetaData metaData = rs.getMetaData(); int columnCount = metaData.getColumnCount(); for (int j = 0; j < columnCount; j++) { @@ -285,7 +285,7 @@ public class JdbcProvider extends DatasourceProvider { if (datasourceRequest.getDatasource().getType().equalsIgnoreCase(DatasourceTypes.hive.name()) && l.contains("\\.")) { l = l.split("\\.")[1]; } - TableFiled field = new TableFiled(); + TableField field = new TableField(); field.setFieldName(l); field.setRemarks(l); field.setFieldType(t); @@ -362,14 +362,15 @@ public class JdbcProvider extends DatasourceProvider { } @Override - public void checkStatus(DatasourceRequest datasourceRequest) throws Exception { + public String checkStatus(DatasourceRequest datasourceRequest) throws Exception { String queryStr = getTablesSql(datasourceRequest); try (Connection con = getConnection(datasourceRequest); Statement statement = con.createStatement(); ResultSet resultSet = statement.executeQuery(queryStr)) { - + return "Success"; } catch (Exception e) { e.printStackTrace(); DataEaseException.throwException(e.getMessage()); } + return "Error"; } @Override diff --git a/backend/src/main/java/io/dataease/provider/query/QueryProviderImpl.java b/backend/src/main/java/io/dataease/provider/query/QueryProviderImpl.java new file mode 100644 index 0000000000..548713b458 --- /dev/null +++ b/backend/src/main/java/io/dataease/provider/query/QueryProviderImpl.java @@ -0,0 +1,122 @@ +package io.dataease.provider.query; + +import io.dataease.base.domain.ChartViewWithBLOBs; +import io.dataease.base.domain.DatasetTableField; +import io.dataease.base.domain.Datasource; +import io.dataease.controller.request.chart.ChartExtFilterRequest; +import io.dataease.dto.chart.ChartFieldCustomFilterDTO; +import io.dataease.dto.chart.ChartViewFieldDTO; + +import java.util.List; + +public class QueryProviderImpl extends QueryProvider { + @Override + public Integer transFieldType(String field) { + return null; + } + + @Override + public String createSQLPreview(String sql, String orderBy) { + return null; + } + + @Override + public String createQuerySQL(String table, List fields, boolean isGroup, Datasource ds, List fieldCustomFilter) { + return null; + } + + @Override + public String createQuerySQLAsTmp(String sql, List fields, boolean isGroup, List fieldCustomFilter) { + return null; + } + + @Override + public String createQueryTableWithPage(String table, List fields, Integer page, Integer pageSize, Integer realSize, boolean isGroup, Datasource ds, List fieldCustomFilter) { + return null; + } + + @Override + public String createQuerySQLWithPage(String sql, List fields, Integer page, Integer pageSize, Integer realSize, boolean isGroup, List fieldCustomFilter) { + return null; + } + + @Override + public String createQueryTableWithLimit(String table, List fields, Integer limit, boolean isGroup, Datasource ds, List fieldCustomFilter) { + return null; + } + + @Override + public String createQuerySqlWithLimit(String sql, List fields, Integer limit, boolean isGroup, List fieldCustomFilter) { + return null; + } + + @Override + public String getSQL(String table, List xAxis, List yAxis, List fieldCustomFilter, List extFilterRequestList, Datasource ds, ChartViewWithBLOBs view) { + return null; + } + + @Override + public String getSQLAsTmp(String table, List xAxis, List yAxis, List fieldCustomFilter, List extFilterRequestList, ChartViewWithBLOBs view) { + return null; + } + + @Override + public String getSQLTableInfo(String table, List xAxis, List fieldCustomFilter, List extFilterRequestList, Datasource ds, ChartViewWithBLOBs view) { + return null; + } + + @Override + public String getSQLAsTmpTableInfo(String sql, List xAxis, List fieldCustomFilter, List extFilterRequestList, Datasource ds, ChartViewWithBLOBs view) { + return null; + } + + @Override + public String getSQLStack(String table, List xAxis, List yAxis, List fieldCustomFilter, List extFilterRequestList, List extStack, Datasource ds, ChartViewWithBLOBs view) { + return null; + } + + @Override + public String getSQLAsTmpStack(String table, List xAxis, List yAxis, List fieldCustomFilter, List extFilterRequestList, List extStack, ChartViewWithBLOBs view) { + return null; + } + + @Override + public String getSQLScatter(String table, List xAxis, List yAxis, List fieldCustomFilter, List extFilterRequestList, List extBubble, Datasource ds, ChartViewWithBLOBs view) { + return null; + } + + @Override + public String getSQLAsTmpScatter(String table, List xAxis, List yAxis, List fieldCustomFilter, List extFilterRequestList, List extBubble, ChartViewWithBLOBs view) { + return null; + } + + @Override + public String searchTable(String table) { + return null; + } + + @Override + public String getSQLSummary(String table, List yAxis, List fieldCustomFilter, List extFilterRequestList, ChartViewWithBLOBs view) { + return null; + } + + @Override + public String getSQLSummaryAsTmp(String sql, List yAxis, List fieldCustomFilter, List extFilterRequestList, ChartViewWithBLOBs view) { + return null; + } + + @Override + public String wrapSql(String sql) { + return null; + } + + @Override + public String createRawQuerySQL(String table, List fields, Datasource ds) { + return null; + } + + @Override + public String createRawQuerySQLAsTmp(String sql, List fields) { + return null; + } +} diff --git a/backend/src/main/java/io/dataease/provider/query/api/ApiProvider.java b/backend/src/main/java/io/dataease/provider/query/api/ApiProvider.java new file mode 100644 index 0000000000..80bfe8f3c4 --- /dev/null +++ b/backend/src/main/java/io/dataease/provider/query/api/ApiProvider.java @@ -0,0 +1,25 @@ +package io.dataease.provider.query.api; + +import io.dataease.provider.query.QueryProviderImpl; +import org.springframework.stereotype.Service; + +@Service("apiQuery") +public class ApiProvider extends QueryProviderImpl { + @Override + public Integer transFieldType(String field) { + switch (field) { + case "0": + return 0;// 文本 + case "1": + return 1;// 时间 + case "2": + return 2;// 整型 + case "3": + return 3;// 浮点 + case "4": + return 4;// 布尔 + default: + return 0; + } + } +} diff --git a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java index dca5d3ecfa..c250176e74 100644 --- a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java +++ b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java @@ -242,9 +242,9 @@ public class ChartViewService { //列权限 List desensitizationList = new ArrayList<>(); - fields = permissionService.filterColumnPermissons(fields, desensitizationList, datasetTable.getId(), requestList.getUser()); + List columnPermissionFields = permissionService.filterColumnPermissons(fields, desensitizationList, datasetTable.getId(), requestList.getUser()); //将没有权限的列删掉 - List dataeaseNames = fields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList()); + List dataeaseNames = columnPermissionFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList()); dataeaseNames.add("*"); fieldCustomFilter = fieldCustomFilter.stream().filter(item -> !desensitizationList.contains(item.getDataeaseName()) && dataeaseNames.contains(item.getDataeaseName())).collect(Collectors.toList()); extStack = extStack.stream().filter(item -> !desensitizationList.contains(item.getDataeaseName()) && dataeaseNames.contains(item.getDataeaseName())).collect(Collectors.toList()); @@ -253,8 +253,8 @@ public class ChartViewService { //行权限 - List permissionFields = permissionService.getCustomFilters(fields, datasetTable, requestList.getUser()); - fieldCustomFilter.addAll(permissionFields); + List rowPermissionFields = permissionService.getCustomFilters(columnPermissionFields, datasetTable, requestList.getUser()); + fieldCustomFilter.addAll(rowPermissionFields); for (ChartFieldCustomFilterDTO ele : fieldCustomFilter) { ele.setField(dataSetTableFieldsService.get(ele.getId())); @@ -485,7 +485,7 @@ public class ChartViewService { // 仪表板有参数不实用缓存 if (!cache || CollectionUtils.isNotEmpty(requestList.getFilter()) || CollectionUtils.isNotEmpty(requestList.getLinkageFilters()) - || CollectionUtils.isNotEmpty(requestList.getDrill())) { + || CollectionUtils.isNotEmpty(requestList.getDrill()) || CollectionUtils.isNotEmpty(rowPermissionFields) || fields.size() != columnPermissionFields.size()) { data = datasourceProvider.getData(datasourceRequest); } else { try { diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index e3357d878b..63c1a4dbdf 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -19,7 +19,7 @@ import io.dataease.dto.dataset.*; import io.dataease.dto.dataset.union.UnionDTO; import io.dataease.dto.dataset.union.UnionItemDTO; import io.dataease.dto.dataset.union.UnionParamDTO; -import io.dataease.dto.datasource.TableFiled; +import io.dataease.dto.datasource.TableField; import io.dataease.exception.DataEaseException; import io.dataease.i18n.Translator; import io.dataease.plugins.loader.ClassloaderResponsity; @@ -143,7 +143,7 @@ public class DataSetTableService { sheetTable.setName(excelSheetDataList.get(0).getDatasetName()); checkName(sheetTable); excelSheetDataList.forEach(excelSheetData -> { - String[] fieldArray = excelSheetData.getFields().stream().map(TableFiled::getFieldName) + String[] fieldArray = excelSheetData.getFields().stream().map(TableField::getFieldName) .toArray(String[]::new); if (checkIsRepeat(fieldArray)) { DataEaseException.throwException(Translator.get("i18n_excel_field_repeat")); @@ -164,7 +164,7 @@ public class DataSetTableService { }); } else { for (ExcelSheetData sheet : datasetTable.getSheets()) { - String[] fieldArray = sheet.getFields().stream().map(TableFiled::getFieldName) + String[] fieldArray = sheet.getFields().stream().map(TableField::getFieldName) .toArray(String[]::new); if (checkIsRepeat(fieldArray)) { DataEaseException.throwException(Translator.get("i18n_excel_field_repeat")); @@ -197,12 +197,12 @@ public class DataSetTableService { } List excelSheetDataList = new ArrayList<>(); - List oldFields = datasetTable.getSheets().get(0).getFields().stream().map(TableFiled::getRemarks) + List oldFields = datasetTable.getSheets().get(0).getFields().stream().map(TableField::getRemarks) .collect(Collectors.toList()); for (ExcelSheetData sheet : datasetTable.getSheets()) { // 替换时， if (datasetTable.getEditType() == 0) { - List newFields = sheet.getFields().stream().map(TableFiled::getRemarks) + List newFields = sheet.getFields().stream().map(TableField::getRemarks) .collect(Collectors.toList()); if (!oldFields.equals(newFields)) { DataEaseException.throwException(Translator.get("i18n_excel_column_inconsistent")); @@ -210,7 +210,7 @@ public class DataSetTableService { oldFields = newFields; } - String[] fieldArray = sheet.getFields().stream().map(TableFiled::getFieldName).toArray(String[]::new); + String[] fieldArray = sheet.getFields().stream().map(TableField::getFieldName).toArray(String[]::new); if (checkIsRepeat(fieldArray)) { DataEaseException.throwException(Translator.get("i18n_excel_field_repeat")); } @@ -405,7 +405,7 @@ public class DataSetTableService { return extDataSetTableMapper.searchOne(dataSetTableRequest); } - public List getFields(DatasetTable datasetTable) throws Exception { + public List getFields(DatasetTable datasetTable) throws Exception { Datasource ds = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId()); DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); DatasourceRequest datasourceRequest = new DatasourceRequest(); @@ -496,7 +496,7 @@ public class DataSetTableService { if (page == Integer.parseInt(dataSetTableRequest.getRow()) / pageSize + 1) { realSize = Integer.parseInt(dataSetTableRequest.getRow()) % pageSize; } - if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "db")) { + if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "db") || StringUtils.equalsIgnoreCase(datasetTable.getType(), "api")) { if (datasetTable.getMode() == 0) { Datasource ds = datasourceMapper.selectByPrimaryKey(dataSetTableRequest.getDataSourceId()); if (ObjectUtils.isEmpty(ds)) { @@ -847,8 +847,8 @@ public class DataSetTableService { datasourceRequest.setQuery(sqlAsTable); Map result = datasourceProvider.fetchResultAndField(datasourceRequest); List data = result.get("dataList"); - List fields = result.get("fieldList"); - String[] fieldArray = fields.stream().map(TableFiled::getFieldName).toArray(String[]::new); + List fields = result.get("fieldList"); + String[] fieldArray = fields.stream().map(TableField::getFieldName).toArray(String[]::new); if (checkIsRepeat(fieldArray)) { DataEaseException.throwException(Translator.get("i18n_excel_field_repeat")); } @@ -896,8 +896,8 @@ public class DataSetTableService { datasourceRequest.setQuery(qp.createSQLPreview(sql, null)); Map result = datasourceProvider.fetchResultAndField(datasourceRequest); List data = result.get("dataList"); - List fields = result.get("fieldList"); - String[] fieldArray = fields.stream().map(TableFiled::getFieldName).toArray(String[]::new); + List fields = result.get("fieldList"); + String[] fieldArray = fields.stream().map(TableField::getFieldName).toArray(String[]::new); List> jsonArray = new ArrayList<>(); if (CollectionUtils.isNotEmpty(data)) { @@ -912,14 +912,14 @@ public class DataSetTableService { // 获取每个字段在当前de数据库中的name，作为sql查询后的remarks返回前端展示 for (DatasetTableField datasetTableField : fieldList) { - for (TableFiled tableFiled : fields) { - if (StringUtils.equalsIgnoreCase(tableFiled.getFieldName(), + for (TableField tableField : fields) { + if (StringUtils.equalsIgnoreCase(tableField.getFieldName(), DorisTableUtils.dorisFieldName( datasetTableField.getTableId() + "_" + datasetTableField.getDataeaseName())) - || StringUtils.equalsIgnoreCase(tableFiled.getFieldName(), + || StringUtils.equalsIgnoreCase(tableField.getFieldName(), DorisTableUtils.dorisFieldNameShort(datasetTableField.getTableId() + "_" + datasetTableField.getOriginName()))) { - tableFiled.setRemarks(datasetTableField.getName()); + tableField.setRemarks(datasetTableField.getName()); break; } } @@ -957,8 +957,8 @@ public class DataSetTableService { datasourceRequest.setQuery(qp.createSQLPreview(sql, null)); Map result = datasourceProvider.fetchResultAndField(datasourceRequest); List data = result.get("dataList"); - List fields = result.get("fieldList"); - String[] fieldArray = fields.stream().map(TableFiled::getFieldName).toArray(String[]::new); + List fields = result.get("fieldList"); + String[] fieldArray = fields.stream().map(TableField::getFieldName).toArray(String[]::new); List> jsonArray = new ArrayList<>(); if (CollectionUtils.isNotEmpty(data)) { @@ -976,13 +976,13 @@ public class DataSetTableService { dataTableInfoDTO.getList().forEach( ele -> checkedFieldList.addAll(dataSetTableFieldsService.getListByIds(ele.getCheckedFields()))); for (DatasetTableField datasetTableField : checkedFieldList) { - for (TableFiled tableFiled : fields) { - if (StringUtils.equalsIgnoreCase(tableFiled.getFieldName(), + for (TableField tableField : fields) { + if (StringUtils.equalsIgnoreCase(tableField.getFieldName(), DorisTableUtils.dorisFieldName( datasetTableField.getTableId() + "_" + datasetTableField.getDataeaseName())) - || StringUtils.equalsIgnoreCase(tableFiled.getFieldName(), DorisTableUtils.dorisFieldName( + || StringUtils.equalsIgnoreCase(tableField.getFieldName(), DorisTableUtils.dorisFieldName( datasetTableField.getTableId() + "_" + datasetTableField.getOriginName()))) { - tableFiled.setRemarks(datasetTableField.getName()); + tableField.setRemarks(datasetTableField.getName()); break; } } @@ -1417,11 +1417,11 @@ public class DataSetTableService { } } - public List saveExcelTableField(String datasetTableId, List fields, boolean insert) { + public List saveExcelTableField(String datasetTableId, List fields, boolean insert) { List datasetTableFields = new ArrayList<>(); if (CollectionUtils.isNotEmpty(fields)) { for (int i = 0; i < fields.size(); i++) { - TableFiled filed = fields.get(i); + TableField filed = fields.get(i); DatasetTableField datasetTableField = DatasetTableField.builder().build(); datasetTableField.setTableId(datasetTableId); datasetTableField.setOriginName(filed.getFieldName()); @@ -1450,9 +1450,9 @@ public class DataSetTableService { DataSetTableRequest dataSetTableRequest = new DataSetTableRequest(); BeanUtils.copyBean(dataSetTableRequest, datasetTable); - List fields = new ArrayList<>(); + List fields = new ArrayList<>(); long syncTime = System.currentTimeMillis(); - if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "db")) { + if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "db") || StringUtils.equalsIgnoreCase(datasetTable.getType(), "api")) { fields = getFields(datasetTable); } else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "sql")) { DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType()); @@ -1514,11 +1514,11 @@ public class DataSetTableService { fieldList.addAll(listByIds); }); for (DatasetTableField field : fieldList) { - for (TableFiled tableFiled : fields) { + for (TableField tableField : fields) { if (StringUtils.equalsIgnoreCase( DorisTableUtils.dorisFieldName(field.getTableId() + "_" + field.getOriginName()), - tableFiled.getFieldName())) { - tableFiled.setRemarks(field.getName()); + tableField.getFieldName())) { + tableField.setRemarks(field.getName()); break; } } @@ -1544,11 +1544,11 @@ public class DataSetTableService { datasourceRequest.setQuery(sql); fields = datasourceProvider.fetchResultField(datasourceRequest); for (DatasetTableField field : fieldList) { - for (TableFiled tableFiled : fields) { + for (TableField tableField : fields) { if (StringUtils.equalsIgnoreCase( DorisTableUtils.dorisFieldName(field.getTableId() + "_" + field.getDataeaseName()), - tableFiled.getFieldName())) { - tableFiled.setRemarks(field.getName()); + tableField.getFieldName())) { + tableField.setRemarks(field.getName()); break; } } @@ -1568,11 +1568,11 @@ public class DataSetTableService { fields = datasourceProvider.fetchResultField(datasourceRequest); for (DatasetTableField field : fieldList) { - for (TableFiled tableFiled : fields) { + for (TableField tableField : fields) { if (StringUtils.equalsIgnoreCase( DorisTableUtils.dorisFieldNameShort(field.getTableId() + "_" + field.getOriginName()), - tableFiled.getFieldName())) { - tableFiled.setRemarks(field.getName()); + tableField.getFieldName())) { + tableField.setRemarks(field.getName()); break; } } @@ -1586,16 +1586,14 @@ public class DataSetTableService { if (CollectionUtils.isNotEmpty(fields)) { List originNameList = new ArrayList<>(); for (int i = 0; i < fields.size(); i++) { - TableFiled filed = fields.get(i); + TableField filed = fields.get(i); originNameList.add(filed.getFieldName()); DatasetTableField datasetTableField = DatasetTableField.builder().build(); // 物理字段名设定为唯一，查询当前数据集下是否已存在该字段，存在则update，不存在则insert DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); // 字段名一致，认为字段没有改变 - datasetTableFieldExample.createCriteria().andTableIdEqualTo(datasetTable.getId()) - .andOriginNameEqualTo(filed.getFieldName()); - List datasetTableFields = datasetTableFieldMapper - .selectByExample(datasetTableFieldExample); + datasetTableFieldExample.createCriteria().andTableIdEqualTo(datasetTable.getId()).andOriginNameEqualTo(filed.getFieldName()); + List datasetTableFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); if (CollectionUtils.isNotEmpty(datasetTableFields)) { datasetTableField.setId(datasetTableFields.get(0).getId()); datasetTableField.setOriginName(filed.getFieldName()); @@ -1636,8 +1634,7 @@ public class DataSetTableService { } // delete 数据库中多余的字段 DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); - datasetTableFieldExample.createCriteria().andTableIdEqualTo(datasetTable.getId()).andExtFieldEqualTo(0) - .andOriginNameNotIn(originNameList); + datasetTableFieldExample.createCriteria().andTableIdEqualTo(datasetTable.getId()).andExtFieldEqualTo(0).andOriginNameNotIn(originNameList); datasetTableFieldMapper.deleteByExample(datasetTableFieldExample); } } @@ -1736,7 +1733,7 @@ public class DataSetTableService { datasourceRequest.setQuery(qp.wrapSql(sql)); List sqlFileds = new ArrayList<>(); try { - datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName) + datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableField::getFieldName) .forEach(filed -> { sqlFileds.add(filed); }); @@ -1756,7 +1753,7 @@ public class DataSetTableService { datasourceRequest.setQuery(qp.wrapSql(sql)); List sqlFileds = new ArrayList<>(); try { - datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName) + datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableField::getFieldName) .forEach(filed -> sqlFileds.add(filed)); } catch (Exception e) { DataEaseException.throwException(Translator.get("i18n_check_sql_error") + e.getMessage()); @@ -1820,8 +1817,8 @@ public class DataSetTableService { List oldFields = datasetTableFields.stream().map(DatasetTableField::getOriginName) .collect(Collectors.toList()); for (ExcelSheetData excelSheetData : excelSheetDataList) { - List fields = excelSheetData.getFields(); - List newFields = fields.stream().map(TableFiled::getRemarks).collect(Collectors.toList()); + List fields = excelSheetData.getFields(); + List newFields = fields.stream().map(TableField::getRemarks).collect(Collectors.toList()); if (oldFields.equals(newFields)) { retrunSheetDataList.add(excelSheetData); } @@ -1879,7 +1876,7 @@ public class DataSetTableService { inputStream.close(); excelSheetDataList.forEach(excelSheetData -> { List> data = excelSheetData.getData(); - String[] fieldArray = excelSheetData.getFields().stream().map(TableFiled::getFieldName) + String[] fieldArray = excelSheetData.getFields().stream().map(TableField::getFieldName) .toArray(String[]::new); List> jsonArray = new ArrayList<>(); if (CollectionUtils.isNotEmpty(data)) { @@ -1901,7 +1898,7 @@ public class DataSetTableService { private Map parseExcel(String filename, InputStream inputStream, boolean isPreview) throws Exception { String suffix = filename.substring(filename.lastIndexOf(".") + 1); - List fields = new ArrayList<>(); + List fields = new ArrayList<>(); List data = new ArrayList<>(); List> jsonArray = new ArrayList<>(); List sheets = new ArrayList<>(); @@ -1933,16 +1930,16 @@ public class DataSetTableService { String[] r = new String[columnNum]; for (int j = 0; j < columnNum; j++) { if (i == 0) { - TableFiled tableFiled = new TableFiled(); - tableFiled.setFieldType("TEXT"); - tableFiled.setFieldSize(1024); + TableField tableField = new TableField(); + tableField.setFieldType("TEXT"); + tableField.setFieldSize(1024); String columnName = readCell(row.getCell(j), false, null); if (StringUtils.isEmpty(columnName)) { columnName = "NONE_" + String.valueOf(j); } - tableFiled.setFieldName(columnName); - tableFiled.setRemarks(columnName); - fields.add(tableFiled); + tableField.setFieldName(columnName); + tableField.setRemarks(columnName); + fields.add(tableField); } else { if (row == null) { break; @@ -1981,17 +1978,17 @@ public class DataSetTableService { String[] r = new String[columnNum]; for (int j = 0; j < columnNum; j++) { if (i == 0) { - TableFiled tableFiled = new TableFiled(); - tableFiled.setFieldType("TEXT"); - tableFiled.setFieldSize(1024); + TableField tableField = new TableField(); + tableField.setFieldType("TEXT"); + tableField.setFieldSize(1024); String columnName = readCell(row.getCell(j), false, null); if (StringUtils.isEmpty(columnName)) { columnName = "NONE_" + String.valueOf(j); } - tableFiled.setFieldName(columnName); - tableFiled.setRemarks(columnName); - fields.add(tableFiled); + tableField.setFieldName(columnName); + tableField.setRemarks(columnName); + fields.add(tableField); } else { if (row == null) { break; @@ -2008,11 +2005,11 @@ public class DataSetTableService { String s = reader.readLine();// first line String[] split = s.split(","); for (String s1 : split) { - TableFiled tableFiled = new TableFiled(); - tableFiled.setFieldName(s1); - tableFiled.setRemarks(s1); - tableFiled.setFieldType("TEXT"); - fields.add(tableFiled); + TableField tableField = new TableField(); + tableField.setFieldName(s1); + tableField.setRemarks(s1); + tableField.setFieldType("TEXT"); + fields.add(tableField); } int num = 1; String line; @@ -2027,7 +2024,7 @@ public class DataSetTableService { } } - String[] fieldArray = fields.stream().map(TableFiled::getFieldName).toArray(String[]::new); + String[] fieldArray = fields.stream().map(TableField::getFieldName).toArray(String[]::new); // 校验excel字段是否重名 if (checkIsRepeat(fieldArray)) { @@ -2052,7 +2049,7 @@ public class DataSetTableService { return map; } - private String readCell(Cell cell, boolean cellType, TableFiled tableFiled) { + private String readCell(Cell cell, boolean cellType, TableField tableField) { if (cell == null) { return ""; } @@ -2065,15 +2062,15 @@ public class DataSetTableService { double eps = 1e-10; if (value - Math.floor(value) < eps) { if (cellType) { - if (StringUtils.isEmpty(tableFiled.getFieldType()) - || tableFiled.getFieldType().equalsIgnoreCase("TEXT")) { - tableFiled.setFieldType("LONG"); + if (StringUtils.isEmpty(tableField.getFieldType()) + || tableField.getFieldType().equalsIgnoreCase("TEXT")) { + tableField.setFieldType("LONG"); } } return value.longValue() + ""; } else { if (cellType) { - tableFiled.setFieldType("DOUBLE"); + tableField.setFieldType("DOUBLE"); } NumberFormat nf = NumberFormat.getInstance(); nf.setGroupingUsed(false); @@ -2086,23 +2083,23 @@ public class DataSetTableService { } catch (IllegalStateException e) { String s = String.valueOf(cell.getRichStringCellValue()); if (cellType) { - tableFiled.setFieldType("TEXT"); - tableFiled.setFieldSize(65533); + tableField.setFieldType("TEXT"); + tableField.setFieldSize(65533); } return s; } } if (cellTypeEnum.equals(CellType.STRING)) { if (cellType) { - tableFiled.setFieldType("TEXT"); - tableFiled.setFieldSize(65533); + tableField.setFieldType("TEXT"); + tableField.setFieldSize(65533); } return cell.getStringCellValue(); } if (cellTypeEnum.equals(CellType.NUMERIC)) { if (HSSFDateUtil.isCellDateFormatted(cell)) { if (cellType) { - tableFiled.setFieldType("DATETIME"); + tableField.setFieldType("DATETIME"); } SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); try { @@ -2117,15 +2114,15 @@ public class DataSetTableService { double eps = 1e-10; if (value - Math.floor(value) < eps) { if (cellType) { - if (StringUtils.isEmpty(tableFiled.getFieldType()) - || tableFiled.getFieldType().equalsIgnoreCase("TEXT")) { - tableFiled.setFieldType("LONG"); + if (StringUtils.isEmpty(tableField.getFieldType()) + || tableField.getFieldType().equalsIgnoreCase("TEXT")) { + tableField.setFieldType("LONG"); } } return value.longValue() + ""; } else { if (cellType) { - tableFiled.setFieldType("DOUBLE"); + tableField.setFieldType("DOUBLE"); } NumberFormat nf = NumberFormat.getInstance(); nf.setGroupingUsed(false); diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index c1cbd64ef5..92f91e172c 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -1,5 +1,6 @@ package io.dataease.service.dataset; +import com.alibaba.fastjson.JSONObject; import com.google.gson.Gson; import io.dataease.base.domain.*; import io.dataease.base.mapper.DatasetTableMapper; @@ -10,6 +11,7 @@ import io.dataease.commons.constants.*; import io.dataease.commons.model.AuthURD; import io.dataease.commons.utils.*; import io.dataease.commons.constants.DatasourceTypes; +import io.dataease.controller.request.datasource.ApiDefinition; import io.dataease.provider.datasource.DatasourceProvider; import io.dataease.provider.datasource.JdbcProvider; import io.dataease.provider.ProviderFactory; @@ -63,7 +65,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import javax.annotation.Resource; -import java.io.File; +import java.io.*; import java.net.InetAddress; import java.util.*; import java.util.stream.Collectors; @@ -314,16 +316,13 @@ public class ExtractDataService { try { createDorisTable(DorisTableUtils.dorisName(datasetTableId), dorisTableColumnSql); createDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)), dorisTableColumnSql); - generateTransFile("all_scope", datasetTable, datasource, datasetTableFields, null); - generateJobFile("all_scope", datasetTable, datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(","))); execTime = System.currentTimeMillis(); - extractData(datasetTable, "all_scope"); + extractData(datasetTable, datasource, datasetTableFields, "all_scope", null); replaceTable(DorisTableUtils.dorisName(datasetTableId)); saveSuccessLog(datasetTableTaskLog); msg = true; lastExecStatus = JobStatus.Completed; } catch (Exception e) { - e.printStackTrace(); saveErrorLog(datasetTableId, taskId, e); msg = false; lastExecStatus = JobStatus.Error; @@ -341,6 +340,10 @@ public class ExtractDataService { case add_scope: // 增量更新 try { + if(datasource.getType().equalsIgnoreCase(DatasourceTypes.api.name())){ + extractData(datasetTable, datasource, datasetTableFields, "incremental_add", null); + return; + } DatasetTableIncrementalConfig datasetTableIncrementalConfig = dataSetTableService.incrementalConfig(datasetTableId); if (datasetTableIncrementalConfig == null || StringUtils.isEmpty(datasetTableIncrementalConfig.getTableId())) { updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, null); @@ -357,17 +360,13 @@ public class ExtractDataService { if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) {// 增量添加 String sql = datasetTableIncrementalConfig.getIncrementalAdd().replace(lastUpdateTime, datasetTable.getLastUpdateTime().toString()) .replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()); - generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, sql); - generateJobFile("incremental_add", datasetTable, fetchSqlField(sql, datasource)); - extractData(datasetTable, "incremental_add"); + extractData(datasetTable, datasource, datasetTableFields, "incremental_add", sql); } if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete().replace(" ", ""))) {// 增量删除 String sql = datasetTableIncrementalConfig.getIncrementalDelete().replace(lastUpdateTime, datasetTable.getLastUpdateTime().toString()) .replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString()); - generateTransFile("incremental_delete", datasetTable, datasource, datasetTableFields, sql); - generateJobFile("incremental_delete", datasetTable, fetchSqlField(sql, datasource)); - extractData(datasetTable, "incremental_delete"); + extractData(datasetTable, datasource, datasetTableFields, "incremental_delete", sql); } saveSuccessLog(datasetTableTaskLog); @@ -394,6 +393,97 @@ public class ExtractDataService { } + private void extractData(DatasetTable datasetTable, Datasource datasource, List datasetTableFields, String extractType, String selectSQL) throws Exception{ + if(datasource.getType().equalsIgnoreCase(DatasourceTypes.api.name())){ + extractDataByDE(datasetTable, datasource, datasetTableFields, extractType); + return; + } + extractDataByKettle(datasetTable, datasource, datasetTableFields, extractType, selectSQL); + } + + private void extractDataByDE(DatasetTable datasetTable, Datasource datasource, List datasetTableFields, String extractType)throws Exception{ + List lists = JSONObject.parseArray(datasource.getConfiguration(), ApiDefinition.class); + lists = lists.stream().filter(item -> item.getName().equalsIgnoreCase(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable())).collect(Collectors.toList()); + if(CollectionUtils.isEmpty(lists)){ + throw new Exception("未找到API数据表"); + } + if(lists.size() > 1 ){ + throw new Exception("存在重名的API数据表"); + } + DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType()); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(datasource); + datasourceRequest.setTable(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable()); + Map result = datasourceProvider.fetchResultAndField(datasourceRequest); + List dataList = result.get("dataList"); + Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); + DorisConfiguration dorisConfiguration = new Gson().fromJson(dorisDatasource.getConfiguration(), DorisConfiguration.class); + String columns = datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(",")) + ",dataease_uuid"; + + String dataFile = null; + String script = null; + switch (extractType) { + case "all_scope": + dataFile = root_path + DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId())) + "." + extention; + script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId()))); + break; + default: + dataFile = root_path + DorisTableUtils.dorisAddName(DorisTableUtils.dorisName(datasetTable.getId())) + "." + extention; + script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), DorisTableUtils.dorisName(datasetTable.getId())); + break; + } + + + BufferedWriter bw = new BufferedWriter(new FileWriter(dataFile)); + for (String[] strings : dataList) { + String content = ""; + for (int i=0;i< strings.length;i++){ + content = content + strings[i] + separator; + } + content = content + Md5Utils.md5(content); + bw.write(content); + bw.newLine(); + } + bw.close(); + + File scriptFile = new File(root_path + datasetTable.getId() + ".sh"); + scriptFile.setExecutable(true); + scriptFile.createNewFile(); + + BufferedWriter scriptFileBw = new BufferedWriter(new FileWriter(root_path + datasetTable.getId() + ".sh")); + scriptFileBw.write("#!/bin/sh"); + scriptFileBw.newLine(); + scriptFileBw.write(script); + scriptFileBw.newLine(); + scriptFileBw.close(); + + try { + Process process = Runtime.getRuntime().exec(root_path + datasetTable.getId() + ".sh"); + process.waitFor(); + if(process.waitFor() != 0){ + BufferedReader input = new BufferedReader(new InputStreamReader(process.getErrorStream())); + String errMsg = ""; + String line = ""; + while ((line = input.readLine()) != null) { + errMsg = errMsg + line + System.getProperty("line.separator"); + } + throw new Exception(errMsg); + } + }catch (Exception e){ + throw e; + }finally { + File deleteFile = new File(root_path + datasetTable.getId() + ".sh"); + FileUtils.forceDelete(deleteFile); + } + + } + + private void extractDataByKettle(DatasetTable datasetTable, Datasource datasource, List datasetTableFields, String extractType, String selectSQL)throws Exception{ + generateTransFile(extractType, datasetTable, datasource, datasetTableFields, selectSQL); + generateJobFile(extractType, datasetTable, datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(","))); + extractData(datasetTable, extractType); + } + private void sendWebMsg(DatasetTable datasetTable, DatasetTableTask datasetTableTask, DatasetTableTaskLog datasetTableTaskLog, Boolean status) { String taskId = datasetTableTask.getId(); String msg = status ? "成功" : "失败"; @@ -729,7 +819,7 @@ public class ExtractDataService { datasourceRequest.setDatasource(ds); datasourceRequest.setQuery(qp.wrapSql(sql)); List dorisFields = new ArrayList<>(); - datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName).forEach(filed -> { + datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableField::getFieldName).forEach(filed -> { dorisFields.add(DorisTableUtils.columnName(filed)); }); return String.join(",", dorisFields); diff --git a/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java b/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java index c709074ebf..ef659854cf 100644 --- a/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java +++ b/backend/src/main/java/io/dataease/service/datasource/DatasourceService.java @@ -1,7 +1,9 @@ package io.dataease.service.datasource; +import cn.hutool.json.JSONArray; import com.alibaba.fastjson.JSONObject; import com.google.gson.Gson; +import com.jayway.jsonpath.JsonPath; import io.dataease.base.domain.*; import io.dataease.base.mapper.*; import io.dataease.base.mapper.ext.ExtDataSourceMapper; @@ -10,12 +12,16 @@ import io.dataease.commons.exception.DEException; import io.dataease.commons.model.AuthURD; import io.dataease.commons.utils.AuthUtils; import io.dataease.commons.utils.CommonThreadPool; +import io.dataease.commons.utils.HttpClientUtil; import io.dataease.commons.utils.LogUtil; import io.dataease.controller.ResultHolder; import io.dataease.controller.request.DatasourceUnionRequest; +import io.dataease.controller.request.datasource.ApiDefinition; import io.dataease.controller.sys.base.BaseGridRequest; import io.dataease.controller.sys.base.ConditionEntity; import io.dataease.commons.constants.DatasourceTypes; +import io.dataease.exception.ExcelException; +import io.dataease.provider.datasource.ApiProvider; import io.dataease.provider.datasource.DatasourceProvider; import io.dataease.provider.ProviderFactory; import io.dataease.controller.request.datasource.DatasourceRequest; @@ -103,6 +109,9 @@ public class DatasourceService { case ck: datasourceDTO.setConfiguration(JSONObject.toJSONString(new Gson().fromJson(datasourceDTO.getConfiguration(), CHConfiguration.class)) ); break; + case api: + datasourceDTO.setApiConfiguration(JSONObject.parseArray(datasourceDTO.getConfiguration())); + break; default: break; } @@ -260,13 +269,45 @@ public class DatasourceService { datasources.forEach(datasource -> checkAndUpdateDatasourceStatus(datasource, true)); } + public ApiDefinition checkApiDatasource(ApiDefinition apiDefinition) throws Exception { + String response = ApiProvider.execHttpRequest(apiDefinition); + + List datas = JsonPath.read(response,apiDefinition.getDataPath()); + List dataList = new ArrayList<>(); + for (LinkedHashMap data : datas) { + JSONObject jsonObject = new JSONObject(); + Iterator it = data.entrySet().iterator(); + while (it.hasNext()){ + Map.Entry entry = (Map.Entry)it.next(); + jsonObject.put((String) entry.getKey(), entry.getValue()); + } + dataList.add(jsonObject); + } + List fields = new ArrayList<>(); + if(CollectionUtils.isNotEmpty(dataList)){ + for (Map.Entry stringObjectEntry : dataList.get(0).entrySet()) { + DatasetTableField tableField = new DatasetTableField(); + tableField.setOriginName(stringObjectEntry.getKey()); + tableField.setName(stringObjectEntry.getKey()); + tableField.setSize(65535); + tableField.setDeExtractType(0); + tableField.setDeType(0); + tableField.setExtField(0); + fields.add(tableField); + } + } + apiDefinition.setDatas(dataList); + apiDefinition.setFields(fields); + return apiDefinition; + } + private void checkAndUpdateDatasourceStatus(Datasource datasource){ try { DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType()); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(datasource); - datasourceProvider.checkStatus(datasourceRequest); - datasource.setStatus("Success"); + String status = datasourceProvider.checkStatus(datasourceRequest); + datasource.setStatus(status); } catch (Exception e) { datasource.setStatus("Error"); } @@ -277,8 +318,8 @@ public class DatasourceService { DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType()); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(datasource); - datasourceProvider.checkStatus(datasourceRequest); - datasource.setStatus("Success"); + String status = datasourceProvider.checkStatus(datasourceRequest); + datasource.setStatus(status); datasourceMapper.updateByPrimaryKeySelective(datasource); } catch (Exception e) { Datasource temp = datasourceMapper.selectByPrimaryKey(datasource.getId()); diff --git a/frontend/package.json b/frontend/package.json index 5d9f2444c6..38ab90cbea 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -33,6 +33,13 @@ "jsencrypt": "^3.0.0-rc.1", "jspdf": "^2.3.1", "lodash": "^4.17.4", + "lodash.isboolean": "^3.0.3", + "lodash.isempty": "^4.4.0", + "lodash.isinteger": "^4.0.4", + "lodash.isnull": "^3.0.0", + "lodash.isnumber": "^3.0.3", + "lodash.isobject": "^3.0.2", + "lodash.isstring": "^4.0.1", "normalize.css": "7.0.0", "nprogress": "0.2.0", "screenfull": "4.2.0", @@ -53,6 +60,7 @@ "vue-to-pdf": "^1.0.0", "vue-uuid": "2.0.2", "vue-video-player": "^5.0.2", + "vue2-ace-editor": "0.0.15", "vuedraggable": "^2.24.3", "vuex": "3.1.0", "webpack": "^4.46.0", diff --git a/frontend/src/api/dataset/dataset.js b/frontend/src/api/dataset/dataset.js index 6167a87d2d..4eca52a04d 100644 --- a/frontend/src/api/dataset/dataset.js +++ b/frontend/src/api/dataset/dataset.js @@ -77,6 +77,14 @@ export function listDatasource() { }) } +export function listApiDatasource() { + return request({ + url: '/datasource/list/api', + loading: true, + method: 'get' + }) +} + export function getTable(id, hideMsg = false) { return request({ url: '/dataset/table/get/' + id, diff --git a/frontend/src/api/system/datasource.js b/frontend/src/api/system/datasource.js index 19452d0d67..b1c7535d0f 100644 --- a/frontend/src/api/system/datasource.js +++ b/frontend/src/api/system/datasource.js @@ -74,4 +74,13 @@ export function getSchema(data) { }) } +export function checkApiDatasource(data){ + return request({ + url: 'datasource/checkApiDatasource', + method: 'post', + loading: false, + data + }) +} + export default { dsGrid, addDs, editDs, delDs, validateDs, listDatasource, getSchema } diff --git a/frontend/src/icons/svg/exclamationmark2.svg b/frontend/src/icons/svg/exclamationmark2.svg new file mode 100644 index 0000000000..6e45d421cd --- /dev/null +++ b/frontend/src/icons/svg/exclamationmark2.svg @@ -0,0 +1 @@ + diff --git a/frontend/src/lang/en.js b/frontend/src/lang/en.js index 3135ae9403..708081a2f1 100644 --- a/frontend/src/lang/en.js +++ b/frontend/src/lang/en.js @@ -1049,6 +1049,7 @@ export default { custom_data: 'Custom Dataset', pls_slc_tbl_left: 'Please select the chart from the left', add_db_table: 'Add Database Dataset', + add_api_table: 'Add API Dataset', pls_slc_data_source: 'Please select data source', table: 'Table', edit: 'Edit', @@ -1298,7 +1299,35 @@ export default { no_less_then_0: 'Parameters in advanced settings cannot be less than zero', port_no_less_then_0: 'Port cannot be less than zero', priority: 'Advanced setting', - extra_params: 'Extra JDBC connection string' + extra_params: 'Extra JDBC connection string', + please_input_dataPath: '请输入 JsonPath 数据路径', + warning: 'Contains invalid datasets', + data_table: 'Dataset Table', + data_table_name: 'Dataset Table name', + method: 'Request mode', + url: 'URL', + add_api_table: 'Add API table', + edit_api_table: 'Edit API table', + base_info: 'Basic information', + request: 'Request', + path_all_info: 'Please fill in the full address', + req_param: 'Request parameters', + headers: 'Request header', + key: 'Key', + value: 'Value', + data_path: 'Extract data', + data_path_desc: 'Please fill in the data path with Jsonpath', + body_form_data: 'form-data', + body_x_www_from_urlencoded: 'x-www-form-urlencoded', + body_json: 'json', + body_xml: 'xml', + body_raw: 'row', + request_body: 'Request Body', + auth_config: 'Auth config', + auth_config_info: 'Permission verification is required for the request', + verified: 'Verified', + verification_method: 'Verification Method', + username: 'Username' }, pblink: { key_pwd: 'Please enter the password to open the link', diff --git a/frontend/src/lang/tw.js b/frontend/src/lang/tw.js index 39d208e5fc..fb9592c5ef 100644 --- a/frontend/src/lang/tw.js +++ b/frontend/src/lang/tw.js @@ -1049,6 +1049,7 @@ export default { custom_data: '自定義數據集', pls_slc_tbl_left: '請從左側選擇錶', add_db_table: '添加數據庫數據集', + add_api_table: '添加API數據集', pls_slc_data_source: '請選擇數據源', table: '錶', edit: '編輯', @@ -1299,7 +1300,35 @@ export default { no_less_then_0: '高級設置中的參數不能小於零', port_no_less_then_0: '端口不能小於零', priority: '高級設置', - extra_params: '額外的JDBC連接字符串' + extra_params: '額外的JDBC連接字符串', + please_input_dataPath: '請輸入 JsonPath 數據路徑', + warning: '包含無效數據機', + data_table: '數據表', + data_table_name: '數據表名稱', + method: '請求方式', + url: 'URL', + add_api_table: '添加 API 數據表', + edit_api_table: '編輯 API 數據表', + base_info: '基礎信息', + request: '請求', + path_all_info: '請輸入完整地址', + req_param: '請求參數', + headers: '請求頭', + key: '鍵', + value: '值', + data_path: '提取數據', + data_path_desc: '請用 JsonPath 填寫數據路徑', + body_form_data: 'form-data', + body_x_www_from_urlencoded: 'x-www-form-urlencoded', + body_json: 'json', + body_xml: 'xml', + body_raw: 'row', + request_body: '請求提', + auth_config: '認證配置', + auth_config_info: '請求需要進行權限校驗', + verified: '認證', + verification_method: '認證方式', + username: '用戶名' }, pblink: { key_pwd: '請輸入密碼打開鏈接', diff --git a/frontend/src/lang/zh.js b/frontend/src/lang/zh.js index 32bfd2af3e..6db0149988 100644 --- a/frontend/src/lang/zh.js +++ b/frontend/src/lang/zh.js @@ -678,6 +678,7 @@ export default { custom_data: '自定义数据集', pls_slc_tbl_left: '请从左侧选视图', add_db_table: '添加数据库数据集', + add_api_table: '添加API数据集', pls_slc_data_source: '请选择数据源', table: '表', edit: '编辑', @@ -1307,7 +1308,35 @@ export default { direct: '直连模式', extract: '抽取模式', all_compute_mode: '直连、抽取模式', - extra_params: '额外的JDBC连接字符串' + extra_params: '额外的JDBC连接字符串', + please_input_dataPath: '请输入 JsonPath 数据路径', + warning: '包含无效数据集', + data_table: '数据表', + data_table_name: '数据表名称', + method: '请求方式', + url: 'URL', + add_api_table: '添加API数据表', + edit_api_table: '编辑API数据表', + base_info: '基础信息', + request: '请求', + path_all_info: '请填入完整地址', + req_param: '请求参数', + headers: '请求头', + key: '键', + value: '值', + data_path: '提取数据', + data_path_desc: '请用JsonPath填写数据路径', + body_form_data: 'form-data', + body_x_www_from_urlencoded: 'x-www-form-urlencoded', + body_json: 'json', + body_xml: 'xml', + body_raw: 'row', + request_body: '请求体', + auth_config: '认证配置', + auth_config_info: '请求需要进行权限校验', + verified: '认证', + verification_method: '认证方式', + username: '用户名' }, pblink: { key_pwd: '请输入密码打开链接', diff --git a/frontend/src/views/dataset/add/AddApi.vue b/frontend/src/views/dataset/add/AddApi.vue new file mode 100644 index 0000000000..044cb6971e --- /dev/null +++ b/frontend/src/views/dataset/add/AddApi.vue @@ -0,0 +1,209 @@ + + + + + diff --git a/frontend/src/views/dataset/data/UpdateInfo.vue b/frontend/src/views/dataset/data/UpdateInfo.vue index 0d1bc81ec0..59c50484c9 100644 --- a/frontend/src/views/dataset/data/UpdateInfo.vue +++ b/frontend/src/views/dataset/data/UpdateInfo.vue @@ -263,7 +263,7 @@ - + {{ $t('dataset.incremental_update_type') }}: @@ -275,7 +275,7 @@ - + {{ $t('dataset.param') }}: @@ -285,7 +285,7 @@ - + - + diff --git a/frontend/src/views/dataset/group/Group.vue b/frontend/src/views/dataset/group/Group.vue index 456f8a01fd..c548c4d58e 100644 --- a/frontend/src/views/dataset/group/Group.vue +++ b/frontend/src/views/dataset/group/Group.vue @@ -99,6 +99,10 @@ {{ $t('dataset.union_data') }} + + + {{ $t('dataset.union_data') }} + @@ -562,6 +566,9 @@ export default { case 'union': this.addData('AddUnion') break + case 'api': + this.addData('AddApi') + break } }, diff --git a/frontend/src/views/dataset/index.vue b/frontend/src/views/dataset/index.vue index 7cb9e9d9d3..328d47a505 100644 --- a/frontend/src/views/dataset/index.vue +++ b/frontend/src/views/dataset/index.vue @@ -20,6 +20,7 @@ import Group from './group/Group' import DataHome from './data/DataHome' import ViewTable from './data/ViewTable' import AddDB from './add/AddDB' +import AddApi from './add/AddApi' import AddSQL from './add/AddSQL' import AddExcel from './add/AddExcel' import AddCustom from './add/AddCustom' @@ -29,7 +30,7 @@ import { removeClass } from '@/utils' import { checkCustomDs } from '@/api/dataset/dataset' export default { name: 'DataSet', - components: { DeMainContainer, DeContainer, DeAsideContainer, Group, DataHome, ViewTable, AddDB, AddSQL, AddExcel, AddCustom }, + components: { DeMainContainer, DeContainer, DeAsideContainer, Group, DataHome, ViewTable, AddDB, AddSQL, AddExcel, AddCustom, AddApi}, data() { return { component: DataHome, @@ -76,6 +77,9 @@ export default { case 'FieldEdit': this.component = FieldEdit break + case 'AddApi': + this.component = AddApi + break default: this.component = DataHome break diff --git a/frontend/src/views/system/datasource/ApiAuthConfig.vue b/frontend/src/views/system/datasource/ApiAuthConfig.vue new file mode 100644 index 0000000000..2448500372 --- /dev/null +++ b/frontend/src/views/system/datasource/ApiAuthConfig.vue @@ -0,0 +1,91 @@ + + + + + diff --git a/frontend/src/views/system/datasource/ApiBody.vue b/frontend/src/views/system/datasource/ApiBody.vue new file mode 100644 index 0000000000..424f2ead17 --- /dev/null +++ b/frontend/src/views/system/datasource/ApiBody.vue @@ -0,0 +1,331 @@ + + + + + diff --git a/frontend/src/views/system/datasource/ApiHttpRequestForm.vue b/frontend/src/views/system/datasource/ApiHttpRequestForm.vue new file mode 100644 index 0000000000..1ae75e35a7 --- /dev/null +++ b/frontend/src/views/system/datasource/ApiHttpRequestForm.vue @@ -0,0 +1,394 @@ + + + + + diff --git a/frontend/src/views/system/datasource/ApiKeyValue.vue b/frontend/src/views/system/datasource/ApiKeyValue.vue new file mode 100644 index 0000000000..5bfc74c9e9 --- /dev/null +++ b/frontend/src/views/system/datasource/ApiKeyValue.vue @@ -0,0 +1,218 @@ + + + + + diff --git a/frontend/src/views/system/datasource/ApiTestModel.js b/frontend/src/views/system/datasource/ApiTestModel.js new file mode 100644 index 0000000000..690f4263aa --- /dev/null +++ b/frontend/src/views/system/datasource/ApiTestModel.js @@ -0,0 +1,165 @@ +export class BaseConfig { + + set(options, notUndefined) { + options = this.initOptions(options) + for (let name in options) { + if (options.hasOwnProperty(name)) { + if (!(this[name] instanceof Array)) { + if (notUndefined === true) { + this[name] = options[name] === undefined ? this[name] : options[name]; + } else { + this[name] = options[name]; + } + } + } + } + } + + sets(types, options) { + options = this.initOptions(options) + if (types) { + for (let name in types) { + if (types.hasOwnProperty(name) && options.hasOwnProperty(name)) { + options[name].forEach(o => { + this[name].push(new types[name](o)); + }) + } + } + } + } + + initOptions(options) { + return options || {}; + } + + isValid() { + return true; + } +} + +export class KeyValue extends BaseConfig { + constructor(options) { + options = options || {}; + options.enable = options.enable === undefined ? true : options.enable; + + super(); + this.name = undefined; + this.value = undefined; + this.type = undefined; + this.files = undefined; + this.enable = undefined; + this.uuid = undefined; + this.time = undefined; + this.contentType = undefined; + this.set(options); + } + + isValid() { + return (!!this.name || !!this.value) && this.type !== 'file'; + } + + isFile() { + return (!!this.name || !!this.value) && this.type === 'file'; + } +} + +export class Body extends BaseConfig { + constructor(options) { + super(); + this.type = "KeyValue"; + this.raw = undefined; + this.kvs = []; + this.binary = []; + this.set(options); + this.sets({kvs: KeyValue}, {binary: KeyValue}, options); + } + + isValid() { + if (this.isKV()) { + return this.kvs.some(kv => { + return kv.isValid(); + }) + } else { + return !!this.raw; + } + } + + isKV() { + return [BODY_TYPE.FORM_DATA, BODY_TYPE.WWW_FORM, BODY_TYPE.BINARY].indexOf(this.type) > 0; + } +} + +export const createComponent = function (name) { + let component = MODELS[name]; + if (component) { + return new component(); + } else { + return new UnsupportedComponent() + } +} + +export const BODY_TYPE = { + KV: "KeyValue", + FORM_DATA: "Form_Data", + RAW: "Raw", + WWW_FORM: "WWW_FORM", + XML: "XML", + JSON: "JSON" +} + +export class Scenario extends BaseConfig { + constructor(options = {}) { + super(); + this.id = undefined; + this.name = undefined; + this.url = undefined; + this.variables = []; + this.headers = []; + this.requests = []; + this.environmentId = undefined; + this.dubboConfig = undefined; + this.environment = undefined; + this.enableCookieShare = false; + this.enable = true; + this.databaseConfigs = []; + this.tcpConfig = undefined; + this.set(options); + this.sets({ + variables: KeyValue, + headers: KeyValue, + requests: RequestFactory, + databaseConfigs: DatabaseConfig + }, options); + } + + initOptions(options = {}) { + options.id = options.id || uuid(); + options.requests = options.requests || [new RequestFactory()]; + options.databaseConfigs = options.databaseConfigs || []; + options.dubboConfig = new DubboConfig(options.dubboConfig); + options.tcpConfig = new TCPConfig(options.tcpConfig); + return options; + } + + clone() { + let clone = new Scenario(this); + clone.id = uuid(); + return clone; + } + + isValid() { + if (this.enable) { + for (let i = 0; i < this.requests.length; i++) { + let validator = this.requests[i].isValid(this.environmentId, this.environment); + if (!validator.isValid) { + return validator; + } + } + } + return {isValid: true}; + } + + isReference() { + return this.id.indexOf("#") !== -1 + } +} diff --git a/frontend/src/views/system/datasource/ApiVariable.vue b/frontend/src/views/system/datasource/ApiVariable.vue new file mode 100644 index 0000000000..5205f32d39 --- /dev/null +++ b/frontend/src/views/system/datasource/ApiVariable.vue @@ -0,0 +1,314 @@ + + + + + diff --git a/frontend/src/views/system/datasource/CodeEdit.vue b/frontend/src/views/system/datasource/CodeEdit.vue new file mode 100644 index 0000000000..da46d707ed --- /dev/null +++ b/frontend/src/views/system/datasource/CodeEdit.vue @@ -0,0 +1,108 @@ + + + + + diff --git a/frontend/src/views/system/datasource/DialogFooter.vue b/frontend/src/views/system/datasource/DialogFooter.vue new file mode 100644 index 0000000000..5195b4f6be --- /dev/null +++ b/frontend/src/views/system/datasource/DialogFooter.vue @@ -0,0 +1,43 @@ + + + + + diff --git a/frontend/src/views/system/datasource/DsMain.vue b/frontend/src/views/system/datasource/DsMain.vue index e4464ab297..19eb5076d3 100644 --- a/frontend/src/views/system/datasource/DsMain.vue +++ b/frontend/src/views/system/datasource/DsMain.vue @@ -37,7 +37,6 @@ export default { methods: { // 切换main区内容 switchMain(param) { - console.log(param) const {component, componentParam, tData} = param this.component = DataHome this.param = null diff --git a/frontend/src/views/system/datasource/DsTree.vue b/frontend/src/views/system/datasource/DsTree.vue index 11151b86ff..eb3af3ec23 100644 --- a/frontend/src/views/system/datasource/DsTree.vue +++ b/frontend/src/views/system/datasource/DsTree.vue @@ -39,23 +39,35 @@ > - + + + + - + {{ data.name }} - + + + {{ data.name }} + + + + {{ data.name }} @@ -140,14 +152,14 @@ export default { let typeData = [] listDatasourceByType(datasource.type).then(res => { typeData = this.buildTree(res.data) - if(typeData.length === 0){ + if (typeData.length === 0) { let index = this.tData.findIndex(item => { - if ( item.id === datasource.type) { + if (item.id === datasource.type) { return true; } }) - this.tData.splice(index,1) - }else { + this.tData.splice(index, 1) + } else { let find = false; for (let index = 0; index < this.tData.length; index++) { if (typeData[0].id === this.tData[index].id) { @@ -155,7 +167,7 @@ export default { find = true } } - if(!find){ + if (!find) { this.tData.push(typeData[0]) } } @@ -212,6 +224,8 @@ export default { return 'Apache Hive' } else if (type === 'db2') { return 'Db2' + } else if (type === 'api') { + return 'API' } }, @@ -268,7 +282,6 @@ export default { }) }, switchMain(component, componentParam, tData) { - console.log(tData) this.$emit('switch-main', { component, componentParam, diff --git a/frontend/src/views/system/datasource/EditDialog.vue b/frontend/src/views/system/datasource/EditDialog.vue new file mode 100644 index 0000000000..4e6f8bb3de --- /dev/null +++ b/frontend/src/views/system/datasource/EditDialog.vue @@ -0,0 +1,86 @@ + + + + + diff --git a/frontend/src/views/system/datasource/ImportJson.vue b/frontend/src/views/system/datasource/ImportJson.vue new file mode 100644 index 0000000000..4950467595 --- /dev/null +++ b/frontend/src/views/system/datasource/ImportJson.vue @@ -0,0 +1,118 @@ + + + + + diff --git a/frontend/src/views/system/datasource/convert.js b/frontend/src/views/system/datasource/convert.js new file mode 100644 index 0000000000..e339e31dce --- /dev/null +++ b/frontend/src/views/system/datasource/convert.js @@ -0,0 +1,206 @@ +const isBoolean = require("lodash.isboolean"); +const isEmpty = require("lodash.isempty"); +const isInteger = require("lodash.isinteger"); +const isNull = require("lodash.isnull"); +const isNumber = require("lodash.isnumber"); +const isObject = require("lodash.isobject"); +const isString = require("lodash.isstring"); +const isArray = Array.isArray; + + +class Convert { + constructor() { + this._option = { + $id: "http://example.com/root.json", + $schema: "http://json-schema.org/draft-07/schema#", + } + this._object = null; + } + + /** + * 转换函数 + * @param {*} object 需要转换的对象 + * @param {*} ?option 可选参数，目前只有能设置 root 节点的 $id 和 $schema + */ + format(object, option = {}) { + // 数据校验，确保传入的的object只能是对象或数组 + if (!isObject(object)) { + throw new TypeError("传入参数只能是对象或数组"); + } + // 合并属性 + this._option = Object.assign(this._option, option); + // 需要转换的对象 + this._object = object; + let convertRes; + // 数组类型和对象类型结构不一样 + if (isArray(object)) { + convertRes = this._arrayToSchema(); + } else { + convertRes = this._objectToSchema(); + } + // 释放 + this._object = null; + return convertRes; + } + + /** + * 数组类型转换成JSONSCHEMA + */ + _arrayToSchema() { + // root节点基本信息 + let result = this._value2object(this._object, this._option.$id, "", true); + if (this._object.length > 0) { + let itemArr = []; + for (let index = 0; index < this._object.length; index++) { + // 创建items对象的基本信息 + let objectItem = this._object[index] + let item = this._value2object(objectItem, `#/items`, 'items'); + if (isObject(objectItem) && !isEmpty(objectItem)) { + // 递归遍历 + let objectItemSchema = this._json2schema(objectItem, `#/items`); + // 合并对象 + item = Object.assign(item, objectItemSchema); + } + itemArr.push(item); + } + result["items"] = itemArr; + } + return result + } + + /** + * 对象类型转换成JSONSCHEMA + */ + _objectToSchema() { + let baseResult = this._value2object(this._object, this._option.$id, "", true) + let objectSchema = this._json2schema(this._object) + baseResult = Object.assign(baseResult, objectSchema) + return baseResult + } + + /** + * 递归函数，转换object对象为json schmea 格式 + * @param {*} object 需要转换对象 + * @param {*} name $id值 + */ + _json2schema(object, name = "") { + // 如果递归值不是对象，那么return掉 + if (!isObject(object)) { + return; + } + // 处理当前路径$id + if (name === "" || name == undefined) { + name = "#" + } + let result = {}; + // 判断传入object是对象还是数组。 + if (isArray(object)) { + result.items = {}; + } else { + result.properties = {}; + } + // 遍历传入的对象 + for (const key in object) { + if (object.hasOwnProperty(key)) { + const element = object[key]; + // 如果只是undefined。跳过 + if (element === undefined) { + continue; + } + let $id = `${name}/properties/${key}` + // 判断当前 element 的值 是否也是对象，如果是就继续递归，不是就赋值给result + if(!result["properties"]){ + continue; + } + if (isObject(element)) { + // 创建当前属性的基本信息 + result["properties"][key] = this._value2object(element, $id, key) + if (isArray(element)) { + // 针对空数组和有值的数组做不同处理 + if (element.length > 0) { + // 是数组 + let itemArr = []; + for (let index = 0; index < element.length; index++) { + let elementItem = element[index]; + // 创建items对象的基本信息 + let item = this._value2object(elementItem, `${$id}/items`, key + 'items'); + // 判断第一项是否是对象,且对象属性不为空 + if (isObject(elementItem) && !isEmpty(elementItem)) { + // 新增的properties才合并进来 + item = Object.assign(item, this._json2schema(elementItem, `${$id}/items`)); + } + itemArr.push(item); + } + result["properties"][key]["items"] = itemArr; + } + } else { + // 不是数组，递归遍历获取，然后合并对象属性 + result["properties"][key] = Object.assign(result["properties"][key], this._json2schema(element, $id)); + } + } else { + // 一般属性直接获取基本信息 + if (result["properties"]) { + result["properties"][key] = this._value2object(element, $id, key); + } + } + } + } + return result; + } + + /** + * 把json的值转换成对象类型 + * @param {*} value + * @param {*} $id + * @param {*} key + */ + _value2object(value, $id, key = '', root = false) { + let objectTemplate = { + $id: $id, + title: `The ${key} Schema`, + mock: { + "mock": value + }, + } + + // 判断是否为初始化root数据 + if (root) { + objectTemplate["$schema"] = this._option.$schema; + objectTemplate["title"] = `The Root Schema`; + objectTemplate["mock"] = undefined; + } + if (isBoolean(value)) { + objectTemplate.type = "boolean"; + } else if (isInteger(value)) { + objectTemplate.type = "integer"; + } else if (isNumber(value)) { + objectTemplate.type = "number"; + } else if (isString(value)) { + objectTemplate.type = "string"; + } else if (isNull(value)) { + objectTemplate.type = "null"; + } else if (isArray(value)) { + objectTemplate.type = "array"; + objectTemplate["mock"] = undefined; + } else if (isObject(value)) { + objectTemplate.type = "object" + objectTemplate["mock"] = undefined; + } + + return objectTemplate; + } + + /** + * 后台转换 + * @param callback + */ + // schemaToJsonStr(schema, callback) { + // post('/api/definition/preview', schema, (response) => { + // if (callback) { + // callback(response.data); + // } + // }); + // } +} + +module.exports = Convert; diff --git a/frontend/src/views/system/datasource/form.vue b/frontend/src/views/system/datasource/form.vue index 5a4813f571..5286fa3d77 100644 --- a/frontend/src/views/system/datasource/form.vue +++ b/frontend/src/views/system/datasource/form.vue @@ -30,6 +30,7 @@ class="select-width" :disabled="formType=='modify' || (formType==='add' && params && !!params.type)" @change="changeType()" + filterable > - + + + + + + + + + + + + + + + + + + + + + + +

{{ $t('datasource.base_info') }}

+ + + + + + + + + + + + + +
+

{{ $t('datasource.req_param') }}

+ + + + +
+ + + + + {{ $t('commons.validate') }} + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ {{ $t('fu.steps.next') }} + {{ $t('fu.steps.prev') }} + {{ $t('commons.save') }} +
+ + + + + - - + + + - + + - + {{ $t('datasource.oracle_sid') }} {{ $t('datasource.oracle_service_name') }} - + - + + - + + - + + - + - + - - - {{ $t('datasource.get_schema') }} - + + + {{ $t('datasource.get_schema') }} - - + :label="$t('datasource.schema')"> + + - + + - + @@ -145,34 +252,23 @@ -
- {{ $t('commons.validate') }} + {{ $t('commons.validate') }} - {{ $t('commons.save') }} + {{ $t('commons.save') }}
- {{ $t('commons.validate') }} + {{ $t('commons.validate') }} - {{ $t('commons.edit') }} + {{ $t('commons.edit') }}
@@ -181,13 +277,17 @@