feat: 支持API數據機

backend/pom.xml

@@ -268,6 +268,12 @@
             <artifactId>jsoup</artifactId>
             <version>1.14.3</version>
         </dependency>
+
+        <dependency>
+            <groupId>com.jayway.jsonpath</groupId>
+            <artifactId>json-path</artifactId>
+            <version>2.4.0</version>
+        </dependency>
     </dependencies>
 
     <build>

backend/src/main/java/io/dataease/auth/api/demo.java

@@ -0,0 +1,31 @@
+package io.dataease.auth.api;
+
+
+import cn.hutool.json.JSONArray;
+import cn.hutool.json.JSONObject;
+import com.github.xiaoymin.knife4j.annotations.ApiSupport;
+import io.dataease.base.domain.DatasetTableFunction;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import org.springframework.web.bind.annotation.*;
+
+import java.util.List;
+
+
+@RestController
+@RequestMapping("api/data")
+public class demo {
+    @ApiOperation("查询")
+    @GetMapping("demo")
+    public Object listByTableId() {
+        JSONArray jsonArray = new JSONArray();
+        for(int i=0;i<100;i++){
+            JSONObject jsonObject = new JSONObject();
+            for(int j=0;j<10;j++){
+                jsonObject.set("column" + j, "value"+j);
+            }
+            jsonArray.put(jsonObject);
+        }
+        return jsonArray;
+    }
+}

backend/src/main/java/io/dataease/commons/constants/DatasourceTypes.java

@@ -14,7 +14,8 @@ public enum DatasourceTypes {
     ck("ch", "ch", "ru.yandex.clickhouse.ClickHouseDriver", "`", "`", "'", "'"),
     db2("db2", "db2", "com.ibm.db2.jcc.DB2Driver", "\"", "\"", "\"", "\""),
     es("es", "es", "", "\"", "\"", "\"", "\""),
-    redshift("redshift", "redshift", "org.postgresql.Driver", "\"", "\"", "\"", "\"");
+    redshift("redshift", "redshift", "org.postgresql.Driver", "\"", "\"", "\"", "\""),
+    api("api", "api", "", "\"", "\"", "\"", "\"");
 
 
     private String feature;

backend/src/main/java/io/dataease/commons/utils/ExcelXlsReader.java

@@ -1,6 +1,6 @@
 package io.dataease.commons.utils;
 
-import io.dataease.dto.datasource.TableFiled;
+import io.dataease.dto.datasource.TableField;
 import io.dataease.dto.dataset.ExcelSheetData;
 import io.dataease.i18n.Translator;
 import org.apache.poi.hssf.eventusermodel.*;
@@ -94,7 +94,7 @@ public class ExcelXlsReader implements HSSFListener {
     @SuppressWarnings("unused")
     private String sheetName;
 
-    public List<TableFiled> fields = new ArrayList<>();
+    public List<TableField> fields = new ArrayList<>();
     public List<List<String>> data = new ArrayList<>();
     public List<ExcelSheetData> totalSheets = new ArrayList<>();
     /**
@@ -103,11 +103,11 @@ public class ExcelXlsReader implements HSSFListener {
     private boolean isDateFormat = false;
 
 
-    public List<TableFiled> getFields() {
+    public List<TableField> getFields() {
         return fields;
     }
 
-    public void setFields(List<TableFiled> fields) {
+    public void setFields(List<TableField> fields) {
         this.fields = fields;
     }
 
@@ -308,13 +308,13 @@ public class ExcelXlsReader implements HSSFListener {
 
             if(curRow == 0){
                 for (String s : cellList) {
-                    TableFiled tableFiled = new TableFiled();
+                    TableField tableField = new TableField();
-                    tableFiled.setFieldType("TEXT");
+                    tableField.setFieldType("TEXT");
-                    tableFiled.setFieldSize(65533);
+                    tableField.setFieldSize(65533);
-                    tableFiled.setFieldName(s);
+                    tableField.setFieldName(s);
-                    tableFiled.setRemarks(s);
+                    tableField.setRemarks(s);
-                    this.fields.add(tableFiled);
+                    this.fields.add(tableField);
-                    totalSheets.get(totalSheets.size() -1).getFields().add(tableFiled);
+                    totalSheets.get(totalSheets.size() -1).getFields().add(tableField);
                 }
             }
 

backend/src/main/java/io/dataease/commons/utils/ExcelXlsxReader.java

@@ -1,5 +1,5 @@
 package io.dataease.commons.utils;
-import io.dataease.dto.datasource.TableFiled;
+import io.dataease.dto.datasource.TableField;
 import io.dataease.dto.dataset.ExcelSheetData;
 import io.dataease.i18n.Translator;
 import org.apache.commons.collections4.CollectionUtils;
@@ -111,7 +111,7 @@ public class ExcelXlsxReader extends DefaultHandler {
      */
     private StylesTable stylesTable;
 
-    public List<TableFiled> fields = new ArrayList<>();
+    public List<TableField> fields = new ArrayList<>();
     public List<List<String>> data = new ArrayList<>();
     public List<ExcelSheetData> totalSheets = new ArrayList<>();
     /**
@@ -120,11 +120,11 @@ public class ExcelXlsxReader extends DefaultHandler {
     private boolean isDateFormat = false;
 
 
-    public List<TableFiled> getFields() {
+    public List<TableField> getFields() {
         return fields;
     }
 
-    public void setFields(List<TableFiled> fields) {
+    public void setFields(List<TableField> fields) {
         this.fields = fields;
     }
 
@@ -238,12 +238,12 @@ public class ExcelXlsxReader extends DefaultHandler {
             //将单元格内容加入rowlist中，在这之前先去掉字符串前后的空白符
             String value = lastIndex.trim();
             if(curRow==1){
-                TableFiled tableFiled = new TableFiled();
+                TableField tableField = new TableField();
-                tableFiled.setFieldType("TEXT");
+                tableField.setFieldType("TEXT");
-                tableFiled.setFieldSize(65533);
+                tableField.setFieldSize(65533);
-                tableFiled.setFieldName(value);
+                tableField.setFieldName(value);
-                tableFiled.setRemarks(value);
+                tableField.setRemarks(value);
-                this.fields.add(tableFiled);
+                this.fields.add(tableField);
             }
             cellList.add(curCol, value);
             curCol++;
@@ -451,15 +451,15 @@ public class ExcelXlsxReader extends DefaultHandler {
     }
 
     private void addField(String columeName, Integer index){
-        TableFiled tableFiled = new TableFiled();
+        TableField tableField = new TableField();
-        tableFiled.setFieldType("TEXT");
+        tableField.setFieldType("TEXT");
-        tableFiled.setFieldSize(65533);
+        tableField.setFieldSize(65533);
-        tableFiled.setFieldName(columeName);
+        tableField.setFieldName(columeName);
-        tableFiled.setRemarks(columeName);
+        tableField.setRemarks(columeName);
         if(index != null){
-            this.fields.add(index, tableFiled);
+            this.fields.add(index, tableField);
         }else {
-            this.fields.add(tableFiled);
+            this.fields.add(tableField);
         }
     }
     private String getType(String thisStr){

backend/src/main/java/io/dataease/commons/utils/HttpClientUtil.java

@@ -2,6 +2,7 @@ package io.dataease.commons.utils;
 
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
 import org.apache.http.NameValuePair;
 import org.apache.http.client.entity.EntityBuilder;
 import org.apache.http.client.entity.UrlEncodedFormEntity;
@@ -90,10 +91,10 @@ public class HttpClientUtil {
 
             HttpResponse response = httpClient.execute(httpGet);
             HttpEntity entity = response.getEntity();
-            return EntityUtils.toString(entity, config.getCharset());
+            return getResponseStr(response, entity, config);
         } catch (Exception e) {
             logger.error("HttpClient查询失败", e);
-            throw new RuntimeException("HttpClient查询失败", e);
+            throw new RuntimeException("HttpClient查询失败: " + e.getMessage());
         } finally {
             try {
                 httpClient.close();
@@ -136,10 +137,10 @@ public class HttpClientUtil {
 
             HttpResponse response = httpClient.execute(httpPost);
             HttpEntity entity = response.getEntity();
-            return EntityUtils.toString(entity, config.getCharset());
+            return getResponseStr(response, entity, config);
         } catch (Exception e) {
             logger.error("HttpClient查询失败", e);
-            throw new RuntimeException("HttpClient查询失败", e);
+            throw new RuntimeException("HttpClient查询失败: " + e.getMessage());
         } finally {
             try {
                 httpClient.close();
@@ -198,10 +199,10 @@ public class HttpClientUtil {
 
             HttpResponse response = httpClient.execute(httpPost);
             HttpEntity entity = response.getEntity();
-            return EntityUtils.toString(entity, config.getCharset());
+            return getResponseStr(response, entity, config);
         } catch (Exception e) {
             logger.error("HttpClient查询失败", e);
-            throw new RuntimeException("HttpClient查询失败", e);
+            throw new RuntimeException("HttpClient查询失败: " + e.getMessage());
         } finally {
             try {
                 httpClient.close();
@@ -211,5 +212,10 @@ public class HttpClientUtil {
         }
     }
 
-
+    private static String getResponseStr(HttpResponse response, HttpEntity entity, HttpClientConfig config) throws Exception{
+        if(response.getStatusLine().getStatusCode() >= 400){
+            throw new Exception(EntityUtils.toString(entity, config.getCharset()));
+        }
+        return EntityUtils.toString(entity, config.getCharset());
+    }
 }

backend/src/main/java/io/dataease/controller/dataset/DataSetTableController.java

@@ -6,7 +6,7 @@ import io.dataease.base.domain.DatasetTableField;
 import io.dataease.base.domain.DatasetTableIncrementalConfig;
 import io.dataease.controller.request.dataset.DataSetTableRequest;
 import io.dataease.controller.response.DataSetDetail;
-import io.dataease.dto.datasource.TableFiled;
+import io.dataease.dto.datasource.TableField;
 import io.dataease.dto.dataset.DataSetTableDTO;
 import io.dataease.dto.dataset.ExcelFileData;
 import io.dataease.service.dataset.DataSetTableService;
@@ -84,7 +84,7 @@ public class DataSetTableController {
 
     @ApiOperation("查询原始字段")
     @PostMapping("getFields")
-    public List<TableFiled> getFields(@RequestBody DatasetTable datasetTable) throws Exception {
+    public List<TableField> getFields(@RequestBody DatasetTable datasetTable) throws Exception {
         return dataSetTableService.getFields(datasetTable);
     }
 

backend/src/main/java/io/dataease/controller/datasource/DatasourceController.java

@@ -9,6 +9,7 @@ import io.dataease.commons.utils.PageUtils;
 import io.dataease.commons.utils.Pager;
 import io.dataease.controller.ResultHolder;
 import io.dataease.controller.request.DatasourceUnionRequest;
+import io.dataease.controller.request.datasource.ApiDefinition;
 import io.dataease.controller.sys.base.BaseGridRequest;
 import io.dataease.dto.datasource.DBTableDTO;
 import io.dataease.service.datasource.DatasourceService;
@@ -95,5 +96,11 @@ public class DatasourceController {
        return datasourceService.getSchema(datasource);
     }
 
+    @ApiOperation("校验API数据源")
+    @PostMapping("/checkApiDatasource")
+    public ApiDefinition checkApiDatasource(@RequestBody ApiDefinition apiDefinition) throws Exception {
+        return datasourceService.checkApiDatasource(apiDefinition);
+    }
+
 
 }

backend/src/main/java/io/dataease/controller/request/dataset/DataSetTableRequest.java

@@ -1,7 +1,7 @@
 package io.dataease.controller.request.dataset;
 
 import io.dataease.base.domain.DatasetTable;
-import io.dataease.dto.datasource.TableFiled;
+import io.dataease.dto.datasource.TableField;
 import io.dataease.dto.dataset.ExcelSheetData;
 import io.swagger.annotations.ApiModelProperty;
 import lombok.Getter;
@@ -33,7 +33,7 @@ public class DataSetTableRequest extends DatasetTable {
     @ApiModelProperty("类型过滤条件集合")
     private List<String> typeFilter;
     @ApiModelProperty("字段集合")
-    private List<TableFiled> fields;
+    private List<TableField> fields;
     @ApiModelProperty("excel sheet集合")
     private List<ExcelSheetData> sheets;
     @ApiModelProperty("是否合并sheet")

backend/src/main/java/io/dataease/controller/request/datasource/ApiDefinition.java

@@ -0,0 +1,20 @@
+package io.dataease.controller.request.datasource;
+
+import com.alibaba.fastjson.JSONObject;
+import io.dataease.base.domain.DatasetTableField;
+import lombok.Data;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Data
+public class ApiDefinition {
+    private String name;
+    private String desc;
+    private String url;
+    private String method = "GET";
+    private List<DatasetTableField> fields;
+    private String request;
+    private String dataPath;
+    private List<JSONObject> datas = new ArrayList<>();
+}

backend/src/main/java/io/dataease/controller/request/datasource/ApiDefinitionRequest.java

@@ -0,0 +1,21 @@
+package io.dataease.controller.request.datasource;
+
+import com.alibaba.fastjson.JSONObject;
+import lombok.Data;
+import java.util.ArrayList;
+import java.util.List;
+
+@Data
+public class ApiDefinitionRequest {
+    private List<JSONObject> headers = new ArrayList<>();
+    private JSONObject body = new JSONObject();
+    private AuthManager authManager = new AuthManager();
+
+
+    @Data
+    public class AuthManager{
+        private String password;
+        private String username;
+        private String verification = "";
+    }
+}

backend/src/main/java/io/dataease/dto/DatasourceDTO.java

@@ -1,5 +1,6 @@
 package io.dataease.dto;
 
+import com.alibaba.fastjson.JSONArray;
 import io.dataease.base.domain.Datasource;
 import io.swagger.annotations.ApiModelProperty;
 import lombok.Data;
@@ -14,5 +15,5 @@ public class DatasourceDTO extends Datasource {
 
     @ApiModelProperty("权限")
     private String privileges;
-
+    private JSONArray apiConfiguration;
 }

backend/src/main/java/io/dataease/dto/dataset/ExcelSheetData.java

@@ -1,6 +1,6 @@
 package io.dataease.dto.dataset;
 
-import io.dataease.dto.datasource.TableFiled;
+import io.dataease.dto.datasource.TableField;
 import io.swagger.annotations.ApiModelProperty;
 import lombok.Data;
 
@@ -14,7 +14,7 @@ public class ExcelSheetData {
     @ApiModelProperty("数据集合")
     private List<List<String>> data;
     @ApiModelProperty("字段集合")
-    private List<TableFiled> fields;
+    private List<TableField> fields;
     @ApiModelProperty("是否sheet")
     private boolean isSheet = true;
     @ApiModelProperty("json数组")

backend/src/main/java/io/dataease/dto/datasource/TableField.java

@@ -6,7 +6,7 @@ import lombok.Setter;
 
 @Setter
 @Getter
-public class TableFiled {
+public class TableField {
     @ApiModelProperty("字段名称")
     private String fieldName;
     @ApiModelProperty("重新标记")

backend/src/main/java/io/dataease/provider/ProviderFactory.java

@@ -4,6 +4,7 @@ import io.dataease.commons.constants.DatasourceTypes;
 import io.dataease.provider.datasource.DatasourceProvider;
 import io.dataease.provider.query.DDLProvider;
 import io.dataease.provider.query.QueryProvider;
+import io.dataease.provider.query.api.ApiProvider;
 import org.springframework.beans.BeansException;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.ApplicationContextAware;
@@ -25,6 +26,8 @@ public class ProviderFactory implements ApplicationContextAware {
         switch (datasourceType) {
             case es:
                 return context.getBean("es", DatasourceProvider.class);
+            case api:
+                return context.getBean("api", DatasourceProvider.class);
             default:
                 return context.getBean("jdbc", DatasourceProvider.class);
         }
@@ -57,6 +60,8 @@ public class ProviderFactory implements ApplicationContextAware {
                 return context.getBean("hiveQuery", QueryProvider.class);
             case db2:
                 return context.getBean("db2Query", QueryProvider.class);
+            case api:
+                return context.getBean("apiQuery", ApiProvider.class);
             default:
                 return context.getBean("mysqlQuery", QueryProvider.class);
         }

backend/src/main/java/io/dataease/provider/datasource/ApiProvider.java

@@ -0,0 +1,214 @@
+package io.dataease.provider.datasource;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.google.gson.Gson;
+import com.jayway.jsonpath.JsonPath;
+import io.dataease.base.domain.DatasetTableField;
+import io.dataease.commons.utils.HttpClientConfig;
+import io.dataease.commons.utils.HttpClientUtil;
+import io.dataease.controller.request.datasource.ApiDefinition;
+import io.dataease.controller.request.datasource.ApiDefinitionRequest;
+import io.dataease.controller.request.datasource.DatasourceRequest;
+import io.dataease.dto.datasource.TableDesc;
+import io.dataease.dto.datasource.TableField;
+import io.dataease.exception.DataEaseException;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.HttpHeaders;
+import org.springframework.stereotype.Service;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Service("api")
+public class ApiProvider extends DatasourceProvider{
+    @Override
+    public List<String[]> getData(DatasourceRequest datasourceRequest) throws Exception {
+        ApiDefinition apiDefinition = checkApiDefinition(datasourceRequest);
+        String response = execHttpRequest(apiDefinition);
+        return fetchResult(response, apiDefinition.getDataPath());
+    }
+
+    @Override
+    public List<TableDesc> getTables(DatasourceRequest datasourceRequest) throws Exception {
+        List<TableDesc> tableDescs = new ArrayList<>();
+        List<ApiDefinition> lists = JSONObject.parseArray(datasourceRequest.getDatasource().getConfiguration(), ApiDefinition.class);
+        for (ApiDefinition apiDefinition : lists) {
+            TableDesc tableDesc = new TableDesc();
+            tableDesc.setName(apiDefinition.getName());
+            tableDesc.setRemark(apiDefinition.getDesc());
+            tableDescs.add(tableDesc);
+        }
+        return tableDescs;
+    }
+
+    @Override
+    public List<String[]> fetchResult(DatasourceRequest datasourceRequest) throws Exception {
+        return null;
+    }
+
+    @Override
+    public List<TableField> fetchResultField(DatasourceRequest datasourceRequest) throws Exception {
+        return null;
+    }
+
+    @Override
+    public Map<String, List> fetchResultAndField(DatasourceRequest datasourceRequest) throws Exception {
+        Map<String, List> result = new HashMap<>();
+        List<String[]> dataList = new ArrayList<>();
+        List<TableField> fieldList = new ArrayList<>();;
+        ApiDefinition apiDefinition = checkApiDefinition(datasourceRequest);
+        String response = execHttpRequest(apiDefinition);
+
+        fieldList = getTableFileds(datasourceRequest);
+        result.put("fieldList", fieldList);
+        dataList = fetchResult(response, apiDefinition.getDataPath());
+        result.put("dataList", dataList);
+        return result;
+    }
+
+    @Override
+    public void handleDatasource(DatasourceRequest datasourceRequest, String type) throws Exception {
+
+    }
+
+    @Override
+    public List<String> getSchema(DatasourceRequest datasourceRequest) throws Exception {
+        return null;
+    }
+
+    @Override
+    public List<TableField> getTableFileds(DatasourceRequest datasourceRequest) throws Exception {
+        List<ApiDefinition> lists = JSONObject.parseArray(datasourceRequest.getDatasource().getConfiguration(), ApiDefinition.class);
+        List<TableField> tableFields = new ArrayList<>();
+        for (ApiDefinition list : lists) {
+            if(datasourceRequest.getTable().equalsIgnoreCase(list.getName())){
+                for (DatasetTableField field : list.getFields()) {
+                    TableField tableField = new TableField();
+                    tableField.setFieldName(field.getOriginName());
+                    tableField.setRemarks(field.getName());
+                    tableField.setFieldSize(field.getSize());
+                    tableField.setFieldType(field.getDeExtractType().toString());
+                    tableFields.add(tableField);
+                }
+            }
+        }
+        return tableFields;
+    }
+
+    @Override
+    public String checkStatus(DatasourceRequest datasourceRequest) throws Exception {
+        List<ApiDefinition> apiDefinitionList = JSONObject.parseArray(datasourceRequest.getDatasource().getConfiguration(), ApiDefinition.class).stream().filter(item -> item.getName().equalsIgnoreCase(datasourceRequest.getTable())).collect(Collectors.toList());
+        int success = 0;
+        for (ApiDefinition apiDefinition : apiDefinitionList) {
+            datasourceRequest.setTable(apiDefinition.getName());
+           try {
+                getData(datasourceRequest);
+                success++;
+           }catch (Exception ignore){}
+        }
+        if(success == apiDefinitionList.size()){
+            return "Success";
+        }
+        if(success > 0 && success < apiDefinitionList.size() ){
+            return "Warning";
+        }
+        return "Error";
+    }
+
+    static public String execHttpRequest(ApiDefinition apiDefinition) throws Exception{
+        String response = "";
+        HttpClientConfig httpClientConfig = new HttpClientConfig();
+        ApiDefinitionRequest apiDefinitionRequest = JSONObject.parseObject(apiDefinition.getRequest(), ApiDefinitionRequest.class);
+        System.out.println(new Gson().toJson(apiDefinitionRequest.getAuthManager()));
+        //headers
+        for (JSONObject header : apiDefinitionRequest.getHeaders()) {
+            if(StringUtils.isNotEmpty(header.getString("name")) && StringUtils.isNotEmpty(header.getString("value"))){
+                httpClientConfig.addHeader(header.getString("name"), header.getString("value"));
+            }
+        }
+
+        if(apiDefinitionRequest.getAuthManager() != null
+                && StringUtils.isNotBlank(apiDefinitionRequest.getAuthManager().getUsername())
+                && StringUtils.isNotBlank(apiDefinitionRequest.getAuthManager().getPassword())
+                && apiDefinitionRequest.getAuthManager().getVerification().equals("Basic Auth")){
+            String authValue = "Basic " + Base64.getUrlEncoder().encodeToString((apiDefinitionRequest.getAuthManager().getUsername()
+                    + ":" + apiDefinitionRequest.getAuthManager().getPassword()).getBytes());
+            httpClientConfig.addHeader("Authorization", authValue);
+        }
+
+        switch (apiDefinition.getMethod()){
+            case "GET":
+                response = HttpClientUtil.get(apiDefinition.getUrl(), httpClientConfig);
+                break;
+            case "POST":
+                if (!apiDefinitionRequest.getBody().containsKey("type")) {
+                   throw new Exception("请求类型不能为空");
+                }
+                String type = apiDefinitionRequest.getBody().getString("type");
+                if (StringUtils.equalsAny(type, "JSON", "XML", "Raw")) {
+                    String raw = null;
+                    if (apiDefinitionRequest.getBody().containsKey("raw")) {
+                        raw = apiDefinitionRequest.getBody().getString("raw");
+                        response = HttpClientUtil.post(apiDefinition.getUrl(), raw, httpClientConfig);
+                    }
+                }
+                if (StringUtils.equalsAny(type, "Form_Data", "WWW_FORM")) {
+                    if (apiDefinitionRequest.getBody().containsKey("kvs")) {
+                        Map<String, String> body = new HashMap<>();
+                        JSONArray kvsArr = apiDefinitionRequest.getBody().getJSONArray("kvs");
+                        for (int i = 0; i < kvsArr.size(); i++) {
+                            JSONObject kv = kvsArr.getJSONObject(i);
+                            if (kv.containsKey("name")) {
+                                body.put(kv.getString("name"), kv.getString("value"));
+                            }
+                        }
+                        response = HttpClientUtil.post(apiDefinition.getUrl(), body, httpClientConfig);
+                    }
+                }
+                break;
+            default:
+                break;
+        }
+        System.out.println("response: " + response);
+        return response;
+    }
+
+    private List<String[]> fetchResult(String result, String path){
+        List<String[]> dataList = new LinkedList<>();
+        List<LinkedHashMap> datas = JsonPath.read(result, path);
+        for (LinkedHashMap data : datas) {
+            String[] row = new String[data.entrySet().size()];
+            Iterator it = data.entrySet().iterator();
+            int i = 0;
+            while (it.hasNext()){
+                Map.Entry entry = (Map.Entry)it.next();
+                row[i] = entry.getValue().toString();
+                i++;
+            }
+            dataList.add(row);
+        }
+        return dataList;
+    }
+
+    private ApiDefinition checkApiDefinition(DatasourceRequest datasourceRequest)throws Exception{
+        List<ApiDefinition> apiDefinitionList = JSONObject.parseArray(datasourceRequest.getDatasource().getConfiguration(), ApiDefinition.class).stream().filter(item -> item.getName().equalsIgnoreCase(datasourceRequest.getTable())).collect(Collectors.toList());
+        if(CollectionUtils.isEmpty(apiDefinitionList)){
+            throw new Exception("未找到API数据表");
+        }
+        if(apiDefinitionList.size() > 1 ){
+            throw new Exception("存在重名的API数据表");
+        }
+        for (ApiDefinition apiDefinition : apiDefinitionList) {
+            if (apiDefinition.getName().equalsIgnoreCase(datasourceRequest.getTable())){
+                return apiDefinition;
+            }
+        }
+        throw new Exception("未找到API数据表");
+    }
+
+}

backend/src/main/java/io/dataease/provider/datasource/DatasourceProvider.java

@@ -1,7 +1,7 @@
 package io.dataease.provider.datasource;
 
 import io.dataease.dto.datasource.TableDesc;
-import io.dataease.dto.datasource.TableFiled;
+import io.dataease.dto.datasource.TableField;
 import io.dataease.controller.request.datasource.DatasourceRequest;
 
 import java.util.List;
@@ -15,13 +15,14 @@ public abstract class DatasourceProvider {
 
     abstract public List<TableDesc> getTables(DatasourceRequest datasourceRequest) throws Exception;
 
-    public void checkStatus(DatasourceRequest datasourceRequest) throws Exception {
+    public String checkStatus(DatasourceRequest datasourceRequest) throws Exception {
         getData(datasourceRequest);
+        return "Success";
     }
 
     abstract public List<String[]> fetchResult(DatasourceRequest datasourceRequest) throws Exception;
 
-    abstract public List<TableFiled> fetchResultField(DatasourceRequest datasourceRequest) throws Exception;
+    abstract public List<TableField> fetchResultField(DatasourceRequest datasourceRequest) throws Exception;
 
     abstract public Map<String, List> fetchResultAndField(DatasourceRequest datasourceRequest) throws Exception;
 
@@ -29,5 +30,5 @@ public abstract class DatasourceProvider {
 
     abstract public List<String> getSchema(DatasourceRequest datasourceRequest) throws Exception;
 
-    public abstract List<TableFiled> getTableFileds(DatasourceRequest datasourceRequest) throws Exception;
+    public abstract List<TableField> getTableFileds(DatasourceRequest datasourceRequest) throws Exception;
 }

backend/src/main/java/io/dataease/provider/datasource/EsProvider.java

@@ -10,7 +10,7 @@ import io.dataease.controller.request.datasource.es.RequstWithCursor;
 import io.dataease.controller.request.datasource.DatasourceRequest;
 import io.dataease.dto.datasource.EsConfiguration;
 import io.dataease.dto.datasource.TableDesc;
-import io.dataease.dto.datasource.TableFiled;
+import io.dataease.dto.datasource.TableField;
 import io.dataease.exception.DataEaseException;
 import io.dataease.i18n.Translator;
 import io.dataease.provider.ProviderFactory;
@@ -98,7 +98,7 @@ public class EsProvider extends DatasourceProvider {
     }
 
     @Override
-    public List<TableFiled> getTableFileds(DatasourceRequest datasourceRequest) throws Exception {
+    public List<TableField> getTableFileds(DatasourceRequest datasourceRequest) throws Exception {
         QueryProvider qp = ProviderFactory.getQueryProvider(datasourceRequest.getDatasource().getType());
         datasourceRequest.setQuery(qp.convertTableToSql(datasourceRequest.getTable(), datasourceRequest.getDatasource()));
         return fetchResultField(datasourceRequest);
@@ -119,26 +119,26 @@ public class EsProvider extends DatasourceProvider {
     }
 
     @Override
-    public List<TableFiled> fetchResultField(DatasourceRequest datasourceRequest) throws Exception {
+    public List<TableField> fetchResultField(DatasourceRequest datasourceRequest) throws Exception {
-        List<TableFiled> tableFileds = new ArrayList<>();
+        List<TableField> tableFields = new ArrayList<>();
         try {
             String response = exexQuery(datasourceRequest, datasourceRequest.getQuery(), "?format=json");
-            tableFileds = fetchResultField4Sql(response);
+            tableFields = fetchResultField4Sql(response);
         } catch (Exception e) {
             DataEaseException.throwException(e);
         }
-        return tableFileds;
+        return tableFields;
     }
 
-    private List<TableFiled> fetchResultField(String response) throws Exception {
+    private List<TableField> fetchResultField(String response) throws Exception {
-        List<TableFiled> fieldList = new ArrayList<>();
+        List<TableField> fieldList = new ArrayList<>();
         EsReponse esReponse = new Gson().fromJson(response, EsReponse.class);
         if (esReponse.getError() != null) {
             throw new Exception(esReponse.getError().getReason());
         }
 
         for (String[] row : esReponse.getRows()) {
-            TableFiled field = new TableFiled();
+            TableField field = new TableField();
             field.setFieldName(row[0]);
             field.setRemarks(row[0]);
             field.setFieldType(row[2]);
@@ -148,15 +148,15 @@ public class EsProvider extends DatasourceProvider {
         return fieldList;
     }
 
-    private List<TableFiled> fetchResultField4Sql(String response) throws Exception {
+    private List<TableField> fetchResultField4Sql(String response) throws Exception {
-        List<TableFiled> fieldList = new ArrayList<>();
+        List<TableField> fieldList = new ArrayList<>();
         EsReponse esReponse = new Gson().fromJson(response, EsReponse.class);
         if (esReponse.getError() != null) {
             throw new Exception(esReponse.getError().getReason());
         }
 
         for (EsReponse.Column column : esReponse.getColumns()) {
-            TableFiled field = new TableFiled();
+            TableField field = new TableField();
             field.setFieldName(column.getName());
             field.setRemarks(column.getName());
             field.setFieldType(column.getType());
@@ -225,7 +225,7 @@ public class EsProvider extends DatasourceProvider {
 
 
     @Override
-    public void checkStatus(DatasourceRequest datasourceRequest) throws Exception {
+    public String checkStatus(DatasourceRequest datasourceRequest) throws Exception {
         EsConfiguration esConfiguration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), EsConfiguration.class);
         String response = exexGetQuery(datasourceRequest);
 
@@ -246,6 +246,7 @@ public class EsProvider extends DatasourceProvider {
         }
         datasourceRequest.getDatasource().setConfiguration(new Gson().toJson(esConfiguration));
         getTables(datasourceRequest);
+        return "Success";
     }
 
     private String exexQuery(DatasourceRequest datasourceRequest, String sql, String uri) {

backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java

@@ -138,12 +138,12 @@ public class JdbcProvider extends DatasourceProvider {
     }
 
     @Override
-    public List<TableFiled> getTableFileds(DatasourceRequest datasourceRequest) throws Exception {
+    public List<TableField> getTableFileds(DatasourceRequest datasourceRequest) throws Exception {
         if(datasourceRequest.getDatasource().getType().equalsIgnoreCase("mongo")){
             datasourceRequest.setQuery("select * from " + datasourceRequest.getTable());
             return fetchResultField(datasourceRequest);
         }
-        List<TableFiled> list = new LinkedList<>();
+        List<TableField> list = new LinkedList<>();
         try (Connection connection = getConnectionFromPool(datasourceRequest)) {
             if (datasourceRequest.getDatasource().getType().equalsIgnoreCase("oracle")) {
                 Method setRemarksReporting = extendedJdbcClassLoader.loadClass("oracle.jdbc.driver.OracleConnection").getMethod("setRemarksReporting",boolean.class);
@@ -161,13 +161,13 @@ public class JdbcProvider extends DatasourceProvider {
                 }
                 if (database != null) {
                     if (tableName.equals(datasourceRequest.getTable()) && database.equalsIgnoreCase(getDatabase(datasourceRequest))) {
-                        TableFiled tableFiled = getTableFiled(resultSet, datasourceRequest);
+                        TableField tableField = getTableFiled(resultSet, datasourceRequest);
-                        list.add(tableFiled);
+                        list.add(tableField);
                     }
                 } else {
                     if (tableName.equals(datasourceRequest.getTable())) {
-                        TableFiled tableFiled = getTableFiled(resultSet, datasourceRequest);
+                        TableField tableField = getTableFiled(resultSet, datasourceRequest);
-                        list.add(tableFiled);
+                        list.add(tableField);
                     }
                 }
             }
@@ -186,40 +186,40 @@ public class JdbcProvider extends DatasourceProvider {
         return list;
     }
 
-    private TableFiled getTableFiled(ResultSet resultSet, DatasourceRequest datasourceRequest) throws SQLException {
+    private TableField getTableFiled(ResultSet resultSet, DatasourceRequest datasourceRequest) throws SQLException {
-        TableFiled tableFiled = new TableFiled();
+        TableField tableField = new TableField();
         String colName = resultSet.getString("COLUMN_NAME");
-        tableFiled.setFieldName(colName);
+        tableField.setFieldName(colName);
         String remarks = resultSet.getString("REMARKS");
         if (remarks == null || remarks.equals("")) {
             remarks = colName;
         }
-        tableFiled.setRemarks(remarks);
+        tableField.setRemarks(remarks);
         String dbType = resultSet.getString("TYPE_NAME").toUpperCase();
-        tableFiled.setFieldType(dbType);
+        tableField.setFieldType(dbType);
         if (dbType.equalsIgnoreCase("LONG")) {
-            tableFiled.setFieldSize(65533);
+            tableField.setFieldSize(65533);
         }
-        if (StringUtils.isNotEmpty(dbType) && dbType.toLowerCase().contains("date") && tableFiled.getFieldSize() < 50) {
+        if (StringUtils.isNotEmpty(dbType) && dbType.toLowerCase().contains("date") && tableField.getFieldSize() < 50) {
-            tableFiled.setFieldSize(50);
+            tableField.setFieldSize(50);
         }
 
         if (datasourceRequest.getDatasource().getType().equalsIgnoreCase(DatasourceTypes.ck.name())) {
             QueryProvider qp = ProviderFactory.getQueryProvider(datasourceRequest.getDatasource().getType());
-            tableFiled.setFieldSize(qp.transFieldSize(dbType));
+            tableField.setFieldSize(qp.transFieldSize(dbType));
         } else {
-            if (datasourceRequest.getDatasource().getType().equalsIgnoreCase(DatasourceTypes.hive.name()) && tableFiled.getFieldType().equalsIgnoreCase("BOOLEAN")) {
+            if (datasourceRequest.getDatasource().getType().equalsIgnoreCase(DatasourceTypes.hive.name()) && tableField.getFieldType().equalsIgnoreCase("BOOLEAN")) {
-                tableFiled.setFieldSize(1);
+                tableField.setFieldSize(1);
             } else {
                 String size = resultSet.getString("COLUMN_SIZE");
                 if (size == null) {
-                    tableFiled.setFieldSize(1);
+                    tableField.setFieldSize(1);
                 } else {
-                    tableFiled.setFieldSize(Integer.valueOf(size));
+                    tableField.setFieldSize(Integer.valueOf(size));
                 }
             }
         }
-        return tableFiled;
+        return tableField;
     }
 
     private String getDatabase(DatasourceRequest datasourceRequest) {
@@ -244,7 +244,7 @@ public class JdbcProvider extends DatasourceProvider {
     }
 
     @Override
-    public List<TableFiled> fetchResultField(DatasourceRequest datasourceRequest) throws Exception {
+    public List<TableField> fetchResultField(DatasourceRequest datasourceRequest) throws Exception {
         try (Connection connection = getConnectionFromPool(datasourceRequest); Statement stat = connection.createStatement(); ResultSet rs = stat.executeQuery(rebuildSqlWithFragment(datasourceRequest.getQuery()))) {
             return fetchResultField(rs, datasourceRequest);
         } catch (SQLException e) {
@@ -259,7 +259,7 @@ public class JdbcProvider extends DatasourceProvider {
     public Map<String, List> fetchResultAndField(DatasourceRequest datasourceRequest) throws Exception {
         Map<String, List> result = new HashMap<>();
         List<String[]> dataList;
-        List<TableFiled> fieldList;
+        List<TableField> fieldList;
         try (Connection connection = getConnectionFromPool(datasourceRequest); Statement stat = connection.createStatement(); ResultSet rs = stat.executeQuery(rebuildSqlWithFragment(datasourceRequest.getQuery()))) {
             fieldList = fetchResultField(rs, datasourceRequest);
             result.put("fieldList", fieldList);
@@ -274,8 +274,8 @@ public class JdbcProvider extends DatasourceProvider {
         return new HashMap<>();
     }
 
-    private List<TableFiled> fetchResultField(ResultSet rs, DatasourceRequest datasourceRequest) throws Exception {
+    private List<TableField> fetchResultField(ResultSet rs, DatasourceRequest datasourceRequest) throws Exception {
-        List<TableFiled> fieldList = new ArrayList<>();
+        List<TableField> fieldList = new ArrayList<>();
         ResultSetMetaData metaData = rs.getMetaData();
         int columnCount = metaData.getColumnCount();
         for (int j = 0; j < columnCount; j++) {
@@ -285,7 +285,7 @@ public class JdbcProvider extends DatasourceProvider {
             if (datasourceRequest.getDatasource().getType().equalsIgnoreCase(DatasourceTypes.hive.name()) && l.contains("\\.")) {
                 l = l.split("\\.")[1];
             }
-            TableFiled field = new TableFiled();
+            TableField field = new TableField();
             field.setFieldName(l);
             field.setRemarks(l);
             field.setFieldType(t);
@@ -362,14 +362,15 @@ public class JdbcProvider extends DatasourceProvider {
     }
 
     @Override
-    public void checkStatus(DatasourceRequest datasourceRequest) throws Exception {
+    public String checkStatus(DatasourceRequest datasourceRequest) throws Exception {
         String queryStr = getTablesSql(datasourceRequest);
         try (Connection con = getConnection(datasourceRequest); Statement statement = con.createStatement(); ResultSet resultSet = statement.executeQuery(queryStr)) {
-
+            return "Success";
         } catch (Exception e) {
             e.printStackTrace();
             DataEaseException.throwException(e.getMessage());
         }
+        return "Error";
     }
 
     @Override

backend/src/main/java/io/dataease/provider/query/QueryProviderImpl.java

@@ -0,0 +1,122 @@
+package io.dataease.provider.query;
+
+import io.dataease.base.domain.ChartViewWithBLOBs;
+import io.dataease.base.domain.DatasetTableField;
+import io.dataease.base.domain.Datasource;
+import io.dataease.controller.request.chart.ChartExtFilterRequest;
+import io.dataease.dto.chart.ChartFieldCustomFilterDTO;
+import io.dataease.dto.chart.ChartViewFieldDTO;
+
+import java.util.List;
+
+public class QueryProviderImpl extends QueryProvider {
+    @Override
+    public Integer transFieldType(String field) {
+        return null;
+    }
+
+    @Override
+    public String createSQLPreview(String sql, String orderBy) {
+        return null;
+    }
+
+    @Override
+    public String createQuerySQL(String table, List<DatasetTableField> fields, boolean isGroup, Datasource ds, List<ChartFieldCustomFilterDTO> fieldCustomFilter) {
+        return null;
+    }
+
+    @Override
+    public String createQuerySQLAsTmp(String sql, List<DatasetTableField> fields, boolean isGroup, List<ChartFieldCustomFilterDTO> fieldCustomFilter) {
+        return null;
+    }
+
+    @Override
+    public String createQueryTableWithPage(String table, List<DatasetTableField> fields, Integer page, Integer pageSize, Integer realSize, boolean isGroup, Datasource ds, List<ChartFieldCustomFilterDTO> fieldCustomFilter) {
+        return null;
+    }
+
+    @Override
+    public String createQuerySQLWithPage(String sql, List<DatasetTableField> fields, Integer page, Integer pageSize, Integer realSize, boolean isGroup, List<ChartFieldCustomFilterDTO> fieldCustomFilter) {
+        return null;
+    }
+
+    @Override
+    public String createQueryTableWithLimit(String table, List<DatasetTableField> fields, Integer limit, boolean isGroup, Datasource ds, List<ChartFieldCustomFilterDTO> fieldCustomFilter) {
+        return null;
+    }
+
+    @Override
+    public String createQuerySqlWithLimit(String sql, List<DatasetTableField> fields, Integer limit, boolean isGroup, List<ChartFieldCustomFilterDTO> fieldCustomFilter) {
+        return null;
+    }
+
+    @Override
+    public String getSQL(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, Datasource ds, ChartViewWithBLOBs view) {
+        return null;
+    }
+
+    @Override
+    public String getSQLAsTmp(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, ChartViewWithBLOBs view) {
+        return null;
+    }
+
+    @Override
+    public String getSQLTableInfo(String table, List<ChartViewFieldDTO> xAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, Datasource ds, ChartViewWithBLOBs view) {
+        return null;
+    }
+
+    @Override
+    public String getSQLAsTmpTableInfo(String sql, List<ChartViewFieldDTO> xAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, Datasource ds, ChartViewWithBLOBs view) {
+        return null;
+    }
+
+    @Override
+    public String getSQLStack(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, List<ChartViewFieldDTO> extStack, Datasource ds, ChartViewWithBLOBs view) {
+        return null;
+    }
+
+    @Override
+    public String getSQLAsTmpStack(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, List<ChartViewFieldDTO> extStack, ChartViewWithBLOBs view) {
+        return null;
+    }
+
+    @Override
+    public String getSQLScatter(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, List<ChartViewFieldDTO> extBubble, Datasource ds, ChartViewWithBLOBs view) {
+        return null;
+    }
+
+    @Override
+    public String getSQLAsTmpScatter(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, List<ChartViewFieldDTO> extBubble, ChartViewWithBLOBs view) {
+        return null;
+    }
+
+    @Override
+    public String searchTable(String table) {
+        return null;
+    }
+
+    @Override
+    public String getSQLSummary(String table, List<ChartViewFieldDTO> yAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, ChartViewWithBLOBs view) {
+        return null;
+    }
+
+    @Override
+    public String getSQLSummaryAsTmp(String sql, List<ChartViewFieldDTO> yAxis, List<ChartFieldCustomFilterDTO> fieldCustomFilter, List<ChartExtFilterRequest> extFilterRequestList, ChartViewWithBLOBs view) {
+        return null;
+    }
+
+    @Override
+    public String wrapSql(String sql) {
+        return null;
+    }
+
+    @Override
+    public String createRawQuerySQL(String table, List<DatasetTableField> fields, Datasource ds) {
+        return null;
+    }
+
+    @Override
+    public String createRawQuerySQLAsTmp(String sql, List<DatasetTableField> fields) {
+        return null;
+    }
+}

backend/src/main/java/io/dataease/provider/query/api/ApiProvider.java

@@ -0,0 +1,25 @@
+package io.dataease.provider.query.api;
+
+import io.dataease.provider.query.QueryProviderImpl;
+import org.springframework.stereotype.Service;
+
+@Service("apiQuery")
+public class ApiProvider extends QueryProviderImpl {
+    @Override
+    public Integer transFieldType(String field) {
+        switch (field) {
+            case "0":
+                return 0;// 文本
+            case "1":
+                return 1;// 时间
+            case "2":
+                return 2;// 整型
+            case "3":
+                return 3;// 浮点
+            case "4":
+                return 4;// 布尔
+            default:
+                return 0;
+        }
+    }
+}

backend/src/main/java/io/dataease/service/chart/ChartViewService.java

@@ -242,9 +242,9 @@ public class ChartViewService {
 
         //列权限
         List<String> desensitizationList = new ArrayList<>();
-        fields = permissionService.filterColumnPermissons(fields, desensitizationList, datasetTable.getId(), requestList.getUser());
+        List<DatasetTableField> columnPermissionFields = permissionService.filterColumnPermissons(fields, desensitizationList, datasetTable.getId(), requestList.getUser());
         //将没有权限的列删掉
-        List<String> dataeaseNames = fields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList());
+        List<String> dataeaseNames = columnPermissionFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.toList());
         dataeaseNames.add("*");
         fieldCustomFilter = fieldCustomFilter.stream().filter(item -> !desensitizationList.contains(item.getDataeaseName()) && dataeaseNames.contains(item.getDataeaseName())).collect(Collectors.toList());
         extStack = extStack.stream().filter(item -> !desensitizationList.contains(item.getDataeaseName()) && dataeaseNames.contains(item.getDataeaseName())).collect(Collectors.toList());
@@ -253,8 +253,8 @@ public class ChartViewService {
 
 
         //行权限
-        List<ChartFieldCustomFilterDTO> permissionFields = permissionService.getCustomFilters(fields, datasetTable, requestList.getUser());
+        List<ChartFieldCustomFilterDTO> rowPermissionFields = permissionService.getCustomFilters(columnPermissionFields, datasetTable, requestList.getUser());
-        fieldCustomFilter.addAll(permissionFields);
+        fieldCustomFilter.addAll(rowPermissionFields);
 
         for (ChartFieldCustomFilterDTO ele : fieldCustomFilter) {
             ele.setField(dataSetTableFieldsService.get(ele.getId()));
@@ -485,7 +485,7 @@ public class ChartViewService {
             // 仪表板有参数不实用缓存
             if (!cache || CollectionUtils.isNotEmpty(requestList.getFilter())
                     || CollectionUtils.isNotEmpty(requestList.getLinkageFilters())
-                    || CollectionUtils.isNotEmpty(requestList.getDrill())) {
+                    || CollectionUtils.isNotEmpty(requestList.getDrill()) || CollectionUtils.isNotEmpty(rowPermissionFields) || fields.size() != columnPermissionFields.size()) {
                 data = datasourceProvider.getData(datasourceRequest);
             } else {
                 try {

backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java

@@ -19,7 +19,7 @@ import io.dataease.dto.dataset.*;
 import io.dataease.dto.dataset.union.UnionDTO;
 import io.dataease.dto.dataset.union.UnionItemDTO;
 import io.dataease.dto.dataset.union.UnionParamDTO;
-import io.dataease.dto.datasource.TableFiled;
+import io.dataease.dto.datasource.TableField;
 import io.dataease.exception.DataEaseException;
 import io.dataease.i18n.Translator;
 import io.dataease.plugins.loader.ClassloaderResponsity;
@@ -143,7 +143,7 @@ public class DataSetTableService {
                     sheetTable.setName(excelSheetDataList.get(0).getDatasetName());
                     checkName(sheetTable);
                     excelSheetDataList.forEach(excelSheetData -> {
-                        String[] fieldArray = excelSheetData.getFields().stream().map(TableFiled::getFieldName)
+                        String[] fieldArray = excelSheetData.getFields().stream().map(TableField::getFieldName)
                                 .toArray(String[]::new);
                         if (checkIsRepeat(fieldArray)) {
                             DataEaseException.throwException(Translator.get("i18n_excel_field_repeat"));
@@ -164,7 +164,7 @@ public class DataSetTableService {
                 });
             } else {
                 for (ExcelSheetData sheet : datasetTable.getSheets()) {
-                    String[] fieldArray = sheet.getFields().stream().map(TableFiled::getFieldName)
+                    String[] fieldArray = sheet.getFields().stream().map(TableField::getFieldName)
                             .toArray(String[]::new);
                     if (checkIsRepeat(fieldArray)) {
                         DataEaseException.throwException(Translator.get("i18n_excel_field_repeat"));
@@ -197,12 +197,12 @@ public class DataSetTableService {
         }
 
         List<ExcelSheetData> excelSheetDataList = new ArrayList<>();
-        List<String> oldFields = datasetTable.getSheets().get(0).getFields().stream().map(TableFiled::getRemarks)
+        List<String> oldFields = datasetTable.getSheets().get(0).getFields().stream().map(TableField::getRemarks)
                 .collect(Collectors.toList());
         for (ExcelSheetData sheet : datasetTable.getSheets()) {
             // 替换时，
             if (datasetTable.getEditType() == 0) {
-                List<String> newFields = sheet.getFields().stream().map(TableFiled::getRemarks)
+                List<String> newFields = sheet.getFields().stream().map(TableField::getRemarks)
                         .collect(Collectors.toList());
                 if (!oldFields.equals(newFields)) {
                     DataEaseException.throwException(Translator.get("i18n_excel_column_inconsistent"));
@@ -210,7 +210,7 @@ public class DataSetTableService {
                 oldFields = newFields;
             }
 
-            String[] fieldArray = sheet.getFields().stream().map(TableFiled::getFieldName).toArray(String[]::new);
+            String[] fieldArray = sheet.getFields().stream().map(TableField::getFieldName).toArray(String[]::new);
             if (checkIsRepeat(fieldArray)) {
                 DataEaseException.throwException(Translator.get("i18n_excel_field_repeat"));
             }
@@ -405,7 +405,7 @@ public class DataSetTableService {
         return extDataSetTableMapper.searchOne(dataSetTableRequest);
     }
 
-    public List<TableFiled> getFields(DatasetTable datasetTable) throws Exception {
+    public List<TableField> getFields(DatasetTable datasetTable) throws Exception {
         Datasource ds = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId());
         DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
         DatasourceRequest datasourceRequest = new DatasourceRequest();
@@ -496,7 +496,7 @@ public class DataSetTableService {
         if (page == Integer.parseInt(dataSetTableRequest.getRow()) / pageSize + 1) {
             realSize = Integer.parseInt(dataSetTableRequest.getRow()) % pageSize;
         }
-        if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "db")) {
+        if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "db") || StringUtils.equalsIgnoreCase(datasetTable.getType(), "api")) {
             if (datasetTable.getMode() == 0) {
                 Datasource ds = datasourceMapper.selectByPrimaryKey(dataSetTableRequest.getDataSourceId());
                 if (ObjectUtils.isEmpty(ds)) {
@@ -847,8 +847,8 @@ public class DataSetTableService {
         datasourceRequest.setQuery(sqlAsTable);
         Map<String, List> result = datasourceProvider.fetchResultAndField(datasourceRequest);
         List<String[]> data = result.get("dataList");
-        List<TableFiled> fields = result.get("fieldList");
+        List<TableField> fields = result.get("fieldList");
-        String[] fieldArray = fields.stream().map(TableFiled::getFieldName).toArray(String[]::new);
+        String[] fieldArray = fields.stream().map(TableField::getFieldName).toArray(String[]::new);
         if (checkIsRepeat(fieldArray)) {
             DataEaseException.throwException(Translator.get("i18n_excel_field_repeat"));
         }
@@ -896,8 +896,8 @@ public class DataSetTableService {
             datasourceRequest.setQuery(qp.createSQLPreview(sql, null));
             Map<String, List> result = datasourceProvider.fetchResultAndField(datasourceRequest);
             List<String[]> data = result.get("dataList");
-            List<TableFiled> fields = result.get("fieldList");
+            List<TableField> fields = result.get("fieldList");
-            String[] fieldArray = fields.stream().map(TableFiled::getFieldName).toArray(String[]::new);
+            String[] fieldArray = fields.stream().map(TableField::getFieldName).toArray(String[]::new);
 
             List<Map<String, Object>> jsonArray = new ArrayList<>();
             if (CollectionUtils.isNotEmpty(data)) {
@@ -912,14 +912,14 @@ public class DataSetTableService {
 
             // 获取每个字段在当前de数据库中的name，作为sql查询后的remarks返回前端展示
             for (DatasetTableField datasetTableField : fieldList) {
-                for (TableFiled tableFiled : fields) {
+                for (TableField tableField : fields) {
-                    if (StringUtils.equalsIgnoreCase(tableFiled.getFieldName(),
+                    if (StringUtils.equalsIgnoreCase(tableField.getFieldName(),
                             DorisTableUtils.dorisFieldName(
                                     datasetTableField.getTableId() + "_" + datasetTableField.getDataeaseName()))
-                            || StringUtils.equalsIgnoreCase(tableFiled.getFieldName(),
+                            || StringUtils.equalsIgnoreCase(tableField.getFieldName(),
                                     DorisTableUtils.dorisFieldNameShort(datasetTableField.getTableId() + "_"
                                             + datasetTableField.getOriginName()))) {
-                        tableFiled.setRemarks(datasetTableField.getName());
+                        tableField.setRemarks(datasetTableField.getName());
                         break;
                     }
                 }
@@ -957,8 +957,8 @@ public class DataSetTableService {
             datasourceRequest.setQuery(qp.createSQLPreview(sql, null));
             Map<String, List> result = datasourceProvider.fetchResultAndField(datasourceRequest);
             List<String[]> data = result.get("dataList");
-            List<TableFiled> fields = result.get("fieldList");
+            List<TableField> fields = result.get("fieldList");
-            String[] fieldArray = fields.stream().map(TableFiled::getFieldName).toArray(String[]::new);
+            String[] fieldArray = fields.stream().map(TableField::getFieldName).toArray(String[]::new);
 
             List<Map<String, Object>> jsonArray = new ArrayList<>();
             if (CollectionUtils.isNotEmpty(data)) {
@@ -976,13 +976,13 @@ public class DataSetTableService {
             dataTableInfoDTO.getList().forEach(
                     ele -> checkedFieldList.addAll(dataSetTableFieldsService.getListByIds(ele.getCheckedFields())));
             for (DatasetTableField datasetTableField : checkedFieldList) {
-                for (TableFiled tableFiled : fields) {
+                for (TableField tableField : fields) {
-                    if (StringUtils.equalsIgnoreCase(tableFiled.getFieldName(),
+                    if (StringUtils.equalsIgnoreCase(tableField.getFieldName(),
                             DorisTableUtils.dorisFieldName(
                                     datasetTableField.getTableId() + "_" + datasetTableField.getDataeaseName()))
-                            || StringUtils.equalsIgnoreCase(tableFiled.getFieldName(), DorisTableUtils.dorisFieldName(
+                            || StringUtils.equalsIgnoreCase(tableField.getFieldName(), DorisTableUtils.dorisFieldName(
                                     datasetTableField.getTableId() + "_" + datasetTableField.getOriginName()))) {
-                        tableFiled.setRemarks(datasetTableField.getName());
+                        tableField.setRemarks(datasetTableField.getName());
                         break;
                     }
                 }
@@ -1417,11 +1417,11 @@ public class DataSetTableService {
         }
     }
 
-    public List<DatasetTableField> saveExcelTableField(String datasetTableId, List<TableFiled> fields, boolean insert) {
+    public List<DatasetTableField> saveExcelTableField(String datasetTableId, List<TableField> fields, boolean insert) {
         List<DatasetTableField> datasetTableFields = new ArrayList<>();
         if (CollectionUtils.isNotEmpty(fields)) {
             for (int i = 0; i < fields.size(); i++) {
-                TableFiled filed = fields.get(i);
+                TableField filed = fields.get(i);
                 DatasetTableField datasetTableField = DatasetTableField.builder().build();
                 datasetTableField.setTableId(datasetTableId);
                 datasetTableField.setOriginName(filed.getFieldName());
@@ -1450,9 +1450,9 @@ public class DataSetTableService {
         DataSetTableRequest dataSetTableRequest = new DataSetTableRequest();
         BeanUtils.copyBean(dataSetTableRequest, datasetTable);
 
-        List<TableFiled> fields = new ArrayList<>();
+        List<TableField> fields = new ArrayList<>();
         long syncTime = System.currentTimeMillis();
-        if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "db")) {
+        if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "db") || StringUtils.equalsIgnoreCase(datasetTable.getType(), "api")) {
             fields = getFields(datasetTable);
         } else if (StringUtils.equalsIgnoreCase(datasetTable.getType(), "sql")) {
             DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
@@ -1514,11 +1514,11 @@ public class DataSetTableService {
                     fieldList.addAll(listByIds);
                 });
                 for (DatasetTableField field : fieldList) {
-                    for (TableFiled tableFiled : fields) {
+                    for (TableField tableField : fields) {
                         if (StringUtils.equalsIgnoreCase(
                                 DorisTableUtils.dorisFieldName(field.getTableId() + "_" + field.getOriginName()),
-                                tableFiled.getFieldName())) {
+                                tableField.getFieldName())) {
-                            tableFiled.setRemarks(field.getName());
+                            tableField.setRemarks(field.getName());
                             break;
                         }
                     }
@@ -1544,11 +1544,11 @@ public class DataSetTableService {
                 datasourceRequest.setQuery(sql);
                 fields = datasourceProvider.fetchResultField(datasourceRequest);
                 for (DatasetTableField field : fieldList) {
-                    for (TableFiled tableFiled : fields) {
+                    for (TableField tableField : fields) {
                         if (StringUtils.equalsIgnoreCase(
                                 DorisTableUtils.dorisFieldName(field.getTableId() + "_" + field.getDataeaseName()),
-                                tableFiled.getFieldName())) {
+                                tableField.getFieldName())) {
-                            tableFiled.setRemarks(field.getName());
+                            tableField.setRemarks(field.getName());
                             break;
                         }
                     }
@@ -1568,11 +1568,11 @@ public class DataSetTableService {
                 fields = datasourceProvider.fetchResultField(datasourceRequest);
 
                 for (DatasetTableField field : fieldList) {
-                    for (TableFiled tableFiled : fields) {
+                    for (TableField tableField : fields) {
                         if (StringUtils.equalsIgnoreCase(
                                 DorisTableUtils.dorisFieldNameShort(field.getTableId() + "_" + field.getOriginName()),
-                                tableFiled.getFieldName())) {
+                                tableField.getFieldName())) {
-                            tableFiled.setRemarks(field.getName());
+                            tableField.setRemarks(field.getName());
                             break;
                         }
                     }
@@ -1586,16 +1586,14 @@ public class DataSetTableService {
         if (CollectionUtils.isNotEmpty(fields)) {
             List<String> originNameList = new ArrayList<>();
             for (int i = 0; i < fields.size(); i++) {
-                TableFiled filed = fields.get(i);
+                TableField filed = fields.get(i);
                 originNameList.add(filed.getFieldName());
                 DatasetTableField datasetTableField = DatasetTableField.builder().build();
                 // 物理字段名设定为唯一，查询当前数据集下是否已存在该字段，存在则update，不存在则insert
                 DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample();
                 // 字段名一致，认为字段没有改变
-                datasetTableFieldExample.createCriteria().andTableIdEqualTo(datasetTable.getId())
+                datasetTableFieldExample.createCriteria().andTableIdEqualTo(datasetTable.getId()).andOriginNameEqualTo(filed.getFieldName());
-                        .andOriginNameEqualTo(filed.getFieldName());
+                List<DatasetTableField> datasetTableFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample);
-                List<DatasetTableField> datasetTableFields = datasetTableFieldMapper
-                        .selectByExample(datasetTableFieldExample);
                 if (CollectionUtils.isNotEmpty(datasetTableFields)) {
                     datasetTableField.setId(datasetTableFields.get(0).getId());
                     datasetTableField.setOriginName(filed.getFieldName());
@@ -1636,8 +1634,7 @@ public class DataSetTableService {
             }
             // delete 数据库中多余的字段
             DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample();
-            datasetTableFieldExample.createCriteria().andTableIdEqualTo(datasetTable.getId()).andExtFieldEqualTo(0)
+            datasetTableFieldExample.createCriteria().andTableIdEqualTo(datasetTable.getId()).andExtFieldEqualTo(0).andOriginNameNotIn(originNameList);
-                    .andOriginNameNotIn(originNameList);
             datasetTableFieldMapper.deleteByExample(datasetTableFieldExample);
         }
     }
@@ -1736,7 +1733,7 @@ public class DataSetTableService {
             datasourceRequest.setQuery(qp.wrapSql(sql));
             List<String> sqlFileds = new ArrayList<>();
             try {
-                datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName)
+                datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableField::getFieldName)
                         .forEach(filed -> {
                             sqlFileds.add(filed);
                         });
@@ -1756,7 +1753,7 @@ public class DataSetTableService {
             datasourceRequest.setQuery(qp.wrapSql(sql));
             List<String> sqlFileds = new ArrayList<>();
             try {
-                datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName)
+                datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableField::getFieldName)
                         .forEach(filed -> sqlFileds.add(filed));
             } catch (Exception e) {
                 DataEaseException.throwException(Translator.get("i18n_check_sql_error") + e.getMessage());
@@ -1820,8 +1817,8 @@ public class DataSetTableService {
             List<String> oldFields = datasetTableFields.stream().map(DatasetTableField::getOriginName)
                     .collect(Collectors.toList());
             for (ExcelSheetData excelSheetData : excelSheetDataList) {
-                List<TableFiled> fields = excelSheetData.getFields();
+                List<TableField> fields = excelSheetData.getFields();
-                List<String> newFields = fields.stream().map(TableFiled::getRemarks).collect(Collectors.toList());
+                List<String> newFields = fields.stream().map(TableField::getRemarks).collect(Collectors.toList());
                 if (oldFields.equals(newFields)) {
                     retrunSheetDataList.add(excelSheetData);
                 }
@@ -1879,7 +1876,7 @@ public class DataSetTableService {
         inputStream.close();
         excelSheetDataList.forEach(excelSheetData -> {
             List<List<String>> data = excelSheetData.getData();
-            String[] fieldArray = excelSheetData.getFields().stream().map(TableFiled::getFieldName)
+            String[] fieldArray = excelSheetData.getFields().stream().map(TableField::getFieldName)
                     .toArray(String[]::new);
             List<Map<String, Object>> jsonArray = new ArrayList<>();
             if (CollectionUtils.isNotEmpty(data)) {
@@ -1901,7 +1898,7 @@ public class DataSetTableService {
     private Map<String, Object> parseExcel(String filename, InputStream inputStream, boolean isPreview)
             throws Exception {
         String suffix = filename.substring(filename.lastIndexOf(".") + 1);
-        List<TableFiled> fields = new ArrayList<>();
+        List<TableField> fields = new ArrayList<>();
         List<String[]> data = new ArrayList<>();
         List<Map<String, Object>> jsonArray = new ArrayList<>();
         List<String> sheets = new ArrayList<>();
@@ -1933,16 +1930,16 @@ public class DataSetTableService {
                 String[] r = new String[columnNum];
                 for (int j = 0; j < columnNum; j++) {
                     if (i == 0) {
-                        TableFiled tableFiled = new TableFiled();
+                        TableField tableField = new TableField();
-                        tableFiled.setFieldType("TEXT");
+                        tableField.setFieldType("TEXT");
-                        tableFiled.setFieldSize(1024);
+                        tableField.setFieldSize(1024);
                         String columnName = readCell(row.getCell(j), false, null);
                         if (StringUtils.isEmpty(columnName)) {
                             columnName = "NONE_" + String.valueOf(j);
                         }
-                        tableFiled.setFieldName(columnName);
+                        tableField.setFieldName(columnName);
-                        tableFiled.setRemarks(columnName);
+                        tableField.setRemarks(columnName);
-                        fields.add(tableFiled);
+                        fields.add(tableField);
                     } else {
                         if (row == null) {
                             break;
@@ -1981,17 +1978,17 @@ public class DataSetTableService {
                 String[] r = new String[columnNum];
                 for (int j = 0; j < columnNum; j++) {
                     if (i == 0) {
-                        TableFiled tableFiled = new TableFiled();
+                        TableField tableField = new TableField();
-                        tableFiled.setFieldType("TEXT");
+                        tableField.setFieldType("TEXT");
-                        tableFiled.setFieldSize(1024);
+                        tableField.setFieldSize(1024);
                         String columnName = readCell(row.getCell(j), false, null);
                         if (StringUtils.isEmpty(columnName)) {
                             columnName = "NONE_" + String.valueOf(j);
                         }
 
-                        tableFiled.setFieldName(columnName);
+                        tableField.setFieldName(columnName);
-                        tableFiled.setRemarks(columnName);
+                        tableField.setRemarks(columnName);
-                        fields.add(tableFiled);
+                        fields.add(tableField);
                     } else {
                         if (row == null) {
                             break;
@@ -2008,11 +2005,11 @@ public class DataSetTableService {
             String s = reader.readLine();// first line
             String[] split = s.split(",");
             for (String s1 : split) {
-                TableFiled tableFiled = new TableFiled();
+                TableField tableField = new TableField();
-                tableFiled.setFieldName(s1);
+                tableField.setFieldName(s1);
-                tableFiled.setRemarks(s1);
+                tableField.setRemarks(s1);
-                tableFiled.setFieldType("TEXT");
+                tableField.setFieldType("TEXT");
-                fields.add(tableFiled);
+                fields.add(tableField);
             }
             int num = 1;
             String line;
@@ -2027,7 +2024,7 @@ public class DataSetTableService {
             }
         }
 
-        String[] fieldArray = fields.stream().map(TableFiled::getFieldName).toArray(String[]::new);
+        String[] fieldArray = fields.stream().map(TableField::getFieldName).toArray(String[]::new);
 
         // 校验excel字段是否重名
         if (checkIsRepeat(fieldArray)) {
@@ -2052,7 +2049,7 @@ public class DataSetTableService {
         return map;
     }
 
-    private String readCell(Cell cell, boolean cellType, TableFiled tableFiled) {
+    private String readCell(Cell cell, boolean cellType, TableField tableField) {
         if (cell == null) {
             return "";
         }
@@ -2065,15 +2062,15 @@ public class DataSetTableService {
                     double eps = 1e-10;
                     if (value - Math.floor(value) < eps) {
                         if (cellType) {
-                            if (StringUtils.isEmpty(tableFiled.getFieldType())
+                            if (StringUtils.isEmpty(tableField.getFieldType())
-                                    || tableFiled.getFieldType().equalsIgnoreCase("TEXT")) {
+                                    || tableField.getFieldType().equalsIgnoreCase("TEXT")) {
-                                tableFiled.setFieldType("LONG");
+                                tableField.setFieldType("LONG");
                             }
                         }
                         return value.longValue() + "";
                     } else {
                         if (cellType) {
-                            tableFiled.setFieldType("DOUBLE");
+                            tableField.setFieldType("DOUBLE");
                         }
                         NumberFormat nf = NumberFormat.getInstance();
                         nf.setGroupingUsed(false);
@@ -2086,23 +2083,23 @@ public class DataSetTableService {
             } catch (IllegalStateException e) {
                 String s = String.valueOf(cell.getRichStringCellValue());
                 if (cellType) {
-                    tableFiled.setFieldType("TEXT");
+                    tableField.setFieldType("TEXT");
-                    tableFiled.setFieldSize(65533);
+                    tableField.setFieldSize(65533);
                 }
                 return s;
             }
         }
         if (cellTypeEnum.equals(CellType.STRING)) {
             if (cellType) {
-                tableFiled.setFieldType("TEXT");
+                tableField.setFieldType("TEXT");
-                tableFiled.setFieldSize(65533);
+                tableField.setFieldSize(65533);
             }
             return cell.getStringCellValue();
         }
         if (cellTypeEnum.equals(CellType.NUMERIC)) {
             if (HSSFDateUtil.isCellDateFormatted(cell)) {
                 if (cellType) {
-                    tableFiled.setFieldType("DATETIME");
+                    tableField.setFieldType("DATETIME");
                 }
                 SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                 try {
@@ -2117,15 +2114,15 @@ public class DataSetTableService {
                     double eps = 1e-10;
                     if (value - Math.floor(value) < eps) {
                         if (cellType) {
-                            if (StringUtils.isEmpty(tableFiled.getFieldType())
+                            if (StringUtils.isEmpty(tableField.getFieldType())
-                                    || tableFiled.getFieldType().equalsIgnoreCase("TEXT")) {
+                                    || tableField.getFieldType().equalsIgnoreCase("TEXT")) {
-                                tableFiled.setFieldType("LONG");
+                                tableField.setFieldType("LONG");
                             }
                         }
                         return value.longValue() + "";
                     } else {
                         if (cellType) {
-                            tableFiled.setFieldType("DOUBLE");
+                            tableField.setFieldType("DOUBLE");
                         }
                         NumberFormat nf = NumberFormat.getInstance();
                         nf.setGroupingUsed(false);

backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java

@@ -1,5 +1,6 @@
 package io.dataease.service.dataset;
 
+import com.alibaba.fastjson.JSONObject;
 import com.google.gson.Gson;
 import io.dataease.base.domain.*;
 import io.dataease.base.mapper.DatasetTableMapper;
@@ -10,6 +11,7 @@ import io.dataease.commons.constants.*;
 import io.dataease.commons.model.AuthURD;
 import io.dataease.commons.utils.*;
 import io.dataease.commons.constants.DatasourceTypes;
+import io.dataease.controller.request.datasource.ApiDefinition;
 import io.dataease.provider.datasource.DatasourceProvider;
 import io.dataease.provider.datasource.JdbcProvider;
 import io.dataease.provider.ProviderFactory;
@@ -63,7 +65,7 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Lazy;
 import org.springframework.stereotype.Service;
 import javax.annotation.Resource;
-import java.io.File;
+import java.io.*;
 import java.net.InetAddress;
 import java.util.*;
 import java.util.stream.Collectors;
@@ -314,16 +316,13 @@ public class ExtractDataService {
                 try {
                     createDorisTable(DorisTableUtils.dorisName(datasetTableId), dorisTableColumnSql);
                     createDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId)), dorisTableColumnSql);
-                    generateTransFile("all_scope", datasetTable, datasource, datasetTableFields, null);
-                    generateJobFile("all_scope", datasetTable, datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(",")));
                     execTime = System.currentTimeMillis();
-                    extractData(datasetTable, "all_scope");
+                    extractData(datasetTable, datasource, datasetTableFields, "all_scope", null);
                     replaceTable(DorisTableUtils.dorisName(datasetTableId));
                     saveSuccessLog(datasetTableTaskLog);
                     msg = true;
                     lastExecStatus = JobStatus.Completed;
                 } catch (Exception e) {
-                    e.printStackTrace();
                     saveErrorLog(datasetTableId, taskId, e);
                     msg = false;
                     lastExecStatus = JobStatus.Error;
@@ -341,6 +340,10 @@ public class ExtractDataService {
 
             case add_scope: // 增量更新
                 try {
+                    if(datasource.getType().equalsIgnoreCase(DatasourceTypes.api.name())){
+                        extractData(datasetTable, datasource, datasetTableFields, "incremental_add", null);
+                        return;
+                    }
                     DatasetTableIncrementalConfig datasetTableIncrementalConfig = dataSetTableService.incrementalConfig(datasetTableId);
                     if (datasetTableIncrementalConfig == null || StringUtils.isEmpty(datasetTableIncrementalConfig.getTableId())) {
                         updateTableStatus(datasetTableId, datasetTable, JobStatus.Completed, null);
@@ -357,17 +360,13 @@ public class ExtractDataService {
                     if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalAdd().replace(" ", ""))) {// 增量添加
                         String sql = datasetTableIncrementalConfig.getIncrementalAdd().replace(lastUpdateTime, datasetTable.getLastUpdateTime().toString())
                                 .replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString());
-                        generateTransFile("incremental_add", datasetTable, datasource, datasetTableFields, sql);
+                        extractData(datasetTable, datasource, datasetTableFields, "incremental_add", sql);
-                        generateJobFile("incremental_add", datasetTable, fetchSqlField(sql, datasource));
-                        extractData(datasetTable, "incremental_add");
                     }
 
                     if (StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete()) && StringUtils.isNotEmpty(datasetTableIncrementalConfig.getIncrementalDelete().replace(" ", ""))) {// 增量删除
                         String sql = datasetTableIncrementalConfig.getIncrementalDelete().replace(lastUpdateTime, datasetTable.getLastUpdateTime().toString())
                                 .replace(currentUpdateTime, Long.valueOf(System.currentTimeMillis()).toString());
-                        generateTransFile("incremental_delete", datasetTable, datasource, datasetTableFields, sql);
+                        extractData(datasetTable, datasource, datasetTableFields, "incremental_delete", sql);
-                        generateJobFile("incremental_delete", datasetTable, fetchSqlField(sql, datasource));
-                        extractData(datasetTable, "incremental_delete");
                     }
                     saveSuccessLog(datasetTableTaskLog);
 
@@ -394,6 +393,97 @@ public class ExtractDataService {
 
     }
 
+    private void extractData(DatasetTable datasetTable, Datasource datasource, List<DatasetTableField> datasetTableFields, String extractType, String selectSQL) throws Exception{
+        if(datasource.getType().equalsIgnoreCase(DatasourceTypes.api.name())){
+            extractDataByDE(datasetTable, datasource, datasetTableFields, extractType);
+            return;
+        }
+        extractDataByKettle(datasetTable, datasource, datasetTableFields, extractType, selectSQL);
+    }
+
+    private void  extractDataByDE(DatasetTable datasetTable, Datasource datasource, List<DatasetTableField> datasetTableFields, String extractType)throws Exception{
+        List<ApiDefinition> lists = JSONObject.parseArray(datasource.getConfiguration(), ApiDefinition.class);
+        lists = lists.stream().filter(item -> item.getName().equalsIgnoreCase(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable())).collect(Collectors.toList());
+        if(CollectionUtils.isEmpty(lists)){
+            throw new Exception("未找到API数据表");
+        }
+        if(lists.size() > 1 ){
+            throw new Exception("存在重名的API数据表");
+        }
+        DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType());
+        DatasourceRequest datasourceRequest = new DatasourceRequest();
+        datasourceRequest.setDatasource(datasource);
+        datasourceRequest.setTable(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable());
+        Map<String, List> result  = datasourceProvider.fetchResultAndField(datasourceRequest);
+        List<String[]> dataList = result.get("dataList");
+        Datasource dorisDatasource = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
+        DorisConfiguration dorisConfiguration = new Gson().fromJson(dorisDatasource.getConfiguration(), DorisConfiguration.class);
+        String columns = datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(",")) + ",dataease_uuid";
+
+        String dataFile = null;
+        String script = null;
+        switch (extractType) {
+            case "all_scope":
+                dataFile = root_path + DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId())) + "." + extention;
+                script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTable.getId())));
+                break;
+            default:
+                dataFile = root_path + DorisTableUtils.dorisAddName(DorisTableUtils.dorisName(datasetTable.getId())) + "." + extention;
+                script = String.format(shellScript, dorisConfiguration.getUsername(), dorisConfiguration.getPassword(), System.currentTimeMillis(), separator, columns, "APPEND", dataFile, dorisConfiguration.getHost(), dorisConfiguration.getHttpPort(), dorisConfiguration.getDataBase(), DorisTableUtils.dorisName(datasetTable.getId()));
+                break;
+        }
+
+
+        BufferedWriter bw = new BufferedWriter(new FileWriter(dataFile));
+        for (String[] strings : dataList) {
+            String content = "";
+            for (int i=0;i< strings.length;i++){
+                content = content +  strings[i] + separator;
+            }
+            content = content + Md5Utils.md5(content);
+            bw.write(content);
+            bw.newLine();
+        }
+        bw.close();
+
+        File scriptFile = new File(root_path + datasetTable.getId() + ".sh");
+        scriptFile.setExecutable(true);
+        scriptFile.createNewFile();
+
+        BufferedWriter scriptFileBw = new BufferedWriter(new FileWriter(root_path + datasetTable.getId() + ".sh"));
+        scriptFileBw.write("#!/bin/sh");
+        scriptFileBw.newLine();
+        scriptFileBw.write(script);
+        scriptFileBw.newLine();
+        scriptFileBw.close();
+
+        try {
+            Process process = Runtime.getRuntime().exec(root_path + datasetTable.getId() + ".sh");
+            process.waitFor();
+            if(process.waitFor() != 0){
+                BufferedReader input = new BufferedReader(new InputStreamReader(process.getErrorStream()));
+                String errMsg = "";
+                String line = "";
+                while ((line = input.readLine()) != null) {
+                    errMsg = errMsg + line + System.getProperty("line.separator");
+                }
+                throw new Exception(errMsg);
+            }
+        }catch (Exception e){
+            throw e;
+        }finally {
+            File deleteFile = new File(root_path + datasetTable.getId() + ".sh");
+            FileUtils.forceDelete(deleteFile);
+        }
+
+    }
+
+    private void extractDataByKettle(DatasetTable datasetTable, Datasource datasource, List<DatasetTableField> datasetTableFields, String extractType, String selectSQL)throws Exception{
+        generateTransFile(extractType, datasetTable, datasource, datasetTableFields, selectSQL);
+        generateJobFile(extractType, datasetTable, datasetTableFields.stream().map(DatasetTableField::getDataeaseName).collect(Collectors.joining(",")));
+        extractData(datasetTable, extractType);
+    }
+
     private void sendWebMsg(DatasetTable datasetTable, DatasetTableTask datasetTableTask, DatasetTableTaskLog datasetTableTaskLog, Boolean status) {
         String taskId = datasetTableTask.getId();
         String msg = status ? "成功" : "失败";
@@ -729,7 +819,7 @@ public class ExtractDataService {
         datasourceRequest.setDatasource(ds);
         datasourceRequest.setQuery(qp.wrapSql(sql));
         List<String> dorisFields = new ArrayList<>();
-        datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableFiled::getFieldName).forEach(filed -> {
+        datasourceProvider.fetchResultField(datasourceRequest).stream().map(TableField::getFieldName).forEach(filed -> {
             dorisFields.add(DorisTableUtils.columnName(filed));
         });
         return String.join(",", dorisFields);

backend/src/main/java/io/dataease/service/datasource/DatasourceService.java

@@ -1,7 +1,9 @@
 package io.dataease.service.datasource;
 
+import cn.hutool.json.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.google.gson.Gson;
+import com.jayway.jsonpath.JsonPath;
 import io.dataease.base.domain.*;
 import io.dataease.base.mapper.*;
 import io.dataease.base.mapper.ext.ExtDataSourceMapper;
@@ -10,12 +12,16 @@ import io.dataease.commons.exception.DEException;
 import io.dataease.commons.model.AuthURD;
 import io.dataease.commons.utils.AuthUtils;
 import io.dataease.commons.utils.CommonThreadPool;
+import io.dataease.commons.utils.HttpClientUtil;
 import io.dataease.commons.utils.LogUtil;
 import io.dataease.controller.ResultHolder;
 import io.dataease.controller.request.DatasourceUnionRequest;
+import io.dataease.controller.request.datasource.ApiDefinition;
 import io.dataease.controller.sys.base.BaseGridRequest;
 import io.dataease.controller.sys.base.ConditionEntity;
 import io.dataease.commons.constants.DatasourceTypes;
+import io.dataease.exception.ExcelException;
+import io.dataease.provider.datasource.ApiProvider;
 import io.dataease.provider.datasource.DatasourceProvider;
 import io.dataease.provider.ProviderFactory;
 import io.dataease.controller.request.datasource.DatasourceRequest;
@@ -103,6 +109,9 @@ public class DatasourceService {
                     case ck:
                         datasourceDTO.setConfiguration(JSONObject.toJSONString(new Gson().fromJson(datasourceDTO.getConfiguration(), CHConfiguration.class)) );
                         break;
+                    case api:
+                        datasourceDTO.setApiConfiguration(JSONObject.parseArray(datasourceDTO.getConfiguration()));
+                        break;
                     default:
                         break;
                 }
@@ -260,13 +269,45 @@ public class DatasourceService {
         datasources.forEach(datasource -> checkAndUpdateDatasourceStatus(datasource, true));
     }
 
+    public ApiDefinition checkApiDatasource(ApiDefinition apiDefinition) throws Exception {
+        String response = ApiProvider.execHttpRequest(apiDefinition);
+
+        List<LinkedHashMap> datas = JsonPath.read(response,apiDefinition.getDataPath());
+        List<JSONObject> dataList = new ArrayList<>();
+        for (LinkedHashMap data : datas) {
+            JSONObject jsonObject = new JSONObject();
+            Iterator it = data.entrySet().iterator();
+            while (it.hasNext()){
+                Map.Entry entry = (Map.Entry)it.next();
+                jsonObject.put((String) entry.getKey(), entry.getValue());
+            }
+            dataList.add(jsonObject);
+        }
+        List<DatasetTableField> fields = new ArrayList<>();
+        if(CollectionUtils.isNotEmpty(dataList)){
+            for (Map.Entry<String, Object> stringObjectEntry : dataList.get(0).entrySet()) {
+                DatasetTableField tableField = new DatasetTableField();
+                tableField.setOriginName(stringObjectEntry.getKey());
+                tableField.setName(stringObjectEntry.getKey());
+                tableField.setSize(65535);
+                tableField.setDeExtractType(0);
+                tableField.setDeType(0);
+                tableField.setExtField(0);
+                fields.add(tableField);
+            }
+        }
+        apiDefinition.setDatas(dataList);
+        apiDefinition.setFields(fields);
+        return apiDefinition;
+    }
+
     private void checkAndUpdateDatasourceStatus(Datasource datasource){
         try {
             DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType());
             DatasourceRequest datasourceRequest = new DatasourceRequest();
             datasourceRequest.setDatasource(datasource);
-            datasourceProvider.checkStatus(datasourceRequest);
+            String status = datasourceProvider.checkStatus(datasourceRequest);
-            datasource.setStatus("Success");
+            datasource.setStatus(status);
         } catch (Exception e) {
             datasource.setStatus("Error");
         }
@@ -277,8 +318,8 @@ public class DatasourceService {
             DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType());
             DatasourceRequest datasourceRequest = new DatasourceRequest();
             datasourceRequest.setDatasource(datasource);
-            datasourceProvider.checkStatus(datasourceRequest);
+            String status = datasourceProvider.checkStatus(datasourceRequest);
-            datasource.setStatus("Success");
+            datasource.setStatus(status);
             datasourceMapper.updateByPrimaryKeySelective(datasource);
         } catch (Exception e) {
             Datasource temp = datasourceMapper.selectByPrimaryKey(datasource.getId());

frontend/package.json

@@ -33,6 +33,13 @@
     "jsencrypt": "^3.0.0-rc.1",
     "jspdf": "^2.3.1",
     "lodash": "^4.17.4",
+    "lodash.isboolean": "^3.0.3",
+    "lodash.isempty": "^4.4.0",
+    "lodash.isinteger": "^4.0.4",
+    "lodash.isnull": "^3.0.0",
+    "lodash.isnumber": "^3.0.3",
+    "lodash.isobject": "^3.0.2",
+    "lodash.isstring": "^4.0.1",
     "normalize.css": "7.0.0",
     "nprogress": "0.2.0",
     "screenfull": "4.2.0",
@@ -53,6 +60,7 @@
     "vue-to-pdf": "^1.0.0",
     "vue-uuid": "2.0.2",
     "vue-video-player": "^5.0.2",
+    "vue2-ace-editor": "0.0.15",
     "vuedraggable": "^2.24.3",
     "vuex": "3.1.0",
     "webpack": "^4.46.0",

frontend/src/api/dataset/dataset.js

@@ -77,6 +77,14 @@ export function listDatasource() {
   })
 }
 
+export function listApiDatasource() {
+  return request({
+    url: '/datasource/list/api',
+    loading: true,
+    method: 'get'
+  })
+}
+
 export function getTable(id, hideMsg = false) {
   return request({
     url: '/dataset/table/get/' + id,

frontend/src/api/system/datasource.js

@@ -74,4 +74,13 @@ export function getSchema(data) {
   })
 }
 
+export function checkApiDatasource(data){
+  return request({
+    url: 'datasource/checkApiDatasource',
+    method: 'post',
+    loading: false,
+    data
+  })
+}
+
 export default { dsGrid, addDs, editDs, delDs, validateDs, listDatasource, getSchema }

frontend/src/icons/svg/exclamationmark2.svg

@@ -0,0 +1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1629085981011" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="16882" xmlns:xlink="http://www.w3.org/1999/xlink" width="200" height="200"><defs><style type="text/css"></style></defs><path d="M637.6 82.3L610.9 643c-1.6 34.1-29.8 61-63.9 61h-70c-34.2 0-62.3-26.8-63.9-61L386.4 82.3c-1.3-27.4 20.5-50.3 48-50.3h155.3c27.4 0 49.2 22.9 47.9 50.3zM640 864c0 35.3-14.3 67.3-37.5 90.5-23.2 23.2-55.2 37.5-90.5 37.5s-67.3-14.3-90.5-37.5C398.3 931.3 384 899.3 384 864c0-70.7 57.3-128 128-128 35.3 0 67.3 14.3 90.5 37.5 23.2 23.2 37.5 55.2 37.5 90.5z" p-id="16883" fill="#FFA500"></path></svg>

frontend/src/lang/en.js

@@ -1049,6 +1049,7 @@ export default {
     custom_data: 'Custom Dataset',
     pls_slc_tbl_left: 'Please select the chart from the left',
     add_db_table: 'Add Database Dataset',
+    add_api_table: 'Add API Dataset',
     pls_slc_data_source: 'Please select data source',
     table: 'Table',
     edit: 'Edit',
@@ -1298,7 +1299,35 @@ export default {
     no_less_then_0: 'Parameters in advanced settings cannot be less than zero',
     port_no_less_then_0: 'Port cannot be less than zero',
     priority: 'Advanced setting',
-    extra_params: 'Extra JDBC connection string'
+    extra_params: 'Extra JDBC connection string',
+    please_input_dataPath: '请输入 JsonPath 数据路径',
+    warning: 'Contains invalid datasets',
+    data_table: 'Dataset Table',
+    data_table_name: 'Dataset Table name',
+    method: 'Request mode',
+    url: 'URL',
+    add_api_table: 'Add API table',
+    edit_api_table: 'Edit API table',
+    base_info: 'Basic information',
+    request: 'Request',
+    path_all_info: 'Please fill in the full address',
+    req_param: 'Request parameters',
+    headers: 'Request header',
+    key: 'Key',
+    value: 'Value',
+    data_path: 'Extract data',
+    data_path_desc: 'Please fill in the data path with Jsonpath',
+    body_form_data: 'form-data',
+    body_x_www_from_urlencoded: 'x-www-form-urlencoded',
+    body_json: 'json',
+    body_xml: 'xml',
+    body_raw: 'row',
+    request_body: 'Request Body',
+    auth_config: 'Auth config',
+    auth_config_info: 'Permission verification is required for the request',
+    verified: 'Verified',
+    verification_method: 'Verification Method',
+    username: 'Username'
   },
   pblink: {
     key_pwd: 'Please enter the password to open the link',

frontend/src/lang/tw.js

@@ -1049,6 +1049,7 @@ export default {
     custom_data: '自定義數據集',
     pls_slc_tbl_left: '請從左側選擇錶',
     add_db_table: '添加數據庫數據集',
+    add_api_table: '添加API數據集',
     pls_slc_data_source: '請選擇數據源',
     table: '錶',
     edit: '編輯',
@@ -1299,7 +1300,35 @@ export default {
     no_less_then_0: '高級設置中的參數不能小於零',
     port_no_less_then_0: '端口不能小於零',
     priority: '高級設置',
-    extra_params: '額外的JDBC連接字符串'
+    extra_params: '額外的JDBC連接字符串',
+    please_input_dataPath: '請輸入 JsonPath 數據路徑',
+    warning: '包含無效數據機',
+    data_table: '數據表',
+    data_table_name: '數據表名稱',
+    method: '請求方式',
+    url: 'URL',
+    add_api_table: '添加 API 數據表',
+    edit_api_table: '編輯 API 數據表',
+    base_info: '基礎信息',
+    request: '請求',
+    path_all_info: '請輸入完整地址',
+    req_param: '請求參數',
+    headers: '請求頭',
+    key: '鍵',
+    value: '值',
+    data_path: '提取數據',
+    data_path_desc: '請用 JsonPath 填寫數據路徑',
+    body_form_data: 'form-data',
+    body_x_www_from_urlencoded: 'x-www-form-urlencoded',
+    body_json: 'json',
+    body_xml: 'xml',
+    body_raw: 'row',
+    request_body: '請求提',
+    auth_config: '認證配置',
+    auth_config_info: '請求需要進行權限校驗',
+    verified: '認證',
+    verification_method: '認證方式',
+    username: '用戶名'
   },
   pblink: {
     key_pwd: '請輸入密碼打開鏈接',

frontend/src/lang/zh.js

@@ -678,6 +678,7 @@ export default {
     custom_data: '自定义数据集',
     pls_slc_tbl_left: '请从左侧选视图',
     add_db_table: '添加数据库数据集',
+    add_api_table: '添加API数据集',
     pls_slc_data_source: '请选择数据源',
     table: '表',
     edit: '编辑',
@@ -1307,7 +1308,35 @@ export default {
     direct: '直连模式',
     extract: '抽取模式',
     all_compute_mode: '直连、抽取模式',
-    extra_params: '额外的JDBC连接字符串'
+    extra_params: '额外的JDBC连接字符串',
+    please_input_dataPath: '请输入 JsonPath 数据路径',
+    warning: '包含无效数据集',
+    data_table: '数据表',
+    data_table_name: '数据表名称',
+    method: '请求方式',
+    url: 'URL',
+    add_api_table: '添加API数据表',
+    edit_api_table: '编辑API数据表',
+    base_info: '基础信息',
+    request: '请求',
+    path_all_info: '请填入完整地址',
+    req_param: '请求参数',
+    headers: '请求头',
+    key: '键',
+    value: '值',
+    data_path: '提取数据',
+    data_path_desc: '请用JsonPath填写数据路径',
+    body_form_data: 'form-data',
+    body_x_www_from_urlencoded: 'x-www-form-urlencoded',
+    body_json: 'json',
+    body_xml: 'xml',
+    body_raw: 'row',
+    request_body: '请求体',
+    auth_config: '认证配置',
+    auth_config_info: '请求需要进行权限校验',
+    verified: '认证',
+    verification_method: '认证方式',
+    username: '用户名'
   },
   pblink: {
     key_pwd: '请输入密码打开链接',

frontend/src/views/dataset/add/AddApi.vue

@@ -0,0 +1,209 @@
+<template>
+  <el-row style="display: flex;flex-direction: column;height: 100%">
+    <el-row style="height: 26px;" class="title-text">
+      <span style="line-height: 26px;">
+        {{ $t('dataset.add_api_table') }}
+      </span>
+      <el-row style="float: right">
+        <el-button size="mini" @click="cancel">
+          {{ $t('dataset.cancel') }}
+        </el-button>
+        <el-button size="mini" type="primary" :disabled="checkTableList.length < 1" @click="save">
+          {{ $t('dataset.confirm') }}
+        </el-button>
+      </el-row>
+    </el-row>
+    <el-divider />
+    <el-row>
+      <el-form :inline="true">
+        <el-form-item class="form-item">
+          <el-select v-model="dataSource" filterable :placeholder="$t('dataset.pls_slc_data_source')" size="mini">
+            <el-option
+              v-for="item in options"
+              :key="item.id"
+              :label="item.name"
+              :value="item.id"
+            />
+          </el-select>
+        </el-form-item>
+        <el-form-item class="form-item">
+          <el-select v-model="mode" filterable :placeholder="$t('dataset.connect_mode')" size="mini">
+            <el-option :label="$t('dataset.sync_data')" value="1" :disabled="!kettleRunning || selectedDatasource.type==='es' || selectedDatasource.type==='ck' || selectedDatasource.type==='mongo' || selectedDatasource.type==='redshift' || selectedDatasource.type==='hive'" />
+          </el-select>
+        </el-form-item>
+
+        <el-form-item v-if="mode === '1'" class="form-item">
+          <el-select v-model="syncType" filterable :placeholder="$t('dataset.connect_mode')" size="mini">
+            <el-option :label="$t('dataset.sync_now')" value="sync_now" />
+            <el-option :label="$t('dataset.sync_latter')" value="sync_latter" />
+          </el-select>
+        </el-form-item>
+
+        <el-form-item class="form-item" style="float: right;">
+          <el-input
+            v-model="searchTable"
+            size="mini"
+            :placeholder="$t('dataset.search')"
+            prefix-icon="el-icon-search"
+            clearable
+          />
+        </el-form-item>
+      </el-form>
+    </el-row>
+    <el-col style="overflow-y: auto;">
+      <el-checkbox-group v-model="checkTableList" size="small">
+        <el-tooltip v-for="t in tableData" :key="t.name" :disabled="t.enableCheck" effect="dark" :content="$t('dataset.table_already_add_to')+': '+t.datasetPath" placement="bottom">
+          <el-checkbox
+            border
+            :label="t.name"
+            :disabled="!t.enableCheck"
+          >{{ showTableNameWithComment(t) }}</el-checkbox>
+        </el-tooltip>
+      </el-checkbox-group>
+    </el-col>
+  </el-row>
+</template>
+
+<script>
+import { listApiDatasource, post, isKettleRunning } from '@/api/dataset/dataset'
+
+export default {
+  name: 'AddApi',
+  props: {
+    param: {
+      type: Object,
+      default: null
+    }
+  },
+  data() {
+    return {
+      searchTable: '',
+      options: [],
+      dataSource: '',
+      tables: [],
+      checkTableList: [],
+      mode: '1',
+      syncType: 'sync_now',
+      tableData: [],
+      kettleRunning: false,
+      selectedDatasource: {}
+    }
+  },
+  watch: {
+    dataSource(val) {
+      if (val) {
+        post('/datasource/getTables', { id: val }).then(response => {
+          this.tables = response.data
+          this.tableData = JSON.parse(JSON.stringify(this.tables))
+        })
+        for (let i = 0; i < this.options.length; i++) {
+          if (this.options[i].id === val) {
+            this.selectedDatasource = this.options[i]
+          }
+        }
+      }
+    },
+    searchTable(val) {
+      if (val && val !== '') {
+        this.tableData = JSON.parse(JSON.stringify(this.tables.filter(ele => { return ele.name.toLocaleLowerCase().includes(val.toLocaleLowerCase()) })))
+      } else {
+        this.tableData = JSON.parse(JSON.stringify(this.tables))
+      }
+    }
+  },
+  mounted() {
+    this.initDataSource()
+  },
+  activated() {
+    this.initDataSource()
+  },
+  created() {
+    this.kettleState()
+  },
+  methods: {
+    initDataSource() {
+      listApiDatasource().then(response => {
+        this.options = response.data
+      })
+    },
+    kettleState() {
+      isKettleRunning().then(res => {
+        this.kettleRunning = res.data
+      })
+    },
+    showTableNameWithComment(t) {
+      if (t.remark) {
+        return `${t.name}(${t.remark})`
+      } else {
+        return `${t.name}`
+      }
+    },
+    save() {
+      let ds = {}
+      this.options.forEach(ele => {
+        if (ele.id === this.dataSource) {
+          ds = ele
+        }
+      })
+      const sceneId = this.param.id
+      const dataSourceId = this.dataSource
+      const tables = []
+      const mode = this.mode
+      const syncType = this.syncType
+      this.checkTableList.forEach(function(name) {
+        tables.push({
+          name: ds.name + '_' + name,
+          sceneId: sceneId,
+          dataSourceId: dataSourceId,
+          type: 'api',
+          syncType: syncType,
+          mode: parseInt(mode),
+          info: JSON.stringify({ table: name })
+        })
+      })
+      post('/dataset/table/batchAdd', tables).then(response => {
+        this.$emit('saveSuccess', tables[0])
+        this.cancel()
+      })
+    },
+
+    cancel() {
+      this.dataReset()
+      this.$emit('switchComponent', { name: '' })
+    },
+
+    dataReset() {
+      this.searchTable = ''
+      this.options = []
+      this.dataSource = ''
+      this.tables = []
+      this.checkTableList = []
+    }
+  }
+
+}
+</script>
+
+<style scoped>
+  .el-divider--horizontal {
+    margin: 12px 0;
+  }
+
+  .form-item {
+    margin-bottom: 6px;
+  }
+
+  .el-checkbox {
+    margin-bottom: 14px;
+    margin-left: 0;
+    margin-right: 14px;
+  }
+
+  .el-checkbox.is-bordered + .el-checkbox.is-bordered {
+    margin-left: 0;
+  }
+
+  span{
+    font-size: 14px;
+  }
+</style>

frontend/src/views/dataset/data/UpdateInfo.vue

@@ -263,7 +263,7 @@
 
       <el-divider />
 
-      <el-row style="height: 26px;">
+      <el-row style="height: 26px;" v-if="table.type !== 'api'">
         <el-row>
           <el-col :span="4"><span>{{ $t('dataset.incremental_update_type') }}:</span></el-col>
           <el-col :span="18">
@@ -275,7 +275,7 @@
         </el-row>
       </el-row>
 
-      <el-row style="height: 26px;">
+      <el-row style="height: 26px;" v-if="table.type !== 'api'">
         <el-row>
           <el-col :span="4" style="height: 26px;"><span style="display: inline-block;height: 26px;line-height: 26px;">{{ $t('dataset.param') }}:</span></el-col>
           <el-col :span="18">
@@ -285,7 +285,7 @@
         </el-row>
       </el-row>
 
-      <el-row>
+      <el-row v-if="table.type !== 'api'">
         <el-col style="min-width: 200px;">
           <codemirror
             ref="myCm"

frontend/src/views/dataset/data/ViewTable.vue

@@ -52,7 +52,7 @@
       <el-tab-pane v-if="!hideCustomDs && table.type !== 'union' && table.type !== 'custom' && !(table.type === 'sql' && table.mode === 0)" :label="$t('dataset.join_view')" name="joinView">
         <union-view :param="param" :table="table" />
       </el-tab-pane>
-      <el-tab-pane v-if="table.mode === 1 && (table.type === 'excel' || table.type === 'db' || table.type === 'sql')" :label="$t('dataset.update_info')" name="updateInfo">
+      <el-tab-pane v-if="table.mode === 1 && (table.type === 'excel' || table.type === 'db' || table.type === 'sql' || table.type === 'api')" :label="$t('dataset.update_info')" name="updateInfo">
         <update-info v-if="tabActive=='updateInfo'" :param="param" :table="table" />
       </el-tab-pane>
       <el-tab-pane v-if="isPluginLoaded && hasDataPermission('manage',param.privileges)" :lazy="true" :label="$t('dataset.row_permissions')" name="rowPermissions">

frontend/src/views/dataset/group/Group.vue

@@ -99,6 +99,10 @@
                               <svg-icon icon-class="ds-union" class="ds-icon-union" />
                               {{ $t('dataset.union_data') }}
                             </el-dropdown-item>
+                            <el-dropdown-item :command="beforeClickAddData('api',data)">
+                              <svg-icon icon-class="ds-union" class="ds-icon-union" />
+                              {{ $t('dataset.union_data') }}
+                            </el-dropdown-item>
                           </el-dropdown-menu>
                         </el-dropdown>
                       </el-dropdown-item>
@@ -562,6 +566,9 @@ export default {
         case 'union':
           this.addData('AddUnion')
           break
+        case 'api':
+          this.addData('AddApi')
+          break
       }
     },
 

frontend/src/views/dataset/index.vue

@@ -20,6 +20,7 @@ import Group from './group/Group'
 import DataHome from './data/DataHome'
 import ViewTable from './data/ViewTable'
 import AddDB from './add/AddDB'
+import AddApi from './add/AddApi'
 import AddSQL from './add/AddSQL'
 import AddExcel from './add/AddExcel'
 import AddCustom from './add/AddCustom'
@@ -29,7 +30,7 @@ import { removeClass } from '@/utils'
 import { checkCustomDs } from '@/api/dataset/dataset'
 export default {
   name: 'DataSet',
-  components: { DeMainContainer, DeContainer, DeAsideContainer, Group, DataHome, ViewTable, AddDB, AddSQL, AddExcel, AddCustom },
+  components: { DeMainContainer, DeContainer, DeAsideContainer, Group, DataHome, ViewTable, AddDB, AddSQL, AddExcel, AddCustom, AddApi},
   data() {
     return {
       component: DataHome,
@@ -76,6 +77,9 @@ export default {
         case 'FieldEdit':
           this.component = FieldEdit
           break
+        case 'AddApi':
+          this.component = AddApi
+          break
         default:
           this.component = DataHome
           break

frontend/src/views/system/datasource/ApiAuthConfig.vue

@@ -0,0 +1,91 @@
+<template xmlns:v-slot="http://www.w3.org/1999/XSL/Transform">
+  <el-tabs v-model="activeName">
+    <!-- 认证-->
+    <el-tab-pane :label="$t('datasource.verified')" name="verified">
+
+      <el-form :model="authConfig" :rules="rule" ref="authConfig" label-position="right">
+        <el-form-item :label="$t('datasource.verification_method')" prop="verification">
+          <el-select v-model="authConfig.verification" @change="change"
+                     :placeholder="$t('datasource.verification_method')" filterable size="small">
+            <el-option
+                v-for="item in options"
+                :key="item.name"
+                :label="item.name"
+                :value="item.name">
+            </el-option>
+          </el-select>
+
+        </el-form-item>
+
+        <el-form-item :label="$t('datasource.username')" prop="username"
+                      v-if="authConfig.verification!=undefined && authConfig.verification !='No Auth'">
+          <el-input :placeholder="$t('datasource.username')" v-model="authConfig.username"
+                    class="ms-http-input" size="small">
+          </el-input>
+        </el-form-item>
+
+        <el-form-item :label="$t('datasource.password')" prop="password"
+                      v-if=" authConfig.verification!=undefined && authConfig.verification !='No Auth'">
+          <el-input v-model="authConfig.password" :placeholder="$t('datasource.password')" show-password autocomplete="off"
+                    maxlength="50" show-word-limit/>
+        </el-form-item>
+
+      </el-form>
+
+    </el-tab-pane>
+  </el-tabs>
+</template>
+
+<script>
+
+export default {
+  name: "ApiAuthConfig",
+  components: {},
+  props: {
+    request: {},
+    encryptShow: {
+      type: Boolean,
+      default: true,
+    }
+  },
+  watch: {
+    request() {
+      this.initData();
+    }
+  },
+  created() {
+    this.initData();
+  },
+  data() {
+    return {
+      options: [{name: "No Auth"}, {name: "Basic Auth"}],
+      encryptOptions: [{id: false, name: this.$t('commons.encrypted')}],
+      activeName: "verified",
+      rule: {},
+      authConfig: {}
+    }
+  },
+  methods: {
+    change() {
+      if (this.authConfig.verification === "Basic Auth") {
+        this.authConfig.verification = "Basic Auth";
+        this.request.authManager = this.authConfig;
+      } else {
+        this.authConfig.verification = "No Auth";
+        this.request.authManager = this.authConfig;
+      }
+    },
+    initData() {
+      if (this.request.authManager) {
+        this.authConfig = this.request.authManager;
+      }
+    }
+  }
+}
+</script>
+
+<style scoped>
+/deep/ .el-tabs__nav-wrap::after {
+  height: 0px;
+}
+</style>

frontend/src/views/system/datasource/ApiBody.vue

@@ -0,0 +1,331 @@
+<template>
+  <div>
+    <el-radio-group v-model="body.type" size="mini">
+      <el-radio :disabled="isReadOnly" :label="type.FORM_DATA" @change="modeChange">
+        {{ $t('datasource.body_form_data') }}
+      </el-radio>
+
+      <el-radio :disabled="isReadOnly" :label="type.WWW_FORM" @change="modeChange">
+        {{ $t('datasource.body_x_www_from_urlencoded') }}
+      </el-radio>
+
+      <el-radio :disabled="isReadOnly" :label="type.JSON" @change="modeChange">
+        {{ $t('datasource.body_json') }}
+      </el-radio>
+
+      <el-radio :disabled="isReadOnly" :label="type.XML" @change="modeChange">
+        {{ $t('datasource.body_xml') }}
+      </el-radio>
+
+      <el-radio :disabled="isReadOnly" :label="type.RAW" @change="modeChange">
+        {{ $t('datasource.body_raw') }}
+      </el-radio>
+
+    </el-radio-group>
+    <div style="min-width: 1200px;" v-if="body.type == 'Form_Data' || body.type == 'WWW_FORM'">
+      <api-variable
+        :with-mor-setting="true"
+        :is-read-only="isReadOnly"
+        :parameters="body.kvs"
+        :isShowEnable="isShowEnable"
+        type="body"/>
+    </div>
+    <div v-if="body.type == 'JSON'">
+      <code-edit
+        :read-only="isReadOnly"
+        :data.sync="body.raw"
+        :modes="modes"
+        :mode="'json'"
+        height="400px"
+        ref="codeEdit"/>
+    </div>
+
+    <div class="ms-body" v-if="body.type == 'XML'">
+      <code-edit
+        :read-only="isReadOnly"
+        :data.sync="body.raw"
+        :modes="modes"
+        :mode="'text'"
+        ref="codeEdit"/>
+    </div>
+
+    <div class="ms-body" v-if="body.type == 'Raw'">
+      <code-edit
+        :read-only="isReadOnly"
+        :data.sync="body.raw"
+        :modes="modes"
+        ref="codeEdit"/>
+    </div>
+  </div>
+</template>
+
+<script>
+import ApiKeyValue from "./ApiKeyValue";
+import {BODY_TYPE, KeyValue} from "./ApiTestModel";
+import CodeEdit from "./CodeEdit";
+import ApiVariable from "./ApiVariable";
+import Convert from "./convert";
+
+
+export default {
+  name: "ApiBody",
+  components: {
+    ApiVariable,
+    CodeEdit,
+    ApiKeyValue
+  },
+  props: {
+    body: {},
+    headers: Array,
+    isReadOnly: {
+      type: Boolean,
+      default: false
+    },
+    isShowEnable: {
+      type: Boolean,
+      default: false
+    }
+  },
+  data() {
+    return {
+      type: BODY_TYPE,
+      modes: ['text', 'json', 'xml', 'html'],
+      jsonSchema: "JSON",
+      codeEditActive: true,
+      hasOwnProperty: Object.prototype.hasOwnProperty,
+      propIsEnumerable: Object.prototype.propertyIsEnumerable
+    };
+  },
+
+  watch: {
+    'body.typeChange'() {
+      this.reloadCodeEdit();
+    },
+    'body.raw'() {
+      if (this.body.format !== 'JSON-SCHEMA' && this.body.raw) {
+        try {
+          const MsConvert = new Convert();
+          let data = MsConvert.format(JSON.parse(this.body.raw));
+          if (this.body.jsonSchema) {
+            this.body.jsonSchema = this.deepAssign(data);
+          } else {
+            this.body.jsonSchema = data;
+          }
+        } catch (ex) {
+          this.body.jsonSchema = "";
+        }
+      }
+    },
+  },
+  methods: {
+    isObj(x) {
+      let type = typeof x;
+      return x !== null && (type === 'object' || type === 'function');
+    },
+
+    toObject(val) {
+      if (val === null || val === undefined) {
+        return;
+      }
+      return Object(val);
+    },
+
+    assignKey(to, from, key) {
+      let val = from[key];
+      if (val === undefined || val === null) {
+        return;
+      }
+
+      if (!this.hasOwnProperty.call(to, key) || !this.isObj(val)) {
+        to[key] = val;
+      } else {
+        to[key] = this.assign(Object(to[key]), from[key]);
+      }
+    },
+
+    assign(to, from) {
+      if (to === from) {
+        return to;
+      }
+      from = Object(from);
+      for (let key in from) {
+        if (this.hasOwnProperty.call(from, key)) {
+          this.assignKey(to, from, key);
+        }
+      }
+
+      if (Object.getOwnPropertySymbols) {
+        let symbols = Object.getOwnPropertySymbols(from);
+
+        for (let i = 0; i < symbols.length; i++) {
+          if (this.propIsEnumerable.call(from, symbols[i])) {
+            this.assignKey(to, from, symbols[i]);
+          }
+        }
+      }
+
+      return to;
+    },
+
+    deepAssign(target) {
+      target = this.toObject(target);
+      for (let s = 1; s < arguments.length; s++) {
+        this.assign(target, arguments[s]);
+      }
+      return target;
+    },
+    reloadCodeEdit() {
+      this.codeEditActive = false;
+      this.$nextTick(() => {
+        this.codeEditActive = true;
+      });
+    },
+    formatChange() {
+      const MsConvert = new Convert();
+
+      if (this.body.format === 'JSON-SCHEMA') {
+        if (this.body.raw && !this.body.jsonSchema) {
+          this.body.jsonSchema = MsConvert.format(JSON.parse(this.body.raw));
+        }
+      } else {
+        if (this.body.jsonSchema) {
+          MsConvert.schemaToJsonStr(this.body.jsonSchema, (result) => {
+            this.$set(this.body, 'raw', result);
+            this.reloadCodeEdit();
+          });
+        }
+      }
+    },
+    modeChange(mode) {
+      switch (this.body.type) {
+        case "JSON":
+          this.setContentType("application/json");
+          break;
+        case "XML":
+          this.setContentType("text/xml");
+          break;
+        case "WWW_FORM":
+          this.setContentType("application/x-www-form-urlencoded");
+          break;
+        // todo from data
+        case "BINARY":
+          this.setContentType("application/octet-stream");
+          break;
+        default:
+          this.removeContentType();
+          break;
+      }
+    },
+    setContentType(value) {
+      let isType = false;
+      this.headers.forEach(item => {
+        if (item.name === "Content-Type") {
+          item.value = value;
+          isType = true;
+        }
+      })
+      if (!isType) {
+        this.headers.unshift(new KeyValue({name: "Content-Type", value: value}));
+        this.$emit('headersChange');
+      }
+    },
+    removeContentType() {
+      for (let index in this.headers) {
+        if (this.headers[index].name === "Content-Type") {
+          this.headers.splice(index, 1);
+          this.$emit('headersChange');
+          return;
+        }
+      }
+    },
+    batchAdd() {
+      this.$refs.batchAddParameter.open();
+    },
+    format(array, obj) {
+      if (array) {
+        let isAdd = true;
+        for (let i in array) {
+          let item = array[i];
+          if (item.name === obj.name) {
+            item.value = obj.value;
+            isAdd = false;
+          }
+        }
+        if (isAdd) {
+          this.body.kvs.unshift(obj);
+        }
+      }
+    },
+    batchSave(data) {
+      if (data) {
+        let params = data.split("\n");
+        let keyValues = [];
+        params.forEach(item => {
+          let line = [];
+          line[0] = item.substring(0, item.indexOf(":"));
+          line[1] = item.substring(item.indexOf(":") + 1, item.length);
+          let required = false;
+          keyValues.unshift(new KeyValue({
+            name: line[0],
+            required: required,
+            value: line[1],
+            description: line[2],
+            type: "text",
+            valid: false,
+            file: false,
+            encode: true,
+            enable: true,
+            contentType: "text/plain"
+          }));
+        })
+        keyValues.forEach(item => {
+          this.format(this.body.kvs, item);
+        })
+      }
+    },
+  },
+  created() {
+    if (!this.body.type) {
+      this.body.type = BODY_TYPE.FORM_DATA;
+    }
+    if (this.body.kvs) {
+      this.body.kvs.forEach(param => {
+        if (!param.type) {
+          param.type = 'text';
+        }
+      });
+    }
+
+  }
+}
+</script>
+
+<style scoped>
+.textarea {
+  margin-top: 10px;
+}
+
+.ms-body {
+  padding: 15px 0;
+  height: 400px;
+}
+
+.el-dropdown {
+  margin-left: 20px;
+  line-height: 30px;
+}
+
+.ace_editor {
+  border-radius: 5px;
+}
+
+.el-radio-group {
+  margin: 10px 10px;
+  margin-top: 15px;
+}
+
+.ms-el-link {
+  float: right;
+  margin-right: 45px;
+}
+</style>

frontend/src/views/system/datasource/ApiHttpRequestForm.vue

@@ -0,0 +1,394 @@
+<template xmlns:v-slot="http://www.w3.org/1999/XSL/Transform">
+  <div>
+    <el-row>
+      <el-col :span="spanCount">
+        <!-- HTTP 请求参数 -->
+        <div style="border:1px #DCDFE6 solid; height: 100%;border-radius: 4px ;width: 100%">
+          <el-tabs v-model="activeName" class="request-tabs">
+            <!-- 请求头-->
+            <el-tab-pane :label="$t('datasource.headers')" name="headers">
+              <el-tooltip class="item-tabs" effect="dark" :content="$t('datasource.headers')" placement="top-start" slot="label">
+              <span>{{ $t('datasource.headers') }}
+                <div class="el-step__icon is-text ms-api-col ms-header" v-if="headers.length>1">
+                  <div class="el-step__icon-inner">{{ headers.length - 1 }}</div>
+                </div>
+              </span>
+              </el-tooltip>
+              <api-key-value :show-desc="true"  :suggestions="headerSuggestions" :items="headers"/>
+            </el-tab-pane>
+
+            <!--请求体-->
+            <el-tab-pane v-if="isBodyShow" :label="$t('datasource.request_body')" name="body" style="overflow: auto">
+              <api-body @headersChange="reloadBody" :isShowEnable="isShowEnable" :headers="headers" :body="request.body"/>
+            </el-tab-pane>
+
+            <!-- 认证配置 -->
+            <el-tab-pane :label="$t('datasource.auth_config')" name="authConfig">
+              <el-tooltip class="item-tabs" effect="dark" :content="$t('datasource.auth_config_info')" placement="top-start" slot="label">
+                <span>{{ $t('datasource.auth_config') }}</span>
+              </el-tooltip>
+              <api-auth-config :request="request"/>
+            </el-tab-pane>
+
+          </el-tabs>
+        </div>
+      </el-col>
+    </el-row>
+  </div>
+</template>
+
+<script>
+import ApiKeyValue from "@/views/system/datasource/ApiKeyValue";
+import ApiBody from "@/views/system/datasource/ApiBody";
+import ApiAuthConfig from "@/views/system/datasource/ApiAuthConfig";
+import {Body, KeyValue} from "@/views/system/datasource/ApiTestModel";
+import Convert from "@/views/system/datasource/convert";
+
+export default {
+  name: "ApiHttpRequestForm",
+  components: {
+    ApiAuthConfig,
+    ApiBody,
+    ApiKeyValue
+  },
+  props: {
+    method: String,
+    request: {},
+    response: {},
+    definitionTest: {
+      type: Boolean,
+      default() {
+        return false;
+      }
+    },
+    showScript: {
+      type: Boolean,
+      default: true,
+    },
+    headers: {
+      type: Array,
+      default() {
+        return [];
+      }
+    },
+    referenced: {
+      type: Boolean,
+      default: false,
+    },
+    isShowEnable: {
+      type: Boolean,
+      default: false,
+    },
+    jsonPathList: Array,
+    isReadOnly: {
+      type: Boolean,
+      default: false
+    },
+    type: String,
+  },
+  data() {
+    let validateURL = (rule, value, callback) => {
+      try {
+        new URL(this.addProtocol(this.request.url));
+      } catch (e) {
+        callback(this.$t('api_test.request.url_invalid'));
+      }
+    };
+    return {
+      activeName: 'headers',
+      rules: {
+        name: [
+          {max: 300, message: this.$t('commons.input_limit', [1, 300]), trigger: 'blur'}
+        ],
+        url: [
+          {max: 500, required: true, message: this.$t('commons.input_limit', [1, 500]), trigger: 'blur'},
+          {validator: validateURL, trigger: 'blur'}
+        ],
+        path: [
+          {max: 500, message: this.$t('commons.input_limit', [0, 500]), trigger: 'blur'},
+        ]
+      },
+      spanCount: 21,
+      isReloadData: false,
+      isBodyShow: true,
+      dialogVisible: false,
+      hasOwnProperty: Object.prototype.hasOwnProperty,
+      propIsEnumerable: Object.prototype.propertyIsEnumerable,
+      headerSuggestions: [
+        {value: 'Accept'},
+        {value: 'Accept-Charset'},
+        {value: 'Accept-Language'},
+        {value: 'Accept-Datetime'},
+        {value: 'Authorization'},
+        {value: 'Cache-Control'},
+        {value: 'Connection'},
+        {value: 'Cookie'},
+        {value: 'Content-Length'},
+        {value: 'Content-MD5'},
+        {value: 'Content-Type'},
+        {value: 'Date'},
+        {value: 'Expect'},
+        {value: 'From'},
+        {value: 'Host'},
+        {value: 'If-Match'},
+        {value: 'If-Modified-Since'},
+        {value: 'If-None-Match'},
+        {value: 'If-Range'},
+        {value: 'If-Unmodified-Since'},
+        {value: 'Max-Forwards'},
+        {value: 'Origin'},
+        {value: 'Pragma'},
+        {value: 'Proxy-Authorization'},
+        {value: 'Range'},
+        {value: 'Referer'},
+        {value: 'TE'},
+        {value: 'User-Agent'},
+        {value: 'Upgrade'},
+        {value: 'Via'},
+        {value: 'Warning'}
+      ]
+
+    }
+  },
+  created() {
+    if (!this.referenced && this.showScript) {
+      this.spanCount = 21;
+    } else {
+      this.spanCount = 24;
+    }
+    this.init();
+  },
+  watch: {
+    'request.changeId'() {
+      if (this.request.headers && this.request.headers.length > 1) {
+        this.activeName = 'headers';
+      }
+      if (this.request.rest && this.request.rest.length > 1) {
+        this.activeName = 'rest';
+      }
+      if (this.request.arguments && this.request.arguments.length > 1) {
+        this.activeName = 'parameters';
+      }
+      if(this.request.body) {
+        this.request.body.typeChange = this.request.changeId;
+      }
+      this.reload();
+    }
+  },
+  methods: {
+    generate() {
+      if (this.request.body && (this.request.body.jsonSchema || this.request.body.raw)) {
+        if (!this.request.body.jsonSchema) {
+          const MsConvert = new Convert();
+          this.request.body.jsonSchema = MsConvert.format(JSON.parse(this.request.body.raw));
+        }
+        this.$post('/api/test/data/generator', this.request.body.jsonSchema, response => {
+          if (response.data) {
+            if (this.request.body.format !== 'JSON-SCHEMA') {
+              this.request.body.raw = response.data;
+            } else {
+              const MsConvert = new Convert();
+              let data = MsConvert.format(JSON.parse(response.data));
+              this.request.body.jsonSchema = this.deepAssign(this.request.body.jsonSchema, data);
+            }
+            this.reloadBody();
+          }
+        });
+      }
+    },
+    remove(row) {
+      let index = this.request.hashTree.indexOf(row);
+      this.request.hashTree.splice(index, 1);
+      this.reload();
+    },
+    copyRow(row) {
+      let obj = JSON.parse(JSON.stringify(row));
+      obj.id = getUUID();
+      this.request.hashTree.push(obj);
+      this.reload();
+    },
+    reload() {
+      this.isReloadData = true
+      this.$nextTick(() => {
+        this.isReloadData = false
+      })
+    },
+    init() {
+      if (Object.prototype.toString.call(this.request).match(/\[object (\w+)\]/)[1].toLowerCase() !== 'object') {
+        this.request = JSON.parse(this.request);
+      }
+      if (!this.request.body) {
+        this.request.body = new Body();
+      }
+      if (!this.request.body.kvs) {
+        this.request.body.kvs = [];
+      }
+      if (!this.request.rest) {
+        this.request.rest = [];
+      }
+      if (!this.request.arguments) {
+        this.request.arguments = [];
+      }
+    },
+    reloadBody() {
+      // 解决修改请求头后 body 显示错位
+      this.isBodyShow = false;
+      this.$nextTick(() => {
+        this.isBodyShow = true;
+      });
+    },
+    batchAdd() {
+      this.$refs.batchAddParameter.open();
+    },
+    format(array, obj) {
+      if (array) {
+        let isAdd = true;
+        for (let i in array) {
+          let item = array[i];
+          if (item.name === obj.name) {
+            item.value = obj.value;
+            isAdd = false;
+          }
+        }
+        if (isAdd) {
+          switch (this.activeName) {
+            case "parameters":
+              this.request.arguments.unshift(obj);
+              break;
+            case "rest":
+              this.request.rest.unshift(obj);
+              break;
+            case "headers":
+              this.request.headers.unshift(obj);
+              break;
+            default:
+              break;
+          }
+        }
+      }
+    },
+    batchSave(data) {
+      if (data) {
+        let params = data.split("\n");
+        let keyValues = [];
+        params.forEach(item => {
+          let line = item.split(/：|:/);
+          let required = false;
+          keyValues.unshift(new KeyValue({
+            name: line[0],
+            required: !required,
+            value: line[1],
+            description: line[2],
+            type: "text",
+            valid: false,
+            file: false,
+            encode: true,
+            enable: true,
+            contentType: "text/plain"
+          }));
+        })
+
+        keyValues.forEach(item => {
+          switch (this.activeName) {
+            case "parameters":
+              this.format(this.request.arguments, item);
+              break;
+            case "rest":
+              this.format(this.request.rest, item);
+              break;
+            case "headers":
+              this.format(this.request.headers, item);
+              break;
+            default:
+              break;
+          }
+        })
+      }
+    },
+
+    isObj(x) {
+      let type = typeof x;
+      return x !== null && (type === 'object' || type === 'function');
+    },
+
+    toObject(val) {
+      if (val === null || val === undefined) {
+        return;
+      }
+
+      return Object(val);
+    },
+
+    assignKey(to, from, key) {
+      let val = from[key];
+
+      if (val === undefined || val === null) {
+        return;
+      }
+      if (!this.hasOwnProperty.call(to, key) || !this.isObj(val)) {
+        to[key] = val;
+      } else {
+        to[key] = this.assign(Object(to[key]), from[key]);
+      }
+    },
+
+    assign(to, from) {
+      if (to === from) {
+        return to;
+      }
+      from = Object(from);
+      for (let key in from) {
+        if (this.hasOwnProperty.call(from, key)) {
+          this.assignKey(to, from, key);
+        }
+      }
+
+      if (Object.getOwnPropertySymbols) {
+        let symbols = Object.getOwnPropertySymbols(from);
+
+        for (let i = 0; i < symbols.length; i++) {
+          if (this.propIsEnumerable.call(from, symbols[i])) {
+            this.assignKey(to, from, symbols[i]);
+          }
+        }
+      }
+
+      return to;
+    },
+
+    deepAssign(target) {
+      target = this.toObject(target);
+      for (let s = 1; s < arguments.length; s++) {
+        this.assign(target, arguments[s]);
+      }
+      return target;
+    }
+  }
+}
+</script>
+
+<style scoped>
+.ms-query {
+  background: #783887;
+  color: white;
+  height: 18px;
+  border-radius: 42%;
+}
+
+.ms-header {
+  background: #783887;
+  color: white;
+  height: 18px;
+  border-radius: 42%;
+}
+
+.request-tabs {
+  margin: 20px;
+  min-height: 200px;
+}
+
+.ms-el-link {
+  float: right;
+  margin-right: 45px;
+}
+</style>

frontend/src/views/system/datasource/ApiKeyValue.vue

@@ -0,0 +1,218 @@
+<template>
+  <div v-loading="loading">
+    <div class="kv-row item" v-for="(item, index) in items" :key="index">
+
+      <el-row type="flex" :gutter="20" justify="space-between" align="middle">
+        <i class="el-icon-top" style="cursor:pointer" @click="moveTop(index)"/>
+        <i class="el-icon-bottom" style="cursor:pointer;" @click="moveBottom(index)"/>
+
+        <el-col class="item" v-if="unShowSelect===false">
+          <el-input v-if="!suggestions" :disabled="isReadOnly" v-model="item.name" size="small" maxlength="200"
+                    @change="change"
+                    :placeholder="keyText" show-word-limit/>
+          <el-autocomplete :disabled="isReadOnly" :maxlength="400" v-if="suggestions" v-model="item.name" size="small"
+                           :fetch-suggestions="querySearch" @change="change" :placeholder="keyText"
+                           show-word-limit/>
+        </el-col>
+
+        <el-col class="item" v-if="unShowSelect===true">
+          <el-input v-if="suggestions" :disabled="isReadOnly" v-model="item.name" size="small" maxlength="200" :placeholder="keyText"  show-word-limit/>
+        </el-col>
+
+        <el-col class="item">
+          <el-input v-if="!needMock" :disabled="isReadOnly" v-model="item.value" size="small" @change="change"
+                    :placeholder="unShowSelect?$t('commons.description'):valueText" show-word-limit/>
+        </el-col>
+
+        <el-col class="item" v-if="showDesc">
+          <el-input v-model="item.description" size="small" maxlength="200" :placeholder="$t('commons.description')" show-word-limit></el-input>
+        </el-col>
+
+        <el-col class="item kv-delete">
+          <el-button size="mini" class="el-icon-delete-solid" circle @click="remove(index)" :disabled="isDisable(index)"/>
+        </el-col>
+      </el-row>
+    </div>
+
+  </div>
+</template>
+
+<script>
+  import {KeyValue} from "./ApiTestModel";
+  import Vue from 'vue';
+
+  export default {
+    name: "ApiKeyValue",
+    components: {},
+    props: {
+      keyPlaceholder: String,
+      valuePlaceholder: String,
+      isShowEnable: {
+        type: Boolean,
+        default: false
+      },
+      unShowSelect:{
+        type: Boolean,
+        default: false
+      },
+      description: String,
+      items: Array,
+      isReadOnly: {
+        type: Boolean,
+        default: false
+      },
+      suggestions: Array,
+      needMock: {
+        type: Boolean,
+        default: false
+      },
+      showDesc: Boolean,
+      appendToBody: {
+        type: Boolean,
+        default() {
+          return false;
+        }
+      },
+    },
+    data() {
+      return {
+        keyValues: [],
+        loading: false,
+        currentItem: {},
+        isSelectAll: true
+      }
+    },
+    computed: {
+      keyText() {
+        return this.keyPlaceholder || this.$t("datasource.key");
+      },
+      valueText() {
+        return this.valuePlaceholder || this.$t("datasource.value");
+      }
+    },
+    watch: {
+      isSelectAll: function (to, from) {
+        if (from == false && to == true) {
+          this.selectAll();
+        } else if (from == true && to == false) {
+          this.invertSelect();
+        }
+      }
+    },
+    methods: {
+      advanced(item) {
+        this.currentItem = item;
+        this.$refs.variableAdvance.open();
+      },
+      funcFilter(queryString) {
+        return (func) => {
+          return (func.name.toLowerCase().indexOf(queryString.toLowerCase()) > -1);
+        };
+      },
+      moveBottom(index) {
+        if (this.items.length < 2 || index === this.items.length - 2) {
+          return;
+        }
+        let thisRow = this.items[index];
+        let nextRow = this.items[index + 1];
+        Vue.set(this.items, index + 1, thisRow);
+        Vue.set(this.items, index, nextRow)
+      },
+      moveTop(index) {
+        if (index === 0) {
+          return;
+        }
+        let thisRow = this.items[index];
+        let lastRow = this.items[index - 1];
+        Vue.set(this.items, index - 1, thisRow);
+        Vue.set(this.items, index, lastRow)
+
+      },
+      reload() {
+        this.loading = true
+        this.$nextTick(() => {
+          this.loading = false
+        })
+      },
+      remove: function (index) {
+        // 移除整行输入控件及内容
+        this.items.splice(index, 1);
+        this.$emit('change', this.items);
+      },
+      change: function () {
+        let isNeedCreate = true;
+        let removeIndex = -1;
+        this.items.forEach((item, index) => {
+          if (!item.name && !item.value) {
+            // 多余的空行
+            if (index !== this.items.length - 1) {
+              removeIndex = index;
+            }
+            // 没有空行，需要创建空行
+            isNeedCreate = false;
+          }
+        });
+        if (isNeedCreate) {
+          this.items.push(new KeyValue({enable: true}));
+        }
+        this.$emit('change', this.items);
+        // TODO 检查key重复
+      },
+      isDisable: function (index) {
+        return this.items.length - 1 === index;
+      },
+      querySearch(queryString, cb) {
+        let suggestions = this.suggestions;
+        let results = queryString ? suggestions.filter(this.createFilter(queryString)) : suggestions;
+        cb(results);
+      },
+      createFilter(queryString) {
+        return (restaurant) => {
+          return (restaurant.value.toLowerCase().indexOf(queryString.toLowerCase()) === 0);
+        };
+      },
+      selectAll() {
+        this.items.forEach(item => {
+          item.enable = true;
+        });
+      },
+      invertSelect() {
+        this.items.forEach(item => {
+          item.enable = false;
+        });
+      },
+    },
+    created() {
+      if (this.items.length === 0 || this.items[this.items.length - 1].name) {
+        this.items.push(new KeyValue({enable: true, name: '', value: ''}));
+      }
+    }
+  }
+</script>
+
+<style scoped>
+  .kv-description {
+    font-size: 13px;
+  }
+
+  .kv-row {
+    margin-top: 10px;
+  }
+
+  .kv-checkbox {
+    width: 20px;
+    margin-right: 10px;
+  }
+
+  .kv-delete {
+    width: 60px;
+  }
+
+  .el-autocomplete {
+    width: 100%;
+  }
+
+  i:hover {
+    color: #783887;
+  }
+</style>

frontend/src/views/system/datasource/ApiTestModel.js

@@ -0,0 +1,165 @@
+export class BaseConfig {
+
+  set(options, notUndefined) {
+    options = this.initOptions(options)
+    for (let name in options) {
+      if (options.hasOwnProperty(name)) {
+        if (!(this[name] instanceof Array)) {
+          if (notUndefined === true) {
+            this[name] = options[name] === undefined ? this[name] : options[name];
+          } else {
+            this[name] = options[name];
+          }
+        }
+      }
+    }
+  }
+
+  sets(types, options) {
+    options = this.initOptions(options)
+    if (types) {
+      for (let name in types) {
+        if (types.hasOwnProperty(name) && options.hasOwnProperty(name)) {
+          options[name].forEach(o => {
+            this[name].push(new types[name](o));
+          })
+        }
+      }
+    }
+  }
+
+  initOptions(options) {
+    return options || {};
+  }
+
+  isValid() {
+    return true;
+  }
+}
+
+export class KeyValue extends BaseConfig {
+  constructor(options) {
+    options = options || {};
+    options.enable = options.enable === undefined ? true : options.enable;
+
+    super();
+    this.name = undefined;
+    this.value = undefined;
+    this.type = undefined;
+    this.files = undefined;
+    this.enable = undefined;
+    this.uuid = undefined;
+    this.time = undefined;
+    this.contentType = undefined;
+    this.set(options);
+  }
+
+  isValid() {
+    return (!!this.name || !!this.value) && this.type !== 'file';
+  }
+
+  isFile() {
+    return (!!this.name || !!this.value) && this.type === 'file';
+  }
+}
+
+export class Body extends BaseConfig {
+  constructor(options) {
+    super();
+    this.type = "KeyValue";
+    this.raw = undefined;
+    this.kvs = [];
+    this.binary = [];
+    this.set(options);
+    this.sets({kvs: KeyValue}, {binary: KeyValue}, options);
+  }
+
+  isValid() {
+    if (this.isKV()) {
+      return this.kvs.some(kv => {
+        return kv.isValid();
+      })
+    } else {
+      return !!this.raw;
+    }
+  }
+
+  isKV() {
+    return [BODY_TYPE.FORM_DATA, BODY_TYPE.WWW_FORM, BODY_TYPE.BINARY].indexOf(this.type) > 0;
+  }
+}
+
+export const createComponent = function (name) {
+  let component = MODELS[name];
+  if (component) {
+    return new component();
+  } else {
+    return new UnsupportedComponent()
+  }
+}
+
+export const BODY_TYPE = {
+  KV: "KeyValue",
+  FORM_DATA: "Form_Data",
+  RAW: "Raw",
+  WWW_FORM: "WWW_FORM",
+  XML: "XML",
+  JSON: "JSON"
+}
+
+export class Scenario extends BaseConfig {
+  constructor(options = {}) {
+    super();
+    this.id = undefined;
+    this.name = undefined;
+    this.url = undefined;
+    this.variables = [];
+    this.headers = [];
+    this.requests = [];
+    this.environmentId = undefined;
+    this.dubboConfig = undefined;
+    this.environment = undefined;
+    this.enableCookieShare = false;
+    this.enable = true;
+    this.databaseConfigs = [];
+    this.tcpConfig = undefined;
+    this.set(options);
+    this.sets({
+      variables: KeyValue,
+      headers: KeyValue,
+      requests: RequestFactory,
+      databaseConfigs: DatabaseConfig
+    }, options);
+  }
+
+  initOptions(options = {}) {
+    options.id = options.id || uuid();
+    options.requests = options.requests || [new RequestFactory()];
+    options.databaseConfigs = options.databaseConfigs || [];
+    options.dubboConfig = new DubboConfig(options.dubboConfig);
+    options.tcpConfig = new TCPConfig(options.tcpConfig);
+    return options;
+  }
+
+  clone() {
+    let clone = new Scenario(this);
+    clone.id = uuid();
+    return clone;
+  }
+
+  isValid() {
+    if (this.enable) {
+      for (let i = 0; i < this.requests.length; i++) {
+        let validator = this.requests[i].isValid(this.environmentId, this.environment);
+        if (!validator.isValid) {
+          return validator;
+        }
+      }
+    }
+    return {isValid: true};
+  }
+
+  isReference() {
+    return this.id.indexOf("#") !== -1
+  }
+}

frontend/src/views/system/datasource/ApiVariable.vue

@@ -0,0 +1,314 @@
+<template>
+  <div style="margin-bottom: 20px">
+    <span class="kv-description" v-if="description">
+      {{ description }}
+    </span>
+    <div class="item kv-row" v-for="(item, index) in parameters" :key="index">
+      <el-row type="flex" :gutter="20" justify="space-between" align="middle">
+        <span style="margin-left: 10px"></span>
+        <i class="el-icon-top" style="cursor:pointer" @click="moveTop(index)"/>
+        <i class="el-icon-bottom" style="cursor:pointer;" @click="moveBottom(index)"/>
+
+        <el-col class="item">
+          <el-input v-if="!suggestions" :disabled="isReadOnly" v-model="item.name" size="small" maxlength="200"
+                    @change="change" :placeholder="keyText" show-word-limit>
+            <template v-slot:prepend>
+              <el-select v-if="type === 'body'" :disabled="isReadOnly" class="kv-type" v-model="item.type"
+                         @change="typeChange(item)">
+                <el-option value="text"/>
+                <el-option value="json"/>
+              </el-select>
+            </template>
+          </el-input>
+
+          <el-autocomplete :disabled="isReadOnly" v-if="suggestions" v-model="item.name" size="small"
+                           :fetch-suggestions="querySearch" @change="change" :placeholder="keyText" show-word-limit/>
+
+        </el-col>
+
+        <el-col class="item" v-if="isActive && item.type !== 'file'">
+          <el-input :disabled="isReadOnly"
+                    size="small"
+                    class="input-with-autocomplete"
+                    v-model="item.value"
+                    :placeholder="valueText"
+                    value-key="name"
+                    highlight-first-item
+                    @select="change">
+          </el-input>
+<!--          <el-autocomplete-->
+<!--            :disabled="isReadOnly"-->
+<!--            size="small"-->
+<!--            class="input-with-autocomplete"-->
+<!--            v-model="item.value"-->
+<!--            placeholder="valueText"-->
+<!--            value-key="name"-->
+<!--            highlight-first-item-->
+<!--            @select="change">-->
+<!--          </el-autocomplete>-->
+        </el-col>
+
+        <el-col class="item">
+          <el-input v-model="item.description" size="small" maxlength="200"
+                    :placeholder="$t('commons.description')" show-word-limit>
+          </el-input>
+
+          <el-autocomplete :disabled="isReadOnly" v-if="suggestions" v-model="item.name" size="small"
+                           :fetch-suggestions="querySearch" @change="change" :placeholder="keyText" show-word-limit/>
+
+        </el-col>
+
+        <el-col v-if="type === 'body'" class="item kv-select">
+          <el-input :disabled="isReadOnly" v-model="item.contentType" size="small"
+                    @change="change" :placeholder="$t('api_test.request.content_type')" show-word-limit>
+          </el-input>
+        </el-col>
+
+        <el-col class="item kv-delete">
+          <el-button size="mini" class="el-icon-delete-solid" circle @click="remove(index)"
+                     :disabled="isDisable(index) || isReadOnly"/>
+        </el-col>
+
+      </el-row>
+    </div>
+  </div>
+</template>
+
+<script>
+import {KeyValue, Scenario} from "./ApiTestModel";
+
+import Vue from 'vue';
+
+export default {
+  name: "ApiVariable",
+  components: {},
+  props: {
+    keyPlaceholder: String,
+    valuePlaceholder: String,
+    description: String,
+    parameters: Array,
+    rest: Array,
+    environment: Object,
+    scenario: Scenario,
+    type: {
+      type: String,
+      default: ''
+    },
+    appendDialogToBody: {
+      type: Boolean,
+      default() {
+        return false;
+      }
+    },
+    isReadOnly: {
+      type: Boolean,
+      default: false
+    },
+    isShowEnable: {
+      type: Boolean,
+      default: true
+    },
+    suggestions: Array,
+    withMorSetting: Boolean
+  },
+  data() {
+    return {
+      currentItem: null,
+      requireds: [
+        {name: this.$t('commons.selector.required'), id: true},
+        {name: this.$t('commons.selector.not_required'), id: false}
+      ],
+      isSelectAll: true,
+      isActive: true,
+    }
+  },
+  watch: {
+    isSelectAll: function (to, from) {
+      if (from == false && to == true) {
+        this.selectAll();
+      } else if (from == true && to == false) {
+        this.invertSelect();
+      }
+    },
+  },
+  computed: {
+    keyText() {
+      return this.keyPlaceholder || this.$t("datasource.key");
+    },
+    valueText() {
+      return this.valuePlaceholder || this.$t("datasource.value");
+    }
+  },
+  methods: {
+    moveBottom(index) {
+      if (this.parameters.length < 2 || index === this.parameters.length - 2) {
+        return;
+      }
+      let thisRow = this.parameters[index];
+      let nextRow = this.parameters[index + 1];
+      Vue.set(this.parameters, index + 1, thisRow);
+      Vue.set(this.parameters, index, nextRow)
+    },
+    moveTop(index) {
+      if (index === 0) {
+        return;
+      }
+      let thisRow = this.parameters[index];
+      let lastRow = this.parameters[index - 1];
+      Vue.set(this.parameters, index - 1, thisRow);
+      Vue.set(this.parameters, index, lastRow)
+
+    },
+    remove: function (index) {
+      // 移除整行输入控件及内容
+      this.parameters.splice(index, 1);
+      this.$emit('change', this.parameters);
+    },
+    change: function () {
+      let isNeedCreate = true;
+      let removeIndex = -1;
+      this.parameters.forEach((item, index) => {
+        if (!item.name && !item.value) {
+          // 多余的空行
+          if (index !== this.parameters.length - 1) {
+            removeIndex = index;
+          }
+          // 没有空行，需要创建空行
+          isNeedCreate = false;
+        }
+      });
+      if (isNeedCreate) {
+        this.parameters.push(new KeyValue({
+          type: 'text',
+          enable: true,
+          uuid: this.uuid(),
+          contentType: 'text/plain'
+        }));
+      }
+      this.$emit('change', this.parameters);
+      // TODO 检查key重复
+    },
+    isDisable: function (index) {
+      return this.parameters.length - 1 == index;
+    },
+    querySearch(queryString, cb) {
+      let suggestions = this.suggestions;
+      let results = queryString ? suggestions.filter(this.createFilter(queryString)) : suggestions;
+      cb(results);
+    },
+    createFilter(queryString) {
+      return (restaurant) => {
+        return (restaurant.value.toLowerCase().indexOf(queryString.toLowerCase()) === 0);
+      };
+    },
+    funcFilter(queryString) {
+      return (func) => {
+        return (func.name.toLowerCase().indexOf(queryString.toLowerCase()) > -1);
+      };
+    },
+    uuid: function () {
+      return (((1 + Math.random()) * 0x100000) | 0).toString(16).substring(1);
+    },
+    advanced(item) {
+      if (item.type === 'json') {
+        this.$refs.variableJson.open(item);
+        this.currentItem = item;
+      } else {
+        this.$refs.variableAdvance.open();
+        this.currentItem = item;
+      }
+
+    },
+    typeChange(item) {
+      if (item.type === 'file') {
+        item.contentType = 'application/octet-stream';
+      } else if (item.type === 'text') {
+        item.contentType = 'text/plain';
+      } else {
+        item.contentType = 'application/json'
+      }
+      this.reload();
+    },
+    selectAll() {
+      this.parameters.forEach(item => {
+        item.enable = true;
+      });
+    },
+    invertSelect() {
+      this.parameters.forEach(item => {
+        item.enable = false;
+      });
+    },
+    reload() {
+      this.isActive = false;
+      this.$nextTick(() => {
+        this.isActive = true;
+      });
+    },
+    callback(item) {
+      this.currentItem.value = item;
+      this.currentItem = null;
+    }
+  },
+  created() {
+    if (this.parameters.length === 0 || this.parameters[this.parameters.length - 1].name) {
+      this.parameters.push(new KeyValue({
+        type: 'text',
+        enable: true,
+        required: true,
+        uuid: this.uuid(),
+        contentType: 'text/plain'
+      }));
+    }
+  }
+}
+</script>
+
+<style scoped>
+.kv-description {
+  font-size: 13px;
+}
+
+.kv-row {
+  margin-top: 10px;
+}
+
+.kv-delete {
+  width: 60px;
+}
+
+.kv-select {
+  width: 50%;
+}
+
+.el-autocomplete {
+  width: 100%;
+}
+
+.kv-checkbox {
+  width: 20px;
+  margin-right: 10px;
+}
+
+.advanced-item-value >>> .el-dialog__body {
+  padding: 15px 25px;
+}
+
+.el-row {
+  margin-bottom: 5px;
+}
+
+.kv-type {
+  width: 70px;
+}
+
+.pointer {
+  cursor: pointer;
+  color: #1E90FF;
+}
+
+.kv-setting {
+  width: 40px;
+  padding: 0px !important;
+}
+</style>

frontend/src/views/system/datasource/CodeEdit.vue

@@ -0,0 +1,108 @@
+<template>
+    <editor v-model="formatData" :lang="mode" @init="editorInit" :theme="theme" :height="height"/>
+</template>
+
+<script>
+  import {formatJson, formatXml} from "./format-utils";
+
+    export default {
+      name: "CodeEdit",
+      components: { editor: require('vue2-ace-editor')},
+      data() {
+        return {
+          formatData: ''
+        }
+      },
+      props: {
+        height: [String, Number],
+        data: {
+          type: String
+        },
+        theme: {
+          type: String,
+          default() {
+            return 'chrome'
+          }
+        },
+        init: {
+          type: Function
+        },
+        enableFormat: {
+          type: Boolean,
+          default() {
+            return true;
+          }
+        },
+        readOnly: {
+          type: Boolean,
+          default() {
+            return false;
+          }
+        },
+        mode: {
+          type: String,
+          default() {
+            return 'text';
+          }
+        },
+        modes: {
+          type: Array,
+          default() {
+            return ['text', 'json', 'xml'];
+          }
+        }
+      },
+      mounted() {
+        if (!this.data) {
+          this.formatData = "";
+        }
+        this.format();
+      },
+      watch: {
+        formatData() {
+          this.$emit('update:data', this.formatData);
+        },
+        mode() {
+          this.format();
+        }
+      },
+      methods: {
+        editorInit: function (editor) {
+          require('brace/ext/language_tools') //language extension prerequsite...
+          this.modes.forEach(mode => {
+            require('brace/mode/' + mode); //language
+          });
+          require('brace/theme/' + this.theme)
+          require('brace/snippets/javascript') //snippet
+          if (this.readOnly) {
+            editor.setReadOnly(true);
+          }
+          if (this.init) {
+            this.init(editor);
+          }
+        },
+        format() {
+          if (this.enableFormat) {
+            if (this.data) {
+              switch (this.mode) {
+                case 'json':
+                  this.formatData = formatJson(this.data);
+                  break;
+                case 'xml':
+                  this.formatData = formatXml(this.data);
+                  break;
+                default:
+                  this.formatData = this.data;
+              }
+            }
+          } else {
+            this.formatData = this.data;
+          }
+        }
+      }
+    }
+</script>
+
+<style scoped>
+
+</style>

frontend/src/views/system/datasource/DialogFooter.vue

@@ -0,0 +1,43 @@
+<template>
+
+  <div class="dialog-footer">
+    <el-button @click="cancel" :size="btnSize">{{$t('commons.cancel')}}</el-button>
+    <el-button type="primary" @click="confirm" @keydown.enter.native.prevent :size="btnSize">{{$t('commons.confirm')}}</el-button>
+    <el-button type="primary" v-if="isShow" @click="saveAsEdit" @keydown.enter.native.prevent>{{title}}</el-button>
+  </div>
+
+</template>
+
+<script>
+  export default {
+    name: "DialogFooter",
+    props: {
+      isShow: {
+        type: Boolean,
+        default: false,
+      },
+      title:String,
+      btnSize: {
+        type: String,
+        default() {
+          return '';
+        }
+      }
+    },
+    methods: {
+      cancel() {
+        this.$emit("cancel");
+      },
+      confirm() {
+        this.$emit("confirm");
+      },
+      saveAsEdit() {
+        this.$emit("saveAsEdit");
+      }
+    }
+  }
+</script>
+
+<style scoped>
+
+</style>

frontend/src/views/system/datasource/DsMain.vue

@@ -37,7 +37,6 @@ export default {
   methods: {
     // 切换main区内容
     switchMain(param) {
-      console.log(param)
       const {component, componentParam, tData} = param
       this.component = DataHome
       this.param = null

frontend/src/views/system/datasource/DsTree.vue

@@ -39,23 +39,35 @@
           >
             <span slot-scope="{ node, data }" class="custom-tree-node-list father">
               <span style="display: flex;flex: 1;width: 0;">
-                <span v-if="data.type !== 'folder' && data.status !== 'Error'">
+                <span v-if="data.type !== 'folder' && data.status !== 'Error' && data.status !== 'Warning'">
                   <svg-icon icon-class="datasource" class="ds-icon-scene"/>
                 </span>
                 <span v-if="data.status === 'Error'">
                   <svg-icon icon-class="exclamationmark" class="ds-icon-scene"/>
                 </span>
+                <span v-if="data.status === 'Warning'">
+                  <svg-icon icon-class="exclamationmark2" class="ds-icon-scene"/>
+                </span>
                 <span v-if="data.type === 'folder'">
                   <i class="el-icon-folder"/>
                 </span>
-                <span v-if=" data.status === 'Error'" style="margin-left: 6px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">
+                <span v-if=" data.status === 'Error'"
+                      style="margin-left: 6px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">
                   <el-tooltip effect="dark" :content="$t('datasource.in_valid')" placement="right">
                     <span>
                       {{ data.name }}
                     </span>
                   </el-tooltip>
                 </span>
-                <span v-if=" data.status !== 'Error'"
+                <span v-if=" data.status === 'Warning'"
+                      style="margin-left: 6px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">
+                  <el-tooltip effect="dark" :content="$t('datasource.warning')" placement="right">
+                    <span>
+                      {{ data.name }}
+                    </span>
+                  </el-tooltip>
+                </span>
+                <span v-if="data.status !== 'Error' && data.status !== 'Warning'"
                       style="margin-left: 6px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">
                   {{ data.name }}
                 </span>
@@ -140,14 +152,14 @@ export default {
       let typeData = []
       listDatasourceByType(datasource.type).then(res => {
         typeData = this.buildTree(res.data)
-        if(typeData.length === 0){
+        if (typeData.length === 0) {
           let index = this.tData.findIndex(item => {
-            if ( item.id === datasource.type) {
+            if (item.id === datasource.type) {
               return true;
             }
           })
-          this.tData.splice(index,1)
+          this.tData.splice(index, 1)
-        }else {
+        } else {
           let find = false;
           for (let index = 0; index < this.tData.length; index++) {
             if (typeData[0].id === this.tData[index].id) {
@@ -155,7 +167,7 @@ export default {
               find = true
             }
           }
-          if(!find){
+          if (!find) {
             this.tData.push(typeData[0])
           }
         }
@@ -212,6 +224,8 @@ export default {
         return 'Apache Hive'
       } else if (type === 'db2') {
         return 'Db2'
+      } else if (type === 'api') {
+        return 'API'
       }
     },
 
@@ -268,7 +282,6 @@ export default {
       })
     },
     switchMain(component, componentParam, tData) {
-      console.log(tData)
       this.$emit('switch-main', {
         component,
         componentParam,

frontend/src/views/system/datasource/EditDialog.vue

@@ -0,0 +1,86 @@
+<template>
+  <el-dialog :close-on-click-modal="closeOnClickModal"
+             :title="title"
+             :width="width"
+             :visible="visible"
+             destroy-on-close
+             :append-to-body="appendToBody"
+             @close="handleClose">
+
+    <slot name="header"></slot>
+
+    <slot></slot>
+
+    <template v-slot:footer>
+      <slot name="footer">
+        <div v-if="withFooter" class="dialog-footer">
+          <ms-dialog-footer
+            @cancel="handleCancel"
+            @confirm="handleConfirm"/>
+        </div>
+      </slot>
+    </template>
+
+  </el-dialog>
+</template>
+
+<script>
+import DialogFooter from "./DialogFooter";
+export default {
+  name: "EditDialog",
+  components: {DialogFooter},
+  props: {
+    title: {
+      type: String,
+      default() {
+        return 'title';
+      }
+    },
+    visible: {
+      type: Boolean,
+      default() {
+        return false;
+      }
+    },
+    appendToBody: {
+      type: Boolean,
+      default() {
+        return false;
+      }
+    },
+    width: {
+      type: String,
+      default() {
+        return "50%";
+      }
+    },
+    withFooter: {
+      type: Boolean,
+      default() {
+        return true;
+      }
+    },
+    closeOnClickModal: {
+      type: Boolean,
+      default: false
+    },
+  },
+  methods: {
+    handleConfirm() {
+      this.$emit('confirm');
+    },
+    handleCancel() {
+      this.handleClose();
+      this.$emit('cancel');
+    },
+    handleClose() {
+      this.$emit('update:visible', false);
+      this.$emit('close');
+    },
+  }
+};
+</script>
+
+<style scoped>
+
+</style>

frontend/src/views/system/datasource/ImportJson.vue

@@ -0,0 +1,118 @@
+<template>
+  <el-dialog
+    :title="$t('commons.import')"
+    :visible.sync="importVisible"
+    width="50%"
+    append-to-body
+    show-close
+    :close-on-click-modal="false"
+    @closed="handleClose">
+
+    <el-tabs v-model="activeName">
+      <el-tab-pane label="JSON" name="JSON">
+        <div style="height: 400px">
+          <ms-code-edit :mode="mode"
+                        :data.sync="json" theme="eclipse" :modes="[]"
+                        ref="codeEdit"/>
+        </div>
+      </el-tab-pane>
+      <el-tab-pane label="JSON-SCHEMA" name="JSON-SCHEMA">
+        <div style="height: 400px">
+          <ms-code-edit :mode="mode"
+                        :data.sync="jsonSchema" theme="eclipse" :modes="[]"
+                        ref="codeEdit"/>
+        </div>
+      </el-tab-pane>
+    </el-tabs>
+    <span slot="footer" class="dialog-footer">
+     <ms-dialog-footer
+       @cancel="importVisible = false"
+       @confirm="saveConfirm"/>
+    </span>
+  </el-dialog>
+</template>
+
+<script>
+  import DialogFooter from './DialogFooter'
+  import CodeEdit from "./CodeEdit";
+  import json5 from 'json5';
+  const Convert = require('./convert.js');
+  const MsConvert = new Convert();
+
+  export default {
+    name: "MsImportJson",
+    components: {DialogFooter, CodeEdit},
+    data() {
+      return {
+        importVisible: false,
+        activeName: "JSON",
+        mode: "json",
+        json: "",
+        jsonSchema: "",
+      };
+    },
+    watch: {},
+    props: {},
+    methods: {
+      openOneClickOperation() {
+        this.importVisible = true;
+      },
+      checkIsJson(json) {
+        try {
+          json5.parse(json);
+          return true;
+        } catch (e) {
+          return false;
+        }
+      },
+      checkIsJsonSchema(json) {
+        try {
+          json = json5.parse(json);
+          if (json.properties && typeof json.properties === 'object' && !json.type) {
+            json.type = 'object';
+          }
+          if (json.items && typeof json.items === 'object' && !json.type) {
+            json.type = 'array';
+          }
+          if (!json.type) {
+            return false;
+          }
+          json.type = json.type.toLowerCase();
+          let types = ['object', 'string', 'number', 'array', 'boolean', 'integer'];
+          if (types.indexOf(json.type) === -1) {
+            return false;
+          }
+          return JSON.stringify(json);
+        } catch (e) {
+          return false;
+        }
+      },
+      saveConfirm() {
+        if (this.activeName === 'JSON') {
+          if (!this.checkIsJson(this.json)) {
+            this.$error(this.$t('schema.json_warning'));
+            return;
+          }
+          let jsonData = MsConvert.format(json5.parse(this.json));
+          //let jsonData = GenerateSchema(json5.parse(this.json));
+          this.$emit('jsonData', jsonData);
+        } else {
+          if (!this.checkIsJsonSchema(this.jsonSchema)) {
+            this.$error(this.$t('schema.json_schema_warning'));
+            return;
+          }
+          let obj = json5.parse(this.jsonSchema);
+          this.$emit('jsonData', obj);
+        }
+        this.importVisible = false;
+      },
+      handleClose() {
+        this.importVisible = false;
+      },
+    }
+  }
+</script>
+
+<style scoped>
+
+</style>

frontend/src/views/system/datasource/convert.js

@@ -0,0 +1,206 @@
+const isBoolean = require("lodash.isboolean");
+const isEmpty = require("lodash.isempty");
+const isInteger = require("lodash.isinteger");
+const isNull = require("lodash.isnull");
+const isNumber = require("lodash.isnumber");
+const isObject = require("lodash.isobject");
+const isString = require("lodash.isstring");
+const isArray = Array.isArray;
+
+
+class Convert {
+  constructor() {
+    this._option = {
+      $id: "http://example.com/root.json",
+      $schema: "http://json-schema.org/draft-07/schema#",
+    }
+    this._object = null;
+  }
+
+  /**
+   * 转换函数
+   * @param {*} object 需要转换的对象
+   * @param {*} ?option 可选参数，目前只有能设置 root 节点的 $id 和 $schema
+   */
+  format(object, option = {}) {
+    // 数据校验，确保传入的的object只能是对象或数组
+    if (!isObject(object)) {
+      throw new TypeError("传入参数只能是对象或数组");
+    }
+    // 合并属性
+    this._option = Object.assign(this._option, option);
+    // 需要转换的对象
+    this._object = object;
+    let convertRes;
+    // 数组类型和对象类型结构不一样
+    if (isArray(object)) {
+      convertRes = this._arrayToSchema();
+    } else {
+      convertRes = this._objectToSchema();
+    }
+    // 释放
+    this._object = null;
+    return convertRes;
+  }
+
+  /**
+   * 数组类型转换成JSONSCHEMA
+   */
+  _arrayToSchema() {
+    // root节点基本信息
+    let result = this._value2object(this._object, this._option.$id, "", true);
+    if (this._object.length > 0) {
+      let itemArr = [];
+      for (let index = 0; index < this._object.length; index++) {
+        // 创建items对象的基本信息
+        let objectItem = this._object[index]
+        let item = this._value2object(objectItem, `#/items`, 'items');
+        if (isObject(objectItem) && !isEmpty(objectItem)) {
+          // 递归遍历
+          let objectItemSchema = this._json2schema(objectItem, `#/items`);
+          // 合并对象
+          item = Object.assign(item, objectItemSchema);
+        }
+        itemArr.push(item);
+      }
+      result["items"] = itemArr;
+    }
+    return result
+  }
+
+  /**
+   * 对象类型转换成JSONSCHEMA
+   */
+  _objectToSchema() {
+    let baseResult = this._value2object(this._object, this._option.$id, "", true)
+    let objectSchema = this._json2schema(this._object)
+    baseResult = Object.assign(baseResult, objectSchema)
+    return baseResult
+  }
+
+  /**
+   * 递归函数，转换object对象为json schmea 格式
+   * @param {*} object 需要转换对象
+   * @param {*} name $id值
+   */
+  _json2schema(object, name = "") {
+    // 如果递归值不是对象，那么return掉
+    if (!isObject(object)) {
+      return;
+    }
+    // 处理当前路径$id
+    if (name === "" || name == undefined) {
+      name = "#"
+    }
+    let result = {};
+    // 判断传入object是对象还是数组。
+    if (isArray(object)) {
+      result.items = {};
+    } else {
+      result.properties = {};
+    }
+    // 遍历传入的对象
+    for (const key in object) {
+      if (object.hasOwnProperty(key)) {
+        const element = object[key];
+        // 如果只是undefined。跳过
+        if (element === undefined) {
+          continue;
+        }
+        let $id = `${name}/properties/${key}`
+        // 判断当前 element 的值 是否也是对象，如果是就继续递归，不是就赋值给result
+        if(!result["properties"]){
+          continue;
+        }
+        if (isObject(element)) {
+          // 创建当前属性的基本信息
+          result["properties"][key] = this._value2object(element, $id, key)
+          if (isArray(element)) {
+            // 针对空数组和有值的数组做不同处理
+            if (element.length > 0) {
+              // 是数组
+              let itemArr = [];
+              for (let index = 0; index < element.length; index++) {
+                let elementItem = element[index];
+                // 创建items对象的基本信息
+                let item = this._value2object(elementItem, `${$id}/items`, key + 'items');
+                // 判断第一项是否是对象,且对象属性不为空
+                if (isObject(elementItem) && !isEmpty(elementItem)) {
+                  // 新增的properties才合并进来
+                  item = Object.assign(item, this._json2schema(elementItem, `${$id}/items`));
+                }
+                itemArr.push(item);
+              }
+              result["properties"][key]["items"] = itemArr;
+            }
+          } else {
+            // 不是数组，递归遍历获取，然后合并对象属性
+            result["properties"][key] = Object.assign(result["properties"][key], this._json2schema(element, $id));
+          }
+        } else {
+          // 一般属性直接获取基本信息
+          if (result["properties"]) {
+            result["properties"][key] = this._value2object(element, $id, key);
+          }
+        }
+      }
+    }
+    return result;
+  }
+
+  /**
+   * 把json的值转换成对象类型
+   * @param {*} value
+   * @param {*} $id
+   * @param {*} key
+   */
+  _value2object(value, $id, key = '', root = false) {
+    let objectTemplate = {
+      $id: $id,
+      title: `The ${key} Schema`,
+      mock: {
+        "mock": value
+      },
+    }
+
+    // 判断是否为初始化root数据
+    if (root) {
+      objectTemplate["$schema"] = this._option.$schema;
+      objectTemplate["title"] = `The Root Schema`;
+      objectTemplate["mock"] = undefined;
+    }
+    if (isBoolean(value)) {
+      objectTemplate.type = "boolean";
+    } else if (isInteger(value)) {
+      objectTemplate.type = "integer";
+    } else if (isNumber(value)) {
+      objectTemplate.type = "number";
+    } else if (isString(value)) {
+      objectTemplate.type = "string";
+    } else if (isNull(value)) {
+      objectTemplate.type = "null";
+    } else if (isArray(value)) {
+      objectTemplate.type = "array";
+      objectTemplate["mock"] = undefined;
+    } else if (isObject(value)) {
+      objectTemplate.type = "object"
+      objectTemplate["mock"] = undefined;
+    }
+
+    return objectTemplate;
+  }
+
+  /**
+   *  后台转换
+   * @param callback
+   */
+  // schemaToJsonStr(schema, callback) {
+  //   post('/api/definition/preview', schema, (response) => {
+  //     if (callback) {
+  //       callback(response.data);
+  //     }
+  //   });
+  // }
+}
+
+module.exports = Convert;

frontend/src/views/system/datasource/form.vue

@@ -30,6 +30,7 @@
             class="select-width"
             :disabled="formType=='modify' || (formType==='add' && params && !!params.type)"
             @change="changeType()"
+            filterable
           >
             <el-option
               v-for="item in allTypes"
@@ -40,104 +41,210 @@
           </el-select>
         </el-form-item>
 
-        <el-form-item
+        <el-form-item v-if="form.type == 'api'" :label="$t('datasource.data_table')">
-          v-if="form.configuration.dataSourceType=='jdbc'"
+          <el-col>
-          :label="$t('datasource.host')"
+            <el-button size="mini" icon="el-icon-plus" type="text" @click="addApiItem(undefined)"/>
-          prop="configuration.host"
+            <el-table :data="form.apiConfiguration" class="my_table" max-height="300" height="300">
-        >
+              <el-table-column prop="name" :label="$t('datasource.data_table_name')" width="150" show-overflow-tooltip></el-table-column>
+              <el-table-column prop="method" :label="$t('datasource.method')" width="150" show-overflow-tooltip></el-table-column>
+              <el-table-column prop="url" :label="$t('datasource.url')" width="150" show-overflow-tooltip></el-table-column>
+              <el-table-column :label="$t('dataset.operate')">
+                <template slot-scope="scope" style="float: right">
+                  <el-button size="mini" type="primary" icon="el-icon-edit" circle @click="addApiItem(scope.row)"/>
+                  <el-button size="mini" type="danger" icon="el-icon-delete" circle @click="deleteItem(scope.row)"/>
+                </template>
+              </el-table-column>
+            </el-table>
+          </el-col>
+        </el-form-item>
+
+        <el-dialog v-dialogDrag :title="api_table_title" :visible="edit_api_item" :before-close="closeEditItem" width="70%" class="dialog-css" append-to-body>
+          <el-steps :active="active" align-center>
+            <el-step title="步骤 1"></el-step>
+            <el-step title="步骤 2"></el-step>
+          </el-steps>
+
+          <el-row  v-show="active === 1">
+            <el-form ref="apiItem" :model="apiItem" label-width="100px" :rules="rule">
+              <p class="tip">{{ $t('datasource.base_info') }} </p>
+
+              <el-form-item :label="$t('commons.name')" prop="name">
+                <el-input v-model="apiItem.name" autocomplete="off"/>
+              </el-form-item>
+
+              <el-form-item :label="$t('datasource.request')" prop="url">
+                <el-input :placeholder="$t('datasource.path_all_info')" v-model="apiItem.url" class="ms-http-input" size="small"  >
+                  <el-select v-model="apiItem.method" slot="prepend" style="width: 100px" size="small">
+                    <el-option v-for="item in reqOptions" :key="item.id" :label="item.label" :value="item.id"/>
+                  </el-select>
+                </el-input>
+              </el-form-item>
+
+              <div v-loading="loading">
+                <p class="tip">{{ $t('datasource.req_param') }} </p>
+                <!-- HTTP 请求参数 -->
+                <el-form-item>
+                  <api-http-request-form  :headers="apiItem.request.headers" :request="apiItem.request" :response="responseData"/>
+                </el-form-item>
+              </div>
+
+              <el-form-item :label="$t('datasource.data_path')" pprop="dataPath">
+                <el-input :placeholder="$t('datasource.data_path_desc')" v-model="apiItem.dataPath" autocomplete="off"/>
+              </el-form-item>
+              <el-button style="margin-top: 12px;" @click="validateApi(undefined)" v-show="active === 1">{{ $t('commons.validate') }}</el-button>
+            </el-form>
+          </el-row>
+          <el-row  v-show="active === 2">
+            <el-tabs v-model="api_step2_active_name" @tab-click="handleClick">
+              <el-tab-pane label="数据预览" name="first">
+                <ux-grid ref="plxTable" size="mini" style="width: 100%;" :height="height" :checkbox-config="{highlight: true}" :width-resize="true" >
+                  <ux-table-column v-for="field in apiItem.fields" :key="field.originName" min-width="200px" :field="field.originName" :resizable="true">
+                    <template slot="header">
+                      <svg-icon v-if="field.deExtractType === 0" icon-class="field_text" class="field-icon-text" />
+                      <svg-icon v-if="field.deExtractType === 1" icon-class="field_time" class="field-icon-time" />
+                      <svg-icon v-if="field.deExtractType === 2 || field.deExtractType === 3" icon-class="field_value" class="field-icon-value" />
+                      <svg-icon v-if="field.deExtractType === 5" icon-class="field_location" class="field-icon-location" />
+                      <span>{{ field.name }}</span>
+                    </template>
+                  </ux-table-column>
+                </ux-grid>
+              </el-tab-pane>
+              <el-tab-pane label="字段管理" name="second">
+                <el-table :data="apiItem.fields" size="mini">
+                  <el-table-column property="name" :label="$t('dataset.field_name')" width="180">
+                    <template slot-scope="scope">
+                      <el-input v-model="scope.row.name" size="mini"/>
+                    </template>
+                  </el-table-column>
+
+                  <el-table-column property="originName" :label="$t('dataset.field_origin_name')" width="100">
+                    <template slot-scope="scope">
+                      <span v-if="scope.row.extField === 0" :title="scope.row.originName" class="field-class" style="width: 100%;white-space: nowrap;overflow: hidden;text-overflow: ellipsis;">
+                        <span style="font-size: 12px;">{{ scope.row.originName }}</span>
+                      </span>
+                    </template>
+                  </el-table-column>
+
+                  <el-table-column property="deExtractType" :label="$t('dataset.field_type')" width="140">
+                    <template slot-scope="scope">
+                      <el-select v-model="scope.row.deExtractType" size="mini" style="display: inline-block;width: 26px;">
+                        <el-option v-for="item in fieldTypes" :key="item.value" :label="item.label" :value="item.value">
+                        <span style="float: left">
+                          <svg-icon v-if="item.value === 0" icon-class="field_text" class="field-icon-text" />
+                          <svg-icon v-if="item.value === 1" icon-class="field_time" class="field-icon-time" />
+                          <svg-icon v-if="item.value === 2 || item.value === 3" icon-class="field_value" class="field-icon-value" />
+                          <svg-icon v-if="item.value === 5" icon-class="field_location" class="field-icon-location" />
+                        </span>
+                          <span style="float: left; color: #8492a6; font-size: 12px">{{ item.label }}</span>
+                        </el-option>
+                      </el-select>
+                      <span style="margin-left: 8px;">
+                      <span v-if="scope.row.deExtractType === 0">
+                        <svg-icon icon-class="field_text" class="field-icon-text" />
+                        <span class="field-class">{{ $t('dataset.text') }}</span>
+                      </span>
+                      <span v-if="scope.row.deExtractType === 1">
+                        <svg-icon v-if="scope.row.deExtractType === 1" icon-class="field_time" class="field-icon-time" />
+                        <span class="field-class">{{ $t('dataset.time') }}</span>
+                      </span>
+                      <span v-if="scope.row.deExtractType === 2 || scope.row.deExtractType === 3">
+                        <svg-icon v-if="scope.row.deExtractType === 2 || scope.row.deExtractType === 3" icon-class="field_value" class="field-icon-value" />
+                        <span v-if="scope.row.deExtractType === 2" class="field-class">{{ $t('dataset.value') }}</span>
+                        <span v-if="scope.row.deExtractType === 3" class="field-class">{{ $t('dataset.value') + '(' + $t('dataset.float') + ')' }}</span>
+                      </span>
+                      <span v-if="scope.row.deExtractType === 5">
+                        <svg-icon v-if="scope.row.deExtractType === 5" icon-class="field_location" class="field-icon-location" />
+                        <span class="field-class">{{ $t('dataset.location') }}</span>
+                      </span>
+                    </span>
+                    </template>
+                  </el-table-column>
+                </el-table>
+              </el-tab-pane>
+            </el-tabs>
+          </el-row>
+          <div slot="footer" class="dialog-footer">
+            <el-button style="margin-top: 12px;" @click="next" v-show="active === 1">{{ $t('fu.steps.next') }}</el-button>
+            <el-button style="margin-top: 12px;" @click="before" v-show="active === 2">{{ $t('fu.steps.prev') }}</el-button>
+            <el-button style="margin-top: 12px;" @click="saveItem" v-show="active === 2">{{ $t('commons.save') }}</el-button>
+          </div>
+
+        </el-dialog>
+
+
+        <el-form-item v-if="form.configuration.dataSourceType=='jdbc' && form.type !== 'api'"
+                      :label="$t('datasource.host')" prop="configuration.host">
           <el-input v-model="form.configuration.host" autocomplete="off"/>
         </el-form-item>
-        <el-form-item
+
-          v-if="form.configuration.dataSourceType=='es'"
+        <el-form-item v-if="form.configuration.dataSourceType=='es' && form.type !== 'api'"
-          :label="$t('datasource.datasource_url')"
+                      :label="$t('datasource.datasource_url')" prop="configuration.url">
-          prop="configuration.url"
+          <el-input v-model="form.configuration.url" :placeholder="$t('datasource.please_input_datasource_url')"
-        >
+                    autocomplete="off"/>
-          <el-input
-            v-model="form.configuration.url"
-            :placeholder="$t('datasource.please_input_datasource_url')"
-            autocomplete="off"
-          />
         </el-form-item>
-        <el-form-item
+
-          v-if="form.configuration.dataSourceType=='jdbc'"
+        <el-form-item v-if="form.configuration.dataSourceType=='jdbc' && form.type !== 'api'"
-          :label="$t('datasource.data_base')"
+                      :label="$t('datasource.data_base')" prop="configuration.dataBase">
-          prop="configuration.dataBase"
-        >
           <el-input v-model="form.configuration.dataBase" autocomplete="off"/>
         </el-form-item>
 
-        <el-form-item
+        <el-form-item v-if="form.type=='oracle' && form.type !== 'api'" :label="$t('datasource.oracle_connection_type')"
-          v-if="form.type=='oracle'"
+                      prop="configuration.connectionType">
-          :label="$t('datasource.oracle_connection_type')"
-          prop="configuration.connectionType"
-        >
           <el-radio v-model="form.configuration.connectionType" label="sid">{{ $t('datasource.oracle_sid') }}</el-radio>
           <el-radio v-model="form.configuration.connectionType" label="serviceName">
             {{ $t('datasource.oracle_service_name') }}
           </el-radio>
         </el-form-item>
 
-        <el-form-item v-if="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.user_name')">
+        <el-form-item v-if="form.configuration.dataSourceType=='jdbc' && form.type !== 'api'"
+                      :label="$t('datasource.user_name')">
           <el-input v-model="form.configuration.username" autocomplete="off"/>
         </el-form-item>
-        <el-form-item v-if="form.configuration.dataSourceType=='jdbc'" :label="$t('datasource.password')">
+
+        <el-form-item v-if="form.configuration.dataSourceType=='jdbc' && form.type !== 'api'"
+                      :label="$t('datasource.password')">
           <el-input v-model="form.configuration.password" autocomplete="off" show-password/>
         </el-form-item>
-        <el-form-item v-if="form.configuration.dataSourceType=='es'" :label="$t('datasource.user_name')">
+
+        <el-form-item v-if="form.configuration.dataSourceType=='es' && form.type !== 'api'"
+                      :label="$t('datasource.user_name')">
           <el-input v-model="form.configuration.esUsername" autocomplete="off"/>
         </el-form-item>
-        <el-form-item v-if="form.configuration.dataSourceType=='es'" :label="$t('datasource.password')">
+
+        <el-form-item v-if="form.configuration.dataSourceType=='es' && form.type !== 'api'"
+                      :label="$t('datasource.password')">
           <el-input v-model="form.configuration.esPassword" autocomplete="off" show-password/>
         </el-form-item>
 
-        <el-form-item
+        <el-form-item v-if="form.configuration.dataSourceType=='jdbc' && form.type!=='oracle' && form.type !== 'api'"
-          v-if="form.configuration.dataSourceType=='jdbc' && form.type!=='oracle'"
+                      :label="$t('datasource.extra_params')">
-          :label="$t('datasource.extra_params')"
-        >
           <el-input v-model="form.configuration.extraParams" autocomplete="off"/>
         </el-form-item>
 
-        <el-form-item
+        <el-form-item v-if="form.configuration.dataSourceType=='jdbc' && form.type !== 'api'"
-          v-if="form.configuration.dataSourceType=='jdbc'"
+                      :label="$t('datasource.port')" prop="configuration.port">
-          :label="$t('datasource.port')"
-          prop="configuration.port"
-        >
           <el-input v-model="form.configuration.port" autocomplete="off" type="number" min="0"/>
         </el-form-item>
-        <el-form-item v-if="form.type=='oracle' || form.type=='sqlServer' || form.type=='pg' || form.type=='redshift' || form.type=='db2'">
+
-          <el-button icon="el-icon-plus" size="mini" @click="getSchema()">
+        <el-form-item
-            {{ $t('datasource.get_schema') }}
+          v-if="form.type=='oracle' || form.type=='sqlServer' || form.type=='pg' || form.type=='redshift' || form.type=='db2'">
-          </el-button>
+          <el-button icon="el-icon-plus" size="mini" @click="getSchema()">{{ $t('datasource.get_schema') }}</el-button>
         </el-form-item>
 
         <el-form-item
           v-if="form.type=='oracle' || form.type=='sqlServer' || form.type=='pg' || form.type=='redshift' || form.type=='db2'"
-          :label="$t('datasource.schema')"
+          :label="$t('datasource.schema')">
-        >
+          <el-select v-model="form.configuration.schema" filterable :placeholder="$t('datasource.please_choose_schema')"
-          <el-select
+                     class="select-width">
-            v-model="form.configuration.schema"
+            <el-option v-for="item in schemas" :key="item" :label="item" :value="item"/>
-            filterable
-            :placeholder="$t('datasource.please_choose_schema')"
-            class="select-width"
-          >
-            <el-option
-              v-for="item in schemas"
-              :key="item"
-              :label="item"
-              :value="item"
-            />
           </el-select>
         </el-form-item>
-        <el-collapse v-if="form.configuration.dataSourceType=='jdbc'">
+
+        <el-collapse v-if="form.configuration.dataSourceType=='jdbc' && form.type !== 'api'">
           <el-collapse-item :title="$t('datasource.priority')" name="1">
             <el-form-item :label="$t('datasource.initial_pool_size')" prop="configuration.initialPoolSize">
-              <el-input
+              <el-input v-model="form.configuration.initialPoolSize" autocomplete="off" type="number" min="0"
-                v-model="form.configuration.initialPoolSize"
+                        size="small"/>
-                autocomplete="off"
-                type="number"
-                min="0"
-                size="small"
-              />
             </el-form-item>
             <el-form-item :label="$t('datasource.min_pool_size')" prop="configuration.minPoolSize">
               <el-input v-model="form.configuration.minPoolSize" autocomplete="off" type="number" min="0"/>
@@ -145,34 +252,23 @@
             <el-form-item :label="$t('datasource.max_pool_size')" prop="configuration.maxPoolSize">
               <el-input v-model="form.configuration.maxPoolSize" autocomplete="off" type="number" min="0"/>
             </el-form-item>
-
           </el-collapse-item>
         </el-collapse>
       </el-form>
       <div v-if="canEdit" slot="footer" class="dialog-footer">
-        <el-button
+        <el-button v-if="formType==='add'?true: hasDataPermission('manage',params.privileges)"
-          v-if="formType==='add'?true: hasDataPermission('manage',params.privileges)"
+                   @click="validaDatasource">{{ $t('commons.validate') }}
-          @click="validaDatasource"
-        >{{ $t('commons.validate') }}
         </el-button>
-        <el-button
+        <el-button v-if="formType==='add'?true: hasDataPermission('manage',params.privileges)" type="primary"
-          v-if="formType==='add'?true: hasDataPermission('manage',params.privileges)"
+                   @click="save">{{ $t('commons.save') }}
-          type="primary"
-          @click="save"
-        >{{ $t('commons.save') }}
         </el-button>
       </div>
       <div v-else slot="footer" class="dialog-footer">
-        <el-button
+        <el-button v-if="formType==='add'?true: hasDataPermission('manage',params.privileges)"
-          v-if="formType==='add'?true: hasDataPermission('manage',params.privileges)"
+                   @click="validaDatasource">{{ $t('commons.validate') }}
-          @click="validaDatasource"
-        >{{ $t('commons.validate') }}
         </el-button>
-        <el-button
+        <el-button v-if="formType==='add'?true: hasDataPermission('manage',params.privileges)" type="primary"
-          v-if="formType==='add'?true: hasDataPermission('manage',params.privileges)"
+                   @click="changeEdit">{{ $t('commons.edit') }}
-          type="primary"
-          @click="changeEdit"
-        >{{ $t('commons.edit') }}
         </el-button>
       </div>
     </div>
@@ -181,13 +277,17 @@
 
 <script>
 import LayoutContent from '@/components/business/LayoutContent'
-import {addDs, editDs, getSchema, validateDs, validateDsById} from '@/api/system/datasource'
+import {addDs, editDs, getSchema, validateDs, validateDsById, checkApiDatasource} from '@/api/system/datasource'
 import {$confirm} from '@/utils/message'
 import i18n from '@/lang/index'
+import ApiHttpRequestForm from '@/views/system/datasource/ApiHttpRequestForm'
 
 export default {
   name: 'DsForm',
-  components: {LayoutContent},
+  components: {
+    LayoutContent,
+    ApiHttpRequestForm
+  },
   props: {
     params: {
       type: Object,
@@ -210,7 +310,8 @@ export default {
           acquireIncrement: 5,
           idleConnectionTestPeriod: 5,
           connectTimeout: 5
-        }
+        },
+        apiConfiguration: []
       },
       rule: {
         name: [{required: true, message: i18n.t('datasource.input_name'), trigger: 'blur'},
@@ -269,26 +370,81 @@ export default {
           required: true,
           message: i18n.t('datasource.please_input_connect_timeout'),
           trigger: 'change'
-        }]
+        }],
+        'url': [{required: true, message: i18n.t('datasource.please_input_url'), trigger: 'change'}],
+        'dataPath': [{required: true, message: i18n.t('datasource.please_input_dataPath'), trigger: 'change'}]
       },
       allTypes: [
-        {name: 'mysql', label: 'MySQL', type: 'jdbc', extraParams: 'characterEncoding=UTF-8&connectTimeout=5000&useSSL=false&allowPublicKeyRetrieval=true'
+        {
+          name: 'mysql',
+          label: 'MySQL',
+          type: 'jdbc',
+          extraParams: 'characterEncoding=UTF-8&connectTimeout=5000&useSSL=false&allowPublicKeyRetrieval=true'
         },
         {name: 'hive', label: 'Apache Hive', type: 'jdbc', extraParams: ''},
         {name: 'oracle', label: 'Oracle', type: 'jdbc'},
         {name: 'sqlServer', label: 'SQL Server', type: 'jdbc', extraParams: ''},
         {name: 'pg', label: 'PostgreSQL', type: 'jdbc', extraParams: ''},
         {name: 'es', label: 'Elasticsearch', type: 'es'},
-        {name: 'mariadb', label: 'MariaDB', type: 'jdbc', extraParams: 'characterEncoding=UTF-8&connectTimeout=5000&useSSL=false&allowPublicKeyRetrieval=true'},
+        {
-        {name: 'ds_doris', label: 'Doris', type: 'jdbc', extraParams: 'characterEncoding=UTF-8&connectTimeout=5000&useSSL=false&allowPublicKeyRetrieval=true'},
+          name: 'mariadb',
+          label: 'MariaDB',
+          type: 'jdbc',
+          extraParams: 'characterEncoding=UTF-8&connectTimeout=5000&useSSL=false&allowPublicKeyRetrieval=true'
+        },
+        {
+          name: 'ds_doris',
+          label: 'Doris',
+          type: 'jdbc',
+          extraParams: 'characterEncoding=UTF-8&connectTimeout=5000&useSSL=false&allowPublicKeyRetrieval=true'
+        },
         {name: 'ck', label: 'ClickHouse', type: 'jdbc', extraParams: ''},
         {name: 'redshift', label: 'AWS Redshift', type: 'jdbc'},
         {name: 'mongo', label: 'MongoDB', type: 'jdbc', extraParams: ''},
-        {name: 'db2', label: 'Db2', type: 'jdbc', extraParams: ''}
+        {name: 'db2', label: 'Db2', type: 'jdbc', extraParams: ''},
+        {name: 'api', label: 'API', type: 'api', extraParams: ''}
       ],
       schemas: [],
       canEdit: false,
-      originConfiguration: {}
+      originConfiguration: {},
+      edit_api_item: false,
+      add_api_item: false,
+      active: 0,
+      defaultApiItem: {
+        name: '',
+        url: '',
+        method: 'GET',
+        request: {
+          headers: [],
+          body: {}
+        },
+        fields: []
+      },
+      apiItem: {
+        name: '',
+        url: '',
+        method: 'GET',
+        dataPath: '',
+        request: {
+          headers: [],
+          body: {},
+          authManager: {}
+        },
+        fields: []
+      },
+      reqOptions: [{id: 'GET', label: 'GET'}, {id: 'POST', label: 'POST'}],
+      loading: false,
+      responseData: {type: 'HTTP', responseResult: {}, subRequestResults: []},
+      api_table_title: '',
+      api_step2_active_name: 'first',
+      fieldTypes: [
+        { label: this.$t('dataset.text'), value: 0 },
+        { label: this.$t('dataset.time'), value: 1 },
+        { label: this.$t('dataset.value'), value: 2 },
+        { label: this.$t('dataset.value') + '(' + this.$t('dataset.float') + ')', value: 3 },
+        { label: this.$t('dataset.location'), value: 5 }
+      ],
+      height: 500
     }
   },
 
@@ -330,9 +486,13 @@ export default {
     },
     edit(row) {
       this.formType = 'modify'
-      this.form = Object.assign({}, row)
+      this.form = JSON.parse(JSON.stringify(row))
+
       this.originConfiguration = this.form.configuration
-      this.form.configuration = JSON.parse(this.form.configuration)
+      if(row.type === 'api'){
+      }else {
+        this.form.configuration = JSON.parse(this.form.configuration)
+      }
     },
     reset() {
       this.$refs.dsForm.resetFields()
@@ -353,9 +513,9 @@ export default {
       let repeat = false
       let repeatDsName = []
       this.tData.forEach(item => {
-        if(item.id === this.form.type){
+        if (item.id === this.form.type) {
           item.children.forEach(child => {
-            if(this.formType === 'modify' && child.id===this.form.id){
+            if (this.formType === 'modify' && child.id === this.form.id) {
               return
             }
             let configuration = JSON.parse(child.configuration)
@@ -367,7 +527,7 @@ export default {
               case 'ck':
               case 'mongo':
               case 'mariadb':
-                if(configuration.host == this.form.configuration.host && configuration.dataBase == this.form.configuration.dataBase && configuration.port == this.form.configuration.port){
+                if (configuration.host == this.form.configuration.host && configuration.dataBase == this.form.configuration.dataBase && configuration.port == this.form.configuration.port) {
                   repeat = true
                   repeatDsName.push(child.name)
                 }
@@ -377,13 +537,13 @@ export default {
               case 'redshift':
               case 'oracle':
               case 'db2':
-                if(configuration.host == this.form.configuration.host && configuration.dataBase == this.form.configuration.dataBase && configuration.port == this.form.configuration.port && configuration.schema == this.form.configuration.schema){
+                if (configuration.host == this.form.configuration.host && configuration.dataBase == this.form.configuration.dataBase && configuration.port == this.form.configuration.port && configuration.schema == this.form.configuration.schema) {
                   repeatDsName.push(child.name)
                   repeat = true
                 }
                 break
               case 'es':
-                if(configuration.url == this.form.configuration.url){
+                if (configuration.url == this.form.configuration.url) {
                   repeatDsName.push(child.name)
                   repeat = true
                 }
@@ -401,31 +561,36 @@ export default {
         }
         const method = this.formType === 'add' ? addDs : editDs
         const form = JSON.parse(JSON.stringify(this.form))
-        form.configuration = JSON.stringify(form.configuration)
+        if(form.type === 'api'){
+          form.configuration = JSON.stringify(form.apiConfiguration)
+        }else {
+          form.configuration = JSON.stringify(form.configuration)
+        }
+
         if (this.formType === 'modify' && this.originConfiguration !== form.configuration) {
-          if(repeat){
+          if (repeat) {
             $confirm(i18n.t('datasource.repeat_datasource_msg') + '[' + repeatDsName.join(',') + '], ' + i18n.t('datasource.confirm_save'), () => {
               $confirm(i18n.t('datasource.edit_datasource_msg'), () => {
                 this.method(method, form)
               })
             })
-          }else {
+          } else {
             $confirm(i18n.t('datasource.edit_datasource_msg'), () => {
               this.method(method, form)
             })
           }
           return
         }
-        if(repeat){
+        if (repeat) {
           $confirm(i18n.t('datasource.repeat_datasource_msg') + '[' + repeatDsName.join(',') + '], ' + i18n.t('datasource.confirm_save'), () => {
             this.method(method, form)
           })
-        }else {
+        } else {
           this.method(method, form)
         }
       })
     },
-    method(method, form){
+    method(method, form) {
       method(form).then(res => {
         this.$success(i18n.t('commons.save_success'))
         this.refreshType(form)
@@ -464,10 +629,10 @@ export default {
               if (res.success) {
                 this.$success(i18n.t('datasource.validate_success'))
               } else {
-                if(res.message.length < 2500){
+                if (res.message.length < 2500) {
                   this.$error(res.message)
-                }else {
+                } else {
-                  this.$error(res.message.substring(0,2500) + '......')
+                  this.$error(res.message.substring(0, 2500) + '......')
                 }
               }
               this.refreshType(data)
@@ -477,10 +642,10 @@ export default {
               if (res.success) {
                 this.$success(i18n.t('datasource.validate_success'))
               } else {
-                if(res.message.length < 2500){
+                if (res.message.length < 2500) {
                   this.$error(res.message)
-                }else {
+                } else {
-                  this.$error(res.message.substring(0,2500) + '......')
+                  this.$error(res.message.substring(0, 2500) + '......')
                 }
               }
             }).catch(res => {
@@ -505,6 +670,104 @@ export default {
     },
     refreshType(form) {
       this.$emit('refresh-type', form)
+    },
+    next() {
+      if(this.active === 1){
+        this.$refs.apiItem.validate(valid => {
+          if (valid) {
+            console.log(this.apiItem)
+            const data = JSON.parse(JSON.stringify(this.apiItem))
+            data.request = JSON.stringify(data.request)
+            this.loading = true
+            checkApiDatasource(data).then(res => {
+              this.loading = false
+              console.log(res)
+              this.$success(i18n.t('commons.success'))
+              this.active++
+              this.apiItem.fields = res.data.fields
+              this.$refs.plxTable.reloadData(res.data.datas)
+            }).catch(res => {
+              this.apiItem.fields = []
+              this.loading = false
+            })
+          } else {
+            this.apiItem.fields = []
+            return false
+          }
+        })
+      }
+    },
+    before() {
+      this.active--
+    },
+    closeEditItem() {
+      this.active = 0
+      this.edit_api_item = false
+    },
+    saveItem() {
+      this.active = 0
+      this.edit_api_item = false
+      if(!this.add_api_item){
+        this.form.apiConfiguration.push(this.apiItem)
+      }
+    },
+    addApiItem(item) {
+      if (item) {
+        this.add_api_item = true
+        this.api_table_title = this.$t('datasource.edit_api_table')
+        this.apiItem = item
+      }else {
+        this.add_api_item = false
+        this.apiItem = JSON.parse(JSON.stringify(this.defaultApiItem))
+        this.api_table_title = this.$t('datasource.add_api_table')
+      }
+      this.active = 1
+      this.edit_api_item = true
+    },
+    deleteItem(item) {
+      this.form.apiConfiguration.splice(this.form.apiConfiguration.indexOf(item), 1)
+    },
+    runDebug() {
+      this.$refs['debugForm'].validate((valid) => {
+        if (valid) {
+          this.loading = true;
+          this.isStop = true;
+          this.request.url = this.debugForm.url;
+          this.request.method = this.debugForm.method;
+          this.request.name = getUUID().substring(0, 8);
+          this.runData = [];
+          this.runData.push(this.request);
+          /*触发执行操作*/
+          this.reportId = getUUID().substring(0, 8);
+        }
+      })
+    },
+    validateApi(item) {
+      if(undefined){
+
+      }else {
+        this.$refs.apiItem.validate(valid => {
+          if (valid) {
+            const data = JSON.parse(JSON.stringify(this.apiItem))
+            data.request = JSON.stringify(data.request)
+            this.loading = true
+            checkApiDatasource(data).then(res => {
+              this.loading = false
+              console.log(res)
+              this.$success(i18n.t('commons.success'))
+              this.apiItem.fields = res.data.fields
+              this.$refs.plxTable.reloadData(res.data.datas)
+            }).catch(res => {
+              this.loading = false
+            })
+          } else {
+            return false
+          }
+        })
+      }
+    },
+    handleClick(tab, event) {
+      console.log(tab, event);
     }
   }
 }
@@ -520,12 +783,27 @@ export default {
     transform: scale(0.85);
   }
 }
-
 .el-input {
   width: 300px;
 }
-
 .el-select {
   width: 300px;
 }
+.ms-http-input {
+  width: 500px;
+  margin-top: 5px;
+}
+.tip {
+  padding: 3px 5px;
+  font-size: 16px;
+  border-radius: 0;
+  border-left: 4px solid #783887;
+  margin: 5px 5px 10px 5px;
+}
+.el-select>>>input{
+  padding-right: 10px;
+}
+.el-select>>>.el-input__suffix{
+  right: 0;
+}
 </style>

frontend/src/views/system/datasource/format-utils.js

@@ -0,0 +1,185 @@
+export function formatJson (json) {
+  let i = 0,
+    il = 0,
+    tab = "    ",
+    newJson = "",
+    indentLevel = 0,
+    inString = false,
+    currentChar = null;
+  let flag = false;
+  for (i = 0, il = json.length; i < il; i += 1) {
+    currentChar = json.charAt(i);
+    switch (currentChar) {
+      case '{':
+        if (i != 0 && json.charAt(i - 1) === '$') {
+          newJson += currentChar;
+          flag = true;
+        } else if (!inString) {
+          newJson += currentChar + "\n" + repeat(tab, indentLevel + 1);
+          indentLevel += 1
+        } else {
+          newJson += currentChar
+        }
+        break;
+      case '[':
+        if (!inString) {
+          newJson += currentChar + "\n" + repeat(tab, indentLevel + 1);
+          indentLevel += 1
+        } else {
+          newJson += currentChar
+        }
+        break;
+      case '}':
+        if (flag) {
+          newJson += currentChar;
+          flag = false;
+        } else if (!inString) {
+          indentLevel -= 1;
+          newJson += "\n" + repeat(tab, indentLevel) + currentChar
+        } else {
+          newJson += currentChar
+        }
+        break;
+      case ']':
+        if (!inString) {
+          indentLevel -= 1;
+          newJson += "\n" + repeat(tab, indentLevel) + currentChar
+        } else {
+          newJson += currentChar
+        }
+        break;
+      case ',':
+        if (!inString) {
+          newJson += ",\n" + repeat(tab, indentLevel)
+        } else {
+          newJson += currentChar
+        }
+        break;
+      case ':':
+        if (!inString) {
+          newJson += ": "
+        } else {
+          newJson += currentChar
+        }
+        break;
+      case ' ':
+      case "\n":
+      case "\t":
+        if (inString) {
+          newJson += currentChar
+        }
+        break;
+      case '"':
+        if (i > 0 && json.charAt(i - 1) !== '\\') {
+          inString = !inString
+        }
+        newJson += currentChar;
+        break;
+      default:
+        newJson += currentChar;
+        break
+    }
+  }
+  return newJson;
+}
+
+function repeat(s, count) {
+  return new Array(count + 1).join(s)
+}
+
+
+
+export function formatXml(text) {
+  //去掉多余的空格
+  text = '\n' + text.replace(/(<\w+)(\s.*?>)/g, function ($0, name, props) {
+    return name + ' ' + props.replace(/\s+(\w+=)/g, " $1");
+  });
+  //把注释编码
+  text = text.replace(/<!--(.+?)-->/g, function ($0, text) {
+    var ret = '<!--' + escape(text) + '-->';
+    //alert(ret);
+    return ret;
+  });
+  //调整格式
+  var rgx = /\n(<(([^\?]).+?)(?:\s|\s*?>|\s*?(\/)>)(?:.*?(?:(?:(\/)>)|(?:<(\/)\2>)))?)/mg;
+  var nodeStack = [];
+  var output = text.replace(rgx, function ($0, all, name, isBegin, isCloseFull1, isCloseFull2, isFull1, isFull2) {
+    var isClosed = (isCloseFull1 == '/') || (isCloseFull2 == '/' ) || (isFull1 == '/') || (isFull2 == '/');
+    //alert([all,isClosed].join('='));
+    var prefix = '';
+    if (isBegin == '!') {
+      prefix = getPrefix(nodeStack.length);
+    }
+    else {
+      if (isBegin != '/') {
+        prefix = getPrefix(nodeStack.length);
+        if (!isClosed) {
+          nodeStack.push(name);
+        }
+      }
+      else {
+        nodeStack.pop();
+        prefix = getPrefix(nodeStack.length);
+      }
+    }
+    var ret = '\n' + prefix + all;
+    return ret;
+  });
+  var prefixSpace = -1;
+  var outputText = output.substring(1);
+  //把注释还原并解码，调格式
+  outputText = outputText.replace(/(\s*)<!--(.+?)-->/g, function ($0, prefix, text) {
+    if (prefix.charAt(0) == '\r')
+      prefix = prefix.substring(1);
+    text = unescape(text).replace(/\r/g, '\n');
+    var ret = '\n' + prefix + '<!--' + text.replace(/^\s*/mg, prefix) + '-->';
+    //alert(ret);
+    return ret;
+  });
+  return outputText.replace(/\s+$/g, '').replace(/\r/g, '\r\n');
+}
+
+/**
+ * @param time 时间
+ * @param cFormat 格式
+ * @returns {string|null} 字符串
+ * @example formatTime('2018-1-29', '{y}/{m}/{d} {h}:{i}:{s}') // -> 2018/01/29 00:00:00
+ */
+export function formatTime(time, cFormat) {
+  if (arguments.length === 0) return null;
+  if ((time + '').length === 10) {
+    time = +time * 1000;
+  }
+  let format = cFormat || '{y}-{m}-{d} {h}:{i}:{s}', date;
+  if (typeof time === 'object') {
+    date = time;
+  } else {
+    date = new Date(time);
+  }
+  let formatObj = {
+    y: date.getFullYear(),
+    m: date.getMonth() + 1,
+    d: date.getDate(),
+    h: date.getHours(),
+    i: date.getMinutes(),
+    s: date.getSeconds(),
+    a: date.getDay()
+  };
+  return format.replace(/{([ymdhisa])+}/g, (result, key) => {
+    let value = formatObj[key];
+    if (key === 'a') return ['一', '二', '三', '四', '五', '六', '日'][value - 1];
+    if (result.length > 0 && value < 10) {
+      value = '0' + value;
+    }
+    return value || 0;
+  });
+}
+
+function getPrefix(prefixIndex) {
+  var span = ' ';
+  var output = [];
+  for (var i = 0; i < prefixIndex; ++i) {
+    output.push(span);
+  }
+  return output.join('');
+}